From 56760150e42751ec80e4692a567013d536255d43 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Thu, 27 Aug 2026 17:35:42 -0700 Subject: [PATCH 01/15] ADFA-5240: Reindent the plugin documentation package to tabs Spotless's ratchet is file-level: editing one line of these space-indented files pulls each whole file under it. Doing the reformat on its own keeps the behavioral diff that follows reviewable. No logic change -- token-identical to the previous revision apart from the trailing commas ktlint adds. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01M2Bzxv38NbXWPMQVckoSNC --- .../manager/documentation/BrotliCompressor.kt | 19 +- .../PluginDocumentationManager.kt | 1171 +++++++++-------- .../manager/documentation/Tier3AssetWalker.kt | 132 +- 3 files changed, 680 insertions(+), 642 deletions(-) diff --git a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/BrotliCompressor.kt b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/BrotliCompressor.kt index 445bf3df30..7be3704192 100644 --- a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/BrotliCompressor.kt +++ b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/BrotliCompressor.kt @@ -6,15 +6,14 @@ import com.aayushatharva.brotli4j.encoder.Encoder import java.io.ByteArrayOutputStream internal object BrotliCompressor { + private val params: Encoder.Parameters by lazy { + Brotli4jLoader.ensureAvailability() + Encoder.Parameters().setQuality(11).setWindow(24) + } - private val params: Encoder.Parameters by lazy { - Brotli4jLoader.ensureAvailability() - Encoder.Parameters().setQuality(11).setWindow(24) - } - - fun compress(input: ByteArray): ByteArray { - val out = ByteArrayOutputStream(input.size) - BrotliOutputStream(out, params).use { it.write(input) } - return out.toByteArray() - } + fun compress(input: ByteArray): ByteArray { + val out = ByteArrayOutputStream(input.size) + BrotliOutputStream(out, params).use { it.write(input) } + return out.toByteArray() + } } diff --git a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt index c079d106b8..a2b0249fb9 100644 --- a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt +++ b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt @@ -20,574 +20,605 @@ import kotlin.coroutines.cancellation.CancellationException * differentiated by a "plugin_" category prefix so they never conflict with * built-in documentation. */ -class PluginDocumentationManager(private val context: Context) { - - companion object { - private const val TAG = "PluginDocManager" - } - - private val databaseName = "documentation.db" - - private suspend fun getPluginDatabase(): SQLiteDatabase? = withContext(Dispatchers.IO) { - try { - val dbFile = context.getDatabasePath(databaseName) - if (!dbFile.exists()) { - Log.w(TAG, "documentation.db not yet available at: ${dbFile.absolutePath}") - return@withContext null - } - SQLiteDatabase.openDatabase(dbFile.absolutePath, null, SQLiteDatabase.OPEN_READWRITE) - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Failed to open documentation.db for plugin writes", e) - null - } - } - - - /** - * Initialize plugin documentation system. - * Also cleans up the legacy plugin_documentation.db if present. - */ - suspend fun initialize() = withContext(Dispatchers.IO) { - val legacyDb = context.getDatabasePath("plugin_documentation.db") - if (legacyDb.exists()) { - if (legacyDb.delete()) { - Log.d(TAG, "Removed legacy plugin_documentation.db") - } else { - Log.w(TAG, "Failed to remove legacy plugin_documentation.db") - } - } - Log.d(TAG, "Plugin documentation system initialized") - } - - /** - * Install documentation from a plugin into documentation.db. - */ - suspend fun installPluginDocumentation( - pluginId: String, - plugin: DocumentationExtension - ): Boolean = withContext(Dispatchers.IO) { - - if (!plugin.onDocumentationInstall()) { - Log.d(TAG, "Plugin $pluginId declined documentation installation") - return@withContext false - } - - val db = getPluginDatabase() - if (db == null) { - Log.w(TAG, "Cannot install documentation for $pluginId - database not available") - return@withContext false - } - - val entries = plugin.getTooltipEntries() - - if (entries.isEmpty()) { - Log.d(TAG, "Plugin $pluginId has no tooltip entries") - db.close() - return@withContext true - } - - Log.d(TAG, "Installing ${entries.size} tooltip entries for plugin $pluginId") - - db.beginTransaction() - try { - removePluginDocumentationInternal(db, pluginId) - - val categoryId = insertOrGetCategoryId(db, pluginCategory(pluginId)) - - for (entry in entries) { - val tooltipId = insertTooltip(db, categoryId, entry) - entry.buttons.sortedBy { it.order }.forEachIndexed { index, button -> - val resolvedUri = resolvePluginButtonUri(pluginId, button.uri, button.directPath) - insertTooltipButton(db, tooltipId, button.description, resolvedUri, index) - } - } - - db.setTransactionSuccessful() - Log.d(TAG, "Successfully installed documentation for plugin $pluginId") - true - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Failed to install documentation for plugin $pluginId", e) - false - } finally { - db.endTransaction() - db.close() - } - } - - /** - * Remove all documentation for a plugin from documentation.db. - */ - suspend fun removePluginDocumentation( - pluginId: String, - plugin: DocumentationExtension? = null - ): Boolean = withContext(Dispatchers.IO) { - - try { - plugin?.onDocumentationUninstall() - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Plugin onDocumentationUninstall() threw during removal: $pluginId", e) - } - - val db = getPluginDatabase() - if (db == null) { - Log.w(TAG, "Cannot remove documentation for $pluginId - database not available") - return@withContext false - } - - db.beginTransaction() - try { - removePluginDocumentationInternal(db, pluginId) - db.setTransactionSuccessful() - Log.d(TAG, "Successfully removed documentation for plugin $pluginId") - true - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Failed to remove documentation for plugin $pluginId", e) - false - } finally { - db.endTransaction() - db.close() - } - } - - /** - * Install Tier 3 documentation (full help pages) contributed by a plugin. - * - * Walks the plugin-declared asset subdirectory, compresses each file per - * the existing ContentTypes.compression column, chunks blobs at 1 MB to - * match WebServer's read loop, and inserts everything under the reserved - * path namespace "plugin//..." inside a single transaction. - */ - suspend fun installPluginTier3Documentation( - pluginId: String, - plugin: DocumentationExtension, - pluginApkPath: String - ): Boolean = withContext(Dispatchers.IO) { - - val assetPath = plugin.getTier3DocsAssetPath() - if (assetPath.isNullOrBlank()) { - return@withContext true - } - - val pluginAssets = try { - openPluginOnlyAssets(pluginApkPath) - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Failed to open plugin APK assets for $pluginId", e) - return@withContext false - } - - val db = getPluginDatabase() - if (db == null) { - Log.w(TAG, "Cannot install Tier 3 docs for $pluginId - database not available") - pluginAssets.close() - return@withContext false - } - - val resolver = ExtensionToContentTypeResolver() - var inserted = 0 - var skipped = 0 - - db.beginTransaction() - try { - removePluginTier3Internal(db, pluginId) - - for (asset in Tier3AssetWalker.walk(pluginAssets, assetPath)) { - val ext = asset.relativePath.substringAfterLast('.', "") - if (ext.isEmpty()) { - Log.w(TAG, "Skipping Tier 3 asset without extension: ${asset.relativePath}") - skipped++ - continue - } - val row = resolver.resolve(db, ext) - if (row == null) { - Log.w(TAG, "No ContentType for .$ext (${asset.relativePath}); skipping") - skipped++ - continue - } - - val payload = if (row.compression == "brotli") { - BrotliCompressor.compress(asset.bytes) - } else { - asset.bytes - } - - val safeRelative = try { - normalizeLocalDocumentationPath(asset.relativePath) - } catch (e: IllegalArgumentException) { - Log.w(TAG, "Skipping Tier 3 asset with invalid path '${asset.relativePath}': ${e.message}") - skipped++ - continue - } - val basePath = "plugin/$pluginId/$safeRelative" - insertContentChunked(db, basePath, payload, row.id) - inserted++ - } - - db.setTransactionSuccessful() - Log.d(TAG, "Installed $inserted Tier 3 documents for plugin $pluginId (skipped=$skipped)") - true - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Failed to install Tier 3 docs for plugin $pluginId", e) - false - } finally { - db.endTransaction() - db.close() - pluginAssets.close() - } - } - - /** - * Remove all Tier 3 documentation rows owned by the given plugin. - */ - suspend fun removePluginTier3Documentation( - pluginId: String - ): Boolean = withContext(Dispatchers.IO) { - val db = getPluginDatabase() ?: return@withContext false - db.beginTransaction() - try { - val deleted = removePluginTier3Internal(db, pluginId) - db.setTransactionSuccessful() - Log.d(TAG, "Removed $deleted Tier 3 rows for plugin $pluginId") - true - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Failed to remove Tier 3 docs for plugin $pluginId", e) - false - } finally { - db.endTransaction() - db.close() - } - } - - /** - * Verify that Tier 3 content exists for this plugin; reinstall if missing. - * Mirrors [verifyAndRecreateDocumentation] for the Tier 1/2 pipeline. - */ - suspend fun verifyAndRecreateTier3Documentation( - pluginId: String, - plugin: DocumentationExtension, - pluginApkPath: String - ): Boolean = withContext(Dispatchers.IO) { - if (plugin.getTier3DocsAssetPath().isNullOrBlank()) { - return@withContext true - } - if (!isDatabaseAvailable()) { - Log.d(TAG, "documentation.db not available yet for Tier 3 verify of $pluginId") - return@withContext false - } - if (isPluginTier3DocumentationInstalled(pluginId)) { - Log.d(TAG, "Tier 3 docs already present for $pluginId") - return@withContext true - } - Log.d(TAG, "Tier 3 docs missing for $pluginId, installing...") - installPluginTier3Documentation(pluginId, plugin, pluginApkPath) - } - - /** - * Check if any Tier 3 content rows exist for this plugin. - */ - suspend fun isPluginTier3DocumentationInstalled(pluginId: String): Boolean = withContext(Dispatchers.IO) { - val db = getPluginDatabase() ?: return@withContext false - try { - val prefix = "plugin/$pluginId" - db.rawQuery( - "SELECT 1 FROM Content WHERE path = ? OR path LIKE ? ESCAPE '\\' LIMIT 1", - arrayOf(prefix, "${escapeLike(prefix)}/%") - ).use { cursor -> cursor.moveToFirst() } - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Failed to probe Tier 3 installation for $pluginId", e) - false - } finally { - db.close() - } - } - - /** - * Build an AssetManager that sees ONLY the plugin APK, so walking a top-level - * asset directory cannot pick up collisions with the host app's assets. - */ - private fun openPluginOnlyAssets(pluginApkPath: String): AssetManager { - @Suppress("DEPRECATION") - val am = AssetManager::class.java.getDeclaredConstructor().newInstance() - val addAssetPath = AssetManager::class.java.getMethod("addAssetPath", String::class.java) - val cookie = addAssetPath.invoke(am, pluginApkPath) as? Int ?: 0 - if (cookie == 0) { - throw IllegalStateException("addAssetPath returned 0 for $pluginApkPath") - } - return am - } - - private fun removePluginTier3Internal(db: SQLiteDatabase, pluginId: String): Int { - val prefix = "plugin/$pluginId" - return db.delete( - "Content", - "path = ? OR path LIKE ? ESCAPE '\\'", - arrayOf(prefix, "${escapeLike(prefix)}/%") - ) - } - - private fun insertContentChunked( - db: SQLiteDatabase, - basePath: String, - payload: ByteArray, - contentTypeId: Long - ) { - val chunkSize = 1024 * 1024 - if (payload.size < chunkSize) { - insertContentRow(db, basePath, payload, contentTypeId) - return - } - - var offset = 0 - var fragment = 0 - while (offset < payload.size) { - val end = minOf(offset + chunkSize, payload.size) - val slice = payload.copyOfRange(offset, end) - val path = if (fragment == 0) basePath else "$basePath-$fragment" - insertContentRow(db, path, slice, contentTypeId) - offset = end - fragment++ - } - if (payload.size % chunkSize == 0) { - insertContentRow(db, "$basePath-$fragment", ByteArray(0), contentTypeId) - } - } - - private fun insertContentRow( - db: SQLiteDatabase, - path: String, - blob: ByteArray, - contentTypeId: Long - ) { - val values = ContentValues().apply { - put("path", path) - put("content", blob) - put("contentTypeID", contentTypeId) - put("languageId", 1) - } - db.insertOrThrow("Content", null, values) - } - - private fun removePluginDocumentationInternal(db: SQLiteDatabase, pluginId: String) { - val category = pluginCategory(pluginId) - - val cursor = db.rawQuery( - """ - SELECT T.id FROM Tooltips AS T - INNER JOIN TooltipCategories AS TC ON T.categoryId = TC.id - WHERE TC.category = ? - """.trimIndent(), - arrayOf(category) - ) - - val tooltipIds = mutableListOf() - while (cursor.moveToNext()) { - tooltipIds.add(cursor.getLong(0)) - } - cursor.close() - - if (tooltipIds.isNotEmpty()) { - val placeholders = tooltipIds.joinToString(",") { "?" } - val args = tooltipIds.map { it.toString() }.toTypedArray() - db.delete("TooltipButtons", "tooltipId IN ($placeholders)", args) - db.delete("Tooltips", "id IN ($placeholders)", args) - } - - db.delete("TooltipCategories", "category = ?", arrayOf(category)) - } - - private fun insertOrGetCategoryId(db: SQLiteDatabase, category: String): Long { - val cursor = db.query( - "TooltipCategories", - arrayOf("id"), - "category = ?", - arrayOf(category), - null, null, null - ) - - if (cursor.moveToFirst()) { - val id = cursor.getLong(0) - cursor.close() - return id - } - cursor.close() - - val values = ContentValues().apply { - put("category", category) - } - return db.insert("TooltipCategories", null, values) - } - - private fun insertTooltip( - db: SQLiteDatabase, - categoryId: Long, - entry: PluginTooltipEntry - ): Long { - val disclaimer = context.getString(R.string.plugin_documentation_third_party_disclaimer) - - val existingCursor = db.query( - "Tooltips", - arrayOf("id"), - "categoryId = ? AND tag = ?", - arrayOf(categoryId.toString(), entry.tag), - null, null, null - ) - - if (existingCursor.moveToFirst()) { - val existingId = existingCursor.getLong(0) - existingCursor.close() - - val updateValues = ContentValues().apply { - put("summary", entry.summary + disclaimer) - put("detail", if (entry.detail.isNotBlank()) entry.detail + disclaimer else "") - } - db.update("Tooltips", updateValues, "id = ?", arrayOf(existingId.toString())) - db.delete("TooltipButtons", "tooltipId = ?", arrayOf(existingId.toString())) - return existingId - } - existingCursor.close() - - val values = ContentValues().apply { - put("categoryId", categoryId) - put("tag", entry.tag) - put("summary", entry.summary + disclaimer) - put("detail", if (entry.detail.isNotBlank()) entry.detail + disclaimer else "") - } - return db.insert("Tooltips", null, values) - } - - private fun escapeLike(value: String): String = - value - .replace("\\", "\\\\") - .replace("%", "\\%") - .replace("_", "\\_") - - private fun normalizeLocalDocumentationPath(path: String): String { - val segments = path.split('/').filter { it.isNotEmpty() && it != "." } - require(segments.none { it == ".." }) { - "Documentation paths must not contain '..' segments: $path" - } - return segments.joinToString("/") - } - - private fun resolvePluginButtonUri( - pluginId: String, - rawUri: String, - directPath: Boolean - ): String { - if (rawUri.isEmpty()) return rawUri - if (rawUri.contains("://")) return rawUri - val absolute = directPath || rawUri.startsWith("/") - val normalized = normalizeLocalDocumentationPath(rawUri.trimStart('/')) - return if (absolute) normalized else "plugin/$pluginId/$normalized" - } - - private fun insertTooltipButton( - db: SQLiteDatabase, - tooltipId: Long, - description: String, - uri: String, - order: Int - ) { - val values = ContentValues().apply { - put("tooltipId", tooltipId) - put("description", description) - put("uri", uri) - put("buttonNumberId", order) - } - db.insert("TooltipButtons", null, values) - } - - /** - * Check if the plugin documentation database is accessible. - */ - suspend fun isDatabaseAvailable(): Boolean = withContext(Dispatchers.IO) { - context.getDatabasePath(databaseName).exists() - } - - /** - * Check if documentation for a specific plugin exists in documentation.db. - */ - suspend fun isPluginDocumentationInstalled(pluginId: String): Boolean = withContext(Dispatchers.IO) { - val db = getPluginDatabase() ?: return@withContext false - - try { - val cursor = db.rawQuery( - "SELECT COUNT(*) FROM TooltipCategories WHERE category = ?", - arrayOf(pluginCategory(pluginId)) - ) - val installed = cursor.moveToFirst() && cursor.getInt(0) > 0 - cursor.close() - installed - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Failed to check plugin documentation for $pluginId", e) - false - } finally { - db.close() - } - } - - /** - * Verify and recreate plugin documentation if missing. - */ - suspend fun verifyAndRecreateDocumentation( - pluginId: String, - plugin: DocumentationExtension - ): Boolean = withContext(Dispatchers.IO) { - if (!isDatabaseAvailable()) { - Log.d(TAG, "documentation.db not available yet for $pluginId, skipping") - return@withContext false - } - - if (!isPluginDocumentationInstalled(pluginId)) { - Log.d(TAG, "Plugin documentation missing for $pluginId, recreating...") - return@withContext installPluginDocumentation(pluginId, plugin) - } - - Log.d(TAG, "Plugin documentation already exists for $pluginId") - true - } - - /** - * Verify and recreate documentation for all plugins that support it. - */ - suspend fun verifyAllPluginDocumentation( - plugins: Map - ): Int = withContext(Dispatchers.IO) { - if (plugins.isEmpty()) return@withContext 0 - - if (!isDatabaseAvailable()) { - Log.d(TAG, "documentation.db not available yet, skipping verification") - return@withContext 0 - } - - var recreatedCount = 0 - - for ((pluginId, plugin) in plugins) { - try { - if (!isPluginDocumentationInstalled(pluginId)) { - Log.d(TAG, "Recreating missing documentation for plugin: $pluginId") - if (installPluginDocumentation(pluginId, plugin)) { - recreatedCount++ - } - } - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Failed to verify/recreate documentation for $pluginId", e) - } - } - - if (recreatedCount > 0) { - Log.i(TAG, "Recreated documentation for $recreatedCount plugins") - } - - recreatedCount - } +class PluginDocumentationManager( + private val context: Context, +) { + companion object { + private const val TAG = "PluginDocManager" + } + + private val databaseName = "documentation.db" + + private suspend fun getPluginDatabase(): SQLiteDatabase? = + withContext(Dispatchers.IO) { + try { + val dbFile = context.getDatabasePath(databaseName) + if (!dbFile.exists()) { + Log.w(TAG, "documentation.db not yet available at: ${dbFile.absolutePath}") + return@withContext null + } + SQLiteDatabase.openDatabase(dbFile.absolutePath, null, SQLiteDatabase.OPEN_READWRITE) + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to open documentation.db for plugin writes", e) + null + } + } + + /** + * Initialize plugin documentation system. + * Also cleans up the legacy plugin_documentation.db if present. + */ + suspend fun initialize() = + withContext(Dispatchers.IO) { + val legacyDb = context.getDatabasePath("plugin_documentation.db") + if (legacyDb.exists()) { + if (legacyDb.delete()) { + Log.d(TAG, "Removed legacy plugin_documentation.db") + } else { + Log.w(TAG, "Failed to remove legacy plugin_documentation.db") + } + } + Log.d(TAG, "Plugin documentation system initialized") + } + + /** + * Install documentation from a plugin into documentation.db. + */ + suspend fun installPluginDocumentation( + pluginId: String, + plugin: DocumentationExtension, + ): Boolean = + withContext(Dispatchers.IO) { + if (!plugin.onDocumentationInstall()) { + Log.d(TAG, "Plugin $pluginId declined documentation installation") + return@withContext false + } + + val db = getPluginDatabase() + if (db == null) { + Log.w(TAG, "Cannot install documentation for $pluginId - database not available") + return@withContext false + } + + val entries = plugin.getTooltipEntries() + + if (entries.isEmpty()) { + Log.d(TAG, "Plugin $pluginId has no tooltip entries") + db.close() + return@withContext true + } + + Log.d(TAG, "Installing ${entries.size} tooltip entries for plugin $pluginId") + + db.beginTransaction() + try { + removePluginDocumentationInternal(db, pluginId) + + val categoryId = insertOrGetCategoryId(db, pluginCategory(pluginId)) + + for (entry in entries) { + val tooltipId = insertTooltip(db, categoryId, entry) + entry.buttons.sortedBy { it.order }.forEachIndexed { index, button -> + val resolvedUri = resolvePluginButtonUri(pluginId, button.uri, button.directPath) + insertTooltipButton(db, tooltipId, button.description, resolvedUri, index) + } + } + + db.setTransactionSuccessful() + Log.d(TAG, "Successfully installed documentation for plugin $pluginId") + true + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to install documentation for plugin $pluginId", e) + false + } finally { + db.endTransaction() + db.close() + } + } + + /** + * Remove all documentation for a plugin from documentation.db. + */ + suspend fun removePluginDocumentation( + pluginId: String, + plugin: DocumentationExtension? = null, + ): Boolean = + withContext(Dispatchers.IO) { + try { + plugin?.onDocumentationUninstall() + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Plugin onDocumentationUninstall() threw during removal: $pluginId", e) + } + + val db = getPluginDatabase() + if (db == null) { + Log.w(TAG, "Cannot remove documentation for $pluginId - database not available") + return@withContext false + } + + db.beginTransaction() + try { + removePluginDocumentationInternal(db, pluginId) + db.setTransactionSuccessful() + Log.d(TAG, "Successfully removed documentation for plugin $pluginId") + true + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to remove documentation for plugin $pluginId", e) + false + } finally { + db.endTransaction() + db.close() + } + } + + /** + * Install Tier 3 documentation (full help pages) contributed by a plugin. + * + * Walks the plugin-declared asset subdirectory, compresses each file per + * the existing ContentTypes.compression column, chunks blobs at 1 MB to + * match WebServer's read loop, and inserts everything under the reserved + * path namespace "plugin//..." inside a single transaction. + */ + suspend fun installPluginTier3Documentation( + pluginId: String, + plugin: DocumentationExtension, + pluginApkPath: String, + ): Boolean = + withContext(Dispatchers.IO) { + val assetPath = plugin.getTier3DocsAssetPath() + if (assetPath.isNullOrBlank()) { + return@withContext true + } + + val pluginAssets = + try { + openPluginOnlyAssets(pluginApkPath) + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to open plugin APK assets for $pluginId", e) + return@withContext false + } + + val db = getPluginDatabase() + if (db == null) { + Log.w(TAG, "Cannot install Tier 3 docs for $pluginId - database not available") + pluginAssets.close() + return@withContext false + } + + val resolver = ExtensionToContentTypeResolver() + var inserted = 0 + var skipped = 0 + + db.beginTransaction() + try { + removePluginTier3Internal(db, pluginId) + + for (asset in Tier3AssetWalker.walk(pluginAssets, assetPath)) { + val ext = asset.relativePath.substringAfterLast('.', "") + if (ext.isEmpty()) { + Log.w(TAG, "Skipping Tier 3 asset without extension: ${asset.relativePath}") + skipped++ + continue + } + val row = resolver.resolve(db, ext) + if (row == null) { + Log.w(TAG, "No ContentType for .$ext (${asset.relativePath}); skipping") + skipped++ + continue + } + + val payload = + if (row.compression == "brotli") { + BrotliCompressor.compress(asset.bytes) + } else { + asset.bytes + } + + val safeRelative = + try { + normalizeLocalDocumentationPath(asset.relativePath) + } catch (e: IllegalArgumentException) { + Log.w(TAG, "Skipping Tier 3 asset with invalid path '${asset.relativePath}': ${e.message}") + skipped++ + continue + } + val basePath = "plugin/$pluginId/$safeRelative" + insertContentChunked(db, basePath, payload, row.id) + inserted++ + } + + db.setTransactionSuccessful() + Log.d(TAG, "Installed $inserted Tier 3 documents for plugin $pluginId (skipped=$skipped)") + true + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to install Tier 3 docs for plugin $pluginId", e) + false + } finally { + db.endTransaction() + db.close() + pluginAssets.close() + } + } + + /** + * Remove all Tier 3 documentation rows owned by the given plugin. + */ + suspend fun removePluginTier3Documentation(pluginId: String): Boolean = + withContext(Dispatchers.IO) { + val db = getPluginDatabase() ?: return@withContext false + db.beginTransaction() + try { + val deleted = removePluginTier3Internal(db, pluginId) + db.setTransactionSuccessful() + Log.d(TAG, "Removed $deleted Tier 3 rows for plugin $pluginId") + true + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to remove Tier 3 docs for plugin $pluginId", e) + false + } finally { + db.endTransaction() + db.close() + } + } + + /** + * Verify that Tier 3 content exists for this plugin; reinstall if missing. + * Mirrors [verifyAndRecreateDocumentation] for the Tier 1/2 pipeline. + */ + suspend fun verifyAndRecreateTier3Documentation( + pluginId: String, + plugin: DocumentationExtension, + pluginApkPath: String, + ): Boolean = + withContext(Dispatchers.IO) { + if (plugin.getTier3DocsAssetPath().isNullOrBlank()) { + return@withContext true + } + if (!isDatabaseAvailable()) { + Log.d(TAG, "documentation.db not available yet for Tier 3 verify of $pluginId") + return@withContext false + } + if (isPluginTier3DocumentationInstalled(pluginId)) { + Log.d(TAG, "Tier 3 docs already present for $pluginId") + return@withContext true + } + Log.d(TAG, "Tier 3 docs missing for $pluginId, installing...") + installPluginTier3Documentation(pluginId, plugin, pluginApkPath) + } + + /** + * Check if any Tier 3 content rows exist for this plugin. + */ + suspend fun isPluginTier3DocumentationInstalled(pluginId: String): Boolean = + withContext(Dispatchers.IO) { + val db = getPluginDatabase() ?: return@withContext false + try { + val prefix = "plugin/$pluginId" + db + .rawQuery( + "SELECT 1 FROM Content WHERE path = ? OR path LIKE ? ESCAPE '\\' LIMIT 1", + arrayOf(prefix, "${escapeLike(prefix)}/%"), + ).use { cursor -> cursor.moveToFirst() } + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to probe Tier 3 installation for $pluginId", e) + false + } finally { + db.close() + } + } + + /** + * Build an AssetManager that sees ONLY the plugin APK, so walking a top-level + * asset directory cannot pick up collisions with the host app's assets. + */ + private fun openPluginOnlyAssets(pluginApkPath: String): AssetManager { + @Suppress("DEPRECATION") + val am = AssetManager::class.java.getDeclaredConstructor().newInstance() + val addAssetPath = AssetManager::class.java.getMethod("addAssetPath", String::class.java) + val cookie = addAssetPath.invoke(am, pluginApkPath) as? Int ?: 0 + if (cookie == 0) { + throw IllegalStateException("addAssetPath returned 0 for $pluginApkPath") + } + return am + } + + private fun removePluginTier3Internal( + db: SQLiteDatabase, + pluginId: String, + ): Int { + val prefix = "plugin/$pluginId" + return db.delete( + "Content", + "path = ? OR path LIKE ? ESCAPE '\\'", + arrayOf(prefix, "${escapeLike(prefix)}/%"), + ) + } + + private fun insertContentChunked( + db: SQLiteDatabase, + basePath: String, + payload: ByteArray, + contentTypeId: Long, + ) { + val chunkSize = 1024 * 1024 + if (payload.size < chunkSize) { + insertContentRow(db, basePath, payload, contentTypeId) + return + } + + var offset = 0 + var fragment = 0 + while (offset < payload.size) { + val end = minOf(offset + chunkSize, payload.size) + val slice = payload.copyOfRange(offset, end) + val path = if (fragment == 0) basePath else "$basePath-$fragment" + insertContentRow(db, path, slice, contentTypeId) + offset = end + fragment++ + } + if (payload.size % chunkSize == 0) { + insertContentRow(db, "$basePath-$fragment", ByteArray(0), contentTypeId) + } + } + + private fun insertContentRow( + db: SQLiteDatabase, + path: String, + blob: ByteArray, + contentTypeId: Long, + ) { + val values = + ContentValues().apply { + put("path", path) + put("content", blob) + put("contentTypeID", contentTypeId) + put("languageId", 1) + } + db.insertOrThrow("Content", null, values) + } + + private fun removePluginDocumentationInternal( + db: SQLiteDatabase, + pluginId: String, + ) { + val category = pluginCategory(pluginId) + + val cursor = + db.rawQuery( + """ + SELECT T.id FROM Tooltips AS T + INNER JOIN TooltipCategories AS TC ON T.categoryId = TC.id + WHERE TC.category = ? + """.trimIndent(), + arrayOf(category), + ) + + val tooltipIds = mutableListOf() + while (cursor.moveToNext()) { + tooltipIds.add(cursor.getLong(0)) + } + cursor.close() + + if (tooltipIds.isNotEmpty()) { + val placeholders = tooltipIds.joinToString(",") { "?" } + val args = tooltipIds.map { it.toString() }.toTypedArray() + db.delete("TooltipButtons", "tooltipId IN ($placeholders)", args) + db.delete("Tooltips", "id IN ($placeholders)", args) + } + + db.delete("TooltipCategories", "category = ?", arrayOf(category)) + } + + private fun insertOrGetCategoryId( + db: SQLiteDatabase, + category: String, + ): Long { + val cursor = + db.query( + "TooltipCategories", + arrayOf("id"), + "category = ?", + arrayOf(category), + null, + null, + null, + ) + + if (cursor.moveToFirst()) { + val id = cursor.getLong(0) + cursor.close() + return id + } + cursor.close() + + val values = + ContentValues().apply { + put("category", category) + } + return db.insert("TooltipCategories", null, values) + } + + private fun insertTooltip( + db: SQLiteDatabase, + categoryId: Long, + entry: PluginTooltipEntry, + ): Long { + val disclaimer = context.getString(R.string.plugin_documentation_third_party_disclaimer) + + val existingCursor = + db.query( + "Tooltips", + arrayOf("id"), + "categoryId = ? AND tag = ?", + arrayOf(categoryId.toString(), entry.tag), + null, + null, + null, + ) + + if (existingCursor.moveToFirst()) { + val existingId = existingCursor.getLong(0) + existingCursor.close() + + val updateValues = + ContentValues().apply { + put("summary", entry.summary + disclaimer) + put("detail", if (entry.detail.isNotBlank()) entry.detail + disclaimer else "") + } + db.update("Tooltips", updateValues, "id = ?", arrayOf(existingId.toString())) + db.delete("TooltipButtons", "tooltipId = ?", arrayOf(existingId.toString())) + return existingId + } + existingCursor.close() + + val values = + ContentValues().apply { + put("categoryId", categoryId) + put("tag", entry.tag) + put("summary", entry.summary + disclaimer) + put("detail", if (entry.detail.isNotBlank()) entry.detail + disclaimer else "") + } + return db.insert("Tooltips", null, values) + } + + private fun escapeLike(value: String): String = + value + .replace("\\", "\\\\") + .replace("%", "\\%") + .replace("_", "\\_") + + private fun normalizeLocalDocumentationPath(path: String): String { + val segments = path.split('/').filter { it.isNotEmpty() && it != "." } + require(segments.none { it == ".." }) { + "Documentation paths must not contain '..' segments: $path" + } + return segments.joinToString("/") + } + + private fun resolvePluginButtonUri( + pluginId: String, + rawUri: String, + directPath: Boolean, + ): String { + if (rawUri.isEmpty()) return rawUri + if (rawUri.contains("://")) return rawUri + val absolute = directPath || rawUri.startsWith("/") + val normalized = normalizeLocalDocumentationPath(rawUri.trimStart('/')) + return if (absolute) normalized else "plugin/$pluginId/$normalized" + } + + private fun insertTooltipButton( + db: SQLiteDatabase, + tooltipId: Long, + description: String, + uri: String, + order: Int, + ) { + val values = + ContentValues().apply { + put("tooltipId", tooltipId) + put("description", description) + put("uri", uri) + put("buttonNumberId", order) + } + db.insert("TooltipButtons", null, values) + } + + /** + * Check if the plugin documentation database is accessible. + */ + suspend fun isDatabaseAvailable(): Boolean = + withContext(Dispatchers.IO) { + context.getDatabasePath(databaseName).exists() + } + + /** + * Check if documentation for a specific plugin exists in documentation.db. + */ + suspend fun isPluginDocumentationInstalled(pluginId: String): Boolean = + withContext(Dispatchers.IO) { + val db = getPluginDatabase() ?: return@withContext false + + try { + val cursor = + db.rawQuery( + "SELECT COUNT(*) FROM TooltipCategories WHERE category = ?", + arrayOf(pluginCategory(pluginId)), + ) + val installed = cursor.moveToFirst() && cursor.getInt(0) > 0 + cursor.close() + installed + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to check plugin documentation for $pluginId", e) + false + } finally { + db.close() + } + } + + /** + * Verify and recreate plugin documentation if missing. + */ + suspend fun verifyAndRecreateDocumentation( + pluginId: String, + plugin: DocumentationExtension, + ): Boolean = + withContext(Dispatchers.IO) { + if (!isDatabaseAvailable()) { + Log.d(TAG, "documentation.db not available yet for $pluginId, skipping") + return@withContext false + } + + if (!isPluginDocumentationInstalled(pluginId)) { + Log.d(TAG, "Plugin documentation missing for $pluginId, recreating...") + return@withContext installPluginDocumentation(pluginId, plugin) + } + + Log.d(TAG, "Plugin documentation already exists for $pluginId") + true + } + + /** + * Verify and recreate documentation for all plugins that support it. + */ + suspend fun verifyAllPluginDocumentation(plugins: Map): Int = + withContext(Dispatchers.IO) { + if (plugins.isEmpty()) return@withContext 0 + + if (!isDatabaseAvailable()) { + Log.d(TAG, "documentation.db not available yet, skipping verification") + return@withContext 0 + } + + var recreatedCount = 0 + + for ((pluginId, plugin) in plugins) { + try { + if (!isPluginDocumentationInstalled(pluginId)) { + Log.d(TAG, "Recreating missing documentation for plugin: $pluginId") + if (installPluginDocumentation(pluginId, plugin)) { + recreatedCount++ + } + } + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to verify/recreate documentation for $pluginId", e) + } + } + + if (recreatedCount > 0) { + Log.i(TAG, "Recreated documentation for $recreatedCount plugins") + } + + recreatedCount + } } diff --git a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/Tier3AssetWalker.kt b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/Tier3AssetWalker.kt index 6159e65ea6..f915df355f 100644 --- a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/Tier3AssetWalker.kt +++ b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/Tier3AssetWalker.kt @@ -7,77 +7,85 @@ import java.io.IOException import java.io.InputStream internal data class Tier3Asset( - val relativePath: String, - val bytes: ByteArray + val relativePath: String, + val bytes: ByteArray, ) { - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (javaClass != other?.javaClass) return false + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false - other as Tier3Asset + other as Tier3Asset - if (relativePath != other.relativePath) return false - if (!bytes.contentEquals(other.bytes)) return false + if (relativePath != other.relativePath) return false + if (!bytes.contentEquals(other.bytes)) return false - return true - } + return true + } - override fun hashCode(): Int { - var result = relativePath.hashCode() - result = 31 * result + bytes.contentHashCode() - return result - } + override fun hashCode(): Int { + var result = relativePath.hashCode() + result = 31 * result + bytes.contentHashCode() + return result + } } internal object Tier3AssetWalker { + private const val TAG = "Tier3AssetWalker" + private const val MAX_ASSET_BYTES = 10L * 1024L * 1024L - private const val TAG = "Tier3AssetWalker" - private const val MAX_ASSET_BYTES = 10L * 1024L * 1024L + fun walk( + assets: AssetManager, + rootAssetPath: String, + ): Sequence = + sequence { + val root = rootAssetPath.trim('/') + yieldAll(walkDir(assets, root, relative = "")) + } - fun walk(assets: AssetManager, rootAssetPath: String): Sequence = sequence { - val root = rootAssetPath.trim('/') - yieldAll(walkDir(assets, root, relative = "")) - } + private fun walkDir( + assets: AssetManager, + absolute: String, + relative: String, + ): Sequence = + sequence { + val children = assets.list(absolute) ?: emptyArray() + if (children.isEmpty()) { + val bytes = + try { + assets.open(absolute).use { readBounded(it, MAX_ASSET_BYTES) } + } catch (_: IOException) { + return@sequence + } + if (bytes == null) { + Log.w(TAG, "Skipping Tier 3 asset '$absolute' — exceeds $MAX_ASSET_BYTES byte limit") + return@sequence + } + if (relative.isNotEmpty()) { + yield(Tier3Asset(relative, bytes)) + } + return@sequence + } + for (child in children) { + val childAbs = if (absolute.isEmpty()) child else "$absolute/$child" + val childRel = if (relative.isEmpty()) child else "$relative/$child" + yieldAll(walkDir(assets, childAbs, childRel)) + } + } - private fun walkDir( - assets: AssetManager, - absolute: String, - relative: String - ): Sequence = sequence { - val children = assets.list(absolute) ?: emptyArray() - if (children.isEmpty()) { - val bytes = try { - assets.open(absolute).use { readBounded(it, MAX_ASSET_BYTES) } - } catch (_: IOException) { - return@sequence - } - if (bytes == null) { - Log.w(TAG, "Skipping Tier 3 asset '$absolute' — exceeds $MAX_ASSET_BYTES byte limit") - return@sequence - } - if (relative.isNotEmpty()) { - yield(Tier3Asset(relative, bytes)) - } - return@sequence - } - for (child in children) { - val childAbs = if (absolute.isEmpty()) child else "$absolute/$child" - val childRel = if (relative.isEmpty()) child else "$relative/$child" - yieldAll(walkDir(assets, childAbs, childRel)) - } - } - - private fun readBounded(stream: InputStream, limit: Long): ByteArray? { - val buffer = ByteArrayOutputStream() - val tmp = ByteArray(64 * 1024) - var total = 0L - while (true) { - val n = stream.read(tmp) - if (n < 0) break - total += n - if (total > limit) return null - buffer.write(tmp, 0, n) - } - return buffer.toByteArray() - } + private fun readBounded( + stream: InputStream, + limit: Long, + ): ByteArray? { + val buffer = ByteArrayOutputStream() + val tmp = ByteArray(64 * 1024) + var total = 0L + while (true) { + val n = stream.read(tmp) + if (n < 0) break + total += n + if (total > limit) return null + buffer.write(tmp, 0, n) + } + return buffer.toByteArray() + } } From edb5af2812d419fab6844088694aad0abcdab5a5 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Thu, 27 Aug 2026 17:40:10 -0700 Subject: [PATCH 02/15] ADFA-5240: Move the compression-dictionary loader into :common WebServer decided on its own whether a database's brotli Content rows carry the shared dictionary. PluginDocumentationManager, in another module, is about to need the identical decision when it writes those rows -- and a writer that disagrees with the reader produces content nothing can decode. Move loadCompressionDictionary and toDirectByteBuffer to :common, next to the DatabaseVersionResolver they gate on, so both sides read the one implementation. Pure move: the version gate, the CompressionDictionary checks and the throw-vs-null contract are unchanged, as is WebServer's retry. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01M2Bzxv38NbXWPMQVckoSNC --- .../androidide/localWebServer/WebServer.kt | 80 +---------------- .../BrotliDictionaryDecodeTest.kt | 1 + .../utils/DocumentationCompression.kt | 90 +++++++++++++++++++ 3 files changed, 93 insertions(+), 78 deletions(-) create mode 100644 common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt diff --git a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt index d374f351e4..2812a61bed 100644 --- a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt +++ b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt @@ -13,6 +13,8 @@ import com.google.gson.annotations.SerializedName import com.google.gson.reflect.TypeToken import com.itsaky.androidide.utils.ContentTypeHeaders import com.itsaky.androidide.utils.DatabaseVersionResolver +import com.itsaky.androidide.utils.loadCompressionDictionary +import com.itsaky.androidide.utils.toDirectByteBuffer import io.pebbletemplates.pebble.PebbleEngine import io.pebbletemplates.pebble.loader.StringLoader import io.pebbletemplates.pebble.template.PebbleTemplate @@ -104,20 +106,6 @@ data class JavaExecutionResult( val timeoutLimit: Long, ) -/** - * Copies [bytes] into a direct [ByteBuffer] -- brotli4j's `attachDictionary` requires a direct - * buffer, a heap-backed one throws `IllegalArgumentException`. - * - * The capacity must be exactly [bytes]`.size`: `attachDictionary` reads the whole capacity and - * ignores position/limit, so trailing slack from an over-allocated buffer is treated as dictionary - * content and every decode then fails with `IOException: corrupted input`. - */ -internal fun toDirectByteBuffer(bytes: ByteArray): ByteBuffer = - ByteBuffer.allocateDirect(bytes.size).apply { - put(bytes) - flip() - } - /** * Reads [chunks] back to back as one stream, without concatenating them into a new array. * Cheap to build twice, which the no-dictionary retry in `decompressBrotli` relies on. @@ -237,70 +225,6 @@ class WebServer( } } - /** - * Loads the shared Brotli dictionary most Content rows are compressed against (see ADFA-5153). - * Returns null (logged) when the database *definitively* has no dictionary -- so callers fall - * back to plain, dictionary-free brotli decode (see [decompressBrotli]). - * - * The gate is the MAJOR version the database declares in ADFA-5220's version table, not the - * presence of a `CompressionDictionary` table: table sniffing infers a whole content format - * from one table's existence, and gets it wrong in both directions -- a database carrying the - * table but *unmigrated* content makes every plain row pay a failed dictionary decode before - * its plain one, on every request. Below - * [DatabaseVersionResolver.MAJOR_VERSION_WITH_COMPRESSION_DICTIONARY] the dictionary is neither - * read nor attached. - * - * The `CompressionDictionary` checks below still run, for a database that declares a new-enough - * version but has no usable dictionary row: without them the data query would raise "no such - * table", which the caller correctly reads as transient and would then retry on every request. - * - * Deliberately does *not* catch exceptions itself: an unexpected `SQLiteException`/IO failure is - * likely transient, and the caller (see [handleClient]) must not cache that as "no dictionary" - * the way it does a definitive absence, or a transient failure would permanently disable - * dictionary decoding for the rest of this database's lifetime. - */ - private fun loadCompressionDictionary(db: SQLiteDatabase): ByteBuffer? { - val majorVersion = DatabaseVersionResolver.resolveMajorVersion(db) - if (majorVersion == null || majorVersion < DatabaseVersionResolver.MAJOR_VERSION_WITH_COMPRESSION_DICTIONARY) { - log.warn( - "Database declares documentation version {}, below {}; decoding brotli content without a dictionary.", - majorVersion ?: "none", - DatabaseVersionResolver.MAJOR_VERSION_WITH_COMPRESSION_DICTIONARY, - ) - return null - } - - val tableExists = - db - .rawQuery( - "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'CompressionDictionary'", - null, - ).use { it.moveToFirst() } - if (!tableExists) { - log.warn("CompressionDictionary table not found; decoding brotli content without a dictionary.") - return null - } - - return db.rawQuery("SELECT data FROM CompressionDictionary WHERE id = 1", null).use { cursor -> - if (!cursor.moveToFirst()) { - log.warn("CompressionDictionary table is empty; decoding brotli content without a dictionary.") - return null - } - val bytes = cursor.getBlob(0) - if (bytes == null) { - log.warn("CompressionDictionary row has a NULL data column; decoding brotli content without a dictionary.") - return null - } - // An empty blob would yield a 0-capacity buffer, which attachDictionary rejects -- - // every row's dictionary decode would then fail with nothing above DEBUG to say why. - if (bytes.isEmpty()) { - log.warn("CompressionDictionary row has an empty data column; decoding brotli content without a dictionary.") - return null - } - toDirectByteBuffer(bytes) - } - } - /** * Opens [path] as the active database, refreshing every piece of state that depends on which * database file is active -- [databaseTimestamp] and the per-database caches diff --git a/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt index 80a1ac152c..6716531564 100644 --- a/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt +++ b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt @@ -4,6 +4,7 @@ import com.aayushatharva.brotli4j.Brotli4jLoader import com.aayushatharva.brotli4j.decoder.BrotliInputStream import com.aayushatharva.brotli4j.encoder.BrotliOutputStream import com.aayushatharva.brotli4j.encoder.Encoder +import com.itsaky.androidide.utils.toDirectByteBuffer import org.junit.Assert.assertArrayEquals import org.junit.Assert.assertEquals import org.junit.Assert.assertSame diff --git a/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt b/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt new file mode 100644 index 0000000000..7284810f9a --- /dev/null +++ b/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt @@ -0,0 +1,90 @@ +package com.itsaky.androidide.utils + +import android.database.sqlite.SQLiteDatabase +import android.util.Log +import java.nio.ByteBuffer + +private const val TAG = "DocumentationCompression" + +/** + * Copies [bytes] into a direct [ByteBuffer] -- brotli4j's `attachDictionary` requires a direct + * buffer, a heap-backed one throws `IllegalArgumentException`. + * + * The capacity must be exactly [bytes]`.size`: `attachDictionary` reads the whole capacity and + * ignores position/limit, so trailing slack from an over-allocated buffer is treated as dictionary + * content and every decode then fails with `IOException: corrupted input`. + */ +fun toDirectByteBuffer(bytes: ByteArray): ByteBuffer = + ByteBuffer.allocateDirect(bytes.size).apply { + put(bytes) + flip() + } + +/** + * Loads the shared Brotli dictionary every `compression = 'brotli'` Content row in [db] is + * compressed against (see ADFA-5153). Returns null (logged) when the database *definitively* has + * no dictionary, which tells callers to read and write plain, dictionary-free brotli instead. + * + * Both the reader and the writer of `Content` call this, so the two cannot disagree about whether + * a given database's brotli rows carry a dictionary -- which is the whole point: a row compressed + * against a dictionary the reader does not attach is undecodable, and vice versa. + * + * The gate is the MAJOR version the database declares in ADFA-5220's version table, not the + * presence of a `CompressionDictionary` table: table sniffing infers a whole content format from + * one table's existence, and gets it wrong in both directions -- a database carrying the table but + * *unmigrated* content would have every plain row read as dictionary-compressed. Below + * [DatabaseVersionResolver.MAJOR_VERSION_WITH_COMPRESSION_DICTIONARY] the dictionary is neither + * read nor attached. + * + * The `CompressionDictionary` checks below still run, for a database that declares a new-enough + * version but has no usable dictionary row: without them the data query would raise "no such + * table", which callers correctly read as transient and would then retry forever. + * + * Deliberately does *not* catch exceptions itself: an unexpected `SQLiteException`/IO failure is + * likely transient, and callers must be able to tell that apart from a definitive absence. Caching + * a transient failure as "no dictionary" would permanently disable dictionary handling for the + * rest of this database's lifetime; writing plain rows because of one would corrupt content the + * reader can never decode. + */ +fun loadCompressionDictionary(db: SQLiteDatabase): ByteBuffer? { + val majorVersion = DatabaseVersionResolver.resolveMajorVersion(db) + if (majorVersion == null || majorVersion < DatabaseVersionResolver.MAJOR_VERSION_WITH_COMPRESSION_DICTIONARY) { + Log.w( + TAG, + "Database declares documentation version ${majorVersion ?: "none"}, below " + + "${DatabaseVersionResolver.MAJOR_VERSION_WITH_COMPRESSION_DICTIONARY}; " + + "brotli content is handled without a dictionary.", + ) + return null + } + + val tableExists = + db + .rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'CompressionDictionary'", + null, + ).use { it.moveToFirst() } + if (!tableExists) { + Log.w(TAG, "CompressionDictionary table not found; brotli content is handled without a dictionary.") + return null + } + + return db.rawQuery("SELECT data FROM CompressionDictionary WHERE id = 1", null).use { cursor -> + if (!cursor.moveToFirst()) { + Log.w(TAG, "CompressionDictionary table is empty; brotli content is handled without a dictionary.") + return null + } + val bytes = cursor.getBlob(0) + if (bytes == null) { + Log.w(TAG, "CompressionDictionary row has a NULL data column; brotli content is handled without a dictionary.") + return null + } + // An empty blob would yield a 0-capacity buffer, which attachDictionary rejects -- + // every row's dictionary decode would then fail with nothing above DEBUG to say why. + if (bytes.isEmpty()) { + Log.w(TAG, "CompressionDictionary row has an empty data column; brotli content is handled without a dictionary.") + return null + } + toDirectByteBuffer(bytes) + } +} From 84772f0221d1ee6632fb8f0df725a099c2390311 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Thu, 27 Aug 2026 17:43:58 -0700 Subject: [PATCH 03/15] ADFA-5240: Compress plugin Tier 3 content with the shared dictionary Plugin-contributed Content rows were plain brotli while every other brotli row in the same table was compressed against the database's CompressionDictionary. WebServer could only tell the two apart by attempting a dictionary decode and catching the failure -- behavior that is documented nowhere in the brotli spec. Compress them the same way instead. BrotliDictionaryCodec (in :common, beside the loader) prepares the dictionary once per install and reuses it across the plugin's assets. Same quality 11 / window 24 as the offline pipeline, so a row written on-device is indistinguishable in size from one built ahead of time. When the dictionary cannot be read the install is abandoned rather than written plain: guessing produces rows WebServer cannot decode, and verifyAndRecreateTier3Documentation retries on the next activation. A database that declares no dictionary still gets plain brotli, which is what its reader expects. brotli4j drops out of plugin-manager's dependencies with BrotliCompressor. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01M2Bzxv38NbXWPMQVckoSNC --- .../BrotliDictionaryDecodeTest.kt | 77 +++++++++++++---- .../utils/DocumentationCompression.kt | 83 +++++++++++++++++++ plugin-manager/build.gradle.kts | 1 - .../manager/documentation/BrotliCompressor.kt | 19 ----- .../PluginDocumentationManager.kt | 26 +++++- 5 files changed, 167 insertions(+), 39 deletions(-) delete mode 100644 plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/BrotliCompressor.kt diff --git a/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt index 6716531564..a79d64063c 100644 --- a/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt +++ b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt @@ -2,8 +2,7 @@ package com.itsaky.androidide.localWebServer import com.aayushatharva.brotli4j.Brotli4jLoader import com.aayushatharva.brotli4j.decoder.BrotliInputStream -import com.aayushatharva.brotli4j.encoder.BrotliOutputStream -import com.aayushatharva.brotli4j.encoder.Encoder +import com.itsaky.androidide.utils.BrotliDictionaryCodec import com.itsaky.androidide.utils.toDirectByteBuffer import org.junit.Assert.assertArrayEquals import org.junit.Assert.assertEquals @@ -12,7 +11,6 @@ import org.junit.Assert.assertThrows import org.junit.BeforeClass import org.junit.Test import java.io.ByteArrayInputStream -import java.io.ByteArrayOutputStream import java.io.IOException import java.nio.ByteBuffer import java.nio.charset.StandardCharsets @@ -181,29 +179,72 @@ class BrotliDictionaryDecodeTest { } @Test - fun `dictionary-free plugin content fails with a dictionary attached but decodes plain`() { - // Regression coverage for the WebServer.decompressBrotli fallback: plugin-contributed - // Tier 3 docs (PluginDocumentationManager/BrotliCompressor) are compressed with the same - // encoder params (quality 11, window 24) but no dictionary, coexisting in the same Content - // table as ADFA-5153-migrated, dictionary-compressed rows. + fun `plugin content compressed against the dictionary decodes the way WebServer reads it`() { + // The ADFA-5240 contract: PluginDocumentationManager encodes Tier 3 rows through + // BrotliDictionaryCodec, and WebServer reads every brotli row by attaching the same + // dictionary. The dictionary here is the CLI-trained fixture, so this also covers the + // cross-tool half -- brotli4j's encoder against a dictionary the Python pipeline produced. val dictionary = decodeBase64ToDirectBuffer(dictionaryBase64) - val plaintext = "plugin-contributed Tier 3 content, compressed with no dictionary" - val expected = plaintext.toByteArray(StandardCharsets.UTF_8) - val compressed = - ByteArrayOutputStream() - .apply { - BrotliOutputStream(this, Encoder.Parameters().setQuality(11).setWindow(24)).use { it.write(expected) } - }.toByteArray() + val expected = "plugin-contributed Tier 3 content, compressed against the shared dictionary".toByteArray(StandardCharsets.UTF_8) - assertThrows(IOException::class.java) { + val compressed = BrotliDictionaryCodec(dictionary).compress(expected) + + val result = BrotliInputStream(ByteArrayInputStream(compressed)).use { stream -> stream.attachDictionary(dictionary) stream.readBytes() } + assertArrayEquals(expected, result) + } + + @Test + fun `the codec reuses one dictionary buffer across compress and decompress`() { + // One codec serves a whole plugin install, so preparing the dictionary for the encoder + // must not disturb the buffer the decoder side attaches (`generate` drains its argument's + // position, hence the defensive duplicate in BrotliDictionaryCodec). + val dictionary = decodeBase64ToDirectBuffer(dictionaryBase64) + val codec = BrotliDictionaryCodec(dictionary) + + repeat(3) { round -> + val expected = "round $round: docs-sidebar toc-element kotlin interface".toByteArray(StandardCharsets.UTF_8) + val result = codec.decompress(ByteArrayInputStream(codec.compress(expected))) + assertArrayEquals(expected, result) } - val plainResult = BrotliInputStream(ByteArrayInputStream(compressed)).use { it.readBytes() } - assertArrayEquals(expected, plainResult) + // The fixture from the offline pipeline must still decode through the same buffer. + assertArrayEquals( + Base64.getDecoder().decode(expectedBase64), + codec.decompress(ByteArrayInputStream(Base64.getDecoder().decode(compressedBase64))), + ) + } + + @Test + fun `dictionary-compressed plugin content is not readable without the dictionary`() { + // WebServer decodes a brotli row exactly once, with the dictionary attached or not + // according to the database's declared version -- there is no retry to paper over a + // mismatch. This is what makes that safe: a row written against the dictionary cannot be + // silently misread as a plain one, it fails loudly. + val dictionary = decodeBase64ToDirectBuffer(dictionaryBase64) + val expected = "plugin-contributed Tier 3 content".toByteArray(StandardCharsets.UTF_8) + + val compressed = BrotliDictionaryCodec(dictionary).compress(expected) + + assertThrows(IOException::class.java) { + BrotliDictionaryCodec(null).decompress(ByteArrayInputStream(compressed)) + } + } + + @Test + fun `a null dictionary round-trips as plain brotli`() { + // A database predating ADFA-5153 declares no dictionary, so both sides fall to plain + // brotli -- the writer must not attach one there either. + val expected = "content for a database with no CompressionDictionary".toByteArray(StandardCharsets.UTF_8) + val codec = BrotliDictionaryCodec(null) + + val compressed = codec.compress(expected) + + assertArrayEquals(expected, codec.decompress(ByteArrayInputStream(compressed))) + assertArrayEquals(expected, BrotliInputStream(ByteArrayInputStream(compressed)).use { it.readBytes() }) } @Test diff --git a/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt b/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt index 7284810f9a..7ee1ff33cf 100644 --- a/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt +++ b/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt @@ -2,6 +2,15 @@ package com.itsaky.androidide.utils import android.database.sqlite.SQLiteDatabase import android.util.Log +import com.aayushatharva.brotli4j.Brotli4jLoader +import com.aayushatharva.brotli4j.decoder.BrotliInputStream +import com.aayushatharva.brotli4j.encoder.BrotliOutputStream +import com.aayushatharva.brotli4j.encoder.Encoder +import com.aayushatharva.brotli4j.encoder.PreparedDictionary +import com.aayushatharva.brotli4j.encoder.PreparedDictionaryGenerator +import java.io.ByteArrayOutputStream +import java.io.IOException +import java.io.InputStream import java.nio.ByteBuffer private const val TAG = "DocumentationCompression" @@ -88,3 +97,77 @@ fun loadCompressionDictionary(db: SQLiteDatabase): ByteBuffer? { toDirectByteBuffer(bytes) } } + +/** + * Compresses and decompresses documentation `Content` blobs against [dictionary], the shared Brotli + * dictionary from the database those blobs live in (see [loadCompressionDictionary]). A null + * [dictionary] means plain, dictionary-free brotli, which is what a database predating ADFA-5153 + * needs. + * + * One instance per database, reused across rows: preparing the dictionary for the encoder costs a + * few milliseconds and the result is safe to share across streams. + */ +class BrotliDictionaryCodec( + private val dictionary: ByteBuffer?, +) { + // Only the encoder needs the dictionary in prepared form, so a decode-only user (WebServer) + // never pays for building it. `generate` advances the buffer's position to its limit, so it + // gets a duplicate: the original is shared with attachDictionary, which reads the whole + // capacity and would be unaffected, but leaving a caller's long-lived buffer drained is a trap + // for the next reader of it. + private val preparedDictionary: PreparedDictionary? by lazy { + dictionary?.let { PreparedDictionaryGenerator.generate(it.duplicate()) } + } + + /** + * Compresses [input] at the quality and window size the offline documentation pipeline uses, + * so content contributed at runtime is stored the same way as content built ahead of time. + */ + fun compress(input: ByteArray): ByteArray { + ensureBrotliAvailable() + val out = ByteArrayOutputStream(input.size) + BrotliOutputStream(out, encoderParameters).use { stream -> + preparedDictionary?.let { stream.attachDictionary(it) } + stream.write(input) + } + return out.toByteArray() + } + + /** + * Decompresses a `Content` blob read from the same database [dictionary] came from. + * + * Throws `IOException` when [input] was not compressed the way that database's rows are -- + * a dictionary-compressed stream decoded without one leaves backward distances out of bounds, + * which any spec-compliant decoder rejects. Note the converse is *not* detectable: attaching + * the *wrong* dictionary decodes without error to different bytes than were compressed, so + * decode success is never evidence that the right dictionary was used. + */ + fun decompress(input: InputStream): ByteArray { + ensureBrotliAvailable() + return BrotliInputStream(input).use { stream -> + dictionary?.let { stream.attachDictionary(it) } + stream.readBytes() + } + } + + /** + * Loads brotli4j's native library if nothing else has yet, and turns its absence into a failed + * request rather than an `Error` that would take down the calling thread. + */ + private fun ensureBrotliAvailable() { + try { + Brotli4jLoader.ensureAvailability() + } catch (e: UnsatisfiedLinkError) { + throw IOException("brotli4j's native library is unavailable, so brotli content cannot be handled", e) + } + } + + companion object { + // Matches OfflineDocumentationTools' encode settings, so a row written on-device is + // indistinguishable in size and decodability from one built by that pipeline. + private val encoderParameters: Encoder.Parameters by lazy { + Brotli4jLoader.ensureAvailability() + Encoder.Parameters().setQuality(11).setWindow(24) + } + } +} diff --git a/plugin-manager/build.gradle.kts b/plugin-manager/build.gradle.kts index 13462875e2..e75526fe2d 100644 --- a/plugin-manager/build.gradle.kts +++ b/plugin-manager/build.gradle.kts @@ -34,7 +34,6 @@ dependencies { implementation(libs.androidx.appcompat) implementation(libs.gson.v2101) - implementation(libs.brotli4j) implementation(libs.commons.compress) implementation(libs.tukaani.xz) diff --git a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/BrotliCompressor.kt b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/BrotliCompressor.kt deleted file mode 100644 index 7be3704192..0000000000 --- a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/BrotliCompressor.kt +++ /dev/null @@ -1,19 +0,0 @@ -package com.itsaky.androidide.plugins.manager.documentation - -import com.aayushatharva.brotli4j.Brotli4jLoader -import com.aayushatharva.brotli4j.encoder.BrotliOutputStream -import com.aayushatharva.brotli4j.encoder.Encoder -import java.io.ByteArrayOutputStream - -internal object BrotliCompressor { - private val params: Encoder.Parameters by lazy { - Brotli4jLoader.ensureAvailability() - Encoder.Parameters().setQuality(11).setWindow(24) - } - - fun compress(input: ByteArray): ByteArray { - val out = ByteArrayOutputStream(input.size) - BrotliOutputStream(out, params).use { it.write(input) } - return out.toByteArray() - } -} diff --git a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt index a2b0249fb9..4a66ec7ca7 100644 --- a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt +++ b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt @@ -9,6 +9,8 @@ import com.itsaky.androidide.plugins.extensions.DocumentationExtension import com.itsaky.androidide.plugins.extensions.PluginTooltipEntry import com.itsaky.androidide.plugins.manager.pluginCategory import com.itsaky.androidide.resources.R +import com.itsaky.androidide.utils.BrotliDictionaryCodec +import com.itsaky.androidide.utils.loadCompressionDictionary import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import java.io.File @@ -162,6 +164,13 @@ class PluginDocumentationManager( * the existing ContentTypes.compression column, chunks blobs at 1 MB to * match WebServer's read loop, and inserts everything under the reserved * path namespace "plugin//..." inside a single transaction. + * + * Brotli assets are compressed against the database's own shared dictionary + * (ADFA-5240), so every brotli row in Content -- contributed here or built + * offline -- decodes the same way. The dictionary is read from the database + * being written, which keeps content and dictionary version-locked: replacing + * documentation.db drops these rows, and they are reinstalled against the new + * file's dictionary. */ suspend fun installPluginTier3Documentation( pluginId: String, @@ -190,6 +199,21 @@ class PluginDocumentationManager( return@withContext false } + // A definitive null means this database has no dictionary, so its brotli rows are plain + // and these must be too. A throw means the answer is merely unavailable right now: + // guessing either way writes rows WebServer cannot decode, so abandon the install and + // let verifyAndRecreateTier3Documentation retry on the next activation. + val codec = + try { + BrotliDictionaryCodec(loadCompressionDictionary(db)) + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Cannot read the compression dictionary; deferring Tier 3 install for $pluginId", e) + db.close() + pluginAssets.close() + return@withContext false + } + val resolver = ExtensionToContentTypeResolver() var inserted = 0 var skipped = 0 @@ -214,7 +238,7 @@ class PluginDocumentationManager( val payload = if (row.compression == "brotli") { - BrotliCompressor.compress(asset.bytes) + codec.compress(asset.bytes) } else { asset.bytes } From 3436417878c5fe130fb7b2b81fa12445dcadb62a Mon Sep 17 00:00:00 2001 From: David Schachter Date: Thu, 27 Aug 2026 17:45:05 -0700 Subject: [PATCH 04/15] ADFA-5240: Reinstall Tier 3 rows left plain by an earlier build The verify step reinstalled only when a plugin's rows were missing, so an app upgrade over an unchanged documentation.db would leave plain brotli rows behind -- unreadable once WebServer stops retrying without the dictionary. Track the compression generation each plugin's rows were written at and reinstall when it is behind. The rows cannot say this themselves: the schema belongs to OfflineDocumentationTools and has no column for it, and probing by decode is the guesswork this ticket removes. Only reachable when documentation.db survives an upgrade. Replacing that file drops every plugin row with it, which the existing missing-rows check already handles. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01M2Bzxv38NbXWPMQVckoSNC --- .../PluginDocumentationManager.kt | 48 +++++++++++++++++-- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt index 4a66ec7ca7..c03b2325df 100644 --- a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt +++ b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt @@ -2,6 +2,7 @@ package com.itsaky.androidide.plugins.manager.documentation import android.content.ContentValues import android.content.Context +import android.content.SharedPreferences import android.content.res.AssetManager import android.database.sqlite.SQLiteDatabase import android.util.Log @@ -27,6 +28,12 @@ class PluginDocumentationManager( ) { companion object { private const val TAG = "PluginDocManager" + + private const val TIER3_PREFS = "plugin_tier3_docs" + + // Bumped whenever previously written Tier 3 rows can no longer be read as they are. + // 1 was plain brotli; 2 compresses against the database's shared dictionary (ADFA-5240). + private const val TIER3_COMPRESSION_GENERATION = 2 } private val databaseName = "documentation.db" @@ -280,6 +287,7 @@ class PluginDocumentationManager( try { val deleted = removePluginTier3Internal(db, pluginId) db.setTransactionSuccessful() + forgetInstalledGeneration(pluginId) Log.d(TAG, "Removed $deleted Tier 3 rows for plugin $pluginId") true } catch (e: Exception) { @@ -293,8 +301,9 @@ class PluginDocumentationManager( } /** - * Verify that Tier 3 content exists for this plugin; reinstall if missing. - * Mirrors [verifyAndRecreateDocumentation] for the Tier 1/2 pipeline. + * Verify that Tier 3 content exists for this plugin, and was written the way the current + * build reads it; reinstall if either is untrue. Mirrors [verifyAndRecreateDocumentation] + * for the Tier 1/2 pipeline. */ suspend fun verifyAndRecreateTier3Documentation( pluginId: String, @@ -309,14 +318,43 @@ class PluginDocumentationManager( Log.d(TAG, "documentation.db not available yet for Tier 3 verify of $pluginId") return@withContext false } - if (isPluginTier3DocumentationInstalled(pluginId)) { + if (isPluginTier3DocumentationInstalled(pluginId) && installedGeneration(pluginId) == TIER3_COMPRESSION_GENERATION) { Log.d(TAG, "Tier 3 docs already present for $pluginId") return@withContext true } - Log.d(TAG, "Tier 3 docs missing for $pluginId, installing...") - installPluginTier3Documentation(pluginId, plugin, pluginApkPath) + Log.d(TAG, "Tier 3 docs missing or stale for $pluginId, installing...") + val installed = installPluginTier3Documentation(pluginId, plugin, pluginApkPath) + if (installed) { + recordInstalledGeneration(pluginId) + } + installed } + /** + * The compression generation [pluginId]'s Tier 3 rows were last written at, or 0 for rows + * this build has never written. + * + * Rows written before [TIER3_COMPRESSION_GENERATION] are plain brotli, which WebServer no + * longer accepts from a database that declares a dictionary. They are not detectable from the + * rows themselves -- the schema is owned by OfflineDocumentationTools and has no column to + * mark them with, and probing by decode is exactly the guesswork ADFA-5240 removes -- so the + * generation is tracked here instead. + * + * Only needed when documentation.db survives an app upgrade. Replacing that file drops every + * plugin row with it, and the missing-rows check above already covers that case. + */ + private fun installedGeneration(pluginId: String): Int = tier3Preferences().getInt(pluginId, 0) + + private fun recordInstalledGeneration(pluginId: String) { + tier3Preferences().edit().putInt(pluginId, TIER3_COMPRESSION_GENERATION).apply() + } + + private fun forgetInstalledGeneration(pluginId: String) { + tier3Preferences().edit().remove(pluginId).apply() + } + + private fun tier3Preferences(): SharedPreferences = context.getSharedPreferences(TIER3_PREFS, Context.MODE_PRIVATE) + /** * Check if any Tier 3 content rows exist for this plugin. */ From 9949a55d7296162d92a91c5d1e9ab2fff4dd912c Mon Sep 17 00:00:00 2001 From: David Schachter Date: Thu, 27 Aug 2026 17:47:39 -0700 Subject: [PATCH 05/15] ADFA-5240: Decode each brotli row once, with no plain-decode retry Now that every brotli row in a dictionary-declaring database is compressed against that dictionary, WebServer can attach it and read the row, full stop. The try-dictionary-then-retry-plain dance existed only to sort out rows written the other way, and it inferred which was which from a decode failure -- behavior the brotli spec does not promise. A decode failure now means the row is damaged, and says so, instead of being quietly retried into a second failure. Documents the same in docs/documentation-database.md, which described the mixed state as intended, and corrects its claim that nothing in the app ever writes to this database. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01M2Bzxv38NbXWPMQVckoSNC --- .../androidide/localWebServer/WebServer.kt | 97 +++++-------------- .../utils/DocumentationCompression.kt | 14 ++- docs/documentation-database.md | 8 +- 3 files changed, 41 insertions(+), 78 deletions(-) diff --git a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt index 2812a61bed..6741dfc557 100644 --- a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt +++ b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt @@ -4,17 +4,15 @@ import android.database.Cursor import android.database.sqlite.SQLiteDatabase import android.net.TrafficStats import android.os.Environment.getExternalStorageDirectory -import com.aayushatharva.brotli4j.Brotli4jLoader -import com.aayushatharva.brotli4j.decoder.BrotliInputStream import com.google.gson.Gson import com.google.gson.GsonBuilder import com.google.gson.ToNumberPolicy import com.google.gson.annotations.SerializedName import com.google.gson.reflect.TypeToken +import com.itsaky.androidide.utils.BrotliDictionaryCodec import com.itsaky.androidide.utils.ContentTypeHeaders import com.itsaky.androidide.utils.DatabaseVersionResolver import com.itsaky.androidide.utils.loadCompressionDictionary -import com.itsaky.androidide.utils.toDirectByteBuffer import io.pebbletemplates.pebble.PebbleEngine import io.pebbletemplates.pebble.loader.StringLoader import io.pebbletemplates.pebble.template.PebbleTemplate @@ -32,7 +30,6 @@ import java.net.InetSocketAddress import java.net.ServerSocket import java.net.Socket import java.net.URLDecoder -import java.nio.ByteBuffer import java.sql.Date import java.text.SimpleDateFormat import java.util.Collections @@ -108,7 +105,6 @@ data class JavaExecutionResult( /** * Reads [chunks] back to back as one stream, without concatenating them into a new array. - * Cheap to build twice, which the no-dictionary retry in `decompressBrotli` relies on. */ internal fun chunksAsStream(chunks: List): InputStream = SequenceInputStream(Collections.enumeration(chunks.map { ByteArrayInputStream(it) })) @@ -152,19 +148,18 @@ class WebServer( // replacing the file is exactly how they'd fix it. private var failedDebugSwapTimestamp: Long = -1 - // The shared dictionary Content's brotli-compressed rows are compressed against (see - // ADFA-5153). Lazily (re)loaded on demand, right before the first content fetch that needs - // it after `database` changes -- see compressionDictionaryStale -- rather than eagerly at - // database-open/swap time, but still cached (not reloaded per-request) once loaded for the - // currently active database. Null (no dictionary attached, plain-brotli decode) unless the - // active database declares MAJOR >= MAJOR_VERSION_WITH_COMPRESSION_DICTIONARY in ADFA-5220's - // version table. - private var compressionDictionary: ByteBuffer? = null - - // Set whenever `database` changes (see switchToDatabase); cleared once compressionDictionary - // has been (re)loaded for that database. Lets the dictionary stay lazily loaded -- only right - // before the first content fetch that actually needs it -- while still loading at most once - // per database change rather than once per request. + // Decodes Content's brotli rows against the shared dictionary they were compressed with (see + // ADFA-5153). Lazily (re)built on demand, right before the first content fetch that needs it + // after `database` changes -- see compressionDictionaryStale -- rather than eagerly at + // database-open/swap time, but still cached (not rebuilt per-request) for the currently active + // database. Holds no dictionary, and so decodes plain brotli, unless the active database + // declares MAJOR >= MAJOR_VERSION_WITH_COMPRESSION_DICTIONARY in ADFA-5220's version table. + private var codec = BrotliDictionaryCodec(null) + + // Set whenever `database` changes (see switchToDatabase); cleared once codec has been + // (re)built for that database. Lets the dictionary stay lazily loaded -- only right before the + // first content fetch that actually needs it -- while still loading at most once per database + // change rather than once per request. private var compressionDictionaryStale = true private val log = LoggerFactory.getLogger(WebServer::class.java) private val debugEnabled: Boolean = File(config.debugEnablePath).exists() @@ -256,56 +251,13 @@ class WebServer( } /** - * Loads brotli4j's native library if nothing else has yet, and turns its absence into a failed - * request rather than a dead app. - * - * Nothing here owns that load: it happens as a side effect of `AssetsInstallationHelper`'s - * install or `ToolsManager`'s tooling-jar update, neither of which runs on an ordinary cold - * start. A process that skips both -- Android restarting the app straight into the editor, say -- - * reaches the first brotli row with the natives unregistered, and `DecoderJNI.nativeCreate` - * raises `UnsatisfiedLinkError`. Being an Error rather than an Exception, that escapes - * [handleClient]'s catch and kills the app from a coroutine worker instead of failing one - * request (observed on-device, 20-Aug). - * - * Referencing [Brotli4jLoader] triggers the static init that performs the load, so this call is - * the warm-up; afterwards `ensureAvailability` is a single static null-check, cheap enough to - * leave on the per-decode path rather than tracking "warmed" state of our own. - */ - private fun ensureBrotliAvailable() { - try { - Brotli4jLoader.ensureAvailability() - } catch (e: UnsatisfiedLinkError) { - throw IOException("brotli4j's native library is unavailable, so brotli content cannot be decoded", e) - } - } - - /** - * Decompresses one Brotli-compressed Content row. Tries the shared dictionary first, since every - * ADFA-5153-migrated row requires it, then falls back to a plain decode for rows that were never - * dictionary-compressed: plugin-contributed Tier 3 docs (PluginDocumentationManager/BrotliCompressor - * compress with no dictionary) or any row served from a pre-migration database. Attaching a - * dictionary to a stream that wasn't compressed against one reliably fails to decode rather than - * silently producing wrong bytes (verified empirically -- see docs/documentation-database.md), so - * this ordering never lets a dictionary-compressed row fall through to the plain path by accident. + * Decompresses one Brotli-compressed Content row, attaching the shared dictionary when the + * active database declares one. Every brotli row in such a database is compressed against it, + * whether built offline or contributed by a plugin (ADFA-5240), so a single decode is enough + * and a failure is a real failure -- not, as it once was, a row that might simply have been + * written the other way. */ - private fun decompressBrotli(chunks: List): ByteArray { - ensureBrotliAvailable() - val dictionary = compressionDictionary - if (dictionary != null) { - try { - return BrotliInputStream(chunksAsStream(chunks)).use { stream -> - stream.attachDictionary(dictionary) - stream.readBytes() - } - } catch (e: IOException) { - log.debug( - "Dictionary decode failed for a brotli row (likely dictionary-free plugin content); retrying without a dictionary: {}", - e.message, - ) - } - } - return BrotliInputStream(chunksAsStream(chunks)).use { it.readBytes() } - } + private fun decompressBrotli(chunks: List): ByteArray = codec.decompress(chunksAsStream(chunks)) /** * Stops the server by closing the listening socket. Safe to call from any thread. @@ -544,7 +496,7 @@ class WebServer( // a transient failure as "no dictionary" for the rest of this database's lifetime. if (compressionDictionaryStale) { try { - compressionDictionary = loadCompressionDictionary(database) + codec = BrotliDictionaryCodec(loadCompressionDictionary(database)) compressionDictionaryStale = false } catch (e: Exception) { log.error("Could not load compression dictionary; will retry on the next request: {}", e.message) @@ -613,11 +565,10 @@ class WebServer( } } - // Content is compressed at rest with brotli -- most rows against the shared dictionary - // loaded into compressionDictionary (see ADFA-5153), but plugin-contributed Tier 3 docs - // (PluginDocumentationManager/BrotliCompressor) are plain brotli with no dictionary. - // This server always decompresses before responding, so it never needs to negotiate - // Content-Encoding with the client. + // Content is compressed at rest with brotli, against the shared dictionary in databases + // that declare one (see ADFA-5153) and plain in those that don't. This server always + // decompresses before responding, so it never needs to negotiate Content-Encoding with + // the client. var dbContent = if (compression == "brotli") { compression = "none" diff --git a/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt b/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt index 7ee1ff33cf..bdb24fc6c4 100644 --- a/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt +++ b/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt @@ -152,7 +152,19 @@ class BrotliDictionaryCodec( /** * Loads brotli4j's native library if nothing else has yet, and turns its absence into a failed - * request rather than an `Error` that would take down the calling thread. + * request rather than a dead app. + * + * Nothing here owns that load: it happens as a side effect of `AssetsInstallationHelper`'s + * install or `ToolsManager`'s tooling-jar update, neither of which runs on an ordinary cold + * start. A process that skips both -- Android restarting the app straight into the editor, say + * -- reaches the first brotli row with the natives unregistered, and `DecoderJNI.nativeCreate` + * raises `UnsatisfiedLinkError`. Being an Error rather than an Exception, that escapes the + * caller's catch and kills the app from a coroutine worker instead of failing one request + * (observed on-device, 20-Aug). + * + * Referencing [Brotli4jLoader] triggers the static init that performs the load, so this call is + * the warm-up; afterwards `ensureAvailability` is a single static null-check, cheap enough to + * leave on the per-row path rather than tracking "warmed" state of our own. */ private fun ensureBrotliAvailable() { try { diff --git a/docs/documentation-database.md b/docs/documentation-database.md index 76c6b8a3ed..c13e872289 100644 --- a/docs/documentation-database.md +++ b/docs/documentation-database.md @@ -34,7 +34,7 @@ CREATE TABLE Content ( One row per file the web server can serve (HTML, CSS, JS, image, video, PDF, ...) — 30,000+ rows. Key points: - **`path`** is the lookup key (indexed via the `UNIQUE` constraint) and is what `WebServer` matches the HTTP request path against. Paths carry a short source prefix to avoid collisions between doc sets, e.g. `k/index.html` (Kotlin) vs `j/index.html` (Java). -- **`content`** is compressed — Brotli for text-like formats, format-specific compression otherwise (images/video/fonts). `ContentTypes.compression` says which. Every migrated `Content` row with `ContentTypes.compression = 'brotli'` is Brotli-compressed against the single shared dictionary in `CompressionDictionary` (see below), converted in one pass by ADFA-5153 — but plugin-contributed Tier 3 rows (`PluginDocumentationManager`/`BrotliCompressor`, see below) are plain, dictionary-free Brotli, and there is no per-row flag distinguishing the two, because a dictionary-compressed stream and a plain one are not distinguishable at decode time by inspection. They *are* distinguishable by attempting the decode: attaching the *wrong* dictionary decodes without error to different bytes than were compressed (its backward distances resolve into real, just incorrect, bytes) — but attaching *no* dictionary to a stream that needs one reliably throws (`IOException`, "corrupted input"), since distances into the dictionary region are then out of bounds for any spec-compliant decoder. `WebServer` relies on exactly this: it tries the dictionary first and falls back to a plain decode on `IOException`, which correctly handles both dictionary-compressed and plain rows — but never rely on decode success/failure to detect a *wrong* dictionary, since that case is silent. Content over 1 MB is split across multiple rows: the first row's path is the base path, continuation rows are `path-1`, `path-2`, ... (`languageId = 1`), reassembled by `WebServer` before returning. +- **`content`** is compressed — Brotli for text-like formats, format-specific compression otherwise (images/video/fonts). `ContentTypes.compression` says which. In a database declaring `MAJOR >= 2`, **every** row with `ContentTypes.compression = 'brotli'` is compressed against the single shared dictionary in `CompressionDictionary` (see below): the bulk of them converted in one pass by ADFA-5153, and plugin-contributed Tier 3 rows written that way at install time by `PluginDocumentationManager` (ADFA-5240). So the compression of a row follows from the database's declared version alone — there is no per-row flag, and none is needed. That matters because the two are not distinguishable by inspection, and only half-distinguishable by decode: attaching *no* dictionary to a stream that needs one reliably throws (`IOException`, "corrupted input"), since distances into the dictionary region are out of bounds for any spec-compliant decoder, but attaching the *wrong* dictionary decodes without error to different bytes than were compressed. Never use decode success as evidence that the right dictionary was used. Content over 1 MB is split across multiple rows: the first row's path is the base path, continuation rows are `path-1`, `path-2`, ... (`languageId = 1`), reassembled by `WebServer` before returning. - **`templateId`**: `0` (or unset) means `content` is legacy HTML with presentation baked in (the pre-CMS Release 0/1 format). A positive value means `content` is JSON *facts only*, rendered through the matching row in `Templates` (a Pebble template) — the ongoing move to a proper CMS that de-duplicates presentation across near-identical pages (e.g. `sin`/`cos` docs). - The `UNIQUE(path)` constraint rejects any duplicate `path`, regardless of `languageID` — a second language for an existing path isn't supported yet (only `EN-us` currently exists). Getting there needs an upstream schema change to composite uniqueness on `(path, languageID)` (see *Known rough edges* below). @@ -63,7 +63,7 @@ CREATE TABLE Tooltips ( ### Supporting tables - **`DocumentationDatabaseVersion(major, minor, patch, who, comment, changeTime)`** — the database's own semver (ADFA-5220), replacing the heuristics that used to infer the format from which tables happened to exist. Append-only: each change is another `INSERT`, so the **row inserted last** is the current version, not the highest one ever recorded — a rebuild from an older content set is a downgrade and has to read as one (`DatabaseVersionResolver.resolveMajorVersion`, which returns null for a database predating the table). `MAJOR >= 2` is what tells the app its brotli `Content` rows are dictionary-compressed; below that, `WebServer` neither reads nor attaches `CompressionDictionary`. Gating on the declared version rather than on the table's presence matters in both directions: a database can carry the dictionary table while its content is still plain brotli (every row would then pay a failed dictionary decode before its plain one, on every request), and a migrated database that lost the table fails loudly instead of quietly. -- **`CompressionDictionary(id, data)`** — single-row table (`id INTEGER PRIMARY KEY CHECK (id = 1)`) holding the raw Brotli dictionary every ADFA-5153-migrated `compression = 'brotli'` `Content` row is compressed against. Trained once, from a representative sample across the whole `Content` table, by `OfflineDocumentationTools`' `migrate_content_to_dictionary_brotli.py` / `populate_db.py` (never retrained after that — a dictionary-compressed row is only decodable against the exact dictionary it was compressed with, so replacing it would silently orphan every already-migrated row). Shipping the dictionary inside `documentation.db` itself, rather than as a separate bundled asset, keeps it version-locked to the content compressed against it. `WebServer` loads it lazily -- not merely from starting the server or swapping databases, but on the first content fetch that needs it after `database` changes, and only when `DocumentationDatabaseVersion` declares `MAJOR >= 2` (see above) -- and caches it from then on, reloading again only on the next database change (a swap can bring in a database with a different dictionary or none, so it can't stay cached across one). Per row, it tries decoding with the dictionary attached first via brotli4j's `attachDictionary`, falling back to a plain decode on failure — needed both for a database predating this migration (no `CompressionDictionary` table at all) and for plugin-contributed rows within an otherwise-migrated database (see `PluginDocumentationManager` below). +- **`CompressionDictionary(id, data)`** — single-row table (`id INTEGER PRIMARY KEY CHECK (id = 1)`) holding the raw Brotli dictionary every ADFA-5153-migrated `compression = 'brotli'` `Content` row is compressed against. Trained once, from a representative sample across the whole `Content` table, by `OfflineDocumentationTools`' `migrate_content_to_dictionary_brotli.py` / `populate_db.py` (never retrained after that — a dictionary-compressed row is only decodable against the exact dictionary it was compressed with, so replacing it would silently orphan every already-migrated row). Shipping the dictionary inside `documentation.db` itself, rather than as a separate bundled asset, keeps it version-locked to the content compressed against it. Both the reader and the writer load it through `loadCompressionDictionary` in `:common`, so they cannot disagree about whether a given database's brotli rows carry a dictionary — a row compressed against one the reader won't attach is undecodable, and vice versa. `WebServer` loads it lazily -- not merely from starting the server or swapping databases, but on the first content fetch that needs it after `database` changes, and only when `DocumentationDatabaseVersion` declares `MAJOR >= 2` (see above) -- and caches it from then on, reloading again only on the next database change (a swap can bring in a database with a different dictionary or none, so it can't stay cached across one). Each row is then decoded exactly once, with the dictionary attached or not according to that version; there is no retry, because there is no longer a second way a row might have been written. - **`Templates(id, name, content)`** — Pebble template source, keyed by id (and by `name` for well-known templates like `bookshelf`). Referenced by `Content.templateId`. - **`Bookshelf(contentID, bookCategoryID, title, description)`** / **`BookCategories(id, category, description)`** — the Dynamic Bookshelf: one row per "book" (PDF or similar), linked to its Tier 3 page via `contentID` -> `Content.id`. Two DB triggers keep `Bookshelf` in sync when a PDF row is inserted/deleted from `Content`; `title`/`description` don't come from those triggers and must be set by hand. Non-PDF books need a separate ingestion path (plugin-provided, e.g. via `PluginDocumentationManager`). - **`LastChange(documentationSet, changeTime, who)`** — audit trail for edits made through `docdb-studio`; not shown to end users. `DatabaseVersionResolver` reads the `documentationSet = 'wholedb'` row to report the DB's build/edit stamp in debug logging, falling back to the most recent row of any set if `'wholedb'` is missing. @@ -71,7 +71,7 @@ CREATE TABLE Tooltips ( ## How CoGo talks to this database -All three sites below open the file with `SQLiteDatabase.openDatabase(..., OPEN_READONLY)` — no writes, ever, from this app (see ADR 0001 for why raw SQLite is justified here instead of Room). +The two reader sites below open the file with `SQLiteDatabase.openDatabase(..., OPEN_READONLY)`; `PluginDocumentationManager` is the one writer, and opens `OPEN_READWRITE` to insert the rows a plugin contributes (see ADR 0001 for why raw SQLite is justified here instead of Room). - **`app/.../localWebServer/WebServer.kt`** — serves Tier 3. On each `GET`, runs: @@ -84,7 +84,7 @@ All three sites below open the file with `SQLiteDatabase.openDatabase(..., OPEN_ then reassembles chunked blobs, always decompresses Brotli content (attaching `CompressionDictionary`'s bytes first, if loaded — see above) since this server never negotiates `Content-Encoding` with the client, and instantiates the template if `templateId > 0`. Also serves a Dynamic Bookshelf JSON payload (joining `Content`/`Bookshelf`/`BookCategories`, rendered through the `bookshelf` template) and debug-only HTML dumps at `/pr/db` (`LastChange`, last 20 rows) and `/pr/pr` (recent projects, from a *different* database). - **`idetooltips/.../ToolTipManager.kt`** — serves Tier 1/2. Looks up `Tooltips` joined to `TooltipCategories` by `(category, tag)`, then `TooltipButtons` for the Tier 3 links shown at the bottom. -- **`plugin-manager/.../documentation/PluginDocumentationManager.kt`** (with `Tier3AssetWalker.kt`, and the `DocumentationExtension` contract in `plugin-api`) — lets plugins contribute their own help content into the same lookup paths. +- **`plugin-manager/.../documentation/PluginDocumentationManager.kt`** (with `Tier3AssetWalker.kt`, and the `DocumentationExtension` contract in `plugin-api`) — lets plugins contribute their own help content into the same lookup paths, under the reserved `plugin//...` prefix. Brotli assets are compressed against the same `CompressionDictionary` the rest of the table uses, read from the database being written. When that dictionary can't be read the install is abandoned rather than written plain, and retried on the plugin's next activation. ## Editing the database From 666fb0a1cc082107a74ef05e44d8982e0bbc5589 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Thu, 27 Aug 2026 18:38:42 -0700 Subject: [PATCH 06/15] ADFA-5240: Fix four defects found in review A database swap marked the dictionary stale but left the old codec in place. The reload that clears that flag can throw, and handleClient serves the request anyway -- so the new database's rows would be decoded against the previous database's dictionary. That does not reliably fail; with a same-length dictionary holding plausible bytes it returns 200 with the wrong content. Reset the codec at swap time so the worst case is a loud failure instead. beginTransaction() sat outside the try, so a throw there -- the database is open elsewhere for reading, so a lock exception is not hypothetical -- leaked both the database handle and the reflectively built AssetManager. It moves inside, with endTransaction guarded on inTransaction(). The generation marker was stamped by verifyAndRecreateTier3Documentation rather than by the function that writes the rows. Since that function is public, any other caller would write generation-2 rows and record nothing, and every later verify would then delete and recompress the whole asset set. It now happens beside the write it describes. The marker used SharedPreferences.apply(). Losing that write to a process death costs a full quality-11 recompress of every asset on the next launch, and this already runs on Dispatchers.IO, so commit() is the right trade. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01M2Bzxv38NbXWPMQVckoSNC --- .../androidide/localWebServer/WebServer.kt | 14 +++++++---- .../PluginDocumentationManager.kt | 23 +++++++++++-------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt index 6741dfc557..1b2e6d7b62 100644 --- a/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt +++ b/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt @@ -223,13 +223,20 @@ class WebServer( /** * Opens [path] as the active database, refreshing every piece of state that depends on which * database file is active -- [databaseTimestamp] and the per-database caches - * [bookshelfTemplateId]/[templateCache] -- as one atomic operation. Does *not* load - * [compressionDictionary] itself -- a different database can have a different dictionary (or + * [bookshelfTemplateId]/[templateCache] -- as one atomic operation. Does *not* load the new + * database's dictionary itself -- a different database can have a different dictionary (or * none) -- it only marks [compressionDictionaryStale] so the next content fetch that needs it * loads it lazily then (see [handleClient]), at most once per database change rather than * once per request. Only closes the previous database once the new one has opened * successfully, so a failed swap (this throws) leaves the previous, still-open database * serving requests rather than leaving [database] referencing an already-closed handle. + * + * [codec] is reset here rather than merely marked stale, because the reload that clears the + * flag can throw (a transient SQLite failure) and [handleClient] then serves the request + * anyway. Carrying the previous database's codec into that request would decode the new + * database's rows against the old database's dictionary -- which, per [BrotliDictionaryCodec], + * succeeds and returns the wrong bytes rather than failing. A dictionary-free codec fails those + * rows loudly instead, which is the only safe way to be wrong here. */ private fun switchToDatabase( path: String, @@ -245,6 +252,7 @@ class WebServer( } database = newDatabase databaseTimestamp = timestamp + codec = BrotliDictionaryCodec(null) compressionDictionaryStale = true bookshelfTemplateId = -1 templateCache.clear() @@ -549,8 +557,6 @@ class WebServer( while (nextChunk.size == contentChunkSize) { val path2 = "$path-$fragmentNumber" val cursor2 = database.rawQuery(query2, arrayOf(path2)) - // Whether the client has already been told 200; see the assignment below. - var responseStarted = false try { if (cursor2.moveToFirst()) { nextChunk = cursor2.getBlob(0) diff --git a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt index c03b2325df..b90d6f2c1c 100644 --- a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt +++ b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt @@ -225,8 +225,8 @@ class PluginDocumentationManager( var inserted = 0 var skipped = 0 - db.beginTransaction() try { + db.beginTransaction() removePluginTier3Internal(db, pluginId) for (asset in Tier3AssetWalker.walk(pluginAssets, assetPath)) { @@ -264,6 +264,11 @@ class PluginDocumentationManager( } db.setTransactionSuccessful() + // Recorded here, beside the write it describes, rather than in + // verifyAndRecreateTier3Documentation: this function is public, and a caller that + // wrote generation-2 rows without stamping them would be re-detected as stale and + // recompressed on every activation from then on. + recordInstalledGeneration(pluginId) Log.d(TAG, "Installed $inserted Tier 3 documents for plugin $pluginId (skipped=$skipped)") true } catch (e: Exception) { @@ -271,7 +276,9 @@ class PluginDocumentationManager( Log.e(TAG, "Failed to install Tier 3 docs for plugin $pluginId", e) false } finally { - db.endTransaction() + if (db.inTransaction()) { + db.endTransaction() + } db.close() pluginAssets.close() } @@ -323,11 +330,7 @@ class PluginDocumentationManager( return@withContext true } Log.d(TAG, "Tier 3 docs missing or stale for $pluginId, installing...") - val installed = installPluginTier3Documentation(pluginId, plugin, pluginApkPath) - if (installed) { - recordInstalledGeneration(pluginId) - } - installed + installPluginTier3Documentation(pluginId, plugin, pluginApkPath) } /** @@ -345,12 +348,14 @@ class PluginDocumentationManager( */ private fun installedGeneration(pluginId: String): Int = tier3Preferences().getInt(pluginId, 0) + // commit(), not apply(): this already runs on Dispatchers.IO, and losing the write to a process + // death would delete and recompress every one of the plugin's assets at quality 11 next launch. private fun recordInstalledGeneration(pluginId: String) { - tier3Preferences().edit().putInt(pluginId, TIER3_COMPRESSION_GENERATION).apply() + tier3Preferences().edit().putInt(pluginId, TIER3_COMPRESSION_GENERATION).commit() } private fun forgetInstalledGeneration(pluginId: String) { - tier3Preferences().edit().remove(pluginId).apply() + tier3Preferences().edit().remove(pluginId).commit() } private fun tier3Preferences(): SharedPreferences = context.getSharedPreferences(TIER3_PREFS, Context.MODE_PRIVATE) From e4efa915ab19149fdf3ed2dea784e81fd5f1a741 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Thu, 27 Aug 2026 18:38:57 -0700 Subject: [PATCH 07/15] ADFA-5240: Correct the wrong-dictionary claim, and tidy what the move left Both the docs and the new KDoc said a wrong dictionary "decodes without error to different bytes". Testing that turned out to be only conditionally true: a wrong dictionary of a different length, or with nothing valid at the offsets the stream references, throws. It decodes cleanly to wrong content only when it is the same length and holds plausible bytes there -- which is exactly the shape two builds of the same documentation.db have. Both statements now say so, and a test pins both halves. That test is also the evidence for the codec reset in the previous commit, which is otherwise justified only by a comment. Loose ends from the move into :common: - The dictionary diagnostics went through slf4j from WebServer and became android.util.Log on the way over, so they stopped reaching the app's own log pipeline -- the one line explaining a "plugin docs 500" report. Back to slf4j, which is also what most of this package uses, and which sheds a 24-character log tag that was over Android's 23-character cap. - encoderParameters re-loaded the brotli native, unreachable behind compress()'s own ensureBrotliAvailable(). It made sense in BrotliCompressor, which had no such guard. - WebServer's switchToDatabase KDoc still linked a property this branch deleted; DatabaseVersionResolver still pointed at loadCompressionDictionary's old home. - The version bullet in documentation-database.md still described the retry removed two commits ago, contradicting the bullet above it. - An em dash in a Tier3AssetWalker log string, and a shadowed, never-read copy of responseStarted in the chunk loop. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01M2Bzxv38NbXWPMQVckoSNC --- .../BrotliDictionaryDecodeTest.kt | 33 +++++++++++++++++ .../utils/DatabaseVersionResolver.kt | 2 +- .../utils/DocumentationCompression.kt | 36 ++++++++++--------- docs/documentation-database.md | 4 +-- .../manager/documentation/Tier3AssetWalker.kt | 2 +- 5 files changed, 56 insertions(+), 21 deletions(-) diff --git a/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt index a79d64063c..5f9d994031 100644 --- a/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt +++ b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt @@ -6,6 +6,7 @@ import com.itsaky.androidide.utils.BrotliDictionaryCodec import com.itsaky.androidide.utils.toDirectByteBuffer import org.junit.Assert.assertArrayEquals import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse import org.junit.Assert.assertSame import org.junit.Assert.assertThrows import org.junit.BeforeClass @@ -234,6 +235,38 @@ class BrotliDictionaryDecodeTest { } } + @Test + fun `the wrong dictionary can decode without error to the wrong bytes`() { + // The reason WebServer.switchToDatabase resets its codec instead of only marking it stale: + // if a failed dictionary reload let the previous database's codec serve the new database's + // rows, this is what a client could get -- a 200 carrying different bytes than were stored, + // with nothing thrown. A dictionary-free codec fails such a row loudly instead, which is + // why that is the safe thing to fall back to. + // + // The two dictionaries here are the same length and differ only in bytes the payload + // actually references, which is what makes the failure silent. A wrong dictionary of a + // different length, or one whose referenced offsets hold nothing valid, throws instead -- + // so a throw does not prove the dictionary was right, and a success does not either. + val base = "docs-sidebar toc-element kotlin interface companion object page.peb ".repeat(400) + val dictionary = toDirectByteBuffer(base.toByteArray(StandardCharsets.UTF_8)) + val lookalike = toDirectByteBuffer(base.replace("kotlin", "scalax").toByteArray(StandardCharsets.UTF_8)) + val expected = "docs-sidebar toc-element kotlin interface companion object page.peb ".repeat(6).toByteArray(StandardCharsets.UTF_8) + + val compressed = BrotliDictionaryCodec(dictionary).compress(expected) + + val decoded = BrotliDictionaryCodec(lookalike).decompress(ByteArrayInputStream(compressed)) + assertEquals("the wrong dictionary still produced a full-length result", expected.size, decoded.size) + assertFalse( + "decoding with the wrong dictionary must not be assumed detectable by failure", + expected.contentEquals(decoded), + ) + + // ... while no dictionary at all is reliably rejected, which is what the writer relies on. + assertThrows(IOException::class.java) { + BrotliDictionaryCodec(null).decompress(ByteArrayInputStream(compressed)) + } + } + @Test fun `a null dictionary round-trips as plain brotli`() { // A database predating ADFA-5153 declares no dictionary, so both sides fall to plain diff --git a/common/src/main/java/com/itsaky/androidide/utils/DatabaseVersionResolver.kt b/common/src/main/java/com/itsaky/androidide/utils/DatabaseVersionResolver.kt index 225ffd6a39..08109f8f08 100644 --- a/common/src/main/java/com/itsaky/androidide/utils/DatabaseVersionResolver.kt +++ b/common/src/main/java/com/itsaky/androidide/utils/DatabaseVersionResolver.kt @@ -85,7 +85,7 @@ object DatabaseVersionResolver { * identifies itself. * * Deliberately does *not* catch exceptions, unlike [resolveDatabaseVersion]: callers cache the - * answer for the lifetime of a database (see `WebServer.loadCompressionDictionary`), so a + * answer for the lifetime of a database (see [loadCompressionDictionary]), so a * transient `SQLiteException` has to stay distinguishable from a definitive "no version table", * or one hiccup would pin the database at unversioned until it is swapped. */ diff --git a/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt b/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt index bdb24fc6c4..9745ae8f0f 100644 --- a/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt +++ b/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt @@ -1,19 +1,19 @@ package com.itsaky.androidide.utils import android.database.sqlite.SQLiteDatabase -import android.util.Log import com.aayushatharva.brotli4j.Brotli4jLoader import com.aayushatharva.brotli4j.decoder.BrotliInputStream import com.aayushatharva.brotli4j.encoder.BrotliOutputStream import com.aayushatharva.brotli4j.encoder.Encoder import com.aayushatharva.brotli4j.encoder.PreparedDictionary import com.aayushatharva.brotli4j.encoder.PreparedDictionaryGenerator +import org.slf4j.LoggerFactory import java.io.ByteArrayOutputStream import java.io.IOException import java.io.InputStream import java.nio.ByteBuffer -private const val TAG = "DocumentationCompression" +private val log = LoggerFactory.getLogger("DocumentationCompression") /** * Copies [bytes] into a direct [ByteBuffer] -- brotli4j's `attachDictionary` requires a direct @@ -58,11 +58,10 @@ fun toDirectByteBuffer(bytes: ByteArray): ByteBuffer = fun loadCompressionDictionary(db: SQLiteDatabase): ByteBuffer? { val majorVersion = DatabaseVersionResolver.resolveMajorVersion(db) if (majorVersion == null || majorVersion < DatabaseVersionResolver.MAJOR_VERSION_WITH_COMPRESSION_DICTIONARY) { - Log.w( - TAG, - "Database declares documentation version ${majorVersion ?: "none"}, below " + - "${DatabaseVersionResolver.MAJOR_VERSION_WITH_COMPRESSION_DICTIONARY}; " + - "brotli content is handled without a dictionary.", + log.warn( + "Database declares documentation version {}, below {}; brotli content is handled without a dictionary.", + majorVersion ?: "none", + DatabaseVersionResolver.MAJOR_VERSION_WITH_COMPRESSION_DICTIONARY, ) return null } @@ -74,24 +73,24 @@ fun loadCompressionDictionary(db: SQLiteDatabase): ByteBuffer? { null, ).use { it.moveToFirst() } if (!tableExists) { - Log.w(TAG, "CompressionDictionary table not found; brotli content is handled without a dictionary.") + log.warn("CompressionDictionary table not found; brotli content is handled without a dictionary.") return null } return db.rawQuery("SELECT data FROM CompressionDictionary WHERE id = 1", null).use { cursor -> if (!cursor.moveToFirst()) { - Log.w(TAG, "CompressionDictionary table is empty; brotli content is handled without a dictionary.") + log.warn("CompressionDictionary table is empty; brotli content is handled without a dictionary.") return null } val bytes = cursor.getBlob(0) if (bytes == null) { - Log.w(TAG, "CompressionDictionary row has a NULL data column; brotli content is handled without a dictionary.") + log.warn("CompressionDictionary row has a NULL data column; brotli content is handled without a dictionary.") return null } // An empty blob would yield a 0-capacity buffer, which attachDictionary rejects -- // every row's dictionary decode would then fail with nothing above DEBUG to say why. if (bytes.isEmpty()) { - Log.w(TAG, "CompressionDictionary row has an empty data column; brotli content is handled without a dictionary.") + log.warn("CompressionDictionary row has an empty data column; brotli content is handled without a dictionary.") return null } toDirectByteBuffer(bytes) @@ -136,11 +135,15 @@ class BrotliDictionaryCodec( /** * Decompresses a `Content` blob read from the same database [dictionary] came from. * - * Throws `IOException` when [input] was not compressed the way that database's rows are -- - * a dictionary-compressed stream decoded without one leaves backward distances out of bounds, - * which any spec-compliant decoder rejects. Note the converse is *not* detectable: attaching - * the *wrong* dictionary decodes without error to different bytes than were compressed, so - * decode success is never evidence that the right dictionary was used. + * Throws `IOException` when [input] needed a dictionary and none was attached: the backward + * distances then reach outside the window, which any spec-compliant decoder rejects. That + * direction is reliable, and the writer depends on it. + * + * A *wrong* dictionary is a different matter, and is not reliably detectable. One of a + * different length, or with nothing valid at the offsets the stream references, usually + * throws -- but one of the same length holding plausible bytes there decodes cleanly to + * content that is simply wrong (verified: see BrotliDictionaryDecodeTest). So neither a throw + * nor a success is evidence about *which* dictionary was used. */ fun decompress(input: InputStream): ByteArray { ensureBrotliAvailable() @@ -178,7 +181,6 @@ class BrotliDictionaryCodec( // Matches OfflineDocumentationTools' encode settings, so a row written on-device is // indistinguishable in size and decodability from one built by that pipeline. private val encoderParameters: Encoder.Parameters by lazy { - Brotli4jLoader.ensureAvailability() Encoder.Parameters().setQuality(11).setWindow(24) } } diff --git a/docs/documentation-database.md b/docs/documentation-database.md index c13e872289..05e861a4e8 100644 --- a/docs/documentation-database.md +++ b/docs/documentation-database.md @@ -34,7 +34,7 @@ CREATE TABLE Content ( One row per file the web server can serve (HTML, CSS, JS, image, video, PDF, ...) — 30,000+ rows. Key points: - **`path`** is the lookup key (indexed via the `UNIQUE` constraint) and is what `WebServer` matches the HTTP request path against. Paths carry a short source prefix to avoid collisions between doc sets, e.g. `k/index.html` (Kotlin) vs `j/index.html` (Java). -- **`content`** is compressed — Brotli for text-like formats, format-specific compression otherwise (images/video/fonts). `ContentTypes.compression` says which. In a database declaring `MAJOR >= 2`, **every** row with `ContentTypes.compression = 'brotli'` is compressed against the single shared dictionary in `CompressionDictionary` (see below): the bulk of them converted in one pass by ADFA-5153, and plugin-contributed Tier 3 rows written that way at install time by `PluginDocumentationManager` (ADFA-5240). So the compression of a row follows from the database's declared version alone — there is no per-row flag, and none is needed. That matters because the two are not distinguishable by inspection, and only half-distinguishable by decode: attaching *no* dictionary to a stream that needs one reliably throws (`IOException`, "corrupted input"), since distances into the dictionary region are out of bounds for any spec-compliant decoder, but attaching the *wrong* dictionary decodes without error to different bytes than were compressed. Never use decode success as evidence that the right dictionary was used. Content over 1 MB is split across multiple rows: the first row's path is the base path, continuation rows are `path-1`, `path-2`, ... (`languageId = 1`), reassembled by `WebServer` before returning. +- **`content`** is compressed — Brotli for text-like formats, format-specific compression otherwise (images/video/fonts). `ContentTypes.compression` says which. In a database declaring `MAJOR >= 2`, **every** row with `ContentTypes.compression = 'brotli'` is compressed against the single shared dictionary in `CompressionDictionary` (see below): the bulk of them converted in one pass by ADFA-5153, and plugin-contributed Tier 3 rows written that way at install time by `PluginDocumentationManager` (ADFA-5240). So the compression of a row follows from the database's declared version alone — there is no per-row flag, and none is needed. That matters because the two are not distinguishable by inspection, and only half-distinguishable by decode: attaching *no* dictionary to a stream that needs one reliably throws (`IOException`, "corrupted input"), since distances into the dictionary region are out of bounds for any spec-compliant decoder. Attaching the *wrong* dictionary is not reliable in either direction -- one of a different length, or with nothing valid at the offsets the stream references, usually throws, but one of the same length holding plausible bytes there decodes cleanly to content that is simply wrong (both cases verified in `BrotliDictionaryDecodeTest`). Never use decode success, or failure, as evidence about *which* dictionary was used. Content over 1 MB is split across multiple rows: the first row's path is the base path, continuation rows are `path-1`, `path-2`, ... (`languageId = 1`), reassembled by `WebServer` before returning. - **`templateId`**: `0` (or unset) means `content` is legacy HTML with presentation baked in (the pre-CMS Release 0/1 format). A positive value means `content` is JSON *facts only*, rendered through the matching row in `Templates` (a Pebble template) — the ongoing move to a proper CMS that de-duplicates presentation across near-identical pages (e.g. `sin`/`cos` docs). - The `UNIQUE(path)` constraint rejects any duplicate `path`, regardless of `languageID` — a second language for an existing path isn't supported yet (only `EN-us` currently exists). Getting there needs an upstream schema change to composite uniqueness on `(path, languageID)` (see *Known rough edges* below). @@ -62,7 +62,7 @@ CREATE TABLE Tooltips ( ### Supporting tables -- **`DocumentationDatabaseVersion(major, minor, patch, who, comment, changeTime)`** — the database's own semver (ADFA-5220), replacing the heuristics that used to infer the format from which tables happened to exist. Append-only: each change is another `INSERT`, so the **row inserted last** is the current version, not the highest one ever recorded — a rebuild from an older content set is a downgrade and has to read as one (`DatabaseVersionResolver.resolveMajorVersion`, which returns null for a database predating the table). `MAJOR >= 2` is what tells the app its brotli `Content` rows are dictionary-compressed; below that, `WebServer` neither reads nor attaches `CompressionDictionary`. Gating on the declared version rather than on the table's presence matters in both directions: a database can carry the dictionary table while its content is still plain brotli (every row would then pay a failed dictionary decode before its plain one, on every request), and a migrated database that lost the table fails loudly instead of quietly. +- **`DocumentationDatabaseVersion(major, minor, patch, who, comment, changeTime)`** — the database's own semver (ADFA-5220), replacing the heuristics that used to infer the format from which tables happened to exist. Append-only: each change is another `INSERT`, so the **row inserted last** is the current version, not the highest one ever recorded — a rebuild from an older content set is a downgrade and has to read as one (`DatabaseVersionResolver.resolveMajorVersion`, which returns null for a database predating the table). `MAJOR >= 2` is what tells the app its brotli `Content` rows are dictionary-compressed; below that, `WebServer` neither reads nor attaches `CompressionDictionary`. Gating on the declared version rather than on the table's presence matters in both directions: a database can carry the dictionary table while its content is still plain brotli (every brotli row would then be decoded with a dictionary attached and fail outright, since ADFA-5240 removed the plain-decode retry that used to mask this), and a migrated database that lost the table fails loudly instead of quietly. - **`CompressionDictionary(id, data)`** — single-row table (`id INTEGER PRIMARY KEY CHECK (id = 1)`) holding the raw Brotli dictionary every ADFA-5153-migrated `compression = 'brotli'` `Content` row is compressed against. Trained once, from a representative sample across the whole `Content` table, by `OfflineDocumentationTools`' `migrate_content_to_dictionary_brotli.py` / `populate_db.py` (never retrained after that — a dictionary-compressed row is only decodable against the exact dictionary it was compressed with, so replacing it would silently orphan every already-migrated row). Shipping the dictionary inside `documentation.db` itself, rather than as a separate bundled asset, keeps it version-locked to the content compressed against it. Both the reader and the writer load it through `loadCompressionDictionary` in `:common`, so they cannot disagree about whether a given database's brotli rows carry a dictionary — a row compressed against one the reader won't attach is undecodable, and vice versa. `WebServer` loads it lazily -- not merely from starting the server or swapping databases, but on the first content fetch that needs it after `database` changes, and only when `DocumentationDatabaseVersion` declares `MAJOR >= 2` (see above) -- and caches it from then on, reloading again only on the next database change (a swap can bring in a database with a different dictionary or none, so it can't stay cached across one). Each row is then decoded exactly once, with the dictionary attached or not according to that version; there is no retry, because there is no longer a second way a row might have been written. - **`Templates(id, name, content)`** — Pebble template source, keyed by id (and by `name` for well-known templates like `bookshelf`). Referenced by `Content.templateId`. - **`Bookshelf(contentID, bookCategoryID, title, description)`** / **`BookCategories(id, category, description)`** — the Dynamic Bookshelf: one row per "book" (PDF or similar), linked to its Tier 3 page via `contentID` -> `Content.id`. Two DB triggers keep `Bookshelf` in sync when a PDF row is inserted/deleted from `Content`; `title`/`description` don't come from those triggers and must be set by hand. Non-PDF books need a separate ingestion path (plugin-provided, e.g. via `PluginDocumentationManager`). diff --git a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/Tier3AssetWalker.kt b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/Tier3AssetWalker.kt index f915df355f..ef46ffd59c 100644 --- a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/Tier3AssetWalker.kt +++ b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/Tier3AssetWalker.kt @@ -57,7 +57,7 @@ internal object Tier3AssetWalker { return@sequence } if (bytes == null) { - Log.w(TAG, "Skipping Tier 3 asset '$absolute' — exceeds $MAX_ASSET_BYTES byte limit") + Log.w(TAG, "Skipping Tier 3 asset '$absolute' - exceeds $MAX_ASSET_BYTES byte limit") return@sequence } if (relative.isNotEmpty()) { From 073aa641d436813945e8017e3f27ce5a2b31dad6 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Thu, 27 Aug 2026 18:47:55 -0700 Subject: [PATCH 08/15] ADFA-5240: Note where the codec's tests actually live :common ships BrotliDictionaryCodec but its tests sit in :app, so `:common:test` is green whether or not the codec works. Moving them needs brotli4j's host-native dispatch extracted into build-logic first -- a third copy of it already exists there -- which is its own change, not one to make inside a content-compression PR. A comment beside the test dependencies, so the misleading green is at least documented where someone would look. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01M2Bzxv38NbXWPMQVckoSNC --- common/build.gradle.kts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/common/build.gradle.kts b/common/build.gradle.kts index dc01fac8e9..6aa6b7c77e 100755 --- a/common/build.gradle.kts +++ b/common/build.gradle.kts @@ -51,6 +51,12 @@ dependencies { api(projects.subprojects.flashbar) implementation(libs.monitor) + // BrotliDictionaryCodec's tests live in :app (BrotliDictionaryDecodeTest), not here, so + // `:common:test` passing says nothing about it. They need brotli4j's per-OS/arch desktop + // native, and the host dispatch for that is wired only in app/build.gradle.kts -- moving them + // means extracting that into build-logic first, where a third copy of the same dispatch + // already lives. Tracked separately; jacocoAggregateReport runs both modules' suites, so CI + // coverage is unaffected either way. testImplementation(projects.testing.common) testImplementation(libs.tests.kotlinx.coroutines) testImplementation(libs.tests.google.truth) From d5595578f0750db1c3fa831f41150ff74c7f35cb Mon Sep 17 00:00:00 2001 From: David Schachter Date: Thu, 27 Aug 2026 18:53:02 -0700 Subject: [PATCH 09/15] ADFA-5240: Record the generation marker only once the rows commit setTransactionSuccessful only marks intent; endTransaction, in the finally, is what commits. The marker was written between the two, and with commit() rather than apply() it lands on disk immediately -- so a process death in that window left generation 2 standing against rows that then rolled back. That direction is the dangerous one. The install opens by deleting the plugin's existing rows, so a rollback restores the legacy plain rows it was replacing. The next verify would find rows present at the current generation, skip the reinstall, and serve content that no longer decodes now that the plain-decode retry is gone -- the exact failure the marker exists to prevent. Both paths now stamp after the transaction closes. The removal path had the same ordering; its failure direction is harmless (a redundant reinstall), but leaving the two written differently is how the install path came to be wrong in the first place. Closing the database is also nested under its own finally, so a throw from endTransaction can no longer leak the handle. No automated coverage for this: plugin-manager has no Robolectric and no test touching PluginDocumentationManager, so a rollback test needs that harness first. Noted on the PR. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01M2Bzxv38NbXWPMQVckoSNC --- .../PluginDocumentationManager.kt | 150 +++++++++++------- 1 file changed, 89 insertions(+), 61 deletions(-) diff --git a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt index b90d6f2c1c..5af08aa158 100644 --- a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt +++ b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt @@ -225,63 +225,77 @@ class PluginDocumentationManager( var inserted = 0 var skipped = 0 - try { - db.beginTransaction() - removePluginTier3Internal(db, pluginId) - - for (asset in Tier3AssetWalker.walk(pluginAssets, assetPath)) { - val ext = asset.relativePath.substringAfterLast('.', "") - if (ext.isEmpty()) { - Log.w(TAG, "Skipping Tier 3 asset without extension: ${asset.relativePath}") - skipped++ - continue - } - val row = resolver.resolve(db, ext) - if (row == null) { - Log.w(TAG, "No ContentType for .$ext (${asset.relativePath}); skipping") - skipped++ - continue - } + val installed = + try { + db.beginTransaction() + removePluginTier3Internal(db, pluginId) - val payload = - if (row.compression == "brotli") { - codec.compress(asset.bytes) - } else { - asset.bytes + for (asset in Tier3AssetWalker.walk(pluginAssets, assetPath)) { + val ext = asset.relativePath.substringAfterLast('.', "") + if (ext.isEmpty()) { + Log.w(TAG, "Skipping Tier 3 asset without extension: ${asset.relativePath}") + skipped++ + continue } - - val safeRelative = - try { - normalizeLocalDocumentationPath(asset.relativePath) - } catch (e: IllegalArgumentException) { - Log.w(TAG, "Skipping Tier 3 asset with invalid path '${asset.relativePath}': ${e.message}") + val row = resolver.resolve(db, ext) + if (row == null) { + Log.w(TAG, "No ContentType for .$ext (${asset.relativePath}); skipping") skipped++ continue } - val basePath = "plugin/$pluginId/$safeRelative" - insertContentChunked(db, basePath, payload, row.id) - inserted++ + + val payload = + if (row.compression == "brotli") { + codec.compress(asset.bytes) + } else { + asset.bytes + } + + val safeRelative = + try { + normalizeLocalDocumentationPath(asset.relativePath) + } catch (e: IllegalArgumentException) { + Log.w(TAG, "Skipping Tier 3 asset with invalid path '${asset.relativePath}': ${e.message}") + skipped++ + continue + } + val basePath = "plugin/$pluginId/$safeRelative" + insertContentChunked(db, basePath, payload, row.id) + inserted++ + } + + db.setTransactionSuccessful() + Log.d(TAG, "Installed $inserted Tier 3 documents for plugin $pluginId (skipped=$skipped)") + true + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to install Tier 3 docs for plugin $pluginId", e) + false + } finally { + try { + if (db.inTransaction()) { + db.endTransaction() + } + } finally { + db.close() + pluginAssets.close() + } } - db.setTransactionSuccessful() - // Recorded here, beside the write it describes, rather than in - // verifyAndRecreateTier3Documentation: this function is public, and a caller that - // wrote generation-2 rows without stamping them would be re-detected as stale and - // recompressed on every activation from then on. + // Stamped only once the rows are durably committed: setTransactionSuccessful above marks + // intent, endTransaction is what commits, and the marker is written with commit() rather + // than apply(), so it lands on disk immediately. Recording it inside the transaction would + // let a process death in between leave generation 2 standing against rows that then rolled + // back -- and since the rollback restores the legacy plain rows this install had deleted, + // the next verify would see rows present at the current generation and skip the reinstall + // those rows need. Still inside this function rather than in + // verifyAndRecreateTier3Documentation, though: this one is public, and a caller that wrote + // generation-2 rows without stamping them would be re-detected as stale and recompressed + // on every activation from then on. + if (installed) { recordInstalledGeneration(pluginId) - Log.d(TAG, "Installed $inserted Tier 3 documents for plugin $pluginId (skipped=$skipped)") - true - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Failed to install Tier 3 docs for plugin $pluginId", e) - false - } finally { - if (db.inTransaction()) { - db.endTransaction() - } - db.close() - pluginAssets.close() } + installed } /** @@ -290,21 +304,35 @@ class PluginDocumentationManager( suspend fun removePluginTier3Documentation(pluginId: String): Boolean = withContext(Dispatchers.IO) { val db = getPluginDatabase() ?: return@withContext false - db.beginTransaction() - try { - val deleted = removePluginTier3Internal(db, pluginId) - db.setTransactionSuccessful() + val removed = + try { + db.beginTransaction() + val deleted = removePluginTier3Internal(db, pluginId) + db.setTransactionSuccessful() + Log.d(TAG, "Removed $deleted Tier 3 rows for plugin $pluginId") + true + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to remove Tier 3 docs for plugin $pluginId", e) + false + } finally { + try { + if (db.inTransaction()) { + db.endTransaction() + } + } finally { + db.close() + } + } + + // Same ordering as the install path: the marker only describes rows that actually + // committed. This direction fails safe -- a marker cleared against rows that rolled back + // just costs one redundant reinstall -- but the two paths reading differently is how the + // install path's version got written the wrong way round in the first place. + if (removed) { forgetInstalledGeneration(pluginId) - Log.d(TAG, "Removed $deleted Tier 3 rows for plugin $pluginId") - true - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Failed to remove Tier 3 docs for plugin $pluginId", e) - false - } finally { - db.endTransaction() - db.close() } + removed } /** From c4f1db069411a4b057729a0e876d061d1bca366f Mon Sep 17 00:00:00 2001 From: David Schachter Date: Thu, 27 Aug 2026 19:44:05 -0700 Subject: [PATCH 10/15] ADFA-5240: Harden the codec's contract and the installer's failure paths A max-effort review found eleven contained defects. The measurements behind two of them changed what the documentation should say. "Attaching no dictionary to a stream that needs one reliably throws" is only true when the stream actually referenced the dictionary. Measured both directions: doc-like content fails as documented, but a 300 KB incompressible payload round-trips identically with or without one, because it carries no backward matches for the dictionary to shift. So a mixed row set fails non-uniformly -- a plugin's HTML 500s while its images keep serving. The KDoc, docs and a restored test now say that; the test's other half, covering plain rows read with a dictionary attached, had been dropped when its obsolete half was replaced. The buffer-reuse test passed with the duplicate() it names removed, because attachDictionary ignores position. It now asserts the caller's buffer position directly, and fails without the fix. Codec contract: - A heap dictionary compressed fine and then threw IllegalArgumentException on every read; both that and a dictionary under 8 bytes (brotli4j's floor, measured) are now rejected at construction rather than at first use. - loadCompressionDictionary treats a truncated blob as absent, so reader and writer keep agreeing. - decompress documents that it closes the stream it is given, and no longer leaks it when the decoder fails to start. - warmUp() lets a caller build the prepared dictionary before taking a lock, rather than inside one. Installer: - A database that declares a dictionary but has no usable one is damaged, not dictionary-free. Writing plain rows into it left them undecodable once the dictionary row was repaired in place, with no missing-rows check to catch them. It now defers, like the throw path. - An install where every asset was skipped committed the delete that opened the transaction and recorded success -- destroying content that was serving. It rolls back. - The codec load closes its resources through finally, so an OutOfMemoryError from the 256 KB direct allocation cannot leak a read-write handle on documentation.db. - Paths are validated before compression rather than after. - The prefs write result is checked; dropping it silently recreated the recompress-forever loop commit() was chosen to avoid. Docs: ADR 0001 justified PluginDocumentationManager under "prebuilt and opened read-only", which is the one thing it is not -- it is the sole writer of documentation.db. Corrected to condition 3, schema owned across a boundary, with ARCHITECTURE.md and this file's own lede brought in line. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01M2Bzxv38NbXWPMQVckoSNC --- ARCHITECTURE.md | 4 +- .../BrotliDictionaryDecodeTest.kt | 40 +++++++- .../utils/DocumentationCompression.kt | 92 +++++++++++++++++-- docs/adr/0001-prefer-room-for-persistence.md | 3 +- docs/documentation-database.md | 2 +- .../PluginDocumentationManager.kt | 89 ++++++++++++++---- 6 files changed, 194 insertions(+), 36 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 1002ead379..a6f59127f8 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -102,9 +102,9 @@ These structural facts shape every module. Day-to-day build *commands* live in ` > > **Recent Projects** is the reference example of the default: `app/src/main/java/com/itsaky/androidide/roomData/recentproject/` (`RecentProjectRoomDatabase`, `@Database version = 4` with migrations 1→4; `RecentProjectDao`; the `RecentProject` `@Entity` → table `recent_project_table`). It's provided via Koin in `di/AppModule.kt` and consumed by `MainViewModel`, `RecentProjectsViewModel`, `MainActivity`, `ProjectInfoBottomSheet`, and `ProjectCreationManager`. > -> **Raw SQLite is allowed only when** the database is prebuilt and opened read-only, the data is performance/allocation-critical and needs granular schema control, or the schema is shared across a process/component boundary. Current exceptions: symbol indexing (`lsp/indexing/SQLiteIndex.kt`), tooltips (`idetooltips/ToolTipManager.kt`), in-app/plugin help (`plugin-manager/.../documentation/PluginDocumentationManager.kt`), and the local web server (`app/.../localWebServer/WebServer.kt`). The `androidx.room:*` strings in `editor`'s `GroovyAutoComplete` are autocomplete suggestions for the *user's* code, not CoGo persistence. +> **Raw SQLite is allowed only when** the database is prebuilt and opened read-only, the data is performance/allocation-critical and needs granular schema control, or the schema is shared across a process/component boundary. Current exceptions: symbol indexing (`lsp/indexing/SQLiteIndex.kt`), tooltips (`idetooltips/ToolTipManager.kt`), in-app/plugin help (`plugin-manager/.../documentation/PluginDocumentationManager.kt` — the one *writer*, inserting plugin-contributed rows into a schema owned elsewhere), the shared compression-dictionary loader (`common/.../utils/DocumentationCompression.kt`), and the local web server (`app/.../localWebServer/WebServer.kt`). The `androidx.room:*` strings in `editor`'s `GroovyAutoComplete` are autocomplete suggestions for the *user's* code, not CoGo persistence. > -> The tooltip, in-app/plugin-help, and local-web-server exceptions all read `documentation.db`, the prebuilt Tier 1/2/3 help database — see [docs/documentation-database.md](docs/documentation-database.md) for its schema and how each consumer queries it. +> The tooltip, in-app/plugin-help, compression-dictionary and local-web-server exceptions all use `documentation.db`, the prebuilt Tier 1/2/3 help database — see [docs/documentation-database.md](docs/documentation-database.md) for its schema and how each consumer queries it. ## State Management diff --git a/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt index 5f9d994031..95e18ca5f3 100644 --- a/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt +++ b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt @@ -198,11 +198,23 @@ class BrotliDictionaryDecodeTest { assertArrayEquals(expected, result) } + @Test + fun `compressing does not drain the caller's dictionary buffer`() { + // PreparedDictionaryGenerator.generate consumes its argument, leaving position at limit. + // attachDictionary happens to ignore position -- it reads the whole capacity -- so a round + // trip alone cannot tell whether the codec drained the caller's buffer. Assert the buffer + // state directly, or the defensive duplicate() in BrotliDictionaryCodec is unpinned and a + // future brotli4j that honours position breaks silently. + val dictionary = decodeBase64ToDirectBuffer(dictionaryBase64) + val positionBefore = dictionary.position() + + BrotliDictionaryCodec(dictionary).compress("docs-sidebar toc-element".toByteArray(StandardCharsets.UTF_8)) + + assertEquals("compress() consumed the dictionary buffer it was given", positionBefore, dictionary.position()) + } + @Test fun `the codec reuses one dictionary buffer across compress and decompress`() { - // One codec serves a whole plugin install, so preparing the dictionary for the encoder - // must not disturb the buffer the decoder side attaches (`generate` drains its argument's - // position, hence the defensive duplicate in BrotliDictionaryCodec). val dictionary = decodeBase64ToDirectBuffer(dictionaryBase64) val codec = BrotliDictionaryCodec(dictionary) @@ -267,6 +279,28 @@ class BrotliDictionaryDecodeTest { } } + @Test + fun `a legacy plain row fails when the dictionary is attached, unless it never referenced one`() { + // What makes rows left plain by an earlier build fail loudly rather than serve wrong bytes, + // now that WebServer decodes once with no retry. The failure is content-dependent, which is + // the part worth pinning: a stream only breaks on the dictionary if it matched into it, so + // a plugin's HTML 500s while its incompressible assets keep serving. Anyone reading a + // half-broken plugin's pages needs to know that is one cause, not two. + val dictionary = decodeBase64ToDirectBuffer(dictionaryBase64) + val plainCodec = BrotliDictionaryCodec(null) + + val docLike = "docs-sidebar toc-element kotlin interface companion ".repeat(200).toByteArray(StandardCharsets.UTF_8) + val asLegacyRow = plainCodec.compress(docLike) + assertThrows(IOException::class.java) { + BrotliDictionaryCodec(dictionary).decompress(ByteArrayInputStream(asLegacyRow)) + } + + // Nothing here matches the dictionary, so attaching one changes nothing. + val noise = ByteArray(64 * 1024).also { java.util.Random(5240).nextBytes(it) } + val noiseRow = plainCodec.compress(noise) + assertArrayEquals(noise, BrotliDictionaryCodec(dictionary).decompress(ByteArrayInputStream(noiseRow))) + } + @Test fun `a null dictionary round-trips as plain brotli`() { // A database predating ADFA-5153 declares no dictionary, so both sides fall to plain diff --git a/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt b/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt index 9745ae8f0f..8eb43c5880 100644 --- a/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt +++ b/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt @@ -15,6 +15,11 @@ import java.nio.ByteBuffer private val log = LoggerFactory.getLogger("DocumentationCompression") +// brotli4j's PreparedDictionaryGenerator rejects a shorter dictionary with "src is too short" +// (measured: 7 bytes throws, 8 round-trips). The decoder has no such floor, so without this the +// failure lands only on the writer, as an IllegalArgumentException from inside a lazy. +private const val MIN_DICTIONARY_BYTES = 8 + /** * Copies [bytes] into a direct [ByteBuffer] -- brotli4j's `attachDictionary` requires a direct * buffer, a heap-backed one throws `IllegalArgumentException`. @@ -87,16 +92,40 @@ fun loadCompressionDictionary(db: SQLiteDatabase): ByteBuffer? { log.warn("CompressionDictionary row has a NULL data column; brotli content is handled without a dictionary.") return null } - // An empty blob would yield a 0-capacity buffer, which attachDictionary rejects -- - // every row's dictionary decode would then fail with nothing above DEBUG to say why. - if (bytes.isEmpty()) { - log.warn("CompressionDictionary row has an empty data column; brotli content is handled without a dictionary.") + // An empty blob would yield a 0-capacity buffer, which attachDictionary rejects, and the + // encoder refuses anything under MIN_DICTIONARY_BYTES outright. Either way a truncated + // dictionary is not usable, and treating it as absent keeps reader and writer agreeing -- + // where letting it through would fail every decode, or every compress, with nothing above + // DEBUG to say why. + if (bytes.size < MIN_DICTIONARY_BYTES) { + log.warn( + "CompressionDictionary row holds {} bytes, below the {} the encoder requires; " + + "brotli content is handled without a dictionary.", + bytes.size, + MIN_DICTIONARY_BYTES, + ) return null } toDirectByteBuffer(bytes) } } +/** + * Whether [db] declares a version whose brotli `Content` rows are dictionary-compressed. + * + * [loadCompressionDictionary] returns null both for a database that legitimately has no dictionary + * (below [DatabaseVersionResolver.MAJOR_VERSION_WITH_COMPRESSION_DICTIONARY]) and for one that + * should have a usable dictionary but does not -- table dropped, row missing, blob truncated. A + * reader cannot tell those apart usefully; it decodes plain either way and every dictionary- + * compressed row simply fails. A *writer* must, because the second case is a damaged database that + * can be repaired in place: rows written plain into it would still be plain afterwards, and would + * then be undecodable with no missing-rows check to catch them. + */ +fun expectsCompressionDictionary(db: SQLiteDatabase): Boolean { + val majorVersion = DatabaseVersionResolver.resolveMajorVersion(db) + return majorVersion != null && majorVersion >= DatabaseVersionResolver.MAJOR_VERSION_WITH_COMPRESSION_DICTIONARY +} + /** * Compresses and decompresses documentation `Content` blobs against [dictionary], the shared Brotli * dictionary from the database those blobs live in (see [loadCompressionDictionary]). A null @@ -109,6 +138,20 @@ fun loadCompressionDictionary(db: SQLiteDatabase): ByteBuffer? { class BrotliDictionaryCodec( private val dictionary: ByteBuffer?, ) { + init { + // The encoder would quietly accept a heap buffer (it copies into a direct one of its own), + // but attachDictionary on the decode side throws IllegalArgumentException. Without this a + // heap dictionary writes genuinely dictionary-compressed rows that every later read + // rejects -- and unchecked, so handleClient turns it into a 500 naming neither the row nor + // the dictionary. Fail at construction, where the caller can see why. + require(dictionary == null || dictionary.isDirect) { + "dictionary must be a direct ByteBuffer (see toDirectByteBuffer); a heap buffer compresses but cannot decompress" + } + require(dictionary == null || dictionary.capacity() >= MIN_DICTIONARY_BYTES) { + "dictionary must be at least $MIN_DICTIONARY_BYTES bytes, was ${dictionary?.capacity()}" + } + } + // Only the encoder needs the dictionary in prepared form, so a decode-only user (WebServer) // never pays for building it. `generate` advances the buffer's position to its limit, so it // gets a duplicate: the original is shared with attachDictionary, which reads the whole @@ -118,6 +161,19 @@ class BrotliDictionaryCodec( dictionary?.let { PreparedDictionaryGenerator.generate(it.duplicate()) } } + /** + * Builds the encoder's prepared dictionary now rather than on the first [compress]. + * + * It is `by lazy` so a decode-only user never pays for it, but that defers a ~780 KB direct + * allocation to whenever compression first happens -- which for the plugin installer is inside + * an open write transaction. Callers holding a lock around their compression should warm it + * first. + */ + fun warmUp() { + ensureBrotliAvailable() + preparedDictionary + } + /** * Compresses [input] at the quality and window size the offline documentation pipeline uses, * so content contributed at runtime is stored the same way as content built ahead of time. @@ -135,21 +191,37 @@ class BrotliDictionaryCodec( /** * Decompresses a `Content` blob read from the same database [dictionary] came from. * - * Throws `IOException` when [input] needed a dictionary and none was attached: the backward - * distances then reach outside the window, which any spec-compliant decoder rejects. That - * direction is reliable, and the writer depends on it. + * Throws `IOException` when [input] *referenced* a dictionary that is not attached: those + * backward distances reach outside the window, which any spec-compliant decoder rejects. + * + * Note the qualifier. What matters is whether the stream actually matched into the dictionary, + * not whether one was attached when it was written. Content with no such matches -- an already + * compressed image, a block of noise -- round-trips identically either way (measured both + * directions; see BrotliDictionaryDecodeTest). So a mismatched row set fails *non-uniformly*: + * a plugin's HTML raises IOException while its incompressible assets keep serving. * * A *wrong* dictionary is a different matter, and is not reliably detectable. One of a * different length, or with nothing valid at the offsets the stream references, usually * throws -- but one of the same length holding plausible bytes there decodes cleanly to * content that is simply wrong (verified: see BrotliDictionaryDecodeTest). So neither a throw * nor a success is evidence about *which* dictionary was used. + * + * Takes ownership of [input] and closes it, including when the decoder fails to start. */ fun decompress(input: InputStream): ByteArray { ensureBrotliAvailable() - return BrotliInputStream(input).use { stream -> - dictionary?.let { stream.attachDictionary(it) } - stream.readBytes() + // BrotliInputStream's constructor allocates native state and can throw, which would leave + // `input` open if it were built inside the use{} it is the subject of. + val stream = + try { + BrotliInputStream(input) + } catch (e: Throwable) { + input.close() + throw e + } + return stream.use { + dictionary?.let { dict -> it.attachDictionary(dict) } + it.readBytes() } } diff --git a/docs/adr/0001-prefer-room-for-persistence.md b/docs/adr/0001-prefer-room-for-persistence.md index 0944a429bd..6e53918192 100644 --- a/docs/adr/0001-prefer-room-for-persistence.md +++ b/docs/adr/0001-prefer-room-for-persistence.md @@ -32,8 +32,9 @@ If none of these hold, use Room. "It's a small table" or "I already know SQL" ar |---|---|---| | Symbol indexing | `lsp/indexing/.../SQLiteIndex.kt` | Performance/allocation-critical; granular schema & query control (condition 2). | | In-app tooltips | `idetooltips/.../ToolTipManager.kt` | Prebuilt DB opened read-only (condition 1). | -| In-app / plugin help | `plugin-manager/.../documentation/PluginDocumentationManager.kt` | Prebuilt help-content DB (condition 1). | +| In-app / plugin help | `plugin-manager/.../documentation/PluginDocumentationManager.kt` | Schema owned by `OfflineDocumentationTools`, shared across that boundary (condition 3). **Not** condition 1: this is the one component that writes to `documentation.db`, opening it `OPEN_READWRITE` to insert plugin-contributed tooltip and Tier 3 rows. Room is still wrong here — it would want to own and migrate a schema this repo is forbidden to change. | | Local web server | `app/.../localWebServer/WebServer.kt` | Reads databases (incl. project data) it doesn't own, read-only (conditions 1 & 3). | +| Documentation compression | `common/.../utils/DocumentationCompression.kt` | Reads `CompressionDictionary` from whichever `documentation.db` its caller opened, so the reader and the writer above cannot disagree about how a row is compressed (conditions 1 & 3). | The tooltips and in-app/plugin-help rows, plus the local web server's Tier 3 serving, all read `documentation.db` — see [docs/documentation-database.md](../documentation-database.md) for its schema and consumers. diff --git a/docs/documentation-database.md b/docs/documentation-database.md index 05e861a4e8..6ab520569c 100644 --- a/docs/documentation-database.md +++ b/docs/documentation-database.md @@ -2,7 +2,7 @@ Reference for `documentation.db`, the SQLite database backing all in-app help: Tier 1/2 tooltips, plus the Tier 3 web content they link to, served by `WebServer`. Read this before touching anything under `localWebServer/`, `idetooltips/`, or `plugin-manager/.../documentation/`, or before writing/editing SQL against this database. -This is a **read-only, prebuilt** database — CoGo never creates or migrates its schema at runtime (see [ADR 0001](adr/0001-prefer-room-for-persistence.md), exception 1). The schema is owned by the separate `OfflineDocumentationTools` project (the `docdb-studio` tool); **never change it from this repo.** +This is a **prebuilt** database — CoGo never creates or migrates its schema at runtime (see [ADR 0001](adr/0001-prefer-room-for-persistence.md), exception 3). Every consumer reads it; one, `PluginDocumentationManager`, also *writes* rows into it, inserting the tooltips and Tier 3 content a plugin contributes into tables it does not own (see *How CoGo talks to this database* below). The schema is owned by the separate `OfflineDocumentationTools` project (the `docdb-studio` tool); **never change it from this repo.** ## Where it lives diff --git a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt index 5af08aa158..8ec5a807b3 100644 --- a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt +++ b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt @@ -11,6 +11,7 @@ import com.itsaky.androidide.plugins.extensions.PluginTooltipEntry import com.itsaky.androidide.plugins.manager.pluginCategory import com.itsaky.androidide.resources.R import com.itsaky.androidide.utils.BrotliDictionaryCodec +import com.itsaky.androidide.utils.expectsCompressionDictionary import com.itsaky.androidide.utils.loadCompressionDictionary import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -206,21 +207,48 @@ class PluginDocumentationManager( return@withContext false } - // A definitive null means this database has no dictionary, so its brotli rows are plain - // and these must be too. A throw means the answer is merely unavailable right now: - // guessing either way writes rows WebServer cannot decode, so abandon the install and - // let verifyAndRecreateTier3Documentation retry on the next activation. + // Three outcomes, and only one of them is "write plain rows". + // + // A dictionary: compress against it. No dictionary in a database that never declared + // one: plain, which is what its reader expects. But a database that *should* have a + // usable dictionary and does not is damaged, not plain -- writing plain rows into it + // would leave them undecodable once the dictionary row is repaired in place, with no + // missing-rows check to catch them, since repairing a row does not drop this plugin's. + // A throw means the answer is merely unavailable right now. The last two both defer to + // the next activation rather than guessing. + // + // Closing runs through finally, not the catch: toDirectByteBuffer and Cursor.getBlob + // can raise OutOfMemoryError, which is an Error and would slip past catch(Exception), + // leaking a read-write handle on documentation.db and the plugin's AssetManager. + var codecLoaded = false val codec = try { - BrotliDictionaryCodec(loadCompressionDictionary(db)) + val dictionary = loadCompressionDictionary(db) + if (dictionary == null && expectsCompressionDictionary(db)) { + Log.e( + TAG, + "Database declares a compression dictionary but has no usable one; " + + "deferring Tier 3 install for $pluginId rather than writing plain rows", + ) + return@withContext false + } + BrotliDictionaryCodec(dictionary).also { codecLoaded = true } } catch (e: Exception) { if (e is CancellationException) throw e Log.e(TAG, "Cannot read the compression dictionary; deferring Tier 3 install for $pluginId", e) - db.close() - pluginAssets.close() return@withContext false + } finally { + if (!codecLoaded) { + db.close() + pluginAssets.close() + } } + // Force the encoder's prepared dictionary to be built now. It is `by lazy`, and its + // first use would otherwise land on the first compressed asset -- inside the write + // transaction opened below, holding the exclusive lock through a ~780 KB allocation. + codec.warmUp() + val resolver = ExtensionToContentTypeResolver() var inserted = 0 var skipped = 0 @@ -244,13 +272,8 @@ class PluginDocumentationManager( continue } - val payload = - if (row.compression == "brotli") { - codec.compress(asset.bytes) - } else { - asset.bytes - } - + // Validated before compressing: quality 11 on a large asset is seconds of work, and + // an asset about to be rejected for its path should not cost any of it. val safeRelative = try { normalizeLocalDocumentationPath(asset.relativePath) @@ -259,14 +282,35 @@ class PluginDocumentationManager( skipped++ continue } + + val payload = + if (row.compression == "brotli") { + codec.compress(asset.bytes) + } else { + asset.bytes + } + val basePath = "plugin/$pluginId/$safeRelative" insertContentChunked(db, basePath, payload, row.id) inserted++ } - db.setTransactionSuccessful() - Log.d(TAG, "Installed $inserted Tier 3 documents for plugin $pluginId (skipped=$skipped)") - true + if (inserted == 0 && skipped > 0) { + // The transaction opened by deleting this plugin's existing rows. Committing now + // would destroy content that was serving and replace it with nothing, then record + // that as a successful install. Roll back instead: the plugin ships assets this + // build cannot type, which is a packaging problem to surface, not one to apply. + Log.e( + TAG, + "Every Tier 3 asset for plugin $pluginId was skipped ($skipped); " + + "rolling back rather than leaving it with no content", + ) + false + } else { + db.setTransactionSuccessful() + Log.d(TAG, "Installed $inserted Tier 3 documents for plugin $pluginId (skipped=$skipped)") + true + } } catch (e: Exception) { if (e is CancellationException) throw e Log.e(TAG, "Failed to install Tier 3 docs for plugin $pluginId", e) @@ -379,11 +423,18 @@ class PluginDocumentationManager( // commit(), not apply(): this already runs on Dispatchers.IO, and losing the write to a process // death would delete and recompress every one of the plugin's assets at quality 11 next launch. private fun recordInstalledGeneration(pluginId: String) { - tier3Preferences().edit().putInt(pluginId, TIER3_COMPRESSION_GENERATION).commit() + // A dropped write recreates the exact loop commit() was chosen to avoid -- the next + // activation reads the 0 default and recompresses everything, and again after that -- so + // say so rather than discarding the result. + if (!tier3Preferences().edit().putInt(pluginId, TIER3_COMPRESSION_GENERATION).commit()) { + Log.w(TAG, "Could not record the Tier 3 compression generation for $pluginId; it will reinstall next activation") + } } private fun forgetInstalledGeneration(pluginId: String) { - tier3Preferences().edit().remove(pluginId).commit() + if (!tier3Preferences().edit().remove(pluginId).commit()) { + Log.w(TAG, "Could not clear the Tier 3 compression generation for $pluginId") + } } private fun tier3Preferences(): SharedPreferences = context.getSharedPreferences(TIER3_PREFS, Context.MODE_PRIVATE) From 9c15dc6dea210a77d7388d0922763e584e97c381 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 05:20:28 +0000 Subject: [PATCH 11/15] ADFA-5240: Close db and pluginAssets when codec warm-up fails codec.warmUp() ran between the codec-load block and the install try/finally, so a throw there -- an IOException when brotli's natives are unavailable, or a failure building the prepared dictionary -- leaked the read-write documentation.db handle and the plugin's AssetManager, and escaped to the caller instead of returning the normal failed-install false. Move the call inside the install try, before beginTransaction(). The existing finally already handles that path: db.inTransaction() is false so no endTransaction runs, both handles close, and the generation marker stays unstamped. Addresses the coderabbitai review finding on PR #1756 (discussion r3877541424). No regression test: plugin-manager has no harness for PluginDocumentationManager (see the rollback thread on this PR). Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01UQ38gkja1cYRakgfaddstz --- .../documentation/PluginDocumentationManager.kt | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt index 8ec5a807b3..eaf9d33d40 100644 --- a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt +++ b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt @@ -244,17 +244,21 @@ class PluginDocumentationManager( } } - // Force the encoder's prepared dictionary to be built now. It is `by lazy`, and its - // first use would otherwise land on the first compressed asset -- inside the write - // transaction opened below, holding the exclusive lock through a ~780 KB allocation. - codec.warmUp() - val resolver = ExtensionToContentTypeResolver() var inserted = 0 var skipped = 0 val installed = try { + // Force the encoder's prepared dictionary to be built now, before the + // transaction: it is `by lazy`, and its first use would otherwise land on the + // first compressed asset -- inside the write transaction, holding the exclusive + // lock through a ~780 KB allocation. Inside this try because warmUp can throw + // (an IOException when brotli's natives are unavailable), and a throw must + // close db and pluginAssets through the finally below and fail the install + // normally rather than leak both handles. + codec.warmUp() + db.beginTransaction() removePluginTier3Internal(db, pluginId) From a97b673c3010ecb5c9fa6351dcb8f85392f52570 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 14:48:06 +0000 Subject: [PATCH 12/15] ADFA-5240: Compress Tier 3 assets before opening the write transaction Quality-11 brotli on assets of up to 10 MB each ran inside the open write transaction, holding documentation.db's exclusive lock through seconds of CPU per asset while WebServer's readers stall -- the same lock cost the warmUp hoist in 9c15dc6 moved out, left in for the far larger encode. Walk and compress every asset into an in-memory list first, then open the transaction only for the delete and inserts. Failure semantics are unchanged: a compress failure now fails the install before the transaction ever opens (previously it rolled back), the all-skipped case leaves existing rows untouched instead of deleting and rolling back, the empty-asset-directory case still deletes and commits, and the generation marker is still stamped only after a commit. Addresses jatezzz's review finding 2 of 2 on PR #1756. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01UQ38gkja1cYRakgfaddstz --- .../PluginDocumentationManager.kt | 55 ++++++++++++------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt index eaf9d33d40..639dba989a 100644 --- a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt +++ b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt @@ -245,23 +245,24 @@ class PluginDocumentationManager( } val resolver = ExtensionToContentTypeResolver() - var inserted = 0 var skipped = 0 val installed = try { - // Force the encoder's prepared dictionary to be built now, before the - // transaction: it is `by lazy`, and its first use would otherwise land on the - // first compressed asset -- inside the write transaction, holding the exclusive - // lock through a ~780 KB allocation. Inside this try because warmUp can throw - // (an IOException when brotli's natives are unavailable), and a throw must - // close db and pluginAssets through the finally below and fail the install - // normally rather than leak both handles. + // Force the encoder's prepared dictionary to be built now rather than on the + // first compressed asset, and surface a missing brotli native as an IOException + // before any asset is walked. Inside this try because warmUp can throw, and a + // throw must close db and pluginAssets through the finally below and fail the + // install normally rather than leak both handles. codec.warmUp() - db.beginTransaction() - removePluginTier3Internal(db, pluginId) - + // Compress before the transaction, not inside it: quality 11 on assets of up + // to 10 MB each is seconds of CPU per asset, and an open write transaction + // holds documentation.db's exclusive lock, stalling WebServer's readers for + // all of it. Only the delete and inserts need the lock. The cost is holding + // every compressed payload in memory at once -- the same bytes the inserts + // below hand to SQLite. + val prepared = ArrayList() for (asset in Tier3AssetWalker.walk(pluginAssets, assetPath)) { val ext = asset.relativePath.substringAfterLast('.', "") if (ext.isEmpty()) { @@ -294,25 +295,29 @@ class PluginDocumentationManager( asset.bytes } - val basePath = "plugin/$pluginId/$safeRelative" - insertContentChunked(db, basePath, payload, row.id) - inserted++ + prepared.add(PreparedTier3Row("plugin/$pluginId/$safeRelative", payload, row.id)) } - if (inserted == 0 && skipped > 0) { - // The transaction opened by deleting this plugin's existing rows. Committing now - // would destroy content that was serving and replace it with nothing, then record - // that as a successful install. Roll back instead: the plugin ships assets this - // build cannot type, which is a packaging problem to surface, not one to apply. + if (prepared.isEmpty() && skipped > 0) { + // Deleting this plugin's existing rows and committing would destroy content + // that was serving and replace it with nothing, then record that as a + // successful install. Leave the rows untouched instead: the plugin ships + // assets this build cannot type, which is a packaging problem to surface, + // not one to apply. Log.e( TAG, "Every Tier 3 asset for plugin $pluginId was skipped ($skipped); " + - "rolling back rather than leaving it with no content", + "leaving its existing content in place", ) false } else { + db.beginTransaction() + removePluginTier3Internal(db, pluginId) + for (prep in prepared) { + insertContentChunked(db, prep.path, prep.payload, prep.contentTypeId) + } db.setTransactionSuccessful() - Log.d(TAG, "Installed $inserted Tier 3 documents for plugin $pluginId (skipped=$skipped)") + Log.d(TAG, "Installed ${prepared.size} Tier 3 documents for plugin $pluginId (skipped=$skipped)") true } } catch (e: Exception) { @@ -772,3 +777,11 @@ class PluginDocumentationManager( recreatedCount } } + +// One Tier 3 asset ready to insert, compressed outside the write transaction so the +// exclusive lock is never held through a quality-11 encode. +private class PreparedTier3Row( + val path: String, + val payload: ByteArray, + val contentTypeId: Long, +) From 5a850001874140d1e14bed6bb102fbde7eb04fa6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 18:39:23 +0000 Subject: [PATCH 13/15] ADFA-5240: Cap the bytes staged in memory before the insert transaction Compressing ahead of the transaction (a97b673) staged every payload in an unbounded list. Tier3AssetWalker caps each asset at 10 MiB but not the sum, so a plugin shipping enough valid assets could exhaust the heap before the transaction opens -- as an OutOfMemoryError, which escapes catch (Exception). Track the staged total and skip any asset that would push it past 32 MiB, through the same warn-and-skip path as an oversized asset. The first asset always fits (per-asset cap is 10 MiB), so the all-skipped failure path cannot trigger from this limit alone. Addresses the coderabbitai finding on PR #1756 (review 5053989580). Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01UQ38gkja1cYRakgfaddstz --- .../PluginDocumentationManager.kt | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt index 639dba989a..20d111ba6b 100644 --- a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt +++ b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt @@ -35,6 +35,13 @@ class PluginDocumentationManager( // Bumped whenever previously written Tier 3 rows can no longer be read as they are. // 1 was plain brotli; 2 compresses against the database's shared dictionary (ADFA-5240). private const val TIER3_COMPRESSION_GENERATION = 2 + + // Aggregate cap on the payload bytes staged in memory before the insert transaction + // opens. Tier3AssetWalker caps each asset at 10 MiB but nothing bounded the sum, and + // enough valid assets would exhaust the heap -- as an OutOfMemoryError, which escapes + // catch (Exception). A small multiple of the per-asset cap: past it, an asset takes + // the same skip path as an oversized one. + private const val MAX_STAGED_BYTES = 32L * 1024L * 1024L } private val databaseName = "documentation.db" @@ -261,8 +268,9 @@ class PluginDocumentationManager( // holds documentation.db's exclusive lock, stalling WebServer's readers for // all of it. Only the delete and inserts need the lock. The cost is holding // every compressed payload in memory at once -- the same bytes the inserts - // below hand to SQLite. + // below hand to SQLite -- bounded by MAX_STAGED_BYTES. val prepared = ArrayList() + var stagedBytes = 0L for (asset in Tier3AssetWalker.walk(pluginAssets, assetPath)) { val ext = asset.relativePath.substringAfterLast('.', "") if (ext.isEmpty()) { @@ -295,6 +303,16 @@ class PluginDocumentationManager( asset.bytes } + if (stagedBytes + payload.size > MAX_STAGED_BYTES) { + Log.w( + TAG, + "Skipping Tier 3 asset '${asset.relativePath}' - staging it would exceed " + + "the $MAX_STAGED_BYTES byte aggregate limit", + ) + skipped++ + continue + } + stagedBytes += payload.size prepared.add(PreparedTier3Row("plugin/$pluginId/$safeRelative", payload, row.id)) } From c73ea242f0af139fd4a23d2f6c4727ffbcecf78f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 18:47:33 +0000 Subject: [PATCH 14/15] ADFA-5240: Fail the install when staged Tier 3 payloads exceed the cap 5a85000 routed aggregate overflow through the per-asset skip path, which is wrong for this limit: skipping a later asset leaves the earlier ones in the staging list, the transaction then deletes every existing plugin row and commits only that subset, and the generation marker records it as current -- so verification never reinstalls the dropped documents. Abort instead, before the transaction opens: existing rows and the generation stay as they are, the install returns false through the same path as a warm-up or compression failure, and it retries on the plugin's next activation. The per-asset oversized skip in Tier3AssetWalker is unchanged. Addresses the coderabbitai finding on PR #1756 (discussion r3883162508). No unit test: plugin-manager has no harness for PluginDocumentationManager (settled earlier on this PR). Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01UQ38gkja1cYRakgfaddstz --- .../PluginDocumentationManager.kt | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt index 20d111ba6b..b60f15ed32 100644 --- a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt +++ b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt @@ -39,8 +39,8 @@ class PluginDocumentationManager( // Aggregate cap on the payload bytes staged in memory before the insert transaction // opens. Tier3AssetWalker caps each asset at 10 MiB but nothing bounded the sum, and // enough valid assets would exhaust the heap -- as an OutOfMemoryError, which escapes - // catch (Exception). A small multiple of the per-asset cap: past it, an asset takes - // the same skip path as an oversized one. + // catch (Exception). A small multiple of the per-asset cap: past it, the whole install + // fails rather than committing a subset of the plugin's documents. private const val MAX_STAGED_BYTES = 32L * 1024L * 1024L } @@ -271,6 +271,7 @@ class PluginDocumentationManager( // below hand to SQLite -- bounded by MAX_STAGED_BYTES. val prepared = ArrayList() var stagedBytes = 0L + var overLimit = false for (asset in Tier3AssetWalker.walk(pluginAssets, assetPath)) { val ext = asset.relativePath.substringAfterLast('.', "") if (ext.isEmpty()) { @@ -304,19 +305,26 @@ class PluginDocumentationManager( } if (stagedBytes + payload.size > MAX_STAGED_BYTES) { - Log.w( - TAG, - "Skipping Tier 3 asset '${asset.relativePath}' - staging it would exceed " + - "the $MAX_STAGED_BYTES byte aggregate limit", - ) - skipped++ - continue + overLimit = true + break } stagedBytes += payload.size prepared.add(PreparedTier3Row("plugin/$pluginId/$safeRelative", payload, row.id)) } - if (prepared.isEmpty() && skipped > 0) { + if (overLimit) { + // Skipping the overflow and committing what fit would delete every existing + // row and record the subset as current -- the dropped documents would then + // never be reinstalled, since verification trusts the generation marker. + // Fail before any delete instead; existing rows and the generation stay as + // they are, and the install retries on the plugin's next activation. + Log.e( + TAG, + "Tier 3 assets for plugin $pluginId exceed the $MAX_STAGED_BYTES byte " + + "staging limit; failing the install rather than committing a subset", + ) + false + } else if (prepared.isEmpty() && skipped > 0) { // Deleting this plugin's existing rows and committing would destroy content // that was serving and replace it with nothing, then record that as a // successful install. Leave the rows untouched instead: the plugin ships From 16ed562fa2bdb0f4d68207f56823ea08256d0612 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Aug 2026 00:08:43 +0000 Subject: [PATCH 15/15] ADFA-5240: Fix the dictionary-load doc claim and four exception logs documentation-database.md said the dictionary loads on the first fetch that needs it; readContent primes the codec before it reads the row's compression -- deliberately, so a database's dictionary loads on its first lookup rather than its first brotli lookup -- so the doc now says that. DocumentationContentSource's four one-shot error logs passed e.message as a placeholder, dropping the exception type and stack trace; they now pass e as the final SLF4J argument, matching the swap-failure logs in the same file. The best-effort priming warn stays message-only: it can recur per lookup, and a row that genuinely needs the dictionary still fails loudly from the decode itself. Addresses the coderabbitai findings on the 20df552 merge (PR #1756). Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01UQ38gkja1cYRakgfaddstz --- .../documentation/DocumentationContentSource.kt | 8 ++++---- docs/documentation-database.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/common/src/main/java/com/itsaky/androidide/documentation/DocumentationContentSource.kt b/common/src/main/java/com/itsaky/androidide/documentation/DocumentationContentSource.kt index 291bf573b7..b63a347017 100644 --- a/common/src/main/java/com/itsaky/androidide/documentation/DocumentationContentSource.kt +++ b/common/src/main/java/com/itsaky/androidide/documentation/DocumentationContentSource.kt @@ -236,7 +236,7 @@ class DocumentationContentSource( try { readContent(database, path) } catch (e: Exception) { - log.error("Cannot read '{}': {}", path, e.message) + log.error("Cannot read '{}'", path, e) DocumentationLookup.Failed(e) } } @@ -347,7 +347,7 @@ class DocumentationContentSource( try { database?.close() } catch (e: Exception) { - log.error("Cannot close the documentation database: {}", e.message) + log.error("Cannot close the documentation database", e) } database = null } @@ -366,7 +366,7 @@ class DocumentationContentSource( open() database != null } catch (e: Exception) { - log.error("Cannot open the documentation database '{}': {}", databaseFile, e.message) + log.error("Cannot open the documentation database '{}'", databaseFile, e) false } } @@ -657,7 +657,7 @@ class DocumentationContentSource( try { previous?.close() } catch (e: Exception) { - log.error("Cannot close previous database: {}", e.message) + log.error("Cannot close previous database", e) } } diff --git a/docs/documentation-database.md b/docs/documentation-database.md index 73e44f6bd0..793cd11a3e 100644 --- a/docs/documentation-database.md +++ b/docs/documentation-database.md @@ -63,7 +63,7 @@ CREATE TABLE Tooltips ( ### Supporting tables - **`DocumentationDatabaseVersion(major, minor, patch, who, comment, changeTime)`** — the database's own semver (ADFA-5220), replacing the heuristics that used to infer the format from which tables happened to exist. Append-only: each change is another `INSERT`, so the **row inserted last** is the current version, not the highest one ever recorded — a rebuild from an older content set is a downgrade and has to read as one (`DatabaseVersionResolver.resolveMajorVersion`, which returns null for a database predating the table). `MAJOR >= 2` is what tells the app its brotli `Content` rows are dictionary-compressed; below that, `DocumentationContentSource` neither reads nor attaches `CompressionDictionary`. Gating on the declared version rather than on the table's presence matters in both directions: a database can carry the dictionary table while its content is still plain brotli (every brotli row would then be decoded with a dictionary attached and fail outright, since ADFA-5240 removed the plain-decode retry that used to mask this), and a migrated database that lost the table fails loudly instead of quietly. -- **`CompressionDictionary(id, data)`** — single-row table (`id INTEGER PRIMARY KEY CHECK (id = 1)`) holding the raw Brotli dictionary every ADFA-5153-migrated `compression = 'brotli'` `Content` row is compressed against. Trained once, from a representative sample across the whole `Content` table, by `OfflineDocumentationTools`' `migrate_content_to_dictionary_brotli.py` / `populate_db.py` (never retrained after that — a dictionary-compressed row is only decodable against the exact dictionary it was compressed with, so replacing it would silently orphan every already-migrated row). Shipping the dictionary inside `documentation.db` itself, rather than as a separate bundled asset, keeps it version-locked to the content compressed against it. Both the reader and the writer load it through `loadCompressionDictionary` in `:common`, so they cannot disagree about whether a given database's brotli rows carry a dictionary — a row compressed against one the reader won't attach is undecodable, and vice versa. `DocumentationContentSource` loads it lazily -- not merely from opening or swapping databases, but on the first content fetch that needs it after the active database changes, and only when `DocumentationDatabaseVersion` declares `MAJOR >= 2` (see above) -- and caches it from then on, reloading again only on the next database change (a swap can bring in a database with a different dictionary or none, so it can't stay cached across one). Each row is then decoded exactly once, with the dictionary attached or not according to that version; there is no retry, because there is no longer a second way a row might have been written. +- **`CompressionDictionary(id, data)`** — single-row table (`id INTEGER PRIMARY KEY CHECK (id = 1)`) holding the raw Brotli dictionary every ADFA-5153-migrated `compression = 'brotli'` `Content` row is compressed against. Trained once, from a representative sample across the whole `Content` table, by `OfflineDocumentationTools`' `migrate_content_to_dictionary_brotli.py` / `populate_db.py` (never retrained after that — a dictionary-compressed row is only decodable against the exact dictionary it was compressed with, so replacing it would silently orphan every already-migrated row). Shipping the dictionary inside `documentation.db` itself, rather than as a separate bundled asset, keeps it version-locked to the content compressed against it. Both the reader and the writer load it through `loadCompressionDictionary` in `:common`, so they cannot disagree about whether a given database's brotli rows carry a dictionary — a row compressed against one the reader won't attach is undecodable, and vice versa. `DocumentationContentSource` loads it lazily -- not at database open or swap time, but on the first content lookup after the active database changes, whatever that row's own compression, and only when `DocumentationDatabaseVersion` declares `MAJOR >= 2` (see above) -- and caches it from then on, reloading again only on the next database change (a swap can bring in a database with a different dictionary or none, so it can't stay cached across one). Each row is then decoded exactly once, with the dictionary attached or not according to that version; there is no retry, because there is no longer a second way a row might have been written. - **`Templates(id, name, content)`** — Pebble template source, keyed by id (and by `name` for well-known templates like `bookshelf`). Referenced by `Content.templateId`. - **`Bookshelf(contentID, bookCategoryID, title, description)`** / **`BookCategories(id, category, description)`** — the Dynamic Bookshelf: one row per "book" (PDF or similar), linked to its Tier 3 page via `contentID` -> `Content.id`. Two DB triggers keep `Bookshelf` in sync when a PDF row is inserted/deleted from `Content`; `title`/`description` don't come from those triggers and must be set by hand. Non-PDF books need a separate ingestion path (plugin-provided, e.g. via `PluginDocumentationManager`). - **`LastChange(documentationSet, changeTime, who)`** — audit trail for edits made through `docdb-studio`; not shown to end users. `DatabaseVersionResolver` reads the `documentationSet = 'wholedb'` row to report the DB's build/edit stamp in debug logging, falling back to the most recent row of any set if `'wholedb'` is missing.