diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index c20febe171..178fae9afa 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -102,9 +102,9 @@ These structural facts shape every module. Day-to-day build *commands* live in ` > > **Recent Projects** is the reference example of the default: `app/src/main/java/com/itsaky/androidide/roomData/recentproject/` (`RecentProjectRoomDatabase`, `@Database version = 4` with migrations 1→4; `RecentProjectDao`; the `RecentProject` `@Entity` → table `recent_project_table`). It's provided via Koin in `di/AppModule.kt` and consumed by `MainViewModel`, `RecentProjectsViewModel`, `MainActivity`, `ProjectInfoBottomSheet`, and `ProjectCreationManager`. > -> **Raw SQLite is allowed only when** the database is prebuilt and opened read-only, the data is performance/allocation-critical and needs granular schema control, or the schema is shared across a process/component boundary. Current exceptions: symbol indexing (`lsp/indexing/SQLiteIndex.kt`), tooltips (`idetooltips/ToolTipManager.kt`), in-app/plugin help (`plugin-manager/.../documentation/PluginDocumentationManager.kt`), and documentation serving (`common/.../documentation/DocumentationContentSource.kt`, the one pipeline behind both the in-process WebView transport and `app/.../localWebServer/WebServer.kt`). The `androidx.room:*` strings in `editor`'s `GroovyAutoComplete` are autocomplete suggestions for the *user's* code, not CoGo persistence. +> **Raw SQLite is allowed only when** the database is prebuilt and opened read-only, the data is performance/allocation-critical and needs granular schema control, or the schema is shared across a process/component boundary. Current exceptions: symbol indexing (`lsp/indexing/SQLiteIndex.kt`), tooltips (`idetooltips/ToolTipManager.kt`), in-app/plugin help (`plugin-manager/.../documentation/PluginDocumentationManager.kt` — the one *writer*, inserting plugin-contributed rows into a schema owned elsewhere), the shared compression-dictionary loader (`common/.../utils/DocumentationCompression.kt`), and documentation serving (`common/.../documentation/DocumentationContentSource.kt`, the one read pipeline behind both the in-process WebView transport and `app/.../localWebServer/WebServer.kt`). The `androidx.room:*` strings in `editor`'s `GroovyAutoComplete` are autocomplete suggestions for the *user's* code, not CoGo persistence. > -> The tooltip, in-app/plugin-help, and local-web-server exceptions all read `documentation.db`, the prebuilt Tier 1/2/3 help database — see [docs/documentation-database.md](docs/documentation-database.md) for its schema and how each consumer queries it. +> The tooltip, in-app/plugin-help, compression-dictionary and documentation-serving exceptions all use `documentation.db`, the prebuilt Tier 1/2/3 help database — see [docs/documentation-database.md](docs/documentation-database.md) for its schema and how each consumer queries it. ## State Management diff --git a/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt index 3f7ebc27ba..0ac31b039a 100644 --- a/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt +++ b/app/src/test/java/com/itsaky/androidide/localWebServer/BrotliDictionaryDecodeTest.kt @@ -1,22 +1,21 @@ package com.itsaky.androidide.localWebServer -// The decode helpers moved to common with the shared content source (ADFA-5176); these tests stay +// The codec and chunk helpers under test live in common (ADFA-5176/ADFA-5240); these tests stay // here, where the brotli4j host-native test wiring lives. import com.aayushatharva.brotli4j.Brotli4jLoader import com.aayushatharva.brotli4j.decoder.BrotliInputStream -import com.aayushatharva.brotli4j.encoder.BrotliOutputStream -import com.aayushatharva.brotli4j.encoder.Encoder import com.itsaky.androidide.documentation.chunksAsStream import com.itsaky.androidide.documentation.joinChunks -import com.itsaky.androidide.documentation.toDirectByteBuffer +import com.itsaky.androidide.utils.BrotliDictionaryCodec +import com.itsaky.androidide.utils.toDirectByteBuffer import org.junit.Assert.assertArrayEquals import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse import org.junit.Assert.assertSame import org.junit.Assert.assertThrows import org.junit.BeforeClass import org.junit.Test import java.io.ByteArrayInputStream -import java.io.ByteArrayOutputStream import java.io.IOException import java.nio.ByteBuffer import java.nio.charset.StandardCharsets @@ -185,29 +184,138 @@ class BrotliDictionaryDecodeTest { } @Test - fun `dictionary-free plugin content fails with a dictionary attached but decodes plain`() { - // Regression coverage for the WebServer.decompressBrotli fallback: plugin-contributed - // Tier 3 docs (PluginDocumentationManager/BrotliCompressor) are compressed with the same - // encoder params (quality 11, window 24) but no dictionary, coexisting in the same Content - // table as ADFA-5153-migrated, dictionary-compressed rows. + fun `plugin content compressed against the dictionary decodes the way WebServer reads it`() { + // The ADFA-5240 contract: PluginDocumentationManager encodes Tier 3 rows through + // BrotliDictionaryCodec, and WebServer reads every brotli row by attaching the same + // dictionary. The dictionary here is the CLI-trained fixture, so this also covers the + // cross-tool half -- brotli4j's encoder against a dictionary the Python pipeline produced. val dictionary = decodeBase64ToDirectBuffer(dictionaryBase64) - val plaintext = "plugin-contributed Tier 3 content, compressed with no dictionary" - val expected = plaintext.toByteArray(StandardCharsets.UTF_8) - val compressed = - ByteArrayOutputStream() - .apply { - BrotliOutputStream(this, Encoder.Parameters().setQuality(11).setWindow(24)).use { it.write(expected) } - }.toByteArray() + val expected = "plugin-contributed Tier 3 content, compressed against the shared dictionary".toByteArray(StandardCharsets.UTF_8) - assertThrows(IOException::class.java) { + val compressed = BrotliDictionaryCodec(dictionary).compress(expected) + + val result = BrotliInputStream(ByteArrayInputStream(compressed)).use { stream -> stream.attachDictionary(dictionary) stream.readBytes() } + assertArrayEquals(expected, result) + } + + @Test + fun `compressing does not drain the caller's dictionary buffer`() { + // PreparedDictionaryGenerator.generate consumes its argument, leaving position at limit. + // attachDictionary happens to ignore position -- it reads the whole capacity -- so a round + // trip alone cannot tell whether the codec drained the caller's buffer. Assert the buffer + // state directly, or the defensive duplicate() in BrotliDictionaryCodec is unpinned and a + // future brotli4j that honours position breaks silently. + val dictionary = decodeBase64ToDirectBuffer(dictionaryBase64) + val positionBefore = dictionary.position() + + BrotliDictionaryCodec(dictionary).compress("docs-sidebar toc-element".toByteArray(StandardCharsets.UTF_8)) + + assertEquals("compress() consumed the dictionary buffer it was given", positionBefore, dictionary.position()) + } + + @Test + fun `the codec reuses one dictionary buffer across compress and decompress`() { + val dictionary = decodeBase64ToDirectBuffer(dictionaryBase64) + val codec = BrotliDictionaryCodec(dictionary) + + repeat(3) { round -> + val expected = "round $round: docs-sidebar toc-element kotlin interface".toByteArray(StandardCharsets.UTF_8) + val result = codec.decompress(ByteArrayInputStream(codec.compress(expected))) + assertArrayEquals(expected, result) } - val plainResult = BrotliInputStream(ByteArrayInputStream(compressed)).use { it.readBytes() } - assertArrayEquals(expected, plainResult) + // The fixture from the offline pipeline must still decode through the same buffer. + assertArrayEquals( + Base64.getDecoder().decode(expectedBase64), + codec.decompress(ByteArrayInputStream(Base64.getDecoder().decode(compressedBase64))), + ) + } + + @Test + fun `dictionary-compressed plugin content is not readable without the dictionary`() { + // WebServer decodes a brotli row exactly once, with the dictionary attached or not + // according to the database's declared version -- there is no retry to paper over a + // mismatch. This is what makes that safe: a row written against the dictionary cannot be + // silently misread as a plain one, it fails loudly. + val dictionary = decodeBase64ToDirectBuffer(dictionaryBase64) + val expected = "plugin-contributed Tier 3 content".toByteArray(StandardCharsets.UTF_8) + + val compressed = BrotliDictionaryCodec(dictionary).compress(expected) + + assertThrows(IOException::class.java) { + BrotliDictionaryCodec(null).decompress(ByteArrayInputStream(compressed)) + } + } + + @Test + fun `the wrong dictionary can decode without error to the wrong bytes`() { + // The reason WebServer.switchToDatabase resets its codec instead of only marking it stale: + // if a failed dictionary reload let the previous database's codec serve the new database's + // rows, this is what a client could get -- a 200 carrying different bytes than were stored, + // with nothing thrown. A dictionary-free codec fails such a row loudly instead, which is + // why that is the safe thing to fall back to. + // + // The two dictionaries here are the same length and differ only in bytes the payload + // actually references, which is what makes the failure silent. A wrong dictionary of a + // different length, or one whose referenced offsets hold nothing valid, throws instead -- + // so a throw does not prove the dictionary was right, and a success does not either. + val base = "docs-sidebar toc-element kotlin interface companion object page.peb ".repeat(400) + val dictionary = toDirectByteBuffer(base.toByteArray(StandardCharsets.UTF_8)) + val lookalike = toDirectByteBuffer(base.replace("kotlin", "scalax").toByteArray(StandardCharsets.UTF_8)) + val expected = "docs-sidebar toc-element kotlin interface companion object page.peb ".repeat(6).toByteArray(StandardCharsets.UTF_8) + + val compressed = BrotliDictionaryCodec(dictionary).compress(expected) + + val decoded = BrotliDictionaryCodec(lookalike).decompress(ByteArrayInputStream(compressed)) + assertEquals("the wrong dictionary still produced a full-length result", expected.size, decoded.size) + assertFalse( + "decoding with the wrong dictionary must not be assumed detectable by failure", + expected.contentEquals(decoded), + ) + + // ... while no dictionary at all is reliably rejected, which is what the writer relies on. + assertThrows(IOException::class.java) { + BrotliDictionaryCodec(null).decompress(ByteArrayInputStream(compressed)) + } + } + + @Test + fun `a legacy plain row fails when the dictionary is attached, unless it never referenced one`() { + // What makes rows left plain by an earlier build fail loudly rather than serve wrong bytes, + // now that WebServer decodes once with no retry. The failure is content-dependent, which is + // the part worth pinning: a stream only breaks on the dictionary if it matched into it, so + // a plugin's HTML 500s while its incompressible assets keep serving. Anyone reading a + // half-broken plugin's pages needs to know that is one cause, not two. + val dictionary = decodeBase64ToDirectBuffer(dictionaryBase64) + val plainCodec = BrotliDictionaryCodec(null) + + val docLike = "docs-sidebar toc-element kotlin interface companion ".repeat(200).toByteArray(StandardCharsets.UTF_8) + val asLegacyRow = plainCodec.compress(docLike) + assertThrows(IOException::class.java) { + BrotliDictionaryCodec(dictionary).decompress(ByteArrayInputStream(asLegacyRow)) + } + + // Nothing here matches the dictionary, so attaching one changes nothing. + val noise = ByteArray(64 * 1024).also { java.util.Random(5240).nextBytes(it) } + val noiseRow = plainCodec.compress(noise) + assertArrayEquals(noise, BrotliDictionaryCodec(dictionary).decompress(ByteArrayInputStream(noiseRow))) + } + + @Test + fun `a null dictionary round-trips as plain brotli`() { + // A database predating ADFA-5153 declares no dictionary, so both sides fall to plain + // brotli -- the writer must not attach one there either. + val expected = "content for a database with no CompressionDictionary".toByteArray(StandardCharsets.UTF_8) + val codec = BrotliDictionaryCodec(null) + + val compressed = codec.compress(expected) + + assertArrayEquals(expected, codec.decompress(ByteArrayInputStream(compressed))) + assertArrayEquals(expected, BrotliInputStream(ByteArrayInputStream(compressed)).use { it.readBytes() }) } @Test diff --git a/common/build.gradle.kts b/common/build.gradle.kts index 7dbd920da8..e9b8e9df92 100755 --- a/common/build.gradle.kts +++ b/common/build.gradle.kts @@ -51,6 +51,12 @@ dependencies { api(projects.subprojects.flashbar) implementation(libs.monitor) + // BrotliDictionaryCodec's tests live in :app (BrotliDictionaryDecodeTest), not here, so + // `:common:test` passing says nothing about it. They need brotli4j's per-OS/arch desktop + // native, and the host dispatch for that is wired only in app/build.gradle.kts -- moving them + // means extracting that into build-logic first, where a third copy of the same dispatch + // already lives. Tracked separately; jacocoAggregateReport runs both modules' suites, so CI + // coverage is unaffected either way. testImplementation(projects.testing.common) testImplementation(libs.tests.kotlinx.coroutines) testImplementation(libs.tests.google.truth) diff --git a/common/src/main/java/com/itsaky/androidide/documentation/DocumentationContentSource.kt b/common/src/main/java/com/itsaky/androidide/documentation/DocumentationContentSource.kt index 79d5c574b3..b63a347017 100644 --- a/common/src/main/java/com/itsaky/androidide/documentation/DocumentationContentSource.kt +++ b/common/src/main/java/com/itsaky/androidide/documentation/DocumentationContentSource.kt @@ -18,13 +18,12 @@ package com.itsaky.androidide.documentation import android.database.sqlite.SQLiteDatabase -import com.aayushatharva.brotli4j.Brotli4jLoader -import com.aayushatharva.brotli4j.decoder.BrotliInputStream import com.google.gson.Gson import com.google.gson.GsonBuilder import com.google.gson.ToNumberPolicy import com.google.gson.reflect.TypeToken -import com.itsaky.androidide.utils.DatabaseVersionResolver +import com.itsaky.androidide.utils.BrotliDictionaryCodec +import com.itsaky.androidide.utils.loadCompressionDictionary import io.pebbletemplates.pebble.PebbleEngine import io.pebbletemplates.pebble.loader.StringLoader import io.pebbletemplates.pebble.template.PebbleTemplate @@ -32,12 +31,10 @@ import org.slf4j.LoggerFactory import java.io.ByteArrayInputStream import java.io.Closeable import java.io.File -import java.io.IOException import java.io.InputStream import java.io.SequenceInputStream import java.io.StringWriter import java.net.URLDecoder -import java.nio.ByteBuffer import java.text.SimpleDateFormat import java.util.Collections import java.util.Date @@ -49,26 +46,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock import kotlin.concurrent.read import kotlin.concurrent.write -/** - * Copies [bytes] into a direct [ByteBuffer] -- brotli4j's `attachDictionary` requires a direct - * buffer, a heap-backed one throws `IllegalArgumentException`. - * - * The capacity must be exactly [bytes]`.size`: `attachDictionary` reads the whole capacity and - * ignores position/limit, so trailing slack from an over-allocated buffer is treated as dictionary - * content and every decode then fails with `IOException: corrupted input`. - * - * @param bytes The bytes to copy. - * @return A direct byte buffer containing the copied bytes, positioned at the beginning. - */ -fun toDirectByteBuffer(bytes: ByteArray): ByteBuffer = - ByteBuffer.allocateDirect(bytes.size).apply { - put(bytes) - flip() - } - /** * Reads [chunks] back to back as one stream, without concatenating them into a new array. - * Cheap to build twice, which the no-dictionary retry in [DocumentationContentSource] relies on. */ fun chunksAsStream(chunks: List): InputStream = SequenceInputStream(Collections.enumeration(chunks.map { ByteArrayInputStream(it) })) @@ -136,8 +115,10 @@ data class RequestLookup( /** * Reads documentation content out of `documentation.db`: the row lookup, reassembly of chunked - * rows, the shared-dictionary Brotli decode (ADFA-5153), and the swap to a newer database dropped - * on the sdcard. + * rows, the shared-dictionary Brotli decode (ADFA-5153) -- one pass through + * [BrotliDictionaryCodec], with no plain-decode retry, since every brotli row in a + * dictionary-declaring database is compressed against that dictionary, plugin-contributed rows + * included (ADFA-5240) -- and the swap to a newer database dropped on the sdcard. * * One pipeline with two callers (ADFA-5176): `WebServer`, which wraps it in HTTP, and * [DocumentationRequestInterceptor], which answers a WebView in-process with no socket at all. A row @@ -197,11 +178,13 @@ class DocumentationContentSource( @Volatile private var failedInstalledSwapTimestamp: Long = -1 - // The dictionary the Content rows are compressed against. Loaded on the first decode that - // needs it after a swap rather than eagerly, and then cached for that database. Null when the - // active database predates the dictionary migration -- CompressionDictionary won't exist. - private var compressionDictionary: ByteBuffer? = null - private var compressionDictionaryStale = true + // Decodes Content's brotli rows against the shared dictionary they were compressed with (see + // ADFA-5153). Built on the first read that needs it after a swap rather than eagerly, then + // cached for that database. Holds no dictionary -- and so decodes plain brotli -- when the + // active database declares a version below the dictionary migration. Access only through + // [codec], which rebuilds it when stale. + private var codec: BrotliDictionaryCodec? = null + private var codecStale = true private val pebbleEngine = PebbleEngine.Builder().loader(StringLoader()).build() @@ -253,7 +236,7 @@ class DocumentationContentSource( try { readContent(database, path) } catch (e: Exception) { - log.error("Cannot read '{}': {}", path, e.message) + log.error("Cannot read '{}'", path, e) DocumentationLookup.Failed(e) } } @@ -364,7 +347,7 @@ class DocumentationContentSource( try { database?.close() } catch (e: Exception) { - log.error("Cannot close the documentation database: {}", e.message) + log.error("Cannot close the documentation database", e) } database = null } @@ -383,7 +366,7 @@ class DocumentationContentSource( open() database != null } catch (e: Exception) { - log.error("Cannot open the documentation database '{}': {}", databaseFile, e.message) + log.error("Cannot open the documentation database '{}'", databaseFile, e) false } } @@ -408,7 +391,7 @@ class DocumentationContentSource( // the next read retries, and a brotli row that genuinely cannot resolve its dictionary still // fails loudly from decompressBrotli. try { - compressionDictionary(database) + codec(database) } catch (e: Exception) { log.warn("Could not prime the compression dictionary; will retry on the next read: {}", e.message) } @@ -519,20 +502,11 @@ class DocumentationContentSource( } /** - * Ensures Brotli native support is available for content decoding. - * - * @throws IOException If the Brotli native library cannot be loaded. - */ - private fun ensureBrotliAvailable() { - try { - Brotli4jLoader.ensureAvailability() - } catch (e: UnsatisfiedLinkError) { - throw IOException("brotli4j's native library is unavailable, so brotli content cannot be decoded", e) - } - } - - /** - * Decompresses Brotli-compressed content, retrying without the database dictionary when dictionary-based decoding fails. + * Decompresses one Brotli-compressed Content row, attaching the shared dictionary when the + * active database declares one. Every brotli row in such a database is compressed against it, + * whether built offline or contributed by a plugin (ADFA-5240), so a single decode is enough + * and a failure is a real failure -- not, as it once was, a row that might simply have been + * written the other way. * * @param database The database used to obtain the Brotli dictionary. * @param chunks The compressed content chunks. @@ -541,97 +515,30 @@ class DocumentationContentSource( private fun decompressBrotli( database: SQLiteDatabase, chunks: List, - ): ByteArray { - ensureBrotliAvailable() - val dictionary = compressionDictionary(database) - if (dictionary != null) { - try { - return BrotliInputStream(chunksAsStream(chunks)).use { stream -> - stream.attachDictionary(dictionary) - stream.readBytes() - } - } catch (e: IOException) { - log.debug( - "Dictionary decode failed for a brotli row (likely dictionary-free plugin content); retrying without a dictionary: {}", - e.message, - ) - } - } - - return BrotliInputStream(chunksAsStream(chunks)).use { it.readBytes() } - } + ): ByteArray = codec(database).decompress(chunksAsStream(chunks)) /** - * Loads the active database's shared compression dictionary when needed. + * The codec for the active database, (re)built on the first use after a swap. + * + * Only clears the staleness flag on a clean build -- a definitive dictionary or a definitive + * absence, per [loadCompressionDictionary]'s contract -- so an unexpected exception leaves it + * set and the next read retries, rather than caching a transient failure as "no dictionary" + * for the rest of this database's lifetime. * * @param database The active documentation database. - * @return The dictionary as a direct byte buffer, or `null` when the database has no usable dictionary. + * @return The codec holding that database's dictionary, dictionary-free when it declares none. */ - private fun compressionDictionary(database: SQLiteDatabase): ByteBuffer? = + private fun codec(database: SQLiteDatabase): BrotliDictionaryCodec = synchronized(this) { - if (compressionDictionaryStale) { - compressionDictionary = dictionaryBytes(database)?.let { toDirectByteBuffer(it) } - compressionDictionaryStale = false + var current = codec + if (codecStale || current == null) { + current = BrotliDictionaryCodec(loadCompressionDictionary(database)) + codec = current + codecStale = false } - compressionDictionary - } - - /** - * Loads the Brotli compression dictionary declared by the database. - * - * @return The dictionary bytes, or `null` when the database does not support a dictionary or has no usable dictionary row. - */ - private fun dictionaryBytes(database: SQLiteDatabase): ByteArray? { - val majorVersion = DatabaseVersionResolver.resolveMajorVersion(database) - if (majorVersion == null || - majorVersion < DatabaseVersionResolver.MAJOR_VERSION_WITH_COMPRESSION_DICTIONARY - ) { - log.warn( - "Database declares documentation version {}, below {}; decoding brotli content without a dictionary.", - majorVersion ?: "none", - DatabaseVersionResolver.MAJOR_VERSION_WITH_COMPRESSION_DICTIONARY, - ) - return null + current } - val tableExists = - database - .rawQuery( - "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'CompressionDictionary'", - null, - ).use { it.moveToFirst() } - if (!tableExists) { - log.warn("CompressionDictionary table not found; decoding brotli content without a dictionary.") - return null - } - - return database.rawQuery("SELECT data FROM CompressionDictionary WHERE id = 1", null).use { cursor -> - if (!cursor.moveToFirst()) { - log.warn("CompressionDictionary table is empty; decoding brotli content without a dictionary.") - return null - } - - val bytes = cursor.getBlob(0) - when { - bytes == null -> { - log.warn("CompressionDictionary row has a NULL data column; decoding brotli content without a dictionary.") - null - } - - // An empty blob yields a 0-capacity buffer, which attachDictionary rejects -- every - // decode would then fail with nothing above DEBUG to say why. - bytes.isEmpty() -> { - log.warn("CompressionDictionary row has an empty data column; decoding brotli content without a dictionary.") - null - } - - else -> { - bytes - } - } - } - } - /** * Applies a pending database replacement when the active database file is stale. * @@ -738,14 +645,19 @@ class DocumentationContentSource( database = opened activeDatabasePath = path databaseTimestamp = timestamp - compressionDictionaryStale = true + // Nulled as well as marked stale: a different database can carry a different dictionary + // (or none), and decoding its rows against the previous one can succeed with wrong bytes + // rather than fail (see BrotliDictionaryCodec). A null codec can only be rebuilt, never + // reused. + codec = null + codecStale = true templateCache.clear() generation++ try { previous?.close() } catch (e: Exception) { - log.error("Cannot close previous database: {}", e.message) + log.error("Cannot close previous database", e) } } diff --git a/common/src/main/java/com/itsaky/androidide/utils/DatabaseVersionResolver.kt b/common/src/main/java/com/itsaky/androidide/utils/DatabaseVersionResolver.kt index 225ffd6a39..08109f8f08 100644 --- a/common/src/main/java/com/itsaky/androidide/utils/DatabaseVersionResolver.kt +++ b/common/src/main/java/com/itsaky/androidide/utils/DatabaseVersionResolver.kt @@ -85,7 +85,7 @@ object DatabaseVersionResolver { * identifies itself. * * Deliberately does *not* catch exceptions, unlike [resolveDatabaseVersion]: callers cache the - * answer for the lifetime of a database (see `WebServer.loadCompressionDictionary`), so a + * answer for the lifetime of a database (see [loadCompressionDictionary]), so a * transient `SQLiteException` has to stay distinguishable from a definitive "no version table", * or one hiccup would pin the database at unversioned until it is swapped. */ diff --git a/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt b/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt new file mode 100644 index 0000000000..8eb43c5880 --- /dev/null +++ b/common/src/main/java/com/itsaky/androidide/utils/DocumentationCompression.kt @@ -0,0 +1,259 @@ +package com.itsaky.androidide.utils + +import android.database.sqlite.SQLiteDatabase +import com.aayushatharva.brotli4j.Brotli4jLoader +import com.aayushatharva.brotli4j.decoder.BrotliInputStream +import com.aayushatharva.brotli4j.encoder.BrotliOutputStream +import com.aayushatharva.brotli4j.encoder.Encoder +import com.aayushatharva.brotli4j.encoder.PreparedDictionary +import com.aayushatharva.brotli4j.encoder.PreparedDictionaryGenerator +import org.slf4j.LoggerFactory +import java.io.ByteArrayOutputStream +import java.io.IOException +import java.io.InputStream +import java.nio.ByteBuffer + +private val log = LoggerFactory.getLogger("DocumentationCompression") + +// brotli4j's PreparedDictionaryGenerator rejects a shorter dictionary with "src is too short" +// (measured: 7 bytes throws, 8 round-trips). The decoder has no such floor, so without this the +// failure lands only on the writer, as an IllegalArgumentException from inside a lazy. +private const val MIN_DICTIONARY_BYTES = 8 + +/** + * Copies [bytes] into a direct [ByteBuffer] -- brotli4j's `attachDictionary` requires a direct + * buffer, a heap-backed one throws `IllegalArgumentException`. + * + * The capacity must be exactly [bytes]`.size`: `attachDictionary` reads the whole capacity and + * ignores position/limit, so trailing slack from an over-allocated buffer is treated as dictionary + * content and every decode then fails with `IOException: corrupted input`. + */ +fun toDirectByteBuffer(bytes: ByteArray): ByteBuffer = + ByteBuffer.allocateDirect(bytes.size).apply { + put(bytes) + flip() + } + +/** + * Loads the shared Brotli dictionary every `compression = 'brotli'` Content row in [db] is + * compressed against (see ADFA-5153). Returns null (logged) when the database *definitively* has + * no dictionary, which tells callers to read and write plain, dictionary-free brotli instead. + * + * Both the reader and the writer of `Content` call this, so the two cannot disagree about whether + * a given database's brotli rows carry a dictionary -- which is the whole point: a row compressed + * against a dictionary the reader does not attach is undecodable, and vice versa. + * + * The gate is the MAJOR version the database declares in ADFA-5220's version table, not the + * presence of a `CompressionDictionary` table: table sniffing infers a whole content format from + * one table's existence, and gets it wrong in both directions -- a database carrying the table but + * *unmigrated* content would have every plain row read as dictionary-compressed. Below + * [DatabaseVersionResolver.MAJOR_VERSION_WITH_COMPRESSION_DICTIONARY] the dictionary is neither + * read nor attached. + * + * The `CompressionDictionary` checks below still run, for a database that declares a new-enough + * version but has no usable dictionary row: without them the data query would raise "no such + * table", which callers correctly read as transient and would then retry forever. + * + * Deliberately does *not* catch exceptions itself: an unexpected `SQLiteException`/IO failure is + * likely transient, and callers must be able to tell that apart from a definitive absence. Caching + * a transient failure as "no dictionary" would permanently disable dictionary handling for the + * rest of this database's lifetime; writing plain rows because of one would corrupt content the + * reader can never decode. + */ +fun loadCompressionDictionary(db: SQLiteDatabase): ByteBuffer? { + val majorVersion = DatabaseVersionResolver.resolveMajorVersion(db) + if (majorVersion == null || majorVersion < DatabaseVersionResolver.MAJOR_VERSION_WITH_COMPRESSION_DICTIONARY) { + log.warn( + "Database declares documentation version {}, below {}; brotli content is handled without a dictionary.", + majorVersion ?: "none", + DatabaseVersionResolver.MAJOR_VERSION_WITH_COMPRESSION_DICTIONARY, + ) + return null + } + + val tableExists = + db + .rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'CompressionDictionary'", + null, + ).use { it.moveToFirst() } + if (!tableExists) { + log.warn("CompressionDictionary table not found; brotli content is handled without a dictionary.") + return null + } + + return db.rawQuery("SELECT data FROM CompressionDictionary WHERE id = 1", null).use { cursor -> + if (!cursor.moveToFirst()) { + log.warn("CompressionDictionary table is empty; brotli content is handled without a dictionary.") + return null + } + val bytes = cursor.getBlob(0) + if (bytes == null) { + log.warn("CompressionDictionary row has a NULL data column; brotli content is handled without a dictionary.") + return null + } + // An empty blob would yield a 0-capacity buffer, which attachDictionary rejects, and the + // encoder refuses anything under MIN_DICTIONARY_BYTES outright. Either way a truncated + // dictionary is not usable, and treating it as absent keeps reader and writer agreeing -- + // where letting it through would fail every decode, or every compress, with nothing above + // DEBUG to say why. + if (bytes.size < MIN_DICTIONARY_BYTES) { + log.warn( + "CompressionDictionary row holds {} bytes, below the {} the encoder requires; " + + "brotli content is handled without a dictionary.", + bytes.size, + MIN_DICTIONARY_BYTES, + ) + return null + } + toDirectByteBuffer(bytes) + } +} + +/** + * Whether [db] declares a version whose brotli `Content` rows are dictionary-compressed. + * + * [loadCompressionDictionary] returns null both for a database that legitimately has no dictionary + * (below [DatabaseVersionResolver.MAJOR_VERSION_WITH_COMPRESSION_DICTIONARY]) and for one that + * should have a usable dictionary but does not -- table dropped, row missing, blob truncated. A + * reader cannot tell those apart usefully; it decodes plain either way and every dictionary- + * compressed row simply fails. A *writer* must, because the second case is a damaged database that + * can be repaired in place: rows written plain into it would still be plain afterwards, and would + * then be undecodable with no missing-rows check to catch them. + */ +fun expectsCompressionDictionary(db: SQLiteDatabase): Boolean { + val majorVersion = DatabaseVersionResolver.resolveMajorVersion(db) + return majorVersion != null && majorVersion >= DatabaseVersionResolver.MAJOR_VERSION_WITH_COMPRESSION_DICTIONARY +} + +/** + * Compresses and decompresses documentation `Content` blobs against [dictionary], the shared Brotli + * dictionary from the database those blobs live in (see [loadCompressionDictionary]). A null + * [dictionary] means plain, dictionary-free brotli, which is what a database predating ADFA-5153 + * needs. + * + * One instance per database, reused across rows: preparing the dictionary for the encoder costs a + * few milliseconds and the result is safe to share across streams. + */ +class BrotliDictionaryCodec( + private val dictionary: ByteBuffer?, +) { + init { + // The encoder would quietly accept a heap buffer (it copies into a direct one of its own), + // but attachDictionary on the decode side throws IllegalArgumentException. Without this a + // heap dictionary writes genuinely dictionary-compressed rows that every later read + // rejects -- and unchecked, so handleClient turns it into a 500 naming neither the row nor + // the dictionary. Fail at construction, where the caller can see why. + require(dictionary == null || dictionary.isDirect) { + "dictionary must be a direct ByteBuffer (see toDirectByteBuffer); a heap buffer compresses but cannot decompress" + } + require(dictionary == null || dictionary.capacity() >= MIN_DICTIONARY_BYTES) { + "dictionary must be at least $MIN_DICTIONARY_BYTES bytes, was ${dictionary?.capacity()}" + } + } + + // Only the encoder needs the dictionary in prepared form, so a decode-only user (WebServer) + // never pays for building it. `generate` advances the buffer's position to its limit, so it + // gets a duplicate: the original is shared with attachDictionary, which reads the whole + // capacity and would be unaffected, but leaving a caller's long-lived buffer drained is a trap + // for the next reader of it. + private val preparedDictionary: PreparedDictionary? by lazy { + dictionary?.let { PreparedDictionaryGenerator.generate(it.duplicate()) } + } + + /** + * Builds the encoder's prepared dictionary now rather than on the first [compress]. + * + * It is `by lazy` so a decode-only user never pays for it, but that defers a ~780 KB direct + * allocation to whenever compression first happens -- which for the plugin installer is inside + * an open write transaction. Callers holding a lock around their compression should warm it + * first. + */ + fun warmUp() { + ensureBrotliAvailable() + preparedDictionary + } + + /** + * Compresses [input] at the quality and window size the offline documentation pipeline uses, + * so content contributed at runtime is stored the same way as content built ahead of time. + */ + fun compress(input: ByteArray): ByteArray { + ensureBrotliAvailable() + val out = ByteArrayOutputStream(input.size) + BrotliOutputStream(out, encoderParameters).use { stream -> + preparedDictionary?.let { stream.attachDictionary(it) } + stream.write(input) + } + return out.toByteArray() + } + + /** + * Decompresses a `Content` blob read from the same database [dictionary] came from. + * + * Throws `IOException` when [input] *referenced* a dictionary that is not attached: those + * backward distances reach outside the window, which any spec-compliant decoder rejects. + * + * Note the qualifier. What matters is whether the stream actually matched into the dictionary, + * not whether one was attached when it was written. Content with no such matches -- an already + * compressed image, a block of noise -- round-trips identically either way (measured both + * directions; see BrotliDictionaryDecodeTest). So a mismatched row set fails *non-uniformly*: + * a plugin's HTML raises IOException while its incompressible assets keep serving. + * + * A *wrong* dictionary is a different matter, and is not reliably detectable. One of a + * different length, or with nothing valid at the offsets the stream references, usually + * throws -- but one of the same length holding plausible bytes there decodes cleanly to + * content that is simply wrong (verified: see BrotliDictionaryDecodeTest). So neither a throw + * nor a success is evidence about *which* dictionary was used. + * + * Takes ownership of [input] and closes it, including when the decoder fails to start. + */ + fun decompress(input: InputStream): ByteArray { + ensureBrotliAvailable() + // BrotliInputStream's constructor allocates native state and can throw, which would leave + // `input` open if it were built inside the use{} it is the subject of. + val stream = + try { + BrotliInputStream(input) + } catch (e: Throwable) { + input.close() + throw e + } + return stream.use { + dictionary?.let { dict -> it.attachDictionary(dict) } + it.readBytes() + } + } + + /** + * Loads brotli4j's native library if nothing else has yet, and turns its absence into a failed + * request rather than a dead app. + * + * Nothing here owns that load: it happens as a side effect of `AssetsInstallationHelper`'s + * install or `ToolsManager`'s tooling-jar update, neither of which runs on an ordinary cold + * start. A process that skips both -- Android restarting the app straight into the editor, say + * -- reaches the first brotli row with the natives unregistered, and `DecoderJNI.nativeCreate` + * raises `UnsatisfiedLinkError`. Being an Error rather than an Exception, that escapes the + * caller's catch and kills the app from a coroutine worker instead of failing one request + * (observed on-device, 20-Aug). + * + * Referencing [Brotli4jLoader] triggers the static init that performs the load, so this call is + * the warm-up; afterwards `ensureAvailability` is a single static null-check, cheap enough to + * leave on the per-row path rather than tracking "warmed" state of our own. + */ + private fun ensureBrotliAvailable() { + try { + Brotli4jLoader.ensureAvailability() + } catch (e: UnsatisfiedLinkError) { + throw IOException("brotli4j's native library is unavailable, so brotli content cannot be handled", e) + } + } + + companion object { + // Matches OfflineDocumentationTools' encode settings, so a row written on-device is + // indistinguishable in size and decodability from one built by that pipeline. + private val encoderParameters: Encoder.Parameters by lazy { + Encoder.Parameters().setQuality(11).setWindow(24) + } + } +} diff --git a/docs/adr/0001-prefer-room-for-persistence.md b/docs/adr/0001-prefer-room-for-persistence.md index 0944a429bd..6e53918192 100644 --- a/docs/adr/0001-prefer-room-for-persistence.md +++ b/docs/adr/0001-prefer-room-for-persistence.md @@ -32,8 +32,9 @@ If none of these hold, use Room. "It's a small table" or "I already know SQL" ar |---|---|---| | Symbol indexing | `lsp/indexing/.../SQLiteIndex.kt` | Performance/allocation-critical; granular schema & query control (condition 2). | | In-app tooltips | `idetooltips/.../ToolTipManager.kt` | Prebuilt DB opened read-only (condition 1). | -| In-app / plugin help | `plugin-manager/.../documentation/PluginDocumentationManager.kt` | Prebuilt help-content DB (condition 1). | +| In-app / plugin help | `plugin-manager/.../documentation/PluginDocumentationManager.kt` | Schema owned by `OfflineDocumentationTools`, shared across that boundary (condition 3). **Not** condition 1: this is the one component that writes to `documentation.db`, opening it `OPEN_READWRITE` to insert plugin-contributed tooltip and Tier 3 rows. Room is still wrong here — it would want to own and migrate a schema this repo is forbidden to change. | | Local web server | `app/.../localWebServer/WebServer.kt` | Reads databases (incl. project data) it doesn't own, read-only (conditions 1 & 3). | +| Documentation compression | `common/.../utils/DocumentationCompression.kt` | Reads `CompressionDictionary` from whichever `documentation.db` its caller opened, so the reader and the writer above cannot disagree about how a row is compressed (conditions 1 & 3). | The tooltips and in-app/plugin-help rows, plus the local web server's Tier 3 serving, all read `documentation.db` — see [docs/documentation-database.md](../documentation-database.md) for its schema and consumers. diff --git a/docs/documentation-database.md b/docs/documentation-database.md index bf01e16d1f..793cd11a3e 100644 --- a/docs/documentation-database.md +++ b/docs/documentation-database.md @@ -2,7 +2,7 @@ Reference for `documentation.db`, the SQLite database backing all in-app help: Tier 1/2 tooltips, plus the Tier 3 web content they link to, served by `WebServer`. Read this before touching anything under `localWebServer/`, `idetooltips/`, or `plugin-manager/.../documentation/`, or before writing/editing SQL against this database. -This is a **read-only, prebuilt** database — CoGo never creates or migrates its schema at runtime (see [ADR 0001](adr/0001-prefer-room-for-persistence.md), exception 1). The schema is owned by the separate `OfflineDocumentationTools` project (the `docdb-studio` tool); **never change it from this repo.** +This is a **prebuilt** database — CoGo never creates or migrates its schema at runtime (see [ADR 0001](adr/0001-prefer-room-for-persistence.md), exception 3). Every consumer reads it; one, `PluginDocumentationManager`, also *writes* rows into it, inserting the tooltips and Tier 3 content a plugin contributes into tables it does not own (see *How CoGo talks to this database* below). The schema is owned by the separate `OfflineDocumentationTools` project (the `docdb-studio` tool); **never change it from this repo.** ## Where it lives @@ -34,7 +34,7 @@ CREATE TABLE Content ( One row per file the web server can serve (HTML, CSS, JS, image, video, PDF, ...) — 30,000+ rows. Key points: - **`path`** is the lookup key (indexed via the `UNIQUE` constraint) and is what `WebServer` matches the HTTP request path against. Paths carry a short source prefix to avoid collisions between doc sets, e.g. `k/index.html` (Kotlin) vs `j/index.html` (Java). -- **`content`** is compressed — Brotli for text-like formats, format-specific compression otherwise (images/video/fonts). `ContentTypes.compression` says which. Every migrated `Content` row with `ContentTypes.compression = 'brotli'` is Brotli-compressed against the single shared dictionary in `CompressionDictionary` (see below), converted in one pass by ADFA-5153 — but plugin-contributed Tier 3 rows (`PluginDocumentationManager`/`BrotliCompressor`, see below) are plain, dictionary-free Brotli, and there is no per-row flag distinguishing the two, because a dictionary-compressed stream and a plain one are not distinguishable at decode time by inspection. They *are* distinguishable by attempting the decode: attaching the *wrong* dictionary decodes without error to different bytes than were compressed (its backward distances resolve into real, just incorrect, bytes) — but attaching *no* dictionary to a stream that needs one reliably throws (`IOException`, "corrupted input"), since distances into the dictionary region are then out of bounds for any spec-compliant decoder. `DocumentationContentSource` (which both Tier 3 transports read through) relies on exactly this: its decode tries the dictionary first and falls back to a plain decode on `IOException`, which correctly handles both dictionary-compressed and plain rows — but never rely on decode success/failure to detect a *wrong* dictionary, since that case is silent. Content over 1 MB is split across multiple rows: the first row's path is the base path, continuation rows are `path-1`, `path-2`, ... (`languageId = 1`), reassembled by `DocumentationContentSource` before returning. +- **`content`** is compressed — Brotli for text-like formats, format-specific compression otherwise (images/video/fonts). `ContentTypes.compression` says which. In a database declaring `MAJOR >= 2`, **every** row with `ContentTypes.compression = 'brotli'` is compressed against the single shared dictionary in `CompressionDictionary` (see below): the bulk of them converted in one pass by ADFA-5153, and plugin-contributed Tier 3 rows written that way at install time by `PluginDocumentationManager` (ADFA-5240). So the compression of a row follows from the database's declared version alone — there is no per-row flag, and none is needed. That matters because the two are not distinguishable by inspection, and only half-distinguishable by decode: attaching *no* dictionary to a stream that needs one reliably throws (`IOException`, "corrupted input"), since distances into the dictionary region are out of bounds for any spec-compliant decoder. Attaching the *wrong* dictionary is not reliable in either direction -- one of a different length, or with nothing valid at the offsets the stream references, usually throws, but one of the same length holding plausible bytes there decodes cleanly to content that is simply wrong (both cases verified in `BrotliDictionaryDecodeTest`). Never use decode success, or failure, as evidence about *which* dictionary was used. Content over 1 MB is split across multiple rows: the first row's path is the base path, continuation rows are `path-1`, `path-2`, ... (`languageId = 1`), reassembled by `DocumentationContentSource` before returning. - **`templateId`**: `0` (or unset) means `content` is legacy HTML with presentation baked in (the pre-CMS Release 0/1 format). A positive value means `content` is JSON *facts only*, rendered through the matching row in `Templates` (a Pebble template) — the ongoing move to a proper CMS that de-duplicates presentation across near-identical pages (e.g. `sin`/`cos` docs). - The `UNIQUE(path)` constraint rejects any duplicate `path`, regardless of `languageID` — a second language for an existing path isn't supported yet (only `EN-us` currently exists). Getting there needs an upstream schema change to composite uniqueness on `(path, languageID)` (see *Known rough edges* below). @@ -62,8 +62,8 @@ CREATE TABLE Tooltips ( ### Supporting tables -- **`DocumentationDatabaseVersion(major, minor, patch, who, comment, changeTime)`** — the database's own semver (ADFA-5220), replacing the heuristics that used to infer the format from which tables happened to exist. Append-only: each change is another `INSERT`, so the **row inserted last** is the current version, not the highest one ever recorded — a rebuild from an older content set is a downgrade and has to read as one (`DatabaseVersionResolver.resolveMajorVersion`, which returns null for a database predating the table). `MAJOR >= 2` is what tells the app its brotli `Content` rows are dictionary-compressed; below that, `DocumentationContentSource` neither reads nor attaches `CompressionDictionary`. Gating on the declared version rather than on the table's presence matters in both directions: a database can carry the dictionary table while its content is still plain brotli (every row would then pay a failed dictionary decode before its plain one, on every request), and a migrated database that lost the table fails loudly instead of quietly. -- **`CompressionDictionary(id, data)`** — single-row table (`id INTEGER PRIMARY KEY CHECK (id = 1)`) holding the raw Brotli dictionary every ADFA-5153-migrated `compression = 'brotli'` `Content` row is compressed against. Trained once, from a representative sample across the whole `Content` table, by `OfflineDocumentationTools`' `migrate_content_to_dictionary_brotli.py` / `populate_db.py` (never retrained after that — a dictionary-compressed row is only decodable against the exact dictionary it was compressed with, so replacing it would silently orphan every already-migrated row). Shipping the dictionary inside `documentation.db` itself, rather than as a separate bundled asset, keeps it version-locked to the content compressed against it. `DocumentationContentSource` loads it lazily -- not merely from opening or swapping databases, but on the first content fetch that needs it after the active database changes, and only when `DocumentationDatabaseVersion` declares `MAJOR >= 2` (see above) -- and caches it from then on, reloading again only on the next database change (a swap can bring in a database with a different dictionary or none, so it can't stay cached across one). Per row, it tries decoding with the dictionary attached first via brotli4j's `attachDictionary`, falling back to a plain decode on failure — needed both for a database predating this migration (no `CompressionDictionary` table at all) and for plugin-contributed rows within an otherwise-migrated database (see `PluginDocumentationManager` below). +- **`DocumentationDatabaseVersion(major, minor, patch, who, comment, changeTime)`** — the database's own semver (ADFA-5220), replacing the heuristics that used to infer the format from which tables happened to exist. Append-only: each change is another `INSERT`, so the **row inserted last** is the current version, not the highest one ever recorded — a rebuild from an older content set is a downgrade and has to read as one (`DatabaseVersionResolver.resolveMajorVersion`, which returns null for a database predating the table). `MAJOR >= 2` is what tells the app its brotli `Content` rows are dictionary-compressed; below that, `DocumentationContentSource` neither reads nor attaches `CompressionDictionary`. Gating on the declared version rather than on the table's presence matters in both directions: a database can carry the dictionary table while its content is still plain brotli (every brotli row would then be decoded with a dictionary attached and fail outright, since ADFA-5240 removed the plain-decode retry that used to mask this), and a migrated database that lost the table fails loudly instead of quietly. +- **`CompressionDictionary(id, data)`** — single-row table (`id INTEGER PRIMARY KEY CHECK (id = 1)`) holding the raw Brotli dictionary every ADFA-5153-migrated `compression = 'brotli'` `Content` row is compressed against. Trained once, from a representative sample across the whole `Content` table, by `OfflineDocumentationTools`' `migrate_content_to_dictionary_brotli.py` / `populate_db.py` (never retrained after that — a dictionary-compressed row is only decodable against the exact dictionary it was compressed with, so replacing it would silently orphan every already-migrated row). Shipping the dictionary inside `documentation.db` itself, rather than as a separate bundled asset, keeps it version-locked to the content compressed against it. Both the reader and the writer load it through `loadCompressionDictionary` in `:common`, so they cannot disagree about whether a given database's brotli rows carry a dictionary — a row compressed against one the reader won't attach is undecodable, and vice versa. `DocumentationContentSource` loads it lazily -- not at database open or swap time, but on the first content lookup after the active database changes, whatever that row's own compression, and only when `DocumentationDatabaseVersion` declares `MAJOR >= 2` (see above) -- and caches it from then on, reloading again only on the next database change (a swap can bring in a database with a different dictionary or none, so it can't stay cached across one). Each row is then decoded exactly once, with the dictionary attached or not according to that version; there is no retry, because there is no longer a second way a row might have been written. - **`Templates(id, name, content)`** — Pebble template source, keyed by id (and by `name` for well-known templates like `bookshelf`). Referenced by `Content.templateId`. - **`Bookshelf(contentID, bookCategoryID, title, description)`** / **`BookCategories(id, category, description)`** — the Dynamic Bookshelf: one row per "book" (PDF or similar), linked to its Tier 3 page via `contentID` -> `Content.id`. Two DB triggers keep `Bookshelf` in sync when a PDF row is inserted/deleted from `Content`; `title`/`description` don't come from those triggers and must be set by hand. Non-PDF books need a separate ingestion path (plugin-provided, e.g. via `PluginDocumentationManager`). - **`LastChange(documentationSet, changeTime, who)`** — audit trail for edits made through `docdb-studio`; not shown to end users. `DatabaseVersionResolver` reads the `documentationSet = 'wholedb'` row to report the DB's build/edit stamp in debug logging, falling back to the most recent row of any set if `'wholedb'` is missing. @@ -77,7 +77,7 @@ Of the five sites below, only `DocumentationContentSource` and `ToolTipManager` - **`common/.../documentation/DocumentationRequestInterceptor.kt`** — serves Tier 3 *in-process* for the app's WebViews (`HelpActivity`, the tooltip fragment, `FAQActivity`), through `WebViewClient.shouldInterceptRequest`, so a page's assets cost a database read instead of a TCP connection each (ADFA-5176). It matches the same `http://localhost:6174/...` URL space, so the strings.xml entries, `ToolTipManager`'s link builder and the `DocumentationExtension` contract need no changes; anything it declines — a `/pr/` endpoint, an unknown path, a failed read — falls through to `WebServer` unchanged. Both transports get their `Content-Type` charset from the same place, `ContentTypeHeaders` (ADFA-5241), so a row does not describe itself differently depending on which one served it. Set `/sdcard/Download/CodeOnTheGo.nointercept` to force documentation back onto the server. - **`app/.../localWebServer/WebServer.kt`** — serves Tier 3 over HTTP on port 6174, for WebViews that are not wired to the interceptor above and for the `/pr/` developer endpoints. It reads through `DocumentationContentSource`, so it holds no database, template engine or decode logic of its own; what remains here is HTTP: request parsing, the `/pr/` pages, error responses, and the CSS/asset shortcuts. Also serves a Dynamic Bookshelf JSON payload (joining `Content`/`Bookshelf`/`BookCategories`, rendered through the `bookshelf` template) and debug-only HTML dumps at `/pr/db` (`LastChange`, last 20 rows) and `/pr/pr` (recent projects, from a *different* database). - **`idetooltips/.../ToolTipManager.kt`** — serves Tier 1/2. Looks up `Tooltips` joined to `TooltipCategories` by `(category, tag)`, then `TooltipButtons` for the Tier 3 links shown at the bottom. -- **`plugin-manager/.../documentation/PluginDocumentationManager.kt`** (with `Tier3AssetWalker.kt`, and the `DocumentationExtension` contract in `plugin-api`) — lets plugins contribute their own help content into the same lookup paths. +- **`plugin-manager/.../documentation/PluginDocumentationManager.kt`** (with `Tier3AssetWalker.kt`, and the `DocumentationExtension` contract in `plugin-api`) — lets plugins contribute their own help content into the same lookup paths, under the reserved `plugin//...` prefix. Brotli assets are compressed against the same `CompressionDictionary` the rest of the table uses, read from the database being written. When that dictionary can't be read the install is abandoned rather than written plain, and retried on the plugin's next activation. ## Editing the database diff --git a/plugin-manager/build.gradle.kts b/plugin-manager/build.gradle.kts index 13462875e2..e75526fe2d 100644 --- a/plugin-manager/build.gradle.kts +++ b/plugin-manager/build.gradle.kts @@ -34,7 +34,6 @@ dependencies { implementation(libs.androidx.appcompat) implementation(libs.gson.v2101) - implementation(libs.brotli4j) implementation(libs.commons.compress) implementation(libs.tukaani.xz) diff --git a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/BrotliCompressor.kt b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/BrotliCompressor.kt deleted file mode 100644 index 445bf3df30..0000000000 --- a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/BrotliCompressor.kt +++ /dev/null @@ -1,20 +0,0 @@ -package com.itsaky.androidide.plugins.manager.documentation - -import com.aayushatharva.brotli4j.Brotli4jLoader -import com.aayushatharva.brotli4j.encoder.BrotliOutputStream -import com.aayushatharva.brotli4j.encoder.Encoder -import java.io.ByteArrayOutputStream - -internal object BrotliCompressor { - - private val params: Encoder.Parameters by lazy { - Brotli4jLoader.ensureAvailability() - Encoder.Parameters().setQuality(11).setWindow(24) - } - - fun compress(input: ByteArray): ByteArray { - val out = ByteArrayOutputStream(input.size) - BrotliOutputStream(out, params).use { it.write(input) } - return out.toByteArray() - } -} diff --git a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt index c079d106b8..b60f15ed32 100644 --- a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt +++ b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/PluginDocumentationManager.kt @@ -2,6 +2,7 @@ package com.itsaky.androidide.plugins.manager.documentation import android.content.ContentValues import android.content.Context +import android.content.SharedPreferences import android.content.res.AssetManager import android.database.sqlite.SQLiteDatabase import android.util.Log @@ -9,6 +10,9 @@ import com.itsaky.androidide.plugins.extensions.DocumentationExtension import com.itsaky.androidide.plugins.extensions.PluginTooltipEntry import com.itsaky.androidide.plugins.manager.pluginCategory import com.itsaky.androidide.resources.R +import com.itsaky.androidide.utils.BrotliDictionaryCodec +import com.itsaky.androidide.utils.expectsCompressionDictionary +import com.itsaky.androidide.utils.loadCompressionDictionary import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import java.io.File @@ -20,574 +24,790 @@ import kotlin.coroutines.cancellation.CancellationException * differentiated by a "plugin_" category prefix so they never conflict with * built-in documentation. */ -class PluginDocumentationManager(private val context: Context) { - - companion object { - private const val TAG = "PluginDocManager" - } - - private val databaseName = "documentation.db" - - private suspend fun getPluginDatabase(): SQLiteDatabase? = withContext(Dispatchers.IO) { - try { - val dbFile = context.getDatabasePath(databaseName) - if (!dbFile.exists()) { - Log.w(TAG, "documentation.db not yet available at: ${dbFile.absolutePath}") - return@withContext null - } - SQLiteDatabase.openDatabase(dbFile.absolutePath, null, SQLiteDatabase.OPEN_READWRITE) - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Failed to open documentation.db for plugin writes", e) - null - } - } - - - /** - * Initialize plugin documentation system. - * Also cleans up the legacy plugin_documentation.db if present. - */ - suspend fun initialize() = withContext(Dispatchers.IO) { - val legacyDb = context.getDatabasePath("plugin_documentation.db") - if (legacyDb.exists()) { - if (legacyDb.delete()) { - Log.d(TAG, "Removed legacy plugin_documentation.db") - } else { - Log.w(TAG, "Failed to remove legacy plugin_documentation.db") - } - } - Log.d(TAG, "Plugin documentation system initialized") - } - - /** - * Install documentation from a plugin into documentation.db. - */ - suspend fun installPluginDocumentation( - pluginId: String, - plugin: DocumentationExtension - ): Boolean = withContext(Dispatchers.IO) { - - if (!plugin.onDocumentationInstall()) { - Log.d(TAG, "Plugin $pluginId declined documentation installation") - return@withContext false - } - - val db = getPluginDatabase() - if (db == null) { - Log.w(TAG, "Cannot install documentation for $pluginId - database not available") - return@withContext false - } - - val entries = plugin.getTooltipEntries() - - if (entries.isEmpty()) { - Log.d(TAG, "Plugin $pluginId has no tooltip entries") - db.close() - return@withContext true - } - - Log.d(TAG, "Installing ${entries.size} tooltip entries for plugin $pluginId") - - db.beginTransaction() - try { - removePluginDocumentationInternal(db, pluginId) - - val categoryId = insertOrGetCategoryId(db, pluginCategory(pluginId)) - - for (entry in entries) { - val tooltipId = insertTooltip(db, categoryId, entry) - entry.buttons.sortedBy { it.order }.forEachIndexed { index, button -> - val resolvedUri = resolvePluginButtonUri(pluginId, button.uri, button.directPath) - insertTooltipButton(db, tooltipId, button.description, resolvedUri, index) - } - } - - db.setTransactionSuccessful() - Log.d(TAG, "Successfully installed documentation for plugin $pluginId") - true - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Failed to install documentation for plugin $pluginId", e) - false - } finally { - db.endTransaction() - db.close() - } - } - - /** - * Remove all documentation for a plugin from documentation.db. - */ - suspend fun removePluginDocumentation( - pluginId: String, - plugin: DocumentationExtension? = null - ): Boolean = withContext(Dispatchers.IO) { - - try { - plugin?.onDocumentationUninstall() - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Plugin onDocumentationUninstall() threw during removal: $pluginId", e) - } - - val db = getPluginDatabase() - if (db == null) { - Log.w(TAG, "Cannot remove documentation for $pluginId - database not available") - return@withContext false - } - - db.beginTransaction() - try { - removePluginDocumentationInternal(db, pluginId) - db.setTransactionSuccessful() - Log.d(TAG, "Successfully removed documentation for plugin $pluginId") - true - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Failed to remove documentation for plugin $pluginId", e) - false - } finally { - db.endTransaction() - db.close() - } - } - - /** - * Install Tier 3 documentation (full help pages) contributed by a plugin. - * - * Walks the plugin-declared asset subdirectory, compresses each file per - * the existing ContentTypes.compression column, chunks blobs at 1 MB to - * match WebServer's read loop, and inserts everything under the reserved - * path namespace "plugin//..." inside a single transaction. - */ - suspend fun installPluginTier3Documentation( - pluginId: String, - plugin: DocumentationExtension, - pluginApkPath: String - ): Boolean = withContext(Dispatchers.IO) { - - val assetPath = plugin.getTier3DocsAssetPath() - if (assetPath.isNullOrBlank()) { - return@withContext true - } - - val pluginAssets = try { - openPluginOnlyAssets(pluginApkPath) - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Failed to open plugin APK assets for $pluginId", e) - return@withContext false - } - - val db = getPluginDatabase() - if (db == null) { - Log.w(TAG, "Cannot install Tier 3 docs for $pluginId - database not available") - pluginAssets.close() - return@withContext false - } - - val resolver = ExtensionToContentTypeResolver() - var inserted = 0 - var skipped = 0 - - db.beginTransaction() - try { - removePluginTier3Internal(db, pluginId) - - for (asset in Tier3AssetWalker.walk(pluginAssets, assetPath)) { - val ext = asset.relativePath.substringAfterLast('.', "") - if (ext.isEmpty()) { - Log.w(TAG, "Skipping Tier 3 asset without extension: ${asset.relativePath}") - skipped++ - continue - } - val row = resolver.resolve(db, ext) - if (row == null) { - Log.w(TAG, "No ContentType for .$ext (${asset.relativePath}); skipping") - skipped++ - continue - } - - val payload = if (row.compression == "brotli") { - BrotliCompressor.compress(asset.bytes) - } else { - asset.bytes - } - - val safeRelative = try { - normalizeLocalDocumentationPath(asset.relativePath) - } catch (e: IllegalArgumentException) { - Log.w(TAG, "Skipping Tier 3 asset with invalid path '${asset.relativePath}': ${e.message}") - skipped++ - continue - } - val basePath = "plugin/$pluginId/$safeRelative" - insertContentChunked(db, basePath, payload, row.id) - inserted++ - } - - db.setTransactionSuccessful() - Log.d(TAG, "Installed $inserted Tier 3 documents for plugin $pluginId (skipped=$skipped)") - true - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Failed to install Tier 3 docs for plugin $pluginId", e) - false - } finally { - db.endTransaction() - db.close() - pluginAssets.close() - } - } - - /** - * Remove all Tier 3 documentation rows owned by the given plugin. - */ - suspend fun removePluginTier3Documentation( - pluginId: String - ): Boolean = withContext(Dispatchers.IO) { - val db = getPluginDatabase() ?: return@withContext false - db.beginTransaction() - try { - val deleted = removePluginTier3Internal(db, pluginId) - db.setTransactionSuccessful() - Log.d(TAG, "Removed $deleted Tier 3 rows for plugin $pluginId") - true - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Failed to remove Tier 3 docs for plugin $pluginId", e) - false - } finally { - db.endTransaction() - db.close() - } - } - - /** - * Verify that Tier 3 content exists for this plugin; reinstall if missing. - * Mirrors [verifyAndRecreateDocumentation] for the Tier 1/2 pipeline. - */ - suspend fun verifyAndRecreateTier3Documentation( - pluginId: String, - plugin: DocumentationExtension, - pluginApkPath: String - ): Boolean = withContext(Dispatchers.IO) { - if (plugin.getTier3DocsAssetPath().isNullOrBlank()) { - return@withContext true - } - if (!isDatabaseAvailable()) { - Log.d(TAG, "documentation.db not available yet for Tier 3 verify of $pluginId") - return@withContext false - } - if (isPluginTier3DocumentationInstalled(pluginId)) { - Log.d(TAG, "Tier 3 docs already present for $pluginId") - return@withContext true - } - Log.d(TAG, "Tier 3 docs missing for $pluginId, installing...") - installPluginTier3Documentation(pluginId, plugin, pluginApkPath) - } - - /** - * Check if any Tier 3 content rows exist for this plugin. - */ - suspend fun isPluginTier3DocumentationInstalled(pluginId: String): Boolean = withContext(Dispatchers.IO) { - val db = getPluginDatabase() ?: return@withContext false - try { - val prefix = "plugin/$pluginId" - db.rawQuery( - "SELECT 1 FROM Content WHERE path = ? OR path LIKE ? ESCAPE '\\' LIMIT 1", - arrayOf(prefix, "${escapeLike(prefix)}/%") - ).use { cursor -> cursor.moveToFirst() } - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Failed to probe Tier 3 installation for $pluginId", e) - false - } finally { - db.close() - } - } - - /** - * Build an AssetManager that sees ONLY the plugin APK, so walking a top-level - * asset directory cannot pick up collisions with the host app's assets. - */ - private fun openPluginOnlyAssets(pluginApkPath: String): AssetManager { - @Suppress("DEPRECATION") - val am = AssetManager::class.java.getDeclaredConstructor().newInstance() - val addAssetPath = AssetManager::class.java.getMethod("addAssetPath", String::class.java) - val cookie = addAssetPath.invoke(am, pluginApkPath) as? Int ?: 0 - if (cookie == 0) { - throw IllegalStateException("addAssetPath returned 0 for $pluginApkPath") - } - return am - } - - private fun removePluginTier3Internal(db: SQLiteDatabase, pluginId: String): Int { - val prefix = "plugin/$pluginId" - return db.delete( - "Content", - "path = ? OR path LIKE ? ESCAPE '\\'", - arrayOf(prefix, "${escapeLike(prefix)}/%") - ) - } - - private fun insertContentChunked( - db: SQLiteDatabase, - basePath: String, - payload: ByteArray, - contentTypeId: Long - ) { - val chunkSize = 1024 * 1024 - if (payload.size < chunkSize) { - insertContentRow(db, basePath, payload, contentTypeId) - return - } - - var offset = 0 - var fragment = 0 - while (offset < payload.size) { - val end = minOf(offset + chunkSize, payload.size) - val slice = payload.copyOfRange(offset, end) - val path = if (fragment == 0) basePath else "$basePath-$fragment" - insertContentRow(db, path, slice, contentTypeId) - offset = end - fragment++ - } - if (payload.size % chunkSize == 0) { - insertContentRow(db, "$basePath-$fragment", ByteArray(0), contentTypeId) - } - } - - private fun insertContentRow( - db: SQLiteDatabase, - path: String, - blob: ByteArray, - contentTypeId: Long - ) { - val values = ContentValues().apply { - put("path", path) - put("content", blob) - put("contentTypeID", contentTypeId) - put("languageId", 1) - } - db.insertOrThrow("Content", null, values) - } - - private fun removePluginDocumentationInternal(db: SQLiteDatabase, pluginId: String) { - val category = pluginCategory(pluginId) - - val cursor = db.rawQuery( - """ - SELECT T.id FROM Tooltips AS T - INNER JOIN TooltipCategories AS TC ON T.categoryId = TC.id - WHERE TC.category = ? - """.trimIndent(), - arrayOf(category) - ) - - val tooltipIds = mutableListOf() - while (cursor.moveToNext()) { - tooltipIds.add(cursor.getLong(0)) - } - cursor.close() - - if (tooltipIds.isNotEmpty()) { - val placeholders = tooltipIds.joinToString(",") { "?" } - val args = tooltipIds.map { it.toString() }.toTypedArray() - db.delete("TooltipButtons", "tooltipId IN ($placeholders)", args) - db.delete("Tooltips", "id IN ($placeholders)", args) - } - - db.delete("TooltipCategories", "category = ?", arrayOf(category)) - } - - private fun insertOrGetCategoryId(db: SQLiteDatabase, category: String): Long { - val cursor = db.query( - "TooltipCategories", - arrayOf("id"), - "category = ?", - arrayOf(category), - null, null, null - ) - - if (cursor.moveToFirst()) { - val id = cursor.getLong(0) - cursor.close() - return id - } - cursor.close() - - val values = ContentValues().apply { - put("category", category) - } - return db.insert("TooltipCategories", null, values) - } - - private fun insertTooltip( - db: SQLiteDatabase, - categoryId: Long, - entry: PluginTooltipEntry - ): Long { - val disclaimer = context.getString(R.string.plugin_documentation_third_party_disclaimer) - - val existingCursor = db.query( - "Tooltips", - arrayOf("id"), - "categoryId = ? AND tag = ?", - arrayOf(categoryId.toString(), entry.tag), - null, null, null - ) - - if (existingCursor.moveToFirst()) { - val existingId = existingCursor.getLong(0) - existingCursor.close() - - val updateValues = ContentValues().apply { - put("summary", entry.summary + disclaimer) - put("detail", if (entry.detail.isNotBlank()) entry.detail + disclaimer else "") - } - db.update("Tooltips", updateValues, "id = ?", arrayOf(existingId.toString())) - db.delete("TooltipButtons", "tooltipId = ?", arrayOf(existingId.toString())) - return existingId - } - existingCursor.close() - - val values = ContentValues().apply { - put("categoryId", categoryId) - put("tag", entry.tag) - put("summary", entry.summary + disclaimer) - put("detail", if (entry.detail.isNotBlank()) entry.detail + disclaimer else "") - } - return db.insert("Tooltips", null, values) - } - - private fun escapeLike(value: String): String = - value - .replace("\\", "\\\\") - .replace("%", "\\%") - .replace("_", "\\_") - - private fun normalizeLocalDocumentationPath(path: String): String { - val segments = path.split('/').filter { it.isNotEmpty() && it != "." } - require(segments.none { it == ".." }) { - "Documentation paths must not contain '..' segments: $path" - } - return segments.joinToString("/") - } - - private fun resolvePluginButtonUri( - pluginId: String, - rawUri: String, - directPath: Boolean - ): String { - if (rawUri.isEmpty()) return rawUri - if (rawUri.contains("://")) return rawUri - val absolute = directPath || rawUri.startsWith("/") - val normalized = normalizeLocalDocumentationPath(rawUri.trimStart('/')) - return if (absolute) normalized else "plugin/$pluginId/$normalized" - } - - private fun insertTooltipButton( - db: SQLiteDatabase, - tooltipId: Long, - description: String, - uri: String, - order: Int - ) { - val values = ContentValues().apply { - put("tooltipId", tooltipId) - put("description", description) - put("uri", uri) - put("buttonNumberId", order) - } - db.insert("TooltipButtons", null, values) - } - - /** - * Check if the plugin documentation database is accessible. - */ - suspend fun isDatabaseAvailable(): Boolean = withContext(Dispatchers.IO) { - context.getDatabasePath(databaseName).exists() - } - - /** - * Check if documentation for a specific plugin exists in documentation.db. - */ - suspend fun isPluginDocumentationInstalled(pluginId: String): Boolean = withContext(Dispatchers.IO) { - val db = getPluginDatabase() ?: return@withContext false - - try { - val cursor = db.rawQuery( - "SELECT COUNT(*) FROM TooltipCategories WHERE category = ?", - arrayOf(pluginCategory(pluginId)) - ) - val installed = cursor.moveToFirst() && cursor.getInt(0) > 0 - cursor.close() - installed - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Failed to check plugin documentation for $pluginId", e) - false - } finally { - db.close() - } - } - - /** - * Verify and recreate plugin documentation if missing. - */ - suspend fun verifyAndRecreateDocumentation( - pluginId: String, - plugin: DocumentationExtension - ): Boolean = withContext(Dispatchers.IO) { - if (!isDatabaseAvailable()) { - Log.d(TAG, "documentation.db not available yet for $pluginId, skipping") - return@withContext false - } - - if (!isPluginDocumentationInstalled(pluginId)) { - Log.d(TAG, "Plugin documentation missing for $pluginId, recreating...") - return@withContext installPluginDocumentation(pluginId, plugin) - } - - Log.d(TAG, "Plugin documentation already exists for $pluginId") - true - } - - /** - * Verify and recreate documentation for all plugins that support it. - */ - suspend fun verifyAllPluginDocumentation( - plugins: Map - ): Int = withContext(Dispatchers.IO) { - if (plugins.isEmpty()) return@withContext 0 - - if (!isDatabaseAvailable()) { - Log.d(TAG, "documentation.db not available yet, skipping verification") - return@withContext 0 - } - - var recreatedCount = 0 - - for ((pluginId, plugin) in plugins) { - try { - if (!isPluginDocumentationInstalled(pluginId)) { - Log.d(TAG, "Recreating missing documentation for plugin: $pluginId") - if (installPluginDocumentation(pluginId, plugin)) { - recreatedCount++ - } - } - } catch (e: Exception) { - if (e is CancellationException) throw e - Log.e(TAG, "Failed to verify/recreate documentation for $pluginId", e) - } - } - - if (recreatedCount > 0) { - Log.i(TAG, "Recreated documentation for $recreatedCount plugins") - } - - recreatedCount - } +class PluginDocumentationManager( + private val context: Context, +) { + companion object { + private const val TAG = "PluginDocManager" + + private const val TIER3_PREFS = "plugin_tier3_docs" + + // Bumped whenever previously written Tier 3 rows can no longer be read as they are. + // 1 was plain brotli; 2 compresses against the database's shared dictionary (ADFA-5240). + private const val TIER3_COMPRESSION_GENERATION = 2 + + // Aggregate cap on the payload bytes staged in memory before the insert transaction + // opens. Tier3AssetWalker caps each asset at 10 MiB but nothing bounded the sum, and + // enough valid assets would exhaust the heap -- as an OutOfMemoryError, which escapes + // catch (Exception). A small multiple of the per-asset cap: past it, the whole install + // fails rather than committing a subset of the plugin's documents. + private const val MAX_STAGED_BYTES = 32L * 1024L * 1024L + } + + private val databaseName = "documentation.db" + + private suspend fun getPluginDatabase(): SQLiteDatabase? = + withContext(Dispatchers.IO) { + try { + val dbFile = context.getDatabasePath(databaseName) + if (!dbFile.exists()) { + Log.w(TAG, "documentation.db not yet available at: ${dbFile.absolutePath}") + return@withContext null + } + SQLiteDatabase.openDatabase(dbFile.absolutePath, null, SQLiteDatabase.OPEN_READWRITE) + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to open documentation.db for plugin writes", e) + null + } + } + + /** + * Initialize plugin documentation system. + * Also cleans up the legacy plugin_documentation.db if present. + */ + suspend fun initialize() = + withContext(Dispatchers.IO) { + val legacyDb = context.getDatabasePath("plugin_documentation.db") + if (legacyDb.exists()) { + if (legacyDb.delete()) { + Log.d(TAG, "Removed legacy plugin_documentation.db") + } else { + Log.w(TAG, "Failed to remove legacy plugin_documentation.db") + } + } + Log.d(TAG, "Plugin documentation system initialized") + } + + /** + * Install documentation from a plugin into documentation.db. + */ + suspend fun installPluginDocumentation( + pluginId: String, + plugin: DocumentationExtension, + ): Boolean = + withContext(Dispatchers.IO) { + if (!plugin.onDocumentationInstall()) { + Log.d(TAG, "Plugin $pluginId declined documentation installation") + return@withContext false + } + + val db = getPluginDatabase() + if (db == null) { + Log.w(TAG, "Cannot install documentation for $pluginId - database not available") + return@withContext false + } + + val entries = plugin.getTooltipEntries() + + if (entries.isEmpty()) { + Log.d(TAG, "Plugin $pluginId has no tooltip entries") + db.close() + return@withContext true + } + + Log.d(TAG, "Installing ${entries.size} tooltip entries for plugin $pluginId") + + db.beginTransaction() + try { + removePluginDocumentationInternal(db, pluginId) + + val categoryId = insertOrGetCategoryId(db, pluginCategory(pluginId)) + + for (entry in entries) { + val tooltipId = insertTooltip(db, categoryId, entry) + entry.buttons.sortedBy { it.order }.forEachIndexed { index, button -> + val resolvedUri = resolvePluginButtonUri(pluginId, button.uri, button.directPath) + insertTooltipButton(db, tooltipId, button.description, resolvedUri, index) + } + } + + db.setTransactionSuccessful() + Log.d(TAG, "Successfully installed documentation for plugin $pluginId") + true + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to install documentation for plugin $pluginId", e) + false + } finally { + db.endTransaction() + db.close() + } + } + + /** + * Remove all documentation for a plugin from documentation.db. + */ + suspend fun removePluginDocumentation( + pluginId: String, + plugin: DocumentationExtension? = null, + ): Boolean = + withContext(Dispatchers.IO) { + try { + plugin?.onDocumentationUninstall() + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Plugin onDocumentationUninstall() threw during removal: $pluginId", e) + } + + val db = getPluginDatabase() + if (db == null) { + Log.w(TAG, "Cannot remove documentation for $pluginId - database not available") + return@withContext false + } + + db.beginTransaction() + try { + removePluginDocumentationInternal(db, pluginId) + db.setTransactionSuccessful() + Log.d(TAG, "Successfully removed documentation for plugin $pluginId") + true + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to remove documentation for plugin $pluginId", e) + false + } finally { + db.endTransaction() + db.close() + } + } + + /** + * Install Tier 3 documentation (full help pages) contributed by a plugin. + * + * Walks the plugin-declared asset subdirectory, compresses each file per + * the existing ContentTypes.compression column, chunks blobs at 1 MB to + * match WebServer's read loop, and inserts everything under the reserved + * path namespace "plugin//..." inside a single transaction. + * + * Brotli assets are compressed against the database's own shared dictionary + * (ADFA-5240), so every brotli row in Content -- contributed here or built + * offline -- decodes the same way. The dictionary is read from the database + * being written, which keeps content and dictionary version-locked: replacing + * documentation.db drops these rows, and they are reinstalled against the new + * file's dictionary. + */ + suspend fun installPluginTier3Documentation( + pluginId: String, + plugin: DocumentationExtension, + pluginApkPath: String, + ): Boolean = + withContext(Dispatchers.IO) { + val assetPath = plugin.getTier3DocsAssetPath() + if (assetPath.isNullOrBlank()) { + return@withContext true + } + + val pluginAssets = + try { + openPluginOnlyAssets(pluginApkPath) + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to open plugin APK assets for $pluginId", e) + return@withContext false + } + + val db = getPluginDatabase() + if (db == null) { + Log.w(TAG, "Cannot install Tier 3 docs for $pluginId - database not available") + pluginAssets.close() + return@withContext false + } + + // Three outcomes, and only one of them is "write plain rows". + // + // A dictionary: compress against it. No dictionary in a database that never declared + // one: plain, which is what its reader expects. But a database that *should* have a + // usable dictionary and does not is damaged, not plain -- writing plain rows into it + // would leave them undecodable once the dictionary row is repaired in place, with no + // missing-rows check to catch them, since repairing a row does not drop this plugin's. + // A throw means the answer is merely unavailable right now. The last two both defer to + // the next activation rather than guessing. + // + // Closing runs through finally, not the catch: toDirectByteBuffer and Cursor.getBlob + // can raise OutOfMemoryError, which is an Error and would slip past catch(Exception), + // leaking a read-write handle on documentation.db and the plugin's AssetManager. + var codecLoaded = false + val codec = + try { + val dictionary = loadCompressionDictionary(db) + if (dictionary == null && expectsCompressionDictionary(db)) { + Log.e( + TAG, + "Database declares a compression dictionary but has no usable one; " + + "deferring Tier 3 install for $pluginId rather than writing plain rows", + ) + return@withContext false + } + BrotliDictionaryCodec(dictionary).also { codecLoaded = true } + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Cannot read the compression dictionary; deferring Tier 3 install for $pluginId", e) + return@withContext false + } finally { + if (!codecLoaded) { + db.close() + pluginAssets.close() + } + } + + val resolver = ExtensionToContentTypeResolver() + var skipped = 0 + + val installed = + try { + // Force the encoder's prepared dictionary to be built now rather than on the + // first compressed asset, and surface a missing brotli native as an IOException + // before any asset is walked. Inside this try because warmUp can throw, and a + // throw must close db and pluginAssets through the finally below and fail the + // install normally rather than leak both handles. + codec.warmUp() + + // Compress before the transaction, not inside it: quality 11 on assets of up + // to 10 MB each is seconds of CPU per asset, and an open write transaction + // holds documentation.db's exclusive lock, stalling WebServer's readers for + // all of it. Only the delete and inserts need the lock. The cost is holding + // every compressed payload in memory at once -- the same bytes the inserts + // below hand to SQLite -- bounded by MAX_STAGED_BYTES. + val prepared = ArrayList() + var stagedBytes = 0L + var overLimit = false + for (asset in Tier3AssetWalker.walk(pluginAssets, assetPath)) { + val ext = asset.relativePath.substringAfterLast('.', "") + if (ext.isEmpty()) { + Log.w(TAG, "Skipping Tier 3 asset without extension: ${asset.relativePath}") + skipped++ + continue + } + val row = resolver.resolve(db, ext) + if (row == null) { + Log.w(TAG, "No ContentType for .$ext (${asset.relativePath}); skipping") + skipped++ + continue + } + + // Validated before compressing: quality 11 on a large asset is seconds of work, and + // an asset about to be rejected for its path should not cost any of it. + val safeRelative = + try { + normalizeLocalDocumentationPath(asset.relativePath) + } catch (e: IllegalArgumentException) { + Log.w(TAG, "Skipping Tier 3 asset with invalid path '${asset.relativePath}': ${e.message}") + skipped++ + continue + } + + val payload = + if (row.compression == "brotli") { + codec.compress(asset.bytes) + } else { + asset.bytes + } + + if (stagedBytes + payload.size > MAX_STAGED_BYTES) { + overLimit = true + break + } + stagedBytes += payload.size + prepared.add(PreparedTier3Row("plugin/$pluginId/$safeRelative", payload, row.id)) + } + + if (overLimit) { + // Skipping the overflow and committing what fit would delete every existing + // row and record the subset as current -- the dropped documents would then + // never be reinstalled, since verification trusts the generation marker. + // Fail before any delete instead; existing rows and the generation stay as + // they are, and the install retries on the plugin's next activation. + Log.e( + TAG, + "Tier 3 assets for plugin $pluginId exceed the $MAX_STAGED_BYTES byte " + + "staging limit; failing the install rather than committing a subset", + ) + false + } else if (prepared.isEmpty() && skipped > 0) { + // Deleting this plugin's existing rows and committing would destroy content + // that was serving and replace it with nothing, then record that as a + // successful install. Leave the rows untouched instead: the plugin ships + // assets this build cannot type, which is a packaging problem to surface, + // not one to apply. + Log.e( + TAG, + "Every Tier 3 asset for plugin $pluginId was skipped ($skipped); " + + "leaving its existing content in place", + ) + false + } else { + db.beginTransaction() + removePluginTier3Internal(db, pluginId) + for (prep in prepared) { + insertContentChunked(db, prep.path, prep.payload, prep.contentTypeId) + } + db.setTransactionSuccessful() + Log.d(TAG, "Installed ${prepared.size} Tier 3 documents for plugin $pluginId (skipped=$skipped)") + true + } + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to install Tier 3 docs for plugin $pluginId", e) + false + } finally { + try { + if (db.inTransaction()) { + db.endTransaction() + } + } finally { + db.close() + pluginAssets.close() + } + } + + // Stamped only once the rows are durably committed: setTransactionSuccessful above marks + // intent, endTransaction is what commits, and the marker is written with commit() rather + // than apply(), so it lands on disk immediately. Recording it inside the transaction would + // let a process death in between leave generation 2 standing against rows that then rolled + // back -- and since the rollback restores the legacy plain rows this install had deleted, + // the next verify would see rows present at the current generation and skip the reinstall + // those rows need. Still inside this function rather than in + // verifyAndRecreateTier3Documentation, though: this one is public, and a caller that wrote + // generation-2 rows without stamping them would be re-detected as stale and recompressed + // on every activation from then on. + if (installed) { + recordInstalledGeneration(pluginId) + } + installed + } + + /** + * Remove all Tier 3 documentation rows owned by the given plugin. + */ + suspend fun removePluginTier3Documentation(pluginId: String): Boolean = + withContext(Dispatchers.IO) { + val db = getPluginDatabase() ?: return@withContext false + val removed = + try { + db.beginTransaction() + val deleted = removePluginTier3Internal(db, pluginId) + db.setTransactionSuccessful() + Log.d(TAG, "Removed $deleted Tier 3 rows for plugin $pluginId") + true + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to remove Tier 3 docs for plugin $pluginId", e) + false + } finally { + try { + if (db.inTransaction()) { + db.endTransaction() + } + } finally { + db.close() + } + } + + // Same ordering as the install path: the marker only describes rows that actually + // committed. This direction fails safe -- a marker cleared against rows that rolled back + // just costs one redundant reinstall -- but the two paths reading differently is how the + // install path's version got written the wrong way round in the first place. + if (removed) { + forgetInstalledGeneration(pluginId) + } + removed + } + + /** + * Verify that Tier 3 content exists for this plugin, and was written the way the current + * build reads it; reinstall if either is untrue. Mirrors [verifyAndRecreateDocumentation] + * for the Tier 1/2 pipeline. + */ + suspend fun verifyAndRecreateTier3Documentation( + pluginId: String, + plugin: DocumentationExtension, + pluginApkPath: String, + ): Boolean = + withContext(Dispatchers.IO) { + if (plugin.getTier3DocsAssetPath().isNullOrBlank()) { + return@withContext true + } + if (!isDatabaseAvailable()) { + Log.d(TAG, "documentation.db not available yet for Tier 3 verify of $pluginId") + return@withContext false + } + if (isPluginTier3DocumentationInstalled(pluginId) && installedGeneration(pluginId) == TIER3_COMPRESSION_GENERATION) { + Log.d(TAG, "Tier 3 docs already present for $pluginId") + return@withContext true + } + Log.d(TAG, "Tier 3 docs missing or stale for $pluginId, installing...") + installPluginTier3Documentation(pluginId, plugin, pluginApkPath) + } + + /** + * The compression generation [pluginId]'s Tier 3 rows were last written at, or 0 for rows + * this build has never written. + * + * Rows written before [TIER3_COMPRESSION_GENERATION] are plain brotli, which WebServer no + * longer accepts from a database that declares a dictionary. They are not detectable from the + * rows themselves -- the schema is owned by OfflineDocumentationTools and has no column to + * mark them with, and probing by decode is exactly the guesswork ADFA-5240 removes -- so the + * generation is tracked here instead. + * + * Only needed when documentation.db survives an app upgrade. Replacing that file drops every + * plugin row with it, and the missing-rows check above already covers that case. + */ + private fun installedGeneration(pluginId: String): Int = tier3Preferences().getInt(pluginId, 0) + + // commit(), not apply(): this already runs on Dispatchers.IO, and losing the write to a process + // death would delete and recompress every one of the plugin's assets at quality 11 next launch. + private fun recordInstalledGeneration(pluginId: String) { + // A dropped write recreates the exact loop commit() was chosen to avoid -- the next + // activation reads the 0 default and recompresses everything, and again after that -- so + // say so rather than discarding the result. + if (!tier3Preferences().edit().putInt(pluginId, TIER3_COMPRESSION_GENERATION).commit()) { + Log.w(TAG, "Could not record the Tier 3 compression generation for $pluginId; it will reinstall next activation") + } + } + + private fun forgetInstalledGeneration(pluginId: String) { + if (!tier3Preferences().edit().remove(pluginId).commit()) { + Log.w(TAG, "Could not clear the Tier 3 compression generation for $pluginId") + } + } + + private fun tier3Preferences(): SharedPreferences = context.getSharedPreferences(TIER3_PREFS, Context.MODE_PRIVATE) + + /** + * Check if any Tier 3 content rows exist for this plugin. + */ + suspend fun isPluginTier3DocumentationInstalled(pluginId: String): Boolean = + withContext(Dispatchers.IO) { + val db = getPluginDatabase() ?: return@withContext false + try { + val prefix = "plugin/$pluginId" + db + .rawQuery( + "SELECT 1 FROM Content WHERE path = ? OR path LIKE ? ESCAPE '\\' LIMIT 1", + arrayOf(prefix, "${escapeLike(prefix)}/%"), + ).use { cursor -> cursor.moveToFirst() } + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to probe Tier 3 installation for $pluginId", e) + false + } finally { + db.close() + } + } + + /** + * Build an AssetManager that sees ONLY the plugin APK, so walking a top-level + * asset directory cannot pick up collisions with the host app's assets. + */ + private fun openPluginOnlyAssets(pluginApkPath: String): AssetManager { + @Suppress("DEPRECATION") + val am = AssetManager::class.java.getDeclaredConstructor().newInstance() + val addAssetPath = AssetManager::class.java.getMethod("addAssetPath", String::class.java) + val cookie = addAssetPath.invoke(am, pluginApkPath) as? Int ?: 0 + if (cookie == 0) { + throw IllegalStateException("addAssetPath returned 0 for $pluginApkPath") + } + return am + } + + private fun removePluginTier3Internal( + db: SQLiteDatabase, + pluginId: String, + ): Int { + val prefix = "plugin/$pluginId" + return db.delete( + "Content", + "path = ? OR path LIKE ? ESCAPE '\\'", + arrayOf(prefix, "${escapeLike(prefix)}/%"), + ) + } + + private fun insertContentChunked( + db: SQLiteDatabase, + basePath: String, + payload: ByteArray, + contentTypeId: Long, + ) { + val chunkSize = 1024 * 1024 + if (payload.size < chunkSize) { + insertContentRow(db, basePath, payload, contentTypeId) + return + } + + var offset = 0 + var fragment = 0 + while (offset < payload.size) { + val end = minOf(offset + chunkSize, payload.size) + val slice = payload.copyOfRange(offset, end) + val path = if (fragment == 0) basePath else "$basePath-$fragment" + insertContentRow(db, path, slice, contentTypeId) + offset = end + fragment++ + } + if (payload.size % chunkSize == 0) { + insertContentRow(db, "$basePath-$fragment", ByteArray(0), contentTypeId) + } + } + + private fun insertContentRow( + db: SQLiteDatabase, + path: String, + blob: ByteArray, + contentTypeId: Long, + ) { + val values = + ContentValues().apply { + put("path", path) + put("content", blob) + put("contentTypeID", contentTypeId) + put("languageId", 1) + } + db.insertOrThrow("Content", null, values) + } + + private fun removePluginDocumentationInternal( + db: SQLiteDatabase, + pluginId: String, + ) { + val category = pluginCategory(pluginId) + + val cursor = + db.rawQuery( + """ + SELECT T.id FROM Tooltips AS T + INNER JOIN TooltipCategories AS TC ON T.categoryId = TC.id + WHERE TC.category = ? + """.trimIndent(), + arrayOf(category), + ) + + val tooltipIds = mutableListOf() + while (cursor.moveToNext()) { + tooltipIds.add(cursor.getLong(0)) + } + cursor.close() + + if (tooltipIds.isNotEmpty()) { + val placeholders = tooltipIds.joinToString(",") { "?" } + val args = tooltipIds.map { it.toString() }.toTypedArray() + db.delete("TooltipButtons", "tooltipId IN ($placeholders)", args) + db.delete("Tooltips", "id IN ($placeholders)", args) + } + + db.delete("TooltipCategories", "category = ?", arrayOf(category)) + } + + private fun insertOrGetCategoryId( + db: SQLiteDatabase, + category: String, + ): Long { + val cursor = + db.query( + "TooltipCategories", + arrayOf("id"), + "category = ?", + arrayOf(category), + null, + null, + null, + ) + + if (cursor.moveToFirst()) { + val id = cursor.getLong(0) + cursor.close() + return id + } + cursor.close() + + val values = + ContentValues().apply { + put("category", category) + } + return db.insert("TooltipCategories", null, values) + } + + private fun insertTooltip( + db: SQLiteDatabase, + categoryId: Long, + entry: PluginTooltipEntry, + ): Long { + val disclaimer = context.getString(R.string.plugin_documentation_third_party_disclaimer) + + val existingCursor = + db.query( + "Tooltips", + arrayOf("id"), + "categoryId = ? AND tag = ?", + arrayOf(categoryId.toString(), entry.tag), + null, + null, + null, + ) + + if (existingCursor.moveToFirst()) { + val existingId = existingCursor.getLong(0) + existingCursor.close() + + val updateValues = + ContentValues().apply { + put("summary", entry.summary + disclaimer) + put("detail", if (entry.detail.isNotBlank()) entry.detail + disclaimer else "") + } + db.update("Tooltips", updateValues, "id = ?", arrayOf(existingId.toString())) + db.delete("TooltipButtons", "tooltipId = ?", arrayOf(existingId.toString())) + return existingId + } + existingCursor.close() + + val values = + ContentValues().apply { + put("categoryId", categoryId) + put("tag", entry.tag) + put("summary", entry.summary + disclaimer) + put("detail", if (entry.detail.isNotBlank()) entry.detail + disclaimer else "") + } + return db.insert("Tooltips", null, values) + } + + private fun escapeLike(value: String): String = + value + .replace("\\", "\\\\") + .replace("%", "\\%") + .replace("_", "\\_") + + private fun normalizeLocalDocumentationPath(path: String): String { + val segments = path.split('/').filter { it.isNotEmpty() && it != "." } + require(segments.none { it == ".." }) { + "Documentation paths must not contain '..' segments: $path" + } + return segments.joinToString("/") + } + + private fun resolvePluginButtonUri( + pluginId: String, + rawUri: String, + directPath: Boolean, + ): String { + if (rawUri.isEmpty()) return rawUri + if (rawUri.contains("://")) return rawUri + val absolute = directPath || rawUri.startsWith("/") + val normalized = normalizeLocalDocumentationPath(rawUri.trimStart('/')) + return if (absolute) normalized else "plugin/$pluginId/$normalized" + } + + private fun insertTooltipButton( + db: SQLiteDatabase, + tooltipId: Long, + description: String, + uri: String, + order: Int, + ) { + val values = + ContentValues().apply { + put("tooltipId", tooltipId) + put("description", description) + put("uri", uri) + put("buttonNumberId", order) + } + db.insert("TooltipButtons", null, values) + } + + /** + * Check if the plugin documentation database is accessible. + */ + suspend fun isDatabaseAvailable(): Boolean = + withContext(Dispatchers.IO) { + context.getDatabasePath(databaseName).exists() + } + + /** + * Check if documentation for a specific plugin exists in documentation.db. + */ + suspend fun isPluginDocumentationInstalled(pluginId: String): Boolean = + withContext(Dispatchers.IO) { + val db = getPluginDatabase() ?: return@withContext false + + try { + val cursor = + db.rawQuery( + "SELECT COUNT(*) FROM TooltipCategories WHERE category = ?", + arrayOf(pluginCategory(pluginId)), + ) + val installed = cursor.moveToFirst() && cursor.getInt(0) > 0 + cursor.close() + installed + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to check plugin documentation for $pluginId", e) + false + } finally { + db.close() + } + } + + /** + * Verify and recreate plugin documentation if missing. + */ + suspend fun verifyAndRecreateDocumentation( + pluginId: String, + plugin: DocumentationExtension, + ): Boolean = + withContext(Dispatchers.IO) { + if (!isDatabaseAvailable()) { + Log.d(TAG, "documentation.db not available yet for $pluginId, skipping") + return@withContext false + } + + if (!isPluginDocumentationInstalled(pluginId)) { + Log.d(TAG, "Plugin documentation missing for $pluginId, recreating...") + return@withContext installPluginDocumentation(pluginId, plugin) + } + + Log.d(TAG, "Plugin documentation already exists for $pluginId") + true + } + + /** + * Verify and recreate documentation for all plugins that support it. + */ + suspend fun verifyAllPluginDocumentation(plugins: Map): Int = + withContext(Dispatchers.IO) { + if (plugins.isEmpty()) return@withContext 0 + + if (!isDatabaseAvailable()) { + Log.d(TAG, "documentation.db not available yet, skipping verification") + return@withContext 0 + } + + var recreatedCount = 0 + + for ((pluginId, plugin) in plugins) { + try { + if (!isPluginDocumentationInstalled(pluginId)) { + Log.d(TAG, "Recreating missing documentation for plugin: $pluginId") + if (installPluginDocumentation(pluginId, plugin)) { + recreatedCount++ + } + } + } catch (e: Exception) { + if (e is CancellationException) throw e + Log.e(TAG, "Failed to verify/recreate documentation for $pluginId", e) + } + } + + if (recreatedCount > 0) { + Log.i(TAG, "Recreated documentation for $recreatedCount plugins") + } + + recreatedCount + } } + +// One Tier 3 asset ready to insert, compressed outside the write transaction so the +// exclusive lock is never held through a quality-11 encode. +private class PreparedTier3Row( + val path: String, + val payload: ByteArray, + val contentTypeId: Long, +) diff --git a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/Tier3AssetWalker.kt b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/Tier3AssetWalker.kt index 6159e65ea6..ef46ffd59c 100644 --- a/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/Tier3AssetWalker.kt +++ b/plugin-manager/src/main/kotlin/com/itsaky/androidide/plugins/manager/documentation/Tier3AssetWalker.kt @@ -7,77 +7,85 @@ import java.io.IOException import java.io.InputStream internal data class Tier3Asset( - val relativePath: String, - val bytes: ByteArray + val relativePath: String, + val bytes: ByteArray, ) { - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (javaClass != other?.javaClass) return false + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false - other as Tier3Asset + other as Tier3Asset - if (relativePath != other.relativePath) return false - if (!bytes.contentEquals(other.bytes)) return false + if (relativePath != other.relativePath) return false + if (!bytes.contentEquals(other.bytes)) return false - return true - } + return true + } - override fun hashCode(): Int { - var result = relativePath.hashCode() - result = 31 * result + bytes.contentHashCode() - return result - } + override fun hashCode(): Int { + var result = relativePath.hashCode() + result = 31 * result + bytes.contentHashCode() + return result + } } internal object Tier3AssetWalker { + private const val TAG = "Tier3AssetWalker" + private const val MAX_ASSET_BYTES = 10L * 1024L * 1024L - private const val TAG = "Tier3AssetWalker" - private const val MAX_ASSET_BYTES = 10L * 1024L * 1024L + fun walk( + assets: AssetManager, + rootAssetPath: String, + ): Sequence = + sequence { + val root = rootAssetPath.trim('/') + yieldAll(walkDir(assets, root, relative = "")) + } - fun walk(assets: AssetManager, rootAssetPath: String): Sequence = sequence { - val root = rootAssetPath.trim('/') - yieldAll(walkDir(assets, root, relative = "")) - } + private fun walkDir( + assets: AssetManager, + absolute: String, + relative: String, + ): Sequence = + sequence { + val children = assets.list(absolute) ?: emptyArray() + if (children.isEmpty()) { + val bytes = + try { + assets.open(absolute).use { readBounded(it, MAX_ASSET_BYTES) } + } catch (_: IOException) { + return@sequence + } + if (bytes == null) { + Log.w(TAG, "Skipping Tier 3 asset '$absolute' - exceeds $MAX_ASSET_BYTES byte limit") + return@sequence + } + if (relative.isNotEmpty()) { + yield(Tier3Asset(relative, bytes)) + } + return@sequence + } + for (child in children) { + val childAbs = if (absolute.isEmpty()) child else "$absolute/$child" + val childRel = if (relative.isEmpty()) child else "$relative/$child" + yieldAll(walkDir(assets, childAbs, childRel)) + } + } - private fun walkDir( - assets: AssetManager, - absolute: String, - relative: String - ): Sequence = sequence { - val children = assets.list(absolute) ?: emptyArray() - if (children.isEmpty()) { - val bytes = try { - assets.open(absolute).use { readBounded(it, MAX_ASSET_BYTES) } - } catch (_: IOException) { - return@sequence - } - if (bytes == null) { - Log.w(TAG, "Skipping Tier 3 asset '$absolute' — exceeds $MAX_ASSET_BYTES byte limit") - return@sequence - } - if (relative.isNotEmpty()) { - yield(Tier3Asset(relative, bytes)) - } - return@sequence - } - for (child in children) { - val childAbs = if (absolute.isEmpty()) child else "$absolute/$child" - val childRel = if (relative.isEmpty()) child else "$relative/$child" - yieldAll(walkDir(assets, childAbs, childRel)) - } - } - - private fun readBounded(stream: InputStream, limit: Long): ByteArray? { - val buffer = ByteArrayOutputStream() - val tmp = ByteArray(64 * 1024) - var total = 0L - while (true) { - val n = stream.read(tmp) - if (n < 0) break - total += n - if (total > limit) return null - buffer.write(tmp, 0, n) - } - return buffer.toByteArray() - } + private fun readBounded( + stream: InputStream, + limit: Long, + ): ByteArray? { + val buffer = ByteArrayOutputStream() + val tmp = ByteArray(64 * 1024) + var total = 0L + while (true) { + val n = stream.read(tmp) + if (n < 0) break + total += n + if (total > limit) return null + buffer.write(tmp, 0, n) + } + return buffer.toByteArray() + } }