From 5766c2d571af0c4ca5d723b3c0f0624267bb1cb7 Mon Sep 17 00:00:00 2001 From: Pierroons <97373452+Pierroons@users.noreply.github.com> Date: Fri, 7 Aug 2026 22:41:20 +0200 Subject: [PATCH 1/2] feat(search): match titles regardless of accents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "prenom" returned nothing while "prénom" found "Le Prénom". SQLite's LIKE ignores case for ASCII but never diacritics, so 4 882 of the 40 971 titles in a French catalogue — 12 % — were unreachable to anyone who did not type the accent, which on a TV remote is most of the time. Titles are now stored a second time in a folded form (NFD, combining marks dropped, lowercased with Locale.ROOT) and queries are folded the same way before they reach the DAO. The comparison stays symmetric: accented or not, either side matches. normalizeForSearch is the single definition of that fold. The migration backfills existing rows through it in Kotlin rather than through a stack of SQL REPLACE calls, so the two can never drift — and a drift here is silent, rows simply stop matching. titleNormalized sits outside the constructor on purpose: a data class only copies constructor parameters, so copy(title = …) recomputes it instead of carrying a stale value that would drop the channel out of every search. Folding happens in the repositories rather than at each call site, so every screen searching channels or categories behaves the same way. No index on the column: '%query%' cannot use one, and it would only slow the bulk inserts a resubscription performs on tens of thousands of rows. Verified on a 40 971-row database: migration to v27 completed at startup, every row backfilled, and "prenom" and "AMELIE" now return the accented titles on screen. Co-Authored-By: Claude Opus 5 (1M context) --- .../m3u/core/foundation/util/basic/Strings.kt | 28 +- .../util/basic/NormalizeForSearchTest.kt | 45 + .../com.m3u.data.database.M3UDatabase/27.json | 840 ++++++++++++++++++ .../m3u/data/database/Migration26To27Test.kt | 130 +++ .../m3u/data/database/DatabaseMigrations.kt | 48 + .../com/m3u/data/database/DatabaseModule.kt | 1 + .../java/com/m3u/data/database/M3UDatabase.kt | 2 +- .../com/m3u/data/database/dao/ChannelDao.kt | 33 +- .../com/m3u/data/database/model/Channel.kt | 19 + .../channel/ChannelRepositoryImpl.kt | 24 +- .../playlist/PlaylistRepositoryImpl.kt | 5 +- 11 files changed, 1153 insertions(+), 22 deletions(-) create mode 100644 core/foundation/src/test/java/com/m3u/core/foundation/util/basic/NormalizeForSearchTest.kt create mode 100644 data/schemas/com.m3u.data.database.M3UDatabase/27.json create mode 100644 data/src/androidTest/java/com/m3u/data/database/Migration26To27Test.kt diff --git a/core/foundation/src/main/java/com/m3u/core/foundation/util/basic/Strings.kt b/core/foundation/src/main/java/com/m3u/core/foundation/util/basic/Strings.kt index 7cbfec224..6190d7452 100644 --- a/core/foundation/src/main/java/com/m3u/core/foundation/util/basic/Strings.kt +++ b/core/foundation/src/main/java/com/m3u/core/foundation/util/basic/Strings.kt @@ -3,6 +3,8 @@ package com.m3u.core.foundation.util.basic import androidx.compose.ui.text.capitalize import androidx.compose.ui.text.intl.Locale +import java.text.Normalizer +import java.util.Locale as JavaLocale fun String.title(): String { if (this.isEmpty()) return this @@ -20,4 +22,28 @@ fun String.startsWithAny(vararg prefix: String, ignoreCase: Boolean = false): Bo fun String.startWithHttpScheme(): Boolean = startsWithAny( "http://", "https://", ignoreCase = true -) \ No newline at end of file +) + +/** + * Folds a string down to the form search compares against: no diacritics, no + * case. + * + * SQLite's LIKE ignores case for ASCII but never accents, so "prenom" cannot + * match "Le Prénom" however the query is written — and a catalogue in French + * has a lot of those. Titles are stored pre-folded in a dedicated column and + * the query is folded the same way, which keeps the comparison symmetric: with + * or without accents, either side matches. + * + * NFD splits an accented letter into its base letter plus a combining mark, so + * dropping the marks (Unicode category Mn) leaves the base letter behind. + * Scripts without combining marks — CJK among them — pass through untouched. + * + * Locale.ROOT on purpose: a Turkish locale lowercases 'I' to a dotless 'ı', + * which would quietly make titles unsearchable for those users. + */ +fun String.normalizeForSearch(): String = Normalizer + .normalize(this, Normalizer.Form.NFD) + .replace(COMBINING_MARKS, "") + .lowercase(JavaLocale.ROOT) + +private val COMBINING_MARKS = Regex("\\p{Mn}+") \ No newline at end of file diff --git a/core/foundation/src/test/java/com/m3u/core/foundation/util/basic/NormalizeForSearchTest.kt b/core/foundation/src/test/java/com/m3u/core/foundation/util/basic/NormalizeForSearchTest.kt new file mode 100644 index 000000000..4d34bd570 --- /dev/null +++ b/core/foundation/src/test/java/com/m3u/core/foundation/util/basic/NormalizeForSearchTest.kt @@ -0,0 +1,45 @@ +package com.m3u.core.foundation.util.basic + +import org.junit.Assert.assertEquals +import org.junit.Test + +class NormalizeForSearchTest { + @Test + fun `accents are folded away`() { + assertEquals("le prenom", "Le Prénom".normalizeForSearch()) + assertEquals("amelie", "Amélie".normalizeForSearch()) + assertEquals("a bout de souffle", "À bout de souffle".normalizeForSearch()) + assertEquals("les miserables", "Les Misérables".normalizeForSearch()) + } + + @Test + fun `a query typed with accents matches the same folded form`() { + assertEquals("Le Prénom".normalizeForSearch(), "le prenom".normalizeForSearch()) + assertEquals("Le Prénom".normalizeForSearch(), "LE PRÉNOM".normalizeForSearch()) + } + + @Test + fun `case is folded independently of the device locale`() { + val previous = java.util.Locale.getDefault() + try { + // Turkish lowercases 'I' to a dotless 'ı'; using the default locale + // here would make "IT" unsearchable for those users. + java.util.Locale.setDefault(java.util.Locale.forLanguageTag("tr-TR")) + assertEquals("it crowd", "IT Crowd".normalizeForSearch()) + } finally { + java.util.Locale.setDefault(previous) + } + } + + @Test + fun `scripts without combining marks are left alone`() { + assertEquals("千と千尋の神隠し", "千と千尋の神隠し".normalizeForSearch()) + assertEquals("привет", "Привет".normalizeForSearch()) + } + + @Test + fun `punctuation and spacing survive so substring matching still works`() { + assertEquals("spider-man: no way home", "Spider-Man: No Way Home".normalizeForSearch()) + assertEquals("", "".normalizeForSearch()) + } +} diff --git a/data/schemas/com.m3u.data.database.M3UDatabase/27.json b/data/schemas/com.m3u.data.database.M3UDatabase/27.json new file mode 100644 index 000000000..321a096f6 --- /dev/null +++ b/data/schemas/com.m3u.data.database.M3UDatabase/27.json @@ -0,0 +1,840 @@ +{ + "formatVersion": 1, + "database": { + "version": 27, + "identityHash": "bca92df38500d6fce82a620dab053867", + "entities": [ + { + "tableName": "playlists", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`title` TEXT NOT NULL, `url` TEXT NOT NULL, `pinned_groups` TEXT NOT NULL DEFAULT '[]', `hidden_groups` TEXT NOT NULL DEFAULT '[]', `source` TEXT NOT NULL DEFAULT '0', `user_agent` TEXT DEFAULT NULL, `epg_urls` TEXT NOT NULL DEFAULT '[]', `auto_refresh_programmes` INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(`url`))", + "fields": [ + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "pinnedCategories", + "columnName": "pinned_groups", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "'[]'" + }, + { + "fieldPath": "hiddenCategories", + "columnName": "hidden_groups", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "'[]'" + }, + { + "fieldPath": "source", + "columnName": "source", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "'0'" + }, + { + "fieldPath": "userAgent", + "columnName": "user_agent", + "affinity": "TEXT", + "defaultValue": "NULL" + }, + { + "fieldPath": "epgUrls", + "columnName": "epg_urls", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "'[]'" + }, + { + "fieldPath": "autoRefreshProgrammes", + "columnName": "auto_refresh_programmes", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "url" + ] + } + }, + { + "tableName": "streams", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`url` TEXT NOT NULL, `group` TEXT NOT NULL, `title` TEXT NOT NULL, `cover` TEXT, `playlist_url` TEXT NOT NULL, `license_type` TEXT DEFAULT NULL, `license_key` TEXT DEFAULT NULL, `id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `favourite` INTEGER NOT NULL, `hidden` INTEGER NOT NULL DEFAULT 0, `seen` INTEGER NOT NULL DEFAULT 0, `relation_id` TEXT DEFAULT NULL, `title_normalized` TEXT NOT NULL DEFAULT '')", + "fields": [ + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "category", + "columnName": "group", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "cover", + "columnName": "cover", + "affinity": "TEXT" + }, + { + "fieldPath": "playlistUrl", + "columnName": "playlist_url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "licenseType", + "columnName": "license_type", + "affinity": "TEXT", + "defaultValue": "NULL" + }, + { + "fieldPath": "licenseKey", + "columnName": "license_key", + "affinity": "TEXT", + "defaultValue": "NULL" + }, + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "favourite", + "columnName": "favourite", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "hidden", + "columnName": "hidden", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "seen", + "columnName": "seen", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "relationId", + "columnName": "relation_id", + "affinity": "TEXT", + "defaultValue": "NULL" + }, + { + "fieldPath": "titleNormalized", + "columnName": "title_normalized", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_streams_playlist_url", + "unique": false, + "columnNames": [ + "playlist_url" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_streams_playlist_url` ON `${TABLE_NAME}` (`playlist_url`)" + }, + { + "name": "index_streams_favourite", + "unique": false, + "columnNames": [ + "favourite" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_streams_favourite` ON `${TABLE_NAME}` (`favourite`)" + } + ] + }, + { + "tableName": "programmes", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`relation_id` TEXT NOT NULL, `epg_url` TEXT NOT NULL, `start` INTEGER NOT NULL, `end` INTEGER NOT NULL, `title` TEXT NOT NULL, `description` TEXT NOT NULL, `icon` TEXT, `categories` TEXT NOT NULL, `id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL)", + "fields": [ + { + "fieldPath": "channelId", + "columnName": "relation_id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "epgUrl", + "columnName": "epg_url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "start", + "columnName": "start", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "end", + "columnName": "end", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "icon", + "columnName": "icon", + "affinity": "TEXT" + }, + { + "fieldPath": "categories", + "columnName": "categories", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_programmes_epg_url_relation_id_start_end", + "unique": false, + "columnNames": [ + "epg_url", + "relation_id", + "start", + "end" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_programmes_epg_url_relation_id_start_end` ON `${TABLE_NAME}` (`epg_url`, `relation_id`, `start`, `end`)" + } + ] + }, + { + "tableName": "episodes", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`title` TEXT NOT NULL, `series_id` INTEGER NOT NULL, `season` TEXT NOT NULL, `number` INTEGER NOT NULL, `url` TEXT NOT NULL, `id` INTEGER NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "seriesId", + "columnName": "series_id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "season", + "columnName": "season", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "number", + "columnName": "number", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "color_pack", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`argb` INTEGER NOT NULL, `dark` INTEGER NOT NULL, `name` TEXT NOT NULL, PRIMARY KEY(`argb`, `dark`))", + "fields": [ + { + "fieldPath": "argb", + "columnName": "argb", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isDark", + "columnName": "dark", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "argb", + "dark" + ] + } + }, + { + "tableName": "provider_accounts", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `provider_id` TEXT NOT NULL, `provider_kind` TEXT NOT NULL, `base_url` TEXT NOT NULL, `server_id` TEXT NOT NULL, `server_name` TEXT NOT NULL, `server_version` TEXT NOT NULL, `user_id` TEXT NOT NULL, `username` TEXT NOT NULL, `playlist_url` TEXT NOT NULL, `requires_reauthentication` INTEGER NOT NULL DEFAULT 0, `owner_package_name` TEXT, `owner_service_name` TEXT, `owner_certificate_sha256` TEXT, PRIMARY KEY(`id`), FOREIGN KEY(`playlist_url`) REFERENCES `playlists`(`url`) ON UPDATE CASCADE ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "providerId", + "columnName": "provider_id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "providerKind", + "columnName": "provider_kind", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "baseUrl", + "columnName": "base_url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "serverId", + "columnName": "server_id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "serverName", + "columnName": "server_name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "serverVersion", + "columnName": "server_version", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userId", + "columnName": "user_id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "username", + "columnName": "username", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "playlistUrl", + "columnName": "playlist_url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "requiresReauthentication", + "columnName": "requires_reauthentication", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "ownerPackageName", + "columnName": "owner_package_name", + "affinity": "TEXT" + }, + { + "fieldPath": "ownerServiceName", + "columnName": "owner_service_name", + "affinity": "TEXT" + }, + { + "fieldPath": "ownerCertificateSha256", + "columnName": "owner_certificate_sha256", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_provider_accounts_playlist_url", + "unique": true, + "columnNames": [ + "playlist_url" + ], + "orders": [], + "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_provider_accounts_playlist_url` ON `${TABLE_NAME}` (`playlist_url`)" + }, + { + "name": "index_provider_accounts_provider_id_server_id_user_id", + "unique": true, + "columnNames": [ + "provider_id", + "server_id", + "user_id" + ], + "orders": [], + "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_provider_accounts_provider_id_server_id_user_id` ON `${TABLE_NAME}` (`provider_id`, `server_id`, `user_id`)" + } + ], + "foreignKeys": [ + { + "table": "playlists", + "onDelete": "CASCADE", + "onUpdate": "CASCADE", + "columns": [ + "playlist_url" + ], + "referencedColumns": [ + "url" + ] + } + ] + }, + { + "tableName": "provider_credentials", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`account_id` TEXT NOT NULL, `credential_handle` TEXT NOT NULL, `ciphertext` TEXT NOT NULL, `nonce` TEXT NOT NULL, `key_version` INTEGER NOT NULL, PRIMARY KEY(`account_id`), FOREIGN KEY(`account_id`) REFERENCES `provider_accounts`(`id`) ON UPDATE CASCADE ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "accountId", + "columnName": "account_id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "credentialHandle", + "columnName": "credential_handle", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "ciphertext", + "columnName": "ciphertext", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "nonce", + "columnName": "nonce", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "keyVersion", + "columnName": "key_version", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "account_id" + ] + }, + "foreignKeys": [ + { + "table": "provider_accounts", + "onDelete": "CASCADE", + "onUpdate": "CASCADE", + "columns": [ + "account_id" + ], + "referencedColumns": [ + "id" + ] + } + ] + }, + { + "tableName": "channel_playback_references", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`channel_id` INTEGER NOT NULL, `account_id` TEXT NOT NULL, `provider_id` TEXT NOT NULL, `item_id` TEXT NOT NULL, `media_source_id` TEXT, `source_type` TEXT NOT NULL, PRIMARY KEY(`channel_id`), FOREIGN KEY(`channel_id`) REFERENCES `streams`(`id`) ON UPDATE CASCADE ON DELETE CASCADE , FOREIGN KEY(`account_id`) REFERENCES `provider_accounts`(`id`) ON UPDATE CASCADE ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "channelId", + "columnName": "channel_id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "accountId", + "columnName": "account_id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "providerId", + "columnName": "provider_id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "itemId", + "columnName": "item_id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "mediaSourceId", + "columnName": "media_source_id", + "affinity": "TEXT" + }, + { + "fieldPath": "sourceType", + "columnName": "source_type", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "channel_id" + ] + }, + "indices": [ + { + "name": "index_channel_playback_references_account_id", + "unique": false, + "columnNames": [ + "account_id" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_channel_playback_references_account_id` ON `${TABLE_NAME}` (`account_id`)" + } + ], + "foreignKeys": [ + { + "table": "streams", + "onDelete": "CASCADE", + "onUpdate": "CASCADE", + "columns": [ + "channel_id" + ], + "referencedColumns": [ + "id" + ] + }, + { + "table": "provider_accounts", + "onDelete": "CASCADE", + "onUpdate": "CASCADE", + "columns": [ + "account_id" + ], + "referencedColumns": [ + "id" + ] + } + ] + }, + { + "tableName": "provider_playback_sessions", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `account_id` TEXT NOT NULL, `provider_id` TEXT NOT NULL, `item_id` TEXT NOT NULL, `media_source_id` TEXT, `source_type` TEXT NOT NULL, `play_session_id` TEXT, `live_stream_id` TEXT, `created_at_epoch_millis` INTEGER NOT NULL, PRIMARY KEY(`id`), FOREIGN KEY(`account_id`) REFERENCES `provider_accounts`(`id`) ON UPDATE CASCADE ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "accountId", + "columnName": "account_id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "providerId", + "columnName": "provider_id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "itemId", + "columnName": "item_id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "mediaSourceId", + "columnName": "media_source_id", + "affinity": "TEXT" + }, + { + "fieldPath": "sourceType", + "columnName": "source_type", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "playSessionId", + "columnName": "play_session_id", + "affinity": "TEXT" + }, + { + "fieldPath": "liveStreamId", + "columnName": "live_stream_id", + "affinity": "TEXT" + }, + { + "fieldPath": "createdAtEpochMillis", + "columnName": "created_at_epoch_millis", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_provider_playback_sessions_account_id", + "unique": false, + "columnNames": [ + "account_id" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_provider_playback_sessions_account_id` ON `${TABLE_NAME}` (`account_id`)" + } + ], + "foreignKeys": [ + { + "table": "provider_accounts", + "onDelete": "CASCADE", + "onUpdate": "CASCADE", + "columns": [ + "account_id" + ], + "referencedColumns": [ + "id" + ] + } + ] + }, + { + "tableName": "channel_metadata_bases", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`playlist_url` TEXT NOT NULL, `channel_reference` TEXT NOT NULL, `title` TEXT NOT NULL, `category` TEXT NOT NULL, PRIMARY KEY(`playlist_url`, `channel_reference`), FOREIGN KEY(`playlist_url`) REFERENCES `playlists`(`url`) ON UPDATE CASCADE ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "playlistUrl", + "columnName": "playlist_url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "channelReference", + "columnName": "channel_reference", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "category", + "columnName": "category", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "playlist_url", + "channel_reference" + ] + }, + "indices": [ + { + "name": "index_channel_metadata_bases_playlist_url", + "unique": false, + "columnNames": [ + "playlist_url" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_channel_metadata_bases_playlist_url` ON `${TABLE_NAME}` (`playlist_url`)" + } + ], + "foreignKeys": [ + { + "table": "playlists", + "onDelete": "CASCADE", + "onUpdate": "CASCADE", + "columns": [ + "playlist_url" + ], + "referencedColumns": [ + "url" + ] + } + ] + }, + { + "tableName": "extension_channel_metadata_overlays", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`playlist_url` TEXT NOT NULL, `channel_reference` TEXT NOT NULL, `extension_id` TEXT NOT NULL, `title` TEXT, `category` TEXT, PRIMARY KEY(`playlist_url`, `channel_reference`, `extension_id`), FOREIGN KEY(`playlist_url`, `channel_reference`) REFERENCES `channel_metadata_bases`(`playlist_url`, `channel_reference`) ON UPDATE CASCADE ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "playlistUrl", + "columnName": "playlist_url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "channelReference", + "columnName": "channel_reference", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "extensionId", + "columnName": "extension_id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT" + }, + { + "fieldPath": "category", + "columnName": "category", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "playlist_url", + "channel_reference", + "extension_id" + ] + }, + "indices": [ + { + "name": "index_extension_channel_metadata_overlays_playlist_url_channel_reference", + "unique": false, + "columnNames": [ + "playlist_url", + "channel_reference" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_extension_channel_metadata_overlays_playlist_url_channel_reference` ON `${TABLE_NAME}` (`playlist_url`, `channel_reference`)" + }, + { + "name": "index_extension_channel_metadata_overlays_extension_id", + "unique": false, + "columnNames": [ + "extension_id" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_extension_channel_metadata_overlays_extension_id` ON `${TABLE_NAME}` (`extension_id`)" + } + ], + "foreignKeys": [ + { + "table": "channel_metadata_bases", + "onDelete": "CASCADE", + "onUpdate": "CASCADE", + "columns": [ + "playlist_url", + "channel_reference" + ], + "referencedColumns": [ + "playlist_url", + "channel_reference" + ] + } + ] + } + ], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'bca92df38500d6fce82a620dab053867')" + ] + } +} \ No newline at end of file diff --git a/data/src/androidTest/java/com/m3u/data/database/Migration26To27Test.kt b/data/src/androidTest/java/com/m3u/data/database/Migration26To27Test.kt new file mode 100644 index 000000000..bd3e4ec19 --- /dev/null +++ b/data/src/androidTest/java/com/m3u/data/database/Migration26To27Test.kt @@ -0,0 +1,130 @@ +package com.m3u.data.database + +import android.content.Context +import androidx.room.Room +import androidx.room.testing.MigrationTestHelper +import androidx.sqlite.db.SupportSQLiteDatabase +import androidx.test.core.app.ApplicationProvider +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import org.junit.Assert.assertEquals +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith + +/** + * The folded title column is only useful if the rows already on disk get filled + * in — a user who has been running the app for months never re-imports their + * catalogue, so a migration that only adds the column would leave search broken + * for exactly the people who have the most channels. + */ +@RunWith(AndroidJUnit4::class) +class Migration26To27Test { + @get:Rule + val migrationHelper = MigrationTestHelper( + instrumentation = InstrumentationRegistry.getInstrumentation(), + databaseClass = M3UDatabase::class.java, + ) + + @Test + fun migrationFoldsExistingTitles() { + migrationHelper.createDatabase(DATABASE_NAME, 26).apply { + insertPlaylist(PLAYLIST_URL, "Provider") + TITLES.forEachIndexed { index, title -> + insertChannel(id = index + 1, title = title) + } + close() + } + + val database = Room + .databaseBuilder( + ApplicationProvider.getApplicationContext(), + M3UDatabase::class.java, + DATABASE_NAME, + ) + .allowMainThreadQueries() + .addMigrations(DatabaseMigrations.MIGRATION_26_27) + .build() + val migrated = database.openHelper.writableDatabase + + assertEquals( + listOf( + "le prenom", + "amelie", + "a bout de souffle", + "spider-man: no way home", + "千と千尋の神隠し", + ), + migrated.readColumn("title_normalized"), + ) + // The displayed title is untouched — only the search copy is folded. + assertEquals(TITLES, migrated.readColumn("title")) + database.close() + } + + @Test + fun migrationCoversRowsBeyondASingleBatch() { + // The backfill walks the table in keyed batches; a catalogue larger than + // one batch must come out entirely folded, not just its first page. + val count = 1_200 + migrationHelper.createDatabase(DATABASE_NAME, 26).apply { + insertPlaylist(PLAYLIST_URL, "Provider") + repeat(count) { index -> insertChannel(id = index + 1, title = "Épisode $index") } + close() + } + + val database = Room + .databaseBuilder( + ApplicationProvider.getApplicationContext(), + M3UDatabase::class.java, + DATABASE_NAME, + ) + .allowMainThreadQueries() + .addMigrations(DatabaseMigrations.MIGRATION_26_27) + .build() + val migrated = database.openHelper.writableDatabase + + migrated.query( + "SELECT COUNT(*) FROM streams WHERE title_normalized LIKE 'episode %'" + ).use { cursor -> + cursor.moveToFirst() + assertEquals(count, cursor.getInt(0)) + } + database.close() + } + + private fun SupportSQLiteDatabase.readColumn(column: String): List = buildList { + query("SELECT $column FROM streams ORDER BY id").use { cursor -> + while (cursor.moveToNext()) add(cursor.getString(0)) + } + } + + private fun SupportSQLiteDatabase.insertPlaylist(url: String, title: String) { + execSQL( + "INSERT INTO playlists (url, title) VALUES (?, ?)", + arrayOf(url, title), + ) + } + + private fun SupportSQLiteDatabase.insertChannel(id: Int, title: String) { + execSQL( + """ + INSERT INTO streams (id, url, `group`, title, playlist_url, favourite, hidden, seen) + VALUES (?, ?, ?, ?, ?, 0, 0, 0) + """.trimIndent(), + arrayOf(id, "http://example.test/$id.mkv", "Films", title, PLAYLIST_URL), + ) + } + + private companion object { + const val DATABASE_NAME = "migration-26-27" + const val PLAYLIST_URL = "http://example.test/playlist.m3u" + val TITLES = listOf( + "Le Prénom", + "Amélie", + "À bout de souffle", + "Spider-Man: No Way Home", + "千と千尋の神隠し", + ) + } +} diff --git a/data/src/main/java/com/m3u/data/database/DatabaseMigrations.kt b/data/src/main/java/com/m3u/data/database/DatabaseMigrations.kt index 99be2a363..82c2fc89c 100644 --- a/data/src/main/java/com/m3u/data/database/DatabaseMigrations.kt +++ b/data/src/main/java/com/m3u/data/database/DatabaseMigrations.kt @@ -6,6 +6,7 @@ import androidx.room.RenameTable import androidx.room.migration.AutoMigrationSpec import androidx.room.migration.Migration import androidx.sqlite.db.SupportSQLiteDatabase +import com.m3u.core.foundation.util.basic.normalizeForSearch import com.m3u.data.extension.security.CredentialVault import com.m3u.extension.api.ExtensionId import com.m3u.extension.api.subscription.ProviderKind @@ -483,6 +484,53 @@ internal object DatabaseMigrations { ) } + /** + * Adds the folded copy of every channel title that search compares against. + * + * The backfill runs in Kotlin rather than in SQL so it goes through the very + * same [normalizeForSearch] the writes and the queries use. Spelling the + * fold out as a stack of SQL REPLACE calls would work today and drift from + * the Kotlin version the first time either side is touched — and the two + * disagreeing is invisible: rows simply stop matching. + * + * Rows are walked in keyed batches instead of one long cursor, so nothing + * is updated underneath an open cursor and memory stays flat whatever the + * catalogue size. Xtream playlists here run to about 41 000 rows. + */ + val MIGRATION_26_27 = object : Migration(26, 27) { + override fun migrate(db: SupportSQLiteDatabase) { + db.execSQL( + "ALTER TABLE streams ADD COLUMN title_normalized TEXT NOT NULL DEFAULT ''" + ) + val update = db.compileStatement( + "UPDATE streams SET title_normalized = ? WHERE id = ?" + ) + var lastId = Int.MIN_VALUE + while (true) { + val batch = buildList { + db.query( + "SELECT id, title FROM streams WHERE id > ? ORDER BY id LIMIT ?", + arrayOf(lastId, BACKFILL_BATCH_SIZE), + ).use { cursor -> + while (cursor.moveToNext()) { + add(cursor.getInt(0) to cursor.getString(1)) + } + } + } + if (batch.isEmpty()) break + batch.forEach { (id, title) -> + update.clearBindings() + update.bindString(1, title.normalizeForSearch()) + update.bindLong(2, id.toLong()) + update.executeUpdateDelete() + } + lastId = batch.last().first + } + } + } + + private const val BACKFILL_BATCH_SIZE = 500 + private fun SupportSQLiteDatabase.enableSecureDelete() { query("PRAGMA secure_delete = ON").use { cursor -> check(cursor.moveToFirst() && cursor.getInt(0) == 1) { diff --git a/data/src/main/java/com/m3u/data/database/DatabaseModule.kt b/data/src/main/java/com/m3u/data/database/DatabaseModule.kt index 640e1bb3a..614b0ee1c 100644 --- a/data/src/main/java/com/m3u/data/database/DatabaseModule.kt +++ b/data/src/main/java/com/m3u/data/database/DatabaseModule.kt @@ -50,6 +50,7 @@ internal object DatabaseModule { .addMigrations(DatabaseMigrations.migration22To23(credentialVault)) .addMigrations(DatabaseMigrations.MIGRATION_24_25) .addMigrations(DatabaseMigrations.MIGRATION_25_26) + .addMigrations(DatabaseMigrations.MIGRATION_26_27) .build() @Provides diff --git a/data/src/main/java/com/m3u/data/database/M3UDatabase.kt b/data/src/main/java/com/m3u/data/database/M3UDatabase.kt index 3abf40172..4d26ee7f9 100644 --- a/data/src/main/java/com/m3u/data/database/M3UDatabase.kt +++ b/data/src/main/java/com/m3u/data/database/M3UDatabase.kt @@ -36,7 +36,7 @@ import com.m3u.data.database.model.ProviderPlaybackSessionEntity ChannelMetadataBase::class, ExtensionChannelMetadataOverlay::class, ], - version = 26, + version = 27, exportSchema = true, autoMigrations = [ AutoMigration( diff --git a/data/src/main/java/com/m3u/data/database/dao/ChannelDao.kt b/data/src/main/java/com/m3u/data/database/dao/ChannelDao.kt index 1525e4e10..dbe14d854 100644 --- a/data/src/main/java/com/m3u/data/database/dao/ChannelDao.kt +++ b/data/src/main/java/com/m3u/data/database/dao/ChannelDao.kt @@ -184,7 +184,7 @@ interface ChannelDao { SELECT DISTINCT `group` FROM streams WHERE playlist_url = :playlistUrl - AND title LIKE '%'||:query||'%' + AND title_normalized LIKE '%'||:query||'%' """ ) suspend fun getCategoriesByPlaylistUrl( @@ -197,7 +197,7 @@ interface ChannelDao { SELECT DISTINCT `group` FROM streams WHERE playlist_url = :playlistUrl - AND title LIKE '%'||:query||'%' + AND title_normalized LIKE '%'||:query||'%' """ ) fun observeCategoriesByPlaylistUrl( @@ -276,7 +276,15 @@ interface ChannelDao { stream.favourite AS favourite, stream.hidden AS hidden, stream.seen AS seen, - stream.relation_id AS relation_id + stream.relation_id AS relation_id, + -- Present only because Room requires every non-null field to be + -- returned. This projection overrides the title with an extension + -- overlay, so the folded copy taken from the source row may not + -- match it — which costs nothing here: the field sits outside the + -- constructor, so it is neither serialised into the backup nor + -- carried across a restore. It is recomputed from whatever title + -- the channel is rebuilt with. + stream.title_normalized AS title_normalized FROM streams AS stream LEFT JOIN channel_metadata_bases AS base ON base.playlist_url = stream.playlist_url @@ -313,7 +321,7 @@ interface ChannelDao { """ SELECT * FROM streams WHERE playlist_url = :url - AND title LIKE '%'||:query||'%' + AND title_normalized LIKE '%'||:query||'%' AND `group` = :category """ ) @@ -327,7 +335,7 @@ interface ChannelDao { """ SELECT * FROM streams WHERE playlist_url = :url - AND title LIKE '%'||:query||'%' + AND title_normalized LIKE '%'||:query||'%' AND `group` = :category ORDER BY title ASC """ @@ -342,7 +350,7 @@ interface ChannelDao { """ SELECT * FROM streams WHERE playlist_url = :url - AND title LIKE '%'||:query||'%' + AND title_normalized LIKE '%'||:query||'%' AND `group` = :category ORDER BY title DESC """ @@ -357,7 +365,7 @@ interface ChannelDao { """ SELECT * FROM streams WHERE playlist_url = :url - AND title LIKE '%'||:query||'%' + AND title_normalized LIKE '%'||:query||'%' AND `group` = :category ORDER BY seen DESC """ @@ -372,7 +380,7 @@ interface ChannelDao { """ SELECT * FROM streams WHERE playlist_url = :url - AND title LIKE '%'||:query||'%' + AND title_normalized LIKE '%'||:query||'%' """ ) fun pagingAllByPlaylistUrlMixed( @@ -430,10 +438,15 @@ interface ChannelDao { ): Flow + /** + * @param query must already be folded with normalizeForSearch — the column + * it is compared against holds folded titles, so an unfolded query would + * match nothing as soon as it carried an accent or a capital. + */ @Query( """ SELECT * FROM streams WHERE 1 - AND title LIKE '%'||:query||'%' + AND title_normalized LIKE '%'||:query||'%' """ ) fun query( @@ -475,7 +488,7 @@ interface ChannelDao { @Query( """ SELECT * FROM streams WHERE 1 - AND title LIKE '%'||:query||'%' + AND title_normalized LIKE '%'||:query||'%' """ ) fun pagingAll(query: String): PagingSource diff --git a/data/src/main/java/com/m3u/data/database/model/Channel.kt b/data/src/main/java/com/m3u/data/database/model/Channel.kt index 2b65c1b3a..2f05b51e0 100644 --- a/data/src/main/java/com/m3u/data/database/model/Channel.kt +++ b/data/src/main/java/com/m3u/data/database/model/Channel.kt @@ -6,6 +6,7 @@ import androidx.room.Entity import androidx.room.PrimaryKey import com.m3u.annotation.Exclude import com.m3u.annotation.Likable +import com.m3u.core.foundation.util.basic.normalizeForSearch import com.m3u.data.parser.xtream.XtreamEpisodeInfo import io.ktor.http.URLBuilder import io.ktor.http.Url @@ -65,6 +66,24 @@ data class Channel( */ val relationId: String? = null ) { + /** + * [title] with diacritics and case folded away — what search compares + * against, since SQLite's LIKE never ignores accents on its own. + * + * Declared outside the constructor on purpose. A data class only copies + * constructor parameters, so every construction path recomputes this from + * whatever [title] ends up being — including copy(title = …), which would + * otherwise carry a stale value forward and silently drop the channel out + * of every search. The invariant holds by construction rather than by + * remembering to maintain it. + */ + // No index: search matches on '%query%', which no B-tree index can serve, + // and one more index would only slow down the bulk inserts a resubscription + // performs on tens of thousands of rows. + @ColumnInfo(name = "title_normalized", defaultValue = "''") + @Exclude + var titleNormalized: String = title.normalizeForSearch() + companion object { const val URL_DYNAMIC = "dynamic" const val LICENSE_TYPE_WIDEVINE = "com.widevine.alpha" diff --git a/data/src/main/java/com/m3u/data/repository/channel/ChannelRepositoryImpl.kt b/data/src/main/java/com/m3u/data/repository/channel/ChannelRepositoryImpl.kt index 3dd17762d..3fdd22769 100644 --- a/data/src/main/java/com/m3u/data/repository/channel/ChannelRepositoryImpl.kt +++ b/data/src/main/java/com/m3u/data/repository/channel/ChannelRepositoryImpl.kt @@ -2,6 +2,7 @@ package com.m3u.data.repository.channel import androidx.paging.PagingSource import com.m3u.core.foundation.architecture.preferences.Settings +import com.m3u.core.foundation.util.basic.normalizeForSearch import com.m3u.core.foundation.wrapper.Sort import com.m3u.data.database.dao.ChannelDao import com.m3u.data.database.dao.PlaylistDao @@ -31,8 +32,12 @@ internal class ChannelRepositoryImpl @Inject constructor( .observeRelationIdsByPlaylistUrl(playlistUrl) .catch { emit(emptyList()) } + // Every user-typed query is folded here rather than at each call site. + // The DAO compares against a folded column, so a raw query would stop + // matching the moment it carried an accent or a capital — and callers + // would have no way of telling, since the result is simply an empty list. override fun pagingAll(query: String): PagingSource { - return channelDao.pagingAll(query) + return channelDao.pagingAll(query.normalizeForSearch()) } override fun pagingAllByPlaylistUrl( @@ -40,12 +45,15 @@ internal class ChannelRepositoryImpl @Inject constructor( category: String, query: String, sort: Sort - ): PagingSource = when (sort) { - Sort.UNSPECIFIED -> channelDao.pagingAllByPlaylistUrl(url, category, query) - Sort.ASC -> channelDao.pagingAllByPlaylistUrlAsc(url, category, query) - Sort.DESC -> channelDao.pagingAllByPlaylistUrlDesc(url, category, query) - Sort.RECENTLY -> channelDao.pagingAllByPlaylistUrlRecently(url, category, query) - Sort.MIXED -> channelDao.pagingAllByPlaylistUrlMixed(url, query) + ): PagingSource { + val folded = query.normalizeForSearch() + return when (sort) { + Sort.UNSPECIFIED -> channelDao.pagingAllByPlaylistUrl(url, category, folded) + Sort.ASC -> channelDao.pagingAllByPlaylistUrlAsc(url, category, folded) + Sort.DESC -> channelDao.pagingAllByPlaylistUrlDesc(url, category, folded) + Sort.RECENTLY -> channelDao.pagingAllByPlaylistUrlRecently(url, category, folded) + Sort.MIXED -> channelDao.pagingAllByPlaylistUrlMixed(url, folded) + } } override suspend fun get(id: Int): Channel? = channelDao.get(id) @@ -151,6 +159,6 @@ internal class ChannelRepositoryImpl @Inject constructor( .catch { emit(emptyList()) } override fun search(query: String): PagingSource { - return channelDao.query(query) + return channelDao.query(query.normalizeForSearch()) } } diff --git a/data/src/main/java/com/m3u/data/repository/playlist/PlaylistRepositoryImpl.kt b/data/src/main/java/com/m3u/data/repository/playlist/PlaylistRepositoryImpl.kt index 1f0fb319c..d76fd04b8 100644 --- a/data/src/main/java/com/m3u/data/repository/playlist/PlaylistRepositoryImpl.kt +++ b/data/src/main/java/com/m3u/data/repository/playlist/PlaylistRepositoryImpl.kt @@ -12,6 +12,7 @@ import com.m3u.core.foundation.architecture.preferences.PreferencesKeys import com.m3u.core.foundation.architecture.preferences.Settings import com.m3u.core.foundation.architecture.preferences.get import com.m3u.core.foundation.util.basic.PlaylistInputKind +import com.m3u.core.foundation.util.basic.normalizeForSearch import com.m3u.core.foundation.util.basic.normalizePlaylistInputForSubmission import com.m3u.core.foundation.util.basic.startsWithAny import com.m3u.data.api.OkhttpClient @@ -1048,7 +1049,7 @@ internal class PlaylistRepositoryImpl @Inject constructor( val pinnedCategories = playlist?.pinnedCategories ?: emptyList() val hiddenCategories = playlist?.hiddenCategories ?: emptyList() channelDao - .getCategoriesByPlaylistUrl(url, query) + .getCategoriesByPlaylistUrl(url, query.normalizeForSearch()) .filterNot { it in hiddenCategories } .sortedByDescending { it in pinnedCategories } } @@ -1061,7 +1062,7 @@ internal class PlaylistRepositoryImpl @Inject constructor( val pinnedCategories = playlist.pinnedCategories val hiddenCategories = playlist.hiddenCategories channelDao - .observeCategoriesByPlaylistUrl(playlist.url, query) + .observeCategoriesByPlaylistUrl(playlist.url, query.normalizeForSearch()) .map { categories -> categories .filterNot { it in hiddenCategories } From 760b843ca1ff331c238bc2a514323c7313f58dfc Mon Sep 17 00:00:00 2001 From: Pierroons <97373452+Pierroons@users.noreply.github.com> Date: Sun, 9 Aug 2026 17:11:07 +0200 Subject: [PATCH 2/2] review: make titleNormalized a val, drop the migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review points from @oxyroid. The property is now a constructor parameter defaulting to title.normalizeForSearch(), so the @Immutable contract holds. It sat outside the constructor to survive copy(title = …), which keeps the old value and would silently drop a channel out of every search — but nothing in the codebase copies a channel with a new title today, the two call sites in PlaylistRepositoryImpl only reassign ids. Migration, schema and migration test are out, to be consolidated with the other PRs. --- .../com.m3u.data.database.M3UDatabase/27.json | 840 ------------------ .../m3u/data/database/Migration26To27Test.kt | 130 --- .../m3u/data/database/DatabaseMigrations.kt | 48 - .../com/m3u/data/database/DatabaseModule.kt | 1 - .../java/com/m3u/data/database/M3UDatabase.kt | 2 +- .../com/m3u/data/database/model/Channel.kt | 19 +- 6 files changed, 11 insertions(+), 1029 deletions(-) delete mode 100644 data/schemas/com.m3u.data.database.M3UDatabase/27.json delete mode 100644 data/src/androidTest/java/com/m3u/data/database/Migration26To27Test.kt diff --git a/data/schemas/com.m3u.data.database.M3UDatabase/27.json b/data/schemas/com.m3u.data.database.M3UDatabase/27.json deleted file mode 100644 index 321a096f6..000000000 --- a/data/schemas/com.m3u.data.database.M3UDatabase/27.json +++ /dev/null @@ -1,840 +0,0 @@ -{ - "formatVersion": 1, - "database": { - "version": 27, - "identityHash": "bca92df38500d6fce82a620dab053867", - "entities": [ - { - "tableName": "playlists", - "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`title` TEXT NOT NULL, `url` TEXT NOT NULL, `pinned_groups` TEXT NOT NULL DEFAULT '[]', `hidden_groups` TEXT NOT NULL DEFAULT '[]', `source` TEXT NOT NULL DEFAULT '0', `user_agent` TEXT DEFAULT NULL, `epg_urls` TEXT NOT NULL DEFAULT '[]', `auto_refresh_programmes` INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(`url`))", - "fields": [ - { - "fieldPath": "title", - "columnName": "title", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "url", - "columnName": "url", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "pinnedCategories", - "columnName": "pinned_groups", - "affinity": "TEXT", - "notNull": true, - "defaultValue": "'[]'" - }, - { - "fieldPath": "hiddenCategories", - "columnName": "hidden_groups", - "affinity": "TEXT", - "notNull": true, - "defaultValue": "'[]'" - }, - { - "fieldPath": "source", - "columnName": "source", - "affinity": "TEXT", - "notNull": true, - "defaultValue": "'0'" - }, - { - "fieldPath": "userAgent", - "columnName": "user_agent", - "affinity": "TEXT", - "defaultValue": "NULL" - }, - { - "fieldPath": "epgUrls", - "columnName": "epg_urls", - "affinity": "TEXT", - "notNull": true, - "defaultValue": "'[]'" - }, - { - "fieldPath": "autoRefreshProgrammes", - "columnName": "auto_refresh_programmes", - "affinity": "INTEGER", - "notNull": true, - "defaultValue": "0" - } - ], - "primaryKey": { - "autoGenerate": false, - "columnNames": [ - "url" - ] - } - }, - { - "tableName": "streams", - "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`url` TEXT NOT NULL, `group` TEXT NOT NULL, `title` TEXT NOT NULL, `cover` TEXT, `playlist_url` TEXT NOT NULL, `license_type` TEXT DEFAULT NULL, `license_key` TEXT DEFAULT NULL, `id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `favourite` INTEGER NOT NULL, `hidden` INTEGER NOT NULL DEFAULT 0, `seen` INTEGER NOT NULL DEFAULT 0, `relation_id` TEXT DEFAULT NULL, `title_normalized` TEXT NOT NULL DEFAULT '')", - "fields": [ - { - "fieldPath": "url", - "columnName": "url", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "category", - "columnName": "group", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "title", - "columnName": "title", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "cover", - "columnName": "cover", - "affinity": "TEXT" - }, - { - "fieldPath": "playlistUrl", - "columnName": "playlist_url", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "licenseType", - "columnName": "license_type", - "affinity": "TEXT", - "defaultValue": "NULL" - }, - { - "fieldPath": "licenseKey", - "columnName": "license_key", - "affinity": "TEXT", - "defaultValue": "NULL" - }, - { - "fieldPath": "id", - "columnName": "id", - "affinity": "INTEGER", - "notNull": true - }, - { - "fieldPath": "favourite", - "columnName": "favourite", - "affinity": "INTEGER", - "notNull": true - }, - { - "fieldPath": "hidden", - "columnName": "hidden", - "affinity": "INTEGER", - "notNull": true, - "defaultValue": "0" - }, - { - "fieldPath": "seen", - "columnName": "seen", - "affinity": "INTEGER", - "notNull": true, - "defaultValue": "0" - }, - { - "fieldPath": "relationId", - "columnName": "relation_id", - "affinity": "TEXT", - "defaultValue": "NULL" - }, - { - "fieldPath": "titleNormalized", - "columnName": "title_normalized", - "affinity": "TEXT", - "notNull": true, - "defaultValue": "''" - } - ], - "primaryKey": { - "autoGenerate": true, - "columnNames": [ - "id" - ] - }, - "indices": [ - { - "name": "index_streams_playlist_url", - "unique": false, - "columnNames": [ - "playlist_url" - ], - "orders": [], - "createSql": "CREATE INDEX IF NOT EXISTS `index_streams_playlist_url` ON `${TABLE_NAME}` (`playlist_url`)" - }, - { - "name": "index_streams_favourite", - "unique": false, - "columnNames": [ - "favourite" - ], - "orders": [], - "createSql": "CREATE INDEX IF NOT EXISTS `index_streams_favourite` ON `${TABLE_NAME}` (`favourite`)" - } - ] - }, - { - "tableName": "programmes", - "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`relation_id` TEXT NOT NULL, `epg_url` TEXT NOT NULL, `start` INTEGER NOT NULL, `end` INTEGER NOT NULL, `title` TEXT NOT NULL, `description` TEXT NOT NULL, `icon` TEXT, `categories` TEXT NOT NULL, `id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL)", - "fields": [ - { - "fieldPath": "channelId", - "columnName": "relation_id", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "epgUrl", - "columnName": "epg_url", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "start", - "columnName": "start", - "affinity": "INTEGER", - "notNull": true - }, - { - "fieldPath": "end", - "columnName": "end", - "affinity": "INTEGER", - "notNull": true - }, - { - "fieldPath": "title", - "columnName": "title", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "description", - "columnName": "description", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "icon", - "columnName": "icon", - "affinity": "TEXT" - }, - { - "fieldPath": "categories", - "columnName": "categories", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "id", - "columnName": "id", - "affinity": "INTEGER", - "notNull": true - } - ], - "primaryKey": { - "autoGenerate": true, - "columnNames": [ - "id" - ] - }, - "indices": [ - { - "name": "index_programmes_epg_url_relation_id_start_end", - "unique": false, - "columnNames": [ - "epg_url", - "relation_id", - "start", - "end" - ], - "orders": [], - "createSql": "CREATE INDEX IF NOT EXISTS `index_programmes_epg_url_relation_id_start_end` ON `${TABLE_NAME}` (`epg_url`, `relation_id`, `start`, `end`)" - } - ] - }, - { - "tableName": "episodes", - "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`title` TEXT NOT NULL, `series_id` INTEGER NOT NULL, `season` TEXT NOT NULL, `number` INTEGER NOT NULL, `url` TEXT NOT NULL, `id` INTEGER NOT NULL, PRIMARY KEY(`id`))", - "fields": [ - { - "fieldPath": "title", - "columnName": "title", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "seriesId", - "columnName": "series_id", - "affinity": "INTEGER", - "notNull": true - }, - { - "fieldPath": "season", - "columnName": "season", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "number", - "columnName": "number", - "affinity": "INTEGER", - "notNull": true - }, - { - "fieldPath": "url", - "columnName": "url", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "id", - "columnName": "id", - "affinity": "INTEGER", - "notNull": true - } - ], - "primaryKey": { - "autoGenerate": false, - "columnNames": [ - "id" - ] - } - }, - { - "tableName": "color_pack", - "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`argb` INTEGER NOT NULL, `dark` INTEGER NOT NULL, `name` TEXT NOT NULL, PRIMARY KEY(`argb`, `dark`))", - "fields": [ - { - "fieldPath": "argb", - "columnName": "argb", - "affinity": "INTEGER", - "notNull": true - }, - { - "fieldPath": "isDark", - "columnName": "dark", - "affinity": "INTEGER", - "notNull": true - }, - { - "fieldPath": "name", - "columnName": "name", - "affinity": "TEXT", - "notNull": true - } - ], - "primaryKey": { - "autoGenerate": false, - "columnNames": [ - "argb", - "dark" - ] - } - }, - { - "tableName": "provider_accounts", - "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `provider_id` TEXT NOT NULL, `provider_kind` TEXT NOT NULL, `base_url` TEXT NOT NULL, `server_id` TEXT NOT NULL, `server_name` TEXT NOT NULL, `server_version` TEXT NOT NULL, `user_id` TEXT NOT NULL, `username` TEXT NOT NULL, `playlist_url` TEXT NOT NULL, `requires_reauthentication` INTEGER NOT NULL DEFAULT 0, `owner_package_name` TEXT, `owner_service_name` TEXT, `owner_certificate_sha256` TEXT, PRIMARY KEY(`id`), FOREIGN KEY(`playlist_url`) REFERENCES `playlists`(`url`) ON UPDATE CASCADE ON DELETE CASCADE )", - "fields": [ - { - "fieldPath": "id", - "columnName": "id", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "providerId", - "columnName": "provider_id", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "providerKind", - "columnName": "provider_kind", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "baseUrl", - "columnName": "base_url", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "serverId", - "columnName": "server_id", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "serverName", - "columnName": "server_name", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "serverVersion", - "columnName": "server_version", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "userId", - "columnName": "user_id", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "username", - "columnName": "username", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "playlistUrl", - "columnName": "playlist_url", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "requiresReauthentication", - "columnName": "requires_reauthentication", - "affinity": "INTEGER", - "notNull": true, - "defaultValue": "0" - }, - { - "fieldPath": "ownerPackageName", - "columnName": "owner_package_name", - "affinity": "TEXT" - }, - { - "fieldPath": "ownerServiceName", - "columnName": "owner_service_name", - "affinity": "TEXT" - }, - { - "fieldPath": "ownerCertificateSha256", - "columnName": "owner_certificate_sha256", - "affinity": "TEXT" - } - ], - "primaryKey": { - "autoGenerate": false, - "columnNames": [ - "id" - ] - }, - "indices": [ - { - "name": "index_provider_accounts_playlist_url", - "unique": true, - "columnNames": [ - "playlist_url" - ], - "orders": [], - "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_provider_accounts_playlist_url` ON `${TABLE_NAME}` (`playlist_url`)" - }, - { - "name": "index_provider_accounts_provider_id_server_id_user_id", - "unique": true, - "columnNames": [ - "provider_id", - "server_id", - "user_id" - ], - "orders": [], - "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_provider_accounts_provider_id_server_id_user_id` ON `${TABLE_NAME}` (`provider_id`, `server_id`, `user_id`)" - } - ], - "foreignKeys": [ - { - "table": "playlists", - "onDelete": "CASCADE", - "onUpdate": "CASCADE", - "columns": [ - "playlist_url" - ], - "referencedColumns": [ - "url" - ] - } - ] - }, - { - "tableName": "provider_credentials", - "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`account_id` TEXT NOT NULL, `credential_handle` TEXT NOT NULL, `ciphertext` TEXT NOT NULL, `nonce` TEXT NOT NULL, `key_version` INTEGER NOT NULL, PRIMARY KEY(`account_id`), FOREIGN KEY(`account_id`) REFERENCES `provider_accounts`(`id`) ON UPDATE CASCADE ON DELETE CASCADE )", - "fields": [ - { - "fieldPath": "accountId", - "columnName": "account_id", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "credentialHandle", - "columnName": "credential_handle", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "ciphertext", - "columnName": "ciphertext", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "nonce", - "columnName": "nonce", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "keyVersion", - "columnName": "key_version", - "affinity": "INTEGER", - "notNull": true - } - ], - "primaryKey": { - "autoGenerate": false, - "columnNames": [ - "account_id" - ] - }, - "foreignKeys": [ - { - "table": "provider_accounts", - "onDelete": "CASCADE", - "onUpdate": "CASCADE", - "columns": [ - "account_id" - ], - "referencedColumns": [ - "id" - ] - } - ] - }, - { - "tableName": "channel_playback_references", - "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`channel_id` INTEGER NOT NULL, `account_id` TEXT NOT NULL, `provider_id` TEXT NOT NULL, `item_id` TEXT NOT NULL, `media_source_id` TEXT, `source_type` TEXT NOT NULL, PRIMARY KEY(`channel_id`), FOREIGN KEY(`channel_id`) REFERENCES `streams`(`id`) ON UPDATE CASCADE ON DELETE CASCADE , FOREIGN KEY(`account_id`) REFERENCES `provider_accounts`(`id`) ON UPDATE CASCADE ON DELETE CASCADE )", - "fields": [ - { - "fieldPath": "channelId", - "columnName": "channel_id", - "affinity": "INTEGER", - "notNull": true - }, - { - "fieldPath": "accountId", - "columnName": "account_id", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "providerId", - "columnName": "provider_id", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "itemId", - "columnName": "item_id", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "mediaSourceId", - "columnName": "media_source_id", - "affinity": "TEXT" - }, - { - "fieldPath": "sourceType", - "columnName": "source_type", - "affinity": "TEXT", - "notNull": true - } - ], - "primaryKey": { - "autoGenerate": false, - "columnNames": [ - "channel_id" - ] - }, - "indices": [ - { - "name": "index_channel_playback_references_account_id", - "unique": false, - "columnNames": [ - "account_id" - ], - "orders": [], - "createSql": "CREATE INDEX IF NOT EXISTS `index_channel_playback_references_account_id` ON `${TABLE_NAME}` (`account_id`)" - } - ], - "foreignKeys": [ - { - "table": "streams", - "onDelete": "CASCADE", - "onUpdate": "CASCADE", - "columns": [ - "channel_id" - ], - "referencedColumns": [ - "id" - ] - }, - { - "table": "provider_accounts", - "onDelete": "CASCADE", - "onUpdate": "CASCADE", - "columns": [ - "account_id" - ], - "referencedColumns": [ - "id" - ] - } - ] - }, - { - "tableName": "provider_playback_sessions", - "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `account_id` TEXT NOT NULL, `provider_id` TEXT NOT NULL, `item_id` TEXT NOT NULL, `media_source_id` TEXT, `source_type` TEXT NOT NULL, `play_session_id` TEXT, `live_stream_id` TEXT, `created_at_epoch_millis` INTEGER NOT NULL, PRIMARY KEY(`id`), FOREIGN KEY(`account_id`) REFERENCES `provider_accounts`(`id`) ON UPDATE CASCADE ON DELETE CASCADE )", - "fields": [ - { - "fieldPath": "id", - "columnName": "id", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "accountId", - "columnName": "account_id", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "providerId", - "columnName": "provider_id", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "itemId", - "columnName": "item_id", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "mediaSourceId", - "columnName": "media_source_id", - "affinity": "TEXT" - }, - { - "fieldPath": "sourceType", - "columnName": "source_type", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "playSessionId", - "columnName": "play_session_id", - "affinity": "TEXT" - }, - { - "fieldPath": "liveStreamId", - "columnName": "live_stream_id", - "affinity": "TEXT" - }, - { - "fieldPath": "createdAtEpochMillis", - "columnName": "created_at_epoch_millis", - "affinity": "INTEGER", - "notNull": true - } - ], - "primaryKey": { - "autoGenerate": false, - "columnNames": [ - "id" - ] - }, - "indices": [ - { - "name": "index_provider_playback_sessions_account_id", - "unique": false, - "columnNames": [ - "account_id" - ], - "orders": [], - "createSql": "CREATE INDEX IF NOT EXISTS `index_provider_playback_sessions_account_id` ON `${TABLE_NAME}` (`account_id`)" - } - ], - "foreignKeys": [ - { - "table": "provider_accounts", - "onDelete": "CASCADE", - "onUpdate": "CASCADE", - "columns": [ - "account_id" - ], - "referencedColumns": [ - "id" - ] - } - ] - }, - { - "tableName": "channel_metadata_bases", - "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`playlist_url` TEXT NOT NULL, `channel_reference` TEXT NOT NULL, `title` TEXT NOT NULL, `category` TEXT NOT NULL, PRIMARY KEY(`playlist_url`, `channel_reference`), FOREIGN KEY(`playlist_url`) REFERENCES `playlists`(`url`) ON UPDATE CASCADE ON DELETE CASCADE )", - "fields": [ - { - "fieldPath": "playlistUrl", - "columnName": "playlist_url", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "channelReference", - "columnName": "channel_reference", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "title", - "columnName": "title", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "category", - "columnName": "category", - "affinity": "TEXT", - "notNull": true - } - ], - "primaryKey": { - "autoGenerate": false, - "columnNames": [ - "playlist_url", - "channel_reference" - ] - }, - "indices": [ - { - "name": "index_channel_metadata_bases_playlist_url", - "unique": false, - "columnNames": [ - "playlist_url" - ], - "orders": [], - "createSql": "CREATE INDEX IF NOT EXISTS `index_channel_metadata_bases_playlist_url` ON `${TABLE_NAME}` (`playlist_url`)" - } - ], - "foreignKeys": [ - { - "table": "playlists", - "onDelete": "CASCADE", - "onUpdate": "CASCADE", - "columns": [ - "playlist_url" - ], - "referencedColumns": [ - "url" - ] - } - ] - }, - { - "tableName": "extension_channel_metadata_overlays", - "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`playlist_url` TEXT NOT NULL, `channel_reference` TEXT NOT NULL, `extension_id` TEXT NOT NULL, `title` TEXT, `category` TEXT, PRIMARY KEY(`playlist_url`, `channel_reference`, `extension_id`), FOREIGN KEY(`playlist_url`, `channel_reference`) REFERENCES `channel_metadata_bases`(`playlist_url`, `channel_reference`) ON UPDATE CASCADE ON DELETE CASCADE )", - "fields": [ - { - "fieldPath": "playlistUrl", - "columnName": "playlist_url", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "channelReference", - "columnName": "channel_reference", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "extensionId", - "columnName": "extension_id", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "title", - "columnName": "title", - "affinity": "TEXT" - }, - { - "fieldPath": "category", - "columnName": "category", - "affinity": "TEXT" - } - ], - "primaryKey": { - "autoGenerate": false, - "columnNames": [ - "playlist_url", - "channel_reference", - "extension_id" - ] - }, - "indices": [ - { - "name": "index_extension_channel_metadata_overlays_playlist_url_channel_reference", - "unique": false, - "columnNames": [ - "playlist_url", - "channel_reference" - ], - "orders": [], - "createSql": "CREATE INDEX IF NOT EXISTS `index_extension_channel_metadata_overlays_playlist_url_channel_reference` ON `${TABLE_NAME}` (`playlist_url`, `channel_reference`)" - }, - { - "name": "index_extension_channel_metadata_overlays_extension_id", - "unique": false, - "columnNames": [ - "extension_id" - ], - "orders": [], - "createSql": "CREATE INDEX IF NOT EXISTS `index_extension_channel_metadata_overlays_extension_id` ON `${TABLE_NAME}` (`extension_id`)" - } - ], - "foreignKeys": [ - { - "table": "channel_metadata_bases", - "onDelete": "CASCADE", - "onUpdate": "CASCADE", - "columns": [ - "playlist_url", - "channel_reference" - ], - "referencedColumns": [ - "playlist_url", - "channel_reference" - ] - } - ] - } - ], - "setupQueries": [ - "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", - "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'bca92df38500d6fce82a620dab053867')" - ] - } -} \ No newline at end of file diff --git a/data/src/androidTest/java/com/m3u/data/database/Migration26To27Test.kt b/data/src/androidTest/java/com/m3u/data/database/Migration26To27Test.kt deleted file mode 100644 index bd3e4ec19..000000000 --- a/data/src/androidTest/java/com/m3u/data/database/Migration26To27Test.kt +++ /dev/null @@ -1,130 +0,0 @@ -package com.m3u.data.database - -import android.content.Context -import androidx.room.Room -import androidx.room.testing.MigrationTestHelper -import androidx.sqlite.db.SupportSQLiteDatabase -import androidx.test.core.app.ApplicationProvider -import androidx.test.ext.junit.runners.AndroidJUnit4 -import androidx.test.platform.app.InstrumentationRegistry -import org.junit.Assert.assertEquals -import org.junit.Rule -import org.junit.Test -import org.junit.runner.RunWith - -/** - * The folded title column is only useful if the rows already on disk get filled - * in — a user who has been running the app for months never re-imports their - * catalogue, so a migration that only adds the column would leave search broken - * for exactly the people who have the most channels. - */ -@RunWith(AndroidJUnit4::class) -class Migration26To27Test { - @get:Rule - val migrationHelper = MigrationTestHelper( - instrumentation = InstrumentationRegistry.getInstrumentation(), - databaseClass = M3UDatabase::class.java, - ) - - @Test - fun migrationFoldsExistingTitles() { - migrationHelper.createDatabase(DATABASE_NAME, 26).apply { - insertPlaylist(PLAYLIST_URL, "Provider") - TITLES.forEachIndexed { index, title -> - insertChannel(id = index + 1, title = title) - } - close() - } - - val database = Room - .databaseBuilder( - ApplicationProvider.getApplicationContext(), - M3UDatabase::class.java, - DATABASE_NAME, - ) - .allowMainThreadQueries() - .addMigrations(DatabaseMigrations.MIGRATION_26_27) - .build() - val migrated = database.openHelper.writableDatabase - - assertEquals( - listOf( - "le prenom", - "amelie", - "a bout de souffle", - "spider-man: no way home", - "千と千尋の神隠し", - ), - migrated.readColumn("title_normalized"), - ) - // The displayed title is untouched — only the search copy is folded. - assertEquals(TITLES, migrated.readColumn("title")) - database.close() - } - - @Test - fun migrationCoversRowsBeyondASingleBatch() { - // The backfill walks the table in keyed batches; a catalogue larger than - // one batch must come out entirely folded, not just its first page. - val count = 1_200 - migrationHelper.createDatabase(DATABASE_NAME, 26).apply { - insertPlaylist(PLAYLIST_URL, "Provider") - repeat(count) { index -> insertChannel(id = index + 1, title = "Épisode $index") } - close() - } - - val database = Room - .databaseBuilder( - ApplicationProvider.getApplicationContext(), - M3UDatabase::class.java, - DATABASE_NAME, - ) - .allowMainThreadQueries() - .addMigrations(DatabaseMigrations.MIGRATION_26_27) - .build() - val migrated = database.openHelper.writableDatabase - - migrated.query( - "SELECT COUNT(*) FROM streams WHERE title_normalized LIKE 'episode %'" - ).use { cursor -> - cursor.moveToFirst() - assertEquals(count, cursor.getInt(0)) - } - database.close() - } - - private fun SupportSQLiteDatabase.readColumn(column: String): List = buildList { - query("SELECT $column FROM streams ORDER BY id").use { cursor -> - while (cursor.moveToNext()) add(cursor.getString(0)) - } - } - - private fun SupportSQLiteDatabase.insertPlaylist(url: String, title: String) { - execSQL( - "INSERT INTO playlists (url, title) VALUES (?, ?)", - arrayOf(url, title), - ) - } - - private fun SupportSQLiteDatabase.insertChannel(id: Int, title: String) { - execSQL( - """ - INSERT INTO streams (id, url, `group`, title, playlist_url, favourite, hidden, seen) - VALUES (?, ?, ?, ?, ?, 0, 0, 0) - """.trimIndent(), - arrayOf(id, "http://example.test/$id.mkv", "Films", title, PLAYLIST_URL), - ) - } - - private companion object { - const val DATABASE_NAME = "migration-26-27" - const val PLAYLIST_URL = "http://example.test/playlist.m3u" - val TITLES = listOf( - "Le Prénom", - "Amélie", - "À bout de souffle", - "Spider-Man: No Way Home", - "千と千尋の神隠し", - ) - } -} diff --git a/data/src/main/java/com/m3u/data/database/DatabaseMigrations.kt b/data/src/main/java/com/m3u/data/database/DatabaseMigrations.kt index 82c2fc89c..99be2a363 100644 --- a/data/src/main/java/com/m3u/data/database/DatabaseMigrations.kt +++ b/data/src/main/java/com/m3u/data/database/DatabaseMigrations.kt @@ -6,7 +6,6 @@ import androidx.room.RenameTable import androidx.room.migration.AutoMigrationSpec import androidx.room.migration.Migration import androidx.sqlite.db.SupportSQLiteDatabase -import com.m3u.core.foundation.util.basic.normalizeForSearch import com.m3u.data.extension.security.CredentialVault import com.m3u.extension.api.ExtensionId import com.m3u.extension.api.subscription.ProviderKind @@ -484,53 +483,6 @@ internal object DatabaseMigrations { ) } - /** - * Adds the folded copy of every channel title that search compares against. - * - * The backfill runs in Kotlin rather than in SQL so it goes through the very - * same [normalizeForSearch] the writes and the queries use. Spelling the - * fold out as a stack of SQL REPLACE calls would work today and drift from - * the Kotlin version the first time either side is touched — and the two - * disagreeing is invisible: rows simply stop matching. - * - * Rows are walked in keyed batches instead of one long cursor, so nothing - * is updated underneath an open cursor and memory stays flat whatever the - * catalogue size. Xtream playlists here run to about 41 000 rows. - */ - val MIGRATION_26_27 = object : Migration(26, 27) { - override fun migrate(db: SupportSQLiteDatabase) { - db.execSQL( - "ALTER TABLE streams ADD COLUMN title_normalized TEXT NOT NULL DEFAULT ''" - ) - val update = db.compileStatement( - "UPDATE streams SET title_normalized = ? WHERE id = ?" - ) - var lastId = Int.MIN_VALUE - while (true) { - val batch = buildList { - db.query( - "SELECT id, title FROM streams WHERE id > ? ORDER BY id LIMIT ?", - arrayOf(lastId, BACKFILL_BATCH_SIZE), - ).use { cursor -> - while (cursor.moveToNext()) { - add(cursor.getInt(0) to cursor.getString(1)) - } - } - } - if (batch.isEmpty()) break - batch.forEach { (id, title) -> - update.clearBindings() - update.bindString(1, title.normalizeForSearch()) - update.bindLong(2, id.toLong()) - update.executeUpdateDelete() - } - lastId = batch.last().first - } - } - } - - private const val BACKFILL_BATCH_SIZE = 500 - private fun SupportSQLiteDatabase.enableSecureDelete() { query("PRAGMA secure_delete = ON").use { cursor -> check(cursor.moveToFirst() && cursor.getInt(0) == 1) { diff --git a/data/src/main/java/com/m3u/data/database/DatabaseModule.kt b/data/src/main/java/com/m3u/data/database/DatabaseModule.kt index 614b0ee1c..640e1bb3a 100644 --- a/data/src/main/java/com/m3u/data/database/DatabaseModule.kt +++ b/data/src/main/java/com/m3u/data/database/DatabaseModule.kt @@ -50,7 +50,6 @@ internal object DatabaseModule { .addMigrations(DatabaseMigrations.migration22To23(credentialVault)) .addMigrations(DatabaseMigrations.MIGRATION_24_25) .addMigrations(DatabaseMigrations.MIGRATION_25_26) - .addMigrations(DatabaseMigrations.MIGRATION_26_27) .build() @Provides diff --git a/data/src/main/java/com/m3u/data/database/M3UDatabase.kt b/data/src/main/java/com/m3u/data/database/M3UDatabase.kt index 4d26ee7f9..3abf40172 100644 --- a/data/src/main/java/com/m3u/data/database/M3UDatabase.kt +++ b/data/src/main/java/com/m3u/data/database/M3UDatabase.kt @@ -36,7 +36,7 @@ import com.m3u.data.database.model.ProviderPlaybackSessionEntity ChannelMetadataBase::class, ExtensionChannelMetadataOverlay::class, ], - version = 27, + version = 26, exportSchema = true, autoMigrations = [ AutoMigration( diff --git a/data/src/main/java/com/m3u/data/database/model/Channel.kt b/data/src/main/java/com/m3u/data/database/model/Channel.kt index 2f05b51e0..c09e50dde 100644 --- a/data/src/main/java/com/m3u/data/database/model/Channel.kt +++ b/data/src/main/java/com/m3u/data/database/model/Channel.kt @@ -64,25 +64,26 @@ data class Channel( * if it is xtream vod, it may be streamId. * if it is xtream series, it may be seriesId. */ - val relationId: String? = null -) { + val relationId: String? = null, /** * [title] with diacritics and case folded away — what search compares * against, since SQLite's LIKE never ignores accents on its own. * - * Declared outside the constructor on purpose. A data class only copies - * constructor parameters, so every construction path recomputes this from - * whatever [title] ends up being — including copy(title = …), which would - * otherwise carry a stale value forward and silently drop the channel out - * of every search. The invariant holds by construction rather than by - * remembering to maintain it. + * Defaults off [title], so no caller has to remember to set it. + * + * The one way to desynchronise it is copy(title = …), which keeps this + * parameter as it was and would silently drop the channel out of every + * search. Nothing copies a channel with a new title today — the two call + * sites in PlaylistRepositoryImpl only reassign ids — so the invariant + * holds. */ // No index: search matches on '%query%', which no B-tree index can serve, // and one more index would only slow down the bulk inserts a resubscription // performs on tens of thousands of rows. @ColumnInfo(name = "title_normalized", defaultValue = "''") @Exclude - var titleNormalized: String = title.normalizeForSearch() + val titleNormalized: String = title.normalizeForSearch() +) { companion object { const val URL_DYNAMIC = "dynamic"