diff --git a/core/foundation/src/main/java/com/m3u/core/foundation/util/basic/Strings.kt b/core/foundation/src/main/java/com/m3u/core/foundation/util/basic/Strings.kt index 7cbfec224..6190d7452 100644 --- a/core/foundation/src/main/java/com/m3u/core/foundation/util/basic/Strings.kt +++ b/core/foundation/src/main/java/com/m3u/core/foundation/util/basic/Strings.kt @@ -3,6 +3,8 @@ package com.m3u.core.foundation.util.basic import androidx.compose.ui.text.capitalize import androidx.compose.ui.text.intl.Locale +import java.text.Normalizer +import java.util.Locale as JavaLocale fun String.title(): String { if (this.isEmpty()) return this @@ -20,4 +22,28 @@ fun String.startsWithAny(vararg prefix: String, ignoreCase: Boolean = false): Bo fun String.startWithHttpScheme(): Boolean = startsWithAny( "http://", "https://", ignoreCase = true -) \ No newline at end of file +) + +/** + * Folds a string down to the form search compares against: no diacritics, no + * case. + * + * SQLite's LIKE ignores case for ASCII but never accents, so "prenom" cannot + * match "Le Prénom" however the query is written — and a catalogue in French + * has a lot of those. Titles are stored pre-folded in a dedicated column and + * the query is folded the same way, which keeps the comparison symmetric: with + * or without accents, either side matches. + * + * NFD splits an accented letter into its base letter plus a combining mark, so + * dropping the marks (Unicode category Mn) leaves the base letter behind. + * Scripts without combining marks — CJK among them — pass through untouched. + * + * Locale.ROOT on purpose: a Turkish locale lowercases 'I' to a dotless 'ı', + * which would quietly make titles unsearchable for those users. + */ +fun String.normalizeForSearch(): String = Normalizer + .normalize(this, Normalizer.Form.NFD) + .replace(COMBINING_MARKS, "") + .lowercase(JavaLocale.ROOT) + +private val COMBINING_MARKS = Regex("\\p{Mn}+") \ No newline at end of file diff --git a/core/foundation/src/test/java/com/m3u/core/foundation/util/basic/NormalizeForSearchTest.kt b/core/foundation/src/test/java/com/m3u/core/foundation/util/basic/NormalizeForSearchTest.kt new file mode 100644 index 000000000..4d34bd570 --- /dev/null +++ b/core/foundation/src/test/java/com/m3u/core/foundation/util/basic/NormalizeForSearchTest.kt @@ -0,0 +1,45 @@ +package com.m3u.core.foundation.util.basic + +import org.junit.Assert.assertEquals +import org.junit.Test + +class NormalizeForSearchTest { + @Test + fun `accents are folded away`() { + assertEquals("le prenom", "Le Prénom".normalizeForSearch()) + assertEquals("amelie", "Amélie".normalizeForSearch()) + assertEquals("a bout de souffle", "À bout de souffle".normalizeForSearch()) + assertEquals("les miserables", "Les Misérables".normalizeForSearch()) + } + + @Test + fun `a query typed with accents matches the same folded form`() { + assertEquals("Le Prénom".normalizeForSearch(), "le prenom".normalizeForSearch()) + assertEquals("Le Prénom".normalizeForSearch(), "LE PRÉNOM".normalizeForSearch()) + } + + @Test + fun `case is folded independently of the device locale`() { + val previous = java.util.Locale.getDefault() + try { + // Turkish lowercases 'I' to a dotless 'ı'; using the default locale + // here would make "IT" unsearchable for those users. + java.util.Locale.setDefault(java.util.Locale.forLanguageTag("tr-TR")) + assertEquals("it crowd", "IT Crowd".normalizeForSearch()) + } finally { + java.util.Locale.setDefault(previous) + } + } + + @Test + fun `scripts without combining marks are left alone`() { + assertEquals("千と千尋の神隠し", "千と千尋の神隠し".normalizeForSearch()) + assertEquals("привет", "Привет".normalizeForSearch()) + } + + @Test + fun `punctuation and spacing survive so substring matching still works`() { + assertEquals("spider-man: no way home", "Spider-Man: No Way Home".normalizeForSearch()) + assertEquals("", "".normalizeForSearch()) + } +} diff --git a/data/src/main/java/com/m3u/data/database/dao/ChannelDao.kt b/data/src/main/java/com/m3u/data/database/dao/ChannelDao.kt index 1525e4e10..dbe14d854 100644 --- a/data/src/main/java/com/m3u/data/database/dao/ChannelDao.kt +++ b/data/src/main/java/com/m3u/data/database/dao/ChannelDao.kt @@ -184,7 +184,7 @@ interface ChannelDao { SELECT DISTINCT `group` FROM streams WHERE playlist_url = :playlistUrl - AND title LIKE '%'||:query||'%' + AND title_normalized LIKE '%'||:query||'%' """ ) suspend fun getCategoriesByPlaylistUrl( @@ -197,7 +197,7 @@ interface ChannelDao { SELECT DISTINCT `group` FROM streams WHERE playlist_url = :playlistUrl - AND title LIKE '%'||:query||'%' + AND title_normalized LIKE '%'||:query||'%' """ ) fun observeCategoriesByPlaylistUrl( @@ -276,7 +276,15 @@ interface ChannelDao { stream.favourite AS favourite, stream.hidden AS hidden, stream.seen AS seen, - stream.relation_id AS relation_id + stream.relation_id AS relation_id, + -- Present only because Room requires every non-null field to be + -- returned. This projection overrides the title with an extension + -- overlay, so the folded copy taken from the source row may not + -- match it — which costs nothing here: the field sits outside the + -- constructor, so it is neither serialised into the backup nor + -- carried across a restore. It is recomputed from whatever title + -- the channel is rebuilt with. + stream.title_normalized AS title_normalized FROM streams AS stream LEFT JOIN channel_metadata_bases AS base ON base.playlist_url = stream.playlist_url @@ -313,7 +321,7 @@ interface ChannelDao { """ SELECT * FROM streams WHERE playlist_url = :url - AND title LIKE '%'||:query||'%' + AND title_normalized LIKE '%'||:query||'%' AND `group` = :category """ ) @@ -327,7 +335,7 @@ interface ChannelDao { """ SELECT * FROM streams WHERE playlist_url = :url - AND title LIKE '%'||:query||'%' + AND title_normalized LIKE '%'||:query||'%' AND `group` = :category ORDER BY title ASC """ @@ -342,7 +350,7 @@ interface ChannelDao { """ SELECT * FROM streams WHERE playlist_url = :url - AND title LIKE '%'||:query||'%' + AND title_normalized LIKE '%'||:query||'%' AND `group` = :category ORDER BY title DESC """ @@ -357,7 +365,7 @@ interface ChannelDao { """ SELECT * FROM streams WHERE playlist_url = :url - AND title LIKE '%'||:query||'%' + AND title_normalized LIKE '%'||:query||'%' AND `group` = :category ORDER BY seen DESC """ @@ -372,7 +380,7 @@ interface ChannelDao { """ SELECT * FROM streams WHERE playlist_url = :url - AND title LIKE '%'||:query||'%' + AND title_normalized LIKE '%'||:query||'%' """ ) fun pagingAllByPlaylistUrlMixed( @@ -430,10 +438,15 @@ interface ChannelDao { ): Flow + /** + * @param query must already be folded with normalizeForSearch — the column + * it is compared against holds folded titles, so an unfolded query would + * match nothing as soon as it carried an accent or a capital. + */ @Query( """ SELECT * FROM streams WHERE 1 - AND title LIKE '%'||:query||'%' + AND title_normalized LIKE '%'||:query||'%' """ ) fun query( @@ -475,7 +488,7 @@ interface ChannelDao { @Query( """ SELECT * FROM streams WHERE 1 - AND title LIKE '%'||:query||'%' + AND title_normalized LIKE '%'||:query||'%' """ ) fun pagingAll(query: String): PagingSource diff --git a/data/src/main/java/com/m3u/data/database/model/Channel.kt b/data/src/main/java/com/m3u/data/database/model/Channel.kt index 2b65c1b3a..c09e50dde 100644 --- a/data/src/main/java/com/m3u/data/database/model/Channel.kt +++ b/data/src/main/java/com/m3u/data/database/model/Channel.kt @@ -6,6 +6,7 @@ import androidx.room.Entity import androidx.room.PrimaryKey import com.m3u.annotation.Exclude import com.m3u.annotation.Likable +import com.m3u.core.foundation.util.basic.normalizeForSearch import com.m3u.data.parser.xtream.XtreamEpisodeInfo import io.ktor.http.URLBuilder import io.ktor.http.Url @@ -63,8 +64,27 @@ data class Channel( * if it is xtream vod, it may be streamId. * if it is xtream series, it may be seriesId. */ - val relationId: String? = null + val relationId: String? = null, + /** + * [title] with diacritics and case folded away — what search compares + * against, since SQLite's LIKE never ignores accents on its own. + * + * Defaults off [title], so no caller has to remember to set it. + * + * The one way to desynchronise it is copy(title = …), which keeps this + * parameter as it was and would silently drop the channel out of every + * search. Nothing copies a channel with a new title today — the two call + * sites in PlaylistRepositoryImpl only reassign ids — so the invariant + * holds. + */ + // No index: search matches on '%query%', which no B-tree index can serve, + // and one more index would only slow down the bulk inserts a resubscription + // performs on tens of thousands of rows. + @ColumnInfo(name = "title_normalized", defaultValue = "''") + @Exclude + val titleNormalized: String = title.normalizeForSearch() ) { + companion object { const val URL_DYNAMIC = "dynamic" const val LICENSE_TYPE_WIDEVINE = "com.widevine.alpha" diff --git a/data/src/main/java/com/m3u/data/repository/channel/ChannelRepositoryImpl.kt b/data/src/main/java/com/m3u/data/repository/channel/ChannelRepositoryImpl.kt index 3dd17762d..3fdd22769 100644 --- a/data/src/main/java/com/m3u/data/repository/channel/ChannelRepositoryImpl.kt +++ b/data/src/main/java/com/m3u/data/repository/channel/ChannelRepositoryImpl.kt @@ -2,6 +2,7 @@ package com.m3u.data.repository.channel import androidx.paging.PagingSource import com.m3u.core.foundation.architecture.preferences.Settings +import com.m3u.core.foundation.util.basic.normalizeForSearch import com.m3u.core.foundation.wrapper.Sort import com.m3u.data.database.dao.ChannelDao import com.m3u.data.database.dao.PlaylistDao @@ -31,8 +32,12 @@ internal class ChannelRepositoryImpl @Inject constructor( .observeRelationIdsByPlaylistUrl(playlistUrl) .catch { emit(emptyList()) } + // Every user-typed query is folded here rather than at each call site. + // The DAO compares against a folded column, so a raw query would stop + // matching the moment it carried an accent or a capital — and callers + // would have no way of telling, since the result is simply an empty list. override fun pagingAll(query: String): PagingSource { - return channelDao.pagingAll(query) + return channelDao.pagingAll(query.normalizeForSearch()) } override fun pagingAllByPlaylistUrl( @@ -40,12 +45,15 @@ internal class ChannelRepositoryImpl @Inject constructor( category: String, query: String, sort: Sort - ): PagingSource = when (sort) { - Sort.UNSPECIFIED -> channelDao.pagingAllByPlaylistUrl(url, category, query) - Sort.ASC -> channelDao.pagingAllByPlaylistUrlAsc(url, category, query) - Sort.DESC -> channelDao.pagingAllByPlaylistUrlDesc(url, category, query) - Sort.RECENTLY -> channelDao.pagingAllByPlaylistUrlRecently(url, category, query) - Sort.MIXED -> channelDao.pagingAllByPlaylistUrlMixed(url, query) + ): PagingSource { + val folded = query.normalizeForSearch() + return when (sort) { + Sort.UNSPECIFIED -> channelDao.pagingAllByPlaylistUrl(url, category, folded) + Sort.ASC -> channelDao.pagingAllByPlaylistUrlAsc(url, category, folded) + Sort.DESC -> channelDao.pagingAllByPlaylistUrlDesc(url, category, folded) + Sort.RECENTLY -> channelDao.pagingAllByPlaylistUrlRecently(url, category, folded) + Sort.MIXED -> channelDao.pagingAllByPlaylistUrlMixed(url, folded) + } } override suspend fun get(id: Int): Channel? = channelDao.get(id) @@ -151,6 +159,6 @@ internal class ChannelRepositoryImpl @Inject constructor( .catch { emit(emptyList()) } override fun search(query: String): PagingSource { - return channelDao.query(query) + return channelDao.query(query.normalizeForSearch()) } } diff --git a/data/src/main/java/com/m3u/data/repository/playlist/PlaylistRepositoryImpl.kt b/data/src/main/java/com/m3u/data/repository/playlist/PlaylistRepositoryImpl.kt index 1f0fb319c..d76fd04b8 100644 --- a/data/src/main/java/com/m3u/data/repository/playlist/PlaylistRepositoryImpl.kt +++ b/data/src/main/java/com/m3u/data/repository/playlist/PlaylistRepositoryImpl.kt @@ -12,6 +12,7 @@ import com.m3u.core.foundation.architecture.preferences.PreferencesKeys import com.m3u.core.foundation.architecture.preferences.Settings import com.m3u.core.foundation.architecture.preferences.get import com.m3u.core.foundation.util.basic.PlaylistInputKind +import com.m3u.core.foundation.util.basic.normalizeForSearch import com.m3u.core.foundation.util.basic.normalizePlaylistInputForSubmission import com.m3u.core.foundation.util.basic.startsWithAny import com.m3u.data.api.OkhttpClient @@ -1048,7 +1049,7 @@ internal class PlaylistRepositoryImpl @Inject constructor( val pinnedCategories = playlist?.pinnedCategories ?: emptyList() val hiddenCategories = playlist?.hiddenCategories ?: emptyList() channelDao - .getCategoriesByPlaylistUrl(url, query) + .getCategoriesByPlaylistUrl(url, query.normalizeForSearch()) .filterNot { it in hiddenCategories } .sortedByDescending { it in pinnedCategories } } @@ -1061,7 +1062,7 @@ internal class PlaylistRepositoryImpl @Inject constructor( val pinnedCategories = playlist.pinnedCategories val hiddenCategories = playlist.hiddenCategories channelDao - .observeCategoriesByPlaylistUrl(playlist.url, query) + .observeCategoriesByPlaylistUrl(playlist.url, query.normalizeForSearch()) .map { categories -> categories .filterNot { it in hiddenCategories }