From 23cd79b3e75b0c8d030a41177badd6ef9fc49989 Mon Sep 17 00:00:00 2001 From: Andrew Polk Date: Tue, 25 Aug 2026 14:44:47 -0700 Subject: [PATCH] +semver:minor Add qps-ploc pseudolocalization support (BL-16748) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lookups for the standard qps-ploc pseudo-locale return the live English text pseudolocalized at runtime — every vowel doubled with an accent on the first of the pair, wrapped in brackets ("Title Missing" -> "[Tîitlée Mîissîing]") — so testers can spot non-internationalized strings (plain English), truncation (missing "]"), concatenation (brackets mid-sentence), and layout problems (~30-40% in-word expansion). Format placeholders ({0}, {0:n0}, named {app_title}-style, and %0-style) and HTML/XML markup pass through untouched. - Derived from English at lookup time: no translation files exist, are loaded, or are written for the pseudo-locale; it never maps to or from a real language; availability and counts report it exactly as complete as English. - Lookups work by simply setting the language code; all lookup funnels are hooked (static, preferred-languages, dynamic, and the WinForms paths — both runtime lookups and designer-created controls, whose component localizers read the string cache directly), transforming exactly once, with code-supplied English winning over the cache as it does for "en". - LocalizationManager.OfferPseudoLocalization (default false) gates only whether the locale is advertised in language lists; display name is hard-coded as "Pseudo-English (qps-ploc)". - LocalizationManager.PseudoLocalize exposes the transform for strings that bypass L10NSharp. - Self-contained in src/L10NSharp/Pseudo/ with no new package dependency; the placeholder-skipping logic is adapted from the MIT-licensed PseudoLocalizer project (attribution in the folder's README). 31 tests pin the transform and lookup semantics. Smoke-tested end to end in Bloom (WinForms chrome incl. designer-created dialogs, React UI, live placeholder substitution; no pseudo files written). Co-Authored-By: Claude Fable 5 Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 1 + .../PseudoLocalizationTests.cs | 395 ++++++++++++++++++ src/L10NSharp/L10NCultureInfo.cs | 10 + src/L10NSharp/LocalizationManager.cs | 28 ++ src/L10NSharp/LocalizationManagerInternal.cs | 59 +++ src/L10NSharp/Pseudo/EscapeHelpers.cs | 76 ++++ src/L10NSharp/Pseudo/PseudoLocalization.cs | 20 + src/L10NSharp/Pseudo/README.md | 61 +++ src/L10NSharp/Pseudo/VowelStretch.cs | 66 +++ .../XLiffUtils/XliffLocalizationManager.cs | 29 +- 10 files changed, 744 insertions(+), 1 deletion(-) create mode 100644 src/L10NSharp.Tests/PseudoLocalizationTests.cs create mode 100644 src/L10NSharp/Pseudo/EscapeHelpers.cs create mode 100644 src/L10NSharp/Pseudo/PseudoLocalization.cs create mode 100644 src/L10NSharp/Pseudo/README.md create mode 100644 src/L10NSharp/Pseudo/VowelStretch.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 305c7b6..7db3a82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Added a repository-level MIT `LICENSE` file, which is now bundled into the NuGet packages (via `PackageLicenseFile`). - [L10NSharp] Added `net8.0` as a target framework, enabling use on non-Windows platforms, and added cross-platform CI/CD coverage for `net8.0`. - [L10NSharp] Added UiLanguageChanged event to ILocalizationManager. This provides a way for clients to deal with changes now that (in Windows) LocalizeItemDlg.StringsLocalized no longer exists. +- [L10NSharp] Added pseudolocalization support: any lookup for the standard `qps-ploc` pseudo-locale (`LocalizationManager.PseudoLocalizationLanguageId`) returns the English text pseudolocalized at runtime — every vowel doubled with an accent on the first of the pair, wrapped in brackets (e.g. `[Tîitlée Mîissîing]`) — so testers can spot non-internationalized strings and layout problems. No translation files exist or are created for it. Set `LocalizationManager.OfferPseudoLocalization = true` to include it in `GetAvailableLocalizedLanguages()`/`GetUILanguages()`; `LocalizationManager.PseudoLocalize(string)` exposes the transform directly. The transform is self-contained (its placeholder/markup-skipping logic was adapted from the MIT-licensed PseudoLocalizer project — see `src/L10NSharp/Pseudo/README.md`), so no new package dependency is added. ### Changed diff --git a/src/L10NSharp.Tests/PseudoLocalizationTests.cs b/src/L10NSharp.Tests/PseudoLocalizationTests.cs new file mode 100644 index 0000000..12c2311 --- /dev/null +++ b/src/L10NSharp.Tests/PseudoLocalizationTests.cs @@ -0,0 +1,395 @@ +using System.IO; +using System.Linq; +using L10NSharp.XLiffUtils; +using NUnit.Framework; + +namespace L10NSharp.Tests +{ + /// + /// Tests for the qps-ploc pseudo-locale: lookups for it return the English text + /// pseudolocalized (vowels doubled and accented, bracketed) at runtime. + /// + [TestFixture] + public class PseudoLocalizationTests + { + private const string AppId = "test"; + private const string AppName = "unit test"; + private const string AppVersion = "1.0.0"; + private const string Pseudo = LocalizationManager.PseudoLocalizationLanguageId; + + [SetUp] + public void Setup() + { + LocalizationManager.TranslationMemoryKind = TranslationMemory.XLiff; + } + + [TearDown] + public void TearDown() + { + LocalizationManager.OfferPseudoLocalization = false; + LocalizationManagerInternal.LoadedManagers.Clear(); + LocalizationManagerInternal.MapToExistingLanguage.Clear(); + LocalizationManager.SetUILanguage(LocalizationManager.kDefaultLang); + } + + #region Transform behavior we rely on + + [Test] + public void PseudoLocalize_PlainEnglish_VowelsDoubledWithLeadingAccentAndBracketed() + { + // Every vowel doubles (expansion), accented on the first of the pair + // (readable but unmistakably transformed), and the string is bracketed. + Assert.That(LocalizationManager.PseudoLocalize("Cook Book"), + Is.EqualTo("[Cöoöok Böoöok]")); // [Cöoöok Böoöok] + Assert.That(LocalizationManager.PseudoLocalize("Edit"), + Is.EqualTo("[ÉEdîit]")); // [ÉEdîit] + } + + [Test] + public void PseudoLocalize_NoVowels_StillBracketed() + { + // An all-consonant string gets no accents or expansion, but the brackets + // still mark it as having gone through localization. + Assert.That(LocalizationManager.PseudoLocalize("PDF"), Is.EqualTo("[PDF]")); + } + + [Test] + public void PseudoLocalize_IsDeterministic() + { + const string english = "The quick brown fox jumps over the lazy dog"; + Assert.That(LocalizationManager.PseudoLocalize(english), + Is.EqualTo(LocalizationManager.PseudoLocalize(english))); + } + + [TestCase("Page {0} of {1}", "{0}", "{1}")] + [TestCase("Showing {0:n0} items", "{0:n0}")] + [TestCase("Save %0 of %1 pages", "%0", "%1")] + [TestCase("Installed {app_title} at {installFolder}", "{app_title}", "{installFolder}")] + [TestCase("Level {N}", "{N}")] + public void PseudoLocalize_FormatPlaceholders_SurviveUntouched(string english, + params string[] placeholders) + { + var result = LocalizationManager.PseudoLocalize(english); + foreach (var placeholder in placeholders) + Assert.That(result, Does.Contain(placeholder)); + } + + [TestCase("Bold text", "", "")] + [TestCase("A link.", "", "")] + public void PseudoLocalize_HtmlTags_SurviveUntouched(string english, + params string[] tags) + { + var result = LocalizationManager.PseudoLocalize(english); + foreach (var tag in tags) + Assert.That(result, Does.Contain(tag)); + } + + [TestCase(null)] + [TestCase("")] + public void PseudoLocalize_NullOrEmpty_ReturnedAsIs(string english) + { + Assert.That(LocalizationManager.PseudoLocalize(english), Is.EqualTo(english)); + } + + #endregion + + #region Manager-level behavior + + [Test] + public void GetString_UILanguageIsPseudo_ReturnsPseudolocalizedEnglishText() + { + using (var folder = new TempFolder()) + { + SetupManager(folder, Pseudo); + Assert.That(LocalizationManager.GetString("blahId", "blah"), + Is.EqualTo(LocalizationManager.PseudoLocalize("blah"))); + } + } + + [Test] + public void GetString_UILanguageIsPseudo_StringMissingEverywhere_PseudolocalizesSuppliedEnglish() + { + using (var folder = new TempFolder()) + { + SetupManager(folder, Pseudo); + Assert.That(LocalizationManager.GetString("no.such.id", "only in code"), + Is.EqualTo(LocalizationManager.PseudoLocalize("only in code"))); + } + } + + [Test] + public void GetString_UILanguageIsRealLanguage_NeverGetsPseudoText() + { + using (var folder = new TempFolder()) + { + SetupManager(folder, "fr"); + Assert.That(LocalizationManager.GetString("blahId", "blah"), + Is.EqualTo("blahInFrench")); + } + } + + [Test] + public void GetDynamicStringOrEnglish_PseudoLangId_PseudolocalizesSuppliedEnglish() + { + using (var folder = new TempFolder()) + { + SetupManager(folder); + Assert.That( + LocalizationManager.GetDynamicStringOrEnglish(AppId, "blahId", "from the code", + null, Pseudo), + Is.EqualTo(LocalizationManager.PseudoLocalize("from the code")), + "the code-supplied English should win over the cache, as for 'en'"); + } + } + + [Test] + public void GetDynamicStringOrEnglish_PseudoLangIdNoDefault_PseudolocalizesCachedEnglish() + { + using (var folder = new TempFolder()) + { + SetupManager(folder); + Assert.That( + LocalizationManager.GetDynamicStringOrEnglish(AppId, "blahId", null, null, + Pseudo), + Is.EqualTo(LocalizationManager.PseudoLocalize("blah"))); + } + } + + [Test] + public void GetDynamicString_UILanguageIsPseudo_DoesNotCreatePseudoTranslationFile() + { + using (var folder = new TempFolder()) + { + SetupManager(folder, Pseudo); + LocalizationManager.GetDynamicString(AppId, "brand.new.id", "novel string"); + var pseudoFiles = Directory + .GetFiles(folder.Path, "*", SearchOption.AllDirectories) + .Where(f => f.IndexOf(Pseudo, System.StringComparison.OrdinalIgnoreCase) >= 0); + Assert.That(pseudoFiles, Is.Empty); + } + } + + [TestCase(Pseudo, null, Description = "pseudo alone")] + [TestCase(Pseudo, "fr", Description = "pseudo preferred over French")] + public void GetString_PreferredLanguagesStartingWithPseudo_ReturnsPseudolocalizedEnglish( + string firstLangId, string secondLangId) + { + var preferredLangIds = secondLangId == null + ? new[] { firstLangId } + : new[] { firstLangId, secondLangId }; + using (var folder = new TempFolder()) + { + SetupManager(folder); + var result = LocalizationManager.GetString("blahId", "blah", null, + preferredLangIds, out var languageIdUsed); + Assert.That(result, Is.EqualTo(LocalizationManager.PseudoLocalize("blah"))); + Assert.That(languageIdUsed, Is.EqualTo(Pseudo)); + } + } + + [Test] + public void GetString_LanguageWithTranslationPreferredOverPseudo_ReturnsTranslation() + { + using (var folder = new TempFolder()) + { + SetupManager(folder); + var result = LocalizationManager.GetString("blahId", "blah", null, + new[] { "fr", Pseudo }, out var languageIdUsed); + Assert.That(result, Is.EqualTo("blahInFrench")); + Assert.That(languageIdUsed, Is.EqualTo("fr")); + } + } + + [Test] + public void GetString_EnglishPreferredOverPseudo_ReturnsPlainEnglish() + { + using (var folder = new TempFolder()) + { + SetupManager(folder); + var result = LocalizationManager.GetString("blahId", "blah", null, + new[] { "en", Pseudo }, out var languageIdUsed); + Assert.That(result, Is.EqualTo("blah")); + Assert.That(languageIdUsed, Is.EqualTo("en")); + } + } + + [Test] + public void GetString_EnglishPreferredOverPseudo_StringMissingFromCache_StillReturnsPlainEnglish() + { + using (var folder = new TempFolder()) + { + SetupManager(folder); + // The id is in no cache at all, but English (always satisfiable from the + // code-supplied text) was preferred over the pseudo-locale, so it must win. + var result = LocalizationManager.GetString("no.such.id", "only in code", null, + new[] { "en", Pseudo }, out var languageIdUsed); + Assert.That(result, Is.EqualTo("only in code")); + Assert.That(languageIdUsed, Is.EqualTo("en")); + } + } + + [Test] + public void GetString_NullEntryInPreferredLanguagesBeforePseudo_DoesNotThrow() + { + using (var folder = new TempFolder()) + { + SetupManager(folder); + var result = LocalizationManager.GetString("blahId", "blah", null, + new[] { null, Pseudo }, out var languageIdUsed); + Assert.That(result, Is.EqualTo(LocalizationManager.PseudoLocalize("blah"))); + Assert.That(languageIdUsed, Is.EqualTo(Pseudo)); + } + } + + [Test] + public void GetIsStringAvailableForLangId_Pseudo_ReportsSameAsEnglish() + { + using (var folder = new TempFolder()) + { + SetupManager(folder); + Assert.That(LocalizationManager.GetIsStringAvailableForLangId("blahId", Pseudo), + Is.True); + Assert.That(LocalizationManager.GetIsStringAvailableForLangId("no.such.id", Pseudo), + Is.False); + } + } + + [Test] + public void GetAvailableLocalizedLanguages_RespectsOfferPseudoLocalization() + { + using (var folder = new TempFolder()) + { + SetupManager(folder); + Assert.That(LocalizationManager.GetAvailableLocalizedLanguages(), + Does.Not.Contain(Pseudo), "pseudo-locale should not be advertised by default"); + + LocalizationManager.OfferPseudoLocalization = true; + Assert.That(LocalizationManager.GetAvailableLocalizedLanguages(), + Does.Contain(Pseudo)); + } + } + + [Test] + public void StringCountAndFractions_Pseudo_ReportEnglishCompleteness() + { + using (var folder = new TempFolder()) + { + SetupManager(folder); + Assert.That(LocalizationManager.StringCount(Pseudo), + Is.EqualTo(LocalizationManager.StringCount(LocalizationManager.kDefaultLang))); + Assert.That(LocalizationManager.FractionApproved(Pseudo), Is.EqualTo(1.0f)); + Assert.That(LocalizationManager.FractionTranslated(Pseudo), Is.EqualTo(1.0f)); + } + } + + [Test] + public void GetUILanguages_PseudoOffered_HasHardCodedDisplayName() + { + using (var folder = new TempFolder()) + { + SetupManager(folder); + LocalizationManager.OfferPseudoLocalization = true; + var pseudoCulture = LocalizationManager.GetUILanguages(true) + .FirstOrDefault(c => c.Name == Pseudo); + Assert.That(pseudoCulture, Is.Not.Null); + Assert.That(pseudoCulture.DisplayName, Is.EqualTo("Pseudo-English (qps-ploc)")); + Assert.That(pseudoCulture.NativeName, Is.EqualTo("Pseudo-English (qps-ploc)")); + } + } + + /// + /// The WinForms component localizers -- what sets the Text of controls, tool strip items + /// and column headers created in the designer -- read the string cache directly rather + /// than going through GetLocalizedString. There is no cache for the pseudo-locale, so + /// without a hook here they found nothing and left the designer's plain English in place, + /// which under this locale means "never internationalized" and so was actively + /// misleading. See BL-16748. + /// + [Test] + public void GetStringFromStringCache_Pseudo_PseudolocalizesTheCachedEnglish() + { + using (var folder = new TempFolder()) + { + SetupManager(folder, Pseudo); + var lm = LocalizationManagerInternal.LoadedManagers[AppId]; + + // Sanity check: the English is there to be derived from. + Assert.That(lm.GetStringFromStringCache(LocalizationManager.kDefaultLang, "blahId"), + Is.EqualTo("blah")); + + Assert.That(lm.GetStringFromStringCache(Pseudo, "blahId"), + Is.EqualTo(LocalizationManager.PseudoLocalize("blah"))); + } + } + + [Test] + public void GetStringFromStringCache_PseudoAndIdNotInEnglish_ReturnsNullSoTheCallerKeepsItsOwnText() + { + using (var folder = new TempFolder()) + { + SetupManager(folder, Pseudo); + var lm = LocalizationManagerInternal.LoadedManagers[AppId]; + Assert.That(lm.GetStringFromStringCache(Pseudo, "no.such.id"), Is.Null); + } + } + + [Test] + public void SetUILanguage_Pseudo_DoesNotThrow() + { + Assert.DoesNotThrow(() => LocalizationManager.SetUILanguage(Pseudo)); + Assert.That(LocalizationManager.UILanguageId, Is.EqualTo(Pseudo)); + } + + #endregion + + /// + /// Installs English (blahId="blah", theId="from English Translation") and French + /// (blahId="blahInFrench") translations, sets the UI language, and loads a manager. + /// + private static void SetupManager(TempFolder folder, + string uiLanguageId = LocalizationManager.kDefaultLang) + { + LocalizationManagerInternal.LoadedManagers.Clear(); + + var englishDoc = CreateDocument(AppVersion, "en"); + englishDoc.AddTransUnit(CreateTransUnit("theId", "en", "from English Translation")); + englishDoc.AddTransUnit(CreateTransUnit("blahId", "en", "blah")); + englishDoc.Save(Path.Combine(folder.Path, + LocalizationManager.GetTranslationFileNameForLanguage(AppId, "en"))); + + var frenchDoc = CreateDocument(null, "en", "fr"); + var frenchTu = CreateTransUnit("blahId", "en", "blah"); + frenchTu.Target = new XLiffTransUnitVariant { Lang = "fr", Value = "blahInFrench" }; + frenchTu.TranslationStatus = TranslationStatus.Approved; + frenchDoc.AddTransUnit(frenchTu); + frenchDoc.Save(Path.Combine(folder.Path, + LocalizationManager.GetTranslationFileNameForLanguage(AppId, "fr"))); + + LocalizationManager.SetUILanguage(uiLanguageId); + var manager = new XliffLocalizationManager(AppId, null, AppName, AppVersion, + folder.Path, folder.Combine("generated"), folder.Combine("userModified"), null); + LocalizationManagerInternal.LoadedManagers[AppId] = manager; + } + + private static XLiffDocument CreateDocument(string productVersion, string sourceLang, + string targetLang = null) + { + var doc = new XLiffDocument { File = { SourceLang = sourceLang } }; + if (!string.IsNullOrEmpty(productVersion)) + doc.File.ProductVersion = productVersion; + if (!string.IsNullOrEmpty(targetLang)) + doc.File.TargetLang = targetLang; + doc.File.Original = "test.dll"; + return doc; + } + + private static XLiffTransUnit CreateTransUnit(string id, string lang, string value) + { + return new XLiffTransUnit + { + Id = id, + Source = new XLiffTransUnitVariant { Lang = lang, Value = value } + }; + } + } +} diff --git a/src/L10NSharp/L10NCultureInfo.cs b/src/L10NSharp/L10NCultureInfo.cs index a9fb38d..a1542ea 100644 --- a/src/L10NSharp/L10NCultureInfo.cs +++ b/src/L10NSharp/L10NCultureInfo.cs @@ -104,6 +104,16 @@ public L10NCultureInfo(string name) else if (name == "id") NativeName = "Bahasa Indonesia"; } + + if (LocalizationManager.IsPseudoLanguageId(name)) + { + // Don't rely on the OS to produce a sensible name (or canonical casing) for the + // pseudo-locale (and Linux typically doesn't know the culture at all). + Name = LocalizationManager.PseudoLocalizationLanguageId; + EnglishName = "Pseudo-English (qps-ploc)"; + DisplayName = EnglishName; + NativeName = EnglishName; + } } private L10NCultureInfo(CultureInfo ci) diff --git a/src/L10NSharp/LocalizationManager.cs b/src/L10NSharp/LocalizationManager.cs index 6168a4b..5c1a4e0 100644 --- a/src/L10NSharp/LocalizationManager.cs +++ b/src/L10NSharp/LocalizationManager.cs @@ -6,6 +6,7 @@ using System.Reflection; using System.Threading; using JetBrains.Annotations; +using L10NSharp.Pseudo; using L10NSharp.TMXUtils; using L10NSharp.XLiffUtils; @@ -14,6 +15,14 @@ namespace L10NSharp public static class LocalizationManager { public const string kDefaultLang = "en"; + + /// + /// The standard pseudo-locale tag. Any lookup for this language returns the English + /// text pseudolocalized (vowels doubled and accented, bracketed) at runtime; no translation files + /// exist or are created for it. + /// + public const string PseudoLocalizationLanguageId = "qps-ploc"; + internal const string kL10NPrefix = "_L10N_:"; internal const string kAppVersionPropTag = "x-appversion"; @@ -41,6 +50,25 @@ public static class LocalizationManager /// public static bool ReturnOnlyApprovedStrings; + /// + /// When true, the qps-ploc pseudo-locale is included in GetAvailableLocalizedLanguages() + /// (and hence in the language lists shown to users). Lookups for qps-ploc work + /// regardless of this setting; it only controls whether the pseudo-locale is advertised. + /// Default: false. + /// + public static bool OfferPseudoLocalization { get; set; } + + /// + /// Applies the same pseudolocalization transform used for qps-ploc lookups to the given + /// English text. Exposed so applications can pseudolocalize strings that take paths + /// around L10NSharp. + /// + [PublicAPI] + public static string PseudoLocalize(string english) => PseudoLocalization.Transform(english); + + internal static bool IsPseudoLanguageId(string langId) => + string.Equals(langId, PseudoLocalizationLanguageId, StringComparison.OrdinalIgnoreCase); + /// ------------------------------------------------------------------------------------ /// /// Creates a new instance of a localization manager for the specified application id. diff --git a/src/L10NSharp/LocalizationManagerInternal.cs b/src/L10NSharp/LocalizationManagerInternal.cs index ac3a325..4b04d37 100644 --- a/src/L10NSharp/LocalizationManagerInternal.cs +++ b/src/L10NSharp/LocalizationManagerInternal.cs @@ -6,6 +6,7 @@ using System.IO; using System.Linq; using System.Reflection; +using L10NSharp.Pseudo; using L10NSharp.XLiffUtils; // ReSharper disable StaticMemberInGenericType - these static fields are parameter-independent @@ -261,6 +262,10 @@ public static List GetAvailableLocalizedLanguages() var langsHavingLocalizations = (LoadedManagers == null ? new List() : LoadedManagers.Values.SelectMany(lm => lm.GetAvailableUILanguageTags()) .Distinct().ToList()); + // The pseudo-locale is derived from the English strings at lookup time, so it is + // available whenever anything is, but is only advertised when the app opts in. + if (LocalizationManager.OfferPseudoLocalization && langsHavingLocalizations.Count > 0) + langsHavingLocalizations.Add(LocalizationManager.PseudoLocalizationLanguageId); return langsHavingLocalizations; } @@ -268,6 +273,9 @@ public static bool IsLocalizationAvailable(string langId) { if (LoadedManagers == null) return false; + // The pseudo-locale is exactly as available as English. + if (LocalizationManager.IsPseudoLanguageId(langId)) + langId = LocalizationManager.kDefaultLang; return LoadedManagers.Values.Any(m => m.IsUILanguageAvailable(langId)); } @@ -346,6 +354,8 @@ orderby ci.DisplayName /// public static int NumberApproved(string lang) { + if (LocalizationManager.IsPseudoLanguageId(lang)) + lang = LocalizationManager.kDefaultLang; // pseudo is exactly as complete as English if (lang == LocalizationManager.kDefaultLang) return StringCount(lang); var approved = 0; @@ -377,6 +387,8 @@ public static float FractionApproved(string lang) /// public static int NumberTranslated(string lang) { + if (LocalizationManager.IsPseudoLanguageId(lang)) + lang = LocalizationManager.kDefaultLang; // pseudo is exactly as complete as English if (lang == LocalizationManager.kDefaultLang) return StringCount(lang); var translated = 0; @@ -408,6 +420,8 @@ public static float FractionTranslated(string lang) /// public static int StringCount(string lang) { + if (LocalizationManager.IsPseudoLanguageId(lang)) + lang = LocalizationManager.kDefaultLang; // pseudo is exactly as complete as English var count = 0; foreach (var lm in s_loadedManagers.Values) { @@ -516,6 +530,16 @@ public static string GetDynamicStringOrEnglish(string appId, string id, string e $"Initialized LMs are {string.Join(", ", LoadedManagers.Keys)}"); } + // For the pseudo-locale, pseudolocalize the English text. As for English, the + // caller-supplied englishText wins over the cache. Note that this never engages the + // dynamic-string collection machinery below: no files exist or are written for the + // pseudo-locale. + if (LocalizationManager.IsPseudoLanguageId(langId)) + { + return PseudoLocalization.Transform(englishText ?? + lm.GetStringFromStringCache(LocalizationManager.kDefaultLang, id)); + } + // If they asked for English, we are going to use the supplied englishText, regardless // of what may be cached, following the rule that the current c# code always wins. In // case we really need to recover the cached version, we will retrieve that only if no @@ -566,6 +590,10 @@ internal static string MapToExistingLanguageIfPossible(string langId) { if (string.IsNullOrEmpty(langId)) return null; + // The pseudo-locale never maps to or from a real language, and no localization + // files exist for it, so don't try to load any. + if (LocalizationManager.IsPseudoLanguageId(langId)) + return langId; // It's a concurrent dictionary, so we can (for performance) try this without a lock. if (MapToExistingLanguage.TryGetValue(langId, out var realId)) return realId; @@ -613,6 +641,10 @@ public static bool GetIsStringAvailableForLangId(string id, string langId) if (string.IsNullOrEmpty(langId) || string.IsNullOrEmpty(id)) return false; + // Every English string is by definition available in the pseudo-locale. + if (LocalizationManager.IsPseudoLanguageId(langId)) + langId = LocalizationManager.kDefaultLang; + var str = MapToExistingLanguageOrAddMapping(id, langId, out _); return !string.IsNullOrEmpty(str); } @@ -710,6 +742,14 @@ public static string GetString(string stringId, string englishText, string comme { if (string.IsNullOrWhiteSpace(stringId)) return LocalizationManager.StripOffLocalizationInfoFromText(englishText); + // For the pseudo-locale, pseudolocalize the English text (as for English, the + // caller-supplied englishText wins over the cache). + if (LocalizationManager.IsPseudoLanguageId(LocalizationManager.UILanguageId)) + { + return PseudoLocalization.Transform( + LocalizationManager.StripOffLocalizationInfoFromText(englishText) ?? + MapToExistingLanguageOrAddMapping(stringId, LocalizationManager.kDefaultLang, out _)); + } return GetStringFromAnyLocalizationManager(stringId) ?? LocalizationManager.StripOffLocalizationInfoFromText(englishText); } @@ -741,12 +781,31 @@ public static string GetString(string stringId, string englishText, string comme if (string.IsNullOrEmpty(englishText)) throw new ArgumentException($"{nameof(englishText)} may not be empty (because common... that can't be what you meant to do..."); + // If the pseudo-locale is in the list, it always has every string (derived from the + // English), so only languages preferred over it can win; anything after it is moot. + // English is also always available (the code-supplied englishText), so note whether + // it too was preferred over the pseudo-locale. + var pseudoIndex = langIds.FindIndex(LocalizationManager.IsPseudoLanguageId); + var englishPreferredOverPseudo = pseudoIndex >= 0 && langIds.Take(pseudoIndex) + .Any(l => l == "en" || + (l != null && l.StartsWith("en-", StringComparison.OrdinalIgnoreCase))); + if (pseudoIndex >= 0) + langIds = langIds.Take(pseudoIndex).ToList(); + var stringFromAnyLocalizationManager = GetStringFromAnyLocalizationManager(stringId, langIds, out languageIdUsed); // Even if found in the English l10n file, we prefer to use the version that came from // the code. if (languageIdUsed == "en" || string.IsNullOrEmpty(stringFromAnyLocalizationManager)) { + // No language preferred over the pseudo-locale had the string (and English was + // not preferred over it), so pseudolocalize the code-supplied English. + if (pseudoIndex >= 0 && !englishPreferredOverPseudo) + { + languageIdUsed = LocalizationManager.PseudoLocalizationLanguageId; + return PseudoLocalization.Transform( + LocalizationManager.StripOffLocalizationInfoFromText(englishText)); + } languageIdUsed = "en"; return LocalizationManager.StripOffLocalizationInfoFromText(englishText); } diff --git a/src/L10NSharp/Pseudo/EscapeHelpers.cs b/src/L10NSharp/Pseudo/EscapeHelpers.cs new file mode 100644 index 0000000..5351535 --- /dev/null +++ b/src/L10NSharp/Pseudo/EscapeHelpers.cs @@ -0,0 +1,76 @@ +// Adapted from the MIT-licensed PseudoLocalizer project, Copyright (C) 2012, Anders Kaplan. +// See README.md in this folder for provenance, license, and local changes. + +namespace L10NSharp.Pseudo +{ + internal static class EscapeHelpers + { + // Local addition (not upstream): placeholder names may be alphanumeric/underscore, + // not just digits — consumers substitute named placeholders like "{app_title}". + private static bool IsPlaceholderNameChar(char c) + => char.IsLetterOrDigit(c) || c == '_'; + + internal static bool ShouldTransform(char[] array, char ch, ref int i) + { + // Are we at the start of a potential placeholder (e.g. "{?...}") + if (ch == '{' && i < array.Length - 2) + { + int j = i; + + while (j < array.Length - 1 && IsPlaceholderNameChar(array[++j])) + { + // Consume the placeholder name (digits for "{0}", or a name like "{lang}") + } + + if (array[j] == ':') + { + while (j < array.Length - 1 && array[++j] != '}') + { + // Consume all of any format specifier (e.g. "{0:yyyy}" for a DateTime) + } + } + + if (array[j] == '}') + { + i = j; + return false; + } + } + else if (ch == '%' && i < array.Length - 1 && char.IsDigit(array[i + 1])) + { + // Local addition (not upstream): "%0"-style placeholders, substituted by + // consumers' front ends (e.g. Bloom's simpleFormat), pass through untouched. + int j = i; + + while (j < array.Length - 1 && char.IsDigit(array[j + 1])) + j++; + + i = j; + return false; + } + else if (ch == '<' && i < array.Length - 2) + { + // Are we at the start of a potential HTML tag (e.g. "") + int j = i; + + char next = array[i + 1]; + + if ((next >= 'a' && next <= 'z') || (next >= 'A' && next <= 'Z') || next == '/') + { + while (j < array.Length - 1 && array[++j] != '>') + { + // Consume all of the tag + } + + if (array[j] == '>') + { + i = j; + return false; + } + } + } + + return true; + } + } +} diff --git a/src/L10NSharp/Pseudo/PseudoLocalization.cs b/src/L10NSharp/Pseudo/PseudoLocalization.cs new file mode 100644 index 0000000..201f1e1 --- /dev/null +++ b/src/L10NSharp/Pseudo/PseudoLocalization.cs @@ -0,0 +1,20 @@ +namespace L10NSharp.Pseudo +{ + /// + /// Produces the pseudolocalized ("qps-ploc") form of English strings: every vowel is + /// doubled (accented on the first of the pair) for ~30-40% expansion, and the whole + /// string is bracketed, with format placeholders and HTML/XML markup passed through + /// untouched. E.g. "Title Missing" becomes "[Tîitlée Mîissîing]". The transform is + /// deterministic, and self-contained in this folder (see its README.md), so L10NSharp + /// carries no extra dependency for this feature. + /// + internal static class PseudoLocalization + { + public static string Transform(string english) + { + if (string.IsNullOrEmpty(english)) + return english; + return "[" + VowelStretch.Transform(english) + "]"; + } + } +} diff --git a/src/L10NSharp/Pseudo/README.md b/src/L10NSharp/Pseudo/README.md new file mode 100644 index 0000000..711c680 --- /dev/null +++ b/src/L10NSharp/Pseudo/README.md @@ -0,0 +1,61 @@ +# Pseudolocalization transform + +Everything in this folder is `internal`, in the `L10NSharp.Pseudo` namespace; the only way +in is `PseudoLocalization.Transform` (exposed publicly as `LocalizationManager.PseudoLocalize`). + +## The transform + +`[Tîitlée Mîissîing]` for "Title Missing": + +- **Every vowel is doubled, accented on the first of the pair** (`VowelStretch`). The + doubling provides the ~30–40% expansion (inside the words, where it stresses layout the + way real translations do), and the accented-plain pairs (`îi`, `öo`, `ée`) make the text + unmistakably transformed while staying easy to read — no real language systematically + produces that pattern. Consonants, digits, and punctuation are untouched. +- **The whole string is bracketed.** A missing `]` reveals truncation; brackets mid-sentence + reveal runtime string concatenation. +- **Placeholders and markup pass through untouched** (`EscapeHelpers`): `{0}`/`{0:fmt}` + format placeholders, named `{app_title}`-style and `%0`-style placeholders (both + substituted by consumers' front ends, e.g. Bloom's), and HTML/XML tags. + +The transform is deterministic, so screenshots are comparable across runs. The behaviors +consumers rely on are pinned by `PseudoLocalizationTests`. + +Design notes: the doubled-vowel expansion follows Mozilla's pseudolocalization approach +(Fluent's "accented" locale); brackets are common to Microsoft's qps-ploc, Android's en-XA, +and others. Earlier iterations of this feature (see git history) used the full accent map +and per-word padding of the PseudoLocalizer project, which we used as a starting point but +replaced for readability. + +## Provenance + +`EscapeHelpers` (the placeholder/markup-skipping logic) is adapted from the MIT-licensed +[PseudoLocalizer](https://github.com/martincostello/Pseudolocalizer) project, Copyright (C) +2012, Anders Kaplan, and extended here to also recognize `%0`-style and named +`{app_title}`-style placeholders. The rest of the folder is original to L10NSharp. + +Upstream license for the adapted code: + +``` +The MIT License (MIT) + +Copyright (C) 2012, Anders Kaplan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +``` diff --git a/src/L10NSharp/Pseudo/VowelStretch.cs b/src/L10NSharp/Pseudo/VowelStretch.cs new file mode 100644 index 0000000..4cbdf33 --- /dev/null +++ b/src/L10NSharp/Pseudo/VowelStretch.cs @@ -0,0 +1,66 @@ +using System.Collections.Generic; +using System.Text; + +namespace L10NSharp.Pseudo +{ + /// + /// Doubles every vowel, accenting the first of each pair, and leaves consonants, + /// digits, and punctuation untouched: "Title Missing" becomes "Tîitlée Mîissîing". + /// This keeps the pseudo text easy to read while still being unmistakably transformed + /// (no real language systematically produces the accented-plain vowel pairs), and it + /// provides the ~30-40% expansion inside the words themselves, with no filler + /// characters. Format placeholders and HTML/XML tags pass through untouched + /// (via EscapeHelpers). + /// + internal static class VowelStretch + { + private static readonly Dictionary AccentedVowels = new Dictionary + { + { 'a', 'å' }, + { 'e', 'é' }, + { 'i', 'î' }, + { 'o', 'ö' }, + { 'u', 'û' }, + { 'A', 'Å' }, + { 'E', 'É' }, + { 'I', 'Î' }, + { 'O', 'Ö' }, + { 'U', 'Û' }, + }; + + public static string Transform(string value) + { + var array = value.ToCharArray(); + var builder = new StringBuilder(value.Length * 2); + + for (int i = 0; i < array.Length; i++) + { + char ch = array[i]; + int indexBefore = i; + + if (EscapeHelpers.ShouldTransform(array, ch, ref i)) + { + if (AccentedVowels.TryGetValue(ch, out var accented)) + { + // Each vowel doubles for expansion, accented on the first of the + // pair so the text stays easy to read. + builder.Append(accented); + builder.Append(ch); + } + else + { + builder.Append(ch); + } + } + else + { + // Skipped span (placeholder or markup): copy it through untouched. + for (int j = indexBefore; j < i + 1; j++) + builder.Append(array[j]); + } + } + + return builder.ToString(); + } + } +} diff --git a/src/L10NSharp/XLiffUtils/XliffLocalizationManager.cs b/src/L10NSharp/XLiffUtils/XliffLocalizationManager.cs index 7f3e6c8..4e35f59 100644 --- a/src/L10NSharp/XLiffUtils/XliffLocalizationManager.cs +++ b/src/L10NSharp/XLiffUtils/XliffLocalizationManager.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.ComponentModel; using System.Diagnostics; @@ -7,6 +7,7 @@ using System.Reflection; using System.Text.RegularExpressions; using System.Xml.Linq; +using L10NSharp.Pseudo; namespace L10NSharp.XLiffUtils { @@ -489,6 +490,15 @@ public string GetLocalizedString(IComponent component, string id, string default /// ------------------------------------------------------------------------------------ public string GetLocalizedString(string id, string defaultText) { + // For the pseudo-locale, pseudolocalize the English text (the code-supplied default + // wins over the cache, as for English). + if (LocalizationManager.IsPseudoLanguageId(UILanguageId)) + { + return PseudoLocalization.Transform( + LocalizationManager.StripOffLocalizationInfoFromText(defaultText) ?? + GetStringFromStringCache(LocalizationManager.kDefaultLang, id)); + } + var text = (UILanguageId != LocalizationManager.kDefaultLang ? GetStringFromStringCache(UILanguageId, id) : null); return text ?? LocalizationManager.StripOffLocalizationInfoFromText(defaultText); @@ -497,6 +507,17 @@ public string GetLocalizedString(string id, string defaultText) /// ------------------------------------------------------------------------------------ public string GetStringFromStringCache(string uiLangId, string id) { + // There is no cache for the pseudo-locale (no files exist for it), so derive it from + // the English entry. Doing it here rather than only in GetLocalizedString matters + // because the WinForms component localizers (which set the Text of designer-created + // controls, tool strip items and column headers) call straight into the string cache; + // without this they would find nothing and leave the designer's plain English in + // place, which is exactly what the pseudo-locale is supposed to mean "not + // internationalized". + if (LocalizationManager.IsPseudoLanguageId(uiLangId)) + return PseudoLocalization.Transform( + GetStringFromStringCache(LocalizationManager.kDefaultLang, id)); + var realLangId = LocalizationManagerInternal.MapToExistingLanguageIfPossible(uiLangId); return StringCache.GetString(realLangId, id); } @@ -504,6 +525,12 @@ public string GetStringFromStringCache(string uiLangId, string id) /// ------------------------------------------------------------------------------------ protected string GetTooltipFromStringCache(string uiLangId, string id) { + // See GetStringFromStringCache: same story for the tooltips of designer-created + // controls. + if (LocalizationManager.IsPseudoLanguageId(uiLangId)) + return PseudoLocalization.Transform( + GetTooltipFromStringCache(LocalizationManager.kDefaultLang, id)); + var realLangId = LocalizationManagerInternal.MapToExistingLanguageIfPossible(uiLangId); return StringCache.GetToolTipText(realLangId, id); }