Add opt-in core bundle with on-demand localization

.gitignore

@@ -263,6 +263,9 @@ FakesAssemblies/
 .ntvs_analysis.dat
 lib/
 node_modules/
+/core.d.ts
+/core.js
+/locales/
 
 # Visual Studio 6 build log
 *.plg

README.md

@@ -8,6 +8,19 @@ A parser for the [Power Query/M](https://docs.microsoft.com/en-us/power-query/)
 
 The most common way to consume the project is to interact with the helper functions found in [taskUtils.ts](src/powerquery-parser/task/taskUtils.ts). There are all-in-one functions, such as `tryLexParse`, which does a full pass on a given document. There are also incremental functions, such as `tryLex` and `tryParse`, which perform one step at a time. Minimal code samples can be found in [example.ts](src/example.ts).
 
+### Localization and bundle size
+
+The package root includes every supported localization template for backwards compatibility. Applications that want a smaller bundle can import the English-only core entrypoint and register only the locales they use:
+
+```ts
+import { LocalizationUtils } from "@microsoft/powerquery-parser/core";
+import { templates as frFR } from "@microsoft/powerquery-parser/locales/fr-FR";
+
+LocalizationUtils.registerLocalizationTemplates("fr-FR", frFR);
+```
+
+An unregistered locale falls back to `en-US`. Import `@microsoft/powerquery-parser/locales/all` to register every supported locale while using the core entrypoint.
+
 ## Related projects
 
 - [powerquery-formatter](https://github.com/microsoft/powerquery-formatter): A code formatter for Power Query which is bundled in the VSCode extension.

package.json

@@ -9,7 +9,7 @@
         "power bi"
     ],
     "scripts": {
-        "build": "node_modules\\.bin\\tsc",
+        "build": "node_modules\\.bin\\tsc && node scripts\\createPackageEntrypoints.js",
         "test": "mocha --reporter mocha-multi-reporters --reporter-options configFile=src/test/mochaConfig.json -r ts-node/register src/test/libraryTest/**/*.ts",
         "test:resources": "mocha --reporter mocha-multi-reporters --reporter-options configFile=src/test/mochaConfig.json -r ts-node/register src/test/resourceTest/**/*.ts",
         "script:benchmark": "npx ts-node src/test/scripts/createBenchmark.ts",
@@ -55,6 +55,9 @@
         "typescript": "^4.5.4"
     },
     "files": [
-        "lib/powerquery-parser/**/*"
+        "lib/powerquery-parser/**/*",
+        "core.js",
+        "core.d.ts",
+        "locales/**/*"
     ]
 }

scripts/createPackageEntrypoints.js

@@ -0,0 +1,44 @@
+const fs = require("fs");
+const path = require("path");
+
+const packageRoot = path.resolve(__dirname, "..");
+const libraryRoot = path.join(packageRoot, "lib", "powerquery-parser");
+const templateLibraryRoot = path.join(libraryRoot, "localization", "templates");
+const localePackageRoot = path.join(packageRoot, "locales");
+const localizationTemplatesModule = "../lib/powerquery-parser/localization/templates";
+
+fs.writeFileSync(path.join(packageRoot, "core.js"), 'module.exports = require("./lib/powerquery-parser/core");\n');
+fs.writeFileSync(path.join(packageRoot, "core.d.ts"), 'export * from "./lib/powerquery-parser/core";\n');
+
+fs.rmSync(localePackageRoot, { recursive: true, force: true });
+fs.mkdirSync(localePackageRoot, { recursive: true });
+
+for (const fileName of fs.readdirSync(templateLibraryRoot)) {
+    const localeMatch = /^template(?:\.([^.]+))?\.json$/.exec(fileName);
+
+    if (localeMatch === null) {
+        continue;
+    }
+
+    const localeName = localeMatch[1] ?? "en-US";
+    const relativeTarget = `../lib/powerquery-parser/localization/templates/${fileName}`;
+
+    fs.writeFileSync(
+        path.join(localePackageRoot, `${localeName}.js`),
+        `exports.templates = require("${relativeTarget}");\n`,
+    );
+    fs.writeFileSync(
+        path.join(localePackageRoot, `${localeName}.d.ts`),
+        `import { ILocalizationTemplates } from "${localizationTemplatesModule}";\n\n` +
+            "export const templates: ILocalizationTemplates;\n",
+    );
+}
+
+fs.writeFileSync(
+    path.join(localePackageRoot, "all.js"),
+    `exports.Templates = require("${localizationTemplatesModule}");\n`,
+);
+fs.writeFileSync(
+    path.join(localePackageRoot, "all.d.ts"),
+    `export * as Templates from "${localizationTemplatesModule}";\n`,
+);

src/powerquery-parser/common/error.ts

@@ -1,9 +1,11 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-import { Localization, LocalizationUtils, Templates } from "../localization";
+import * as LocalizationUtils from "../localization/localizationUtils";
 import { Assert } from ".";
+import { DefaultTemplates } from "../localization/defaultTemplates";
 import { ICancellationToken } from "./cancellationToken/ICancellationToken";
+import { Localization } from "../localization/localization";
 
 export type TInnerCommonError = CancellationError | InvariantError | UnknownError;
 
@@ -16,14 +18,14 @@ export class CommonError extends Error {
 
 export class CancellationError extends Error {
     constructor(readonly cancellationToken: ICancellationToken, readonly reason: string) {
-        super(Localization.error_common_cancellationError(Templates.DefaultTemplates, reason));
+        super(Localization.error_common_cancellationError(DefaultTemplates, reason));
         Object.setPrototypeOf(this, CancellationError.prototype);
     }
 }
 
 export class InvariantError extends Error {
     constructor(readonly invariantBroken: string, readonly details?: object) {
-        super(Localization.error_common_invariantError(Templates.DefaultTemplates, invariantBroken, details));
+        super(Localization.error_common_invariantError(DefaultTemplates, invariantBroken, details));
         Object.setPrototypeOf(this, InvariantError.prototype);
     }
 }

src/powerquery-parser/core.ts

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+import * as LocalizationUtils from "./localization/localizationUtils";
+import * as Templates from "./localization/defaultTemplates";
+
+export { LocalizationUtils, Templates };
+export * as Language from "./language";
+export * as Lexer from "./lexer";
+export * as Parser from "./parser";
+export * from "./common";
+export { Localization } from "./localization/localization";
+export { DefaultLocale, Locale } from "./localization/locale";
+export { DefaultTemplates } from "./localization/defaultTemplates";
+export type { ILocalizationTemplates } from "./localization/templates";
+export * from "./settings";
+export * from "./task";

src/powerquery-parser/lexer/error.ts

@@ -1,9 +1,10 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
+import * as LocalizationUtils from "../localization/localizationUtils";
 import { CommonError, StringUtils } from "../common";
-import { Localization, LocalizationUtils } from "../localization";
 import { Lexer } from "..";
+import { Localization } from "../localization/localization";
 
 export type TLexError = CommonError.CommonError | LexError;
 

src/powerquery-parser/localization/defaultTemplates.ts

@@ -0,0 +1,7 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+import * as en_US from "./templates/template.json";
+import type { ILocalizationTemplates } from "./templates";
+
+export const DefaultTemplates: ILocalizationTemplates = en_US;

src/powerquery-parser/localization/localization.ts

@@ -3,7 +3,7 @@
 
 import { Assert, StringUtils } from "../common";
 import { SequenceKind, TokenWithColumnNumber } from "../parser/error";
-import { ILocalizationTemplates } from "./templates";
+import type { ILocalizationTemplates } from "./templates";
 import { Lexer } from "..";
 import { LexError } from "../lexer";
 import { ParseError } from "../parser";

src/powerquery-parser/localization/localizationUtils.ts

@@ -1,8 +1,18 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-import { DefaultTemplates, ILocalizationTemplates, TemplatesByLocale } from "./templates";
+import { DefaultLocale } from "./locale";
+import { DefaultTemplates } from "./defaultTemplates";
+import type { ILocalizationTemplates } from "./templates";
+
+export const TemplatesByLocale: Map<string, ILocalizationTemplates> = new Map([
+    [DefaultLocale.toLowerCase(), DefaultTemplates],
+]);
 
 export function getLocalizationTemplates(locale: string): ILocalizationTemplates {
     return TemplatesByLocale.get(locale.toLowerCase()) ?? DefaultTemplates;
 }
+
+export function registerLocalizationTemplates(locale: string, templates: ILocalizationTemplates): void {
+    TemplatesByLocale.set(locale.toLowerCase(), templates);
+}

src/powerquery-parser/localization/templates.ts

@@ -7,7 +7,6 @@ import * as cs_CZ from "./templates/template.cs-CZ.json";
 import * as da_DK from "./templates/template.da-DK.json";
 import * as de_DE from "./templates/template.de-DE.json";
 import * as el_GR from "./templates/template.el-GR.json";
-import * as en_US from "./templates/template.json";
 import * as es_ES from "./templates/template.es-ES.json";
 import * as et_EE from "./templates/template.et-EE.json";
 import * as eu_ES from "./templates/template.eu-ES.json";
@@ -43,6 +42,8 @@ import * as uk_UA from "./templates/template.uk-UA.json";
 import * as vi_VN from "./templates/template.vi-VN.json";
 import * as zh_CN from "./templates/template.zh-CN.json";
 import * as zh_TW from "./templates/template.zh-TW.json";
+import { registerLocalizationTemplates, TemplatesByLocale } from "./localizationUtils";
+import { DefaultTemplates } from "./defaultTemplates";
 import { Locale } from "./locale";
 
 export {
@@ -52,7 +53,6 @@ export {
     da_DK,
     de_DE,
     el_GR,
-    en_US,
     es_ES,
     et_EE,
     eu_ES,
@@ -88,16 +88,18 @@ export {
     vi_VN,
     zh_CN,
     zh_TW,
+    DefaultTemplates as en_US,
+    TemplatesByLocale,
 };
 
-export const TemplatesByLocale: Map<string, ILocalizationTemplates> = new Map([
+const templatesByLocaleEntries: ReadonlyArray<readonly [string, ILocalizationTemplates]> = [
     [Locale.bg_BG.toLowerCase(), bg_BG],
     [Locale.ca_EZ.toLowerCase(), ca_ES],
     [Locale.cs_CZ.toLowerCase(), cs_CZ],
     [Locale.da_DK.toLowerCase(), da_DK],
     [Locale.de_DE.toLowerCase(), de_DE],
     [Locale.el_GR.toLowerCase(), el_GR],
-    [Locale.en_US.toLowerCase(), en_US],
+    [Locale.en_US.toLowerCase(), DefaultTemplates],
     [Locale.es_ES.toLowerCase(), es_ES],
     [Locale.et_EE.toLowerCase(), et_EE],
     [Locale.eu_ES.toLowerCase(), eu_ES],
@@ -133,7 +135,11 @@ export const TemplatesByLocale: Map<string, ILocalizationTemplates> = new Map([
     [Locale.vi_VN.toLowerCase(), vi_VN],
     [Locale.zh_CN.toLowerCase(), zh_CN],
     [Locale.zh_TW.toLowerCase(), zh_TW],
-]);
+];
+
+for (const [locale, templates] of templatesByLocaleEntries) {
+    registerLocalizationTemplates(locale, templates);
+}
 
 export interface ILocalizationTemplates {
     readonly error_common_cancellationError: string;
@@ -239,4 +245,4 @@ export interface ILocalizationTemplates {
     readonly tokenKind_textLiteral: string;
 }
 
-export const DefaultTemplates: ILocalizationTemplates = en_US;
+export { DefaultTemplates };

src/powerquery-parser/parser/error.ts

@@ -1,8 +1,9 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
+import * as LocalizationUtils from "../localization/localizationUtils";
 import { Assert, CommonError, StringUtils } from "../common";
-import { Localization, LocalizationUtils } from "../localization";
+import { Localization } from "../localization/localization";
 import { ParseState } from "./parseState";
 import { Token } from "../language";
 

src/powerquery-parser/parser/parseState/parseStateUtils.ts

@@ -5,7 +5,7 @@ import { ArrayUtils, Assert, CommonError, MapUtils } from "../../common";
 import { Ast, Constant, Token } from "../../language";
 import { NodeIdMap, NodeIdMapUtils, ParseContext, ParseContextUtils, ParseError, TXorNode, XorNodeUtils } from "..";
 import { NoOpTraceManagerInstance, Trace } from "../../common/trace";
-import { DefaultLocale } from "../../localization";
+import { DefaultLocale } from "../../localization/locale";
 import { Disambiguation } from "../disambiguation";
 import { LexerSnapshot } from "../../lexer";
 import { ParseState } from "./parseState";

src/powerquery-parser/settings.ts

@@ -3,7 +3,7 @@
 
 import { CombinatorialParserV2, ParseSettings, ParseState, ParseStateUtils } from "./parser";
 import { LexerSnapshot, LexSettings } from "./lexer";
-import { DefaultLocale } from "./localization";
+import { DefaultLocale } from "./localization/locale";
 import { NoOpTraceManagerInstance } from "./common/trace";
 
 export type Settings = LexSettings & ParseSettings;

src/powerquery-parser/task/taskUtils.ts

@@ -1,8 +1,9 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
+import * as Lexer from "../lexer";
+import * as Parser from "../parser";
 import { CommonError, ResultKind, ResultUtils } from "../common";
-import { Lexer, Parser } from "..";
 import {
     LexTaskError,
     LexTaskOk,

src/test/libraryTest/localization/localizationUtils.test.ts

@@ -0,0 +1,39 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+import "mocha";
+import { expect } from "chai";
+
+import { LocalizationUtils as FullLocalizationUtils, Templates as FullTemplates } from "../../..";
+import { LocalizationUtils, Templates } from "../../../powerquery-parser/core";
+
+describe("LocalizationUtils", () => {
+    it("falls back to the default templates for an unregistered locale", () => {
+        expect(LocalizationUtils.getLocalizationTemplates("not-a-locale")).to.equal(Templates.DefaultTemplates);
+    });
+
+    it("registers templates case-insensitively", () => {
+        const locale: string = "test-locale";
+
+        const templates: typeof Templates.DefaultTemplates = {
+            ...Templates.DefaultTemplates,
+            error_lex_endOfStream: "Localized end of stream",
+        };
+
+        LocalizationUtils.registerLocalizationTemplates(locale, templates);
+
+        expect(LocalizationUtils.getLocalizationTemplates(locale.toUpperCase())).to.equal(templates);
+    });
+
+    it("registers every locale from the backwards-compatible package root", () => {
+        expect(FullLocalizationUtils.getLocalizationTemplates("fr-FR")).to.equal(FullTemplates.fr_FR);
+    });
+
+    it("preserves direct access to TemplatesByLocale", () => {
+        const locale: string = "map-locale";
+
+        FullTemplates.TemplatesByLocale.set(locale, FullTemplates.fr_FR);
+
+        expect(FullLocalizationUtils.getLocalizationTemplates(locale)).to.equal(FullTemplates.fr_FR);
+    });
+});