diff --git a/.gitignore b/.gitignore index 8b692db4..c1ddedfc 100644 --- a/.gitignore +++ b/.gitignore @@ -263,6 +263,9 @@ FakesAssemblies/ .ntvs_analysis.dat lib/ node_modules/ +/core.d.ts +/core.js +/locales/ # Visual Studio 6 build log *.plg diff --git a/README.md b/README.md index a1bded50..40d7199b 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,19 @@ A parser for the [Power Query/M](https://docs.microsoft.com/en-us/power-query/) The most common way to consume the project is to interact with the helper functions found in [taskUtils.ts](src/powerquery-parser/task/taskUtils.ts). There are all-in-one functions, such as `tryLexParse`, which does a full pass on a given document. There are also incremental functions, such as `tryLex` and `tryParse`, which perform one step at a time. Minimal code samples can be found in [example.ts](src/example.ts). +### Localization and bundle size + +The package root includes every supported localization template for backwards compatibility. Applications that want a smaller bundle can import the English-only core entrypoint and register only the locales they use: + +```ts +import { LocalizationUtils } from "@microsoft/powerquery-parser/core"; +import { templates as frFR } from "@microsoft/powerquery-parser/locales/fr-FR"; + +LocalizationUtils.registerLocalizationTemplates("fr-FR", frFR); +``` + +An unregistered locale falls back to `en-US`. Import `@microsoft/powerquery-parser/locales/all` to register every supported locale while using the core entrypoint. + ## Related projects - [powerquery-formatter](https://github.com/microsoft/powerquery-formatter): A code formatter for Power Query which is bundled in the VSCode extension. diff --git a/package.json b/package.json index 53f4a668..bd270df1 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "power bi" ], "scripts": { - "build": "node_modules\\.bin\\tsc", + "build": "node_modules\\.bin\\tsc && node scripts\\createPackageEntrypoints.js", "test": "mocha --reporter mocha-multi-reporters --reporter-options configFile=src/test/mochaConfig.json -r ts-node/register src/test/libraryTest/**/*.ts", "test:resources": "mocha --reporter mocha-multi-reporters --reporter-options configFile=src/test/mochaConfig.json -r ts-node/register src/test/resourceTest/**/*.ts", "script:benchmark": "npx ts-node src/test/scripts/createBenchmark.ts", @@ -55,6 +55,9 @@ "typescript": "^4.5.4" }, "files": [ - "lib/powerquery-parser/**/*" + "lib/powerquery-parser/**/*", + "core.js", + "core.d.ts", + "locales/**/*" ] } diff --git a/scripts/createPackageEntrypoints.js b/scripts/createPackageEntrypoints.js new file mode 100644 index 00000000..89ec18b5 --- /dev/null +++ b/scripts/createPackageEntrypoints.js @@ -0,0 +1,44 @@ +const fs = require("fs"); +const path = require("path"); + +const packageRoot = path.resolve(__dirname, ".."); +const libraryRoot = path.join(packageRoot, "lib", "powerquery-parser"); +const templateLibraryRoot = path.join(libraryRoot, "localization", "templates"); +const localePackageRoot = path.join(packageRoot, "locales"); +const localizationTemplatesModule = "../lib/powerquery-parser/localization/templates"; + +fs.writeFileSync(path.join(packageRoot, "core.js"), 'module.exports = require("./lib/powerquery-parser/core");\n'); +fs.writeFileSync(path.join(packageRoot, "core.d.ts"), 'export * from "./lib/powerquery-parser/core";\n'); + +fs.rmSync(localePackageRoot, { recursive: true, force: true }); +fs.mkdirSync(localePackageRoot, { recursive: true }); + +for (const fileName of fs.readdirSync(templateLibraryRoot)) { + const localeMatch = /^template(?:\.([^.]+))?\.json$/.exec(fileName); + + if (localeMatch === null) { + continue; + } + + const localeName = localeMatch[1] ?? "en-US"; + const relativeTarget = `../lib/powerquery-parser/localization/templates/${fileName}`; + + fs.writeFileSync( + path.join(localePackageRoot, `${localeName}.js`), + `exports.templates = require("${relativeTarget}");\n`, + ); + fs.writeFileSync( + path.join(localePackageRoot, `${localeName}.d.ts`), + `import { ILocalizationTemplates } from "${localizationTemplatesModule}";\n\n` + + "export const templates: ILocalizationTemplates;\n", + ); +} + +fs.writeFileSync( + path.join(localePackageRoot, "all.js"), + `exports.Templates = require("${localizationTemplatesModule}");\n`, +); +fs.writeFileSync( + path.join(localePackageRoot, "all.d.ts"), + `export * as Templates from "${localizationTemplatesModule}";\n`, +); diff --git a/src/powerquery-parser/common/error.ts b/src/powerquery-parser/common/error.ts index 307f4f31..983da0f6 100644 --- a/src/powerquery-parser/common/error.ts +++ b/src/powerquery-parser/common/error.ts @@ -1,9 +1,11 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -import { Localization, LocalizationUtils, Templates } from "../localization"; +import * as LocalizationUtils from "../localization/localizationUtils"; import { Assert } from "."; +import { DefaultTemplates } from "../localization/defaultTemplates"; import { ICancellationToken } from "./cancellationToken/ICancellationToken"; +import { Localization } from "../localization/localization"; export type TInnerCommonError = CancellationError | InvariantError | UnknownError; @@ -16,14 +18,14 @@ export class CommonError extends Error { export class CancellationError extends Error { constructor(readonly cancellationToken: ICancellationToken, readonly reason: string) { - super(Localization.error_common_cancellationError(Templates.DefaultTemplates, reason)); + super(Localization.error_common_cancellationError(DefaultTemplates, reason)); Object.setPrototypeOf(this, CancellationError.prototype); } } export class InvariantError extends Error { constructor(readonly invariantBroken: string, readonly details?: object) { - super(Localization.error_common_invariantError(Templates.DefaultTemplates, invariantBroken, details)); + super(Localization.error_common_invariantError(DefaultTemplates, invariantBroken, details)); Object.setPrototypeOf(this, InvariantError.prototype); } } diff --git a/src/powerquery-parser/core.ts b/src/powerquery-parser/core.ts new file mode 100644 index 00000000..052bd122 --- /dev/null +++ b/src/powerquery-parser/core.ts @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import * as LocalizationUtils from "./localization/localizationUtils"; +import * as Templates from "./localization/defaultTemplates"; + +export { LocalizationUtils, Templates }; +export * as Language from "./language"; +export * as Lexer from "./lexer"; +export * as Parser from "./parser"; +export * from "./common"; +export { Localization } from "./localization/localization"; +export { DefaultLocale, Locale } from "./localization/locale"; +export { DefaultTemplates } from "./localization/defaultTemplates"; +export type { ILocalizationTemplates } from "./localization/templates"; +export * from "./settings"; +export * from "./task"; diff --git a/src/powerquery-parser/lexer/error.ts b/src/powerquery-parser/lexer/error.ts index 20269146..3e118e63 100644 --- a/src/powerquery-parser/lexer/error.ts +++ b/src/powerquery-parser/lexer/error.ts @@ -1,9 +1,10 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +import * as LocalizationUtils from "../localization/localizationUtils"; import { CommonError, StringUtils } from "../common"; -import { Localization, LocalizationUtils } from "../localization"; import { Lexer } from ".."; +import { Localization } from "../localization/localization"; export type TLexError = CommonError.CommonError | LexError; diff --git a/src/powerquery-parser/localization/defaultTemplates.ts b/src/powerquery-parser/localization/defaultTemplates.ts new file mode 100644 index 00000000..26e22405 --- /dev/null +++ b/src/powerquery-parser/localization/defaultTemplates.ts @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import * as en_US from "./templates/template.json"; +import type { ILocalizationTemplates } from "./templates"; + +export const DefaultTemplates: ILocalizationTemplates = en_US; diff --git a/src/powerquery-parser/localization/localization.ts b/src/powerquery-parser/localization/localization.ts index fe567eaa..18fa8590 100644 --- a/src/powerquery-parser/localization/localization.ts +++ b/src/powerquery-parser/localization/localization.ts @@ -3,7 +3,7 @@ import { Assert, StringUtils } from "../common"; import { SequenceKind, TokenWithColumnNumber } from "../parser/error"; -import { ILocalizationTemplates } from "./templates"; +import type { ILocalizationTemplates } from "./templates"; import { Lexer } from ".."; import { LexError } from "../lexer"; import { ParseError } from "../parser"; diff --git a/src/powerquery-parser/localization/localizationUtils.ts b/src/powerquery-parser/localization/localizationUtils.ts index 4f70cfa9..f9acc78d 100644 --- a/src/powerquery-parser/localization/localizationUtils.ts +++ b/src/powerquery-parser/localization/localizationUtils.ts @@ -1,8 +1,18 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -import { DefaultTemplates, ILocalizationTemplates, TemplatesByLocale } from "./templates"; +import { DefaultLocale } from "./locale"; +import { DefaultTemplates } from "./defaultTemplates"; +import type { ILocalizationTemplates } from "./templates"; + +export const TemplatesByLocale: Map = new Map([ + [DefaultLocale.toLowerCase(), DefaultTemplates], +]); export function getLocalizationTemplates(locale: string): ILocalizationTemplates { return TemplatesByLocale.get(locale.toLowerCase()) ?? DefaultTemplates; } + +export function registerLocalizationTemplates(locale: string, templates: ILocalizationTemplates): void { + TemplatesByLocale.set(locale.toLowerCase(), templates); +} diff --git a/src/powerquery-parser/localization/templates.ts b/src/powerquery-parser/localization/templates.ts index c94fc63a..8200b998 100644 --- a/src/powerquery-parser/localization/templates.ts +++ b/src/powerquery-parser/localization/templates.ts @@ -7,7 +7,6 @@ import * as cs_CZ from "./templates/template.cs-CZ.json"; import * as da_DK from "./templates/template.da-DK.json"; import * as de_DE from "./templates/template.de-DE.json"; import * as el_GR from "./templates/template.el-GR.json"; -import * as en_US from "./templates/template.json"; import * as es_ES from "./templates/template.es-ES.json"; import * as et_EE from "./templates/template.et-EE.json"; import * as eu_ES from "./templates/template.eu-ES.json"; @@ -43,6 +42,8 @@ import * as uk_UA from "./templates/template.uk-UA.json"; import * as vi_VN from "./templates/template.vi-VN.json"; import * as zh_CN from "./templates/template.zh-CN.json"; import * as zh_TW from "./templates/template.zh-TW.json"; +import { registerLocalizationTemplates, TemplatesByLocale } from "./localizationUtils"; +import { DefaultTemplates } from "./defaultTemplates"; import { Locale } from "./locale"; export { @@ -52,7 +53,6 @@ export { da_DK, de_DE, el_GR, - en_US, es_ES, et_EE, eu_ES, @@ -88,16 +88,18 @@ export { vi_VN, zh_CN, zh_TW, + DefaultTemplates as en_US, + TemplatesByLocale, }; -export const TemplatesByLocale: Map = new Map([ +const templatesByLocaleEntries: ReadonlyArray = [ [Locale.bg_BG.toLowerCase(), bg_BG], [Locale.ca_EZ.toLowerCase(), ca_ES], [Locale.cs_CZ.toLowerCase(), cs_CZ], [Locale.da_DK.toLowerCase(), da_DK], [Locale.de_DE.toLowerCase(), de_DE], [Locale.el_GR.toLowerCase(), el_GR], - [Locale.en_US.toLowerCase(), en_US], + [Locale.en_US.toLowerCase(), DefaultTemplates], [Locale.es_ES.toLowerCase(), es_ES], [Locale.et_EE.toLowerCase(), et_EE], [Locale.eu_ES.toLowerCase(), eu_ES], @@ -133,7 +135,11 @@ export const TemplatesByLocale: Map = new Map([ [Locale.vi_VN.toLowerCase(), vi_VN], [Locale.zh_CN.toLowerCase(), zh_CN], [Locale.zh_TW.toLowerCase(), zh_TW], -]); +]; + +for (const [locale, templates] of templatesByLocaleEntries) { + registerLocalizationTemplates(locale, templates); +} export interface ILocalizationTemplates { readonly error_common_cancellationError: string; @@ -239,4 +245,4 @@ export interface ILocalizationTemplates { readonly tokenKind_textLiteral: string; } -export const DefaultTemplates: ILocalizationTemplates = en_US; +export { DefaultTemplates }; diff --git a/src/powerquery-parser/parser/error.ts b/src/powerquery-parser/parser/error.ts index 00a9c6d9..54b7058b 100644 --- a/src/powerquery-parser/parser/error.ts +++ b/src/powerquery-parser/parser/error.ts @@ -1,8 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +import * as LocalizationUtils from "../localization/localizationUtils"; import { Assert, CommonError, StringUtils } from "../common"; -import { Localization, LocalizationUtils } from "../localization"; +import { Localization } from "../localization/localization"; import { ParseState } from "./parseState"; import { Token } from "../language"; diff --git a/src/powerquery-parser/parser/parseState/parseStateUtils.ts b/src/powerquery-parser/parser/parseState/parseStateUtils.ts index e4c3e5eb..2256c2ea 100644 --- a/src/powerquery-parser/parser/parseState/parseStateUtils.ts +++ b/src/powerquery-parser/parser/parseState/parseStateUtils.ts @@ -5,7 +5,7 @@ import { ArrayUtils, Assert, CommonError, MapUtils } from "../../common"; import { Ast, Constant, Token } from "../../language"; import { NodeIdMap, NodeIdMapUtils, ParseContext, ParseContextUtils, ParseError, TXorNode, XorNodeUtils } from ".."; import { NoOpTraceManagerInstance, Trace } from "../../common/trace"; -import { DefaultLocale } from "../../localization"; +import { DefaultLocale } from "../../localization/locale"; import { Disambiguation } from "../disambiguation"; import { LexerSnapshot } from "../../lexer"; import { ParseState } from "./parseState"; diff --git a/src/powerquery-parser/settings.ts b/src/powerquery-parser/settings.ts index 9980b6a7..3ae8940f 100644 --- a/src/powerquery-parser/settings.ts +++ b/src/powerquery-parser/settings.ts @@ -3,7 +3,7 @@ import { CombinatorialParserV2, ParseSettings, ParseState, ParseStateUtils } from "./parser"; import { LexerSnapshot, LexSettings } from "./lexer"; -import { DefaultLocale } from "./localization"; +import { DefaultLocale } from "./localization/locale"; import { NoOpTraceManagerInstance } from "./common/trace"; export type Settings = LexSettings & ParseSettings; diff --git a/src/powerquery-parser/task/taskUtils.ts b/src/powerquery-parser/task/taskUtils.ts index d38fb21a..93cc5649 100644 --- a/src/powerquery-parser/task/taskUtils.ts +++ b/src/powerquery-parser/task/taskUtils.ts @@ -1,8 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +import * as Lexer from "../lexer"; +import * as Parser from "../parser"; import { CommonError, ResultKind, ResultUtils } from "../common"; -import { Lexer, Parser } from ".."; import { LexTaskError, LexTaskOk, diff --git a/src/test/libraryTest/localization/localizationUtils.test.ts b/src/test/libraryTest/localization/localizationUtils.test.ts new file mode 100644 index 00000000..8a994eeb --- /dev/null +++ b/src/test/libraryTest/localization/localizationUtils.test.ts @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import "mocha"; +import { expect } from "chai"; + +import { LocalizationUtils as FullLocalizationUtils, Templates as FullTemplates } from "../../.."; +import { LocalizationUtils, Templates } from "../../../powerquery-parser/core"; + +describe("LocalizationUtils", () => { + it("falls back to the default templates for an unregistered locale", () => { + expect(LocalizationUtils.getLocalizationTemplates("not-a-locale")).to.equal(Templates.DefaultTemplates); + }); + + it("registers templates case-insensitively", () => { + const locale: string = "test-locale"; + + const templates: typeof Templates.DefaultTemplates = { + ...Templates.DefaultTemplates, + error_lex_endOfStream: "Localized end of stream", + }; + + LocalizationUtils.registerLocalizationTemplates(locale, templates); + + expect(LocalizationUtils.getLocalizationTemplates(locale.toUpperCase())).to.equal(templates); + }); + + it("registers every locale from the backwards-compatible package root", () => { + expect(FullLocalizationUtils.getLocalizationTemplates("fr-FR")).to.equal(FullTemplates.fr_FR); + }); + + it("preserves direct access to TemplatesByLocale", () => { + const locale: string = "map-locale"; + + FullTemplates.TemplatesByLocale.set(locale, FullTemplates.fr_FR); + + expect(FullLocalizationUtils.getLocalizationTemplates(locale)).to.equal(FullTemplates.fr_FR); + }); +});