Make morphology parsers safe to bundle in browser/React Native

CHANGELOG.md

@@ -52,6 +52,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- **Browser/native bundling of the morphology parsers.** `Grid3VerbsParser`
+  statically imported Node's `path` and `adm-zip`, and `TDSnapLexiconParser`
+  statically required `better-sqlite3`, so importing the browser or metrics
+  entry points failed in Metro (React Native/Expo) with
+  `Unable to resolve module path`. Both parsers now take a `FileAdapter`,
+  `Grid3VerbsParser` reads archives through `getZipAdapter` (Node `adm-zip` /
+  browser `jszip`), and `TDSnapLexiconParser` opens databases through
+  `openSqliteDatabase` (`better-sqlite3` on Node, `sql.js` in the browser), so
+  these modules are safe to bundle for web and native.
+
+  **Breaking change:** `Grid3VerbsParser.parseZip`/`parseZipDetailed`/
+  `parseXmlFileDetailed`/`parseCustomDirectory`, `TDSnapLexiconParser.parseDb`,
+  `WordFormGenerator.generateFromPos` and `MetricsCalculator.analyze` are now
+  asynchronous.
+
 - **OBF/OBZ manifest `root` bug.** `ObfProcessor.saveFromTree` and
   `saveModifiedTree` set `manifest.root` to the board **ID** (e.g. `"1"`)
   instead of the board **file path** (e.g. `"1.obf"`). This produced OBZ

src/cli/index.ts

@@ -578,7 +578,7 @@ program
         }
 
         const calculator = new MetricsCalculator();
-        const metrics = calculator.analyze(tree, {
+        const metrics = await calculator.analyze(tree, {
           scanningConfig,
           useSmartGrammar: options.smartGrammar,
         });
@@ -669,7 +669,7 @@ program
           const tree = await processor.loadIntoTree(file);
           // analyze() first: it expands morphological predictions on the tree,
           // which dumpVocabulary() needs for the word-form counts.
-          const metrics = new MetricsCalculator().analyze(tree, {
+          const metrics = await new MetricsCalculator().analyze(tree, {
             useSmartGrammar: options.smartGrammar,
           });
           dump = dumpVocabulary(tree, { metrics });

src/utilities/analytics/metrics/core.ts

@@ -47,7 +47,7 @@ export class MetricsCalculator {
    * @param options - Optional configuration for metrics calculation
    * @returns Complete metrics result
    */
-  analyze(tree: AACTree, options: MetricsOptions = {}): MetricsResult {
+  async analyze(tree: AACTree, options: MetricsOptions = {}): Promise<MetricsResult> {
     // Get root board - prioritize tree.rootId, then fall back to boards with no parentId
     let rootBoard: AACPage | undefined;
     if (tree.rootId) {
@@ -122,7 +122,7 @@ export class MetricsCalculator {
     // Expand morphological predictions from POS tags if enabled or auto-detected
     const useSmartGrammar = options.useSmartGrammar === true || this.treeHasPosTags(tree);
     if (useSmartGrammar) {
-      this.expandMorphologicalPredictions(tree, options);
+      await this.expandMorphologicalPredictions(tree, options);
     }
 
     if (useSmartGrammar) {
@@ -789,15 +789,18 @@ export class MetricsCalculator {
    * button's predictions array. This is done as a pre-processing step
    * before calculateWordFormMetrics assigns effort to each form.
    */
-  private expandMorphologicalPredictions(tree: AACTree, options: MetricsOptions): void {
+  private async expandMorphologicalPredictions(
+    tree: AACTree,
+    options: MetricsOptions
+  ): Promise<void> {
     const locale = options.morphologyLocale || 'en-gb';
     let morph: MorphologyEngine;
 
     if (options.tdsnapLexiconPath) {
       // eslint-disable-next-line @typescript-eslint/no-var-requires
       const { TDSnapLexiconParser } = require('../morphology/tdsnapLexiconParser');
       const parser = new TDSnapLexiconParser();
-      const lexiconData = parser.parseDb(options.tdsnapLexiconPath, locale.replace('-', '_'));
+      const lexiconData = await parser.parseDb(options.tdsnapLexiconPath, locale.replace('-', '_'));
       // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
       morph = MorphologyEngine.fromTDSnapLexicon(lexiconData);
       this.expandTDSnapPredictions(tree, morph);
@@ -808,7 +811,7 @@ export class MetricsCalculator {
       // eslint-disable-next-line @typescript-eslint/no-var-requires
       const { Grid3VerbsParser } = require('../morphology/grid3VerbsParser');
       const parser = new Grid3VerbsParser();
-      const verbForms = parser.parseZip(options.grid3VerbsPath);
+      const verbForms = await parser.parseZip(options.grid3VerbsPath);
       // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
       morph = MorphologyEngine.fromGrid3Verbs(verbForms);
       this.expandGrid3Predictions(tree, morph);

src/utilities/analytics/morphology/grid3VerbsParser.ts

@@ -1,6 +1,7 @@
 import { XMLParser } from 'fast-xml-parser';
-import AdmZip from 'adm-zip';
-import { join, dirname, basename } from 'path';
+import { defaultFileAdapter } from '../../../utils/io';
+import type { FileAdapter } from '../../../utils/io';
+import { getZipAdapter } from '../../../utils/zip';
 import type { VerbFormWithConditions, Grid3VerbFormsDetailed } from './types';
 
 export interface Grid3VerbForms {
@@ -32,6 +33,11 @@ export class Grid3VerbsParser {
     ignoreDeclaration: true,
     textNodeName: '#text',
   });
+  private fileAdapter: FileAdapter;
+
+  constructor(fileAdapter: FileAdapter = defaultFileAdapter) {
+    this.fileAdapter = fileAdapter;
+  }
 
   parseXml(xmlContent: string, locale?: string): Grid3VerbForms {
     const data = this.parser.parse(xmlContent);
@@ -77,74 +83,73 @@ export class Grid3VerbsParser {
     return { locale: detectedLocale, verbs: result };
   }
 
-  /* eslint-disable @typescript-eslint/no-var-requires, @typescript-eslint/no-unsafe-argument */
-  parseXmlFileDetailed(filePath: string): Grid3VerbFormsDetailed {
-    const fs = require('fs');
-    const xml = fs.readFileSync(filePath, 'utf-8');
+  async parseXmlFileDetailed(filePath: string): Promise<Grid3VerbFormsDetailed> {
+    const xml = await this.fileAdapter.readTextFromInput(filePath);
     return this.parseXmlDetailed(xml);
   }
-  /* eslint-enable @typescript-eslint/no-var-requires, @typescript-eslint/no-unsafe-argument */
-
-  parseZip(zipPath: string): Grid3VerbForms {
-    const zip = new AdmZip(zipPath);
-    const entries = zip.getEntries();
-    const verbEntry = entries.find((e) => e.entryName.toLowerCase().endsWith('verbs.xml'));
-    if (!verbEntry) {
-      const locale = basename(dirname(zipPath));
+
+  private async readVerbsXml(zipPath: string): Promise<string | null> {
+    const zip = await getZipAdapter(zipPath, this.fileAdapter);
+    const verbEntry = zip.listFiles().find((entry) => entry.toLowerCase().endsWith('verbs.xml'));
+    if (!verbEntry) return null;
+    return this.fileAdapter.readTextFromInput(await zip.readFile(verbEntry));
+  }
+
+  async parseZip(zipPath: string): Promise<Grid3VerbForms> {
+    const xml = await this.readVerbsXml(zipPath);
+    if (xml === null) {
+      const locale = this.fileAdapter.basename(this.fileAdapter.dirname(zipPath));
       return { locale, verbs: new Map() };
     }
-    const xml = verbEntry.getData().toString('utf-8');
     return this.parseXml(xml);
   }
 
-  parseZipDetailed(zipPath: string): Grid3VerbFormsDetailed {
-    const zip = new AdmZip(zipPath);
-    const entries = zip.getEntries();
-    const verbEntry = entries.find((e) => e.entryName.toLowerCase().endsWith('verbs.xml'));
-    if (!verbEntry) {
-      const locale = basename(dirname(zipPath));
+  async parseZipDetailed(zipPath: string): Promise<Grid3VerbFormsDetailed> {
+    const xml = await this.readVerbsXml(zipPath);
+    if (xml === null) {
+      const locale = this.fileAdapter.basename(this.fileAdapter.dirname(zipPath));
       return { locale, verbs: new Map() };
     }
-    const xml = verbEntry.getData().toString('utf-8');
     return this.parseXmlDetailed(xml);
   }
 
-  // eslint-disable-next-line @typescript-eslint/require-await
   async parseLocale(locale: string, grid3InstallPath?: string): Promise<Grid3VerbForms> {
-    const installPath = grid3InstallPath || this.getDefaultInstallPath();
+    const installPath = grid3InstallPath || (await this.getDefaultInstallPath());
     if (!installPath) {
       return { locale, verbs: new Map() };
     }
-    const zipPath = join(installPath, 'Locale', locale, 'verbs', 'verbs.zip');
+    const zipPath = this.fileAdapter.join(installPath, 'Locale', locale, 'verbs', 'verbs.zip');
     try {
-      return this.parseZip(zipPath);
+      return await this.parseZip(zipPath);
     } catch {
       return { locale, verbs: new Map() };
     }
   }
 
   async parseInstalledLocales(grid3InstallPath?: string): Promise<Map<string, Grid3VerbForms>> {
-    const installPath = grid3InstallPath || this.getDefaultInstallPath();
+    const installPath = grid3InstallPath || (await this.getDefaultInstallPath());
     const results = new Map<string, Grid3VerbForms>();
     if (!installPath) return results;
 
-    const fs = await import('fs');
-    const localeDir = join(installPath, 'Locale');
+    const localeDir = this.fileAdapter.join(installPath, 'Locale');
     let locales: string[];
     try {
-      locales = fs
-        .readdirSync(localeDir)
-        .filter((d) => fs.statSync(join(localeDir, d)).isDirectory());
+      const entries = await this.fileAdapter.listDir(localeDir);
+      locales = [];
+      for (const entry of entries) {
+        if (await this.fileAdapter.isDirectory(this.fileAdapter.join(localeDir, entry))) {
+          locales.push(entry);
+        }
+      }
     } catch {
       return results;
     }
 
     for (const locale of locales) {
-      const verbsZip = join(localeDir, locale, 'verbs', 'verbs.zip');
+      const verbsZip = this.fileAdapter.join(localeDir, locale, 'verbs', 'verbs.zip');
       try {
-        fs.accessSync(verbsZip, fs.constants.R_OK);
-        const forms = this.parseZip(verbsZip);
-        results.set(locale, forms);
+        if (!(await this.fileAdapter.pathExists(verbsZip))) continue;
+        results.set(locale, await this.parseZip(verbsZip));
       } catch {
         // No verbs.zip for this locale
       }
@@ -164,34 +169,35 @@ export class Grid3VerbsParser {
    * This allows users to supply morphology data copied from any Grid 3
    * installation without needing Grid 3 installed on this machine.
    */
-  parseCustomDirectory(dirPath: string, detailed?: false): Map<string, Grid3VerbForms>;
+  parseCustomDirectory(dirPath: string, detailed?: false): Promise<Map<string, Grid3VerbForms>>;
   parseCustomDirectory(
     dirPath: string,
     detailed: true
-  ): Map<string, import('./types').Grid3VerbFormsDetailed>;
-  // eslint-disable-next-line @typescript-eslint/no-var-requires
-  parseCustomDirectory(
+  ): Promise<Map<string, import('./types').Grid3VerbFormsDetailed>>;
+  async parseCustomDirectory(
     dirPath: string,
     detailed = false
-  ): Map<string, Grid3VerbForms> | Map<string, import('./types').Grid3VerbFormsDetailed> {
-    /* eslint-disable @typescript-eslint/no-var-requires, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-return */
-    const fs = require('fs');
+  ): Promise<Map<string, Grid3VerbForms> | Map<string, import('./types').Grid3VerbFormsDetailed>> {
     let locales: string[];
     try {
-      locales = fs
-        .readdirSync(dirPath)
-        .filter((d: string) => fs.statSync(join(dirPath, d)).isDirectory());
+      const entries = await this.fileAdapter.listDir(dirPath);
+      locales = [];
+      for (const entry of entries) {
+        if (await this.fileAdapter.isDirectory(this.fileAdapter.join(dirPath, entry))) {
+          locales.push(entry);
+        }
+      }
     } catch {
       return new Map();
     }
 
     if (detailed) {
       const results = new Map<string, import('./types').Grid3VerbFormsDetailed>();
       for (const locale of locales) {
-        const verbsZip = join(dirPath, locale, 'verbs', 'verbs.zip');
+        const verbsZip = this.fileAdapter.join(dirPath, locale, 'verbs', 'verbs.zip');
         try {
-          fs.accessSync(verbsZip, fs.constants.R_OK);
-          results.set(locale, this.parseZipDetailed(verbsZip));
+          if (!(await this.fileAdapter.pathExists(verbsZip))) continue;
+          results.set(locale, await this.parseZipDetailed(verbsZip));
         } catch {
           // No verbs.zip for this locale
         }
@@ -201,19 +207,18 @@ export class Grid3VerbsParser {
 
     const results = new Map<string, Grid3VerbForms>();
     for (const locale of locales) {
-      const verbsZip = join(dirPath, locale, 'verbs', 'verbs.zip');
+      const verbsZip = this.fileAdapter.join(dirPath, locale, 'verbs', 'verbs.zip');
       try {
-        fs.accessSync(verbsZip, fs.constants.R_OK);
-        results.set(locale, this.parseZip(verbsZip));
+        if (!(await this.fileAdapter.pathExists(verbsZip))) continue;
+        results.set(locale, await this.parseZip(verbsZip));
       } catch {
         // No verbs.zip for this locale
       }
     }
     return results;
-    /* eslint-enable @typescript-eslint/no-var-requires, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-return */
   }
 
-  private getDefaultInstallPath(): string | null {
+  private async getDefaultInstallPath(): Promise<string | null> {
     if (typeof process === 'undefined' || process.platform !== 'win32') {
       return null;
     }
@@ -222,10 +227,8 @@ export class Grid3VerbsParser {
       'C:\\Program Files\\Smartbox\\Grid 3',
     ];
     try {
-      // eslint-disable-next-line @typescript-eslint/no-var-requires
-      const fs = require('fs');
       for (const p of paths) {
-        if (fs.existsSync(p)) return p;
+        if (await this.fileAdapter.pathExists(p)) return p;
       }
     } catch {
       // Grid 3 not installed

src/utilities/analytics/morphology/tdsnapLexiconParser.ts

@@ -1,3 +1,8 @@
+import { defaultFileAdapter } from '../../../utils/io';
+import type { FileAdapter } from '../../../utils/io';
+import { openSqliteDatabase } from '../../../utils/sqlite';
+import type { SqliteDatabaseAdapter } from '../../../utils/sqlite';
+
 export interface TDSnapLexiconForm {
   tag: string;
   form: string;
@@ -20,17 +25,27 @@ interface PosSubclassRow {
 }
 
 export class TDSnapLexiconParser {
-  parseDb(dbPath: string, locale?: string): TDSnapLexiconData {
+  private fileAdapter: FileAdapter;
+
+  constructor(fileAdapter: FileAdapter = defaultFileAdapter) {
+    this.fileAdapter = fileAdapter;
+  }
+
+  async parseDb(dbPath: string, locale?: string): Promise<TDSnapLexiconData> {
     const detectedLocale = locale || this.inferLocale(dbPath);
-    // eslint-disable-next-line @typescript-eslint/no-var-requires
-    const Database = require('better-sqlite3');
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
-    const db = new Database(dbPath, { readonly: true }) as import('better-sqlite3').Database;
+    const { db, cleanup } = await openSqliteDatabase(dbPath, {
+      readonly: true,
+      fileAdapter: this.fileAdapter,
+    });
 
     try {
       return this.extractAll(db, detectedLocale);
     } finally {
-      db.close();
+      if (cleanup) {
+        await cleanup();
+      } else {
+        db.close();
+      }
     }
   }
 
@@ -39,7 +54,7 @@ export class TDSnapLexiconParser {
     return match ? match[1] : 'unknown';
   }
 
-  private extractAll(db: import('better-sqlite3').Database, locale: string): TDSnapLexiconData {
+  private extractAll(db: SqliteDatabaseAdapter, locale: string): TDSnapLexiconData {
     const words = new Map<string, TDSnapLexiconEntry>();
     const subclassCache = new Map<number, string>();
 

src/utilities/analytics/morphology/wordFormGenerator.ts

@@ -62,16 +62,16 @@ export class WordFormGenerator {
     return this.deduplicate(result);
   }
 
-  generateFromPos(
+  async generateFromPos(
     base: string,
     pos: string,
     engine: MorphologyEngine,
     grid3Parser: Grid3VerbsParser,
     verbsZipPath?: string,
     lang: string = 'en'
-  ): AstericsWordForm[] {
+  ): Promise<AstericsWordForm[]> {
     if (verbsZipPath) {
-      const detailed = grid3Parser.parseZipDetailed(verbsZipPath);
+      const detailed = await grid3Parser.parseZipDetailed(verbsZipPath);
       const forms = detailed.verbs.get(base) || detailed.verbs.get(base.toLowerCase());
       if (forms && forms.length > 0) {
         return this.generateFromGrid3Conditions(base, forms, lang);