From a69bf21093f4a783fb84b4a3c12fa18f0ce71282 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Wed, 19 Aug 2026 08:47:46 +0300 Subject: [PATCH 01/29] feat: add namespace entity --- openapi3.yaml | 59 ++++++++++++-- package-lock.json | 53 +++++++++++-- package.json | 2 +- .../api/src/namespace/routes/namespace.ts | 16 ++++ .../src/migrations/1786885317922-Migration.ts | 61 ++++++++++++++ packages/db/src/openapi.d.ts | 79 +++++++++++++++++-- packages/db/src/types/namespace.ts | 3 + 7 files changed, 257 insertions(+), 16 deletions(-) create mode 100644 packages/api/src/namespace/routes/namespace.ts create mode 100644 packages/db/src/migrations/1786885317922-Migration.ts create mode 100644 packages/db/src/types/namespace.ts diff --git a/openapi3.yaml b/openapi3.yaml index f38a4b6..8889fb7 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -4,27 +4,38 @@ info: description: > API for retrieving geographic layer specifications. Layers expose a set of properties; `enum`-typed properties also carry their list of allowed values. - version: 1.2.0 + version: 2.0.0 license: name: Proprietary url: https://example.com/licenses/proprietary paths: - /layers: + /namespaces: get: - summary: Get all layer names + summary: Get all namespaces + operationId: getNamespaces + security: [] + responses: + '200': + $ref: '#/components/responses/NamespaceListResponse' + /namespaces/{namespace}/layers: + get: + summary: Get all layer names within a namespace operationId: getLayers security: [] + parameters: + - $ref: '#/components/parameters/NamespaceParam' responses: '200': $ref: '#/components/responses/LayerListResponse' '404': $ref: '#/components/responses/NotFound' - /layers/{layerName}: + /namespaces/{namespace}/layers/{layerName}: get: - summary: Get layer spec by name + summary: Get layer spec by name within a namespace operationId: getLayerByName security: [] parameters: + - $ref: '#/components/parameters/NamespaceParam' - $ref: '#/components/parameters/LayerNameParam' responses: '200': @@ -33,6 +44,14 @@ paths: $ref: '#/components/responses/NotFound' components: parameters: + NamespaceParam: + name: namespace + in: path + required: true + description: The namespace the layer belongs to. Layer names are only unique within a namespace + schema: + type: string + example: data-2025 LayerNameParam: name: layerName in: path @@ -42,6 +61,16 @@ components: type: string example: buildings_polygon responses: + NamespaceListResponse: + description: A list of the namespaces this deployment serves + content: + application/json: + schema: + $ref: '#/components/schemas/NamespaceList' + example: + namespaces: + - name: data-2024 + - name: data-2025 LayerListResponse: description: A list of layers with their display aliases content: @@ -83,6 +112,26 @@ components: schema: $ref: '#/components/schemas/Error' schemas: + NamespaceSummary: + type: object + description: A namespace served by this deployment. + properties: + name: + type: string + description: Name of the namespace + example: data-2025 + required: + - name + NamespaceList: + type: object + description: Response wrapper for the list of namespaces. + properties: + namespaces: + type: array + items: + $ref: '#/components/schemas/NamespaceSummary' + required: + - namespaces LayerSummary: type: object description: Basic layer info returned in the list endpoint. diff --git a/package-lock.json b/package-lock.json index 95a8e82..fbdf779 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "packages/*" ], "dependencies": { - "@map-colonies/schemas": "1.26.0" + "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-9687e6cb21fe29d10c52aaf9db64a763050ff519.tgz" }, "devDependencies": { "@commitlint/cli": "^20.4.1", @@ -1129,6 +1129,17 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.3.tgz", + "integrity": "sha512-ELEBe8PsLvvJ6QMr0zLt8ffvOHW/dc1m3CEzNMg7aJUv3bMaoDtw2TXyDAwkYBuroxxuHEwhRTLJSe5sya547g==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@epic-web/invariant": { "version": "1.0.0", "dev": true, @@ -3232,8 +3243,8 @@ }, "node_modules/@map-colonies/schemas": { "version": "1.26.0", - "resolved": "https://registry.npmjs.org/@map-colonies/schemas/-/schemas-1.26.0.tgz", - "integrity": "sha512-XUQ2/gvC6d6/nGMKIGVyZo/kemGH5uGOy4xhkYCVLNV/CJsHvJXmhiov7QpqVpb58x+/2WVVUThAStj9ogipug==", + "resolved": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-9687e6cb21fe29d10c52aaf9db64a763050ff519.tgz", + "integrity": "sha512-HvWTHuAd+HcvfiNQWPsmsF/XUQ8ln/DZOeau/az+WvQq8A84mMi9WGDdESTvVTP2FqP04c0zQmdunn8NcjNsDw==", "license": "MIT", "peer": true }, @@ -7339,7 +7350,6 @@ "version": "8.63.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.63.0", @@ -7657,6 +7667,40 @@ "node": ">=14.0.0" } }, + "node_modules/@unrs/resolver-binding-wasm32-wasi/node_modules/@emnapi/core": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", + "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" + } + }, + "node_modules/@unrs/resolver-binding-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@unrs/resolver-binding-wasm32-wasi/node_modules/@emnapi/wasi-threads": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@unrs/resolver-binding-win32-arm64-msvc": { "version": "1.12.2", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-arm64-msvc/-/resolver-binding-win32-arm64-msvc-1.12.2.tgz", @@ -10866,7 +10910,6 @@ "version": "9.39.5", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", diff --git a/package.json b/package.json index 67e5b2b..89ad48f 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,6 @@ "lerna": "^6.6.2" }, "dependencies": { - "@map-colonies/schemas": "1.26.0" + "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-9687e6cb21fe29d10c52aaf9db64a763050ff519.tgz" } } diff --git a/packages/api/src/namespace/routes/namespace.ts b/packages/api/src/namespace/routes/namespace.ts new file mode 100644 index 0000000..e308e0f --- /dev/null +++ b/packages/api/src/namespace/routes/namespace.ts @@ -0,0 +1,16 @@ +import { Router } from 'express'; +import type { FactoryFunction } from 'tsyringe'; +import { NamespaceController } from '../controllers/namespaceController'; + +const namespaceRouterFactory: FactoryFunction = (dependencyContainer) => { + const router = Router(); + const controller = dependencyContainer.resolve(NamespaceController); + + router.get('/', controller.getNamespaces); + + return router; +}; + +export const NAMESPACE_ROUTER_SYMBOL = Symbol('namespaceRouterFactory'); + +export { namespaceRouterFactory }; diff --git a/packages/db/src/migrations/1786885317922-Migration.ts b/packages/db/src/migrations/1786885317922-Migration.ts new file mode 100644 index 0000000..c5f4330 --- /dev/null +++ b/packages/db/src/migrations/1786885317922-Migration.ts @@ -0,0 +1,61 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class Migration1786885317922 implements MigrationInterface { + name = 'Migration1786885317922'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "enum_value" DROP CONSTRAINT "FK_8548a0384cc733008852a8f4e70"`); + await queryRunner.query(`ALTER TABLE "property" DROP CONSTRAINT "FK_34ce6cedb279510bb20da63cf21"`); + await queryRunner.query(`ALTER TABLE "enum_value" ADD "namespace" character varying NOT NULL`); + await queryRunner.query(`ALTER TABLE "enum_value" DROP CONSTRAINT "PK_08cf35141adc487a902990ccff2"`); + await queryRunner.query( + `ALTER TABLE "enum_value" ADD CONSTRAINT "PK_0607639cc6d4c22cad2fa8552e9" PRIMARY KEY ("value", "layer_name", "property", "namespace")` + ); + await queryRunner.query(`ALTER TABLE "layer" ADD "namespace" character varying NOT NULL`); + await queryRunner.query(`ALTER TABLE "layer" DROP CONSTRAINT "PK_ee22a436b5250ea132444cd9baa"`); + await queryRunner.query(`ALTER TABLE "layer" ADD CONSTRAINT "PK_0080403b718d78e8dc9590e56ad" PRIMARY KEY ("layer_name", "namespace")`); + await queryRunner.query(`CREATE TYPE "public"."layer_source" AS ENUM('sharedLua', 'perLayerJson')`); + await queryRunner.query(`ALTER TABLE "layer" ADD "source" "public"."layer_source" NOT NULL`); + await queryRunner.query(`ALTER TABLE "property" ADD "namespace" character varying NOT NULL`); + await queryRunner.query(`ALTER TABLE "property" DROP CONSTRAINT "PK_c92abce6639221fa0e46b3d1380"`); + await queryRunner.query( + `ALTER TABLE "property" ADD CONSTRAINT "PK_f92b1edaa8f549c489fc2de4a05" PRIMARY KEY ("layer_name", "property", "namespace")` + ); + await queryRunner.query(`ALTER TABLE "layer" ALTER COLUMN "layer_id" DROP NOT NULL`); + await queryRunner.query( + `ALTER TABLE "layer" ADD CONSTRAINT "CHK_layer_source_layer_id" CHECK (("source" = 'sharedLua' AND "layer_id" IS NOT NULL) OR ("source" = 'perLayerJson' AND "layer_id" IS NULL))` + ); + await queryRunner.query( + `ALTER TABLE "enum_value" ADD CONSTRAINT "FK_10b4583c2705fa3454d63f05534" FOREIGN KEY ("namespace", "layer_name", "property") REFERENCES "property"("namespace","layer_name","property") ON DELETE CASCADE ON UPDATE NO ACTION` + ); + await queryRunner.query( + `ALTER TABLE "property" ADD CONSTRAINT "FK_d4a43c70a881756a56e51b86b84" FOREIGN KEY ("namespace", "layer_name") REFERENCES "layer"("namespace","layer_name") ON DELETE NO ACTION ON UPDATE NO ACTION` + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "property" DROP CONSTRAINT "FK_d4a43c70a881756a56e51b86b84"`); + await queryRunner.query(`ALTER TABLE "enum_value" DROP CONSTRAINT "FK_10b4583c2705fa3454d63f05534"`); + await queryRunner.query(`ALTER TABLE "layer" DROP CONSTRAINT "CHK_layer_source_layer_id"`); + await queryRunner.query(`ALTER TABLE "layer" ALTER COLUMN "layer_id" SET NOT NULL`); + await queryRunner.query(`ALTER TABLE "property" DROP CONSTRAINT "PK_f92b1edaa8f549c489fc2de4a05"`); + await queryRunner.query(`ALTER TABLE "property" ADD CONSTRAINT "PK_c92abce6639221fa0e46b3d1380" PRIMARY KEY ("layer_name", "property")`); + await queryRunner.query(`ALTER TABLE "property" DROP COLUMN "namespace"`); + await queryRunner.query(`ALTER TABLE "layer" DROP COLUMN "source"`); + await queryRunner.query(`DROP TYPE "public"."layer_source"`); + await queryRunner.query(`ALTER TABLE "layer" DROP CONSTRAINT "PK_0080403b718d78e8dc9590e56ad"`); + await queryRunner.query(`ALTER TABLE "layer" ADD CONSTRAINT "PK_ee22a436b5250ea132444cd9baa" PRIMARY KEY ("layer_name")`); + await queryRunner.query(`ALTER TABLE "layer" DROP COLUMN "namespace"`); + await queryRunner.query(`ALTER TABLE "enum_value" DROP CONSTRAINT "PK_0607639cc6d4c22cad2fa8552e9"`); + await queryRunner.query( + `ALTER TABLE "enum_value" ADD CONSTRAINT "PK_08cf35141adc487a902990ccff2" PRIMARY KEY ("value", "layer_name", "property")` + ); + await queryRunner.query(`ALTER TABLE "enum_value" DROP COLUMN "namespace"`); + await queryRunner.query( + `ALTER TABLE "property" ADD CONSTRAINT "FK_34ce6cedb279510bb20da63cf21" FOREIGN KEY ("layer_name") REFERENCES "layer"("layer_name") ON DELETE NO ACTION ON UPDATE NO ACTION` + ); + await queryRunner.query( + `ALTER TABLE "enum_value" ADD CONSTRAINT "FK_8548a0384cc733008852a8f4e70" FOREIGN KEY ("layer_name", "property") REFERENCES "property"("layer_name","property") ON DELETE CASCADE ON UPDATE NO ACTION` + ); + } +} diff --git a/packages/db/src/openapi.d.ts b/packages/db/src/openapi.d.ts index c6916e9..71613fd 100644 --- a/packages/db/src/openapi.d.ts +++ b/packages/db/src/openapi.d.ts @@ -4,14 +4,31 @@ import type { TypedRequestHandlers as ImportedTypedRequestHandlers } from '@map-colonies/openapi-helpers/typedRequestHandler'; export type paths = { - '/layers': { + '/namespaces': { parameters: { query?: never; header?: never; path?: never; cookie?: never; }; - /** Get all layer names */ + /** Get all namespaces */ + get: operations['getNamespaces']; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + '/namespaces/{namespace}/layers': { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get all layer names within a namespace */ get: operations['getLayers']; put?: never; post?: never; @@ -21,14 +38,14 @@ export type paths = { patch?: never; trace?: never; }; - '/layers/{layerName}': { + '/namespaces/{namespace}/layers/{layerName}': { parameters: { query?: never; header?: never; path?: never; cookie?: never; }; - /** Get layer spec by name */ + /** Get layer spec by name within a namespace */ get: operations['getLayerByName']; put?: never; post?: never; @@ -42,6 +59,18 @@ export type paths = { export type webhooks = Record; export type components = { schemas: { + /** @description A namespace served by this deployment. */ + NamespaceSummary: { + /** + * @description Name of the namespace + * @example data-2025 + */ + name: string; + }; + /** @description Response wrapper for the list of namespaces. */ + NamespaceList: { + namespaces: components['schemas']['NamespaceSummary'][]; + }; /** @description Basic layer info returned in the list endpoint. */ LayerSummary: { /** @@ -103,6 +132,27 @@ export type components = { }; }; responses: { + /** @description A list of the namespaces this deployment serves */ + NamespaceListResponse: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "namespaces": [ + * { + * "name": "data-2024" + * }, + * { + * "name": "data-2025" + * } + * ] + * } + */ + 'application/json': components['schemas']['NamespaceList']; + }; + }; /** @description A list of layers with their display aliases */ LayerListResponse: { headers: { @@ -175,6 +225,8 @@ export type components = { }; }; parameters: { + /** @description The namespace the layer belongs to. Layer names are only unique within a namespace */ + NamespaceParam: string; /** @description The name of the layer (matches `Property.layerName`) */ LayerNameParam: string; }; @@ -184,7 +236,7 @@ export type components = { }; export type $defs = Record; export interface operations { - getLayers: { + getNamespaces: { parameters: { query?: never; header?: never; @@ -192,6 +244,21 @@ export interface operations { cookie?: never; }; requestBody?: never; + responses: { + 200: components['responses']['NamespaceListResponse']; + }; + }; + getLayers: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The namespace the layer belongs to. Layer names are only unique within a namespace */ + namespace: components['parameters']['NamespaceParam']; + }; + cookie?: never; + }; + requestBody?: never; responses: { 200: components['responses']['LayerListResponse']; 404: components['responses']['NotFound']; @@ -202,6 +269,8 @@ export interface operations { query?: never; header?: never; path: { + /** @description The namespace the layer belongs to. Layer names are only unique within a namespace */ + namespace: components['parameters']['NamespaceParam']; /** @description The name of the layer (matches `Property.layerName`) */ layerName: components['parameters']['LayerNameParam']; }; diff --git a/packages/db/src/types/namespace.ts b/packages/db/src/types/namespace.ts new file mode 100644 index 0000000..57b6be8 --- /dev/null +++ b/packages/db/src/types/namespace.ts @@ -0,0 +1,3 @@ +import type { components } from '../openapi'; + +export type NamespaceSummary = components['schemas']['NamespaceSummary']; From 93e93bc1e8cd762c21d0914bf619c73eec8b358e Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Wed, 19 Aug 2026 08:48:18 +0300 Subject: [PATCH 02/29] feat: add namespace to api --- .../controllers/namespaceController.ts | 27 +++++++++++++++++++ .../src/namespace/models/namespaceManager.ts | 24 +++++++++++++++++ .../api/src/namespace/types/namespaceTypes.ts | 5 ++++ 3 files changed, 56 insertions(+) create mode 100644 packages/api/src/namespace/controllers/namespaceController.ts create mode 100644 packages/api/src/namespace/models/namespaceManager.ts create mode 100644 packages/api/src/namespace/types/namespaceTypes.ts diff --git a/packages/api/src/namespace/controllers/namespaceController.ts b/packages/api/src/namespace/controllers/namespaceController.ts new file mode 100644 index 0000000..1c72f3d --- /dev/null +++ b/packages/api/src/namespace/controllers/namespaceController.ts @@ -0,0 +1,27 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { injectable, inject } from 'tsyringe'; +import { RequestHandler } from 'express'; +import { SERVICES } from '@common/constants'; +import { NamespaceManager } from '../models/namespaceManager'; +import { GetNamespacesResponse } from '../types/namespaceTypes'; + +type GetNamespacesHandler = RequestHandler; + +@injectable() +export class NamespaceController { + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(NamespaceManager) private readonly manager: NamespaceManager + ) {} + + public getNamespaces: GetNamespacesHandler = async (req, res, next) => { + try { + const namespaces = await this.manager.getNamespaces(); + this.logger.debug({ msg: `got ${namespaces.length} namespaces` }); + return res.json({ namespaces }); + } catch (error) { + this.logger.error({ msg: 'failed to get namespaces', err: error }); + next(error); + } + }; +} diff --git a/packages/api/src/namespace/models/namespaceManager.ts b/packages/api/src/namespace/models/namespaceManager.ts new file mode 100644 index 0000000..977ebf9 --- /dev/null +++ b/packages/api/src/namespace/models/namespaceManager.ts @@ -0,0 +1,24 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { injectable, inject } from 'tsyringe'; +import { Repository } from 'typeorm'; +import { Layer, LAYER_REPOSITORY_SYMBOL } from '@db'; +import type { NamespaceSummary } from '@db'; +import { SERVICES } from '@common/constants'; + +@injectable() +export class NamespaceManager { + public constructor( + @inject(LAYER_REPOSITORY_SYMBOL) private readonly repository: Repository, + @inject(SERVICES.LOGGER) private readonly logger: Logger + ) {} + + public async getNamespaces(): Promise { + this.logger.debug({ msg: 'getting namespaces' }); + return this.repository + .createQueryBuilder('layer') + .select('layer.namespace', 'name') + .distinct(true) + .orderBy('name', 'ASC') + .getRawMany(); + } +} diff --git a/packages/api/src/namespace/types/namespaceTypes.ts b/packages/api/src/namespace/types/namespaceTypes.ts new file mode 100644 index 0000000..3f2e445 --- /dev/null +++ b/packages/api/src/namespace/types/namespaceTypes.ts @@ -0,0 +1,5 @@ +import type { NamespaceSummary } from '@db'; + +export interface GetNamespacesResponse { + namespaces: NamespaceSummary[]; +} From e56442eb89cba6992a1d87ee102bc24d84b50719 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Wed, 19 Aug 2026 08:50:30 +0300 Subject: [PATCH 03/29] feat: add the namespace syncing --- packages/api/src/containerConfig.ts | 2 + .../src/layer/controllers/layerController.ts | 19 +-- packages/api/src/layer/models/layerManager.ts | 15 +- packages/api/src/layer/routes/layer.ts | 4 +- packages/api/src/layer/types/layerTypes.ts | 5 + packages/api/src/serverBuilder.ts | 7 +- .../layer/helpers/layerRequestSender.ts | 13 +- .../api/tests/integration/layer/layer.spec.ts | 135 +++++++++++++++--- 8 files changed, 154 insertions(+), 46 deletions(-) diff --git a/packages/api/src/containerConfig.ts b/packages/api/src/containerConfig.ts index 8c1f4e3..21f2258 100644 --- a/packages/api/src/containerConfig.ts +++ b/packages/api/src/containerConfig.ts @@ -11,6 +11,7 @@ import { type InjectionObject, registerDependencies } from '@common/dependencyRe import { HEALTHCHECK, ON_SIGNAL, SERVICES, SERVICE_NAME } from '@common/constants'; import { getTracing } from '@common/tracing'; import { LAYER_ROUTER_SYMBOL, layerRouterFactory } from './layer/routes/layer'; +import { NAMESPACE_ROUTER_SYMBOL, namespaceRouterFactory } from './namespace/routes/namespace'; import { dataSourceFactory, healthCheckFactory } from './common/db/connection'; import { type ConfigType, getConfig } from './common/config'; @@ -66,6 +67,7 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise }, }, { token: LAYER_ROUTER_SYMBOL, provider: { useFactory: layerRouterFactory } }, + { token: NAMESPACE_ROUTER_SYMBOL, provider: { useFactory: namespaceRouterFactory } }, { token: LAYER_REPOSITORY_SYMBOL, provider: { diff --git a/packages/api/src/layer/controllers/layerController.ts b/packages/api/src/layer/controllers/layerController.ts index c40c803..bb2e0b9 100644 --- a/packages/api/src/layer/controllers/layerController.ts +++ b/packages/api/src/layer/controllers/layerController.ts @@ -4,9 +4,9 @@ import { injectable, inject } from 'tsyringe'; import { RequestHandler } from 'express'; import { SERVICES } from '@common/constants'; import { LayerManager } from '../models/layerManager'; -import { GetLayerByNameParam, GetLayersResponse } from '../types/layerTypes'; +import { GetLayerByNameParam, GetLayersParam, GetLayersResponse } from '../types/layerTypes'; -type GetLayersHandler = RequestHandler; +type GetLayersHandler = RequestHandler; type GetLayerByNameHandler = RequestHandler; @injectable() @@ -17,24 +17,25 @@ export class LayerController { ) {} public getLayers: GetLayersHandler = async (req, res, next) => { + const { namespace } = req.params; try { - const layers = await this.manager.getLayers(); - this.logger.debug({ msg: `got ${layers.length} layers` }); + const layers = await this.manager.getLayers(namespace); + this.logger.debug({ msg: `got ${layers.length} layers`, namespace }); return res.json({ layers }); } catch (error) { - this.logger.error({ msg: 'failed to get layers', err: error }); + this.logger.error({ msg: 'failed to get layers', namespace, err: error }); next(error); } }; public getLayerByName: GetLayerByNameHandler = async (req, res, next) => { - const { layerName } = req.params; + const { namespace, layerName } = req.params; try { - const layer = await this.manager.getLayerSpecByName(layerName); - this.logger.debug({ msg: `got layer: ${layerName}` }); + const layer = await this.manager.getLayerSpecByName(namespace, layerName); + this.logger.debug({ msg: `got layer: ${layerName}`, namespace }); return res.json(layer); } catch (error) { - this.logger.error({ msg: 'failed to get layer by name', layerName, err: error }); + this.logger.error({ msg: 'failed to get layer by name', namespace, layerName, err: error }); next(error); } }; diff --git a/packages/api/src/layer/models/layerManager.ts b/packages/api/src/layer/models/layerManager.ts index 47ad673..48f658d 100644 --- a/packages/api/src/layer/models/layerManager.ts +++ b/packages/api/src/layer/models/layerManager.ts @@ -14,16 +14,19 @@ export class LayerManager { @inject(SERVICES.LOGGER) private readonly logger: Logger ) {} - public async getLayers(): Promise { - this.logger.debug({ msg: 'getting layers' }); - const layers = await this.repository.find({ select: { layerName: true, alias: true } }); + public async getLayers(namespace: string): Promise { + this.logger.debug({ msg: 'getting layers', namespace }); + const layers = await this.repository.find({ where: { namespace }, select: { layerName: true, alias: true } }); + if (layers.length === 0) { + throw new NotFoundError(`Namespace doesn't exist`); + } return layers.map(({ layerName, alias }) => ({ layerName, alias })); } - public async getLayerSpecByName(name: string): Promise { - this.logger.debug({ msg: 'getting layer spec by name', name }); + public async getLayerSpecByName(namespace: string, name: string): Promise { + this.logger.debug({ msg: 'getting layer spec by name', namespace, name }); const layer = await this.repository.findOne({ - where: { layerName: name }, + where: { namespace, layerName: name }, relations: { properties: { possibleValues: true } }, }); if (!layer) { diff --git a/packages/api/src/layer/routes/layer.ts b/packages/api/src/layer/routes/layer.ts index 09b45d0..d5f3952 100644 --- a/packages/api/src/layer/routes/layer.ts +++ b/packages/api/src/layer/routes/layer.ts @@ -6,8 +6,8 @@ const layerRouterFactory: FactoryFunction = (dependencyContainer) => { const router = Router(); const controller = dependencyContainer.resolve(LayerController); - router.get('/', controller.getLayers); - router.get('/:layerName', controller.getLayerByName); + router.get('/:namespace/layers', controller.getLayers); + router.get('/:namespace/layers/:layerName', controller.getLayerByName); return router; }; diff --git a/packages/api/src/layer/types/layerTypes.ts b/packages/api/src/layer/types/layerTypes.ts index 27d53d1..9d292f0 100644 --- a/packages/api/src/layer/types/layerTypes.ts +++ b/packages/api/src/layer/types/layerTypes.ts @@ -1,6 +1,11 @@ import type { LayerSummary } from '@db'; +export interface GetLayersParam { + namespace: string; +} + export interface GetLayerByNameParam { + namespace: string; layerName: string; } diff --git a/packages/api/src/serverBuilder.ts b/packages/api/src/serverBuilder.ts index f2e28e6..b56043d 100644 --- a/packages/api/src/serverBuilder.ts +++ b/packages/api/src/serverBuilder.ts @@ -11,6 +11,7 @@ import { collectMetricsExpressMiddleware } from '@map-colonies/prometheus'; import { Registry } from 'prom-client'; import { SERVICES } from '@common/constants'; import { LAYER_ROUTER_SYMBOL } from './layer/routes/layer'; +import { NAMESPACE_ROUTER_SYMBOL } from './namespace/routes/namespace'; import { ConfigType } from './common/config'; @injectable() @@ -21,7 +22,8 @@ export class ServerBuilder { @inject(SERVICES.CONFIG) private readonly config: ConfigType, @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.METRICS) private readonly metricsRegistry: Registry, - @inject(LAYER_ROUTER_SYMBOL) private readonly layerRouter: Router + @inject(LAYER_ROUTER_SYMBOL) private readonly layerRouter: Router, + @inject(NAMESPACE_ROUTER_SYMBOL) private readonly namespaceRouter: Router ) { this.serverInstance = express(); } @@ -44,7 +46,8 @@ export class ServerBuilder { } private buildRoutes(): void { - this.serverInstance.use('/layers', this.layerRouter); + this.serverInstance.use('/namespaces', this.namespaceRouter); + this.serverInstance.use('/namespaces', this.layerRouter); this.buildDocsRoutes(); } diff --git a/packages/api/tests/integration/layer/helpers/layerRequestSender.ts b/packages/api/tests/integration/layer/helpers/layerRequestSender.ts index 9a73f66..99f4ad1 100644 --- a/packages/api/tests/integration/layer/helpers/layerRequestSender.ts +++ b/packages/api/tests/integration/layer/helpers/layerRequestSender.ts @@ -3,6 +3,7 @@ import type supertest from 'supertest'; import { agent } from 'supertest'; import type { Layer } from '@db'; import type { GetLayersResponse } from '@src/layer/types/layerTypes'; +import type { GetNamespacesResponse } from '@src/namespace/types/namespaceTypes'; interface TypedResponse extends Omit { body: T; @@ -11,11 +12,15 @@ interface TypedResponse extends Omit { export class LayerRequestSender { public constructor(private readonly app: Application) {} - public async getLayers(): Promise> { - return agent(this.app).get('/layers'); + public async getNamespaces(): Promise> { + return agent(this.app).get('/namespaces'); } - public async getLayerByName(layerName: string): Promise> { - return agent(this.app).get(`/layers/${layerName}`); + public async getLayers(namespace: string): Promise> { + return agent(this.app).get(`/namespaces/${namespace}/layers`); + } + + public async getLayerByName(namespace: string, layerName: string): Promise> { + return agent(this.app).get(`/namespaces/${namespace}/layers/${layerName}`); } } diff --git a/packages/api/tests/integration/layer/layer.spec.ts b/packages/api/tests/integration/layer/layer.spec.ts index 479cada..fbbf18b 100644 --- a/packages/api/tests/integration/layer/layer.spec.ts +++ b/packages/api/tests/integration/layer/layer.spec.ts @@ -3,13 +3,16 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; import type { DataSource, Repository } from 'typeorm'; -import { DATA_SOURCE_PROVIDER, EnumValue, LAYER_REPOSITORY_SYMBOL, Property, columnType } from '@map-colonies/vector-standard-db'; +import { DATA_SOURCE_PROVIDER, EnumValue, LAYER_REPOSITORY_SYMBOL, Property, columnType, layerSource } from '@map-colonies/vector-standard-db'; import type { Layer } from '@map-colonies/vector-standard-db'; import { getApp } from '@src/app'; import { SERVICES } from '@src/common/constants'; import { initConfig } from '@src/common/config'; import { LayerRequestSender } from './helpers/layerRequestSender'; +const NAMESPACE = 'data-2025'; +const OTHER_NAMESPACE = 'data-2024'; + describe('layer', function () { let requestSender: LayerRequestSender; let dataSource: DataSource; @@ -44,37 +47,76 @@ describe('layer', function () { }); describe('Happy Path', function () { - describe('GET /layers', function () { + describe('GET /namespaces', function () { + it('should return 200 with every namespace holding layers, without duplicates', async function () { + await layerRepository.save([ + { namespace: NAMESPACE, layerName: 'buildings_polygon', layerId: 1, source: layerSource.sharedLua, alias: 'Buildings Polygon' }, + { namespace: NAMESPACE, layerName: 'fences_line', layerId: 2, source: layerSource.sharedLua, alias: 'Fences Line' }, + { namespace: OTHER_NAMESPACE, layerName: 'buildings_polygon', layerId: null, source: layerSource.perLayerJson, alias: 'Old Buildings' }, + ]); + + const response = await requestSender.getNamespaces(); + const { namespaces } = response.body; + + expect(response.statusCode).toBe(httpStatusCodes.OK); + expect(namespaces).toEqual([{ name: OTHER_NAMESPACE }, { name: NAMESPACE }]); + }); + }); + + describe('GET /namespaces/:namespace/layers', function () { beforeEach(async function () { await layerRepository.save([ - { layerName: 'buildings_polygon', layerId: 1, alias: 'Buildings Polygon' }, - { layerName: 'fences_line', layerId: 2, alias: 'Fences Line' }, + { namespace: NAMESPACE, layerName: 'buildings_polygon', layerId: 1, source: layerSource.sharedLua, alias: 'Buildings Polygon' }, + { namespace: NAMESPACE, layerName: 'fences_line', layerId: 2, source: layerSource.sharedLua, alias: 'Fences Line' }, ]); await propertyRepository.save([ - { layerName: 'buildings_polygon', property: 'code', type: columnType.text }, - { layerName: 'fences_line', property: 'height', type: columnType.real }, + { namespace: NAMESPACE, layerName: 'buildings_polygon', property: 'code', type: columnType.text }, + { namespace: NAMESPACE, layerName: 'fences_line', property: 'height', type: columnType.real }, ]); }); it('should return 200 with all layers including their aliases', async function () { - const response = await requestSender.getLayers(); + const response = await requestSender.getLayers(NAMESPACE); const { layers } = response.body; expect(response.statusCode).toBe(httpStatusCodes.OK); expect(layers).toContainEqual({ layerName: 'buildings_polygon', alias: 'Buildings Polygon' }); expect(layers).toContainEqual({ layerName: 'fences_line', alias: 'Fences Line' }); }); + + it('should not return layers belonging to another namespace', async function () { + await layerRepository.save({ + namespace: OTHER_NAMESPACE, + layerName: 'rivers_line', + layerId: null, + source: layerSource.perLayerJson, + alias: 'Rivers', + }); + + const response = await requestSender.getLayers(NAMESPACE); + const { layers } = response.body; + + expect(response.statusCode).toBe(httpStatusCodes.OK); + expect(layers).toHaveLength(2); + expect(layers).not.toContainEqual({ layerName: 'rivers_line', alias: 'Rivers' }); + }); }); - describe('GET /layers/:layerName', function () { + describe('GET /namespaces/:namespace/layers/:layerName', function () { it('should return 200 with the layer spec for non-enum properties', async function () { - await layerRepository.save({ layerName: 'buildings_polygon', layerId: 1, alias: 'Buildings Polygon' }); + await layerRepository.save({ + namespace: NAMESPACE, + layerName: 'buildings_polygon', + layerId: 1, + source: layerSource.sharedLua, + alias: 'Buildings Polygon', + }); await propertyRepository.save([ - { layerName: 'buildings_polygon', property: 'code', type: columnType.text }, - { layerName: 'buildings_polygon', property: 'height', type: columnType.real }, + { namespace: NAMESPACE, layerName: 'buildings_polygon', property: 'code', type: columnType.text }, + { namespace: NAMESPACE, layerName: 'buildings_polygon', property: 'height', type: columnType.real }, ]); - const response = await requestSender.getLayerByName('buildings_polygon'); + const response = await requestSender.getLayerByName(NAMESPACE, 'buildings_polygon'); const { layerName, properties } = response.body; expect(response.statusCode).toBe(httpStatusCodes.OK); @@ -84,14 +126,20 @@ describe('layer', function () { }); it('should return 200 with possibleValues for enum properties', async function () { - await layerRepository.save({ layerName: 'buildings_polygon', layerId: 1, alias: 'Buildings Polygon' }); - await propertyRepository.save({ layerName: 'buildings_polygon', property: 'classification', type: columnType.text }); + await layerRepository.save({ + namespace: NAMESPACE, + layerName: 'buildings_polygon', + layerId: 1, + source: layerSource.sharedLua, + alias: 'Buildings Polygon', + }); + await propertyRepository.save({ namespace: NAMESPACE, layerName: 'buildings_polygon', property: 'classification', type: columnType.text }); await enumValueRepository.save([ - { value: 'A', layerName: 'buildings_polygon', property: 'classification' }, - { value: 'B', layerName: 'buildings_polygon', property: 'classification' }, + { namespace: NAMESPACE, value: 'A', layerName: 'buildings_polygon', property: 'classification' }, + { namespace: NAMESPACE, value: 'B', layerName: 'buildings_polygon', property: 'classification' }, ]); - const response = await requestSender.getLayerByName('buildings_polygon'); + const response = await requestSender.getLayerByName(NAMESPACE, 'buildings_polygon'); const { properties } = response.body; const classificationProp = properties.find((p) => p.property === 'classification'); @@ -100,23 +148,64 @@ describe('layer', function () { expect(classificationProp?.possibleValues).toContain('A'); expect(classificationProp?.possibleValues).toContain('B'); }); + + it('should return the spec of the requested namespace when the layer name exists in both', async function () { + await layerRepository.save([ + { namespace: NAMESPACE, layerName: 'buildings_polygon', layerId: 1, source: layerSource.sharedLua, alias: 'New Buildings' }, + { namespace: OTHER_NAMESPACE, layerName: 'buildings_polygon', layerId: null, source: layerSource.perLayerJson, alias: 'Old Buildings' }, + ]); + await propertyRepository.save([ + { namespace: NAMESPACE, layerName: 'buildings_polygon', property: 'code', type: columnType.text }, + { namespace: OTHER_NAMESPACE, layerName: 'buildings_polygon', property: 'legacy_code', type: columnType.text }, + ]); + + const response = await requestSender.getLayerByName(OTHER_NAMESPACE, 'buildings_polygon'); + const { alias, properties } = response.body; + + expect(response.statusCode).toBe(httpStatusCodes.OK); + expect(alias).toBe('Old Buildings'); + expect(properties).toContainEqual({ property: 'legacy_code', type: columnType.text }); + expect(properties).not.toContainEqual({ property: 'code', type: columnType.text }); + }); }); }); describe('Sad Path', function () { - describe('GET /layers', function () { + describe('GET /namespaces', function () { it('should return 200 with an empty list when there are no layers', async function () { - const response = await requestSender.getLayers(); - const { layers } = response.body; + const response = await requestSender.getNamespaces(); + const { namespaces } = response.body; expect(response.statusCode).toBe(httpStatusCodes.OK); - expect(layers).toHaveLength(0); + expect(namespaces).toHaveLength(0); }); }); - describe('GET /layers/:layerName', function () { + describe('GET /namespaces/:namespace/layers', function () { + it('should return 404 not found for a namespace holding no layers', async function () { + const response = await requestSender.getLayers('nonexistent'); + + expect(response.statusCode).toBe(httpStatusCodes.NOT_FOUND); + }); + }); + + describe('GET /namespaces/:namespace/layers/:layerName', function () { it('should return 404 not found for a non-existent layer', async function () { - const response = await requestSender.getLayerByName('nonexistent'); + const response = await requestSender.getLayerByName(NAMESPACE, 'nonexistent'); + + expect(response.statusCode).toBe(httpStatusCodes.NOT_FOUND); + }); + + it('should return 404 not found when the layer exists only in another namespace', async function () { + await layerRepository.save({ + namespace: OTHER_NAMESPACE, + layerName: 'rivers_line', + layerId: null, + source: layerSource.perLayerJson, + alias: 'Rivers', + }); + + const response = await requestSender.getLayerByName(NAMESPACE, 'rivers_line'); expect(response.statusCode).toBe(httpStatusCodes.NOT_FOUND); }); From 9fdc3b5e1ef6a7c0b5e6d3f63e2c82b0850d0573 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Wed, 19 Aug 2026 08:51:15 +0300 Subject: [PATCH 04/29] feat: add final package stuff --- packages/db/src/entities/enumValue.ts | 4 ++++ packages/db/src/entities/layer.ts | 19 ++++++++++++++++--- packages/db/src/entities/property.ts | 8 +++++++- packages/db/src/index.ts | 5 +++-- packages/db/src/types/enums.ts | 7 +++++++ 5 files changed, 37 insertions(+), 6 deletions(-) diff --git a/packages/db/src/entities/enumValue.ts b/packages/db/src/entities/enumValue.ts index d73b465..cdb60c2 100644 --- a/packages/db/src/entities/enumValue.ts +++ b/packages/db/src/entities/enumValue.ts @@ -5,6 +5,9 @@ export const ENUMS_REPOSITORY_SYMBOL = Symbol('EnumsRepository'); @Entity('enum_value') export class EnumValue { + @PrimaryColumn() + public namespace!: string; + @PrimaryColumn() public value!: string; @@ -16,6 +19,7 @@ export class EnumValue { @ManyToOne(() => Property, (p) => p.possibleValues, { onDelete: 'CASCADE' }) @JoinColumn([ + { name: 'namespace', referencedColumnName: 'namespace' }, { name: 'layer_name', referencedColumnName: 'layerName' }, { name: 'property', referencedColumnName: 'property' }, ]) diff --git a/packages/db/src/entities/layer.ts b/packages/db/src/entities/layer.ts index d28902a..fc8beb8 100644 --- a/packages/db/src/entities/layer.ts +++ b/packages/db/src/entities/layer.ts @@ -1,16 +1,29 @@ -import { Column, Entity, OneToMany, PrimaryColumn } from 'typeorm'; +import { Check, Column, Entity, OneToMany, PrimaryColumn } from 'typeorm'; import { Layer as ILayer } from '../types/layer'; +import { LayerSource, layerSource } from '../types/enums'; import { Property } from './property'; export const LAYER_REPOSITORY_SYMBOL = Symbol('LayerRepository'); +export const LAYER_SOURCE_CHECK = 'CHK_layer_source_layer_id'; + @Entity('layer') +@Check( + LAYER_SOURCE_CHECK, + `("source" = '${layerSource.sharedLua}' AND "layer_id" IS NOT NULL) OR ("source" = '${layerSource.perLayerJson}' AND "layer_id" IS NULL)` +) export class Layer implements ILayer { + @PrimaryColumn() + public namespace!: string; + @PrimaryColumn({ name: 'layer_name' }) public layerName!: string; - @Column({ name: 'layer_id' }) - public layerId!: number; + @Column({ name: 'layer_id', type: 'integer', nullable: true }) + public layerId!: number | null; + + @Column({ type: 'enum', enum: Object.values(layerSource), enumName: 'layer_source' }) + public source!: LayerSource; @Column() public alias!: string; diff --git a/packages/db/src/entities/property.ts b/packages/db/src/entities/property.ts index 9f72d9d..d6b50b3 100644 --- a/packages/db/src/entities/property.ts +++ b/packages/db/src/entities/property.ts @@ -7,6 +7,9 @@ export const PROPERTY_REPOSITORY_SYMBOL = Symbol('PropertyRepository'); @Entity('property') export class Property { + @PrimaryColumn() + public namespace!: string; + @PrimaryColumn({ name: 'layer_name' }) public layerName!: string; @@ -27,6 +30,9 @@ export class Property { public possibleValues?: EnumValue[]; @ManyToOne(() => Layer, (layer) => layer.properties) - @JoinColumn({ name: 'layer_name', referencedColumnName: 'layerName' }) + @JoinColumn([ + { name: 'namespace', referencedColumnName: 'namespace' }, + { name: 'layer_name', referencedColumnName: 'layerName' }, + ]) public layerRelation!: Layer; } diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 5cc38c5..21d0280 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -1,8 +1,9 @@ -export { Layer, LAYER_REPOSITORY_SYMBOL } from './entities/layer'; +export { Layer, LAYER_REPOSITORY_SYMBOL, LAYER_SOURCE_CHECK } from './entities/layer'; export type { Layer as ILayer, LayerSpec, LayerSummary } from './types/layer'; +export type { NamespaceSummary } from './types/namespace'; export type { Property as PropertySpec } from './types/property'; export { Property, PROPERTY_REPOSITORY_SYMBOL } from './entities/property'; export { EnumValue, ENUMS_REPOSITORY_SYMBOL } from './entities/enumValue'; -export { ColumnType, columnType } from './types/enums'; +export { ColumnType, columnType, LayerSource, layerSource } from './types/enums'; export { DATA_SOURCE_PROVIDER, createDataSourceOptions, createDataSourceHealthCheck, createSslOptions } from './db/connection'; export { DbConfig } from './interfaces'; diff --git a/packages/db/src/types/enums.ts b/packages/db/src/types/enums.ts index ef26b74..2d0ab34 100644 --- a/packages/db/src/types/enums.ts +++ b/packages/db/src/types/enums.ts @@ -14,3 +14,10 @@ export const columnType = { } as const; export type ColumnType = (typeof columnType)[keyof typeof columnType]; + +export const layerSource = { + sharedLua: 'sharedLua', + perLayerJson: 'perLayerJson', +} as const; + +export type LayerSource = (typeof layerSource)[keyof typeof layerSource]; From 827e0dfcf08d77cdb872de84df09538c6f351a5e Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Wed, 19 Aug 2026 16:31:36 +0300 Subject: [PATCH 05/29] test: fix layerManager spec for namespace-scoped api --- .../unit/layer/models/layerManager.spec.ts | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/packages/api/tests/unit/layer/models/layerManager.spec.ts b/packages/api/tests/unit/layer/models/layerManager.spec.ts index 7894896..3c37a40 100644 --- a/packages/api/tests/unit/layer/models/layerManager.spec.ts +++ b/packages/api/tests/unit/layer/models/layerManager.spec.ts @@ -7,6 +7,8 @@ import { SERVICES } from '@src/common/constants'; import { registerDependencies } from '@src/common/dependencyRegistration'; describe('LayerManager', function () { + const TEST_NAMESPACE = 'data-2026'; + let manager: LayerManager; const find = vi.fn(); @@ -38,7 +40,7 @@ describe('LayerManager', function () { ]; find.mockResolvedValue(layers); - const result = await manager.getLayers(); + const result = await manager.getLayers(TEST_NAMESPACE); expect(result).toEqual(layers); }); @@ -48,9 +50,12 @@ describe('LayerManager', function () { it('should query using the provided layer name', async function () { findOne.mockResolvedValue({ layerName: 'buildings_polygon', alias: 'Buildings', properties: [] }); - await manager.getLayerSpecByName('buildings_polygon'); + await manager.getLayerSpecByName(TEST_NAMESPACE, 'buildings_polygon'); - expect(findOne).toHaveBeenCalledWith({ where: { layerName: 'buildings_polygon' }, relations: { properties: { possibleValues: true } } }); + expect(findOne).toHaveBeenCalledWith({ + where: { layerName: 'buildings_polygon', namespace: TEST_NAMESPACE }, + relations: { properties: { possibleValues: true } }, + }); }); it.each([ @@ -84,7 +89,7 @@ describe('LayerManager', function () { ])('should $name', async function ({ inputProperties, expectedPossibleValues }) { findOne.mockResolvedValue({ layerName: 'buildings_polygon', alias: 'Buildings', properties: inputProperties }); - const { properties } = await manager.getLayerSpecByName('buildings_polygon'); + const { properties } = await manager.getLayerSpecByName(TEST_NAMESPACE, 'buildings_polygon'); for (const { property, possibleValues } of expectedPossibleValues) { expect(properties.find((p: { property: string }) => p.property === property)?.possibleValues).toEqual(possibleValues); @@ -95,12 +100,10 @@ describe('LayerManager', function () { describe('Sad Path', function () { describe('getLayers', function () { - it('should return an empty array when there are no layers', async function () { + it("should throw an error when the namespace doesn't exist", async function () { find.mockResolvedValue([]); - const result = await manager.getLayers(); - - expect(result).toEqual([]); + await expect(manager.getLayers(TEST_NAMESPACE)).rejects.toThrow(); }); }); @@ -108,7 +111,7 @@ describe('LayerManager', function () { it('should throw a NotFoundError when the layer does not exist', async function () { findOne.mockResolvedValue(null); - await expect(manager.getLayerSpecByName('nonexistent')).rejects.toThrow(NotFoundError); + await expect(manager.getLayerSpecByName(TEST_NAMESPACE, 'nonexistent')).rejects.toThrow(NotFoundError); }); }); }); From fa990c38a559ecfc96474dd224c714a0b32b6ecf Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Wed, 19 Aug 2026 16:32:25 +0300 Subject: [PATCH 06/29] feat: add multi-namespace and layer source compatibility to synchronizer --- package-lock.json | 325 +++++++++++--- package.json | 5 +- packages/synchronizer/config/default.json | 54 ++- .../{layers.json => layers/default.json} | 0 packages/synchronizer/config/layers/test.json | 7 + packages/synchronizer/config/test.json | 44 +- packages/synchronizer/package.json | 5 +- packages/synchronizer/src/common/config.ts | 6 +- packages/synchronizer/src/common/constants.ts | 9 +- .../synchronizer/src/common/db/connection.ts | 23 +- .../synchronizer/src/common/interfaces.ts | 8 +- .../synchronizer/src/common/s3/interfaces.ts | 4 +- .../src/common/s3/s3Repository.ts | 22 +- .../synchronizer/src/common/tracing/sync.ts | 1 + packages/synchronizer/src/containerConfig.ts | 63 ++- packages/synchronizer/src/sync/SyncModel.ts | 110 +++-- packages/synchronizer/src/sync/cron.ts | 122 +++--- packages/synchronizer/src/sync/errors.ts | 14 +- packages/synchronizer/src/sync/fileReader.ts | 27 +- packages/synchronizer/src/sync/helpers.ts | 4 +- .../src/sync/layerSource/index.ts | 20 + .../sync/layerSource/perLayerJsonSource.ts | 104 +++++ .../src/sync/layerSource/sharedLuaSource.ts | 62 +++ .../src/sync/layerSource/types.ts | 17 + .../synchronizer/src/sync/namespaceHandle.ts | 47 ++ .../tests/configurations/globalSetup.ts | 8 +- .../integration/syncModel/syncModel.spec.ts | 404 +++++++++--------- .../unit/synchronizer/aliasEnricher.spec.ts | 18 +- .../tests/unit/synchronizer/helpers.spec.ts | 23 +- .../synchronizer/perLayerJsonSource.spec.ts | 141 ++++++ .../unit/synchronizer/sharedLuaSource.spec.ts | 141 ++++++ 31 files changed, 1331 insertions(+), 507 deletions(-) rename packages/synchronizer/config/{layers.json => layers/default.json} (100%) create mode 100644 packages/synchronizer/config/layers/test.json create mode 100644 packages/synchronizer/src/sync/layerSource/index.ts create mode 100644 packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts create mode 100644 packages/synchronizer/src/sync/layerSource/sharedLuaSource.ts create mode 100644 packages/synchronizer/src/sync/layerSource/types.ts create mode 100644 packages/synchronizer/src/sync/namespaceHandle.ts create mode 100644 packages/synchronizer/tests/unit/synchronizer/perLayerJsonSource.spec.ts create mode 100644 packages/synchronizer/tests/unit/synchronizer/sharedLuaSource.spec.ts diff --git a/package-lock.json b/package-lock.json index fbdf779..f946a30 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "packages/*" ], "dependencies": { - "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-9687e6cb21fe29d10c52aaf9db64a763050ff519.tgz" + "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-11bf4605d7a17c723dd85f624c88fe32d9957863.tgz" }, "devDependencies": { "@commitlint/cli": "^20.4.1", @@ -345,6 +345,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=6.9.0" } @@ -383,6 +384,7 @@ "version": "6.3.1", "dev": true, "license": "ISC", + "peer": true, "bin": { "semver": "bin/semver.js" } @@ -391,6 +393,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/parser": "^7.29.7", "@babel/types": "^7.29.7", @@ -406,6 +409,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/compat-data": "^7.29.7", "@babel/helper-validator-option": "^7.29.7", @@ -421,6 +425,7 @@ "version": "6.3.1", "dev": true, "license": "ISC", + "peer": true, "bin": { "semver": "bin/semver.js" } @@ -429,6 +434,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=6.9.0" } @@ -437,6 +443,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/traverse": "^7.29.7", "@babel/types": "^7.29.7" @@ -449,6 +456,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-module-imports": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7", @@ -465,6 +473,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=6.9.0" } @@ -488,6 +497,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=6.9.0" } @@ -496,6 +506,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/template": "^7.29.7", "@babel/types": "^7.29.7" @@ -522,6 +533,7 @@ "version": "7.8.4", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -533,6 +545,7 @@ "version": "7.8.3", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -544,6 +557,7 @@ "version": "7.12.13", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.12.13" }, @@ -555,6 +569,7 @@ "version": "7.14.5", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.14.5" }, @@ -569,6 +584,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.29.7" }, @@ -583,6 +599,7 @@ "version": "7.10.4", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.10.4" }, @@ -594,6 +611,7 @@ "version": "7.8.3", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -605,6 +623,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.29.7" }, @@ -619,6 +638,7 @@ "version": "7.10.4", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.10.4" }, @@ -630,6 +650,7 @@ "version": "7.8.3", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -641,6 +662,7 @@ "version": "7.10.4", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.10.4" }, @@ -652,6 +674,7 @@ "version": "7.8.3", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -663,6 +686,7 @@ "version": "7.8.3", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -674,6 +698,7 @@ "version": "7.8.3", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -685,6 +710,7 @@ "version": "7.14.5", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.14.5" }, @@ -699,6 +725,7 @@ "version": "7.14.5", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.14.5" }, @@ -713,6 +740,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.29.7" }, @@ -734,6 +762,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/parser": "^7.29.7", @@ -747,6 +776,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", @@ -1129,6 +1159,31 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@emnapi/core": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.3.tgz", + "integrity": "sha512-zLpS5asjEb7lq8jYLq37N6XKaE41DIexlY1rF/z4/tIl3wo13Sqm28fRyfIsKZD+NZ8mM5RoKkpW/rBcuoSZSg==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.3", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", + "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@emnapi/wasi-threads": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.3.tgz", @@ -1136,6 +1191,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "tslib": "^2.4.0" } @@ -1553,6 +1609,7 @@ "version": "1.1.0", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "camelcase": "^5.3.1", "find-up": "^4.1.0", @@ -1568,6 +1625,7 @@ "version": "1.0.10", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "sprintf-js": "~1.0.2" } @@ -1576,6 +1634,7 @@ "version": "4.1.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" @@ -1588,6 +1647,7 @@ "version": "3.15.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" @@ -1600,6 +1660,7 @@ "version": "5.0.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "p-locate": "^4.1.0" }, @@ -1611,6 +1672,7 @@ "version": "2.3.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "p-try": "^2.0.0" }, @@ -1625,6 +1687,7 @@ "version": "4.1.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "p-limit": "^2.2.0" }, @@ -1636,6 +1699,7 @@ "version": "0.1.6", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=8" } @@ -1644,6 +1708,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/types": "30.4.1", "@types/node": "*", @@ -1660,6 +1725,7 @@ "version": "30.4.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/console": "30.4.1", "@jest/pattern": "30.4.0", @@ -1706,6 +1772,7 @@ "version": "30.4.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } @@ -1714,6 +1781,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/fake-timers": "30.4.1", "@jest/types": "30.4.1", @@ -1728,6 +1796,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "expect": "30.4.1", "jest-snapshot": "30.4.1" @@ -1740,6 +1809,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/get-type": "30.1.0" }, @@ -1751,6 +1821,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/types": "30.4.1", "@sinonjs/fake-timers": "^15.4.0", @@ -1767,6 +1838,7 @@ "version": "30.1.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } @@ -1775,6 +1847,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/environment": "30.4.1", "@jest/expect": "30.4.1", @@ -1789,6 +1862,7 @@ "version": "30.4.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/node": "*", "jest-regex-util": "30.4.0" @@ -1801,6 +1875,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@bcoe/v8-coverage": "^0.2.3", "@jest/console": "30.4.1", @@ -1841,17 +1916,20 @@ "node_modules/@jest/reporters/node_modules/@bcoe/v8-coverage": { "version": "0.2.3", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@jest/reporters/node_modules/balanced-match": { "version": "1.0.2", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@jest/reporters/node_modules/brace-expansion": { "version": "2.1.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "balanced-match": "^1.0.0" } @@ -1860,6 +1938,7 @@ "version": "10.5.0", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", @@ -1879,6 +1958,7 @@ "version": "9.0.9", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "brace-expansion": "^2.0.2" }, @@ -1893,6 +1973,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@sinclair/typebox": "^0.34.0" }, @@ -1904,6 +1985,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/types": "30.4.1", "chalk": "^4.1.2", @@ -1918,6 +2000,7 @@ "version": "30.0.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jridgewell/trace-mapping": "^0.3.25", "callsites": "^3.1.0", @@ -1931,6 +2014,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/console": "30.4.1", "@jest/types": "30.4.1", @@ -1945,6 +2029,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/test-result": "30.4.1", "graceful-fs": "^4.2.11", @@ -1959,6 +2044,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/core": "^7.27.4", "@jest/types": "30.4.1", @@ -1983,6 +2069,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/pattern": "30.4.0", "@jest/schemas": "30.4.1", @@ -2000,6 +2087,7 @@ "version": "0.3.13", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" @@ -2009,6 +2097,7 @@ "version": "2.3.5", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" @@ -3243,10 +3332,9 @@ }, "node_modules/@map-colonies/schemas": { "version": "1.26.0", - "resolved": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-9687e6cb21fe29d10c52aaf9db64a763050ff519.tgz", - "integrity": "sha512-HvWTHuAd+HcvfiNQWPsmsF/XUQ8ln/DZOeau/az+WvQq8A84mMi9WGDdESTvVTP2FqP04c0zQmdunn8NcjNsDw==", - "license": "MIT", - "peer": true + "resolved": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-11bf4605d7a17c723dd85f624c88fe32d9957863.tgz", + "integrity": "sha512-jGavRY1lGItcyyNlaO/Sa4hpV1TZ15NXH2vFbcZnNlNmHtFtU3ZnGqLpRZtSOxAbBnQY7xga4LRGZpZea+c0VQ==", + "license": "MIT" }, "node_modules/@map-colonies/tracing": { "version": "1.0.0", @@ -4126,7 +4214,6 @@ "version": "4.2.4", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@octokit/auth-token": "^3.0.0", "@octokit/graphql": "^5.0.0", @@ -4312,7 +4399,6 @@ "node_modules/@opentelemetry/api": { "version": "1.9.1", "license": "Apache-2.0", - "peer": true, "engines": { "node": ">=8.0.0" } @@ -4402,7 +4488,6 @@ "node_modules/@opentelemetry/core": { "version": "2.9.0", "license": "Apache-2.0", - "peer": true, "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, @@ -6296,6 +6381,7 @@ "node_modules/@redocly/ajv": { "version": "8.11.2", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "json-schema-traverse": "^1.0.0", @@ -6322,11 +6408,13 @@ }, "node_modules/@redocly/config": { "version": "0.22.0", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@redocly/openapi-core": { "version": "1.34.17", "license": "MIT", + "peer": true, "dependencies": { "@redocly/ajv": "8.11.2", "@redocly/config": "0.22.0", @@ -6346,17 +6434,20 @@ "node_modules/@redocly/openapi-core/node_modules/agent-base": { "version": "7.1.4", "license": "MIT", + "peer": true, "engines": { "node": ">= 14" } }, "node_modules/@redocly/openapi-core/node_modules/balanced-match": { "version": "1.0.2", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@redocly/openapi-core/node_modules/brace-expansion": { "version": "2.1.2", "license": "MIT", + "peer": true, "dependencies": { "balanced-match": "^1.0.0" } @@ -6364,6 +6455,7 @@ "node_modules/@redocly/openapi-core/node_modules/https-proxy-agent": { "version": "7.0.6", "license": "MIT", + "peer": true, "dependencies": { "agent-base": "^7.1.2", "debug": "4" @@ -6385,6 +6477,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "argparse": "^2.0.1" }, @@ -6395,6 +6488,7 @@ "node_modules/@redocly/openapi-core/node_modules/minimatch": { "version": "5.1.9", "license": "ISC", + "peer": true, "dependencies": { "brace-expansion": "^2.0.1" }, @@ -6598,12 +6692,14 @@ "node_modules/@sinclair/typebox": { "version": "0.34.52", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@sinonjs/commons": { "version": "3.0.1", "dev": true, "license": "BSD-3-Clause", + "peer": true, "dependencies": { "type-detect": "4.0.8" } @@ -6612,6 +6708,7 @@ "version": "15.4.0", "dev": true, "license": "BSD-3-Clause", + "peer": true, "dependencies": { "@sinonjs/commons": "^3.0.1" } @@ -6786,6 +6883,7 @@ "version": "7.20.5", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/parser": "^7.20.7", "@babel/types": "^7.20.7", @@ -6798,6 +6896,7 @@ "version": "7.27.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/types": "^7.0.0" } @@ -6806,6 +6905,7 @@ "version": "7.4.4", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/parser": "^7.1.0", "@babel/types": "^7.0.0" @@ -6815,6 +6915,7 @@ "version": "7.28.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/types": "^7.28.2" } @@ -6882,7 +6983,6 @@ "node_modules/@types/express": { "version": "4.17.25", "license": "MIT", - "peer": true, "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^4.17.33", @@ -6927,8 +7027,7 @@ }, "node_modules/@types/json-schema": { "version": "7.0.15", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/lodash": { "version": "4.17.24", @@ -6984,10 +7083,12 @@ } }, "node_modules/@types/node": { - "version": "26.1.1", + "version": "24.13.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", + "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", "license": "MIT", "dependencies": { - "undici-types": "~8.3.0" + "undici-types": "~7.18.0" } }, "node_modules/@types/node-cron": { @@ -7109,7 +7210,8 @@ "node_modules/@types/stack-utils": { "version": "2.0.3", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@types/superagent": { "version": "8.1.10", @@ -7156,6 +7258,7 @@ "version": "17.0.35", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/yargs-parser": "*" } @@ -7398,7 +7501,8 @@ "node_modules/@ungap/structured-clone": { "version": "1.3.3", "dev": true, - "license": "ISC" + "license": "ISC", + "peer": true }, "node_modules/@unrs/resolver-binding-android-arm-eabi": { "version": "1.12.2", @@ -7776,7 +7880,6 @@ "version": "1.6.23", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "^8.58.0", "@typescript-eslint/utils": "^8.58.0" @@ -7998,7 +8101,6 @@ "node_modules/acorn": { "version": "8.17.0", "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -8080,8 +8182,9 @@ }, "node_modules/ajv": { "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -8356,6 +8459,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/transform": "30.4.1", "@types/babel__core": "^7.20.5", @@ -8376,6 +8480,7 @@ "version": "7.0.1", "dev": true, "license": "BSD-3-Clause", + "peer": true, "workspaces": [ "test/babel-8" ], @@ -8394,6 +8499,7 @@ "version": "30.4.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/babel__core": "^7.20.5" }, @@ -8405,6 +8511,7 @@ "version": "1.2.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/plugin-syntax-async-generators": "^7.8.4", "@babel/plugin-syntax-bigint": "^7.8.3", @@ -8430,6 +8537,7 @@ "version": "30.4.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "babel-plugin-jest-hoist": "30.4.0", "babel-preset-current-node-syntax": "^1.2.0" @@ -8622,7 +8730,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.42", "caniuse-lite": "^1.0.30001803", @@ -8652,6 +8759,7 @@ "version": "2.1.1", "dev": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "node-int64": "^0.4.0" } @@ -8940,6 +9048,7 @@ "version": "1.0.2", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=10" } @@ -9128,6 +9237,7 @@ "version": "4.6.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "iojs": ">= 1.0.0", "node": ">= 0.12.0" @@ -9136,7 +9246,8 @@ "node_modules/collect-v8-coverage": { "version": "1.0.3", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/color-convert": { "version": "2.0.1", @@ -9162,7 +9273,8 @@ }, "node_modules/colorette": { "version": "1.4.0", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/columnify": { "version": "1.6.0", @@ -9194,7 +9306,6 @@ "node_modules/commander": { "version": "14.0.3", "license": "MIT", - "peer": true, "engines": { "node": ">=20" } @@ -10314,7 +10425,6 @@ "version": "9.0.2", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "env-paths": "^2.2.1", "import-fresh": "^3.3.0", @@ -10652,6 +10762,7 @@ "version": "3.1.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=8" } @@ -10761,6 +10872,7 @@ "version": "0.13.1", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -10779,6 +10891,29 @@ "node": ">= 0.8" } }, + "node_modules/encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "license": "MIT", + "optional": true, + "dependencies": { + "iconv-lite": "^0.6.2" + } + }, + "node_modules/encoding/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "optional": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/end-of-stream": { "version": "1.4.5", "license": "MIT", @@ -11038,7 +11173,6 @@ "version": "4.17.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/types": "^8.56.0", "comment-parser": "^1.4.1", @@ -11085,7 +11219,6 @@ "version": "28.14.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/utils": "^6.0.0 || ^7.0.0 || ^8.0.0" }, @@ -11376,6 +11509,7 @@ "version": "0.2.2", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">= 0.8.0" } @@ -11384,6 +11518,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/expect-utils": "30.4.1", "@jest/get-type": "30.1.0", @@ -11412,7 +11547,6 @@ "node_modules/express": { "version": "5.2.1", "license": "MIT", - "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -11873,6 +12007,7 @@ "version": "2.0.2", "dev": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "bser": "2.1.1" } @@ -12294,6 +12429,7 @@ "version": "1.0.0-beta.2", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=6.9.0" } @@ -12341,6 +12477,7 @@ "version": "0.1.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=8.0.0" } @@ -12814,7 +12951,6 @@ "version": "14.0.0", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=18" }, @@ -13228,6 +13364,7 @@ "node_modules/index-to-position": { "version": "1.2.0", "license": "MIT", + "peer": true, "engines": { "node": ">=18" }, @@ -13537,6 +13674,7 @@ "version": "2.1.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=6" } @@ -13723,6 +13861,7 @@ "version": "6.0.3", "dev": true, "license": "BSD-3-Clause", + "peer": true, "dependencies": { "@babel/core": "^7.23.9", "@babel/parser": "^7.23.9", @@ -13751,6 +13890,7 @@ "version": "5.0.6", "dev": true, "license": "BSD-3-Clause", + "peer": true, "dependencies": { "@jridgewell/trace-mapping": "^0.3.23", "debug": "^4.1.1", @@ -13831,6 +13971,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "execa": "^5.1.1", "jest-util": "30.4.1", @@ -13844,6 +13985,7 @@ "version": "30.4.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/environment": "30.4.1", "@jest/expect": "30.4.1", @@ -13874,6 +14016,7 @@ "version": "1.7.2", "dev": true, "license": "MIT", + "peer": true, "peerDependencies": { "babel-plugin-macros": "^3.1.0" }, @@ -13887,6 +14030,7 @@ "version": "30.4.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/core": "30.4.2", "@jest/test-result": "30.4.1", @@ -13918,6 +14062,7 @@ "version": "30.4.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/core": "^7.27.4", "@jest/get-type": "30.1.0", @@ -13966,12 +14111,14 @@ "node_modules/jest-config/node_modules/balanced-match": { "version": "1.0.2", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/jest-config/node_modules/brace-expansion": { "version": "2.1.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "balanced-match": "^1.0.0" } @@ -13980,6 +14127,7 @@ "version": "10.5.0", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", @@ -13999,6 +14147,7 @@ "version": "9.0.9", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "brace-expansion": "^2.0.2" }, @@ -14013,6 +14162,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/diff-sequences": "30.4.0", "@jest/get-type": "30.1.0", @@ -14027,6 +14177,7 @@ "version": "30.4.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "detect-newline": "^3.1.0" }, @@ -14038,6 +14189,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/get-type": "30.1.0", "@jest/types": "30.4.1", @@ -14053,6 +14205,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/environment": "30.4.1", "@jest/fake-timers": "30.4.1", @@ -14078,6 +14231,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/types": "30.4.1", "@types/node": "*", @@ -14101,6 +14255,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/get-type": "30.1.0", "pretty-format": "30.4.1" @@ -14113,6 +14268,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/get-type": "30.1.0", "chalk": "^4.1.2", @@ -14127,6 +14283,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@jest/types": "30.4.1", @@ -14147,6 +14304,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/types": "30.4.1", "@types/node": "*", @@ -14239,6 +14397,7 @@ "version": "1.2.3", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=6" }, @@ -14255,6 +14414,7 @@ "version": "30.4.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } @@ -14263,6 +14423,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "chalk": "^4.1.2", "graceful-fs": "^4.2.11", @@ -14281,6 +14442,7 @@ "version": "30.4.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "jest-regex-util": "30.4.0", "jest-snapshot": "30.4.1" @@ -14293,6 +14455,7 @@ "version": "30.4.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/console": "30.4.1", "@jest/environment": "30.4.1", @@ -14325,6 +14488,7 @@ "version": "30.4.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/environment": "30.4.1", "@jest/fake-timers": "30.4.1", @@ -14356,12 +14520,14 @@ "node_modules/jest-runtime/node_modules/balanced-match": { "version": "1.0.2", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/jest-runtime/node_modules/brace-expansion": { "version": "2.1.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "balanced-match": "^1.0.0" } @@ -14370,6 +14536,7 @@ "version": "10.5.0", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", @@ -14389,6 +14556,7 @@ "version": "9.0.9", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "brace-expansion": "^2.0.2" }, @@ -14403,6 +14571,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/core": "^7.27.4", "@babel/generator": "^7.27.5", @@ -14434,6 +14603,7 @@ "version": "7.8.5", "dev": true, "license": "ISC", + "peer": true, "bin": { "semver": "bin/semver.js" }, @@ -14445,6 +14615,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/types": "30.4.1", "@types/node": "*", @@ -14461,6 +14632,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/get-type": "30.1.0", "@jest/types": "30.4.1", @@ -14477,6 +14649,7 @@ "version": "6.3.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=10" }, @@ -14488,6 +14661,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/test-result": "30.4.1", "@jest/types": "30.4.1", @@ -14506,6 +14680,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/node": "*", "@ungap/structured-clone": "^1.3.0", @@ -14521,6 +14696,7 @@ "version": "8.1.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "has-flag": "^4.0.0" }, @@ -14535,7 +14711,6 @@ "version": "2.6.1", "dev": true, "license": "MIT", - "peer": true, "bin": { "jiti": "lib/jiti-cli.mjs" } @@ -14550,6 +14725,7 @@ "node_modules/js-levenshtein": { "version": "1.1.6", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -15417,6 +15593,7 @@ "version": "5.1.1", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "yallist": "^3.0.2" } @@ -15643,6 +15820,7 @@ "version": "1.0.12", "dev": true, "license": "BSD-3-Clause", + "peer": true, "dependencies": { "tmpl": "1.0.5" } @@ -16442,7 +16620,8 @@ "node_modules/node-int64": { "version": "0.4.0", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/node-readfiles": { "version": "0.2.0", @@ -16827,7 +17006,6 @@ "dev": true, "hasInstallScript": true, "license": "MIT", - "peer": true, "dependencies": { "@nrwl/cli": "15.9.7", "@nrwl/tao": "15.9.7", @@ -17213,12 +17391,12 @@ }, "node_modules/openapi-types": { "version": "12.1.3", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/openapi-typescript": { "version": "7.13.0", "license": "MIT", + "peer": true, "dependencies": { "@redocly/openapi-core": "^1.34.6", "ansi-colors": "^4.1.3", @@ -17237,6 +17415,7 @@ "node_modules/openapi-typescript/node_modules/parse-json": { "version": "8.3.0", "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.26.2", "index-to-position": "^1.1.0", @@ -17252,6 +17431,7 @@ "node_modules/openapi-typescript/node_modules/supports-color": { "version": "10.2.2", "license": "MIT", + "peer": true, "engines": { "node": ">=18" }, @@ -17262,6 +17442,7 @@ "node_modules/openapi-typescript/node_modules/yargs-parser": { "version": "21.1.1", "license": "ISC", + "peer": true, "engines": { "node": ">=12" } @@ -18138,7 +18319,6 @@ "node_modules/pg": { "version": "8.22.0", "license": "MIT", - "peer": true, "dependencies": { "pg-connection-string": "^2.14.0", "pg-pool": "^3.14.0", @@ -18238,7 +18418,6 @@ "node_modules/pino": { "version": "10.3.1", "license": "MIT", - "peer": true, "dependencies": { "@pinojs/redact": "^0.4.0", "atomic-sleep": "^1.0.0", @@ -18345,6 +18524,7 @@ "version": "4.0.7", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">= 6" } @@ -18514,7 +18694,6 @@ "node_modules/prettier": { "version": "3.9.5", "license": "MIT", - "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -18529,6 +18708,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/schemas": "30.4.1", "ansi-styles": "^5.2.0", @@ -18543,6 +18723,7 @@ "version": "5.2.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=10" }, @@ -18619,7 +18800,6 @@ "node_modules/prom-client": { "version": "15.1.3", "license": "Apache-2.0", - "peer": true, "dependencies": { "@opentelemetry/api": "^1.4.0", "tdigest": "^0.1.1" @@ -18755,7 +18935,8 @@ "url": "https://opencollective.com/fast-check" } ], - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/q": { "version": "1.5.1", @@ -18852,13 +19033,15 @@ "name": "react-is", "version": "18.3.1", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/react-is-19": { "name": "react-is", "version": "19.2.7", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/read": { "version": "1.0.7", @@ -20074,6 +20257,7 @@ "version": "2.0.6", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "escape-string-regexp": "^2.0.0" }, @@ -20085,6 +20269,7 @@ "version": "2.0.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=8" } @@ -20146,6 +20331,7 @@ "version": "4.0.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "char-regex": "^1.0.2", "strip-ansi": "^6.0.0" @@ -20275,7 +20461,6 @@ "node_modules/supertest": { "version": "7.2.2", "license": "MIT", - "peer": true, "dependencies": { "cookie-signature": "^1.2.2", "methods": "^1.1.2", @@ -20363,6 +20548,7 @@ "version": "0.11.13", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@pkgr/core": "^0.3.6" }, @@ -20377,6 +20563,7 @@ "version": "0.3.6", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": "^14.18.0 || >=16.0.0" }, @@ -20508,6 +20695,7 @@ "version": "6.0.0", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "@istanbuljs/schema": "^0.1.2", "glob": "^7.1.4", @@ -20520,12 +20708,14 @@ "node_modules/test-exclude/node_modules/balanced-match": { "version": "1.0.2", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/test-exclude/node_modules/brace-expansion": { "version": "1.1.16", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -20535,6 +20725,7 @@ "version": "7.2.3", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -20554,6 +20745,7 @@ "version": "3.1.5", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -20680,7 +20872,8 @@ "node_modules/tmpl": { "version": "1.0.5", "dev": true, - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/to-buffer": { "version": "1.2.2", @@ -21054,6 +21247,7 @@ "version": "4.0.8", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=4" } @@ -21374,7 +21568,9 @@ } }, "node_modules/undici-types": { - "version": "8.3.0", + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", "license": "MIT" }, "node_modules/unique-filename": { @@ -21522,7 +21718,8 @@ }, "node_modules/uri-js-replace": { "version": "1.0.1", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/url-value-parser": { "version": "2.2.0", @@ -21567,6 +21764,7 @@ "version": "9.3.0", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "@jridgewell/trace-mapping": "^0.3.12", "@types/istanbul-lib-coverage": "^2.0.1", @@ -21607,7 +21805,6 @@ "version": "8.1.4", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.5", @@ -21684,7 +21881,6 @@ "version": "4.1.10", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@vitest/expect": "4.1.10", "@vitest/mocker": "4.1.10", @@ -21778,6 +21974,7 @@ "version": "1.0.8", "dev": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "makeerror": "1.0.12" } @@ -22105,7 +22302,8 @@ "node_modules/yallist": { "version": "3.1.1", "dev": true, - "license": "ISC" + "license": "ISC", + "peer": true }, "node_modules/yaml": { "version": "2.9.0", @@ -22122,7 +22320,8 @@ }, "node_modules/yaml-ast-parser": { "version": "0.0.43", - "license": "Apache-2.0" + "license": "Apache-2.0", + "peer": true }, "node_modules/yargs": { "version": "17.7.3", @@ -22576,6 +22775,7 @@ "@map-colonies/tracing-utils": "^1.0.0", "@opentelemetry/api": "^1.9.1", "@smithy/node-http-handler": "^4.7.6", + "ajv": "^8.20.0", "axios": "^1.7.0", "express": "^5.2.1", "node-cron": "^4.2.1", @@ -22588,7 +22788,7 @@ }, "devDependencies": { "@types/nock": "^10.0.3", - "@types/node": "^20.11.0", + "@types/node": "^24.0.0", "@types/node-cron": "^3.0.11", "@vitest/coverage-v8": "^4.0.18", "@vitest/ui": "^4.0.18", @@ -22604,19 +22804,6 @@ "peerDependencies": { "typeorm": "^0.3.20" } - }, - "packages/synchronizer/node_modules/@types/node": { - "version": "20.19.43", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~6.21.0" - } - }, - "packages/synchronizer/node_modules/undici-types": { - "version": "6.21.0", - "dev": true, - "license": "MIT" } } } diff --git a/package.json b/package.json index 89ad48f..2b3b9aa 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,9 @@ "lerna": "^6.6.2" }, "dependencies": { - "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-9687e6cb21fe29d10c52aaf9db64a763050ff519.tgz" + "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-11bf4605d7a17c723dd85f624c88fe32d9957863.tgz" + }, + "overrides": { + "@types/node": "^24.0.0" } } diff --git a/packages/synchronizer/config/default.json b/packages/synchronizer/config/default.json index 9067f03..34d28d4 100644 --- a/packages/synchronizer/config/default.json +++ b/packages/synchronizer/config/default.json @@ -3,27 +3,11 @@ "offlineMode": true, "name": "standard-synchronizer" }, - "layersFile": "./config/layers.json", "aliasesFile": "./config/aliases.json", "typeMapFile": "./config/typeMap.json", "schedule": "*/1 * * * *", "indexNameFormat": "{layerName}_{column}_idx", - "enrichment": { - "enabled": false - }, "dbs": { - "source": { - "type": "postgres", - "host": "localhost", - "port": 5432, - "username": "postgres", - "password": "postgres", - "database": "postgres", - "ssl": { - "enabled": false - }, - "schema": "query" - }, "destination": { "type": "postgres", "host": "localhost", @@ -41,15 +25,39 @@ "accessKeyId": "key", "secretAccessKey": "keySecret", "forcePathStyle": true, - "region": "local", - "bucket": "vector-query-example", - "fileName": "mock-query.lua", - "layersVariable": "layers", - "aliasFieldName": "alias", - "idFieldName": "id", - "nameFieldName": "name" + "region": "local" } }, + "namespaces": [ + { + "name": "default", + "bucket": "vector-query-example", + "layersFile": "./config/layers/default.json", + "db": { + "type": "postgres", + "host": "localhost", + "port": 5432, + "username": "postgres", + "password": "postgres", + "database": "postgres", + "ssl": { + "enabled": false + }, + "schema": "query" + }, + "layerSource": { + "type": "sharedLua", + "fileName": "mock-query.lua", + "layersVariable": "layers", + "aliasFieldName": "alias", + "idFieldName": "id", + "nameFieldName": "name", + "enrichment": { + "enabled": false + } + } + } + ], "openapiConfig": { "filePath": "./openapi3.yaml", "basePath": "/docs", diff --git a/packages/synchronizer/config/layers.json b/packages/synchronizer/config/layers/default.json similarity index 100% rename from packages/synchronizer/config/layers.json rename to packages/synchronizer/config/layers/default.json diff --git a/packages/synchronizer/config/layers/test.json b/packages/synchronizer/config/layers/test.json new file mode 100644 index 0000000..23709ec --- /dev/null +++ b/packages/synchronizer/config/layers/test.json @@ -0,0 +1,7 @@ +[ + { + "layerName": "buildings_polygon", + "excludeProperties": ["id"], + "enums": ["code", "attribute"] + } +] diff --git a/packages/synchronizer/config/test.json b/packages/synchronizer/config/test.json index 1884e75..8d2d4ee 100644 --- a/packages/synchronizer/config/test.json +++ b/packages/synchronizer/config/test.json @@ -1,17 +1,41 @@ { "dbs": { - "source": { - "schema": "test_source" - }, "destination": { "schema": "test_dest" } }, - "enrichmentApi": { - "enabled": true, - "api": "https://example.com/{layerName}", - "propertiesPath": "fields_list", - "aliasField": "display_name", - "requestTimeoutMilliseconds": 10000 - } + "namespaces": [ + { + "name": "test", + "bucket": "vector-query-example", + "layersFile": "./config/layers/test.json", + "db": { + "type": "postgres", + "host": "localhost", + "port": 5432, + "username": "postgres", + "password": "postgres", + "database": "postgres", + "ssl": { + "enabled": false + }, + "schema": "test_source" + }, + "layerSource": { + "type": "sharedLua", + "fileName": "mock-query.lua", + "layersVariable": "layers", + "aliasFieldName": "alias", + "idFieldName": "id", + "nameFieldName": "name", + "enrichment": { + "enabled": true, + "api": "https://example.com/{layerName}", + "propertiesPath": "fields_list", + "aliasField": "display_name", + "requestTimeoutMilliseconds": 10000 + } + } + } + ] } diff --git a/packages/synchronizer/package.json b/packages/synchronizer/package.json index 776d0d0..9d35951 100644 --- a/packages/synchronizer/package.json +++ b/packages/synchronizer/package.json @@ -9,7 +9,7 @@ "start": "npm run build && node dist/src/index.js", "start:dev": "npm run build && cd dist && cross-env CONFIG_OFFLINE_MODE=true node --enable-source-maps --import ./instrumentation.mjs ./src/index.js", "dev": "ts-node src/index.ts", - "assets:copy": "copyfiles -f ./config/* ./dist/config && copyfiles -f copyfiles ./package.json dist", + "assets:copy": "copyfiles -u 1 ./config/**/* ./dist/config && copyfiles -f copyfiles ./package.json dist", "watch": "ts-node-dev --respawn src/index.ts", "test:integration": "vitest run --coverage.enabled=false --project integration", "test": "vitest run", @@ -32,6 +32,7 @@ "@map-colonies/tracing-utils": "^1.0.0", "@opentelemetry/api": "^1.9.1", "@smithy/node-http-handler": "^4.7.6", + "ajv": "^8.20.0", "axios": "^1.7.0", "express": "^5.2.1", "node-cron": "^4.2.1", @@ -44,7 +45,7 @@ }, "devDependencies": { "@types/nock": "^10.0.3", - "@types/node": "^20.11.0", + "@types/node": "^24.0.0", "@types/node-cron": "^3.0.11", "@vitest/coverage-v8": "^4.0.18", "@vitest/ui": "^4.0.18", diff --git a/packages/synchronizer/src/common/config.ts b/packages/synchronizer/src/common/config.ts index b472e6d..17bd753 100644 --- a/packages/synchronizer/src/common/config.ts +++ b/packages/synchronizer/src/common/config.ts @@ -1,8 +1,8 @@ import { type ConfigInstance, config } from '@map-colonies/config'; -import { vectorVectorStandardSynchronizerV3, type vectorVectorStandardSynchronizerV3Type } from '@map-colonies/schemas'; +import { vectorVectorStandardSynchronizerV4, type vectorVectorStandardSynchronizerV4Type } from '@map-colonies/schemas'; // Choose here the type of the config instance and import this type from the entire application -type ConfigType = ConfigInstance; +type ConfigType = ConfigInstance; let configInstance: ConfigType | undefined; @@ -13,7 +13,7 @@ let configInstance: ConfigType | undefined; */ async function initConfig(offlineMode?: boolean): Promise { configInstance = await config({ - schema: vectorVectorStandardSynchronizerV3, + schema: vectorVectorStandardSynchronizerV4, offlineMode, }); } diff --git a/packages/synchronizer/src/common/constants.ts b/packages/synchronizer/src/common/constants.ts index 0e6c984..cf85c5c 100644 --- a/packages/synchronizer/src/common/constants.ts +++ b/packages/synchronizer/src/common/constants.ts @@ -20,10 +20,7 @@ export const HEALTHCHECK = Symbol('HealthCheck'); export const s3ConfigPath = 'dbs.s3'; -export const DB_CONFIGS = [ - { configKey: 'dbs.source', token: SOURCE_DATA_SOURCE_PROVIDER }, - { configKey: 'dbs.destination', token: DESTINATION_DATA_SOURCE_PROVIDER }, -] as const; +export const DESTINATION_DB_CONFIG_PATH = 'dbs.destination'; export const REPOSITORIES = [ { entity: Layer, token: LAYER_REPOSITORY_SYMBOL }, @@ -41,3 +38,7 @@ export const SERVICES = { CLEANUP_REGISTRY: Symbol('CleanupRegistry'), } satisfies Record; /* eslint-enable @typescript-eslint/naming-convention */ + +export { DESTINATION_DATA_SOURCE_PROVIDER }; + +export const NAMESPACE_HANDLES = Symbol('NamespaceHandles'); diff --git a/packages/synchronizer/src/common/db/connection.ts b/packages/synchronizer/src/common/db/connection.ts index 3ae1624..0073c42 100644 --- a/packages/synchronizer/src/common/db/connection.ts +++ b/packages/synchronizer/src/common/db/connection.ts @@ -1,22 +1,17 @@ -import type { FactoryFunction, DependencyContainer } from 'tsyringe'; +import type { FactoryFunction, DependencyContainer, InjectionToken } from 'tsyringe'; import { DataSource } from 'typeorm'; import { createDataSourceOptions, createDataSourceHealthCheck, type DbConfig } from '@db'; import type { HealthCheck } from '@godaddy/terminus'; -import { DB_CONFIGS, SERVICE_NAME, SERVICES } from '../constants'; +import { DESTINATION_DATA_SOURCE_PROVIDER, DESTINATION_DB_CONFIG_PATH, SERVICE_NAME, SERVICES } from '../constants'; import type { ConfigType } from '../config'; -type DbConfigKey = (typeof DB_CONFIGS)[number]['configKey']; +const HEALTHCHECK_TOKENS: InjectionToken[] = [DESTINATION_DATA_SOURCE_PROVIDER]; -export const createDataSourceFactory = - (configKey: DbConfigKey): FactoryFunction => - (container: DependencyContainer): DataSource => { - const config = container.resolve(SERVICES.CONFIG); - const dbConfig: DbConfig = config.get(configKey); - return new DataSource(createDataSourceOptions(dbConfig, SERVICE_NAME)); - }; +export const destinationDataSourceFactory: FactoryFunction = (container: DependencyContainer): DataSource => { + const config = container.resolve(SERVICES.CONFIG); + const dbConfig: DbConfig = config.get(DESTINATION_DB_CONFIG_PATH); + return new DataSource(createDataSourceOptions(dbConfig, SERVICE_NAME)); +}; export const healthCheckFactory: FactoryFunction = (container: DependencyContainer): HealthCheck => - createDataSourceHealthCheck( - container, - DB_CONFIGS.map(({ token }) => token) - ); + createDataSourceHealthCheck(container, HEALTHCHECK_TOKENS); diff --git a/packages/synchronizer/src/common/interfaces.ts b/packages/synchronizer/src/common/interfaces.ts index 9dd65fd..997d12d 100644 --- a/packages/synchronizer/src/common/interfaces.ts +++ b/packages/synchronizer/src/common/interfaces.ts @@ -7,6 +7,12 @@ export interface LayerEnums { layerName: string; enums: string[]; excludeProperties?: string[]; + sourceKey?: string; } -export type EnrichmentConfig = ReturnType['enrichment']; +export type NamespacesConfig = ReturnType['namespaces']; +export type NamespaceConfig = NamespacesConfig[number]; +export type LayerSourceConfig = NamespaceConfig['layerSource']; +export type SharedLuaSourceConfig = Extract; +export type PerLayerJsonSourceConfig = Extract; +export type EnrichmentConfig = SharedLuaSourceConfig['enrichment']; diff --git a/packages/synchronizer/src/common/s3/interfaces.ts b/packages/synchronizer/src/common/s3/interfaces.ts index c3e0e53..7f1bb0d 100644 --- a/packages/synchronizer/src/common/s3/interfaces.ts +++ b/packages/synchronizer/src/common/s3/interfaces.ts @@ -1,6 +1,6 @@ import type { S3ClientConfig } from '@aws-sdk/client-s3'; -import { type vectorVectorStandardSynchronizerV1Type } from '@map-colonies/schemas'; +import { type vectorVectorStandardSynchronizerV4Type } from '@map-colonies/schemas'; -export type BaseS3Config = vectorVectorStandardSynchronizerV1Type['dbs']['s3']; +export type BaseS3Config = vectorVectorStandardSynchronizerV4Type['dbs']['s3']; export type S3Config = BaseS3Config & S3ClientConfig; diff --git a/packages/synchronizer/src/common/s3/s3Repository.ts b/packages/synchronizer/src/common/s3/s3Repository.ts index e14a4bd..291bc41 100644 --- a/packages/synchronizer/src/common/s3/s3Repository.ts +++ b/packages/synchronizer/src/common/s3/s3Repository.ts @@ -8,8 +8,7 @@ import { GetObjectCommand } from '@aws-sdk/client-s3'; import { context as contextAPI } from '@opentelemetry/api'; import { startActivePromisifiedSpan } from '@common/tracing/util'; import { S3Attributes, S3SpanName } from '@common/tracing/s3'; -import { SERVICES, s3ConfigPath } from '@common/constants'; -import type { ConfigType } from '../config'; +import { SERVICES } from '@common/constants'; import { FsRepository } from '../fs/fsRepository'; @injectable() @@ -17,7 +16,6 @@ import { FsRepository } from '../fs/fsRepository'; export class S3Repository { public constructor( @inject(SERVICES.S3_CLIENT) private readonly s3Client: S3Client, - @inject(SERVICES.CONFIG) private readonly config: ConfigType, @inject(SERVICES.LOGGER) private readonly logger: Logger, private readonly fsRepository: FsRepository ) {} @@ -42,29 +40,27 @@ export class S3Repository { ); } - public async downloadFile(): Promise { - const { bucket, fileName } = this.config.get(s3ConfigPath); - + public async downloadFile(bucket: string, key: string): Promise { return startActivePromisifiedSpan( S3SpanName.S3_DOWNLOAD_FILE, - { [S3Attributes.BUCKET]: bucket, [S3Attributes.FILE_NAME]: fileName }, + { [S3Attributes.BUCKET]: bucket, [S3Attributes.FILE_NAME]: key }, contextAPI.active(), async () => { try { - this.logger.info(`Downloading ${fileName} file from S3`); + this.logger.info(`Downloading ${key} file from S3 bucket ${bucket}`); - const body = await this.getObjectWrapper(bucket, fileName); + const body = await this.getObjectWrapper(bucket, key); - const filePath = path.join(__dirname, 'downloads', fileName); + const filePath = path.join(__dirname, 'downloads', bucket, key); await this.fsRepository.mkdir(path.dirname(filePath)); await this.fsRepository.writeFile(filePath, await buffer(body)); - this.logger.info(`${fileName} file was downloaded successfully`); + this.logger.info(`${key} file was downloaded successfully`); return filePath; } catch (err) { - this.logger.error({ msg: `Failed to download ${fileName} file from S3 bucket ${bucket}`, err }); - throw new Error(`Failed to download ${fileName} file from S3`, { cause: err }); + this.logger.error({ msg: `Failed to download ${key} file from S3 bucket ${bucket}`, err }); + throw new Error(`Failed to download ${key} file from S3`, { cause: err }); } } ); diff --git a/packages/synchronizer/src/common/tracing/sync.ts b/packages/synchronizer/src/common/tracing/sync.ts index 0458768..3209699 100644 --- a/packages/synchronizer/src/common/tracing/sync.ts +++ b/packages/synchronizer/src/common/tracing/sync.ts @@ -15,6 +15,7 @@ export type SyncSpanName = (typeof SyncSpanName)[keyof typeof SyncSpanName]; export const SyncAttributes = { LAYER_NAME: 'layer.name', LAYER_ID: 'layer.id', + NAMESPACE: 'namespace', LAYERS_COUNT: 'layers.count', LAYERS_CHANGED: 'layers.changed', PROPERTIES_AFFECTED: 'properties.affected', diff --git a/packages/synchronizer/src/containerConfig.ts b/packages/synchronizer/src/containerConfig.ts index 6fc99fb..99def09 100644 --- a/packages/synchronizer/src/containerConfig.ts +++ b/packages/synchronizer/src/containerConfig.ts @@ -12,11 +12,14 @@ import type { S3Client } from '@aws-sdk/client-s3'; import { ListBucketsCommand } from '@aws-sdk/client-s3'; import { type ConfigType, getConfig } from '@common/config'; import { type InjectionObject, registerDependencies } from '@common/dependencyRegistration'; -import { DB_CONFIGS, HEALTHCHECK, ON_SIGNAL, REPOSITORIES, SERVICES, SERVICE_NAME } from '@common/constants'; +import { HEALTHCHECK, NAMESPACE_HANDLES, ON_SIGNAL, REPOSITORIES, SERVICES, SERVICE_NAME } from '@common/constants'; import { getTracing } from '@common/tracing'; -import { createDataSourceFactory, healthCheckFactory } from './common/db/connection'; +import { destinationDataSourceFactory, healthCheckFactory } from './common/db/connection'; import { CRON_MANAGER_SYMBOL, CronManager } from './sync/cron'; import { s3ClientFactory } from './common/s3'; +import { S3Repository } from './common/s3/s3Repository'; +import { FsRepository } from './common/fs/fsRepository'; +import { buildNamespaceHandle } from './sync/namespaceHandle'; export interface RegisterOptions { override?: InjectionObject[]; @@ -92,26 +95,32 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise }); }, }, - ...DB_CONFIGS.map(({ configKey, token }) => { - return { - token, - provider: { - useFactory: instancePerContainerCachingFactory(createDataSourceFactory(configKey)), - }, - postInjectionHook: async (deps: DependencyContainer): Promise => { - const dataSource = deps.resolve(token); + { + token: S3Repository, + provider: { useFactory: instancePerContainerCachingFactory((container) => container.resolve(S3Repository)) }, + postInjectionHook: (deps: DependencyContainer): void => { + const s3Repository = deps.resolve(S3Repository); + deps.register(S3Repository, { useValue: s3Repository }); + }, + }, + { + token: DESTINATION_DATA_SOURCE_PROVIDER, + provider: { useFactory: instancePerContainerCachingFactory(destinationDataSourceFactory) }, + postInjectionHook: async (deps: DependencyContainer): Promise => { + const dataSource = deps.resolve(DESTINATION_DATA_SOURCE_PROVIDER); - if (!dataSource.isInitialized) { - await dataSource.initialize(); - } + if (!dataSource.isInitialized) { + await dataSource.initialize(); + } - cleanupRegistry.register({ - id: token, - func: dataSource.destroy.bind(dataSource), - }); - }, - }; - }), + deps.register(DESTINATION_DATA_SOURCE_PROVIDER, { useValue: dataSource }); + + cleanupRegistry.register({ + id: DESTINATION_DATA_SOURCE_PROVIDER, + func: dataSource.destroy.bind(dataSource), + }); + }, + }, ...REPOSITORIES.map(({ token, entity }) => ({ token, provider: { @@ -121,6 +130,20 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise }, }, })), + { + token: NAMESPACE_HANDLES, + provider: { + useFactory: instancePerContainerCachingFactory((container) => { + const config = container.resolve(SERVICES.CONFIG); + const logger = container.resolve(SERVICES.LOGGER); + const s3Repository = container.resolve(S3Repository); + const fsRepository = container.resolve(FsRepository); + return config + .get('namespaces') + .map((namespaceConfig) => buildNamespaceHandle(container, namespaceConfig, s3Repository, fsRepository, logger, cleanupRegistry)); + }), + }, + }, { token: CRON_MANAGER_SYMBOL, provider: { useClass: CronManager }, diff --git a/packages/synchronizer/src/sync/SyncModel.ts b/packages/synchronizer/src/sync/SyncModel.ts index 7de50f4..c5e4627 100644 --- a/packages/synchronizer/src/sync/SyncModel.ts +++ b/packages/synchronizer/src/sync/SyncModel.ts @@ -1,16 +1,14 @@ -import { readFile } from 'node:fs/promises'; import { inject, injectable } from 'tsyringe'; import { DataSource, Repository } from 'typeorm'; -import { ENUMS_REPOSITORY_SYMBOL, EnumValue, Layer, LAYER_REPOSITORY_SYMBOL, Property, PROPERTY_REPOSITORY_SYMBOL } from '@db'; +import { ENUMS_REPOSITORY_SYMBOL, EnumValue, Layer, LAYER_REPOSITORY_SYMBOL, LayerSource, Property, PROPERTY_REPOSITORY_SYMBOL } from '@db'; import { Logger } from '@map-colonies/js-logger'; import { context as contextAPI } from '@opentelemetry/api'; import { startActivePromisifiedSpan } from '@common/tracing/util'; import { SyncAttributes, SyncSpanName } from '@common/tracing/sync'; -import { SERVICES, SOURCE_DATA_SOURCE_PROVIDER, s3ConfigPath } from '@common/constants'; +import { SERVICES, SOURCE_DATA_SOURCE_PROVIDER } from '@common/constants'; import { ConfigType } from '@common/config'; import { InsertPropertyDTO, LayerEnums } from '@src/common/interfaces'; -import { S3Repository } from '@src/common/s3/s3Repository'; import { CaseInsensitiveMap } from '@src/common/caseInsensitiveMap'; import { columnInfosToProperties, excludedPropertySet, schemaOf } from './helpers'; import { @@ -24,10 +22,8 @@ import { StalePropertiesDeletionError, TableColumnsQueryError, } from './errors'; -import { fetchPropertyAliases } from './aliasEnricher'; -import type { FileAliases } from './fileReader'; +import { resolveFileAliases, type FileAliases } from './fileReader'; import type { TypeMap } from './typeMap'; -import { LuaLayer, parseLuaLayers } from './luaParser'; export interface ColumnInfo { columnName: string; @@ -42,25 +38,24 @@ export class SyncModel { @inject(SOURCE_DATA_SOURCE_PROVIDER) private readonly sourceDataSource: DataSource, @inject(LAYER_REPOSITORY_SYMBOL) private readonly layerRepository: Repository, @inject(PROPERTY_REPOSITORY_SYMBOL) private readonly propertyRepository: Repository, - @inject(ENUMS_REPOSITORY_SYMBOL) private readonly enumsRepository: Repository, - private readonly luaRepository: S3Repository + @inject(ENUMS_REPOSITORY_SYMBOL) private readonly enumsRepository: Repository ) {} - public async syncLayer(layerName: string, layerId: number, alias?: string): Promise { + public async syncLayer(namespace: string, layerName: string, layerId: number | null, source: LayerSource, alias?: string): Promise { return startActivePromisifiedSpan( SyncSpanName.SYNC_LAYER, - { [SyncAttributes.LAYER_NAME]: layerName, [SyncAttributes.LAYER_ID]: layerId }, + { [SyncAttributes.LAYER_NAME]: layerName, [SyncAttributes.NAMESPACE]: namespace }, contextAPI.active(), async () => { try { await this.layerRepository .createQueryBuilder() .insert() - .values({ layerName, layerId, alias: alias ?? layerName }) - .orIgnore() + .values({ namespace, layerName, layerId, source, alias: alias ?? layerName }) + .orUpdate(['layer_id', 'source', 'alias'], ['namespace', 'layer_name']) .execute(); } catch (err) { - this.logger.error({ msg: `Failed to sync layer ${layerName} (${layerId})`, err }); + this.logger.error({ msg: `Failed to sync layer ${namespace}/${layerName}`, err }); throw new LayerSyncError(layerName, layerId, err); } } @@ -102,7 +97,6 @@ export class SyncModel { const schema = schemaOf(this.sourceDataSource); const qualifiedTable = `"${schema}"."${layerName}"`; - // Recursive CTE function const cteParts = enums.map( (col) => `cte_${col} AS ( SELECT MIN("${col}") AS value FROM ${qualifiedTable} @@ -136,7 +130,7 @@ export class SyncModel { public async upsertProperties(properties: InsertPropertyDTO[]): Promise { return startActivePromisifiedSpan(SyncSpanName.UPSERT_PROPERTIES, {}, contextAPI.active(), async (span) => { try { - const result = await this.propertyRepository.upsert(properties, ['layerName', 'property']); + const result = await this.propertyRepository.upsert(properties, ['namespace', 'layerName', 'property']); span.setAttribute(SyncAttributes.PROPERTIES_AFFECTED, result.identifiers.length); return result.identifiers.length; } catch (err) { @@ -146,10 +140,16 @@ export class SyncModel { }); } - public async syncProperties(layer: LayerEnums, layerId: number, typeMap: TypeMap, fileAliases: FileAliases): Promise { + public async syncProperties( + namespace: string, + layer: LayerEnums, + typeMap: TypeMap, + fileAliases: FileAliases, + sourceAliases: Map + ): Promise { return startActivePromisifiedSpan( SyncSpanName.SYNC_PROPERTIES, - { [SyncAttributes.LAYER_NAME]: layer.layerName, [SyncAttributes.LAYER_ID]: layerId }, + { [SyncAttributes.LAYER_NAME]: layer.layerName, [SyncAttributes.NAMESPACE]: namespace }, contextAPI.active(), async (span) => { const excluded = excludedPropertySet(layer.excludeProperties); @@ -158,25 +158,12 @@ export class SyncModel { this.logger.debug({ layerName: layer.layerName, excludeProperties: layer.excludeProperties }, 'Excluding properties from sync'); } - const properties = columnInfosToProperties(columns, layer.layerName, typeMap, (columnName, udtName) => { + const properties = columnInfosToProperties(columns, namespace, layer.layerName, typeMap, (columnName, udtName) => { this.logger.warn({ layerName: layer.layerName, columnName, udtName }, 'Unknown column type, skipping property'); }); - const enrichment = this.config.get('enrichment'); - let apiAliases = new Map(); - if (enrichment.enabled) { - try { - apiAliases = await fetchPropertyAliases(layer.layerName, layerId, enrichment); - } catch (err) { - this.logger.warn( - { layerName: layer.layerName, layerId, err }, - 'Failed to fetch property aliases from enrichment API, continuing without them' - ); - } - } - const globalAliases = fileAliases.get('*') ?? new Map(); - const layerAliases = fileAliases.get(layer.layerName) ?? new Map(); - const aliases = new CaseInsensitiveMap([...apiAliases, ...globalAliases, ...layerAliases]); + const fileLayerAliases = resolveFileAliases(fileAliases, namespace, layer.layerName); + const aliases = new CaseInsensitiveMap([...sourceAliases, ...fileLayerAliases]); for (const property of properties) { const alias = aliases.get(property.property); @@ -192,6 +179,7 @@ export class SyncModel { affected += await this.upsertProperties(withoutAlias); } await this.deleteStaleProperties( + namespace, layer.layerName, properties.map((p) => p.property) ); @@ -201,28 +189,42 @@ export class SyncModel { ); } - public async deleteStaleProperties(layerName: string, currentPropertyNames: string[]): Promise { - const qb = this.propertyRepository.createQueryBuilder().delete().where('"layer_name" = :layerName', { layerName }); + public async deleteStaleProperties(namespace: string, layerName: string, currentPropertyNames: string[]): Promise { + const qb = this.propertyRepository + .createQueryBuilder() + .delete() + .where('"namespace" = :namespace', { namespace }) + .andWhere('"layer_name" = :layerName', { layerName }); if (currentPropertyNames.length > 0) { qb.andWhere('"property" NOT IN (:...properties)', { properties: currentPropertyNames }); } try { await qb.execute(); } catch (err) { - this.logger.error({ msg: `Failed to delete stale properties for ${layerName}`, err }); + this.logger.error({ msg: `Failed to delete stale properties for ${namespace}/${layerName}`, err }); throw new StalePropertiesDeletionError(layerName, err); } } - public async deleteLayersNotIn(layerNames: string[]): Promise { + public async deleteLayersNotIn(namespace: string, layerNames: string[]): Promise { if (layerNames.length === 0) { return; } try { - await this.propertyRepository.createQueryBuilder().delete().where('"layer_name" NOT IN (:...layerNames)', { layerNames }).execute(); - await this.layerRepository.createQueryBuilder().delete().where('"layer_name" NOT IN (:...layerNames)', { layerNames }).execute(); + await this.propertyRepository + .createQueryBuilder() + .delete() + .where('"namespace" = :namespace', { namespace }) + .andWhere('"layer_name" NOT IN (:...layerNames)', { layerNames }) + .execute(); + await this.layerRepository + .createQueryBuilder() + .delete() + .where('"namespace" = :namespace', { namespace }) + .andWhere('"layer_name" NOT IN (:...layerNames)', { layerNames }) + .execute(); } catch (err) { - this.logger.error({ msg: `Failed to delete layers not in [${layerNames.join(', ')}]`, err }); + this.logger.error({ msg: `Failed to delete layers not in [${layerNames.join(', ')}] for namespace ${namespace}`, err }); throw new LayersDeletionError(layerNames, err); } } @@ -247,25 +249,19 @@ export class SyncModel { } } - public async luaFileData(): Promise> { - return startActivePromisifiedSpan(SyncSpanName.LOAD_LUA_DATA, {}, contextAPI.active(), async () => { - this.logger.debug('Loading lua query data'); - const filePath = await this.luaRepository.downloadFile(); - const content = await readFile(filePath, 'utf-8'); - const { layersVariable, idFieldName, nameFieldName, aliasFieldName } = this.config.get(s3ConfigPath); - return parseLuaLayers(content, layersVariable, idFieldName, nameFieldName, aliasFieldName); - }); - } - - public async deleteStaleEnumValues(layerName: string, currentEnums: string[]): Promise { - const qb = this.enumsRepository.createQueryBuilder().delete().where('"layer_name" = :layerName', { layerName }); + public async deleteStaleEnumValues(namespace: string, layerName: string, currentEnums: string[]): Promise { + const qb = this.enumsRepository + .createQueryBuilder() + .delete() + .where('"namespace" = :namespace', { namespace }) + .andWhere('"layer_name" = :layerName', { layerName }); if (currentEnums.length > 0) { qb.andWhere('"property" NOT IN (:...properties)', { properties: currentEnums }); } try { await qb.execute(); } catch (err) { - this.logger.error({ msg: `Failed to delete stale enum values for ${layerName}`, err }); + this.logger.error({ msg: `Failed to delete stale enum values for ${namespace}/${layerName}`, err }); throw new StaleEnumValuesDeletionError(layerName, err); } } @@ -286,7 +282,7 @@ export class SyncModel { return existing; } - public async syncEnum(layer: LayerEnums): Promise { + public async syncEnum(namespace: string, layer: LayerEnums): Promise { return startActivePromisifiedSpan(SyncSpanName.SYNC_ENUM, { [SyncAttributes.LAYER_NAME]: layer.layerName }, contextAPI.active(), async (span) => { const existingLayer: LayerEnums = { layerName: layer.layerName, enums: await this.existingEnumColumns(layer) }; @@ -294,14 +290,14 @@ export class SyncModel { const enumValuesByColumn = await this.getEnumDistinctValues(existingLayer); const entities = [...enumValuesByColumn.entries()].flatMap(([col, values]) => - values.map((value) => this.enumsRepository.create({ layerName: layer.layerName, property: col, value })) + values.map((value) => this.enumsRepository.create({ namespace, layerName: layer.layerName, property: col, value })) ); let affected: number; try { affected = (await this.enumsRepository.save(entities)).length; } catch (err) { - this.logger.error({ msg: `Failed to save enum values for ${layer.layerName}`, err }); + this.logger.error({ msg: `Failed to save enum values for ${namespace}/${layer.layerName}`, err }); throw new EnumSaveError(layer.layerName, err); } span.setAttribute(SyncAttributes.ENUMS_AFFECTED, affected); diff --git a/packages/synchronizer/src/sync/cron.ts b/packages/synchronizer/src/sync/cron.ts index a601e97..34ef1c7 100644 --- a/packages/synchronizer/src/sync/cron.ts +++ b/packages/synchronizer/src/sync/cron.ts @@ -2,83 +2,101 @@ import { inject, injectable } from 'tsyringe'; import type { Logger } from '@map-colonies/js-logger'; import { schedule, type ScheduledTask } from 'node-cron'; import { context as contextAPI } from '@opentelemetry/api'; -import { SERVICES } from '@common/constants'; +import { SERVICES, NAMESPACE_HANDLES } from '@common/constants'; import { ConfigType } from '@common/config'; import { startActivePromisifiedSpan } from '@common/tracing/util'; -import { SyncAttributes, SyncSpanName } from '@common/tracing/sync'; -import { SyncModel } from './SyncModel'; -import { FileReader } from './fileReader'; +import { SyncSpanName } from '@common/tracing/sync'; +import { FileReader, type FileAliases } from './fileReader'; +import type { TypeMap } from './typeMap'; +import type { NamespaceHandle } from './namespaceHandle'; export const CRON_MANAGER_SYMBOL = Symbol('cronManagerSymbol'); @injectable() export class CronManager { private task: ScheduledTask | undefined; - private isRunning = false; - private lastLayersChecksum: string | undefined; + private readonly lastLayersChecksum = new Map(); public constructor( @inject(SERVICES.LOGGER) private readonly logger: Logger, - @inject(SyncModel) private readonly dal: SyncModel, @inject(SERVICES.CONFIG) private readonly config: ConfigType, + @inject(NAMESPACE_HANDLES) private readonly namespaces: NamespaceHandle[], private readonly fileReader: FileReader ) {} public async tick(): Promise { - if (this.isRunning) { - this.logger.warn('previous sync tick still running, skipping'); - return; - } - this.isRunning = true; - try { - await startActivePromisifiedSpan(SyncSpanName.SYNC_TICK, {}, contextAPI.active(), async (span) => { - const { checksum, layers } = await this.fileReader.readLayersWithChecksum(this.config.get('layersFile')); - const layersChanged = checksum !== this.lastLayersChecksum; - span.setAttribute(SyncAttributes.LAYERS_COUNT, layers.length); - span.setAttribute(SyncAttributes.LAYERS_CHANGED, layersChanged); - - const typeMap = await this.fileReader.readTypeMap(this.config.get('typeMapFile')); - const fileAliases = await this.fileReader.readAliases(this.config.get('aliasesFile')); + await startActivePromisifiedSpan(SyncSpanName.SYNC_TICK, {}, contextAPI.active(), async () => { + const typeMap = await this.fileReader.readTypeMap(this.config.get('typeMapFile')); + const fileAliases = await this.fileReader.readAliases(this.config.get('aliasesFile')); - const luaLayersMap = await this.dal.luaFileData(); - for (const layer of layers) { - const luaLayer = luaLayersMap.get(layer.layerName); - if (luaLayer === undefined) { - this.logger.warn({ layerName: layer.layerName }, 'Layer not found in lua file, skipping'); - continue; - } - await this.dal.syncLayer(layer.layerName, luaLayer.layerId, luaLayer.alias); - try { - const affected = await this.dal.syncProperties(layer, luaLayer.layerId, typeMap, fileAliases); - this.logger.info({ layer, affected }, 'synced properties'); - } catch (err) { - this.logger.warn({ layerName: layer.layerName, err }, 'failed to sync properties, skipping to enum sync'); - } - const enumsAffected = await this.dal.syncEnum(layer); - this.logger.info({ layer, enumsAffected }, 'synced enums'); - if (layersChanged) { - await this.dal.deleteStaleEnumValues(layer.layerName, layer.enums); - } + for (const namespace of this.namespaces) { + try { + await this.tickNamespace(namespace, typeMap, fileAliases); + } catch (err) { + this.logger.error({ namespace: namespace.name, err }, 'namespace sync tick failed, continuing with next namespace'); } - if (layersChanged) { - await this.dal.deleteLayersNotIn(layers.map((l) => l.layerName)); - this.lastLayersChecksum = checksum; - } - }); - } catch (err) { - this.logger.error({ err }, 'sync tick failed, waiting for next tick'); - } finally { - this.isRunning = false; - } + } + }); } public start(): void { - this.task = schedule(this.config.get('schedule'), async () => { - await this.tick(); + this.task = schedule( + this.config.get('schedule'), + async () => { + await this.tick(); + }, + { noOverlap: true } + ); + this.task.on('execution:overlap', () => { + this.logger.warn('previous sync tick still running, skipping this execution'); }); } public async stop(): Promise { await this.task?.stop(); } + + private async tickNamespace(namespace: NamespaceHandle, typeMap: TypeMap, fileAliases: FileAliases): Promise { + if (!namespace.sourceDataSource.isInitialized) { + try { + await namespace.sourceDataSource.initialize(); + } catch (err) { + this.logger.warn({ namespace: namespace.name, err }, 'source database unreachable, skipping namespace for this tick'); + return; + } + } + + const { checksum, layers } = await this.fileReader.readLayersWithChecksum(namespace.layersFile); + const layersChanged = checksum !== this.lastLayersChecksum.get(namespace.name); + + const records = await namespace.layerSource.tick(layers); + + for (const layer of layers) { + const record = records.get(layer.layerName); + if (record === undefined) { + continue; + } + + await namespace.dal.syncLayer(namespace.name, layer.layerName, record.layerId ?? null, record.source, record.alias); + try { + const affected = await namespace.dal.syncProperties(namespace.name, layer, typeMap, fileAliases, record.propertyAliases); + this.logger.info({ namespace: namespace.name, layer, affected }, 'synced properties'); + } catch (err) { + this.logger.warn({ namespace: namespace.name, layerName: layer.layerName, err }, 'failed to sync properties, skipping to enum sync'); + } + const enumsAffected = await namespace.dal.syncEnum(namespace.name, layer); + this.logger.info({ namespace: namespace.name, layer, enumsAffected }, 'synced enums'); + if (layersChanged) { + await namespace.dal.deleteStaleEnumValues(namespace.name, layer.layerName, layer.enums); + } + } + + if (layersChanged) { + await namespace.dal.deleteLayersNotIn( + namespace.name, + layers.map((l) => l.layerName) + ); + this.lastLayersChecksum.set(namespace.name, checksum); + } + } } diff --git a/packages/synchronizer/src/sync/errors.ts b/packages/synchronizer/src/sync/errors.ts index 3d76d0b..54b33dd 100644 --- a/packages/synchronizer/src/sync/errors.ts +++ b/packages/synchronizer/src/sync/errors.ts @@ -7,7 +7,7 @@ abstract class SyncError extends Error { } export class LayerSyncError extends SyncError { - public constructor(layerName: string, layerId: number, cause?: unknown) { + public constructor(layerName: string, layerId: number | null, cause?: unknown) { super(`Failed to sync layer ${layerName} (${layerId})`, { cause }); } } @@ -65,3 +65,15 @@ export class EnumSaveError extends SyncError { super(`Failed to save enum values for ${layerName}`, { cause }); } } + +export class LayerJsonFetchError extends SyncError { + public constructor(layerName: string, key: string, cause?: unknown) { + super(`Failed to fetch layer JSON for ${layerName} at key ${key}`, { cause }); + } +} + +export class LayerJsonParseError extends SyncError { + public constructor(layerName: string, key: string, cause?: unknown) { + super(`Failed to parse layer JSON for ${layerName} at key ${key}`, { cause }); + } +} diff --git a/packages/synchronizer/src/sync/fileReader.ts b/packages/synchronizer/src/sync/fileReader.ts index cc1d368..b3709e4 100644 --- a/packages/synchronizer/src/sync/fileReader.ts +++ b/packages/synchronizer/src/sync/fileReader.ts @@ -6,9 +6,21 @@ import type { LayerEnums } from '@common/interfaces'; import { parseTypeMap, type TypeMap } from './typeMap'; import { TypeMapError } from './errors'; -type AliasesFile = Record>; +const ALL = '*'; -export type FileAliases = Map>; +type AliasesFile = Record>>; + +export type FileAliases = Map>>; + +export const resolveFileAliases = (fileAliases: FileAliases, namespace: string, layerName: string): Map => { + const tiers = [ + fileAliases.get(ALL)?.get(ALL), + fileAliases.get(ALL)?.get(layerName), + fileAliases.get(namespace)?.get(ALL), + fileAliases.get(namespace)?.get(layerName), + ]; + return new Map(tiers.filter((tier): tier is Map => tier !== undefined).flatMap((tier) => [...tier])); +}; @injectable() export class FileReader { @@ -38,12 +50,19 @@ export class FileReader { } public async readAliases(filePath: string): Promise { + let parsed: AliasesFile; try { const content = await this.fsRepository.readFile(filePath, 'utf-8'); - const parsed = JSON.parse(content.toString()) as AliasesFile; - return new Map(Object.entries(parsed).map(([layer, props]) => [layer, new Map(Object.entries(props))])); + parsed = JSON.parse(content.toString()) as AliasesFile; } catch (err) { throw new Error(`Failed to read aliases from ${filePath}`, { cause: err }); } + + return new Map( + Object.entries(parsed).map(([namespace, layers]) => [ + namespace, + new Map(Object.entries(layers).map(([layer, props]) => [layer, new Map(Object.entries(props))])), + ]) + ); } } diff --git a/packages/synchronizer/src/sync/helpers.ts b/packages/synchronizer/src/sync/helpers.ts index fad307c..516be7e 100644 --- a/packages/synchronizer/src/sync/helpers.ts +++ b/packages/synchronizer/src/sync/helpers.ts @@ -16,7 +16,6 @@ const normalizeUdtName = (rawUdtName: string, typeMap: TypeMap): ColumnType | un return plain; } - // Strip length/precision modifier e.g. "character varying(255)" → "character varying" const withoutModifier = udtName.replace(/\s*\(\d+(?:,\s*\d+)?\)$/, ''); if (withoutModifier !== udtName) { const stripped = typeMap.types.get(withoutModifier); @@ -52,6 +51,7 @@ export const schemaOf = (dataSource: DataSource): string => { export const columnInfosToProperties = ( columnInfos: ColumnInfo[], + namespace: string, layerName: string, typeMap: TypeMap, onUnknown?: (columnName: string, udtName: string) => void @@ -62,7 +62,7 @@ export const columnInfosToProperties = ( if (type === undefined) { onUnknown?.(columnName, udtName); } else { - result.push({ layerName, property: columnName, type }); + result.push({ namespace, layerName, property: columnName, type }); } } return result; diff --git a/packages/synchronizer/src/sync/layerSource/index.ts b/packages/synchronizer/src/sync/layerSource/index.ts new file mode 100644 index 0000000..3e11a8d --- /dev/null +++ b/packages/synchronizer/src/sync/layerSource/index.ts @@ -0,0 +1,20 @@ +import type { Logger } from '@map-colonies/js-logger'; +import type { NamespaceConfig } from '@common/interfaces'; +import type { S3Repository } from '@common/s3/s3Repository'; +import type { FsRepository } from '@common/fs/fsRepository'; +import { SharedLuaSource } from './sharedLuaSource'; +import { PerLayerJsonSource } from './perLayerJsonSource'; +import type { LayerSourceStrategy } from './types'; + +export const createLayerSourceStrategy = ( + namespaceConfig: NamespaceConfig, + s3Repository: S3Repository, + fsRepository: FsRepository, + logger: Logger +): LayerSourceStrategy => { + const { layerSource, bucket } = namespaceConfig; + if (layerSource.type === 'sharedLua') { + return new SharedLuaSource(s3Repository, fsRepository, logger, bucket, layerSource); + } + return new PerLayerJsonSource(s3Repository, fsRepository, logger, bucket, layerSource); +}; diff --git a/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts b/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts new file mode 100644 index 0000000..029ee5c --- /dev/null +++ b/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts @@ -0,0 +1,104 @@ +import ajvCtor from 'ajv'; +import type { Logger } from '@map-colonies/js-logger'; +import { layerSource } from '@db'; +import type { LayerEnums, PerLayerJsonSourceConfig } from '@common/interfaces'; +import { CaseInsensitiveMap } from '@src/common/caseInsensitiveMap'; +import type { FsRepository } from '@common/fs/fsRepository'; +import { LayerJsonFetchError, LayerJsonParseError } from '../errors'; +import type { FileDownloader, LayerSourceRecord, LayerSourceStrategy } from './types'; + +const ajv = new ajvCtor(); +const isRecord = ajv.compile>({ type: 'object' }); +const isString = ajv.compile({ type: 'string' }); +const isArray = ajv.compile({ type: 'array' }); + +interface ParsedDocument { + alias: string; + propertyAliases: Map; +} + +const isRecordEntry = (value: unknown): value is Record => isRecord(value); + +const buildPropertyAliases = (rawFields: unknown[], config: PerLayerJsonSourceConfig): Map => + new CaseInsensitiveMap( + rawFields + .filter(isRecordEntry) + .map((field) => ({ fieldName: field[config.fieldNameField], aliasFieldName: field[config.aliasFieldNameField] })) + .filter((field): field is { fieldName: string; aliasFieldName: string } => isString(field.fieldName) && isString(field.aliasFieldName)) + .map(({ fieldName, aliasFieldName }): [string, string] => [fieldName, aliasFieldName]) + ); + +const parseDocument = (content: string, config: PerLayerJsonSourceConfig): ParsedDocument => { + const raw: unknown = JSON.parse(content); + if (!isRecord(raw)) { + throw new Error('layer JSON must be an object'); + } + + const alias = raw[config.aliasLayerNameField]; + const fields = raw[config.fieldsField]; + if (!isString(alias) || !isArray(fields)) { + throw new Error(`layer JSON must have a string "${config.aliasLayerNameField}" and an array "${config.fieldsField}"`); + } + + return { alias, propertyAliases: buildPropertyAliases(fields, config) }; +}; + +const resolveOne = async ( + layer: LayerEnums, + sourceKey: string, + s3Repository: FileDownloader, + fsRepository: FsRepository, + bucket: string, + config: PerLayerJsonSourceConfig, + logger: Logger +): Promise => { + const key = `${config.prefix}${sourceKey}`; + + let filePath: string; + try { + filePath = await s3Repository.downloadFile(bucket, key); + } catch (err) { + logger.warn({ err: new LayerJsonFetchError(layer.layerName, key, err) }, 'Failed to fetch layer JSON, skipping layer'); + return undefined; + } + + try { + const content = (await fsRepository.readFile(filePath, 'utf-8')).toString(); + const document = parseDocument(content, config); + return { + layerId: undefined, + alias: document.alias, + propertyAliases: document.propertyAliases, + source: layerSource.perLayerJson, + }; + } catch (err) { + logger.warn({ err: new LayerJsonParseError(layer.layerName, key, err) }, 'Failed to read or parse layer JSON, skipping layer'); + return undefined; + } +}; + +export class PerLayerJsonSource implements LayerSourceStrategy { + public constructor( + private readonly s3Repository: FileDownloader, + private readonly fsRepository: FsRepository, + private readonly logger: Logger, + private readonly bucket: string, + private readonly config: PerLayerJsonSourceConfig + ) {} + + public async tick(layers: LayerEnums[]): Promise> { + const records = new Map(); + for (const layer of layers) { + if (layer.sourceKey === undefined) { + this.logger.warn({ layerName: layer.layerName }, 'No sourceKey configured for perLayerJson layer, skipping'); + continue; + } + + const record = await resolveOne(layer, layer.sourceKey, this.s3Repository, this.fsRepository, this.bucket, this.config, this.logger); + if (record !== undefined) { + records.set(layer.layerName, record); + } + } + return records; + } +} diff --git a/packages/synchronizer/src/sync/layerSource/sharedLuaSource.ts b/packages/synchronizer/src/sync/layerSource/sharedLuaSource.ts new file mode 100644 index 0000000..8d98c62 --- /dev/null +++ b/packages/synchronizer/src/sync/layerSource/sharedLuaSource.ts @@ -0,0 +1,62 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { layerSource } from '@db'; +import type { LayerEnums, SharedLuaSourceConfig } from '@common/interfaces'; +import type { FsRepository } from '@common/fs/fsRepository'; +import { fetchPropertyAliases } from '../aliasEnricher'; +import { parseLuaLayers, type LuaLayer } from '../luaParser'; +import type { LayerSourceRecord, FileDownloader, LayerSourceStrategy } from './types'; + +const resolveOne = async (layer: LayerEnums, luaLayer: LuaLayer, config: SharedLuaSourceConfig, logger: Logger): Promise => { + let propertyAliases = new Map(); + if (config.enrichment.enabled) { + try { + propertyAliases = await fetchPropertyAliases(layer.layerName, luaLayer.layerId, config.enrichment); + } catch (err) { + logger.warn( + { layerName: layer.layerName, layerId: luaLayer.layerId, err }, + 'Failed to fetch property aliases from enrichment API, continuing without them' + ); + } + } + + return { + layerId: luaLayer.layerId, + alias: luaLayer.alias, + propertyAliases, + source: layerSource.sharedLua, + }; +}; + +export class SharedLuaSource implements LayerSourceStrategy { + public constructor( + private readonly s3Repository: FileDownloader, + private readonly fsRepository: FsRepository, + private readonly logger: Logger, + private readonly bucket: string, + private readonly config: SharedLuaSourceConfig + ) {} + + public async tick(layers: LayerEnums[]): Promise> { + this.logger.debug('Loading lua query data'); + const filePath = await this.s3Repository.downloadFile(this.bucket, this.config.fileName); + const content = (await this.fsRepository.readFile(filePath, 'utf-8')).toString(); + const luaLayers = parseLuaLayers( + content, + this.config.layersVariable, + this.config.idFieldName, + this.config.nameFieldName, + this.config.aliasFieldName + ); + + const records = new Map(); + for (const layer of layers) { + const luaLayer = luaLayers.get(layer.layerName); + if (luaLayer === undefined) { + this.logger.warn({ layerName: layer.layerName }, 'Layer not found in lua file, skipping'); + continue; + } + records.set(layer.layerName, await resolveOne(layer, luaLayer, this.config, this.logger)); + } + return records; + } +} diff --git a/packages/synchronizer/src/sync/layerSource/types.ts b/packages/synchronizer/src/sync/layerSource/types.ts new file mode 100644 index 0000000..693498f --- /dev/null +++ b/packages/synchronizer/src/sync/layerSource/types.ts @@ -0,0 +1,17 @@ +import type { LayerSource } from '@db'; +import type { LayerEnums } from '@common/interfaces'; + +export interface LayerSourceRecord { + layerId?: number; + alias: string; + propertyAliases: Map; + source: LayerSource; +} + +export interface LayerSourceStrategy { + tick: (layers: LayerEnums[]) => Promise>; +} + +export interface FileDownloader { + downloadFile: (bucket: string, key: string) => Promise; +} diff --git a/packages/synchronizer/src/sync/namespaceHandle.ts b/packages/synchronizer/src/sync/namespaceHandle.ts new file mode 100644 index 0000000..b0725c7 --- /dev/null +++ b/packages/synchronizer/src/sync/namespaceHandle.ts @@ -0,0 +1,47 @@ +import type { Logger } from '@map-colonies/js-logger'; +import type { CleanupRegistry } from '@map-colonies/cleanup-registry'; +import { DataSource } from 'typeorm'; +import type { DependencyContainer } from 'tsyringe/dist/typings/types'; +import { createDataSourceOptions } from '@db'; +import type { NamespaceConfig } from '@common/interfaces'; +import { SERVICE_NAME, SOURCE_DATA_SOURCE_PROVIDER } from '@common/constants'; +import type { S3Repository } from '@common/s3/s3Repository'; +import type { FsRepository } from '@common/fs/fsRepository'; +import { SyncModel } from './SyncModel'; +import { createLayerSourceStrategy } from './layerSource'; +import type { LayerSourceStrategy } from './layerSource/types'; + +export interface NamespaceHandle { + name: string; + sourceDataSource: DataSource; + layerSource: LayerSourceStrategy; + layersFile: string; + dal: SyncModel; +} + +export const buildNamespaceHandle = ( + parent: DependencyContainer, + namespaceConfig: NamespaceConfig, + s3Repository: S3Repository, + fsRepository: FsRepository, + logger: Logger, + cleanupRegistry: CleanupRegistry +): NamespaceHandle => { + const child = parent.createChildContainer(); + const sourceDataSource = new DataSource(createDataSourceOptions(namespaceConfig.db, `${SERVICE_NAME}-${namespaceConfig.name}`)); + child.register(SOURCE_DATA_SOURCE_PROVIDER, { useValue: sourceDataSource }); + + const layerSource = createLayerSourceStrategy(namespaceConfig, s3Repository, fsRepository, logger); + const dal = child.resolve(SyncModel); + + cleanupRegistry.register({ + id: `${SOURCE_DATA_SOURCE_PROVIDER.toString()}:${namespaceConfig.name}`, + func: async () => { + if (sourceDataSource.isInitialized) { + await sourceDataSource.destroy(); + } + }, + }); + + return { name: namespaceConfig.name, sourceDataSource, layerSource, layersFile: namespaceConfig.layersFile, dal }; +}; diff --git a/packages/synchronizer/tests/configurations/globalSetup.ts b/packages/synchronizer/tests/configurations/globalSetup.ts index dbd6b15..ac4afb3 100644 --- a/packages/synchronizer/tests/configurations/globalSetup.ts +++ b/packages/synchronizer/tests/configurations/globalSetup.ts @@ -4,11 +4,15 @@ import { createSslOptions } from '../../../db/dist/db/connection'; export default async function setup(): Promise { await initConfig(true); - const { source, destination } = getConfig().get('dbs'); + const config = getConfig(); + const destination = config.get('dbs.destination'); + const namespaces = config.get('namespaces'); const pgClient = new Client({ ...destination, user: destination.username, ssl: createSslOptions(destination.ssl) }); await pgClient.connect(); await pgClient.query(`DROP SCHEMA IF EXISTS "${destination.schema}" CASCADE`); - await pgClient.query(`DROP SCHEMA IF EXISTS "${source.schema}" CASCADE`); + for (const namespace of namespaces) { + await pgClient.query(`DROP SCHEMA IF EXISTS "${namespace.db.schema}" CASCADE`); + } await pgClient.end(); } diff --git a/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts b/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts index d93b778..1d49775 100644 --- a/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts +++ b/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts @@ -1,11 +1,10 @@ -/* eslint-disable @typescript-eslint/naming-convention */ import { writeFileSync, mkdtempSync, rmSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { jsLogger } from '@map-colonies/js-logger'; -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'; import { trace } from '@opentelemetry/api'; -import nock, { disableNetConnect, enableNetConnect, cleanAll } from 'nock'; +import { disableNetConnect, enableNetConnect, cleanAll } from 'nock'; import type { DataSource, Repository } from 'typeorm'; import type { EnumValue, Layer, Property } from '@map-colonies/vector-standard-db'; import { @@ -14,54 +13,65 @@ import { LAYER_REPOSITORY_SYMBOL, PROPERTY_REPOSITORY_SYMBOL, columnType, + layerSource, } from '@map-colonies/vector-standard-db'; -import { initConfig, getConfig } from '@src/common/config'; +import { initConfig } from '@src/common/config'; import { registerExternalValues } from '@src/containerConfig'; -import { SERVICES, SOURCE_DATA_SOURCE_PROVIDER } from '@src/common/constants'; +import { SERVICES, NAMESPACE_HANDLES } from '@src/common/constants'; import { S3Repository } from '@src/common/s3/s3Repository'; -import { SyncModel } from '@src/sync/SyncModel'; +import type { SyncModel } from '@src/sync/SyncModel'; import { schemaOf } from '@src/sync/helpers'; import { FileReader, type FileAliases } from '@src/sync/fileReader'; import type { TypeMap } from '@src/sync/typeMap'; +import type { NamespaceHandle } from '@src/sync/namespaceHandle'; const TEST_LAYER = 'test_layer'; -const ENRICHMENT_ORIGIN = 'http://mock-api'; +const NAMESPACE = 'test'; const TYPE_MAP_FILE = './config/typeMap.json'; const noAliases: FileAliases = new Map(); - -const mockEnrichmentApi = (responseBody: object, statusCode = 200): nock.Scope => - nock(ENRICHMENT_ORIGIN).get(`/${TEST_LAYER}`).reply(statusCode, responseBody); +const noSourceAliases = new Map(); const createDestinationSchema = async (dataSource: DataSource, schema: string): Promise => { await dataSource.query(`CREATE SCHEMA IF NOT EXISTS "${schema}"`); await dataSource.query(` CREATE TYPE "${schema}"."column_type" AS ENUM('xsd:long', 'xsd:double', 'xsd:boolean', 'xsd:string', 'xsd:dateTime', 'gml:GeometryPropertyType', 'gml:PointPropertyType', 'gml:LineStringPropertyType', 'gml:PolygonPropertyType', 'gml:MultiPointPropertyType', 'gml:MultiLineStringPropertyType', 'gml:MultiPolygonPropertyType') `); + await dataSource.query(` + CREATE TYPE "${schema}"."layer_source" AS ENUM('sharedLua', 'perLayerJson') + `); await dataSource.query(` CREATE TABLE "${schema}"."layer" ( + "namespace" character varying NOT NULL, "layer_name" character varying NOT NULL, - "layer_id" integer NOT NULL, + "layer_id" integer, + "source" "${schema}"."layer_source" NOT NULL, "alias" character varying NOT NULL, - PRIMARY KEY ("layer_name") + PRIMARY KEY ("namespace", "layer_name"), + CONSTRAINT "CHK_layer_source_layer_id" CHECK (("source" = 'sharedLua' AND "layer_id" IS NOT NULL) OR ("source" = 'perLayerJson' AND "layer_id" IS NULL)) ) `); await dataSource.query(` CREATE TABLE "${schema}"."property" ( + "namespace" character varying NOT NULL, "layer_name" character varying NOT NULL, "property" character varying NOT NULL, "type" "${schema}"."column_type" NOT NULL, "alias" character varying, - PRIMARY KEY ("layer_name", "property") + PRIMARY KEY ("namespace", "layer_name", "property"), + FOREIGN KEY ("namespace", "layer_name") + REFERENCES "${schema}"."layer"("namespace", "layer_name") + ON DELETE NO ACTION ON UPDATE NO ACTION ) `); await dataSource.query(` CREATE TABLE "${schema}"."enum_value" ( "value" character varying NOT NULL, + "namespace" character varying NOT NULL, "layer_name" character varying NOT NULL, "property" character varying NOT NULL, - PRIMARY KEY ("value", "layer_name", "property"), - FOREIGN KEY ("layer_name", "property") - REFERENCES "${schema}"."property"("layer_name","property") + PRIMARY KEY ("value", "namespace", "layer_name", "property"), + FOREIGN KEY ("namespace", "layer_name", "property") + REFERENCES "${schema}"."property"("namespace", "layer_name", "property") ON DELETE CASCADE ON UPDATE NO ACTION ) `); @@ -69,7 +79,6 @@ const createDestinationSchema = async (dataSource: DataSource, schema: string): describe('DAL', function () { let dal: SyncModel; - let enrichedDal: SyncModel; let fileReader: FileReader; let typeMap: TypeMap; let sourceDataSource: DataSource; @@ -88,15 +97,23 @@ describe('DAL', function () { override: [ { token: SERVICES.LOGGER, provider: { useValue: await jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, - { token: S3Repository, provider: { useValue: { downloadFile: vi.fn() } } }, + { token: S3Repository, provider: { useValue: { downloadFile: async () => Promise.reject(new Error('not used in this suite')) } } }, ], useChild: true, }); - dal = container.resolve(SyncModel); + const namespaceHandles = container.resolve(NAMESPACE_HANDLES); + const namespace = namespaceHandles.find((n) => n.name === NAMESPACE); + if (namespace === undefined) { + throw new Error(`Namespace ${NAMESPACE} was not configured for this test run`); + } + + dal = namespace.dal; + sourceDataSource = namespace.sourceDataSource; + await sourceDataSource.initialize(); + fileReader = container.resolve(FileReader); typeMap = await fileReader.readTypeMap(TYPE_MAP_FILE); - sourceDataSource = container.resolve(SOURCE_DATA_SOURCE_PROVIDER); destinationDataSource = container.resolve(DESTINATION_DATA_SOURCE_PROVIDER); layerRepository = container.resolve>(LAYER_REPOSITORY_SYMBOL); propertyRepository = container.resolve>(PROPERTY_REPOSITORY_SYMBOL); @@ -107,38 +124,6 @@ describe('DAL', function () { await sourceDataSource.query(`CREATE SCHEMA IF NOT EXISTS "${sourceSchema}"`); await createDestinationSchema(destinationDataSource, destinationSchema); - - const configWithEnrichment = new Proxy(getConfig(), { - get(target, prop) { - if (prop === 'get') { - return (key: string) => { - if (key === 'enrichment') { - return { - enabled: true as const, - api: 'http://mock-api/{layerName}', - propertiesPath: 'fields_list', - aliasField: 'display_name', - requestTimeoutMilliseconds: 10000, - }; - } - return target.get(key); - }; - } - return target[prop as keyof typeof target]; - }, - }); - - const enrichedContainer = await registerExternalValues({ - override: [ - { token: SERVICES.LOGGER, provider: { useValue: await jsLogger({ enabled: false }) } }, - { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, - { token: SERVICES.CONFIG, provider: { useValue: configWithEnrichment } }, - { token: S3Repository, provider: { useValue: { downloadFile: vi.fn() } } }, - ], - useChild: true, - }); - - enrichedDal = enrichedContainer.resolve(SyncModel); }); beforeEach(async function () { @@ -153,6 +138,7 @@ describe('DAL', function () { shape geometry ) `); + await dal.syncLayer(NAMESPACE, TEST_LAYER, 1, layerSource.sharedLua); }); afterEach(async function () { @@ -174,9 +160,9 @@ describe('DAL', function () { describe('Happy Path', function () { describe('syncLayer', function () { it('should insert a layer row with the provided alias', async function () { - await dal.syncLayer('buildings', 1, 'Buildings Layer'); + await dal.syncLayer(NAMESPACE, 'buildings', 1, layerSource.sharedLua, 'Buildings Layer'); - const layer = await layerRepository.findOne({ where: { layerName: 'buildings' } }); + const layer = await layerRepository.findOne({ where: { namespace: NAMESPACE, layerName: 'buildings' } }); expect(layer?.layerName).toBe('buildings'); expect(layer?.layerId).toBe(1); @@ -184,12 +170,21 @@ describe('DAL', function () { }); it('should use layerName as alias when none is provided', async function () { - await dal.syncLayer('buildings', 1); + await dal.syncLayer(NAMESPACE, 'buildings', 1, layerSource.sharedLua); - const layer = await layerRepository.findOne({ where: { layerName: 'buildings' } }); + const layer = await layerRepository.findOne({ where: { namespace: NAMESPACE, layerName: 'buildings' } }); expect(layer?.alias).toBe('buildings'); }); + + it('should accept a null layerId for a perLayerJson layer', async function () { + await dal.syncLayer(NAMESPACE, 'parks', null, layerSource.perLayerJson, 'Parks'); + + const layer = await layerRepository.findOne({ where: { namespace: NAMESPACE, layerName: 'parks' } }); + + expect(layer?.layerId).toBeNull(); + expect(layer?.source).toBe(layerSource.perLayerJson); + }); }); describe('getTableColumns', function () { @@ -245,9 +240,9 @@ describe('DAL', function () { describe('syncProperties', function () { it('should upsert all columns from the source table into the property repository', async function () { - const affected = await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); + const affected = await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, noSourceAliases); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); expect(affected).toBeGreaterThan(0); expect(properties).toEqual( @@ -263,9 +258,9 @@ describe('DAL', function () { }); it('should mark enum columns with columnType.enum', async function () { - await dal.syncProperties({ layerName: TEST_LAYER, enums: ['category', 'name'] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: ['category', 'name'] }, typeMap, noAliases, noSourceAliases); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); const categoryProp = properties.find((p) => p.property === 'category'); const nameProp = properties.find((p) => p.property === 'name'); const heightProp = properties.find((p) => p.property === 'height'); @@ -276,19 +271,25 @@ describe('DAL', function () { }); it('should upsert on subsequent calls without duplicating rows', async function () { - await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); - await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, noSourceAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, noSourceAliases); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); const uniqueProps = new Set(properties.map((p) => p.property)); expect(properties).toHaveLength(uniqueProps.size); }); it('should not sync properties listed in excludeProperties', async function () { - await dal.syncProperties({ layerName: TEST_LAYER, enums: [], excludeProperties: ['height', 'CREATED_AT'] }, 1, typeMap, noAliases); + await dal.syncProperties( + NAMESPACE, + { layerName: TEST_LAYER, enums: [], excludeProperties: ['height', 'CREATED_AT'] }, + typeMap, + noAliases, + noSourceAliases + ); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); const names = properties.map((p) => p.property); expect(names).not.toContain('height'); @@ -297,11 +298,11 @@ describe('DAL', function () { }); it('should delete properties that were synced before being excluded', async function () { - await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, noSourceAliases); - await dal.syncProperties({ layerName: TEST_LAYER, enums: [], excludeProperties: ['height'] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [], excludeProperties: ['height'] }, typeMap, noAliases, noSourceAliases); - const heightProp = await propertyRepository.findOne({ where: { layerName: TEST_LAYER, property: 'height' } }); + const heightProp = await propertyRepository.findOne({ where: { namespace: NAMESPACE, layerName: TEST_LAYER, property: 'height' } }); expect(heightProp).toBeNull(); }); @@ -311,9 +312,9 @@ describe('DAL', function () { ALTER TABLE "${sourceSchema}"."${TEST_LAYER}" ADD COLUMN geom_param geometry(Point,4326) `); - await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, noSourceAliases); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); const geomProp = properties.find((p) => p.property === 'geom_param'); expect(geomProp?.type).toBe(columnType.point); @@ -325,11 +326,11 @@ describe('DAL', function () { await sourceDataSource.query(` INSERT INTO "${sourceSchema}"."${TEST_LAYER}" (category) VALUES ('A'), ('B'), ('A'), ('C') `); - await dal.syncProperties({ layerName: TEST_LAYER, enums: ['category'] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: ['category'] }, typeMap, noAliases, noSourceAliases); - const affected = await dal.syncEnum({ layerName: TEST_LAYER, enums: ['category'] }); + const affected = await dal.syncEnum(NAMESPACE, { layerName: TEST_LAYER, enums: ['category'] }); - const enums = await enumsRepository.find({ where: { layerName: TEST_LAYER, property: 'category' } }); + const enums = await enumsRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER, property: 'category' } }); const values = enums.map((e) => e.value); expect(affected).toBe(3); @@ -337,8 +338,8 @@ describe('DAL', function () { }); it('should create an index on the enum column in the source table', async function () { - await dal.syncProperties({ layerName: TEST_LAYER, enums: ['category'] }, 1, typeMap, noAliases); - await dal.syncEnum({ layerName: TEST_LAYER, enums: ['category'] }); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: ['category'] }, typeMap, noAliases, noSourceAliases); + await dal.syncEnum(NAMESPACE, { layerName: TEST_LAYER, enums: ['category'] }); const indexName = `${TEST_LAYER}_category_idx`; const rows = await sourceDataSource.query<{ indexname: string }[]>( @@ -350,18 +351,28 @@ describe('DAL', function () { }); }); - describe('syncProperties with enrichment', function () { - it('should write aliases from the enrichment API into the property table', async function () { - mockEnrichmentApi({ - fields_list: { - name: { display_name: 'Layer Name', type: 'TEXT' }, - height: { display_name: 'Building Height', type: 'REAL' }, - }, - }); + describe('syncProperties with source and file aliases', function () { + let tmpDir: string; + let aliasesFilePath: string; + + beforeAll(function () { + tmpDir = mkdtempSync(join(tmpdir(), 'dal-aliases-test-')); + aliasesFilePath = join(tmpDir, 'aliases.json'); + }); + + afterAll(function () { + rmSync(tmpDir, { recursive: true }); + }); + + it('should write aliases from the source aliases map into the property table', async function () { + const sourceAliases = new Map([ + ['name', 'Layer Name'], + ['height', 'Building Height'], + ]); - await enrichedDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, sourceAliases); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); const nameProp = properties.find((p) => p.property === 'name'); const heightProp = properties.find((p) => p.property === 'height'); const idProp = properties.find((p) => p.property === 'id'); @@ -372,127 +383,100 @@ describe('DAL', function () { expect(idProp?.alias).toBeUndefined(); }); - it('should call the enrichment API once with the resolved layer URL', async function () { - const scope = mockEnrichmentApi({ fields_list: {} }); - - await enrichedDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); - - expect(scope.isDone()).toBe(true); - }); - }); + it('should apply aliases from the file when there are no source aliases', async function () { + writeFileSync(aliasesFilePath, JSON.stringify({ [NAMESPACE]: { [TEST_LAYER]: { name: 'File Name', height: 'File Height' } } })); - describe('syncProperties with file aliases', function () { - let tmpDir: string; - let aliasesFilePath: string; - let aliasesFileDal: SyncModel; - let bothDal: SyncModel; + await dal.syncProperties( + NAMESPACE, + { layerName: TEST_LAYER, enums: [] }, + typeMap, + await fileReader.readAliases(aliasesFilePath), + noSourceAliases + ); - beforeAll(async function () { - tmpDir = mkdtempSync(join(tmpdir(), 'dal-aliases-test-')); - aliasesFilePath = join(tmpDir, 'aliases.json'); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); - const makeConfig = (enrichmentEnabled: boolean) => - new Proxy(getConfig(), { - get(target, prop) { - if (prop === 'get') { - return (key: string) => { - if (key === 'enrichment') { - return enrichmentEnabled - ? { - enabled: true as const, - api: 'http://mock-api/{layerName}', - propertiesPath: 'fields_list', - aliasField: 'display_name', - requestTimeoutMilliseconds: 10000, - } - : { enabled: false }; - } - return target.get(key); - }; - } - return target[prop as keyof typeof target]; - }, - }); - - const logger = await jsLogger({ enabled: false }); - const s3Mock = { token: S3Repository, provider: { useValue: { downloadFile: vi.fn() } } }; - const sharedOverrides = [ - { token: SERVICES.LOGGER, provider: { useValue: logger } }, - { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, - s3Mock, - ]; - - aliasesFileDal = ( - await registerExternalValues({ - override: [...sharedOverrides, { token: SERVICES.CONFIG, provider: { useValue: makeConfig(false) } }], - useChild: true, - }) - ).resolve(SyncModel); - - bothDal = ( - await registerExternalValues({ - override: [...sharedOverrides, { token: SERVICES.CONFIG, provider: { useValue: makeConfig(true) } }], - useChild: true, - }) - ).resolve(SyncModel); + expect(properties.find((p) => p.property === 'name')?.alias).toBe('File Name'); + expect(properties.find((p) => p.property === 'height')?.alias).toBe('File Height'); + expect(properties.find((p) => p.property === 'id')?.alias).toBeUndefined(); }); - afterAll(function () { - rmSync(tmpDir, { recursive: true }); - }); + it('should override source aliases with file aliases for the same property', async function () { + writeFileSync(aliasesFilePath, JSON.stringify({ [NAMESPACE]: { [TEST_LAYER]: { name: 'File Name' } } })); - it('should apply aliases from the file when enrichment is disabled', async function () { - writeFileSync(aliasesFilePath, JSON.stringify({ [TEST_LAYER]: { name: 'File Name', height: 'File Height' } })); + const sourceAliases = new Map([ + ['name', 'Source Name'], + ['height', 'Source Height'], + ]); - await aliasesFileDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, await fileReader.readAliases(aliasesFilePath)); + await dal.syncProperties( + NAMESPACE, + { layerName: TEST_LAYER, enums: [] }, + typeMap, + await fileReader.readAliases(aliasesFilePath), + sourceAliases + ); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); expect(properties.find((p) => p.property === 'name')?.alias).toBe('File Name'); - expect(properties.find((p) => p.property === 'height')?.alias).toBe('File Height'); - expect(properties.find((p) => p.property === 'id')?.alias).toBeUndefined(); + expect(properties.find((p) => p.property === 'height')?.alias).toBe('Source Height'); }); - it('should override API aliases with file aliases for the same property', async function () { - writeFileSync(aliasesFilePath, JSON.stringify({ [TEST_LAYER]: { name: 'File Name' } })); + it('should fill in aliases from the file for properties absent from the source aliases', async function () { + writeFileSync(aliasesFilePath, JSON.stringify({ [NAMESPACE]: { [TEST_LAYER]: { height: 'File Height' } } })); - mockEnrichmentApi({ - fields_list: { name: { display_name: 'API Name', type: 'TEXT' }, height: { display_name: 'API Height', type: 'REAL' } }, - }); + const sourceAliases = new Map([['name', 'Source Name']]); - await bothDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, await fileReader.readAliases(aliasesFilePath)); + await dal.syncProperties( + NAMESPACE, + { layerName: TEST_LAYER, enums: [] }, + typeMap, + await fileReader.readAliases(aliasesFilePath), + sourceAliases + ); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); - expect(properties.find((p) => p.property === 'name')?.alias).toBe('File Name'); - expect(properties.find((p) => p.property === 'height')?.alias).toBe('API Height'); + expect(properties.find((p) => p.property === 'name')?.alias).toBe('Source Name'); + expect(properties.find((p) => p.property === 'height')?.alias).toBe('File Height'); }); - it('should fill in aliases from the file for properties absent from the API response', async function () { - writeFileSync(aliasesFilePath, JSON.stringify({ [TEST_LAYER]: { height: 'File Height' } })); - - mockEnrichmentApi({ fields_list: { name: { display_name: 'API Name', type: 'TEXT' } } }); + it('should prefer a namespace-specific file alias over a wildcard one', async function () { + writeFileSync( + aliasesFilePath, + JSON.stringify({ + '*': { '*': { name: 'Global Name' } }, + [NAMESPACE]: { [TEST_LAYER]: { name: 'Specific Name' } }, + }) + ); - await bothDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, await fileReader.readAliases(aliasesFilePath)); + await dal.syncProperties( + NAMESPACE, + { layerName: TEST_LAYER, enums: [] }, + typeMap, + await fileReader.readAliases(aliasesFilePath), + noSourceAliases + ); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); - expect(properties.find((p) => p.property === 'name')?.alias).toBe('API Name'); - expect(properties.find((p) => p.property === 'height')?.alias).toBe('File Height'); + expect(properties.find((p) => p.property === 'name')?.alias).toBe('Specific Name'); }); }); }); describe('Sad Path', function () { describe('syncLayer', function () { - it('should not throw and not update on duplicate layerName', async function () { - await dal.syncLayer('buildings', 1, 'First'); - await dal.syncLayer('buildings', 2, 'Second'); + it('should upsert layerId, source and alias on a duplicate namespace/layerName', async function () { + await dal.syncLayer(NAMESPACE, 'buildings', 1, layerSource.sharedLua, 'First'); + await dal.syncLayer(NAMESPACE, 'buildings', 2, layerSource.sharedLua, 'Second'); - const layers = await layerRepository.find({ where: { layerName: 'buildings' } }); + const layers = await layerRepository.find({ where: { namespace: NAMESPACE, layerName: 'buildings' } }); expect(layers).toHaveLength(1); - expect(layers[0]?.alias).toBe('First'); + expect(layers[0]?.layerId).toBe(2); + expect(layers[0]?.alias).toBe('Second'); }); }); @@ -518,44 +502,15 @@ describe('DAL', function () { describe('syncEnum', function () { it('should return 0 and skip when no enum columns are configured', async function () { - const affected = await dal.syncEnum({ layerName: TEST_LAYER, enums: [] }); + const affected = await dal.syncEnum(NAMESPACE, { layerName: TEST_LAYER, enums: [] }); expect(affected).toBe(0); - const enums = await enumsRepository.find({ where: { layerName: TEST_LAYER } }); + const enums = await enumsRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); expect(enums).toHaveLength(0); }); }); - - describe('syncProperties with enrichment', function () { - it('should leave alias unset for properties absent from the enrichment response', async function () { - mockEnrichmentApi({ fields_list: { name: { display_name: 'Layer Name', type: 'TEXT' } } }); - - await enrichedDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); - - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); - const idProp = properties.find((p) => p.property === 'id'); - - expect(idProp).toBeDefined(); - expect(idProp?.alias).toBeUndefined(); - }); - - it('should sync properties without aliases when the enrichment API returns an error response', async function () { - mockEnrichmentApi({}, 500); - - const affected = await enrichedDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); - - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); - const idProp = properties.find((p) => p.property === 'id'); - const nameProp = properties.find((p) => p.property === 'name'); - - expect(affected).toBeGreaterThan(0); - expect(idProp).toBeDefined(); - expect(idProp?.alias).toBeUndefined(); - expect(nameProp?.alias).toBeUndefined(); - }); - }); }); describe('Bad Path', function () { @@ -565,8 +520,8 @@ describe('DAL', function () { ALTER TABLE "${sourceSchema}"."${TEST_LAYER}" ADD COLUMN lsn pg_lsn `); - const affected = await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const affected = await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, noSourceAliases); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); expect(affected).toBeGreaterThan(0); expect(properties.find((p) => p.property === 'lsn')).toBeUndefined(); @@ -574,4 +529,35 @@ describe('DAL', function () { }); }); }); + + describe('Multi-namespace isolation', function () { + it('should not delete layers belonging to another namespace', async function () { + await dal.syncLayer(NAMESPACE, 'buildings', 1, layerSource.sharedLua); + await layerRepository.insert({ namespace: 'other', layerName: 'buildings', layerId: 1, source: layerSource.sharedLua, alias: 'buildings' }); + + await dal.deleteLayersNotIn(NAMESPACE, ['roads']); + + const otherLayer = await layerRepository.findOne({ where: { namespace: 'other', layerName: 'buildings' } }); + const ownLayer = await layerRepository.findOne({ where: { namespace: NAMESPACE, layerName: 'buildings' } }); + + expect(otherLayer).not.toBeNull(); + expect(ownLayer).toBeNull(); + + await layerRepository.delete({ namespace: 'other', layerName: 'buildings' }); + }); + + it('should not delete stale properties belonging to another namespace', async function () { + await layerRepository.insert({ namespace: 'other', layerName: TEST_LAYER, layerId: 1, source: layerSource.sharedLua, alias: TEST_LAYER }); + await propertyRepository.insert({ namespace: 'other', layerName: TEST_LAYER, property: 'legacy', type: columnType.text }); + + await dal.deleteStaleProperties(NAMESPACE, TEST_LAYER, []); + + const otherProperty = await propertyRepository.findOne({ where: { namespace: 'other', layerName: TEST_LAYER, property: 'legacy' } }); + + expect(otherProperty).not.toBeNull(); + + await propertyRepository.delete({ namespace: 'other', layerName: TEST_LAYER }); + await layerRepository.delete({ namespace: 'other', layerName: TEST_LAYER }); + }); + }); }); diff --git a/packages/synchronizer/tests/unit/synchronizer/aliasEnricher.spec.ts b/packages/synchronizer/tests/unit/synchronizer/aliasEnricher.spec.ts index 0d5ae3f..38ba574 100644 --- a/packages/synchronizer/tests/unit/synchronizer/aliasEnricher.spec.ts +++ b/packages/synchronizer/tests/unit/synchronizer/aliasEnricher.spec.ts @@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, beforeAll, afterEach, afterAll } from 'vitest'; import nock, { disableNetConnect, enableNetConnect, cleanAll } from 'nock'; import { fetchPropertyAliases } from '@src/sync/aliasEnricher'; -import { type ConfigType, getConfig, initConfig } from '@src/common/config'; +import type { EnrichmentConfig } from '@src/common/interfaces'; const ENRICHMENT_ORIGIN = 'https://example.com'; @@ -13,11 +13,15 @@ const makeBody = (fields: Record }); describe('fetchPropertyAliases', function () { - let enrichmentConfig: ReturnType['enrichment'] & { enabled: true }; - - beforeAll(async function () { - await initConfig(true); - enrichmentConfig = getConfig().get('enrichmentApi'); + const enrichmentConfig: Extract = { + enabled: true, + api: `${ENRICHMENT_ORIGIN}/{layerName}`, + propertiesPath: 'fields_list', + aliasField: 'display_name', + requestTimeoutMilliseconds: 10000, + }; + + beforeAll(function () { disableNetConnect(); }); @@ -37,7 +41,7 @@ describe('fetchPropertyAliases', function () { it('should call the API once with the layer name substituted into the URL', async function () { const scope = nock(ENRICHMENT_ORIGIN).get('/buildings').reply(200, makeBody({})); - await fetchPropertyAliases('buildings', 1, getConfig().get('enrichmentApi')); + await fetchPropertyAliases('buildings', 1, enrichmentConfig); expect(scope.isDone()).toBe(true); }); diff --git a/packages/synchronizer/tests/unit/synchronizer/helpers.spec.ts b/packages/synchronizer/tests/unit/synchronizer/helpers.spec.ts index ce520ba..d5d9ec3 100644 --- a/packages/synchronizer/tests/unit/synchronizer/helpers.spec.ts +++ b/packages/synchronizer/tests/unit/synchronizer/helpers.spec.ts @@ -10,6 +10,7 @@ const typeMapFile = './config/typeMap.json'; const typeMap = parseTypeMap(JSON.parse(readFileSync(typeMapFile, 'utf-8')) as JsonValue, typeMapFile); describe('columnInfosToProperties', function () { + const namespace = 'test_namespace'; const layer = 'test_layer'; describe('Happy Path', function () { @@ -53,7 +54,7 @@ describe('columnInfosToProperties', function () { ])('should map udt "%s" to %s', function (udtName, expected) { const columnInfos: ColumnInfo[] = [{ columnName: 'col', udtName }]; - const [result] = columnInfosToProperties(columnInfos, layer, typeMap); + const [result] = columnInfosToProperties(columnInfos, namespace, layer, typeMap); expect(result.type).toBe(expected); }); @@ -72,19 +73,19 @@ describe('columnInfosToProperties', function () { ['geography(MultiPolygon,4326)', columnType.multiPolygon], ['geometry(GeometryCollection,4326)', columnType.geom], ])('should map "%s" to its specific GML type', function (udtName, expected) { - const [result] = columnInfosToProperties([{ columnName: 'shape', udtName }], layer, typeMap); + const [result] = columnInfosToProperties([{ columnName: 'shape', udtName }], namespace, layer, typeMap); expect(result.type).toBe(expected); }); it('should map unqualified geometry to gml:GeometryPropertyType', function () { - const [result] = columnInfosToProperties([{ columnName: 'shape', udtName: 'geometry' }], layer, typeMap); + const [result] = columnInfosToProperties([{ columnName: 'shape', udtName: 'geometry' }], namespace, layer, typeMap); expect(result.type).toBe(columnType.geom); }); it('should match udt names case insensitively', function () { - const [result] = columnInfosToProperties([{ columnName: 'name', udtName: 'CHARACTER VARYING(255)' }], layer, typeMap); + const [result] = columnInfosToProperties([{ columnName: 'name', udtName: 'CHARACTER VARYING(255)' }], namespace, layer, typeMap); expect(result.type).toBe(columnType.text); }); @@ -95,30 +96,30 @@ describe('columnInfosToProperties', function () { { columnName: 'height', udtName: 'real' }, ]; - const result = columnInfosToProperties(columnInfos, layer, typeMap); + const result = columnInfosToProperties(columnInfos, namespace, layer, typeMap); expect(result).toEqual([ - { layerName: layer, property: 'name', type: columnType.text }, - { layerName: layer, property: 'height', type: columnType.real }, + { namespace, layerName: layer, property: 'name', type: columnType.text }, + { namespace, layerName: layer, property: 'height', type: columnType.real }, ]); }); }); describe('Sad Path', function () { it('should return an empty array for no column infos', function () { - expect(columnInfosToProperties([], layer, typeMap)).toEqual([]); + expect(columnInfosToProperties([], namespace, layer, typeMap)).toEqual([]); }); }); describe('Bad Path', function () { it('should map unknown geometry sub-type to gml:GeometryPropertyType', function () { - const [result] = columnInfosToProperties([{ columnName: 'shape', udtName: 'geometry(Curve,4326)' }], layer, typeMap); + const [result] = columnInfosToProperties([{ columnName: 'shape', udtName: 'geometry(Curve,4326)' }], namespace, layer, typeMap); expect(result.type).toBe(columnType.geom); }); it('should return no properties for a udt name not in the type map', function () { - const result = columnInfosToProperties([{ columnName: 'data', udtName: 'pg_lsn' }], layer, typeMap); + const result = columnInfosToProperties([{ columnName: 'data', udtName: 'pg_lsn' }], namespace, layer, typeMap); expect(result).toHaveLength(0); }); @@ -131,7 +132,7 @@ describe('columnInfosToProperties', function () { { columnName: 'height', udtName: 'real' }, ]; - const result = columnInfosToProperties(columnInfos, layer, typeMap, onUnknown); + const result = columnInfosToProperties(columnInfos, namespace, layer, typeMap, onUnknown); expect(result).toHaveLength(2); expect(result.map((p) => p.property)).toEqual(['name', 'height']); diff --git a/packages/synchronizer/tests/unit/synchronizer/perLayerJsonSource.spec.ts b/packages/synchronizer/tests/unit/synchronizer/perLayerJsonSource.spec.ts new file mode 100644 index 0000000..293505d --- /dev/null +++ b/packages/synchronizer/tests/unit/synchronizer/perLayerJsonSource.spec.ts @@ -0,0 +1,141 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { writeFileSync, mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { describe, it, expect, vi, beforeEach, afterEach, beforeAll } from 'vitest'; +import { jsLogger } from '@map-colonies/js-logger'; +import type { Logger } from '@map-colonies/js-logger'; +import { PerLayerJsonSource } from '@src/sync/layerSource/perLayerJsonSource'; +import type { FileDownloader } from '@src/sync/layerSource/types'; +import { FsRepository } from '@src/common/fs/fsRepository'; +import type { PerLayerJsonSourceConfig } from '@src/common/interfaces'; + +const BUCKET = 'test-bucket'; + +const config: PerLayerJsonSourceConfig = { + type: 'perLayerJson', + prefix: '', + aliasLayerNameField: 'alias_layer_name', + fieldsField: 'fields', + fieldNameField: 'field_name', + aliasFieldNameField: 'alias_field_name', +}; + +describe('PerLayerJsonSource', function () { + let tmpDir: string; + let logger: Logger; + let fsRepository: FsRepository; + + const makeS3Repository = (key: string, content: string): { s3Repository: FileDownloader; downloadFile: ReturnType } => { + const filePath = join(tmpDir, 'downloaded.json'); + writeFileSync(filePath, content); + const downloadFile = vi.fn().mockImplementation((_bucket: string, requestedKey: string) => { + if (requestedKey !== key) { + throw new Error('unexpected key'); + } + return filePath; + }); + return { s3Repository: { downloadFile }, downloadFile }; + }; + + beforeAll(async function () { + logger = await jsLogger({ enabled: false }); + fsRepository = new FsRepository(logger); + }); + + beforeEach(function () { + tmpDir = mkdtempSync(join(tmpdir(), 'per-layer-json-source-test-')); + }); + + afterEach(function () { + rmSync(tmpDir, { recursive: true }); + }); + + describe('Happy Path', function () { + it('should resolve a layer alias and case-insensitive property aliases', async function () { + const { s3Repository } = makeS3Repository( + 'BUILDINGS.json', + JSON.stringify({ + alias_layer_name: 'Buildings', + fields: [{ field_name: 'PROPERTY', alias_field_name: 'Property Alias' }], + }) + ); + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, config); + + const records = await source.tick([{ layerName: 'buildings', enums: [], sourceKey: 'BUILDINGS.json' }]); + const record = records.get('buildings'); + + expect(record?.alias).toBe('Buildings'); + expect(record?.layerId).toBeUndefined(); + expect(record?.source).toBe('perLayerJson'); + expect(record?.propertyAliases.get('property')).toBe('Property Alias'); + }); + + it('should prepend the configured prefix to the sourceKey', async function () { + const prefixedConfig: PerLayerJsonSourceConfig = { ...config, prefix: 'layers/' }; + const { s3Repository } = makeS3Repository('layers/BUILDINGS.json', JSON.stringify({ alias_layer_name: 'Buildings', fields: [] })); + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, prefixedConfig); + + const records = await source.tick([{ layerName: 'buildings', enums: [], sourceKey: 'BUILDINGS.json' }]); + + expect(records.get('buildings')?.alias).toBe('Buildings'); + }); + + it('should ignore fields with no matching field_name or alias_field_name', async function () { + const { s3Repository } = makeS3Repository( + 'BUILDINGS.json', + JSON.stringify({ + alias_layer_name: 'Buildings', + fields: [{ field_name: 'PROPERTY' }, { alias_field_name: 'Orphan Alias' }, 'not-an-object'], + }) + ); + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, config); + + const records = await source.tick([{ layerName: 'buildings', enums: [], sourceKey: 'BUILDINGS.json' }]); + + expect(records.get('buildings')?.propertyAliases.size).toBe(0); + }); + }); + + describe('Sad Path', function () { + it('should skip and warn when the layer has no sourceKey', async function () { + const { s3Repository, downloadFile } = makeS3Repository('BUILDINGS.json', '{}'); + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, config); + + const records = await source.tick([{ layerName: 'buildings', enums: [] }]); + + expect(records.has('buildings')).toBe(false); + expect(downloadFile).not.toHaveBeenCalled(); + }); + + it('should skip a layer whose object is missing from S3', async function () { + const downloadFile = vi.fn().mockRejectedValue(new Error('NoSuchKey')); + const s3Repository: FileDownloader = { downloadFile }; + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, config); + + const records = await source.tick([{ layerName: 'buildings', enums: [], sourceKey: 'MISSING.json' }]); + + expect(records.has('buildings')).toBe(false); + }); + }); + + describe('Bad Path', function () { + it('should skip a layer with unparseable JSON', async function () { + const { s3Repository } = makeS3Repository('BUILDINGS.json', 'not json'); + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, config); + + const records = await source.tick([{ layerName: 'buildings', enums: [], sourceKey: 'BUILDINGS.json' }]); + + expect(records.has('buildings')).toBe(false); + }); + + it('should skip a layer missing required fields', async function () { + const { s3Repository } = makeS3Repository('BUILDINGS.json', JSON.stringify({ fields: [] })); + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, config); + + const records = await source.tick([{ layerName: 'buildings', enums: [], sourceKey: 'BUILDINGS.json' }]); + + expect(records.has('buildings')).toBe(false); + }); + }); +}); diff --git a/packages/synchronizer/tests/unit/synchronizer/sharedLuaSource.spec.ts b/packages/synchronizer/tests/unit/synchronizer/sharedLuaSource.spec.ts new file mode 100644 index 0000000..c0004b2 --- /dev/null +++ b/packages/synchronizer/tests/unit/synchronizer/sharedLuaSource.spec.ts @@ -0,0 +1,141 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { writeFileSync, mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { describe, it, expect, vi, beforeEach, afterEach, beforeAll, afterAll } from 'vitest'; +import nock, { disableNetConnect, enableNetConnect, cleanAll } from 'nock'; +import { jsLogger } from '@map-colonies/js-logger'; +import type { Logger } from '@map-colonies/js-logger'; +import { SharedLuaSource } from '@src/sync/layerSource/sharedLuaSource'; +import type { FileDownloader } from '@src/sync/layerSource/types'; +import { FsRepository } from '@src/common/fs/fsRepository'; +import type { SharedLuaSourceConfig } from '@src/common/interfaces'; + +const ID_FIELD = 'layer_id'; +const NAME_FIELD = 'layer_name'; +const ALIAS_FIELD = 'layer_alias'; +const BUCKET = 'test-bucket'; +const FILE_NAME = 'query.lua'; +const ENRICHMENT_ORIGIN = 'http://mock-api'; + +const makeLua = (entries: string): string => `local layers = {\n${entries}\n}`; + +const makeEntry = (layerId: string, layerName: string, alias: string): string => + ` { ${ID_FIELD} = '${layerId}', ${NAME_FIELD} = '${layerName}', ${ALIAS_FIELD} = '${alias}' },`; + +describe('SharedLuaSource', function () { + let tmpDir: string; + let logger: Logger; + let fsRepository: FsRepository; + + const makeS3Repository = (content: string): FileDownloader => { + const filePath = join(tmpDir, FILE_NAME); + writeFileSync(filePath, content); + return { downloadFile: vi.fn().mockResolvedValue(filePath) }; + }; + + const disabledEnrichmentConfig: SharedLuaSourceConfig = { + type: 'sharedLua', + fileName: FILE_NAME, + layersVariable: 'layers', + aliasFieldName: ALIAS_FIELD, + idFieldName: ID_FIELD, + nameFieldName: NAME_FIELD, + enrichment: { enabled: false }, + }; + + beforeAll(async function () { + logger = await jsLogger({ enabled: false }); + fsRepository = new FsRepository(logger); + disableNetConnect(); + }); + + beforeEach(function () { + tmpDir = mkdtempSync(join(tmpdir(), 'shared-lua-source-test-')); + cleanAll(); + }); + + afterEach(function () { + rmSync(tmpDir, { recursive: true }); + cleanAll(); + }); + + afterAll(function () { + enableNetConnect(); + }); + + describe('Happy Path', function () { + it('should resolve a known layer with its id and alias', async function () { + const s3Repository = makeS3Repository(makeLua(makeEntry('1', 'buildings', 'Buildings'))); + const source = new SharedLuaSource(s3Repository, fsRepository, logger, BUCKET, disabledEnrichmentConfig); + + const records = await source.tick([{ layerName: 'buildings', enums: [] }]); + const record = records.get('buildings'); + + expect(record?.layerId).toBe(1); + expect(record?.alias).toBe('Buildings'); + expect(record?.source).toBe('sharedLua'); + expect(record?.propertyAliases.size).toBe(0); + }); + + it('should fetch property aliases from the enrichment API when enabled', async function () { + const enrichmentConfig: SharedLuaSourceConfig = { + ...disabledEnrichmentConfig, + enrichment: { + enabled: true, + api: `${ENRICHMENT_ORIGIN}/{layerName}`, + propertiesPath: 'fields_list', + aliasField: 'display_name', + requestTimeoutMilliseconds: 5000, + }, + }; + nock(ENRICHMENT_ORIGIN) + .get('/buildings') + .reply(200, { fields_list: { name: { display_name: 'Name' } } }); + + const s3Repository = makeS3Repository(makeLua(makeEntry('1', 'buildings', 'Buildings'))); + const source = new SharedLuaSource(s3Repository, fsRepository, logger, BUCKET, enrichmentConfig); + + const records = await source.tick([{ layerName: 'buildings', enums: [] }]); + const record = records.get('buildings'); + + expect(record?.propertyAliases.get('name')).toBe('Name'); + }); + }); + + describe('Sad Path', function () { + it('should not include a layer absent from the lua file', async function () { + const s3Repository = makeS3Repository(makeLua(makeEntry('1', 'buildings', 'Buildings'))); + const source = new SharedLuaSource(s3Repository, fsRepository, logger, BUCKET, disabledEnrichmentConfig); + + const records = await source.tick([{ layerName: 'roads', enums: [] }]); + + expect(records.has('roads')).toBe(false); + }); + }); + + describe('Bad Path', function () { + it('should resolve without property aliases when the enrichment API fails', async function () { + const enrichmentConfig: SharedLuaSourceConfig = { + ...disabledEnrichmentConfig, + enrichment: { + enabled: true, + api: `${ENRICHMENT_ORIGIN}/{layerName}`, + propertiesPath: 'fields_list', + aliasField: 'display_name', + requestTimeoutMilliseconds: 5000, + }, + }; + nock(ENRICHMENT_ORIGIN).get('/buildings').reply(500); + + const s3Repository = makeS3Repository(makeLua(makeEntry('1', 'buildings', 'Buildings'))); + const source = new SharedLuaSource(s3Repository, fsRepository, logger, BUCKET, enrichmentConfig); + + const records = await source.tick([{ layerName: 'buildings', enums: [] }]); + const record = records.get('buildings'); + + expect(record?.layerId).toBe(1); + expect(record?.propertyAliases.size).toBe(0); + }); + }); +}); From d8e1b6c7e44d0c5fea61f464ee0d4042523c6bff Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Wed, 19 Aug 2026 16:32:36 +0300 Subject: [PATCH 07/29] helm: migrate synchronizer chart to multi-namespace config --- .../synchronizer/templates/configmap.yaml | 15 ----- .../synchronizer/templates/deployment.yaml | 29 +++++---- .../templates/layers-configmap.yaml | 4 +- .../templates/namespaces-secret.yaml | 11 ++++ .../charts/synchronizer/templates/secret.yaml | 1 - helm/charts/synchronizer/values.yaml | 65 ++++++++++--------- helm/values.yaml | 64 ++++++++++-------- 7 files changed, 102 insertions(+), 87 deletions(-) create mode 100644 helm/charts/synchronizer/templates/namespaces-secret.yaml diff --git a/helm/charts/synchronizer/templates/configmap.yaml b/helm/charts/synchronizer/templates/configmap.yaml index 66a4cc0..158ddc4 100644 --- a/helm/charts/synchronizer/templates/configmap.yaml +++ b/helm/charts/synchronizer/templates/configmap.yaml @@ -32,8 +32,6 @@ data: INDEX_NAME_FORMAT: {{ .Values.env.indexNameFormat | quote }} - LAYERS_FILE_PATH: {{ .Values.env.layersFile | quote }} - ALIASES_FILE_PATH: {{ .Values.env.aliasesFile | quote }} {{- if .Values.typeMap.enabled }} @@ -47,21 +45,8 @@ data: S3_SSL_ENABLED: {{ .sslEnabled | quote }} S3_SIG_VERSION: {{ .sigVersion | quote }} S3_STORAGE_CLASS: {{ .storageClass | quote }} - S3_BUCKET: {{ .bucket | quote }} - S3_FILE_NAME: {{ .fileName | quote }} - S3_LAYERS_VARIABLE: {{ .layersVariable | quote }} {{- end }} - {{- with .Values.env.enrichment }} - ENRICHMENT_ENABLED: {{ .enabled | quote }} - {{- if .enabled }} - ENRICHMENT_API_URL: {{ .api | quote }} - ENRICHMENT_PROPERTIES_PATH: {{ .propertiesPath | quote }} - ENRICHMENT_ALIAS_FIELD: {{ .aliasField | quote }} - ENRICHMENT_REQUEST_TIMEOUT_MILLISECONDS: {{ .requestTimeoutMilliseconds | quote }} - {{- end }} - {{- end }} - {{- include "synchronizer.dbEnvBlock" (dict "prefix" "SOURCE_DB" "db" .Values.dbs.sourceDb) | nindent 2 }} {{- include "synchronizer.dbEnvBlock" (dict "prefix" "DEST_DB" "db" .Values.dbs.destinationDb) | nindent 2 }} diff --git a/helm/charts/synchronizer/templates/deployment.yaml b/helm/charts/synchronizer/templates/deployment.yaml index c0a319f..f5803b3 100644 --- a/helm/charts/synchronizer/templates/deployment.yaml +++ b/helm/charts/synchronizer/templates/deployment.yaml @@ -93,7 +93,10 @@ spec: {{- end }} volumeMounts: - name: layers-config - mountPath: {{ .Values.env.layersFile | dir | quote }} + mountPath: /usr/src/app/packages/synchronizer/dist/config/layers + - name: namespaces-config + mountPath: /usr/src/app/packages/synchronizer/dist/config/production.json + subPath: production.json - name: aliases-config mountPath: {{ .Values.env.aliasesFile | dir | quote }} {{- if .Values.typeMap.enabled }} @@ -102,11 +105,13 @@ spec: {{- end }} - name: s3-downloads mountPath: /usr/src/app/packages/synchronizer/dist/src/common/s3/downloads - {{- if .Values.dbs.sourceDb.sslAuth.enabled }} - - name: source-db-cert-conf - mountPath: /tmp/certs/source + {{- range .Values.namespaces }} + {{- if .db.ssl.enabled }} + - name: source-db-cert-{{ .name }} + mountPath: {{ printf "/tmp/certs/source-%s" .name | quote }} readOnly: true {{- end }} + {{- end }} {{- if .Values.dbs.destinationDb.sslAuth.enabled }} - name: dest-db-cert-conf mountPath: /tmp/certs/dest @@ -121,6 +126,9 @@ spec: - name: layers-config configMap: name: {{ include "synchronizer.fullname" . }}-layers + - name: namespaces-config + secret: + secretName: {{ include "synchronizer.fullname" . }}-namespaces - name: aliases-config configMap: name: {{ include "synchronizer.fullname" . }}-aliases @@ -131,16 +139,13 @@ spec: {{- end }} - name: s3-downloads emptyDir: {} - {{- if .Values.dbs.sourceDb.sslAuth.enabled }} - - name: source-db-cert-conf + {{- range .Values.namespaces }} + {{- if .db.ssl.enabled }} + - name: source-db-cert-{{ .name }} secret: - secretName: {{ .Values.dbs.sourceDb.sslAuth.secretName }} + secretName: {{ .db.sslSecretName }} + {{- end }} {{- end }} - {{- if .Values.caSecretName }} - - mountPath: {{ printf "%s/%s" .Values.caPath .Values.caKey | quote }} - name: root-ca - subPath: {{ quote .Values.caKey }} - {{- end }} {{- if .Values.dbs.destinationDb.sslAuth.enabled }} - name: dest-db-cert-conf secret: diff --git a/helm/charts/synchronizer/templates/layers-configmap.yaml b/helm/charts/synchronizer/templates/layers-configmap.yaml index 407ae25..5d7aef5 100644 --- a/helm/charts/synchronizer/templates/layers-configmap.yaml +++ b/helm/charts/synchronizer/templates/layers-configmap.yaml @@ -6,5 +6,7 @@ metadata: labels: {{- include "synchronizer.labels" . | nindent 4 }} data: - layers.json: {{ .Values.layers | toPrettyJson | quote }} + {{- range .Values.namespaces }} + {{ .name }}.json: {{ .layers | toPrettyJson | quote }} + {{- end }} {{- end }} diff --git a/helm/charts/synchronizer/templates/namespaces-secret.yaml b/helm/charts/synchronizer/templates/namespaces-secret.yaml new file mode 100644 index 0000000..53b6484 --- /dev/null +++ b/helm/charts/synchronizer/templates/namespaces-secret.yaml @@ -0,0 +1,11 @@ +{{- if .Values.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "synchronizer.fullname" . }}-namespaces + labels: + {{- include "synchronizer.labels" . | nindent 4 }} +type: Opaque +stringData: + production.json: {{ dict "namespaces" .Values.namespaces | toPrettyJson | quote }} +{{- end }} diff --git a/helm/charts/synchronizer/templates/secret.yaml b/helm/charts/synchronizer/templates/secret.yaml index 96277b1..12c441f 100644 --- a/helm/charts/synchronizer/templates/secret.yaml +++ b/helm/charts/synchronizer/templates/secret.yaml @@ -7,7 +7,6 @@ metadata: {{- include "synchronizer.labels" . | nindent 4 }} type: Opaque data: - SOURCE_DB_PASSWORD: {{ .Values.dbs.sourceDb.password | b64enc | quote }} DEST_DB_PASSWORD: {{ .Values.dbs.destinationDb.password | b64enc | quote }} S3_ACCESS_KEY: {{ .Values.dbs.s3.accessKey | b64enc | quote }} S3_SECRET_KEY: {{ .Values.dbs.s3.secretKey | b64enc | quote }} diff --git a/helm/charts/synchronizer/values.yaml b/helm/charts/synchronizer/values.yaml index a9912ee..d5604f5 100644 --- a/helm/charts/synchronizer/values.yaml +++ b/helm/charts/synchronizer/values.yaml @@ -23,7 +23,6 @@ startupProbe: enabled: true periodSeconds: 10 timeoutSeconds: 5 - # failureThreshold * periodSeconds = max startup wait (30 * 10s = 300s) failureThreshold: 30 livenessProbe: @@ -41,11 +40,41 @@ readinessProbe: failureThreshold: 6 path: /liveness -layers: - - layerName: buildings_polygon - enums: [code, attribute, sensitivity, building_type, status] - - layerName: borders_polygon - enums: [code, attribute] +namespaces: + - name: default + bucket: "" + layers: + - layerName: buildings_polygon + enums: [code, attribute, sensitivity, building_type, status] + - layerName: borders_polygon + enums: [code, attribute] + db: + host: localhost + port: 5432 + username: postgres + password: postgres + database: postgres + schema: public + type: postgres + sslSecretName: secret-name + ssl: + enabled: false + ca: "" + cert: "" + key: "" + layerSource: + type: sharedLua + fileName: mock-query.lua + layersVariable: layers + aliasFieldName: alias + idFieldName: id + nameFieldName: name + enrichment: + enabled: false + api: "" + propertiesPath: "" + aliasField: "" + requestTimeoutMilliseconds: 5000 aliases: {} @@ -70,15 +99,8 @@ env: url: http://localhost:55681/v1/metrics schedule: '0 0 * * *' indexNameFormat: '{layerName}_{column}_idx' - layersFile: /usr/src/app/packages/synchronizer/dist/config/layers/layers.json aliasesFile: /usr/src/app/packages/synchronizer/dist/config/aliases/aliases.json typeMapFile: /usr/src/app/packages/synchronizer/dist/config/typeMap/typeMap.json - enrichment: - enabled: false - api: "" - propertiesPath: "" - aliasField: "" - requestTimeoutMilliseconds: 5000 dbs: s3: @@ -88,25 +110,8 @@ dbs: sslEnabled: false sigVersion: v4 storageClass: STANDARD - bucket: "" - fileName: "" - layersVariable: layers accessKey: "" secretKey: "" - sourceDb: - host: localhost - port: 5432 - username: postgres - password: postgres - database: postgres - schema: public - type: postgres - sslAuth: - enabled: false - secretName: secret-name - cert: "" - key: "" - root: "" destinationDb: host: localhost port: 5432 diff --git a/helm/values.yaml b/helm/values.yaml index 51b1cad..0bb40ba 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -163,13 +163,44 @@ synchronizer: pullPolicy: Always tag: "v1.2.0" - layers: - - layerName: buildings_polygon - enums: [ code, attribute ] + namespaces: + - name: default + bucket: "" + layers: + - layerName: buildings_polygon + enums: [code, attribute] + db: + host: localhost + port: 5432 + username: postgres + password: postgres + database: postgres + schema: public + type: postgres + sslSecretName: secret-name + ssl: + enabled: false + ca: "" + cert: "" + key: "" + layerSource: + type: sharedLua + fileName: "" + layersVariable: layers + aliasFieldName: alias + idFieldName: id + nameFieldName: name + enrichment: + enabled: false + api: "" + propertiesPath: "" + aliasField: "" + requestTimeoutMilliseconds: 5000 aliases: "*": - num_floors: "number_of_floors" + "*": + num_floors: "number_of_floors" typeMap: enabled: false @@ -192,15 +223,8 @@ synchronizer: url: http://localhost:55681/v1/metrics schedule: '*/1 * * * *' indexNameFormat: '{layerName}_{column}_idx' - layersFile: /usr/src/app/packages/synchronizer/dist/config/layers/layers.json aliasesFile: /usr/src/app/packages/synchronizer/dist/config/aliases/aliases.json - typeMapFile: /usr/src/app/packages/synchronizer/dist/config/typeMap.json - enrichment: - enabled: false - api: "" - propertiesPath: "" - aliasField: "" - requestTimeoutMilliseconds: 5000 + typeMapFile: /usr/src/app/packages/synchronizer/dist/config/typeMap/typeMap.json dbs: s3: @@ -210,24 +234,8 @@ synchronizer: sslEnabled: false sigVersion: v4 storageClass: STANDARD - bucket: "" - fileName: "" - layersVariable: layers accessKey: "" secretKey: "" - sourceDb: - host: localhost - port: 5432 - username: postgres - password: postgres - database: postgres - schema: public - type: postgres - sslAuth: - enabled: false - cert: '' - key: '' - root: '' destinationDb: *dbConfig resources: From 374f4a16adc95756140095d774241253815a7c1e Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Wed, 19 Aug 2026 16:32:47 +0300 Subject: [PATCH 08/29] fix: resolve postgres ssl type mismatch between typeorm and pg --- packages/db/src/interfaces.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/db/src/interfaces.ts b/packages/db/src/interfaces.ts index d52dbec..9eb298c 100644 --- a/packages/db/src/interfaces.ts +++ b/packages/db/src/interfaces.ts @@ -1,8 +1,7 @@ -import type { DataSourceOptions } from 'typeorm'; import type { ConfigType } from './db/config'; export type DbConfig = ReturnType['db']; export type SslConfig = DbConfig['ssl']; -export type SslOptions = Extract['ssl']; +export type SslOptions = false | { key?: Buffer; cert?: Buffer; ca?: Buffer }; From a7611b4d519333d648637a93731936af07fcae63 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Wed, 19 Aug 2026 16:41:18 +0300 Subject: [PATCH 09/29] chore: move namespace logging to formatted message --- packages/api/src/layer/controllers/layerController.ts | 8 ++++---- packages/api/src/layer/models/layerManager.ts | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/api/src/layer/controllers/layerController.ts b/packages/api/src/layer/controllers/layerController.ts index bb2e0b9..83b03eb 100644 --- a/packages/api/src/layer/controllers/layerController.ts +++ b/packages/api/src/layer/controllers/layerController.ts @@ -20,10 +20,10 @@ export class LayerController { const { namespace } = req.params; try { const layers = await this.manager.getLayers(namespace); - this.logger.debug({ msg: `got ${layers.length} layers`, namespace }); + this.logger.debug({ msg: `got ${layers.length} layers for namespace ${namespace}` }); return res.json({ layers }); } catch (error) { - this.logger.error({ msg: 'failed to get layers', namespace, err: error }); + this.logger.error({ msg: `failed to get layers for namespace ${namespace}`, err: error }); next(error); } }; @@ -32,10 +32,10 @@ export class LayerController { const { namespace, layerName } = req.params; try { const layer = await this.manager.getLayerSpecByName(namespace, layerName); - this.logger.debug({ msg: `got layer: ${layerName}`, namespace }); + this.logger.debug({ msg: `got layer: ${layerName} for namespace ${namespace}` }); return res.json(layer); } catch (error) { - this.logger.error({ msg: 'failed to get layer by name', namespace, layerName, err: error }); + this.logger.error({ msg: `failed to get layer ${layerName} for namespace ${namespace}`, err: error }); next(error); } }; diff --git a/packages/api/src/layer/models/layerManager.ts b/packages/api/src/layer/models/layerManager.ts index 48f658d..e6bce80 100644 --- a/packages/api/src/layer/models/layerManager.ts +++ b/packages/api/src/layer/models/layerManager.ts @@ -15,7 +15,7 @@ export class LayerManager { ) {} public async getLayers(namespace: string): Promise { - this.logger.debug({ msg: 'getting layers', namespace }); + this.logger.debug({ msg: `getting layers for namespace ${namespace}` }); const layers = await this.repository.find({ where: { namespace }, select: { layerName: true, alias: true } }); if (layers.length === 0) { throw new NotFoundError(`Namespace doesn't exist`); @@ -24,7 +24,7 @@ export class LayerManager { } public async getLayerSpecByName(namespace: string, name: string): Promise { - this.logger.debug({ msg: 'getting layer spec by name', namespace, name }); + this.logger.debug({ msg: `getting layer spec ${name} for namespace ${namespace}` }); const layer = await this.repository.findOne({ where: { namespace, layerName: name }, relations: { properties: { possibleValues: true } }, From f04634a1cba221fde4ed0ff14d8b8101c3f20830 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Wed, 19 Aug 2026 17:09:39 +0300 Subject: [PATCH 10/29] feat: collapse standard db creation to one factory --- packages/api/src/common/db/connection.ts | 15 --------------- packages/api/src/containerConfig.ts | 11 +++++++---- packages/db/src/db/connection.ts | 6 +++++- packages/db/src/index.ts | 2 +- .../synchronizer/src/common/db/connection.ts | 17 ----------------- packages/synchronizer/src/containerConfig.ts | 15 ++++++++++----- 6 files changed, 23 insertions(+), 43 deletions(-) delete mode 100644 packages/api/src/common/db/connection.ts delete mode 100644 packages/synchronizer/src/common/db/connection.ts diff --git a/packages/api/src/common/db/connection.ts b/packages/api/src/common/db/connection.ts deleted file mode 100644 index c5b451f..0000000 --- a/packages/api/src/common/db/connection.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { FactoryFunction, DependencyContainer } from 'tsyringe'; -import { DataSource } from 'typeorm'; -import { createDataSourceOptions, createDataSourceHealthCheck, DATA_SOURCE_PROVIDER } from '@db'; -import { type HealthCheck } from '@godaddy/terminus'; -import { SERVICE_NAME, SERVICES } from '../constants'; -import type { ConfigType } from '../config'; - -export const dataSourceFactory: FactoryFunction = (container: DependencyContainer): DataSource => { - const config = container.resolve(SERVICES.CONFIG); - const dbConfig = config.get('db'); - return new DataSource(createDataSourceOptions(dbConfig, SERVICE_NAME)); -}; - -export const healthCheckFactory: FactoryFunction = (container: DependencyContainer): HealthCheck => - createDataSourceHealthCheck(container, [DATA_SOURCE_PROVIDER]); diff --git a/packages/api/src/containerConfig.ts b/packages/api/src/containerConfig.ts index 21f2258..5e579db 100644 --- a/packages/api/src/containerConfig.ts +++ b/packages/api/src/containerConfig.ts @@ -4,7 +4,8 @@ import { Registry } from 'prom-client'; import type { DependencyContainer } from 'tsyringe/dist/typings/types'; import { jsLogger, type Logger } from '@map-colonies/js-logger'; import { CleanupRegistry } from '@map-colonies/cleanup-registry'; -import { DATA_SOURCE_PROVIDER, Layer, LAYER_REPOSITORY_SYMBOL } from '@db'; +import { DATA_SOURCE_PROVIDER, Layer, LAYER_REPOSITORY_SYMBOL, createDataSource, createDataSourceHealthCheck } from '@db'; +import type { HealthCheck } from '@godaddy/terminus'; import type { Repository, DataSource } from 'typeorm'; import { instancePerContainerCachingFactory } from 'tsyringe'; import { type InjectionObject, registerDependencies } from '@common/dependencyRegistration'; @@ -12,7 +13,6 @@ import { HEALTHCHECK, ON_SIGNAL, SERVICES, SERVICE_NAME } from '@common/constant import { getTracing } from '@common/tracing'; import { LAYER_ROUTER_SYMBOL, layerRouterFactory } from './layer/routes/layer'; import { NAMESPACE_ROUTER_SYMBOL, namespaceRouterFactory } from './namespace/routes/namespace'; -import { dataSourceFactory, healthCheckFactory } from './common/db/connection'; import { type ConfigType, getConfig } from './common/config'; export interface RegisterOptions { @@ -80,7 +80,10 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: DATA_SOURCE_PROVIDER, provider: { - useFactory: instancePerContainerCachingFactory(dataSourceFactory), + useFactory: instancePerContainerCachingFactory((container) => { + const config = container.resolve(SERVICES.CONFIG); + return createDataSource(config.get('db'), SERVICE_NAME); + }), }, postInjectionHook: async (container: DependencyContainer): Promise => { const dataSource = container.resolve(DATA_SOURCE_PROVIDER); @@ -98,7 +101,7 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: HEALTHCHECK, provider: { - useFactory: healthCheckFactory, + useFactory: (container: DependencyContainer): HealthCheck => createDataSourceHealthCheck(container, [DATA_SOURCE_PROVIDER]), }, }, ]; diff --git a/packages/db/src/db/connection.ts b/packages/db/src/db/connection.ts index 290d847..c093184 100644 --- a/packages/db/src/db/connection.ts +++ b/packages/db/src/db/connection.ts @@ -1,5 +1,6 @@ import { readFileSync } from 'node:fs'; -import type { DataSource, DataSourceOptions } from 'typeorm'; +import { DataSource } from 'typeorm'; +import type { DataSourceOptions } from 'typeorm'; import type { DependencyContainer, InjectionToken } from 'tsyringe'; import { Property } from '../entities/property'; import { EnumValue } from '../entities/enumValue'; @@ -51,3 +52,6 @@ export const createDataSourceOptions = (dbConfig: DbConfig, applicationName: str ssl: createSslOptions(ssl), }; }; + +export const createDataSource = (dbConfig: DbConfig, applicationName: string): DataSource => + new DataSource(createDataSourceOptions(dbConfig, applicationName)); diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 21d0280..e0807f8 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -5,5 +5,5 @@ export type { Property as PropertySpec } from './types/property'; export { Property, PROPERTY_REPOSITORY_SYMBOL } from './entities/property'; export { EnumValue, ENUMS_REPOSITORY_SYMBOL } from './entities/enumValue'; export { ColumnType, columnType, LayerSource, layerSource } from './types/enums'; -export { DATA_SOURCE_PROVIDER, createDataSourceOptions, createDataSourceHealthCheck, createSslOptions } from './db/connection'; +export { DATA_SOURCE_PROVIDER, createDataSource, createDataSourceOptions, createDataSourceHealthCheck, createSslOptions } from './db/connection'; export { DbConfig } from './interfaces'; diff --git a/packages/synchronizer/src/common/db/connection.ts b/packages/synchronizer/src/common/db/connection.ts deleted file mode 100644 index 0073c42..0000000 --- a/packages/synchronizer/src/common/db/connection.ts +++ /dev/null @@ -1,17 +0,0 @@ -import type { FactoryFunction, DependencyContainer, InjectionToken } from 'tsyringe'; -import { DataSource } from 'typeorm'; -import { createDataSourceOptions, createDataSourceHealthCheck, type DbConfig } from '@db'; -import type { HealthCheck } from '@godaddy/terminus'; -import { DESTINATION_DATA_SOURCE_PROVIDER, DESTINATION_DB_CONFIG_PATH, SERVICE_NAME, SERVICES } from '../constants'; -import type { ConfigType } from '../config'; - -const HEALTHCHECK_TOKENS: InjectionToken[] = [DESTINATION_DATA_SOURCE_PROVIDER]; - -export const destinationDataSourceFactory: FactoryFunction = (container: DependencyContainer): DataSource => { - const config = container.resolve(SERVICES.CONFIG); - const dbConfig: DbConfig = config.get(DESTINATION_DB_CONFIG_PATH); - return new DataSource(createDataSourceOptions(dbConfig, SERVICE_NAME)); -}; - -export const healthCheckFactory: FactoryFunction = (container: DependencyContainer): HealthCheck => - createDataSourceHealthCheck(container, HEALTHCHECK_TOKENS); diff --git a/packages/synchronizer/src/containerConfig.ts b/packages/synchronizer/src/containerConfig.ts index 99def09..aafb500 100644 --- a/packages/synchronizer/src/containerConfig.ts +++ b/packages/synchronizer/src/containerConfig.ts @@ -7,14 +7,14 @@ import { jsLogger } from '@map-colonies/js-logger'; import { instancePerContainerCachingFactory } from 'tsyringe'; import { CleanupRegistry } from '@map-colonies/cleanup-registry'; import type { DataSource, Repository } from 'typeorm'; -import { DATA_SOURCE_PROVIDER as DESTINATION_DATA_SOURCE_PROVIDER } from '@db'; +import type { HealthCheck } from '@godaddy/terminus'; +import { DATA_SOURCE_PROVIDER as DESTINATION_DATA_SOURCE_PROVIDER, createDataSource, createDataSourceHealthCheck } from '@db'; import type { S3Client } from '@aws-sdk/client-s3'; import { ListBucketsCommand } from '@aws-sdk/client-s3'; import { type ConfigType, getConfig } from '@common/config'; import { type InjectionObject, registerDependencies } from '@common/dependencyRegistration'; -import { HEALTHCHECK, NAMESPACE_HANDLES, ON_SIGNAL, REPOSITORIES, SERVICES, SERVICE_NAME } from '@common/constants'; +import { DESTINATION_DB_CONFIG_PATH, HEALTHCHECK, NAMESPACE_HANDLES, ON_SIGNAL, REPOSITORIES, SERVICES, SERVICE_NAME } from '@common/constants'; import { getTracing } from '@common/tracing'; -import { destinationDataSourceFactory, healthCheckFactory } from './common/db/connection'; import { CRON_MANAGER_SYMBOL, CronManager } from './sync/cron'; import { s3ClientFactory } from './common/s3'; import { S3Repository } from './common/s3/s3Repository'; @@ -105,7 +105,12 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise }, { token: DESTINATION_DATA_SOURCE_PROVIDER, - provider: { useFactory: instancePerContainerCachingFactory(destinationDataSourceFactory) }, + provider: { + useFactory: instancePerContainerCachingFactory((container) => { + const config = container.resolve(SERVICES.CONFIG); + return createDataSource(config.get(DESTINATION_DB_CONFIG_PATH), SERVICE_NAME); + }), + }, postInjectionHook: async (deps: DependencyContainer): Promise => { const dataSource = deps.resolve(DESTINATION_DATA_SOURCE_PROVIDER); @@ -159,7 +164,7 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: HEALTHCHECK, provider: { - useFactory: healthCheckFactory, + useFactory: (container: DependencyContainer): HealthCheck => createDataSourceHealthCheck(container, [DESTINATION_DATA_SOURCE_PROVIDER]), }, }, ]; From 559f88c7ec6053b824c6754670dcce2758027f2d Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Wed, 19 Aug 2026 17:19:49 +0300 Subject: [PATCH 11/29] chore: match the key name to the one in master --- .../synchronizer/src/common/s3/s3Repository.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/synchronizer/src/common/s3/s3Repository.ts b/packages/synchronizer/src/common/s3/s3Repository.ts index 291bc41..610b2d5 100644 --- a/packages/synchronizer/src/common/s3/s3Repository.ts +++ b/packages/synchronizer/src/common/s3/s3Repository.ts @@ -40,27 +40,27 @@ export class S3Repository { ); } - public async downloadFile(bucket: string, key: string): Promise { + public async downloadFile(bucket: string, fileName: string): Promise { return startActivePromisifiedSpan( S3SpanName.S3_DOWNLOAD_FILE, - { [S3Attributes.BUCKET]: bucket, [S3Attributes.FILE_NAME]: key }, + { [S3Attributes.BUCKET]: bucket, [S3Attributes.FILE_NAME]: fileName }, contextAPI.active(), async () => { try { - this.logger.info(`Downloading ${key} file from S3 bucket ${bucket}`); + this.logger.info(`Downloading ${fileName} file from S3 bucket ${bucket}`); - const body = await this.getObjectWrapper(bucket, key); + const body = await this.getObjectWrapper(bucket, fileName); - const filePath = path.join(__dirname, 'downloads', bucket, key); + const filePath = path.join(__dirname, 'downloads', bucket, fileName); await this.fsRepository.mkdir(path.dirname(filePath)); await this.fsRepository.writeFile(filePath, await buffer(body)); - this.logger.info(`${key} file was downloaded successfully`); + this.logger.info(`${fileName} file was downloaded successfully`); return filePath; } catch (err) { - this.logger.error({ msg: `Failed to download ${key} file from S3 bucket ${bucket}`, err }); - throw new Error(`Failed to download ${key} file from S3`, { cause: err }); + this.logger.error({ msg: `Failed to download ${fileName} file from S3 bucket ${bucket}`, err }); + throw new Error(`Failed to download ${fileName} file from S3`, { cause: err }); } } ); From 04b1781cb5a6305dc52c0e8a1953c9d59b217c0d Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Wed, 19 Aug 2026 17:27:35 +0300 Subject: [PATCH 12/29] chore: move json parsing to helper --- .../src/sync/layerSource/aliasJsonParser.ts | 39 +++++++++++++++++++ .../sync/layerSource/perLayerJsonSource.ts | 39 +------------------ 2 files changed, 40 insertions(+), 38 deletions(-) create mode 100644 packages/synchronizer/src/sync/layerSource/aliasJsonParser.ts diff --git a/packages/synchronizer/src/sync/layerSource/aliasJsonParser.ts b/packages/synchronizer/src/sync/layerSource/aliasJsonParser.ts new file mode 100644 index 0000000..0af882b --- /dev/null +++ b/packages/synchronizer/src/sync/layerSource/aliasJsonParser.ts @@ -0,0 +1,39 @@ +import ajvCtor from 'ajv'; +import type { PerLayerJsonSourceConfig } from '@common/interfaces'; +import { CaseInsensitiveMap } from '@src/common/caseInsensitiveMap'; + +const ajv = new ajvCtor(); +const isRecord = ajv.compile>({ type: 'object' }); +const isString = ajv.compile({ type: 'string' }); +const isArray = ajv.compile({ type: 'array' }); + +export interface ParsedDocument { + alias: string; + propertyAliases: Map; +} + +const isRecordEntry = (value: unknown): value is Record => isRecord(value); + +const buildPropertyAliases = (rawFields: unknown[], config: PerLayerJsonSourceConfig): Map => + new CaseInsensitiveMap( + rawFields + .filter(isRecordEntry) + .map((field) => ({ fieldName: field[config.fieldNameField], aliasFieldName: field[config.aliasFieldNameField] })) + .filter((field): field is { fieldName: string; aliasFieldName: string } => isString(field.fieldName) && isString(field.aliasFieldName)) + .map(({ fieldName, aliasFieldName }): [string, string] => [fieldName, aliasFieldName]) + ); + +export const parseDocument = (content: string, config: PerLayerJsonSourceConfig): ParsedDocument => { + const raw: unknown = JSON.parse(content); + if (!isRecord(raw)) { + throw new Error('layer JSON must be an object'); + } + + const alias = raw[config.aliasLayerNameField]; + const fields = raw[config.fieldsField]; + if (!isString(alias) || !isArray(fields)) { + throw new Error(`layer JSON must have a string "${config.aliasLayerNameField}" and an array "${config.fieldsField}"`); + } + + return { alias, propertyAliases: buildPropertyAliases(fields, config) }; +}; diff --git a/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts b/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts index 029ee5c..101acc6 100644 --- a/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts +++ b/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts @@ -1,47 +1,10 @@ -import ajvCtor from 'ajv'; import type { Logger } from '@map-colonies/js-logger'; import { layerSource } from '@db'; import type { LayerEnums, PerLayerJsonSourceConfig } from '@common/interfaces'; -import { CaseInsensitiveMap } from '@src/common/caseInsensitiveMap'; import type { FsRepository } from '@common/fs/fsRepository'; import { LayerJsonFetchError, LayerJsonParseError } from '../errors'; import type { FileDownloader, LayerSourceRecord, LayerSourceStrategy } from './types'; - -const ajv = new ajvCtor(); -const isRecord = ajv.compile>({ type: 'object' }); -const isString = ajv.compile({ type: 'string' }); -const isArray = ajv.compile({ type: 'array' }); - -interface ParsedDocument { - alias: string; - propertyAliases: Map; -} - -const isRecordEntry = (value: unknown): value is Record => isRecord(value); - -const buildPropertyAliases = (rawFields: unknown[], config: PerLayerJsonSourceConfig): Map => - new CaseInsensitiveMap( - rawFields - .filter(isRecordEntry) - .map((field) => ({ fieldName: field[config.fieldNameField], aliasFieldName: field[config.aliasFieldNameField] })) - .filter((field): field is { fieldName: string; aliasFieldName: string } => isString(field.fieldName) && isString(field.aliasFieldName)) - .map(({ fieldName, aliasFieldName }): [string, string] => [fieldName, aliasFieldName]) - ); - -const parseDocument = (content: string, config: PerLayerJsonSourceConfig): ParsedDocument => { - const raw: unknown = JSON.parse(content); - if (!isRecord(raw)) { - throw new Error('layer JSON must be an object'); - } - - const alias = raw[config.aliasLayerNameField]; - const fields = raw[config.fieldsField]; - if (!isString(alias) || !isArray(fields)) { - throw new Error(`layer JSON must have a string "${config.aliasLayerNameField}" and an array "${config.fieldsField}"`); - } - - return { alias, propertyAliases: buildPropertyAliases(fields, config) }; -}; +import { parseDocument } from './aliasJsonParser'; const resolveOne = async ( layer: LayerEnums, From 8d115e8725249dc2f8321182fee1ae399824ea25 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Wed, 19 Aug 2026 17:30:08 +0300 Subject: [PATCH 13/29] chore: move exports to end of file --- .../src/sync/layerSource/aliasJsonParser.ts | 10 +++++----- .../src/sync/layerSource/perLayerJsonSource.ts | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/synchronizer/src/sync/layerSource/aliasJsonParser.ts b/packages/synchronizer/src/sync/layerSource/aliasJsonParser.ts index 0af882b..71ac92f 100644 --- a/packages/synchronizer/src/sync/layerSource/aliasJsonParser.ts +++ b/packages/synchronizer/src/sync/layerSource/aliasJsonParser.ts @@ -7,11 +7,6 @@ const isRecord = ajv.compile>({ type: 'object' }); const isString = ajv.compile({ type: 'string' }); const isArray = ajv.compile({ type: 'array' }); -export interface ParsedDocument { - alias: string; - propertyAliases: Map; -} - const isRecordEntry = (value: unknown): value is Record => isRecord(value); const buildPropertyAliases = (rawFields: unknown[], config: PerLayerJsonSourceConfig): Map => @@ -23,6 +18,11 @@ const buildPropertyAliases = (rawFields: unknown[], config: PerLayerJsonSourceCo .map(({ fieldName, aliasFieldName }): [string, string] => [fieldName, aliasFieldName]) ); +export interface ParsedDocument { + alias: string; + propertyAliases: Map; +} + export const parseDocument = (content: string, config: PerLayerJsonSourceConfig): ParsedDocument => { const raw: unknown = JSON.parse(content); if (!isRecord(raw)) { diff --git a/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts b/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts index 101acc6..0c5c324 100644 --- a/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts +++ b/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts @@ -29,7 +29,6 @@ const resolveOne = async ( const content = (await fsRepository.readFile(filePath, 'utf-8')).toString(); const document = parseDocument(content, config); return { - layerId: undefined, alias: document.alias, propertyAliases: document.propertyAliases, source: layerSource.perLayerJson, From dfd9c9fb12f2fe4d8fc15b7e485470e2058b27cc Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Wed, 19 Aug 2026 18:08:56 +0300 Subject: [PATCH 14/29] fix: fix bad s3 repo injection --- packages/synchronizer/package.json | 2 +- packages/synchronizer/src/containerConfig.ts | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/packages/synchronizer/package.json b/packages/synchronizer/package.json index 9d35951..9e8ff0c 100644 --- a/packages/synchronizer/package.json +++ b/packages/synchronizer/package.json @@ -9,7 +9,7 @@ "start": "npm run build && node dist/src/index.js", "start:dev": "npm run build && cd dist && cross-env CONFIG_OFFLINE_MODE=true node --enable-source-maps --import ./instrumentation.mjs ./src/index.js", "dev": "ts-node src/index.ts", - "assets:copy": "copyfiles -u 1 ./config/**/* ./dist/config && copyfiles -f copyfiles ./package.json dist", + "assets:copy": "copyfiles -u 1 \"./config/**/*\" ./dist/config && copyfiles -f copyfiles ./package.json dist", "watch": "ts-node-dev --respawn src/index.ts", "test:integration": "vitest run --coverage.enabled=false --project integration", "test": "vitest run", diff --git a/packages/synchronizer/src/containerConfig.ts b/packages/synchronizer/src/containerConfig.ts index aafb500..a7dbb30 100644 --- a/packages/synchronizer/src/containerConfig.ts +++ b/packages/synchronizer/src/containerConfig.ts @@ -4,7 +4,7 @@ import { Registry } from 'prom-client'; import type { DependencyContainer } from 'tsyringe/dist/typings/types'; import type { Logger } from '@map-colonies/js-logger'; import { jsLogger } from '@map-colonies/js-logger'; -import { instancePerContainerCachingFactory } from 'tsyringe'; +import { Lifecycle, instancePerContainerCachingFactory } from 'tsyringe'; import { CleanupRegistry } from '@map-colonies/cleanup-registry'; import type { DataSource, Repository } from 'typeorm'; import type { HealthCheck } from '@godaddy/terminus'; @@ -97,11 +97,8 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise }, { token: S3Repository, - provider: { useFactory: instancePerContainerCachingFactory((container) => container.resolve(S3Repository)) }, - postInjectionHook: (deps: DependencyContainer): void => { - const s3Repository = deps.resolve(S3Repository); - deps.register(S3Repository, { useValue: s3Repository }); - }, + provider: { useClass: S3Repository }, + options: { lifecycle: Lifecycle.Singleton }, }, { token: DESTINATION_DATA_SOURCE_PROVIDER, From 8878f52584866262e8e15990ba8f471c5d4ee46d Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Wed, 19 Aug 2026 19:22:35 +0300 Subject: [PATCH 15/29] feat: upgrade namespace dependency registration --- packages/synchronizer/src/containerConfig.ts | 13 +--- .../synchronizer/src/sync/namespaceHandle.ts | 67 +++++++++++-------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/packages/synchronizer/src/containerConfig.ts b/packages/synchronizer/src/containerConfig.ts index a7dbb30..9d7da06 100644 --- a/packages/synchronizer/src/containerConfig.ts +++ b/packages/synchronizer/src/containerConfig.ts @@ -18,8 +18,7 @@ import { getTracing } from '@common/tracing'; import { CRON_MANAGER_SYMBOL, CronManager } from './sync/cron'; import { s3ClientFactory } from './common/s3'; import { S3Repository } from './common/s3/s3Repository'; -import { FsRepository } from './common/fs/fsRepository'; -import { buildNamespaceHandle } from './sync/namespaceHandle'; +import { createNamespaceHandles } from './sync/namespaceHandle'; export interface RegisterOptions { override?: InjectionObject[]; @@ -135,15 +134,7 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: NAMESPACE_HANDLES, provider: { - useFactory: instancePerContainerCachingFactory((container) => { - const config = container.resolve(SERVICES.CONFIG); - const logger = container.resolve(SERVICES.LOGGER); - const s3Repository = container.resolve(S3Repository); - const fsRepository = container.resolve(FsRepository); - return config - .get('namespaces') - .map((namespaceConfig) => buildNamespaceHandle(container, namespaceConfig, s3Repository, fsRepository, logger, cleanupRegistry)); - }), + useFactory: instancePerContainerCachingFactory((container) => createNamespaceHandles(container, cleanupRegistry)), }, }, { diff --git a/packages/synchronizer/src/sync/namespaceHandle.ts b/packages/synchronizer/src/sync/namespaceHandle.ts index b0725c7..1efc3e8 100644 --- a/packages/synchronizer/src/sync/namespaceHandle.ts +++ b/packages/synchronizer/src/sync/namespaceHandle.ts @@ -1,12 +1,20 @@ -import type { Logger } from '@map-colonies/js-logger'; import type { CleanupRegistry } from '@map-colonies/cleanup-registry'; -import { DataSource } from 'typeorm'; +import type { Logger } from '@map-colonies/js-logger'; import type { DependencyContainer } from 'tsyringe/dist/typings/types'; -import { createDataSourceOptions } from '@db'; -import type { NamespaceConfig } from '@common/interfaces'; -import { SERVICE_NAME, SOURCE_DATA_SOURCE_PROVIDER } from '@common/constants'; -import type { S3Repository } from '@common/s3/s3Repository'; -import type { FsRepository } from '@common/fs/fsRepository'; +import type { DataSource, Repository } from 'typeorm'; +import { + createDataSource, + ENUMS_REPOSITORY_SYMBOL, + type EnumValue, + type Layer, + LAYER_REPOSITORY_SYMBOL, + type Property, + PROPERTY_REPOSITORY_SYMBOL, +} from '@db'; +import type { ConfigType } from '@common/config'; +import { SERVICES, SERVICE_NAME, SOURCE_DATA_SOURCE_PROVIDER } from '@common/constants'; +import { S3Repository } from '@common/s3/s3Repository'; +import { FsRepository } from '@common/fs/fsRepository'; import { SyncModel } from './SyncModel'; import { createLayerSourceStrategy } from './layerSource'; import type { LayerSourceStrategy } from './layerSource/types'; @@ -19,29 +27,30 @@ export interface NamespaceHandle { dal: SyncModel; } -export const buildNamespaceHandle = ( - parent: DependencyContainer, - namespaceConfig: NamespaceConfig, - s3Repository: S3Repository, - fsRepository: FsRepository, - logger: Logger, - cleanupRegistry: CleanupRegistry -): NamespaceHandle => { - const child = parent.createChildContainer(); - const sourceDataSource = new DataSource(createDataSourceOptions(namespaceConfig.db, `${SERVICE_NAME}-${namespaceConfig.name}`)); - child.register(SOURCE_DATA_SOURCE_PROVIDER, { useValue: sourceDataSource }); +export const createNamespaceHandles = (container: DependencyContainer, cleanupRegistry: CleanupRegistry): NamespaceHandle[] => { + const config = container.resolve(SERVICES.CONFIG); + const logger = container.resolve(SERVICES.LOGGER); + const s3Repository = container.resolve(S3Repository); + const fsRepository = container.resolve(FsRepository); + const layerRepository = container.resolve>(LAYER_REPOSITORY_SYMBOL); + const propertyRepository = container.resolve>(PROPERTY_REPOSITORY_SYMBOL); + const enumsRepository = container.resolve>(ENUMS_REPOSITORY_SYMBOL); - const layerSource = createLayerSourceStrategy(namespaceConfig, s3Repository, fsRepository, logger); - const dal = child.resolve(SyncModel); + return config.get('namespaces').map((namespaceConfig) => { + const sourceDataSource = createDataSource(namespaceConfig.db, `${SERVICE_NAME}-${namespaceConfig.name}`); - cleanupRegistry.register({ - id: `${SOURCE_DATA_SOURCE_PROVIDER.toString()}:${namespaceConfig.name}`, - func: async () => { - if (sourceDataSource.isInitialized) { - await sourceDataSource.destroy(); - } - }, - }); + cleanupRegistry.register({ + id: `${SOURCE_DATA_SOURCE_PROVIDER.toString()}:${namespaceConfig.name}`, + func: async () => { + if (sourceDataSource.isInitialized) { + await sourceDataSource.destroy(); + } + }, + }); - return { name: namespaceConfig.name, sourceDataSource, layerSource, layersFile: namespaceConfig.layersFile, dal }; + const layerSource = createLayerSourceStrategy(namespaceConfig, s3Repository, fsRepository, logger); + const dal = new SyncModel(config, logger, sourceDataSource, layerRepository, propertyRepository, enumsRepository); + + return { name: namespaceConfig.name, sourceDataSource, layerSource, layersFile: namespaceConfig.layersFile, dal }; + }); }; From c7860b23266424df9d1d2d095810b5f2dde9e95c Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Sun, 23 Aug 2026 17:48:27 +0300 Subject: [PATCH 16/29] feat: make api configurable with namespaces --- openapi3.yaml | 39 +++++++++++++------- packages/api/src/common/config.ts | 6 ++-- packages/api/src/common/constants.ts | 11 ++++++ packages/api/src/common/interfaces.ts | 16 +++++++++ packages/api/src/common/openapi.ts | 51 +++++++++++++++++++++++++++ packages/api/src/containerConfig.ts | 2 ++ packages/api/src/docs/routes/docs.ts | 36 +++++++++++++++++++ packages/api/src/serverBuilder.ts | 20 ++++------- 8 files changed, 152 insertions(+), 29 deletions(-) create mode 100644 packages/api/src/common/openapi.ts create mode 100644 packages/api/src/docs/routes/docs.ts diff --git a/openapi3.yaml b/openapi3.yaml index 8889fb7..9278bf8 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -2,47 +2,58 @@ openapi: 3.0.3 info: title: Layers & Specs API description: > - API for retrieving geographic layer specifications. Layers expose a set of - properties; `enum`-typed properties also carry their list of allowed values. + API for retrieving geographic layer specifications. Layers expose a set of properties; `enum`-typed properties also carry their list of allowed values. version: 2.0.0 license: name: Proprietary url: https://example.com/licenses/proprietary +security: + - ApiKeyAuth: [] paths: /namespaces: get: summary: Get all namespaces operationId: getNamespaces - security: [] responses: '200': $ref: '#/components/responses/NamespaceListResponse' + '401': + $ref: '#/components/responses/Unauthorized' /namespaces/{namespace}/layers: get: summary: Get all layer names within a namespace operationId: getLayers - security: [] parameters: - $ref: '#/components/parameters/NamespaceParam' responses: '200': $ref: '#/components/responses/LayerListResponse' + '401': + $ref: '#/components/responses/Unauthorized' '404': $ref: '#/components/responses/NotFound' /namespaces/{namespace}/layers/{layerName}: get: summary: Get layer spec by name within a namespace operationId: getLayerByName - security: [] parameters: - $ref: '#/components/parameters/NamespaceParam' - $ref: '#/components/parameters/LayerNameParam' responses: '200': $ref: '#/components/responses/LayerSpecResponse' + '401': + $ref: '#/components/responses/Unauthorized' '404': $ref: '#/components/responses/NotFound' components: + securitySchemes: + ApiKeyAuth: + type: apiKey + in: header + name: x-api-key + description: > + Enforced by the API gateway, not by this service. The service itself never inspects the key - it only declares the requirement so that clients, and the "Authorize" dialog on this page, send it. parameters: NamespaceParam: name: namespace @@ -111,6 +122,13 @@ components: application/json: schema: $ref: '#/components/schemas/Error' + Unauthorized: + description: > + The `x-api-key` header was missing or rejected. Returned by the API gateway, which never forwards the request to this service. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' schemas: NamespaceSummary: type: object @@ -160,8 +178,7 @@ components: LayerSpec: type: object description: > - A layer and all of its properties. Groups rows from the `Property` - entity by `layerName`. + A layer and all of its properties. Groups rows from the `Property` entity by `layerName`. properties: layerName: type: string @@ -182,10 +199,7 @@ components: Property: type: object description: > - A single property definition for a layer. Mirrors the `Property` entity, - minus `id` and `layerName` (which are implicit on the parent - `LayerSpec`). When the property has coded/enumerated values, - `possibleValues` is populated. + A single property definition for a layer. Mirrors the `Property` entity, minus `id` and `layerName` (which are implicit on the parent `LayerSpec`). When the property has coded/enumerated values, `possibleValues` is populated. properties: property: type: string @@ -200,8 +214,7 @@ components: possibleValues: type: array description: > - Allowed values for properties that have a fixed set of coded values. - Omitted when the property accepts arbitrary values. + Allowed values for properties that have a fixed set of coded values. Omitted when the property accepts arbitrary values. items: type: string required: diff --git a/packages/api/src/common/config.ts b/packages/api/src/common/config.ts index 5e577e4..6d16876 100644 --- a/packages/api/src/common/config.ts +++ b/packages/api/src/common/config.ts @@ -1,8 +1,8 @@ import { type ConfigInstance, config } from '@map-colonies/config'; -import { vectorVectorStandardApiV1, type vectorVectorStandardApiV1Type } from '@map-colonies/schemas'; +import { vectorVectorStandardApiV2, type vectorVectorStandardApiV2Type } from '@map-colonies/schemas'; // Choose here the type of the config instance and import this type from the entire application -type ConfigType = ConfigInstance; +type ConfigType = ConfigInstance; let configInstance: ConfigType | undefined; @@ -13,7 +13,7 @@ let configInstance: ConfigType | undefined; */ async function initConfig(offlineMode?: boolean): Promise { configInstance = await config({ - schema: vectorVectorStandardApiV1, + schema: vectorVectorStandardApiV2, offlineMode, }); } diff --git a/packages/api/src/common/constants.ts b/packages/api/src/common/constants.ts index 65754cc..d1abfad 100644 --- a/packages/api/src/common/constants.ts +++ b/packages/api/src/common/constants.ts @@ -1,10 +1,21 @@ import { readPackageJsonSync } from '@map-colonies/read-pkg'; +import type { PublicPath } from './interfaces'; export const SERVICE_NAME = readPackageJsonSync().name ?? 'unknown_service'; export const IGNORED_OUTGOING_TRACE_ROUTES = [/^.*\/v1\/metrics.*$/]; export const IGNORED_INCOMING_TRACE_ROUTES = [/^.*\/docs.*$/]; +export const PREFIX_HEADER = 'x-forwarded-prefix'; +export const ROOT_HEADER = 'x-gateway-root'; +export const NAMESPACE_PARAM_REF = '#/components/parameters/NamespaceParam'; + +export const PUBLIC_PATHS = new Map([ + ['/namespaces', { path: '/namespaces', scope: 'root' }], + ['/namespaces/{namespace}/layers', { path: '/', scope: 'namespaced' }], + ['/namespaces/{namespace}/layers/{layerName}', { path: '/{layerName}', scope: 'namespaced' }], +]); + export const HEALTHCHECK = Symbol('HealthCheck'); export const ON_SIGNAL = Symbol('onSignal'); diff --git a/packages/api/src/common/interfaces.ts b/packages/api/src/common/interfaces.ts index d27d733..841a161 100644 --- a/packages/api/src/common/interfaces.ts +++ b/packages/api/src/common/interfaces.ts @@ -1,6 +1,22 @@ +import type { OpenAPIV3 } from 'openapi-types'; + export interface OpenApiConfig { filePath: string; basePath: string; jsonPath: string; uiPath: string; } + +export type OpenapiSpec = OpenAPIV3.Document; + +export interface GatewayBases { + namespaced: string; + root: string; +} + +export interface PublicPath { + path: string; + scope: keyof GatewayBases; +} + +export type PathRewrite = (pathItem: OpenAPIV3.PathItemObject, bases: GatewayBases) => OpenAPIV3.PathItemObject; diff --git a/packages/api/src/common/openapi.ts b/packages/api/src/common/openapi.ts new file mode 100644 index 0000000..5bab06f --- /dev/null +++ b/packages/api/src/common/openapi.ts @@ -0,0 +1,51 @@ +import { readFileSync } from 'node:fs'; +import { load } from 'js-yaml'; +import { OpenAPIV3 } from 'openapi-types'; +import { NAMESPACE_PARAM_REF, PUBLIC_PATHS } from './constants'; +import type { GatewayBases, OpenapiSpec, PathRewrite } from './interfaces'; + +const stripNamespaceParam = (pathItem: OpenAPIV3.PathItemObject): OpenAPIV3.PathItemObject => { + const stripped: OpenAPIV3.PathItemObject = { ...pathItem }; + + for (const method of Object.values(OpenAPIV3.HttpMethods)) { + const operation = stripped[method]; + + if (operation?.parameters === undefined) { + continue; + } + + stripped[method] = { + ...operation, + parameters: operation.parameters.filter((param) => !('$ref' in param) || param.$ref !== NAMESPACE_PARAM_REF), + }; + } + + return stripped; +}; + +const SCOPE_REWRITES = { + namespaced: stripNamespaceParam, + root: (pathItem, bases): OpenAPIV3.PathItemObject => ({ ...pathItem, servers: [{ url: bases.root }] }), +} satisfies Record; + +const loadSpec = (filePath: string): OpenapiSpec => load(readFileSync(filePath, 'utf8')) as OpenapiSpec; + +const toPublicSpec = (spec: OpenapiSpec, bases: GatewayBases): OpenapiSpec => { + const paths = Object.entries(spec.paths).flatMap(([path, pathItem]): [string, OpenAPIV3.PathItemObject][] => { + const publicPath = PUBLIC_PATHS.get(path); + + if (publicPath === undefined || pathItem === undefined) { + return []; + } + + return [[publicPath.path, SCOPE_REWRITES[publicPath.scope](pathItem, bases)]]; + }); + + return { + ...spec, + servers: [{ url: bases.namespaced, description: 'The API gateway. It rewrites these URLs onto the routes the service serves' }], + paths: Object.fromEntries(paths), + }; +}; + +export { loadSpec, toPublicSpec }; diff --git a/packages/api/src/containerConfig.ts b/packages/api/src/containerConfig.ts index 5e579db..967f8e2 100644 --- a/packages/api/src/containerConfig.ts +++ b/packages/api/src/containerConfig.ts @@ -13,6 +13,7 @@ import { HEALTHCHECK, ON_SIGNAL, SERVICES, SERVICE_NAME } from '@common/constant import { getTracing } from '@common/tracing'; import { LAYER_ROUTER_SYMBOL, layerRouterFactory } from './layer/routes/layer'; import { NAMESPACE_ROUTER_SYMBOL, namespaceRouterFactory } from './namespace/routes/namespace'; +import { DOCS_ROUTER_SYMBOL, docsRouterFactory } from './docs/routes/docs'; import { type ConfigType, getConfig } from './common/config'; export interface RegisterOptions { @@ -68,6 +69,7 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise }, { token: LAYER_ROUTER_SYMBOL, provider: { useFactory: layerRouterFactory } }, { token: NAMESPACE_ROUTER_SYMBOL, provider: { useFactory: namespaceRouterFactory } }, + { token: DOCS_ROUTER_SYMBOL, provider: { useFactory: docsRouterFactory } }, { token: LAYER_REPOSITORY_SYMBOL, provider: { diff --git a/packages/api/src/docs/routes/docs.ts b/packages/api/src/docs/routes/docs.ts new file mode 100644 index 0000000..655487d --- /dev/null +++ b/packages/api/src/docs/routes/docs.ts @@ -0,0 +1,36 @@ +import { Router } from 'express'; +import type { Request, RequestHandler } from 'express'; +import type { FactoryFunction } from 'tsyringe'; +import { OpenapiViewerRouter } from '@map-colonies/openapi-express-viewer'; +import { PREFIX_HEADER, ROOT_HEADER, SERVICES } from '@common/constants'; +import { loadSpec, toPublicSpec } from '@common/openapi'; +import type { OpenapiSpec } from '@common/interfaces'; +import type { ConfigType } from '@common/config'; + +const docsRouterFactory: FactoryFunction = (dependencyContainer) => { + const config = dependencyContainer.resolve(SERVICES.CONFIG); + const openapiConfig = config.get('openapiConfig'); + const spec = loadSpec(openapiConfig.filePath); + + const viewerRouter = new OpenapiViewerRouter({ ...openapiConfig, filePathOrSpec: spec }); + viewerRouter.setup(); + + const usePublicSpec: RequestHandler = (req, res, next) => { + const namespaced = req.get(PREFIX_HEADER); + const root = req.get(ROOT_HEADER); + const behindGateway = namespaced !== undefined && root !== undefined; + + (req as Request & { swaggerDoc?: OpenapiSpec }).swaggerDoc = behindGateway ? toPublicSpec(spec, { namespaced, root }) : spec; + + next(); + }; + + const router = Router(); + router.use(usePublicSpec, viewerRouter.getRouter()); + + return router; +}; + +const DOCS_ROUTER_SYMBOL = Symbol('docsRouterFactory'); + +export { docsRouterFactory, DOCS_ROUTER_SYMBOL }; diff --git a/packages/api/src/serverBuilder.ts b/packages/api/src/serverBuilder.ts index b56043d..8e9880c 100644 --- a/packages/api/src/serverBuilder.ts +++ b/packages/api/src/serverBuilder.ts @@ -1,7 +1,6 @@ import express, { Router } from 'express'; import bodyParser from 'body-parser'; import compression from 'compression'; -import { OpenapiViewerRouter } from '@map-colonies/openapi-express-viewer'; import { getErrorHandlerMiddleware } from '@map-colonies/error-express-handler'; import { middleware as OpenApiMiddleware } from 'express-openapi-validator'; import { inject, injectable } from 'tsyringe'; @@ -12,6 +11,7 @@ import { Registry } from 'prom-client'; import { SERVICES } from '@common/constants'; import { LAYER_ROUTER_SYMBOL } from './layer/routes/layer'; import { NAMESPACE_ROUTER_SYMBOL } from './namespace/routes/namespace'; +import { DOCS_ROUTER_SYMBOL } from './docs/routes/docs'; import { ConfigType } from './common/config'; @injectable() @@ -23,7 +23,8 @@ export class ServerBuilder { @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.METRICS) private readonly metricsRegistry: Registry, @inject(LAYER_ROUTER_SYMBOL) private readonly layerRouter: Router, - @inject(NAMESPACE_ROUTER_SYMBOL) private readonly namespaceRouter: Router + @inject(NAMESPACE_ROUTER_SYMBOL) private readonly namespaceRouter: Router, + @inject(DOCS_ROUTER_SYMBOL) private readonly docsRouter: Router ) { this.serverInstance = express(); } @@ -36,19 +37,10 @@ export class ServerBuilder { return this.serverInstance; } - private buildDocsRoutes(): void { - const openapiRouter = new OpenapiViewerRouter({ - ...this.config.get('openapiConfig'), - filePathOrSpec: this.config.get('openapiConfig.filePath'), - }); - openapiRouter.setup(); - this.serverInstance.use(this.config.get('openapiConfig.basePath'), openapiRouter.getRouter()); - } - private buildRoutes(): void { this.serverInstance.use('/namespaces', this.namespaceRouter); this.serverInstance.use('/namespaces', this.layerRouter); - this.buildDocsRoutes(); + this.serverInstance.use(this.config.get('openapiConfig.basePath'), this.docsRouter); } private registerPreRoutesMiddleware(): void { @@ -63,7 +55,9 @@ export class ServerBuilder { const ignorePathRegex = new RegExp(`^${this.config.get('openapiConfig.basePath')}/.*`, 'i'); const apiSpecPath = this.config.get('openapiConfig.filePath'); - this.serverInstance.use(OpenApiMiddleware({ apiSpec: apiSpecPath, validateRequests: true, ignorePaths: ignorePathRegex })); + this.serverInstance.use( + OpenApiMiddleware({ apiSpec: apiSpecPath, validateRequests: true, validateSecurity: false, ignorePaths: ignorePathRegex }) + ); } private registerPostRoutesMiddleware(): void { From ec5ac6f90266150dbc7f0120c8c06fc68d698912 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Sun, 23 Aug 2026 17:49:16 +0300 Subject: [PATCH 17/29] feat: add tests to openapi --- package-lock.json | 331 +++++------------- package.json | 2 +- packages/api/config/test.json | 1 + packages/api/package.json | 5 +- .../api/tests/unit/common/openapi.spec.ts | 44 +++ 5 files changed, 136 insertions(+), 247 deletions(-) create mode 100644 packages/api/tests/unit/common/openapi.spec.ts diff --git a/package-lock.json b/package-lock.json index f946a30..69e0e33 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "packages/*" ], "dependencies": { - "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-11bf4605d7a17c723dd85f624c88fe32d9957863.tgz" + "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-3c2d0665e3509bb3e7113b328fc224fe6950895a.tgz" }, "devDependencies": { "@commitlint/cli": "^20.4.1", @@ -345,7 +345,6 @@ "version": "7.29.7", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6.9.0" } @@ -384,7 +383,6 @@ "version": "6.3.1", "dev": true, "license": "ISC", - "peer": true, "bin": { "semver": "bin/semver.js" } @@ -393,7 +391,6 @@ "version": "7.29.7", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/parser": "^7.29.7", "@babel/types": "^7.29.7", @@ -409,7 +406,6 @@ "version": "7.29.7", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/compat-data": "^7.29.7", "@babel/helper-validator-option": "^7.29.7", @@ -425,7 +421,6 @@ "version": "6.3.1", "dev": true, "license": "ISC", - "peer": true, "bin": { "semver": "bin/semver.js" } @@ -434,7 +429,6 @@ "version": "7.29.7", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6.9.0" } @@ -443,7 +437,6 @@ "version": "7.29.7", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/traverse": "^7.29.7", "@babel/types": "^7.29.7" @@ -456,7 +449,6 @@ "version": "7.29.7", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-module-imports": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7", @@ -473,7 +465,6 @@ "version": "7.29.7", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6.9.0" } @@ -497,7 +488,6 @@ "version": "7.29.7", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6.9.0" } @@ -506,7 +496,6 @@ "version": "7.29.7", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/template": "^7.29.7", "@babel/types": "^7.29.7" @@ -533,7 +522,6 @@ "version": "7.8.4", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -545,7 +533,6 @@ "version": "7.8.3", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -557,7 +544,6 @@ "version": "7.12.13", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.12.13" }, @@ -569,7 +555,6 @@ "version": "7.14.5", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.14.5" }, @@ -584,7 +569,6 @@ "version": "7.29.7", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.29.7" }, @@ -599,7 +583,6 @@ "version": "7.10.4", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.10.4" }, @@ -611,7 +594,6 @@ "version": "7.8.3", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -623,7 +605,6 @@ "version": "7.29.7", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.29.7" }, @@ -638,7 +619,6 @@ "version": "7.10.4", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.10.4" }, @@ -650,7 +630,6 @@ "version": "7.8.3", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -662,7 +641,6 @@ "version": "7.10.4", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.10.4" }, @@ -674,7 +652,6 @@ "version": "7.8.3", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -686,7 +663,6 @@ "version": "7.8.3", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -698,7 +674,6 @@ "version": "7.8.3", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -710,7 +685,6 @@ "version": "7.14.5", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.14.5" }, @@ -725,7 +699,6 @@ "version": "7.14.5", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.14.5" }, @@ -740,7 +713,6 @@ "version": "7.29.7", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.29.7" }, @@ -762,7 +734,6 @@ "version": "7.29.7", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/parser": "^7.29.7", @@ -776,7 +747,6 @@ "version": "7.29.7", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", @@ -1159,31 +1129,6 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, - "node_modules/@emnapi/core": { - "version": "1.11.3", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.3.tgz", - "integrity": "sha512-zLpS5asjEb7lq8jYLq37N6XKaE41DIexlY1rF/z4/tIl3wo13Sqm28fRyfIsKZD+NZ8mM5RoKkpW/rBcuoSZSg==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.3", - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.11.3", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", - "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@emnapi/wasi-threads": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.3.tgz", @@ -1191,7 +1136,6 @@ "dev": true, "license": "MIT", "optional": true, - "peer": true, "dependencies": { "tslib": "^2.4.0" } @@ -1609,7 +1553,6 @@ "version": "1.1.0", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "camelcase": "^5.3.1", "find-up": "^4.1.0", @@ -1625,7 +1568,6 @@ "version": "1.0.10", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "sprintf-js": "~1.0.2" } @@ -1634,7 +1576,6 @@ "version": "4.1.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" @@ -1647,7 +1588,6 @@ "version": "3.15.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" @@ -1660,7 +1600,6 @@ "version": "5.0.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "p-locate": "^4.1.0" }, @@ -1672,7 +1611,6 @@ "version": "2.3.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "p-try": "^2.0.0" }, @@ -1687,7 +1625,6 @@ "version": "4.1.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "p-limit": "^2.2.0" }, @@ -1699,7 +1636,6 @@ "version": "0.1.6", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -1708,7 +1644,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/types": "30.4.1", "@types/node": "*", @@ -1725,7 +1660,6 @@ "version": "30.4.2", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/console": "30.4.1", "@jest/pattern": "30.4.0", @@ -1772,7 +1706,6 @@ "version": "30.4.0", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } @@ -1781,7 +1714,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/fake-timers": "30.4.1", "@jest/types": "30.4.1", @@ -1796,7 +1728,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "expect": "30.4.1", "jest-snapshot": "30.4.1" @@ -1809,7 +1740,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/get-type": "30.1.0" }, @@ -1821,7 +1751,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/types": "30.4.1", "@sinonjs/fake-timers": "^15.4.0", @@ -1838,7 +1767,6 @@ "version": "30.1.0", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } @@ -1847,7 +1775,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/environment": "30.4.1", "@jest/expect": "30.4.1", @@ -1862,7 +1789,6 @@ "version": "30.4.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/node": "*", "jest-regex-util": "30.4.0" @@ -1875,7 +1801,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@bcoe/v8-coverage": "^0.2.3", "@jest/console": "30.4.1", @@ -1916,20 +1841,17 @@ "node_modules/@jest/reporters/node_modules/@bcoe/v8-coverage": { "version": "0.2.3", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@jest/reporters/node_modules/balanced-match": { "version": "1.0.2", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@jest/reporters/node_modules/brace-expansion": { "version": "2.1.2", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "balanced-match": "^1.0.0" } @@ -1938,7 +1860,6 @@ "version": "10.5.0", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", @@ -1958,7 +1879,6 @@ "version": "9.0.9", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "brace-expansion": "^2.0.2" }, @@ -1973,7 +1893,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@sinclair/typebox": "^0.34.0" }, @@ -1985,7 +1904,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/types": "30.4.1", "chalk": "^4.1.2", @@ -2000,7 +1918,6 @@ "version": "30.0.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jridgewell/trace-mapping": "^0.3.25", "callsites": "^3.1.0", @@ -2014,7 +1931,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/console": "30.4.1", "@jest/types": "30.4.1", @@ -2029,7 +1945,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/test-result": "30.4.1", "graceful-fs": "^4.2.11", @@ -2044,7 +1959,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/core": "^7.27.4", "@jest/types": "30.4.1", @@ -2069,7 +1983,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/pattern": "30.4.0", "@jest/schemas": "30.4.1", @@ -2087,7 +2000,6 @@ "version": "0.3.13", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" @@ -2097,7 +2009,6 @@ "version": "2.3.5", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" @@ -3332,9 +3243,10 @@ }, "node_modules/@map-colonies/schemas": { "version": "1.26.0", - "resolved": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-11bf4605d7a17c723dd85f624c88fe32d9957863.tgz", - "integrity": "sha512-jGavRY1lGItcyyNlaO/Sa4hpV1TZ15NXH2vFbcZnNlNmHtFtU3ZnGqLpRZtSOxAbBnQY7xga4LRGZpZea+c0VQ==", - "license": "MIT" + "resolved": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-3c2d0665e3509bb3e7113b328fc224fe6950895a.tgz", + "integrity": "sha512-UmMUnSTVagpHi0IwcBG7Vf7GuWothyKa3L/BGDrplR2mfG0DxTq9GtwImx24I/qVzEtqF5OO1WapeLqyIgRpMw==", + "license": "MIT", + "peer": true }, "node_modules/@map-colonies/tracing": { "version": "1.0.0", @@ -4214,6 +4126,7 @@ "version": "4.2.4", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@octokit/auth-token": "^3.0.0", "@octokit/graphql": "^5.0.0", @@ -4399,6 +4312,7 @@ "node_modules/@opentelemetry/api": { "version": "1.9.1", "license": "Apache-2.0", + "peer": true, "engines": { "node": ">=8.0.0" } @@ -4488,6 +4402,7 @@ "node_modules/@opentelemetry/core": { "version": "2.9.0", "license": "Apache-2.0", + "peer": true, "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, @@ -6381,7 +6296,6 @@ "node_modules/@redocly/ajv": { "version": "8.11.2", "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "json-schema-traverse": "^1.0.0", @@ -6408,13 +6322,11 @@ }, "node_modules/@redocly/config": { "version": "0.22.0", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@redocly/openapi-core": { "version": "1.34.17", "license": "MIT", - "peer": true, "dependencies": { "@redocly/ajv": "8.11.2", "@redocly/config": "0.22.0", @@ -6434,20 +6346,17 @@ "node_modules/@redocly/openapi-core/node_modules/agent-base": { "version": "7.1.4", "license": "MIT", - "peer": true, "engines": { "node": ">= 14" } }, "node_modules/@redocly/openapi-core/node_modules/balanced-match": { "version": "1.0.2", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@redocly/openapi-core/node_modules/brace-expansion": { "version": "2.1.2", "license": "MIT", - "peer": true, "dependencies": { "balanced-match": "^1.0.0" } @@ -6455,7 +6364,6 @@ "node_modules/@redocly/openapi-core/node_modules/https-proxy-agent": { "version": "7.0.6", "license": "MIT", - "peer": true, "dependencies": { "agent-base": "^7.1.2", "debug": "4" @@ -6477,7 +6385,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "argparse": "^2.0.1" }, @@ -6488,7 +6395,6 @@ "node_modules/@redocly/openapi-core/node_modules/minimatch": { "version": "5.1.9", "license": "ISC", - "peer": true, "dependencies": { "brace-expansion": "^2.0.1" }, @@ -6692,14 +6598,12 @@ "node_modules/@sinclair/typebox": { "version": "0.34.52", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@sinonjs/commons": { "version": "3.0.1", "dev": true, "license": "BSD-3-Clause", - "peer": true, "dependencies": { "type-detect": "4.0.8" } @@ -6708,7 +6612,6 @@ "version": "15.4.0", "dev": true, "license": "BSD-3-Clause", - "peer": true, "dependencies": { "@sinonjs/commons": "^3.0.1" } @@ -6883,7 +6786,6 @@ "version": "7.20.5", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/parser": "^7.20.7", "@babel/types": "^7.20.7", @@ -6896,7 +6798,6 @@ "version": "7.27.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/types": "^7.0.0" } @@ -6905,7 +6806,6 @@ "version": "7.4.4", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/parser": "^7.1.0", "@babel/types": "^7.0.0" @@ -6915,7 +6815,6 @@ "version": "7.28.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/types": "^7.28.2" } @@ -6983,6 +6882,7 @@ "node_modules/@types/express": { "version": "4.17.25", "license": "MIT", + "peer": true, "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^4.17.33", @@ -7025,9 +6925,17 @@ "@types/istanbul-lib-report": "*" } }, + "node_modules/@types/js-yaml": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz", + "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/json-schema": { "version": "7.0.15", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@types/lodash": { "version": "4.17.24", @@ -7210,8 +7118,7 @@ "node_modules/@types/stack-utils": { "version": "2.0.3", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/superagent": { "version": "8.1.10", @@ -7258,7 +7165,6 @@ "version": "17.0.35", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/yargs-parser": "*" } @@ -7501,8 +7407,7 @@ "node_modules/@ungap/structured-clone": { "version": "1.3.3", "dev": true, - "license": "ISC", - "peer": true + "license": "ISC" }, "node_modules/@unrs/resolver-binding-android-arm-eabi": { "version": "1.12.2", @@ -7880,6 +7785,7 @@ "version": "1.6.23", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "^8.58.0", "@typescript-eslint/utils": "^8.58.0" @@ -8101,6 +8007,7 @@ "node_modules/acorn": { "version": "8.17.0", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -8185,6 +8092,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -8459,7 +8367,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/transform": "30.4.1", "@types/babel__core": "^7.20.5", @@ -8480,7 +8387,6 @@ "version": "7.0.1", "dev": true, "license": "BSD-3-Clause", - "peer": true, "workspaces": [ "test/babel-8" ], @@ -8499,7 +8405,6 @@ "version": "30.4.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/babel__core": "^7.20.5" }, @@ -8511,7 +8416,6 @@ "version": "1.2.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/plugin-syntax-async-generators": "^7.8.4", "@babel/plugin-syntax-bigint": "^7.8.3", @@ -8537,7 +8441,6 @@ "version": "30.4.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "babel-plugin-jest-hoist": "30.4.0", "babel-preset-current-node-syntax": "^1.2.0" @@ -8730,6 +8633,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.42", "caniuse-lite": "^1.0.30001803", @@ -8759,7 +8663,6 @@ "version": "2.1.1", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "node-int64": "^0.4.0" } @@ -9048,7 +8951,6 @@ "version": "1.0.2", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" } @@ -9237,7 +9139,6 @@ "version": "4.6.0", "dev": true, "license": "MIT", - "peer": true, "engines": { "iojs": ">= 1.0.0", "node": ">= 0.12.0" @@ -9246,8 +9147,7 @@ "node_modules/collect-v8-coverage": { "version": "1.0.3", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/color-convert": { "version": "2.0.1", @@ -9273,8 +9173,7 @@ }, "node_modules/colorette": { "version": "1.4.0", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/columnify": { "version": "1.6.0", @@ -9306,6 +9205,7 @@ "node_modules/commander": { "version": "14.0.3", "license": "MIT", + "peer": true, "engines": { "node": ">=20" } @@ -10425,6 +10325,7 @@ "version": "9.0.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "env-paths": "^2.2.1", "import-fresh": "^3.3.0", @@ -10762,7 +10663,6 @@ "version": "3.1.0", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -10872,7 +10772,6 @@ "version": "0.13.1", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -10891,29 +10790,6 @@ "node": ">= 0.8" } }, - "node_modules/encoding": { - "version": "0.1.13", - "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", - "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", - "license": "MIT", - "optional": true, - "dependencies": { - "iconv-lite": "^0.6.2" - } - }, - "node_modules/encoding/node_modules/iconv-lite": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", - "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", - "license": "MIT", - "optional": true, - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/end-of-stream": { "version": "1.4.5", "license": "MIT", @@ -11173,6 +11049,7 @@ "version": "4.17.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/types": "^8.56.0", "comment-parser": "^1.4.1", @@ -11219,6 +11096,7 @@ "version": "28.14.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/utils": "^6.0.0 || ^7.0.0 || ^8.0.0" }, @@ -11509,7 +11387,6 @@ "version": "0.2.2", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">= 0.8.0" } @@ -11518,7 +11395,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/expect-utils": "30.4.1", "@jest/get-type": "30.1.0", @@ -11547,6 +11423,7 @@ "node_modules/express": { "version": "5.2.1", "license": "MIT", + "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -12007,7 +11884,6 @@ "version": "2.0.2", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "bser": "2.1.1" } @@ -12429,7 +12305,6 @@ "version": "1.0.0-beta.2", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6.9.0" } @@ -12477,7 +12352,6 @@ "version": "0.1.0", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8.0.0" } @@ -12951,6 +12825,7 @@ "version": "14.0.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=18" }, @@ -13364,7 +13239,6 @@ "node_modules/index-to-position": { "version": "1.2.0", "license": "MIT", - "peer": true, "engines": { "node": ">=18" }, @@ -13674,7 +13548,6 @@ "version": "2.1.0", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6" } @@ -13861,7 +13734,6 @@ "version": "6.0.3", "dev": true, "license": "BSD-3-Clause", - "peer": true, "dependencies": { "@babel/core": "^7.23.9", "@babel/parser": "^7.23.9", @@ -13890,7 +13762,6 @@ "version": "5.0.6", "dev": true, "license": "BSD-3-Clause", - "peer": true, "dependencies": { "@jridgewell/trace-mapping": "^0.3.23", "debug": "^4.1.1", @@ -13971,7 +13842,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "execa": "^5.1.1", "jest-util": "30.4.1", @@ -13985,7 +13855,6 @@ "version": "30.4.2", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/environment": "30.4.1", "@jest/expect": "30.4.1", @@ -14016,7 +13885,6 @@ "version": "1.7.2", "dev": true, "license": "MIT", - "peer": true, "peerDependencies": { "babel-plugin-macros": "^3.1.0" }, @@ -14030,7 +13898,6 @@ "version": "30.4.2", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/core": "30.4.2", "@jest/test-result": "30.4.1", @@ -14062,7 +13929,6 @@ "version": "30.4.2", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/core": "^7.27.4", "@jest/get-type": "30.1.0", @@ -14111,14 +13977,12 @@ "node_modules/jest-config/node_modules/balanced-match": { "version": "1.0.2", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/jest-config/node_modules/brace-expansion": { "version": "2.1.2", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "balanced-match": "^1.0.0" } @@ -14127,7 +13991,6 @@ "version": "10.5.0", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", @@ -14147,7 +14010,6 @@ "version": "9.0.9", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "brace-expansion": "^2.0.2" }, @@ -14162,7 +14024,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/diff-sequences": "30.4.0", "@jest/get-type": "30.1.0", @@ -14177,7 +14038,6 @@ "version": "30.4.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "detect-newline": "^3.1.0" }, @@ -14189,7 +14049,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/get-type": "30.1.0", "@jest/types": "30.4.1", @@ -14205,7 +14064,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/environment": "30.4.1", "@jest/fake-timers": "30.4.1", @@ -14231,7 +14089,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/types": "30.4.1", "@types/node": "*", @@ -14255,7 +14112,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/get-type": "30.1.0", "pretty-format": "30.4.1" @@ -14268,7 +14124,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/get-type": "30.1.0", "chalk": "^4.1.2", @@ -14283,7 +14138,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@jest/types": "30.4.1", @@ -14304,7 +14158,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/types": "30.4.1", "@types/node": "*", @@ -14397,7 +14250,6 @@ "version": "1.2.3", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6" }, @@ -14414,7 +14266,6 @@ "version": "30.4.0", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } @@ -14423,7 +14274,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "chalk": "^4.1.2", "graceful-fs": "^4.2.11", @@ -14442,7 +14292,6 @@ "version": "30.4.2", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "jest-regex-util": "30.4.0", "jest-snapshot": "30.4.1" @@ -14455,7 +14304,6 @@ "version": "30.4.2", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/console": "30.4.1", "@jest/environment": "30.4.1", @@ -14488,7 +14336,6 @@ "version": "30.4.2", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/environment": "30.4.1", "@jest/fake-timers": "30.4.1", @@ -14520,14 +14367,12 @@ "node_modules/jest-runtime/node_modules/balanced-match": { "version": "1.0.2", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/jest-runtime/node_modules/brace-expansion": { "version": "2.1.2", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "balanced-match": "^1.0.0" } @@ -14536,7 +14381,6 @@ "version": "10.5.0", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", @@ -14556,7 +14400,6 @@ "version": "9.0.9", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "brace-expansion": "^2.0.2" }, @@ -14571,7 +14414,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/core": "^7.27.4", "@babel/generator": "^7.27.5", @@ -14603,7 +14445,6 @@ "version": "7.8.5", "dev": true, "license": "ISC", - "peer": true, "bin": { "semver": "bin/semver.js" }, @@ -14615,7 +14456,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/types": "30.4.1", "@types/node": "*", @@ -14632,7 +14472,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/get-type": "30.1.0", "@jest/types": "30.4.1", @@ -14649,7 +14488,6 @@ "version": "6.3.0", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -14661,7 +14499,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/test-result": "30.4.1", "@jest/types": "30.4.1", @@ -14680,7 +14517,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/node": "*", "@ungap/structured-clone": "^1.3.0", @@ -14696,7 +14532,6 @@ "version": "8.1.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "has-flag": "^4.0.0" }, @@ -14711,6 +14546,7 @@ "version": "2.6.1", "dev": true, "license": "MIT", + "peer": true, "bin": { "jiti": "lib/jiti-cli.mjs" } @@ -14725,7 +14561,6 @@ "node_modules/js-levenshtein": { "version": "1.1.6", "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -15593,7 +15428,6 @@ "version": "5.1.1", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "yallist": "^3.0.2" } @@ -15820,7 +15654,6 @@ "version": "1.0.12", "dev": true, "license": "BSD-3-Clause", - "peer": true, "dependencies": { "tmpl": "1.0.5" } @@ -16620,8 +16453,7 @@ "node_modules/node-int64": { "version": "0.4.0", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/node-readfiles": { "version": "0.2.0", @@ -17006,6 +16838,7 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "peer": true, "dependencies": { "@nrwl/cli": "15.9.7", "@nrwl/tao": "15.9.7", @@ -17391,12 +17224,14 @@ }, "node_modules/openapi-types": { "version": "12.1.3", - "license": "MIT" + "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz", + "integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==", + "license": "MIT", + "peer": true }, "node_modules/openapi-typescript": { "version": "7.13.0", "license": "MIT", - "peer": true, "dependencies": { "@redocly/openapi-core": "^1.34.6", "ansi-colors": "^4.1.3", @@ -17415,7 +17250,6 @@ "node_modules/openapi-typescript/node_modules/parse-json": { "version": "8.3.0", "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.26.2", "index-to-position": "^1.1.0", @@ -17431,7 +17265,6 @@ "node_modules/openapi-typescript/node_modules/supports-color": { "version": "10.2.2", "license": "MIT", - "peer": true, "engines": { "node": ">=18" }, @@ -17442,7 +17275,6 @@ "node_modules/openapi-typescript/node_modules/yargs-parser": { "version": "21.1.1", "license": "ISC", - "peer": true, "engines": { "node": ">=12" } @@ -18319,6 +18151,7 @@ "node_modules/pg": { "version": "8.22.0", "license": "MIT", + "peer": true, "dependencies": { "pg-connection-string": "^2.14.0", "pg-pool": "^3.14.0", @@ -18418,6 +18251,7 @@ "node_modules/pino": { "version": "10.3.1", "license": "MIT", + "peer": true, "dependencies": { "@pinojs/redact": "^0.4.0", "atomic-sleep": "^1.0.0", @@ -18524,7 +18358,6 @@ "version": "4.0.7", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">= 6" } @@ -18694,6 +18527,7 @@ "node_modules/prettier": { "version": "3.9.5", "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -18708,7 +18542,6 @@ "version": "30.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/schemas": "30.4.1", "ansi-styles": "^5.2.0", @@ -18723,7 +18556,6 @@ "version": "5.2.0", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -18800,6 +18632,7 @@ "node_modules/prom-client": { "version": "15.1.3", "license": "Apache-2.0", + "peer": true, "dependencies": { "@opentelemetry/api": "^1.4.0", "tdigest": "^0.1.1" @@ -18935,8 +18768,7 @@ "url": "https://opencollective.com/fast-check" } ], - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/q": { "version": "1.5.1", @@ -19033,15 +18865,13 @@ "name": "react-is", "version": "18.3.1", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/react-is-19": { "name": "react-is", "version": "19.2.7", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/read": { "version": "1.0.7", @@ -20257,7 +20087,6 @@ "version": "2.0.6", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "escape-string-regexp": "^2.0.0" }, @@ -20269,7 +20098,6 @@ "version": "2.0.0", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -20331,7 +20159,6 @@ "version": "4.0.2", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "char-regex": "^1.0.2", "strip-ansi": "^6.0.0" @@ -20461,6 +20288,7 @@ "node_modules/supertest": { "version": "7.2.2", "license": "MIT", + "peer": true, "dependencies": { "cookie-signature": "^1.2.2", "methods": "^1.1.2", @@ -20548,7 +20376,6 @@ "version": "0.11.13", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@pkgr/core": "^0.3.6" }, @@ -20563,7 +20390,6 @@ "version": "0.3.6", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": "^14.18.0 || >=16.0.0" }, @@ -20695,7 +20521,6 @@ "version": "6.0.0", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "@istanbuljs/schema": "^0.1.2", "glob": "^7.1.4", @@ -20708,14 +20533,12 @@ "node_modules/test-exclude/node_modules/balanced-match": { "version": "1.0.2", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/test-exclude/node_modules/brace-expansion": { "version": "1.1.16", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -20725,7 +20548,6 @@ "version": "7.2.3", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -20745,7 +20567,6 @@ "version": "3.1.5", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -20872,8 +20693,7 @@ "node_modules/tmpl": { "version": "1.0.5", "dev": true, - "license": "BSD-3-Clause", - "peer": true + "license": "BSD-3-Clause" }, "node_modules/to-buffer": { "version": "1.2.2", @@ -21247,7 +21067,6 @@ "version": "4.0.8", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=4" } @@ -21718,8 +21537,7 @@ }, "node_modules/uri-js-replace": { "version": "1.0.1", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/url-value-parser": { "version": "2.2.0", @@ -21764,7 +21582,6 @@ "version": "9.3.0", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "@jridgewell/trace-mapping": "^0.3.12", "@types/istanbul-lib-coverage": "^2.0.1", @@ -21805,6 +21622,7 @@ "version": "8.1.4", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.5", @@ -21881,6 +21699,7 @@ "version": "4.1.10", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@vitest/expect": "4.1.10", "@vitest/mocker": "4.1.10", @@ -21974,7 +21793,6 @@ "version": "1.0.8", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "makeerror": "1.0.12" } @@ -22302,8 +22120,7 @@ "node_modules/yallist": { "version": "3.1.1", "dev": true, - "license": "ISC", - "peer": true + "license": "ISC" }, "node_modules/yaml": { "version": "2.9.0", @@ -22320,8 +22137,7 @@ }, "node_modules/yaml-ast-parser": { "version": "0.0.43", - "license": "Apache-2.0", - "peer": true + "license": "Apache-2.0" }, "node_modules/yargs": { "version": "17.7.3", @@ -22436,6 +22252,8 @@ "express": "^4.21.2", "express-openapi-validator": "^5.6.2", "http-status-codes": "^2.3.0", + "js-yaml": "^4.3.1", + "openapi-types": "^12.1.3", "prom-client": "^15.1.3", "reflect-metadata": "^0.2.2", "tsyringe": "^4.8.0", @@ -22450,6 +22268,7 @@ "@redocly/cli": "^2.16.0", "@types/compression": "^1.7.5", "@types/express": "^4.17.21", + "@types/js-yaml": "^4.0.9", "@types/multer": "^1.4.12", "@types/pg": "^8.11.14", "@types/supertest": "^6.0.2", @@ -22619,6 +22438,28 @@ "node": ">=0.10.0" } }, + "packages/api/node_modules/js-yaml": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", + "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "packages/api/node_modules/media-typer": { "version": "0.3.0", "license": "MIT", diff --git a/package.json b/package.json index 2b3b9aa..adf6a26 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "lerna": "^6.6.2" }, "dependencies": { - "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-11bf4605d7a17c723dd85f624c88fe32d9957863.tgz" + "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-3c2d0665e3509bb3e7113b328fc224fe6950895a.tgz" }, "overrides": { "@types/node": "^24.0.0" diff --git a/packages/api/config/test.json b/packages/api/config/test.json index d752fa8..1762bb1 100644 --- a/packages/api/config/test.json +++ b/packages/api/config/test.json @@ -1,4 +1,5 @@ { + "namespaces": ["data-2024", "data-2025"], "openapiConfig": { "filePath": "../../openapi3.yaml" }, diff --git a/packages/api/package.json b/packages/api/package.json index 8fa3aed..a3eaad7 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -46,6 +46,8 @@ "express": "^4.21.2", "express-openapi-validator": "^5.6.2", "http-status-codes": "^2.3.0", + "js-yaml": "^4.3.1", + "openapi-types": "^12.1.3", "prom-client": "^15.1.3", "reflect-metadata": "^0.2.2", "tsyringe": "^4.8.0", @@ -60,6 +62,7 @@ "@redocly/cli": "^2.16.0", "@types/compression": "^1.7.5", "@types/express": "^4.17.21", + "@types/js-yaml": "^4.0.9", "@types/multer": "^1.4.12", "@types/pg": "^8.11.14", "@types/supertest": "^6.0.2", @@ -73,10 +76,10 @@ "eslint-plugin-jest": "^28.11.0", "jest-openapi": "^0.14.2", "lerna": "^6.6.2", + "pg": "^8.20.0", "prettier": "^3.8.1", "pretty-quick": "^4.2.2", "rimraf": "^6.1.2", - "pg": "^8.20.0", "supertest": "^7.2.2", "ts-jest": "^29.2.6", "tsc-alias": "^1.8.11", diff --git a/packages/api/tests/unit/common/openapi.spec.ts b/packages/api/tests/unit/common/openapi.spec.ts new file mode 100644 index 0000000..2920431 --- /dev/null +++ b/packages/api/tests/unit/common/openapi.spec.ts @@ -0,0 +1,44 @@ +import { resolve } from 'node:path'; +import { describe, it, expect } from 'vitest'; +import { loadSpec, toPublicSpec } from '@src/common/openapi'; +import type { OpenapiSpec } from '@src/common/interfaces'; + +describe('openapi', function () { + const specPath = resolve('../../openapi3.yaml'); + const bases = { namespaced: 'https://gateway/vector/{namespace}', root: 'https://gateway/vector' }; + + const toPublic = (): OpenapiSpec => toPublicSpec(loadSpec(specPath), bases); + + const paramRefs = (spec: OpenapiSpec, route: string): (string | undefined)[] => + spec.paths[route]?.get?.parameters?.map((param) => ('$ref' in param ? param.$ref : undefined)) ?? []; + + describe('toPublicSpec', function () { + it('should give every path in the spec a public equivalent', function () { + const spec = loadSpec(specPath); + + expect(Object.keys(toPublicSpec(spec, bases).paths)).toHaveLength(Object.keys(spec.paths).length); + }); + + it('should point the top level servers at the namespaced gateway base', function () { + expect(toPublic().servers?.map((server) => server.url)).toEqual([bases.namespaced]); + }); + + it('should strip the namespace parameter from namespaced paths', function () { + const spec = toPublic(); + + expect(paramRefs(spec, '/')).toEqual([]); + expect(paramRefs(spec, '/{layerName}')).toEqual(['#/components/parameters/LayerNameParam']); + }); + + it('should override the servers of root scoped paths', function () { + expect(toPublic().paths['/namespaces']?.servers).toEqual([{ url: bases.root }]); + }); + + it('should drop paths that have no public equivalent', function () { + const spec = loadSpec(specPath); + spec.paths['/unmapped'] = { get: { responses: {} } }; + + expect(toPublicSpec(spec, bases).paths).not.toHaveProperty('/unmapped'); + }); + }); +}); From 4f3008bafa897e41a22d6ae0c33e7ada0788105b Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Sun, 23 Aug 2026 17:51:43 +0300 Subject: [PATCH 18/29] chore: rename docs test folder name --- packages/api/tests/unit/{common => docs}/openapi.spec.ts | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/api/tests/unit/{common => docs}/openapi.spec.ts (100%) diff --git a/packages/api/tests/unit/common/openapi.spec.ts b/packages/api/tests/unit/docs/openapi.spec.ts similarity index 100% rename from packages/api/tests/unit/common/openapi.spec.ts rename to packages/api/tests/unit/docs/openapi.spec.ts From c672d65f35bd6bd9d5efe8ec6f9af1466519e2ad Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Sun, 23 Aug 2026 18:00:19 +0300 Subject: [PATCH 19/29] feat: make openapi spec file singleton --- packages/api/src/common/constants.ts | 1 + packages/api/src/containerConfig.ts | 13 ++++++++++++- packages/api/src/docs/routes/docs.ts | 8 ++++---- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/packages/api/src/common/constants.ts b/packages/api/src/common/constants.ts index d1abfad..f0300e2 100644 --- a/packages/api/src/common/constants.ts +++ b/packages/api/src/common/constants.ts @@ -18,6 +18,7 @@ export const PUBLIC_PATHS = new Map([ export const HEALTHCHECK = Symbol('HealthCheck'); export const ON_SIGNAL = Symbol('onSignal'); +export const OPENAPI_SPEC = Symbol('OpenapiSpec'); /* eslint-disable @typescript-eslint/naming-convention */ export const SERVICES = { diff --git a/packages/api/src/containerConfig.ts b/packages/api/src/containerConfig.ts index 967f8e2..4ad6777 100644 --- a/packages/api/src/containerConfig.ts +++ b/packages/api/src/containerConfig.ts @@ -9,7 +9,9 @@ import type { HealthCheck } from '@godaddy/terminus'; import type { Repository, DataSource } from 'typeorm'; import { instancePerContainerCachingFactory } from 'tsyringe'; import { type InjectionObject, registerDependencies } from '@common/dependencyRegistration'; -import { HEALTHCHECK, ON_SIGNAL, SERVICES, SERVICE_NAME } from '@common/constants'; +import { HEALTHCHECK, ON_SIGNAL, OPENAPI_SPEC, SERVICES, SERVICE_NAME } from '@common/constants'; +import { loadSpec } from '@common/openapi'; +import type { OpenapiSpec } from '@common/interfaces'; import { getTracing } from '@common/tracing'; import { LAYER_ROUTER_SYMBOL, layerRouterFactory } from './layer/routes/layer'; import { NAMESPACE_ROUTER_SYMBOL, namespaceRouterFactory } from './namespace/routes/namespace'; @@ -67,6 +69,15 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise useValue: cleanupRegistry.trigger.bind(cleanupRegistry), }, }, + { + token: OPENAPI_SPEC, + provider: { + useFactory: instancePerContainerCachingFactory((container): OpenapiSpec => { + const config = container.resolve(SERVICES.CONFIG); + return loadSpec(config.get('openapiConfig.filePath')); + }), + }, + }, { token: LAYER_ROUTER_SYMBOL, provider: { useFactory: layerRouterFactory } }, { token: NAMESPACE_ROUTER_SYMBOL, provider: { useFactory: namespaceRouterFactory } }, { token: DOCS_ROUTER_SYMBOL, provider: { useFactory: docsRouterFactory } }, diff --git a/packages/api/src/docs/routes/docs.ts b/packages/api/src/docs/routes/docs.ts index 655487d..6dee0c5 100644 --- a/packages/api/src/docs/routes/docs.ts +++ b/packages/api/src/docs/routes/docs.ts @@ -2,20 +2,20 @@ import { Router } from 'express'; import type { Request, RequestHandler } from 'express'; import type { FactoryFunction } from 'tsyringe'; import { OpenapiViewerRouter } from '@map-colonies/openapi-express-viewer'; -import { PREFIX_HEADER, ROOT_HEADER, SERVICES } from '@common/constants'; -import { loadSpec, toPublicSpec } from '@common/openapi'; +import { OPENAPI_SPEC, PREFIX_HEADER, ROOT_HEADER, SERVICES } from '@common/constants'; +import { toPublicSpec } from '@common/openapi'; import type { OpenapiSpec } from '@common/interfaces'; import type { ConfigType } from '@common/config'; const docsRouterFactory: FactoryFunction = (dependencyContainer) => { const config = dependencyContainer.resolve(SERVICES.CONFIG); const openapiConfig = config.get('openapiConfig'); - const spec = loadSpec(openapiConfig.filePath); + const spec = dependencyContainer.resolve(OPENAPI_SPEC); const viewerRouter = new OpenapiViewerRouter({ ...openapiConfig, filePathOrSpec: spec }); viewerRouter.setup(); - const usePublicSpec: RequestHandler = (req, res, next) => { + const usePublicSpec: RequestHandler = (req, _, next) => { const namespaced = req.get(PREFIX_HEADER); const root = req.get(ROOT_HEADER); const behindGateway = namespaced !== undefined && root !== undefined; From 06c8a3432d744cdeeac40a30d6c3262733bd361f Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Mon, 24 Aug 2026 09:13:17 +0300 Subject: [PATCH 20/29] feat: add s3 and fs repos to container config for singleton --- packages/synchronizer/src/common/constants.ts | 2 + .../src/common/s3/s3Repository.ts | 7 +-- packages/synchronizer/src/containerConfig.ts | 56 +++++++++++++++++-- packages/synchronizer/src/sync/fileReader.ts | 7 ++- 4 files changed, 61 insertions(+), 11 deletions(-) diff --git a/packages/synchronizer/src/common/constants.ts b/packages/synchronizer/src/common/constants.ts index cf85c5c..27e2490 100644 --- a/packages/synchronizer/src/common/constants.ts +++ b/packages/synchronizer/src/common/constants.ts @@ -33,6 +33,8 @@ export const SERVICES = { LOGGER: Symbol('Logger'), CONFIG: Symbol('Config'), S3_CLIENT: Symbol('S3Client'), + S3_REPOSITORY: Symbol('S3Repository'), + FS_REPOSITORY: Symbol('FsRepository'), TRACER: Symbol('Tracer'), METRICS: Symbol('METRICS'), CLEANUP_REGISTRY: Symbol('CleanupRegistry'), diff --git a/packages/synchronizer/src/common/s3/s3Repository.ts b/packages/synchronizer/src/common/s3/s3Repository.ts index 610b2d5..4b27df1 100644 --- a/packages/synchronizer/src/common/s3/s3Repository.ts +++ b/packages/synchronizer/src/common/s3/s3Repository.ts @@ -2,22 +2,21 @@ import path from 'node:path'; import { Readable } from 'node:stream'; import { buffer } from 'node:stream/consumers'; import type { Logger } from '@map-colonies/js-logger'; -import { inject, injectable, singleton } from 'tsyringe'; +import { inject, injectable } from 'tsyringe'; import type { S3Client } from '@aws-sdk/client-s3'; import { GetObjectCommand } from '@aws-sdk/client-s3'; import { context as contextAPI } from '@opentelemetry/api'; import { startActivePromisifiedSpan } from '@common/tracing/util'; import { S3Attributes, S3SpanName } from '@common/tracing/s3'; import { SERVICES } from '@common/constants'; -import { FsRepository } from '../fs/fsRepository'; +import type { FsRepository } from '../fs/fsRepository'; @injectable() -@singleton() export class S3Repository { public constructor( @inject(SERVICES.S3_CLIENT) private readonly s3Client: S3Client, @inject(SERVICES.LOGGER) private readonly logger: Logger, - private readonly fsRepository: FsRepository + @inject(SERVICES.FS_REPOSITORY) private readonly fsRepository: FsRepository ) {} public async getObjectWrapper(bucket: string, key: string): Promise { diff --git a/packages/synchronizer/src/containerConfig.ts b/packages/synchronizer/src/containerConfig.ts index 9d7da06..91f6466 100644 --- a/packages/synchronizer/src/containerConfig.ts +++ b/packages/synchronizer/src/containerConfig.ts @@ -8,17 +8,38 @@ import { Lifecycle, instancePerContainerCachingFactory } from 'tsyringe'; import { CleanupRegistry } from '@map-colonies/cleanup-registry'; import type { DataSource, Repository } from 'typeorm'; import type { HealthCheck } from '@godaddy/terminus'; -import { DATA_SOURCE_PROVIDER as DESTINATION_DATA_SOURCE_PROVIDER, createDataSource, createDataSourceHealthCheck } from '@db'; +import { + DATA_SOURCE_PROVIDER as DESTINATION_DATA_SOURCE_PROVIDER, + ENUMS_REPOSITORY_SYMBOL, + type EnumValue, + LAYER_REPOSITORY_SYMBOL, + type Layer, + PROPERTY_REPOSITORY_SYMBOL, + type Property, + createDataSource, + createDataSourceHealthCheck, +} from '@db'; import type { S3Client } from '@aws-sdk/client-s3'; import { ListBucketsCommand } from '@aws-sdk/client-s3'; import { type ConfigType, getConfig } from '@common/config'; import { type InjectionObject, registerDependencies } from '@common/dependencyRegistration'; -import { DESTINATION_DB_CONFIG_PATH, HEALTHCHECK, NAMESPACE_HANDLES, ON_SIGNAL, REPOSITORIES, SERVICES, SERVICE_NAME } from '@common/constants'; +import { + DESTINATION_DB_CONFIG_PATH, + HEALTHCHECK, + NAMESPACE_HANDLES, + ON_SIGNAL, + REPOSITORIES, + SERVICES, + SERVICE_NAME, + SOURCE_DATA_SOURCE_PROVIDER, +} from '@common/constants'; import { getTracing } from '@common/tracing'; import { CRON_MANAGER_SYMBOL, CronManager } from './sync/cron'; import { s3ClientFactory } from './common/s3'; import { S3Repository } from './common/s3/s3Repository'; +import { FsRepository } from './common/fs/fsRepository'; import { createNamespaceHandles } from './sync/namespaceHandle'; +import type { NamespaceHandle } from './sync/namespaceHandle/types'; export interface RegisterOptions { override?: InjectionObject[]; @@ -95,7 +116,12 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise }, }, { - token: S3Repository, + token: SERVICES.FS_REPOSITORY, + provider: { useClass: FsRepository }, + options: { lifecycle: Lifecycle.Singleton }, + }, + { + token: SERVICES.S3_REPOSITORY, provider: { useClass: S3Repository }, options: { lifecycle: Lifecycle.Singleton }, }, @@ -134,7 +160,29 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: NAMESPACE_HANDLES, provider: { - useFactory: instancePerContainerCachingFactory((container) => createNamespaceHandles(container, cleanupRegistry)), + useFactory: instancePerContainerCachingFactory((container) => + createNamespaceHandles({ + config: container.resolve(SERVICES.CONFIG), + logger: container.resolve(SERVICES.LOGGER), + s3Repository: container.resolve(SERVICES.S3_REPOSITORY), + fsRepository: container.resolve(SERVICES.FS_REPOSITORY), + layerRepository: container.resolve>(LAYER_REPOSITORY_SYMBOL), + propertyRepository: container.resolve>(PROPERTY_REPOSITORY_SYMBOL), + enumsRepository: container.resolve>(ENUMS_REPOSITORY_SYMBOL), + }) + ), + }, + postInjectionHook: (deps: DependencyContainer): void => { + for (const { name, sourceDataSource } of deps.resolve(NAMESPACE_HANDLES)) { + cleanupRegistry.register({ + id: `${SOURCE_DATA_SOURCE_PROVIDER.toString()}:${name}`, + func: async () => { + if (sourceDataSource.isInitialized) { + await sourceDataSource.destroy(); + } + }, + }); + } }, }, { diff --git a/packages/synchronizer/src/sync/fileReader.ts b/packages/synchronizer/src/sync/fileReader.ts index b3709e4..cf978b4 100644 --- a/packages/synchronizer/src/sync/fileReader.ts +++ b/packages/synchronizer/src/sync/fileReader.ts @@ -1,8 +1,9 @@ import { createHash } from 'node:crypto'; import type { JsonValue } from 'type-fest'; -import { injectable } from 'tsyringe'; -import { FsRepository } from '@common/fs/fsRepository'; +import { inject, injectable } from 'tsyringe'; +import type { FsRepository } from '@common/fs/fsRepository'; import type { LayerEnums } from '@common/interfaces'; +import { SERVICES } from '@common/constants'; import { parseTypeMap, type TypeMap } from './typeMap'; import { TypeMapError } from './errors'; @@ -24,7 +25,7 @@ export const resolveFileAliases = (fileAliases: FileAliases, namespace: string, @injectable() export class FileReader { - public constructor(private readonly fsRepository: FsRepository) {} + public constructor(@inject(SERVICES.FS_REPOSITORY) private readonly fsRepository: FsRepository) {} public async readLayersWithChecksum(filePath: string): Promise<{ checksum: string; layers: LayerEnums[] }> { try { From 2a88ca0c6607a2ce4b74ae3b4c71feb18630ae65 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Mon, 24 Aug 2026 09:16:05 +0300 Subject: [PATCH 21/29] chore: move types to logical files --- packages/synchronizer/src/sync/cron.ts | 2 +- .../synchronizer/src/sync/namespaceHandle.ts | 56 ------------------- .../src/sync/namespaceHandle/index.ts | 17 ++++++ .../src/sync/namespaceHandle/types.ts | 26 +++++++++ 4 files changed, 44 insertions(+), 57 deletions(-) delete mode 100644 packages/synchronizer/src/sync/namespaceHandle.ts create mode 100644 packages/synchronizer/src/sync/namespaceHandle/index.ts create mode 100644 packages/synchronizer/src/sync/namespaceHandle/types.ts diff --git a/packages/synchronizer/src/sync/cron.ts b/packages/synchronizer/src/sync/cron.ts index 34ef1c7..a354305 100644 --- a/packages/synchronizer/src/sync/cron.ts +++ b/packages/synchronizer/src/sync/cron.ts @@ -8,7 +8,7 @@ import { startActivePromisifiedSpan } from '@common/tracing/util'; import { SyncSpanName } from '@common/tracing/sync'; import { FileReader, type FileAliases } from './fileReader'; import type { TypeMap } from './typeMap'; -import type { NamespaceHandle } from './namespaceHandle'; +import type { NamespaceHandle } from './namespaceHandle/types'; export const CRON_MANAGER_SYMBOL = Symbol('cronManagerSymbol'); diff --git a/packages/synchronizer/src/sync/namespaceHandle.ts b/packages/synchronizer/src/sync/namespaceHandle.ts deleted file mode 100644 index 1efc3e8..0000000 --- a/packages/synchronizer/src/sync/namespaceHandle.ts +++ /dev/null @@ -1,56 +0,0 @@ -import type { CleanupRegistry } from '@map-colonies/cleanup-registry'; -import type { Logger } from '@map-colonies/js-logger'; -import type { DependencyContainer } from 'tsyringe/dist/typings/types'; -import type { DataSource, Repository } from 'typeorm'; -import { - createDataSource, - ENUMS_REPOSITORY_SYMBOL, - type EnumValue, - type Layer, - LAYER_REPOSITORY_SYMBOL, - type Property, - PROPERTY_REPOSITORY_SYMBOL, -} from '@db'; -import type { ConfigType } from '@common/config'; -import { SERVICES, SERVICE_NAME, SOURCE_DATA_SOURCE_PROVIDER } from '@common/constants'; -import { S3Repository } from '@common/s3/s3Repository'; -import { FsRepository } from '@common/fs/fsRepository'; -import { SyncModel } from './SyncModel'; -import { createLayerSourceStrategy } from './layerSource'; -import type { LayerSourceStrategy } from './layerSource/types'; - -export interface NamespaceHandle { - name: string; - sourceDataSource: DataSource; - layerSource: LayerSourceStrategy; - layersFile: string; - dal: SyncModel; -} - -export const createNamespaceHandles = (container: DependencyContainer, cleanupRegistry: CleanupRegistry): NamespaceHandle[] => { - const config = container.resolve(SERVICES.CONFIG); - const logger = container.resolve(SERVICES.LOGGER); - const s3Repository = container.resolve(S3Repository); - const fsRepository = container.resolve(FsRepository); - const layerRepository = container.resolve>(LAYER_REPOSITORY_SYMBOL); - const propertyRepository = container.resolve>(PROPERTY_REPOSITORY_SYMBOL); - const enumsRepository = container.resolve>(ENUMS_REPOSITORY_SYMBOL); - - return config.get('namespaces').map((namespaceConfig) => { - const sourceDataSource = createDataSource(namespaceConfig.db, `${SERVICE_NAME}-${namespaceConfig.name}`); - - cleanupRegistry.register({ - id: `${SOURCE_DATA_SOURCE_PROVIDER.toString()}:${namespaceConfig.name}`, - func: async () => { - if (sourceDataSource.isInitialized) { - await sourceDataSource.destroy(); - } - }, - }); - - const layerSource = createLayerSourceStrategy(namespaceConfig, s3Repository, fsRepository, logger); - const dal = new SyncModel(config, logger, sourceDataSource, layerRepository, propertyRepository, enumsRepository); - - return { name: namespaceConfig.name, sourceDataSource, layerSource, layersFile: namespaceConfig.layersFile, dal }; - }); -}; diff --git a/packages/synchronizer/src/sync/namespaceHandle/index.ts b/packages/synchronizer/src/sync/namespaceHandle/index.ts new file mode 100644 index 0000000..733d928 --- /dev/null +++ b/packages/synchronizer/src/sync/namespaceHandle/index.ts @@ -0,0 +1,17 @@ +import { createDataSource } from '@db'; +import { SERVICE_NAME } from '@common/constants'; +import { SyncModel } from '../SyncModel'; +import { createLayerSourceStrategy } from '../layerSource'; +import type { NamespaceHandle, NamespaceHandleDependencies } from './types'; + +export const createNamespaceHandles = (dependencies: NamespaceHandleDependencies): NamespaceHandle[] => { + const { config, logger, s3Repository, fsRepository, layerRepository, propertyRepository, enumsRepository } = dependencies; + + return config.get('namespaces').map((namespaceConfig) => { + const sourceDataSource = createDataSource(namespaceConfig.db, `${SERVICE_NAME}-${namespaceConfig.name}`); + const layerSource = createLayerSourceStrategy(namespaceConfig, s3Repository, fsRepository, logger); + const dal = new SyncModel(config, logger, sourceDataSource, layerRepository, propertyRepository, enumsRepository); + + return { name: namespaceConfig.name, sourceDataSource, layerSource, layersFile: namespaceConfig.layersFile, dal }; + }); +}; diff --git a/packages/synchronizer/src/sync/namespaceHandle/types.ts b/packages/synchronizer/src/sync/namespaceHandle/types.ts new file mode 100644 index 0000000..e1dc2f5 --- /dev/null +++ b/packages/synchronizer/src/sync/namespaceHandle/types.ts @@ -0,0 +1,26 @@ +import type { Logger } from '@map-colonies/js-logger'; +import type { DataSource, Repository } from 'typeorm'; +import type { EnumValue, Layer, Property } from '@db'; +import type { ConfigType } from '@common/config'; +import type { S3Repository } from '@common/s3/s3Repository'; +import type { FsRepository } from '@common/fs/fsRepository'; +import type { SyncModel } from '../SyncModel'; +import type { LayerSourceStrategy } from '../layerSource/types'; + +export interface NamespaceHandle { + name: string; + sourceDataSource: DataSource; + layerSource: LayerSourceStrategy; + layersFile: string; + dal: SyncModel; +} + +export interface NamespaceHandleDependencies { + config: ConfigType; + logger: Logger; + s3Repository: S3Repository; + fsRepository: FsRepository; + layerRepository: Repository; + propertyRepository: Repository; + enumsRepository: Repository; +} From e811214ad3533cc8fbd0b0aa1c84de35aa951660 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Mon, 24 Aug 2026 09:16:22 +0300 Subject: [PATCH 22/29] test: update tests configuration --- .../tests/integration/syncModel/syncModel.spec.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts b/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts index 1d49775..68d8e48 100644 --- a/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts +++ b/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts @@ -18,12 +18,11 @@ import { import { initConfig } from '@src/common/config'; import { registerExternalValues } from '@src/containerConfig'; import { SERVICES, NAMESPACE_HANDLES } from '@src/common/constants'; -import { S3Repository } from '@src/common/s3/s3Repository'; import type { SyncModel } from '@src/sync/SyncModel'; import { schemaOf } from '@src/sync/helpers'; import { FileReader, type FileAliases } from '@src/sync/fileReader'; import type { TypeMap } from '@src/sync/typeMap'; -import type { NamespaceHandle } from '@src/sync/namespaceHandle'; +import type { NamespaceHandle } from '@src/sync/namespaceHandle/types'; const TEST_LAYER = 'test_layer'; const NAMESPACE = 'test'; @@ -97,7 +96,7 @@ describe('DAL', function () { override: [ { token: SERVICES.LOGGER, provider: { useValue: await jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, - { token: S3Repository, provider: { useValue: { downloadFile: async () => Promise.reject(new Error('not used in this suite')) } } }, + { token: SERVICES.S3_REPOSITORY, provider: { useValue: { downloadFile: async () => Promise.reject(new Error('not used in this suite')) } } }, ], useChild: true, }); From 18c43fc41c7392d679724717d1d06fc506a92003 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Mon, 24 Aug 2026 09:19:46 +0300 Subject: [PATCH 23/29] helm: update helms for namespaces --- helm/charts/api/templates/deployment.yaml | 6 ++ .../api/templates/namespaces-configmap.yaml | 14 ++++ helm/charts/api/values.yaml | 2 + .../synchronizer/templates/deployment.yaml | 2 +- .../templates/namespaces-secret.yaml | 7 +- helm/charts/synchronizer/values.yaml | 1 + helm/values.yaml | 66 +++++++++---------- 7 files changed, 63 insertions(+), 35 deletions(-) create mode 100644 helm/charts/api/templates/namespaces-configmap.yaml diff --git a/helm/charts/api/templates/deployment.yaml b/helm/charts/api/templates/deployment.yaml index 5d3d58f..2529833 100644 --- a/helm/charts/api/templates/deployment.yaml +++ b/helm/charts/api/templates/deployment.yaml @@ -93,6 +93,9 @@ spec: {{- end }} volumeMounts: + - name: namespaces-config + mountPath: /usr/src/app/packages/api/dist/config/production.json + subPath: production.json {{- if .Values.caSecretName }} - mountPath: {{ printf "%s/%s" .Values.caPath .Values.caKey | quote }} name: root-ca @@ -104,6 +107,9 @@ spec: readOnly: true {{- end }} volumes: + - name: namespaces-config + configMap: + name: {{ include "api.fullname" . }}-namespaces {{- if .Values.db.sslAuth.enabled }} - name: cert-conf secret: diff --git a/helm/charts/api/templates/namespaces-configmap.yaml b/helm/charts/api/templates/namespaces-configmap.yaml new file mode 100644 index 0000000..b0d9d60 --- /dev/null +++ b/helm/charts/api/templates/namespaces-configmap.yaml @@ -0,0 +1,14 @@ +{{- if .Values.enabled -}} +{{- $names := list -}} +{{- range .Values.namespaces -}} +{{- $names = append $names .name -}} +{{- end -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "api.fullname" . }}-namespaces + labels: + {{- include "api.labels" . | nindent 4 }} +data: + production.json: {{ dict "namespaces" $names | toPrettyJson | quote }} +{{- end }} diff --git a/helm/charts/api/values.yaml b/helm/charts/api/values.yaml index 8c23878..72e2fbf 100644 --- a/helm/charts/api/values.yaml +++ b/helm/charts/api/values.yaml @@ -52,6 +52,8 @@ env: metrics: enabled: false url: http://localhost:55681/v1/metrics +namespaces: [] + db: host: localhost port: 5432 diff --git a/helm/charts/synchronizer/templates/deployment.yaml b/helm/charts/synchronizer/templates/deployment.yaml index f5803b3..aa77bc5 100644 --- a/helm/charts/synchronizer/templates/deployment.yaml +++ b/helm/charts/synchronizer/templates/deployment.yaml @@ -93,7 +93,7 @@ spec: {{- end }} volumeMounts: - name: layers-config - mountPath: /usr/src/app/packages/synchronizer/dist/config/layers + mountPath: {{ .Values.env.layersDir | quote }} - name: namespaces-config mountPath: /usr/src/app/packages/synchronizer/dist/config/production.json subPath: production.json diff --git a/helm/charts/synchronizer/templates/namespaces-secret.yaml b/helm/charts/synchronizer/templates/namespaces-secret.yaml index 53b6484..f5c1376 100644 --- a/helm/charts/synchronizer/templates/namespaces-secret.yaml +++ b/helm/charts/synchronizer/templates/namespaces-secret.yaml @@ -1,4 +1,9 @@ {{- if .Values.enabled -}} +{{- $layersDir := .Values.env.layersDir -}} +{{- $namespaces := list -}} +{{- range .Values.namespaces -}} +{{- $namespaces = append $namespaces (merge (dict "layersFile" (printf "%s/%s.json" $layersDir .name)) (omit . "layers")) -}} +{{- end -}} apiVersion: v1 kind: Secret metadata: @@ -7,5 +12,5 @@ metadata: {{- include "synchronizer.labels" . | nindent 4 }} type: Opaque stringData: - production.json: {{ dict "namespaces" .Values.namespaces | toPrettyJson | quote }} + production.json: {{ dict "namespaces" $namespaces | toPrettyJson | quote }} {{- end }} diff --git a/helm/charts/synchronizer/values.yaml b/helm/charts/synchronizer/values.yaml index d5604f5..55d4cd2 100644 --- a/helm/charts/synchronizer/values.yaml +++ b/helm/charts/synchronizer/values.yaml @@ -99,6 +99,7 @@ env: url: http://localhost:55681/v1/metrics schedule: '0 0 * * *' indexNameFormat: '{layerName}_{column}_idx' + layersDir: /usr/src/app/packages/synchronizer/dist/config/layers aliasesFile: /usr/src/app/packages/synchronizer/dist/config/aliases/aliases.json typeMapFile: /usr/src/app/packages/synchronizer/dist/config/typeMap/typeMap.json diff --git a/helm/values.yaml b/helm/values.yaml index 0bb40ba..f3cb874 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -36,7 +36,38 @@ dbConfig: &dbConfig key: '' root: '' +namespaces: &namespaces + - name: default + bucket: "" + layers: + - layerName: buildings_polygon + enums: [ code, attribute ] + db: + host: localhost + port: 5432 + username: postgres + password: postgres + database: postgres + schema: public + type: postgres + sslSecretName: secret-name + ssl: + enabled: false + ca: "" + cert: "" + key: "" + layerSource: + type: sharedLua + fileName: "" + layersVariable: layers + aliasFieldName: alias + idFieldName: id + nameFieldName: name + enrichment: + enabled: false + api: + namespaces: *namespaces enabled: true replicaCount: 1 revisionHistoryLimit: 5 @@ -163,39 +194,7 @@ synchronizer: pullPolicy: Always tag: "v1.2.0" - namespaces: - - name: default - bucket: "" - layers: - - layerName: buildings_polygon - enums: [code, attribute] - db: - host: localhost - port: 5432 - username: postgres - password: postgres - database: postgres - schema: public - type: postgres - sslSecretName: secret-name - ssl: - enabled: false - ca: "" - cert: "" - key: "" - layerSource: - type: sharedLua - fileName: "" - layersVariable: layers - aliasFieldName: alias - idFieldName: id - nameFieldName: name - enrichment: - enabled: false - api: "" - propertiesPath: "" - aliasField: "" - requestTimeoutMilliseconds: 5000 + namespaces: *namespaces aliases: "*": @@ -223,6 +222,7 @@ synchronizer: url: http://localhost:55681/v1/metrics schedule: '*/1 * * * *' indexNameFormat: '{layerName}_{column}_idx' + layersDir: /usr/src/app/packages/synchronizer/dist/config/layers aliasesFile: /usr/src/app/packages/synchronizer/dist/config/aliases/aliases.json typeMapFile: /usr/src/app/packages/synchronizer/dist/config/typeMap/typeMap.json From 1d9977144ef76ceedf5706bbf2b93d99c4d25ea9 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Mon, 24 Aug 2026 09:32:30 +0300 Subject: [PATCH 24/29] docs: make logs match --- packages/synchronizer/src/sync/SyncModel.ts | 6 +++--- packages/synchronizer/src/sync/cron.ts | 12 ++++++------ .../src/sync/layerSource/perLayerJsonSource.ts | 6 +++--- .../src/sync/layerSource/sharedLuaSource.ts | 7 ++----- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/packages/synchronizer/src/sync/SyncModel.ts b/packages/synchronizer/src/sync/SyncModel.ts index c5e4627..5dc020b 100644 --- a/packages/synchronizer/src/sync/SyncModel.ts +++ b/packages/synchronizer/src/sync/SyncModel.ts @@ -155,11 +155,11 @@ export class SyncModel { const excluded = excludedPropertySet(layer.excludeProperties); const columns = (await this.getTableColumns(layer.layerName)).filter(({ columnName }) => !excluded.has(columnName.toLowerCase())); if (excluded.size > 0) { - this.logger.debug({ layerName: layer.layerName, excludeProperties: layer.excludeProperties }, 'Excluding properties from sync'); + this.logger.debug({ msg: `Excluding properties from sync for ${layer.layerName}`, excludeProperties: layer.excludeProperties }); } const properties = columnInfosToProperties(columns, namespace, layer.layerName, typeMap, (columnName, udtName) => { - this.logger.warn({ layerName: layer.layerName, columnName, udtName }, 'Unknown column type, skipping property'); + this.logger.warn({ msg: `Unknown column type ${udtName} for ${layer.layerName}.${columnName}, skipping property` }); }); const fileLayerAliases = resolveFileAliases(fileAliases, namespace, layer.layerName); @@ -276,7 +276,7 @@ export class SyncModel { if (existing.length !== layer.enums.length) { const missing = layer.enums.filter((col) => !tableColumns.has(col)); - this.logger.warn({ layerName: layer.layerName, columns: missing }, 'Enum columns do not exist in table, skipping'); + this.logger.warn({ msg: `Enum columns do not exist in table ${layer.layerName}, skipping`, columns: missing }); } return existing; diff --git a/packages/synchronizer/src/sync/cron.ts b/packages/synchronizer/src/sync/cron.ts index a354305..f10eb15 100644 --- a/packages/synchronizer/src/sync/cron.ts +++ b/packages/synchronizer/src/sync/cron.ts @@ -33,7 +33,7 @@ export class CronManager { try { await this.tickNamespace(namespace, typeMap, fileAliases); } catch (err) { - this.logger.error({ namespace: namespace.name, err }, 'namespace sync tick failed, continuing with next namespace'); + this.logger.error({ msg: `Failed to sync namespace ${namespace.name}, skipping`, err }); } } }); @@ -61,8 +61,8 @@ export class CronManager { try { await namespace.sourceDataSource.initialize(); } catch (err) { - this.logger.warn({ namespace: namespace.name, err }, 'source database unreachable, skipping namespace for this tick'); - return; + this.logger.warn({ msg: `Source database unreachable for namespace ${namespace.name}, skipping`, err }); + throw err; } } @@ -80,12 +80,12 @@ export class CronManager { await namespace.dal.syncLayer(namespace.name, layer.layerName, record.layerId ?? null, record.source, record.alias); try { const affected = await namespace.dal.syncProperties(namespace.name, layer, typeMap, fileAliases, record.propertyAliases); - this.logger.info({ namespace: namespace.name, layer, affected }, 'synced properties'); + this.logger.info({ msg: `Synced properties for ${namespace.name}/${layer.layerName}`, affected }); } catch (err) { - this.logger.warn({ namespace: namespace.name, layerName: layer.layerName, err }, 'failed to sync properties, skipping to enum sync'); + this.logger.warn({ msg: `Failed to sync properties for ${namespace.name}/${layer.layerName}, skipping to enum sync`, err }); } const enumsAffected = await namespace.dal.syncEnum(namespace.name, layer); - this.logger.info({ namespace: namespace.name, layer, enumsAffected }, 'synced enums'); + this.logger.info({ msg: `Synced enums for ${namespace.name}/${layer.layerName}`, enumsAffected }); if (layersChanged) { await namespace.dal.deleteStaleEnumValues(namespace.name, layer.layerName, layer.enums); } diff --git a/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts b/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts index 0c5c324..9650fd9 100644 --- a/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts +++ b/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts @@ -21,7 +21,7 @@ const resolveOne = async ( try { filePath = await s3Repository.downloadFile(bucket, key); } catch (err) { - logger.warn({ err: new LayerJsonFetchError(layer.layerName, key, err) }, 'Failed to fetch layer JSON, skipping layer'); + logger.warn({ msg: `Failed to fetch layer JSON ${key}, skipping layer`, err: new LayerJsonFetchError(layer.layerName, key, err) }); return undefined; } @@ -34,7 +34,7 @@ const resolveOne = async ( source: layerSource.perLayerJson, }; } catch (err) { - logger.warn({ err: new LayerJsonParseError(layer.layerName, key, err) }, 'Failed to read or parse layer JSON, skipping layer'); + logger.warn({ msg: `Failed to read or parse layer JSON ${key}, skipping layer`, err: new LayerJsonParseError(layer.layerName, key, err) }); return undefined; } }; @@ -52,7 +52,7 @@ export class PerLayerJsonSource implements LayerSourceStrategy { const records = new Map(); for (const layer of layers) { if (layer.sourceKey === undefined) { - this.logger.warn({ layerName: layer.layerName }, 'No sourceKey configured for perLayerJson layer, skipping'); + this.logger.warn({ msg: `No sourceKey configured for perLayerJson layer ${layer.layerName}, skipping` }); continue; } diff --git a/packages/synchronizer/src/sync/layerSource/sharedLuaSource.ts b/packages/synchronizer/src/sync/layerSource/sharedLuaSource.ts index 8d98c62..ebdcab5 100644 --- a/packages/synchronizer/src/sync/layerSource/sharedLuaSource.ts +++ b/packages/synchronizer/src/sync/layerSource/sharedLuaSource.ts @@ -12,10 +12,7 @@ const resolveOne = async (layer: LayerEnums, luaLayer: LuaLayer, config: SharedL try { propertyAliases = await fetchPropertyAliases(layer.layerName, luaLayer.layerId, config.enrichment); } catch (err) { - logger.warn( - { layerName: layer.layerName, layerId: luaLayer.layerId, err }, - 'Failed to fetch property aliases from enrichment API, continuing without them' - ); + logger.warn({ msg: `Failed to fetch property aliases for ${layer.layerName} (${luaLayer.layerId}), continuing without them`, err }); } } @@ -52,7 +49,7 @@ export class SharedLuaSource implements LayerSourceStrategy { for (const layer of layers) { const luaLayer = luaLayers.get(layer.layerName); if (luaLayer === undefined) { - this.logger.warn({ layerName: layer.layerName }, 'Layer not found in lua file, skipping'); + this.logger.warn({ msg: `Layer ${layer.layerName} not found in lua file, skipping` }); continue; } records.set(layer.layerName, await resolveOne(layer, luaLayer, this.config, this.logger)); From 8393633054f1c34d84b8911accb3471292ef97fd Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Mon, 24 Aug 2026 09:40:17 +0300 Subject: [PATCH 25/29] chore: make sync own layer sync flow --- .../synchronizer/src/common/tracing/sync.ts | 1 + packages/synchronizer/src/sync/SyncModel.ts | 34 +++++++++++++++++++ packages/synchronizer/src/sync/cron.ts | 11 ++---- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/packages/synchronizer/src/common/tracing/sync.ts b/packages/synchronizer/src/common/tracing/sync.ts index 3209699..9c76b50 100644 --- a/packages/synchronizer/src/common/tracing/sync.ts +++ b/packages/synchronizer/src/common/tracing/sync.ts @@ -2,6 +2,7 @@ export const SyncSpanName = { SYNC_TICK: 'sync.tick', SYNC_LAYER: 'sync.layer', + SYNC_FULL_LAYER: 'sync.layer.full', SYNC_PROPERTIES: 'sync.properties', SYNC_ENUM: 'sync.enum', LOAD_LUA_DATA: 'sync.lua.load', diff --git a/packages/synchronizer/src/sync/SyncModel.ts b/packages/synchronizer/src/sync/SyncModel.ts index 5dc020b..7b911b1 100644 --- a/packages/synchronizer/src/sync/SyncModel.ts +++ b/packages/synchronizer/src/sync/SyncModel.ts @@ -24,12 +24,19 @@ import { } from './errors'; import { resolveFileAliases, type FileAliases } from './fileReader'; import type { TypeMap } from './typeMap'; +import type { LayerSourceRecord } from './layerSource/types'; export interface ColumnInfo { columnName: string; udtName: string; } +export interface SyncFullLayerContext { + typeMap: TypeMap; + fileAliases: FileAliases; + pruneStale: boolean; +} + @injectable() export class SyncModel { public constructor( @@ -41,6 +48,33 @@ export class SyncModel { @inject(ENUMS_REPOSITORY_SYMBOL) private readonly enumsRepository: Repository ) {} + public async syncFullLayer(namespace: string, layer: LayerEnums, record: LayerSourceRecord, context: SyncFullLayerContext): Promise { + return startActivePromisifiedSpan( + SyncSpanName.SYNC_FULL_LAYER, + { [SyncAttributes.LAYER_NAME]: layer.layerName, [SyncAttributes.NAMESPACE]: namespace }, + contextAPI.active(), + async () => { + const { typeMap, fileAliases, pruneStale } = context; + + await this.syncLayer(namespace, layer.layerName, record.layerId ?? null, record.source, record.alias); + + try { + const affected = await this.syncProperties(namespace, layer, typeMap, fileAliases, record.propertyAliases); + this.logger.info({ msg: `Synced properties for ${namespace}/${layer.layerName}`, affected }); + } catch (err) { + this.logger.warn({ msg: `Failed to sync properties for ${namespace}/${layer.layerName}, skipping to enum sync`, err }); + } + + const enumsAffected = await this.syncEnum(namespace, layer); + this.logger.info({ msg: `Synced enums for ${namespace}/${layer.layerName}`, enumsAffected }); + + if (pruneStale) { + await this.deleteStaleEnumValues(namespace, layer.layerName, layer.enums); + } + } + ); + } + public async syncLayer(namespace: string, layerName: string, layerId: number | null, source: LayerSource, alias?: string): Promise { return startActivePromisifiedSpan( SyncSpanName.SYNC_LAYER, diff --git a/packages/synchronizer/src/sync/cron.ts b/packages/synchronizer/src/sync/cron.ts index f10eb15..2714bed 100644 --- a/packages/synchronizer/src/sync/cron.ts +++ b/packages/synchronizer/src/sync/cron.ts @@ -77,17 +77,10 @@ export class CronManager { continue; } - await namespace.dal.syncLayer(namespace.name, layer.layerName, record.layerId ?? null, record.source, record.alias); try { - const affected = await namespace.dal.syncProperties(namespace.name, layer, typeMap, fileAliases, record.propertyAliases); - this.logger.info({ msg: `Synced properties for ${namespace.name}/${layer.layerName}`, affected }); + await namespace.dal.syncFullLayer(namespace.name, layer, record, { typeMap, fileAliases, pruneStale: layersChanged }); } catch (err) { - this.logger.warn({ msg: `Failed to sync properties for ${namespace.name}/${layer.layerName}, skipping to enum sync`, err }); - } - const enumsAffected = await namespace.dal.syncEnum(namespace.name, layer); - this.logger.info({ msg: `Synced enums for ${namespace.name}/${layer.layerName}`, enumsAffected }); - if (layersChanged) { - await namespace.dal.deleteStaleEnumValues(namespace.name, layer.layerName, layer.enums); + this.logger.warn({ msg: `Failed to sync ${namespace.name}/${layer.layerName}, continuing with next layer`, err }); } } From d06ab36d616f5e943f56ed768bc44198cdd22117 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Mon, 24 Aug 2026 09:59:01 +0300 Subject: [PATCH 26/29] chore: move all keys selector to contss --- packages/synchronizer/src/common/constants.ts | 2 ++ packages/synchronizer/src/sync/fileReader.ts | 10 ++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/synchronizer/src/common/constants.ts b/packages/synchronizer/src/common/constants.ts index 27e2490..f4a3239 100644 --- a/packages/synchronizer/src/common/constants.ts +++ b/packages/synchronizer/src/common/constants.ts @@ -28,6 +28,8 @@ export const REPOSITORIES = [ { entity: EnumValue, token: ENUMS_REPOSITORY_SYMBOL }, ] as const; +export const ALL_KEYS_SELECTOR = '*'; + /* eslint-disable @typescript-eslint/naming-convention */ export const SERVICES = { LOGGER: Symbol('Logger'), diff --git a/packages/synchronizer/src/sync/fileReader.ts b/packages/synchronizer/src/sync/fileReader.ts index cf978b4..f1f3f6b 100644 --- a/packages/synchronizer/src/sync/fileReader.ts +++ b/packages/synchronizer/src/sync/fileReader.ts @@ -3,21 +3,19 @@ import type { JsonValue } from 'type-fest'; import { inject, injectable } from 'tsyringe'; import type { FsRepository } from '@common/fs/fsRepository'; import type { LayerEnums } from '@common/interfaces'; -import { SERVICES } from '@common/constants'; +import { ALL_KEYS_SELECTOR, SERVICES } from '@common/constants'; import { parseTypeMap, type TypeMap } from './typeMap'; import { TypeMapError } from './errors'; -const ALL = '*'; - type AliasesFile = Record>>; export type FileAliases = Map>>; export const resolveFileAliases = (fileAliases: FileAliases, namespace: string, layerName: string): Map => { const tiers = [ - fileAliases.get(ALL)?.get(ALL), - fileAliases.get(ALL)?.get(layerName), - fileAliases.get(namespace)?.get(ALL), + fileAliases.get(ALL_KEYS_SELECTOR)?.get(ALL_KEYS_SELECTOR), + fileAliases.get(ALL_KEYS_SELECTOR)?.get(layerName), + fileAliases.get(namespace)?.get(ALL_KEYS_SELECTOR), fileAliases.get(namespace)?.get(layerName), ]; return new Map(tiers.filter((tier): tier is Map => tier !== undefined).flatMap((tier) => [...tier])); From 2d3e85c2d36cc52e01b8a4df2201bbfe08c693e6 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Mon, 24 Aug 2026 10:07:56 +0300 Subject: [PATCH 27/29] feat: improve file parsing type inferring --- packages/synchronizer/src/sync/errors.ts | 12 +++++ packages/synchronizer/src/sync/fileReader.ts | 52 ++++++++++++++++--- .../src/sync/layerSource/sharedLuaSource.ts | 4 +- .../src/sync/layerSource/types.ts | 6 +++ packages/synchronizer/src/sync/luaParser.ts | 8 +-- 5 files changed, 66 insertions(+), 16 deletions(-) diff --git a/packages/synchronizer/src/sync/errors.ts b/packages/synchronizer/src/sync/errors.ts index 54b33dd..dd20ebc 100644 --- a/packages/synchronizer/src/sync/errors.ts +++ b/packages/synchronizer/src/sync/errors.ts @@ -60,6 +60,18 @@ export class TypeMapError extends SyncError { } } +export class LayersFileError extends SyncError { + public constructor(filePath: string, reason: string, cause?: unknown) { + super(`Invalid layers file ${filePath}: ${reason}`, { cause }); + } +} + +export class AliasesFileError extends SyncError { + public constructor(filePath: string, reason: string, cause?: unknown) { + super(`Invalid aliases file ${filePath}: ${reason}`, { cause }); + } +} + export class EnumSaveError extends SyncError { public constructor(layerName: string, cause?: unknown) { super(`Failed to save enum values for ${layerName}`, { cause }); diff --git a/packages/synchronizer/src/sync/fileReader.ts b/packages/synchronizer/src/sync/fileReader.ts index f1f3f6b..a0312ed 100644 --- a/packages/synchronizer/src/sync/fileReader.ts +++ b/packages/synchronizer/src/sync/fileReader.ts @@ -1,14 +1,39 @@ import { createHash } from 'node:crypto'; +import ajvCtor from 'ajv'; import type { JsonValue } from 'type-fest'; import { inject, injectable } from 'tsyringe'; import type { FsRepository } from '@common/fs/fsRepository'; import type { LayerEnums } from '@common/interfaces'; import { ALL_KEYS_SELECTOR, SERVICES } from '@common/constants'; import { parseTypeMap, type TypeMap } from './typeMap'; -import { TypeMapError } from './errors'; +import { AliasesFileError, LayersFileError, TypeMapError } from './errors'; type AliasesFile = Record>>; +const ajv = new ajvCtor(); + +const isLayersFile = ajv.compile({ + type: 'array', + items: { + type: 'object', + properties: { + layerName: { type: 'string' }, + enums: { type: 'array', items: { type: 'string' } }, + excludeProperties: { type: 'array', items: { type: 'string' } }, + sourceKey: { type: 'string' }, + }, + required: ['layerName', 'enums'], + }, +}); + +const isAliasesFile = ajv.compile({ + type: 'object', + additionalProperties: { + type: 'object', + additionalProperties: { type: 'object', additionalProperties: { type: 'string' } }, + }, +}); + export type FileAliases = Map>>; export const resolveFileAliases = (fileAliases: FileAliases, namespace: string, layerName: string): Map => { @@ -26,14 +51,21 @@ export class FileReader { public constructor(@inject(SERVICES.FS_REPOSITORY) private readonly fsRepository: FsRepository) {} public async readLayersWithChecksum(filePath: string): Promise<{ checksum: string; layers: LayerEnums[] }> { + let checksum: string; + let parsed: unknown; try { const buffer = await this.fsRepository.readFile(filePath); - const checksum = createHash('sha256').update(buffer).digest('hex'); - const layers = JSON.parse(buffer.toString()) as LayerEnums[]; - return { checksum, layers }; + checksum = createHash('sha256').update(buffer).digest('hex'); + parsed = JSON.parse(buffer.toString()); } catch (err) { - throw new Error(`Failed to read layers from ${filePath}`, { cause: err }); + throw new LayersFileError(filePath, 'failed to read or parse the file', err); + } + + if (!isLayersFile(parsed)) { + throw new LayersFileError(filePath, ajv.errorsText(isLayersFile.errors)); } + + return { checksum, layers: parsed }; } public async readTypeMap(filePath: string): Promise { @@ -49,12 +81,16 @@ export class FileReader { } public async readAliases(filePath: string): Promise { - let parsed: AliasesFile; + let parsed: unknown; try { const content = await this.fsRepository.readFile(filePath, 'utf-8'); - parsed = JSON.parse(content.toString()) as AliasesFile; + parsed = JSON.parse(content.toString()); } catch (err) { - throw new Error(`Failed to read aliases from ${filePath}`, { cause: err }); + throw new AliasesFileError(filePath, 'failed to read or parse the file', err); + } + + if (!isAliasesFile(parsed)) { + throw new AliasesFileError(filePath, ajv.errorsText(isAliasesFile.errors)); } return new Map( diff --git a/packages/synchronizer/src/sync/layerSource/sharedLuaSource.ts b/packages/synchronizer/src/sync/layerSource/sharedLuaSource.ts index ebdcab5..97bb5db 100644 --- a/packages/synchronizer/src/sync/layerSource/sharedLuaSource.ts +++ b/packages/synchronizer/src/sync/layerSource/sharedLuaSource.ts @@ -3,8 +3,8 @@ import { layerSource } from '@db'; import type { LayerEnums, SharedLuaSourceConfig } from '@common/interfaces'; import type { FsRepository } from '@common/fs/fsRepository'; import { fetchPropertyAliases } from '../aliasEnricher'; -import { parseLuaLayers, type LuaLayer } from '../luaParser'; -import type { LayerSourceRecord, FileDownloader, LayerSourceStrategy } from './types'; +import { parseLuaLayers } from '../luaParser'; +import type { LayerSourceRecord, FileDownloader, LayerSourceStrategy, LuaLayer } from './types'; const resolveOne = async (layer: LayerEnums, luaLayer: LuaLayer, config: SharedLuaSourceConfig, logger: Logger): Promise => { let propertyAliases = new Map(); diff --git a/packages/synchronizer/src/sync/layerSource/types.ts b/packages/synchronizer/src/sync/layerSource/types.ts index 693498f..fdbdc1a 100644 --- a/packages/synchronizer/src/sync/layerSource/types.ts +++ b/packages/synchronizer/src/sync/layerSource/types.ts @@ -15,3 +15,9 @@ export interface LayerSourceStrategy { export interface FileDownloader { downloadFile: (bucket: string, key: string) => Promise; } + +export interface LuaLayer { + layerId: number; + layerName: string; + alias: string; +} diff --git a/packages/synchronizer/src/sync/luaParser.ts b/packages/synchronizer/src/sync/luaParser.ts index abe7427..f0d8f75 100644 --- a/packages/synchronizer/src/sync/luaParser.ts +++ b/packages/synchronizer/src/sync/luaParser.ts @@ -1,3 +1,5 @@ +import type { LuaLayer } from './layerSource/types'; + const NOT_FOUND = -1; export const extractBlock = (content: string, variableName: string): string | null => { @@ -26,12 +28,6 @@ export const extractBlock = (content: string, variableName: string): string | nu return null; }; -export interface LuaLayer { - layerId: number; - layerName: string; - alias: string; -} - export const parseLuaLayers = ( content: string, variableName: string, From 4e82de8eedd678091f396b6c5f5bc627cfd891f2 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Mon, 24 Aug 2026 10:13:35 +0300 Subject: [PATCH 28/29] chore: split helpers to logical files --- packages/synchronizer/src/sync/SyncModel.ts | 2 +- packages/synchronizer/src/sync/aliasesFile.ts | 40 ++++++++++ packages/synchronizer/src/sync/cron.ts | 3 +- packages/synchronizer/src/sync/fileReader.ts | 75 +++---------------- packages/synchronizer/src/sync/layersFile.ts | 27 +++++++ .../integration/syncModel/syncModel.spec.ts | 3 +- 6 files changed, 84 insertions(+), 66 deletions(-) create mode 100644 packages/synchronizer/src/sync/aliasesFile.ts create mode 100644 packages/synchronizer/src/sync/layersFile.ts diff --git a/packages/synchronizer/src/sync/SyncModel.ts b/packages/synchronizer/src/sync/SyncModel.ts index 7b911b1..79dcac8 100644 --- a/packages/synchronizer/src/sync/SyncModel.ts +++ b/packages/synchronizer/src/sync/SyncModel.ts @@ -22,7 +22,7 @@ import { StalePropertiesDeletionError, TableColumnsQueryError, } from './errors'; -import { resolveFileAliases, type FileAliases } from './fileReader'; +import { resolveFileAliases, type FileAliases } from './aliasesFile'; import type { TypeMap } from './typeMap'; import type { LayerSourceRecord } from './layerSource/types'; diff --git a/packages/synchronizer/src/sync/aliasesFile.ts b/packages/synchronizer/src/sync/aliasesFile.ts new file mode 100644 index 0000000..d3be78d --- /dev/null +++ b/packages/synchronizer/src/sync/aliasesFile.ts @@ -0,0 +1,40 @@ +import ajvCtor from 'ajv'; +import { ALL_KEYS_SELECTOR } from '@common/constants'; +import { AliasesFileError } from './errors'; + +type AliasesFile = Record>>; + +const ajv = new ajvCtor(); + +const isAliasesFile = ajv.compile({ + type: 'object', + additionalProperties: { + type: 'object', + additionalProperties: { type: 'object', additionalProperties: { type: 'string' } }, + }, +}); + +export type FileAliases = Map>>; + +export const parseAliases = (raw: unknown, source: string): FileAliases => { + if (!isAliasesFile(raw)) { + throw new AliasesFileError(source, ajv.errorsText(isAliasesFile.errors)); + } + + return new Map( + Object.entries(raw).map(([namespace, layers]) => [ + namespace, + new Map(Object.entries(layers).map(([layer, props]) => [layer, new Map(Object.entries(props))])), + ]) + ); +}; + +export const resolveFileAliases = (fileAliases: FileAliases, namespace: string, layerName: string): Map => { + const tiers = [ + fileAliases.get(ALL_KEYS_SELECTOR)?.get(ALL_KEYS_SELECTOR), + fileAliases.get(ALL_KEYS_SELECTOR)?.get(layerName), + fileAliases.get(namespace)?.get(ALL_KEYS_SELECTOR), + fileAliases.get(namespace)?.get(layerName), + ]; + return new Map(tiers.filter((tier): tier is Map => tier !== undefined).flatMap((tier) => [...tier])); +}; diff --git a/packages/synchronizer/src/sync/cron.ts b/packages/synchronizer/src/sync/cron.ts index 2714bed..cd0fd63 100644 --- a/packages/synchronizer/src/sync/cron.ts +++ b/packages/synchronizer/src/sync/cron.ts @@ -6,7 +6,8 @@ import { SERVICES, NAMESPACE_HANDLES } from '@common/constants'; import { ConfigType } from '@common/config'; import { startActivePromisifiedSpan } from '@common/tracing/util'; import { SyncSpanName } from '@common/tracing/sync'; -import { FileReader, type FileAliases } from './fileReader'; +import { FileReader } from './fileReader'; +import type { FileAliases } from './aliasesFile'; import type { TypeMap } from './typeMap'; import type { NamespaceHandle } from './namespaceHandle/types'; diff --git a/packages/synchronizer/src/sync/fileReader.ts b/packages/synchronizer/src/sync/fileReader.ts index a0312ed..2d615c4 100644 --- a/packages/synchronizer/src/sync/fileReader.ts +++ b/packages/synchronizer/src/sync/fileReader.ts @@ -1,71 +1,29 @@ import { createHash } from 'node:crypto'; -import ajvCtor from 'ajv'; import type { JsonValue } from 'type-fest'; import { inject, injectable } from 'tsyringe'; import type { FsRepository } from '@common/fs/fsRepository'; import type { LayerEnums } from '@common/interfaces'; -import { ALL_KEYS_SELECTOR, SERVICES } from '@common/constants'; +import { SERVICES } from '@common/constants'; import { parseTypeMap, type TypeMap } from './typeMap'; +import { parseLayers } from './layersFile'; +import { parseAliases, type FileAliases } from './aliasesFile'; import { AliasesFileError, LayersFileError, TypeMapError } from './errors'; -type AliasesFile = Record>>; - -const ajv = new ajvCtor(); - -const isLayersFile = ajv.compile({ - type: 'array', - items: { - type: 'object', - properties: { - layerName: { type: 'string' }, - enums: { type: 'array', items: { type: 'string' } }, - excludeProperties: { type: 'array', items: { type: 'string' } }, - sourceKey: { type: 'string' }, - }, - required: ['layerName', 'enums'], - }, -}); - -const isAliasesFile = ajv.compile({ - type: 'object', - additionalProperties: { - type: 'object', - additionalProperties: { type: 'object', additionalProperties: { type: 'string' } }, - }, -}); - -export type FileAliases = Map>>; - -export const resolveFileAliases = (fileAliases: FileAliases, namespace: string, layerName: string): Map => { - const tiers = [ - fileAliases.get(ALL_KEYS_SELECTOR)?.get(ALL_KEYS_SELECTOR), - fileAliases.get(ALL_KEYS_SELECTOR)?.get(layerName), - fileAliases.get(namespace)?.get(ALL_KEYS_SELECTOR), - fileAliases.get(namespace)?.get(layerName), - ]; - return new Map(tiers.filter((tier): tier is Map => tier !== undefined).flatMap((tier) => [...tier])); -}; - @injectable() export class FileReader { public constructor(@inject(SERVICES.FS_REPOSITORY) private readonly fsRepository: FsRepository) {} public async readLayersWithChecksum(filePath: string): Promise<{ checksum: string; layers: LayerEnums[] }> { - let checksum: string; - let parsed: unknown; try { const buffer = await this.fsRepository.readFile(filePath); - checksum = createHash('sha256').update(buffer).digest('hex'); - parsed = JSON.parse(buffer.toString()); + const checksum = createHash('sha256').update(buffer).digest('hex'); + return { checksum, layers: parseLayers(JSON.parse(buffer.toString()), filePath) }; } catch (err) { + if (err instanceof LayersFileError) { + throw err; + } throw new LayersFileError(filePath, 'failed to read or parse the file', err); } - - if (!isLayersFile(parsed)) { - throw new LayersFileError(filePath, ajv.errorsText(isLayersFile.errors)); - } - - return { checksum, layers: parsed }; } public async readTypeMap(filePath: string): Promise { @@ -81,23 +39,14 @@ export class FileReader { } public async readAliases(filePath: string): Promise { - let parsed: unknown; try { const content = await this.fsRepository.readFile(filePath, 'utf-8'); - parsed = JSON.parse(content.toString()); + return parseAliases(JSON.parse(content.toString()), filePath); } catch (err) { + if (err instanceof AliasesFileError) { + throw err; + } throw new AliasesFileError(filePath, 'failed to read or parse the file', err); } - - if (!isAliasesFile(parsed)) { - throw new AliasesFileError(filePath, ajv.errorsText(isAliasesFile.errors)); - } - - return new Map( - Object.entries(parsed).map(([namespace, layers]) => [ - namespace, - new Map(Object.entries(layers).map(([layer, props]) => [layer, new Map(Object.entries(props))])), - ]) - ); } } diff --git a/packages/synchronizer/src/sync/layersFile.ts b/packages/synchronizer/src/sync/layersFile.ts new file mode 100644 index 0000000..764a405 --- /dev/null +++ b/packages/synchronizer/src/sync/layersFile.ts @@ -0,0 +1,27 @@ +import ajvCtor from 'ajv'; +import type { LayerEnums } from '@common/interfaces'; +import { LayersFileError } from './errors'; + +const ajv = new ajvCtor(); + +const isLayersFile = ajv.compile({ + type: 'array', + items: { + type: 'object', + properties: { + layerName: { type: 'string' }, + enums: { type: 'array', items: { type: 'string' } }, + excludeProperties: { type: 'array', items: { type: 'string' } }, + sourceKey: { type: 'string' }, + }, + required: ['layerName', 'enums'], + }, +}); + +export const parseLayers = (raw: unknown, source: string): LayerEnums[] => { + if (!isLayersFile(raw)) { + throw new LayersFileError(source, ajv.errorsText(isLayersFile.errors)); + } + + return raw; +}; diff --git a/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts b/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts index 68d8e48..99d0fe1 100644 --- a/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts +++ b/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts @@ -20,7 +20,8 @@ import { registerExternalValues } from '@src/containerConfig'; import { SERVICES, NAMESPACE_HANDLES } from '@src/common/constants'; import type { SyncModel } from '@src/sync/SyncModel'; import { schemaOf } from '@src/sync/helpers'; -import { FileReader, type FileAliases } from '@src/sync/fileReader'; +import { FileReader } from '@src/sync/fileReader'; +import type { FileAliases } from '@src/sync/aliasesFile'; import type { TypeMap } from '@src/sync/typeMap'; import type { NamespaceHandle } from '@src/sync/namespaceHandle/types'; From ce7bdc20b9e686fe04882b987b714380f5c095e0 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Mon, 24 Aug 2026 10:20:08 +0300 Subject: [PATCH 29/29] chore: remove override in package --- package.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/package.json b/package.json index adf6a26..c49c59d 100644 --- a/package.json +++ b/package.json @@ -25,8 +25,5 @@ }, "dependencies": { "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-3c2d0665e3509bb3e7113b328fc224fe6950895a.tgz" - }, - "overrides": { - "@types/node": "^24.0.0" } }