diff --git a/helm/charts/api/templates/deployment.yaml b/helm/charts/api/templates/deployment.yaml index 5d3d58f..2529833 100644 --- a/helm/charts/api/templates/deployment.yaml +++ b/helm/charts/api/templates/deployment.yaml @@ -93,6 +93,9 @@ spec: {{- end }} volumeMounts: + - name: namespaces-config + mountPath: /usr/src/app/packages/api/dist/config/production.json + subPath: production.json {{- if .Values.caSecretName }} - mountPath: {{ printf "%s/%s" .Values.caPath .Values.caKey | quote }} name: root-ca @@ -104,6 +107,9 @@ spec: readOnly: true {{- end }} volumes: + - name: namespaces-config + configMap: + name: {{ include "api.fullname" . }}-namespaces {{- if .Values.db.sslAuth.enabled }} - name: cert-conf secret: diff --git a/helm/charts/api/templates/namespaces-configmap.yaml b/helm/charts/api/templates/namespaces-configmap.yaml new file mode 100644 index 0000000..b0d9d60 --- /dev/null +++ b/helm/charts/api/templates/namespaces-configmap.yaml @@ -0,0 +1,14 @@ +{{- if .Values.enabled -}} +{{- $names := list -}} +{{- range .Values.namespaces -}} +{{- $names = append $names .name -}} +{{- end -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "api.fullname" . }}-namespaces + labels: + {{- include "api.labels" . | nindent 4 }} +data: + production.json: {{ dict "namespaces" $names | toPrettyJson | quote }} +{{- end }} diff --git a/helm/charts/api/values.yaml b/helm/charts/api/values.yaml index 8c23878..72e2fbf 100644 --- a/helm/charts/api/values.yaml +++ b/helm/charts/api/values.yaml @@ -52,6 +52,8 @@ env: metrics: enabled: false url: http://localhost:55681/v1/metrics +namespaces: [] + db: host: localhost port: 5432 diff --git a/helm/charts/synchronizer/templates/configmap.yaml b/helm/charts/synchronizer/templates/configmap.yaml index 66a4cc0..158ddc4 100644 --- a/helm/charts/synchronizer/templates/configmap.yaml +++ b/helm/charts/synchronizer/templates/configmap.yaml @@ -32,8 +32,6 @@ data: INDEX_NAME_FORMAT: {{ .Values.env.indexNameFormat | quote }} - LAYERS_FILE_PATH: {{ .Values.env.layersFile | quote }} - ALIASES_FILE_PATH: {{ .Values.env.aliasesFile | quote }} {{- if .Values.typeMap.enabled }} @@ -47,21 +45,8 @@ data: S3_SSL_ENABLED: {{ .sslEnabled | quote }} S3_SIG_VERSION: {{ .sigVersion | quote }} S3_STORAGE_CLASS: {{ .storageClass | quote }} - S3_BUCKET: {{ .bucket | quote }} - S3_FILE_NAME: {{ .fileName | quote }} - S3_LAYERS_VARIABLE: {{ .layersVariable | quote }} {{- end }} - {{- with .Values.env.enrichment }} - ENRICHMENT_ENABLED: {{ .enabled | quote }} - {{- if .enabled }} - ENRICHMENT_API_URL: {{ .api | quote }} - ENRICHMENT_PROPERTIES_PATH: {{ .propertiesPath | quote }} - ENRICHMENT_ALIAS_FIELD: {{ .aliasField | quote }} - ENRICHMENT_REQUEST_TIMEOUT_MILLISECONDS: {{ .requestTimeoutMilliseconds | quote }} - {{- end }} - {{- end }} - {{- include "synchronizer.dbEnvBlock" (dict "prefix" "SOURCE_DB" "db" .Values.dbs.sourceDb) | nindent 2 }} {{- include "synchronizer.dbEnvBlock" (dict "prefix" "DEST_DB" "db" .Values.dbs.destinationDb) | nindent 2 }} diff --git a/helm/charts/synchronizer/templates/deployment.yaml b/helm/charts/synchronizer/templates/deployment.yaml index c0a319f..aa77bc5 100644 --- a/helm/charts/synchronizer/templates/deployment.yaml +++ b/helm/charts/synchronizer/templates/deployment.yaml @@ -93,7 +93,10 @@ spec: {{- end }} volumeMounts: - name: layers-config - mountPath: {{ .Values.env.layersFile | dir | quote }} + mountPath: {{ .Values.env.layersDir | quote }} + - name: namespaces-config + mountPath: /usr/src/app/packages/synchronizer/dist/config/production.json + subPath: production.json - name: aliases-config mountPath: {{ .Values.env.aliasesFile | dir | quote }} {{- if .Values.typeMap.enabled }} @@ -102,11 +105,13 @@ spec: {{- end }} - name: s3-downloads mountPath: /usr/src/app/packages/synchronizer/dist/src/common/s3/downloads - {{- if .Values.dbs.sourceDb.sslAuth.enabled }} - - name: source-db-cert-conf - mountPath: /tmp/certs/source + {{- range .Values.namespaces }} + {{- if .db.ssl.enabled }} + - name: source-db-cert-{{ .name }} + mountPath: {{ printf "/tmp/certs/source-%s" .name | quote }} readOnly: true {{- end }} + {{- end }} {{- if .Values.dbs.destinationDb.sslAuth.enabled }} - name: dest-db-cert-conf mountPath: /tmp/certs/dest @@ -121,6 +126,9 @@ spec: - name: layers-config configMap: name: {{ include "synchronizer.fullname" . }}-layers + - name: namespaces-config + secret: + secretName: {{ include "synchronizer.fullname" . }}-namespaces - name: aliases-config configMap: name: {{ include "synchronizer.fullname" . }}-aliases @@ -131,16 +139,13 @@ spec: {{- end }} - name: s3-downloads emptyDir: {} - {{- if .Values.dbs.sourceDb.sslAuth.enabled }} - - name: source-db-cert-conf + {{- range .Values.namespaces }} + {{- if .db.ssl.enabled }} + - name: source-db-cert-{{ .name }} secret: - secretName: {{ .Values.dbs.sourceDb.sslAuth.secretName }} + secretName: {{ .db.sslSecretName }} + {{- end }} {{- end }} - {{- if .Values.caSecretName }} - - mountPath: {{ printf "%s/%s" .Values.caPath .Values.caKey | quote }} - name: root-ca - subPath: {{ quote .Values.caKey }} - {{- end }} {{- if .Values.dbs.destinationDb.sslAuth.enabled }} - name: dest-db-cert-conf secret: diff --git a/helm/charts/synchronizer/templates/layers-configmap.yaml b/helm/charts/synchronizer/templates/layers-configmap.yaml index 407ae25..5d7aef5 100644 --- a/helm/charts/synchronizer/templates/layers-configmap.yaml +++ b/helm/charts/synchronizer/templates/layers-configmap.yaml @@ -6,5 +6,7 @@ metadata: labels: {{- include "synchronizer.labels" . | nindent 4 }} data: - layers.json: {{ .Values.layers | toPrettyJson | quote }} + {{- range .Values.namespaces }} + {{ .name }}.json: {{ .layers | toPrettyJson | quote }} + {{- end }} {{- end }} diff --git a/helm/charts/synchronizer/templates/namespaces-secret.yaml b/helm/charts/synchronizer/templates/namespaces-secret.yaml new file mode 100644 index 0000000..f5c1376 --- /dev/null +++ b/helm/charts/synchronizer/templates/namespaces-secret.yaml @@ -0,0 +1,16 @@ +{{- if .Values.enabled -}} +{{- $layersDir := .Values.env.layersDir -}} +{{- $namespaces := list -}} +{{- range .Values.namespaces -}} +{{- $namespaces = append $namespaces (merge (dict "layersFile" (printf "%s/%s.json" $layersDir .name)) (omit . "layers")) -}} +{{- end -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "synchronizer.fullname" . }}-namespaces + labels: + {{- include "synchronizer.labels" . | nindent 4 }} +type: Opaque +stringData: + production.json: {{ dict "namespaces" $namespaces | toPrettyJson | quote }} +{{- end }} diff --git a/helm/charts/synchronizer/templates/secret.yaml b/helm/charts/synchronizer/templates/secret.yaml index 96277b1..12c441f 100644 --- a/helm/charts/synchronizer/templates/secret.yaml +++ b/helm/charts/synchronizer/templates/secret.yaml @@ -7,7 +7,6 @@ metadata: {{- include "synchronizer.labels" . | nindent 4 }} type: Opaque data: - SOURCE_DB_PASSWORD: {{ .Values.dbs.sourceDb.password | b64enc | quote }} DEST_DB_PASSWORD: {{ .Values.dbs.destinationDb.password | b64enc | quote }} S3_ACCESS_KEY: {{ .Values.dbs.s3.accessKey | b64enc | quote }} S3_SECRET_KEY: {{ .Values.dbs.s3.secretKey | b64enc | quote }} diff --git a/helm/charts/synchronizer/values.yaml b/helm/charts/synchronizer/values.yaml index a9912ee..55d4cd2 100644 --- a/helm/charts/synchronizer/values.yaml +++ b/helm/charts/synchronizer/values.yaml @@ -23,7 +23,6 @@ startupProbe: enabled: true periodSeconds: 10 timeoutSeconds: 5 - # failureThreshold * periodSeconds = max startup wait (30 * 10s = 300s) failureThreshold: 30 livenessProbe: @@ -41,11 +40,41 @@ readinessProbe: failureThreshold: 6 path: /liveness -layers: - - layerName: buildings_polygon - enums: [code, attribute, sensitivity, building_type, status] - - layerName: borders_polygon - enums: [code, attribute] +namespaces: + - name: default + bucket: "" + layers: + - layerName: buildings_polygon + enums: [code, attribute, sensitivity, building_type, status] + - layerName: borders_polygon + enums: [code, attribute] + db: + host: localhost + port: 5432 + username: postgres + password: postgres + database: postgres + schema: public + type: postgres + sslSecretName: secret-name + ssl: + enabled: false + ca: "" + cert: "" + key: "" + layerSource: + type: sharedLua + fileName: mock-query.lua + layersVariable: layers + aliasFieldName: alias + idFieldName: id + nameFieldName: name + enrichment: + enabled: false + api: "" + propertiesPath: "" + aliasField: "" + requestTimeoutMilliseconds: 5000 aliases: {} @@ -70,15 +99,9 @@ env: url: http://localhost:55681/v1/metrics schedule: '0 0 * * *' indexNameFormat: '{layerName}_{column}_idx' - layersFile: /usr/src/app/packages/synchronizer/dist/config/layers/layers.json + layersDir: /usr/src/app/packages/synchronizer/dist/config/layers aliasesFile: /usr/src/app/packages/synchronizer/dist/config/aliases/aliases.json typeMapFile: /usr/src/app/packages/synchronizer/dist/config/typeMap/typeMap.json - enrichment: - enabled: false - api: "" - propertiesPath: "" - aliasField: "" - requestTimeoutMilliseconds: 5000 dbs: s3: @@ -88,25 +111,8 @@ dbs: sslEnabled: false sigVersion: v4 storageClass: STANDARD - bucket: "" - fileName: "" - layersVariable: layers accessKey: "" secretKey: "" - sourceDb: - host: localhost - port: 5432 - username: postgres - password: postgres - database: postgres - schema: public - type: postgres - sslAuth: - enabled: false - secretName: secret-name - cert: "" - key: "" - root: "" destinationDb: host: localhost port: 5432 diff --git a/helm/values.yaml b/helm/values.yaml index 51b1cad..f3cb874 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -36,7 +36,38 @@ dbConfig: &dbConfig key: '' root: '' +namespaces: &namespaces + - name: default + bucket: "" + layers: + - layerName: buildings_polygon + enums: [ code, attribute ] + db: + host: localhost + port: 5432 + username: postgres + password: postgres + database: postgres + schema: public + type: postgres + sslSecretName: secret-name + ssl: + enabled: false + ca: "" + cert: "" + key: "" + layerSource: + type: sharedLua + fileName: "" + layersVariable: layers + aliasFieldName: alias + idFieldName: id + nameFieldName: name + enrichment: + enabled: false + api: + namespaces: *namespaces enabled: true replicaCount: 1 revisionHistoryLimit: 5 @@ -163,13 +194,12 @@ synchronizer: pullPolicy: Always tag: "v1.2.0" - layers: - - layerName: buildings_polygon - enums: [ code, attribute ] + namespaces: *namespaces aliases: "*": - num_floors: "number_of_floors" + "*": + num_floors: "number_of_floors" typeMap: enabled: false @@ -192,15 +222,9 @@ synchronizer: url: http://localhost:55681/v1/metrics schedule: '*/1 * * * *' indexNameFormat: '{layerName}_{column}_idx' - layersFile: /usr/src/app/packages/synchronizer/dist/config/layers/layers.json + layersDir: /usr/src/app/packages/synchronizer/dist/config/layers aliasesFile: /usr/src/app/packages/synchronizer/dist/config/aliases/aliases.json - typeMapFile: /usr/src/app/packages/synchronizer/dist/config/typeMap.json - enrichment: - enabled: false - api: "" - propertiesPath: "" - aliasField: "" - requestTimeoutMilliseconds: 5000 + typeMapFile: /usr/src/app/packages/synchronizer/dist/config/typeMap/typeMap.json dbs: s3: @@ -210,24 +234,8 @@ synchronizer: sslEnabled: false sigVersion: v4 storageClass: STANDARD - bucket: "" - fileName: "" - layersVariable: layers accessKey: "" secretKey: "" - sourceDb: - host: localhost - port: 5432 - username: postgres - password: postgres - database: postgres - schema: public - type: postgres - sslAuth: - enabled: false - cert: '' - key: '' - root: '' destinationDb: *dbConfig resources: diff --git a/openapi3.yaml b/openapi3.yaml index f38a4b6..9278bf8 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -2,37 +2,67 @@ openapi: 3.0.3 info: title: Layers & Specs API description: > - API for retrieving geographic layer specifications. Layers expose a set of - properties; `enum`-typed properties also carry their list of allowed values. - version: 1.2.0 + API for retrieving geographic layer specifications. Layers expose a set of properties; `enum`-typed properties also carry their list of allowed values. + version: 2.0.0 license: name: Proprietary url: https://example.com/licenses/proprietary +security: + - ApiKeyAuth: [] paths: - /layers: + /namespaces: get: - summary: Get all layer names + summary: Get all namespaces + operationId: getNamespaces + responses: + '200': + $ref: '#/components/responses/NamespaceListResponse' + '401': + $ref: '#/components/responses/Unauthorized' + /namespaces/{namespace}/layers: + get: + summary: Get all layer names within a namespace operationId: getLayers - security: [] + parameters: + - $ref: '#/components/parameters/NamespaceParam' responses: '200': $ref: '#/components/responses/LayerListResponse' + '401': + $ref: '#/components/responses/Unauthorized' '404': $ref: '#/components/responses/NotFound' - /layers/{layerName}: + /namespaces/{namespace}/layers/{layerName}: get: - summary: Get layer spec by name + summary: Get layer spec by name within a namespace operationId: getLayerByName - security: [] parameters: + - $ref: '#/components/parameters/NamespaceParam' - $ref: '#/components/parameters/LayerNameParam' responses: '200': $ref: '#/components/responses/LayerSpecResponse' + '401': + $ref: '#/components/responses/Unauthorized' '404': $ref: '#/components/responses/NotFound' components: + securitySchemes: + ApiKeyAuth: + type: apiKey + in: header + name: x-api-key + description: > + Enforced by the API gateway, not by this service. The service itself never inspects the key - it only declares the requirement so that clients, and the "Authorize" dialog on this page, send it. parameters: + NamespaceParam: + name: namespace + in: path + required: true + description: The namespace the layer belongs to. Layer names are only unique within a namespace + schema: + type: string + example: data-2025 LayerNameParam: name: layerName in: path @@ -42,6 +72,16 @@ components: type: string example: buildings_polygon responses: + NamespaceListResponse: + description: A list of the namespaces this deployment serves + content: + application/json: + schema: + $ref: '#/components/schemas/NamespaceList' + example: + namespaces: + - name: data-2024 + - name: data-2025 LayerListResponse: description: A list of layers with their display aliases content: @@ -82,7 +122,34 @@ components: application/json: schema: $ref: '#/components/schemas/Error' + Unauthorized: + description: > + The `x-api-key` header was missing or rejected. Returned by the API gateway, which never forwards the request to this service. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' schemas: + NamespaceSummary: + type: object + description: A namespace served by this deployment. + properties: + name: + type: string + description: Name of the namespace + example: data-2025 + required: + - name + NamespaceList: + type: object + description: Response wrapper for the list of namespaces. + properties: + namespaces: + type: array + items: + $ref: '#/components/schemas/NamespaceSummary' + required: + - namespaces LayerSummary: type: object description: Basic layer info returned in the list endpoint. @@ -111,8 +178,7 @@ components: LayerSpec: type: object description: > - A layer and all of its properties. Groups rows from the `Property` - entity by `layerName`. + A layer and all of its properties. Groups rows from the `Property` entity by `layerName`. properties: layerName: type: string @@ -133,10 +199,7 @@ components: Property: type: object description: > - A single property definition for a layer. Mirrors the `Property` entity, - minus `id` and `layerName` (which are implicit on the parent - `LayerSpec`). When the property has coded/enumerated values, - `possibleValues` is populated. + A single property definition for a layer. Mirrors the `Property` entity, minus `id` and `layerName` (which are implicit on the parent `LayerSpec`). When the property has coded/enumerated values, `possibleValues` is populated. properties: property: type: string @@ -151,8 +214,7 @@ components: possibleValues: type: array description: > - Allowed values for properties that have a fixed set of coded values. - Omitted when the property accepts arbitrary values. + Allowed values for properties that have a fixed set of coded values. Omitted when the property accepts arbitrary values. items: type: string required: diff --git a/package-lock.json b/package-lock.json index 95a8e82..69e0e33 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "packages/*" ], "dependencies": { - "@map-colonies/schemas": "1.26.0" + "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-3c2d0665e3509bb3e7113b328fc224fe6950895a.tgz" }, "devDependencies": { "@commitlint/cli": "^20.4.1", @@ -1129,6 +1129,17 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.3.tgz", + "integrity": "sha512-ELEBe8PsLvvJ6QMr0zLt8ffvOHW/dc1m3CEzNMg7aJUv3bMaoDtw2TXyDAwkYBuroxxuHEwhRTLJSe5sya547g==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@epic-web/invariant": { "version": "1.0.0", "dev": true, @@ -3232,8 +3243,8 @@ }, "node_modules/@map-colonies/schemas": { "version": "1.26.0", - "resolved": "https://registry.npmjs.org/@map-colonies/schemas/-/schemas-1.26.0.tgz", - "integrity": "sha512-XUQ2/gvC6d6/nGMKIGVyZo/kemGH5uGOy4xhkYCVLNV/CJsHvJXmhiov7QpqVpb58x+/2WVVUThAStj9ogipug==", + "resolved": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-3c2d0665e3509bb3e7113b328fc224fe6950895a.tgz", + "integrity": "sha512-UmMUnSTVagpHi0IwcBG7Vf7GuWothyKa3L/BGDrplR2mfG0DxTq9GtwImx24I/qVzEtqF5OO1WapeLqyIgRpMw==", "license": "MIT", "peer": true }, @@ -6914,6 +6925,13 @@ "@types/istanbul-lib-report": "*" } }, + "node_modules/@types/js-yaml": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz", + "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/json-schema": { "version": "7.0.15", "license": "MIT", @@ -6973,10 +6991,12 @@ } }, "node_modules/@types/node": { - "version": "26.1.1", + "version": "24.13.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", + "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", "license": "MIT", "dependencies": { - "undici-types": "~8.3.0" + "undici-types": "~7.18.0" } }, "node_modules/@types/node-cron": { @@ -7339,7 +7359,6 @@ "version": "8.63.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.63.0", @@ -7657,6 +7676,40 @@ "node": ">=14.0.0" } }, + "node_modules/@unrs/resolver-binding-wasm32-wasi/node_modules/@emnapi/core": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", + "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" + } + }, + "node_modules/@unrs/resolver-binding-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@unrs/resolver-binding-wasm32-wasi/node_modules/@emnapi/wasi-threads": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@unrs/resolver-binding-win32-arm64-msvc": { "version": "1.12.2", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-arm64-msvc/-/resolver-binding-win32-arm64-msvc-1.12.2.tgz", @@ -8036,6 +8089,8 @@ }, "node_modules/ajv": { "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "license": "MIT", "peer": true, "dependencies": { @@ -10866,7 +10921,6 @@ "version": "9.39.5", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -17170,6 +17224,8 @@ }, "node_modules/openapi-types": { "version": "12.1.3", + "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz", + "integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==", "license": "MIT", "peer": true }, @@ -21331,7 +21387,9 @@ } }, "node_modules/undici-types": { - "version": "8.3.0", + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", "license": "MIT" }, "node_modules/unique-filename": { @@ -22194,6 +22252,8 @@ "express": "^4.21.2", "express-openapi-validator": "^5.6.2", "http-status-codes": "^2.3.0", + "js-yaml": "^4.3.1", + "openapi-types": "^12.1.3", "prom-client": "^15.1.3", "reflect-metadata": "^0.2.2", "tsyringe": "^4.8.0", @@ -22208,6 +22268,7 @@ "@redocly/cli": "^2.16.0", "@types/compression": "^1.7.5", "@types/express": "^4.17.21", + "@types/js-yaml": "^4.0.9", "@types/multer": "^1.4.12", "@types/pg": "^8.11.14", "@types/supertest": "^6.0.2", @@ -22377,6 +22438,28 @@ "node": ">=0.10.0" } }, + "packages/api/node_modules/js-yaml": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", + "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "packages/api/node_modules/media-typer": { "version": "0.3.0", "license": "MIT", @@ -22533,6 +22616,7 @@ "@map-colonies/tracing-utils": "^1.0.0", "@opentelemetry/api": "^1.9.1", "@smithy/node-http-handler": "^4.7.6", + "ajv": "^8.20.0", "axios": "^1.7.0", "express": "^5.2.1", "node-cron": "^4.2.1", @@ -22545,7 +22629,7 @@ }, "devDependencies": { "@types/nock": "^10.0.3", - "@types/node": "^20.11.0", + "@types/node": "^24.0.0", "@types/node-cron": "^3.0.11", "@vitest/coverage-v8": "^4.0.18", "@vitest/ui": "^4.0.18", @@ -22561,19 +22645,6 @@ "peerDependencies": { "typeorm": "^0.3.20" } - }, - "packages/synchronizer/node_modules/@types/node": { - "version": "20.19.43", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~6.21.0" - } - }, - "packages/synchronizer/node_modules/undici-types": { - "version": "6.21.0", - "dev": true, - "license": "MIT" } } } diff --git a/package.json b/package.json index 67e5b2b..c49c59d 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,6 @@ "lerna": "^6.6.2" }, "dependencies": { - "@map-colonies/schemas": "1.26.0" + "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-3c2d0665e3509bb3e7113b328fc224fe6950895a.tgz" } } diff --git a/packages/api/config/test.json b/packages/api/config/test.json index d752fa8..1762bb1 100644 --- a/packages/api/config/test.json +++ b/packages/api/config/test.json @@ -1,4 +1,5 @@ { + "namespaces": ["data-2024", "data-2025"], "openapiConfig": { "filePath": "../../openapi3.yaml" }, diff --git a/packages/api/package.json b/packages/api/package.json index 8fa3aed..a3eaad7 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -46,6 +46,8 @@ "express": "^4.21.2", "express-openapi-validator": "^5.6.2", "http-status-codes": "^2.3.0", + "js-yaml": "^4.3.1", + "openapi-types": "^12.1.3", "prom-client": "^15.1.3", "reflect-metadata": "^0.2.2", "tsyringe": "^4.8.0", @@ -60,6 +62,7 @@ "@redocly/cli": "^2.16.0", "@types/compression": "^1.7.5", "@types/express": "^4.17.21", + "@types/js-yaml": "^4.0.9", "@types/multer": "^1.4.12", "@types/pg": "^8.11.14", "@types/supertest": "^6.0.2", @@ -73,10 +76,10 @@ "eslint-plugin-jest": "^28.11.0", "jest-openapi": "^0.14.2", "lerna": "^6.6.2", + "pg": "^8.20.0", "prettier": "^3.8.1", "pretty-quick": "^4.2.2", "rimraf": "^6.1.2", - "pg": "^8.20.0", "supertest": "^7.2.2", "ts-jest": "^29.2.6", "tsc-alias": "^1.8.11", diff --git a/packages/api/src/common/config.ts b/packages/api/src/common/config.ts index 5e577e4..6d16876 100644 --- a/packages/api/src/common/config.ts +++ b/packages/api/src/common/config.ts @@ -1,8 +1,8 @@ import { type ConfigInstance, config } from '@map-colonies/config'; -import { vectorVectorStandardApiV1, type vectorVectorStandardApiV1Type } from '@map-colonies/schemas'; +import { vectorVectorStandardApiV2, type vectorVectorStandardApiV2Type } from '@map-colonies/schemas'; // Choose here the type of the config instance and import this type from the entire application -type ConfigType = ConfigInstance; +type ConfigType = ConfigInstance; let configInstance: ConfigType | undefined; @@ -13,7 +13,7 @@ let configInstance: ConfigType | undefined; */ async function initConfig(offlineMode?: boolean): Promise { configInstance = await config({ - schema: vectorVectorStandardApiV1, + schema: vectorVectorStandardApiV2, offlineMode, }); } diff --git a/packages/api/src/common/constants.ts b/packages/api/src/common/constants.ts index 65754cc..f0300e2 100644 --- a/packages/api/src/common/constants.ts +++ b/packages/api/src/common/constants.ts @@ -1,12 +1,24 @@ import { readPackageJsonSync } from '@map-colonies/read-pkg'; +import type { PublicPath } from './interfaces'; export const SERVICE_NAME = readPackageJsonSync().name ?? 'unknown_service'; export const IGNORED_OUTGOING_TRACE_ROUTES = [/^.*\/v1\/metrics.*$/]; export const IGNORED_INCOMING_TRACE_ROUTES = [/^.*\/docs.*$/]; +export const PREFIX_HEADER = 'x-forwarded-prefix'; +export const ROOT_HEADER = 'x-gateway-root'; +export const NAMESPACE_PARAM_REF = '#/components/parameters/NamespaceParam'; + +export const PUBLIC_PATHS = new Map([ + ['/namespaces', { path: '/namespaces', scope: 'root' }], + ['/namespaces/{namespace}/layers', { path: '/', scope: 'namespaced' }], + ['/namespaces/{namespace}/layers/{layerName}', { path: '/{layerName}', scope: 'namespaced' }], +]); + export const HEALTHCHECK = Symbol('HealthCheck'); export const ON_SIGNAL = Symbol('onSignal'); +export const OPENAPI_SPEC = Symbol('OpenapiSpec'); /* eslint-disable @typescript-eslint/naming-convention */ export const SERVICES = { diff --git a/packages/api/src/common/db/connection.ts b/packages/api/src/common/db/connection.ts deleted file mode 100644 index c5b451f..0000000 --- a/packages/api/src/common/db/connection.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { FactoryFunction, DependencyContainer } from 'tsyringe'; -import { DataSource } from 'typeorm'; -import { createDataSourceOptions, createDataSourceHealthCheck, DATA_SOURCE_PROVIDER } from '@db'; -import { type HealthCheck } from '@godaddy/terminus'; -import { SERVICE_NAME, SERVICES } from '../constants'; -import type { ConfigType } from '../config'; - -export const dataSourceFactory: FactoryFunction = (container: DependencyContainer): DataSource => { - const config = container.resolve(SERVICES.CONFIG); - const dbConfig = config.get('db'); - return new DataSource(createDataSourceOptions(dbConfig, SERVICE_NAME)); -}; - -export const healthCheckFactory: FactoryFunction = (container: DependencyContainer): HealthCheck => - createDataSourceHealthCheck(container, [DATA_SOURCE_PROVIDER]); diff --git a/packages/api/src/common/interfaces.ts b/packages/api/src/common/interfaces.ts index d27d733..841a161 100644 --- a/packages/api/src/common/interfaces.ts +++ b/packages/api/src/common/interfaces.ts @@ -1,6 +1,22 @@ +import type { OpenAPIV3 } from 'openapi-types'; + export interface OpenApiConfig { filePath: string; basePath: string; jsonPath: string; uiPath: string; } + +export type OpenapiSpec = OpenAPIV3.Document; + +export interface GatewayBases { + namespaced: string; + root: string; +} + +export interface PublicPath { + path: string; + scope: keyof GatewayBases; +} + +export type PathRewrite = (pathItem: OpenAPIV3.PathItemObject, bases: GatewayBases) => OpenAPIV3.PathItemObject; diff --git a/packages/api/src/common/openapi.ts b/packages/api/src/common/openapi.ts new file mode 100644 index 0000000..5bab06f --- /dev/null +++ b/packages/api/src/common/openapi.ts @@ -0,0 +1,51 @@ +import { readFileSync } from 'node:fs'; +import { load } from 'js-yaml'; +import { OpenAPIV3 } from 'openapi-types'; +import { NAMESPACE_PARAM_REF, PUBLIC_PATHS } from './constants'; +import type { GatewayBases, OpenapiSpec, PathRewrite } from './interfaces'; + +const stripNamespaceParam = (pathItem: OpenAPIV3.PathItemObject): OpenAPIV3.PathItemObject => { + const stripped: OpenAPIV3.PathItemObject = { ...pathItem }; + + for (const method of Object.values(OpenAPIV3.HttpMethods)) { + const operation = stripped[method]; + + if (operation?.parameters === undefined) { + continue; + } + + stripped[method] = { + ...operation, + parameters: operation.parameters.filter((param) => !('$ref' in param) || param.$ref !== NAMESPACE_PARAM_REF), + }; + } + + return stripped; +}; + +const SCOPE_REWRITES = { + namespaced: stripNamespaceParam, + root: (pathItem, bases): OpenAPIV3.PathItemObject => ({ ...pathItem, servers: [{ url: bases.root }] }), +} satisfies Record; + +const loadSpec = (filePath: string): OpenapiSpec => load(readFileSync(filePath, 'utf8')) as OpenapiSpec; + +const toPublicSpec = (spec: OpenapiSpec, bases: GatewayBases): OpenapiSpec => { + const paths = Object.entries(spec.paths).flatMap(([path, pathItem]): [string, OpenAPIV3.PathItemObject][] => { + const publicPath = PUBLIC_PATHS.get(path); + + if (publicPath === undefined || pathItem === undefined) { + return []; + } + + return [[publicPath.path, SCOPE_REWRITES[publicPath.scope](pathItem, bases)]]; + }); + + return { + ...spec, + servers: [{ url: bases.namespaced, description: 'The API gateway. It rewrites these URLs onto the routes the service serves' }], + paths: Object.fromEntries(paths), + }; +}; + +export { loadSpec, toPublicSpec }; diff --git a/packages/api/src/containerConfig.ts b/packages/api/src/containerConfig.ts index 8c1f4e3..4ad6777 100644 --- a/packages/api/src/containerConfig.ts +++ b/packages/api/src/containerConfig.ts @@ -4,14 +4,18 @@ import { Registry } from 'prom-client'; import type { DependencyContainer } from 'tsyringe/dist/typings/types'; import { jsLogger, type Logger } from '@map-colonies/js-logger'; import { CleanupRegistry } from '@map-colonies/cleanup-registry'; -import { DATA_SOURCE_PROVIDER, Layer, LAYER_REPOSITORY_SYMBOL } from '@db'; +import { DATA_SOURCE_PROVIDER, Layer, LAYER_REPOSITORY_SYMBOL, createDataSource, createDataSourceHealthCheck } from '@db'; +import type { HealthCheck } from '@godaddy/terminus'; import type { Repository, DataSource } from 'typeorm'; import { instancePerContainerCachingFactory } from 'tsyringe'; import { type InjectionObject, registerDependencies } from '@common/dependencyRegistration'; -import { HEALTHCHECK, ON_SIGNAL, SERVICES, SERVICE_NAME } from '@common/constants'; +import { HEALTHCHECK, ON_SIGNAL, OPENAPI_SPEC, SERVICES, SERVICE_NAME } from '@common/constants'; +import { loadSpec } from '@common/openapi'; +import type { OpenapiSpec } from '@common/interfaces'; import { getTracing } from '@common/tracing'; import { LAYER_ROUTER_SYMBOL, layerRouterFactory } from './layer/routes/layer'; -import { dataSourceFactory, healthCheckFactory } from './common/db/connection'; +import { NAMESPACE_ROUTER_SYMBOL, namespaceRouterFactory } from './namespace/routes/namespace'; +import { DOCS_ROUTER_SYMBOL, docsRouterFactory } from './docs/routes/docs'; import { type ConfigType, getConfig } from './common/config'; export interface RegisterOptions { @@ -65,7 +69,18 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise useValue: cleanupRegistry.trigger.bind(cleanupRegistry), }, }, + { + token: OPENAPI_SPEC, + provider: { + useFactory: instancePerContainerCachingFactory((container): OpenapiSpec => { + const config = container.resolve(SERVICES.CONFIG); + return loadSpec(config.get('openapiConfig.filePath')); + }), + }, + }, { token: LAYER_ROUTER_SYMBOL, provider: { useFactory: layerRouterFactory } }, + { token: NAMESPACE_ROUTER_SYMBOL, provider: { useFactory: namespaceRouterFactory } }, + { token: DOCS_ROUTER_SYMBOL, provider: { useFactory: docsRouterFactory } }, { token: LAYER_REPOSITORY_SYMBOL, provider: { @@ -78,7 +93,10 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: DATA_SOURCE_PROVIDER, provider: { - useFactory: instancePerContainerCachingFactory(dataSourceFactory), + useFactory: instancePerContainerCachingFactory((container) => { + const config = container.resolve(SERVICES.CONFIG); + return createDataSource(config.get('db'), SERVICE_NAME); + }), }, postInjectionHook: async (container: DependencyContainer): Promise => { const dataSource = container.resolve(DATA_SOURCE_PROVIDER); @@ -96,7 +114,7 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: HEALTHCHECK, provider: { - useFactory: healthCheckFactory, + useFactory: (container: DependencyContainer): HealthCheck => createDataSourceHealthCheck(container, [DATA_SOURCE_PROVIDER]), }, }, ]; diff --git a/packages/api/src/docs/routes/docs.ts b/packages/api/src/docs/routes/docs.ts new file mode 100644 index 0000000..6dee0c5 --- /dev/null +++ b/packages/api/src/docs/routes/docs.ts @@ -0,0 +1,36 @@ +import { Router } from 'express'; +import type { Request, RequestHandler } from 'express'; +import type { FactoryFunction } from 'tsyringe'; +import { OpenapiViewerRouter } from '@map-colonies/openapi-express-viewer'; +import { OPENAPI_SPEC, PREFIX_HEADER, ROOT_HEADER, SERVICES } from '@common/constants'; +import { toPublicSpec } from '@common/openapi'; +import type { OpenapiSpec } from '@common/interfaces'; +import type { ConfigType } from '@common/config'; + +const docsRouterFactory: FactoryFunction = (dependencyContainer) => { + const config = dependencyContainer.resolve(SERVICES.CONFIG); + const openapiConfig = config.get('openapiConfig'); + const spec = dependencyContainer.resolve(OPENAPI_SPEC); + + const viewerRouter = new OpenapiViewerRouter({ ...openapiConfig, filePathOrSpec: spec }); + viewerRouter.setup(); + + const usePublicSpec: RequestHandler = (req, _, next) => { + const namespaced = req.get(PREFIX_HEADER); + const root = req.get(ROOT_HEADER); + const behindGateway = namespaced !== undefined && root !== undefined; + + (req as Request & { swaggerDoc?: OpenapiSpec }).swaggerDoc = behindGateway ? toPublicSpec(spec, { namespaced, root }) : spec; + + next(); + }; + + const router = Router(); + router.use(usePublicSpec, viewerRouter.getRouter()); + + return router; +}; + +const DOCS_ROUTER_SYMBOL = Symbol('docsRouterFactory'); + +export { docsRouterFactory, DOCS_ROUTER_SYMBOL }; diff --git a/packages/api/src/layer/controllers/layerController.ts b/packages/api/src/layer/controllers/layerController.ts index c40c803..83b03eb 100644 --- a/packages/api/src/layer/controllers/layerController.ts +++ b/packages/api/src/layer/controllers/layerController.ts @@ -4,9 +4,9 @@ import { injectable, inject } from 'tsyringe'; import { RequestHandler } from 'express'; import { SERVICES } from '@common/constants'; import { LayerManager } from '../models/layerManager'; -import { GetLayerByNameParam, GetLayersResponse } from '../types/layerTypes'; +import { GetLayerByNameParam, GetLayersParam, GetLayersResponse } from '../types/layerTypes'; -type GetLayersHandler = RequestHandler; +type GetLayersHandler = RequestHandler; type GetLayerByNameHandler = RequestHandler; @injectable() @@ -17,24 +17,25 @@ export class LayerController { ) {} public getLayers: GetLayersHandler = async (req, res, next) => { + const { namespace } = req.params; try { - const layers = await this.manager.getLayers(); - this.logger.debug({ msg: `got ${layers.length} layers` }); + const layers = await this.manager.getLayers(namespace); + this.logger.debug({ msg: `got ${layers.length} layers for namespace ${namespace}` }); return res.json({ layers }); } catch (error) { - this.logger.error({ msg: 'failed to get layers', err: error }); + this.logger.error({ msg: `failed to get layers for namespace ${namespace}`, err: error }); next(error); } }; public getLayerByName: GetLayerByNameHandler = async (req, res, next) => { - const { layerName } = req.params; + const { namespace, layerName } = req.params; try { - const layer = await this.manager.getLayerSpecByName(layerName); - this.logger.debug({ msg: `got layer: ${layerName}` }); + const layer = await this.manager.getLayerSpecByName(namespace, layerName); + this.logger.debug({ msg: `got layer: ${layerName} for namespace ${namespace}` }); return res.json(layer); } catch (error) { - this.logger.error({ msg: 'failed to get layer by name', layerName, err: error }); + this.logger.error({ msg: `failed to get layer ${layerName} for namespace ${namespace}`, err: error }); next(error); } }; diff --git a/packages/api/src/layer/models/layerManager.ts b/packages/api/src/layer/models/layerManager.ts index 47ad673..e6bce80 100644 --- a/packages/api/src/layer/models/layerManager.ts +++ b/packages/api/src/layer/models/layerManager.ts @@ -14,16 +14,19 @@ export class LayerManager { @inject(SERVICES.LOGGER) private readonly logger: Logger ) {} - public async getLayers(): Promise { - this.logger.debug({ msg: 'getting layers' }); - const layers = await this.repository.find({ select: { layerName: true, alias: true } }); + public async getLayers(namespace: string): Promise { + this.logger.debug({ msg: `getting layers for namespace ${namespace}` }); + const layers = await this.repository.find({ where: { namespace }, select: { layerName: true, alias: true } }); + if (layers.length === 0) { + throw new NotFoundError(`Namespace doesn't exist`); + } return layers.map(({ layerName, alias }) => ({ layerName, alias })); } - public async getLayerSpecByName(name: string): Promise { - this.logger.debug({ msg: 'getting layer spec by name', name }); + public async getLayerSpecByName(namespace: string, name: string): Promise { + this.logger.debug({ msg: `getting layer spec ${name} for namespace ${namespace}` }); const layer = await this.repository.findOne({ - where: { layerName: name }, + where: { namespace, layerName: name }, relations: { properties: { possibleValues: true } }, }); if (!layer) { diff --git a/packages/api/src/layer/routes/layer.ts b/packages/api/src/layer/routes/layer.ts index 09b45d0..d5f3952 100644 --- a/packages/api/src/layer/routes/layer.ts +++ b/packages/api/src/layer/routes/layer.ts @@ -6,8 +6,8 @@ const layerRouterFactory: FactoryFunction = (dependencyContainer) => { const router = Router(); const controller = dependencyContainer.resolve(LayerController); - router.get('/', controller.getLayers); - router.get('/:layerName', controller.getLayerByName); + router.get('/:namespace/layers', controller.getLayers); + router.get('/:namespace/layers/:layerName', controller.getLayerByName); return router; }; diff --git a/packages/api/src/layer/types/layerTypes.ts b/packages/api/src/layer/types/layerTypes.ts index 27d53d1..9d292f0 100644 --- a/packages/api/src/layer/types/layerTypes.ts +++ b/packages/api/src/layer/types/layerTypes.ts @@ -1,6 +1,11 @@ import type { LayerSummary } from '@db'; +export interface GetLayersParam { + namespace: string; +} + export interface GetLayerByNameParam { + namespace: string; layerName: string; } diff --git a/packages/api/src/namespace/controllers/namespaceController.ts b/packages/api/src/namespace/controllers/namespaceController.ts new file mode 100644 index 0000000..1c72f3d --- /dev/null +++ b/packages/api/src/namespace/controllers/namespaceController.ts @@ -0,0 +1,27 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { injectable, inject } from 'tsyringe'; +import { RequestHandler } from 'express'; +import { SERVICES } from '@common/constants'; +import { NamespaceManager } from '../models/namespaceManager'; +import { GetNamespacesResponse } from '../types/namespaceTypes'; + +type GetNamespacesHandler = RequestHandler; + +@injectable() +export class NamespaceController { + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(NamespaceManager) private readonly manager: NamespaceManager + ) {} + + public getNamespaces: GetNamespacesHandler = async (req, res, next) => { + try { + const namespaces = await this.manager.getNamespaces(); + this.logger.debug({ msg: `got ${namespaces.length} namespaces` }); + return res.json({ namespaces }); + } catch (error) { + this.logger.error({ msg: 'failed to get namespaces', err: error }); + next(error); + } + }; +} diff --git a/packages/api/src/namespace/models/namespaceManager.ts b/packages/api/src/namespace/models/namespaceManager.ts new file mode 100644 index 0000000..977ebf9 --- /dev/null +++ b/packages/api/src/namespace/models/namespaceManager.ts @@ -0,0 +1,24 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { injectable, inject } from 'tsyringe'; +import { Repository } from 'typeorm'; +import { Layer, LAYER_REPOSITORY_SYMBOL } from '@db'; +import type { NamespaceSummary } from '@db'; +import { SERVICES } from '@common/constants'; + +@injectable() +export class NamespaceManager { + public constructor( + @inject(LAYER_REPOSITORY_SYMBOL) private readonly repository: Repository, + @inject(SERVICES.LOGGER) private readonly logger: Logger + ) {} + + public async getNamespaces(): Promise { + this.logger.debug({ msg: 'getting namespaces' }); + return this.repository + .createQueryBuilder('layer') + .select('layer.namespace', 'name') + .distinct(true) + .orderBy('name', 'ASC') + .getRawMany(); + } +} diff --git a/packages/api/src/namespace/routes/namespace.ts b/packages/api/src/namespace/routes/namespace.ts new file mode 100644 index 0000000..e308e0f --- /dev/null +++ b/packages/api/src/namespace/routes/namespace.ts @@ -0,0 +1,16 @@ +import { Router } from 'express'; +import type { FactoryFunction } from 'tsyringe'; +import { NamespaceController } from '../controllers/namespaceController'; + +const namespaceRouterFactory: FactoryFunction = (dependencyContainer) => { + const router = Router(); + const controller = dependencyContainer.resolve(NamespaceController); + + router.get('/', controller.getNamespaces); + + return router; +}; + +export const NAMESPACE_ROUTER_SYMBOL = Symbol('namespaceRouterFactory'); + +export { namespaceRouterFactory }; diff --git a/packages/api/src/namespace/types/namespaceTypes.ts b/packages/api/src/namespace/types/namespaceTypes.ts new file mode 100644 index 0000000..3f2e445 --- /dev/null +++ b/packages/api/src/namespace/types/namespaceTypes.ts @@ -0,0 +1,5 @@ +import type { NamespaceSummary } from '@db'; + +export interface GetNamespacesResponse { + namespaces: NamespaceSummary[]; +} diff --git a/packages/api/src/serverBuilder.ts b/packages/api/src/serverBuilder.ts index f2e28e6..8e9880c 100644 --- a/packages/api/src/serverBuilder.ts +++ b/packages/api/src/serverBuilder.ts @@ -1,7 +1,6 @@ import express, { Router } from 'express'; import bodyParser from 'body-parser'; import compression from 'compression'; -import { OpenapiViewerRouter } from '@map-colonies/openapi-express-viewer'; import { getErrorHandlerMiddleware } from '@map-colonies/error-express-handler'; import { middleware as OpenApiMiddleware } from 'express-openapi-validator'; import { inject, injectable } from 'tsyringe'; @@ -11,6 +10,8 @@ import { collectMetricsExpressMiddleware } from '@map-colonies/prometheus'; import { Registry } from 'prom-client'; import { SERVICES } from '@common/constants'; import { LAYER_ROUTER_SYMBOL } from './layer/routes/layer'; +import { NAMESPACE_ROUTER_SYMBOL } from './namespace/routes/namespace'; +import { DOCS_ROUTER_SYMBOL } from './docs/routes/docs'; import { ConfigType } from './common/config'; @injectable() @@ -21,7 +22,9 @@ export class ServerBuilder { @inject(SERVICES.CONFIG) private readonly config: ConfigType, @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.METRICS) private readonly metricsRegistry: Registry, - @inject(LAYER_ROUTER_SYMBOL) private readonly layerRouter: Router + @inject(LAYER_ROUTER_SYMBOL) private readonly layerRouter: Router, + @inject(NAMESPACE_ROUTER_SYMBOL) private readonly namespaceRouter: Router, + @inject(DOCS_ROUTER_SYMBOL) private readonly docsRouter: Router ) { this.serverInstance = express(); } @@ -34,18 +37,10 @@ export class ServerBuilder { return this.serverInstance; } - private buildDocsRoutes(): void { - const openapiRouter = new OpenapiViewerRouter({ - ...this.config.get('openapiConfig'), - filePathOrSpec: this.config.get('openapiConfig.filePath'), - }); - openapiRouter.setup(); - this.serverInstance.use(this.config.get('openapiConfig.basePath'), openapiRouter.getRouter()); - } - private buildRoutes(): void { - this.serverInstance.use('/layers', this.layerRouter); - this.buildDocsRoutes(); + this.serverInstance.use('/namespaces', this.namespaceRouter); + this.serverInstance.use('/namespaces', this.layerRouter); + this.serverInstance.use(this.config.get('openapiConfig.basePath'), this.docsRouter); } private registerPreRoutesMiddleware(): void { @@ -60,7 +55,9 @@ export class ServerBuilder { const ignorePathRegex = new RegExp(`^${this.config.get('openapiConfig.basePath')}/.*`, 'i'); const apiSpecPath = this.config.get('openapiConfig.filePath'); - this.serverInstance.use(OpenApiMiddleware({ apiSpec: apiSpecPath, validateRequests: true, ignorePaths: ignorePathRegex })); + this.serverInstance.use( + OpenApiMiddleware({ apiSpec: apiSpecPath, validateRequests: true, validateSecurity: false, ignorePaths: ignorePathRegex }) + ); } private registerPostRoutesMiddleware(): void { diff --git a/packages/api/tests/integration/layer/helpers/layerRequestSender.ts b/packages/api/tests/integration/layer/helpers/layerRequestSender.ts index 9a73f66..99f4ad1 100644 --- a/packages/api/tests/integration/layer/helpers/layerRequestSender.ts +++ b/packages/api/tests/integration/layer/helpers/layerRequestSender.ts @@ -3,6 +3,7 @@ import type supertest from 'supertest'; import { agent } from 'supertest'; import type { Layer } from '@db'; import type { GetLayersResponse } from '@src/layer/types/layerTypes'; +import type { GetNamespacesResponse } from '@src/namespace/types/namespaceTypes'; interface TypedResponse extends Omit { body: T; @@ -11,11 +12,15 @@ interface TypedResponse extends Omit { export class LayerRequestSender { public constructor(private readonly app: Application) {} - public async getLayers(): Promise> { - return agent(this.app).get('/layers'); + public async getNamespaces(): Promise> { + return agent(this.app).get('/namespaces'); } - public async getLayerByName(layerName: string): Promise> { - return agent(this.app).get(`/layers/${layerName}`); + public async getLayers(namespace: string): Promise> { + return agent(this.app).get(`/namespaces/${namespace}/layers`); + } + + public async getLayerByName(namespace: string, layerName: string): Promise> { + return agent(this.app).get(`/namespaces/${namespace}/layers/${layerName}`); } } diff --git a/packages/api/tests/integration/layer/layer.spec.ts b/packages/api/tests/integration/layer/layer.spec.ts index 479cada..fbbf18b 100644 --- a/packages/api/tests/integration/layer/layer.spec.ts +++ b/packages/api/tests/integration/layer/layer.spec.ts @@ -3,13 +3,16 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; import type { DataSource, Repository } from 'typeorm'; -import { DATA_SOURCE_PROVIDER, EnumValue, LAYER_REPOSITORY_SYMBOL, Property, columnType } from '@map-colonies/vector-standard-db'; +import { DATA_SOURCE_PROVIDER, EnumValue, LAYER_REPOSITORY_SYMBOL, Property, columnType, layerSource } from '@map-colonies/vector-standard-db'; import type { Layer } from '@map-colonies/vector-standard-db'; import { getApp } from '@src/app'; import { SERVICES } from '@src/common/constants'; import { initConfig } from '@src/common/config'; import { LayerRequestSender } from './helpers/layerRequestSender'; +const NAMESPACE = 'data-2025'; +const OTHER_NAMESPACE = 'data-2024'; + describe('layer', function () { let requestSender: LayerRequestSender; let dataSource: DataSource; @@ -44,37 +47,76 @@ describe('layer', function () { }); describe('Happy Path', function () { - describe('GET /layers', function () { + describe('GET /namespaces', function () { + it('should return 200 with every namespace holding layers, without duplicates', async function () { + await layerRepository.save([ + { namespace: NAMESPACE, layerName: 'buildings_polygon', layerId: 1, source: layerSource.sharedLua, alias: 'Buildings Polygon' }, + { namespace: NAMESPACE, layerName: 'fences_line', layerId: 2, source: layerSource.sharedLua, alias: 'Fences Line' }, + { namespace: OTHER_NAMESPACE, layerName: 'buildings_polygon', layerId: null, source: layerSource.perLayerJson, alias: 'Old Buildings' }, + ]); + + const response = await requestSender.getNamespaces(); + const { namespaces } = response.body; + + expect(response.statusCode).toBe(httpStatusCodes.OK); + expect(namespaces).toEqual([{ name: OTHER_NAMESPACE }, { name: NAMESPACE }]); + }); + }); + + describe('GET /namespaces/:namespace/layers', function () { beforeEach(async function () { await layerRepository.save([ - { layerName: 'buildings_polygon', layerId: 1, alias: 'Buildings Polygon' }, - { layerName: 'fences_line', layerId: 2, alias: 'Fences Line' }, + { namespace: NAMESPACE, layerName: 'buildings_polygon', layerId: 1, source: layerSource.sharedLua, alias: 'Buildings Polygon' }, + { namespace: NAMESPACE, layerName: 'fences_line', layerId: 2, source: layerSource.sharedLua, alias: 'Fences Line' }, ]); await propertyRepository.save([ - { layerName: 'buildings_polygon', property: 'code', type: columnType.text }, - { layerName: 'fences_line', property: 'height', type: columnType.real }, + { namespace: NAMESPACE, layerName: 'buildings_polygon', property: 'code', type: columnType.text }, + { namespace: NAMESPACE, layerName: 'fences_line', property: 'height', type: columnType.real }, ]); }); it('should return 200 with all layers including their aliases', async function () { - const response = await requestSender.getLayers(); + const response = await requestSender.getLayers(NAMESPACE); const { layers } = response.body; expect(response.statusCode).toBe(httpStatusCodes.OK); expect(layers).toContainEqual({ layerName: 'buildings_polygon', alias: 'Buildings Polygon' }); expect(layers).toContainEqual({ layerName: 'fences_line', alias: 'Fences Line' }); }); + + it('should not return layers belonging to another namespace', async function () { + await layerRepository.save({ + namespace: OTHER_NAMESPACE, + layerName: 'rivers_line', + layerId: null, + source: layerSource.perLayerJson, + alias: 'Rivers', + }); + + const response = await requestSender.getLayers(NAMESPACE); + const { layers } = response.body; + + expect(response.statusCode).toBe(httpStatusCodes.OK); + expect(layers).toHaveLength(2); + expect(layers).not.toContainEqual({ layerName: 'rivers_line', alias: 'Rivers' }); + }); }); - describe('GET /layers/:layerName', function () { + describe('GET /namespaces/:namespace/layers/:layerName', function () { it('should return 200 with the layer spec for non-enum properties', async function () { - await layerRepository.save({ layerName: 'buildings_polygon', layerId: 1, alias: 'Buildings Polygon' }); + await layerRepository.save({ + namespace: NAMESPACE, + layerName: 'buildings_polygon', + layerId: 1, + source: layerSource.sharedLua, + alias: 'Buildings Polygon', + }); await propertyRepository.save([ - { layerName: 'buildings_polygon', property: 'code', type: columnType.text }, - { layerName: 'buildings_polygon', property: 'height', type: columnType.real }, + { namespace: NAMESPACE, layerName: 'buildings_polygon', property: 'code', type: columnType.text }, + { namespace: NAMESPACE, layerName: 'buildings_polygon', property: 'height', type: columnType.real }, ]); - const response = await requestSender.getLayerByName('buildings_polygon'); + const response = await requestSender.getLayerByName(NAMESPACE, 'buildings_polygon'); const { layerName, properties } = response.body; expect(response.statusCode).toBe(httpStatusCodes.OK); @@ -84,14 +126,20 @@ describe('layer', function () { }); it('should return 200 with possibleValues for enum properties', async function () { - await layerRepository.save({ layerName: 'buildings_polygon', layerId: 1, alias: 'Buildings Polygon' }); - await propertyRepository.save({ layerName: 'buildings_polygon', property: 'classification', type: columnType.text }); + await layerRepository.save({ + namespace: NAMESPACE, + layerName: 'buildings_polygon', + layerId: 1, + source: layerSource.sharedLua, + alias: 'Buildings Polygon', + }); + await propertyRepository.save({ namespace: NAMESPACE, layerName: 'buildings_polygon', property: 'classification', type: columnType.text }); await enumValueRepository.save([ - { value: 'A', layerName: 'buildings_polygon', property: 'classification' }, - { value: 'B', layerName: 'buildings_polygon', property: 'classification' }, + { namespace: NAMESPACE, value: 'A', layerName: 'buildings_polygon', property: 'classification' }, + { namespace: NAMESPACE, value: 'B', layerName: 'buildings_polygon', property: 'classification' }, ]); - const response = await requestSender.getLayerByName('buildings_polygon'); + const response = await requestSender.getLayerByName(NAMESPACE, 'buildings_polygon'); const { properties } = response.body; const classificationProp = properties.find((p) => p.property === 'classification'); @@ -100,23 +148,64 @@ describe('layer', function () { expect(classificationProp?.possibleValues).toContain('A'); expect(classificationProp?.possibleValues).toContain('B'); }); + + it('should return the spec of the requested namespace when the layer name exists in both', async function () { + await layerRepository.save([ + { namespace: NAMESPACE, layerName: 'buildings_polygon', layerId: 1, source: layerSource.sharedLua, alias: 'New Buildings' }, + { namespace: OTHER_NAMESPACE, layerName: 'buildings_polygon', layerId: null, source: layerSource.perLayerJson, alias: 'Old Buildings' }, + ]); + await propertyRepository.save([ + { namespace: NAMESPACE, layerName: 'buildings_polygon', property: 'code', type: columnType.text }, + { namespace: OTHER_NAMESPACE, layerName: 'buildings_polygon', property: 'legacy_code', type: columnType.text }, + ]); + + const response = await requestSender.getLayerByName(OTHER_NAMESPACE, 'buildings_polygon'); + const { alias, properties } = response.body; + + expect(response.statusCode).toBe(httpStatusCodes.OK); + expect(alias).toBe('Old Buildings'); + expect(properties).toContainEqual({ property: 'legacy_code', type: columnType.text }); + expect(properties).not.toContainEqual({ property: 'code', type: columnType.text }); + }); }); }); describe('Sad Path', function () { - describe('GET /layers', function () { + describe('GET /namespaces', function () { it('should return 200 with an empty list when there are no layers', async function () { - const response = await requestSender.getLayers(); - const { layers } = response.body; + const response = await requestSender.getNamespaces(); + const { namespaces } = response.body; expect(response.statusCode).toBe(httpStatusCodes.OK); - expect(layers).toHaveLength(0); + expect(namespaces).toHaveLength(0); }); }); - describe('GET /layers/:layerName', function () { + describe('GET /namespaces/:namespace/layers', function () { + it('should return 404 not found for a namespace holding no layers', async function () { + const response = await requestSender.getLayers('nonexistent'); + + expect(response.statusCode).toBe(httpStatusCodes.NOT_FOUND); + }); + }); + + describe('GET /namespaces/:namespace/layers/:layerName', function () { it('should return 404 not found for a non-existent layer', async function () { - const response = await requestSender.getLayerByName('nonexistent'); + const response = await requestSender.getLayerByName(NAMESPACE, 'nonexistent'); + + expect(response.statusCode).toBe(httpStatusCodes.NOT_FOUND); + }); + + it('should return 404 not found when the layer exists only in another namespace', async function () { + await layerRepository.save({ + namespace: OTHER_NAMESPACE, + layerName: 'rivers_line', + layerId: null, + source: layerSource.perLayerJson, + alias: 'Rivers', + }); + + const response = await requestSender.getLayerByName(NAMESPACE, 'rivers_line'); expect(response.statusCode).toBe(httpStatusCodes.NOT_FOUND); }); diff --git a/packages/api/tests/unit/docs/openapi.spec.ts b/packages/api/tests/unit/docs/openapi.spec.ts new file mode 100644 index 0000000..2920431 --- /dev/null +++ b/packages/api/tests/unit/docs/openapi.spec.ts @@ -0,0 +1,44 @@ +import { resolve } from 'node:path'; +import { describe, it, expect } from 'vitest'; +import { loadSpec, toPublicSpec } from '@src/common/openapi'; +import type { OpenapiSpec } from '@src/common/interfaces'; + +describe('openapi', function () { + const specPath = resolve('../../openapi3.yaml'); + const bases = { namespaced: 'https://gateway/vector/{namespace}', root: 'https://gateway/vector' }; + + const toPublic = (): OpenapiSpec => toPublicSpec(loadSpec(specPath), bases); + + const paramRefs = (spec: OpenapiSpec, route: string): (string | undefined)[] => + spec.paths[route]?.get?.parameters?.map((param) => ('$ref' in param ? param.$ref : undefined)) ?? []; + + describe('toPublicSpec', function () { + it('should give every path in the spec a public equivalent', function () { + const spec = loadSpec(specPath); + + expect(Object.keys(toPublicSpec(spec, bases).paths)).toHaveLength(Object.keys(spec.paths).length); + }); + + it('should point the top level servers at the namespaced gateway base', function () { + expect(toPublic().servers?.map((server) => server.url)).toEqual([bases.namespaced]); + }); + + it('should strip the namespace parameter from namespaced paths', function () { + const spec = toPublic(); + + expect(paramRefs(spec, '/')).toEqual([]); + expect(paramRefs(spec, '/{layerName}')).toEqual(['#/components/parameters/LayerNameParam']); + }); + + it('should override the servers of root scoped paths', function () { + expect(toPublic().paths['/namespaces']?.servers).toEqual([{ url: bases.root }]); + }); + + it('should drop paths that have no public equivalent', function () { + const spec = loadSpec(specPath); + spec.paths['/unmapped'] = { get: { responses: {} } }; + + expect(toPublicSpec(spec, bases).paths).not.toHaveProperty('/unmapped'); + }); + }); +}); diff --git a/packages/api/tests/unit/layer/models/layerManager.spec.ts b/packages/api/tests/unit/layer/models/layerManager.spec.ts index 7894896..3c37a40 100644 --- a/packages/api/tests/unit/layer/models/layerManager.spec.ts +++ b/packages/api/tests/unit/layer/models/layerManager.spec.ts @@ -7,6 +7,8 @@ import { SERVICES } from '@src/common/constants'; import { registerDependencies } from '@src/common/dependencyRegistration'; describe('LayerManager', function () { + const TEST_NAMESPACE = 'data-2026'; + let manager: LayerManager; const find = vi.fn(); @@ -38,7 +40,7 @@ describe('LayerManager', function () { ]; find.mockResolvedValue(layers); - const result = await manager.getLayers(); + const result = await manager.getLayers(TEST_NAMESPACE); expect(result).toEqual(layers); }); @@ -48,9 +50,12 @@ describe('LayerManager', function () { it('should query using the provided layer name', async function () { findOne.mockResolvedValue({ layerName: 'buildings_polygon', alias: 'Buildings', properties: [] }); - await manager.getLayerSpecByName('buildings_polygon'); + await manager.getLayerSpecByName(TEST_NAMESPACE, 'buildings_polygon'); - expect(findOne).toHaveBeenCalledWith({ where: { layerName: 'buildings_polygon' }, relations: { properties: { possibleValues: true } } }); + expect(findOne).toHaveBeenCalledWith({ + where: { layerName: 'buildings_polygon', namespace: TEST_NAMESPACE }, + relations: { properties: { possibleValues: true } }, + }); }); it.each([ @@ -84,7 +89,7 @@ describe('LayerManager', function () { ])('should $name', async function ({ inputProperties, expectedPossibleValues }) { findOne.mockResolvedValue({ layerName: 'buildings_polygon', alias: 'Buildings', properties: inputProperties }); - const { properties } = await manager.getLayerSpecByName('buildings_polygon'); + const { properties } = await manager.getLayerSpecByName(TEST_NAMESPACE, 'buildings_polygon'); for (const { property, possibleValues } of expectedPossibleValues) { expect(properties.find((p: { property: string }) => p.property === property)?.possibleValues).toEqual(possibleValues); @@ -95,12 +100,10 @@ describe('LayerManager', function () { describe('Sad Path', function () { describe('getLayers', function () { - it('should return an empty array when there are no layers', async function () { + it("should throw an error when the namespace doesn't exist", async function () { find.mockResolvedValue([]); - const result = await manager.getLayers(); - - expect(result).toEqual([]); + await expect(manager.getLayers(TEST_NAMESPACE)).rejects.toThrow(); }); }); @@ -108,7 +111,7 @@ describe('LayerManager', function () { it('should throw a NotFoundError when the layer does not exist', async function () { findOne.mockResolvedValue(null); - await expect(manager.getLayerSpecByName('nonexistent')).rejects.toThrow(NotFoundError); + await expect(manager.getLayerSpecByName(TEST_NAMESPACE, 'nonexistent')).rejects.toThrow(NotFoundError); }); }); }); diff --git a/packages/db/src/db/connection.ts b/packages/db/src/db/connection.ts index 290d847..c093184 100644 --- a/packages/db/src/db/connection.ts +++ b/packages/db/src/db/connection.ts @@ -1,5 +1,6 @@ import { readFileSync } from 'node:fs'; -import type { DataSource, DataSourceOptions } from 'typeorm'; +import { DataSource } from 'typeorm'; +import type { DataSourceOptions } from 'typeorm'; import type { DependencyContainer, InjectionToken } from 'tsyringe'; import { Property } from '../entities/property'; import { EnumValue } from '../entities/enumValue'; @@ -51,3 +52,6 @@ export const createDataSourceOptions = (dbConfig: DbConfig, applicationName: str ssl: createSslOptions(ssl), }; }; + +export const createDataSource = (dbConfig: DbConfig, applicationName: string): DataSource => + new DataSource(createDataSourceOptions(dbConfig, applicationName)); diff --git a/packages/db/src/entities/enumValue.ts b/packages/db/src/entities/enumValue.ts index d73b465..cdb60c2 100644 --- a/packages/db/src/entities/enumValue.ts +++ b/packages/db/src/entities/enumValue.ts @@ -5,6 +5,9 @@ export const ENUMS_REPOSITORY_SYMBOL = Symbol('EnumsRepository'); @Entity('enum_value') export class EnumValue { + @PrimaryColumn() + public namespace!: string; + @PrimaryColumn() public value!: string; @@ -16,6 +19,7 @@ export class EnumValue { @ManyToOne(() => Property, (p) => p.possibleValues, { onDelete: 'CASCADE' }) @JoinColumn([ + { name: 'namespace', referencedColumnName: 'namespace' }, { name: 'layer_name', referencedColumnName: 'layerName' }, { name: 'property', referencedColumnName: 'property' }, ]) diff --git a/packages/db/src/entities/layer.ts b/packages/db/src/entities/layer.ts index d28902a..fc8beb8 100644 --- a/packages/db/src/entities/layer.ts +++ b/packages/db/src/entities/layer.ts @@ -1,16 +1,29 @@ -import { Column, Entity, OneToMany, PrimaryColumn } from 'typeorm'; +import { Check, Column, Entity, OneToMany, PrimaryColumn } from 'typeorm'; import { Layer as ILayer } from '../types/layer'; +import { LayerSource, layerSource } from '../types/enums'; import { Property } from './property'; export const LAYER_REPOSITORY_SYMBOL = Symbol('LayerRepository'); +export const LAYER_SOURCE_CHECK = 'CHK_layer_source_layer_id'; + @Entity('layer') +@Check( + LAYER_SOURCE_CHECK, + `("source" = '${layerSource.sharedLua}' AND "layer_id" IS NOT NULL) OR ("source" = '${layerSource.perLayerJson}' AND "layer_id" IS NULL)` +) export class Layer implements ILayer { + @PrimaryColumn() + public namespace!: string; + @PrimaryColumn({ name: 'layer_name' }) public layerName!: string; - @Column({ name: 'layer_id' }) - public layerId!: number; + @Column({ name: 'layer_id', type: 'integer', nullable: true }) + public layerId!: number | null; + + @Column({ type: 'enum', enum: Object.values(layerSource), enumName: 'layer_source' }) + public source!: LayerSource; @Column() public alias!: string; diff --git a/packages/db/src/entities/property.ts b/packages/db/src/entities/property.ts index 9f72d9d..d6b50b3 100644 --- a/packages/db/src/entities/property.ts +++ b/packages/db/src/entities/property.ts @@ -7,6 +7,9 @@ export const PROPERTY_REPOSITORY_SYMBOL = Symbol('PropertyRepository'); @Entity('property') export class Property { + @PrimaryColumn() + public namespace!: string; + @PrimaryColumn({ name: 'layer_name' }) public layerName!: string; @@ -27,6 +30,9 @@ export class Property { public possibleValues?: EnumValue[]; @ManyToOne(() => Layer, (layer) => layer.properties) - @JoinColumn({ name: 'layer_name', referencedColumnName: 'layerName' }) + @JoinColumn([ + { name: 'namespace', referencedColumnName: 'namespace' }, + { name: 'layer_name', referencedColumnName: 'layerName' }, + ]) public layerRelation!: Layer; } diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 5cc38c5..e0807f8 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -1,8 +1,9 @@ -export { Layer, LAYER_REPOSITORY_SYMBOL } from './entities/layer'; +export { Layer, LAYER_REPOSITORY_SYMBOL, LAYER_SOURCE_CHECK } from './entities/layer'; export type { Layer as ILayer, LayerSpec, LayerSummary } from './types/layer'; +export type { NamespaceSummary } from './types/namespace'; export type { Property as PropertySpec } from './types/property'; export { Property, PROPERTY_REPOSITORY_SYMBOL } from './entities/property'; export { EnumValue, ENUMS_REPOSITORY_SYMBOL } from './entities/enumValue'; -export { ColumnType, columnType } from './types/enums'; -export { DATA_SOURCE_PROVIDER, createDataSourceOptions, createDataSourceHealthCheck, createSslOptions } from './db/connection'; +export { ColumnType, columnType, LayerSource, layerSource } from './types/enums'; +export { DATA_SOURCE_PROVIDER, createDataSource, createDataSourceOptions, createDataSourceHealthCheck, createSslOptions } from './db/connection'; export { DbConfig } from './interfaces'; diff --git a/packages/db/src/interfaces.ts b/packages/db/src/interfaces.ts index d52dbec..9eb298c 100644 --- a/packages/db/src/interfaces.ts +++ b/packages/db/src/interfaces.ts @@ -1,8 +1,7 @@ -import type { DataSourceOptions } from 'typeorm'; import type { ConfigType } from './db/config'; export type DbConfig = ReturnType['db']; export type SslConfig = DbConfig['ssl']; -export type SslOptions = Extract['ssl']; +export type SslOptions = false | { key?: Buffer; cert?: Buffer; ca?: Buffer }; diff --git a/packages/db/src/migrations/1786885317922-Migration.ts b/packages/db/src/migrations/1786885317922-Migration.ts new file mode 100644 index 0000000..c5f4330 --- /dev/null +++ b/packages/db/src/migrations/1786885317922-Migration.ts @@ -0,0 +1,61 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class Migration1786885317922 implements MigrationInterface { + name = 'Migration1786885317922'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "enum_value" DROP CONSTRAINT "FK_8548a0384cc733008852a8f4e70"`); + await queryRunner.query(`ALTER TABLE "property" DROP CONSTRAINT "FK_34ce6cedb279510bb20da63cf21"`); + await queryRunner.query(`ALTER TABLE "enum_value" ADD "namespace" character varying NOT NULL`); + await queryRunner.query(`ALTER TABLE "enum_value" DROP CONSTRAINT "PK_08cf35141adc487a902990ccff2"`); + await queryRunner.query( + `ALTER TABLE "enum_value" ADD CONSTRAINT "PK_0607639cc6d4c22cad2fa8552e9" PRIMARY KEY ("value", "layer_name", "property", "namespace")` + ); + await queryRunner.query(`ALTER TABLE "layer" ADD "namespace" character varying NOT NULL`); + await queryRunner.query(`ALTER TABLE "layer" DROP CONSTRAINT "PK_ee22a436b5250ea132444cd9baa"`); + await queryRunner.query(`ALTER TABLE "layer" ADD CONSTRAINT "PK_0080403b718d78e8dc9590e56ad" PRIMARY KEY ("layer_name", "namespace")`); + await queryRunner.query(`CREATE TYPE "public"."layer_source" AS ENUM('sharedLua', 'perLayerJson')`); + await queryRunner.query(`ALTER TABLE "layer" ADD "source" "public"."layer_source" NOT NULL`); + await queryRunner.query(`ALTER TABLE "property" ADD "namespace" character varying NOT NULL`); + await queryRunner.query(`ALTER TABLE "property" DROP CONSTRAINT "PK_c92abce6639221fa0e46b3d1380"`); + await queryRunner.query( + `ALTER TABLE "property" ADD CONSTRAINT "PK_f92b1edaa8f549c489fc2de4a05" PRIMARY KEY ("layer_name", "property", "namespace")` + ); + await queryRunner.query(`ALTER TABLE "layer" ALTER COLUMN "layer_id" DROP NOT NULL`); + await queryRunner.query( + `ALTER TABLE "layer" ADD CONSTRAINT "CHK_layer_source_layer_id" CHECK (("source" = 'sharedLua' AND "layer_id" IS NOT NULL) OR ("source" = 'perLayerJson' AND "layer_id" IS NULL))` + ); + await queryRunner.query( + `ALTER TABLE "enum_value" ADD CONSTRAINT "FK_10b4583c2705fa3454d63f05534" FOREIGN KEY ("namespace", "layer_name", "property") REFERENCES "property"("namespace","layer_name","property") ON DELETE CASCADE ON UPDATE NO ACTION` + ); + await queryRunner.query( + `ALTER TABLE "property" ADD CONSTRAINT "FK_d4a43c70a881756a56e51b86b84" FOREIGN KEY ("namespace", "layer_name") REFERENCES "layer"("namespace","layer_name") ON DELETE NO ACTION ON UPDATE NO ACTION` + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "property" DROP CONSTRAINT "FK_d4a43c70a881756a56e51b86b84"`); + await queryRunner.query(`ALTER TABLE "enum_value" DROP CONSTRAINT "FK_10b4583c2705fa3454d63f05534"`); + await queryRunner.query(`ALTER TABLE "layer" DROP CONSTRAINT "CHK_layer_source_layer_id"`); + await queryRunner.query(`ALTER TABLE "layer" ALTER COLUMN "layer_id" SET NOT NULL`); + await queryRunner.query(`ALTER TABLE "property" DROP CONSTRAINT "PK_f92b1edaa8f549c489fc2de4a05"`); + await queryRunner.query(`ALTER TABLE "property" ADD CONSTRAINT "PK_c92abce6639221fa0e46b3d1380" PRIMARY KEY ("layer_name", "property")`); + await queryRunner.query(`ALTER TABLE "property" DROP COLUMN "namespace"`); + await queryRunner.query(`ALTER TABLE "layer" DROP COLUMN "source"`); + await queryRunner.query(`DROP TYPE "public"."layer_source"`); + await queryRunner.query(`ALTER TABLE "layer" DROP CONSTRAINT "PK_0080403b718d78e8dc9590e56ad"`); + await queryRunner.query(`ALTER TABLE "layer" ADD CONSTRAINT "PK_ee22a436b5250ea132444cd9baa" PRIMARY KEY ("layer_name")`); + await queryRunner.query(`ALTER TABLE "layer" DROP COLUMN "namespace"`); + await queryRunner.query(`ALTER TABLE "enum_value" DROP CONSTRAINT "PK_0607639cc6d4c22cad2fa8552e9"`); + await queryRunner.query( + `ALTER TABLE "enum_value" ADD CONSTRAINT "PK_08cf35141adc487a902990ccff2" PRIMARY KEY ("value", "layer_name", "property")` + ); + await queryRunner.query(`ALTER TABLE "enum_value" DROP COLUMN "namespace"`); + await queryRunner.query( + `ALTER TABLE "property" ADD CONSTRAINT "FK_34ce6cedb279510bb20da63cf21" FOREIGN KEY ("layer_name") REFERENCES "layer"("layer_name") ON DELETE NO ACTION ON UPDATE NO ACTION` + ); + await queryRunner.query( + `ALTER TABLE "enum_value" ADD CONSTRAINT "FK_8548a0384cc733008852a8f4e70" FOREIGN KEY ("layer_name", "property") REFERENCES "property"("layer_name","property") ON DELETE CASCADE ON UPDATE NO ACTION` + ); + } +} diff --git a/packages/db/src/openapi.d.ts b/packages/db/src/openapi.d.ts index c6916e9..71613fd 100644 --- a/packages/db/src/openapi.d.ts +++ b/packages/db/src/openapi.d.ts @@ -4,14 +4,31 @@ import type { TypedRequestHandlers as ImportedTypedRequestHandlers } from '@map-colonies/openapi-helpers/typedRequestHandler'; export type paths = { - '/layers': { + '/namespaces': { parameters: { query?: never; header?: never; path?: never; cookie?: never; }; - /** Get all layer names */ + /** Get all namespaces */ + get: operations['getNamespaces']; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + '/namespaces/{namespace}/layers': { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get all layer names within a namespace */ get: operations['getLayers']; put?: never; post?: never; @@ -21,14 +38,14 @@ export type paths = { patch?: never; trace?: never; }; - '/layers/{layerName}': { + '/namespaces/{namespace}/layers/{layerName}': { parameters: { query?: never; header?: never; path?: never; cookie?: never; }; - /** Get layer spec by name */ + /** Get layer spec by name within a namespace */ get: operations['getLayerByName']; put?: never; post?: never; @@ -42,6 +59,18 @@ export type paths = { export type webhooks = Record; export type components = { schemas: { + /** @description A namespace served by this deployment. */ + NamespaceSummary: { + /** + * @description Name of the namespace + * @example data-2025 + */ + name: string; + }; + /** @description Response wrapper for the list of namespaces. */ + NamespaceList: { + namespaces: components['schemas']['NamespaceSummary'][]; + }; /** @description Basic layer info returned in the list endpoint. */ LayerSummary: { /** @@ -103,6 +132,27 @@ export type components = { }; }; responses: { + /** @description A list of the namespaces this deployment serves */ + NamespaceListResponse: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "namespaces": [ + * { + * "name": "data-2024" + * }, + * { + * "name": "data-2025" + * } + * ] + * } + */ + 'application/json': components['schemas']['NamespaceList']; + }; + }; /** @description A list of layers with their display aliases */ LayerListResponse: { headers: { @@ -175,6 +225,8 @@ export type components = { }; }; parameters: { + /** @description The namespace the layer belongs to. Layer names are only unique within a namespace */ + NamespaceParam: string; /** @description The name of the layer (matches `Property.layerName`) */ LayerNameParam: string; }; @@ -184,7 +236,7 @@ export type components = { }; export type $defs = Record; export interface operations { - getLayers: { + getNamespaces: { parameters: { query?: never; header?: never; @@ -192,6 +244,21 @@ export interface operations { cookie?: never; }; requestBody?: never; + responses: { + 200: components['responses']['NamespaceListResponse']; + }; + }; + getLayers: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The namespace the layer belongs to. Layer names are only unique within a namespace */ + namespace: components['parameters']['NamespaceParam']; + }; + cookie?: never; + }; + requestBody?: never; responses: { 200: components['responses']['LayerListResponse']; 404: components['responses']['NotFound']; @@ -202,6 +269,8 @@ export interface operations { query?: never; header?: never; path: { + /** @description The namespace the layer belongs to. Layer names are only unique within a namespace */ + namespace: components['parameters']['NamespaceParam']; /** @description The name of the layer (matches `Property.layerName`) */ layerName: components['parameters']['LayerNameParam']; }; diff --git a/packages/db/src/types/enums.ts b/packages/db/src/types/enums.ts index ef26b74..2d0ab34 100644 --- a/packages/db/src/types/enums.ts +++ b/packages/db/src/types/enums.ts @@ -14,3 +14,10 @@ export const columnType = { } as const; export type ColumnType = (typeof columnType)[keyof typeof columnType]; + +export const layerSource = { + sharedLua: 'sharedLua', + perLayerJson: 'perLayerJson', +} as const; + +export type LayerSource = (typeof layerSource)[keyof typeof layerSource]; diff --git a/packages/db/src/types/namespace.ts b/packages/db/src/types/namespace.ts new file mode 100644 index 0000000..57b6be8 --- /dev/null +++ b/packages/db/src/types/namespace.ts @@ -0,0 +1,3 @@ +import type { components } from '../openapi'; + +export type NamespaceSummary = components['schemas']['NamespaceSummary']; diff --git a/packages/synchronizer/config/default.json b/packages/synchronizer/config/default.json index 9067f03..34d28d4 100644 --- a/packages/synchronizer/config/default.json +++ b/packages/synchronizer/config/default.json @@ -3,27 +3,11 @@ "offlineMode": true, "name": "standard-synchronizer" }, - "layersFile": "./config/layers.json", "aliasesFile": "./config/aliases.json", "typeMapFile": "./config/typeMap.json", "schedule": "*/1 * * * *", "indexNameFormat": "{layerName}_{column}_idx", - "enrichment": { - "enabled": false - }, "dbs": { - "source": { - "type": "postgres", - "host": "localhost", - "port": 5432, - "username": "postgres", - "password": "postgres", - "database": "postgres", - "ssl": { - "enabled": false - }, - "schema": "query" - }, "destination": { "type": "postgres", "host": "localhost", @@ -41,15 +25,39 @@ "accessKeyId": "key", "secretAccessKey": "keySecret", "forcePathStyle": true, - "region": "local", - "bucket": "vector-query-example", - "fileName": "mock-query.lua", - "layersVariable": "layers", - "aliasFieldName": "alias", - "idFieldName": "id", - "nameFieldName": "name" + "region": "local" } }, + "namespaces": [ + { + "name": "default", + "bucket": "vector-query-example", + "layersFile": "./config/layers/default.json", + "db": { + "type": "postgres", + "host": "localhost", + "port": 5432, + "username": "postgres", + "password": "postgres", + "database": "postgres", + "ssl": { + "enabled": false + }, + "schema": "query" + }, + "layerSource": { + "type": "sharedLua", + "fileName": "mock-query.lua", + "layersVariable": "layers", + "aliasFieldName": "alias", + "idFieldName": "id", + "nameFieldName": "name", + "enrichment": { + "enabled": false + } + } + } + ], "openapiConfig": { "filePath": "./openapi3.yaml", "basePath": "/docs", diff --git a/packages/synchronizer/config/layers.json b/packages/synchronizer/config/layers/default.json similarity index 100% rename from packages/synchronizer/config/layers.json rename to packages/synchronizer/config/layers/default.json diff --git a/packages/synchronizer/config/layers/test.json b/packages/synchronizer/config/layers/test.json new file mode 100644 index 0000000..23709ec --- /dev/null +++ b/packages/synchronizer/config/layers/test.json @@ -0,0 +1,7 @@ +[ + { + "layerName": "buildings_polygon", + "excludeProperties": ["id"], + "enums": ["code", "attribute"] + } +] diff --git a/packages/synchronizer/config/test.json b/packages/synchronizer/config/test.json index 1884e75..8d2d4ee 100644 --- a/packages/synchronizer/config/test.json +++ b/packages/synchronizer/config/test.json @@ -1,17 +1,41 @@ { "dbs": { - "source": { - "schema": "test_source" - }, "destination": { "schema": "test_dest" } }, - "enrichmentApi": { - "enabled": true, - "api": "https://example.com/{layerName}", - "propertiesPath": "fields_list", - "aliasField": "display_name", - "requestTimeoutMilliseconds": 10000 - } + "namespaces": [ + { + "name": "test", + "bucket": "vector-query-example", + "layersFile": "./config/layers/test.json", + "db": { + "type": "postgres", + "host": "localhost", + "port": 5432, + "username": "postgres", + "password": "postgres", + "database": "postgres", + "ssl": { + "enabled": false + }, + "schema": "test_source" + }, + "layerSource": { + "type": "sharedLua", + "fileName": "mock-query.lua", + "layersVariable": "layers", + "aliasFieldName": "alias", + "idFieldName": "id", + "nameFieldName": "name", + "enrichment": { + "enabled": true, + "api": "https://example.com/{layerName}", + "propertiesPath": "fields_list", + "aliasField": "display_name", + "requestTimeoutMilliseconds": 10000 + } + } + } + ] } diff --git a/packages/synchronizer/package.json b/packages/synchronizer/package.json index 776d0d0..9e8ff0c 100644 --- a/packages/synchronizer/package.json +++ b/packages/synchronizer/package.json @@ -9,7 +9,7 @@ "start": "npm run build && node dist/src/index.js", "start:dev": "npm run build && cd dist && cross-env CONFIG_OFFLINE_MODE=true node --enable-source-maps --import ./instrumentation.mjs ./src/index.js", "dev": "ts-node src/index.ts", - "assets:copy": "copyfiles -f ./config/* ./dist/config && copyfiles -f copyfiles ./package.json dist", + "assets:copy": "copyfiles -u 1 \"./config/**/*\" ./dist/config && copyfiles -f copyfiles ./package.json dist", "watch": "ts-node-dev --respawn src/index.ts", "test:integration": "vitest run --coverage.enabled=false --project integration", "test": "vitest run", @@ -32,6 +32,7 @@ "@map-colonies/tracing-utils": "^1.0.0", "@opentelemetry/api": "^1.9.1", "@smithy/node-http-handler": "^4.7.6", + "ajv": "^8.20.0", "axios": "^1.7.0", "express": "^5.2.1", "node-cron": "^4.2.1", @@ -44,7 +45,7 @@ }, "devDependencies": { "@types/nock": "^10.0.3", - "@types/node": "^20.11.0", + "@types/node": "^24.0.0", "@types/node-cron": "^3.0.11", "@vitest/coverage-v8": "^4.0.18", "@vitest/ui": "^4.0.18", diff --git a/packages/synchronizer/src/common/config.ts b/packages/synchronizer/src/common/config.ts index b472e6d..17bd753 100644 --- a/packages/synchronizer/src/common/config.ts +++ b/packages/synchronizer/src/common/config.ts @@ -1,8 +1,8 @@ import { type ConfigInstance, config } from '@map-colonies/config'; -import { vectorVectorStandardSynchronizerV3, type vectorVectorStandardSynchronizerV3Type } from '@map-colonies/schemas'; +import { vectorVectorStandardSynchronizerV4, type vectorVectorStandardSynchronizerV4Type } from '@map-colonies/schemas'; // Choose here the type of the config instance and import this type from the entire application -type ConfigType = ConfigInstance; +type ConfigType = ConfigInstance; let configInstance: ConfigType | undefined; @@ -13,7 +13,7 @@ let configInstance: ConfigType | undefined; */ async function initConfig(offlineMode?: boolean): Promise { configInstance = await config({ - schema: vectorVectorStandardSynchronizerV3, + schema: vectorVectorStandardSynchronizerV4, offlineMode, }); } diff --git a/packages/synchronizer/src/common/constants.ts b/packages/synchronizer/src/common/constants.ts index 0e6c984..f4a3239 100644 --- a/packages/synchronizer/src/common/constants.ts +++ b/packages/synchronizer/src/common/constants.ts @@ -20,10 +20,7 @@ export const HEALTHCHECK = Symbol('HealthCheck'); export const s3ConfigPath = 'dbs.s3'; -export const DB_CONFIGS = [ - { configKey: 'dbs.source', token: SOURCE_DATA_SOURCE_PROVIDER }, - { configKey: 'dbs.destination', token: DESTINATION_DATA_SOURCE_PROVIDER }, -] as const; +export const DESTINATION_DB_CONFIG_PATH = 'dbs.destination'; export const REPOSITORIES = [ { entity: Layer, token: LAYER_REPOSITORY_SYMBOL }, @@ -31,13 +28,21 @@ export const REPOSITORIES = [ { entity: EnumValue, token: ENUMS_REPOSITORY_SYMBOL }, ] as const; +export const ALL_KEYS_SELECTOR = '*'; + /* eslint-disable @typescript-eslint/naming-convention */ export const SERVICES = { LOGGER: Symbol('Logger'), CONFIG: Symbol('Config'), S3_CLIENT: Symbol('S3Client'), + S3_REPOSITORY: Symbol('S3Repository'), + FS_REPOSITORY: Symbol('FsRepository'), TRACER: Symbol('Tracer'), METRICS: Symbol('METRICS'), CLEANUP_REGISTRY: Symbol('CleanupRegistry'), } satisfies Record; /* eslint-enable @typescript-eslint/naming-convention */ + +export { DESTINATION_DATA_SOURCE_PROVIDER }; + +export const NAMESPACE_HANDLES = Symbol('NamespaceHandles'); diff --git a/packages/synchronizer/src/common/db/connection.ts b/packages/synchronizer/src/common/db/connection.ts deleted file mode 100644 index 3ae1624..0000000 --- a/packages/synchronizer/src/common/db/connection.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type { FactoryFunction, DependencyContainer } from 'tsyringe'; -import { DataSource } from 'typeorm'; -import { createDataSourceOptions, createDataSourceHealthCheck, type DbConfig } from '@db'; -import type { HealthCheck } from '@godaddy/terminus'; -import { DB_CONFIGS, SERVICE_NAME, SERVICES } from '../constants'; -import type { ConfigType } from '../config'; - -type DbConfigKey = (typeof DB_CONFIGS)[number]['configKey']; - -export const createDataSourceFactory = - (configKey: DbConfigKey): FactoryFunction => - (container: DependencyContainer): DataSource => { - const config = container.resolve(SERVICES.CONFIG); - const dbConfig: DbConfig = config.get(configKey); - return new DataSource(createDataSourceOptions(dbConfig, SERVICE_NAME)); - }; - -export const healthCheckFactory: FactoryFunction = (container: DependencyContainer): HealthCheck => - createDataSourceHealthCheck( - container, - DB_CONFIGS.map(({ token }) => token) - ); diff --git a/packages/synchronizer/src/common/interfaces.ts b/packages/synchronizer/src/common/interfaces.ts index 9dd65fd..997d12d 100644 --- a/packages/synchronizer/src/common/interfaces.ts +++ b/packages/synchronizer/src/common/interfaces.ts @@ -7,6 +7,12 @@ export interface LayerEnums { layerName: string; enums: string[]; excludeProperties?: string[]; + sourceKey?: string; } -export type EnrichmentConfig = ReturnType['enrichment']; +export type NamespacesConfig = ReturnType['namespaces']; +export type NamespaceConfig = NamespacesConfig[number]; +export type LayerSourceConfig = NamespaceConfig['layerSource']; +export type SharedLuaSourceConfig = Extract; +export type PerLayerJsonSourceConfig = Extract; +export type EnrichmentConfig = SharedLuaSourceConfig['enrichment']; diff --git a/packages/synchronizer/src/common/s3/interfaces.ts b/packages/synchronizer/src/common/s3/interfaces.ts index c3e0e53..7f1bb0d 100644 --- a/packages/synchronizer/src/common/s3/interfaces.ts +++ b/packages/synchronizer/src/common/s3/interfaces.ts @@ -1,6 +1,6 @@ import type { S3ClientConfig } from '@aws-sdk/client-s3'; -import { type vectorVectorStandardSynchronizerV1Type } from '@map-colonies/schemas'; +import { type vectorVectorStandardSynchronizerV4Type } from '@map-colonies/schemas'; -export type BaseS3Config = vectorVectorStandardSynchronizerV1Type['dbs']['s3']; +export type BaseS3Config = vectorVectorStandardSynchronizerV4Type['dbs']['s3']; export type S3Config = BaseS3Config & S3ClientConfig; diff --git a/packages/synchronizer/src/common/s3/s3Repository.ts b/packages/synchronizer/src/common/s3/s3Repository.ts index e14a4bd..4b27df1 100644 --- a/packages/synchronizer/src/common/s3/s3Repository.ts +++ b/packages/synchronizer/src/common/s3/s3Repository.ts @@ -2,24 +2,21 @@ import path from 'node:path'; import { Readable } from 'node:stream'; import { buffer } from 'node:stream/consumers'; import type { Logger } from '@map-colonies/js-logger'; -import { inject, injectable, singleton } from 'tsyringe'; +import { inject, injectable } from 'tsyringe'; import type { S3Client } from '@aws-sdk/client-s3'; import { GetObjectCommand } from '@aws-sdk/client-s3'; import { context as contextAPI } from '@opentelemetry/api'; import { startActivePromisifiedSpan } from '@common/tracing/util'; import { S3Attributes, S3SpanName } from '@common/tracing/s3'; -import { SERVICES, s3ConfigPath } from '@common/constants'; -import type { ConfigType } from '../config'; -import { FsRepository } from '../fs/fsRepository'; +import { SERVICES } from '@common/constants'; +import type { FsRepository } from '../fs/fsRepository'; @injectable() -@singleton() export class S3Repository { public constructor( @inject(SERVICES.S3_CLIENT) private readonly s3Client: S3Client, - @inject(SERVICES.CONFIG) private readonly config: ConfigType, @inject(SERVICES.LOGGER) private readonly logger: Logger, - private readonly fsRepository: FsRepository + @inject(SERVICES.FS_REPOSITORY) private readonly fsRepository: FsRepository ) {} public async getObjectWrapper(bucket: string, key: string): Promise { @@ -42,20 +39,18 @@ export class S3Repository { ); } - public async downloadFile(): Promise { - const { bucket, fileName } = this.config.get(s3ConfigPath); - + public async downloadFile(bucket: string, fileName: string): Promise { return startActivePromisifiedSpan( S3SpanName.S3_DOWNLOAD_FILE, { [S3Attributes.BUCKET]: bucket, [S3Attributes.FILE_NAME]: fileName }, contextAPI.active(), async () => { try { - this.logger.info(`Downloading ${fileName} file from S3`); + this.logger.info(`Downloading ${fileName} file from S3 bucket ${bucket}`); const body = await this.getObjectWrapper(bucket, fileName); - const filePath = path.join(__dirname, 'downloads', fileName); + const filePath = path.join(__dirname, 'downloads', bucket, fileName); await this.fsRepository.mkdir(path.dirname(filePath)); await this.fsRepository.writeFile(filePath, await buffer(body)); diff --git a/packages/synchronizer/src/common/tracing/sync.ts b/packages/synchronizer/src/common/tracing/sync.ts index 0458768..9c76b50 100644 --- a/packages/synchronizer/src/common/tracing/sync.ts +++ b/packages/synchronizer/src/common/tracing/sync.ts @@ -2,6 +2,7 @@ export const SyncSpanName = { SYNC_TICK: 'sync.tick', SYNC_LAYER: 'sync.layer', + SYNC_FULL_LAYER: 'sync.layer.full', SYNC_PROPERTIES: 'sync.properties', SYNC_ENUM: 'sync.enum', LOAD_LUA_DATA: 'sync.lua.load', @@ -15,6 +16,7 @@ export type SyncSpanName = (typeof SyncSpanName)[keyof typeof SyncSpanName]; export const SyncAttributes = { LAYER_NAME: 'layer.name', LAYER_ID: 'layer.id', + NAMESPACE: 'namespace', LAYERS_COUNT: 'layers.count', LAYERS_CHANGED: 'layers.changed', PROPERTIES_AFFECTED: 'properties.affected', diff --git a/packages/synchronizer/src/containerConfig.ts b/packages/synchronizer/src/containerConfig.ts index 6fc99fb..91f6466 100644 --- a/packages/synchronizer/src/containerConfig.ts +++ b/packages/synchronizer/src/containerConfig.ts @@ -4,19 +4,42 @@ import { Registry } from 'prom-client'; import type { DependencyContainer } from 'tsyringe/dist/typings/types'; import type { Logger } from '@map-colonies/js-logger'; import { jsLogger } from '@map-colonies/js-logger'; -import { instancePerContainerCachingFactory } from 'tsyringe'; +import { Lifecycle, instancePerContainerCachingFactory } from 'tsyringe'; import { CleanupRegistry } from '@map-colonies/cleanup-registry'; import type { DataSource, Repository } from 'typeorm'; -import { DATA_SOURCE_PROVIDER as DESTINATION_DATA_SOURCE_PROVIDER } from '@db'; +import type { HealthCheck } from '@godaddy/terminus'; +import { + DATA_SOURCE_PROVIDER as DESTINATION_DATA_SOURCE_PROVIDER, + ENUMS_REPOSITORY_SYMBOL, + type EnumValue, + LAYER_REPOSITORY_SYMBOL, + type Layer, + PROPERTY_REPOSITORY_SYMBOL, + type Property, + createDataSource, + createDataSourceHealthCheck, +} from '@db'; import type { S3Client } from '@aws-sdk/client-s3'; import { ListBucketsCommand } from '@aws-sdk/client-s3'; import { type ConfigType, getConfig } from '@common/config'; import { type InjectionObject, registerDependencies } from '@common/dependencyRegistration'; -import { DB_CONFIGS, HEALTHCHECK, ON_SIGNAL, REPOSITORIES, SERVICES, SERVICE_NAME } from '@common/constants'; +import { + DESTINATION_DB_CONFIG_PATH, + HEALTHCHECK, + NAMESPACE_HANDLES, + ON_SIGNAL, + REPOSITORIES, + SERVICES, + SERVICE_NAME, + SOURCE_DATA_SOURCE_PROVIDER, +} from '@common/constants'; import { getTracing } from '@common/tracing'; -import { createDataSourceFactory, healthCheckFactory } from './common/db/connection'; import { CRON_MANAGER_SYMBOL, CronManager } from './sync/cron'; import { s3ClientFactory } from './common/s3'; +import { S3Repository } from './common/s3/s3Repository'; +import { FsRepository } from './common/fs/fsRepository'; +import { createNamespaceHandles } from './sync/namespaceHandle'; +import type { NamespaceHandle } from './sync/namespaceHandle/types'; export interface RegisterOptions { override?: InjectionObject[]; @@ -92,26 +115,39 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise }); }, }, - ...DB_CONFIGS.map(({ configKey, token }) => { - return { - token, - provider: { - useFactory: instancePerContainerCachingFactory(createDataSourceFactory(configKey)), - }, - postInjectionHook: async (deps: DependencyContainer): Promise => { - const dataSource = deps.resolve(token); + { + token: SERVICES.FS_REPOSITORY, + provider: { useClass: FsRepository }, + options: { lifecycle: Lifecycle.Singleton }, + }, + { + token: SERVICES.S3_REPOSITORY, + provider: { useClass: S3Repository }, + options: { lifecycle: Lifecycle.Singleton }, + }, + { + token: DESTINATION_DATA_SOURCE_PROVIDER, + provider: { + useFactory: instancePerContainerCachingFactory((container) => { + const config = container.resolve(SERVICES.CONFIG); + return createDataSource(config.get(DESTINATION_DB_CONFIG_PATH), SERVICE_NAME); + }), + }, + postInjectionHook: async (deps: DependencyContainer): Promise => { + const dataSource = deps.resolve(DESTINATION_DATA_SOURCE_PROVIDER); - if (!dataSource.isInitialized) { - await dataSource.initialize(); - } + if (!dataSource.isInitialized) { + await dataSource.initialize(); + } - cleanupRegistry.register({ - id: token, - func: dataSource.destroy.bind(dataSource), - }); - }, - }; - }), + deps.register(DESTINATION_DATA_SOURCE_PROVIDER, { useValue: dataSource }); + + cleanupRegistry.register({ + id: DESTINATION_DATA_SOURCE_PROVIDER, + func: dataSource.destroy.bind(dataSource), + }); + }, + }, ...REPOSITORIES.map(({ token, entity }) => ({ token, provider: { @@ -121,6 +157,34 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise }, }, })), + { + token: NAMESPACE_HANDLES, + provider: { + useFactory: instancePerContainerCachingFactory((container) => + createNamespaceHandles({ + config: container.resolve(SERVICES.CONFIG), + logger: container.resolve(SERVICES.LOGGER), + s3Repository: container.resolve(SERVICES.S3_REPOSITORY), + fsRepository: container.resolve(SERVICES.FS_REPOSITORY), + layerRepository: container.resolve>(LAYER_REPOSITORY_SYMBOL), + propertyRepository: container.resolve>(PROPERTY_REPOSITORY_SYMBOL), + enumsRepository: container.resolve>(ENUMS_REPOSITORY_SYMBOL), + }) + ), + }, + postInjectionHook: (deps: DependencyContainer): void => { + for (const { name, sourceDataSource } of deps.resolve(NAMESPACE_HANDLES)) { + cleanupRegistry.register({ + id: `${SOURCE_DATA_SOURCE_PROVIDER.toString()}:${name}`, + func: async () => { + if (sourceDataSource.isInitialized) { + await sourceDataSource.destroy(); + } + }, + }); + } + }, + }, { token: CRON_MANAGER_SYMBOL, provider: { useClass: CronManager }, @@ -136,7 +200,7 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: HEALTHCHECK, provider: { - useFactory: healthCheckFactory, + useFactory: (container: DependencyContainer): HealthCheck => createDataSourceHealthCheck(container, [DESTINATION_DATA_SOURCE_PROVIDER]), }, }, ]; diff --git a/packages/synchronizer/src/sync/SyncModel.ts b/packages/synchronizer/src/sync/SyncModel.ts index 7de50f4..79dcac8 100644 --- a/packages/synchronizer/src/sync/SyncModel.ts +++ b/packages/synchronizer/src/sync/SyncModel.ts @@ -1,16 +1,14 @@ -import { readFile } from 'node:fs/promises'; import { inject, injectable } from 'tsyringe'; import { DataSource, Repository } from 'typeorm'; -import { ENUMS_REPOSITORY_SYMBOL, EnumValue, Layer, LAYER_REPOSITORY_SYMBOL, Property, PROPERTY_REPOSITORY_SYMBOL } from '@db'; +import { ENUMS_REPOSITORY_SYMBOL, EnumValue, Layer, LAYER_REPOSITORY_SYMBOL, LayerSource, Property, PROPERTY_REPOSITORY_SYMBOL } from '@db'; import { Logger } from '@map-colonies/js-logger'; import { context as contextAPI } from '@opentelemetry/api'; import { startActivePromisifiedSpan } from '@common/tracing/util'; import { SyncAttributes, SyncSpanName } from '@common/tracing/sync'; -import { SERVICES, SOURCE_DATA_SOURCE_PROVIDER, s3ConfigPath } from '@common/constants'; +import { SERVICES, SOURCE_DATA_SOURCE_PROVIDER } from '@common/constants'; import { ConfigType } from '@common/config'; import { InsertPropertyDTO, LayerEnums } from '@src/common/interfaces'; -import { S3Repository } from '@src/common/s3/s3Repository'; import { CaseInsensitiveMap } from '@src/common/caseInsensitiveMap'; import { columnInfosToProperties, excludedPropertySet, schemaOf } from './helpers'; import { @@ -24,16 +22,21 @@ import { StalePropertiesDeletionError, TableColumnsQueryError, } from './errors'; -import { fetchPropertyAliases } from './aliasEnricher'; -import type { FileAliases } from './fileReader'; +import { resolveFileAliases, type FileAliases } from './aliasesFile'; import type { TypeMap } from './typeMap'; -import { LuaLayer, parseLuaLayers } from './luaParser'; +import type { LayerSourceRecord } from './layerSource/types'; export interface ColumnInfo { columnName: string; udtName: string; } +export interface SyncFullLayerContext { + typeMap: TypeMap; + fileAliases: FileAliases; + pruneStale: boolean; +} + @injectable() export class SyncModel { public constructor( @@ -42,25 +45,51 @@ export class SyncModel { @inject(SOURCE_DATA_SOURCE_PROVIDER) private readonly sourceDataSource: DataSource, @inject(LAYER_REPOSITORY_SYMBOL) private readonly layerRepository: Repository, @inject(PROPERTY_REPOSITORY_SYMBOL) private readonly propertyRepository: Repository, - @inject(ENUMS_REPOSITORY_SYMBOL) private readonly enumsRepository: Repository, - private readonly luaRepository: S3Repository + @inject(ENUMS_REPOSITORY_SYMBOL) private readonly enumsRepository: Repository ) {} - public async syncLayer(layerName: string, layerId: number, alias?: string): Promise { + public async syncFullLayer(namespace: string, layer: LayerEnums, record: LayerSourceRecord, context: SyncFullLayerContext): Promise { + return startActivePromisifiedSpan( + SyncSpanName.SYNC_FULL_LAYER, + { [SyncAttributes.LAYER_NAME]: layer.layerName, [SyncAttributes.NAMESPACE]: namespace }, + contextAPI.active(), + async () => { + const { typeMap, fileAliases, pruneStale } = context; + + await this.syncLayer(namespace, layer.layerName, record.layerId ?? null, record.source, record.alias); + + try { + const affected = await this.syncProperties(namespace, layer, typeMap, fileAliases, record.propertyAliases); + this.logger.info({ msg: `Synced properties for ${namespace}/${layer.layerName}`, affected }); + } catch (err) { + this.logger.warn({ msg: `Failed to sync properties for ${namespace}/${layer.layerName}, skipping to enum sync`, err }); + } + + const enumsAffected = await this.syncEnum(namespace, layer); + this.logger.info({ msg: `Synced enums for ${namespace}/${layer.layerName}`, enumsAffected }); + + if (pruneStale) { + await this.deleteStaleEnumValues(namespace, layer.layerName, layer.enums); + } + } + ); + } + + public async syncLayer(namespace: string, layerName: string, layerId: number | null, source: LayerSource, alias?: string): Promise { return startActivePromisifiedSpan( SyncSpanName.SYNC_LAYER, - { [SyncAttributes.LAYER_NAME]: layerName, [SyncAttributes.LAYER_ID]: layerId }, + { [SyncAttributes.LAYER_NAME]: layerName, [SyncAttributes.NAMESPACE]: namespace }, contextAPI.active(), async () => { try { await this.layerRepository .createQueryBuilder() .insert() - .values({ layerName, layerId, alias: alias ?? layerName }) - .orIgnore() + .values({ namespace, layerName, layerId, source, alias: alias ?? layerName }) + .orUpdate(['layer_id', 'source', 'alias'], ['namespace', 'layer_name']) .execute(); } catch (err) { - this.logger.error({ msg: `Failed to sync layer ${layerName} (${layerId})`, err }); + this.logger.error({ msg: `Failed to sync layer ${namespace}/${layerName}`, err }); throw new LayerSyncError(layerName, layerId, err); } } @@ -102,7 +131,6 @@ export class SyncModel { const schema = schemaOf(this.sourceDataSource); const qualifiedTable = `"${schema}"."${layerName}"`; - // Recursive CTE function const cteParts = enums.map( (col) => `cte_${col} AS ( SELECT MIN("${col}") AS value FROM ${qualifiedTable} @@ -136,7 +164,7 @@ export class SyncModel { public async upsertProperties(properties: InsertPropertyDTO[]): Promise { return startActivePromisifiedSpan(SyncSpanName.UPSERT_PROPERTIES, {}, contextAPI.active(), async (span) => { try { - const result = await this.propertyRepository.upsert(properties, ['layerName', 'property']); + const result = await this.propertyRepository.upsert(properties, ['namespace', 'layerName', 'property']); span.setAttribute(SyncAttributes.PROPERTIES_AFFECTED, result.identifiers.length); return result.identifiers.length; } catch (err) { @@ -146,37 +174,30 @@ export class SyncModel { }); } - public async syncProperties(layer: LayerEnums, layerId: number, typeMap: TypeMap, fileAliases: FileAliases): Promise { + public async syncProperties( + namespace: string, + layer: LayerEnums, + typeMap: TypeMap, + fileAliases: FileAliases, + sourceAliases: Map + ): Promise { return startActivePromisifiedSpan( SyncSpanName.SYNC_PROPERTIES, - { [SyncAttributes.LAYER_NAME]: layer.layerName, [SyncAttributes.LAYER_ID]: layerId }, + { [SyncAttributes.LAYER_NAME]: layer.layerName, [SyncAttributes.NAMESPACE]: namespace }, contextAPI.active(), async (span) => { const excluded = excludedPropertySet(layer.excludeProperties); const columns = (await this.getTableColumns(layer.layerName)).filter(({ columnName }) => !excluded.has(columnName.toLowerCase())); if (excluded.size > 0) { - this.logger.debug({ layerName: layer.layerName, excludeProperties: layer.excludeProperties }, 'Excluding properties from sync'); + this.logger.debug({ msg: `Excluding properties from sync for ${layer.layerName}`, excludeProperties: layer.excludeProperties }); } - const properties = columnInfosToProperties(columns, layer.layerName, typeMap, (columnName, udtName) => { - this.logger.warn({ layerName: layer.layerName, columnName, udtName }, 'Unknown column type, skipping property'); + const properties = columnInfosToProperties(columns, namespace, layer.layerName, typeMap, (columnName, udtName) => { + this.logger.warn({ msg: `Unknown column type ${udtName} for ${layer.layerName}.${columnName}, skipping property` }); }); - const enrichment = this.config.get('enrichment'); - let apiAliases = new Map(); - if (enrichment.enabled) { - try { - apiAliases = await fetchPropertyAliases(layer.layerName, layerId, enrichment); - } catch (err) { - this.logger.warn( - { layerName: layer.layerName, layerId, err }, - 'Failed to fetch property aliases from enrichment API, continuing without them' - ); - } - } - const globalAliases = fileAliases.get('*') ?? new Map(); - const layerAliases = fileAliases.get(layer.layerName) ?? new Map(); - const aliases = new CaseInsensitiveMap([...apiAliases, ...globalAliases, ...layerAliases]); + const fileLayerAliases = resolveFileAliases(fileAliases, namespace, layer.layerName); + const aliases = new CaseInsensitiveMap([...sourceAliases, ...fileLayerAliases]); for (const property of properties) { const alias = aliases.get(property.property); @@ -192,6 +213,7 @@ export class SyncModel { affected += await this.upsertProperties(withoutAlias); } await this.deleteStaleProperties( + namespace, layer.layerName, properties.map((p) => p.property) ); @@ -201,28 +223,42 @@ export class SyncModel { ); } - public async deleteStaleProperties(layerName: string, currentPropertyNames: string[]): Promise { - const qb = this.propertyRepository.createQueryBuilder().delete().where('"layer_name" = :layerName', { layerName }); + public async deleteStaleProperties(namespace: string, layerName: string, currentPropertyNames: string[]): Promise { + const qb = this.propertyRepository + .createQueryBuilder() + .delete() + .where('"namespace" = :namespace', { namespace }) + .andWhere('"layer_name" = :layerName', { layerName }); if (currentPropertyNames.length > 0) { qb.andWhere('"property" NOT IN (:...properties)', { properties: currentPropertyNames }); } try { await qb.execute(); } catch (err) { - this.logger.error({ msg: `Failed to delete stale properties for ${layerName}`, err }); + this.logger.error({ msg: `Failed to delete stale properties for ${namespace}/${layerName}`, err }); throw new StalePropertiesDeletionError(layerName, err); } } - public async deleteLayersNotIn(layerNames: string[]): Promise { + public async deleteLayersNotIn(namespace: string, layerNames: string[]): Promise { if (layerNames.length === 0) { return; } try { - await this.propertyRepository.createQueryBuilder().delete().where('"layer_name" NOT IN (:...layerNames)', { layerNames }).execute(); - await this.layerRepository.createQueryBuilder().delete().where('"layer_name" NOT IN (:...layerNames)', { layerNames }).execute(); + await this.propertyRepository + .createQueryBuilder() + .delete() + .where('"namespace" = :namespace', { namespace }) + .andWhere('"layer_name" NOT IN (:...layerNames)', { layerNames }) + .execute(); + await this.layerRepository + .createQueryBuilder() + .delete() + .where('"namespace" = :namespace', { namespace }) + .andWhere('"layer_name" NOT IN (:...layerNames)', { layerNames }) + .execute(); } catch (err) { - this.logger.error({ msg: `Failed to delete layers not in [${layerNames.join(', ')}]`, err }); + this.logger.error({ msg: `Failed to delete layers not in [${layerNames.join(', ')}] for namespace ${namespace}`, err }); throw new LayersDeletionError(layerNames, err); } } @@ -247,25 +283,19 @@ export class SyncModel { } } - public async luaFileData(): Promise> { - return startActivePromisifiedSpan(SyncSpanName.LOAD_LUA_DATA, {}, contextAPI.active(), async () => { - this.logger.debug('Loading lua query data'); - const filePath = await this.luaRepository.downloadFile(); - const content = await readFile(filePath, 'utf-8'); - const { layersVariable, idFieldName, nameFieldName, aliasFieldName } = this.config.get(s3ConfigPath); - return parseLuaLayers(content, layersVariable, idFieldName, nameFieldName, aliasFieldName); - }); - } - - public async deleteStaleEnumValues(layerName: string, currentEnums: string[]): Promise { - const qb = this.enumsRepository.createQueryBuilder().delete().where('"layer_name" = :layerName', { layerName }); + public async deleteStaleEnumValues(namespace: string, layerName: string, currentEnums: string[]): Promise { + const qb = this.enumsRepository + .createQueryBuilder() + .delete() + .where('"namespace" = :namespace', { namespace }) + .andWhere('"layer_name" = :layerName', { layerName }); if (currentEnums.length > 0) { qb.andWhere('"property" NOT IN (:...properties)', { properties: currentEnums }); } try { await qb.execute(); } catch (err) { - this.logger.error({ msg: `Failed to delete stale enum values for ${layerName}`, err }); + this.logger.error({ msg: `Failed to delete stale enum values for ${namespace}/${layerName}`, err }); throw new StaleEnumValuesDeletionError(layerName, err); } } @@ -280,13 +310,13 @@ export class SyncModel { if (existing.length !== layer.enums.length) { const missing = layer.enums.filter((col) => !tableColumns.has(col)); - this.logger.warn({ layerName: layer.layerName, columns: missing }, 'Enum columns do not exist in table, skipping'); + this.logger.warn({ msg: `Enum columns do not exist in table ${layer.layerName}, skipping`, columns: missing }); } return existing; } - public async syncEnum(layer: LayerEnums): Promise { + public async syncEnum(namespace: string, layer: LayerEnums): Promise { return startActivePromisifiedSpan(SyncSpanName.SYNC_ENUM, { [SyncAttributes.LAYER_NAME]: layer.layerName }, contextAPI.active(), async (span) => { const existingLayer: LayerEnums = { layerName: layer.layerName, enums: await this.existingEnumColumns(layer) }; @@ -294,14 +324,14 @@ export class SyncModel { const enumValuesByColumn = await this.getEnumDistinctValues(existingLayer); const entities = [...enumValuesByColumn.entries()].flatMap(([col, values]) => - values.map((value) => this.enumsRepository.create({ layerName: layer.layerName, property: col, value })) + values.map((value) => this.enumsRepository.create({ namespace, layerName: layer.layerName, property: col, value })) ); let affected: number; try { affected = (await this.enumsRepository.save(entities)).length; } catch (err) { - this.logger.error({ msg: `Failed to save enum values for ${layer.layerName}`, err }); + this.logger.error({ msg: `Failed to save enum values for ${namespace}/${layer.layerName}`, err }); throw new EnumSaveError(layer.layerName, err); } span.setAttribute(SyncAttributes.ENUMS_AFFECTED, affected); diff --git a/packages/synchronizer/src/sync/aliasesFile.ts b/packages/synchronizer/src/sync/aliasesFile.ts new file mode 100644 index 0000000..d3be78d --- /dev/null +++ b/packages/synchronizer/src/sync/aliasesFile.ts @@ -0,0 +1,40 @@ +import ajvCtor from 'ajv'; +import { ALL_KEYS_SELECTOR } from '@common/constants'; +import { AliasesFileError } from './errors'; + +type AliasesFile = Record>>; + +const ajv = new ajvCtor(); + +const isAliasesFile = ajv.compile({ + type: 'object', + additionalProperties: { + type: 'object', + additionalProperties: { type: 'object', additionalProperties: { type: 'string' } }, + }, +}); + +export type FileAliases = Map>>; + +export const parseAliases = (raw: unknown, source: string): FileAliases => { + if (!isAliasesFile(raw)) { + throw new AliasesFileError(source, ajv.errorsText(isAliasesFile.errors)); + } + + return new Map( + Object.entries(raw).map(([namespace, layers]) => [ + namespace, + new Map(Object.entries(layers).map(([layer, props]) => [layer, new Map(Object.entries(props))])), + ]) + ); +}; + +export const resolveFileAliases = (fileAliases: FileAliases, namespace: string, layerName: string): Map => { + const tiers = [ + fileAliases.get(ALL_KEYS_SELECTOR)?.get(ALL_KEYS_SELECTOR), + fileAliases.get(ALL_KEYS_SELECTOR)?.get(layerName), + fileAliases.get(namespace)?.get(ALL_KEYS_SELECTOR), + fileAliases.get(namespace)?.get(layerName), + ]; + return new Map(tiers.filter((tier): tier is Map => tier !== undefined).flatMap((tier) => [...tier])); +}; diff --git a/packages/synchronizer/src/sync/cron.ts b/packages/synchronizer/src/sync/cron.ts index a601e97..cd0fd63 100644 --- a/packages/synchronizer/src/sync/cron.ts +++ b/packages/synchronizer/src/sync/cron.ts @@ -2,83 +2,95 @@ import { inject, injectable } from 'tsyringe'; import type { Logger } from '@map-colonies/js-logger'; import { schedule, type ScheduledTask } from 'node-cron'; import { context as contextAPI } from '@opentelemetry/api'; -import { SERVICES } from '@common/constants'; +import { SERVICES, NAMESPACE_HANDLES } from '@common/constants'; import { ConfigType } from '@common/config'; import { startActivePromisifiedSpan } from '@common/tracing/util'; -import { SyncAttributes, SyncSpanName } from '@common/tracing/sync'; -import { SyncModel } from './SyncModel'; +import { SyncSpanName } from '@common/tracing/sync'; import { FileReader } from './fileReader'; +import type { FileAliases } from './aliasesFile'; +import type { TypeMap } from './typeMap'; +import type { NamespaceHandle } from './namespaceHandle/types'; export const CRON_MANAGER_SYMBOL = Symbol('cronManagerSymbol'); @injectable() export class CronManager { private task: ScheduledTask | undefined; - private isRunning = false; - private lastLayersChecksum: string | undefined; + private readonly lastLayersChecksum = new Map(); public constructor( @inject(SERVICES.LOGGER) private readonly logger: Logger, - @inject(SyncModel) private readonly dal: SyncModel, @inject(SERVICES.CONFIG) private readonly config: ConfigType, + @inject(NAMESPACE_HANDLES) private readonly namespaces: NamespaceHandle[], private readonly fileReader: FileReader ) {} public async tick(): Promise { - if (this.isRunning) { - this.logger.warn('previous sync tick still running, skipping'); - return; - } - this.isRunning = true; - try { - await startActivePromisifiedSpan(SyncSpanName.SYNC_TICK, {}, contextAPI.active(), async (span) => { - const { checksum, layers } = await this.fileReader.readLayersWithChecksum(this.config.get('layersFile')); - const layersChanged = checksum !== this.lastLayersChecksum; - span.setAttribute(SyncAttributes.LAYERS_COUNT, layers.length); - span.setAttribute(SyncAttributes.LAYERS_CHANGED, layersChanged); - - const typeMap = await this.fileReader.readTypeMap(this.config.get('typeMapFile')); - const fileAliases = await this.fileReader.readAliases(this.config.get('aliasesFile')); + await startActivePromisifiedSpan(SyncSpanName.SYNC_TICK, {}, contextAPI.active(), async () => { + const typeMap = await this.fileReader.readTypeMap(this.config.get('typeMapFile')); + const fileAliases = await this.fileReader.readAliases(this.config.get('aliasesFile')); - const luaLayersMap = await this.dal.luaFileData(); - for (const layer of layers) { - const luaLayer = luaLayersMap.get(layer.layerName); - if (luaLayer === undefined) { - this.logger.warn({ layerName: layer.layerName }, 'Layer not found in lua file, skipping'); - continue; - } - await this.dal.syncLayer(layer.layerName, luaLayer.layerId, luaLayer.alias); - try { - const affected = await this.dal.syncProperties(layer, luaLayer.layerId, typeMap, fileAliases); - this.logger.info({ layer, affected }, 'synced properties'); - } catch (err) { - this.logger.warn({ layerName: layer.layerName, err }, 'failed to sync properties, skipping to enum sync'); - } - const enumsAffected = await this.dal.syncEnum(layer); - this.logger.info({ layer, enumsAffected }, 'synced enums'); - if (layersChanged) { - await this.dal.deleteStaleEnumValues(layer.layerName, layer.enums); - } + for (const namespace of this.namespaces) { + try { + await this.tickNamespace(namespace, typeMap, fileAliases); + } catch (err) { + this.logger.error({ msg: `Failed to sync namespace ${namespace.name}, skipping`, err }); } - if (layersChanged) { - await this.dal.deleteLayersNotIn(layers.map((l) => l.layerName)); - this.lastLayersChecksum = checksum; - } - }); - } catch (err) { - this.logger.error({ err }, 'sync tick failed, waiting for next tick'); - } finally { - this.isRunning = false; - } + } + }); } public start(): void { - this.task = schedule(this.config.get('schedule'), async () => { - await this.tick(); + this.task = schedule( + this.config.get('schedule'), + async () => { + await this.tick(); + }, + { noOverlap: true } + ); + this.task.on('execution:overlap', () => { + this.logger.warn('previous sync tick still running, skipping this execution'); }); } public async stop(): Promise { await this.task?.stop(); } + + private async tickNamespace(namespace: NamespaceHandle, typeMap: TypeMap, fileAliases: FileAliases): Promise { + if (!namespace.sourceDataSource.isInitialized) { + try { + await namespace.sourceDataSource.initialize(); + } catch (err) { + this.logger.warn({ msg: `Source database unreachable for namespace ${namespace.name}, skipping`, err }); + throw err; + } + } + + const { checksum, layers } = await this.fileReader.readLayersWithChecksum(namespace.layersFile); + const layersChanged = checksum !== this.lastLayersChecksum.get(namespace.name); + + const records = await namespace.layerSource.tick(layers); + + for (const layer of layers) { + const record = records.get(layer.layerName); + if (record === undefined) { + continue; + } + + try { + await namespace.dal.syncFullLayer(namespace.name, layer, record, { typeMap, fileAliases, pruneStale: layersChanged }); + } catch (err) { + this.logger.warn({ msg: `Failed to sync ${namespace.name}/${layer.layerName}, continuing with next layer`, err }); + } + } + + if (layersChanged) { + await namespace.dal.deleteLayersNotIn( + namespace.name, + layers.map((l) => l.layerName) + ); + this.lastLayersChecksum.set(namespace.name, checksum); + } + } } diff --git a/packages/synchronizer/src/sync/errors.ts b/packages/synchronizer/src/sync/errors.ts index 3d76d0b..dd20ebc 100644 --- a/packages/synchronizer/src/sync/errors.ts +++ b/packages/synchronizer/src/sync/errors.ts @@ -7,7 +7,7 @@ abstract class SyncError extends Error { } export class LayerSyncError extends SyncError { - public constructor(layerName: string, layerId: number, cause?: unknown) { + public constructor(layerName: string, layerId: number | null, cause?: unknown) { super(`Failed to sync layer ${layerName} (${layerId})`, { cause }); } } @@ -60,8 +60,32 @@ export class TypeMapError extends SyncError { } } +export class LayersFileError extends SyncError { + public constructor(filePath: string, reason: string, cause?: unknown) { + super(`Invalid layers file ${filePath}: ${reason}`, { cause }); + } +} + +export class AliasesFileError extends SyncError { + public constructor(filePath: string, reason: string, cause?: unknown) { + super(`Invalid aliases file ${filePath}: ${reason}`, { cause }); + } +} + export class EnumSaveError extends SyncError { public constructor(layerName: string, cause?: unknown) { super(`Failed to save enum values for ${layerName}`, { cause }); } } + +export class LayerJsonFetchError extends SyncError { + public constructor(layerName: string, key: string, cause?: unknown) { + super(`Failed to fetch layer JSON for ${layerName} at key ${key}`, { cause }); + } +} + +export class LayerJsonParseError extends SyncError { + public constructor(layerName: string, key: string, cause?: unknown) { + super(`Failed to parse layer JSON for ${layerName} at key ${key}`, { cause }); + } +} diff --git a/packages/synchronizer/src/sync/fileReader.ts b/packages/synchronizer/src/sync/fileReader.ts index cc1d368..2d615c4 100644 --- a/packages/synchronizer/src/sync/fileReader.ts +++ b/packages/synchronizer/src/sync/fileReader.ts @@ -1,27 +1,28 @@ import { createHash } from 'node:crypto'; import type { JsonValue } from 'type-fest'; -import { injectable } from 'tsyringe'; -import { FsRepository } from '@common/fs/fsRepository'; +import { inject, injectable } from 'tsyringe'; +import type { FsRepository } from '@common/fs/fsRepository'; import type { LayerEnums } from '@common/interfaces'; +import { SERVICES } from '@common/constants'; import { parseTypeMap, type TypeMap } from './typeMap'; -import { TypeMapError } from './errors'; - -type AliasesFile = Record>; - -export type FileAliases = Map>; +import { parseLayers } from './layersFile'; +import { parseAliases, type FileAliases } from './aliasesFile'; +import { AliasesFileError, LayersFileError, TypeMapError } from './errors'; @injectable() export class FileReader { - public constructor(private readonly fsRepository: FsRepository) {} + public constructor(@inject(SERVICES.FS_REPOSITORY) private readonly fsRepository: FsRepository) {} public async readLayersWithChecksum(filePath: string): Promise<{ checksum: string; layers: LayerEnums[] }> { try { const buffer = await this.fsRepository.readFile(filePath); const checksum = createHash('sha256').update(buffer).digest('hex'); - const layers = JSON.parse(buffer.toString()) as LayerEnums[]; - return { checksum, layers }; + return { checksum, layers: parseLayers(JSON.parse(buffer.toString()), filePath) }; } catch (err) { - throw new Error(`Failed to read layers from ${filePath}`, { cause: err }); + if (err instanceof LayersFileError) { + throw err; + } + throw new LayersFileError(filePath, 'failed to read or parse the file', err); } } @@ -40,10 +41,12 @@ export class FileReader { public async readAliases(filePath: string): Promise { try { const content = await this.fsRepository.readFile(filePath, 'utf-8'); - const parsed = JSON.parse(content.toString()) as AliasesFile; - return new Map(Object.entries(parsed).map(([layer, props]) => [layer, new Map(Object.entries(props))])); + return parseAliases(JSON.parse(content.toString()), filePath); } catch (err) { - throw new Error(`Failed to read aliases from ${filePath}`, { cause: err }); + if (err instanceof AliasesFileError) { + throw err; + } + throw new AliasesFileError(filePath, 'failed to read or parse the file', err); } } } diff --git a/packages/synchronizer/src/sync/helpers.ts b/packages/synchronizer/src/sync/helpers.ts index fad307c..516be7e 100644 --- a/packages/synchronizer/src/sync/helpers.ts +++ b/packages/synchronizer/src/sync/helpers.ts @@ -16,7 +16,6 @@ const normalizeUdtName = (rawUdtName: string, typeMap: TypeMap): ColumnType | un return plain; } - // Strip length/precision modifier e.g. "character varying(255)" → "character varying" const withoutModifier = udtName.replace(/\s*\(\d+(?:,\s*\d+)?\)$/, ''); if (withoutModifier !== udtName) { const stripped = typeMap.types.get(withoutModifier); @@ -52,6 +51,7 @@ export const schemaOf = (dataSource: DataSource): string => { export const columnInfosToProperties = ( columnInfos: ColumnInfo[], + namespace: string, layerName: string, typeMap: TypeMap, onUnknown?: (columnName: string, udtName: string) => void @@ -62,7 +62,7 @@ export const columnInfosToProperties = ( if (type === undefined) { onUnknown?.(columnName, udtName); } else { - result.push({ layerName, property: columnName, type }); + result.push({ namespace, layerName, property: columnName, type }); } } return result; diff --git a/packages/synchronizer/src/sync/layerSource/aliasJsonParser.ts b/packages/synchronizer/src/sync/layerSource/aliasJsonParser.ts new file mode 100644 index 0000000..71ac92f --- /dev/null +++ b/packages/synchronizer/src/sync/layerSource/aliasJsonParser.ts @@ -0,0 +1,39 @@ +import ajvCtor from 'ajv'; +import type { PerLayerJsonSourceConfig } from '@common/interfaces'; +import { CaseInsensitiveMap } from '@src/common/caseInsensitiveMap'; + +const ajv = new ajvCtor(); +const isRecord = ajv.compile>({ type: 'object' }); +const isString = ajv.compile({ type: 'string' }); +const isArray = ajv.compile({ type: 'array' }); + +const isRecordEntry = (value: unknown): value is Record => isRecord(value); + +const buildPropertyAliases = (rawFields: unknown[], config: PerLayerJsonSourceConfig): Map => + new CaseInsensitiveMap( + rawFields + .filter(isRecordEntry) + .map((field) => ({ fieldName: field[config.fieldNameField], aliasFieldName: field[config.aliasFieldNameField] })) + .filter((field): field is { fieldName: string; aliasFieldName: string } => isString(field.fieldName) && isString(field.aliasFieldName)) + .map(({ fieldName, aliasFieldName }): [string, string] => [fieldName, aliasFieldName]) + ); + +export interface ParsedDocument { + alias: string; + propertyAliases: Map; +} + +export const parseDocument = (content: string, config: PerLayerJsonSourceConfig): ParsedDocument => { + const raw: unknown = JSON.parse(content); + if (!isRecord(raw)) { + throw new Error('layer JSON must be an object'); + } + + const alias = raw[config.aliasLayerNameField]; + const fields = raw[config.fieldsField]; + if (!isString(alias) || !isArray(fields)) { + throw new Error(`layer JSON must have a string "${config.aliasLayerNameField}" and an array "${config.fieldsField}"`); + } + + return { alias, propertyAliases: buildPropertyAliases(fields, config) }; +}; diff --git a/packages/synchronizer/src/sync/layerSource/index.ts b/packages/synchronizer/src/sync/layerSource/index.ts new file mode 100644 index 0000000..3e11a8d --- /dev/null +++ b/packages/synchronizer/src/sync/layerSource/index.ts @@ -0,0 +1,20 @@ +import type { Logger } from '@map-colonies/js-logger'; +import type { NamespaceConfig } from '@common/interfaces'; +import type { S3Repository } from '@common/s3/s3Repository'; +import type { FsRepository } from '@common/fs/fsRepository'; +import { SharedLuaSource } from './sharedLuaSource'; +import { PerLayerJsonSource } from './perLayerJsonSource'; +import type { LayerSourceStrategy } from './types'; + +export const createLayerSourceStrategy = ( + namespaceConfig: NamespaceConfig, + s3Repository: S3Repository, + fsRepository: FsRepository, + logger: Logger +): LayerSourceStrategy => { + const { layerSource, bucket } = namespaceConfig; + if (layerSource.type === 'sharedLua') { + return new SharedLuaSource(s3Repository, fsRepository, logger, bucket, layerSource); + } + return new PerLayerJsonSource(s3Repository, fsRepository, logger, bucket, layerSource); +}; diff --git a/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts b/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts new file mode 100644 index 0000000..9650fd9 --- /dev/null +++ b/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts @@ -0,0 +1,66 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { layerSource } from '@db'; +import type { LayerEnums, PerLayerJsonSourceConfig } from '@common/interfaces'; +import type { FsRepository } from '@common/fs/fsRepository'; +import { LayerJsonFetchError, LayerJsonParseError } from '../errors'; +import type { FileDownloader, LayerSourceRecord, LayerSourceStrategy } from './types'; +import { parseDocument } from './aliasJsonParser'; + +const resolveOne = async ( + layer: LayerEnums, + sourceKey: string, + s3Repository: FileDownloader, + fsRepository: FsRepository, + bucket: string, + config: PerLayerJsonSourceConfig, + logger: Logger +): Promise => { + const key = `${config.prefix}${sourceKey}`; + + let filePath: string; + try { + filePath = await s3Repository.downloadFile(bucket, key); + } catch (err) { + logger.warn({ msg: `Failed to fetch layer JSON ${key}, skipping layer`, err: new LayerJsonFetchError(layer.layerName, key, err) }); + return undefined; + } + + try { + const content = (await fsRepository.readFile(filePath, 'utf-8')).toString(); + const document = parseDocument(content, config); + return { + alias: document.alias, + propertyAliases: document.propertyAliases, + source: layerSource.perLayerJson, + }; + } catch (err) { + logger.warn({ msg: `Failed to read or parse layer JSON ${key}, skipping layer`, err: new LayerJsonParseError(layer.layerName, key, err) }); + return undefined; + } +}; + +export class PerLayerJsonSource implements LayerSourceStrategy { + public constructor( + private readonly s3Repository: FileDownloader, + private readonly fsRepository: FsRepository, + private readonly logger: Logger, + private readonly bucket: string, + private readonly config: PerLayerJsonSourceConfig + ) {} + + public async tick(layers: LayerEnums[]): Promise> { + const records = new Map(); + for (const layer of layers) { + if (layer.sourceKey === undefined) { + this.logger.warn({ msg: `No sourceKey configured for perLayerJson layer ${layer.layerName}, skipping` }); + continue; + } + + const record = await resolveOne(layer, layer.sourceKey, this.s3Repository, this.fsRepository, this.bucket, this.config, this.logger); + if (record !== undefined) { + records.set(layer.layerName, record); + } + } + return records; + } +} diff --git a/packages/synchronizer/src/sync/layerSource/sharedLuaSource.ts b/packages/synchronizer/src/sync/layerSource/sharedLuaSource.ts new file mode 100644 index 0000000..97bb5db --- /dev/null +++ b/packages/synchronizer/src/sync/layerSource/sharedLuaSource.ts @@ -0,0 +1,59 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { layerSource } from '@db'; +import type { LayerEnums, SharedLuaSourceConfig } from '@common/interfaces'; +import type { FsRepository } from '@common/fs/fsRepository'; +import { fetchPropertyAliases } from '../aliasEnricher'; +import { parseLuaLayers } from '../luaParser'; +import type { LayerSourceRecord, FileDownloader, LayerSourceStrategy, LuaLayer } from './types'; + +const resolveOne = async (layer: LayerEnums, luaLayer: LuaLayer, config: SharedLuaSourceConfig, logger: Logger): Promise => { + let propertyAliases = new Map(); + if (config.enrichment.enabled) { + try { + propertyAliases = await fetchPropertyAliases(layer.layerName, luaLayer.layerId, config.enrichment); + } catch (err) { + logger.warn({ msg: `Failed to fetch property aliases for ${layer.layerName} (${luaLayer.layerId}), continuing without them`, err }); + } + } + + return { + layerId: luaLayer.layerId, + alias: luaLayer.alias, + propertyAliases, + source: layerSource.sharedLua, + }; +}; + +export class SharedLuaSource implements LayerSourceStrategy { + public constructor( + private readonly s3Repository: FileDownloader, + private readonly fsRepository: FsRepository, + private readonly logger: Logger, + private readonly bucket: string, + private readonly config: SharedLuaSourceConfig + ) {} + + public async tick(layers: LayerEnums[]): Promise> { + this.logger.debug('Loading lua query data'); + const filePath = await this.s3Repository.downloadFile(this.bucket, this.config.fileName); + const content = (await this.fsRepository.readFile(filePath, 'utf-8')).toString(); + const luaLayers = parseLuaLayers( + content, + this.config.layersVariable, + this.config.idFieldName, + this.config.nameFieldName, + this.config.aliasFieldName + ); + + const records = new Map(); + for (const layer of layers) { + const luaLayer = luaLayers.get(layer.layerName); + if (luaLayer === undefined) { + this.logger.warn({ msg: `Layer ${layer.layerName} not found in lua file, skipping` }); + continue; + } + records.set(layer.layerName, await resolveOne(layer, luaLayer, this.config, this.logger)); + } + return records; + } +} diff --git a/packages/synchronizer/src/sync/layerSource/types.ts b/packages/synchronizer/src/sync/layerSource/types.ts new file mode 100644 index 0000000..fdbdc1a --- /dev/null +++ b/packages/synchronizer/src/sync/layerSource/types.ts @@ -0,0 +1,23 @@ +import type { LayerSource } from '@db'; +import type { LayerEnums } from '@common/interfaces'; + +export interface LayerSourceRecord { + layerId?: number; + alias: string; + propertyAliases: Map; + source: LayerSource; +} + +export interface LayerSourceStrategy { + tick: (layers: LayerEnums[]) => Promise>; +} + +export interface FileDownloader { + downloadFile: (bucket: string, key: string) => Promise; +} + +export interface LuaLayer { + layerId: number; + layerName: string; + alias: string; +} diff --git a/packages/synchronizer/src/sync/layersFile.ts b/packages/synchronizer/src/sync/layersFile.ts new file mode 100644 index 0000000..764a405 --- /dev/null +++ b/packages/synchronizer/src/sync/layersFile.ts @@ -0,0 +1,27 @@ +import ajvCtor from 'ajv'; +import type { LayerEnums } from '@common/interfaces'; +import { LayersFileError } from './errors'; + +const ajv = new ajvCtor(); + +const isLayersFile = ajv.compile({ + type: 'array', + items: { + type: 'object', + properties: { + layerName: { type: 'string' }, + enums: { type: 'array', items: { type: 'string' } }, + excludeProperties: { type: 'array', items: { type: 'string' } }, + sourceKey: { type: 'string' }, + }, + required: ['layerName', 'enums'], + }, +}); + +export const parseLayers = (raw: unknown, source: string): LayerEnums[] => { + if (!isLayersFile(raw)) { + throw new LayersFileError(source, ajv.errorsText(isLayersFile.errors)); + } + + return raw; +}; diff --git a/packages/synchronizer/src/sync/luaParser.ts b/packages/synchronizer/src/sync/luaParser.ts index abe7427..f0d8f75 100644 --- a/packages/synchronizer/src/sync/luaParser.ts +++ b/packages/synchronizer/src/sync/luaParser.ts @@ -1,3 +1,5 @@ +import type { LuaLayer } from './layerSource/types'; + const NOT_FOUND = -1; export const extractBlock = (content: string, variableName: string): string | null => { @@ -26,12 +28,6 @@ export const extractBlock = (content: string, variableName: string): string | nu return null; }; -export interface LuaLayer { - layerId: number; - layerName: string; - alias: string; -} - export const parseLuaLayers = ( content: string, variableName: string, diff --git a/packages/synchronizer/src/sync/namespaceHandle/index.ts b/packages/synchronizer/src/sync/namespaceHandle/index.ts new file mode 100644 index 0000000..733d928 --- /dev/null +++ b/packages/synchronizer/src/sync/namespaceHandle/index.ts @@ -0,0 +1,17 @@ +import { createDataSource } from '@db'; +import { SERVICE_NAME } from '@common/constants'; +import { SyncModel } from '../SyncModel'; +import { createLayerSourceStrategy } from '../layerSource'; +import type { NamespaceHandle, NamespaceHandleDependencies } from './types'; + +export const createNamespaceHandles = (dependencies: NamespaceHandleDependencies): NamespaceHandle[] => { + const { config, logger, s3Repository, fsRepository, layerRepository, propertyRepository, enumsRepository } = dependencies; + + return config.get('namespaces').map((namespaceConfig) => { + const sourceDataSource = createDataSource(namespaceConfig.db, `${SERVICE_NAME}-${namespaceConfig.name}`); + const layerSource = createLayerSourceStrategy(namespaceConfig, s3Repository, fsRepository, logger); + const dal = new SyncModel(config, logger, sourceDataSource, layerRepository, propertyRepository, enumsRepository); + + return { name: namespaceConfig.name, sourceDataSource, layerSource, layersFile: namespaceConfig.layersFile, dal }; + }); +}; diff --git a/packages/synchronizer/src/sync/namespaceHandle/types.ts b/packages/synchronizer/src/sync/namespaceHandle/types.ts new file mode 100644 index 0000000..e1dc2f5 --- /dev/null +++ b/packages/synchronizer/src/sync/namespaceHandle/types.ts @@ -0,0 +1,26 @@ +import type { Logger } from '@map-colonies/js-logger'; +import type { DataSource, Repository } from 'typeorm'; +import type { EnumValue, Layer, Property } from '@db'; +import type { ConfigType } from '@common/config'; +import type { S3Repository } from '@common/s3/s3Repository'; +import type { FsRepository } from '@common/fs/fsRepository'; +import type { SyncModel } from '../SyncModel'; +import type { LayerSourceStrategy } from '../layerSource/types'; + +export interface NamespaceHandle { + name: string; + sourceDataSource: DataSource; + layerSource: LayerSourceStrategy; + layersFile: string; + dal: SyncModel; +} + +export interface NamespaceHandleDependencies { + config: ConfigType; + logger: Logger; + s3Repository: S3Repository; + fsRepository: FsRepository; + layerRepository: Repository; + propertyRepository: Repository; + enumsRepository: Repository; +} diff --git a/packages/synchronizer/tests/configurations/globalSetup.ts b/packages/synchronizer/tests/configurations/globalSetup.ts index dbd6b15..ac4afb3 100644 --- a/packages/synchronizer/tests/configurations/globalSetup.ts +++ b/packages/synchronizer/tests/configurations/globalSetup.ts @@ -4,11 +4,15 @@ import { createSslOptions } from '../../../db/dist/db/connection'; export default async function setup(): Promise { await initConfig(true); - const { source, destination } = getConfig().get('dbs'); + const config = getConfig(); + const destination = config.get('dbs.destination'); + const namespaces = config.get('namespaces'); const pgClient = new Client({ ...destination, user: destination.username, ssl: createSslOptions(destination.ssl) }); await pgClient.connect(); await pgClient.query(`DROP SCHEMA IF EXISTS "${destination.schema}" CASCADE`); - await pgClient.query(`DROP SCHEMA IF EXISTS "${source.schema}" CASCADE`); + for (const namespace of namespaces) { + await pgClient.query(`DROP SCHEMA IF EXISTS "${namespace.db.schema}" CASCADE`); + } await pgClient.end(); } diff --git a/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts b/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts index d93b778..99d0fe1 100644 --- a/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts +++ b/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts @@ -1,11 +1,10 @@ -/* eslint-disable @typescript-eslint/naming-convention */ import { writeFileSync, mkdtempSync, rmSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { jsLogger } from '@map-colonies/js-logger'; -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'; import { trace } from '@opentelemetry/api'; -import nock, { disableNetConnect, enableNetConnect, cleanAll } from 'nock'; +import { disableNetConnect, enableNetConnect, cleanAll } from 'nock'; import type { DataSource, Repository } from 'typeorm'; import type { EnumValue, Layer, Property } from '@map-colonies/vector-standard-db'; import { @@ -14,54 +13,65 @@ import { LAYER_REPOSITORY_SYMBOL, PROPERTY_REPOSITORY_SYMBOL, columnType, + layerSource, } from '@map-colonies/vector-standard-db'; -import { initConfig, getConfig } from '@src/common/config'; +import { initConfig } from '@src/common/config'; import { registerExternalValues } from '@src/containerConfig'; -import { SERVICES, SOURCE_DATA_SOURCE_PROVIDER } from '@src/common/constants'; -import { S3Repository } from '@src/common/s3/s3Repository'; -import { SyncModel } from '@src/sync/SyncModel'; +import { SERVICES, NAMESPACE_HANDLES } from '@src/common/constants'; +import type { SyncModel } from '@src/sync/SyncModel'; import { schemaOf } from '@src/sync/helpers'; -import { FileReader, type FileAliases } from '@src/sync/fileReader'; +import { FileReader } from '@src/sync/fileReader'; +import type { FileAliases } from '@src/sync/aliasesFile'; import type { TypeMap } from '@src/sync/typeMap'; +import type { NamespaceHandle } from '@src/sync/namespaceHandle/types'; const TEST_LAYER = 'test_layer'; -const ENRICHMENT_ORIGIN = 'http://mock-api'; +const NAMESPACE = 'test'; const TYPE_MAP_FILE = './config/typeMap.json'; const noAliases: FileAliases = new Map(); - -const mockEnrichmentApi = (responseBody: object, statusCode = 200): nock.Scope => - nock(ENRICHMENT_ORIGIN).get(`/${TEST_LAYER}`).reply(statusCode, responseBody); +const noSourceAliases = new Map(); const createDestinationSchema = async (dataSource: DataSource, schema: string): Promise => { await dataSource.query(`CREATE SCHEMA IF NOT EXISTS "${schema}"`); await dataSource.query(` CREATE TYPE "${schema}"."column_type" AS ENUM('xsd:long', 'xsd:double', 'xsd:boolean', 'xsd:string', 'xsd:dateTime', 'gml:GeometryPropertyType', 'gml:PointPropertyType', 'gml:LineStringPropertyType', 'gml:PolygonPropertyType', 'gml:MultiPointPropertyType', 'gml:MultiLineStringPropertyType', 'gml:MultiPolygonPropertyType') `); + await dataSource.query(` + CREATE TYPE "${schema}"."layer_source" AS ENUM('sharedLua', 'perLayerJson') + `); await dataSource.query(` CREATE TABLE "${schema}"."layer" ( + "namespace" character varying NOT NULL, "layer_name" character varying NOT NULL, - "layer_id" integer NOT NULL, + "layer_id" integer, + "source" "${schema}"."layer_source" NOT NULL, "alias" character varying NOT NULL, - PRIMARY KEY ("layer_name") + PRIMARY KEY ("namespace", "layer_name"), + CONSTRAINT "CHK_layer_source_layer_id" CHECK (("source" = 'sharedLua' AND "layer_id" IS NOT NULL) OR ("source" = 'perLayerJson' AND "layer_id" IS NULL)) ) `); await dataSource.query(` CREATE TABLE "${schema}"."property" ( + "namespace" character varying NOT NULL, "layer_name" character varying NOT NULL, "property" character varying NOT NULL, "type" "${schema}"."column_type" NOT NULL, "alias" character varying, - PRIMARY KEY ("layer_name", "property") + PRIMARY KEY ("namespace", "layer_name", "property"), + FOREIGN KEY ("namespace", "layer_name") + REFERENCES "${schema}"."layer"("namespace", "layer_name") + ON DELETE NO ACTION ON UPDATE NO ACTION ) `); await dataSource.query(` CREATE TABLE "${schema}"."enum_value" ( "value" character varying NOT NULL, + "namespace" character varying NOT NULL, "layer_name" character varying NOT NULL, "property" character varying NOT NULL, - PRIMARY KEY ("value", "layer_name", "property"), - FOREIGN KEY ("layer_name", "property") - REFERENCES "${schema}"."property"("layer_name","property") + PRIMARY KEY ("value", "namespace", "layer_name", "property"), + FOREIGN KEY ("namespace", "layer_name", "property") + REFERENCES "${schema}"."property"("namespace", "layer_name", "property") ON DELETE CASCADE ON UPDATE NO ACTION ) `); @@ -69,7 +79,6 @@ const createDestinationSchema = async (dataSource: DataSource, schema: string): describe('DAL', function () { let dal: SyncModel; - let enrichedDal: SyncModel; let fileReader: FileReader; let typeMap: TypeMap; let sourceDataSource: DataSource; @@ -88,15 +97,23 @@ describe('DAL', function () { override: [ { token: SERVICES.LOGGER, provider: { useValue: await jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, - { token: S3Repository, provider: { useValue: { downloadFile: vi.fn() } } }, + { token: SERVICES.S3_REPOSITORY, provider: { useValue: { downloadFile: async () => Promise.reject(new Error('not used in this suite')) } } }, ], useChild: true, }); - dal = container.resolve(SyncModel); + const namespaceHandles = container.resolve(NAMESPACE_HANDLES); + const namespace = namespaceHandles.find((n) => n.name === NAMESPACE); + if (namespace === undefined) { + throw new Error(`Namespace ${NAMESPACE} was not configured for this test run`); + } + + dal = namespace.dal; + sourceDataSource = namespace.sourceDataSource; + await sourceDataSource.initialize(); + fileReader = container.resolve(FileReader); typeMap = await fileReader.readTypeMap(TYPE_MAP_FILE); - sourceDataSource = container.resolve(SOURCE_DATA_SOURCE_PROVIDER); destinationDataSource = container.resolve(DESTINATION_DATA_SOURCE_PROVIDER); layerRepository = container.resolve>(LAYER_REPOSITORY_SYMBOL); propertyRepository = container.resolve>(PROPERTY_REPOSITORY_SYMBOL); @@ -107,38 +124,6 @@ describe('DAL', function () { await sourceDataSource.query(`CREATE SCHEMA IF NOT EXISTS "${sourceSchema}"`); await createDestinationSchema(destinationDataSource, destinationSchema); - - const configWithEnrichment = new Proxy(getConfig(), { - get(target, prop) { - if (prop === 'get') { - return (key: string) => { - if (key === 'enrichment') { - return { - enabled: true as const, - api: 'http://mock-api/{layerName}', - propertiesPath: 'fields_list', - aliasField: 'display_name', - requestTimeoutMilliseconds: 10000, - }; - } - return target.get(key); - }; - } - return target[prop as keyof typeof target]; - }, - }); - - const enrichedContainer = await registerExternalValues({ - override: [ - { token: SERVICES.LOGGER, provider: { useValue: await jsLogger({ enabled: false }) } }, - { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, - { token: SERVICES.CONFIG, provider: { useValue: configWithEnrichment } }, - { token: S3Repository, provider: { useValue: { downloadFile: vi.fn() } } }, - ], - useChild: true, - }); - - enrichedDal = enrichedContainer.resolve(SyncModel); }); beforeEach(async function () { @@ -153,6 +138,7 @@ describe('DAL', function () { shape geometry ) `); + await dal.syncLayer(NAMESPACE, TEST_LAYER, 1, layerSource.sharedLua); }); afterEach(async function () { @@ -174,9 +160,9 @@ describe('DAL', function () { describe('Happy Path', function () { describe('syncLayer', function () { it('should insert a layer row with the provided alias', async function () { - await dal.syncLayer('buildings', 1, 'Buildings Layer'); + await dal.syncLayer(NAMESPACE, 'buildings', 1, layerSource.sharedLua, 'Buildings Layer'); - const layer = await layerRepository.findOne({ where: { layerName: 'buildings' } }); + const layer = await layerRepository.findOne({ where: { namespace: NAMESPACE, layerName: 'buildings' } }); expect(layer?.layerName).toBe('buildings'); expect(layer?.layerId).toBe(1); @@ -184,12 +170,21 @@ describe('DAL', function () { }); it('should use layerName as alias when none is provided', async function () { - await dal.syncLayer('buildings', 1); + await dal.syncLayer(NAMESPACE, 'buildings', 1, layerSource.sharedLua); - const layer = await layerRepository.findOne({ where: { layerName: 'buildings' } }); + const layer = await layerRepository.findOne({ where: { namespace: NAMESPACE, layerName: 'buildings' } }); expect(layer?.alias).toBe('buildings'); }); + + it('should accept a null layerId for a perLayerJson layer', async function () { + await dal.syncLayer(NAMESPACE, 'parks', null, layerSource.perLayerJson, 'Parks'); + + const layer = await layerRepository.findOne({ where: { namespace: NAMESPACE, layerName: 'parks' } }); + + expect(layer?.layerId).toBeNull(); + expect(layer?.source).toBe(layerSource.perLayerJson); + }); }); describe('getTableColumns', function () { @@ -245,9 +240,9 @@ describe('DAL', function () { describe('syncProperties', function () { it('should upsert all columns from the source table into the property repository', async function () { - const affected = await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); + const affected = await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, noSourceAliases); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); expect(affected).toBeGreaterThan(0); expect(properties).toEqual( @@ -263,9 +258,9 @@ describe('DAL', function () { }); it('should mark enum columns with columnType.enum', async function () { - await dal.syncProperties({ layerName: TEST_LAYER, enums: ['category', 'name'] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: ['category', 'name'] }, typeMap, noAliases, noSourceAliases); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); const categoryProp = properties.find((p) => p.property === 'category'); const nameProp = properties.find((p) => p.property === 'name'); const heightProp = properties.find((p) => p.property === 'height'); @@ -276,19 +271,25 @@ describe('DAL', function () { }); it('should upsert on subsequent calls without duplicating rows', async function () { - await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); - await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, noSourceAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, noSourceAliases); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); const uniqueProps = new Set(properties.map((p) => p.property)); expect(properties).toHaveLength(uniqueProps.size); }); it('should not sync properties listed in excludeProperties', async function () { - await dal.syncProperties({ layerName: TEST_LAYER, enums: [], excludeProperties: ['height', 'CREATED_AT'] }, 1, typeMap, noAliases); + await dal.syncProperties( + NAMESPACE, + { layerName: TEST_LAYER, enums: [], excludeProperties: ['height', 'CREATED_AT'] }, + typeMap, + noAliases, + noSourceAliases + ); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); const names = properties.map((p) => p.property); expect(names).not.toContain('height'); @@ -297,11 +298,11 @@ describe('DAL', function () { }); it('should delete properties that were synced before being excluded', async function () { - await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, noSourceAliases); - await dal.syncProperties({ layerName: TEST_LAYER, enums: [], excludeProperties: ['height'] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [], excludeProperties: ['height'] }, typeMap, noAliases, noSourceAliases); - const heightProp = await propertyRepository.findOne({ where: { layerName: TEST_LAYER, property: 'height' } }); + const heightProp = await propertyRepository.findOne({ where: { namespace: NAMESPACE, layerName: TEST_LAYER, property: 'height' } }); expect(heightProp).toBeNull(); }); @@ -311,9 +312,9 @@ describe('DAL', function () { ALTER TABLE "${sourceSchema}"."${TEST_LAYER}" ADD COLUMN geom_param geometry(Point,4326) `); - await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, noSourceAliases); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); const geomProp = properties.find((p) => p.property === 'geom_param'); expect(geomProp?.type).toBe(columnType.point); @@ -325,11 +326,11 @@ describe('DAL', function () { await sourceDataSource.query(` INSERT INTO "${sourceSchema}"."${TEST_LAYER}" (category) VALUES ('A'), ('B'), ('A'), ('C') `); - await dal.syncProperties({ layerName: TEST_LAYER, enums: ['category'] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: ['category'] }, typeMap, noAliases, noSourceAliases); - const affected = await dal.syncEnum({ layerName: TEST_LAYER, enums: ['category'] }); + const affected = await dal.syncEnum(NAMESPACE, { layerName: TEST_LAYER, enums: ['category'] }); - const enums = await enumsRepository.find({ where: { layerName: TEST_LAYER, property: 'category' } }); + const enums = await enumsRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER, property: 'category' } }); const values = enums.map((e) => e.value); expect(affected).toBe(3); @@ -337,8 +338,8 @@ describe('DAL', function () { }); it('should create an index on the enum column in the source table', async function () { - await dal.syncProperties({ layerName: TEST_LAYER, enums: ['category'] }, 1, typeMap, noAliases); - await dal.syncEnum({ layerName: TEST_LAYER, enums: ['category'] }); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: ['category'] }, typeMap, noAliases, noSourceAliases); + await dal.syncEnum(NAMESPACE, { layerName: TEST_LAYER, enums: ['category'] }); const indexName = `${TEST_LAYER}_category_idx`; const rows = await sourceDataSource.query<{ indexname: string }[]>( @@ -350,18 +351,28 @@ describe('DAL', function () { }); }); - describe('syncProperties with enrichment', function () { - it('should write aliases from the enrichment API into the property table', async function () { - mockEnrichmentApi({ - fields_list: { - name: { display_name: 'Layer Name', type: 'TEXT' }, - height: { display_name: 'Building Height', type: 'REAL' }, - }, - }); + describe('syncProperties with source and file aliases', function () { + let tmpDir: string; + let aliasesFilePath: string; + + beforeAll(function () { + tmpDir = mkdtempSync(join(tmpdir(), 'dal-aliases-test-')); + aliasesFilePath = join(tmpDir, 'aliases.json'); + }); + + afterAll(function () { + rmSync(tmpDir, { recursive: true }); + }); + + it('should write aliases from the source aliases map into the property table', async function () { + const sourceAliases = new Map([ + ['name', 'Layer Name'], + ['height', 'Building Height'], + ]); - await enrichedDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, sourceAliases); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); const nameProp = properties.find((p) => p.property === 'name'); const heightProp = properties.find((p) => p.property === 'height'); const idProp = properties.find((p) => p.property === 'id'); @@ -372,127 +383,100 @@ describe('DAL', function () { expect(idProp?.alias).toBeUndefined(); }); - it('should call the enrichment API once with the resolved layer URL', async function () { - const scope = mockEnrichmentApi({ fields_list: {} }); - - await enrichedDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); - - expect(scope.isDone()).toBe(true); - }); - }); + it('should apply aliases from the file when there are no source aliases', async function () { + writeFileSync(aliasesFilePath, JSON.stringify({ [NAMESPACE]: { [TEST_LAYER]: { name: 'File Name', height: 'File Height' } } })); - describe('syncProperties with file aliases', function () { - let tmpDir: string; - let aliasesFilePath: string; - let aliasesFileDal: SyncModel; - let bothDal: SyncModel; + await dal.syncProperties( + NAMESPACE, + { layerName: TEST_LAYER, enums: [] }, + typeMap, + await fileReader.readAliases(aliasesFilePath), + noSourceAliases + ); - beforeAll(async function () { - tmpDir = mkdtempSync(join(tmpdir(), 'dal-aliases-test-')); - aliasesFilePath = join(tmpDir, 'aliases.json'); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); - const makeConfig = (enrichmentEnabled: boolean) => - new Proxy(getConfig(), { - get(target, prop) { - if (prop === 'get') { - return (key: string) => { - if (key === 'enrichment') { - return enrichmentEnabled - ? { - enabled: true as const, - api: 'http://mock-api/{layerName}', - propertiesPath: 'fields_list', - aliasField: 'display_name', - requestTimeoutMilliseconds: 10000, - } - : { enabled: false }; - } - return target.get(key); - }; - } - return target[prop as keyof typeof target]; - }, - }); - - const logger = await jsLogger({ enabled: false }); - const s3Mock = { token: S3Repository, provider: { useValue: { downloadFile: vi.fn() } } }; - const sharedOverrides = [ - { token: SERVICES.LOGGER, provider: { useValue: logger } }, - { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, - s3Mock, - ]; - - aliasesFileDal = ( - await registerExternalValues({ - override: [...sharedOverrides, { token: SERVICES.CONFIG, provider: { useValue: makeConfig(false) } }], - useChild: true, - }) - ).resolve(SyncModel); - - bothDal = ( - await registerExternalValues({ - override: [...sharedOverrides, { token: SERVICES.CONFIG, provider: { useValue: makeConfig(true) } }], - useChild: true, - }) - ).resolve(SyncModel); + expect(properties.find((p) => p.property === 'name')?.alias).toBe('File Name'); + expect(properties.find((p) => p.property === 'height')?.alias).toBe('File Height'); + expect(properties.find((p) => p.property === 'id')?.alias).toBeUndefined(); }); - afterAll(function () { - rmSync(tmpDir, { recursive: true }); - }); + it('should override source aliases with file aliases for the same property', async function () { + writeFileSync(aliasesFilePath, JSON.stringify({ [NAMESPACE]: { [TEST_LAYER]: { name: 'File Name' } } })); - it('should apply aliases from the file when enrichment is disabled', async function () { - writeFileSync(aliasesFilePath, JSON.stringify({ [TEST_LAYER]: { name: 'File Name', height: 'File Height' } })); + const sourceAliases = new Map([ + ['name', 'Source Name'], + ['height', 'Source Height'], + ]); - await aliasesFileDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, await fileReader.readAliases(aliasesFilePath)); + await dal.syncProperties( + NAMESPACE, + { layerName: TEST_LAYER, enums: [] }, + typeMap, + await fileReader.readAliases(aliasesFilePath), + sourceAliases + ); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); expect(properties.find((p) => p.property === 'name')?.alias).toBe('File Name'); - expect(properties.find((p) => p.property === 'height')?.alias).toBe('File Height'); - expect(properties.find((p) => p.property === 'id')?.alias).toBeUndefined(); + expect(properties.find((p) => p.property === 'height')?.alias).toBe('Source Height'); }); - it('should override API aliases with file aliases for the same property', async function () { - writeFileSync(aliasesFilePath, JSON.stringify({ [TEST_LAYER]: { name: 'File Name' } })); + it('should fill in aliases from the file for properties absent from the source aliases', async function () { + writeFileSync(aliasesFilePath, JSON.stringify({ [NAMESPACE]: { [TEST_LAYER]: { height: 'File Height' } } })); - mockEnrichmentApi({ - fields_list: { name: { display_name: 'API Name', type: 'TEXT' }, height: { display_name: 'API Height', type: 'REAL' } }, - }); + const sourceAliases = new Map([['name', 'Source Name']]); - await bothDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, await fileReader.readAliases(aliasesFilePath)); + await dal.syncProperties( + NAMESPACE, + { layerName: TEST_LAYER, enums: [] }, + typeMap, + await fileReader.readAliases(aliasesFilePath), + sourceAliases + ); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); - expect(properties.find((p) => p.property === 'name')?.alias).toBe('File Name'); - expect(properties.find((p) => p.property === 'height')?.alias).toBe('API Height'); + expect(properties.find((p) => p.property === 'name')?.alias).toBe('Source Name'); + expect(properties.find((p) => p.property === 'height')?.alias).toBe('File Height'); }); - it('should fill in aliases from the file for properties absent from the API response', async function () { - writeFileSync(aliasesFilePath, JSON.stringify({ [TEST_LAYER]: { height: 'File Height' } })); - - mockEnrichmentApi({ fields_list: { name: { display_name: 'API Name', type: 'TEXT' } } }); + it('should prefer a namespace-specific file alias over a wildcard one', async function () { + writeFileSync( + aliasesFilePath, + JSON.stringify({ + '*': { '*': { name: 'Global Name' } }, + [NAMESPACE]: { [TEST_LAYER]: { name: 'Specific Name' } }, + }) + ); - await bothDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, await fileReader.readAliases(aliasesFilePath)); + await dal.syncProperties( + NAMESPACE, + { layerName: TEST_LAYER, enums: [] }, + typeMap, + await fileReader.readAliases(aliasesFilePath), + noSourceAliases + ); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); - expect(properties.find((p) => p.property === 'name')?.alias).toBe('API Name'); - expect(properties.find((p) => p.property === 'height')?.alias).toBe('File Height'); + expect(properties.find((p) => p.property === 'name')?.alias).toBe('Specific Name'); }); }); }); describe('Sad Path', function () { describe('syncLayer', function () { - it('should not throw and not update on duplicate layerName', async function () { - await dal.syncLayer('buildings', 1, 'First'); - await dal.syncLayer('buildings', 2, 'Second'); + it('should upsert layerId, source and alias on a duplicate namespace/layerName', async function () { + await dal.syncLayer(NAMESPACE, 'buildings', 1, layerSource.sharedLua, 'First'); + await dal.syncLayer(NAMESPACE, 'buildings', 2, layerSource.sharedLua, 'Second'); - const layers = await layerRepository.find({ where: { layerName: 'buildings' } }); + const layers = await layerRepository.find({ where: { namespace: NAMESPACE, layerName: 'buildings' } }); expect(layers).toHaveLength(1); - expect(layers[0]?.alias).toBe('First'); + expect(layers[0]?.layerId).toBe(2); + expect(layers[0]?.alias).toBe('Second'); }); }); @@ -518,44 +502,15 @@ describe('DAL', function () { describe('syncEnum', function () { it('should return 0 and skip when no enum columns are configured', async function () { - const affected = await dal.syncEnum({ layerName: TEST_LAYER, enums: [] }); + const affected = await dal.syncEnum(NAMESPACE, { layerName: TEST_LAYER, enums: [] }); expect(affected).toBe(0); - const enums = await enumsRepository.find({ where: { layerName: TEST_LAYER } }); + const enums = await enumsRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); expect(enums).toHaveLength(0); }); }); - - describe('syncProperties with enrichment', function () { - it('should leave alias unset for properties absent from the enrichment response', async function () { - mockEnrichmentApi({ fields_list: { name: { display_name: 'Layer Name', type: 'TEXT' } } }); - - await enrichedDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); - - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); - const idProp = properties.find((p) => p.property === 'id'); - - expect(idProp).toBeDefined(); - expect(idProp?.alias).toBeUndefined(); - }); - - it('should sync properties without aliases when the enrichment API returns an error response', async function () { - mockEnrichmentApi({}, 500); - - const affected = await enrichedDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); - - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); - const idProp = properties.find((p) => p.property === 'id'); - const nameProp = properties.find((p) => p.property === 'name'); - - expect(affected).toBeGreaterThan(0); - expect(idProp).toBeDefined(); - expect(idProp?.alias).toBeUndefined(); - expect(nameProp?.alias).toBeUndefined(); - }); - }); }); describe('Bad Path', function () { @@ -565,8 +520,8 @@ describe('DAL', function () { ALTER TABLE "${sourceSchema}"."${TEST_LAYER}" ADD COLUMN lsn pg_lsn `); - const affected = await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const affected = await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, noSourceAliases); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); expect(affected).toBeGreaterThan(0); expect(properties.find((p) => p.property === 'lsn')).toBeUndefined(); @@ -574,4 +529,35 @@ describe('DAL', function () { }); }); }); + + describe('Multi-namespace isolation', function () { + it('should not delete layers belonging to another namespace', async function () { + await dal.syncLayer(NAMESPACE, 'buildings', 1, layerSource.sharedLua); + await layerRepository.insert({ namespace: 'other', layerName: 'buildings', layerId: 1, source: layerSource.sharedLua, alias: 'buildings' }); + + await dal.deleteLayersNotIn(NAMESPACE, ['roads']); + + const otherLayer = await layerRepository.findOne({ where: { namespace: 'other', layerName: 'buildings' } }); + const ownLayer = await layerRepository.findOne({ where: { namespace: NAMESPACE, layerName: 'buildings' } }); + + expect(otherLayer).not.toBeNull(); + expect(ownLayer).toBeNull(); + + await layerRepository.delete({ namespace: 'other', layerName: 'buildings' }); + }); + + it('should not delete stale properties belonging to another namespace', async function () { + await layerRepository.insert({ namespace: 'other', layerName: TEST_LAYER, layerId: 1, source: layerSource.sharedLua, alias: TEST_LAYER }); + await propertyRepository.insert({ namespace: 'other', layerName: TEST_LAYER, property: 'legacy', type: columnType.text }); + + await dal.deleteStaleProperties(NAMESPACE, TEST_LAYER, []); + + const otherProperty = await propertyRepository.findOne({ where: { namespace: 'other', layerName: TEST_LAYER, property: 'legacy' } }); + + expect(otherProperty).not.toBeNull(); + + await propertyRepository.delete({ namespace: 'other', layerName: TEST_LAYER }); + await layerRepository.delete({ namespace: 'other', layerName: TEST_LAYER }); + }); + }); }); diff --git a/packages/synchronizer/tests/unit/synchronizer/aliasEnricher.spec.ts b/packages/synchronizer/tests/unit/synchronizer/aliasEnricher.spec.ts index 0d5ae3f..38ba574 100644 --- a/packages/synchronizer/tests/unit/synchronizer/aliasEnricher.spec.ts +++ b/packages/synchronizer/tests/unit/synchronizer/aliasEnricher.spec.ts @@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, beforeAll, afterEach, afterAll } from 'vitest'; import nock, { disableNetConnect, enableNetConnect, cleanAll } from 'nock'; import { fetchPropertyAliases } from '@src/sync/aliasEnricher'; -import { type ConfigType, getConfig, initConfig } from '@src/common/config'; +import type { EnrichmentConfig } from '@src/common/interfaces'; const ENRICHMENT_ORIGIN = 'https://example.com'; @@ -13,11 +13,15 @@ const makeBody = (fields: Record }); describe('fetchPropertyAliases', function () { - let enrichmentConfig: ReturnType['enrichment'] & { enabled: true }; - - beforeAll(async function () { - await initConfig(true); - enrichmentConfig = getConfig().get('enrichmentApi'); + const enrichmentConfig: Extract = { + enabled: true, + api: `${ENRICHMENT_ORIGIN}/{layerName}`, + propertiesPath: 'fields_list', + aliasField: 'display_name', + requestTimeoutMilliseconds: 10000, + }; + + beforeAll(function () { disableNetConnect(); }); @@ -37,7 +41,7 @@ describe('fetchPropertyAliases', function () { it('should call the API once with the layer name substituted into the URL', async function () { const scope = nock(ENRICHMENT_ORIGIN).get('/buildings').reply(200, makeBody({})); - await fetchPropertyAliases('buildings', 1, getConfig().get('enrichmentApi')); + await fetchPropertyAliases('buildings', 1, enrichmentConfig); expect(scope.isDone()).toBe(true); }); diff --git a/packages/synchronizer/tests/unit/synchronizer/helpers.spec.ts b/packages/synchronizer/tests/unit/synchronizer/helpers.spec.ts index ce520ba..d5d9ec3 100644 --- a/packages/synchronizer/tests/unit/synchronizer/helpers.spec.ts +++ b/packages/synchronizer/tests/unit/synchronizer/helpers.spec.ts @@ -10,6 +10,7 @@ const typeMapFile = './config/typeMap.json'; const typeMap = parseTypeMap(JSON.parse(readFileSync(typeMapFile, 'utf-8')) as JsonValue, typeMapFile); describe('columnInfosToProperties', function () { + const namespace = 'test_namespace'; const layer = 'test_layer'; describe('Happy Path', function () { @@ -53,7 +54,7 @@ describe('columnInfosToProperties', function () { ])('should map udt "%s" to %s', function (udtName, expected) { const columnInfos: ColumnInfo[] = [{ columnName: 'col', udtName }]; - const [result] = columnInfosToProperties(columnInfos, layer, typeMap); + const [result] = columnInfosToProperties(columnInfos, namespace, layer, typeMap); expect(result.type).toBe(expected); }); @@ -72,19 +73,19 @@ describe('columnInfosToProperties', function () { ['geography(MultiPolygon,4326)', columnType.multiPolygon], ['geometry(GeometryCollection,4326)', columnType.geom], ])('should map "%s" to its specific GML type', function (udtName, expected) { - const [result] = columnInfosToProperties([{ columnName: 'shape', udtName }], layer, typeMap); + const [result] = columnInfosToProperties([{ columnName: 'shape', udtName }], namespace, layer, typeMap); expect(result.type).toBe(expected); }); it('should map unqualified geometry to gml:GeometryPropertyType', function () { - const [result] = columnInfosToProperties([{ columnName: 'shape', udtName: 'geometry' }], layer, typeMap); + const [result] = columnInfosToProperties([{ columnName: 'shape', udtName: 'geometry' }], namespace, layer, typeMap); expect(result.type).toBe(columnType.geom); }); it('should match udt names case insensitively', function () { - const [result] = columnInfosToProperties([{ columnName: 'name', udtName: 'CHARACTER VARYING(255)' }], layer, typeMap); + const [result] = columnInfosToProperties([{ columnName: 'name', udtName: 'CHARACTER VARYING(255)' }], namespace, layer, typeMap); expect(result.type).toBe(columnType.text); }); @@ -95,30 +96,30 @@ describe('columnInfosToProperties', function () { { columnName: 'height', udtName: 'real' }, ]; - const result = columnInfosToProperties(columnInfos, layer, typeMap); + const result = columnInfosToProperties(columnInfos, namespace, layer, typeMap); expect(result).toEqual([ - { layerName: layer, property: 'name', type: columnType.text }, - { layerName: layer, property: 'height', type: columnType.real }, + { namespace, layerName: layer, property: 'name', type: columnType.text }, + { namespace, layerName: layer, property: 'height', type: columnType.real }, ]); }); }); describe('Sad Path', function () { it('should return an empty array for no column infos', function () { - expect(columnInfosToProperties([], layer, typeMap)).toEqual([]); + expect(columnInfosToProperties([], namespace, layer, typeMap)).toEqual([]); }); }); describe('Bad Path', function () { it('should map unknown geometry sub-type to gml:GeometryPropertyType', function () { - const [result] = columnInfosToProperties([{ columnName: 'shape', udtName: 'geometry(Curve,4326)' }], layer, typeMap); + const [result] = columnInfosToProperties([{ columnName: 'shape', udtName: 'geometry(Curve,4326)' }], namespace, layer, typeMap); expect(result.type).toBe(columnType.geom); }); it('should return no properties for a udt name not in the type map', function () { - const result = columnInfosToProperties([{ columnName: 'data', udtName: 'pg_lsn' }], layer, typeMap); + const result = columnInfosToProperties([{ columnName: 'data', udtName: 'pg_lsn' }], namespace, layer, typeMap); expect(result).toHaveLength(0); }); @@ -131,7 +132,7 @@ describe('columnInfosToProperties', function () { { columnName: 'height', udtName: 'real' }, ]; - const result = columnInfosToProperties(columnInfos, layer, typeMap, onUnknown); + const result = columnInfosToProperties(columnInfos, namespace, layer, typeMap, onUnknown); expect(result).toHaveLength(2); expect(result.map((p) => p.property)).toEqual(['name', 'height']); diff --git a/packages/synchronizer/tests/unit/synchronizer/perLayerJsonSource.spec.ts b/packages/synchronizer/tests/unit/synchronizer/perLayerJsonSource.spec.ts new file mode 100644 index 0000000..293505d --- /dev/null +++ b/packages/synchronizer/tests/unit/synchronizer/perLayerJsonSource.spec.ts @@ -0,0 +1,141 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { writeFileSync, mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { describe, it, expect, vi, beforeEach, afterEach, beforeAll } from 'vitest'; +import { jsLogger } from '@map-colonies/js-logger'; +import type { Logger } from '@map-colonies/js-logger'; +import { PerLayerJsonSource } from '@src/sync/layerSource/perLayerJsonSource'; +import type { FileDownloader } from '@src/sync/layerSource/types'; +import { FsRepository } from '@src/common/fs/fsRepository'; +import type { PerLayerJsonSourceConfig } from '@src/common/interfaces'; + +const BUCKET = 'test-bucket'; + +const config: PerLayerJsonSourceConfig = { + type: 'perLayerJson', + prefix: '', + aliasLayerNameField: 'alias_layer_name', + fieldsField: 'fields', + fieldNameField: 'field_name', + aliasFieldNameField: 'alias_field_name', +}; + +describe('PerLayerJsonSource', function () { + let tmpDir: string; + let logger: Logger; + let fsRepository: FsRepository; + + const makeS3Repository = (key: string, content: string): { s3Repository: FileDownloader; downloadFile: ReturnType } => { + const filePath = join(tmpDir, 'downloaded.json'); + writeFileSync(filePath, content); + const downloadFile = vi.fn().mockImplementation((_bucket: string, requestedKey: string) => { + if (requestedKey !== key) { + throw new Error('unexpected key'); + } + return filePath; + }); + return { s3Repository: { downloadFile }, downloadFile }; + }; + + beforeAll(async function () { + logger = await jsLogger({ enabled: false }); + fsRepository = new FsRepository(logger); + }); + + beforeEach(function () { + tmpDir = mkdtempSync(join(tmpdir(), 'per-layer-json-source-test-')); + }); + + afterEach(function () { + rmSync(tmpDir, { recursive: true }); + }); + + describe('Happy Path', function () { + it('should resolve a layer alias and case-insensitive property aliases', async function () { + const { s3Repository } = makeS3Repository( + 'BUILDINGS.json', + JSON.stringify({ + alias_layer_name: 'Buildings', + fields: [{ field_name: 'PROPERTY', alias_field_name: 'Property Alias' }], + }) + ); + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, config); + + const records = await source.tick([{ layerName: 'buildings', enums: [], sourceKey: 'BUILDINGS.json' }]); + const record = records.get('buildings'); + + expect(record?.alias).toBe('Buildings'); + expect(record?.layerId).toBeUndefined(); + expect(record?.source).toBe('perLayerJson'); + expect(record?.propertyAliases.get('property')).toBe('Property Alias'); + }); + + it('should prepend the configured prefix to the sourceKey', async function () { + const prefixedConfig: PerLayerJsonSourceConfig = { ...config, prefix: 'layers/' }; + const { s3Repository } = makeS3Repository('layers/BUILDINGS.json', JSON.stringify({ alias_layer_name: 'Buildings', fields: [] })); + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, prefixedConfig); + + const records = await source.tick([{ layerName: 'buildings', enums: [], sourceKey: 'BUILDINGS.json' }]); + + expect(records.get('buildings')?.alias).toBe('Buildings'); + }); + + it('should ignore fields with no matching field_name or alias_field_name', async function () { + const { s3Repository } = makeS3Repository( + 'BUILDINGS.json', + JSON.stringify({ + alias_layer_name: 'Buildings', + fields: [{ field_name: 'PROPERTY' }, { alias_field_name: 'Orphan Alias' }, 'not-an-object'], + }) + ); + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, config); + + const records = await source.tick([{ layerName: 'buildings', enums: [], sourceKey: 'BUILDINGS.json' }]); + + expect(records.get('buildings')?.propertyAliases.size).toBe(0); + }); + }); + + describe('Sad Path', function () { + it('should skip and warn when the layer has no sourceKey', async function () { + const { s3Repository, downloadFile } = makeS3Repository('BUILDINGS.json', '{}'); + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, config); + + const records = await source.tick([{ layerName: 'buildings', enums: [] }]); + + expect(records.has('buildings')).toBe(false); + expect(downloadFile).not.toHaveBeenCalled(); + }); + + it('should skip a layer whose object is missing from S3', async function () { + const downloadFile = vi.fn().mockRejectedValue(new Error('NoSuchKey')); + const s3Repository: FileDownloader = { downloadFile }; + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, config); + + const records = await source.tick([{ layerName: 'buildings', enums: [], sourceKey: 'MISSING.json' }]); + + expect(records.has('buildings')).toBe(false); + }); + }); + + describe('Bad Path', function () { + it('should skip a layer with unparseable JSON', async function () { + const { s3Repository } = makeS3Repository('BUILDINGS.json', 'not json'); + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, config); + + const records = await source.tick([{ layerName: 'buildings', enums: [], sourceKey: 'BUILDINGS.json' }]); + + expect(records.has('buildings')).toBe(false); + }); + + it('should skip a layer missing required fields', async function () { + const { s3Repository } = makeS3Repository('BUILDINGS.json', JSON.stringify({ fields: [] })); + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, config); + + const records = await source.tick([{ layerName: 'buildings', enums: [], sourceKey: 'BUILDINGS.json' }]); + + expect(records.has('buildings')).toBe(false); + }); + }); +}); diff --git a/packages/synchronizer/tests/unit/synchronizer/sharedLuaSource.spec.ts b/packages/synchronizer/tests/unit/synchronizer/sharedLuaSource.spec.ts new file mode 100644 index 0000000..c0004b2 --- /dev/null +++ b/packages/synchronizer/tests/unit/synchronizer/sharedLuaSource.spec.ts @@ -0,0 +1,141 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { writeFileSync, mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { describe, it, expect, vi, beforeEach, afterEach, beforeAll, afterAll } from 'vitest'; +import nock, { disableNetConnect, enableNetConnect, cleanAll } from 'nock'; +import { jsLogger } from '@map-colonies/js-logger'; +import type { Logger } from '@map-colonies/js-logger'; +import { SharedLuaSource } from '@src/sync/layerSource/sharedLuaSource'; +import type { FileDownloader } from '@src/sync/layerSource/types'; +import { FsRepository } from '@src/common/fs/fsRepository'; +import type { SharedLuaSourceConfig } from '@src/common/interfaces'; + +const ID_FIELD = 'layer_id'; +const NAME_FIELD = 'layer_name'; +const ALIAS_FIELD = 'layer_alias'; +const BUCKET = 'test-bucket'; +const FILE_NAME = 'query.lua'; +const ENRICHMENT_ORIGIN = 'http://mock-api'; + +const makeLua = (entries: string): string => `local layers = {\n${entries}\n}`; + +const makeEntry = (layerId: string, layerName: string, alias: string): string => + ` { ${ID_FIELD} = '${layerId}', ${NAME_FIELD} = '${layerName}', ${ALIAS_FIELD} = '${alias}' },`; + +describe('SharedLuaSource', function () { + let tmpDir: string; + let logger: Logger; + let fsRepository: FsRepository; + + const makeS3Repository = (content: string): FileDownloader => { + const filePath = join(tmpDir, FILE_NAME); + writeFileSync(filePath, content); + return { downloadFile: vi.fn().mockResolvedValue(filePath) }; + }; + + const disabledEnrichmentConfig: SharedLuaSourceConfig = { + type: 'sharedLua', + fileName: FILE_NAME, + layersVariable: 'layers', + aliasFieldName: ALIAS_FIELD, + idFieldName: ID_FIELD, + nameFieldName: NAME_FIELD, + enrichment: { enabled: false }, + }; + + beforeAll(async function () { + logger = await jsLogger({ enabled: false }); + fsRepository = new FsRepository(logger); + disableNetConnect(); + }); + + beforeEach(function () { + tmpDir = mkdtempSync(join(tmpdir(), 'shared-lua-source-test-')); + cleanAll(); + }); + + afterEach(function () { + rmSync(tmpDir, { recursive: true }); + cleanAll(); + }); + + afterAll(function () { + enableNetConnect(); + }); + + describe('Happy Path', function () { + it('should resolve a known layer with its id and alias', async function () { + const s3Repository = makeS3Repository(makeLua(makeEntry('1', 'buildings', 'Buildings'))); + const source = new SharedLuaSource(s3Repository, fsRepository, logger, BUCKET, disabledEnrichmentConfig); + + const records = await source.tick([{ layerName: 'buildings', enums: [] }]); + const record = records.get('buildings'); + + expect(record?.layerId).toBe(1); + expect(record?.alias).toBe('Buildings'); + expect(record?.source).toBe('sharedLua'); + expect(record?.propertyAliases.size).toBe(0); + }); + + it('should fetch property aliases from the enrichment API when enabled', async function () { + const enrichmentConfig: SharedLuaSourceConfig = { + ...disabledEnrichmentConfig, + enrichment: { + enabled: true, + api: `${ENRICHMENT_ORIGIN}/{layerName}`, + propertiesPath: 'fields_list', + aliasField: 'display_name', + requestTimeoutMilliseconds: 5000, + }, + }; + nock(ENRICHMENT_ORIGIN) + .get('/buildings') + .reply(200, { fields_list: { name: { display_name: 'Name' } } }); + + const s3Repository = makeS3Repository(makeLua(makeEntry('1', 'buildings', 'Buildings'))); + const source = new SharedLuaSource(s3Repository, fsRepository, logger, BUCKET, enrichmentConfig); + + const records = await source.tick([{ layerName: 'buildings', enums: [] }]); + const record = records.get('buildings'); + + expect(record?.propertyAliases.get('name')).toBe('Name'); + }); + }); + + describe('Sad Path', function () { + it('should not include a layer absent from the lua file', async function () { + const s3Repository = makeS3Repository(makeLua(makeEntry('1', 'buildings', 'Buildings'))); + const source = new SharedLuaSource(s3Repository, fsRepository, logger, BUCKET, disabledEnrichmentConfig); + + const records = await source.tick([{ layerName: 'roads', enums: [] }]); + + expect(records.has('roads')).toBe(false); + }); + }); + + describe('Bad Path', function () { + it('should resolve without property aliases when the enrichment API fails', async function () { + const enrichmentConfig: SharedLuaSourceConfig = { + ...disabledEnrichmentConfig, + enrichment: { + enabled: true, + api: `${ENRICHMENT_ORIGIN}/{layerName}`, + propertiesPath: 'fields_list', + aliasField: 'display_name', + requestTimeoutMilliseconds: 5000, + }, + }; + nock(ENRICHMENT_ORIGIN).get('/buildings').reply(500); + + const s3Repository = makeS3Repository(makeLua(makeEntry('1', 'buildings', 'Buildings'))); + const source = new SharedLuaSource(s3Repository, fsRepository, logger, BUCKET, enrichmentConfig); + + const records = await source.tick([{ layerName: 'buildings', enums: [] }]); + const record = records.get('buildings'); + + expect(record?.layerId).toBe(1); + expect(record?.propertyAliases.size).toBe(0); + }); + }); +});