Feat!: Add namespaces and s3 json enrichment (MAPCO-11463)

helm/charts/api/templates/deployment.yaml

@@ -93,6 +93,9 @@ spec:
           {{- end }}
 
           volumeMounts:
+            - name: namespaces-config
+              mountPath: /usr/src/app/packages/api/dist/config/production.json
+              subPath: production.json
             {{- if .Values.caSecretName }}
             - mountPath: {{ printf "%s/%s" .Values.caPath .Values.caKey | quote }}
               name: root-ca
@@ -104,6 +107,9 @@ spec:
               readOnly: true
             {{- end }}
       volumes:
+        - name: namespaces-config
+          configMap:
+            name: {{ include "api.fullname" . }}-namespaces
       {{- if .Values.db.sslAuth.enabled }}
         - name: cert-conf
           secret:

helm/charts/api/templates/namespaces-configmap.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.enabled -}}
+{{- $names := list -}}
+{{- range .Values.namespaces -}}
+{{- $names = append $names .name -}}
+{{- end -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "api.fullname" . }}-namespaces
+  labels:
+    {{- include "api.labels" . | nindent 4 }}
+data:
+  production.json: {{ dict "namespaces" $names | toPrettyJson | quote }}
+{{- end }}

helm/charts/api/values.yaml

@@ -52,6 +52,8 @@ env:
   metrics:
     enabled: false
     url: http://localhost:55681/v1/metrics
+namespaces: []
+
 db:
   host: localhost
   port: 5432

helm/charts/synchronizer/templates/configmap.yaml

@@ -32,8 +32,6 @@ data:
 
   INDEX_NAME_FORMAT: {{ .Values.env.indexNameFormat | quote }}
 
-  LAYERS_FILE_PATH: {{ .Values.env.layersFile | quote }}
-
   ALIASES_FILE_PATH: {{ .Values.env.aliasesFile | quote }}
 
   {{- if .Values.typeMap.enabled }}
@@ -47,21 +45,8 @@ data:
   S3_SSL_ENABLED: {{ .sslEnabled | quote }}
   S3_SIG_VERSION: {{ .sigVersion | quote }}
   S3_STORAGE_CLASS: {{ .storageClass | quote }}
-  S3_BUCKET: {{ .bucket | quote }}
-  S3_FILE_NAME: {{ .fileName | quote }}
-  S3_LAYERS_VARIABLE: {{ .layersVariable | quote }}
   {{- end }}
 
-  {{- with .Values.env.enrichment }}
-  ENRICHMENT_ENABLED: {{ .enabled | quote }}
-  {{- if .enabled }}
-  ENRICHMENT_API_URL: {{ .api | quote }}
-  ENRICHMENT_PROPERTIES_PATH: {{ .propertiesPath | quote }}
-  ENRICHMENT_ALIAS_FIELD: {{ .aliasField | quote }}
-  ENRICHMENT_REQUEST_TIMEOUT_MILLISECONDS: {{ .requestTimeoutMilliseconds | quote }}
-  {{- end }}
-  {{- end }}
-  {{- include "synchronizer.dbEnvBlock" (dict "prefix" "SOURCE_DB" "db" .Values.dbs.sourceDb) | nindent 2 }}
   {{- include "synchronizer.dbEnvBlock" (dict "prefix" "DEST_DB" "db" .Values.dbs.destinationDb) | nindent 2 }}
 
 

helm/charts/synchronizer/templates/deployment.yaml

@@ -93,7 +93,10 @@ spec:
           {{- end }}
           volumeMounts:
             - name: layers-config
-              mountPath: {{ .Values.env.layersFile | dir | quote }}
+              mountPath: {{ .Values.env.layersDir | quote }}
+            - name: namespaces-config
+              mountPath: /usr/src/app/packages/synchronizer/dist/config/production.json
+              subPath: production.json
             - name: aliases-config
               mountPath: {{ .Values.env.aliasesFile | dir | quote }}
             {{- if .Values.typeMap.enabled }}
@@ -102,11 +105,13 @@ spec:
             {{- end }}
             - name: s3-downloads
               mountPath: /usr/src/app/packages/synchronizer/dist/src/common/s3/downloads
-            {{- if .Values.dbs.sourceDb.sslAuth.enabled }}
-            - name: source-db-cert-conf
-              mountPath: /tmp/certs/source
+            {{- range .Values.namespaces }}
+            {{- if .db.ssl.enabled }}
+            - name: source-db-cert-{{ .name }}
+              mountPath: {{ printf "/tmp/certs/source-%s" .name | quote }}
               readOnly: true
             {{- end }}
+            {{- end }}
             {{- if .Values.dbs.destinationDb.sslAuth.enabled }}
             - name: dest-db-cert-conf
               mountPath: /tmp/certs/dest
@@ -121,6 +126,9 @@ spec:
         - name: layers-config
           configMap:
             name: {{ include "synchronizer.fullname" . }}-layers
+        - name: namespaces-config
+          secret:
+            secretName: {{ include "synchronizer.fullname" . }}-namespaces
         - name: aliases-config
           configMap:
             name: {{ include "synchronizer.fullname" . }}-aliases
@@ -131,16 +139,13 @@ spec:
         {{- end }}
         - name: s3-downloads
           emptyDir: {}
-        {{- if .Values.dbs.sourceDb.sslAuth.enabled }}
-        - name: source-db-cert-conf
+        {{- range .Values.namespaces }}
+        {{- if .db.ssl.enabled }}
+        - name: source-db-cert-{{ .name }}
           secret:
-            secretName: {{ .Values.dbs.sourceDb.sslAuth.secretName }}
+            secretName: {{ .db.sslSecretName }}
+        {{- end }}
         {{- end }}
-         {{- if .Values.caSecretName }}
-            - mountPath: {{ printf "%s/%s" .Values.caPath .Values.caKey | quote }}
-              name: root-ca
-              subPath: {{ quote .Values.caKey }}
-            {{- end }}
         {{- if .Values.dbs.destinationDb.sslAuth.enabled }}
         - name: dest-db-cert-conf
           secret:

helm/charts/synchronizer/templates/layers-configmap.yaml

@@ -6,5 +6,7 @@ metadata:
   labels:
     {{- include "synchronizer.labels" . | nindent 4 }}
 data:
-  layers.json: {{ .Values.layers | toPrettyJson | quote }}
+  {{- range .Values.namespaces }}
+  {{ .name }}.json: {{ .layers | toPrettyJson | quote }}
+  {{- end }}
 {{- end }}

helm/charts/synchronizer/templates/namespaces-secret.yaml

@@ -0,0 +1,16 @@
+{{- if .Values.enabled -}}
+{{- $layersDir := .Values.env.layersDir -}}
+{{- $namespaces := list -}}
+{{- range .Values.namespaces -}}
+{{- $namespaces = append $namespaces (merge (dict "layersFile" (printf "%s/%s.json" $layersDir .name)) (omit . "layers")) -}}
+{{- end -}}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "synchronizer.fullname" . }}-namespaces
+  labels:
+    {{- include "synchronizer.labels" . | nindent 4 }}
+type: Opaque
+stringData:
+  production.json: {{ dict "namespaces" $namespaces | toPrettyJson | quote }}
+{{- end }}

helm/charts/synchronizer/templates/secret.yaml

@@ -7,7 +7,6 @@ metadata:
     {{- include "synchronizer.labels" . | nindent 4 }}
 type: Opaque
 data:
-  SOURCE_DB_PASSWORD: {{ .Values.dbs.sourceDb.password | b64enc | quote }}
   DEST_DB_PASSWORD: {{ .Values.dbs.destinationDb.password | b64enc | quote }}
   S3_ACCESS_KEY: {{ .Values.dbs.s3.accessKey | b64enc | quote }}
   S3_SECRET_KEY: {{ .Values.dbs.s3.secretKey | b64enc | quote }}

helm/charts/synchronizer/values.yaml

@@ -23,7 +23,6 @@ startupProbe:
   enabled: true
   periodSeconds: 10
   timeoutSeconds: 5
-  # failureThreshold * periodSeconds = max startup wait (30 * 10s = 300s)
   failureThreshold: 30
 
 livenessProbe:
@@ -41,11 +40,41 @@ readinessProbe:
   failureThreshold: 6
   path: /liveness
 
-layers:
-  - layerName: buildings_polygon
-    enums: [code, attribute, sensitivity, building_type, status]
-  - layerName: borders_polygon
-    enums: [code, attribute]
+namespaces:
+  - name: default
+    bucket: ""
+    layers:
+      - layerName: buildings_polygon
+        enums: [code, attribute, sensitivity, building_type, status]
+      - layerName: borders_polygon
+        enums: [code, attribute]
+    db:
+      host: localhost
+      port: 5432
+      username: postgres
+      password: postgres
+      database: postgres
+      schema: public
+      type: postgres
+      sslSecretName: secret-name
+      ssl:
+        enabled: false
+        ca: ""
+        cert: ""
+        key: ""
+    layerSource:
+      type: sharedLua
+      fileName: mock-query.lua
+      layersVariable: layers
+      aliasFieldName: alias
+      idFieldName: id
+      nameFieldName: name
+      enrichment:
+        enabled: false
+        api: ""
+        propertiesPath: ""
+        aliasField: ""
+        requestTimeoutMilliseconds: 5000
 
 aliases: {}
 
@@ -70,15 +99,9 @@ env:
     url: http://localhost:55681/v1/metrics
   schedule: '0 0 * * *'
   indexNameFormat: '{layerName}_{column}_idx'
-  layersFile: /usr/src/app/packages/synchronizer/dist/config/layers/layers.json
+  layersDir: /usr/src/app/packages/synchronizer/dist/config/layers
   aliasesFile: /usr/src/app/packages/synchronizer/dist/config/aliases/aliases.json
   typeMapFile: /usr/src/app/packages/synchronizer/dist/config/typeMap/typeMap.json
-  enrichment:
-    enabled: false
-    api: ""
-    propertiesPath: ""
-    aliasField: ""
-    requestTimeoutMilliseconds: 5000
 
 dbs:
   s3:
@@ -88,25 +111,8 @@ dbs:
     sslEnabled: false
     sigVersion: v4
     storageClass: STANDARD
-    bucket: ""
-    fileName: ""
-    layersVariable: layers
     accessKey: ""
     secretKey: ""
-  sourceDb:
-    host: localhost
-    port: 5432
-    username: postgres
-    password: postgres
-    database: postgres
-    schema: public
-    type: postgres
-    sslAuth:
-      enabled: false
-      secretName: secret-name
-      cert: ""
-      key: ""
-      root: ""
   destinationDb:
     host: localhost
     port: 5432

helm/values.yaml

@@ -36,7 +36,38 @@ dbConfig: &dbConfig
     key: ''
     root: ''
 
+namespaces: &namespaces
+  - name: default
+    bucket: ""
+    layers:
+    - layerName: buildings_polygon
+      enums: [ code, attribute ]
+    db:
+      host: localhost
+      port: 5432
+      username: postgres
+      password: postgres
+      database: postgres
+      schema: public
+      type: postgres
+      sslSecretName: secret-name
+      ssl:
+        enabled: false
+        ca: ""
+        cert: ""
+        key: ""
+    layerSource:
+      type: sharedLua
+      fileName: ""
+      layersVariable: layers
+      aliasFieldName: alias
+      idFieldName: id
+      nameFieldName: name
+      enrichment:
+        enabled: false
+
 api:
+  namespaces: *namespaces
   enabled: true
   replicaCount: 1
   revisionHistoryLimit: 5
@@ -163,13 +194,12 @@ synchronizer:
     pullPolicy: Always
     tag: "v1.2.0"
 
-  layers:
-  - layerName: buildings_polygon
-    enums: [ code, attribute ]
+  namespaces: *namespaces
 
   aliases:
     "*":
-      num_floors: "number_of_floors"
+      "*":
+        num_floors: "number_of_floors"
 
   typeMap:
     enabled: false
@@ -192,15 +222,9 @@ synchronizer:
       url: http://localhost:55681/v1/metrics
     schedule: '*/1 * * * *'
     indexNameFormat: '{layerName}_{column}_idx'
-    layersFile: /usr/src/app/packages/synchronizer/dist/config/layers/layers.json
+    layersDir: /usr/src/app/packages/synchronizer/dist/config/layers
     aliasesFile: /usr/src/app/packages/synchronizer/dist/config/aliases/aliases.json
-    typeMapFile: /usr/src/app/packages/synchronizer/dist/config/typeMap.json
-    enrichment:
-      enabled: false
-      api: ""
-      propertiesPath: ""
-      aliasField: ""
-      requestTimeoutMilliseconds: 5000
+    typeMapFile: /usr/src/app/packages/synchronizer/dist/config/typeMap/typeMap.json
 
   dbs:
     s3:
@@ -210,24 +234,8 @@ synchronizer:
       sslEnabled: false
       sigVersion: v4
       storageClass: STANDARD
-      bucket: ""
-      fileName: ""
-      layersVariable: layers
       accessKey: ""
       secretKey: ""
-    sourceDb:
-      host: localhost
-      port: 5432
-      username: postgres
-      password: postgres
-      database: postgres
-      schema: public
-      type: postgres
-      sslAuth:
-        enabled: false
-        cert: ''
-        key: ''
-        root: ''
     destinationDb: *dbConfig
 
   resources: