diff --git a/.gitignore b/.gitignore index f89ab76f2..d405200d5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,9 @@ config.mk .env */.npm_install_done -.vscode +.vscode/ +.claude/ +.devcontainer/ .DS_Store /edge-ads/node_modules spirit-schemas/ diff --git a/acs-openmetadata-server/Dockerfile b/acs-openmetadata-server/Dockerfile new file mode 100644 index 000000000..b95d2fc40 --- /dev/null +++ b/acs-openmetadata-server/Dockerfile @@ -0,0 +1,14 @@ +FROM docker.getcollate.io/openmetadata/server:1.13.3 + +# The upstream image bundles log4j-core/log4j-api 2.25.5 under +# /opt/openmetadata/libs. Swap in a patched 2.x release - log4j2's public API +# is stable across patch versions, so this is a safe drop-in without needing +# to touch the OpenMetadata server itself. ADD's remote-URL support is used +# instead of curl/wget since the Alpine base image doesn't ship either. +ARG LOG4J_VERSION=2.26.1 + +USER root +RUN rm -f /opt/openmetadata/libs/log4j-core-*.jar /opt/openmetadata/libs/log4j-api-*.jar +ADD --chown=openmetadata:openmetadata https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/${LOG4J_VERSION}/log4j-core-${LOG4J_VERSION}.jar /opt/openmetadata/libs/log4j-core-${LOG4J_VERSION}.jar +ADD --chown=openmetadata:openmetadata https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/${LOG4J_VERSION}/log4j-api-${LOG4J_VERSION}.jar /opt/openmetadata/libs/log4j-api-${LOG4J_VERSION}.jar +USER openmetadata diff --git a/acs-openmetadata-server/Makefile b/acs-openmetadata-server/Makefile new file mode 100644 index 000000000..352d257e6 --- /dev/null +++ b/acs-openmetadata-server/Makefile @@ -0,0 +1,16 @@ +top=.. +include ${top}/mk/acs.init.mk + +repo?=openmetadata-server +# Hard-set (not ?=): this tracks the upstream openmetadata/server version +# this Dockerfile patches, not ACS's own release version, so it must not +# follow config.mk's version= override for ACS's own services. +version=1.13.3 +suffix?=-patched +# Flatten after push: the Dockerfile removes the bundled server log4j jars, +# but a plain layered image only whiteouts them - the old bytes are still on +# disk in the base image's layer. Flattening collapses that away. See +# mk/acs.docker.mk and README.md. +flatten=1 + +include ${mk}/acs.docker.mk diff --git a/acs-openmetadata-server/README.md b/acs-openmetadata-server/README.md new file mode 100644 index 000000000..e02fce568 --- /dev/null +++ b/acs-openmetadata-server/README.md @@ -0,0 +1,84 @@ +# OpenMetadata server image (log4j patch) + +The OpenMetadata server itself is deployed via the `openmetadata` Helm chart +(see `/deploy/values.yaml`'s `openmetadata` block and +[`acs-openmetadata/README.md`](../acs-openmetadata/README.md) for the wider +OpenMetadata deployment). This is a separate image from the ingestion image +patched in `acs-openmetadata` - the server chart's default image is +`docker.getcollate.io/openmetadata/server`, built from OpenMetadata's own +Maven release tarball, not `openmetadata/ingestion`. + +That upstream image bundles `log4j-core`/`log4j-api` 2.25.5 under +`/opt/openmetadata/libs`. This directory's `Dockerfile` patches the jars in +directly, the same way `acs-opensearch/Dockerfile` patches OpenSearch's +bundled log4j: + +```dockerfile +FROM docker.getcollate.io/openmetadata/server:1.13.3 + +ARG LOG4J_VERSION=2.26.1 + +USER root +RUN rm -f /opt/openmetadata/libs/log4j-core-*.jar /opt/openmetadata/libs/log4j-api-*.jar +ADD --chown=openmetadata:openmetadata https://.../log4j-core-${LOG4J_VERSION}.jar /opt/openmetadata/libs/log4j-core-${LOG4J_VERSION}.jar +ADD --chown=openmetadata:openmetadata https://.../log4j-api-${LOG4J_VERSION}.jar /opt/openmetadata/libs/log4j-api-${LOG4J_VERSION}.jar +USER openmetadata +``` + +log4j2's public API is stable across patch versions, so this is a safe +drop-in swap that doesn't require touching the OpenMetadata server itself. +`LOG4J_VERSION` is kept in sync with the version pinned in +`acs-openmetadata/Dockerfile` and `acs-opensearch/Dockerfile` - there's no +reason to track three different patched log4j versions across the three +images. + +`ADD`'s remote-URL support is used to fetch the jars (rather than `curl`/ +`wget`) because the upstream image is built on Alpine and doesn't ship +either tool. `USER root`/`USER openmetadata` bracket the patch because the +upstream image already drops privileges to the `openmetadata` user, and +that user doesn't own `/opt/openmetadata/libs` by default. + +## Building and pushing the patched image + +Requires the [`crane`](https://github.com/google/go-containerregistry) CLI on +`PATH` in addition to `docker buildx` - see below. + +```sh +cd acs-openmetadata-server +make build +``` + +`make build` (via `mk/acs.docker.mk`) runs +`docker buildx build --push --platform linux/amd64 -t /openmetadata-server:1.13.3-patched .`, +then flattens the pushed image with `crane flatten`. `rm`-ing a file in a +Dockerfile only hides it behind a whiteout - the bytes are still present in +the upstream base image's layer underneath, which file-level vulnerability +scanners that walk a node's disk (rather than asking a registry to resolve +the image) will still flag. Flattening merges the layers into one via the +registry API so the removed jars are actually gone, not just masked; `crane` +preserves the image's config (`ENV`/`ENTRYPOINT`/`CMD`/`USER`/etc.) +unchanged - only the filesystem layers are affected. This is opted into via +`flatten=1` in this directory's `Makefile` and is a no-op for every other +ACS service's `make build`, since `mk/acs.docker.mk` only runs it when +`flatten` is set. + +The `version` in the `Makefile` is pinned to `1.13.3` on purpose (not the +usual `?=` override) - it tracks the upstream `openmetadata/server` version +this Dockerfile patches, not ACS's own release version. The resulting tag is +what `deploy/values.yaml` references: + +```yaml +openmetadata: + image: + repository: /openmetadata-server + tag: "1.13.3-patched" +``` + +When upstream OpenMetadata ships a new release, bump the `FROM` tag here +(keeping it in sync with the `openmetadata`/`openmetadata-dependencies` +chart versions in `/deploy/Chart.yaml`, since the server image version and +chart `appVersion` are expected to match), rebuild, and update the tag in +`values.yaml` to match. If a future upstream release already bundles a fixed +log4j, this Dockerfile (and the custom image) can be dropped and +`values.yaml` pointed back at the stock +`docker.getcollate.io/openmetadata/server` image. diff --git a/acs-openmetadata/Dockerfile b/acs-openmetadata/Dockerfile new file mode 100644 index 000000000..b41012830 --- /dev/null +++ b/acs-openmetadata/Dockerfile @@ -0,0 +1,15 @@ +FROM openmetadata/ingestion:1.13.3 + +# Pre-install the package during image build time +RUN pip install --no-cache-dir "apache-airflow-providers-fab==2.4.4" + +# pyspark bundles its own log4j-core/log4j-api (2.20.0), which is in the +# affected range for CVE-2025-68161/CVE-2026-34477/CVE-2026-34480. Swap in a +# patched 2.x release - log4j2's public API is stable across patch +# versions, so this is a safe drop-in without needing to touch pyspark +# itself or know whether the Spark profiling engine is actually used. +ARG LOG4J_VERSION=2.26.1 +RUN PYSPARK_JARS="$(python -c 'import os, pyspark; print(os.path.join(os.path.dirname(pyspark.__file__), "jars"))')" \ + && rm -f "${PYSPARK_JARS}"/log4j-core-*.jar "${PYSPARK_JARS}"/log4j-api-*.jar \ + && python -c "import urllib.request as r; r.urlretrieve('https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/${LOG4J_VERSION}/log4j-core-${LOG4J_VERSION}.jar', '${PYSPARK_JARS}/log4j-core-${LOG4J_VERSION}.jar')" \ + && python -c "import urllib.request as r; r.urlretrieve('https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/${LOG4J_VERSION}/log4j-api-${LOG4J_VERSION}.jar', '${PYSPARK_JARS}/log4j-api-${LOG4J_VERSION}.jar')" \ No newline at end of file diff --git a/acs-openmetadata/Makefile b/acs-openmetadata/Makefile new file mode 100644 index 000000000..015dba2af --- /dev/null +++ b/acs-openmetadata/Makefile @@ -0,0 +1,17 @@ +top=.. +include ${top}/mk/acs.init.mk + +repo?=openmetadata-ingestion +# Hard-set (not ?=): this tracks the upstream openmetadata/ingestion +# version this Dockerfile patches, not ACS's own release version, so it +# must not follow config.mk's version= override for ACS's own services. +version=1.13.3 +suffix?=-patched +k8s.deployment?=openmetadata +# Flatten after push: the Dockerfile removes the bundled pyspark log4j jars, +# but a plain layered image only whiteouts them - the old bytes are still on +# disk in the base image's layer. Flattening collapses that away. See +# mk/acs.docker.mk and README.md. +flatten=1 + +include ${mk}/acs.docker.mk diff --git a/acs-openmetadata/README.md b/acs-openmetadata/README.md new file mode 100644 index 000000000..7073d0db1 --- /dev/null +++ b/acs-openmetadata/README.md @@ -0,0 +1,273 @@ +# OpenMetadata deployment + +[OpenMetadata](https://open-metadata.org/) has been deployed into ACS as a set +of Helm chart dependencies from +[open-metadata/openmetadata-helm-charts](https://github.com/open-metadata/openmetadata-helm-charts), +declared in `/deploy/Chart.yaml` and configured in `/deploy/values.yaml`. +This directory (`acs-openmetadata`) holds the ingestion image patch: a +patched ingestion image, built with the `Dockerfile`/`Makefile` here, that +works around two problems in the upstream image (see +[Troubleshooting](#troubleshooting) below). The OpenSearch image gets a +similar log4j patch, but lives in its own `acs-opensearch` directory since +it isn't part of the ingestion image - see +[acs-opensearch/README.md](../acs-opensearch/README.md). + +## Components + +OpenMetadata is not a single deployment - the upstream charts bring in several +moving parts. What's actually running in the cluster: + +| Component | Chart / image | Function | +| --------------------------------------------------- | ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **OpenMetadata server** | `openmetadata` chart | The metadata catalogue itself - the web UI and API that stores and serves schema/lineage/ownership metadata for data assets. Runs a log4j-patched image built from `acs-openmetadata-server` (see [its README](../acs-openmetadata-server/README.md)). | +| **MySQL** | `openmetadata-dependencies` → `mysql` (Bitnami) | OpenMetadata's own relational store, holding both the `openmetadata_db` (catalogue metadata) and `airflow_db` (ingestion pipeline scheduling state) databases. | +| **OpenSearch** | `openmetadata-dependencies` → `opensearch` | Search/index backend behind OpenMetadata's search UI and discovery features. The upstream chart calls this dependency "elasticsearch" throughout (config keys, secret names) for historical reasons, but the image actually deployed is OpenSearch - a drop-in, license-compatible fork. Don't be misled by the naming when reading `values.yaml`. Runs a log4j-patched image built from `acs-opensearch` (see [its README](../acs-opensearch/README.md)). | +| **Airflow** | `openmetadata-dependencies` → `airflow` | Runs OpenMetadata's ingestion pipelines (metadata/profiler/lineage extraction jobs) on a schedule. OpenMetadata talks to it over its "pipeline service client" API rather than the user interacting with Airflow directly. | +| **Ingestion image** (`acs-openmetadata/Dockerfile`) | `{{registry}}/openmetadata-ingestion:1.13.3-patched` | The image Airflow's workers actually run. Built here from upstream's `openmetadata/ingestion` image with two fixes baked in - see [Troubleshooting](#troubleshooting). | + +An already-deployed ACS PostgreSQL database was **not** reused for +OpenMetadata's own storage. ACS's shared Postgres is only reachable via +Kerberos authentication, and the upstream OpenMetadata/Airflow charts only +know how to authenticate to a database with a username and password - there's +no way to plug Kerberos into them. Rather than fork the charts to add +Kerberos support, a separate MySQL instance is deployed via +`openmetadata-dependencies` for OpenMetadata's exclusive use, authenticated +with a generated username/password instead. + +## Installation + +The OpenMetadata Helm repo was added: + +```sh +helm repo add open-metadata https://helm.open-metadata.org/ +``` + +The two charts were then added as dependencies in `/deploy/Chart.yaml`: + +```yaml +- name: openmetadata + version: 1.13.3 + repository: https://helm.open-metadata.org/ + condition: openmetadata.enabled +- name: openmetadata-dependencies + version: 1.13.3 + repository: https://helm.open-metadata.org/ + condition: openmetadata-dependencies.enabled +``` + +Running `helm dependency update` from `/deploy` pulls both charts down as +`.tgz` archives into `/deploy/charts`, where they're picked up automatically +by the parent ACS chart. Both are gated by `enabled` flags in `values.yaml` +so OpenMetadata can be switched off entirely for deployments that don't want +it. + +## Secrets + +The upstream charts expect a handful of passwords (MySQL root/user, the +separate Airflow MySQL user, Airflow's web UI admin user, OpenSearch's admin +user) to already exist as Kubernetes Secrets rather than generating them +internally. `/deploy/templates/openmetadata/openmetadata-secrets.yaml` +generates all of these with `randAlphaNum`/`randAlpha`/`randNumeric` the +first time the chart is installed, and is written so that re-running +`helm upgrade` never regenerates or overwrites them: + +- Every secret is guarded with + `{{- if not (lookup "v1" "Secret" .Release.Namespace "") }}` - if the + secret already exists in the cluster, the block is skipped and the + existing value (and therefore the existing password) survives the upgrade. +- Each generated secret carries `helm.sh/resource-policy: keep`, so `helm +uninstall` doesn't delete it either. Passwords only disappear if the + namespace itself is deleted. +- The Airflow MySQL password is a special case: it has to end up in _two_ + places that must agree - the password the MySQL `initdbScripts` actually + sets for `airflow_user`, and the SQLAlchemy connection string in + `airflow-metadata-db` that the Airflow chart uses to reach that same user. + There's no per-field secret reference for just the password on the Airflow + side, only a full pre-built connection string, so the template reads the + password back out of the first secret (if it already exists) and reuses it + when building the second, rather than generating it twice and having the + two drift apart. +- The OpenSearch/`elasticsearch-secrets` password is built from + `randAlpha`/`randNumeric` plus a fixed trailing symbol, because OpenSearch's + admin password policy requires upper+lower+digit+symbol and Sprig has no + single "random alphanumeric-with-symbols" helper. + +`values.yaml` then wires these secrets into the charts via each chart's own +`secretRef`/`existingSecret` contract (`openmetadata.openmetadata.config.*.password.secretRef`, +`openmetadata-dependencies.mysql.auth.existingSecret`, Airflow's +`metadataSecretName`, `createUserJob.defaultUser.password`, etc.). + +## Troubleshooting + +These are the encountered problems that were worked around by configuring `values.yaml` and `Dockerfile`s. + +### Fix: large / fragile pip installation + +By default the `openmetadata-dependencies` Airflow chart installs +`apache-airflow-providers-fab==2.4.4` at **container start** via the +`_PIP_ADDITIONAL_REQUIREMENTS` env var (a workaround for an Airflow 3 + MySQL +`CREATE INDEX IF NOT EXISTS` incompatibility). Doing this at runtime is slow +(a multi-minute pip resolve/install on every pod start) and fragile (any +transient PyPI/network hiccup fails the pod). Instead, `acs-openmetadata/Dockerfile` +pre-installs the same package at **build time**: + +```dockerfile +FROM openmetadata/ingestion:1.13.3 +RUN pip install --no-cache-dir "apache-airflow-providers-fab==2.4.4" +``` + +and `deploy/values.yaml` disables the runtime install so it doesn't happen +twice: + +```yaml +openmetadata-dependencies: + airflow: + env: + - name: _PIP_ADDITIONAL_REQUIREMENTS + value: "" # Empty string disables the fragile runtime pip installation step +``` + +### Fix: `log4j-core` vulnerability + +The upstream ingestion image bundles PySpark, which in turn bundles its own +older copy of `log4j-core`/`log4j-api` under its `jars/` directory - affected +by known log4j2 CVEs. The Dockerfile removes those jars and drops in a +patched 2.x release instead: + +```dockerfile +ARG LOG4J_VERSION=2.26.1 +RUN PYSPARK_JARS="$(python -c 'import os, pyspark; print(os.path.join(os.path.dirname(pyspark.__file__), "jars"))')" \ + && rm -f "${PYSPARK_JARS}"/log4j-core-*.jar "${PYSPARK_JARS}"/log4j-api-*.jar \ + && python -c "...urlretrieve(.../log4j-core-${LOG4J_VERSION}.jar...)" \ + && python -c "...urlretrieve(.../log4j-api-${LOG4J_VERSION}.jar...)" +``` + +log4j2's public API is stable across patch versions, so this is a safe +drop-in swap that doesn't require touching PySpark itself or knowing whether +the Spark profiling engine is actually exercised. + +This is a separate fix from the OpenMetadata **server** image's own bundled +log4j (`/opt/openmetadata/libs`) - that's a different upstream image +(`docker.getcollate.io/openmetadata/server`, not `openmetadata/ingestion`), +patched independently. See +[acs-openmetadata-server/README.md](../acs-openmetadata-server/README.md). + +### Building and pushing the patched image + +Requires the [`crane`](https://github.com/google/go-containerregistry) CLI on +`PATH` in addition to `docker buildx` - see below. + +```sh +cd acs-openmetadata +make build +``` + +`make build` (via `mk/acs.docker.mk`) runs +`docker buildx build --push --platform linux/amd64 -t /openmetadata-ingestion:1.13.3-patched .`, +then flattens the pushed image with `crane flatten`. `rm`-ing a file in a +Dockerfile only hides it behind a whiteout - the bytes are still present in +the upstream base image's layer underneath, which file-level vulnerability +scanners that walk a node's disk (rather than asking a registry to resolve +the image) will still flag. Flattening merges the layers into one via the +registry API so the removed jar is actually gone, not just masked; `crane` +preserves the image's config (`ENV`/`ENTRYPOINT`/`CMD`/`USER`/etc.) +unchanged - only the filesystem layers are affected. This is opted into via +`flatten=1` in this directory's `Makefile` and is a no-op for every other +ACS service's `make build`, since `mk/acs.docker.mk` only runs it when +`flatten` is set. + +The `version` in the `Makefile` is pinned to `1.13.3` on purpose (not the +usual `?=` override) - it tracks the upstream `openmetadata/ingestion` +version this Dockerfile patches, not ACS's own release version, so it must +not follow `config.mk`'s `version=` override for ACS's own services. The +resulting tag is what `deploy/values.yaml` references: + +```yaml +openmetadata-dependencies: + airflow: + images: + airflow: + repository: /openmetadata-ingestion + tag: 1.13.3-patched +``` + +### Fix: init DB scripts + +The `openmetadata-dependencies` chart's own default `values.yaml` ships two +init scripts under `mysql.initdbScripts` +(`init_openmetadata_db_scripts.sql`, `init_airflow_db_scripts.sql`) with +hard-coded passwords baked in. Helm deep-merges map values rather than +replacing them, so simply adding our own `initdbScripts` entry on top left +the upstream scripts in place too, and MySQL ran _all_ of them on first +boot - creating users with passwords that didn't match the ones in our +generated secrets. `values.yaml` explicitly `null`s out both upstream keys +(Helm's "delete this key on merge" syntax) and replaces the Airflow one with +a `.sh` script (not `.sql`), so the container's own env vars +(`MYSQL_ROOT_PASSWORD`, set by the chart from `auth.existingSecret`, and +`AIRFLOW_MYSQL_PASSWORD`, injected via `primary.extraEnvVars`) can be +expanded into the script at runtime instead of a password being hard-coded +in the template: + +```yaml +mysql: + initdbScripts: + init_openmetadata_db_scripts.sql: null + init_airflow_db_scripts.sql: null + init_airflow_db_scripts.sh: | + #!/bin/bash + set -e + mysql -uroot -p"${MYSQL_ROOT_PASSWORD}" <<-EOSQL + CREATE DATABASE IF NOT EXISTS airflow_db ... + CREATE USER IF NOT EXISTS 'airflow_user'@'%' IDENTIFIED BY '${AIRFLOW_MYSQL_PASSWORD}'; + GRANT ALL PRIVILEGES ON airflow_db.* TO 'airflow_user'@'%'; + EOSQL +``` + +The `openmetadata_db` database/user don't need a custom script at all - +`mysql.auth.database`/`mysql.auth.username` (pointed at the same +`openmetadata-mysql-secrets` used everywhere else) make the Bitnami chart +create them itself. + +### Fix: `ReadWriteMany` not available + +Airflow's chart defaults to `CeleryExecutor`, which needs its DAGs and logs +directories mounted `ReadWriteMany` so the scheduler, webserver, triggerer +and every worker pod can all read/write them concurrently. The storage +classes available to ACS deployments don't support `ReadWriteMany`. Since +OpenMetadata only needs Airflow to run its own scheduled ingestion DAGs +(not arbitrary user workloads at scale), `values.yaml` switches to +`LocalExecutor` and scales workers to zero instead of trying to make RWX +work: + +```yaml +openmetadata-dependencies: + airflow: + executor: "LocalExecutor" + workers: + replicas: 0 + airflow: + config: + AIRFLOW__CORE__EXECUTOR: "LocalExecutor" +``` + +With `LocalExecutor`, only the scheduler pod ever touches the DAGs/logs +volumes, so a plain `ReadWriteOnce` PVC is sufficient. + +### `openmetadata-logs.pvc.yaml` + +Because the executor change above means DAGs/logs only need `ReadWriteOnce`, +`deploy/templates/openmetadata/openmetadata-logs.pvc.yaml` defines plain +PVCs directly rather than relying on the Airflow chart's own +persistence-template defaults, and `values.yaml` points the chart at them +via `existingClaim`: + +```yaml +logs: + persistence: + enabled: true + existingClaim: "manual-airflow-logs" +dags: + persistence: + enabled: true + existingClaim: "manual-airflow-dags" +``` diff --git a/acs-opensearch/Dockerfile b/acs-opensearch/Dockerfile new file mode 100644 index 000000000..6fbd53afc --- /dev/null +++ b/acs-opensearch/Dockerfile @@ -0,0 +1,25 @@ +FROM opensearchproject/opensearch:3.8.0 + +# The upstream image bundles log4j-core/log4j-api 2.25.4. Swap in a patched +# 2.x release - log4j2's public API is stable across patch versions, so +# this is a safe drop-in without needing to touch OpenSearch itself. ADD's +# remote-URL support is used instead of curl/wget since the AL2023 base +# image doesn't ship either. +ARG LOG4J_VERSION=2.26.1 + +USER root +RUN rm -f /usr/share/opensearch/lib/log4j-core-*.jar /usr/share/opensearch/lib/log4j-api-*.jar +ADD --chown=opensearch:opensearch https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/${LOG4J_VERSION}/log4j-core-${LOG4J_VERSION}.jar /usr/share/opensearch/lib/log4j-core-${LOG4J_VERSION}.jar +ADD --chown=opensearch:opensearch https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/${LOG4J_VERSION}/log4j-api-${LOG4J_VERSION}.jar /usr/share/opensearch/lib/log4j-api-${LOG4J_VERSION}.jar + +# The Performance Analyzer agent runs as a separate JVM and loads its own +# classpath from performance-analyzer-rca/lib (see +# OPENSEARCH_ADDITIONAL_CLASSPATH_DIRECTORIES in performance-analyzer-agent-cli), +# independent of the main /usr/share/opensearch/lib classpath patched above. +# It bundles its own log4j-core/log4j-api, so it needs patching separately - +# otherwise it stays on the vulnerable version no matter how many times the +# image above is rebuilt. +RUN rm -f /usr/share/opensearch/performance-analyzer-rca/lib/log4j-core-*.jar /usr/share/opensearch/performance-analyzer-rca/lib/log4j-api-*.jar +ADD --chown=opensearch:opensearch https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/${LOG4J_VERSION}/log4j-core-${LOG4J_VERSION}.jar /usr/share/opensearch/performance-analyzer-rca/lib/log4j-core-${LOG4J_VERSION}.jar +ADD --chown=opensearch:opensearch https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/${LOG4J_VERSION}/log4j-api-${LOG4J_VERSION}.jar /usr/share/opensearch/performance-analyzer-rca/lib/log4j-api-${LOG4J_VERSION}.jar +USER opensearch \ No newline at end of file diff --git a/acs-opensearch/Makefile b/acs-opensearch/Makefile new file mode 100644 index 000000000..79dccc4a9 --- /dev/null +++ b/acs-opensearch/Makefile @@ -0,0 +1,16 @@ +top=.. +include ${top}/mk/acs.init.mk + +repo?=opensearch +# Hard-set (not ?=): this tracks the upstream opensearchproject/opensearch +# version this Dockerfile patches, not ACS's own release version, so it +# must not follow config.mk's version= override for ACS's own services. +version=3.8.0 +suffix?=-patched +# Flatten after push: the Dockerfile removes the upstream log4j jars, but a +# plain layered image only whiteouts them - the old bytes are still on disk +# in the base image's layer. Flattening collapses that away. See +# mk/acs.docker.mk and README.md. +flatten=1 + +include ${mk}/acs.docker.mk diff --git a/acs-opensearch/README.md b/acs-opensearch/README.md new file mode 100644 index 000000000..d6b2ddee1 --- /dev/null +++ b/acs-opensearch/README.md @@ -0,0 +1,99 @@ +# OpenSearch image (log4j patch) + +OpenSearch is deployed as part of the `openmetadata-dependencies` Helm chart +(see `/deploy/values.yaml`'s `openmetadata-dependencies.opensearch` block and +[`acs-openmetadata/README.md`](../acs-openmetadata/README.md) for the wider +OpenMetadata deployment). It isn't built by ACS - the chart just pulls the +upstream `opensearchproject/opensearch` image directly. + +That upstream image bundles two independent, older copies of +`log4j-core`/`log4j-api` - affected by known log4j2 CVEs - and this +directory's `Dockerfile` patches both, the same way +`acs-openmetadata/Dockerfile` patches PySpark's bundled log4j for the +ingestion image: + +- The main OpenSearch classpath under `/usr/share/opensearch/lib` (2.25.4). +- The Performance Analyzer agent's own classpath under + `/usr/share/opensearch/performance-analyzer-rca/lib`. Performance Analyzer + runs as a separate JVM process and loads its dependencies from + `performance-analyzer-rca/lib` (set via + `OPENSEARCH_ADDITIONAL_CLASSPATH_DIRECTORIES` in the upstream + `performance-analyzer-agent-cli` script) rather than the main OpenSearch + classpath, so it bundles its own separate log4j copy that patching + `/usr/share/opensearch/lib` alone doesn't touch. + +There is no newer upstream OpenSearch release (3.8.0 is current at time of +writing) that bundles a fixed log4j in either location, so both are patched +in directly: + +```dockerfile +FROM opensearchproject/opensearch:3.8.0 + +ARG LOG4J_VERSION=2.26.1 + +USER root +RUN rm -f /usr/share/opensearch/lib/log4j-core-*.jar /usr/share/opensearch/lib/log4j-api-*.jar +ADD --chown=opensearch:opensearch https://.../log4j-core-${LOG4J_VERSION}.jar /usr/share/opensearch/lib/log4j-core-${LOG4J_VERSION}.jar +ADD --chown=opensearch:opensearch https://.../log4j-api-${LOG4J_VERSION}.jar /usr/share/opensearch/lib/log4j-api-${LOG4J_VERSION}.jar + +RUN rm -f /usr/share/opensearch/performance-analyzer-rca/lib/log4j-core-*.jar /usr/share/opensearch/performance-analyzer-rca/lib/log4j-api-*.jar +ADD --chown=opensearch:opensearch https://.../log4j-core-${LOG4J_VERSION}.jar /usr/share/opensearch/performance-analyzer-rca/lib/log4j-core-${LOG4J_VERSION}.jar +ADD --chown=opensearch:opensearch https://.../log4j-api-${LOG4J_VERSION}.jar /usr/share/opensearch/performance-analyzer-rca/lib/log4j-api-${LOG4J_VERSION}.jar +USER opensearch +``` + +log4j2's public API is stable across patch versions, so this is a safe +drop-in swap that doesn't require touching OpenSearch itself. `LOG4J_VERSION` +is kept in sync with the version pinned in `acs-openmetadata/Dockerfile` - +there's no reason to track two different patched log4j versions across the +two images. + +`ADD`'s remote-URL support is used to fetch the jars (rather than `curl`/ +`wget`) because the upstream image is built on Amazon Linux 2023 and doesn't +ship either tool. `USER root`/`USER opensearch` bracket the patch because the +upstream image already drops privileges to the `opensearch` user, and that +user doesn't own `/usr/share/opensearch/lib` by default. + +## Building and pushing the patched image + +Requires the [`crane`](https://github.com/google/go-containerregistry) CLI on +`PATH` in addition to `docker buildx` - see below. + +```sh +cd acs-opensearch +make build +``` + +`make build` (via `mk/acs.docker.mk`) runs +`docker buildx build --push --platform linux/amd64 -t /opensearch:3.8.0-patched .`, +then flattens the pushed image with +`crane flatten -t /opensearch:3.8.0-patched /opensearch:3.8.0-patched`. +`rm`-ing a file in a Dockerfile only hides it behind a whiteout - the bytes +are still present in the upstream base image's layer underneath, which +file-level vulnerability scanners that walk a node's disk (rather than +asking a registry to resolve the image) will still flag. Flattening merges +the layers into one via the registry API so the removed jars are actually +gone, not just masked; `crane` preserves the image's config +(`ENV`/`ENTRYPOINT`/`CMD`/`USER`/etc.) unchanged - only the filesystem layers +are affected. This is opted into via `flatten=1` in this directory's +`Makefile` and is a no-op for every other ACS service's `make build`, since +`mk/acs.docker.mk` only runs it when `flatten` is set. + +The `version` in the `Makefile` is pinned to `3.8.0` on purpose (not the +usual `?=` override) - it tracks the upstream `opensearchproject/opensearch` +version this Dockerfile patches, not ACS's own release version. The +resulting tag is what `deploy/values.yaml` references: + +```yaml +openmetadata-dependencies: + opensearch: + image: + repository: opensearch + tag: "3.8.0-patched" +``` + +When upstream OpenSearch ships a new minor/patch release, bump the `FROM` +tag here, rebuild, and update the tag in `values.yaml` to match. If a future +upstream release already bundles a fixed log4j, this Dockerfile (and the +custom image) can be dropped and `values.yaml` pointed back at the stock +`opensearchproject/opensearch` image. diff --git a/acs-service-setup/dumps/openmetadata.yaml b/acs-service-setup/dumps/openmetadata.yaml new file mode 100644 index 000000000..e44ad80d8 --- /dev/null +++ b/acs-service-setup/dumps/openmetadata.yaml @@ -0,0 +1,27 @@ +#-REQUIRE: auth +service: !u UUIDs.Service.ConfigDB +version: 2 +objects: + !u UUIDs.Class.Service: + !u Openmetadata.Service.Openmetadata: + name: "Openmetadata service" + + !u Auth.Class.ServiceRole: + !u Openmetadata.Requirement.ServiceRole: + name: "Openmetadata service role" + subclassOf: + - !u Auth.Class.CentralService + + !u Auth.Class.ServicePerms: + !u Openmetadata.Perm.All: + name: "Openmetadata permission" + subclassOf: + - !u Auth.Class.Permission + +# --- +# service: !u UUIDs.Service.Authentication +# version: 2 +# grants: +# !u Openmetadata.Requirement.ServiceRole: +# !u UUIDs.Permission.Directory.AdvertiseService: +# !u Openmetadata.Service.Openmetadata: false diff --git a/acs-service-setup/lib/uuids.js b/acs-service-setup/lib/uuids.js index 37daaaadb..a2a049f68 100644 --- a/acs-service-setup/lib/uuids.js +++ b/acs-service-setup/lib/uuids.js @@ -559,3 +559,14 @@ export const UNS = { }, }; +export const Openmetadata = { + Requirement: { + ServiceRole: "449166c8-68eb-4381-bd98-af20ed12c5f2", + }, + Service: { + Openmetadata: "c4660c38-0e10-444a-b5f2-bdab1e417a1e", + }, + Perm: { + All: "aed3139d-ec43-47bf-bd73-586308fa88a7", + }, +}; \ No newline at end of file diff --git a/deploy/Chart.lock b/deploy/Chart.lock index 9042b9c7f..b1ae05a8e 100644 --- a/deploy/Chart.lock +++ b/deploy/Chart.lock @@ -11,5 +11,11 @@ dependencies: - name: cert-manager repository: https://charts.jetstack.io version: v1.14.4 -digest: sha256:0c6fd4a51ef2b5a1ce99e72ade9a967d91793fa52d7f1969ef6cb7e5fbb4e143 -generated: "2026-04-13T11:01:47.894885542+01:00" +- name: openmetadata + repository: https://helm.open-metadata.org/ + version: 1.13.3 +- name: openmetadata-dependencies + repository: https://helm.open-metadata.org/ + version: 1.13.3 +digest: sha256:04c0c0923d51e2c6656cdd8fdc4f561e25001db34c8dbc2147728604bc35e5a7 +generated: "2026-08-03T10:59:27.447561316+01:00" diff --git a/deploy/Chart.yaml b/deploy/Chart.yaml index 22af73c66..2eb2e0095 100644 --- a/deploy/Chart.yaml +++ b/deploy/Chart.yaml @@ -32,3 +32,11 @@ dependencies: repository: https://charts.jetstack.io version: v1.14.4 condition: acs.letsEncrypt.enabled + - name: openmetadata + version: 1.13.3 + repository: https://helm.open-metadata.org/ + condition: openmetadata.enabled + - name: openmetadata-dependencies + version: 1.13.3 + repository: https://helm.open-metadata.org/ + condition: openmetadata-dependencies.enabled diff --git a/deploy/templates/lets-encrypt/tls.yaml b/deploy/templates/lets-encrypt/tls.yaml index 8b030c5a8..780662c69 100644 --- a/deploy/templates/lets-encrypt/tls.yaml +++ b/deploy/templates/lets-encrypt/tls.yaml @@ -54,6 +54,9 @@ spec: {{- if .Values.dataAccess.enabled }} - data-access.{{ .Values.acs.baseUrl}} {{- end }} + {{- if .Values.openmetadata.enabled }} + - openmetadata.{{ .Values.acs.baseUrl}} + {{- end }} {{- range .Values.acs.letsEncrypt.additionalDnsNames }} - {{ . }} {{- end }} diff --git a/deploy/templates/openmetadata/ingress.yaml b/deploy/templates/openmetadata/ingress.yaml new file mode 100644 index 000000000..15bb53d81 --- /dev/null +++ b/deploy/templates/openmetadata/ingress.yaml @@ -0,0 +1,23 @@ +{{ if .Values.openmetadata.enabled }} +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: openmetadata-ingressroute + namespace: {{ .Release.Namespace }} +spec: + entryPoints: + - {{ .Values.acs.secure | ternary "websecure" "web" }} + routes: + - match: Host(`openmetadata.{{.Values.acs.baseUrl | required "values.acs.baseUrl is required"}}`) + kind: Rule + services: + - name: openmetadata + port: 8585 + namespace: {{ .Release.Namespace }} + {{- if .Values.acs.secure }} + tls: + secretName: {{ coalesce .Values.openmetadata.tlsSecretName .Values.acs.tlsSecretName }} + domains: + - main: openmetadata.{{.Values.acs.baseUrl | required "values.acs.baseUrl is required"}} + {{- end -}} +{{- end -}} \ No newline at end of file diff --git a/deploy/templates/openmetadata/openmetadata-logs.pvc.yaml b/deploy/templates/openmetadata/openmetadata-logs.pvc.yaml new file mode 100644 index 000000000..3657b5f95 --- /dev/null +++ b/deploy/templates/openmetadata/openmetadata-logs.pvc.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: manual-airflow-dags +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: acs-dags +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: manual-airflow-logs +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi \ No newline at end of file diff --git a/deploy/templates/openmetadata/openmetadata-secrets.yaml b/deploy/templates/openmetadata/openmetadata-secrets.yaml new file mode 100644 index 000000000..60fab99c2 --- /dev/null +++ b/deploy/templates/openmetadata/openmetadata-secrets.yaml @@ -0,0 +1,102 @@ +{{ if .Values.openmetadata.enabled }} +{{- if not (lookup "v1" "Secret" .Release.Namespace "openmetadata-mysql-secrets") }} + +apiVersion: v1 +kind: Secret +metadata: + name: "openmetadata-mysql-secrets" + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/resource-policy": "keep" +type: Opaque +data: + # Key names are fixed by the bitnami mysql chart's auth.existingSecret contract. + mysql-root-password: {{ randAlphaNum 32 | b64enc | quote }} + mysql-replication-password: {{ randAlphaNum 32 | b64enc | quote }} + mysql-password: {{ randAlphaNum 32 | b64enc | quote }} + +{{- end }} +--- +{{- /* + airflow-mysql-password needs to end up in two places that must always + agree: the password the mysql initdb script actually sets for + airflow_user (via airflow-mysql-secrets, read by mysql.primary.extraEnvVars), + and the connection string the apache/airflow chart uses to reach that + same user (data.metadataSecretName below - the chart has no per-field + secretRef for just the password, only a full pre-built connection + string). So this value is computed once here and reused for both, + instead of two independent randAlphaNum calls that would drift apart. + Read back the existing value across upgrades rather than the usual + "skip the whole block if lookup finds it" pattern, since the second + secret needs the value even when the first secret already exists. +*/}} +{{- $airflowMysqlSecret := lookup "v1" "Secret" .Release.Namespace "airflow-mysql-secrets" }} +{{- $airflowMysqlPassword := randAlphaNum 32 }} +{{- if $airflowMysqlSecret }} +{{- $airflowMysqlPassword = index $airflowMysqlSecret.data "airflow-mysql-password" | b64dec }} +{{- end }} +{{- if not $airflowMysqlSecret }} + +apiVersion: v1 +kind: Secret +metadata: + name: "airflow-mysql-secrets" + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/resource-policy": "keep" +type: Opaque +data: + airflow-mysql-password: {{ $airflowMysqlPassword | b64enc | quote }} + +{{- end }} +--- +{{- if not (lookup "v1" "Secret" .Release.Namespace "airflow-metadata-db") }} + +apiVersion: v1 +kind: Secret +metadata: + name: "airflow-metadata-db" + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/resource-policy": "keep" +type: Opaque +data: + # Key name and format ("connection" holding a full SQLAlchemy URI) are + # fixed by the apache/airflow chart's data.metadataSecretName contract. + connection: {{ printf "mysql://airflow_user:%s@mysql:3306/airflow_db" $airflowMysqlPassword | b64enc | quote }} + +{{- end }} +--- +{{- if not (lookup "v1" "Secret" .Release.Namespace "airflow-secrets") }} + +apiVersion: v1 +kind: Secret +metadata: + name: "airflow-secrets" + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/resource-policy": "keep" +type: Opaque +data: + openmetadata-airflow-password: {{ randAlphaNum 32 | b64enc | quote }} + +{{- end }} +--- +{{- if not (lookup "v1" "Secret" .Release.Namespace "elasticsearch-secrets") }} + +apiVersion: v1 +kind: Secret +metadata: + name: "elasticsearch-secrets" + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/resource-policy": "keep" +type: Opaque +data: + # OpenSearch's admin password policy requires upper/lower/digit/symbol. + # Sprig has no combined alphanum+symbols random function, so build one + # from randAlpha/randNumeric and append a fixed symbol to guarantee it. + openmetadata-elasticsearch-password: {{ printf "%s%s!" (randAlpha 24) (randNumeric 7) | b64enc | quote }} + +{{- end }} +{{- end -}} diff --git a/deploy/templates/service-setup.yaml b/deploy/templates/service-setup.yaml index 756020f3e..e22696ceb 100644 --- a/deploy/templates/service-setup.yaml +++ b/deploy/templates/service-setup.yaml @@ -269,4 +269,8 @@ data: - service: !u UUIDs.Service.i3x url: !acs "http://i3x.${namespace}.svc.${k8sdomain}" {{ end }} +{{ if .Values.openmetadata.enabled }} + - service: !u Openmetadata.Service.Openmetadata + url: !acs "http${secure}://openmetadata.${domain}" +{{ end }} {{ end }} diff --git a/deploy/values.yaml b/deploy/values.yaml index 5f0fc9a5d..7df8431c5 100644 --- a/deploy/values.yaml +++ b/deploy/values.yaml @@ -689,3 +689,156 @@ influxdb2: cert-manager: fullnameOverride: "cert-manager" + +openmetadata: + enabled: true + fullnameOverride: "openmetadata" + image: + # Patched to replace the bundled log4j-core/log4j-api 2.25.5 jars under + # /opt/openmetadata/libs - built from acs-openmetadata-server/Dockerfile. + # See acs-openmetadata-server/README.md. + repository: /openmetadata-server + tag: "1.13.3-patched" + openmetadata: + config: + elasticsearch: + auth: + enabled: true + username: "admin" + password: + secretRef: "elasticsearch-secrets" + secretKey: "openmetadata-elasticsearch-password" + database: + enabled: true + host: mysql + port: 3306 + driverClass: com.mysql.cj.jdbc.Driver + dbScheme: mysql + databaseName: openmetadata_db + auth: + username: openmetadata_user + password: + secretRef: openmetadata-mysql-secrets + secretKey: mysql-password + pipelineServiceClientConfig: + airflow: + auth: + enabled: true + username: admin + password: + secretRef: airflow-secrets + secretKey: openmetadata-airflow-password + +openmetadata-dependencies: + enabled: true + + mysql: + enabled: true + auth: + # Random root/user passwords generated once by + # deploy/templates/openmetadata/openmetadata-secrets.yaml and kept + # stable across upgrades - see the existingSecret contract in that + # file. This also creates openmetadata_user/openmetadata_db for us, + # so no custom initdb script is needed for that database. + existingSecret: openmetadata-mysql-secrets + database: openmetadata_db + username: openmetadata_user + primary: + extraEnvVars: + - name: AIRFLOW_MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: airflow-mysql-secrets + key: airflow-mysql-password + + initdbScripts: + # openmetadata-dependencies' own default values.yaml ships BOTH of + # these keys with hardcoded passwords baked in. Helm deep-merges map + # values rather than replacing them, so without explicitly nulling + # these out here, they'd survive alongside our own key below and + # mysql would try to run all of them - which is exactly what kept + # happening. null is Helm's "delete this key during merge" syntax, + # not an empty script. + init_openmetadata_db_scripts.sql: null + init_airflow_db_scripts.sql: null + # .sh (not .sql) so the container's own env vars - MYSQL_ROOT_PASSWORD + # (set by the chart from auth.existingSecret) and AIRFLOW_MYSQL_PASSWORD + # above - can be expanded into the script at runtime instead of being + # hardcoded here. + init_airflow_db_scripts.sh: | + #!/bin/bash + set -e + mysql -uroot -p"${MYSQL_ROOT_PASSWORD}" <<-EOSQL + CREATE DATABASE IF NOT EXISTS airflow_db + CHARACTER SET utf8mb4 + COLLATE utf8mb4_unicode_ci; + CREATE USER IF NOT EXISTS 'airflow_user'@'%' IDENTIFIED BY '${AIRFLOW_MYSQL_PASSWORD}'; + GRANT ALL PRIVILEGES ON airflow_db.* TO 'airflow_user'@'%'; + FLUSH PRIVILEGES; + EOSQL + + airflow: + images: + airflow: + repository: /openmetadata-ingestion + tag: 1.13.3-patched + data: + # The chart's own default (data.metadataConnection.pass: airflow_pass) + # is a hardcoded literal with no per-field secretRef - metadataSecretName + # is the only way to point it at a real password instead. Must contain + # a "connection" key with the full SQLAlchemy URI - see + # deploy/templates/openmetadata/openmetadata-secrets.yaml, which builds + # this from the same password airflow_user is actually created with. + metadataSecretName: airflow-metadata-db + env: + - name: _PIP_ADDITIONAL_REQUIREMENTS + value: "" # Empty string disables the fragile runtime pip installation step + + executor: "LocalExecutor" + workers: + replicas: 0 + airflow: + config: + AIRFLOW__CORE__EXECUTOR: "LocalExecutor" + + createUserJob: + # Reset the webserver admin user's password to the same random value + # openmetadata's pipelineServiceClientConfig.airflow.auth uses (secret + # airflow-secrets/openmetadata-airflow-password), so the two stay in + # sync without either side hardcoding it. $(OM_AIRFLOW_PASSWORD) is + # Kubernetes' own command/args env substitution, not Helm templating - + # it resolves at container start from the env var below. + env: + - name: OM_AIRFLOW_PASSWORD + valueFrom: + secretKeyRef: + name: airflow-secrets + key: openmetadata-airflow-password + defaultUser: + password: "$(OM_AIRFLOW_PASSWORD)" + + logs: + persistence: + enabled: true + existingClaim: "manual-airflow-logs" + dags: + persistence: + enabled: true + existingClaim: "manual-airflow-dags" + + opensearch: + image: + # Patched to replace the bundled log4j-core/log4j-api 2.25.4 jars - + # built from acs-opensearch/Dockerfile. See acs-openmetadata/README.md + # for the equivalent log4j fix applied to the ingestion image. + repository: /opensearch + tag: "3.8.0-patched" + persistence: + accessModes: + - ReadWriteOnce + extraEnvs: + - name: OPENSEARCH_INITIAL_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: elasticsearch-secrets + key: openmetadata-elasticsearch-password diff --git a/mk/acs.docker.mk b/mk/acs.docker.mk index fd229ac28..b4c3d1b25 100644 --- a/mk/acs.docker.mk +++ b/mk/acs.docker.mk @@ -22,6 +22,16 @@ all: build build: git.prepare docker buildx build --push --platform "${platform}" -t "${image}" ${build_args} . +ifdef flatten +# `flatten` opts a service out of Docker's normal layer sharing: it +# collapses the just-pushed image into a single layer via the registry +# API, so a `rm` earlier in the Dockerfile actually removes the file's +# bytes rather than just hiding them behind a whiteout. `crane` preserves +# the image config (ENV/ENTRYPOINT/CMD/USER/etc.) unchanged - only the +# filesystem layers are merged. Unset (the default) for every service that +# doesn't need this. + crane flatten -t "${image}" "${image}" +endif pull: docker pull "${image}"