From 7e522fb389b52a6e076caab56c6e06ae4db20a17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Florian=20M=C3=BCller?= <florian@tomueller.de>
Date: Thu, 9 Jul 2026 15:24:22 +0200
Subject: [PATCH 1/4] feat: add openlineage to spark image

---
 CHANGELOG.md               |  2 ++
 spark-k8s/Dockerfile.4     | 11 +++++++++++
 spark-k8s/boil-config.toml |  2 ++
 3 files changed, 15 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2999a4f99..e10f47a2b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,6 +24,7 @@ All notable changes to this project will be documented in this file.
 - druid: Add `37.0.0` ([#1535]).
 - hbase: Add `2.6.6` ([#1547]).
 - spark: Add `4.1.2` ([#1550]).
+- spark: Bake the OpenLineage Spark listener (`openlineage-spark_2.13` `1.51.0`) into the `4.1.x` images under the dedicated directory `/stackable/spark/openlineage/` (off the `extra-jars` classpath), enabling the OpenLineage MVP in the operator ([#XXXX]).
 - opensearch: Add `3.6.0` ([#1549]).
 - opensearch-dashboards: Add `3.6.0` ([#1551]).
 
@@ -95,6 +96,7 @@ All notable changes to this project will be documented in this file.
 [#1548]: https://github.com/stackabletech/docker-images/pull/1548
 [#1549]: https://github.com/stackabletech/docker-images/pull/1549
 [#1550]: https://github.com/stackabletech/docker-images/pull/1550
+[#XXXX]: https://github.com/stackabletech/docker-images/pull/XXXX
 [#1551]: https://github.com/stackabletech/docker-images/pull/1551
 [#1559]: https://github.com/stackabletech/docker-images/pull/1559
 [#1561]: https://github.com/stackabletech/docker-images/pull/1561
diff --git a/spark-k8s/Dockerfile.4 b/spark-k8s/Dockerfile.4
index fff71258e..c8602ac8f 100644
--- a/spark-k8s/Dockerfile.4
+++ b/spark-k8s/Dockerfile.4
@@ -39,6 +39,7 @@ ENV HADOOP_VERSION=${HADOOP_HADOOP_VERSION}
 ARG JACKSON_DATAFORMAT_XML_VERSION
 ARG STAX2_API_VERSION
 ARG WOODSTOX_CORE_VERSION
+ARG OPENLINEAGE_SPARK_VERSION
 ARG JMX_EXPORTER_VERSION
 ARG TARGETARCH
 ARG TINI_VERSION
@@ -122,6 +123,16 @@ mvn dependency:get -Dartifact=mvn dependency:get -Dartifact=com.fasterxml.woodst
 cp /root/.m2/repository/com/fasterxml/woodstox/woodstox-core/${WOODSTOX_CORE_VERSION}/woodstox-core-${WOODSTOX_CORE_VERSION}.jar \
   /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/extra-jars/woodstox-core-${WOODSTOX_CORE_VERSION}.jar
 
+# Download the OpenLineage Spark listener jar into a DEDICATED directory (dist/openlineage),
+# NOT dist/extra-jars. The operator references it via `--jars local:///stackable/spark/openlineage/...`
+# so it loads on Spark's child/user classloader, sharing it with connectors delivered via `--packages`
+# (e.g. Iceberg). If it landed on extra-jars it would be added to the system classloader via
+# extraClassPath, OpenLineage's connector probe would fail, and no datasets would be emitted.
+mkdir -p /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage
+mvn dependency:get -Dartifact=io.openlineage:openlineage-spark_2.13:${OPENLINEAGE_SPARK_VERSION}
+cp /root/.m2/repository/io/openlineage/openlineage-spark_2.13/${OPENLINEAGE_SPARK_VERSION}/openlineage-spark_2.13-${OPENLINEAGE_SPARK_VERSION}.jar \
+  /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage/openlineage-spark_2.13-${OPENLINEAGE_SPARK_VERSION}.jar
+
 # Get the correct `tini` binary for our architecture.
 curl --fail "https://repo.stackable.tech/repository/packages/tini/tini-${TINI_VERSION}-${TARGETARCH}" \
   -o /usr/bin/tini
diff --git a/spark-k8s/boil-config.toml b/spark-k8s/boil-config.toml
index 724c436d9..b8559480c 100644
--- a/spark-k8s/boil-config.toml
+++ b/spark-k8s/boil-config.toml
@@ -37,6 +37,7 @@ stax2-api-version = "4.2.2"               # Needs to match https://mvnrepository
 woodstox-core-version = "7.1.1"           # Needs to match https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.20.0
 jmx-exporter-version = "1.3.0"
 tini-version = "0.19.0"
+openlineage-spark-version = "1.51.0"      # Scala 2.13 build for Spark 4.1.x; see https://mvnrepository.com/artifact/io.openlineage/openlineage-spark_2.13
 
 # LTS since 26.7.0
 [versions."4.1.2"]
@@ -54,3 +55,4 @@ stax2-api-version = "4.2.2"               # Needs to match https://mvnrepository
 woodstox-core-version = "7.1.1"           # Needs to match https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.21.2
 jmx-exporter-version = "1.3.0"
 tini-version = "0.19.0"
+openlineage-spark-version = "1.51.0"      # Scala 2.13 build for Spark 4.1.x; see https://mvnrepository.com/artifact/io.openlineage/openlineage-spark_2.13

From 200aa12ccd8d8acdfb55462ec5a0161a9942faa8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Florian=20M=C3=BCller?= <florian@tomueller.de>
Date: Thu, 16 Jul 2026 15:54:44 +0200
Subject: [PATCH 2/4] feat: add openlineage for 3.5.8 image

---
 CHANGELOG.md               |  2 +-
 spark-k8s/Dockerfile.3     | 12 ++++++++++++
 spark-k8s/boil-config.toml |  1 +
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e10f47a2b..bc01f6e8a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,7 +24,7 @@ All notable changes to this project will be documented in this file.
 - druid: Add `37.0.0` ([#1535]).
 - hbase: Add `2.6.6` ([#1547]).
 - spark: Add `4.1.2` ([#1550]).
-- spark: Bake the OpenLineage Spark listener (`openlineage-spark_2.13` `1.51.0`) into the `4.1.x` images under the dedicated directory `/stackable/spark/openlineage/` (off the `extra-jars` classpath), enabling the OpenLineage MVP in the operator ([#XXXX]).
+- spark: Bake the OpenLineage Spark listener (`1.51.0`) into the Spark images under the dedicated directory `/stackable/spark/openlineage/` (off the `extra-jars` classpath), enabling the OpenLineage MVP in the operator. The Scala 2.13 build (`openlineage-spark_2.13`) is used for the `4.1.x` images and the Scala 2.12 build (`openlineage-spark_2.12`) for the `3.5.x` images ([#XXXX]).
 - opensearch: Add `3.6.0` ([#1549]).
 - opensearch-dashboards: Add `3.6.0` ([#1551]).
 
diff --git a/spark-k8s/Dockerfile.3 b/spark-k8s/Dockerfile.3
index 7c3bbdcea..fd1138430 100644
--- a/spark-k8s/Dockerfile.3
+++ b/spark-k8s/Dockerfile.3
@@ -47,6 +47,7 @@ ARG HBASE_VERSION
 ARG JACKSON_DATAFORMAT_XML_VERSION
 ARG STAX2_API_VERSION
 ARG WOODSTOX_CORE_VERSION
+ARG OPENLINEAGE_SPARK_VERSION
 ARG JMX_EXPORTER_VERSION
 ARG TARGETARCH
 ARG TINI_VERSION
@@ -148,6 +149,17 @@ mvn dependency:get -Dartifact=mvn dependency:get -Dartifact=com.fasterxml.woodst
 cp /root/.m2/repository/com/fasterxml/woodstox/woodstox-core/${WOODSTOX_CORE_VERSION}/woodstox-core-${WOODSTOX_CORE_VERSION}.jar \
   /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/extra-jars/woodstox-core-${WOODSTOX_CORE_VERSION}.jar
 
+# Download the OpenLineage Spark listener jar into a DEDICATED directory (dist/openlineage),
+# NOT dist/extra-jars. The operator references it via `--jars local:///stackable/spark/openlineage/...`
+# so it loads on Spark's child/user classloader, sharing it with connectors delivered via `--packages`
+# (e.g. Iceberg). If it landed on extra-jars it would be added to the system classloader via
+# extraClassPath, OpenLineage's connector probe would fail, and no datasets would be emitted.
+# Spark 3.5.x is a Scala 2.12 build, so the 2.12 artifact is used (Spark 4.x uses 2.13).
+mkdir -p /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage
+mvn dependency:get -Dartifact=io.openlineage:openlineage-spark_2.12:${OPENLINEAGE_SPARK_VERSION}
+cp /root/.m2/repository/io/openlineage/openlineage-spark_2.12/${OPENLINEAGE_SPARK_VERSION}/openlineage-spark_2.12-${OPENLINEAGE_SPARK_VERSION}.jar \
+  /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage/openlineage-spark_2.12-${OPENLINEAGE_SPARK_VERSION}.jar
+
 # Get the correct `tini` binary for our architecture.
 curl --fail "https://repo.stackable.tech/repository/packages/tini/tini-${TINI_VERSION}-${TARGETARCH}" \
   -o /usr/bin/tini
diff --git a/spark-k8s/boil-config.toml b/spark-k8s/boil-config.toml
index b8559480c..2e90f459e 100644
--- a/spark-k8s/boil-config.toml
+++ b/spark-k8s/boil-config.toml
@@ -20,6 +20,7 @@ woodstox-core-version = "6.5.1"           # Needs to match https://mvnrepository
 jmx-exporter-version = "1.3.0"
 tini-version = "0.19.0"
 hbase-connector-version = "1.0.1_3.5.8"
+openlineage-spark-version = "1.51.0"      # Scala 2.12 build for Spark 3.5.x; see https://mvnrepository.com/artifact/io.openlineage/openlineage-spark_2.12
 
 # Deprecated since 26.7
 [versions."4.1.1"]

From 039c6e1f562dcca71e917d2775f781b8639df2fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Florian=20M=C3=BCller?= <florian@tomueller.de>
Date: Thu, 16 Jul 2026 16:51:24 +0200
Subject: [PATCH 3/4] feat: symlinks to harmonize openlineage filenames

---
 CHANGELOG.md           | 2 +-
 spark-k8s/Dockerfile.3 | 5 +++++
 spark-k8s/Dockerfile.4 | 5 +++++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index bc01f6e8a..817277a85 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,7 +24,7 @@ All notable changes to this project will be documented in this file.
 - druid: Add `37.0.0` ([#1535]).
 - hbase: Add `2.6.6` ([#1547]).
 - spark: Add `4.1.2` ([#1550]).
-- spark: Bake the OpenLineage Spark listener (`1.51.0`) into the Spark images under the dedicated directory `/stackable/spark/openlineage/` (off the `extra-jars` classpath), enabling the OpenLineage MVP in the operator. The Scala 2.13 build (`openlineage-spark_2.13`) is used for the `4.1.x` images and the Scala 2.12 build (`openlineage-spark_2.12`) for the `3.5.x` images ([#XXXX]).
+- spark: Bake the OpenLineage Spark listener (`1.51.0`) into the Spark images under the dedicated directory `/stackable/spark/openlineage/` (off the `extra-jars` classpath), enabling the OpenLineage MVP in the operator. The Scala 2.13 build (`openlineage-spark_2.13`) is used for the `4.1.x` images and the Scala 2.12 build (`openlineage-spark_2.12`) for the `3.5.x` images, both exposed through a stable `openlineage-spark.jar` symlink so the operator need not track the version or Scala suffix ([#XXXX]).
 - opensearch: Add `3.6.0` ([#1549]).
 - opensearch-dashboards: Add `3.6.0` ([#1551]).
 
diff --git a/spark-k8s/Dockerfile.3 b/spark-k8s/Dockerfile.3
index fd1138430..344a20933 100644
--- a/spark-k8s/Dockerfile.3
+++ b/spark-k8s/Dockerfile.3
@@ -159,6 +159,11 @@ mkdir -p /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/op
 mvn dependency:get -Dartifact=io.openlineage:openlineage-spark_2.12:${OPENLINEAGE_SPARK_VERSION}
 cp /root/.m2/repository/io/openlineage/openlineage-spark_2.12/${OPENLINEAGE_SPARK_VERSION}/openlineage-spark_2.12-${OPENLINEAGE_SPARK_VERSION}.jar \
   /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage/openlineage-spark_2.12-${OPENLINEAGE_SPARK_VERSION}.jar
+# Symlink to a stable, version- and Scala-independent name (mirroring the jmx_prometheus_javaagent.jar
+# pattern) so the operator can reference the jar without knowing the version or Scala build baked here.
+# The relative target keeps the link valid after `dist` is copied into the final image.
+ln -s openlineage-spark_2.12-${OPENLINEAGE_SPARK_VERSION}.jar \
+  /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage/openlineage-spark.jar
 
 # Get the correct `tini` binary for our architecture.
 curl --fail "https://repo.stackable.tech/repository/packages/tini/tini-${TINI_VERSION}-${TARGETARCH}" \
diff --git a/spark-k8s/Dockerfile.4 b/spark-k8s/Dockerfile.4
index c8602ac8f..dae6b2046 100644
--- a/spark-k8s/Dockerfile.4
+++ b/spark-k8s/Dockerfile.4
@@ -132,6 +132,11 @@ mkdir -p /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/op
 mvn dependency:get -Dartifact=io.openlineage:openlineage-spark_2.13:${OPENLINEAGE_SPARK_VERSION}
 cp /root/.m2/repository/io/openlineage/openlineage-spark_2.13/${OPENLINEAGE_SPARK_VERSION}/openlineage-spark_2.13-${OPENLINEAGE_SPARK_VERSION}.jar \
   /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage/openlineage-spark_2.13-${OPENLINEAGE_SPARK_VERSION}.jar
+# Symlink to a stable, version- and Scala-independent name (mirroring the jmx_prometheus_javaagent.jar
+# pattern) so the operator can reference the jar without knowing the version or Scala build baked here.
+# The relative target keeps the link valid after `dist` is copied into the final image.
+ln -s openlineage-spark_2.13-${OPENLINEAGE_SPARK_VERSION}.jar \
+  /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage/openlineage-spark.jar
 
 # Get the correct `tini` binary for our architecture.
 curl --fail "https://repo.stackable.tech/repository/packages/tini/tini-${TINI_VERSION}-${TARGETARCH}" \

From d90a00f7fc47b01b2b2de29ca583c5c5014796a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Florian=20M=C3=BCller?= <florian@tomueller.de>
Date: Thu, 16 Jul 2026 17:12:03 +0200
Subject: [PATCH 4/4] Apply suggestions from code review

Co-authored-by: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>
---
 CHANGELOG.md           | 4 ++--
 spark-k8s/Dockerfile.3 | 2 --
 spark-k8s/Dockerfile.4 | 5 ++---
 3 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 817277a85..a8b21d927 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,7 +24,7 @@ All notable changes to this project will be documented in this file.
 - druid: Add `37.0.0` ([#1535]).
 - hbase: Add `2.6.6` ([#1547]).
 - spark: Add `4.1.2` ([#1550]).
-- spark: Bake the OpenLineage Spark listener (`1.51.0`) into the Spark images under the dedicated directory `/stackable/spark/openlineage/` (off the `extra-jars` classpath), enabling the OpenLineage MVP in the operator. The Scala 2.13 build (`openlineage-spark_2.13`) is used for the `4.1.x` images and the Scala 2.12 build (`openlineage-spark_2.12`) for the `3.5.x` images, both exposed through a stable `openlineage-spark.jar` symlink so the operator need not track the version or Scala suffix ([#XXXX]).
+- spark: Add the OpenLineage listener under the dedicated directory `/stackable/spark/openlineage/` ([#1577]).
 - opensearch: Add `3.6.0` ([#1549]).
 - opensearch-dashboards: Add `3.6.0` ([#1551]).
 
@@ -96,7 +96,7 @@ All notable changes to this project will be documented in this file.
 [#1548]: https://github.com/stackabletech/docker-images/pull/1548
 [#1549]: https://github.com/stackabletech/docker-images/pull/1549
 [#1550]: https://github.com/stackabletech/docker-images/pull/1550
-[#XXXX]: https://github.com/stackabletech/docker-images/pull/XXXX
+[#1577]: https://github.com/stackabletech/docker-images/pull/1577
 [#1551]: https://github.com/stackabletech/docker-images/pull/1551
 [#1559]: https://github.com/stackabletech/docker-images/pull/1559
 [#1561]: https://github.com/stackabletech/docker-images/pull/1561
diff --git a/spark-k8s/Dockerfile.3 b/spark-k8s/Dockerfile.3
index 344a20933..ff858c5c4 100644
--- a/spark-k8s/Dockerfile.3
+++ b/spark-k8s/Dockerfile.3
@@ -154,14 +154,12 @@ cp /root/.m2/repository/com/fasterxml/woodstox/woodstox-core/${WOODSTOX_CORE_VER
 # so it loads on Spark's child/user classloader, sharing it with connectors delivered via `--packages`
 # (e.g. Iceberg). If it landed on extra-jars it would be added to the system classloader via
 # extraClassPath, OpenLineage's connector probe would fail, and no datasets would be emitted.
-# Spark 3.5.x is a Scala 2.12 build, so the 2.12 artifact is used (Spark 4.x uses 2.13).
 mkdir -p /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage
 mvn dependency:get -Dartifact=io.openlineage:openlineage-spark_2.12:${OPENLINEAGE_SPARK_VERSION}
 cp /root/.m2/repository/io/openlineage/openlineage-spark_2.12/${OPENLINEAGE_SPARK_VERSION}/openlineage-spark_2.12-${OPENLINEAGE_SPARK_VERSION}.jar \
   /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage/openlineage-spark_2.12-${OPENLINEAGE_SPARK_VERSION}.jar
 # Symlink to a stable, version- and Scala-independent name (mirroring the jmx_prometheus_javaagent.jar
 # pattern) so the operator can reference the jar without knowing the version or Scala build baked here.
-# The relative target keeps the link valid after `dist` is copied into the final image.
 ln -s openlineage-spark_2.12-${OPENLINEAGE_SPARK_VERSION}.jar \
   /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage/openlineage-spark.jar
 
diff --git a/spark-k8s/Dockerfile.4 b/spark-k8s/Dockerfile.4
index dae6b2046..29f0eeded 100644
--- a/spark-k8s/Dockerfile.4
+++ b/spark-k8s/Dockerfile.4
@@ -132,9 +132,8 @@ mkdir -p /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/op
 mvn dependency:get -Dartifact=io.openlineage:openlineage-spark_2.13:${OPENLINEAGE_SPARK_VERSION}
 cp /root/.m2/repository/io/openlineage/openlineage-spark_2.13/${OPENLINEAGE_SPARK_VERSION}/openlineage-spark_2.13-${OPENLINEAGE_SPARK_VERSION}.jar \
   /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage/openlineage-spark_2.13-${OPENLINEAGE_SPARK_VERSION}.jar
-# Symlink to a stable, version- and Scala-independent name (mirroring the jmx_prometheus_javaagent.jar
-# pattern) so the operator can reference the jar without knowing the version or Scala build baked here.
-# The relative target keeps the link valid after `dist` is copied into the final image.
+# Symlink to a stable, version- and Scala-independent name
+# so the operator can reference the jar without knowing the version or Scala build baked here.
 ln -s openlineage-spark_2.13-${OPENLINEAGE_SPARK_VERSION}.jar \
   /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage/openlineage-spark.jar