diff --git a/CHANGELOG.md b/CHANGELOG.md index 961f1b29c..5811ff1e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ All notable changes to this project will be documented in this file. - druid: Add `37.0.0` ([#1535]). - hbase: Add `2.6.6` ([#1547]). - spark: Add `4.1.2` ([#1550]). +- spark: Add the OpenLineage listener under the dedicated directory `/stackable/spark/openlineage/` ([#1577]). - opensearch: Add `3.6.0` ([#1549]). - opensearch-dashboards: Add `3.6.0` ([#1551]). - spark: Add hbase-connector protobuf classes ([#1573]). @@ -110,6 +111,7 @@ All notable changes to this project will be documented in this file. [#1548]: https://github.com/stackabletech/docker-images/pull/1548 [#1549]: https://github.com/stackabletech/docker-images/pull/1549 [#1550]: https://github.com/stackabletech/docker-images/pull/1550 +[#1577]: https://github.com/stackabletech/docker-images/pull/1577 [#1551]: https://github.com/stackabletech/docker-images/pull/1551 [#1559]: https://github.com/stackabletech/docker-images/pull/1559 [#1561]: https://github.com/stackabletech/docker-images/pull/1561 diff --git a/spark-k8s/Dockerfile.3 b/spark-k8s/Dockerfile.3 index 7c3bbdcea..ff858c5c4 100644 --- a/spark-k8s/Dockerfile.3 +++ b/spark-k8s/Dockerfile.3 @@ -47,6 +47,7 @@ ARG HBASE_VERSION ARG JACKSON_DATAFORMAT_XML_VERSION ARG STAX2_API_VERSION ARG WOODSTOX_CORE_VERSION +ARG OPENLINEAGE_SPARK_VERSION ARG JMX_EXPORTER_VERSION ARG TARGETARCH ARG TINI_VERSION @@ -148,6 +149,20 @@ mvn dependency:get -Dartifact=mvn dependency:get -Dartifact=com.fasterxml.woodst cp /root/.m2/repository/com/fasterxml/woodstox/woodstox-core/${WOODSTOX_CORE_VERSION}/woodstox-core-${WOODSTOX_CORE_VERSION}.jar \ /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/extra-jars/woodstox-core-${WOODSTOX_CORE_VERSION}.jar +# Download the OpenLineage Spark listener jar into a DEDICATED directory (dist/openlineage), +# NOT dist/extra-jars. The operator references it via `--jars local:///stackable/spark/openlineage/...` +# so it loads on Spark's child/user classloader, sharing it with connectors delivered via `--packages` +# (e.g. Iceberg). If it landed on extra-jars it would be added to the system classloader via +# extraClassPath, OpenLineage's connector probe would fail, and no datasets would be emitted. +mkdir -p /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage +mvn dependency:get -Dartifact=io.openlineage:openlineage-spark_2.12:${OPENLINEAGE_SPARK_VERSION} +cp /root/.m2/repository/io/openlineage/openlineage-spark_2.12/${OPENLINEAGE_SPARK_VERSION}/openlineage-spark_2.12-${OPENLINEAGE_SPARK_VERSION}.jar \ + /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage/openlineage-spark_2.12-${OPENLINEAGE_SPARK_VERSION}.jar +# Symlink to a stable, version- and Scala-independent name (mirroring the jmx_prometheus_javaagent.jar +# pattern) so the operator can reference the jar without knowing the version or Scala build baked here. +ln -s openlineage-spark_2.12-${OPENLINEAGE_SPARK_VERSION}.jar \ + /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage/openlineage-spark.jar + # Get the correct `tini` binary for our architecture. curl --fail "https://repo.stackable.tech/repository/packages/tini/tini-${TINI_VERSION}-${TARGETARCH}" \ -o /usr/bin/tini diff --git a/spark-k8s/Dockerfile.4 b/spark-k8s/Dockerfile.4 index fff71258e..29f0eeded 100644 --- a/spark-k8s/Dockerfile.4 +++ b/spark-k8s/Dockerfile.4 @@ -39,6 +39,7 @@ ENV HADOOP_VERSION=${HADOOP_HADOOP_VERSION} ARG JACKSON_DATAFORMAT_XML_VERSION ARG STAX2_API_VERSION ARG WOODSTOX_CORE_VERSION +ARG OPENLINEAGE_SPARK_VERSION ARG JMX_EXPORTER_VERSION ARG TARGETARCH ARG TINI_VERSION @@ -122,6 +123,20 @@ mvn dependency:get -Dartifact=mvn dependency:get -Dartifact=com.fasterxml.woodst cp /root/.m2/repository/com/fasterxml/woodstox/woodstox-core/${WOODSTOX_CORE_VERSION}/woodstox-core-${WOODSTOX_CORE_VERSION}.jar \ /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/extra-jars/woodstox-core-${WOODSTOX_CORE_VERSION}.jar +# Download the OpenLineage Spark listener jar into a DEDICATED directory (dist/openlineage), +# NOT dist/extra-jars. The operator references it via `--jars local:///stackable/spark/openlineage/...` +# so it loads on Spark's child/user classloader, sharing it with connectors delivered via `--packages` +# (e.g. Iceberg). If it landed on extra-jars it would be added to the system classloader via +# extraClassPath, OpenLineage's connector probe would fail, and no datasets would be emitted. +mkdir -p /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage +mvn dependency:get -Dartifact=io.openlineage:openlineage-spark_2.13:${OPENLINEAGE_SPARK_VERSION} +cp /root/.m2/repository/io/openlineage/openlineage-spark_2.13/${OPENLINEAGE_SPARK_VERSION}/openlineage-spark_2.13-${OPENLINEAGE_SPARK_VERSION}.jar \ + /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage/openlineage-spark_2.13-${OPENLINEAGE_SPARK_VERSION}.jar +# Symlink to a stable, version- and Scala-independent name +# so the operator can reference the jar without knowing the version or Scala build baked here. +ln -s openlineage-spark_2.13-${OPENLINEAGE_SPARK_VERSION}.jar \ + /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/openlineage/openlineage-spark.jar + # Get the correct `tini` binary for our architecture. curl --fail "https://repo.stackable.tech/repository/packages/tini/tini-${TINI_VERSION}-${TARGETARCH}" \ -o /usr/bin/tini diff --git a/spark-k8s/boil-config.toml b/spark-k8s/boil-config.toml index a6e78665a..321841bf0 100644 --- a/spark-k8s/boil-config.toml +++ b/spark-k8s/boil-config.toml @@ -20,6 +20,7 @@ woodstox-core-version = "6.5.1" # Needs to match https://mvnrepository jmx-exporter-version = "1.3.0" tini-version = "0.19.0" hbase-connector-version = "1.0.1_3.5.8" +openlineage-spark-version = "1.51.0" # Scala 2.12 build for Spark 3.5.x; see https://mvnrepository.com/artifact/io.openlineage/openlineage-spark_2.12 # Deprecated since 26.7 [versions."4.1.1"] @@ -37,6 +38,7 @@ stax2-api-version = "4.2.2" # Needs to match https://mvnrepository woodstox-core-version = "7.1.1" # Needs to match https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.20.0 jmx-exporter-version = "1.3.0" tini-version = "0.19.0" +openlineage-spark-version = "1.51.0" # Scala 2.13 build for Spark 4.1.x; see https://mvnrepository.com/artifact/io.openlineage/openlineage-spark_2.13 # LTS since 26.7.0 [versions."4.1.2"] @@ -54,3 +56,4 @@ stax2-api-version = "4.2.2" # Needs to match https://mvnrepository woodstox-core-version = "7.1.1" # Needs to match https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.21.2 jmx-exporter-version = "1.3.0" tini-version = "0.19.0" +openlineage-spark-version = "1.51.0" # Scala 2.13 build for Spark 4.1.x; see https://mvnrepository.com/artifact/io.openlineage/openlineage-spark_2.13