diff --git a/CHANGELOG.md b/CHANGELOG.md index efaf4921..91618c6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file. - BREAKING: Add required CLI argument and env var to set the image repository used to construct final product image names: `IMAGE_REPOSITORY` (`--image-repository`), eg. `oci.example.org/my/namespace` ([#684]). - Support for Spark `4.1.2` ([#708]) +- Add `spec.openLineage` to enable OpenLineage lineage emission: injects the OpenLineage Spark listener, HTTP transport (endpoint built from the `host` and `port` fields), a stable job name, and the required `--add-opens` JVM flag. When the connection configures TLS server verification against a `secretClass` CA, that certificate is mounted into the driver and added to its trust store ([#XXXX]). ### Fixed diff --git a/Cargo.lock b/Cargo.lock index 0fce88c0..44044a33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -158,7 +158,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -169,7 +169,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -288,9 +288,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "block-buffer" @@ -325,9 +325,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.2.66" +version = "1.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" dependencies = [ "find-msvc-tools", "jobserver", @@ -354,9 +354,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011" dependencies = [ "clap_builder", "clap_derive", @@ -364,9 +364,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" dependencies = [ "anstream", "anstyle", @@ -383,7 +383,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -543,7 +543,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -554,7 +554,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -576,7 +576,7 @@ dependencies = [ "defmt-parser", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -596,7 +596,7 @@ checksum = "780eb241654bf097afb00fc5f054a09b687dad862e485fdcf8399bb056565370" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -620,7 +620,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -647,7 +647,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -670,7 +670,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -722,7 +722,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -795,7 +795,7 @@ checksum = "42e528e2d34ba8a67a1a650b86beae8ef69fc5fdb638016f386b973226590432" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -967,7 +967,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1060,7 +1060,7 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "libc", "libgit2-sys", "log", @@ -1179,9 +1179,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -1189,9 +1189,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", @@ -1494,9 +1494,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccfe6121cbe750cf81efa362d85c0bde7ea298ec43092d3a193baca59cdbd634" +checksum = "961d16382652bfdd8c6f68b223b26a8c93e0d475c672f414411db31c6c5c900e" dependencies = [ "defmt", "jiff-static", @@ -1510,20 +1510,20 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2" +checksum = "d0879bd39df99c4c5e2c6615ccc026391a423dde10532c573e6086eb94a802cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "jiff-tzdb" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6142247df1a93c2b3587402a19710be3e6e942f1581a1702e76408f2c21d6590" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" [[package]] name = "jiff-tzdb-platform" @@ -1570,9 +1570,9 @@ dependencies = [ [[package]] name = "jsonpath-rust" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "633a7320c4bb672863a3782e89b9094ad70285e097ff6832cddd0ec615beadfa" +checksum = "4222e00941bfe18bf81b79fa23ad933e233bfa18f3ee27254c5dd9b1543b5d5f" dependencies = [ "pest", "pest_derive", @@ -1607,7 +1607,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#013bbf43f7006a4ddfc08a147f68441ed88b462b" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fopenlineage-crd#01535bc1acb05e8a28f098d40bf5bef3105e075e" dependencies = [ "darling", "regex", @@ -1707,7 +1707,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1842,9 +1842,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi", @@ -1877,7 +1877,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.6", + "rand 0.8.7", "smallvec", "zeroize", ] @@ -2024,7 +2024,7 @@ dependencies = [ "opentelemetry", "percent-encoding", "portable-atomic", - "rand 0.9.4", + "rand 0.9.5", "thiserror 2.0.18", "tokio", "tokio-stream", @@ -2135,7 +2135,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2164,7 +2164,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2302,7 +2302,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2337,9 +2337,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" dependencies = [ "rand_chacha 0.3.1", "rand_core 0.6.4", @@ -2347,9 +2347,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -2399,7 +2399,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] @@ -2419,14 +2419,14 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "regex" -version = "1.12.4" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -2436,9 +2436,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -2558,7 +2558,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.118", + "syn 2.0.119", "unicode-ident", ] @@ -2573,9 +2573,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.41" +version = "0.23.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ "log", "once_cell", @@ -2662,7 +2662,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2700,7 +2700,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "core-foundation", "core-foundation-sys", "libc", @@ -2779,7 +2779,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2790,7 +2790,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2844,9 +2844,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" dependencies = [ "cfg-if", "cpufeatures", @@ -2901,9 +2901,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" [[package]] name = "slab" @@ -2965,7 +2965,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2977,14 +2977,14 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "socket2" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -2992,9 +2992,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" [[package]] name = "spki" @@ -3015,14 +3015,14 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#013bbf43f7006a4ddfc08a147f68441ed88b462b" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fopenlineage-crd#01535bc1acb05e8a28f098d40bf5bef3105e075e" dependencies = [ "const-oid", "ecdsa", "k8s-openapi", "kube", "p256", - "rand 0.9.4", + "rand 0.9.5", "rand_core 0.6.4", "rsa", "sha2", @@ -3038,8 +3038,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.113.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#013bbf43f7006a4ddfc08a147f68441ed88b462b" +version = "0.113.4" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fopenlineage-crd#01535bc1acb05e8a28f098d40bf5bef3105e075e" dependencies = [ "base64", "clap", @@ -3057,7 +3057,7 @@ dependencies = [ "k8s-openapi", "kube", "product-config", - "rand 0.9.4", + "rand 0.9.5", "regex", "schemars", "semver", @@ -3084,18 +3084,18 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#013bbf43f7006a4ddfc08a147f68441ed88b462b" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fopenlineage-crd#01535bc1acb05e8a28f098d40bf5bef3105e075e" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "stackable-shared" version = "0.1.2" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#013bbf43f7006a4ddfc08a147f68441ed88b462b" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fopenlineage-crd#01535bc1acb05e8a28f098d40bf5bef3105e075e" dependencies = [ "jiff", "k8s-openapi", @@ -3136,7 +3136,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.5" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#013bbf43f7006a4ddfc08a147f68441ed88b462b" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fopenlineage-crd#01535bc1acb05e8a28f098d40bf5bef3105e075e" dependencies = [ "axum", "clap", @@ -3160,7 +3160,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#013bbf43f7006a4ddfc08a147f68441ed88b462b" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fopenlineage-crd#01535bc1acb05e8a28f098d40bf5bef3105e075e" dependencies = [ "kube", "schemars", @@ -3174,7 +3174,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#013bbf43f7006a4ddfc08a147f68441ed88b462b" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fopenlineage-crd#01535bc1acb05e8a28f098d40bf5bef3105e075e" dependencies = [ "convert_case", "convert_case_extras", @@ -3186,13 +3186,13 @@ dependencies = [ "kube", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "stackable-webhook" version = "0.9.2" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#013bbf43f7006a4ddfc08a147f68441ed88b462b" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fopenlineage-crd#01535bc1acb05e8a28f098d40bf5bef3105e075e" dependencies = [ "arc-swap", "async-trait", @@ -3205,7 +3205,7 @@ dependencies = [ "kube", "opentelemetry", "opentelemetry-semantic-conventions", - "rand 0.9.4", + "rand 0.9.5", "serde", "serde_json", "snafu 0.9.1", @@ -3245,7 +3245,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3273,9 +3273,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" dependencies = [ "proc-macro2", "quote", @@ -3299,7 +3299,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3328,7 +3328,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3339,14 +3339,14 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "thread_local" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" dependencies = [ "cfg-if", ] @@ -3409,14 +3409,14 @@ checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee" dependencies = [ "bytes", "libc", @@ -3431,13 +3431,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3486,9 +3486,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.12+spec-1.1.0" +version = "0.25.13+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" dependencies = [ "indexmap", "toml_datetime", @@ -3580,7 +3580,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "base64", - "bitflags 2.13.0", + "bitflags 2.13.1", "bytes", "futures-util", "http", @@ -3600,7 +3600,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "bytes", "http", "http-body", @@ -3656,7 +3656,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3820,9 +3820,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.4" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "js-sys", "wasm-bindgen", @@ -3912,7 +3912,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "wasm-bindgen-shared", ] @@ -3966,7 +3966,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3977,7 +3977,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4088,9 +4088,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" dependencies = [ "memchr", ] @@ -4146,28 +4146,28 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.53" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75726053136156d419e285b9b7eddaaea9e3fea6ce32eed44a89901f0bd98de1" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.53" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4714fd92cf900833d49538023a9b3915155210801d1c1169eba513b2addefd71" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4187,7 +4187,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "synstructure", ] @@ -4208,7 +4208,7 @@ checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4241,11 +4241,11 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.nix b/Cargo.nix index fd10413f..46030bb8 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -481,7 +481,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" "visit-mut" ]; } ]; @@ -508,7 +508,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; usesDefaultFeatures = false; features = [ "clone-impls" "full" "parsing" "printing" "proc-macro" "visit-mut" ]; } @@ -921,11 +921,11 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; - "bitflags 2.13.0" = rec { + "bitflags 2.13.1" = rec { crateName = "bitflags"; - version = "2.13.0"; + version = "2.13.1"; edition = "2021"; - sha256 = "1y239gpvl061rfvav7jds8mjs42kmwi39is7yx5d1qw3hvp8nf5l"; + sha256 = "1nl76mpykmwmb8rq1l5vw1azdh1wvxdrnsk4sy3rdrzx01nvg25m"; authors = [ "The Rust Project Developers" ]; @@ -1019,9 +1019,9 @@ rec { }; "cc" = rec { crateName = "cc"; - version = "1.2.66"; + version = "1.2.67"; edition = "2018"; - sha256 = "15nr9bpbcinb9z7zvr1d70xyivv8969v490001qdjywrjg3wmmpm"; + sha256 = "0f0srhm5s5371nqk7dlini2knpfhml1hxcp18ksixwyhlxjx4zg1"; authors = [ "Alex Crichton " ]; @@ -1118,10 +1118,10 @@ rec { }; "clap" = rec { crateName = "clap"; - version = "4.6.1"; + version = "4.6.2"; edition = "2024"; crateBin = []; - sha256 = "0lcf88l7vlg796rrqr7wipbbmfa5sgsgx4211b7xmxxv8dz13nqx"; + sha256 = "04ah42lfd7imr2mqj7xh0xgk3k5imb63hpgnhwvnphzmljfry1fx"; dependencies = [ { name = "clap_builder"; @@ -1160,9 +1160,9 @@ rec { }; "clap_builder" = rec { crateName = "clap_builder"; - version = "4.6.0"; + version = "4.6.2"; edition = "2024"; - sha256 = "17q6np22yxhh5y5v53y4l31ps3hlaz45mvz2n2nicr7n3c056jki"; + sha256 = "12sl6fyj6w2djxj0lsc1lkj1h3wpx74fjhb37izvaf65vjpji5ph"; dependencies = [ { name = "anstream"; @@ -1218,7 +1218,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" ]; } ]; @@ -1649,7 +1649,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" "extra-traits" ]; } ]; @@ -1680,7 +1680,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; @@ -1734,7 +1734,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" "extra-traits" ]; } ]; @@ -1780,7 +1780,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" "visit-mut" ]; } ]; @@ -1857,7 +1857,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "extra-traits" ]; } ]; @@ -1951,7 +1951,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; buildDependencies = [ @@ -2048,7 +2048,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; features = { @@ -2214,13 +2214,13 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; devDependencies = [ { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" ]; } ]; @@ -2434,7 +2434,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; features = { @@ -2935,7 +2935,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" ]; } ]; @@ -3325,7 +3325,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.0"; + packageId = "bitflags 2.13.1"; } { name = "libc"; @@ -3680,9 +3680,9 @@ rec { }; "http-body" = rec { crateName = "http-body"; - version = "1.0.1"; + version = "1.1.0"; edition = "2018"; - sha256 = "111ir5k2b9ihz5nr9cz7cwm7fnydca7dx4hc7vr16scfzghxrzhy"; + sha256 = "0b5wj0rdj8p03k20q8x0jy249amg2db919fnmh7zcrgf2clqyana"; libName = "http_body"; authors = [ "Carl Lerche " @@ -3703,9 +3703,9 @@ rec { }; "http-body-util" = rec { crateName = "http-body-util"; - version = "0.1.3"; + version = "0.1.4"; edition = "2018"; - sha256 = "0jm6jv4gxsnlsi1kzdyffjrj8cfr3zninnxpw73mvkxy4qzdj8dh"; + sha256 = "1wizkqx9a75x8v5lm7cawpammz8sfvd7cngnkp34wkcfl3b1zx79"; libName = "http_body_util"; authors = [ "Carl Lerche " @@ -4731,9 +4731,9 @@ rec { }; "jiff" = rec { crateName = "jiff"; - version = "0.2.31"; + version = "0.2.32"; edition = "2021"; - sha256 = "0d6nvffabb1v34x2s2a3xjca4zny1dfdhqm3xy0wyl77rchn3znc"; + sha256 = "03lhbin1rcqx84ag8wn6fpaf14wcdar27ck8dy6dvgsj4qw1c7cn"; authors = [ "Andrew Gallant " ]; @@ -4817,9 +4817,9 @@ rec { }; "jiff-static" = rec { crateName = "jiff-static"; - version = "0.2.31"; + version = "0.2.32"; edition = "2021"; - sha256 = "1lpdrmzd5yafwzniw0xi5gac4rz0vccsja1qrprjim32ysbyhrg1"; + sha256 = "1k02m2afp1k07rbjqlqhvqyl46ir4v0cq5b65ig4r77rkp9rp1yh"; procMacro = true; libName = "jiff_static"; authors = [ @@ -4836,7 +4836,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; features = { @@ -4846,9 +4846,9 @@ rec { }; "jiff-tzdb" = rec { crateName = "jiff-tzdb"; - version = "0.1.7"; + version = "0.1.8"; edition = "2021"; - sha256 = "14353p1g4234ww11f6jqy51fkrp31dqijaj0hwsjng59y5yj8hk1"; + sha256 = "07hl9sgzfb9as1x0n5bjk1qxishzcriapy9xa481y8xd6acx6aql"; libName = "jiff_tzdb"; libPath = "lib.rs"; authors = [ @@ -4984,9 +4984,9 @@ rec { }; "jsonpath-rust" = rec { crateName = "jsonpath-rust"; - version = "1.0.4"; + version = "1.0.5"; edition = "2021"; - sha256 = "1ymdpqawc3nxrlr6izwpw22h5msa16wqjbkqldijhrxvqhh76fk3"; + sha256 = "0psx7dab3nax9hjjgvpk33x3n8ryjfnj7ykr3gw8pqdz844y08j2"; libName = "jsonpath_rust"; authors = [ "BorisZhguchev " @@ -5107,9 +5107,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "054p2mcinq3x4iykmc72nhga7kapc8ihkx9mnzqq3qdm2bk2qcps"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "01535bc1acb05e8a28f098d40bf5bef3105e075e"; + sha256 = "0p43kywyfjq9l3j0fapzpdcdphy8nyn7l9n4i9isd1iwpyn5ix3f"; }; libName = "k8s_version"; authors = [ @@ -5606,7 +5606,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "extra-traits" ]; } ]; @@ -6039,9 +6039,9 @@ rec { }; "mio" = rec { crateName = "mio"; - version = "1.2.1"; + version = "1.2.2"; edition = "2021"; - sha256 = "1nkggmrlnjs93w8rja4lvjj4aml1xqahgimv1h0p7d373kvhmg82"; + sha256 = "09y4b7gc42ymgssshh8sz6gs3y5r8bbigqaw2c4snh6fy5qmrmih"; authors = [ "Carl Lerche " "Thomas de Zeeuw " @@ -6154,7 +6154,7 @@ rec { } { name = "rand"; - packageId = "rand 0.8.6"; + packageId = "rand 0.8.7"; optional = true; usesDefaultFeatures = false; } @@ -6173,7 +6173,7 @@ rec { devDependencies = [ { name = "rand"; - packageId = "rand 0.8.6"; + packageId = "rand 0.8.7"; features = [ "small_rng" ]; } ]; @@ -6716,7 +6716,7 @@ rec { } { name = "rand"; - packageId = "rand 0.9.4"; + packageId = "rand 0.9.5"; optional = true; usesDefaultFeatures = false; features = [ "std" "std_rng" "small_rng" "os_rng" "thread_rng" ]; @@ -7107,7 +7107,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; features = { @@ -7167,7 +7167,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; usesDefaultFeatures = false; features = [ "parsing" "printing" "clone-impls" "proc-macro" "full" "visit-mut" ]; } @@ -7534,7 +7534,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "extra-traits" ]; } ]; @@ -7612,11 +7612,11 @@ rec { "rustc-dep-of-std" = [ "core" ]; }; }; - "rand 0.8.6" = rec { + "rand 0.8.7" = rec { crateName = "rand"; - version = "0.8.6"; + version = "0.8.7"; edition = "2018"; - sha256 = "12kd4rljn86m00rcaz4c1rcya4mb4gk5ig6i8xq00a8wjgxfr82w"; + sha256 = "06iaf16fr0z8zly7anmn8ky0p80xnx9yv0gdcm30fwn9vqmigxi2"; authors = [ "The Rand Project Developers" "The Rust Project Developers" @@ -7646,11 +7646,11 @@ rec { }; resolvedDefaultFeatures = [ "rand_chacha" "std_rng" ]; }; - "rand 0.9.4" = rec { + "rand 0.9.5" = rec { crateName = "rand"; - version = "0.9.4"; + version = "0.9.5"; edition = "2021"; - sha256 = "1sknbxgs6nfg0nxdd7689lwbyr2i4vaswchrv4b34z8vpc3azia4"; + sha256 = "0hbvllk8g28mqjld6hqmckk69w296qpzg95whm3didsyg46ivvxr"; authors = [ "The Rand Project Developers" "The Rust Project Developers" @@ -7803,7 +7803,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.0"; + packageId = "bitflags 2.13.1"; } ]; features = { @@ -7851,16 +7851,16 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; }; "regex" = rec { crateName = "regex"; - version = "1.12.4"; + version = "1.13.1"; edition = "2021"; - sha256 = "1fm6si2xpmhwqflabdqsakc0qkq718wx2ljl37nbj75fb5vjnagi"; + sha256 = "1391a0a4100ik8cp7l577p3ip3haqq03rd9c5vdr7vcfdixj687h"; authors = [ "The Rust Project Developers" "Andrew Gallant " @@ -7916,9 +7916,9 @@ rec { }; "regex-automata" = rec { crateName = "regex-automata"; - version = "0.4.14"; + version = "0.4.16"; edition = "2021"; - sha256 = "13xf7hhn4qmgfh784llcp2kzrvljd13lb2b1ca0mwnf15w9d87bf"; + sha256 = "1b8ihxq99g3hr8mr37bvhib4bfn8rlmpmp0wjg2q1j50plvdpkwg"; libName = "regex_automata"; authors = [ "The Rust Project Developers" @@ -8482,7 +8482,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" "parsing" "extra-traits" "visit" "visit-mut" ]; } { @@ -8517,9 +8517,9 @@ rec { }; "rustls" = rec { crateName = "rustls"; - version = "0.23.41"; + version = "0.23.42"; edition = "2021"; - sha256 = "07vbs2935a7xjqqvy8w3ndzmmw8dg769d9zcgdg7k6sdccjv34kb"; + sha256 = "0f619dq1izpl40glcqgfjbqzpmwg8g5iffjx4429sh4v06mzqm1w"; dependencies = [ { name = "log"; @@ -8824,13 +8824,13 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; devDependencies = [ { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "extra-traits" ]; } ]; @@ -8941,7 +8941,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.0"; + packageId = "bitflags 2.13.1"; } { name = "core-foundation"; @@ -9202,7 +9202,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "parsing" "printing" "proc-macro" ]; } @@ -9234,7 +9234,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "parsing" "printing" ]; } @@ -9388,9 +9388,9 @@ rec { }; "sha1" = rec { crateName = "sha1"; - version = "0.10.6"; + version = "0.10.7"; edition = "2018"; - sha256 = "1fnnxlfg08xhkmwf2ahv634as30l1i3xhlhkvxflmasi5nd85gz3"; + sha256 = "1f632d529qzz95yrprr632w1fxqkrv6b6jksjc11vnzl049lay59"; authors = [ "RustCrypto Developers" ]; @@ -9417,10 +9417,8 @@ rec { } ]; features = { - "asm" = [ "sha1-asm" ]; "default" = [ "std" ]; "oid" = [ "digest/oid" ]; - "sha1-asm" = [ "dep:sha1-asm" ]; "std" = [ "digest/std" ]; }; resolvedDefaultFeatures = [ "default" "std" ]; @@ -9556,9 +9554,9 @@ rec { }; "simd-adler32" = rec { crateName = "simd-adler32"; - version = "0.3.9"; + version = "0.3.10"; edition = "2018"; - sha256 = "0532ysdwcvzyp2bwpk8qz0hijplcdwpssr5gy5r7qwqqy5z5qgbh"; + sha256 = "1sny4y2qa5mwyxx5x59ln2p02vsdh92004njlslnx98imjc9489s"; libName = "simd_adler32"; authors = [ "Marvin Countryman " @@ -9745,7 +9743,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" ]; } ]; @@ -9781,7 +9779,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "full" "parsing" "printing" "proc-macro" "visit-mut" ]; } @@ -9791,9 +9789,9 @@ rec { }; "socket2" = rec { crateName = "socket2"; - version = "0.6.4"; + version = "0.6.5"; edition = "2021"; - sha256 = "0ldyp5rhba15spwxj1n94xh7sjks1398c3vwpwkxkd1087nwzlaj"; + sha256 = "1m7diygswpvlpvrxd6ap169nxgax014jr8220nqlr3bzyb3y5lf3"; authors = [ "Alex Crichton " "Thomas de Zeeuw " @@ -9817,9 +9815,9 @@ rec { }; "spin" = rec { crateName = "spin"; - version = "0.9.8"; + version = "0.9.9"; edition = "2015"; - sha256 = "0rvam5r0p3a6qhc18scqpvpgb3ckzyqxpgdfyjnghh8ja7byi039"; + sha256 = "03psal0vh1xdxp7agphw09p7kf50v3bj1zshijq1s5bkdd7jcqrp"; authors = [ "Mathijs van de Nes " "John Ericson " @@ -9891,9 +9889,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "054p2mcinq3x4iykmc72nhga7kapc8ihkx9mnzqq3qdm2bk2qcps"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "01535bc1acb05e8a28f098d40bf5bef3105e075e"; + sha256 = "0p43kywyfjq9l3j0fapzpdcdphy8nyn7l9n4i9isd1iwpyn5ix3f"; }; libName = "stackable_certs"; authors = [ @@ -9929,7 +9927,7 @@ rec { } { name = "rand"; - packageId = "rand 0.9.4"; + packageId = "rand 0.9.5"; } { name = "rand_core"; @@ -9990,13 +9988,13 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.113.3"; + version = "0.113.4"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "054p2mcinq3x4iykmc72nhga7kapc8ihkx9mnzqq3qdm2bk2qcps"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "01535bc1acb05e8a28f098d40bf5bef3105e075e"; + sha256 = "0p43kywyfjq9l3j0fapzpdcdphy8nyn7l9n4i9isd1iwpyn5ix3f"; }; libName = "stackable_operator"; authors = [ @@ -10077,7 +10075,7 @@ rec { } { name = "rand"; - packageId = "rand 0.9.4"; + packageId = "rand 0.9.5"; } { name = "regex"; @@ -10193,9 +10191,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "054p2mcinq3x4iykmc72nhga7kapc8ihkx9mnzqq3qdm2bk2qcps"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "01535bc1acb05e8a28f098d40bf5bef3105e075e"; + sha256 = "0p43kywyfjq9l3j0fapzpdcdphy8nyn7l9n4i9isd1iwpyn5ix3f"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -10217,7 +10215,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; @@ -10228,9 +10226,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "054p2mcinq3x4iykmc72nhga7kapc8ihkx9mnzqq3qdm2bk2qcps"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "01535bc1acb05e8a28f098d40bf5bef3105e075e"; + sha256 = "0p43kywyfjq9l3j0fapzpdcdphy8nyn7l9n4i9isd1iwpyn5ix3f"; }; libName = "stackable_shared"; authors = [ @@ -10415,9 +10413,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "054p2mcinq3x4iykmc72nhga7kapc8ihkx9mnzqq3qdm2bk2qcps"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "01535bc1acb05e8a28f098d40bf5bef3105e075e"; + sha256 = "0p43kywyfjq9l3j0fapzpdcdphy8nyn7l9n4i9isd1iwpyn5ix3f"; }; libName = "stackable_telemetry"; authors = [ @@ -10525,9 +10523,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "054p2mcinq3x4iykmc72nhga7kapc8ihkx9mnzqq3qdm2bk2qcps"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "01535bc1acb05e8a28f098d40bf5bef3105e075e"; + sha256 = "0p43kywyfjq9l3j0fapzpdcdphy8nyn7l9n4i9isd1iwpyn5ix3f"; }; libName = "stackable_versioned"; authors = [ @@ -10575,9 +10573,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "054p2mcinq3x4iykmc72nhga7kapc8ihkx9mnzqq3qdm2bk2qcps"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "01535bc1acb05e8a28f098d40bf5bef3105e075e"; + sha256 = "0p43kywyfjq9l3j0fapzpdcdphy8nyn7l9n4i9isd1iwpyn5ix3f"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -10632,7 +10630,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; @@ -10643,9 +10641,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "013bbf43f7006a4ddfc08a147f68441ed88b462b"; - sha256 = "054p2mcinq3x4iykmc72nhga7kapc8ihkx9mnzqq3qdm2bk2qcps"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "01535bc1acb05e8a28f098d40bf5bef3105e075e"; + sha256 = "0p43kywyfjq9l3j0fapzpdcdphy8nyn7l9n4i9isd1iwpyn5ix3f"; }; libName = "stackable_webhook"; authors = [ @@ -10704,7 +10702,7 @@ rec { } { name = "rand"; - packageId = "rand 0.9.4"; + packageId = "rand 0.9.5"; } { name = "serde"; @@ -10827,7 +10825,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "parsing" ]; } ]; @@ -10891,11 +10889,11 @@ rec { }; resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "full" "parsing" "printing" "proc-macro" "quote" ]; }; - "syn 2.0.118" = rec { + "syn 2.0.119" = rec { crateName = "syn"; - version = "2.0.118"; + version = "2.0.119"; edition = "2021"; - sha256 = "08hlbc32lqd5d67p26ck7chg0rkclsw9as6f96vfn4s2j1zyb6hv"; + sha256 = "15vjy620l91a3q4n4f4gzhnflmdr6pnm38v2m6cpk86i8av32a47"; authors = [ "David Tolnay " ]; @@ -10967,7 +10965,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; usesDefaultFeatures = false; features = [ "derive" "parsing" "printing" "clone-impls" "visit" "extra-traits" ]; } @@ -11034,7 +11032,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; @@ -11060,16 +11058,16 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; } ]; }; "thread_local" = rec { crateName = "thread_local"; - version = "1.1.9"; + version = "1.1.10"; edition = "2021"; - sha256 = "1191jvl8d63agnq06pcnarivf63qzgpws5xa33hgc92gjjj4c0pn"; + sha256 = "0w20g2pfdcp8pz3gds0bzksv6mxk802szca8qlr3701jdm69rn8s"; authors = [ "Amanieu d'Antras " ]; @@ -11272,7 +11270,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "parsing" ]; } ]; @@ -11284,9 +11282,9 @@ rec { }; "tokio" = rec { crateName = "tokio"; - version = "1.52.3"; + version = "1.53.0"; edition = "2021"; - sha256 = "1zpzazypkg61sw91na1m85x5s4rsjym335fwwhwm1hcs70dz1iwg"; + sha256 = "1vpzc93iaiaqk90jh54vqji8fawiiksk4cwh8qyns1xy5pavr26r"; authors = [ "Tokio Contributors " ]; @@ -11405,9 +11403,9 @@ rec { }; "tokio-macros" = rec { crateName = "tokio-macros"; - version = "2.7.0"; + version = "2.7.1"; edition = "2021"; - sha256 = "15m4f37mdafs0gg36sh0rskm1i768lb7zmp8bw67kaxr3avnqniq"; + sha256 = "1fj2h3gysqzwqchyhcyyvslwdj7qjgyzlc20d6sajwqf949sya33"; procMacro = true; libName = "tokio_macros"; authors = [ @@ -11424,7 +11422,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" ]; } ]; @@ -11596,9 +11594,9 @@ rec { }; "toml_edit" = rec { crateName = "toml_edit"; - version = "0.25.12+spec-1.1.0"; + version = "0.25.13+spec-1.1.0"; edition = "2024"; - sha256 = "1mx5paq837rjw7w51zprrjynk1vaig9yzxfqz9ac79jmd7f3w5fj"; + sha256 = "16xgmjdnxssdpj7rjyimsk4fqbv29g8zl7zhdbc6dxrf9mz3cxb9"; dependencies = [ { name = "indexmap"; @@ -11966,7 +11964,7 @@ rec { } { name = "bitflags"; - packageId = "bitflags 2.13.0"; + packageId = "bitflags 2.13.1"; } { name = "bytes"; @@ -12091,7 +12089,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.0"; + packageId = "bitflags 2.13.1"; } { name = "bytes"; @@ -12308,7 +12306,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; usesDefaultFeatures = false; features = [ "full" "parsing" "printing" "visit-mut" "clone-impls" "extra-traits" "proc-macro" ]; } @@ -12848,9 +12846,9 @@ rec { }; "uuid" = rec { crateName = "uuid"; - version = "1.23.4"; + version = "1.24.0"; edition = "2021"; - sha256 = "0lws65rrqncssdz1rk8g8ww7xg6k4d3l6avzkslzwni78llag05z"; + sha256 = "0faj5x0zgri8m3i8dv9qgyhiwqwdyhbl2g351cp3iin4ynk26fdz"; authors = [ "Ashley Mannix" "Dylan DPC" @@ -13118,7 +13116,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "visit" "visit-mut" "full" "extra-traits" ]; } { @@ -13727,7 +13725,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; usesDefaultFeatures = false; features = [ "parsing" "proc-macro" "printing" "full" "clone-impls" ]; } @@ -13754,7 +13752,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; usesDefaultFeatures = false; features = [ "parsing" "proc-macro" "printing" "full" "clone-impls" ]; } @@ -14450,9 +14448,9 @@ rec { }; "winnow" = rec { crateName = "winnow"; - version = "1.0.3"; + version = "1.0.4"; edition = "2021"; - sha256 = "1wajycd3krn6h699vydjv7hm0ll5l31p899qzpk59y2is74y34h5"; + sha256 = "10fzxipa7lx16172p3aca9j60hzbqgjki2f95kqksd5qywcp7f93"; dependencies = [ { name = "memchr"; @@ -14629,7 +14627,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "fold" ]; } { @@ -14641,9 +14639,9 @@ rec { }; "zerocopy" = rec { crateName = "zerocopy"; - version = "0.8.53"; + version = "0.8.54"; edition = "2021"; - sha256 = "1qcdv45iz4499bafwcnflvzf7adfvbnvgfc5w8cx8mk12d9n0wkm"; + sha256 = "06cxymy8i9q9a93xdins9ayakx9b1nc2arb7qdfd03ssf05brjxp"; authors = [ "Joshua Liebow-Feeser " "Jack Wrenn " @@ -14677,9 +14675,9 @@ rec { }; "zerocopy-derive" = rec { crateName = "zerocopy-derive"; - version = "0.8.53"; + version = "0.8.54"; edition = "2021"; - sha256 = "0wgxvsnv44x5xdli270xh085458m76dkl0iqjpa3624hry9gs527"; + sha256 = "1xb292dhgb0d4fs05cdj2s0v3srmk7bajv94sdc7631dnnvigs72"; procMacro = true; libName = "zerocopy_derive"; authors = [ @@ -14697,14 +14695,14 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" ]; } ]; devDependencies = [ { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "visit" ]; } ]; @@ -14753,7 +14751,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "fold" ]; } { @@ -14807,7 +14805,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "full" "extra-traits" "visit" ]; } ]; @@ -14920,7 +14918,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.118"; + packageId = "syn 2.0.119"; features = [ "extra-traits" ]; } ]; @@ -14928,9 +14926,9 @@ rec { }; "zmij" = rec { crateName = "zmij"; - version = "1.0.21"; + version = "1.0.23"; edition = "2021"; - sha256 = "1amb5i6gz7yjb0dnmz5y669674pqmwbj44p4yfxfv2ncgvk8x15q"; + sha256 = "06zwri21nnrl34rwinmvbciap8yk1mrl8qfg9pff7lgspc56sri9"; authors = [ "David Tolnay " ]; diff --git a/Cargo.toml b/Cargo.toml index b7cb1efb..12fc1fc6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,9 @@ edition = "2024" repository = "https://github.com/stackabletech/spark-k8s-operator" [workspace.dependencies] -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.113.3", features = ["webhook"] } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.113.3", features = [ + "webhook", +] } anyhow = "1.0.103" built = { version = "0.8", features = ["chrono", "git2"] } @@ -31,5 +33,5 @@ indoc = "2" regex = "1.12" [patch."https://github.com/stackabletech/operator-rs.git"] -# stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch="main" } -# stackable-operator = { path = "../operator-rs/crates/stackable-operator" } +# TODO: remove this before merge +stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "feat/openlineage-crd" } diff --git a/crate-hashes.json b/crate-hashes.json index cd3917c3..24806b5c 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#k8s-version@0.1.3": "054p2mcinq3x4iykmc72nhga7kapc8ihkx9mnzqq3qdm2bk2qcps", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-certs@0.4.1": "054p2mcinq3x4iykmc72nhga7kapc8ihkx9mnzqq3qdm2bk2qcps", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-operator-derive@0.3.1": "054p2mcinq3x4iykmc72nhga7kapc8ihkx9mnzqq3qdm2bk2qcps", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-operator@0.113.3": "054p2mcinq3x4iykmc72nhga7kapc8ihkx9mnzqq3qdm2bk2qcps", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-shared@0.1.2": "054p2mcinq3x4iykmc72nhga7kapc8ihkx9mnzqq3qdm2bk2qcps", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-telemetry@0.6.5": "054p2mcinq3x4iykmc72nhga7kapc8ihkx9mnzqq3qdm2bk2qcps", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-versioned-macros@0.11.1": "054p2mcinq3x4iykmc72nhga7kapc8ihkx9mnzqq3qdm2bk2qcps", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-versioned@0.11.1": "054p2mcinq3x4iykmc72nhga7kapc8ihkx9mnzqq3qdm2bk2qcps", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.113.3#stackable-webhook@0.9.2": "054p2mcinq3x4iykmc72nhga7kapc8ihkx9mnzqq3qdm2bk2qcps", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fopenlineage-crd#k8s-version@0.1.3": "0p43kywyfjq9l3j0fapzpdcdphy8nyn7l9n4i9isd1iwpyn5ix3f", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fopenlineage-crd#stackable-certs@0.4.1": "0p43kywyfjq9l3j0fapzpdcdphy8nyn7l9n4i9isd1iwpyn5ix3f", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fopenlineage-crd#stackable-operator-derive@0.3.1": "0p43kywyfjq9l3j0fapzpdcdphy8nyn7l9n4i9isd1iwpyn5ix3f", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fopenlineage-crd#stackable-operator@0.113.4": "0p43kywyfjq9l3j0fapzpdcdphy8nyn7l9n4i9isd1iwpyn5ix3f", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fopenlineage-crd#stackable-shared@0.1.2": "0p43kywyfjq9l3j0fapzpdcdphy8nyn7l9n4i9isd1iwpyn5ix3f", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fopenlineage-crd#stackable-telemetry@0.6.5": "0p43kywyfjq9l3j0fapzpdcdphy8nyn7l9n4i9isd1iwpyn5ix3f", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fopenlineage-crd#stackable-versioned-macros@0.11.1": "0p43kywyfjq9l3j0fapzpdcdphy8nyn7l9n4i9isd1iwpyn5ix3f", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fopenlineage-crd#stackable-versioned@0.11.1": "0p43kywyfjq9l3j0fapzpdcdphy8nyn7l9n4i9isd1iwpyn5ix3f", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fopenlineage-crd#stackable-webhook@0.9.2": "0p43kywyfjq9l3j0fapzpdcdphy8nyn7l9n4i9isd1iwpyn5ix3f", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/deploy/helm/spark-k8s-operator/templates/roles.yaml b/deploy/helm/spark-k8s-operator/templates/roles.yaml index e3fab4ff..3217f5e8 100644 --- a/deploy/helm/spark-k8s-operator/templates/roles.yaml +++ b/deploy/helm/spark-k8s-operator/templates/roles.yaml @@ -193,6 +193,15 @@ rules: - get - list - watch + # OpenLineage backend connections referenced by SparkApplications. + - apiGroups: + - openlineage.stackable.tech + resources: + - openlineageconnections + verbs: + - get + - list + - watch # Required for managing how the History Server and Connect Server are exposed # outside of the cluster. - apiGroups: diff --git a/docs/modules/spark-k8s/pages/usage-guide/openlineage.adoc b/docs/modules/spark-k8s/pages/usage-guide/openlineage.adoc new file mode 100644 index 00000000..d1ff89c3 --- /dev/null +++ b/docs/modules/spark-k8s/pages/usage-guide/openlineage.adoc @@ -0,0 +1,134 @@ += OpenLineage + +https://openlineage.io/[OpenLineage] is an open standard for data lineage collection. +The Spark operator can automatically emit lineage events for a `SparkApplication` to an OpenLineage-compatible backend such as https://marquezproject.github.io/marquez/[Marquez], without any manual `sparkConf` wiring. + +When configured, the operator injects everything required to make the https://openlineage.io/docs/integrations/spark/[OpenLineage Spark listener] work: + +* the OpenLineage Spark listener (appended to `spark.extraListeners`, so any listener you configure yourself is preserved), +* the OpenLineage jar baked into the Spark image, referenced via `spark.jars` so it shares the same classloader as connectors pulled in through xref:usage-guide/job-dependencies.adoc[`deps.packages`] (for example Apache Iceberg), +* an HTTP transport pointing at the backend endpoint, +* a stable lineage namespace and job name (see <>), and +* on Spark 4.x only, the `--add-opens java.base/java.security=ALL-UNNAMED` JVM flag the listener requires on the driver and executors. This flag is not added on the Spark 3.5.x images, where it is unnecessary (and, on their JVM, potentially harmful). + +== Enabling OpenLineage + +The backend is configured through the `openLineage.connection`, which either inlines an OpenLineage +connection or references a standalone `OpenLineageConnection` resource by name. +The operator points the transport at `://:`: + +[source,yaml] +---- +apiVersion: spark.stackable.tech/v1alpha1 +kind: SparkApplication +metadata: + name: orders-by-nation +spec: + openLineage: # <1> + connection: # <2> + inline: + host: marquez + port: 5000 + # tls: + # verification: + # server: + # caCert: + # secretClass: marquez-ca + # namespace: orders-lineage # <3> + # appName: orders-by-nation # <4> + ... +---- +<1> Adding the `openLineage` block enables OpenLineage event emission. Omit it entirely to leave OpenLineage off (the default), in which case the operator emits nothing OpenLineage-related and behaviour is unchanged. +<2> The OpenLineage backend connection. Use `inline` (shown here) to define the connection directly, + or `reference: ` to point at an `OpenLineageConnection` resource in the same namespace. + Required. +<3> Optional. The OpenLineage namespace events are reported under. + Defaults to the application's Kubernetes namespace. +<4> Optional. The stable OpenLineage job name (see <>). + +The `OpenLineageConnection` resource (`openlineage.stackable.tech`) is a reusable definition of a +connection to an OpenLineage backend, provided by the Stackable operator library. Reference it to +share one backend definition across multiple applications: + +[source,yaml] +---- +apiVersion: openlineage.stackable.tech/v1alpha1 +kind: OpenLineageConnection +metadata: + name: marquez +spec: + host: marquez + port: 5000 +---- + +The transport scheme is `https` when `tls.verification.server` is configured, otherwise `http`. +When the connection verifies the server against a `secretClass` CA (as in the commented `tls` block +above), the operator mounts that SecretClass certificate into the driver and adds it to the driver's +trust store, so the OpenLineage listener trusts the backend's certificate. + +You can still override any of the injected values — including the full `spark.openlineage.transport.url` — through `sparkConf`. + +[#authentication] +== Authentication + +Backends that require authentication (for example DataHub, or a Marquez instance behind a gateway) expect a bearer token on each request. +Configure this by setting `credentialsSecretName` on the connection to the name of a Secret holding the token: + +[source,yaml] +---- +apiVersion: spark.stackable.tech/v1alpha1 +kind: SparkApplication +metadata: + name: orders-by-nation +spec: + openLineage: + connection: + inline: + host: marquez + port: 5000 + credentialsSecretName: openlineage-token # <1> + ... +--- +apiVersion: v1 +kind: Secret +metadata: + name: openlineage-token + namespace: default # <2> +stringData: + apiKey: "s3cr3t-bearer-token" # <3> +---- +<1> Name of the Secret holding the token. If omitted, no authentication is used. The Secret must exist in the same namespace as the `SparkApplication`. +<2> The Secret must be in the workload's namespace so it can be projected into the driver. +<3> The token must be stored under the fixed key `apiKey`. The operator never reads the Secret; it is injected into the driver's OpenLineage listener via a `secretKeyRef` environment variable, so the token never appears in the spark-submit arguments or the pod spec. + +The operator translates this into the OpenLineage HTTP transport's `api_key` (bearer) authentication. +Combine it with the `tls` block above when the backend also needs TLS server verification. + +[NOTE] +==== +When `credentialsSecretName` is set, the operator delivers the *entire* transport configuration +(type, URL and auth) to the driver through `OPENLINEAGE__*` environment variables rather than +`spark.openlineage.transport.*` `--conf` properties. +This is required: OpenLineage resolves the `transport` from a single source, so a transport defined +via `sparkConf` would discard the environment-provided authentication. +Consequently, do **not** override `spark.openlineage.transport.*` via `sparkConf` when using +`credentialsSecretName` — doing so drops the credentials and the backend will reject the events. +==== + +[#job-name] +== Job name + +The OpenLineage job name is resolved in the following order: + +. `spec.openLineage.appName`, if set. +. `spark.app.name` from `sparkConf`, if set. +. `metadata.name` as a last resort. + +The last case additionally emits a Kubernetes warning event. +Falling back to `metadata.name` is discouraged: if that name carries a timestamp or other run-specific suffix (as generated names often do), every run becomes a separate job in the backend and the run history fragments. +Set `spec.openLineage.appName` (or `spark.app.name`) to a stable value to keep all runs of the same logical job together. + +== Overriding injected configuration + +Every injected value is a default that goes into the submit configuration *before* your xref:usage-guide/overrides.adoc[`sparkConf`] is merged, so any key you set yourself wins. +`spark.extraListeners` and `spark.jars` are the exception: the operator *appends* to any value you provide (comma-merged) rather than overwriting it, so your own listeners and jars are kept alongside OpenLineage's. diff --git a/docs/modules/spark-k8s/partials/nav.adoc b/docs/modules/spark-k8s/partials/nav.adoc index 71f2278e..815c58dd 100644 --- a/docs/modules/spark-k8s/partials/nav.adoc +++ b/docs/modules/spark-k8s/partials/nav.adoc @@ -9,6 +9,7 @@ ** xref:spark-k8s:usage-guide/app_templates.adoc[] ** xref:spark-k8s:usage-guide/security.adoc[] ** xref:spark-k8s:usage-guide/logging.adoc[] +** xref:spark-k8s:usage-guide/openlineage.adoc[] ** xref:spark-k8s:usage-guide/history-server.adoc[] ** xref:spark-k8s:usage-guide/spark-connect.adoc[] ** xref:spark-k8s:usage-guide/examples.adoc[] diff --git a/extra/crds.yaml b/extra/crds.yaml index ab28308f..9d4a6080 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -1957,6 +1957,114 @@ spec: - cluster - client type: string + openLineage: + description: |- + Emit [OpenLineage](https://openlineage.io/) lineage events for this application. + The OpenLineage Spark listener runs on the driver and describes the whole application, + so this is application-scoped config (not per driver/executor role). + The backend connection is either inlined or references an `OpenLineageConnection` resource. + See the [OpenLineage usage guide](https://docs.stackable.tech/home/nightly/spark-k8s/usage-guide/openlineage). + nullable: true + properties: + appName: + description: |- + A stable OpenLineage job/application name. Setting this prevents fragmented run history. + If unset, operators resolve a name from workload-specific configuration. + nullable: true + type: string + connection: + description: |- + The OpenLineage backend connection, either inlined or referencing an + `OpenLineageConnection` resource by name. + oneOf: + - required: + - inline + - required: + - reference + properties: + inline: + description: |- + OpenLineage connection definition as a resource. + Learn more about [OpenLineage](https://openlineage.io/). + properties: + credentialsSecretName: + description: |- + Name of a Secret containing the API key used to authenticate against the OpenLineage + backend. The API key must be stored under the key `apiKey`. The Secret must be located in + the same namespace as the workload using this connection. If not specified, no + authentication is used. + nullable: true + type: string + host: + description: 'Host of the OpenLineage backend without any protocol or port. For example: `marquez`.' + type: string + port: + description: 'Port the OpenLineage backend listens on. For example: `5000`.' + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + tls: + description: Use a TLS connection. If not specified no TLS will be used. + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client. + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: Use TLS but don't verify certificates. + type: object + server: + description: Use TLS and a CA certificate to verify the server. + properties: + caCert: + description: CA cert to verify the server. + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: |- + Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. + Note that a SecretClass does not need to have a key but can also work with just a CA certificate, + so if you got provided with a CA cert but don't have access to the key you can still use this method. + type: string + webPki: + description: |- + Use TLS and the CA certificates trusted by the common web browsers to verify the server. + This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + required: + - host + - port + type: object + reference: + type: string + type: object + namespace: + description: |- + The OpenLineage namespace lineage is reported under. + If unset, operators typically default to the workload's Kubernetes namespace. + nullable: true + type: string + required: + - connection + type: object s3connection: description: |- Configure an S3 connection that the SparkApplication has access to. @@ -6563,6 +6671,114 @@ spec: - cluster - client type: string + openLineage: + description: |- + Emit [OpenLineage](https://openlineage.io/) lineage events for this application. + The OpenLineage Spark listener runs on the driver and describes the whole application, + so this is application-scoped config (not per driver/executor role). + The backend connection is either inlined or references an `OpenLineageConnection` resource. + See the [OpenLineage usage guide](https://docs.stackable.tech/home/nightly/spark-k8s/usage-guide/openlineage). + nullable: true + properties: + appName: + description: |- + A stable OpenLineage job/application name. Setting this prevents fragmented run history. + If unset, operators resolve a name from workload-specific configuration. + nullable: true + type: string + connection: + description: |- + The OpenLineage backend connection, either inlined or referencing an + `OpenLineageConnection` resource by name. + oneOf: + - required: + - inline + - required: + - reference + properties: + inline: + description: |- + OpenLineage connection definition as a resource. + Learn more about [OpenLineage](https://openlineage.io/). + properties: + credentialsSecretName: + description: |- + Name of a Secret containing the API key used to authenticate against the OpenLineage + backend. The API key must be stored under the key `apiKey`. The Secret must be located in + the same namespace as the workload using this connection. If not specified, no + authentication is used. + nullable: true + type: string + host: + description: 'Host of the OpenLineage backend without any protocol or port. For example: `marquez`.' + type: string + port: + description: 'Port the OpenLineage backend listens on. For example: `5000`.' + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + tls: + description: Use a TLS connection. If not specified no TLS will be used. + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client. + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: Use TLS but don't verify certificates. + type: object + server: + description: Use TLS and a CA certificate to verify the server. + properties: + caCert: + description: CA cert to verify the server. + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: |- + Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. + Note that a SecretClass does not need to have a key but can also work with just a CA certificate, + so if you got provided with a CA cert but don't have access to the key you can still use this method. + type: string + webPki: + description: |- + Use TLS and the CA certificates trusted by the common web browsers to verify the server. + This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + required: + - host + - port + type: object + reference: + type: string + type: object + namespace: + description: |- + The OpenLineage namespace lineage is reported under. + If unset, operators typically default to the workload's Kubernetes namespace. + nullable: true + type: string + required: + - connection + type: object s3connection: description: |- Configure an S3 connection that the SparkApplication has access to. diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/config/jvm.rs index cc5388b8..aab8fd4d 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/config/jvm.rs @@ -1,11 +1,13 @@ -use stackable_operator::crd::s3; +use stackable_operator::crd::{openlineage, s3}; use crate::crd::{ + Error, constants::{ - JVM_SECURITY_PROPERTIES_FILE, STACKABLE_TLS_STORE_PASSWORD, STACKABLE_TRUST_STORE, - VOLUME_MOUNT_PATH_LOG_CONFIG, + JVM_SECURITY_PROPERTIES_FILE, OPENLINEAGE_ADD_OPENS, STACKABLE_TLS_STORE_PASSWORD, + STACKABLE_TRUST_STORE, VOLUME_MOUNT_PATH_LOG_CONFIG, }, logdir::ResolvedLogDir, + spark_major_version, tlscerts::tls_secret_names, v1alpha1::SparkApplication, }; @@ -16,11 +18,16 @@ use crate::crd::{ /// /// Returns `(driver, executor)`: the operator-generated base arguments with the role's /// `jvmArgumentOverrides` applied on top. +/// +/// `product_version` is the resolved Spark product version (e.g. `4.1.2`); it gates the +/// version-specific OpenLineage `--add-opens` flag. pub fn construct_extra_java_options( spark_application: &SparkApplication, s3_conn: &Option, log_dir: &Option, -) -> (String, String) { + product_version: &str, + open_lineage_conn: Option<&openlineage::ResolvedOpenLineageConnection>, +) -> Result<(String, String), Error> { // Note (@sbernauer): As of 2025-03-04, we did not set any heap related JVM arguments, so I // kept the implementation as is. We can always re-visit this as needed. @@ -28,7 +35,7 @@ pub fn construct_extra_java_options( "-Djava.security.properties={VOLUME_MOUNT_PATH_LOG_CONFIG}/{JVM_SECURITY_PROPERTIES_FILE}" )]; - if tls_secret_names(s3_conn, log_dir).is_some() { + if tls_secret_names(s3_conn, log_dir, open_lineage_conn).is_some() { jvm_args.extend([ format!("-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}/truststore.p12"), format!("-Djavax.net.ssl.trustStorePassword={STACKABLE_TLS_STORE_PASSWORD}"), @@ -36,6 +43,16 @@ pub fn construct_extra_java_options( ]); } + // OpenLineage on Spark 4.x needs `java.base/java.security` opened to the unnamed module, + // otherwise the driver throws a non-fatal `InaccessibleObjectException` and silently degrades + // extension-interface lineage (MVP §7). Added to both driver and executor. + // + // This is scoped to Spark 4.x: the flag is unnecessary — and on the JDK 17 Spark 3.5.x images the + // operator also ships, potentially a startup error — so it must not be emitted there. + if spark_application.openlineage_enabled() && spark_major_version(product_version)? >= 4 { + jvm_args.push(OPENLINEAGE_ADD_OPENS.to_string()); + } + // The role's `jvmArgumentOverrides` are applied on top of the operator-generated arguments // above. Note this is not purely additive: a role may also remove or replace operator-set // arguments (e.g. a `removeRegex` dropping the `-Djava.security.properties` default) — see the @@ -56,16 +73,24 @@ pub fn construct_extra_java_options( None => jvm_args.clone(), }; - (driver.join(" "), executor.join(" ")) + Ok((driver.join(" "), executor.join(" "))) } #[cfg(test)] mod tests { + use rstest::rstest; + use super::*; + fn spark_app_from_yaml(input: &str) -> SparkApplication { + let deserializer = serde_yaml::Deserializer::from_str(input); + serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap() + } + #[test] fn test_construct_jvm_arguments_defaults() { - let input = r#" + let spark_app = spark_app_from_yaml( + r#" apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication metadata: @@ -74,14 +99,11 @@ mod tests { mode: cluster mainApplicationFile: test.py sparkImage: - productVersion: 1.2.3 - "#; - - let deserializer = serde_yaml::Deserializer::from_str(input); - let spark_app: SparkApplication = - serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); + productVersion: 4.1.2 + "#, + ); let (driver_extra_java_options, executor_extra_java_options) = - construct_extra_java_options(&spark_app, &None, &None); + construct_extra_java_options(&spark_app, &None, &None, "4.1.2", None).unwrap(); assert_eq!( driver_extra_java_options, @@ -95,7 +117,8 @@ mod tests { #[test] fn test_construct_jvm_argument_overrides() { - let input = r#" + let spark_app = spark_app_from_yaml( + r#" apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication metadata: @@ -104,7 +127,7 @@ mod tests { mode: cluster mainApplicationFile: test.py sparkImage: - productVersion: 1.2.3 + productVersion: 4.1.2 driver: jvmArgumentOverrides: add: @@ -115,13 +138,10 @@ mod tests { - -Dhttps.proxyHost=from-executor removeRegex: - -Djava.security.properties=.* - "#; - - let deserializer = serde_yaml::Deserializer::from_str(input); - let spark_app: SparkApplication = - serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); + "#, + ); let (driver_extra_java_options, executor_extra_java_options) = - construct_extra_java_options(&spark_app, &None, &None); + construct_extra_java_options(&spark_app, &None, &None, "4.1.2", None).unwrap(); assert_eq!( driver_extra_java_options, @@ -132,4 +152,60 @@ mod tests { "-Dhttps.proxyHost=from-executor" ); } + + const OPENLINEAGE_ENABLED: &str = r#" + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: spark-example + spec: + mode: cluster + mainApplicationFile: test.py + sparkImage: + productVersion: PLACEHOLDER + openLineage: + connection: + inline: + host: marquez + port: 5000 + "#; + + const OPENLINEAGE_ABSENT: &str = r#" + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: spark-example + spec: + mode: cluster + mainApplicationFile: test.py + sparkImage: + productVersion: PLACEHOLDER + "#; + + /// `--add-opens` is emitted only on Spark 4.x with OpenLineage enabled — never on the Scala + /// 2.12 / JDK 17 Spark 3.5.x images, and never when OpenLineage is absent. + #[rstest] + #[case::enabled_spark_3(OPENLINEAGE_ENABLED, "3.5.8", false)] + #[case::enabled_spark_4(OPENLINEAGE_ENABLED, "4.1.2", true)] + #[case::absent_spark_4(OPENLINEAGE_ABSENT, "4.1.2", false)] + fn test_openlineage_add_opens_is_version_gated( + #[case] yaml_template: &str, + #[case] product_version: &str, + #[case] expect_add_opens: bool, + ) { + let spark_app = spark_app_from_yaml(&yaml_template.replace("PLACEHOLDER", product_version)); + let (driver_extra_java_options, executor_extra_java_options) = + construct_extra_java_options(&spark_app, &None, &None, product_version, None).unwrap(); + + assert_eq!( + driver_extra_java_options.contains(OPENLINEAGE_ADD_OPENS), + expect_add_opens, + "driver --add-opens presence mismatch for Spark {product_version}" + ); + assert_eq!( + executor_extra_java_options.contains(OPENLINEAGE_ADD_OPENS), + expect_add_opens, + "executor --add-opens presence mismatch for Spark {product_version}" + ); + } } diff --git a/rust/operator-binary/src/crd/constants.rs b/rust/operator-binary/src/crd/constants.rs index efef0f14..09e584b5 100644 --- a/rust/operator-binary/src/crd/constants.rs +++ b/rust/operator-binary/src/crd/constants.rs @@ -103,6 +103,60 @@ pub const DEFAULT_LISTENER_CLASS: &str = "cluster-internal"; pub const DEFAULT_SUBMIT_JOB_RETRY_ON_FAILURE_COUNT: u16 = 0; +// --- OpenLineage (see the OpenLineage usage guide) --- + +/// The OpenLineage Spark listener, appended to `spark.extraListeners` when OpenLineage is enabled. +pub const OPENLINEAGE_LISTENER_CLASS: &str = "io.openlineage.spark.agent.OpenLineageSparkListener"; + +/// `local://` URI of the OpenLineage jar baked into the Spark image, referenced via `spark.jars` so +/// it shares the (child) classloader with `--packages` connectors. Delivering it this way — rather +/// than on `extraClassPath` (system classloader) — is what lets OpenLineage's connector probes +/// succeed; see the classpath discussion in the OpenLineage usage guide. +/// +/// This is a stable, version- and Scala-independent symlink the image maintains (mirroring the +/// `jmx_prometheus_javaagent.jar` pattern): each Spark image bakes the correct +/// `openlineage-spark_-.jar` for its build (Scala 2.13 for Spark 4.x, Scala 2.12 for +/// Spark 3.5.x) and symlinks it here. Referencing the symlink decouples the operator from the jar's +/// version and Scala suffix, so bumping either only touches `docker-images/spark-k8s`. +pub const OPENLINEAGE_JAR_LOCAL_URI: &str = + "local:///stackable/spark/openlineage/openlineage-spark.jar"; + +/// Java module-system flag OpenLineage requires on Spark 4.x: without it the driver throws a +/// non-fatal `InaccessibleObjectException` and silently degrades extension-interface lineage. +/// Appended to both driver and executor `extraJavaOptions`. +pub const OPENLINEAGE_ADD_OPENS: &str = "--add-opens java.base/java.security=ALL-UNNAMED"; + +/// Env vars delivering the OpenLineage HTTP transport config via the OpenLineage Java client's +/// `OPENLINEAGE__` env-var configuration (`__` separates the config hierarchy). +/// +/// When an `AuthenticationClass` is configured, the operator delivers the **whole** transport this +/// way rather than via `spark.openlineage.transport.*` `--conf`. This is required, not cosmetic: +/// OpenLineage resolves `transport` from a single source, so a `--conf`-defined transport would +/// drop the env-provided `auth` sub-tree entirely (verified against openlineage-spark 1.51.0). +/// Delivering the token as a `secretKeyRef` env var also keeps it out of the spark-submit args. +pub const OPENLINEAGE_TRANSPORT_TYPE_ENV: &str = "OPENLINEAGE__TRANSPORT__TYPE"; + +/// Env var carrying the OpenLineage HTTP transport URL. See [`OPENLINEAGE_TRANSPORT_TYPE_ENV`]. +pub const OPENLINEAGE_TRANSPORT_URL_ENV: &str = "OPENLINEAGE__TRANSPORT__URL"; + +/// The OpenLineage transport type the operator uses (the HTTP transport). +pub const OPENLINEAGE_TRANSPORT_TYPE_HTTP: &str = "http"; + +/// Env var selecting the OpenLineage HTTP transport auth type. See [`OPENLINEAGE_TRANSPORT_TYPE_ENV`]. +pub const OPENLINEAGE_AUTH_TYPE_ENV: &str = "OPENLINEAGE__TRANSPORT__AUTH__TYPE"; + +/// Env var carrying the OpenLineage HTTP transport bearer token. Sourced from the referenced +/// Secret via `secretKeyRef` (key [`OPENLINEAGE_AUTH_SECRET_KEY`]), so the operator never reads +/// the token and it stays out of the Job/pod spec. +pub const OPENLINEAGE_AUTH_API_KEY_ENV: &str = "OPENLINEAGE__TRANSPORT__AUTH__API_KEY"; + +/// The only OpenLineage HTTP transport auth type the operator supports: a static bearer token. +pub const OPENLINEAGE_AUTH_TYPE_API_KEY: &str = "api_key"; + +/// Fixed key that must hold the bearer token inside the Secret referenced by the Static +/// `AuthenticationClass`. +pub const OPENLINEAGE_AUTH_SECRET_KEY: &str = "apiKey"; + /// The JVM `security.properties` entries the operator sets by default (DNS cache TTLs). pub fn default_jvm_security_properties() -> BTreeMap { [ diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index a456b1d9..637fde55 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -24,7 +24,7 @@ use stackable_operator::{ fragment::{self, ValidationError}, merge::Merge, }, - crd::s3, + crd::{openlineage, s3}, k8s_openapi::{ api::core::v1::{EmptyDirVolumeSource, EnvVar, PodTemplateSpec, Volume, VolumeMount}, apimachinery::pkg::api::resource::Quantity, @@ -52,6 +52,7 @@ use crate::{ SubmitConfig, SubmitConfigFragment, VolumeMounts, }, }, + openlineage::{append_conf_csv, openlineage_transport_env_vars}, }; pub mod affinity; @@ -76,7 +77,7 @@ pub enum Error { #[snafu(display("object defines no application artifact"))] ObjectHasNoArtifact, - #[snafu(display("object has no name"))] + #[snafu(display("object has no name"), visibility(pub(crate)))] ObjectHasNoName, #[snafu(display("application has no Spark image"))] @@ -114,6 +115,28 @@ pub enum Error { #[snafu(display("failed to configure log directory"))] ConfigureLogDir { source: logdir::Error }, + + #[snafu(display( + "failed to parse the Spark major version from the resolved product version {product_version:?}" + ))] + UnparseableSparkVersion { + source: std::num::ParseIntError, + product_version: String, + }, +} + +/// Parses the Spark major version from a resolved product version string such as `4.1.2` or +/// `3.5.8`. Used to gate version-specific OpenLineage wiring — the `--add-opens` JVM flag, which is +/// only needed on (and safe for) the JDK 17+/Spark 4.x line and can break the Spark 3.5.x JVMs. +pub(crate) fn spark_major_version(product_version: &str) -> Result { + product_version + .split('.') + .next() + .unwrap_or_default() + .parse::() + .context(UnparseableSparkVersionSnafu { + product_version: product_version.to_string(), + }) } pub type SparkApplicationJobRoleType = @@ -248,6 +271,14 @@ pub mod versioned { /// The log file directory definition used by the Spark history server. #[serde(default, skip_serializing_if = "Option::is_none")] pub log_file_directory: Option, + + /// Emit [OpenLineage](https://openlineage.io/) lineage events for this application. + /// The OpenLineage Spark listener runs on the driver and describes the whole application, + /// so this is application-scoped config (not per driver/executor role). + /// The backend connection is either inlined or references an `OpenLineageConnection` resource. + /// See the [OpenLineage usage guide](DOCS_BASE_URL_PLACEHOLDER/spark-k8s/usage-guide/openlineage). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub open_lineage: Option, } #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize, Merge)] @@ -306,6 +337,7 @@ impl v1alpha1::SparkApplication { logdir: &Option, log_config_map: Option<&str>, requested_secret_lifetime: &Duration, + open_lineage_conn: Option<&openlineage::ResolvedOpenLineageConnection>, ) -> Result, Error> { // Collect the volumes in a map keyed by name to avoid duplicates. // Duplicates can happen when the the S3 credentials volume and the history server log directory use the same secret class. @@ -386,7 +418,7 @@ impl v1alpha1::SparkApplication { .build(), ); } - if let Some(cert_secrets) = tlscerts::tls_secret_names(s3conn, logdir) { + if let Some(cert_secrets) = tlscerts::tls_secret_names(s3conn, logdir, open_lineage_conn) { result.insert( STACKABLE_TRUST_STORE_NAME.to_string(), VolumeBuilder::new(STACKABLE_TRUST_STORE_NAME) @@ -439,6 +471,7 @@ impl v1alpha1::SparkApplication { &self, s3conn: &Option, logdir: &Option, + open_lineage_conn: Option<&openlineage::ResolvedOpenLineageConnection>, ) -> Vec { let mut tmpl_mounts = vec![ VolumeMount { @@ -453,7 +486,8 @@ impl v1alpha1::SparkApplication { }, ]; - tmpl_mounts = self.add_common_volume_mounts(tmpl_mounts, s3conn, logdir, false); + tmpl_mounts = + self.add_common_volume_mounts(tmpl_mounts, s3conn, logdir, false, open_lineage_conn); if let Some(CommonConfiguration { config: @@ -479,6 +513,7 @@ impl v1alpha1::SparkApplication { s3conn: &Option, logdir: &Option, logging_enabled: bool, + open_lineage_conn: Option<&openlineage::ResolvedOpenLineageConnection>, ) -> Vec { if self.spec.image.is_some() { mounts.push(VolumeMount { @@ -537,7 +572,7 @@ impl v1alpha1::SparkApplication { ..VolumeMount::default() }); } - if let Some(cert_secrets) = tlscerts::tls_secret_names(s3conn, logdir) { + if let Some(cert_secrets) = tlscerts::tls_secret_names(s3conn, logdir, open_lineage_conn) { mounts.push(VolumeMount { name: STACKABLE_TRUST_STORE_NAME.into(), mount_path: STACKABLE_TRUST_STORE.into(), @@ -556,28 +591,37 @@ impl v1alpha1::SparkApplication { mounts } + /// True when OpenLineage emission is configured for this application. OpenLineage is off exactly + /// when the `openLineage` block is absent — its presence is the enable switch. + pub fn openlineage_enabled(&self) -> bool { + self.spec.open_lineage.is_some() + } + pub fn build_command( &self, s3conn: &Option, log_dir: &Option, spark_image: &str, + product_version: &str, + open_lineage_conn: Option<&openlineage::ResolvedOpenLineageConnection>, ) -> Result, Error> { // mandatory properties let mode = &self.spec.mode; let name = self.metadata.name.clone().context(ObjectHasNoNameSnafu)?; // Commands needed to build the p12 trust store from the secret class certs configured for - // S3 connections. - let build_truststore_commands = match tlscerts::tls_secret_names(s3conn, log_dir) { - Some(cert_secrets) => { - let mut build_truststore_str = - vec![tlscerts::convert_system_trust_store_to_pkcs12()]; - build_truststore_str - .extend(cert_secrets.into_iter().map(tlscerts::import_truststore)); - format!("{};", build_truststore_str.join(" && ")) - } - None => "".to_string(), - }; + // S3 connections, the log directory, and the OpenLineage backend connection. + let build_truststore_commands = + match tlscerts::tls_secret_names(s3conn, log_dir, open_lineage_conn) { + Some(cert_secrets) => { + let mut build_truststore_str = + vec![tlscerts::convert_system_trust_store_to_pkcs12()]; + build_truststore_str + .extend(cert_secrets.into_iter().map(tlscerts::import_truststore)); + format!("{};", build_truststore_str.join(" && ")) + } + None => "".to_string(), + }; let mut submit_cmd = vec![ format!( @@ -659,7 +703,13 @@ impl v1alpha1::SparkApplication { } let (driver_extra_java_options, executor_extra_java_options) = - construct_extra_java_options(self, s3conn, log_dir); + construct_extra_java_options( + self, + s3conn, + log_dir, + product_version, + open_lineage_conn, + )?; submit_cmd.extend(vec![ format!("--conf spark.driver.extraJavaOptions=\"{driver_extra_java_options}\""), format!("--conf spark.executor.extraJavaOptions=\"{executor_extra_java_options}\""), @@ -733,9 +783,62 @@ impl v1alpha1::SparkApplication { submit_cmd.push(format!("--conf spark.jars.ivy={VOLUME_MOUNT_PATH_IVY2}")) } + // OpenLineage: inject the transport + job-name config that can be overridden by the user's + // `sparkConf` (so it goes in BEFORE the merge below). The two append-merge keys + // (`spark.jars`, `spark.extraListeners`) are handled AFTER the merge so a user value is + // combined with — not clobbered by — ours. See the OpenLineage usage guide. + if let Some(open_lineage) = &self.spec.open_lineage { + // The backend connection is resolved (inline or referenced) before `build_command`. + // The transport type is always `http` (the OpenLineage HTTP transport); the URL scheme + // it points at is `https` when the connection configures TLS server verification. + // + // When authentication is configured (the connection sets `credentialsSecretName`), the + // transport is delivered entirely via `OPENLINEAGE__` env vars instead (see `env`) — it + // MUST come from a single source, or OpenLineage drops the env-provided `auth`. So skip + // the `--conf` transport in that case. + if let Some(connection) = open_lineage_conn + && connection.credentials_secret_name.is_none() + { + submit_conf.insert( + "spark.openlineage.transport.type".to_string(), + "http".to_string(), + ); + submit_conf.insert( + "spark.openlineage.transport.url".to_string(), + connection.transport_url(), + ); + } + + let namespace = match &open_lineage.namespace { + Some(namespace) => namespace.clone(), + None => self.metadata.namespace.clone().context(NoNamespaceSnafu)?, + }; + submit_conf.insert("spark.openlineage.namespace".to_string(), namespace); + + // Stable job name — fixes the intermittent `unknown` bug (see the usage guide). + submit_conf.insert( + "spark.openlineage.appName".to_string(), + self.resolved_openlineage_app_name()?, + ); + } + // conf arguments: these should follow - and thus override - values set from resource limits above submit_conf.extend(self.spec.spark_conf.clone()); + // OpenLineage append-merge keys: combine our values with any user-provided ones (already + // merged into `submit_conf` above) so neither clobbers the other. + if self.openlineage_enabled() { + append_conf_csv( + &mut submit_conf, + "spark.extraListeners", + OPENLINEAGE_LISTENER_CLASS, + ); + // Reference the stable symlink the image maintains; it points at the correct + // Scala/version build for this image (2.13 on Spark 4.x, 2.12 on Spark 3.5.x), so the + // operator needs to track neither the jar version nor its Scala suffix. + append_conf_csv(&mut submit_conf, "spark.jars", OPENLINEAGE_JAR_LOCAL_URI); + } + // ...before being added to the command collection for (key, value) in submit_conf { submit_cmd.push(format!("--conf \"{key}={value}\"")); @@ -760,6 +863,7 @@ impl v1alpha1::SparkApplication { &self, s3conn: &Option, logdir: &Option, + open_lineage_conn: Option<&openlineage::ResolvedOpenLineageConnection>, ) -> Vec { let mut e: Vec = self.spec.env.clone(); @@ -792,13 +896,26 @@ impl v1alpha1::SparkApplication { value_from: None, }); } - if tlscerts::tls_secret_names(s3conn, logdir).is_some() { + if tlscerts::tls_secret_names(s3conn, logdir, open_lineage_conn).is_some() { e.push(EnvVar { name: "STACKABLE_TLS_STORE_PASSWORD".to_string(), value: Some(STACKABLE_TLS_STORE_PASSWORD.to_string()), value_from: None, }); } + // OpenLineage HTTP transport auth: when the connection sets `credentialsSecretName`, deliver + // the WHOLE transport (type, URL and bearer-token auth) to the driver's OpenLineage listener + // via the client's `OPENLINEAGE__` env-var configuration — see `openlineage_transport_env_vars` + // and the matching skip of the `spark.openlineage.transport.*` `--conf` in `build_command`. + // The token is sourced from the referenced Secret, so it never lands in the `--conf` args. + if let Some(connection) = open_lineage_conn + && let Some(secret_name) = &connection.credentials_secret_name + { + e.extend(openlineage_transport_env_vars( + &connection.transport_url(), + secret_name, + )); + } e } @@ -1542,7 +1659,7 @@ spec: "#}) .unwrap(); - let got = spark_application.spark_job_volume_mounts(&None, &None); + let got = spark_application.spark_job_volume_mounts(&None, &None, None); let expected = vec![ VolumeMount { @@ -1574,6 +1691,477 @@ spec: assert_eq!(got, expected); } + /// Builds a `SparkApplication` from YAML and returns the assembled spark-submit command string. + /// + /// Test fixtures always use an inline OpenLineage connection, so the connection is extracted + /// directly (no async `resolve()` against a k8s API) and passed to `build_command` — exactly the + /// resolved value the controller would compute in `dereference`. + fn build_command_with_openlineage(yaml: &str, product_version: &str) -> String { + use stackable_operator::crd::openlineage::v1alpha1::InlineConnectionOrReference; + + // `singleton_map_recursive` so externally-tagged enums (the OpenLineage `connection`) use + // the `inline:`/`reference:` map form that k8s/JSON uses, rather than serde_yaml's `!tag`. + let deserializer = serde_yaml::Deserializer::from_str(yaml); + let spark_application: v1alpha1::SparkApplication = + serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); + let open_lineage_conn = spark_application + .spec + .open_lineage + .as_ref() + .map(|job| match &job.connection { + InlineConnectionOrReference::Inline(spec) => spec.clone(), + InlineConnectionOrReference::Reference(name) => { + panic!("test fixtures must use an inline connection, got reference {name:?}") + } + }); + spark_application + .build_command( + &None, + &None, + "test-image", + product_version, + open_lineage_conn.as_ref(), + ) + .unwrap() + .join(" ") + } + + const MINIMAL_APP: &str = indoc! {r#" + --- + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: test + namespace: default + spec: + mode: cluster + mainApplicationFile: test.py + sparkImage: + productVersion: 1.2.3 + "#}; + + fn test_openlineage_connection( + credentials_secret_name: Option, + ) -> openlineage::ResolvedOpenLineageConnection { + openlineage::v1alpha1::OpenLineageConnectionSpec { + host: "marquez".to_string(), + port: 5000, + tls: stackable_operator::commons::tls_verification::TlsClientDetails { tls: None }, + credentials_secret_name, + } + } + + #[test] + fn test_openlineage_full_transport_delivered_via_env_when_authenticated() { + let spark_application = + serde_yaml::from_str::(MINIMAL_APP).unwrap(); + let connection = test_openlineage_connection(Some("ol-token".to_string())); + + let env = spark_application.env(&None, &None, Some(&connection)); + let value = |name| { + env.iter() + .find(|v| v.name == name) + .and_then(|v| v.value.as_deref()) + }; + + // The whole transport is delivered via env (not --conf) so OpenLineage keeps the auth. + assert_eq!(value(OPENLINEAGE_TRANSPORT_TYPE_ENV), Some("http")); + assert_eq!( + value(OPENLINEAGE_TRANSPORT_URL_ENV), + Some("http://marquez:5000") + ); + assert_eq!( + value(OPENLINEAGE_AUTH_TYPE_ENV), + Some(OPENLINEAGE_AUTH_TYPE_API_KEY) + ); + + let key_var = env + .iter() + .find(|v| v.name == OPENLINEAGE_AUTH_API_KEY_ENV) + .expect("auth api key env var present"); + assert!( + key_var.value.is_none(), + "the token must never be a literal env value" + ); + let selector = key_var + .value_from + .as_ref() + .and_then(|source| source.secret_key_ref.as_ref()) + .expect("api key sourced from a secretKeyRef"); + assert_eq!(selector.name, "ol-token"); + assert_eq!(selector.key, OPENLINEAGE_AUTH_SECRET_KEY); + } + + #[test] + fn test_no_openlineage_transport_env_vars_when_unauthenticated() { + let spark_application = + serde_yaml::from_str::(MINIMAL_APP).unwrap(); + let connection = test_openlineage_connection(None); + + // A connection but no credentialsSecretName: transport stays in --conf, no OL env vars. + let env = spark_application.env(&None, &None, Some(&connection)); + + for name in [ + OPENLINEAGE_TRANSPORT_TYPE_ENV, + OPENLINEAGE_TRANSPORT_URL_ENV, + OPENLINEAGE_AUTH_TYPE_ENV, + OPENLINEAGE_AUTH_API_KEY_ENV, + ] { + assert!( + !env.iter().any(|v| v.name == name), + "unexpected OpenLineage env var {name} when unauthenticated" + ); + } + } + + #[test] + fn test_openlineage_authenticated_omits_transport_conf() { + // With auth present the transport moves to env vars, so build_command must NOT emit the + // `spark.openlineage.transport.*` --conf (that would make OpenLineage drop the env auth). + // namespace / appName / listener stay in --conf (OPENLINEAGE_ENABLED_APP has the block). + let spark_application: v1alpha1::SparkApplication = + serde_yaml::with::singleton_map_recursive::deserialize( + serde_yaml::Deserializer::from_str(OPENLINEAGE_ENABLED_APP), + ) + .unwrap(); + let connection = test_openlineage_connection(Some("ol-token".to_string())); + + let command = spark_application + .build_command(&None, &None, "test-image", "3.5.8", Some(&connection)) + .unwrap() + .join(" "); + + assert!( + !command.contains("spark.openlineage.transport.type"), + "transport.type must not be in --conf when authenticated" + ); + assert!( + !command.contains("spark.openlineage.transport.url"), + "transport.url must not be in --conf when authenticated" + ); + assert!(command.contains(r#"--conf "spark.openlineage.namespace="#)); + assert!(command.contains(r#"--conf "spark.openlineage.appName="#)); + assert!(command.contains(&format!( + r#"--conf "spark.extraListeners={OPENLINEAGE_LISTENER_CLASS}""# + ))); + } + + const OPENLINEAGE_ENABLED_APP: &str = indoc! {r#" + --- + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: spark-examples + namespace: default + spec: + mode: cluster + mainApplicationFile: test.py + sparkImage: + productVersion: 1.2.3 + openLineage: + connection: + inline: + host: marquez + port: 5000 + "#}; + + #[test] + fn test_openlineage_injects_conf_when_enabled() { + let command = build_command_with_openlineage(OPENLINEAGE_ENABLED_APP, "4.1.2"); + + assert!(command.contains(r#"--conf "spark.openlineage.transport.type=http""#)); + assert!( + command.contains(r#"--conf "spark.openlineage.transport.url=http://marquez:5000""#) + ); + // Namespace defaults to metadata.namespace. + assert!(command.contains(r#"--conf "spark.openlineage.namespace=default""#)); + // appName falls back to metadata.name (no appName / spark.app.name set). + assert!(command.contains(r#"--conf "spark.openlineage.appName=spark-examples""#)); + assert!(command.contains(&format!( + r#"--conf "spark.extraListeners={OPENLINEAGE_LISTENER_CLASS}""# + ))); + // The jar is referenced via the stable, Scala/version-independent image symlink. + assert!(command.contains(&format!( + r#"--conf "spark.jars={OPENLINEAGE_JAR_LOCAL_URI}""# + ))); + // --add-opens reaches both driver and executor on Spark 4.x. + assert_eq!( + command.matches(OPENLINEAGE_ADD_OPENS).count(), + 2, + "expected --add-opens on both driver and executor extraJavaOptions" + ); + } + + #[test] + fn test_openlineage_stable_jar_uri_and_no_add_opens_on_spark_3() { + // On the JDK 17 Spark 3.5.x images the operator references the same stable jar symlink (the + // image points it at the Scala 2.12 build) and must NOT emit `--add-opens`. + let command = build_command_with_openlineage(OPENLINEAGE_ENABLED_APP, "3.5.8"); + + assert!(command.contains(&format!( + r#"--conf "spark.jars={OPENLINEAGE_JAR_LOCAL_URI}""# + ))); + assert!( + !command.contains(OPENLINEAGE_ADD_OPENS), + "--add-opens must not be emitted on Spark 3.x" + ); + // The rest of the OpenLineage wiring is still present on Spark 3.x. + assert!(command.contains(&format!( + r#"--conf "spark.extraListeners={OPENLINEAGE_LISTENER_CLASS}""# + ))); + assert!(command.contains(r#"--conf "spark.openlineage.transport.type=http""#)); + } + + #[test] + fn test_openlineage_injects_nothing_when_absent() { + // No `openLineage` block → OpenLineage is off (its absence is the disable switch). + let yaml = indoc! {r#" + --- + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: spark-examples + namespace: default + spec: + mode: cluster + mainApplicationFile: test.py + sparkImage: + productVersion: 1.2.3 + "#}; + let command = build_command_with_openlineage(yaml, "4.1.2"); + + assert!(!command.contains("openlineage")); + assert!(!command.contains("OpenLineageSparkListener")); + assert!(!command.contains("--add-opens")); + assert!(!command.contains("spark.jars=")); + } + + #[test] + fn test_openlineage_appends_to_existing_extra_listeners() { + let yaml = indoc! {r#" + --- + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: spark-examples + namespace: default + spec: + mode: cluster + mainApplicationFile: test.py + sparkImage: + productVersion: 1.2.3 + openLineage: + connection: + inline: + host: marquez + port: 5000 + sparkConf: + spark.extraListeners: com.example.CustomListener + "#}; + let command = build_command_with_openlineage(yaml, "4.1.2"); + + assert!(command.contains(&format!( + r#"--conf "spark.extraListeners=com.example.CustomListener,{OPENLINEAGE_LISTENER_CLASS}""# + ))); + } + + #[test] + fn test_openlineage_conf_is_overridable_via_spark_conf() { + let yaml = indoc! {r#" + --- + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: spark-examples + namespace: default + spec: + mode: cluster + mainApplicationFile: test.py + sparkImage: + productVersion: 1.2.3 + openLineage: + connection: + inline: + host: marquez + port: 5000 + sparkConf: + spark.openlineage.transport.url: http://custom:1234 + spark.openlineage.namespace: custom-ns + "#}; + let command = build_command_with_openlineage(yaml, "4.1.2"); + + assert!(command.contains(r#"--conf "spark.openlineage.transport.url=http://custom:1234""#)); + assert!(!command.contains("http://marquez:5000")); + assert!(command.contains(r#"--conf "spark.openlineage.namespace=custom-ns""#)); + } + + #[test] + fn test_openlineage_uses_https_when_tls_verification_set() { + let yaml = indoc! {r#" + --- + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: spark-examples + namespace: default + spec: + mode: cluster + mainApplicationFile: test.py + sparkImage: + productVersion: 1.2.3 + openLineage: + connection: + inline: + host: marquez + port: 5000 + tls: + verification: + server: + caCert: + secretClass: marquez-ca + "#}; + let command = build_command_with_openlineage(yaml, "4.1.2"); + + assert!( + command.contains(r#"--conf "spark.openlineage.transport.url=https://marquez:5000""#) + ); + } + + /// The OpenLineage connection's `caCert.secretClass` must be wired into the driver truststore, + /// so the OpenLineage listener's HTTPS POST to the backend trusts the secret-operator cert. + /// Without this the `https` scheme above is unusable (handshake fails against the default JVM + /// truststore). + #[test] + fn test_openlineage_tls_secret_class_wired_into_truststore() { + let yaml = indoc! {r#" + --- + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: spark-examples + namespace: default + spec: + mode: cluster + mainApplicationFile: test.py + sparkImage: + productVersion: 1.2.3 + openLineage: + connection: + inline: + host: marquez + port: 5000 + tls: + verification: + server: + caCert: + secretClass: marquez-ca + "#}; + let command = build_command_with_openlineage(yaml, "4.1.2"); + + // The mounted secret-operator truststore for `marquez-ca` is imported into the shared + // PKCS12 truststore. + assert!(command.contains(&format!( + "{STACKABLE_MOUNT_PATH_TLS}/marquez-ca/truststore.p12" + ))); + // The driver + executor JVMs are pointed at that truststore. + assert_eq!( + command + .matches(&format!( + "-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}/truststore.p12" + )) + .count(), + 2, + ); + } + + /// Without an OpenLineage TLS `secretClass` (and no S3/logdir), no truststore is built and the + /// driver JVM gets no truststore options. + #[test] + fn test_openlineage_without_tls_has_no_truststore() { + let command = build_command_with_openlineage(OPENLINEAGE_ENABLED_APP, "4.1.2"); + + assert!(!command.contains("-Djavax.net.ssl.trustStore=")); + assert!(!command.contains(STACKABLE_MOUNT_PATH_TLS)); + } + + #[rstest] + #[case::explicit_app_name( + indoc! {r#" + --- + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: metadata-name + namespace: default + spec: + mode: cluster + mainApplicationFile: test.py + sparkImage: + productVersion: 1.2.3 + openLineage: + connection: + inline: + host: marquez + port: 5000 + appName: explicit-name + sparkConf: + spark.app.name: spark-conf-name + "#}, + "explicit-name" + )] + #[case::spark_app_name( + indoc! {r#" + --- + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: metadata-name + namespace: default + spec: + mode: cluster + mainApplicationFile: test.py + sparkImage: + productVersion: 1.2.3 + openLineage: + connection: + inline: + host: marquez + port: 5000 + sparkConf: + spark.app.name: spark-conf-name + "#}, + "spark-conf-name" + )] + #[case::metadata_name_fallback( + indoc! {r#" + --- + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: metadata-name + namespace: default + spec: + mode: cluster + mainApplicationFile: test.py + sparkImage: + productVersion: 1.2.3 + openLineage: + connection: + inline: + host: marquez + port: 5000 + "#}, + "metadata-name" + )] + fn test_openlineage_app_name_resolution(#[case] yaml: &str, #[case] expected_name: &str) { + let deserializer = serde_yaml::Deserializer::from_str(yaml); + let spark_application: v1alpha1::SparkApplication = + serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); + let resolved = spark_application.resolved_openlineage_app_name().unwrap(); + + assert_eq!(resolved, expected_name); + } + impl RoundtripTestData for v1alpha1::SparkApplicationSpec { fn roundtrip_test_data() -> Vec { stackable_operator::utils::yaml_from_str_singleton_map(indoc! {r#" diff --git a/rust/operator-binary/src/crd/roles.rs b/rust/operator-binary/src/crd/roles.rs index b2eb26f8..62c47d52 100644 --- a/rust/operator-binary/src/crd/roles.rs +++ b/rust/operator-binary/src/crd/roles.rs @@ -28,7 +28,7 @@ use stackable_operator::{ fragment::Fragment, merge::{Atomic, Merge}, }, - crd::s3, + crd::{openlineage, s3}, k8s_openapi::{api::core::v1::VolumeMount, apimachinery::pkg::api::resource::Quantity}, product_logging::{self, spec::Logging}, schemars::{self, JsonSchema}, @@ -167,9 +167,16 @@ impl RoleConfig { spark_application: &v1alpha1::SparkApplication, s3conn: &Option, logdir: &Option, + open_lineage_conn: Option<&openlineage::ResolvedOpenLineageConnection>, ) -> Vec { let volume_mounts = self.volume_mounts.clone().into(); - spark_application.add_common_volume_mounts(volume_mounts, s3conn, logdir, true) + spark_application.add_common_volume_mounts( + volume_mounts, + s3conn, + logdir, + true, + open_lineage_conn, + ) } } diff --git a/rust/operator-binary/src/crd/template_merger.rs b/rust/operator-binary/src/crd/template_merger.rs index 7f8afbba..b12bff0f 100644 --- a/rust/operator-binary/src/crd/template_merger.rs +++ b/rust/operator-binary/src/crd/template_merger.rs @@ -55,6 +55,11 @@ pub fn deep_merge(base: &SparkApplication, overlay: &SparkApplication) -> SparkA .vector_aggregator_config_map_name .clone() .or_else(|| base.spec.vector_aggregator_config_map_name.clone()), + open_lineage: overlay + .spec + .open_lineage + .clone() + .or_else(|| base.spec.open_lineage.clone()), s3connection: overlay .spec .s3connection diff --git a/rust/operator-binary/src/crd/tlscerts.rs b/rust/operator-binary/src/crd/tlscerts.rs index a31bda07..f380c75a 100644 --- a/rust/operator-binary/src/crd/tlscerts.rs +++ b/rust/operator-binary/src/crd/tlscerts.rs @@ -4,7 +4,7 @@ use stackable_operator::{ commons::tls_verification::{ CaCert, Tls, TlsClientDetails, TlsServerVerification, TlsVerification, }, - crd::s3, + crd::{openlineage, s3}, }; use crate::crd::{ @@ -33,9 +33,32 @@ pub fn tls_secret_name(s3conn: &s3::v1alpha1::ConnectionSpec) -> Option<&str> { None } +/// Extracts the SecretClass name backing the CA cert for a resolved OpenLineage connection, if the +/// connection verifies the server's TLS certificate against a SecretClass CA. Mirrors +/// [`tls_secret_name`] for S3 connections — both carry a flattened [`TlsClientDetails`]. +pub fn openlineage_tls_secret_name( + conn: &openlineage::ResolvedOpenLineageConnection, +) -> Option<&str> { + if let TlsClientDetails { + tls: + Some(Tls { + verification: + TlsVerification::Server(TlsServerVerification { + ca_cert: CaCert::SecretClass(secret_name), + }), + }), + } = &conn.tls + { + return Some(secret_name); + } + + None +} + pub fn tls_secret_names<'a>( s3conn: &'a Option, logdir: &'a Option, + open_lineage_conn: Option<&'a openlineage::ResolvedOpenLineageConnection>, ) -> Option> { // Ensure there are no duplicate secret names. let mut names = BTreeSet::new(); @@ -49,6 +72,10 @@ pub fn tls_secret_names<'a>( { names.insert(secret_name); } + + if let Some(secret_name) = open_lineage_conn.and_then(openlineage_tls_secret_name) { + names.insert(secret_name); + } if names.is_empty() { None } else { diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 9f14b776..c96d8bec 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -51,6 +51,7 @@ mod config; mod connect; mod crd; mod history; +mod openlineage; mod pod_driver_controller; mod product_logging; mod spark_k8s_controller; @@ -70,6 +71,7 @@ struct Opts { pub struct Ctx { pub client: stackable_operator::client::Client, pub operator_environment: OperatorEnvironmentOptions, + pub event_recorder: Recorder, } #[tokio::main] @@ -136,11 +138,6 @@ async fn main() -> anyhow::Result<()> { .run(sigterm_watcher.handle()) .map_err(|err| anyhow!(err).context("failed to run webhook server")); - let ctx = Ctx { - client: client.clone(), - operator_environment: operator_environment.clone(), - }; - let spark_event_recorder = Arc::new(Recorder::new( client.as_kube_client(), Reporter { @@ -148,6 +145,14 @@ async fn main() -> anyhow::Result<()> { instance: None, }, )); + + let ctx = Ctx { + client: client.clone(), + operator_environment: operator_environment.clone(), + // Shares the recorder (and thus its event-aggregation cache) with the + // report_controller_reconciled reporting stream below; `Recorder` is `Clone`. + event_recorder: (*spark_event_recorder).clone(), + }; let app_controller = Controller::new( watch_namespace .get_api::>(&client), @@ -224,11 +229,6 @@ async fn main() -> anyhow::Result<()> { }, ).map(anyhow::Ok); - // Create new object because Ctx cannot be cloned - let ctx = Ctx { - client: client.clone(), - operator_environment: operator_environment.clone(), - }; let history_event_recorder = Arc::new(Recorder::new( client.as_kube_client(), Reporter { @@ -236,6 +236,12 @@ async fn main() -> anyhow::Result<()> { instance: None, }, )); + // Create new object because Ctx cannot be cloned + let ctx = Ctx { + client: client.clone(), + operator_environment: operator_environment.clone(), + event_recorder: (*history_event_recorder).clone(), + }; let history_controller = Controller::new( watch_namespace .get_api::>( @@ -289,11 +295,6 @@ async fn main() -> anyhow::Result<()> { .map(anyhow::Ok); // ============================== - // Create new object because Ctx cannot be cloned - let ctx = Ctx { - client: client.clone(), - operator_environment, - }; let connect_event_recorder = Arc::new(Recorder::new( client.as_kube_client(), Reporter { @@ -301,6 +302,12 @@ async fn main() -> anyhow::Result<()> { instance: None, }, )); + // Create new object because Ctx cannot be cloned + let ctx = Ctx { + client: client.clone(), + operator_environment, + event_recorder: (*connect_event_recorder).clone(), + }; let connect_controller = Controller::new( watch_namespace .get_api::>( diff --git a/rust/operator-binary/src/openlineage.rs b/rust/operator-binary/src/openlineage.rs new file mode 100644 index 00000000..24b23546 --- /dev/null +++ b/rust/operator-binary/src/openlineage.rs @@ -0,0 +1,149 @@ +//! Operator-side OpenLineage helpers for [`SparkApplication`]. +//! +//! The reusable OpenLineage types (`OpenLineageConnectionSpec`, `InlineConnectionOrReference`, +//! `OpenLineageJob`) live in the `stackable_operator::crd::openlineage` library module. This module +//! only holds the bits that depend on the operator's own `SparkApplication` (job-name resolution) +//! plus a spark-submit conf helper. +//! +//! [`SparkApplication`]: crate::crd::v1alpha1::SparkApplication + +use std::collections::BTreeMap; + +use snafu::OptionExt; +use stackable_operator::k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, SecretKeySelector}; + +use crate::crd::{ + Error, ObjectHasNoNameSnafu, + constants::{ + OPENLINEAGE_AUTH_API_KEY_ENV, OPENLINEAGE_AUTH_SECRET_KEY, OPENLINEAGE_AUTH_TYPE_API_KEY, + OPENLINEAGE_AUTH_TYPE_ENV, OPENLINEAGE_TRANSPORT_TYPE_ENV, OPENLINEAGE_TRANSPORT_TYPE_HTTP, + OPENLINEAGE_TRANSPORT_URL_ENV, + }, + v1alpha1, +}; + +/// Appends `value` to a comma-separated `--conf` value in `submit_conf`, preserving any existing +/// (e.g. user-supplied) entries and skipping `value` if it is already present. Used for the +/// OpenLineage keys that must accumulate rather than clobber (`spark.extraListeners`, `spark.jars`). +pub(crate) fn append_conf_csv(submit_conf: &mut BTreeMap, key: &str, value: &str) { + match submit_conf.get_mut(key) { + Some(existing) if !existing.is_empty() => { + if !existing.split(',').any(|entry| entry.trim() == value) { + existing.push(','); + existing.push_str(value); + } + } + _ => { + submit_conf.insert(key.to_string(), value.to_string()); + } + } +} + +/// Builds the driver env vars that deliver the **entire** OpenLineage HTTP transport — type, URL and +/// bearer-token auth — via the OpenLineage Java client's `OPENLINEAGE__` env-var configuration. +/// +/// This is used only when `credentialsSecretName` is configured. The whole transport must come from +/// one source: OpenLineage resolves `transport` as a unit, so if `spark.openlineage.transport.type` +/// or `.url` were set via `--conf`, the transport would be taken entirely from SparkConf and the +/// env-provided `auth` sub-tree would be silently dropped (verified against openlineage-spark +/// 1.51.0). Delivering the token as a `secretKeyRef` also keeps it out of the spark-submit `--conf` +/// args and the Job/pod spec — the operator never reads it. +pub(crate) fn openlineage_transport_env_vars( + transport_url: &str, + secret_name: &str, +) -> Vec { + let literal = |name: &str, value: &str| EnvVar { + name: name.to_string(), + value: Some(value.to_string()), + value_from: None, + }; + + vec![ + literal( + OPENLINEAGE_TRANSPORT_TYPE_ENV, + OPENLINEAGE_TRANSPORT_TYPE_HTTP, + ), + literal(OPENLINEAGE_TRANSPORT_URL_ENV, transport_url), + literal(OPENLINEAGE_AUTH_TYPE_ENV, OPENLINEAGE_AUTH_TYPE_API_KEY), + EnvVar { + name: OPENLINEAGE_AUTH_API_KEY_ENV.to_string(), + value: None, + value_from: Some(EnvVarSource { + secret_key_ref: Some(SecretKeySelector { + name: secret_name.to_string(), + key: OPENLINEAGE_AUTH_SECRET_KEY.to_string(), + optional: None, + }), + ..EnvVarSource::default() + }), + }, + ] +} + +impl v1alpha1::SparkApplication { + /// Resolves the stable OpenLineage job/app name (MVP §5), in priority order: + /// 1. `spec.openLineage.appName`, else + /// 2. `spark.app.name` from `sparkConf`, else + /// 3. `metadata.name`. + /// + /// Always yields a non-blank name — which is exactly what fixes the intermittent `unknown` bug. + pub fn resolved_openlineage_app_name(&self) -> Result { + if let Some(app_name) = self + .spec + .open_lineage + .as_ref() + .and_then(|open_lineage| open_lineage.app_name.clone()) + { + return Ok(app_name); + } + + if let Some(app_name) = self.spec.spark_conf.get("spark.app.name") { + return Ok(app_name.clone()); + } + + self.metadata.name.clone().context(ObjectHasNoNameSnafu) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn transport_env_vars_carry_full_transport_with_token_secret_ref() { + let vars = openlineage_transport_env_vars("https://marquez:5000", "my-secret"); + + let get = |name: &str| { + vars.iter() + .find(|v| v.name == name) + .unwrap_or_else(|| panic!("env var {name} present")) + }; + + // The whole transport is delivered via env so OpenLineage keeps the auth sub-tree. + assert_eq!( + get(OPENLINEAGE_TRANSPORT_TYPE_ENV).value.as_deref(), + Some(OPENLINEAGE_TRANSPORT_TYPE_HTTP) + ); + assert_eq!( + get(OPENLINEAGE_TRANSPORT_URL_ENV).value.as_deref(), + Some("https://marquez:5000") + ); + assert_eq!( + get(OPENLINEAGE_AUTH_TYPE_ENV).value.as_deref(), + Some(OPENLINEAGE_AUTH_TYPE_API_KEY) + ); + + let key_var = get(OPENLINEAGE_AUTH_API_KEY_ENV); + assert!( + key_var.value.is_none(), + "the token must never be delivered as a literal value" + ); + let selector = key_var + .value_from + .as_ref() + .and_then(|source| source.secret_key_ref.as_ref()) + .expect("api key sourced from a secretKeyRef"); + assert_eq!(selector.name, "my-secret"); + assert_eq!(selector.key, OPENLINEAGE_AUTH_SECRET_KEY); + } +} diff --git a/rust/operator-binary/src/spark_k8s_controller.rs b/rust/operator-binary/src/spark_k8s_controller.rs index a7a128b7..c7a2bb7b 100644 --- a/rust/operator-binary/src/spark_k8s_controller.rs +++ b/rust/operator-binary/src/spark_k8s_controller.rs @@ -148,6 +148,7 @@ pub async fn reconcile( let spark_application = &validated.spark_application; let opt_s3conn = &validated.cluster_config.s3_connection; + let opt_open_lineage_conn = validated.cluster_config.open_lineage_connection.as_ref(); let logdir = &validated.cluster_config.log_dir; let resolved_product_image = &validated.resolved_product_image; // This is the final version of the spark app to reconcile. @@ -165,7 +166,7 @@ pub async fn reconcile( .await .context(ApplyRoleBindingSnafu)?; - let env_vars = spark_application.env(opt_s3conn, logdir); + let env_vars = spark_application.env(opt_s3conn, logdir, opt_open_lineage_conn); let driver_config = spark_application .driver_config() @@ -224,7 +225,13 @@ pub async fn reconcile( .context(ApplyApplicationSnafu)?; let job_commands = spark_application - .build_command(opt_s3conn, logdir, &resolved_product_image.image) + .build_command( + opt_s3conn, + logdir, + &resolved_product_image.image, + &resolved_product_image.product_version, + opt_open_lineage_conn, + ) .context(BuildCommandSnafu)?; let submit_config = spark_application diff --git a/rust/operator-binary/src/spark_k8s_controller/build/pod.rs b/rust/operator-binary/src/spark_k8s_controller/build/pod.rs index 9abf83ad..fe27f3da 100644 --- a/rust/operator-binary/src/spark_k8s_controller/build/pod.rs +++ b/rust/operator-binary/src/spark_k8s_controller/build/pod.rs @@ -51,6 +51,7 @@ fn init_containers( let spark_application = &validated.spark_application; let s3conn = &validated.cluster_config.s3_connection; let logdir = &validated.cluster_config.log_dir; + let open_lineage_conn = validated.cluster_config.open_lineage_connection.as_ref(); let spark_image = &validated.resolved_product_image; let mut jcb = new_container_builder(&SparkContainer::Job.to_container_name()); let job_container = match &spark_application.spec.image { @@ -153,7 +154,7 @@ fn init_containers( let mut tcb = new_container_builder(&SparkContainer::Tls.to_container_name()); let mut args = Vec::new(); - let tls_container = match tlscerts::tls_secret_names(s3conn, logdir) { + let tls_container = match tlscerts::tls_secret_names(s3conn, logdir, open_lineage_conn) { Some(cert_secrets) => { args.push(tlscerts::convert_system_trust_store_to_pkcs12()); for cert_secret in cert_secrets { @@ -207,16 +208,22 @@ pub(crate) fn pod_template( let spark_application = &validated.spark_application; let s3conn = &validated.cluster_config.s3_connection; let logdir = &validated.cluster_config.log_dir; + let open_lineage_conn = validated.cluster_config.open_lineage_connection.as_ref(); let spark_image = &validated.resolved_product_image; let container_name = SparkContainer::Spark.to_string(); let mut cb = new_container_builder(&SparkContainer::Spark.to_container_name()); let merged_env = spark_application.merged_env(role.clone(), env); - cb.add_volume_mounts(config.volume_mounts(spark_application, s3conn, logdir)) - .context(AddVolumeMountSnafu)? - .add_env_vars(merged_env) - .resources(config.resources.clone().into()) - .image_from_product_image(spark_image); + cb.add_volume_mounts(config.volume_mounts( + spark_application, + s3conn, + logdir, + open_lineage_conn, + )) + .context(AddVolumeMountSnafu)? + .add_env_vars(merged_env) + .resources(config.resources.clone().into()) + .image_from_product_image(spark_image); if config.logging.enable_vector_agent { cb.add_env_var( diff --git a/rust/operator-binary/src/spark_k8s_controller/build/resource/config_map.rs b/rust/operator-binary/src/spark_k8s_controller/build/resource/config_map.rs index 3993fee3..f18f7efd 100644 --- a/rust/operator-binary/src/spark_k8s_controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/spark_k8s_controller/build/resource/config_map.rs @@ -37,6 +37,7 @@ pub(crate) fn pod_template_config_map( let spark_application = &validated.spark_application; let s3conn = &validated.cluster_config.s3_connection; let logdir = &validated.cluster_config.log_dir; + let open_lineage_conn = validated.cluster_config.open_lineage_connection.as_ref(); let cm_name = spark_application.pod_template_config_map_name(role.clone()); let log_config_map = if let Some(ContainerLogConfig { @@ -60,6 +61,7 @@ pub(crate) fn pod_template_config_map( logdir, Some(&log_config_map), &requested_secret_lifetime, + open_lineage_conn, ) .context(CreateVolumesSnafu)?; volumes.push( diff --git a/rust/operator-binary/src/spark_k8s_controller/build/resource/job.rs b/rust/operator-binary/src/spark_k8s_controller/build/resource/job.rs index d078df55..8d9d5c2f 100644 --- a/rust/operator-binary/src/spark_k8s_controller/build/resource/job.rs +++ b/rust/operator-binary/src/spark_k8s_controller/build/resource/job.rs @@ -34,6 +34,7 @@ pub(crate) fn spark_job( let spark_image = &validated.resolved_product_image; let s3conn = &validated.cluster_config.s3_connection; let logdir = &validated.cluster_config.log_dir; + let open_lineage_conn = validated.cluster_config.open_lineage_connection.as_ref(); let mut cb = new_container_builder(&SparkContainer::SparkSubmit.to_container_name()); let merged_env = spark_application.merged_env(SparkApplicationRole::Submit, env); @@ -45,7 +46,7 @@ pub(crate) fn spark_job( let mut spark_submit_opts_env = vec![format!( "-Dlog4j.configurationFile={VOLUME_MOUNT_PATH_LOG_CONFIG}/{LOG4J2_CONFIG_FILE}" )]; - if tlscerts::tls_secret_names(s3conn, logdir).is_some() { + if tlscerts::tls_secret_names(s3conn, logdir, open_lineage_conn).is_some() { spark_submit_opts_env.push(format!( "-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}/truststore.p12" )); @@ -63,7 +64,11 @@ pub(crate) fn spark_job( ]) .args(vec![job_commands.join("\n")]) .resources(job_config.resources.clone().into()) - .add_volume_mounts(spark_application.spark_job_volume_mounts(s3conn, logdir)) + .add_volume_mounts(spark_application.spark_job_volume_mounts( + s3conn, + logdir, + open_lineage_conn, + )) .context(AddVolumeMountSnafu)? .add_env_vars(merged_env) .add_env_var("SPARK_SUBMIT_OPTS", spark_submit_opts_env.join(" ")) @@ -90,7 +95,13 @@ pub(crate) fn spark_job( .context(MissingSecretLifetimeSnafu)?; volumes.extend( spark_application - .volumes(s3conn, logdir, None, &requested_secret_lifetime) + .volumes( + s3conn, + logdir, + None, + &requested_secret_lifetime, + open_lineage_conn, + ) .context(CreateVolumesSnafu)?, ); diff --git a/rust/operator-binary/src/spark_k8s_controller/dereference.rs b/rust/operator-binary/src/spark_k8s_controller/dereference.rs index aa68ea14..908647cc 100644 --- a/rust/operator-binary/src/spark_k8s_controller/dereference.rs +++ b/rust/operator-binary/src/spark_k8s_controller/dereference.rs @@ -5,7 +5,10 @@ //! Synchronous validation belongs in the sibling [`super::validate`] module. use snafu::{ResultExt, Snafu}; -use stackable_operator::{client::Client, crd::s3}; +use stackable_operator::{ + client::Client, + crd::{openlineage, s3}, +}; use crate::crd::{ logdir::ResolvedLogDir, @@ -23,6 +26,11 @@ pub enum Error { source: stackable_operator::crd::s3::v1alpha1::ConnectionError, }, + #[snafu(display("failed to resolve OpenLineage connection"))] + ResolveOpenLineageConnection { + source: stackable_operator::crd::openlineage::v1alpha1::OpenLineageError, + }, + #[snafu(display("failed to resolve log directory"))] LogDir { source: crate::crd::logdir::Error }, @@ -40,6 +48,9 @@ pub struct DereferencedSparkApplication { pub resolved_template_refs: Vec, /// Resolved S3 connection, if `spec.s3connection` is set. pub s3_connection: Option, + /// Resolved OpenLineage backend connection, if `spec.openLineage` is set. Its + /// `credentialsSecretName` (when set) carries the bearer-token Secret for authentication. + pub open_lineage_connection: Option, /// Resolved log directory, if `spec.log_file_directory` is set. pub log_dir: Option, } @@ -73,7 +84,19 @@ pub async fn dereference( None => None, }; - // 3. Log directory (also pulls S3Bucket + TLS secret internally). + // 3. OpenLineage connection (inline or referenced `OpenLineageConnection`). + let open_lineage_connection = match merged_app.spec.open_lineage.as_ref() { + Some(job) => Some( + job.connection + .clone() + .resolve(client, namespace) + .await + .context(ResolveOpenLineageConnectionSnafu)?, + ), + None => None, + }; + + // 4. Log directory (also pulls S3Bucket + TLS secret internally). let log_dir = match merged_app.spec.log_file_directory.as_ref() { Some(log_file_dir) => Some( ResolvedLogDir::resolve(log_file_dir, merged_app.metadata.namespace.clone(), client) @@ -87,6 +110,7 @@ pub async fn dereference( spark_application: merged_app, resolved_template_refs, s3_connection, + open_lineage_connection, log_dir, }) } diff --git a/rust/operator-binary/src/spark_k8s_controller/validate.rs b/rust/operator-binary/src/spark_k8s_controller/validate.rs index 8c0d47bc..30e90b19 100644 --- a/rust/operator-binary/src/spark_k8s_controller/validate.rs +++ b/rust/operator-binary/src/spark_k8s_controller/validate.rs @@ -13,7 +13,7 @@ use stackable_operator::{ product_image_selection::{self, ResolvedProductImage}, tls_verification::TlsVerification, }, - crd::s3, + crd::{openlineage, s3}, k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta, kube::Resource, kvp::Labels, @@ -95,6 +95,7 @@ pub struct ValidatedSparkApplication { pub struct ValidatedClusterConfig { pub resolved_template_refs: Vec, pub s3_connection: Option, + pub open_lineage_connection: Option, pub log_dir: Option, } @@ -240,6 +241,7 @@ pub fn validate( cluster_config: ValidatedClusterConfig { resolved_template_refs: dereferenced.resolved_template_refs, s3_connection: dereferenced.s3_connection, + open_lineage_connection: dereferenced.open_lineage_connection, log_dir: dereferenced.log_dir, }, }) diff --git a/tests/templates/kuttl/openlineage/00-assert.yaml b/tests/templates/kuttl/openlineage/00-assert.yaml new file mode 100644 index 00000000..5baf8caa --- /dev/null +++ b/tests/templates/kuttl/openlineage/00-assert.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 900 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: integration-tests-sa diff --git a/tests/templates/kuttl/openlineage/00-patch-ns.yaml.j2 b/tests/templates/kuttl/openlineage/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/openlineage/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/openlineage/00-serviceaccount.yaml.j2 b/tests/templates/kuttl/openlineage/00-serviceaccount.yaml.j2 new file mode 100644 index 00000000..9cbf0351 --- /dev/null +++ b/tests/templates/kuttl/openlineage/00-serviceaccount.yaml.j2 @@ -0,0 +1,29 @@ +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: use-integration-tests-scc +rules: +{% if test_scenario['values']['openshift'] == "true" %} + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: integration-tests-sa +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: use-integration-tests-scc +subjects: + - kind: ServiceAccount + name: integration-tests-sa +roleRef: + kind: Role + name: use-integration-tests-scc + apiGroup: rbac.authorization.k8s.io diff --git a/tests/templates/kuttl/openlineage/01-assert.yaml.j2 b/tests/templates/kuttl/openlineage/01-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/openlineage/01-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/openlineage/01-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/openlineage/01-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/openlineage/01-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/openlineage/10-assert.yaml b/tests/templates/kuttl/openlineage/10-assert.yaml new file mode 100644 index 00000000..188a11e8 --- /dev/null +++ b/tests/templates/kuttl/openlineage/10-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: openlineage-receiver +status: + availableReplicas: 1 + readyReplicas: 1 diff --git a/tests/templates/kuttl/openlineage/10-install-receiver.yaml.j2 b/tests/templates/kuttl/openlineage/10-install-receiver.yaml.j2 new file mode 100644 index 00000000..c3390131 --- /dev/null +++ b/tests/templates/kuttl/openlineage/10-install-receiver.yaml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # `envsubst` injects the kuttl-assigned $NAMESPACE (only $NAMESPACE, so the nginx config's own + # `$` tokens are left untouched) into the SecretClass name / cert scope. + - script: envsubst '$NAMESPACE' < 10_receiver.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/openlineage/10_receiver.yaml.j2 b/tests/templates/kuttl/openlineage/10_receiver.yaml.j2 new file mode 100644 index 00000000..d880244c --- /dev/null +++ b/tests/templates/kuttl/openlineage/10_receiver.yaml.j2 @@ -0,0 +1,163 @@ +# Lightweight OpenLineage receiver. +# +# Instead of a full Marquez, this is a plain nginx that accepts the OpenLineage HTTP transport's +# `POST /api/v1/lineage` requests and persists each request body as a flat file. The assertion in +# `30-assert.yaml` then greps those files to prove events were delivered (and, in the TLS scenario, +# that the driver trusted the secret-operator-issued server cert end-to-end). +# +# nginx cannot write a request body to disk on a `return` (the body is discarded), so the receive +# location sets `client_body_in_file_only on` and `proxy_pass`es to a local sink server; nginx then +# buffers the full body to a file (kept, not cleaned) before forwarding. Bodies land in +# `/tmp/lineage/`. +# +# `$NAMESPACE` is substituted by the `10-install-receiver` step via `envsubst` (kuttl assigns the +# namespace at run time; beku leaves the literal in place). +{% if test_scenario['values']['openlineage-use-tls'] == 'true' %} +--- +# A genuine autoTls SecretClass: the secret operator generates the CA and issues the nginx server +# cert. Suffixed with the (cluster-scoped) namespace to avoid collisions across parallel scenarios. +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: openlineage-tls-$NAMESPACE +spec: + backend: + autoTls: + ca: + autoGenerate: true + secret: + name: openlineage-tls-ca + namespace: $NAMESPACE +{% endif %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: openlineage-receiver-config +data: + nginx.conf: | + worker_processes 1; + error_log /dev/stderr notice; + pid /tmp/nginx.pid; + events { worker_connections 1024; } + http { + access_log /dev/stdout; + client_body_temp_path /tmp/nginx-body; + proxy_temp_path /tmp/nginx-proxy; + fastcgi_temp_path /tmp/nginx-fastcgi; + uwsgi_temp_path /tmp/nginx-uwsgi; + scgi_temp_path /tmp/nginx-scgi; + + # Internal sink: acknowledges the forwarded lineage event. + server { + listen 127.0.0.1:8080; + location / { return 201; } + } + + server { + listen 5000{% if test_scenario['values']['openlineage-use-tls'] == 'true' %} ssl{% endif %}; +{% if test_scenario['values']['openlineage-use-tls'] == 'true' %} + ssl_certificate /stackable/tls/tls.crt; + ssl_certificate_key /stackable/tls/tls.key; +{% endif %} + client_max_body_size 16m; + + # Persist each OpenLineage event body to /tmp/lineage/, then acknowledge via the sink. + location = /api/v1/lineage { +{% if test_scenario['values']['openlineage-use-auth'] == 'true' %} + # Require the bearer token the operator wires in from the credentialsSecretName Secret. + # Without a valid token nginx returns 401 and never persists a body, so the 30-assert + # event checks only pass if the driver actually authenticated. + if ($http_authorization != "Bearer openlineage-test-token") { return 401; } +{% endif %} + client_body_in_file_only on; + client_body_temp_path /tmp/lineage; + proxy_pass http://127.0.0.1:8080; + } + + location / { return 204; } + } + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: openlineage-receiver + labels: + app: openlineage-receiver +spec: + replicas: 1 + selector: + matchLabels: + app: openlineage-receiver + template: + metadata: + labels: + app: openlineage-receiver + spec: +{% if test_scenario['values']['openshift'] != 'true' %} + # On plain Kubernetes, make mounted emptyDirs writable by the nginx-unprivileged group (101). + # On OpenShift this is omitted so the restricted SCC can assign an fsGroup from the project range. + securityContext: + fsGroup: 101 +{% endif %} + containers: + - name: nginx + image: docker.io/nginxinc/nginx-unprivileged:1.27-alpine + ports: + - containerPort: 5000 + volumeMounts: + - name: config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + - name: tmp + mountPath: /tmp +{% if test_scenario['values']['openlineage-use-tls'] == 'true' %} + - name: tls + mountPath: /stackable/tls +{% endif %} + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 128Mi + volumes: + - name: config + configMap: + name: openlineage-receiver-config + - name: tmp + emptyDir: {} +{% if test_scenario['values']['openlineage-use-tls'] == 'true' %} + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: openlineage-tls-$NAMESPACE + # `service` scope puts openlineage-receiver..svc.cluster.local in the cert SANs, + # which is exactly the host the Spark driver dials (see the SparkApplication). + secrets.stackable.tech/scope: service=openlineage-receiver + secrets.stackable.tech/format: tls-pem + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech +{% endif %} +--- +apiVersion: v1 +kind: Service +metadata: + name: openlineage-receiver + labels: + app: openlineage-receiver +spec: + selector: + app: openlineage-receiver + ports: + - name: http + port: 5000 + targetPort: 5000 diff --git a/tests/templates/kuttl/openlineage/20-assert.yaml b/tests/templates/kuttl/openlineage/20-assert.yaml new file mode 100644 index 00000000..ac15a9a5 --- /dev/null +++ b/tests/templates/kuttl/openlineage/20-assert.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 900 +--- +apiVersion: spark.stackable.tech/v1alpha1 +kind: SparkApplication +metadata: + name: openlineage +status: + phase: Succeeded diff --git a/tests/templates/kuttl/openlineage/20-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/openlineage/20-deploy-spark-app.yaml.j2 new file mode 100644 index 00000000..6519d8a4 --- /dev/null +++ b/tests/templates/kuttl/openlineage/20-deploy-spark-app.yaml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # `envsubst '$NAMESPACE'` fills in the connection host FQDN (and the TLS secretClass name) while + # leaving any other `$` tokens in the PySpark script untouched. + - script: envsubst '$NAMESPACE' < 20_spark-app.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/openlineage/20_spark-app.yaml.j2 b/tests/templates/kuttl/openlineage/20_spark-app.yaml.j2 new file mode 100644 index 00000000..bdeb39e6 --- /dev/null +++ b/tests/templates/kuttl/openlineage/20_spark-app.yaml.j2 @@ -0,0 +1,97 @@ +# The SparkApplication under test plus its PySpark job. +# +# The job reads/writes a small DataFrame so the OpenLineage Spark listener emits START/COMPLETE run +# events. The operator injects the listener, the jar, and the `spark.openlineage.*` config from the +# `openLineage` block below; the events are POSTed to the nginx receiver. +# +# `$NAMESPACE` is substituted by the `20-deploy-spark-app` step so the connection host is the fully +# qualified Service DNS name — which is what the secret-operator `service` scope puts in the cert +# SANs, so TLS hostname verification on the driver succeeds. +{% if test_scenario['values']['openlineage-use-auth'] == 'true' %} +--- +# Token Secret for the auth scenario. The Secret is namespaced and must live in the workload +# namespace; the operator wires its `apiKey` entry into the driver as the OpenLineage bearer token +# via a secretKeyRef env var (referenced by `credentialsSecretName` on the connection). +apiVersion: v1 +kind: Secret +metadata: + name: openlineage-token +stringData: + apiKey: "openlineage-test-token" +{% endif %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: openlineage-job +data: + openlineage-job.py: | + from pyspark.sql import SparkSession + + spark = SparkSession.builder.appName("openlineage-test-app").getOrCreate() + + df = spark.createDataFrame( + [(1, "a"), (2, "b"), (2, "c"), (3, "d")], + ["id", "grp"], + ) + # An aggregation + collect triggers a SQL execution, which makes the OpenLineage listener emit + # START/COMPLETE run events. We deliberately avoid a filesystem write: in cluster mode the driver + # and executors have separate local filesystems, so a write to a local path would fail the commit. + result = df.groupBy("grp").count().collect() + print(f"aggregation result: {result}") + + spark.stop() +--- +apiVersion: spark.stackable.tech/v1alpha1 +kind: SparkApplication +metadata: + name: openlineage +spec: +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + sparkImage: +{% if test_scenario['values']['spark'].find(",") > 0 %} + custom: "{{ test_scenario['values']['spark'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['spark'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['spark'] }}" +{% endif %} + pullPolicy: IfNotPresent + mode: cluster + mainApplicationFile: "local:///stackable/spark/jobs/openlineage-job.py" + openLineage: + appName: openlineage-test-app + connection: + inline: + host: openlineage-receiver.$NAMESPACE.svc.cluster.local + port: 5000 +{% if test_scenario['values']['openlineage-use-tls'] == 'true' %} + tls: + verification: + server: + caCert: + secretClass: openlineage-tls-$NAMESPACE +{% endif %} +{% if test_scenario['values']['openlineage-use-auth'] == 'true' %} + credentialsSecretName: openlineage-token +{% endif %} + driver: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + volumeMounts: + - name: script + mountPath: /stackable/spark/jobs + executor: + replicas: 1 + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + volumeMounts: + - name: script + mountPath: /stackable/spark/jobs + volumes: + - name: script + configMap: + name: openlineage-job diff --git a/tests/templates/kuttl/openlineage/30-assert.yaml b/tests/templates/kuttl/openlineage/30-assert.yaml new file mode 100644 index 00000000..0120b54b --- /dev/null +++ b/tests/templates/kuttl/openlineage/30-assert.yaml @@ -0,0 +1,14 @@ +--- +# Verify the OpenLineage events actually reached the receiver. In the TLS scenario this also proves +# the driver trusted the secret-operator-issued cert end-to-end (a failed handshake would mean no +# body files were written). +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 120 +commands: + - script: | + POD=$(kubectl get -n "$NAMESPACE" pod -l app=openlineage-receiver -o jsonpath='{.items[0].metadata.name}') + # At least one lineage event body was persisted, mentioning our job name ... + kubectl exec -n "$NAMESPACE" "$POD" -- sh -c 'cat /tmp/lineage/* 2>/dev/null | grep -q "openlineage-test-app"' + # ... and a COMPLETE run event was received (the job finished and reported lineage). + kubectl exec -n "$NAMESPACE" "$POD" -- sh -c 'cat /tmp/lineage/* 2>/dev/null | grep -q "COMPLETE"' diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 5d387b63..ed45c568 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -70,6 +70,14 @@ dimensions: values: - "false" - "true" + - name: openlineage-use-tls + values: + - "false" + - "true" + - name: openlineage-use-auth + values: + - "false" + - "true" tests: - name: smoke dimensions: @@ -153,6 +161,12 @@ tests: - kerberos-realm - openshift - s3-use-tls + - name: openlineage + dimensions: + - spark + - openlineage-use-tls + - openlineage-use-auth + - openshift suites: - name: nightly @@ -161,6 +175,8 @@ suites: - expr: last - name: s3-use-tls expr: "true" + - name: openlineage-use-tls + expr: "true" - name: smoke-latest select: - smoke @@ -175,3 +191,5 @@ suites: expr: "true" - name: s3-use-tls expr: "true" + - name: openlineage-use-tls + expr: "true"