feat(kafka): Add 3.9.2 and 4.2.1

CHANGELOG.md

@@ -12,6 +12,7 @@ All notable changes to this project will be documented in this file.
 - nifi: Backport NIFI-15901 to 2.x versions ([#1481]).
 - testing-tools: Added grpcurl utility ([#1493]).
 - opa: Add `1.16.2` ([#1509]).
+- kafka: Add `3.9.2` and `4.2.1` ([#1483])
 
 ### Changed
 
@@ -39,6 +40,7 @@ All notable changes to this project will be documented in this file.
 [#1474]: https://github.com/stackabletech/docker-images/pull/1474
 [#1476]: https://github.com/stackabletech/docker-images/pull/1476
 [#1481]: https://github.com/stackabletech/docker-images/pull/1481
+[#1483]: https://github.com/stackabletech/docker-images/pull/1483
 [#1493]: https://github.com/stackabletech/docker-images/pull/1493
 [#1509]: https://github.com/stackabletech/docker-images/pull/1509
 [#1510]: https://github.com/stackabletech/docker-images/pull/1510

kafka/boil-config.toml

@@ -12,6 +12,17 @@ java-devel = "21"
 scala-version = "2.13"
 jmx-exporter-version = "1.3.0"
 
+[versions."3.9.2".local-images]
+java-base = "21"
+java-devel = "21"
+"kafka/kcat" = "1.7.0"
+"kafka/kafka-opa-plugin" = "1.5.1"
+"shared/reload4j" = "1.2.25"
+
+[versions."3.9.2".build-arguments]
+scala-version = "2.13"
+jmx-exporter-version = "1.3.0"
+
 [versions."4.1.1".local-images]
 java-base = "24"
 java-devel = "24"
@@ -25,3 +36,18 @@ java-devel = "24"
 [versions."4.1.1".build-arguments]
 scala-version = "2.13"
 jmx-exporter-version = "1.3.0"
+
+[versions."4.2.1".local-images]
+# For JVM compatibility see README.md and docs/getting-started/compatibility.md in the Kafka repository
+java-base = "25"
+java-devel = "25"
+"kafka/kcat" = "1.7.0"
+"kafka/kafka-opa-plugin" = "1.5.1"
+# TODO: this is not used in this version but it's added
+# to avoid major changes to the Kafka image build on short notice.
+# Building this image is quick and in CI should not even be noticed.
+"shared/reload4j" = "1.2.25"
+
+[versions."4.2.1".build-arguments]
+scala-version = "2.13"
+jmx-exporter-version = "1.3.0"

kafka/stackable/patches/3.9.2/0001-Add-CycloneDX-plugin.patch

@@ -0,0 +1,62 @@
+From c19817ee82f7b90aac0a8be66e3efe17f268eb35 Mon Sep 17 00:00:00 2001
+From: Lukas Voetmand <lukas.voetmand@stackable.tech>
+Date: Thu, 17 Oct 2024 11:01:40 +0200
+Subject: Add CycloneDX plugin
+
+---
+ build.gradle | 42 ++++++++++++++++++++++++++++++++++++++++++
+ 1 file changed, 42 insertions(+)
+
+diff --git a/build.gradle b/build.gradle
+index 381e964d13..f51f410858 100644
+--- a/build.gradle
++++ b/build.gradle
+@@ -48,6 +48,48 @@ plugins {
+   //  We are going to drop JDK8 support. Hence, the spotless is upgrade to newest version and be applied only if the build env is compatible with JDK 11.
+   //  spotless 6.15.0+ has issue in runtime with JDK8 even through we define it with `apply:false`. see https://github.com/diffplug/spotless/issues/2156 for more details
+   id 'com.diffplug.spotless' version "6.14.0" apply false
++  id 'org.cyclonedx.bom' version '1.10.0'
++}
++
++cyclonedxBom {
++    // Specified the type of project being built. Defaults to 'library'
++    projectType = "application"
++    // Specified the version of the CycloneDX specification to use. Defaults to '1.5'
++    schemaVersion = "1.5"
++    // Boms destination directory. Defaults to 'build/reports'
++    destination = file("build/reports")
++    // The file name for the generated BOMs (before the file format suffix). Defaults to 'bom'
++    outputName = "bom"
++    // The file format generated, can be xml, json or all for generating both. Defaults to 'all'
++    outputFormat = "json"
++    includeConfigs = ["runtimeClasspath"]
++    // Exclude test components. This list needs to be checked and, if it changed, updated for every new Kafka version.
++    // The list can be obtained by running `gradle projects | grep upgrade-system-tests`
++    skipProjects = [
++      'upgrade-system-tests-0100',
++      'upgrade-system-tests-0101',
++      'upgrade-system-tests-0102',
++      'upgrade-system-tests-0110',
++      'upgrade-system-tests-10',
++      'upgrade-system-tests-11',
++      'upgrade-system-tests-20',
++      'upgrade-system-tests-21',
++      'upgrade-system-tests-22',
++      'upgrade-system-tests-23',
++      'upgrade-system-tests-24',
++      'upgrade-system-tests-25',
++      'upgrade-system-tests-26',
++      'upgrade-system-tests-27',
++      'upgrade-system-tests-28',
++      'upgrade-system-tests-30',
++      'upgrade-system-tests-31',
++      'upgrade-system-tests-32',
++      'upgrade-system-tests-33',
++      'upgrade-system-tests-34',
++      'upgrade-system-tests-35',
++      'upgrade-system-tests-36',
++      'upgrade-system-tests-37'
++    ]
+ }
+
+ ext {

kafka/stackable/patches/3.9.2/0002-Change-Gradle-to-use-the-Nexus-Build-Repo.patch

@@ -0,0 +1,35 @@
+From 343021661d7ed166a7555e6bce9da4805340f6cb Mon Sep 17 00:00:00 2001
+From: Lars Francke <git@lars-francke.de>
+Date: Thu, 12 Dec 2024 10:09:47 +0100
+Subject: Change Gradle to use the Nexus Build Repo
+
+---
+ build.gradle | 8 ++++++--
+ 1 file changed, 6 insertions(+), 2 deletions(-)
+
+diff --git a/build.gradle b/build.gradle
+index f51f410858..5ccd99e024 100644
+--- a/build.gradle
++++ b/build.gradle
+@@ -20,7 +20,9 @@ import java.nio.charset.StandardCharsets
+
+ buildscript {
+   repositories {
+-    mavenCentral()
++    maven {
++      url 'https://build-repo.stackable.tech/repository/maven-public/'
++    }
+   }
+   apply from: "$rootDir/gradle/dependencies.gradle"
+
+@@ -168,7 +170,9 @@ ext {
+ allprojects {
+
+   repositories {
+-    mavenCentral()
++    maven {
++      url 'https://build-repo.stackable.tech/repository/maven-public/'
++    }
+   }
+
+   dependencyUpdates {

kafka/stackable/patches/3.9.2/0003-Build-custom-Stackable-version.patch

@@ -0,0 +1,140 @@
+From 944ef05a7468a0437c0da923898aecae983eb5f6 Mon Sep 17 00:00:00 2001
+From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>
+Date: Mon, 1 Jun 2026 08:52:19 +0200
+Subject: Build custom Stackable version
+
+How it was done:
+
+Replace in files in root dir (do not garble .git):
+
+for d in $(ls -p|grep -v /); do find $d -type f -exec sed -i 's/3\.9\.2/3.9.2-stackable0.0.0-dev/g' {} +; done
+
+Replace in all subdirs:
+
+for d in $(ls -d */); do find $d -type f -exec sed -i 's/3\.9\.2/3.9.2-stackable0.0.0-dev/g' {} +; done
+---
+ docs/getting-started/upgrade.md                                 | 2 +-
+ docs/operations/kraft.md                                        | 2 +-
+ gradle.properties                                               | 2 +-
+ kafka-merge-pr.py                                               | 2 +-
+ streams/quickstart/java/pom.xml                                 | 2 +-
+ .../java/src/main/resources/archetype-resources/pom.xml         | 2 +-
+ streams/quickstart/pom.xml                                      | 2 +-
+ tests/kafkatest/__init__.py                                     | 2 +-
+ tests/kafkatest/version.py                                      | 2 +-
+ 9 files changed, 9 insertions(+), 9 deletions(-)
+
+diff --git a/docs/getting-started/upgrade.md b/docs/getting-started/upgrade.md
+index d7347b3ee8..7f888fc90a 100644
+--- a/docs/getting-started/upgrade.md
++++ b/docs/getting-started/upgrade.md
+@@ -58,7 +58,7 @@ If you are upgrading from version 2.4.0 or above, and you have not overridden th
+
+
+
+-### Notable changes in 3.9.2
++### Notable changes in 3.9.2-stackable0.0.0-dev
+
+   * There is a new server configuration, `alter.config.policy.kraft.compatibility.enable`, which ensures that an AlterConfigPolicy is invoked consistently between KRaft and ZooKeeper modes. To enable this behavior, set the property to `true` in `server.properties`. For backward compatibility, the default value is `false`. See [KIP-1252](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=399279475) for more details.
+
+diff --git a/docs/operations/kraft.md b/docs/operations/kraft.md
+index a63ce05d4e..a8caef2d2b 100644
+--- a/docs/operations/kraft.md
++++ b/docs/operations/kraft.md
+@@ -275,7 +275,7 @@ In general, the migration process passes through several phases.
+   * After the migration has been finalized, it is not possible to revert back to ZooKeeper mode.
+   * During the migration, if a ZK broker is running with multiple log directories, any directory failure will cause the broker to shutdown. Brokers with broken log directories will only be able to migrate to KRaft once the directories are repaired. For further details refer to [KAFKA-16431](https://issues.apache.org/jira/browse/KAFKA-16431). 
+   * As noted above, some features are not fully implemented in KRaft mode. If you are using one of those features, you will not be able to migrate to KRaft yet.
+-  * There is a known inconsistency between ZK and KRaft modes in the arguments passed to an `AlterConfigPolicy`, when operations of type `SUBTRACT`, `DELETE` or `APPEND` are processed. This has been addressed with a compatibility flag in version 3.9.2. For further details see [KIP-1252](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=399279475).
++  * There is a known inconsistency between ZK and KRaft modes in the arguments passed to an `AlterConfigPolicy`, when operations of type `SUBTRACT`, `DELETE` or `APPEND` are processed. This has been addressed with a compatibility flag in version 3.9.2-stackable0.0.0-dev. For further details see [KIP-1252](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=399279475).
+
+
+
+diff --git a/gradle.properties b/gradle.properties
+index 3c674c89c1..24bf3d2259 100644
+--- a/gradle.properties
++++ b/gradle.properties
+@@ -22,7 +22,7 @@ group=org.apache.kafka
+ #  - streams/quickstart/pom.xml
+ #  - streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
+ #  - streams/quickstart/java/pom.xml
+-version=3.9.2
++version=3.9.2-stackable0.0.0-dev
+ scalaVersion=2.13.15
+ # Adding swaggerVersion in gradle.properties to have a single version in place for swagger
+ # New version of Swagger 2.2.14 requires minimum JDK 11.
+diff --git a/kafka-merge-pr.py b/kafka-merge-pr.py
+index 4ac7434065..4f5f69d872 100755
+--- a/kafka-merge-pr.py
++++ b/kafka-merge-pr.py
+@@ -70,7 +70,7 @@ TEMP_BRANCH_PREFIX = "PR_TOOL"
+
+ DEV_BRANCH_NAME = "trunk"
+
+-DEFAULT_FIX_VERSION = os.environ.get("DEFAULT_FIX_VERSION", "3.9.2")
++DEFAULT_FIX_VERSION = os.environ.get("DEFAULT_FIX_VERSION", "3.9.2-stackable0.0.0-dev")
+
+ ORIGINAL_HEAD = ""
+
+diff --git a/streams/quickstart/java/pom.xml b/streams/quickstart/java/pom.xml
+index f8d1bd9e54..2ee0861269 100644
+--- a/streams/quickstart/java/pom.xml
++++ b/streams/quickstart/java/pom.xml
+@@ -26,7 +26,7 @@
+     <parent>
+         <groupId>org.apache.kafka</groupId>
+         <artifactId>streams-quickstart</artifactId>
+-        <version>3.9.2</version>
++        <version>3.9.2-stackable0.0.0-dev</version>
+         <relativePath>..</relativePath>
+     </parent>
+
+diff --git a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
+index b62ab64acf..3d3d881096 100644
+--- a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
++++ b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
+@@ -29,7 +29,7 @@
+
+     <properties>
+         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+-        <kafka.version>3.9.2</kafka.version>
++        <kafka.version>3.9.2-stackable0.0.0-dev</kafka.version>
+         <slf4j.version>1.7.36</slf4j.version>
+     </properties>
+
+diff --git a/streams/quickstart/pom.xml b/streams/quickstart/pom.xml
+index ac81ea06bb..1686b2bd56 100644
+--- a/streams/quickstart/pom.xml
++++ b/streams/quickstart/pom.xml
+@@ -22,7 +22,7 @@
+     <groupId>org.apache.kafka</groupId>
+     <artifactId>streams-quickstart</artifactId>
+     <packaging>pom</packaging>
+-    <version>3.9.2</version>
++    <version>3.9.2-stackable0.0.0-dev</version>
+
+     <name>Kafka Streams :: Quickstart</name>
+
+diff --git a/tests/kafkatest/__init__.py b/tests/kafkatest/__init__.py
+index 1362dad9ca..e20c655bbf 100644
+--- a/tests/kafkatest/__init__.py
++++ b/tests/kafkatest/__init__.py
+@@ -22,4 +22,4 @@
+ # Instead, in development branches, the version should have a suffix of the form ".devN"
+ #
+ # For example, when Kafka is at version 1.0.0-SNAPSHOT, this should be something like "1.0.0.dev0"
+-__version__ = '3.9.2'
++__version__ = '3.9.2-stackable0.0.0-dev'
+diff --git a/tests/kafkatest/version.py b/tests/kafkatest/version.py
+index fa301e6a8d..1980c35d11 100644
+--- a/tests/kafkatest/version.py
++++ b/tests/kafkatest/version.py
+@@ -122,7 +122,7 @@ def get_version(node=None):
+         return DEV_BRANCH
+
+ DEV_BRANCH = KafkaVersion("dev")
+-DEV_VERSION = KafkaVersion("3.9.2-SNAPSHOT")
++DEV_VERSION = KafkaVersion("3.9.2-stackable0.0.0-dev-SNAPSHOT")
+
+ # This should match the LATEST_PRODUCTION version defined in MetadataVersion.java
+ LATEST_STABLE_METADATA_VERSION = "3.9-IV0"

kafka/stackable/patches/3.9.2/patchable.toml

@@ -0,0 +1 @@
+base = "5e9866f43ab8e7e41ef39e5584ac50019381328d"

kafka/stackable/patches/4.2.1/0001-Add-CycloneDX-plugin.patch

@@ -0,0 +1,67 @@
+From 5ef351ca945bcaa8346ef066ba00fd9288a4f443 Mon Sep 17 00:00:00 2001
+From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>
+Date: Wed, 13 May 2026 16:30:33 +0300
+Subject: Add CycloneDX plugin
+
+---
+ build.gradle | 45 +++++++++++++++++++++++++++++++++++++++++++++
+ 1 file changed, 45 insertions(+)
+
+diff --git a/build.gradle b/build.gradle
+index c07cd3b047..effbe0f65d 100644
+--- a/build.gradle
++++ b/build.gradle
+@@ -41,8 +41,53 @@ plugins {
+   id 'org.scoverage' version '8.1' apply false
+   id 'com.gradleup.shadow' version '8.3.9' apply false
+   id 'com.diffplug.spotless' version "8.0.0"
++  id 'org.cyclonedx.bom' version '3.2.4'
+ }
+
++cyclonedxBom {
++    // Specified the type of project being built. Defaults to 'library'
++    projectType = org.cyclonedx.model.Component.Type.APPLICATION
++    // Specified the version of the CycloneDX specification to use. Defaults to VERSION_16.
++    schemaVersion = org.cyclonedx.Version.VERSION_15
++    jsonOutput = file("build/reports/bom.json")
++    xmlOutput.unsetConvention()
++}
++
++// Exclude test components. This list needs to be checked and, if it changed, updated for every new Kafka version.
++// The list can be obtained by running `gradle projects | grep upgrade-system-tests`
++def skipCyclonedxProjects = [
++  ':streams:upgrade-system-tests-0110',
++  ':streams:upgrade-system-tests-10',
++  ':streams:upgrade-system-tests-11',
++  ':streams:upgrade-system-tests-20',
++  ':streams:upgrade-system-tests-21',
++  ':streams:upgrade-system-tests-22',
++  ':streams:upgrade-system-tests-23',
++  ':streams:upgrade-system-tests-24',
++  ':streams:upgrade-system-tests-25',
++  ':streams:upgrade-system-tests-26',
++  ':streams:upgrade-system-tests-27',
++  ':streams:upgrade-system-tests-28',
++  ':streams:upgrade-system-tests-30',
++  ':streams:upgrade-system-tests-31',
++  ':streams:upgrade-system-tests-32',
++  ':streams:upgrade-system-tests-33',
++  ':streams:upgrade-system-tests-34',
++  ':streams:upgrade-system-tests-35',
++  ':streams:upgrade-system-tests-36',
++  ':streams:upgrade-system-tests-37',
++  ':streams:upgrade-system-tests-38',
++  ':streams:upgrade-system-tests-39',
++  ':streams:upgrade-system-tests-40',
++  ':streams:upgrade-system-tests-41',
++]
++
++configure(subprojects.findAll { skipCyclonedxProjects.contains(it.path) }) {
++  tasks.named('cyclonedxDirectBom') {
++    enabled = false
++  }
++}
++ 
+ ext {
+   minClientJavaVersion = 11
+   minNonClientJavaVersion = 17

kafka/stackable/patches/4.2.1/0002-Change-Gradle-to-use-the-Nexus-Build-Repo.patch

@@ -0,0 +1,35 @@
+From e4e184668e99d71e2cbdebfd55ef4ba3322a6508 Mon Sep 17 00:00:00 2001
+From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>
+Date: Wed, 13 May 2026 16:31:44 +0300
+Subject: Change Gradle to use the Nexus Build Repo
+
+---
+ build.gradle | 8 ++++++--
+ 1 file changed, 6 insertions(+), 2 deletions(-)
+
+diff --git a/build.gradle b/build.gradle
+index effbe0f65d..bd840210bd 100644
+--- a/build.gradle
++++ b/build.gradle
+@@ -18,7 +18,9 @@ import java.nio.charset.StandardCharsets
+
+ buildscript {
+   repositories {
+-    mavenCentral()
++    maven {
++      url 'https://build-repo.stackable.tech/repository/maven-public/'
++    }
+   }
+   apply from: "$rootDir/gradle/dependencies.gradle"
+
+@@ -212,7 +214,9 @@ ext {
+ allprojects {
+
+   repositories {
+-    mavenCentral()
++    maven {
++      url 'https://build-repo.stackable.tech/repository/maven-public/'
++    }
+   }
+
+   dependencyUpdates {