chore(airflow): Bump to 3.3.1

CHANGELOG.md

@@ -12,6 +12,7 @@ All notable changes to this project will be documented in this file.
 - trino: Add SBOMs for the web UI, both for the two npm projects behind it and for the pre-built JavaScript vendored into the source tree ([#1620]).
 - hadoop, spark: Add SBOMs for the pre-built JavaScript that is vendored into the source tree for the HDFS and Spark web UIs ([#1620]).
 - stats-exporter: Add `0.31.0` ([#1664]).
+- airflow: Add `3.3.1`, deprecate `3.2.2` ([#1665]).
 
 ### Changed
 
@@ -57,6 +58,7 @@ All notable changes to this project will be documented in this file.
 
 - omid: remove 1.1.2 ([#1593]).
 - ci: Remove SLSA build provenance generation for published image indexes ([#1596]).
+- airflow: Remove `3.1.6` ([#1665]).
 
 [#1593]: https://github.com/stackabletech/docker-images/pull/1593
 [#1595]: https://github.com/stackabletech/docker-images/pull/1595
@@ -71,6 +73,7 @@ All notable changes to this project will be documented in this file.
 [#1638]: https://github.com/stackabletech/docker-images/pull/1638
 [#1641]: https://github.com/stackabletech/docker-images/pull/1641
 [#1664]: https://github.com/stackabletech/docker-images/pull/1664
+[#1665]: https://github.com/stackabletech/docker-images/pull/1665
 
 ## [26.7.0] - 2026-07-21
 

airflow/Dockerfile

@@ -17,23 +17,46 @@ FROM local-image/shared/statsd-exporter AS statsd_exporter-builder
 FROM local-image/vector AS opa-auth-manager-builder
 
 ARG OPA_AUTH_MANAGER
+ARG PRODUCT_VERSION
 ARG PYTHON_VERSION
 ARG STACKABLE_USER_UID
 
 COPY airflow/opa-auth-manager/${OPA_AUTH_MANAGER} /tmp/opa-auth-manager
 
+# The tests need the same constraints the image is built with, see the comment
+# in the RUN block below.
+COPY airflow/stackable/constraints/${PRODUCT_VERSION}/constraints-python${PYTHON_VERSION}.txt /tmp/constraints.txt
+
 WORKDIR /tmp/opa-auth-manager
 
 COPY --from=uv-image --chown=${STACKABLE_USER_UID}:0 /uv /uvx /bin/
 
 RUN <<EOF
-
 # This folder is required by the tests to set up an sqlite database
 mkdir /root/airflow
 
+# Provides the test tooling, and the project itself so the tests can import it.
+# --locked fails the build if uv.lock is out of date with pyproject.toml, rather
+# than silently re-resolving it here.
+uv sync --locked --python "${PYTHON_VERSION}"
+
+# The OPA auth manager subclasses FabAuthManager, so a signature change in the
+# FAB provider breaks it at import time. One source tree is shared by several
+# Airflow versions (see opa-auth-manager in airflow/boil-config.toml), so the
+# only way for the tests to see what a given image ships is to install Airflow
+# at PRODUCT_VERSION and let that version's constraints file pin the FAB
+# provider. Without this, a provider signature change only surfaces at runtime.
+uv pip install \
+  --python .venv/bin/python \
+  --constraint /tmp/constraints.txt \
+  "apache-airflow==${PRODUCT_VERSION}" \
+  apache-airflow-providers-fab
+
 # Warnings are disabled because they come from various third party testing libraries
 # that we have no control over.
-uv run pytest --disable-warnings
+# --no-sync keeps uv from reverting the versions installed just above.
+uv run --no-sync pytest --disable-warnings
+
 uv build
 EOF
 

airflow/README.md

@@ -7,14 +7,15 @@ script resides. This example will assume you are running from the repository roo
 
 ```sh
 # Specify Airflow and Python versions:
-./airflow/download_constraints.sh 3.0.6 3.12
+./airflow/download_constraints.sh 3.3.1 3.14
 ```
 
 Example output:
 
 ```output
-Downloading constraints file for Airflow 3.0.6 (Python 3.12)
-Successfully pulled new constraints file: constraints-3.0.6-python3.12.txt
+Downloading constraints file for Airflow 3.3.1 (Python 3.14)
+Successfully pulled new constraints file: constraints-3.3.1-python3.14.txt
+
 ```
 
 ## Airflow providers/extras
@@ -24,7 +25,7 @@ The list of provider packages are listed in the build configuration file, matchi
 The expected versions are listed in the constraints files, but these can change over time.
 To keep the installation tightly coupled to the associated constraints it is best to only use providers listed in the relevant constraints file.
 
-### Version 3.0.6
+### Version 3.3.1
 
 Applying the filter above results in the omission of the following providers:
 

airflow/boil-config.toml

@@ -3,7 +3,7 @@
 
 # Deprecated since SDP 25.11
 [versions."2.9.3".local-images]
-"shared/statsd-exporter" = "0.30.0"
+"shared/statsd-exporter" = "0.31.0"
 vector = "0.55.0"
 stackable-devel = "1.0.0"
 
@@ -43,7 +43,7 @@ nodejs-version = "22"
 
 # LTS
 [versions."3.0.6".local-images]
-"shared/statsd-exporter" = "0.30.0"
+"shared/statsd-exporter" = "0.31.0"
 vector = "0.55.0"
 stackable-devel = "1.0.0"
 
@@ -103,21 +103,21 @@ nodejs-version = "22"
 # TODO: Specify the pnpm version here
 
 # Deprecated
-[versions."3.1.6".local-images]
-"shared/statsd-exporter" = "0.30.0"
+[versions."3.2.2".local-images]
+"shared/statsd-exporter" = "0.31.0"
 vector = "0.55.0"
 stackable-devel = "1.0.0"
 
-[versions."3.1.6".build-arguments]
+[versions."3.2.2".build-arguments]
 # NOTE: This can be bumped for new airflow versions, but once set, do not updated it, as it could
 # break many customer DAGs. We generally try to pick the latest supported version unless there are
 # reasons against it.
-# This version of Airflow supports: 3.10 - 3.13 (see https://airflow.apache.org/docs/apache-airflow/3.1.6/installation/prerequisites.html)
-python-version = "3.12"
+# This version of Airflow supports: 3.10 - 3.14 (see https://airflow.apache.org/docs/apache-airflow/3.2.2/installation/prerequisites.html)
+python-version = "3.14"
 git-sync-version = "v4.6.0"
 # Must match the version in the constraints file (search for s3fs==)
 # TODO: Check if this can be removed in favour of letting it come in via the s3fs extra and constraints.
-s3fs-version = "2026.1.0"
+s3fs-version = "2026.4.0"
 cyclonedx-bom-version = "7.3.0"
 # Used to create the SBOM of the frontend (npm) dependencies.
 # Note: 13.x and later are published as @cdxgen/cdxgen, 12.x and earlier as @cyclonedx/cdxgen.
@@ -139,22 +139,22 @@ uv-version = "0.11.19"
 # Airflow extras are defined in separate lists to make them easier to check against the links below.
 # The lists will be concatenated and duplicates removed in the dockerfile.
 # TODO: Add policy for when to add extras.
-# See https://airflow.apache.org/docs/apache-airflow/3.1.6/extra-packages-ref.html#core-airflow-extras
+# See https://airflow.apache.org/docs/apache-airflow/3.2.2/extra-packages-ref.html#core-airflow-extras
 airflow-extras-core = "async,graphviz,kerberos,otel,sentry,standard,statsd"
 
-# See https://airflow.apache.org/docs/apache-airflow/3.1.6/extra-packages-ref.html#meta-airflow-package-extras
+# See https://airflow.apache.org/docs/apache-airflow/3.2.2/extra-packages-ref.html#meta-airflow-package-extras
 airflow-extras-meta = "aiobotocore,cloudpickle,github-enterprise,google-auth,graphviz,ldap,leveldb,pandas,polars,rabbitmq,s3fs,saml,uv"
 
-# See https://airflow.apache.org/docs/apache-airflow/3.1.6/extra-packages-ref.html#apache-software-extras
+# See https://airflow.apache.org/docs/apache-airflow/3.2.2/extra-packages-ref.html#apache-software-extras
 airflow-extras-provider-apache = "apache-beam,apache-cassandra,apache-drill,apache-druid,apache-flink,apache-hdfs,apache-hive,apache-iceberg,apache-impala,apache-kafka,apache-kylin,apache-livy,apache-pig,apache-pinot"
 
-# See https://airflow.apache.org/docs/apache-airflow/3.1.6/extra-packages-ref.html#external-services-extras
+# See https://airflow.apache.org/docs/apache-airflow/3.2.2/extra-packages-ref.html#external-services-extras
 airflow-extras-external-services = "airbyte,alibaba,apprise,amazon,asana,atlassian-jira,microsoft-azure,cloudant,cohere,databricks,datadog,dbt-cloud,dingding,discord,facebook,github,google,hashicorp,openai,opsgenie,pagerduty,pgvector,pinecone,qdrant,salesforce,sendgrid,segment,slack,snowflake,tableau,tabular,telegram,vertica,weaviate,yandex,ydb,zendesk"
 
-# See https://airflow.apache.org/docs/apache-airflow/3.1.6/extra-packages-ref.html#locally-installed-software-extras
+# See https://airflow.apache.org/docs/apache-airflow/3.2.2/extra-packages-ref.html#locally-installed-software-extras
 airflow-extras-locally-installed-software = "arangodb,celery,cncf-kubernetes,docker,edge3,elasticsearch,exasol,fab,git,github,influxdb,jenkins,mongo,microsoft-mssql,neo4j,odbc,openfaas,oracle,postgres,presto,redis,samba,singularity,teradata,trino"
 
-# See https://airflow.apache.org/docs/apache-airflow/3.1.6/extra-packages-ref.html#other-extras
+# See https://airflow.apache.org/docs/apache-airflow/3.2.2/extra-packages-ref.html#other-extras
 airflow-extras-other = "common-compat,common-io,common-messaging,common-sql,ftp,grpc,http,imap,jdbc,microsoft-psrp,microsoft-winrm,openlineage,opensearch,papermill,sftp,smtp,sqlite,ssh"
 
 opa-auth-manager = "airflow-3"
@@ -164,18 +164,18 @@ nodejs-version = "22"
 # TODO: Specify the pnpm version here
 
 # Supported
-[versions."3.2.2".local-images]
-"shared/statsd-exporter" = "0.30.0"
+[versions."3.3.1".local-images]
+"shared/statsd-exporter" = "0.31.0"
 vector = "0.55.0"
 stackable-devel = "1.0.0"
 
-[versions."3.2.2".build-arguments]
+[versions."3.3.1".build-arguments]
 # NOTE: This can be bumped for new airflow versions, but once set, do not updated it, as it could
 # break many customer DAGs. We generally try to pick the latest supported version unless there are
 # reasons against it.
-# This version of Airflow supports: 3.10 - 3.14 (see https://airflow.apache.org/docs/apache-airflow/3.2.2/installation/prerequisites.html)
+# This version of Airflow supports: 3.10 - 3.14 (see https://airflow.apache.org/docs/apache-airflow/3.3.1/installation/prerequisites.html)
 python-version = "3.14"
-git-sync-version = "v4.6.0"
+git-sync-version = "v4.7.1"
 # Must match the version in the constraints file (search for s3fs==)
 # TODO: Check if this can be removed in favour of letting it come in via the s3fs extra and constraints.
 s3fs-version = "2026.4.0"
@@ -184,7 +184,7 @@ cyclonedx-bom-version = "7.3.0"
 # Note: 13.x and later are published as @cdxgen/cdxgen, 12.x and earlier as @cyclonedx/cdxgen.
 # Find the latest version here: https://www.npmjs.com/package/@cdxgen/cdxgen
 # renovate: datasource=npm packageName=@cdxgen/cdxgen
-cdxgen-version = "13.0.1"
+cdxgen-version = "13.1.0"
 # cdxgen requires Node >= 24, independent of the Node version used to build the frontend
 # assets, so it gets its own Node installation.
 # Find the latest release here: https://github.com/nodejs/node/releases
@@ -200,23 +200,23 @@ uv-version = "0.11.19"
 # Airflow extras are defined in separate lists to make them easier to check against the links below.
 # The lists will be concatenated and duplicates removed in the dockerfile.
 # TODO: Add policy for when to add extras.
-# See https://airflow.apache.org/docs/apache-airflow/3.2.2/extra-packages-ref.html#core-airflow-extras
-airflow-extras-core = "async,graphviz,kerberos,otel,sentry,standard,statsd"
+# See https://airflow.apache.org/docs/apache-airflow/3.3.1/extra-packages-ref.html#core-airflow-extras
+airflow-extras-core = "async,graphviz,kerberos,memray,otel,sentry,standard,statsd"
 
-# See https://airflow.apache.org/docs/apache-airflow/3.2.2/extra-packages-ref.html#meta-airflow-package-extras
-airflow-extras-meta = "aiobotocore,cloudpickle,github-enterprise,google-auth,graphviz,ldap,leveldb,pandas,polars,rabbitmq,s3fs,saml,uv"
+# See https://airflow.apache.org/docs/apache-airflow/3.3.1/extra-packages-ref.html#meta-airflow-package-extras
+airflow-extras-meta = "aiobotocore,amazon-aws-auth,cloudpickle,github-enterprise,google-auth,graphviz,gunicorn,ldap,leveldb,pandas,polars,rabbitmq,s3fs,saml,uv"
 
-# See https://airflow.apache.org/docs/apache-airflow/3.2.2/extra-packages-ref.html#apache-software-extras
-airflow-extras-provider-apache = "apache-beam,apache-cassandra,apache-drill,apache-druid,apache-flink,apache-hdfs,apache-hive,apache-iceberg,apache-impala,apache-kafka,apache-kylin,apache-livy,apache-pig,apache-pinot"
+# See https://airflow.apache.org/docs/apache-airflow/3.3.1/extra-packages-ref.html#apache-software-extras
+airflow-extras-provider-apache = "apache-beam,apache-cassandra,apache-drill,apache-druid,apache-flink,apache-hdfs,apache-hive,apache-iceberg,apache-impala,apache-kafka,apache-kylin,apache-livy,apache-pig,apache-pinot,apache-tinkerpop,apache-webhdfs"
 
-# See https://airflow.apache.org/docs/apache-airflow/3.2.2/extra-packages-ref.html#external-services-extras
-airflow-extras-external-services = "airbyte,alibaba,apprise,amazon,asana,atlassian-jira,microsoft-azure,cloudant,cohere,databricks,datadog,dbt-cloud,dingding,discord,facebook,github,google,hashicorp,openai,opsgenie,pagerduty,pgvector,pinecone,qdrant,salesforce,sendgrid,segment,slack,snowflake,tableau,tabular,telegram,vertica,weaviate,yandex,ydb,zendesk"
+# See https://airflow.apache.org/docs/apache-airflow/3.3.1/extra-packages-ref.html#external-services-extras
+airflow-extras-external-services = "airbyte,akeyless,alibaba,apprise,amazon,asana,atlassian-jira,microsoft-azure,clickhousedb,cloudant,cohere,databricks,datadog,dbt-cloud,dingding,discord,facebook,github,google,hashicorp,openai,opsgenie,pagerduty,pgvector,pinecone,qdrant,salesforce,sendgrid,segment,slack,snowflake,tableau,tabular,telegram,vertica,vespa,weaviate,yandex,ydb,zendesk"
 
-# See https://airflow.apache.org/docs/apache-airflow/3.2.2/extra-packages-ref.html#locally-installed-software-extras
-airflow-extras-locally-installed-software = "arangodb,celery,cncf-kubernetes,docker,edge3,elasticsearch,exasol,fab,git,github,influxdb,jenkins,mongo,microsoft-mssql,neo4j,odbc,openfaas,oracle,postgres,presto,redis,samba,singularity,teradata,trino"
+# See https://airflow.apache.org/docs/apache-airflow/3.3.1/extra-packages-ref.html#locally-installed-software-extras
+airflow-extras-locally-installed-software = "arangodb,celery,cncf-kubernetes,docker,edge3,elasticsearch,exasol,fab,git,github,influxdb,ibm-mq,jenkins,mongo,microsoft-mssql,neo4j,odbc,openfaas,oracle,postgres,presto,redis,samba,singularity,teradata,trino"
 
-# See https://airflow.apache.org/docs/apache-airflow/3.2.2/extra-packages-ref.html#other-extras
-airflow-extras-other = "common-compat,common-io,common-messaging,common-sql,ftp,grpc,http,imap,jdbc,microsoft-psrp,microsoft-winrm,openlineage,opensearch,papermill,sftp,smtp,sqlite,ssh"
+# See https://airflow.apache.org/docs/apache-airflow/3.3.1/extra-packages-ref.html#other-extras
+airflow-extras-other = "common-ai,common-compat,common-io,common-messaging,common-sql,ftp,grpc,http,imap,jdbc,keycloak,microsoft-psrp,microsoft-winrm,openlineage,opensearch,papermill,sftp,smtp,sqlite,ssh,informatica"
 
 opa-auth-manager = "airflow-3"
 # Node module stream used to install pnpm/yarn and build the frontend assets.

airflow/opa-auth-manager/airflow-2/README.md

@@ -3,10 +3,27 @@
 Auth manager for Airflow 2 which delegates the authorization to an Open Policy
 Agent
 
-[uv](https://docs.astral.sh/uv/) is used to build the project:
+Build:
 
     uv build
 
-The unit tests can be run as follows:
+Test:
 
-    uv run pytest
+    uv run --python 3.9 --group local pytest --disable-warnings
+
+- `--python` must match the image's `python-version` in
+  `airflow/boil-config.toml`; wheel availability differs between Python
+  versions.
+- The `local` group holds the Airflow and FAB versions to test against. The
+  image build ignores it and installs both from
+  `airflow/stackable/constraints/<PRODUCT_VERSION>/` instead.
+
+## Updating `uv.lock`
+
+The build runs `uv sync --locked`, so the lock must match `pyproject.toml`.
+
+Run `uv lock --upgrade` after any change to it.
+
+Always `--upgrade`: uv otherwise keeps already-locked versions, which can leave
+packages with no wheel for a newer Python — and the builder stage has no C
+compiler to build them from source.

airflow/opa-auth-manager/airflow-2/pyproject.toml

@@ -9,20 +9,32 @@ authors = [
 readme = "README.md"
 requires-python = ">=3.9,<3.13"
 
+# Deliberately loose. The wheel is installed into the environment Airflow was
+# installed into under airflow/stackable/constraints/, which pins requests and
+# cachetools itself; exact pins here would fight what Airflow resolved.
+# overrides is not in the constraints file, so it stays pinned.
 dependencies = [
-    "requests~=2.32.3",
-    "cachetools~=5.5.0",
+    "requests>=2.31",
+    "cachetools>=5.3",
     "overrides~=7.7.0",
 ]
 
 [dependency-groups]
+# Test tooling only, and uv's default group; see the airflow-3 project for why
+# Airflow itself is not in here.
 dev = [
-    "apache-airflow~=2.9.3",
     "pylint~=3.3.1",
     "pytest~=8.3.3",
     "flask-limiter~=3.9.0",
 ]
 
+# Adds an Airflow to run the tests against outside the image build, where there
+# is no constraints file: `uv run --group local pytest`. Unused by the build.
+local = [
+    { include-group = "dev" },
+    "apache-airflow~=2.9.3",
+]
+
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"