ci: move the shipped binaries to AlmaLinux 8, get the test images off EOL Debian

.github/workflows/ci.yml

@@ -45,6 +45,29 @@ jobs:
             .github/scripts/tests/test_release_automation.sh
           .github/scripts/tests/test_release_automation.sh
 
+  ci-script-tests:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Validate pipeline scripts without network access
+        run: |
+          bash -n \
+            .gitlab/scripts/build.sh \
+            .gitlab/scripts/check-abi-floor.sh \
+            .gitlab/scripts/tests/check_abi_floor_test.sh \
+            .gitlab/scripts/tests/includes_test.sh
+          shellcheck \
+            .gitlab/scripts/check-abi-floor.sh \
+            .gitlab/scripts/tests/check_abi_floor_test.sh
+          bash .gitlab/scripts/tests/check_abi_floor_test.sh
+          bash .gitlab/scripts/tests/includes_test.sh
+
   check-for-pr:
     runs-on: ubuntu-latest
     outputs:

.gitlab/base/Dockerfile

@@ -1,11 +1,27 @@
-ARG BASE_IMAGE=debian:bullseye-slim@sha256:0083feb8da4f624e3a0245e7752af2517d4b81d8b8db50c725644672a132a31b
+# CI always passes BASE_IMAGE explicitly (OPENJDK_BASE_IMAGE /
+# OPENJDK_BASE_IMAGE_ARM64), both of which are this same digest: the shipped
+# binaries are built elsewhere (.gitlab/base/el8/Dockerfile), so neither arch
+# of this image sets a customer-facing glibc floor. The default matches them,
+# so a hand-built image -- or rebuild-images.sh reaching its unset-variable
+# fallback -- gets a release whose packages still resolve.
+ARG BASE_IMAGE=debian:trixie-slim@sha256:d7e12182ce18b85b93007c1dedf31f2d29e01ccf3182cc4017c709b6259bc132
 FROM ${BASE_IMAGE} as base
 ARG CI_JOB_TOKEN
 WORKDIR /root
 
 RUN mkdir -p /usr/share/man/man1 # https://github.com/debuerreotype/docker-debian-artifacts/issues/24
-RUN (apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends curl git moreutils awscli amazon-ecr-credential-helper gnupg2 npm build-essential wget bsdmainutils clang jq zip unzip) || true
-RUN (apk update && apk add curl git moreutils aws-cli docker-credential-ecr-login gnupg alpine-sdk build-base wget npm hexdump linux-headers clang compiler-rt bash jq gradle zip unzip) || true
+# Alpine/musl images build from Dockerfile.musl, so this file installs only the
+# Debian toolchain.
+#
+# The second group is what the C++ test and sanitizer jobs need. They live in
+# the image rather than in a job's before_script so that a job does not depend
+# on the distro archive still serving those packages at run time.
+# libclang-rt-dev is the unversioned metapackage: it provides
+# sanitizer/asan_interface.h from whichever clang runtime matches this release.
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        curl git moreutils awscli amazon-ecr-credential-helper gnupg2 npm build-essential \
+        wget bsdmainutils clang jq zip unzip \
+        cmake libgtest-dev libgmock-dev libc6-dbg llvm libclang-rt-dev
 # Install JDK 21 and Maven via SDKMAN (glibc image only; musl uses Dockerfile.musl).
 RUN curl -s "https://get.sdkman.io" | bash && \
     bash -c "source /root/.sdkman/bin/sdkman-init.sh && sdk install java 21.0.3-tem && sdk install maven"

.gitlab/base/el8/Dockerfile

@@ -0,0 +1,37 @@
+# Build image for the shipped glibc binaries, both architectures.
+#
+# AlmaLinux 8 (glibc 2.28) with gcc-toolset-14. Release builds link
+# -static-libstdc++ -static-libgcc (see ConfigurationPresets.kt), so the C++
+# runtime and unwinder in the shipped libjavaProfiler.so come out of this
+# image: the toolchain has to be one that still receives security rebuilds,
+# which rules out the EL7 software collections (frozen at CentOS 7.9, and with
+# no gcc newer than devtoolset-10 ever built for aarch64).
+#
+# almalinux:8 is a multi-arch manifest, so BASE_IMAGE is the same for both
+# platforms (BASE_IMAGE_GLIBC_2_28 in .gitlab/build-deploy/images.yml) and
+# `docker build --platform` selects the architecture. gcc-toolset-14 is
+# published for x86_64 and aarch64 alike, so both arches compile with one
+# compiler version.
+ARG BASE_IMAGE=almalinux:8
+FROM ${BASE_IMAGE} as base
+ARG CI_JOB_TOKEN
+WORKDIR /root
+
+# gcc-toolset-14 carries its own libstdc++.a/libgcc.a, which is what
+# -static-libstdc++ resolves against; no separate *-static package is needed.
+RUN dnf -y install gcc-toolset-14 \
+                   git make cmake zip unzip which wget jq \
+ && dnf -y clean all
+
+# Put the toolset ahead of the distro gcc for every shell, including the
+# non-login shells GitLab uses to run job scripts (the `scl enable` /
+# /opt/rh/gcc-toolset-14/enable wrappers only affect interactive use).
+ENV PATH="/opt/rh/gcc-toolset-14/root/usr/bin:${PATH}" \
+    LD_LIBRARY_PATH="/opt/rh/gcc-toolset-14/root/usr/lib64:${LD_LIBRARY_PATH}"
+
+# The build scripts locate a JDK through JAVA_HOME, falling back to SDKMAN's
+# current candidate (.gitlab/scripts/build.sh).
+RUN curl -s "https://get.sdkman.io" | bash && \
+    bash -c "source /root/.sdkman/bin/sdkman-init.sh && sdk install java 21.0.3-tem && sdk install maven"
+ENV JAVA_HOME=/root/.sdkman/candidates/java/current
+ENV PATH="/root/.sdkman/candidates/java/current/bin:/root/.sdkman/candidates/maven/current/bin:${PATH}"

.gitlab/benchmarks/images.yml

@@ -5,7 +5,7 @@ variables:
   BASE_BENCHMARK_IMAGE_NAME: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:newest
   BASE_CI_IMAGE_NAME: registry.ddbuild.io/ci/async-profiler-build
 
-  # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/1600939090
-  BENCHMARK_IMAGE_AMD64: registry.ddbuild.io/ci/async-profiler-build-amd64:v107978918-amd64-benchmarks@sha256:95d4e3719717a6af63ed62437b985e36a95b4552bac04a39cd7b82bafa09271d
-  # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/1600939090
-  BENCHMARK_IMAGE_ARM64: registry.ddbuild.io/ci/async-profiler-build-arm64:v107978918-arm64-benchmarks@sha256:b534640f415e2c1af44611c1531a0435a387c85f435876fa2e7e5011e4749221
+  # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/2040944172
+  BENCHMARK_IMAGE_AMD64: registry.ddbuild.io/ci/async-profiler-build-amd64:v137281788-amd64-benchmarks@sha256:28e10be33f32e73ba3850796b3fc7abe747b9e1c71046ea1b91ead84ae7d4ed6
+  # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/2040944172
+  BENCHMARK_IMAGE_ARM64: registry.ddbuild.io/ci/async-profiler-build-arm64:v137281788-arm64-benchmarks@sha256:c03d7ac565f4b36fc3086a00fe9dcd5d89385b799d78bab7e16b1a8cff713f70

.gitlab/build-deploy/.gitlab-ci.yml

@@ -1,18 +1,20 @@
 image: alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1
 
 variables:
-  # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/1600939090
-  BUILD_IMAGE_X64: registry.ddbuild.io/ci/async-profiler-build:v107978918-x64-base@sha256:4712b562eb75cf27c5d494081f459fdbd7578ef46a99162e1051ef7d1f79da4f
-  # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/1600939090
-  BUILD_IMAGE_ARM64: registry.ddbuild.io/ci/async-profiler-build:v107978918-arm64-base@sha256:15670752fa8e67c81c1b7fd4d93ef34c668649f117584ab29327d8d7733d64d6
-  # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/1600939090
-  BUILD_IMAGE_X64_2_17: registry.ddbuild.io/ci/async-profiler-build:v107978918-x64-2.17-base@sha256:03fb65c5c7fb2866397ceaf18ccb5a0b91bb377a9f89c94264427b62d48cfffd
-  # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/1600939090
-  BUILD_IMAGE_X64_MUSL: registry.ddbuild.io/ci/async-profiler-build:v107978918-x64-musl-base@sha256:0efe0c5fbd04e5647a53235d4c4942a2dd305b3d76bf5afdc052d64a7c0b9505
-  # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/1600939090
-  BUILD_IMAGE_ARM64_MUSL: registry.ddbuild.io/ci/async-profiler-build:v107978918-arm64-musl-base@sha256:1c8a96ded178bfa130077166db049cbea422c94c96a8c5172d66f8c3581d57cf
-  # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/1600939090
-  DATADOG_CI_IMAGE: registry.ddbuild.io/ci/async-profiler-build:v107978918-datadog-ci@sha256:1353737e433ef58b37ae34daf6edc0d6a5fc4706bac76ac97729e981869ea680
+  # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/2040944172
+  BUILD_IMAGE_X64: registry.ddbuild.io/ci/async-profiler-build:v137281788-x64-base@sha256:81eab385c41ac74a4f71140df9972883ca20bb9d97756f443f9001e9d415d832
+  # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/2040944172
+  BUILD_IMAGE_ARM64: registry.ddbuild.io/ci/async-profiler-build:v137281788-arm64-base@sha256:281a24ae99326041711ed4f63e5528f439091ddc8c19f79c490e9021434486be
+  # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/2040944172
+  BUILD_IMAGE_X64_GLIBC: registry.ddbuild.io/ci/async-profiler-build:v137281788-x64-glibc-base@sha256:270a9101716a0865071182e0ef0e40e98f874535b90722949b33a4a8998fe08d
+  # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/2040944172
+  BUILD_IMAGE_ARM64_GLIBC: registry.ddbuild.io/ci/async-profiler-build:v137281788-arm64-glibc-base@sha256:511d051d37ed99b2ca9cd3af797aa9de63e4b4b952af1e738b2cea913bbcc057
+  # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/2040944172
+  BUILD_IMAGE_X64_MUSL: registry.ddbuild.io/ci/async-profiler-build:v137281788-x64-musl-base@sha256:fd3803695771de3eabf44e4953c0474273f9dbb97d88c22e99ac507b88572481
+  # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/2040944172
+  BUILD_IMAGE_ARM64_MUSL: registry.ddbuild.io/ci/async-profiler-build:v137281788-arm64-musl-base@sha256:edff5b0205db2d1d520debdd35f9ae6b047a6c0f80bd6f3eca02b57cf56c8c98
+  # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/2040944172
+  DATADOG_CI_IMAGE: registry.ddbuild.io/ci/async-profiler-build:v137281788-datadog-ci@sha256:84b7902bec3b4cef3e5dbeb0a8a2c2542f388a4c72bdcd736eb9db4fdd107d9f
 
   LAST_COMMIT_FILE: .last.commit
   CACHE_FALLBACK_KEY: async-profiler
@@ -92,8 +94,7 @@ prepare:start:
 build:x64:
   extends: .build_job
   variables:
-    # default to the libc 2.17 linked version
-    BUILD_IMAGE: ${BUILD_IMAGE_X64_2_17}
+    BUILD_IMAGE: ${BUILD_IMAGE_X64_GLIBC}
     TARGET: linux-x64
 
 build:x64-musl:
@@ -106,7 +107,8 @@ build:arm64:
   extends: .build_job
   tags: [ "arch:arm64" ]
   variables:
-    BUILD_IMAGE: ${BUILD_IMAGE_ARM64}
+    # shipped-binary image, mirroring build:x64
+    BUILD_IMAGE: ${BUILD_IMAGE_ARM64_GLIBC}
     TARGET: linux-arm64
 
 build:arm64-musl:
@@ -125,8 +127,7 @@ stresstest:x64:
     - job: build:x64
       artifacts: true
   variables:
-    # default to the libc 2.17 linked version
-    BUILD_IMAGE: ${BUILD_IMAGE_X64_2_17}
+    BUILD_IMAGE: ${BUILD_IMAGE_X64_GLIBC}
     TARGET: linux-x64
 
 stresstest:x64-musl:
@@ -149,7 +150,8 @@ stresstest:arm64:
       artifacts: true
   tags: [ "arch:arm64" ]
   variables:
-    BUILD_IMAGE: ${BUILD_IMAGE_ARM64}
+    # same image as build:arm64, so the stress run exercises the shipped build
+    BUILD_IMAGE: ${BUILD_IMAGE_ARM64_GLIBC}
     TARGET: linux-arm64
 
 stresstest:arm64-musl:
@@ -181,7 +183,7 @@ chaos:build:
       when: always
     - when: never
   tags: [ "arch:amd64" ]
-  image: ${BUILD_IMAGE_X64_2_17}
+  image: ${BUILD_IMAGE_X64_GLIBC}
   script:
     - |
       if [ -z "${JAVA_HOME}" ] || [ ! -x "${JAVA_HOME}/bin/java" ]; then

.gitlab/build-deploy/images.yml

@@ -2,14 +2,23 @@ stages:
   - images
 variables:
   # Base images for the build docker images
-  # debian:bullseye-slim = Debian 11 (glibc 2.31); pinned by digest for reproducibility.
-  # Multi-arch manifest covers both amd64 and arm64 — no separate arm64 tag needed.
-  # JDK 21 is installed at image-build time via SDKMAN (see .gitlab/base/Dockerfile).
-  OPENJDK_BASE_IMAGE: debian:bullseye-slim@sha256:0083feb8da4f624e3a0245e7752af2517d4b81d8b8db50c725644672a132a31b
-  OPENJDK_BASE_IMAGE_ARM64: debian:bullseye-slim@sha256:0083feb8da4f624e3a0245e7752af2517d4b81d8b8db50c725644672a132a31b
+  # Both pinned by digest for reproducibility. JDK 21 is installed at
+  # image-build time via SDKMAN (see .gitlab/base/Dockerfile).
+  #
+  # Debian 13 trixie (glibc 2.41), current stable. These images run the test,
+  # sanitizer, integration and packaging jobs; the shipped binaries are
+  # compiled in the AlmaLinux 8 image below, so this glibc is not a customer
+  # floor and the release can track stable.
+  OPENJDK_BASE_IMAGE: debian:trixie-slim@sha256:d7e12182ce18b85b93007c1dedf31f2d29e01ccf3182cc4017c709b6259bc132
+  OPENJDK_BASE_IMAGE_ARM64: debian:trixie-slim@sha256:d7e12182ce18b85b93007c1dedf31f2d29e01ccf3182cc4017c709b6259bc132
   # eclipse-temurin:21-jdk-alpine = Alpine 3.x musl; arm64 manifest confirmed present.
   OPENJDK_BASE_IMAGE_MUSL: eclipse-temurin:21-jdk-alpine@sha256:c98f0d2e171c898bf896dc4166815d28a56d428e218190a1f35cdc7d82efd61f
   OPENJDK_BASE_IMAGE_ARM64_MUSL: eclipse-temurin:21-jdk-alpine@sha256:c98f0d2e171c898bf896dc4166815d28a56d428e218190a1f35cdc7d82efd61f
-  BASE_IMAGE_LIBC_2_17: centos:7
+  # AlmaLinux 8 (glibc 2.28) with gcc-toolset-14 compiles the shipped glibc
+  # binaries on both architectures. Release builds link -static-libstdc++
+  # -static-libgcc, so this toolchain's C++ runtime ships inside the library
+  # and must be one that still gets security rebuilds. The resulting .so
+  # requires GLIBC_2.27. Multi-arch manifest, pinned by digest.
+  BASE_IMAGE_GLIBC: almalinux:8@sha256:9f355ae942d6a6c0561f0771dc053a2cfae9580fc45fa4252756db7c7e80c09f
 
   DOCKER_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/images/docker:24.0.4-gbi-focal

.gitlab/config.env

@@ -10,11 +10,35 @@ JAVA_TEST_VERSION=21.0.3-tem
 JAVA_BENCHMARK_VERSION=21
 
 # Docker Base Images
-# debian:bullseye-slim = Debian 11 (glibc 2.31); pinned by digest for reproducibility.
-# Multi-arch manifests cover both amd64 and arm64.
-# JDK 21 is installed at image-build time via SDKMAN (see .gitlab/base/Dockerfile).
-OPENJDK_BASE_IMAGE_AMD64=debian:bullseye-slim@sha256:0083feb8da4f624e3a0245e7752af2517d4b81d8b8db50c725644672a132a31b
-OPENJDK_BASE_IMAGE_ARM64=debian:bullseye-slim@sha256:0083feb8da4f624e3a0245e7752af2517d4b81d8b8db50c725644672a132a31b
+# All pinned by digest for reproducibility; multi-arch manifests cover amd64
+# and arm64. JDK 21 is installed at image-build time via SDKMAN.
+#
+# debian:trixie-slim = Debian 13 (glibc 2.41), current stable. These run the
+# test, sanitizer, integration and packaging jobs (see .gitlab/base/Dockerfile);
+# the shipped binaries come from the AlmaLinux 8 image below, so this glibc
+# is not a customer-facing floor.
+OPENJDK_BASE_IMAGE_AMD64=debian:trixie-slim@sha256:d7e12182ce18b85b93007c1dedf31f2d29e01ccf3182cc4017c709b6259bc132
+OPENJDK_BASE_IMAGE_ARM64=debian:trixie-slim@sha256:d7e12182ce18b85b93007c1dedf31f2d29e01ccf3182cc4017c709b6259bc132
+# almalinux:8 = AlmaLinux 8 (glibc 2.28) with gcc-toolset-14, used to compile
+# the shipped glibc binaries on both architectures (see
+# .gitlab/base/el8/Dockerfile). Release builds link -static-libstdc++
+# -static-libgcc, so this toolchain's C++ runtime ends up inside the shipped
+# library and has to be one that still receives security rebuilds. The
+# resulting .so requires GLIBC_2.27 (expf), so it runs on glibc 2.27 or newer.
+BASE_IMAGE_GLIBC=almalinux:8@sha256:9f355ae942d6a6c0561f0771dc053a2cfae9580fc45fa4252756db7c7e80c09f
+# Highest glibc symbol version the shipped binaries are allowed to require,
+# asserted by .gitlab/scripts/check-abi-floor.sh at the end of every glibc build
+# job. It is the oldest glibc a customer can load the library on, so raising it
+# drops hosts and is a deliberate decision rather than a consequence of changing
+# the image above.
+#
+# 2.17 is the oldest glibc still in support among the distributions this ships
+# to (RHEL 7.9 and its derivatives). Holding it takes two things: expf is pinned
+# to each architecture's oldest interface (glibcCompat.h), and nothing here uses
+# std::random_device, whose libstdc++ implementation would otherwise pull in
+# getentropy@GLIBC_2.25 -- a symbol glibc exports in only one version, so no pin
+# could reach it.
+SHIPPED_GLIBC_FLOOR=2.17
 # eclipse-temurin:21-jdk-alpine = Alpine 3.x musl; arm64 manifest confirmed present.
 OPENJDK_BASE_IMAGE_AMD64_MUSL=eclipse-temurin:21-jdk-alpine@sha256:c98f0d2e171c898bf896dc4166815d28a56d428e218190a1f35cdc7d82efd61f
 OPENJDK_BASE_IMAGE_ARM64_MUSL=eclipse-temurin:21-jdk-alpine@sha256:c98f0d2e171c898bf896dc4166815d28a56d428e218190a1f35cdc7d82efd61f

.gitlab/sanitizer-tests/.gitlab-ci.yml

@@ -20,9 +20,9 @@
       when: never
     - when: on_success
   interruptible: true
-  before_script:
-    - apt-get update -qq
-    - apt-get install -y -qq cmake libgtest-dev libgmock-dev binutils libc6-dbg llvm
+  # No before_script override: the toolchain lives in the build image, so the
+  # pipeline default (CANCELLED guard, maven proxy) applies here like anywhere
+  # else.
   script:
     - ./gradlew :ddprof-lib:buildGtest${SANITIZER_CONFIG} --no-daemon --parallel --build-cache
     - |

.gitlab/scripts/build.sh

@@ -60,3 +60,17 @@ set -x
 
 mkdir -p "${REPO_ROOT}/libs"
 cp -r "${REPO_ROOT}/ddprof-lib/build/native/release/META-INF/native-libs/"* "${REPO_ROOT}/libs/"
+
+# Assert what the artifact requires from the host that will load it. musl
+# targets carry no glibc symbol versions, so the check does not apply to them.
+case "${TARGET}" in
+  *-musl)
+    echo "Skipping ABI floor check for ${TARGET} (musl)"
+    ;;
+  *)
+    source "${REPO_ROOT}/.gitlab/config.env"
+    "${HERE}/check-abi-floor.sh" \
+      "${REPO_ROOT}/libs/${TARGET}/libjavaProfiler.so" \
+      "${SHIPPED_GLIBC_FLOOR}"
+    ;;
+esac