From 1b826d598904e3272d8f6f3a74e5a70126d07c75 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Mon, 21 Sep 2026 15:15:41 +0200 Subject: [PATCH 01/15] fix(ci): make the quarantine machinery run under Python 3.6 Follow-up to #777, for wiring the GitLab EL7 functional job into quarantine: EL7's base-repo python3 is 3.6.8 (no SCL/newer-Python repo exists for Oracle Linux 7), and running the real scripts inside that image crashed twice: add_subparsers(dest="command", required=True) TypeError: __init__() got an unexpected keyword argument 'required' datetime.date.fromisoformat(review_by) AttributeError: type object 'datetime.date' has no attribute 'fromisoformat' Both need Python 3.7+. Drops required=True (checking args.command is None manually afterward) and replaces fromisoformat with strptime(s, "%Y-%m-%d").date() everywhere it was called. Verified end-to-end inside the real Oracle Linux 7 image: a quarantined failure on its own cell now goes green under python3 3.6.8, and the existing 59 test_quarantine.sh assertions (which run under whatever python3 is on the host, typically 3.9+) still pass unchanged. Also adds "el7" to KNOWN_LIBCS: the GitLab functional job's Oracle Linux 7 runtime is technically glibc but must not share the "glibc" cell token with the GitHub Actions Ubuntu matrix -- confirmed by a second smoke test that a quarantine entry scoped to el7-8-release-amd64 does not excuse the identical failure on glibc-8-release-amd64. Co-Authored-By: Claude Sonnet 5 --- .github/scripts/flake_report.py | 6 +++++- .github/scripts/quarantine.py | 30 +++++++++++++++++++++++++----- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/.github/scripts/flake_report.py b/.github/scripts/flake_report.py index 67a6cc8ade..b5d23f4b27 100755 --- a/.github/scripts/flake_report.py +++ b/.github/scripts/flake_report.py @@ -381,7 +381,9 @@ def cmd_report(args): def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--list", default=quarantine.DEFAULT_LIST) - sub = parser.add_subparsers(dest="command", required=True) + # required=True on add_subparsers() needs Python 3.7+; this also has to + # run under Python 3.6 (EL7's base-repo python3), so the check is manual. + sub = parser.add_subparsers(dest="command") count = sub.add_parser("count", help="print the number of distinct failed tests") count.add_argument("--dir", required=True) @@ -408,6 +410,8 @@ def main(): report.set_defaults(func=cmd_report) args = parser.parse_args() + if args.command is None: + parser.error("a command is required") return args.func(args) diff --git a/.github/scripts/quarantine.py b/.github/scripts/quarantine.py index fb048073f1..5f14a28bb0 100755 --- a/.github/scripts/quarantine.py +++ b/.github/scripts/quarantine.py @@ -27,6 +27,15 @@ TICKET_RE = re.compile(r"^PROF-\d+$") DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") FIELDS = ("test", "ticket", "added", "review_by", "cells", "reason") + + +def _parse_date(s): + """A YYYY-MM-DD string as a date, or raises ValueError. + + date.fromisoformat() needs Python 3.7+; this also has to run under + Python 3.6 (EL7's base-repo python3). + """ + return datetime.datetime.strptime(s, "%Y-%m-%d").date() # Long enough not to be busywork, short enough that a quarantine outlives # neither the release it was added in nor the memory of why. DEFAULT_REVIEW_DAYS = 90 @@ -44,8 +53,15 @@ # Cell names are ---. Only libc and arch are a closed # set -- jdk and config come from the workflow inputs and grow without warning # -- so those two are the only axes worth checking a glob against. +# +# "el7" names the GitLab functional job's Oracle Linux 7 runtime, which is +# technically glibc but must not share the "glibc" token: that job runs a +# different environment (container, kernel, network stack) than the GitHub +# Actions "glibc" matrix (Ubuntu), and the two can fail the same test for +# unrelated reasons. Sharing a cell string would let an entry meant to excuse +# one silently excuse the other. KNOWN_ARCHES = ("amd64", "aarch64") -KNOWN_LIBCS = ("glibc", "musl") +KNOWN_LIBCS = ("glibc", "musl", "el7") # Anything that reads like an architecture. A glob naming one that CI never # builds silently quarantines nothing, which is how "*arm64*" shipped in this # file's own example: the arch is spelled aarch64. @@ -199,7 +215,7 @@ def is_expired(entry, today=None): # entry that can be. An expiry that cannot be read has passed. return True try: - due = datetime.date.fromisoformat(review_by) + due = _parse_date(review_by) except ValueError: return True return due < (today or datetime.date.today()) @@ -311,7 +327,7 @@ def complain(line, message): if entry["added"] and DATE_RE.match(entry["added"]): try: - datetime.date.fromisoformat(entry["added"]) + _parse_date(entry["added"]) except ValueError: complain(line, "added '{}' is not a real calendar date".format(entry["added"])) @@ -338,7 +354,7 @@ def complain(line, message): if DATE_RE.match(entry["review_by"]): try: - due = datetime.date.fromisoformat(entry["review_by"]) + due = _parse_date(entry["review_by"]) except ValueError: complain(line, "review_by '{}' is not a real calendar date".format( entry["review_by"])) @@ -370,12 +386,16 @@ def complain(line, message): def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--list", default=DEFAULT_LIST) - sub = parser.add_subparsers(dest="command", required=True) + # required=True on add_subparsers() needs Python 3.7+; this also has to run + # under Python 3.6 (EL7's base-repo python3), so the check is manual. + sub = parser.add_subparsers(dest="command") validate = sub.add_parser("validate", help="check the list's format and review dates") validate.set_defaults(func=cmd_validate) args = parser.parse_args() + if args.command is None: + parser.error("a command is required") return args.func(args) From 4fb6b46d8615916c40e8b933424460ce33eaaf34 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Mon, 21 Sep 2026 15:28:39 +0200 Subject: [PATCH 02/15] ci: add python3 to the EL7 image for the quarantine classifier functional-tests.sh is about to wrap its gradlew invocation in run_tests_with_retry.sh, which shells out to quarantine.py and flake_report.py. Co-Authored-By: Claude Sonnet 5 --- .gitlab/base/el7/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab/base/el7/Dockerfile b/.gitlab/base/el7/Dockerfile index 5c6f261199..0dafec4fa9 100644 --- a/.gitlab/base/el7/Dockerfile +++ b/.gitlab/base/el7/Dockerfile @@ -23,7 +23,11 @@ ARG BASE_IMAGE=oraclelinux:7 FROM ${BASE_IMAGE} as base WORKDIR /root -RUN yum -y install git make zip unzip which wget curl jq gcc-c++ \ +# python3 (3.6.8 on this base repo -- no newer python3/SCL repo exists for +# Oracle Linux 7) runs the quarantine classifier (.github/scripts/ +# run_tests_with_retry.sh, quarantine.py, flake_report.py) that +# functional-tests.sh wraps its test run in. +RUN yum -y install git make zip unzip which wget curl jq gcc-c++ python3 \ && yum -y clean all RUN curl -s "https://get.sdkman.io" | bash && \ From e4a3bf43e52ec09938407b9fe7d741e0253c3820 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Mon, 21 Sep 2026 15:29:03 +0200 Subject: [PATCH 03/15] ci: wire functional-tests.sh into the quarantine machinery functional:x64-el7-jdk* previously just ran gradlew directly and let its raw exit code decide the job -- ddprof-test/quarantine.txt had no effect on it at all, unlike the GH Actions matrix (#777). Wraps the gradlew invocation in run_tests_with_retry.sh with cell "el7---amd64". MAX_ATTEMPTS defaults to 1 (no retry) here specifically: the default of 2 means a full second suite run, and this runner has already OOMKilled the container outright once at the current heap/concurrency (PR #805). Losing the flaky-vs-broken distinction is an acceptable trade for not doubling memory pressure on a runner already this tight; revisit once there's more headroom. Extends .functional_job's artifact paths with flake-evidence/, ci-outcome/ and build/logs/attempt.log so a quarantine decision on this job has evidence to show for it. Verified end-to-end inside the real Oracle Linux 7 image with a stub gradlew: a quarantined failure on its own cell now exits 0 through the full functional-tests.sh-shaped invocation. Co-Authored-By: Claude Sonnet 5 --- .gitlab/build-deploy/.gitlab-ci.yml | 7 +++++++ .gitlab/scripts/functional-tests.sh | 15 ++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/.gitlab/build-deploy/.gitlab-ci.yml b/.gitlab/build-deploy/.gitlab-ci.yml index 6a6455ae02..6cad74a129 100644 --- a/.gitlab/build-deploy/.gitlab-ci.yml +++ b/.gitlab/build-deploy/.gitlab-ci.yml @@ -228,6 +228,13 @@ stresstest:arm64-musl: paths: - functional/$TARGET/jdk$TEST_JDK/reports - functional/$TARGET/jdk$TEST_JDK/logs + # run_tests_with_retry.sh's own evidence: each attempt's JUnit XML + # snapshot, the classifier's gating decision, and the final attempt's + # raw log -- without these, a quarantine decision on this job has + # nothing to show for it besides the console log. + - flake-evidence + - ci-outcome + - build/logs/attempt.log expire_in: 2 weeks # Runs on every pipeline (not just nightly): JDK 8 is the oldest/most diff --git a/.gitlab/scripts/functional-tests.sh b/.gitlab/scripts/functional-tests.sh index 3ad233a191..062d864562 100755 --- a/.gitlab/scripts/functional-tests.sh +++ b/.gitlab/scripts/functional-tests.sh @@ -68,6 +68,19 @@ function onexit { trap onexit EXIT -./gradlew -Pddprof_version="$(get_version)" -Pskip-native=ddprof-lib,malloc-shim -Pwith-libs="$(pwd)/libs" -PCI \ +# "el7", not "glibc": this runtime (container, kernel, network stack) is a +# different environment than the GitHub Actions glibc/Ubuntu matrix and can +# fail the same test for unrelated reasons, so it needs its own cell token +# (quarantine.py's KNOWN_LIBCS) -- sharing "glibc" would let an entry meant to +# excuse one silently excuse the other. +CELL="el7-${TEST_JDK}-$(printf '%s' "${TEST_CONFIG}" | tr '[:upper:]' '[:lower:]')-amd64" + +# Default to no retry: a flaky-vs-broken retry means a full second suite run, +# and this runner has already OOMKilled the container outright at the default +# 2 attempts (see PR #805). Overridable via the job's own variables. +export MAX_ATTEMPTS="${MAX_ATTEMPTS:-1}" + +.github/scripts/run_tests_with_retry.sh "${CELL}" -- \ + ./gradlew -Pddprof_version="$(get_version)" -Pskip-native=ddprof-lib,malloc-shim -Pwith-libs="$(pwd)/libs" -PCI \ -PtestMaxHeap=1536m \ ":ddprof-test:test${TEST_CONFIG}" --max-workers=1 --build-cache --stacktrace --info --no-watch-fs --no-daemon From 8052f5838c54ae85ff50058310fc49d82830a04a Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Mon, 21 Sep 2026 15:29:44 +0200 Subject: [PATCH 04/15] ci: quarantine the NativeSocket* flake on el7-8-release-amd64 PROF-16014. Root-caused in PR #805: NativeSocketSampler's hooks install and fire correctly (confirmed via a debug-config diagnostic build), but the rate limiter rejects nearly every sample because NativeSocketTestBase.doTcpTransfer()'s assumption that the send buffer fills and blocks after ~32 iterations does not hold on this runner's network stack. Scoped to el7-8-release-amd64, the only cell this has actually been observed on -- not el7-*-release-amd64, so the nightly el7-17/-21 cells still gate for real if this doesn't reproduce there the same way. Co-Authored-By: Claude Sonnet 5 --- ddprof-test/quarantine.txt | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/ddprof-test/quarantine.txt b/ddprof-test/quarantine.txt index 7fbd2121da..f11739e2de 100644 --- a/ddprof-test/quarantine.txt +++ b/ddprof-test/quarantine.txt @@ -42,5 +42,21 @@ # When CI sees a flake it prints a ready-made line in the PR comment. The # ticket and the judgement are still yours. # -# Example (delete when the first real entry lands): -# com.datadoghq.profiler.cpu.CpuSamplingTest.testSampling | PROF-12345 | 2026-09-02 | 2026-12-02 | *aarch64* | Under-samples on emulated aarch64; 2 of 40 runs +# NativeSocketSampler's hooks install and fire correctly on this cell +# (confirmed via a debug-config diagnostic build -- send/recv/write/read all +# called, PLT patching succeeds); the rate limiter then rejects nearly every +# sample because doTcpTransfer()'s "the send buffer fills and blocks after +# ~32 iterations" assumption does not hold on this runner's network stack, so +# calls never get slow enough for the time-weighted sampler to keep them. +# Scoped to el7-8-release-amd64 (the only cell this has been observed on) -- +# widen only with fresh evidence from the nightly el7-17/-21 cells. +com.datadoghq.profiler.nativesocket.NativeSocketBytesAccuracyTest.timeWeightedEstimateIsWithinReasonableBounds | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner +com.datadoghq.profiler.nativesocket.NativeSocketEnabledTest.socketEventsProducedWhenFeatureEnabled | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner +com.datadoghq.profiler.nativesocket.NativeSocketEventFieldsTest.allRequiredFieldsPresentAndValid | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner +com.datadoghq.profiler.nativesocket.NativeSocketEventThreadTest.eventThreadIsPopulated | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner +com.datadoghq.profiler.nativesocket.NativeSocketRateLimitTest.eventCountIsSubstantiallyLessThanOperationCount | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner +com.datadoghq.profiler.nativesocket.NativeSocketRemoteAddressTest.remoteAddressIsIpColonPort | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner +com.datadoghq.profiler.nativesocket.NativeSocketRestartTest.testNativeSocketProfilerRestart | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner +com.datadoghq.profiler.nativesocket.NativeSocketSendRecvSeparateTest.sendAndRecvTrackedWithSeparateCounts | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner +com.datadoghq.profiler.nativesocket.NativeSocketUdpExcludedTest.udpTransfersProduceNoSocketEvents | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner +com.datadoghq.profiler.nativesocket.NativeSocketStackTraceTest.* | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | @TestTemplate invocations (cstack=vm/vmx/dwarf/fp) have no method name in the XML; same root cause as the rest of the package From ea659ce01e086c1a5135a17f57633aa9a912f81a Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Tue, 22 Sep 2026 12:40:32 +0200 Subject: [PATCH 05/15] docs: keep the quarantine.txt example even with real entries present The example line was written to be deleted once the first real entry landed, but it's worth keeping as a permanent format reference -- the NativeSocket entries are tied to a bug that will get fixed and removed, and the file shouldn't go back to having no example once that happens. Co-Authored-By: Claude Sonnet 5 --- ddprof-test/quarantine.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ddprof-test/quarantine.txt b/ddprof-test/quarantine.txt index f11739e2de..4d0f9c9ab4 100644 --- a/ddprof-test/quarantine.txt +++ b/ddprof-test/quarantine.txt @@ -42,6 +42,9 @@ # When CI sees a flake it prints a ready-made line in the PR comment. The # ticket and the judgement are still yours. # +# Example: +# com.datadoghq.profiler.cpu.CpuSamplingTest.testSampling | PROF-12345 | 2026-09-02 | 2026-12-02 | *aarch64* | Under-samples on emulated aarch64; 2 of 40 runs +# # NativeSocketSampler's hooks install and fire correctly on this cell # (confirmed via a debug-config diagnostic build -- send/recv/write/read all # called, PLT patching succeeds); the rate limiter then rejects nearly every From d55f44901891e087ec01bafae92d1af5029e0292 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Tue, 22 Sep 2026 16:11:26 +0200 Subject: [PATCH 06/15] style: move _parse_date next to its first use Inserting it in the module-level constants block left it wedged between two densely-packed constant groups with no blank-line separation, breaking the file's own two-blank-lines-around-defs convention. Moves it next to is_expired(), its first caller, matching how the file already places its other small helpers near their use. Co-Authored-By: Claude Sonnet 5 --- .github/scripts/quarantine.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/scripts/quarantine.py b/.github/scripts/quarantine.py index 5f14a28bb0..d0e14d8639 100755 --- a/.github/scripts/quarantine.py +++ b/.github/scripts/quarantine.py @@ -28,14 +28,6 @@ DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") FIELDS = ("test", "ticket", "added", "review_by", "cells", "reason") - -def _parse_date(s): - """A YYYY-MM-DD string as a date, or raises ValueError. - - date.fromisoformat() needs Python 3.7+; this also has to run under - Python 3.6 (EL7's base-repo python3). - """ - return datetime.datetime.strptime(s, "%Y-%m-%d").date() # Long enough not to be busywork, short enough that a quarantine outlives # neither the release it was added in nor the memory of why. DEFAULT_REVIEW_DAYS = 90 @@ -198,6 +190,15 @@ def covers(entry, test_id): return test_id == pattern +def _parse_date(s): + """A YYYY-MM-DD string as a date, or raises ValueError. + + date.fromisoformat() needs Python 3.7+; this also has to run under + Python 3.6 (EL7's base-repo python3). + """ + return datetime.datetime.strptime(s, "%Y-%m-%d").date() + + def is_expired(entry, today=None): """True once this entry's review_by date has passed. From 5a1531d237b86045224e0c8a5eab601689bf14d5 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Tue, 22 Sep 2026 22:04:11 +0200 Subject: [PATCH 07/15] fix: quarantine NativeSocket* by class, not by exact method Flagged in review, and confirmed against both the test source and quarantine.txt's own documented format: every affected class's single test method is @RetryingTest, which -- like @ParameterizedTest and @TestTemplate -- reports in JUnit XML as "Class.[1]", "[2]", ... with no method name at all. The exact-method entries could never match anything; only the already-class-wide NativeSocketStackTraceTest.* entry ever worked. Converts the other 8 to .*. Also drops the NativeSocketUdpExcludedTest entry entirely rather than widening it: that test asserts *zero* events, which the bug's own symptom (near-zero events everywhere) would make pass, not fail. Its earlier observed failure has an unconfirmed, likely different cause, and quarantining it on this evidence risks hiding an unrelated regression later. Verified: find_entry() now matches the real "NativeSocketEnabledTest.[1]"-shaped id against the class-wide entry, and confirms the UDP test is not quarantined. All 59 test_quarantine.sh assertions and quarantine.py validate still pass. Co-Authored-By: Claude Sonnet 5 --- ddprof-test/quarantine.txt | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/ddprof-test/quarantine.txt b/ddprof-test/quarantine.txt index 4d0f9c9ab4..6944b1e105 100644 --- a/ddprof-test/quarantine.txt +++ b/ddprof-test/quarantine.txt @@ -53,13 +53,21 @@ # calls never get slow enough for the time-weighted sampler to keep them. # Scoped to el7-8-release-amd64 (the only cell this has been observed on) -- # widen only with fresh evidence from the nightly el7-17/-21 cells. -com.datadoghq.profiler.nativesocket.NativeSocketBytesAccuracyTest.timeWeightedEstimateIsWithinReasonableBounds | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner -com.datadoghq.profiler.nativesocket.NativeSocketEnabledTest.socketEventsProducedWhenFeatureEnabled | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner -com.datadoghq.profiler.nativesocket.NativeSocketEventFieldsTest.allRequiredFieldsPresentAndValid | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner -com.datadoghq.profiler.nativesocket.NativeSocketEventThreadTest.eventThreadIsPopulated | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner -com.datadoghq.profiler.nativesocket.NativeSocketRateLimitTest.eventCountIsSubstantiallyLessThanOperationCount | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner -com.datadoghq.profiler.nativesocket.NativeSocketRemoteAddressTest.remoteAddressIsIpColonPort | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner -com.datadoghq.profiler.nativesocket.NativeSocketRestartTest.testNativeSocketProfilerRestart | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner -com.datadoghq.profiler.nativesocket.NativeSocketSendRecvSeparateTest.sendAndRecvTrackedWithSeparateCounts | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner -com.datadoghq.profiler.nativesocket.NativeSocketUdpExcludedTest.udpTransfersProduceNoSocketEvents | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner +# +# Every one of these classes' single test method is @RetryingTest, which (like +# @ParameterizedTest/@TestTemplate) reports as "Class.[1]", "[2]", ... in the +# XML -- no method name at all -- so the entry has to be class-wide; an exact +# method name here would never match anything. NativeSocketUdpExcludedTest is +# deliberately not listed: it asserts *zero* events, which the bug's symptom +# (near-zero events everywhere) would make pass, not fail, so a failure there +# has a different, unconfirmed cause and quarantining it on this evidence +# risks hiding an unrelated regression later. +com.datadoghq.profiler.nativesocket.NativeSocketBytesAccuracyTest.* | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner +com.datadoghq.profiler.nativesocket.NativeSocketEnabledTest.* | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner +com.datadoghq.profiler.nativesocket.NativeSocketEventFieldsTest.* | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner +com.datadoghq.profiler.nativesocket.NativeSocketEventThreadTest.* | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner +com.datadoghq.profiler.nativesocket.NativeSocketRateLimitTest.* | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner +com.datadoghq.profiler.nativesocket.NativeSocketRemoteAddressTest.* | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner +com.datadoghq.profiler.nativesocket.NativeSocketRestartTest.* | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner +com.datadoghq.profiler.nativesocket.NativeSocketSendRecvSeparateTest.* | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | NativeSocketEvent count 0: doTcpTransfer() never blocks long enough to be sampled on this runner com.datadoghq.profiler.nativesocket.NativeSocketStackTraceTest.* | PROF-16014 | 2026-09-21 | 2026-12-20 | el7-8-release-amd64 | @TestTemplate invocations (cstack=vm/vmx/dwarf/fp) have no method name in the XML; same root cause as the rest of the package From 50008c32c8a3607aecadb3c85eade2b6895d774a Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Tue, 22 Sep 2026 22:31:16 +0200 Subject: [PATCH 08/15] ci: repin BUILD_IMAGE_X64_EL7 to the python3-enabled image Built and pushed successfully in pipeline 139300629 (the job's overall failure was two unrelated images, arm64-musl and benchmarks-amd64). Confirmed python3 3.6.8 installed in the build log. Co-Authored-By: Claude Sonnet 5 --- .gitlab/build-deploy/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/build-deploy/.gitlab-ci.yml b/.gitlab/build-deploy/.gitlab-ci.yml index 6cad74a129..5727a68990 100644 --- a/.gitlab/build-deploy/.gitlab-ci.yml +++ b/.gitlab/build-deploy/.gitlab-ci.yml @@ -9,8 +9,8 @@ variables: BUILD_IMAGE_X64_GLIBC: registry.ddbuild.io/ci/async-profiler-build:v137281788-x64-glibc-base@sha256:270a9101716a0865071182e0ef0e40e98f874535b90722949b33a4a8998fe08d # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/2040944172 BUILD_IMAGE_ARM64_GLIBC: registry.ddbuild.io/ci/async-profiler-build:v137281788-arm64-glibc-base@sha256:511d051d37ed99b2ca9cd3af797aa9de63e4b4b952af1e738b2cea913bbcc057 - # Generated by pipeline 138438605 - BUILD_IMAGE_X64_EL7: registry.ddbuild.io/ci/async-profiler-build:v138438605-x64-el7-base@sha256:2f4d2852979d7b190dcb997bbc833eb795f2f0db9905300ec239516480e8599d + # Generated by pipeline 139300629 + BUILD_IMAGE_X64_EL7: registry.ddbuild.io/ci/async-profiler-build:v139300629-x64-el7-base@sha256:b3a77ea5db8bd245a3d455e38b7a0e9579f35c5c8099cc0e39fc8de29fb07b47 # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/2040944172 BUILD_IMAGE_X64_MUSL: registry.ddbuild.io/ci/async-profiler-build:v137281788-x64-musl-base@sha256:fd3803695771de3eabf44e4953c0474273f9dbb97d88c22e99ac507b88572481 # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/2040944172 From 447d40d16b015e14ab0d3abe78d53d63f195740f Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Wed, 23 Sep 2026 12:35:09 +0200 Subject: [PATCH 09/15] ci: raise container memory limit for EL7 functional job The x64-el7 functional job was getting OOMKilled (exit 137) at the container level during pod cleanup, after the test suite had already finished and uploaded its artifacts. The JVM under test already has its own -Xmx1536m cap; the pod itself needed more headroom to run Gradle's daemon/workers, the gtest builds, and the JVM together. Co-Authored-By: Claude Sonnet 5 --- .gitlab/build-deploy/.gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab/build-deploy/.gitlab-ci.yml b/.gitlab/build-deploy/.gitlab-ci.yml index 5727a68990..26af68a6c0 100644 --- a/.gitlab/build-deploy/.gitlab-ci.yml +++ b/.gitlab/build-deploy/.gitlab-ci.yml @@ -198,6 +198,12 @@ stresstest:arm64-musl: image: ${BUILD_IMAGE_X64_EL7} variables: TARGET: linux-x64 + # The shared-runner default container memory limit isn't enough for this + # job: Gradle's own daemon/workers, gtest builds, and the JVM under test + # (testMaxHeap=1536m) all coexist in the same pod, and the container was + # getting OOMKilled (exit 137) after the test run had already finished. + KUBERNETES_MEMORY_REQUEST: 4Gi + KUBERNETES_MEMORY_LIMIT: 6Gi # Two independent cache entries rather than extending .cache-config-pull: # the JDK-under-test tarball (functional-tests.sh's JDK_CACHE_DIR) has # nothing to do with the Gradle/Maven cache's key or lifetime. It doesn't From 6f6db9ae865ab3aa71f217df72c0b2170c4856b1 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Wed, 23 Sep 2026 15:19:53 +0200 Subject: [PATCH 10/15] build: compile build-logic in-process instead of a Kotlin daemon The Kotlin compile daemon outlives the build-logic compile and stays resident for the rest of the build, including the whole test run, next to the Gradle process and the test JVMs. Measured on a --no-daemon build: ~580MB Gradle + ~660MB Kotlin daemon with the default strategy, versus ~920MB Gradle and no daemon in-process. The out-of-process strategy was removed in Kotlin 2.4.0. Co-Authored-By: Claude Opus 5.5 (1M context) --- build-logic/gradle.properties | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 build-logic/gradle.properties diff --git a/build-logic/gradle.properties b/build-logic/gradle.properties new file mode 100644 index 0000000000..3517480107 --- /dev/null +++ b/build-logic/gradle.properties @@ -0,0 +1,10 @@ +# Copyright 2026, Datadog, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# Compile build-logic inside the Gradle process instead of a separate Kotlin +# daemon. The daemon outlives the compile (it idles for 2h) and stays resident +# for the rest of the build -- through the whole test run -- next to the Gradle +# process and the test JVMs; measured on this build it costs ~660MB RSS against +# ~340MB of extra Gradle RSS for in-process. That matters on memory-limited CI +# runners. build-logic rarely changes, so a warm daemon buys little locally. +kotlin.compiler.execution.strategy=in-process From 39111711385a70590f8f7c9e49cf855643f681a5 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Wed, 23 Sep 2026 15:57:06 +0200 Subject: [PATCH 11/15] Revert "build: compile build-logic in-process instead of a Kotlin daemon" This reverts commit 7864e74748acc5ae91831aff33f13ccd55bec10c. --- build-logic/gradle.properties | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 build-logic/gradle.properties diff --git a/build-logic/gradle.properties b/build-logic/gradle.properties deleted file mode 100644 index 3517480107..0000000000 --- a/build-logic/gradle.properties +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright 2026, Datadog, Inc. -# SPDX-License-Identifier: Apache-2.0 - -# Compile build-logic inside the Gradle process instead of a separate Kotlin -# daemon. The daemon outlives the compile (it idles for 2h) and stays resident -# for the rest of the build -- through the whole test run -- next to the Gradle -# process and the test JVMs; measured on this build it costs ~660MB RSS against -# ~340MB of extra Gradle RSS for in-process. That matters on memory-limited CI -# runners. build-logic rarely changes, so a warm daemon buys little locally. -kotlin.compiler.execution.strategy=in-process From 390e03a442a37f18f266519f541d6213c49244cc Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Wed, 23 Sep 2026 15:57:37 +0200 Subject: [PATCH 12/15] ci: compile build-logic in-process only in the EL7 functional job The Kotlin compile daemon (~660MB RSS) stays resident through the whole test run in this 6GB pod. Setting in-process globally via build-logic/gradle.properties broke the CodeQL Java job: its Kotlin extractor runs inside the compiler, and in-process that exhausted Gradle's default 512m heap. Scope the setting to this job instead. Co-Authored-By: Claude Opus 5.5 (1M context) --- .gitlab/scripts/functional-tests.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab/scripts/functional-tests.sh b/.gitlab/scripts/functional-tests.sh index 062d864562..9f04779971 100755 --- a/.gitlab/scripts/functional-tests.sh +++ b/.gitlab/scripts/functional-tests.sh @@ -80,7 +80,11 @@ CELL="el7-${TEST_JDK}-$(printf '%s' "${TEST_CONFIG}" | tr '[:upper:]' '[:lower:] # 2 attempts (see PR #805). Overridable via the job's own variables. export MAX_ATTEMPTS="${MAX_ATTEMPTS:-1}" +# Compile build-logic inside the Gradle process: the default Kotlin compile +# daemon (~660MB RSS) otherwise stays resident through the whole test run in +# this memory-limited pod. Not set globally -- under CodeQL's Kotlin extractor +# an in-process compile thrashes Gradle's default 512m heap. .github/scripts/run_tests_with_retry.sh "${CELL}" -- \ ./gradlew -Pddprof_version="$(get_version)" -Pskip-native=ddprof-lib,malloc-shim -Pwith-libs="$(pwd)/libs" -PCI \ - -PtestMaxHeap=1536m \ + -PtestMaxHeap=1536m -Pkotlin.compiler.execution.strategy=in-process \ ":ddprof-test:test${TEST_CONFIG}" --max-workers=1 --build-cache --stacktrace --info --no-watch-fs --no-daemon From 92c4a01f2c12566afc4c19cec746dc7808c1d507 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Wed, 23 Sep 2026 16:24:36 +0200 Subject: [PATCH 13/15] Revert "ci: raise container memory limit for EL7 functional job" This reverts commit a794f69fdc0c9ca6a4fc3c964dfb29429013c034. --- .gitlab/build-deploy/.gitlab-ci.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.gitlab/build-deploy/.gitlab-ci.yml b/.gitlab/build-deploy/.gitlab-ci.yml index 26af68a6c0..5727a68990 100644 --- a/.gitlab/build-deploy/.gitlab-ci.yml +++ b/.gitlab/build-deploy/.gitlab-ci.yml @@ -198,12 +198,6 @@ stresstest:arm64-musl: image: ${BUILD_IMAGE_X64_EL7} variables: TARGET: linux-x64 - # The shared-runner default container memory limit isn't enough for this - # job: Gradle's own daemon/workers, gtest builds, and the JVM under test - # (testMaxHeap=1536m) all coexist in the same pod, and the container was - # getting OOMKilled (exit 137) after the test run had already finished. - KUBERNETES_MEMORY_REQUEST: 4Gi - KUBERNETES_MEMORY_LIMIT: 6Gi # Two independent cache entries rather than extending .cache-config-pull: # the JDK-under-test tarball (functional-tests.sh's JDK_CACHE_DIR) has # nothing to do with the Gradle/Maven cache's key or lifetime. It doesn't From a19c786ed0ec63b51492d0e6442a10a0eda50339 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Thu, 24 Sep 2026 11:33:10 +0200 Subject: [PATCH 14/15] fix: gate a quarantined failure when the final attempt was killed by a signal A signal that kills the Gradle JVM itself (e.g. the OOM killer taking the --no-daemon build process) leaves nothing alive to print a cut-short marker, and with MAX_ATTEMPTS=1 there is no earlier attempt to measure a shortfall against. If a quarantined test had already written its failure XML, the report excused the run and the wrapper turned exit 137 into 0. Treat an exit status above 128 as the attempt being cut short. Gradle never exits above 128 on its own, however many tests fail. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/scripts/flake_report.py | 30 ++++++++++++++++++------ .github/scripts/tests/test_quarantine.sh | 23 ++++++++++++++++++ 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/.github/scripts/flake_report.py b/.github/scripts/flake_report.py index b5d23f4b27..a28594592f 100755 --- a/.github/scripts/flake_report.py +++ b/.github/scripts/flake_report.py @@ -224,16 +224,23 @@ def cmd_report(args): other_task_failures = non_test_task_failures(args.attempt_log, args.test_task_pattern) - # Two independent signs that the final attempt stopped early rather than - # running to completion and failing tests: Gradle saying so in the log, and - # the attempt having reached fewer tests than another attempt managed. The - # latter needs more than one attempt to compare against, which slow suites - # (MAX_ATTEMPTS=1) do not have, so the log is the primary signal. + # Three independent signs that the final attempt stopped early rather than + # running to completion and failing tests: Gradle saying so in the log, the + # command dying to a signal, and the attempt having reached fewer tests than + # another attempt managed. The last needs more than one attempt to compare + # against, which slow suites (MAX_ATTEMPTS=1) do not have. A signal that + # kills Gradle itself leaves nothing alive to log a marker, so the exit + # status is the only evidence in that case. # Cell names are ---[-slow] (quarantine.KNOWN_LIBCS); # the leading token is the only part cut_short_marker needs. final_attempt_cut_short = cut_short_marker( args.attempt_log, musl=args.cell.split("-", 1)[0] == "musl" ) + # Shells report death by signal N as 128+N; Gradle never exits above 128 on + # its own, however many tests fail. + final_attempt_signal = None + if args.final_attempt_exit_code is not None and args.final_attempt_exit_code > 128: + final_attempt_signal = args.final_attempt_exit_code - 128 observed_shortfall = None if final_attempt_ran and ran > 1: best_observed = max(len(seen) for _, seen, _ in attempts) @@ -264,8 +271,9 @@ def cmd_report(args): # must not gate. # final attempt exited -> gate only with evidence that it was cut # non-zero and was cut short short: Gradle reporting a dead test JVM, - # or the attempt having reached fewer - # tests than another attempt managed. A + # the command dying to a signal, or the + # attempt having reached fewer tests + # than another attempt managed. A # quarantined test that fails makes Gradle # exit non-zero all by itself, so treating # every non-zero exit as a crash would @@ -312,6 +320,13 @@ def cmd_report(args): "short ({!r}), so the tests missing from its results cannot be " "read as quarantined" ).format(args.final_attempt_exit_code, final_attempt_cut_short) + elif final_attempt_signal is not None: + gates = True + gate_reason = ( + "the final attempt exited {} (killed by signal {}), so it did not " + "run to completion and the tests missing from its results cannot " + "be read as quarantined" + ).format(args.final_attempt_exit_code, final_attempt_signal) elif args.final_attempt_exit_code not in (None, 0) and observed_shortfall: gates = True gate_reason = ( @@ -351,6 +366,7 @@ def cmd_report(args): "final_attempt_failure_count": final_attempt_failure_count, "other_task_failures": other_task_failures, "final_attempt_cut_short": final_attempt_cut_short, + "final_attempt_signal": final_attempt_signal, "final_attempt_observed_shortfall": observed_shortfall, "final_attempt_exit_code": args.final_attempt_exit_code, "gates": gates, diff --git a/.github/scripts/tests/test_quarantine.sh b/.github/scripts/tests/test_quarantine.sh index 078e5bc238..c2406138a2 100755 --- a/.github/scripts/tests/test_quarantine.sh +++ b/.github/scripts/tests/test_quarantine.sh @@ -675,6 +675,29 @@ assert d['final_attempt_cut_short'], d " "$CASE/out.json" || fail "a dead test JVM was excused by the quarantine list" pass "a cut-short final attempt gates despite its failures being quarantined" +# A signal that kills Gradle itself (e.g. the OOM killer taking the --no-daemon +# build JVM) leaves nothing alive to print a marker, and with a single attempt +# there is no earlier run to measure a shortfall against. The exit status is +# then the only evidence the suite never finished. +CASE="$TEMP_DIR/case-quarantined-killed-by-signal" +mkdir -p "$CASE/flake-evidence/attempt-1" +write_failure_xml "$CASE/flake-evidence/attempt-1" "com.dd.WobblyTest" "sometimesFails" "boom" +cat > "$CASE/attempt.log" <<'EOS' +> Task :ddprof-test:testRelease +EOS +write_list "$CASE/list.txt" "$(entry com.dd.WobblyTest.sometimesFails PROF-1 "$(day_offset 30)")" +python3 "$SCRIPTS/flake_report.py" --list "$CASE/list.txt" report \ + --cell "el7-8-release-amd64" --evidence-dir "$CASE/flake-evidence" \ + --final-attempt 1 --attempt-log "$CASE/attempt.log" \ + --final-attempt-exit-code 137 --test-task-pattern test \ + --out "$CASE/out.json" >/dev/null 2>&1 +python3 -c " +import json,sys +d = json.load(open(sys.argv[1])) +assert d['gates'] is True, 'a final attempt killed by a signal must gate even with every named failure quarantined: %r' % d['gate_reason'] +" "$CASE/out.json" || fail "a final attempt killed by SIGKILL was excused by the quarantine list" +pass "a final attempt killed by a signal gates despite its failures being quarantined" + # musl runs ProfilerTestRunner through a plain Exec task, not Gradle's native # Test task: Gradle prints the exact same "finished with non-zero exit value" # line whenever that process exits non-zero for *any* reason, including an From 090bafbdae27fb8577433a50ea198672ce2f87cf Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Thu, 24 Sep 2026 11:52:27 +0200 Subject: [PATCH 15/15] fix: honor skip-native=true as a global native skip gradle.properties.template documents skip-native=true, but isNativeSkipped() only treated an empty value as the global skip and read anything else as a list of project names. "true" matched no project, so native compilation ran anyway. Treat "true" like a bare -Pskip-native, and "false" as skipping nothing. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../com/datadoghq/native/util/PlatformUtils.kt | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/util/PlatformUtils.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/util/PlatformUtils.kt index f48055ac98..f6c4bbb44f 100644 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/util/PlatformUtils.kt +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/util/PlatformUtils.kt @@ -377,9 +377,11 @@ object PlatformUtils { /** * Whether native compilation should be skipped for [project]. * - * `-Pskip-native` (bare, no value) skips it everywhere — the long-standing - * behavior, used when consuming a prebuilt shipped library via -Pwith-libs - * and never touching a compiler. + * `-Pskip-native` (bare, no value) or `skip-native=true` (the form + * gradle.properties.template documents) skips it everywhere — the + * long-standing behavior, used when consuming a prebuilt shipped library + * via -Pwith-libs and never touching a compiler. `skip-native=false` skips + * nothing. * * `-Pskip-native=` skips it only for those * projects. This lets a caller substitute the prebuilt *shipped* library @@ -392,10 +394,13 @@ object PlatformUtils { if (!project.hasProperty("skip-native")) { return false } - val value = project.property("skip-native") as? String - if (value.isNullOrEmpty()) { + val value = (project.property("skip-native") as? String)?.trim() + if (value.isNullOrEmpty() || value.equals("true", ignoreCase = true)) { return true } + if (value.equals("false", ignoreCase = true)) { + return false + } return value.split(",").map { it.trim() }.contains(project.name) }