diff --git a/.claude/ci/tracer-unit-tests.md b/.claude/ci/tracer-unit-tests.md index 97a1a5974f..75ee86e125 100644 --- a/.claude/ci/tracer-unit-tests.md +++ b/.claude/ci/tracer-unit-tests.md @@ -8,7 +8,7 @@ - `.gitlab/compile_extension.sh` — compiles ddtrace.so (used by the `compile extension: debug` prerequisite) - `Makefile` — defines the `test_c`, `test_unit`, `test_opcache`, - `test_extension_ci`, etc. targets + `test_extension_ci_normal`, `test_extension_ci_valgrind`, etc. targets | CI Job | Image | What it does | |--------|-------|-------------| @@ -16,7 +16,8 @@ | `compile extension: debug-zts-asan` | same | Compiles ddtrace.so with ASAN+ZTS; used by ASAN test jobs | | `Unit tests: [{ver}]` | `dd-trace-ci:php-{ver}_bookworm-6` | Runs PHPUnit `--testsuite=unit` | | `API unit tests: [{ver}]` | same | Runs PHPUnit API unit tests | -| `test_extension_ci: [{ver}]` | same | Runs .phpt extension tests + valgrind wrapper, with test-agent | +| `test_extension_ci: [{ver}]` | same | Runs .phpt extension tests (normal pass), with test-agent | +| `test_extension_ci: [{ver}, valgrind]` | same | Same suite under valgrind for leak checking; much slower, so it is a separate job | | `PHP Language Tests: [{ver}]` | same | Runs the upstream PHP test suite with ddtrace loaded; uses an xfail list | | `Opcache tests: [{ver}]` | same | Runs .phpt tests in `tests/opcache/` with opcache.so loaded | | `xDebug tests: [{ver}, {xdebug_ver}]` | same | Runs xdebug-specific .phpt tests + unit tests with xdebug loaded | @@ -549,14 +550,22 @@ make test_opcache Each new `dockerh` invocation must re-run `make install_all` even when the compiled artifacts are cached. -- **`test_extension_ci` uses a valgrind wrapper.** The Makefile - prepends `tests/ext/valgrind` to `$PATH`, which intercepts `php` - calls to run them under valgrind. This makes the job significantly - slower and is specific to CI. - -- **`PHP Language Tests` has retry:2 in CI.** These tests are - inherently flaky due to timing-sensitive PHP runtime tests. The CI - job retries up to 2 times on script failure. +- **The valgrind pass is a separate job.** `make test_extension_ci_normal` + runs the suite normally; `make test_extension_ci_valgrind` re-runs it + under valgrind (`run-tests.php -m`) for leak checking. Both prepend + `tests/ext/valgrind` to `$PATH` — the shim there intercepts `valgrind` + invocations to add the suppressions file. The valgrind pass is roughly + an order of magnitude slower, which is why it no longer shares a job + with the normal pass. `make test_extension_ci` still runs both + serially for local use. + +- **Retries are infrastructure-only.** `test_extension_ci`, + `ASAN test_c`, and `PHP Language Tests` inherit `default.retry` from + `.gitlab/generate-common.php`, which retries runner/API/timeout + failures but *not* `script_failure`. A genuine test failure is not + retried — retrying it tripled compute without changing the outcome. + These job classes are listed in `flaky-jobs.txt`, so their failures + are already non-gating. - **`test_integration` talks to test-agent on port 9126** and mongodb. `test_composer`, `test_auto_instrumentation`, and diff --git a/.gitlab/generate-tracer.php b/.gitlab/generate-tracer.php index 05aea7c9b6..2d463293c7 100644 --- a/.gitlab/generate-tracer.php +++ b/.gitlab/generate-tracer.php @@ -237,7 +237,6 @@ function before_script_steps($with_docker_auth = false) { - PHP_MAJOR_MINOR: "" ARCH: "" artifacts: true - retry: 2 variables: WAIT_FOR: test-agent:9126 KUBERNETES_CPU_REQUEST: 6 @@ -355,6 +354,8 @@ function before_script_steps($with_docker_auth = false) { + "test_extension_ci: []": extends: .debug_test services: @@ -369,13 +370,46 @@ function before_script_steps($with_docker_auth = false) { variables: WAIT_FOR: test-agent:9126 KUBERNETES_CPU_REQUEST: 12 +=")): ?> + # Match the CPU request. + MAX_TEST_PARALLELISM: 12 + + PHP_MAJOR_MINOR: "" + ARCH: "amd64" + KUBERNETES_POD_ANNOTATIONS_1: "ci.ddbuild.io/enforce-static-cpus=true" +=")): ?> + timeout: 45m + + # run-tests.php only gets -j on PHP >= 7.4 (RUN_TESTS_IS_PARALLEL in the + # Makefile), so these versions run serially and need the larger budget. + timeout: 120m + + script: + - make test_extension_ci_normal + + +"test_extension_ci: [, valgrind]": + extends: .debug_test + services: + + needs: + - job: "compile extension: debug" + parallel: + matrix: + - PHP_MAJOR_MINOR: "" + ARCH: "amd64" + artifacts: true + variables: + WAIT_FOR: test-agent:9126 + KUBERNETES_CPU_REQUEST: 12 + # Below the CPU request: each worker spawns its own valgrind process. MAX_TEST_PARALLELISM: 4 PHP_MAJOR_MINOR: "" ARCH: "amd64" KUBERNETES_POD_ANNOTATIONS_1: "ci.ddbuild.io/enforce-static-cpus=true" timeout: 120m script: - - make test_extension_ci + - make test_extension_ci_valgrind "Unit tests: []": @@ -534,17 +568,6 @@ function before_script_steps($with_docker_auth = false) { DD_INSTRUMENTATION_TELEMETRY_ENABLED: 0 timeout: 40m - retry: - max: 2 - when: - - script_failure - - unknown_failure - - data_integrity_failure - - runner_system_failure - - scheduler_failure - - api_failure - - stuck_or_timeout_failure - - job_execution_timeout script: - make install_all - export XFAIL_LIST="dockerfiles/ci/xfail_tests/${PHP_MAJOR_MINOR}.list" diff --git a/Makefile b/Makefile index aa1ba9dad6..c26c6acba6 100644 --- a/Makefile +++ b/Makefile @@ -202,18 +202,32 @@ test_c2php: $(SO_FILE) $(INIT_HOOK_TEST_FILES) $(BUILD_DIR)/run-tests.php test_with_init_hook: $(SO_FILE) $(INIT_HOOK_TEST_FILES) $(BUILD_DIR)/run-tests.php $(if $(ASAN), USE_ZEND_ALLOC=0 USE_TRACKED_ALLOC=1) $(RUN_TESTS_CMD) -d extension=$(SO_FILE) $(TRACER_SOURCES_INI) $(INIT_HOOK_TEST_FILES); -test_extension_ci: $(SO_FILE) $(TEST_FILES) $(TEST_STUB_FILES) $(BUILD_DIR)/run-tests.php +# The .phpt suite runs twice: normally, and under valgrind for leak checking. +# Separate targets so CI can parallelize them -- valgrind is far slower. +# The PATH shim in tests/ext/valgrind adds the suppressions file. +test_extension_ci_normal: $(SO_FILE) $(TEST_FILES) $(TEST_STUB_FILES) $(BUILD_DIR)/run-tests.php ( \ set -xe; \ export PATH="$(PROJECT_ROOT)/tests/ext/valgrind:$$PATH"; \ export TEST_PHP_JUNIT=$(JUNIT_RESULTS_DIR)/normal-extension-test.xml; \ $(ALL_TEST_ENV_OVERRIDE) $(RUN_TESTS_CMD) -d extension=$(SO_FILE) $(BUILD_DIR)/$(TESTS); \ - \ + ) + +test_extension_ci_valgrind: $(SO_FILE) $(TEST_FILES) $(TEST_STUB_FILES) $(BUILD_DIR)/run-tests.php + ( \ + set -xe; \ + export PATH="$(PROJECT_ROOT)/tests/ext/valgrind:$$PATH"; \ export TEST_PHP_JUNIT=$(JUNIT_RESULTS_DIR)/valgrind-extension-test.xml; \ export TEST_PHP_OUTPUT=$(JUNIT_RESULTS_DIR)/valgrind-run-tests.out; \ DD_SPAWN_WORKER_STABLE_TRAMPOLINE=1 $(ALL_TEST_ENV_OVERRIDE) DD_TRACE_AGENT_TIMEOUT=5000 $(RUN_TESTS_CMD) -d extension=$(SO_FILE) -m -s $$TEST_PHP_OUTPUT $(BUILD_DIR)/$(TESTS) && ! grep -e '^LEAKED TEST SUMMARY' $$TEST_PHP_OUTPUT; \ ) +# Recursive $(MAKE), not prerequisites: under `make -jN` prerequisites would run +# concurrently and both passes share the same .phpt sandbox (.out/.diff/.mem). +test_extension_ci: + $(MAKE) test_extension_ci_normal + $(MAKE) test_extension_ci_valgrind + build_tea: TEA_BUILD_TESTS=ON build_tea: TEA_PREFIX_PATH=/opt/catch2 build_tea: build_tea_common @@ -1637,5 +1651,5 @@ test_internal_api_randomized: $(SO_FILE) composer.lock: composer.json $(call run_composer_with_retry,,) -.PHONY: dev dist_clean clean cores all clang_format_check clang_format_fix install sudo_install test_c test_c_mem test_extension_ci test_zai test_zai_asan test install_ini install_all \ +.PHONY: dev dist_clean clean cores all clang_format_check clang_format_fix install sudo_install test_c test_c_mem test_extension_ci test_extension_ci_normal test_extension_ci_valgrind test_zai test_zai_asan test install_ini install_all \ .apk .rpm .deb .tar.gz sudo debug prod strict run-tests.php verify_pecl_file_definitions verify_package_xml cbindgen cbindgen_binary