diff --git a/test/devloop.sh b/test/devloop.sh index a4054ad..ed713ac 100755 --- a/test/devloop.sh +++ b/test/devloop.sh @@ -69,8 +69,9 @@ for s in "$@"; do PGC_SKIP_BUILD=1 PGC_PORT=$((PGC_PORT_LO + RANDOM % (PGC_PORT_HI - PGC_PORT_LO))) \ bash "test/${s}.sh" "$PGC" _rc=$? - # 2 is pgc_summary's skipped state (#447), not a failure. - if [ "$_rc" != 0 ] && [ "$_rc" != 2 ]; then + # 66 is pgc_summary's skipped state, not a failure. Not 2: bash returns 2 on + # a parse error, so treating 2 as a skip hides a broken suite file. + if [ "$_rc" != 0 ] && [ "$_rc" != 66 ]; then rc=1 fi done diff --git a/test/lib.sh b/test/lib.sh index c6f5001..231fb05 100755 --- a/test/lib.sh +++ b/test/lib.sh @@ -30,6 +30,22 @@ PGC_FAIL=0 PGC_CHECKS=0 +# The status pgc_summary uses for "ran no checks". +# +# NOT 2. #448 used 2 and that was wrong: 2 is a status suites already produce for +# unrelated reasons. bash exits 2 on a parse error in the suite file (verified), +# and the suites that run under `set -euo pipefail` -- smoke, phase2 through +# phase6, audit -- abort with whatever status the failing command returned, so a +# dead postmaster or a typo became "ran no checks" and every runner reported the +# major green. That is precisely the lie #447 was opened to remove, relocated one +# layer down. +# +# 66 is not produced by bash (1, 2, 126, 127, 128+n), by psql (1, 2, 3), or by +# make. It cannot be made collision-proof -- `set -e` propagates any status an +# aborting command returns -- so the runners ALSO require the SKIPPED line in the +# log before believing it. Two independent signals, because one was not enough. +PGC_EXIT_SKIPPED=66 + # ---- cluster identity helpers ---------------------------------------------- # Normalize a directory for comparison. `cd && pwd -P` is POSIX; realpath -m is @@ -639,7 +655,7 @@ pgc_summary() { fi if [ "$PGC_CHECKS" = "0" ]; then echo "$(basename "$0"): SKIPPED (ran no checks)" - exit 2 + exit $PGC_EXIT_SKIPPED fi echo "$(basename "$0"): PASSED" exit 0 diff --git a/test/native_parquet_streaming.sh b/test/native_parquet_streaming.sh index f4206ab..7d95d27 100755 --- a/test/native_parquet_streaming.sh +++ b/test/native_parquet_streaming.sh @@ -60,9 +60,12 @@ if os.path.getsize(big) != SIZE: sys.exit(4) PY then - echo "SKIP could not build the oversized sparse file" - pgc_summary - exit 0 + # An environment that cannot build the file is a box that cannot gate this + # suite, not a major without the feature. The pyarrow gate thirty lines above + # was converted and this one was missed, so a low-space or non-sparse + # filesystem quietly took the 1GB-palloc ceiling and the 1600MB-hole guard out + # of the run while the major still reported PASS. + pgc_skip sparse_file "could not build the oversized sparse file" fi # errtext QUERY -> the raised ERROR line, or "NO ERROR" diff --git a/test/native_repack.sh b/test/native_repack.sh index 9ec7dab..77cccb0 100755 --- a/test/native_repack.sh +++ b/test/native_repack.sh @@ -46,7 +46,20 @@ pgc_setup "${1:-/usr/local/pg19/bin/pg_config}" # Version gate, and it must be VISIBLE. A suite that silently passes on 15 to 18 # is the failure mode this project keeps finding. srv="$(q 'SHOW server_version_num')" -if [ "${srv:-0}" -lt 190000 ]; then +# An empty answer is a cluster that stopped talking, not PostgreSQL 18. ${srv:-0} +# made those identical, so a dead postmaster reported SKIPPED and the major passed. +# +# The failure is raised only when it happens, rather than as a check that always +# runs: a passing check would put PGC_CHECKS at 1, and this suite's whole verdict +# on an older major depends on it being 0. Asserting the premise must not destroy +# the skip it guards. +if ! pgc_is_number "$srv"; then + PGC_CHECKS=$((PGC_CHECKS + 1)) + PGC_FAIL=1 + echo "FAIL the server did not answer 'SHOW server_version_num': got [$srv]" + pgc_summary +fi +if [ "$srv" -lt 190000 ]; then echo "SKIP REPACK requires PostgreSQL 19 (server_version_num=$srv)" pgc_summary fi diff --git a/test/pg19_vacuum_options.sh b/test/pg19_vacuum_options.sh index ec485c7..8fa5399 100755 --- a/test/pg19_vacuum_options.sh +++ b/test/pg19_vacuum_options.sh @@ -32,7 +32,20 @@ set -uo pipefail pgc_setup "${1:-/usr/local/pg19/bin/pg_config}" srv="$(q 'SHOW server_version_num')" -if [ "${srv:-0}" -lt 190000 ]; then +# An empty answer is a cluster that stopped talking, not PostgreSQL 18. ${srv:-0} +# made those identical, so a dead postmaster reported SKIPPED and the major passed. +# +# The failure is raised only when it happens, rather than as a check that always +# runs: a passing check would put PGC_CHECKS at 1, and this suite's whole verdict +# on an older major depends on it being 0. Asserting the premise must not destroy +# the skip it guards. +if ! pgc_is_number "$srv"; then + PGC_CHECKS=$((PGC_CHECKS + 1)) + PGC_FAIL=1 + echo "FAIL the server did not answer 'SHOW server_version_num': got [$srv]" + pgc_summary +fi +if [ "$srv" -lt 190000 ]; then echo "SKIP parallel autovacuum requires PostgreSQL 19 (server_version_num=$srv)" pgc_summary fi diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index 9de01bd..821151c 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -434,10 +434,15 @@ for pgc in "${CONFIGS[@]}"; do # its assertions untrustworthy, only slower. if [ "${PGC_SKIP_TIMING:-0}" = 1 ] && is_timing_suite "$s"; then echo " SKIP $s (PGC_SKIP_TIMING)" - # 2, not 0. This suite did not run, and since #447 the collector has - # a state that says so. Recording it as a pass was the same lie the - # zero-check suites were telling, just written by the driver. - echo 2 >"$builddir/${s}.rc" + # 66, not 0. This suite did not run, and the collector has a state + # that says so. Recording it as a pass was the same lie the zero-check + # suites were telling, just written by the driver. + # + # The log is written too, because the collector requires the marker as + # well as the status: this branch never executes the suite, so nothing + # else would produce one and the run would be classified a failure. + echo 66 >"$builddir/${s}.rc" + echo "$s.sh: SKIPPED (ran no checks)" >"$builddir/${s}.log" continue fi port=$((BASE_PORT++)) @@ -456,7 +461,7 @@ for pgc in "${CONFIGS[@]}"; do echo " PASS $s" results+="$s=PASS " suites_ran=$((suites_ran + 1)) - elif [ "$_rc" = 2 ]; then + elif [ "$_rc" = 66 ] && grep -q 'SKIPPED (ran no checks)' "$builddir/${s}.log" 2>/dev/null; then # Exit 2 is pgc_summary's third state: the suite ran no checks (#447). # Not a pass, because it asserted nothing. Not a failure, because a # major without the feature and a box without an optional dependency diff --git a/test/run_coverage.sh b/test/run_coverage.sh index a3dbaec..e0e8c9d 100755 --- a/test/run_coverage.sh +++ b/test/run_coverage.sh @@ -81,12 +81,17 @@ port="$(pgc_pick_port)" pass=0; fail=0; failed=""; skip=0; skipped="" for s in $SUITES; do port=$((port + 1)) - PGC_SKIP_BUILD=1 PGC_PORT="$port" \ + # PGC_SKIP_TIMING, because a --coverage build is instrumented and its wall + # clock means nothing. The wall-clock suites assert ratios and absolute + # timeouts; run here without the flag they fail for the instrumentation rather + # than for the code, and this runner discovers every test/*.sh including any + # added later. The matrix is where those numbers are taken. + PGC_SKIP_BUILD=1 PGC_SKIP_TIMING=1 PGC_PORT="$port" \ bash "$SRCDIR/test/${s}.sh" "$PGC" >"$OUT/${s}.log" 2>&1 rc=$? if [ "$rc" = 0 ]; then pass=$((pass + 1)) - elif [ "$rc" = 2 ]; then + elif [ "$rc" = 66 ] && grep -q 'SKIPPED (ran no checks)' "$OUT/${s}.log" 2>/dev/null; then # Ran no checks (#447). It contributed no coverage either, so counting it # as a pass overstates what this report measured. skip=$((skip + 1)); skipped="$skipped $s" @@ -95,6 +100,13 @@ for s in $SUITES; do fi done echo "-- suites: $pass passed, $fail failed${failed:+ ($failed)}, $skip skipped${skipped:+ ($skipped)}" +# A coverage report built from nothing is not a coverage report. run_san grew this +# guard and this runner did not, so its only verdict was "nothing failed" -- which +# a box where every suite aborts satisfies perfectly. +if [ "$pass" = 0 ]; then + echo "-- NO SUITE RAN, so this measures no coverage" + fail=$((fail + 1)) +fi echo "-- collect" lcov --directory "$SRCDIR/src" --capture --output-file "$OUT/coverage.info" \ diff --git a/test/run_san.sh b/test/run_san.sh index bf3e35a..a669b2d 100644 --- a/test/run_san.sh +++ b/test/run_san.sh @@ -97,11 +97,14 @@ for s in $SUITES; do # A sanitizer report reaches here two ways: the backend aborts (SIGABRT, so the # suite's own crash checks or a nonzero rc), or the text appears in the output. san="$(printf '%s' "$out" | grep -icE 'runtime error:|AddressSanitizer|UndefinedBehaviorSanitizer|SUMMARY: .*Sanitizer|terminated by signal 6')" - if [ "$rc" = 2 ] && [ "$san" = 0 ]; then - # pgc_summary's skipped state (#447): the suite ran no checks. Five of - # this runner's default subset are pyarrow-gated, so on a sanitizer image - # without pyarrow this is the difference between "no sanitizer findings" - # and "nothing was sanitized". + if [ "$rc" = 66 ] && [ "$san" = 0 ] && \ + printf '%s' "$out" | grep -q 'SKIPPED (ran no checks)'; then + # pgc_summary's skipped state: the suite ran no checks. + # + # This does NOT cover the pyarrow-gated suites, and an earlier version of + # this comment claimed it did. Since a missing dependency FAILS, pgc_skip + # exits 1, not here. What reaches this branch is a suite with nothing to + # assert on this build, which is a different thing and a rarer one. skipped=$((skipped + 1)) echo " SKIP $s (ran no checks)" elif [ "$rc" != 0 ] || [ "$san" != 0 ]; then @@ -116,11 +119,12 @@ done echo "-- $ran suites under sanitizers, $fail failed, $skipped ran no checks" if [ "$fail" != 0 ]; then echo "SANITIZER GATE FAILED" -elif [ "$skipped" = "$ran" ]; then - # Every suite skipped means nothing was sanitized. Five of the default subset - # are pyarrow-gated, so an image without pyarrow could otherwise report the - # gate passed having exercised no decode path at all (#447). - echo "SANITIZER GATE RAN NOTHING, which is not a pass" +elif [ "$skipped" != 0 ]; then + # ANY skip, not just all of them. The previous form fired only when every + # suite skipped, so 22 of 23 skipped still printed PASSED -- a gate reporting + # on a surface it had almost entirely not exercised. A sanitizer run is worth + # what it covered, so an incomplete one is not a pass. + echo "SANITIZER GATE INCOMPLETE: $skipped of $ran suites ran no checks" fail=1 else echo "SANITIZER GATE PASSED"