A suite that ran no checks must stop reporting PASSED (#447)

test/arrow_export.sh

@@ -22,9 +22,7 @@ set -uo pipefail
 pgc_setup "${1:-/usr/local/pg17/bin/pg_config}"
 
 if ! python3 -c 'import pyarrow' 2>/dev/null; then
-	echo "-- pyarrow not available; skipping Arrow export verification"
-	pgc_summary
-	exit 0
+	pgc_skip pyarrow "pyarrow not available; skipping Arrow export verification"
 fi
 
 expect_error() {

test/arrow_nested.sh

@@ -20,9 +20,7 @@ set -uo pipefail
 pgc_setup "${1:-/usr/local/pg17/bin/pg_config}"
 
 if ! python3 -c 'import pyarrow' 2>/dev/null; then
-	echo "-- pyarrow not available; skipping Arrow nested export verification"
-	pgc_summary
-	exit 0
+	pgc_skip pyarrow "pyarrow not available; skipping Arrow nested export verification"
 fi
 
 expect_error() {

test/devloop.sh

@@ -66,8 +66,11 @@ for s in "$@"; do
 	# clusters did. Sourced rather than duplicated as literals -- an earlier
 	# version of this comment claimed the literals were checked against the band
 	# elsewhere, and no such check existed.
-	if ! PGC_SKIP_BUILD=1 PGC_PORT=$((PGC_PORT_LO + RANDOM % (PGC_PORT_HI - PGC_PORT_LO))) \
-		bash "test/${s}.sh" "$PGC"; then
+	PGC_SKIP_BUILD=1 PGC_PORT=$((PGC_PORT_LO + RANDOM % (PGC_PORT_HI - PGC_PORT_LO))) \
+		bash "test/${s}.sh" "$PGC"
+	_rc=$?
+	# 2 is pgc_summary's skipped state (#447), not a failure.
+	if [ "$_rc" != 0 ] && [ "$_rc" != 2 ]; then
 		rc=1
 	fi
 done

test/fuzz_arrow.sh

@@ -39,8 +39,7 @@ set -uo pipefail
 pgc_setup "${1:-/usr/local/pg17/bin/pg_config}"
 
 if ! python3 -c 'import pyarrow.ipc' 2>/dev/null; then
-	echo "SKIP  pyarrow not available; the Arrow fuzzer needs it to build seeds"
-	pgc_summary
+	pgc_skip pyarrow "pyarrow not available; the Arrow fuzzer needs it to build seeds"
 fi
 
 SEED="${PGC_SEED:-20260728}"

test/fuzz_parquet.sh

@@ -41,8 +41,7 @@ set -uo pipefail
 pgc_setup "${1:-/usr/local/pg17/bin/pg_config}"
 
 if ! python3 -c 'import pyarrow.parquet' 2>/dev/null; then
-	echo "SKIP  pyarrow not available; the Parquet fuzzer needs it to build seeds"
-	pgc_summary
+	pgc_skip pyarrow "pyarrow not available; the Parquet fuzzer needs it to build seeds"
 fi
 
 SEED="${PGC_SEED:-20260728}"

test/lib.sh

@@ -571,12 +571,61 @@ chunk_group_count() {
 
 # ---- summary ---------------------------------------------------------------
 
+# A dependency this suite needs is not installed.
+#
+# This FAILS, and that is the point. A skip is a red that nobody has to look at,
+# which is how fifteen suites came to report PASSED while asserting nothing, and
+# how temporal.sh has been green on PG18 without btree_gist. A box that cannot run
+# a suite is not a box that passed it.
+#
+# The opt-out is explicit and per-capability, so a developer without pyarrow can
+# still work, and so the waiver is visible in the command rather than implied by
+# silence:
+#
+#     PGC_ALLOW_MISSING_PYARROW=1 test/native_parquet_units.sh
+#     PGC_ALLOW_MISSING=1         test/run_all_versions.sh
+#
+# Missing DEPENDENCY and not-applicable-to-this-MAJOR are different things and are
+# deliberately not the same code path. PostgreSQL 15 has no WITHOUT OVERLAPS to
+# test and no amount of installing will give it one, so those gates call
+# pgc_summary directly and report SKIPPED. Nothing is broken there. Here it is.
+pgc_skip() {  # pgc_skip <capability> <message>
+	local cap allow_one
+	cap="$(printf '%s' "$1" | tr '[:lower:]-' '[:upper:]_')"
+	allow_one="PGC_ALLOW_MISSING_$cap"
+	if [ "${PGC_ALLOW_MISSING:-0}" = 1 ] || [ "${!allow_one:-0}" = 1 ]; then
+		echo "SKIP  $2 (waived by $allow_one or PGC_ALLOW_MISSING)"
+		pgc_summary
+	fi
+	PGC_CHECKS=$((PGC_CHECKS + 1))
+	PGC_FAIL=1
+	echo "FAIL  $2"
+	echo "      A missing dependency is an environment defect, not a pass. Install"
+	echo "      it, or set $allow_one=1 to run knowingly without this coverage."
+	pgc_summary
+}
+
+# Three states, not two (#447).
+#
+# The verdict used to be a function of PGC_FAIL alone, and PGC_CHECKS was printed
+# and never read. So a suite that asserted NOTHING printed PASSED and exited 0,
+# indistinguishable from one that ran four hundred checks. Fifteen suites do that
+# whenever pyarrow is absent, which is how an entire Parquet and Arrow surface,
+# including both fuzzers, can leave a run with every line still saying PASSED.
+#
+# A suite that ran no checks did not pass. It is also not a failure: PostgreSQL 15
+# genuinely has no WITHOUT OVERLAPS to test, and a developer box without an
+# optional dependency is a supported configuration rather than a defect. Making
+# those red is the "a red everyone knows to ignore is a red nobody reads" failure
+# this tree keeps arguing against.
+#
+# So skipped is its own exit code. 0 passed, 1 failed, 2 ran nothing. The drivers
+# count 2 separately and report how many suites actually ran, which is what #422
+# did one level up for how many VERSIONS actually ran.
 pgc_summary() {
 	echo
 	echo "checks run: $PGC_CHECKS"
-	if [ "$PGC_FAIL" = "0" ]; then
-		echo "$(basename "$0"): PASSED"
-	else
+	if [ "$PGC_FAIL" != "0" ]; then
 		echo "$(basename "$0"): FAILED"
 		# A source-shape suite (wal_envelope, decode_interrupts) never calls
 		# pgc_setup, so there is no cluster and no log. Without this guard the
@@ -586,6 +635,12 @@ pgc_summary() {
 			echo "---- server log tail ----"
 			pgc_pg "tail -40 '$PGC_LOGFILE'" 2>/dev/null || true
 		fi
+		exit 1
+	fi
+	if [ "$PGC_CHECKS" = "0" ]; then
+		echo "$(basename "$0"): SKIPPED (ran no checks)"
+		exit 2
 	fi
-	exit $PGC_FAIL
+	echo "$(basename "$0"): PASSED"
+	exit 0
 }

test/native_parquet_codecs.sh

@@ -16,9 +16,7 @@ set -uo pipefail
 pgc_setup "${1:-/usr/local/pg17/bin/pg_config}"
 
 if ! python3 -c 'import pyarrow.parquet' 2>/dev/null; then
-	echo "SKIP  pyarrow not available; codec suite needs it to write compressed files"
-	pgc_summary
-	exit 0
+	pgc_skip pyarrow "pyarrow not available; codec suite needs it to write compressed files"
 fi
 
 W="$PGC_WORKDIR"

test/native_parquet_hardening.sh

@@ -22,9 +22,7 @@ set -uo pipefail
 pgc_setup "${1:-/usr/local/pg17/bin/pg_config}"
 
 if ! python3 -c 'import pyarrow.parquet' 2>/dev/null; then
-	echo "SKIP  pyarrow not available; Parquet hardening suite needs it"
-	pgc_summary
-	exit 0
+	pgc_skip pyarrow "pyarrow not available; Parquet hardening suite needs it"
 fi
 
 W="$PGC_WORKDIR"

test/native_parquet_multifile.sh

@@ -18,9 +18,7 @@ set -uo pipefail
 pgc_setup "${1:-/usr/local/pg17/bin/pg_config}"
 
 if ! python3 -c 'import pyarrow.parquet' 2>/dev/null; then
-	echo "SKIP  pyarrow not available; multi-file suite needs it"
-	pgc_summary
-	exit 0
+	pgc_skip pyarrow "pyarrow not available; multi-file suite needs it"
 fi
 
 W="$PGC_WORKDIR"

test/native_parquet_partition.sh

@@ -21,9 +21,7 @@ set -uo pipefail
 pgc_setup "${1:-/usr/local/pg17/bin/pg_config}"
 
 if ! python3 -c 'import pyarrow.parquet' 2>/dev/null; then
-	echo "SKIP  pyarrow not available; partition suite needs it"
-	pgc_summary
-	exit 0
+	pgc_skip pyarrow "pyarrow not available; partition suite needs it"
 fi
 
 W="$PGC_WORKDIR"

test/native_parquet_projection.sh

@@ -17,9 +17,7 @@ set -uo pipefail
 pgc_setup "${1:-/usr/local/pg17/bin/pg_config}"
 
 if ! python3 -c 'import pyarrow.parquet' 2>/dev/null; then
-	echo "SKIP  pyarrow not available; projection suite needs it"
-	pgc_summary
-	exit 0
+	pgc_skip pyarrow "pyarrow not available; projection suite needs it"
 fi
 
 W="$PGC_WORKDIR"

test/native_parquet_pushdown.sh

@@ -17,9 +17,7 @@ set -uo pipefail
 pgc_setup "${1:-/usr/local/pg17/bin/pg_config}"
 
 if ! python3 -c 'import pyarrow.parquet' 2>/dev/null; then
-	echo "SKIP  pyarrow not available; predicate-pushdown suite needs it to write stats"
-	pgc_summary
-	exit 0
+	pgc_skip pyarrow "pyarrow not available; predicate-pushdown suite needs it to write stats"
 fi
 
 PARQ="$PGC_WORKDIR/stats.parquet"

test/native_parquet_streaming.sh

@@ -22,9 +22,7 @@ set -uo pipefail
 pgc_setup "${1:-/usr/local/pg17/bin/pg_config}"
 
 if ! python3 -c 'import pyarrow.parquet' 2>/dev/null; then
-	echo "SKIP  pyarrow not available; streaming suite needs it"
-	pgc_summary
-	exit 0
+	pgc_skip pyarrow "pyarrow not available; streaming suite needs it"
 fi
 
 W="$PGC_WORKDIR"

test/native_parquet_units.sh

@@ -19,9 +19,7 @@ set -uo pipefail
 pgc_setup "${1:-/usr/local/pg17/bin/pg_config}"
 
 if ! python3 -c 'import pyarrow.parquet' 2>/dev/null; then
-	echo "SKIP  pyarrow not available; Parquet unit suite needs it"
-	pgc_summary
-	exit 0
+	pgc_skip pyarrow "pyarrow not available; Parquet unit suite needs it"
 fi
 
 W="$PGC_WORKDIR"

test/native_repack.sh

@@ -48,8 +48,7 @@ pgc_setup "${1:-/usr/local/pg19/bin/pg_config}"
 srv="$(q 'SHOW server_version_num')"
 if [ "${srv:-0}" -lt 190000 ]; then
 	echo "SKIP  REPACK requires PostgreSQL 19 (server_version_num=$srv)"
-	echo "native_repack.sh: SKIPPED"
-	exit 0
+	pgc_summary
 fi
 
 ROWS=${PGC_REPACK_ROWS:-20000}

test/parquet_export.sh

@@ -24,9 +24,7 @@ set -uo pipefail
 pgc_setup "${1:-/usr/local/pg17/bin/pg_config}"
 
 if ! python3 -c 'import pyarrow.parquet' 2>/dev/null; then
-	echo "-- pyarrow not available; skipping Parquet export verification"
-	pgc_summary
-	exit 0
+	pgc_skip pyarrow "pyarrow not available; skipping Parquet export verification"
 fi
 
 expect_error() {

test/parquet_import.sh

@@ -20,9 +20,7 @@ set -uo pipefail
 pgc_setup "${1:-/usr/local/pg17/bin/pg_config}"
 
 if ! python3 -c 'import pyarrow' 2>/dev/null; then
-	echo "-- pyarrow not available; skipping Parquet import verification"
-	pgc_summary
-	exit 0
+	pgc_skip pyarrow "pyarrow not available; skipping Parquet import verification"
 fi
 
 expect_error() {

test/parquet_nested.sh

@@ -20,9 +20,7 @@ set -uo pipefail
 pgc_setup "${1:-/usr/local/pg17/bin/pg_config}"
 
 if ! python3 -c 'import pyarrow' 2>/dev/null; then
-	echo "-- pyarrow not available; skipping Parquet nested export verification"
-	pgc_summary
-	exit 0
+	pgc_skip pyarrow "pyarrow not available; skipping Parquet nested export verification"
 fi
 
 expect_error() {

test/pg19_vacuum_options.sh

@@ -34,8 +34,7 @@ pgc_setup "${1:-/usr/local/pg19/bin/pg_config}"
 srv="$(q 'SHOW server_version_num')"
 if [ "${srv:-0}" -lt 190000 ]; then
 	echo "SKIP  parallel autovacuum requires PostgreSQL 19 (server_version_num=$srv)"
-	echo "pg19_vacuum_options.sh: SKIPPED"
-	exit 0
+	pgc_summary
 fi
 
 ROWS=${PGC_AV_ROWS:-100000}

test/run_all_versions.sh

@@ -434,7 +434,10 @@ for pgc in "${CONFIGS[@]}"; do
 		# its assertions untrustworthy, only slower.
 		if [ "${PGC_SKIP_TIMING:-0}" = 1 ] && is_timing_suite "$s"; then
 			echo "  SKIP  $s (PGC_SKIP_TIMING)"
-			echo 0 >"$builddir/${s}.rc"
+			# 2, not 0. This suite did not run, and since #447 the collector has
+			# a state that says so. Recording it as a pass was the same lie the
+			# zero-check suites were telling, just written by the driver.
+			echo 2 >"$builddir/${s}.rc"
 			continue
 		fi
 		port=$((BASE_PORT++))
@@ -444,10 +447,24 @@ for pgc in "${CONFIGS[@]}"; do
 	done
 
 	# collect results in suite order for a stable, readable summary
+	suites_ran=0
+	suites_skipped=0
+	skipped_names=""
 	for s in "${SUITES[@]}"; do
-		if [ "$(cat "$builddir/${s}.rc" 2>/dev/null)" = 0 ]; then
+		_rc="$(cat "$builddir/${s}.rc" 2>/dev/null)"
+		if [ "$_rc" = 0 ]; then
 			echo "  PASS  $s"
 			results+="$s=PASS "
+			suites_ran=$((suites_ran + 1))
+		elif [ "$_rc" = 2 ]; then
+			# Exit 2 is pgc_summary's third state: the suite ran no checks (#447).
+			# Not a pass, because it asserted nothing. Not a failure, because a
+			# major without the feature and a box without an optional dependency
+			# are both supported. Counted, so the total below can say so.
+			echo "  SKIP  $s (ran no checks)"
+			results+="$s=SKIP "
+			suites_skipped=$((suites_skipped + 1))
+			skipped_names="$skipped_names $s"
 		else
 			echo "  FAIL  $s"
 			# The failing check first, then the tail. A suite that prints a
@@ -467,10 +484,26 @@ for pgc in "${CONFIGS[@]}"; do
 		fi
 	done
 
+	# How many suites actually asserted something, said out loud (#447).
+	#
+	# #422 added this one level up, after a matrix reported ALL VERSIONS PASSED
+	# having run none of them. The same hole existed per-suite: fifteen suites
+	# report a verdict without running a check when pyarrow is absent, and the old
+	# per-version line counted them among the passes. A count that includes suites
+	# nobody ran is the thing this project keeps having to unlearn.
+	echo "  suites that ran: $suites_ran of ${#SUITES[@]} (skipped: $suites_skipped)"
+	if [ "$suites_skipped" != 0 ]; then
+		echo "  skipped:${skipped_names}"
+	fi
+	if [ "$suites_ran" = 0 ]; then
+		echo "  NO SUITES RAN on PG$major, which is not a pass"
+		verfail=1
+	fi
+
 	if [ "$verfail" = 0 ]; then
-		SUMMARY+=("PASS   PG$major  ${results}")
+		SUMMARY+=("PASS   PG$major  ($suites_ran ran, $suites_skipped skipped)  ${results}")
 	else
-		SUMMARY+=("FAIL   PG$major  ${results}")
+		SUMMARY+=("FAIL   PG$major  ($suites_ran ran, $suites_skipped skipped)  ${results}")
 		overall=1
 	fi
 	rm -rf "$builddir"

test/run_coverage.sh

@@ -78,17 +78,23 @@ lcov --directory "$SRCDIR/src" --zerocounters >/dev/null 2>&1
 . "$SRCDIR/test/portlib.sh"
 port="$(pgc_pick_port)"
 
-pass=0; fail=0; failed=""
+pass=0; fail=0; failed=""; skip=0; skipped=""
 for s in $SUITES; do
 	port=$((port + 1))
-	if PGC_SKIP_BUILD=1 PGC_PORT="$port" \
-		bash "$SRCDIR/test/${s}.sh" "$PGC" >"$OUT/${s}.log" 2>&1; then
+	PGC_SKIP_BUILD=1 PGC_PORT="$port" \
+		bash "$SRCDIR/test/${s}.sh" "$PGC" >"$OUT/${s}.log" 2>&1
+	rc=$?
+	if [ "$rc" = 0 ]; then
 		pass=$((pass + 1))
+	elif [ "$rc" = 2 ]; then
+		# Ran no checks (#447). It contributed no coverage either, so counting it
+		# as a pass overstates what this report measured.
+		skip=$((skip + 1)); skipped="$skipped $s"
 	else
 		fail=$((fail + 1)); failed="$failed $s"
 	fi
 done
-echo "-- suites: $pass passed, $fail failed${failed:+ ($failed)}"
+echo "-- suites: $pass passed, $fail failed${failed:+ ($failed)}, $skip skipped${skipped:+ ($skipped)}"
 
 echo "-- collect"
 lcov --directory "$SRCDIR/src" --capture --output-file "$OUT/coverage.info" \