From b203f7f76275732930e8789347de34b5c9247398 Mon Sep 17 00:00:00 2001 From: "Joshua (D) Drake" Date: Wed, 5 Aug 2026 06:44:30 -0600 Subject: [PATCH 1/7] Project the index build scan, and pin that it narrowed (#413) jdatcmd's diagnosis: pgcolumnar_index_build_range_scan opened its reader with no projection, so building a one-column index on a wide table decoded every column. It never had to. The callback receives IndexInfo, which carries ii_IndexAttrNumbers and the expression and predicate trees, so the columns were in its own arguments and were thrown away. Measured on 300,000 rows of 20 columns, index on the key alone, non-assert PG18: before columnar 1,403 ms heap 149 ms 9.4x slower than heap after columnar 87 ms heap 144 ms 1.65x faster Three sources feed the projection, and missing any of them reads unset slot values rather than merely reading too much: ii_IndexAttrNumbers the key columns, with 0 marking an expression ii_Expressions an expression index references more ii_Predicate a partial index evaluates against more The "needed attnos to projected set" half is factored out of pgcolumnar_projected_columns as PgColumnarProjectionFromAttnos and shared, per the suggestion on the issue. The system-column and whole-row escapes are the subtle part of that computation and should not exist twice. The build logs what it projected at DEBUG1, because the test needs to assert that the projection NARROWED. A fix here that silently did nothing would pass every correctness check and a wall-clock check on a quiet machine, which is the failure mode this project keeps finding in its own suites. test/native_index_projection.sh asserts the projection for plain, two-column, late-column, expression and partial indexes, then checks each against a forced sequential scan over the full ordered result rather than by point lookup. It runs amcheck where the build has contrib and skips visibly where it does not, since source builds have none. --- src/columnar.h | 4 ++ src/columnar_customscan.c | 33 +++++++-- src/columnar_tableam.c | 56 ++++++++++++++- test/native_index_projection.sh | 117 ++++++++++++++++++++++++++++++++ test/run_all_versions.sh | 2 +- 5 files changed, 203 insertions(+), 9 deletions(-) create mode 100755 test/native_index_projection.sh diff --git a/src/columnar.h b/src/columnar.h index 6741e6c..ad62bbd 100644 --- a/src/columnar.h +++ b/src/columnar.h @@ -451,6 +451,10 @@ extern void PgColumnarCheckFreeSpaceNoOverlap(uint64 storageId); * ------------------------------------------------------------------------- */ extern uint64 PgColumnarNextStorageId(void); extern void PgColumnarInsertNativeStorageRow(const NativeStorageMetadata *s); + +/* projection: needed attnos (pull_varattnos form) -> the reader's 0-based set */ +extern Bitmapset *PgColumnarProjectionFromAttnos(Bitmapset *needed, int natts, + int *nProjected); extern void PgColumnarSetSortedExtent(uint64 storageId, int64 firstGroup, int64 lastGroup); extern void PgColumnarCheckNativeFormatVersion(uint64 storageId, const char *relName); diff --git a/src/columnar_customscan.c b/src/columnar_customscan.c index 6572757..5c57ef1 100644 --- a/src/columnar_customscan.c +++ b/src/columnar_customscan.c @@ -169,17 +169,23 @@ static const CustomExecMethods pgcolumnar_exec_methods = { * carry a ctid system Var). This is the projection pushed into the reader * (spec 9). */ -static Bitmapset * -pgcolumnar_projected_columns(CustomScan *cscan, int natts, int *nProjected) +/* + * PgColumnarProjectionFromAttnos + * Turn a set of needed attribute numbers, in pull_varattnos' offset form, + * into the reader's 0-based projection. Returns NULL for "read every + * column", which is what a system column, a whole-row Var, or an empty set + * all mean. + * + * Shared because index_build_range_scan needs the same computation over a + * different source (#413): its columns come from IndexInfo rather than from + * a plan. The escapes are the interesting part and are worth having once. + */ +Bitmapset * +PgColumnarProjectionFromAttnos(Bitmapset *needed, int natts, int *nProjected) { - Bitmapset *needed = NULL; Bitmapset *projected = NULL; - Index scanrelid = cscan->scan.scanrelid; int attno; - pull_varattnos((Node *) cscan->scan.plan.targetlist, scanrelid, &needed); - pull_varattnos((Node *) cscan->scan.plan.qual, scanrelid, &needed); - /* a system column or whole-row Var forces reading every column */ for (attno = FirstLowInvalidHeapAttributeNumber + 1; attno <= 0; attno++) { @@ -210,6 +216,19 @@ pgcolumnar_projected_columns(CustomScan *cscan, int natts, int *nProjected) return projected; } +static Bitmapset * +pgcolumnar_projected_columns(CustomScan *cscan, int natts, int *nProjected) +{ + Bitmapset *needed = NULL; + Index scanrelid = cscan->scan.scanrelid; + + pull_varattnos((Node *) cscan->scan.plan.targetlist, scanrelid, &needed); + pull_varattnos((Node *) cscan->scan.plan.qual, scanrelid, &needed); + + return PgColumnarProjectionFromAttnos(needed, natts, nProjected); +} + + /* * pgcolumnar_commute_strategy * The btree comparison strategy for "value op column" given the strategy diff --git a/src/columnar_tableam.c b/src/columnar_tableam.c index b497d69..72fbde7 100644 --- a/src/columnar_tableam.c +++ b/src/columnar_tableam.c @@ -29,6 +29,7 @@ #include "executor/tuptable.h" #include "miscadmin.h" #include "nodes/pathnodes.h" +#include "optimizer/optimizer.h" #include "optimizer/pathnode.h" #include "optimizer/plancat.h" #include "port/atomics.h" @@ -1490,7 +1491,60 @@ pgcolumnar_index_build_range_scan(Relation table_rel, Relation index_rel, else snapshot = GetTransactionSnapshot(); - readState = PgColumnarBeginRead(table_rel, snapshot, NULL, NULL, 0, NULL); + /* + * Project. The columns this build needs are all in our own arguments and + * we were throwing them away, so a one-column index on a wide table + * decoded every column (#413). + * + * Three sources, and missing any of them reads unset slot values: + * ii_IndexAttrNumbers the key columns + * ii_Expressions an expression index references more + * ii_Predicate a partial index evaluates against more + * + * PgColumnarProjectionFromAttnos returns NULL for "every column", which + * covers a whole-row or system-column reference, and is what the custom + * scan already does with the same escapes. + */ + { + Bitmapset *needed = NULL; + Bitmapset *projected; + int nProjected = 0; + int i; + + for (i = 0; i < index_info->ii_NumIndexAttrs; i++) + { + AttrNumber attno = index_info->ii_IndexAttrNumbers[i]; + + /* 0 marks an expression column; Vars come from ii_Expressions */ + if (attno != 0) + needed = bms_add_member(needed, + attno - FirstLowInvalidHeapAttributeNumber); + } + pull_varattnos((Node *) index_info->ii_Expressions, 1, &needed); + pull_varattnos((Node *) index_info->ii_Predicate, 1, &needed); + + projected = + PgColumnarProjectionFromAttnos(needed, + RelationGetDescr(table_rel)->natts, + &nProjected); + + /* + * Say what was projected, so a test can assert the projection + * NARROWED rather than infer it from a stopwatch. A fix here that + * silently did nothing would pass every correctness check and a + * wall-clock check on a quiet machine, which is the failure mode + * this projection is being added to avoid. + * + * DEBUG1, so it costs nothing at the default log level. + */ + elog(DEBUG1, + "columnar: index build on \"%s\" projecting %d of %d columns", + RelationGetRelationName(index_rel), nProjected, + RelationGetDescr(table_rel)->natts); + + readState = PgColumnarBeginRead(table_rel, snapshot, NULL, + projected, 0, NULL); + } ownReadState = true; } diff --git a/test/native_index_projection.sh b/test/native_index_projection.sh new file mode 100755 index 0000000..0ff885c --- /dev/null +++ b/test/native_index_projection.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# +# CREATE INDEX projects only the columns the index needs (issue #413). +# +# pgcolumnar_index_build_range_scan opened its reader with no projection, so building a +# one-column index on a wide table decoded every column. It never had to: the callback +# receives IndexInfo, which carries ii_IndexAttrNumbers and the expression and predicate +# trees. The information was in its own arguments and was thrown away. +# +# Measured on 300,000 rows of 20 columns, index on the key alone, non-assert PG18: +# +# before columnar 1,403 ms heap 149 ms 9.4x slower than heap +# after columnar 87 ms heap 144 ms 1.65x faster +# +# What is asserted, in the order the risk sits: +# +# 1. the projection NARROWED, read from the build's own DEBUG1 line. A fix that +# silently did nothing passes every correctness check below, and a wall-clock check +# on a quiet machine, so this is the assertion with teeth. +# 2. expression and partial indexes project their EXTRA columns. Getting those wrong +# evaluates against unset slot values, which is a wrong answer and not a slow one. +# 3. a forced index scan and a forced seq scan agree over the FULL ordered result. +# Point lookups can be satisfied by an index that is wrong for keys nobody asked for. +# +# amcheck runs where the build has contrib and skips VISIBLY where it does not. Source +# builds have no contrib, and a silent skip is the defect this file exists to prevent. +# +# Usage: test/native_index_projection.sh [PG_CONFIG] +# Written fresh for pgColumnar. +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" + +ROWS=${PGC_IDXPROJ_ROWS:-300000} + +# lib.sh's q() emits psql's output for every statement, so "SET ...; SELECT ..." in one +# call returns "SET" as well as the value. Keep the session but read only the result. +qset() { # $1 = SET clause(s), $2 = query + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -At -q -c "$1" -c "$2" 2>/dev/null | tail -1 +} + + +# 20 columns: an int key, an int secondary, 18 wide text. +cols="k int, k2 int" +vals="g, g*2" +for i in $(seq 1 18); do cols="$cols, c$i text"; vals="$vals, repeat('x',80)||g"; done +psql_run "CREATE TABLE w ($cols) USING pgcolumnar;" +psql_run "INSERT INTO w SELECT $vals FROM generate_series(1,$ROWS) g;" +check "fixture rows" "$(q 'SELECT count(*) FROM w')" "$ROWS" +check "fixture is wide" \ + "$(q "SELECT count(*) FROM pg_attribute WHERE attrelid='w'::regclass AND attnum>0")" "20" + +# The projection each build chose, from its own DEBUG1 line. This is the check a +# do-nothing fix would fail and the correctness checks below would not. +proj() { # $1 index name, $2 CREATE INDEX statement -> "n of m" + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \ + -c "SET client_min_messages=debug1;" -c "$2" 2>&1 \ + | grep -oE "index build on \"$1\" projecting [0-9]+ of [0-9]+" \ + | grep -oE '[0-9]+ of [0-9]+' | head -1 +} + +check "plain index projects one column of twenty" \ + "$(proj w_k 'CREATE INDEX w_k ON w (k)')" "1 of 20" +check "two-column index projects two" \ + "$(proj w_k12 'CREATE INDEX w_k12 ON w (k, k2)')" "2 of 20" +check "an index on a late column projects one, not everything before it" \ + "$(proj w_c18 'CREATE INDEX w_c18 ON w (c18)')" "1 of 20" +check "an expression index projects its expression's columns" \ + "$(proj w_expr 'CREATE INDEX w_expr ON w ((k + k2))')" "2 of 20" +check "a partial index projects the predicate's columns too" \ + "$(proj w_part 'CREATE INDEX w_part ON w (k) WHERE k2 > 100')" "2 of 20" +check "an expression over a text column projects that column" \ + "$(proj w_len 'CREATE INDEX w_len ON w ((length(c1)))')" "1 of 20" + +# Correctness. An index built from under-projected data indexes unset slot values, so +# each of these must find what a sequential scan finds. +check "plain index finds the row" \ + "$(qset 'SET enable_seqscan=off' 'SELECT k2 FROM w WHERE k = 12345')" "24690" +check "expression index finds the row" \ + "$(qset 'SET enable_seqscan=off' 'SELECT count(*) FROM w WHERE (k + k2) = 30000')" "1" +check "partial index finds the row inside its predicate" \ + "$(qset 'SET enable_seqscan=off' 'SELECT count(*) FROM w WHERE k = 50000 AND k2 > 100')" "1" + +# The oracle, depending on nothing but PostgreSQL. Compare the FULL ordered result of a +# forced index scan against a forced seq scan. +agree() { # $1 label, $2 predicate, $3 selected expression + local viaix viaseq + viaix=$(qset "SET enable_seqscan=off; SET enable_bitmapscan=off" \ + "SELECT md5(string_agg(t::text, ',' ORDER BY t)) + FROM (SELECT $3 AS t FROM w WHERE $2) s") + viaseq=$(qset "SET enable_indexscan=off; SET enable_bitmapscan=off; SET enable_indexonlyscan=off" \ + "SELECT md5(string_agg(t::text, ',' ORDER BY t)) + FROM (SELECT $3 AS t FROM w WHERE $2) s") + check "$1" "$viaix" "$viaseq" +} +agree "plain index agrees with a sequential scan" "k BETWEEN 1000 AND 9999" "k" +agree "late-column index agrees" "c18 > repeat('x',80)||'99000'" "k" +agree "expression index agrees" "(k + k2) BETWEEN 300 AND 30000" "k" +agree "partial index agrees on its subset" "k2 > 100 AND k < 5000" "k" +agree "text expression index agrees" "length(c1) = 83" "k" + +# amcheck where available. The skip must be visible: a check that reports nothing is +# indistinguishable from a check that passes, which is what this file is about. +if psql_run "CREATE EXTENSION IF NOT EXISTS amcheck;" >/dev/null 2>&1 && + [ "$(q "SELECT count(*) FROM pg_proc WHERE proname='bt_index_check'")" != "0" ]; then + for ix in w_k w_k12 w_c18 w_expr w_part w_len; do + out=$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -c "SELECT bt_index_check('$ix'::regclass)" 2>&1) + check "bt_index_check($ix)" \ + "$(grep -qE 'ERROR' <<<"$out" && echo bad || echo ok)" "ok" + done +else + echo "SKIP amcheck is not installed on this build; the seq-scan oracle above still ran" +fi + +pgc_summary diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index 2290c8b..10cdc0f 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -208,7 +208,7 @@ SRCDIR="${PGC_RUN_SRCDIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" SUITES=(harness_selftest docs_style smoke phase2 phase3 phase4 phase5 phase6 audit concurrency unique_conc \ differential recovery replication native_backend_crash fuzz fuzz_parquet fuzz_arrow hardening concurrent_diff parallel sorted_projection \ arrow_export parquet_export read_stream corruption \ - generated_columns temporal arrow_import index_only projections arrow_nested parquet_import parquet_nested arrow_nested_import parquet_nested_import native_writer native_roundtrip native_encoding native_fastdecode native_zonemap write_minmax_fastpath write_fsst_compressed fsst_margin encode_invariants encode_effort native_skip pushdown_report native_agg native_agg_deletes native_agg_addcolumn native_groupagg ungrouped_vector_agg parallel_vector_agg native_bloom bloom_setting bloom_lazy native_vecskip native_index native_fetch_position native_dml alter_column_type native_ios native_projection native_cluster native_repack native_compact native_recluster recluster_extent native_vacuum_race native_sort_by sort_status native_reclaim native_ownership drop_cleanup pg_dump_roundtrip native_reclaim_cycles native_reclaim_frag native_reclaim_reconcile native_gap native_format native_truncate native_rewrite native_rewrite_conc rewrite_group_scan native_parquet_schema native_read_parquet native_parquet_fdw native_parquet_pushdown native_parquet_hardening server_file_privilege native_parquet_stack native_parquet_units native_parquet_flba native_parquet_codecs native_parquet_projection native_parquet_multifile native_parquet_streaming native_parquet_partition native_cancel cancel_decode wal_envelope decode_interrupts import_exclusion import_deferred parallel_copy parallel_export_parquet fk_referencing row_triggers native_lazy_slot native_ctas native_fetch_cache native_fetch_interrupt analyze_stats analyze_reltuples native_fetch_projection column_projection isolation) + generated_columns temporal arrow_import index_only projections arrow_nested parquet_import parquet_nested arrow_nested_import parquet_nested_import native_writer native_roundtrip native_encoding native_fastdecode native_zonemap write_minmax_fastpath write_fsst_compressed fsst_margin encode_invariants encode_effort native_skip pushdown_report native_agg native_agg_deletes native_agg_addcolumn native_groupagg ungrouped_vector_agg parallel_vector_agg native_bloom bloom_setting bloom_lazy native_vecskip native_index native_index_projection native_fetch_position native_dml alter_column_type native_ios native_projection native_cluster native_repack native_compact native_recluster recluster_extent native_vacuum_race native_sort_by sort_status native_reclaim native_ownership drop_cleanup pg_dump_roundtrip native_reclaim_cycles native_reclaim_frag native_reclaim_reconcile native_gap native_format native_truncate native_rewrite native_rewrite_conc rewrite_group_scan native_parquet_schema native_read_parquet native_parquet_fdw native_parquet_pushdown native_parquet_hardening server_file_privilege native_parquet_stack native_parquet_units native_parquet_flba native_parquet_codecs native_parquet_projection native_parquet_multifile native_parquet_streaming native_parquet_partition native_cancel cancel_decode wal_envelope decode_interrupts import_exclusion import_deferred parallel_copy parallel_export_parquet fk_referencing row_triggers native_lazy_slot native_ctas native_fetch_cache native_fetch_interrupt analyze_stats analyze_reltuples native_fetch_projection column_projection isolation) # Default matrix: one assert-enabled pg_config per major, 15 through 19. DEFAULT_CONFIGS=( From efeff67036ef712d427d7f46ad763837e17290be Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 5 Aug 2026 08:18:25 -0600 Subject: [PATCH 2/7] fix: project the parallel index build too, not only the serial one (#413) pgcolumnar_index_build_range_scan gets its reader two ways. A serial build opens its own; every participant of a parallel build, leader included, arrives with the shared TableScanDesc, whose reader came through the table-AM scan interface and so carries no projection. Projecting only the branch that opens its own reader leaves the parallel build decoding every column. That is not a tuning corner. With every parallel GUC at its default, a 1.5M-row columnar table of incompressible text (459 MB on disk) takes the parallel branch, so the default path for any table of consequential size was the unprojected one. Measured on 300,000 rows of 20 columns, one-column index, PG18: serial-branch fix only parallel arm 568 ms serial arm 71 ms both branches parallel arm 73 ms serial arm 63 ms heap parallel arm 563 ms The wall clock alone reads as "fixed" if you only measure the serial arm, which is why the DEBUG1 line now names the branch it took and the suite asserts it. The projection is computed once, before the branch, and applied to both readers. Each participant derives the same set from the same IndexInfo, so they agree without communicating. PgColumnarReadSetProjection narrows an already-opened reader and is legal only before its first read: colWanted drives what the group loader decodes, and a group already loaded under a wider projection would be reused under a narrower one, returning unset values rather than failing. test/native_index_projection.sh gains a parallel arm that asserts the premise (workers really are used) before asserting the projection, plus a full ordered seq-scan oracle and amcheck over the parallel-built indexes, since one shared reader across participants is where duplicate index entries would come from. Co-authored-by: ChronicallyJD --- src/columnar.h | 2 + src/columnar_reader.c | 37 ++++++++++ src/columnar_tableam.c | 124 ++++++++++++++++++-------------- test/native_index_projection.sh | 49 ++++++++++++- 4 files changed, 157 insertions(+), 55 deletions(-) diff --git a/src/columnar.h b/src/columnar.h index ad62bbd..e34150d 100644 --- a/src/columnar.h +++ b/src/columnar.h @@ -619,6 +619,8 @@ extern void PgColumnarParquetCheckExportable(Relation rel); * stripe indices, so several workers scanning the same relation each claim * distinct stripes. Set by the custom scan's DSM init callbacks. */ +extern void PgColumnarReadSetProjection(PgColumnarReadState *readState, + Bitmapset *projectedColumns); extern void PgColumnarReadSetParallelCounter(PgColumnarReadState *readState, pg_atomic_uint32 *counter); diff --git a/src/columnar_reader.c b/src/columnar_reader.c index 4a1ebe2..dac5892 100644 --- a/src/columnar_reader.c +++ b/src/columnar_reader.c @@ -1565,6 +1565,43 @@ PgColumnarReadSetParallelCounter(PgColumnarReadState *readState, * nothing" and is what the aggregate path relies on when every group is * clean. */ +/* + * PgColumnarReadSetProjection + * Narrow an already-opened reader to a set of columns (issue #413). + * + * The table-AM scan interface has nowhere to put a projection, so a reader + * opened through pgcolumnar_scan_begin reads every column. A caller that + * does know which columns it needs -- an index build knows, from IndexInfo + * -- can say so here instead. + * + * Only legal before the first read. colWanted drives what the group loader + * decodes, and a group already loaded under a wider projection would be + * reused under a narrower one, so changing it mid-scan would silently + * return unset values rather than fail. The caller is expected to do this + * immediately after obtaining the reader; the assertion states the rule and + * the early return keeps a release build honest. + * + * A NULL set means "all columns", matching PgColumnarBeginRead. + */ +void +PgColumnarReadSetProjection(PgColumnarReadState *readState, + Bitmapset *projectedColumns) +{ + int pc; + + Assert(!readState->started); + if (readState->started) + return; + + bms_free(readState->projectedColumns); + readState->projectedColumns = bms_copy(projectedColumns); + readState->allColumnsWanted = (projectedColumns == NULL || + !pgcolumnar_enable_column_projection); + for (pc = 0; pc < readState->natts; pc++) + readState->colWanted[pc] = readState->allColumnsWanted || + bms_is_member(pc, projectedColumns); +} + void PgColumnarReadRestrictToGroups(PgColumnarReadState *readState, const uint64 *groupNumbers, int ngroups) diff --git a/src/columnar_tableam.c b/src/columnar_tableam.c index 72fbde7..7e226ba 100644 --- a/src/columnar_tableam.c +++ b/src/columnar_tableam.c @@ -1432,6 +1432,8 @@ pgcolumnar_index_build_range_scan(Relation table_rel, Relation index_rel, { PgColumnarReadState *readState; bool ownReadState; + Bitmapset *projected; + int nProjected = 0; EState *estate; ExprContext *econtext; ExprState *predicate; @@ -1471,6 +1473,60 @@ pgcolumnar_index_build_range_scan(Relation table_rel, Relation index_rel, * snapshot. The reader advances the command id internally for * read-your-writes. */ + /* + * Project. The columns this build needs are all in our own arguments and we + * were throwing them away, so a one-column index on a wide table decoded + * every column (#413). + * + * Three sources, and missing any of them reads unset slot values: + * ii_IndexAttrNumbers the key columns + * ii_Expressions an expression index references more + * ii_Predicate a partial index evaluates against more + * + * PgColumnarProjectionFromAttnos returns NULL for "every column", which + * covers a whole-row or system-column reference, and is what the custom scan + * already does with the same escapes. + * + * Computed before the branch because BOTH readers need it. Every + * participant in a parallel build computes the same set from the same + * IndexInfo, so they agree without having to communicate. + */ + { + Bitmapset *needed = NULL; + int i; + + for (i = 0; i < index_info->ii_NumIndexAttrs; i++) + { + AttrNumber attno = index_info->ii_IndexAttrNumbers[i]; + + /* 0 marks an expression column; Vars come from ii_Expressions */ + if (attno != 0) + needed = bms_add_member(needed, + attno - FirstLowInvalidHeapAttributeNumber); + } + pull_varattnos((Node *) index_info->ii_Expressions, 1, &needed); + pull_varattnos((Node *) index_info->ii_Predicate, 1, &needed); + + projected = PgColumnarProjectionFromAttnos(needed, + RelationGetDescr(table_rel)->natts, + &nProjected); + } + + /* + * Say what was projected, and which reader it was applied to, so a test can + * assert the projection NARROWED rather than infer it from a stopwatch. A + * fix here that silently did nothing would pass every correctness check and + * a wall-clock check on a quiet machine, which is the failure mode this + * projection is being added to avoid. + * + * DEBUG1, so it costs nothing at the default log level. + */ + elog(DEBUG1, + "columnar: %s index build on \"%s\" projecting %d of %d columns", + scan != NULL ? "parallel" : "serial", + RelationGetRelationName(index_rel), nProjected, + RelationGetDescr(table_rel)->natts); + if (scan != NULL) { /* @@ -1480,6 +1536,18 @@ pgcolumnar_index_build_range_scan(Relation table_rel, Relation index_rel, */ readState = pgcolumnar_scan_read_state((PgColumnarScanDesc) scan, RelationGetDescr(table_rel)); + + /* + * This reader was opened through the table-AM scan interface, which has + * nowhere to carry a projection, so it would decode every column. We + * know better here, so narrow it before the first read. + * + * This branch is not hypothetical and it is not the rare case: with + * parallel maintenance workers available, EVERY participant including + * the leader arrives here, and the serial branch below never runs. Fix + * only the serial branch and a parallel build stays unprojected. + */ + PgColumnarReadSetProjection(readState, projected); ownReadState = false; } else @@ -1491,60 +1559,8 @@ pgcolumnar_index_build_range_scan(Relation table_rel, Relation index_rel, else snapshot = GetTransactionSnapshot(); - /* - * Project. The columns this build needs are all in our own arguments and - * we were throwing them away, so a one-column index on a wide table - * decoded every column (#413). - * - * Three sources, and missing any of them reads unset slot values: - * ii_IndexAttrNumbers the key columns - * ii_Expressions an expression index references more - * ii_Predicate a partial index evaluates against more - * - * PgColumnarProjectionFromAttnos returns NULL for "every column", which - * covers a whole-row or system-column reference, and is what the custom - * scan already does with the same escapes. - */ - { - Bitmapset *needed = NULL; - Bitmapset *projected; - int nProjected = 0; - int i; - - for (i = 0; i < index_info->ii_NumIndexAttrs; i++) - { - AttrNumber attno = index_info->ii_IndexAttrNumbers[i]; - - /* 0 marks an expression column; Vars come from ii_Expressions */ - if (attno != 0) - needed = bms_add_member(needed, - attno - FirstLowInvalidHeapAttributeNumber); - } - pull_varattnos((Node *) index_info->ii_Expressions, 1, &needed); - pull_varattnos((Node *) index_info->ii_Predicate, 1, &needed); - - projected = - PgColumnarProjectionFromAttnos(needed, - RelationGetDescr(table_rel)->natts, - &nProjected); - - /* - * Say what was projected, so a test can assert the projection - * NARROWED rather than infer it from a stopwatch. A fix here that - * silently did nothing would pass every correctness check and a - * wall-clock check on a quiet machine, which is the failure mode - * this projection is being added to avoid. - * - * DEBUG1, so it costs nothing at the default log level. - */ - elog(DEBUG1, - "columnar: index build on \"%s\" projecting %d of %d columns", - RelationGetRelationName(index_rel), nProjected, - RelationGetDescr(table_rel)->natts); - - readState = PgColumnarBeginRead(table_rel, snapshot, NULL, - projected, 0, NULL); - } + readState = PgColumnarBeginRead(table_rel, snapshot, NULL, + projected, 0, NULL); ownReadState = true; } diff --git a/test/native_index_projection.sh b/test/native_index_projection.sh index 0ff885c..8ecbf91 100755 --- a/test/native_index_projection.sh +++ b/test/native_index_projection.sh @@ -100,11 +100,58 @@ agree "expression index agrees" "(k + k2) BETWEEN 300 AND 30 agree "partial index agrees on its subset" "k2 > 100 AND k < 5000" "k" agree "text expression index agrees" "length(c1) = 83" "k" +# The PARALLEL build, which is the default path for any table of consequential size and +# was the one left unprojected. +# +# index_build_range_scan gets its reader two ways. A serial build opens its own, and +# every participant of a parallel build (leader included) arrives with the shared +# TableScanDesc, whose reader came through the table-AM scan interface with nowhere to +# carry a projection. Projecting only the serial branch reads every column exactly when +# it costs most. +# +# This is not a tuning corner. Measured on this fixture at 1.5M rows of incompressible +# text, 459 MB on disk, with every parallel GUC at its default, core chose workers and +# the build took the parallel branch. Forcing it here only keeps the assertion cheap. +# +# Serial-branch-only projection scores 568 ms against heap's 563 ms on the parallel arm, +# and 71 ms on the serial arm: the wall clock alone reads as "fixed" if you measure the +# serial arm, which is why the branch is named in the DEBUG1 line and asserted here. +PAR="SET max_parallel_maintenance_workers=4; SET min_parallel_table_scan_size='0'; + SET maintenance_work_mem='256MB';" + +branch() { # $1 index name, $2 SET clauses, $3 CREATE INDEX -> " of " + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \ + -c "SET client_min_messages=debug1; $2" -c "$3" 2>&1 \ + | grep -oE "(parallel|serial) index build on \"$1\" projecting [0-9]+ of [0-9]+" \ + | sed -E 's/ index build on .* projecting / /' | head -1 +} + +# Assert the PREMISE first. If core declines to go parallel here, the next check would +# pass by reading the serial branch and prove nothing about the one under test. +check "forcing parallel maintenance workers does reach the parallel branch" \ + "$(branch w_par "$PAR" 'CREATE INDEX w_par ON w (k)' | cut -d' ' -f1)" "parallel" +check "a parallel build projects one column of twenty, not all twenty" \ + "$(branch w_par2 "$PAR" 'CREATE INDEX w_par2 ON w (k2)')" "parallel 1 of 20" +check "a parallel expression build projects its expression's columns" \ + "$(branch w_pare "$PAR" 'CREATE INDEX w_pare ON w ((k + k2))')" "parallel 2 of 20" +check "a parallel partial build projects the predicate's columns too" \ + "$(branch w_parp "$PAR" 'CREATE INDEX w_parp ON w (k) WHERE k2 > 100')" "parallel 2 of 20" +check "the serial branch is still reached when workers are refused" \ + "$(branch w_ser 'SET max_parallel_maintenance_workers=0;' \ + 'CREATE INDEX w_ser ON w (k2)' | cut -d' ' -f1)" "serial" + +# Every participant reads through one shared reader, so an under-projected or +# double-counted parallel build shows up as wrong or duplicated index entries. +agree "parallel-built index agrees with a sequential scan" "k2 BETWEEN 2000 AND 19998" "k2" +check "parallel-built index returns each row once" \ + "$(qset 'SET enable_seqscan=off; SET enable_bitmapscan=off' \ + 'SELECT count(*) FROM w WHERE k2 BETWEEN 2 AND 2000')" "1000" + # amcheck where available. The skip must be visible: a check that reports nothing is # indistinguishable from a check that passes, which is what this file is about. if psql_run "CREATE EXTENSION IF NOT EXISTS amcheck;" >/dev/null 2>&1 && [ "$(q "SELECT count(*) FROM pg_proc WHERE proname='bt_index_check'")" != "0" ]; then - for ix in w_k w_k12 w_c18 w_expr w_part w_len; do + for ix in w_k w_k12 w_c18 w_expr w_part w_len w_par w_par2 w_pare w_parp w_ser; do out=$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ -d "$PGC_DB" -c "SELECT bt_index_check('$ix'::regclass)" 2>&1) check "bt_index_check($ix)" \ From 8e067e30ced55a190a0b8edb007a12f0dcdcf40c Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 5 Aug 2026 08:21:15 -0600 Subject: [PATCH 3/7] fix: report the projection the reader will use, not the one we computed The DEBUG1 line the suite asserts on printed nProjected, the count computed before the branch. On the parallel branch that number is right even when the projection is never applied, so the assertion guarding that branch would keep passing while the build read every column: it tested the arithmetic, not the fix. PgColumnarReadProjectedCount reads colWanted, the field the group loader actually consults, and the line is emitted after the reader exists. Removing the setter call on the parallel branch now turns the three parallel projection checks from "1 of 20" to "20 of 20" and the suite fails, which is what a guard is for. Proved by removal on PG18: deleting only PgColumnarReadSetProjection(readState, projected) fails exactly those three checks and leaves the premise and correctness checks passing, since an unprojected build is slow rather than wrong. --- CHANGELOG.md | 12 ++++++++++ src/columnar.h | 1 + src/columnar_reader.c | 24 +++++++++++++++++++ src/columnar_tableam.c | 41 +++++++++++++++++++++------------ test/native_index_projection.sh | 3 +++ 5 files changed, 66 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb4188a..194d561 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,18 @@ which was true until that script existed. setting to the other's storage. - `default_version` moves from `1.0-dev` to `1.0-alpha`, so `SELECT extversion FROM pg_extension` now agrees with `VERSION`. +- `CREATE INDEX` decodes only the columns the index needs (#413). The index + build received an `IndexInfo` carrying the key columns and the expression and + predicate trees, and discarded it, so a one-column index on a wide table read + every column. Both readers are now projected: the one a serial build opens for + itself, and the shared scan a parallel build arrives with, which comes through + the table-access-method scan interface and has nowhere to carry a projection. + The parallel branch is not a corner case. With every parallel setting left at + its default, a 1.5 million row table of incompressible text, 459 MB on disk, + is built in parallel, so that is the branch a table of consequential size + takes. On 300,000 rows of 20 columns on PostgreSQL 18, a one-column index + drops from 568 ms to 73 ms with workers allowed, against 563 ms for the same + index on a heap table. ### Upgrading diff --git a/src/columnar.h b/src/columnar.h index e34150d..f8c83e7 100644 --- a/src/columnar.h +++ b/src/columnar.h @@ -619,6 +619,7 @@ extern void PgColumnarParquetCheckExportable(Relation rel); * stripe indices, so several workers scanning the same relation each claim * distinct stripes. Set by the custom scan's DSM init callbacks. */ +extern int PgColumnarReadProjectedCount(PgColumnarReadState *readState); extern void PgColumnarReadSetProjection(PgColumnarReadState *readState, Bitmapset *projectedColumns); extern void PgColumnarReadSetParallelCounter(PgColumnarReadState *readState, diff --git a/src/columnar_reader.c b/src/columnar_reader.c index dac5892..662457b 100644 --- a/src/columnar_reader.c +++ b/src/columnar_reader.c @@ -1583,6 +1583,30 @@ PgColumnarReadSetParallelCounter(PgColumnarReadState *readState, * * A NULL set means "all columns", matching PgColumnarBeginRead. */ +/* + * PgColumnarReadProjectedCount + * How many columns this reader will actually decode. + * + * Read from colWanted, the field the group loader consults, so a caller + * reporting a projection reports what the reader WILL DO rather than what + * the caller computed and may have failed to apply. That distinction is + * the whole point: a projection computed and then dropped on the floor is + * exactly the bug this accessor exists to make visible (#413). + */ +int +PgColumnarReadProjectedCount(PgColumnarReadState *readState) +{ + int pc; + int n = 0; + + for (pc = 0; pc < readState->natts; pc++) + { + if (readState->colWanted[pc]) + n++; + } + return n; +} + void PgColumnarReadSetProjection(PgColumnarReadState *readState, Bitmapset *projectedColumns) diff --git a/src/columnar_tableam.c b/src/columnar_tableam.c index 7e226ba..5163516 100644 --- a/src/columnar_tableam.c +++ b/src/columnar_tableam.c @@ -1507,26 +1507,16 @@ pgcolumnar_index_build_range_scan(Relation table_rel, Relation index_rel, pull_varattnos((Node *) index_info->ii_Expressions, 1, &needed); pull_varattnos((Node *) index_info->ii_Predicate, 1, &needed); + /* + * nProjected is a required out-parameter, not the number we report. + * The DEBUG1 line below reads the count off the reader instead; see + * the comment there for why. + */ projected = PgColumnarProjectionFromAttnos(needed, RelationGetDescr(table_rel)->natts, &nProjected); } - /* - * Say what was projected, and which reader it was applied to, so a test can - * assert the projection NARROWED rather than infer it from a stopwatch. A - * fix here that silently did nothing would pass every correctness check and - * a wall-clock check on a quiet machine, which is the failure mode this - * projection is being added to avoid. - * - * DEBUG1, so it costs nothing at the default log level. - */ - elog(DEBUG1, - "columnar: %s index build on \"%s\" projecting %d of %d columns", - scan != NULL ? "parallel" : "serial", - RelationGetRelationName(index_rel), nProjected, - RelationGetDescr(table_rel)->natts); - if (scan != NULL) { /* @@ -1564,6 +1554,27 @@ pgcolumnar_index_build_range_scan(Relation table_rel, Relation index_rel, ownReadState = true; } + /* + * Say which branch ran and how wide the reader it produced will actually + * read, so a test can assert the projection NARROWED rather than infer it + * from a stopwatch. A fix that silently did nothing would pass every + * correctness check, and a wall-clock check on a quiet machine, which is + * the failure mode this projection is being added to avoid. + * + * The count comes from the READER, not from nProjected. Reporting what we + * computed would keep printing "1 of 20" if the parallel branch stopped + * applying it, and the assertion guarding that branch would pass while the + * build read every column. Reporting what the reader will decode cannot. + * + * DEBUG1, so it costs nothing at the default log level. + */ + elog(DEBUG1, + "columnar: %s index build on \"%s\" projecting %d of %d columns", + scan != NULL ? "parallel" : "serial", + RelationGetRelationName(index_rel), + PgColumnarReadProjectedCount(readState), + RelationGetDescr(table_rel)->natts); + while (true) { CHECK_FOR_INTERRUPTS(); diff --git a/test/native_index_projection.sh b/test/native_index_projection.sh index 8ecbf91..bc49e54 100755 --- a/test/native_index_projection.sh +++ b/test/native_index_projection.sh @@ -21,6 +21,9 @@ # evaluates against unset slot values, which is a wrong answer and not a slow one. # 3. a forced index scan and a forced seq scan agree over the FULL ordered result. # Point lookups can be satisfied by an index that is wrong for keys nobody asked for. +# 4. the PARALLEL build projects as well. Its reader arrives through the table-AM scan +# interface, which carries no projection, and it is the branch core takes by default +# for any table of consequential size. See the section below for the measurements. # # amcheck runs where the build has contrib and skips VISIBLY where it does not. Source # builds have no contrib, and a silent skip is the defect this file exists to prevent. From 6812f1203f8b8aae8d29482f7c03770a105a5269 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 5 Aug 2026 08:32:14 -0600 Subject: [PATCH 4/7] test: refuse to compare two empty results in the index-projection oracle check "" "" compares nothing with nothing and prints PASS (#418). agree() is the strongest assertion in this file, comparing a forced index scan against a forced sequential scan over the full ordered result, so it is the worst one to have quietly comparing two empty strings. A down cluster, an errored query or a predicate matching nothing all produce that. Both sides must now look like an md5. Proved by substituting a predicate that matches nothing, which is the shape the issue describes: FAIL EMPTY-SHAPE probe (both sides must be a real result, not empty): got [index=[] seq=[]] want [two md5 hashes] Before the guard that comparison passed. Local on purpose. #418 proposes check_num and check_ratio in test/lib.sh and @ChronicallyJD owns that work; this file should adopt the shared helper when it lands and drop the guard. --- test/native_index_projection.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/native_index_projection.sh b/test/native_index_projection.sh index bc49e54..7c3caf1 100755 --- a/test/native_index_projection.sh +++ b/test/native_index_projection.sh @@ -87,6 +87,15 @@ check "partial index finds the row inside its predicate" \ # The oracle, depending on nothing but PostgreSQL. Compare the FULL ordered result of a # forced index scan against a forced seq scan. +# +# Both sides must be a real md5. A down cluster, an errored query or a predicate that +# matches nothing all yield "", and `check "" ""` compares nothing with nothing and +# prints PASS (#418). This oracle is the strongest assertion in the file, so it is the +# worst one to have silently comparing two empty strings. +# +# Local guard on purpose. #418 proposes `check_num` in `test/lib.sh` and +# @ChronicallyJD owns it; this file should adopt that helper when it lands and drop +# the check below. agree() { # $1 label, $2 predicate, $3 selected expression local viaix viaseq viaix=$(qset "SET enable_seqscan=off; SET enable_bitmapscan=off" \ @@ -95,6 +104,11 @@ agree() { # $1 label, $2 predicate, $3 selected expression viaseq=$(qset "SET enable_indexscan=off; SET enable_bitmapscan=off; SET enable_indexonlyscan=off" \ "SELECT md5(string_agg(t::text, ',' ORDER BY t)) FROM (SELECT $3 AS t FROM w WHERE $2) s") + if ! grep -qE '^[0-9a-f]{32}$' <<<"$viaix" || ! grep -qE '^[0-9a-f]{32}$' <<<"$viaseq"; then + check "$1 (both sides must be a real result, not empty)" \ + "index=[$viaix] seq=[$viaseq]" "two md5 hashes" + return + fi check "$1" "$viaix" "$viaseq" } agree "plain index agrees with a sequential scan" "k BETWEEN 1000 AND 9999" "k" From e7cfb0d0f33c17a95d49a947769730372332e238 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 5 Aug 2026 09:13:17 -0600 Subject: [PATCH 5/7] test: make the parallel oracle actually read the parallel-built index (#413) @ChronicallyJD caught that the two checks named for the parallel build did not use a parallel-built index. w_ser was created on (k2), the same column as the parallel-built w_par2, and the planner is free to pick either. It picked w_ser. Confirmed by removal rather than taken on trust. Same suite, only w_ser's column differing: w_ser on (c17) PASS the oracle below really reads the parallel-built index w_ser on (k2) FAIL ... got [w_ser] want [w_par2] In both arms the two "parallel-built index" checks passed, which is the point: they would have stayed green if every parallel-built index were garbage. w_ser only has to show the serial branch is still reachable, so it moves to a column no oracle reads. The premise is now asserted the way the branch checks above already do it: a query names a column, not an index, so "the index scan agrees with the seq scan" proves nothing about a parallel build until the plan is shown to use it. Also from the same review: - The three comment blocks in columnar_reader.c had stacked with the bodies in reverse order, leaving PgColumnarReadSetProjection -- the one carrying the "only legal before the first read" rule -- with no adjacent comment. Each comment now sits on its own function. - Both new externs had landed under the "Parallel scan (gap 23)" block that documents something else. They get their own block. - PgColumnarReadSetProjection's bms_copy ran in the caller's context while PgColumnarBeginRead builds the same field in readContext. No reader of that field exists today so this is consistency, not a fixed bug, but a field owned by a different context depending on which function set it is a trap. --- src/columnar.h | 15 +++++-- src/columnar_reader.c | 73 +++++++++++++++++++-------------- test/native_index_projection.sh | 21 +++++++++- 3 files changed, 74 insertions(+), 35 deletions(-) diff --git a/src/columnar.h b/src/columnar.h index f8c83e7..7aeada7 100644 --- a/src/columnar.h +++ b/src/columnar.h @@ -614,14 +614,23 @@ extern int64 PgColumnarWriteParquetFile(Relation rel, Snapshot snapshot, int nRestrictGroups); extern void PgColumnarParquetCheckExportable(Relation rel); +/* + * Column projection on an already-opened reader (#413). The table-AM scan + * interface has nowhere to carry a projection, so a reader obtained through it + * reads every column; a caller that knows better narrows it here, before the + * first read. PgColumnarReadProjectedCount reports what the reader WILL decode, + * read off colWanted, so a caller reporting a projection cannot report one it + * failed to apply. + */ +extern void PgColumnarReadSetProjection(PgColumnarReadState *readState, + Bitmapset *projectedColumns); +extern int PgColumnarReadProjectedCount(PgColumnarReadState *readState); + /* * Parallel scan (gap 23): point the read state at a shared atomic that hands out * stripe indices, so several workers scanning the same relation each claim * distinct stripes. Set by the custom scan's DSM init callbacks. */ -extern int PgColumnarReadProjectedCount(PgColumnarReadState *readState); -extern void PgColumnarReadSetProjection(PgColumnarReadState *readState, - Bitmapset *projectedColumns); extern void PgColumnarReadSetParallelCounter(PgColumnarReadState *readState, pg_atomic_uint32 *counter); diff --git a/src/columnar_reader.c b/src/columnar_reader.c index 662457b..9684b65 100644 --- a/src/columnar_reader.c +++ b/src/columnar_reader.c @@ -1553,18 +1553,6 @@ PgColumnarReadSetParallelCounter(PgColumnarReadState *readState, readState->parallelCounter = counter; } -/* - * PgColumnarReadRestrictToGroups - * Restrict this scan to the given row group numbers (issue #149). Groups - * outside the set are skipped in the claim loop, so their bytes are never - * read and their column chunks never decoded. The array is copied into the - * read state's own context and sorted there, so the caller may free its own. - * - * Must be called before the first PgColumnarReadNextRow. Passing ngroups == 0 - * makes the scan return no rows, which is the honest reading of "restrict to - * nothing" and is what the aggregate path relies on when every group is - * clean. - */ /* * PgColumnarReadSetProjection * Narrow an already-opened reader to a set of columns (issue #413). @@ -1583,6 +1571,36 @@ PgColumnarReadSetParallelCounter(PgColumnarReadState *readState, * * A NULL set means "all columns", matching PgColumnarBeginRead. */ +void +PgColumnarReadSetProjection(PgColumnarReadState *readState, + Bitmapset *projectedColumns) +{ + int pc; + MemoryContext old; + + Assert(!readState->started); + if (readState->started) + return; + + /* + * Copy into the read state's own context, not the caller's. Nothing reads + * this field after the setter today, so this is consistency rather than a + * fixed bug -- PgColumnarBeginRead builds the same field in readContext and + * PgColumnarReadRestrictToGroups says so in its own comment. A field owned by + * two different contexts depending on which function set it is a trap for + * whoever reads it next. + */ + old = MemoryContextSwitchTo(readState->readContext); + bms_free(readState->projectedColumns); + readState->projectedColumns = bms_copy(projectedColumns); + MemoryContextSwitchTo(old); + readState->allColumnsWanted = (projectedColumns == NULL || + !pgcolumnar_enable_column_projection); + for (pc = 0; pc < readState->natts; pc++) + readState->colWanted[pc] = readState->allColumnsWanted || + bms_is_member(pc, projectedColumns); +} + /* * PgColumnarReadProjectedCount * How many columns this reader will actually decode. @@ -1607,25 +1625,18 @@ PgColumnarReadProjectedCount(PgColumnarReadState *readState) return n; } -void -PgColumnarReadSetProjection(PgColumnarReadState *readState, - Bitmapset *projectedColumns) -{ - int pc; - - Assert(!readState->started); - if (readState->started) - return; - - bms_free(readState->projectedColumns); - readState->projectedColumns = bms_copy(projectedColumns); - readState->allColumnsWanted = (projectedColumns == NULL || - !pgcolumnar_enable_column_projection); - for (pc = 0; pc < readState->natts; pc++) - readState->colWanted[pc] = readState->allColumnsWanted || - bms_is_member(pc, projectedColumns); -} - +/* + * PgColumnarReadRestrictToGroups + * Restrict this scan to the given row group numbers (issue #149). Groups + * outside the set are skipped in the claim loop, so their bytes are never + * read and their column chunks never decoded. The array is copied into the + * read state's own context and sorted there, so the caller may free its own. + * + * Must be called before the first PgColumnarReadNextRow. Passing ngroups == 0 + * makes the scan return no rows, which is the honest reading of "restrict to + * nothing" and is what the aggregate path relies on when every group is + * clean. + */ void PgColumnarReadRestrictToGroups(PgColumnarReadState *readState, const uint64 *groupNumbers, int ngroups) diff --git a/test/native_index_projection.sh b/test/native_index_projection.sh index 7c3caf1..919fa6e 100755 --- a/test/native_index_projection.sh +++ b/test/native_index_projection.sh @@ -153,12 +153,31 @@ check "a parallel expression build projects its expression's columns" \ "$(branch w_pare "$PAR" 'CREATE INDEX w_pare ON w ((k + k2))')" "parallel 2 of 20" check "a parallel partial build projects the predicate's columns too" \ "$(branch w_parp "$PAR" 'CREATE INDEX w_parp ON w (k) WHERE k2 > 100')" "parallel 2 of 20" +# +# On c17, NOT on k2. A serially built index on k2 would sit beside the parallel-built +# w_par2 covering the same column, and the planner is free to pick either, so the two +# oracles below would silently validate the serial build. This check only has to show +# the serial branch is still reachable, so it goes on a column no oracle reads. check "the serial branch is still reached when workers are refused" \ "$(branch w_ser 'SET max_parallel_maintenance_workers=0;' \ - 'CREATE INDEX w_ser ON w (k2)' | cut -d' ' -f1)" "serial" + 'CREATE INDEX w_ser ON w (c17)' | cut -d' ' -f1)" "serial" # Every participant reads through one shared reader, so an under-projected or # double-counted parallel build shows up as wrong or duplicated index entries. +# +# Assert the premise first. "The index scan agrees with the seq scan" proves nothing +# about a parallel build unless the scan actually uses the parallel-built index, and +# nothing in the query text guarantees that: it names a column, not an index. +# qset ends in `tail -1`, which would keep only the last plan line, so read the whole +# EXPLAIN directly. +usesix() { # $1 = query -> the index name the plan scans + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \ + -At -q -c "SET enable_seqscan=off; SET enable_bitmapscan=off;" \ + -c "EXPLAIN (COSTS OFF) $1" 2>/dev/null \ + | grep -oE 'using [a-z0-9_]+' | head -1 | cut -d' ' -f2 +} +check "the oracle below really reads the parallel-built index" \ + "$(usesix 'SELECT k2 FROM w WHERE k2 BETWEEN 2000 AND 19998')" "w_par2" agree "parallel-built index agrees with a sequential scan" "k2 BETWEEN 2000 AND 19998" "k2" check "parallel-built index returns each row once" \ "$(qset 'SET enable_seqscan=off; SET enable_bitmapscan=off' \ From 5ae049b46e3195fcf00006a15b9fc17a09452920 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 5 Aug 2026 10:23:21 -0600 Subject: [PATCH 6/7] fix: put native_index_projection inside the SUITES array, not after it My conflict resolution when merging main appended the suite name after the array's closing paren: ... column_projection isolation) native_index_projection so the array ended at isolation and bash then tried to run the leftover token as a command. CI on PG17 and PG18 said exactly that: run_all_versions.sh: line 211: native_index_projection: command not found The suite is back beside native_index where it was before the merge. Verified by sourcing the array rather than by reading it: 120 elements parse, and native_index_projection, pg19_vacuum_options, harness_selftest, column_projection and isolation are all members. Worth noting how this got past me: `bash -n` passed on the broken file, because a stray command after a closed array is valid syntax. The check that would have caught it is the one that asks whether the array contains what it should, which is the shape of #418 in the runner itself. --- test/run_all_versions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index 524ed51..a647f6d 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -208,7 +208,7 @@ SRCDIR="${PGC_RUN_SRCDIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" SUITES=(harness_selftest docs_style smoke phase2 phase3 phase4 phase5 phase6 audit concurrency unique_conc \ differential recovery replication native_backend_crash fuzz fuzz_parquet fuzz_arrow hardening concurrent_diff parallel sorted_projection \ arrow_export parquet_export read_stream corruption \ - generated_columns temporal arrow_import index_only projections arrow_nested parquet_import parquet_nested arrow_nested_import parquet_nested_import native_writer native_roundtrip native_encoding native_fastdecode native_zonemap write_minmax_fastpath write_fsst_compressed fsst_margin encode_invariants encode_effort native_skip pushdown_report native_agg native_agg_deletes native_agg_addcolumn native_groupagg ungrouped_vector_agg parallel_vector_agg native_bloom bloom_setting bloom_lazy native_vecskip native_index native_fetch_position native_dml alter_column_type native_ios native_projection native_cluster pg19_vacuum_options native_repack native_compact native_recluster recluster_extent native_vacuum_race native_sort_by sort_status native_reclaim native_ownership drop_cleanup pg_dump_roundtrip native_reclaim_cycles native_reclaim_frag native_reclaim_reconcile native_gap native_format native_truncate native_rewrite native_rewrite_conc rewrite_group_scan native_parquet_schema native_read_parquet native_parquet_fdw native_parquet_pushdown native_parquet_hardening server_file_privilege native_parquet_stack native_parquet_units native_parquet_flba native_parquet_codecs native_parquet_projection native_parquet_multifile native_parquet_streaming native_parquet_partition native_cancel cancel_decode wal_envelope decode_interrupts import_exclusion import_deferred parallel_copy parallel_export_parquet fk_referencing row_triggers native_lazy_slot native_ctas native_fetch_cache native_fetch_interrupt analyze_stats analyze_reltuples native_fetch_projection column_projection isolation) native_index_projection + generated_columns temporal arrow_import index_only projections arrow_nested parquet_import parquet_nested arrow_nested_import parquet_nested_import native_writer native_roundtrip native_encoding native_fastdecode native_zonemap write_minmax_fastpath write_fsst_compressed fsst_margin encode_invariants encode_effort native_skip pushdown_report native_agg native_agg_deletes native_agg_addcolumn native_groupagg ungrouped_vector_agg parallel_vector_agg native_bloom bloom_setting bloom_lazy native_vecskip native_index native_index_projection native_fetch_position native_dml alter_column_type native_ios native_projection native_cluster pg19_vacuum_options native_repack native_compact native_recluster recluster_extent native_vacuum_race native_sort_by sort_status native_reclaim native_ownership drop_cleanup pg_dump_roundtrip native_reclaim_cycles native_reclaim_frag native_reclaim_reconcile native_gap native_format native_truncate native_rewrite native_rewrite_conc rewrite_group_scan native_parquet_schema native_read_parquet native_parquet_fdw native_parquet_pushdown native_parquet_hardening server_file_privilege native_parquet_stack native_parquet_units native_parquet_flba native_parquet_codecs native_parquet_projection native_parquet_multifile native_parquet_streaming native_parquet_partition native_cancel cancel_decode wal_envelope decode_interrupts import_exclusion import_deferred parallel_copy parallel_export_parquet fk_referencing row_triggers native_lazy_slot native_ctas native_fetch_cache native_fetch_interrupt analyze_stats analyze_reltuples native_fetch_projection column_projection isolation) # Default matrix: one assert-enabled pg_config per major, 15 through 19. DEFAULT_CONFIGS=( From b646f830c236ef217d44ea5f2ae2e9eb10e1b72b Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 5 Aug 2026 10:36:45 -0600 Subject: [PATCH 7/7] test: drop the #369 probe scripts that a `git add -A` swept into the merge These are throwaway reproduction probes for #369. They were never meant to be committed; resolving the merge conflict with `git add -A` staged them along with the resolution. harness_selftest.sh caught it, which is the registration guard doing exactly its job: FAIL every suite is registered in run_all_versions.sh: got [unregistered: zz_probe_369c zz_probe_369d zz_probe_369e zz_probe_369f zz_probe_369g] want [none] Every other suite passed on both PG17 and PG18, native_index_projection included. The probes live in the session scratchpad, where the #369 work continues. --- test/zz_probe_369c.sh | 94 ------------------------------------------- test/zz_probe_369d.sh | 68 ------------------------------- test/zz_probe_369e.sh | 75 ---------------------------------- test/zz_probe_369f.sh | 53 ------------------------ test/zz_probe_369g.sh | 63 ----------------------------- 5 files changed, 353 deletions(-) delete mode 100644 test/zz_probe_369c.sh delete mode 100755 test/zz_probe_369d.sh delete mode 100755 test/zz_probe_369e.sh delete mode 100755 test/zz_probe_369f.sh delete mode 100755 test/zz_probe_369g.sh diff --git a/test/zz_probe_369c.sh b/test/zz_probe_369c.sh deleted file mode 100644 index 16da7d6..0000000 --- a/test/zz_probe_369c.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env bash -# TEMPORARY PROBE. Reproduce #369 with a fixture that can actually express it. -# -# My first two attempts failed for a reason worth stating: the grouping expression -# has to COLLAPSE a high-cardinality column. I used -# time = start + (g % 250000) * interval '1 second' -# so `time` held 250,000 distinct values and date_trunc('second', time) preserved -# every one. estimate_num_groups came back 251,434 against 250,000 actual, which is -# right, so there was no inflation to trigger the defect. -# -# Here every row gets its own timestamp (20M distinct, 2160us apart, exactly 12 -# hours) and date_trunc('minute') collapses them to 720 groups. That gap between the -# column's distinctness and the expression's is the whole mechanism. -# -# Data is random so it does not compress to nothing: the earlier fixture put 20M -# rows in 34 MB, which made every scan term negligible against the finalize term -# and moved the balance point on its own. -set -uo pipefail -. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -pgc_setup "${1:-/usr/local/pgsql/bin/pg_config}" - -ROWS=${PGC_369_ROWS:-20000000} - -psql_run "CREATE TABLE m (time timestamptz, hostname text, - usage_user float8, usage_system float8, usage_idle float8, - usage_nice float8, usage_iowait float8, usage_irq float8, - usage_softirq float8, usage_steal float8, usage_guest float8, - usage_guest_nice float8) USING pgcolumnar; - INSERT INTO m SELECT - '2026-01-01'::timestamptz + (g * interval '2160 microseconds'), - 'host_' || (g % 4000), - random()*100, random()*100, random()*100, random()*100, - random()*100, random()*100, random()*100, random()*100, - random()*100, random()*100 - FROM generate_series(1,$ROWS) g; - ANALYZE m;" - -got=$(q "SELECT count(*) FROM m") -[ "$got" = "$ROWS" ] || { echo "ABORT: fixture has [$got] rows, expected $ROWS"; pgc_summary; exit 1; } -echo "-- fixture: $got rows, $(q "SELECT pg_size_pretty(pg_total_relation_size('m'))")" -echo "-- ndistinct(time) per ANALYZE: $(q "SELECT n_distinct FROM pg_stats WHERE tablename='m' AND attname='time'")" -echo "-- span: $(q "SELECT max(time)-min(time) FROM m")" - -G="SET pgcolumnar.enable_group_vectorization=on; - SET pgcolumnar.enable_parallel_vector_agg=on; - SET max_parallel_workers_per_gather=4;" -CORE="SET pgcolumnar.enable_group_vectorization=off; - SET pgcolumnar.enable_parallel_vector_agg=off; - SET max_parallel_workers_per_gather=4;" -SER="SET pgcolumnar.enable_group_vectorization=on; - SET pgcolumnar.enable_parallel_vector_agg=off; - SET max_parallel_workers_per_gather=0;" - -ex() { # $1 SETs, $2 query -> whole plan - env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \ - -q -c "$1" -c "EXPLAIN (COSTS ON) $2" 2>&1 -} -t() { local s e; s=$(date +%s%N); psql_run "$1" >/dev/null 2>&1; e=$(date +%s%N); echo $(( (e-s)/1000000 )); } - -shape() { # $1 label, $2 query - local p top est act - p=$(ex "$G" "$2") - # the top node's row estimate is dNumGroups for either arm - est=$(grep -oE 'rows=[0-9]+' <<<"$p" | head -1 | cut -d= -f2) - act=$(q "SELECT count(*) FROM ($2) z" | tail -1) - if grep -q "Parallel Custom Scan" <<<"$p"; then top="PARALLEL arm" - elif grep -q "Columnar Vectorized Group Keys" <<<"$p"; then top="serial node" - else top="core"; fi - echo - echo "=== $1 ===========================================" - printf ' dNumGroups=%s actual=%s inflation=%s chosen=%s\n' \ - "${est:-?}" "$act" \ - "$(awk -v e="${est:-0}" -v a="$act" 'BEGIN{printf (a>0? "%.0fx" : "?"), e/a}')" "$top" - echo " plan:"; grep -E "Custom Scan|HashAggregate|Gather|Vectorized" <<<"$p" | sed 's/^/ /' | head -6 - echo " timings:" - echo " as chosen : $(t "$G $2") ms" - echo " core only : $(t "$CORE $2") ms" - echo " serial forced : $(t "$SER $2") ms" -} - -# G1: expression key over a 12h window. The shape #369 says is declined. -shape "G1 expression key, 12h window" \ - "SELECT date_trunc('minute', time) AS b, avg(usage_user) FROM m - WHERE time >= '2026-01-01' AND time < '2026-01-01 12:00' GROUP BY b" -# G2: same, ten aggregates. -shape "G2 expression key, ten aggregates" \ - "SELECT date_trunc('minute', time) AS b, avg(usage_user), avg(usage_system), - avg(usage_idle), avg(usage_nice), avg(usage_iowait), avg(usage_irq), - avg(usage_softirq), avg(usage_steal), avg(usage_guest), avg(usage_guest_nice) - FROM m WHERE time >= '2026-01-01' AND time < '2026-01-01 12:00' GROUP BY b" -# G3: plain column key. The control: estimate is accurate, arm should be chosen. -shape "G3 plain column key, full scan" \ - "SELECT hostname, avg(usage_user) FROM m GROUP BY hostname" -pgc_summary diff --git a/test/zz_probe_369d.sh b/test/zz_probe_369d.sh deleted file mode 100755 index cb31566..0000000 --- a/test/zz_probe_369d.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# TEMPORARY PROBE. What does the parallel arm cost when the planner is not allowed to -# decline it? Reproducing the wrong CHOICE is only half the case; the other half is -# whether the declined plan is actually better. Requires the throwaway build whose -# add_path for the serial node is behind PGC369_NO_SERIAL. -set -uo pipefail -. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -pgc_setup "${1:-/usr/local/pgsql/bin/pg_config}" - -ROWS=${PGC_369_ROWS:-20000000} -psql_run "CREATE TABLE m (time timestamptz, hostname text, - usage_user float8, usage_system float8, usage_idle float8, - usage_nice float8, usage_iowait float8, usage_irq float8, - usage_softirq float8, usage_steal float8, usage_guest float8, - usage_guest_nice float8) USING pgcolumnar; - INSERT INTO m SELECT - '2026-01-01'::timestamptz + (g * interval '2160 microseconds'), - 'host_' || (g % 4000), - random()*100, random()*100, random()*100, random()*100, - random()*100, random()*100, random()*100, random()*100, - random()*100, random()*100 - FROM generate_series(1,$ROWS) g; - ANALYZE m;" -got=$(q "SELECT count(*) FROM m") -[ "$got" = "$ROWS" ] || { echo "ABORT: fixture has [$got] rows"; pgc_summary; exit 1; } -echo "-- fixture: $got rows, $(q "SELECT pg_size_pretty(pg_total_relation_size('m'))")" - -G="SET pgcolumnar.enable_group_vectorization=on; - SET pgcolumnar.enable_parallel_vector_agg=on; - SET max_parallel_workers_per_gather=4;" - -Q1="SELECT date_trunc('minute', time) AS b, avg(usage_user) FROM m - WHERE time >= '2026-01-01' AND time < '2026-01-01 12:00' GROUP BY b" -Q2="SELECT date_trunc('minute', time) AS b, avg(usage_user), avg(usage_system), - avg(usage_idle), avg(usage_nice), avg(usage_iowait), avg(usage_irq), - avg(usage_softirq), avg(usage_steal), avg(usage_guest), avg(usage_guest_nice) - FROM m WHERE time >= '2026-01-01' AND time < '2026-01-01 12:00' GROUP BY b" - -# The suppression is read by the BACKEND, so it has to be in the server's environment. -# Setting it in this shell would do nothing. Restart the cluster with it set. -restart_with() { # $1 = "" or "1" - pgc_pg "pg_ctl -D '$PGC_PGDATA' -m fast stop" >/dev/null 2>&1 - if [ -n "$1" ]; then - pgc_pg "PGC369_NO_SERIAL=1 pg_ctl -D '$PGC_PGDATA' -l '$PGC_LOGFILE' start -w" >/dev/null 2>&1 - else - pgc_pg "pg_ctl -D '$PGC_PGDATA' -l '$PGC_LOGFILE' start -w" >/dev/null 2>&1 - fi - sleep 2 -} -t() { local s e; s=$(date +%s%N); psql_run "$1" >/dev/null 2>&1; e=$(date +%s%N); echo $(( (e-s)/1000000 )); } -arm() { # -> which node the plan uses - local p; p=$(psql_run "$G EXPLAIN (COSTS OFF) $1" 2>&1) - if grep -q "Parallel Custom Scan" <<<"$p"; then echo "PARALLEL arm" - elif grep -q "Columnar Vectorized Group Keys" <<<"$p"; then echo "serial node" - else echo "core"; fi -} - -for mode in "" "1"; do - restart_with "$mode" - label=$([ -n "$mode" ] && echo "serial node SUPPRESSED" || echo "as shipped") - echo - echo "=== $label ================================================" - # assert the premise: the suppression must actually change the plan - echo " G1 plan: $(arm "$Q1") G2 plan: $(arm "$Q2")" - echo " G1 time: $(t "$G $Q1") ms" - echo " G2 time: $(t "$G $Q2") ms" -done -pgc_summary diff --git a/test/zz_probe_369e.sh b/test/zz_probe_369e.sh deleted file mode 100755 index 1502912..0000000 --- a/test/zz_probe_369e.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash -# TEMPORARY PROBE. The single-run comparison said forcing the parallel arm makes G1 and -# G2 SLOWER, which contradicts #369 and would cancel planned work. A finding that says -# "do not build this" has to be worth more than one measurement each, so: three runs per -# shape per arm, all times reported, plus a warm-up that is discarded. -# -# G3 is the control. Parallelism demonstrably works on this box there, so a slow parallel -# arm on G1/G2 cannot be dismissed as "this container cannot parallelise". -set -uo pipefail -. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -pgc_setup "${1:-/usr/local/pgsql/bin/pg_config}" - -ROWS=${PGC_369_ROWS:-20000000} -psql_run "CREATE TABLE m (time timestamptz, hostname text, - usage_user float8, usage_system float8, usage_idle float8, - usage_nice float8, usage_iowait float8, usage_irq float8, - usage_softirq float8, usage_steal float8, usage_guest float8, - usage_guest_nice float8) USING pgcolumnar; - INSERT INTO m SELECT - '2026-01-01'::timestamptz + (g * interval '2160 microseconds'), - 'host_' || (g % 4000), - random()*100, random()*100, random()*100, random()*100, - random()*100, random()*100, random()*100, random()*100, - random()*100, random()*100 - FROM generate_series(1,$ROWS) g; - ANALYZE m;" -got=$(q "SELECT count(*) FROM m") -[ "$got" = "$ROWS" ] || { echo "ABORT: fixture has [$got] rows"; pgc_summary; exit 1; } -echo "-- fixture: $got rows, $(q "SELECT pg_size_pretty(pg_total_relation_size('m'))")" -echo "-- cores visible to the server: $(nproc)" - -G="SET pgcolumnar.enable_group_vectorization=on; - SET pgcolumnar.enable_parallel_vector_agg=on; - SET max_parallel_workers_per_gather=4;" - -Q1="SELECT date_trunc('minute', time) AS b, avg(usage_user) FROM m - WHERE time >= '2026-01-01' AND time < '2026-01-01 12:00' GROUP BY b" -Q2="SELECT date_trunc('minute', time) AS b, avg(usage_user), avg(usage_system), - avg(usage_idle), avg(usage_nice), avg(usage_iowait), avg(usage_irq), - avg(usage_softirq), avg(usage_steal), avg(usage_guest), avg(usage_guest_nice) - FROM m WHERE time >= '2026-01-01' AND time < '2026-01-01 12:00' GROUP BY b" -Q3="SELECT hostname, avg(usage_user) FROM m GROUP BY hostname" - -restart_with() { - pgc_pg "pg_ctl -D '$PGC_PGDATA' -m fast stop" >/dev/null 2>&1 - if [ -n "$1" ]; then - pgc_pg "PGC369_NO_SERIAL=1 pg_ctl -D '$PGC_PGDATA' -l '$PGC_LOGFILE' start -w" >/dev/null 2>&1 - else - pgc_pg "pg_ctl -D '$PGC_PGDATA' -l '$PGC_LOGFILE' start -w" >/dev/null 2>&1 - fi - sleep 2 -} -t() { local s e; s=$(date +%s%N); psql_run "$1" >/dev/null 2>&1; e=$(date +%s%N); echo $(( (e-s)/1000000 )); } -arm() { - local p; p=$(psql_run "$G EXPLAIN (COSTS OFF) $1" 2>&1) - if grep -q "Parallel Custom Scan" <<<"$p"; then echo "PARALLEL" - elif grep -q "Columnar Vectorized Group Keys" <<<"$p"; then echo "serial" - else echo "core"; fi -} -runs() { # $1 query -> "warm: a b c" - t "$G $1" >/dev/null # discard the cold run - echo "$(t "$G $1") $(t "$G $1") $(t "$G $1")" -} - -for mode in "" "1"; do - restart_with "$mode" - label=$([ -n "$mode" ] && echo "serial node SUPPRESSED" || echo "as shipped") - echo - echo "=== $label ===============================================" - for n in 1 2 3; do - case $n in 1) Q="$Q1";; 2) Q="$Q2";; 3) Q="$Q3";; esac - printf ' G%s plan=%-9s runs(ms)= %s\n' "$n" "$(arm "$Q")" "$(runs "$Q")" - done -done -pgc_summary diff --git a/test/zz_probe_369f.sh b/test/zz_probe_369f.sh deleted file mode 100755 index 186ab2b..0000000 --- a/test/zz_probe_369f.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -# TEMPORARY PROBE. Forced onto the parallel arm, G1 runs 2x SLOWER than the serial node, -# and 8.5x slower than G3's parallel plan over the same 20M rows and the same number of -# columns. Either the workers are not dividing the work, or they are each doing all of -# it. EXPLAIN (ANALYZE) says which. -# -# 5M rows: the question is the ratio and the per-worker row counts, not the absolute time. -set -uo pipefail -. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -pgc_setup "${1:-/usr/local/pgsql/bin/pg_config}" - -ROWS=${PGC_369_ROWS:-5000000} -psql_run "CREATE TABLE m (time timestamptz, hostname text, - usage_user float8, usage_system float8) USING pgcolumnar; - INSERT INTO m SELECT - '2026-01-01'::timestamptz + (g * interval '8640 microseconds'), - 'host_' || (g % 4000), random()*100, random()*100 - FROM generate_series(1,$ROWS) g; - ANALYZE m;" -got=$(q "SELECT count(*) FROM m") -[ "$got" = "$ROWS" ] || { echo "ABORT: fixture has [$got] rows"; pgc_summary; exit 1; } -echo "-- fixture: $got rows, $(q "SELECT pg_size_pretty(pg_total_relation_size('m'))"), cores=$(nproc)" - -G="SET pgcolumnar.enable_group_vectorization=on; - SET pgcolumnar.enable_parallel_vector_agg=on; - SET max_parallel_workers_per_gather=4;" -Q1="SELECT date_trunc('minute', time) AS b, avg(usage_user) FROM m - WHERE time >= '2026-01-01' AND time < '2026-01-01 12:00' GROUP BY b" -Q3="SELECT hostname, avg(usage_user) FROM m GROUP BY hostname" - -restart_with() { - pgc_pg "pg_ctl -D '$PGC_PGDATA' -m fast stop" >/dev/null 2>&1 - if [ -n "$1" ]; then - pgc_pg "PGC369_NO_SERIAL=1 pg_ctl -D '$PGC_PGDATA' -l '$PGC_LOGFILE' start -w" >/dev/null 2>&1 - else - pgc_pg "pg_ctl -D '$PGC_PGDATA' -l '$PGC_LOGFILE' start -w" >/dev/null 2>&1 - fi - sleep 2 -} -ea() { # $1 label, $2 query - echo - echo "--- $1 ---" - psql_run "$G EXPLAIN (ANALYZE, TIMING ON, VERBOSE OFF) $2" 2>&1 | - grep -iE "Custom Scan|HashAggregate|Gather|Workers|actual time|rows=|Execution Time" | - sed 's/^[[:space:]]*/ /' | head -14 -} - -restart_with "" -ea "G1 as shipped (serial node expected)" "$Q1" -ea "G3 as shipped (parallel arm expected)" "$Q3" -restart_with "1" -ea "G1 with the serial node suppressed (parallel arm forced)" "$Q1" -pgc_summary diff --git a/test/zz_probe_369g.sh b/test/zz_probe_369g.sh deleted file mode 100755 index 50204cd..0000000 --- a/test/zz_probe_369g.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -# TEMPORARY PROBE. The decisive run for #369. -# -# At 5M rows, suppressing the serial node did NOT select our parallel arm: the planner -# fell back to core's HashAggregate. So a "forced parallel" timing is only evidence if -# the plan is inspected, not inferred from the suppression. At 20M the classifier did -# report Parallel Custom Scan, and this run proves it with EXPLAIN (ANALYZE) and shows -# Workers Launched and the per-worker row counts, which is what explains WHY our arm is -# slower than our serial node on this shape. -set -uo pipefail -. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -pgc_setup "${1:-/usr/local/pgsql/bin/pg_config}" - -ROWS=${PGC_369_ROWS:-20000000} -psql_run "CREATE TABLE m (time timestamptz, hostname text, - usage_user float8, usage_system float8, usage_idle float8, - usage_nice float8, usage_iowait float8, usage_irq float8, - usage_softirq float8, usage_steal float8, usage_guest float8, - usage_guest_nice float8) USING pgcolumnar; - INSERT INTO m SELECT - '2026-01-01'::timestamptz + (g * interval '2160 microseconds'), - 'host_' || (g % 4000), - random()*100, random()*100, random()*100, random()*100, - random()*100, random()*100, random()*100, random()*100, - random()*100, random()*100 - FROM generate_series(1,$ROWS) g; - ANALYZE m;" -got=$(q "SELECT count(*) FROM m") -[ "$got" = "$ROWS" ] || { echo "ABORT: fixture has [$got] rows"; pgc_summary; exit 1; } -echo "-- fixture: $got rows, $(q "SELECT pg_size_pretty(pg_total_relation_size('m'))"), cores=$(nproc)" - -G="SET pgcolumnar.enable_group_vectorization=on; - SET pgcolumnar.enable_parallel_vector_agg=on; - SET max_parallel_workers_per_gather=4;" -Q1="SELECT date_trunc('minute', time) AS b, avg(usage_user) FROM m - WHERE time >= '2026-01-01' AND time < '2026-01-01 12:00' GROUP BY b" -Q3="SELECT hostname, avg(usage_user) FROM m GROUP BY hostname" - -restart_with() { - pgc_pg "pg_ctl -D '$PGC_PGDATA' -m fast stop" >/dev/null 2>&1 - if [ -n "$1" ]; then - pgc_pg "PGC369_NO_SERIAL=1 pg_ctl -D '$PGC_PGDATA' -l '$PGC_LOGFILE' start -w" >/dev/null 2>&1 - else - pgc_pg "pg_ctl -D '$PGC_PGDATA' -l '$PGC_LOGFILE' start -w" >/dev/null 2>&1 - fi - sleep 2 -} -ea() { # $1 label, $2 query - echo - echo "--- $1 ---" - psql_run "$G $2" >/dev/null 2>&1 # warm - psql_run "$G EXPLAIN (ANALYZE, TIMING ON) $2" 2>&1 | - grep -iE "Custom Scan|HashAggregate|Gather|Workers|Vectorized|Execution Time" | - sed 's/^[[:space:]]*/ /' | head -14 -} - -restart_with "" -ea "G1 as shipped" "$Q1" -ea "G3 as shipped (the arm working correctly, for contrast)" "$Q3" -restart_with "1" -ea "G1, serial node suppressed" "$Q1" -ea "G3, serial node suppressed" "$Q3" -pgc_summary