From 14fd203474a835874341ce7d8d0301241d7e8fe7 Mon Sep 17 00:00:00 2001 From: "Joshua (D) Drake" <136637981+ChronicallyJD@users.noreply.github.com> Date: Sun, 2 Aug 2026 15:24:23 -0600 Subject: [PATCH 1/3] feat: ungrouped vectorized aggregate for filtered and extended shapes (#289) An ungrouped aggregate with a WHERE filter, or a sum/avg over int8/float/numeric, is answerable from no zone map, so it fell to the row-wise core Agg: about 244 ns/row against heap's 48 ns/row on the TSBS q6 shape (the measurement on #289). Give it a dedicated single-pass scan-fold node, the ungrouped sibling of the grouped path #321 built. pgcolumnar.enable_ungrouped_vector_agg (default off) routes such a query to columnar_native_scan_agg, generalized: it builds scan keys from the WHERE for group and vector pruning, rechecks the whole WHERE per row (the keys only prune), and folds every surviving row through columnar_apply_one. That is the same reference fold the grouped and metadata paths use, applied in scan order, so the result is byte-for-byte what core Agg returns, floats included. The zone-map metadata path (count, min, max, and sum/avg over int2/int4 with no filter) is untouched. With the GUC off the behavior is exactly as before. test/ungrouped_vector_agg.sh: 27 checks, on==off across filtered and unfiltered float/int8/numeric sum/avg, min/max, nulls, empty result, an all-null column, and after deletes; it asserts via EXPLAIN that the new node actually runs, so the A/B is never vacuous. Passes on PG18 and PG19; the existing agg suites (native_agg, native_groupagg, deletes, add-column, group rewrite) still pass. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01UX1jrWiQsJJA1t4pkmkb4T --- src/columnar.h | 1 + src/columnar_tableam.c | 11 ++ src/columnar_vector.c | 296 +++++++++++++++++++++++++++++++---- test/ungrouped_vector_agg.sh | 116 ++++++++++++++ 4 files changed, 392 insertions(+), 32 deletions(-) create mode 100644 test/ungrouped_vector_agg.sh diff --git a/src/columnar.h b/src/columnar.h index 81b516e..2b857ad 100644 --- a/src/columnar.h +++ b/src/columnar.h @@ -174,6 +174,7 @@ extern bool columnar_enable_bloom_filter; /* bloom equality skipping (I7) */ /* Phase 6 GUCs (spec 8.3) */ extern bool columnar_enable_vectorization; /* vectorized aggregate path */ extern bool columnar_enable_group_vectorization; /* GROUP BY vectorized agg (#289) */ +extern bool columnar_enable_ungrouped_vector_agg; /* filtered/extended ungrouped agg (#289) */ extern int columnar_groupagg_max_groups; /* plan-time group-count cap (#289) */ extern bool columnar_enable_read_stream; /* stream/prefetch block reads (PG17+) */ extern bool columnar_enable_index_only_scan; /* allow index-only scans (gap 28) */ diff --git a/src/columnar_tableam.c b/src/columnar_tableam.c index 83524a2..1f6194e 100644 --- a/src/columnar_tableam.c +++ b/src/columnar_tableam.c @@ -2254,6 +2254,17 @@ _PG_init(void) 0, NULL, NULL, NULL); + DefineCustomBoolVariable("pgcolumnar.enable_ungrouped_vector_agg", + "Use the vectorized aggregate fast path for ungrouped " + "aggregates with a filter or sum/avg over " + "int8/float/numeric.", + NULL, + &columnar_enable_ungrouped_vector_agg, + false, + PGC_USERSET, + 0, + NULL, NULL, NULL); + DefineCustomIntVariable("pgcolumnar.groupagg_max_groups", "Cap on the actual group count the grouped vectorized " "aggregate builds before it stops with an error.", diff --git a/src/columnar_vector.c b/src/columnar_vector.c index 60d1096..b9e7430 100644 --- a/src/columnar_vector.c +++ b/src/columnar_vector.c @@ -88,6 +88,15 @@ bool columnar_enable_vectorization = true; bool columnar_enable_group_vectorization = false; int columnar_groupagg_max_groups = 1000000; +/* + * GUC: extend the ungrouped vectorized aggregate to the shapes a zone map cannot + * answer -- an aggregate with a WHERE filter, or sum/avg over int8/float/numeric + * (#289). Those fall to the row-wise core Agg today; this folds them in one pass + * over the columnar scan instead. Default off while the path is proven and + * benchmarked; the metadata-answerable no-filter path is unaffected. + */ +bool columnar_enable_ungrouped_vector_agg = false; + /* ------------------------------------------------------------------------- * shared column-at-a-time filter * ------------------------------------------------------------------------- */ @@ -229,9 +238,9 @@ typedef enum ColumnarAggKind COLUMNAR_AGG_MIN, COLUMNAR_AGG_MAX, /* - * Extended kinds used by the grouped path (#289). The ungrouped - * metadata-fold path never produces these (its classifier still rejects - * int8/float/numeric sum/avg), so its switches never see them. + * Extended kinds used by the grouped path and the ungrouped scan-fold path + * (#289). The metadata-fold path (columnar_fill_native_metadata_agg) still + * never produces these: a query using one is routed to scan-fold instead. */ COLUMNAR_AGG_SUM_INT8, /* sum(int8) -> numeric */ COLUMNAR_AGG_SUM_FLOAT, /* sum(float4/float8) -> float8 */ @@ -520,6 +529,17 @@ typedef struct ColumnarAggScanState MemoryContext resultContext; /* holds min/max running values */ bool done; /* the single result row was emitted */ + /* + * Scan-fold mode (#289): the aggregate has a WHERE filter, or a sum/avg a + * zone map cannot answer, so the node scans and folds every surviving row + * instead of answering from metadata. NULL whereState means no residual + * recheck (a pure extended aggregate with no filter). + */ + bool scanFold; + Bitmapset *projected; /* base columns the scan must return */ + TupleTableSlot *baseSlot; /* holds each read row for the qual recheck */ + ExprState *whereState; /* residual WHERE recheck, or NULL */ + /* chunk-group skip counters captured for EXPLAIN */ bool haveStats; uint64 groupsRead; @@ -639,6 +659,37 @@ static const CustomPathMethods columnar_agg_path_methods = { static create_upper_paths_hook_type prev_create_upper_paths_hook = NULL; +/* + * columnar_agg_metadata_answerable + * True when this aggregate kind is answerable from whole-chunk zone maps + * with no data scan: count, count(col), sum/avg over int2/int4 (the zone + * stores an int sum), min and max. The extended kinds (sum/avg over + * int8/float/numeric) are not, so a query using one must scan and fold. See + * columnar_fill_native_metadata_agg. + */ +static bool +columnar_agg_metadata_answerable(ColumnarAggKind kind) +{ + switch (kind) + { + case COLUMNAR_AGG_COUNT_STAR: + case COLUMNAR_AGG_COUNT_COL: + case COLUMNAR_AGG_SUM_INT: + case COLUMNAR_AGG_AVG_INT: + case COLUMNAR_AGG_MIN: + case COLUMNAR_AGG_MAX: + return true; + case COLUMNAR_AGG_SUM_INT8: + case COLUMNAR_AGG_SUM_FLOAT: + case COLUMNAR_AGG_SUM_NUMERIC: + case COLUMNAR_AGG_AVG_INT8: + case COLUMNAR_AGG_AVG_FLOAT: + case COLUMNAR_AGG_AVG_NUMERIC: + return false; + } + return false; +} + /* * ColumnarCreateUpperPaths * create_upper_paths_hook: for a plain SELECT agg(col) FROM columnar_table @@ -663,6 +714,7 @@ ColumnarCreateUpperPaths(PlannerInfo *root, UpperRelationKind stage, List *quals; int npreds; bool allConvertible; + bool needsScan; Path *cheapest; CustomPath *cpath; @@ -729,26 +781,62 @@ ColumnarCreateUpperPaths(PlannerInfo *root, UpperRelationKind stage, if (!IsA(expr, Aggref)) return; if (!columnar_classify_aggref((Aggref *) expr, (int) input_rel->relid, - false, &specs[i])) + true, &specs[i])) return; i++; } /* - * The ungrouped path supports no residual filter: collect any restriction - * clauses so that with any WHERE we fall back to the ordinary Agg. + * The whole WHERE, rechecked per row in scan-fold mode; NIL with no filter. + * A native table answers an ungrouped aggregate from its zone maps with no + * data scan (native spec 7.1), but only with no filter and every aggregate + * zone-map answerable. A filter, or a sum/avg over int8/float/numeric, needs + * the scan-fold path instead (#289). */ quals = extract_actual_clauses(input_rel->baserestrictinfo, false); - /* - * A native table answers an ungrouped aggregate from its zone maps without a - * data scan (native spec 7.1), but only when there is no residual filter; with - * a filter, fall back to the ordinary Agg over the skipping-enabled custom - * scan. - */ - if (quals != NIL) - return; + needsScan = (quals != NIL); + for (i = 0; i < naggs; i++) + if (!columnar_agg_metadata_answerable(specs[i].kind)) + needsScan = true; + if (needsScan) + { + /* + * Scan-fold path (#289): the ungrouped sibling of the grouped path. It + * scans every surviving row once and folds it, so it handles a filter and + * the extended sum/avg kinds a zone map cannot. Opt-in while it is proven. + */ + ListCell *rc; + Bitmapset *whereAtts = NULL; + int m = -1; + + if (!columnar_enable_ungrouped_vector_agg) + return; + + /* + * A pseudoconstant (gating) qual is a one-time filter, not a per-row + * predicate; the recheck this path relies on runs per row and would never + * apply it, so a false gate would wrongly return a row. Rare; fall back. + * (Mirrors the grouped path.) + */ + foreach(rc, input_rel->baserestrictinfo) + if (lfirst_node(RestrictInfo, rc)->pseudoconstant) + return; + + /* + * A WHERE on a system column or a whole-row Var cannot be evaluated from + * the projected data columns the recheck reads; fall back rather than + * evaluate it against unset slot values. + */ + pull_varattnos((Node *) quals, input_rel->relid, &whereAtts); + while ((m = bms_next_member(whereAtts, m)) >= 0) + if (m + FirstLowInvalidHeapAttributeNumber <= 0) + return; + + npreds = 0; /* the EXPLAIN count is filled at execution */ + } + else { Relation rel = table_open(relid, AccessShareLock); TupleDesc tupdesc = RelationGetDescr(rel); @@ -756,9 +844,9 @@ ColumnarCreateUpperPaths(PlannerInfo *root, UpperRelationKind stage, ColumnarCountConvertibleQuals(quals, input_rel->relid, tupdesc, &npreds, &allConvertible); table_close(rel, AccessShareLock); + if (!allConvertible) + return; } - if (!allConvertible) - return; cheapest = input_rel->cheapest_total_path; if (cheapest == NULL) @@ -794,6 +882,34 @@ ColumnarCreateUpperPaths(PlannerInfo *root, UpperRelationKind stage, * execution asks; if the answer changes between planning and execution the * plan is mispriced, never wrong. */ + if (needsScan) + { + /* + * A full columnar scan, priced from the cheapest non-index path (an index + * scan cannot serve this node), plus a token per-row bump so this opt-in + * accelerator is chosen over the ordinary Agg-over-scan when it is enabled. + */ + Path *scanp = NULL; + ListCell *pc; + Cost cost; + + foreach(pc, input_rel->pathlist) + { + Path *p = (Path *) lfirst(pc); + + if (p->pathtype == T_IndexScan || p->pathtype == T_IndexOnlyScan || + p->pathtype == T_BitmapHeapScan) + continue; + if (scanp == NULL || p->total_cost < scanp->total_cost) + scanp = p; + } + if (scanp == NULL) + scanp = cheapest; + cost = scanp->total_cost + cpu_tuple_cost; + cpath->path.startup_cost = cost; + cpath->path.total_cost = cost; + } + else { ColumnarOptions opts; int limit = columnar_stripe_row_limit; @@ -1142,11 +1258,21 @@ ColumnarCreateAggScanState(CustomScan *cscan) TargetEntry *tle = (TargetEntry *) lfirst(lc); /* classified successfully at plan time; -1 skips the varno check */ - (void) columnar_classify_aggref((Aggref *) tle->expr, -1, false, + (void) columnar_classify_aggref((Aggref *) tle->expr, -1, true, &state->specs[i]); i++; } + /* + * Scan-fold mode (#289) matches the planner's routing: a residual filter, or + * any aggregate a zone map cannot answer. The metadata-answerable no-filter + * case keeps the zone-map path in ColumnarExecAggScan. + */ + state->scanFold = (state->quals != NIL); + for (i = 0; i < naggs; i++) + if (!columnar_agg_metadata_answerable(state->specs[i].kind)) + state->scanFold = true; + return (Node *) state; } @@ -1216,6 +1342,40 @@ ColumnarBeginAggScan(CustomScanState *node, EState *estate, int eflags) ColumnarCountConvertibleQuals(state->quals, state->scanrelid, tupdesc, &state->npreds, &allConvertible); + /* + * Scan-fold mode reads and rechecks every surviving row (#289): a virtual + * base slot to hold each row, the residual WHERE recheck (scan keys only + * prune groups and vectors), and the set of columns the aggregates and the + * WHERE reference so the reader returns them. + */ + if (state->scanFold) + { + Bitmapset *proj = NULL; + int x; + + state->baseSlot = MakeSingleTupleTableSlot(CreateTupleDescCopy(tupdesc), + &TTSOpsVirtual); + state->whereState = (state->quals != NIL) + ? ExecInitQual(state->quals, &node->ss.ps) + : NULL; + + pull_varattnos((Node *) state->quals, state->scanrelid, &proj); + x = -1; + while ((x = bms_next_member(proj, x)) >= 0) + { + AttrNumber attno = x + FirstLowInvalidHeapAttributeNumber; + + if (attno > 0) + state->projected = bms_add_member(state->projected, attno - 1); + } + for (a = 0; a < state->naggs; a++) + if (state->specs[a].attidx >= 0) + state->projected = bms_add_member(state->projected, + state->specs[a].attidx); + if (state->projected == NULL) + state->projected = bms_make_singleton(0); + } + table_close(rel, AccessShareLock); } @@ -1852,29 +2012,75 @@ columnar_native_scan_agg(ColumnarAggScanState *state, const uint64 *restrictGroups, int nRestrictGroups) { EState *estate = state->css.ss.ps.state; + ExprContext *econtext = state->css.ss.ps.ps_ExprContext; Relation rel = table_open(state->relid, AccessShareLock); TupleDesc tupdesc = RelationGetDescr(rel); - Bitmapset *projected = NULL; + Bitmapset *projected; ColumnarReadState *rs; + ScanKey keys = NULL; + int nScanKeys = 0; Datum *values = (Datum *) palloc(sizeof(Datum) * tupdesc->natts); bool *nulls = (bool *) palloc(sizeof(bool) * tupdesc->natts); uint64 rowNumber; int a; - for (a = 0; a < state->naggs; a++) - if (state->specs[a].attidx >= 0) - projected = bms_add_member(projected, state->specs[a].attidx); - if (projected == NULL) - projected = bms_make_singleton(0); /* count(*) only: one column */ + /* + * Scan-fold mode carries the projected set (aggregate plus WHERE columns) and + * a base slot for the per-row recheck; the dirty-groups metadata path passes + * neither and projects only the aggregate columns. + */ + if (state->scanFold) + projected = state->projected; + else + { + projected = NULL; + for (a = 0; a < state->naggs; a++) + if (state->specs[a].attidx >= 0) + projected = bms_add_member(projected, state->specs[a].attidx); + if (projected == NULL) + projected = bms_make_singleton(0); /* count(*) only: one column */ + } ColumnarFlushWriteStateForRelation(state->relid); ColumnarFlushDeleteVectorForRelation(rel); - rs = ColumnarBeginRead(rel, estate->es_snapshot, NULL, projected, 0, NULL); + /* push the WHERE down for group and vector pruning; the recheck is exact */ + if (state->scanFold && state->quals != NIL) + keys = ColumnarBuildScanKeys(state->quals, state->scanrelid, tupdesc, + &nScanKeys); + + rs = ColumnarBeginRead(rel, estate->es_snapshot, NULL, projected, + nScanKeys, keys); if (restrictGroups != NULL) ColumnarReadRestrictToGroups(rs, restrictGroups, nRestrictGroups); + + /* columns outside the projection stay null in the recheck slot */ + if (state->whereState != NULL) + memset(state->baseSlot->tts_isnull, true, sizeof(bool) * tupdesc->natts); + while (ColumnarReadNextRow(rs, values, nulls, &rowNumber)) { + /* + * Recheck the whole WHERE per row: the scan keys only prune groups and + * vectors, so a surviving row may still fail the predicate. + */ + if (state->whereState != NULL) + { + int x = -1; + + ResetExprContext(econtext); + ExecClearTuple(state->baseSlot); + while ((x = bms_next_member(state->projected, x)) >= 0) + { + state->baseSlot->tts_values[x] = values[x]; + state->baseSlot->tts_isnull[x] = nulls[x]; + } + ExecStoreVirtualTuple(state->baseSlot); + econtext->ecxt_scantuple = state->baseSlot; + if (!ExecQual(state->whereState, econtext)) + continue; + } + for (a = 0; a < state->naggs; a++) { ColumnarAggSpec *spec = &state->specs[a]; @@ -1886,6 +2092,14 @@ columnar_native_scan_agg(ColumnarAggScanState *state, columnar_apply_one(state->resultContext, spec, (Datum) 0, true); } } + + if (state->scanFold) + { + ColumnarReadStats(rs, &state->groupsRead, &state->groupsSkipped, + &state->groupsTotal); + state->haveStats = true; + } + ColumnarEndRead(rs); table_close(rel, AccessShareLock); } @@ -1923,10 +2137,22 @@ ColumnarExecAggScan(CustomScanState *node) ColumnarFlushDeleteVectorForRelation(frel); table_close(frel, AccessShareLock); - dirtyGroups = columnar_fill_native_metadata_agg(state, &nDirtyGroups); - if (nDirtyGroups > 0) - columnar_native_scan_agg(state, dirtyGroups, nDirtyGroups); - state->haveStats = false; + if (state->scanFold) + { + /* + * A filter, or a sum/avg no zone map answers (#289): scan every row once + * and fold it. columnar_native_scan_agg builds the scan keys, rechecks the + * WHERE, and captures the EXPLAIN stats. + */ + columnar_native_scan_agg(state, NULL, 0); + } + else + { + dirtyGroups = columnar_fill_native_metadata_agg(state, &nDirtyGroups); + if (nDirtyGroups > 0) + columnar_native_scan_agg(state, dirtyGroups, nDirtyGroups); + state->haveStats = false; + } /* build the single result row from the finalized aggregates */ ExecClearTuple(scanSlot); @@ -1954,7 +2180,13 @@ ColumnarExecAggScan(CustomScanState *node) static void ColumnarEndAggScan(CustomScanState *node) { - /* the reader is ended inside ColumnarExecAggScan; nothing else to free */ + ColumnarAggScanState *state = (ColumnarAggScanState *) node; + + if (state->baseSlot != NULL) + ExecDropSingleTupleTableSlot(state->baseSlot); + state->baseSlot = NULL; + /* the reader is ended inside ColumnarExecAggScan; the memory contexts are + * children of es_query_cxt and freed with it */ } static void @@ -1977,9 +2209,9 @@ ColumnarReScanAggScan(CustomScanState *node) /* * Also clear the running float/numeric accumulators. resultContext was * just reset, so nsum's storage is gone; leaving nsumSet true would make - * the next scan add to (and free) a dangling pointer. The ungrouped path - * does not classify these extended kinds today, but reset every field so a - * rescan is safe if it ever does. + * the next scan add to (and free) a dangling pointer. Scan-fold mode + * classifies these extended kinds (#289), so this reset is load-bearing on + * a rescan, not only defensive. */ spec->fsum = 0; spec->fsxx = 0; diff --git a/test/ungrouped_vector_agg.sh b/test/ungrouped_vector_agg.sh new file mode 100644 index 0000000..430ac2f --- /dev/null +++ b/test/ungrouped_vector_agg.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# +# pgColumnar ungrouped vectorized aggregate (#289). +# +# pgcolumnar.enable_ungrouped_vector_agg routes an ungrouped aggregate that a +# zone map cannot answer -- one with a WHERE filter, or sum/avg over +# int8/float/numeric -- to a single-pass scan-fold node instead of the row-wise +# core Agg. This suite proves the new path returns byte-for-byte what core Agg +# returns (the fold reuses columnar_apply_one in scan order, so floats match +# exactly), across types, nulls, filters, empty and all-null inputs; and it +# ASSERTS THE PREMISE that the new path actually runs with the GUC on and does +# not with it off, so the A/B is never vacuous. +# +# Usage: test/ungrouped_vector_agg.sh [PG_CONFIG] +# Written fresh for pgColumnar. + +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" + +GUC=pgcolumnar.enable_ungrouped_vector_agg + +# Pin serial execution in both arms. The new node is serial; a parallel core Agg +# would sum floats in a different order, so an unpinned oracle could differ from +# the serial fold by float reassociation -- a test artifact, not a code defect. +NOPAR="SET max_parallel_workers_per_gather=0" + +# run a scalar query and echo its value, at a given setting of the new GUC +val() { # val + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -Atq -c "$NOPAR" -c "SET $GUC=$1" -c "$2" 2>&1 +} +# echo the plan text at a given setting +plan() { # plan + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -Atq -c "$NOPAR" -c "SET $GUC=$1" -c "EXPLAIN (COSTS OFF) $2" 2>&1 +} +# oracle A/B: the new path (on) must equal core Agg (off), byte for byte +ab() { # ab