diff --git a/CHANGELOG.md b/CHANGELOG.md index f6a0c8a..43dd9d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,20 @@ which was true until that script existed. ### Added +- `EXPLAIN (ANALYZE)` now reports `Columnar Usable Skip Predicates` beside + `Columnar Pushed-Down Filters` (#479). The existing line counts the filters the + scan was handed and is unchanged; the new one counts how many of those the + reader can actually skip chunk groups with. A filter whose types have no + ordering function for the pair is dropped by the reader and excludes nothing, + and until now the plan reported it as pushed down with no way to see the + difference. That is how #477 went unseen for a year, and how + `test/zonemap_cost.sh` validated a cost discount against a fixture that pruned + zero groups. + + All three nodes that print the original line report the new one: the scalar + custom scan and both vectorized aggregate nodes. The new line needs `ANALYZE`, + since it describes what the scan built at execution. + - `pgcolumnar.analyze()` now collects `most_common_vals` and `most_common_freqs`, and excludes those values from `histogram_bounds` (#414). Frequencies are exact counts over the total row count rather than sample estimates. PostgreSQL 18 and diff --git a/docs/user-guide.md b/docs/user-guide.md index 2cd9106..3e853b5 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -136,6 +136,36 @@ controlled by a setting in the [Configuration reference](configuration.md): avg, min, or max on a supported type. - `count(*)` answered from catalog metadata when there is no filter. +#### Reading the filter counters + +`EXPLAIN (ANALYZE)` reports two counters for filters. They answer different +questions and are read together: + +- `Columnar Pushed-Down Filters` is how many filters the scan was given. It + follows the `pgcolumnar.enable_qual_pushdown` setting. +- `Columnar Usable Skip Predicates` is how many of those the scan can skip chunk + groups with. A filter is not usable when the two types being compared have no + ordering function for that pair. The scan then applies the filter to each row + and returns the same result, after it reads every group. + +``` +Columnar Pushed-Down Filters: 1 +Columnar Usable Skip Predicates: 0 +Columnar Chunk Groups Removed by Filter: 0 +``` + +That plan read the whole table. The `1` and the `0` mean that the filter reached +the scan and the scan could not use it. That is a different situation from a +usable filter that matches most rows. + +When the two numbers are equal and no groups are removed, the filter is usable. +The values are then spread across every group, so the scan can skip none of them. +Cluster the table on that column with `pgcolumnar.cluster()` to change that +result. See the [SQL reference](sql-reference.md). + +`Columnar Usable Skip Predicates` requires `ANALYZE`, because it reports what the +scan built at execution. A plain `EXPLAIN` shows only the first counter. + ### Point lookups and indexes Create indexes on columnar tables as usual: diff --git a/src/columnar.h b/src/columnar.h index 79cab46..4321973 100644 --- a/src/columnar.h +++ b/src/columnar.h @@ -644,6 +644,17 @@ extern void PgColumnarReadStats(PgColumnarReadState *readState, uint64 *groupsTotal); extern uint64 PgColumnarVectorsSkipped(PgColumnarReadState *readState); +/* + * How many of the scan keys the reader was handed became skip predicates it can + * actually exclude a chunk group with (#479). Never larger than the scan-key + * count EXPLAIN reports as "Columnar Pushed-Down Filters", and smaller whenever + * pgcolumnar_make_predicates dropped a key -- a cross-type pair the opfamily has + * no ordering proc for, a strategy outside BTLess..BTGreater, a null-test or + * row-comparison key. Those keys are still counted as pushed down, and skip + * nothing. + */ +extern int PgColumnarReadUsablePredicates(PgColumnarReadState *readState); + /* cached base-liveness for a projection scan (gap 26): build once per scan, * probe per row with a binary search instead of a per-row catalog scan */ typedef struct PgColumnarLivenessCache PgColumnarLivenessCache; diff --git a/src/columnar_customscan.c b/src/columnar_customscan.c index 4cc2dd9..8d1c2c6 100644 --- a/src/columnar_customscan.c +++ b/src/columnar_customscan.c @@ -1889,6 +1889,27 @@ PgColumnarExplainCustomScan(CustomScanState *node, List *ancestors, PgColumnarReadStats(cstate->readState, &groupsRead, &groupsSkipped, &groupsTotal); + /* + * How many of those filters the reader can actually exclude a chunk + * group with (#479). The line above counts the scan keys the scan was + * GIVEN; pgcolumnar_make_predicates then drops any it cannot evaluate + * against the stored min/max, and a dropped key skips nothing. + * + * Reported separately rather than replacing the line above, because the + * two answer different questions and #191 shows both get asked: that one + * says whether pgcolumnar.enable_qual_pushdown took effect, this one says + * whether the predicates it pushed can prune. A single number cannot say + * both, and saying only the first is how #477 stayed invisible for a + * year -- "Pushed-Down Filters: 1" beside "Chunk Groups Removed by + * Filter: 0" reads as an unselective predicate and meant an unusable one. + * + * No enable_qual_pushdown ternary here: with the setting off the reader + * builds no predicates, so this is already 0. It describes the run. + */ + ExplainPropertyInteger("Columnar Usable Skip Predicates", NULL, + PgColumnarReadUsablePredicates(cstate->readState), + es); + ExplainPropertyInteger("Columnar Chunk Groups Total", NULL, (int64) groupsTotal, es); ExplainPropertyInteger("Columnar Chunk Groups Read", NULL, diff --git a/src/columnar_reader.c b/src/columnar_reader.c index f73bb60..fdc3a46 100644 --- a/src/columnar_reader.c +++ b/src/columnar_reader.c @@ -2969,6 +2969,25 @@ PgColumnarReadStats(PgColumnarReadState *readState, uint64 *groupsRead, *groupsTotal = readState->groupsRead + readState->groupsSkipped; } +/* + * PgColumnarReadUsablePredicates + * How many skip predicates this read state built, which is how many of its + * scan keys can exclude a chunk group. Used by EXPLAIN (#479). + * + * pgcolumnar_make_predicates drops a key it cannot evaluate against the + * stored min/max, and a dropped key excludes nothing -- but it is still + * counted by "Columnar Pushed-Down Filters", which reports the keys the + * scan was given. Reporting only that number is how #477 stayed invisible: + * a bigint column against a bare integer literal read as a pushed-down + * filter that simply was not selective, when in fact no group could ever + * be skipped. + */ +int +PgColumnarReadUsablePredicates(PgColumnarReadState *readState) +{ + return readState->numPredicates; +} + /* * PgColumnarVectorsSkipped * How many 1024-value vectors the native scan skipped within read row groups diff --git a/src/columnar_vector.c b/src/columnar_vector.c index 1fbd2ff..c54ee97 100644 --- a/src/columnar_vector.c +++ b/src/columnar_vector.c @@ -595,6 +595,13 @@ typedef struct PgColumnarAggScanState uint64 groupsRead; uint64 groupsSkipped; uint64 groupsTotal; + + /* + * How many of npreds the reader could actually exclude a group with (#479). + * Captured from the read state beside the counters above, because the read + * state is ended before EXPLAIN runs. Meaningful only when haveStats. + */ + int usablePreds; } PgColumnarAggScanState; static const CustomExecMethods pgcolumnar_agg_exec_methods; @@ -688,6 +695,7 @@ typedef struct PgColumnarGroupAggScanState uint64 groupsRead; uint64 groupsSkipped; uint64 groupsTotal; + int usablePreds; /* of npreds, how many can exclude (#479) */ } PgColumnarGroupAggScanState; static const CustomExecMethods pgcolumnar_groupagg_exec_methods; @@ -3255,6 +3263,7 @@ pgcolumnar_native_batch_fold(PgColumnarAggScanState *state, Relation rel, PgColumnarReadStats(rs, &state->groupsRead, &state->groupsSkipped, &state->groupsTotal); + state->usablePreds = PgColumnarReadUsablePredicates(rs); state->haveStats = true; state->batchFolded = true; PgColumnarEndRead(rs); @@ -3388,6 +3397,7 @@ pgcolumnar_native_scan_agg(PgColumnarAggScanState *state, { PgColumnarReadStats(rs, &state->groupsRead, &state->groupsSkipped, &state->groupsTotal); + state->usablePreds = PgColumnarReadUsablePredicates(rs); state->haveStats = true; } @@ -3532,6 +3542,18 @@ PgColumnarExplainAggScan(CustomScanState *node, List *ancestors, ExplainState *e if (state->haveStats) { + /* + * See PgColumnarExplainCustomScan: npreds above counts the quals that + * became scan keys, this counts the ones the reader can exclude a chunk + * group with, and only the pair distinguishes an unselective predicate + * from an unusable one (#479). This node fills npreds from + * PgColumnarCountConvertibleQuals, which is the same built-key count the + * scalar node reports, so it has the same gap and needs the same second + * number -- otherwise one line of plan text would mean two different + * things depending on which node ran. + */ + ExplainPropertyInteger("Columnar Usable Skip Predicates", NULL, + state->usablePreds, es); ExplainPropertyInteger("Columnar Chunk Groups Total", NULL, (int64) state->groupsTotal, es); ExplainPropertyInteger("Columnar Chunk Groups Read", NULL, @@ -4095,6 +4117,7 @@ pgcolumnar_groupagg_build(PgColumnarGroupAggScanState *state) PgColumnarReadStats(rs, &state->groupsRead, &state->groupsSkipped, &state->groupsTotal); + state->usablePreds = PgColumnarReadUsablePredicates(rs); state->haveStats = true; PgColumnarEndRead(rs); @@ -4208,6 +4231,9 @@ PgColumnarExplainGroupAggScan(CustomScanState *node, List *ancestors, if (state->haveStats) { + /* of those, the ones that can exclude a chunk group (#479) */ + ExplainPropertyInteger("Columnar Usable Skip Predicates", NULL, + state->usablePreds, es); ExplainPropertyInteger("Columnar Chunk Groups Total", NULL, (int64) state->groupsTotal, es); ExplainPropertyInteger("Columnar Chunk Groups Read", NULL, diff --git a/test/pushdown_report.sh b/test/pushdown_report.sh index 026f11a..0954d63 100755 --- a/test/pushdown_report.sh +++ b/test/pushdown_report.sh @@ -125,4 +125,164 @@ two="$(q "SET pgcolumnar.enable_qual_pushdown = on; check "two quals report two pushed-down filters" \ "$(field "$two" 'Columnar Pushed-Down Filters')" "2" +# --- 6. a filter that is pushed down but cannot exclude anything (#479) --------- +# +# "Columnar Pushed-Down Filters" counts the scan keys the reader was HANDED. +# The reader then converts each into a skip predicate and can drop it -- and a +# dropped key excludes no chunk group at all, while the line above still reports +# it as pushed down. That is how #477 stayed invisible for a year: a bigint +# column against a bare integer literal reported +# +# Columnar Pushed-Down Filters: 1 +# Columnar Chunk Groups Removed by Filter: 0 +# +# which reads as "pushdown works, this predicate is just not selective" and +# actually meant "the predicate was never usable". test/zonemap_cost.sh sat in +# exactly that state for its whole life, and #460's cost discount was validated +# against it. +# +# "Columnar Usable Skip Predicates" is the second number: how many predicates +# the reader built and can exclude with. The two together say which case you are +# in. +# +# The fixture is a DOMAIN column, which the issue's own Test section did not +# propose and which is the shape that survives #478. A domain resolves to its +# base type in GetDefaultOpClass, so pgcolumnar_clause_to_scankey finds a btree +# opfamily and a strategy and builds the key; but in pgcolumnar_make_predicates +# the column type is the domain while the constant's type is the base, so the +# key is cross-type, and integer_ops has no BTORDER proc for (domain, int4). +# #478's fallback drops it. Ordinary SQL, and it prunes nothing. +# +# (The fixture the issue DOES propose -- a same-type predicate on a column with +# no btree comparison -- cannot show this at all: clause_to_scankey rejects such +# a column outright, so no scan key is built and both numbers read 0.) +# +# Both columns hold the same values in the same table, so the two arms differ +# only in the declared type of the column being compared: the physical layout, +# the row groups and their min/max are identical by construction. +psql_run "DROP TABLE IF EXISTS pdr_u; + DROP DOMAIN IF EXISTS pdr_acct; + CREATE DOMAIN pdr_acct AS int; + CREATE TABLE pdr_u (plain int, dom pdr_acct) USING pgcolumnar; + SELECT pgcolumnar.set_options('pdr_u', stripe_row_limit => 10000);" >/dev/null +psql_run "INSERT INTO pdr_u + SELECT g, g::pdr_acct FROM generate_series(1, $ROWS) g;" >/dev/null + +uplan() { # uplan + q "SET pgcolumnar.enable_qual_pushdown = on; + EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) + SELECT count(*) FROM pdr_u WHERE $1 > $((ROWS - 10000));" +} + +usable="$(uplan plain)" +unusable="$(uplan dom)" + +# The node again, before any number is read out of either plan. Neither of these +# queries is the one checked above and the planner is free to choose differently. +check "the usable arm is a columnar custom scan" "$(is_scalar_scan "$usable")" "yes" +check "the unusable arm is a columnar custom scan" "$(is_scalar_scan "$unusable")" "yes" + +# The premise that makes the whole section mean something: these two arms really +# do prune differently. Without this, "1 usable" against "0 usable" could be two +# labels on identical behaviour -- which is the #477 failure repeated one level +# up, asserting a derived number with no physical fact under it. +check "the usable arm removes chunk groups" \ + "$([ "$(field "$usable" 'Columnar Chunk Groups Removed by Filter')" -gt 0 ] \ + && echo yes || echo no)" "yes" +# This one pins a KNOWN-WRONG behaviour deliberately. A domain column ought to +# prune exactly as its base type does, and it does not (#483). This suite needs +# some predicate the reader cannot use, and that is the only one available on +# current main; when #483 is fixed, this check goes red and whoever fixed it must +# supply a new unusable fixture rather than delete the section. Pinned as an +# assertion and not an echo, because nobody reads a passing suite's output. +check "the unusable arm removes none" \ + "$(field "$unusable" 'Columnar Chunk Groups Removed by Filter')" "0" + +# Both are reported as pushed down. This is the defect: the old line alone +# cannot tell these two plans apart. +check "both arms report the filter as pushed down" \ + "$(field "$usable" 'Columnar Pushed-Down Filters')/$(field "$unusable" 'Columnar Pushed-Down Filters')" \ + "1/1" + +# And the new line does tell them apart. +check "the usable arm reports one usable skip predicate" \ + "$(field "$usable" 'Columnar Usable Skip Predicates')" "1" +check "the unusable arm reports none" \ + "$(field "$unusable" 'Columnar Usable Skip Predicates')" "0" + +# With pushdown off the reader builds no predicates at all, so the new line must +# follow the setting too -- otherwise it would report a capability the run did +# not have, which is the #191 complaint about the old line. +check "pushdown off reports no usable skip predicates" \ + "$(field "$off" 'Columnar Usable Skip Predicates')" "0" + +# --- 7. the two vectorized aggregate nodes report it too (#479) ----------------- +# +# "Columnar Pushed-Down Filters" is printed by three nodes, not one: the scalar +# scan above, and both vectorized aggregate nodes, which fill it from +# PgColumnarCountConvertibleQuals -- the same built-key count, with the same gap +# under it. Measured on this same domain fixture before the fix, all three +# reported "1" while removing 0 of 20 chunk groups. +# +# Leaving two of the three unfixed would make one line of plan text mean two +# different things depending on which node ran, which is worse than the defect. +# +# Both nodes are opt-in, so each arm asserts its own node fired FIRST. Without +# that, a query the node quietly declines falls back to core Agg over the scalar +# scan -- which now reports the new line correctly, so the check would pass while +# testing the node it was written for not at all. +aggplan() { # aggplan + q "SET pgcolumnar.enable_qual_pushdown = on; + SET $1 = on; + EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) $2;" +} +has() { echo "$1" | grep -q "$2" && echo yes || echo no; } + +UAGG=pgcolumnar.enable_ungrouped_vector_agg +u_usable="$(aggplan $UAGG "SELECT count(*), sum(plain) FROM pdr_u WHERE plain > $((ROWS - 10000))")" +u_unusable="$(aggplan $UAGG "SELECT count(*), sum(plain) FROM pdr_u WHERE dom > $((ROWS - 10000))")" + +check "the ungrouped vectorized aggregate node ran (usable arm)" \ + "$(has "$u_usable" 'Columnar Vectorized Aggregates')" "yes" +check "the ungrouped vectorized aggregate node ran (unusable arm)" \ + "$(has "$u_unusable" 'Columnar Vectorized Aggregates')" "yes" + +check "ungrouped: the usable arm removes chunk groups" \ + "$([ "$(field "$u_usable" 'Columnar Chunk Groups Removed by Filter')" -gt 0 ] \ + && echo yes || echo no)" "yes" +check "ungrouped: the unusable arm removes none" \ + "$(field "$u_unusable" 'Columnar Chunk Groups Removed by Filter')" "0" + +check "ungrouped: both arms report the filter as pushed down" \ + "$(field "$u_usable" 'Columnar Pushed-Down Filters')/$(field "$u_unusable" 'Columnar Pushed-Down Filters')" \ + "1/1" +check "ungrouped: and the usable skip predicates tell them apart" \ + "$(field "$u_usable" 'Columnar Usable Skip Predicates')/$(field "$u_unusable" 'Columnar Usable Skip Predicates')" \ + "1/0" + +GAGG=pgcolumnar.enable_group_vectorization +g_usable="$(aggplan $GAGG "SELECT dom, count(*) FROM pdr_u WHERE plain > $((ROWS - 10000)) GROUP BY 1")" +g_unusable="$(aggplan $GAGG "SELECT plain, count(*) FROM pdr_u WHERE dom > $((ROWS - 10000)) GROUP BY 1")" + +# "Columnar Vectorized Group Keys" is this node's own marker; no other node emits +# it, so a positive grep proves the node rather than an absence a fallback would +# also satisfy. +check "the grouped vectorized aggregate node ran (usable arm)" \ + "$(has "$g_usable" 'Columnar Vectorized Group Keys')" "yes" +check "the grouped vectorized aggregate node ran (unusable arm)" \ + "$(has "$g_unusable" 'Columnar Vectorized Group Keys')" "yes" + +check "grouped: the usable arm removes chunk groups" \ + "$([ "$(field "$g_usable" 'Columnar Chunk Groups Removed by Filter')" -gt 0 ] \ + && echo yes || echo no)" "yes" +check "grouped: the unusable arm removes none" \ + "$(field "$g_unusable" 'Columnar Chunk Groups Removed by Filter')" "0" + +check "grouped: both arms report the filter as pushed down" \ + "$(field "$g_usable" 'Columnar Pushed-Down Filters')/$(field "$g_unusable" 'Columnar Pushed-Down Filters')" \ + "1/1" +check "grouped: and the usable skip predicates tell them apart" \ + "$(field "$g_usable" 'Columnar Usable Skip Predicates')/$(field "$g_unusable" 'Columnar Usable Skip Predicates')" \ + "1/0" + pgc_summary