From 172cf35cbd952c0d0d86545ab6dec11b96e0d297 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Tue, 7 Jul 2026 20:47:06 +0530 Subject: [PATCH] MDEV-19574: innodb_stats_method is not honored when innodb_stats_persistent=ON Problem: ======= When persistent statistics are enabled (innodb_stats_persistent=ON), the innodb_stats_method setting is not properly utilized during statistics calculation. The statistics collection functions always use a hardcoded default behavior for NULL value comparison instead of respecting the configured stats method (NULLS_EQUAL, NULLS_UNEQUAL, or NULLS_IGNORED). This affects the accuracy of n_diff_key_vals (distinct key count) and n_non_null_key_val estimates, particularly for indexes with nullable columns containing NULL values. This impacts the query optimizer, which makes decisions based on inaccurate cardinality estimates. Solution: ======== Introduced IndexLevelStats to collect statistics at a specific B-tree level during index analysis. Introduced PageStats to collect statistics for leaf page analysis. Refactored the following functions: dict_stats_analyze_index_level() to IndexLevelStats::analyze_level() dict_stats_analyze_index_for_n_prefix() to IndexLevelStats::sample_leaf_pages() dict_stats_analyze_index_below_cur() to PageStats::scan_below() dict_stats_scan_page() to PageStats::scan() Add the stats method name to stat_description when innodb_stats_method has a non-default value. Added the new stat name n_nonnull_fld01, n_nonnull_fld02, etc. with a stats description, to indicate how many non-null values exist for the nth field of the index. This value is retrieved and stored in the index statistics in dict_stats_fetch_index_stats_step(). rec_get_n_blob_pages(): Calculate the number of externally stored pages for a record. It uses ceiling division with the actual usable blob page space (blob_part_size) and now correctly handles both compressed and uncompressed table formats for accurate BLOB page counting. When InnoDB scans the leaf page directly, assign the leaf page count as the number of pages scanned for a multi-level index. For single-page indexes, use 1. This change leads to multiple changes in existing test cases. Non-null values are only counted at the leaf level, since only leaf pages hold actual records. Both nullable and NOT NULL columns are estimated with the same leaf-sampling formula: n_ordinary_leaf_pages * (n_non_null_all_analyzed_pages / n_leaf_pages_to_analyze) For a NOT NULL column every record is counted, so this yields the estimated record count (no NULLs). dict_stats_save(): now static function in dict0stats.cc that takes the innodb_stats_method value, and is removed from dict0stats.h. Replaced btr_rec_get_externally_stored_len() with rec_get_n_blob_pages() in dict0stats.cc. btr_rec_get_field_ref_offs() and btr_rec_get_field_ref(), together with the BTR_BLOB_HDR_* macros, were moved from btr0cur.cc to btr0cur.h so that rec_get_n_blob_pages() can reuse them; btr_rec_get_field_ref_offs() is now a noexcept function returning size_t. Changed stat_n_diff_key_vals and stat_n_non_null_key_vals from ib_uint64_t* to uint64_t* Removed the unused UNIV_STATS_DEBUG build macro (univ.i) and turned the DEBUG_PRINTF() helper in dict0stats.cc into an unconditional no-op. --- mysql-test/main/index_intersect_innodb.result | 8 +- .../main/innodb_ext_key,covering,on.rdiff | 4 +- mysql-test/main/innodb_ext_key,off.rdiff | 14 +- .../main/innodb_ext_key,on,unoptimized.rdiff | 4 +- mysql-test/main/mysqldump-system.result | 6 + .../main/opt_trace_index_merge_innodb.result | 6 +- .../selectivity_innodb_notembedded.result | 4 +- mysql-test/main/stat_tables_innodb.result | 4 +- .../suite/gcol/r/innodb_virtual_stats.result | 42 + .../suite/innodb/r/innodb-index-online.result | 6 +- .../innodb/r/innodb_stats_auto_recalc.result | 9 + .../r/innodb_stats_auto_recalc_ddl.result | 6 + ...db_stats_auto_recalc_on_nonexistent.result | 10 +- .../innodb/r/innodb_stats_drop_locked.result | 7 + .../suite/innodb/r/innodb_stats_fetch.result | 30 + .../innodb/r/innodb_stats_rename_table.result | 8 + ...innodb_stats_rename_table_if_exists.result | 24 + .../r/instant_alter_index_rename.result | 13 + .../suite/innodb/r/records_in_range,4k.rdiff | 9 +- .../suite/innodb/r/records_in_range,8k.rdiff | 9 +- .../suite/innodb/r/records_in_range.result | 2 + .../innodb/r/stats_external_pages.result | 58 + .../innodb/r/stats_method,NULLS_IGNORED.rdiff | 75 + .../innodb/r/stats_method,NULLS_UNEQUAL.rdiff | 75 + mysql-test/suite/innodb/r/stats_method.result | 160 ++ .../suite/innodb/t/records_in_range.test | 2 + .../suite/innodb/t/stats_external_pages.test | 60 + .../suite/innodb/t/stats_method.combinations | 14 + mysql-test/suite/innodb/t/stats_method.test | 192 +++ storage/innobase/btr/btr0cur.cc | 84 +- storage/innobase/dict/dict0stats.cc | 1502 +++++++++-------- storage/innobase/handler/ha_innodb.cc | 6 - storage/innobase/include/btr0cur.h | 34 +- storage/innobase/include/data0type.h | 2 +- storage/innobase/include/dict0mem.h | 4 +- storage/innobase/include/dict0stats.h | 19 +- storage/innobase/include/univ.i | 3 - 37 files changed, 1648 insertions(+), 867 deletions(-) create mode 100644 mysql-test/suite/innodb/r/stats_external_pages.result create mode 100644 mysql-test/suite/innodb/r/stats_method,NULLS_IGNORED.rdiff create mode 100644 mysql-test/suite/innodb/r/stats_method,NULLS_UNEQUAL.rdiff create mode 100644 mysql-test/suite/innodb/r/stats_method.result create mode 100644 mysql-test/suite/innodb/t/stats_external_pages.test create mode 100644 mysql-test/suite/innodb/t/stats_method.combinations create mode 100644 mysql-test/suite/innodb/t/stats_method.test diff --git a/mysql-test/main/index_intersect_innodb.result b/mysql-test/main/index_intersect_innodb.result index 44407dbcd3065..c070b41e54db9 100644 --- a/mysql-test/main/index_intersect_innodb.result +++ b/mysql-test/main/index_intersect_innodb.result @@ -81,7 +81,7 @@ EXPLAIN SELECT * FROM City WHERE Name LIKE 'M%' AND Population > 300000; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE City range Population,Name Name 35 NULL # Using index condition; Using where +1 SIMPLE City index_merge Population,Name Name,Population 35,4 NULL # Using sort_intersect(Name,Population); Using where EXPLAIN SELECT * FROM City WHERE Name LIKE 'M%' AND Population > 7000000; @@ -381,7 +381,7 @@ EXPLAIN SELECT * FROM City WHERE Name BETWEEN 'G' AND 'K' AND Population > 500000 AND Country LIKE 'C%'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE City range Population,Name,Country Name # NULL # Using index condition; Using where +1 SIMPLE City index_merge Population,Name,Country Name,Population,Country # NULL # Using sort_intersect(Name,Population,Country); Using where SELECT * FROM City USE INDEX () WHERE Name BETWEEN 'M' AND 'N' AND Population > 1000000 AND Country LIKE 'C%'; ID Name Country Population @@ -721,7 +721,7 @@ EXPLAIN SELECT * FROM City WHERE Name BETWEEN 'G' AND 'J' AND Population > 500000 AND Country LIKE 'C%'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE City range Population,Country,Name Name 35 NULL # Using index condition; Using where +1 SIMPLE City index_merge Population,Country,Name Name,Population,Country 35,4,3 NULL # Using sort_intersect(Name,Population,Country); Using where EXPLAIN SELECT * FROM City WHERE ID BETWEEN 1 AND 500 AND Population > 700000 AND Country LIKE 'C%'; @@ -732,7 +732,7 @@ SELECT * FROM City WHERE ID BETWEEN 3001 AND 4000 AND Population > 600000 AND Country BETWEEN 'S' AND 'Z'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE City range PRIMARY,Population,Country PRIMARY 4 NULL # Using where +1 SIMPLE City index_merge PRIMARY,Population,Country PRIMARY,Population,Country 4,4,7 NULL # Using sort_intersect(PRIMARY,Population,Country); Using where SELECT * FROM City WHERE Name LIKE 'C%' AND Population > 1000000; ID Name Country Population diff --git a/mysql-test/main/innodb_ext_key,covering,on.rdiff b/mysql-test/main/innodb_ext_key,covering,on.rdiff index 0d8bcc6ce3c2d..70144da745ff9 100644 --- a/mysql-test/main/innodb_ext_key,covering,on.rdiff +++ b/mysql-test/main/innodb_ext_key,covering,on.rdiff @@ -1,5 +1,5 @@ ---- ./main/innodb_ext_key.result -+++ ./main/innodb_ext_key.reject +--- innodb_ext_key.result ++++ innodb_ext_key,covering,on.result @@ -244,7 +244,7 @@ Variable_name Value Handler_read_first 0 diff --git a/mysql-test/main/innodb_ext_key,off.rdiff b/mysql-test/main/innodb_ext_key,off.rdiff index f27df1d081d3c..2a6db3c6070eb 100644 --- a/mysql-test/main/innodb_ext_key,off.rdiff +++ b/mysql-test/main/innodb_ext_key,off.rdiff @@ -1,11 +1,11 @@ ---- main/innodb_ext_key.result -+++ main/innodb_ext_key,off.reject +--- innodb_ext_key.result ++++ innodb_ext_key,OFF.result @@ -9,7 +9,7 @@ explain select count(*) from lineitem where l_orderkey=130 and l_shipdate='1992-07-01'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE lineitem ref PRIMARY,i_l_shipdate,i_l_orderkey,i_l_orderkey_quantity i_l_shipdate 8 const,const 1 Using index -+1 SIMPLE lineitem ref PRIMARY,i_l_shipdate,i_l_orderkey,i_l_orderkey_quantity i_l_shipdate 4 const 6 Using where; Using index ++1 SIMPLE lineitem ref PRIMARY,i_l_shipdate,i_l_orderkey,i_l_orderkey_quantity PRIMARY 4 const 5 Using where flush status; select count(*) from lineitem where l_orderkey=130 and l_shipdate='1992-07-01'; count(*) @@ -14,7 +14,7 @@ Handler_read_key 1 Handler_read_last 0 -Handler_read_next 1 -+Handler_read_next 6 ++Handler_read_next 5 Handler_read_prev 0 Handler_read_retry 0 Handler_read_rnd 0 @@ -95,7 +95,7 @@ where l_shipdate='1992-07-01' and l_orderkey=130; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Select tables optimized away -+1 SIMPLE lineitem ref PRIMARY,i_l_shipdate,i_l_orderkey,i_l_orderkey_quantity i_l_shipdate 4 const 6 Using where; Using index ++1 SIMPLE lineitem ref PRIMARY,i_l_shipdate,i_l_orderkey,i_l_orderkey_quantity PRIMARY 4 const 5 Using where flush status; select max(l_linenumber) from lineitem where l_shipdate='1992-07-01' and l_orderkey=130; @@ -104,7 +104,7 @@ Handler_read_key 1 Handler_read_last 0 -Handler_read_next 0 -+Handler_read_next 6 ++Handler_read_next 5 Handler_read_prev 0 Handler_read_retry 0 Handler_read_rnd 0 @@ -249,7 +249,7 @@ drop table t1,t2,t3; # # Bug mdev-4340: performance regression with extended_keys=on -@@ -714,13 +714,13 @@ +@@ -725,13 +725,13 @@ select * from t1 force index(index_date_updated) where index_date_updated= 10 and index_id < 800; id select_type table type possible_keys key key_len ref rows Extra diff --git a/mysql-test/main/innodb_ext_key,on,unoptimized.rdiff b/mysql-test/main/innodb_ext_key,on,unoptimized.rdiff index 0d8bcc6ce3c2d..31b4abcff8a61 100644 --- a/mysql-test/main/innodb_ext_key,on,unoptimized.rdiff +++ b/mysql-test/main/innodb_ext_key,on,unoptimized.rdiff @@ -1,5 +1,5 @@ ---- ./main/innodb_ext_key.result -+++ ./main/innodb_ext_key.reject +--- innodb_ext_key.result ++++ innodb_ext_key,ON,UNOPTIMIZED.rdiff @@ -244,7 +244,7 @@ Variable_name Value Handler_read_first 0 diff --git a/mysql-test/main/mysqldump-system.result b/mysql-test/main/mysqldump-system.result index c09b6f2500a6f..0742413e40ab7 100644 --- a/mysql-test/main/mysqldump-system.result +++ b/mysql-test/main/mysqldump-system.result @@ -121,6 +121,8 @@ REPLACE INTO `innodb_index_stats` VALUES ('mysql','tz','PRIMARY','2019-12-31 21:00:00','n_diff_pfx01',4,1,'Time_zone_id'), ('mysql','tz','PRIMARY','2019-12-31 21:00:00','n_diff_pfx02',393,1,'Time_zone_id,Transition_time'), ('mysql','tz','PRIMARY','2019-12-31 21:00:00','n_leaf_pages',1,NULL,'Number of leaf pages in the index'), +('mysql','tz','PRIMARY','2019-12-31 21:00:00','n_nonnull_fld01',393,1,'Time_zone_id'), +('mysql','tz','PRIMARY','2019-12-31 21:00:00','n_nonnull_fld02',393,1,'Transition_time'), ('mysql','tz','PRIMARY','2019-12-31 21:00:00','size',1,NULL,'Number of pages in the index'); /*!40000 ALTER TABLE `innodb_index_stats` ENABLE KEYS */; UNLOCK TABLES; @@ -736,6 +738,8 @@ REPLACE INTO `innodb_index_stats` VALUES ('mysql','tz','PRIMARY','2019-12-31 21:00:00','n_diff_pfx01',4,1,'Time_zone_id'), ('mysql','tz','PRIMARY','2019-12-31 21:00:00','n_diff_pfx02',393,1,'Time_zone_id,Transition_time'), ('mysql','tz','PRIMARY','2019-12-31 21:00:00','n_leaf_pages',1,NULL,'Number of leaf pages in the index'), +('mysql','tz','PRIMARY','2019-12-31 21:00:00','n_nonnull_fld01',393,1,'Time_zone_id'), +('mysql','tz','PRIMARY','2019-12-31 21:00:00','n_nonnull_fld02',393,1,'Transition_time'), ('mysql','tz','PRIMARY','2019-12-31 21:00:00','size',1,NULL,'Number of pages in the index'); /*!40000 ALTER TABLE `innodb_index_stats` ENABLE KEYS */; UNLOCK TABLES; @@ -1328,6 +1332,8 @@ INSERT IGNORE INTO `innodb_index_stats` VALUES ('mysql','tz','PRIMARY','2019-12-31 21:00:00','n_diff_pfx01',4,1,'Time_zone_id'), ('mysql','tz','PRIMARY','2019-12-31 21:00:00','n_diff_pfx02',393,1,'Time_zone_id,Transition_time'), ('mysql','tz','PRIMARY','2019-12-31 21:00:00','n_leaf_pages',1,NULL,'Number of leaf pages in the index'), +('mysql','tz','PRIMARY','2019-12-31 21:00:00','n_nonnull_fld01',393,1,'Time_zone_id'), +('mysql','tz','PRIMARY','2019-12-31 21:00:00','n_nonnull_fld02',393,1,'Transition_time'), ('mysql','tz','PRIMARY','2019-12-31 21:00:00','size',1,NULL,'Number of pages in the index'); /*!40000 ALTER TABLE `innodb_index_stats` ENABLE KEYS */; UNLOCK TABLES; diff --git a/mysql-test/main/opt_trace_index_merge_innodb.result b/mysql-test/main/opt_trace_index_merge_innodb.result index bdd45a509c380..09310ba37f84c 100644 --- a/mysql-test/main/opt_trace_index_merge_innodb.result +++ b/mysql-test/main/opt_trace_index_merge_innodb.result @@ -118,7 +118,7 @@ explain select * from t1 where pk1 != 0 and key1 = 1 { "using_mrr": false, "index_only": false, "rows": 1000, - "cost": 204.27, + "cost": 201.6536043, "chosen": true }, { @@ -138,8 +138,8 @@ explain select * from t1 where pk1 != 0 and key1 = 1 { "index": "key1", "index_scan_cost": 1.000146475, "cumulated_index_scan_cost": 1.000146475, - "disk_sweep_cost": 1.004153686, - "cumulative_total_cost": 2.004300162, + "disk_sweep_cost": 1.001383604, + "cumulative_total_cost": 2.00153008, "usable": true, "matching_rows_now": 1, "intersect_covering_with_this_index": false, diff --git a/mysql-test/main/selectivity_innodb_notembedded.result b/mysql-test/main/selectivity_innodb_notembedded.result index 7931e28294e6f..c2437fe21b171 100644 --- a/mysql-test/main/selectivity_innodb_notembedded.result +++ b/mysql-test/main/selectivity_innodb_notembedded.result @@ -282,7 +282,7 @@ JS { "access_type": "range", "resulting_rows": 5, - "cost": 6.83879845, + "cost": 6.666533161, "chosen": true } ], @@ -290,7 +290,7 @@ JS { "type": "range", "records": 5, - "cost": 6.83879845, + "cost": 6.666533161, "uses_join_buffering": false } } diff --git a/mysql-test/main/stat_tables_innodb.result b/mysql-test/main/stat_tables_innodb.result index bce9e884e997e..43e8a5fee0dac 100644 --- a/mysql-test/main/stat_tables_innodb.result +++ b/mysql-test/main/stat_tables_innodb.result @@ -82,7 +82,7 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE supplier ref PRIMARY,i_s_nationkey i_s_nationkey 5 dbt3_s001.nation.n_nationkey 1 Using index 1 SIMPLE customer ref PRIMARY,i_c_nationkey i_c_nationkey 5 dbt3_s001.nation.n_nationkey 6 Using index 1 SIMPLE orders ref|filter PRIMARY,i_o_orderdate,i_o_custkey i_o_custkey|i_o_orderdate 5|4 dbt3_s001.customer.c_custkey 15 (14%) Using where; Using rowid filter -1 SIMPLE lineitem ref PRIMARY,i_l_suppkey,i_l_orderkey,i_l_orderkey_quantity i_l_suppkey 9 dbt3_s001.supplier.s_suppkey,dbt3_s001.orders.o_orderkey 1 +1 SIMPLE lineitem ref PRIMARY,i_l_suppkey,i_l_orderkey,i_l_orderkey_quantity PRIMARY 4 dbt3_s001.orders.o_orderkey 4 Using where select n_name, sum(l_extendedprice * (1 - l_discount)) as revenue from customer, orders, lineitem, supplier, nation, region where c_custkey = o_custkey and l_orderkey = o_orderkey @@ -213,7 +213,7 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE supplier ref PRIMARY,i_s_nationkey i_s_nationkey 5 dbt3_s001.nation.n_nationkey 1 Using index 1 SIMPLE customer ref PRIMARY,i_c_nationkey i_c_nationkey 5 dbt3_s001.nation.n_nationkey 6 Using index 1 SIMPLE orders ref|filter PRIMARY,i_o_orderdate,i_o_custkey i_o_custkey|i_o_orderdate 5|4 dbt3_s001.customer.c_custkey 15 (14%) Using where; Using rowid filter -1 SIMPLE lineitem ref PRIMARY,i_l_suppkey,i_l_orderkey,i_l_orderkey_quantity i_l_suppkey 9 dbt3_s001.supplier.s_suppkey,dbt3_s001.orders.o_orderkey 1 +1 SIMPLE lineitem ref PRIMARY,i_l_suppkey,i_l_orderkey,i_l_orderkey_quantity PRIMARY 4 dbt3_s001.orders.o_orderkey 4 Using where select n_name, sum(l_extendedprice * (1 - l_discount)) as revenue from customer, orders, lineitem, supplier, nation, region where c_custkey = o_custkey and l_orderkey = o_orderkey diff --git a/mysql-test/suite/gcol/r/innodb_virtual_stats.result b/mysql-test/suite/gcol/r/innodb_virtual_stats.result index c0f595263df72..f02c699328825 100644 --- a/mysql-test/suite/gcol/r/innodb_virtual_stats.result +++ b/mysql-test/suite/gcol/r/innodb_virtual_stats.result @@ -13,15 +13,21 @@ WHERE database_name = 'test' AND table_name = 't'; index_name stat_name stat_description GEN_CLUST_INDEX n_diff_pfx01 DB_ROW_ID GEN_CLUST_INDEX n_leaf_pages Number of leaf pages in the index +GEN_CLUST_INDEX n_nonnull_fld01 DB_ROW_ID GEN_CLUST_INDEX size Number of pages in the index idxa n_diff_pfx01 a idxa n_diff_pfx02 a,DB_ROW_ID idxa n_leaf_pages Number of leaf pages in the index +idxa n_nonnull_fld01 a +idxa n_nonnull_fld02 DB_ROW_ID idxa size Number of pages in the index vidxcd n_diff_pfx01 c vidxcd n_diff_pfx02 c,d vidxcd n_diff_pfx03 c,d,DB_ROW_ID vidxcd n_leaf_pages Number of leaf pages in the index +vidxcd n_nonnull_fld01 c +vidxcd n_nonnull_fld02 d +vidxcd n_nonnull_fld03 DB_ROW_ID vidxcd size Number of pages in the index ALTER TABLE t ADD COLUMN e INT GENERATED ALWAYS AS(a+a+b), ADD INDEX idxb (b), ALGORITHM=INPLACE; select count(*) from t; @@ -33,19 +39,27 @@ WHERE database_name = 'test' AND table_name = 't'; index_name stat_name stat_description GEN_CLUST_INDEX n_diff_pfx01 DB_ROW_ID GEN_CLUST_INDEX n_leaf_pages Number of leaf pages in the index +GEN_CLUST_INDEX n_nonnull_fld01 DB_ROW_ID GEN_CLUST_INDEX size Number of pages in the index idxa n_diff_pfx01 a idxa n_diff_pfx02 a,DB_ROW_ID idxa n_leaf_pages Number of leaf pages in the index +idxa n_nonnull_fld01 a +idxa n_nonnull_fld02 DB_ROW_ID idxa size Number of pages in the index idxb n_diff_pfx01 b idxb n_diff_pfx02 b,DB_ROW_ID idxb n_leaf_pages Number of leaf pages in the index +idxb n_nonnull_fld01 b +idxb n_nonnull_fld02 DB_ROW_ID idxb size Number of pages in the index vidxcd n_diff_pfx01 c vidxcd n_diff_pfx02 c,d vidxcd n_diff_pfx03 c,d,DB_ROW_ID vidxcd n_leaf_pages Number of leaf pages in the index +vidxcd n_nonnull_fld01 c +vidxcd n_nonnull_fld02 d +vidxcd n_nonnull_fld03 DB_ROW_ID vidxcd size Number of pages in the index ALTER TABLE t DROP COLUMN c, DROP INDEX idxa, ALGORITHM=INPLACE; select count(*) from t; @@ -57,14 +71,19 @@ WHERE database_name = 'test' AND table_name = 't'; index_name stat_name stat_description GEN_CLUST_INDEX n_diff_pfx01 DB_ROW_ID GEN_CLUST_INDEX n_leaf_pages Number of leaf pages in the index +GEN_CLUST_INDEX n_nonnull_fld01 DB_ROW_ID GEN_CLUST_INDEX size Number of pages in the index idxb n_diff_pfx01 b idxb n_diff_pfx02 b,DB_ROW_ID idxb n_leaf_pages Number of leaf pages in the index +idxb n_nonnull_fld01 b +idxb n_nonnull_fld02 DB_ROW_ID idxb size Number of pages in the index vidxcd n_diff_pfx01 d vidxcd n_diff_pfx02 d,DB_ROW_ID vidxcd n_leaf_pages Number of leaf pages in the index +vidxcd n_nonnull_fld01 d +vidxcd n_nonnull_fld02 DB_ROW_ID vidxcd size Number of pages in the index ALTER TABLE t ADD INDEX vidxe (e), ALGORITHM=INPLACE; select count(*) from t; @@ -76,18 +95,25 @@ WHERE database_name = 'test' AND table_name = 't'; index_name stat_name stat_description GEN_CLUST_INDEX n_diff_pfx01 DB_ROW_ID GEN_CLUST_INDEX n_leaf_pages Number of leaf pages in the index +GEN_CLUST_INDEX n_nonnull_fld01 DB_ROW_ID GEN_CLUST_INDEX size Number of pages in the index idxb n_diff_pfx01 b idxb n_diff_pfx02 b,DB_ROW_ID idxb n_leaf_pages Number of leaf pages in the index +idxb n_nonnull_fld01 b +idxb n_nonnull_fld02 DB_ROW_ID idxb size Number of pages in the index vidxcd n_diff_pfx01 d vidxcd n_diff_pfx02 d,DB_ROW_ID vidxcd n_leaf_pages Number of leaf pages in the index +vidxcd n_nonnull_fld01 d +vidxcd n_nonnull_fld02 DB_ROW_ID vidxcd size Number of pages in the index vidxe n_diff_pfx01 e vidxe n_diff_pfx02 e,DB_ROW_ID vidxe n_leaf_pages Number of leaf pages in the index +vidxe n_nonnull_fld01 e +vidxe n_nonnull_fld02 DB_ROW_ID vidxe size Number of pages in the index ALTER TABLE t ADD COLUMN f INT GENERATED ALWAYS AS(a + a), ADD INDEX vidxf (f), ALGORITHM=INPLACE; select count(*) from t; @@ -99,22 +125,31 @@ WHERE database_name = 'test' AND table_name = 't'; index_name stat_name stat_description GEN_CLUST_INDEX n_diff_pfx01 DB_ROW_ID GEN_CLUST_INDEX n_leaf_pages Number of leaf pages in the index +GEN_CLUST_INDEX n_nonnull_fld01 DB_ROW_ID GEN_CLUST_INDEX size Number of pages in the index idxb n_diff_pfx01 b idxb n_diff_pfx02 b,DB_ROW_ID idxb n_leaf_pages Number of leaf pages in the index +idxb n_nonnull_fld01 b +idxb n_nonnull_fld02 DB_ROW_ID idxb size Number of pages in the index vidxcd n_diff_pfx01 d vidxcd n_diff_pfx02 d,DB_ROW_ID vidxcd n_leaf_pages Number of leaf pages in the index +vidxcd n_nonnull_fld01 d +vidxcd n_nonnull_fld02 DB_ROW_ID vidxcd size Number of pages in the index vidxe n_diff_pfx01 e vidxe n_diff_pfx02 e,DB_ROW_ID vidxe n_leaf_pages Number of leaf pages in the index +vidxe n_nonnull_fld01 e +vidxe n_nonnull_fld02 DB_ROW_ID vidxe size Number of pages in the index vidxf n_diff_pfx01 f vidxf n_diff_pfx02 f,DB_ROW_ID vidxf n_leaf_pages Number of leaf pages in the index +vidxf n_nonnull_fld01 f +vidxf n_nonnull_fld02 DB_ROW_ID vidxf size Number of pages in the index ALTER TABLE t DROP INDEX vidxcd; SELECT index_name, stat_name, stat_description @@ -123,17 +158,24 @@ WHERE database_name = 'test' AND table_name = 't'; index_name stat_name stat_description GEN_CLUST_INDEX n_diff_pfx01 DB_ROW_ID GEN_CLUST_INDEX n_leaf_pages Number of leaf pages in the index +GEN_CLUST_INDEX n_nonnull_fld01 DB_ROW_ID GEN_CLUST_INDEX size Number of pages in the index idxb n_diff_pfx01 b idxb n_diff_pfx02 b,DB_ROW_ID idxb n_leaf_pages Number of leaf pages in the index +idxb n_nonnull_fld01 b +idxb n_nonnull_fld02 DB_ROW_ID idxb size Number of pages in the index vidxe n_diff_pfx01 e vidxe n_diff_pfx02 e,DB_ROW_ID vidxe n_leaf_pages Number of leaf pages in the index +vidxe n_nonnull_fld01 e +vidxe n_nonnull_fld02 DB_ROW_ID vidxe size Number of pages in the index vidxf n_diff_pfx01 f vidxf n_diff_pfx02 f,DB_ROW_ID vidxf n_leaf_pages Number of leaf pages in the index +vidxf n_nonnull_fld01 f +vidxf n_nonnull_fld02 DB_ROW_ID vidxf size Number of pages in the index DROP TABLE t; diff --git a/mysql-test/suite/innodb/r/innodb-index-online.result b/mysql-test/suite/innodb/r/innodb-index-online.result index eb1daa5a2d396..645e903b62e70 100644 --- a/mysql-test/suite/innodb/r/innodb-index-online.result +++ b/mysql-test/suite/innodb/r/innodb-index-online.result @@ -134,9 +134,11 @@ SELECT * FROM mysql.innodb_index_stats WHERE table_name IN ('t1'); database_name table_name index_name last_update stat_name stat_value sample_size stat_description test t1 PRIMARY LAST_UPDATE n_diff_pfx01 5 1 c1 test t1 PRIMARY LAST_UPDATE n_leaf_pages 1 NULL Number of leaf pages in the index +test t1 PRIMARY LAST_UPDATE n_nonnull_fld01 5 1 c1 test t1 PRIMARY LAST_UPDATE size 1 NULL Number of pages in the index test t1 c2 LAST_UPDATE n_diff_pfx01 5 1 c2 test t1 c2 LAST_UPDATE n_leaf_pages 1 NULL Number of leaf pages in the index +test t1 c2 LAST_UPDATE n_nonnull_fld01 5 1 c2 test t1 c2 LAST_UPDATE size 1 NULL Number of pages in the index CREATE TABLE t1_c2_stats SELECT * FROM mysql.innodb_index_stats WHERE database_name = 'test' AND table_name = 't1' and index_name = 'c2'; @@ -150,9 +152,11 @@ SELECT * FROM mysql.innodb_index_stats WHERE table_name IN ('t1', 't1_c2_stats') database_name table_name index_name last_update stat_name stat_value sample_size stat_description test t1 PRIMARY LAST_UPDATE n_diff_pfx01 5 1 c1 test t1 PRIMARY LAST_UPDATE n_leaf_pages 1 NULL Number of leaf pages in the index +test t1 PRIMARY LAST_UPDATE n_nonnull_fld01 5 1 c1 test t1 PRIMARY LAST_UPDATE size 1 NULL Number of pages in the index -test t1_c2_stats GEN_CLUST_INDEX LAST_UPDATE n_diff_pfx01 3 1 DB_ROW_ID +test t1_c2_stats GEN_CLUST_INDEX LAST_UPDATE n_diff_pfx01 4 1 DB_ROW_ID test t1_c2_stats GEN_CLUST_INDEX LAST_UPDATE n_leaf_pages 1 NULL Number of leaf pages in the index +test t1_c2_stats GEN_CLUST_INDEX LAST_UPDATE n_nonnull_fld01 4 1 DB_ROW_ID test t1_c2_stats GEN_CLUST_INDEX LAST_UPDATE size 1 NULL Number of pages in the index connection con1; KILL QUERY @id; diff --git a/mysql-test/suite/innodb/r/innodb_stats_auto_recalc.result b/mysql-test/suite/innodb/r/innodb_stats_auto_recalc.result index 40eae0a938547..9129224a80268 100644 --- a/mysql-test/suite/innodb/r/innodb_stats_auto_recalc.result +++ b/mysql-test/suite/innodb/r/innodb_stats_auto_recalc.result @@ -10,6 +10,9 @@ index_name PRIMARY stat_name n_leaf_pages stat_value 1 index_name PRIMARY +stat_name n_nonnull_fld01 +stat_value 0 +index_name PRIMARY stat_name size stat_value 1 INSERT INTO autorecalc VALUES (1); @@ -25,6 +28,9 @@ index_name PRIMARY stat_name n_leaf_pages stat_value 1 index_name PRIMARY +stat_name n_nonnull_fld01 +stat_value 2 +index_name PRIMARY stat_name size stat_value 1 DELETE FROM autorecalc; @@ -39,6 +45,9 @@ index_name PRIMARY stat_name n_leaf_pages stat_value 1 index_name PRIMARY +stat_name n_nonnull_fld01 +stat_value 0 +index_name PRIMARY stat_name size stat_value 1 DROP TABLE autorecalc; diff --git a/mysql-test/suite/innodb/r/innodb_stats_auto_recalc_ddl.result b/mysql-test/suite/innodb/r/innodb_stats_auto_recalc_ddl.result index 8c68fe745042e..4dc9824a5d841 100644 --- a/mysql-test/suite/innodb/r/innodb_stats_auto_recalc_ddl.result +++ b/mysql-test/suite/innodb/r/innodb_stats_auto_recalc_ddl.result @@ -12,6 +12,9 @@ index_name PRIMARY stat_name n_leaf_pages stat_value 1 index_name PRIMARY +stat_name n_nonnull_fld01 +stat_value 2 +index_name PRIMARY stat_name size stat_value 1 DROP TABLE arddl; @@ -29,6 +32,9 @@ index_name PRIMARY stat_name n_leaf_pages stat_value 1 index_name PRIMARY +stat_name n_nonnull_fld01 +stat_value 2 +index_name PRIMARY stat_name size stat_value 1 DROP TABLE arddl; diff --git a/mysql-test/suite/innodb/r/innodb_stats_auto_recalc_on_nonexistent.result b/mysql-test/suite/innodb/r/innodb_stats_auto_recalc_on_nonexistent.result index 9a216374125f6..aec77b1277433 100644 --- a/mysql-test/suite/innodb/r/innodb_stats_auto_recalc_on_nonexistent.result +++ b/mysql-test/suite/innodb/r/innodb_stats_auto_recalc_on_nonexistent.result @@ -3,7 +3,7 @@ CREATE TABLE t (a INT, PRIMARY KEY (a)) ENGINE=INNODB; SELECT COUNT(*) FROM mysql.innodb_table_stats WHERE table_name = 't'; COUNT(*) 1 SELECT COUNT(*) FROM mysql.innodb_index_stats WHERE table_name = 't'; -COUNT(*) 3 +COUNT(*) 4 SELECT * FROM t; DELETE FROM mysql.innodb_index_stats WHERE table_name = 't'; DELETE FROM mysql.innodb_table_stats WHERE table_name = 't'; @@ -16,14 +16,14 @@ SELECT * FROM t; SELECT COUNT(*) FROM mysql.innodb_table_stats WHERE table_name = 't'; COUNT(*) 1 SELECT COUNT(*) FROM mysql.innodb_index_stats WHERE table_name = 't'; -COUNT(*) 3 +COUNT(*) 4 DROP TABLE t; Test with explicit enable CREATE TABLE t (a INT, PRIMARY KEY (a)) ENGINE=INNODB STATS_AUTO_RECALC=1; SELECT COUNT(*) FROM mysql.innodb_table_stats WHERE table_name = 't'; COUNT(*) 1 SELECT COUNT(*) FROM mysql.innodb_index_stats WHERE table_name = 't'; -COUNT(*) 3 +COUNT(*) 4 SELECT * FROM t; DELETE FROM mysql.innodb_index_stats WHERE table_name = 't'; DELETE FROM mysql.innodb_table_stats WHERE table_name = 't'; @@ -36,14 +36,14 @@ SELECT * FROM t; SELECT COUNT(*) FROM mysql.innodb_table_stats WHERE table_name = 't'; COUNT(*) 1 SELECT COUNT(*) FROM mysql.innodb_index_stats WHERE table_name = 't'; -COUNT(*) 3 +COUNT(*) 4 DROP TABLE t; Test with explicit disable CREATE TABLE t (a INT, PRIMARY KEY (a)) ENGINE=INNODB STATS_AUTO_RECALC=0; SELECT COUNT(*) FROM mysql.innodb_table_stats WHERE table_name = 't'; COUNT(*) 1 SELECT COUNT(*) FROM mysql.innodb_index_stats WHERE table_name = 't'; -COUNT(*) 3 +COUNT(*) 4 SELECT * FROM t; DELETE FROM mysql.innodb_index_stats WHERE table_name = 't'; DELETE FROM mysql.innodb_table_stats WHERE table_name = 't'; diff --git a/mysql-test/suite/innodb/r/innodb_stats_drop_locked.result b/mysql-test/suite/innodb/r/innodb_stats_drop_locked.result index 70f1edabf1178..d06ae12e74b01 100644 --- a/mysql-test/suite/innodb/r/innodb_stats_drop_locked.result +++ b/mysql-test/suite/innodb/r/innodb_stats_drop_locked.result @@ -25,6 +25,9 @@ innodb_stats_drop_locked innodb_stats_drop_locked innodb_stats_drop_locked innodb_stats_drop_locked +innodb_stats_drop_locked +innodb_stats_drop_locked +innodb_stats_drop_locked connect con1,localhost,root,,; SET innodb_lock_wait_timeout=1; ALTER TABLE innodb_stats_drop_locked DROP INDEX c_key; @@ -58,6 +61,9 @@ innodb_stats_drop_locked innodb_stats_drop_locked innodb_stats_drop_locked innodb_stats_drop_locked +innodb_stats_drop_locked +innodb_stats_drop_locked +innodb_stats_drop_locked ALTER TABLE innodb_stats_drop_locked DROP INDEX c_key; SELECT table_name FROM mysql.innodb_index_stats WHERE table_name='innodb_stats_drop_locked'; @@ -65,6 +71,7 @@ table_name innodb_stats_drop_locked innodb_stats_drop_locked innodb_stats_drop_locked +innodb_stats_drop_locked DROP TABLE innodb_stats_drop_locked; SELECT table_name FROM mysql.innodb_table_stats WHERE table_name='innodb_stats_drop_locked'; diff --git a/mysql-test/suite/innodb/r/innodb_stats_fetch.result b/mysql-test/suite/innodb/r/innodb_stats_fetch.result index 772cc5d146a7f..bb85be6de27b0 100644 --- a/mysql-test/suite/innodb/r/innodb_stats_fetch.result +++ b/mysql-test/suite/innodb/r/innodb_stats_fetch.result @@ -36,6 +36,16 @@ stat_value 1 sample_size NULL stat_description Number of leaf pages in the index index_name PRIMARY +stat_name n_nonnull_fld01 +stat_value 0 +sample_size 1 +stat_description a +index_name PRIMARY +stat_name n_nonnull_fld02 +stat_value 0 +sample_size 1 +stat_description b +index_name PRIMARY stat_name size stat_value 1 sample_size NULL @@ -66,6 +76,26 @@ stat_value 1 sample_size NULL stat_description Number of leaf pages in the index index_name idx +stat_name n_nonnull_fld01 +stat_value 0 +sample_size 1 +stat_description c +index_name idx +stat_name n_nonnull_fld02 +stat_value 0 +sample_size 1 +stat_description d +index_name idx +stat_name n_nonnull_fld03 +stat_value 0 +sample_size 1 +stat_description a +index_name idx +stat_name n_nonnull_fld04 +stat_value 0 +sample_size 1 +stat_description b +index_name idx stat_name size stat_value 1 sample_size NULL diff --git a/mysql-test/suite/innodb/r/innodb_stats_rename_table.result b/mysql-test/suite/innodb/r/innodb_stats_rename_table.result index 20d82dff9ba0d..48de2c8243206 100644 --- a/mysql-test/suite/innodb/r/innodb_stats_rename_table.result +++ b/mysql-test/suite/innodb/r/innodb_stats_rename_table.result @@ -24,6 +24,10 @@ stat_name n_leaf_pages stat_value 1 table_name stats_rename_old index_name PRIMARY +stat_name n_nonnull_fld01 +stat_value 0 +table_name stats_rename_old +index_name PRIMARY stat_name size stat_value 1 RENAME TABLE stats_rename_old TO stats_rename_new; @@ -45,6 +49,10 @@ stat_name n_leaf_pages stat_value 1 table_name stats_rename_new index_name PRIMARY +stat_name n_nonnull_fld01 +stat_value 0 +table_name stats_rename_new +index_name PRIMARY stat_name size stat_value 1 DROP TABLE stats_rename_new; diff --git a/mysql-test/suite/innodb/r/innodb_stats_rename_table_if_exists.result b/mysql-test/suite/innodb/r/innodb_stats_rename_table_if_exists.result index a966f629d7ee2..036e1d1c8e436 100644 --- a/mysql-test/suite/innodb/r/innodb_stats_rename_table_if_exists.result +++ b/mysql-test/suite/innodb/r/innodb_stats_rename_table_if_exists.result @@ -44,6 +44,10 @@ stat_name n_leaf_pages stat_value 1 table_name stats_rename1 index_name PRIMARY +stat_name n_nonnull_fld01 +stat_value 0 +table_name stats_rename1 +index_name PRIMARY stat_name size stat_value 1 table_name stats_rename1 @@ -56,6 +60,10 @@ stat_name n_leaf_pages stat_value 1 table_name stats_rename1 index_name b +stat_name n_nonnull_fld01 +stat_value 0 +table_name stats_rename1 +index_name b stat_name size stat_value 1 table_name stats_rename2 @@ -68,6 +76,10 @@ stat_name n_leaf_pages stat_value 567 table_name stats_rename2 index_name PRIMARY +stat_name n_nonnull_fld01 +stat_value 567 +table_name stats_rename2 +index_name PRIMARY stat_name size stat_value 567 table_name stats_rename2 @@ -80,6 +92,10 @@ stat_name n_leaf_pages stat_value 567 table_name stats_rename2 index_name b +stat_name n_nonnull_fld01 +stat_value 567 +table_name stats_rename2 +index_name b stat_name size stat_value 567 RENAME TABLE stats_rename1 TO stats_rename2; @@ -114,6 +130,10 @@ stat_name n_leaf_pages stat_value 1 table_name stats_rename2 index_name PRIMARY +stat_name n_nonnull_fld01 +stat_value 0 +table_name stats_rename2 +index_name PRIMARY stat_name size stat_value 1 table_name stats_rename2 @@ -126,6 +146,10 @@ stat_name n_leaf_pages stat_value 1 table_name stats_rename2 index_name c +stat_name n_nonnull_fld01 +stat_value 0 +table_name stats_rename2 +index_name c stat_name size stat_value 1 DROP TABLE stats_rename2; diff --git a/mysql-test/suite/innodb/r/instant_alter_index_rename.result b/mysql-test/suite/innodb/r/instant_alter_index_rename.result index 49f2213fbaf3f..ebccb27a2e020 100644 --- a/mysql-test/suite/innodb/r/instant_alter_index_rename.result +++ b/mysql-test/suite/innodb/r/instant_alter_index_rename.result @@ -208,14 +208,19 @@ SELECT table_name, index_name, stat_name FROM mysql.innodb_index_stats; table_name index_name stat_name t1 GEN_CLUST_INDEX n_diff_pfx01 t1 GEN_CLUST_INDEX n_leaf_pages +t1 GEN_CLUST_INDEX n_nonnull_fld01 t1 GEN_CLUST_INDEX size t1 ind1 n_diff_pfx01 t1 ind1 n_diff_pfx02 t1 ind1 n_leaf_pages +t1 ind1 n_nonnull_fld01 +t1 ind1 n_nonnull_fld02 t1 ind1 size t1 ind2 n_diff_pfx01 t1 ind2 n_diff_pfx02 t1 ind2 n_leaf_pages +t1 ind2 n_nonnull_fld01 +t1 ind2 n_nonnull_fld02 t1 ind2 size ALTER TABLE t1 DROP INDEX ind2, ADD INDEX ind3(b), DROP INDEX ind1, ADD INDEX ind2(c); @@ -223,24 +228,32 @@ SELECT table_name, index_name, stat_name FROM mysql.innodb_index_stats; table_name index_name stat_name t1 GEN_CLUST_INDEX n_diff_pfx01 t1 GEN_CLUST_INDEX n_leaf_pages +t1 GEN_CLUST_INDEX n_nonnull_fld01 t1 GEN_CLUST_INDEX size t1 ind2 n_diff_pfx01 t1 ind2 n_diff_pfx02 t1 ind2 n_leaf_pages +t1 ind2 n_nonnull_fld01 +t1 ind2 n_nonnull_fld02 t1 ind2 size t1 ind3 n_diff_pfx01 t1 ind3 n_diff_pfx02 t1 ind3 n_leaf_pages +t1 ind3 n_nonnull_fld01 +t1 ind3 n_nonnull_fld02 t1 ind3 size ALTER TABLE t1 DROP b, FORCE; SELECT table_name, index_name, stat_name FROM mysql.innodb_index_stats; table_name index_name stat_name t1 GEN_CLUST_INDEX n_diff_pfx01 t1 GEN_CLUST_INDEX n_leaf_pages +t1 GEN_CLUST_INDEX n_nonnull_fld01 t1 GEN_CLUST_INDEX size t1 ind2 n_diff_pfx01 t1 ind2 n_diff_pfx02 t1 ind2 n_leaf_pages +t1 ind2 n_nonnull_fld01 +t1 ind2 n_nonnull_fld02 t1 ind2 size UPDATE t1 SET a = 1 WHERE c = 'foo'; DROP TABLE t1; diff --git a/mysql-test/suite/innodb/r/records_in_range,4k.rdiff b/mysql-test/suite/innodb/r/records_in_range,4k.rdiff index 12b857ece2239..d1a03aa78ad9b 100644 --- a/mysql-test/suite/innodb/r/records_in_range,4k.rdiff +++ b/mysql-test/suite/innodb/r/records_in_range,4k.rdiff @@ -1,4 +1,11 @@ -@@ -39,7 +39,7 @@ +@@ -33,13 +33,13 @@ + WHERE + table_name='records_in_range_test' AND stat_name = 'n_leaf_pages'; + index_name stat_name stat_value +-PRIMARY n_leaf_pages 1 ++PRIMARY n_leaf_pages 4 + SELECT index_name, stat_name, stat_value + FROM mysql.innodb_index_stats WHERE table_name='records_in_range_test' AND stat_name = 'size'; index_name stat_name stat_value diff --git a/mysql-test/suite/innodb/r/records_in_range,8k.rdiff b/mysql-test/suite/innodb/r/records_in_range,8k.rdiff index bd24af160ba41..8085f2950814c 100644 --- a/mysql-test/suite/innodb/r/records_in_range,8k.rdiff +++ b/mysql-test/suite/innodb/r/records_in_range,8k.rdiff @@ -1,4 +1,11 @@ -@@ -39,7 +39,7 @@ +@@ -33,13 +33,13 @@ + WHERE + table_name='records_in_range_test' AND stat_name = 'n_leaf_pages'; + index_name stat_name stat_value +-PRIMARY n_leaf_pages 1 ++PRIMARY n_leaf_pages 2 + SELECT index_name, stat_name, stat_value + FROM mysql.innodb_index_stats WHERE table_name='records_in_range_test' AND stat_name = 'size'; index_name stat_name stat_value diff --git a/mysql-test/suite/innodb/r/records_in_range.result b/mysql-test/suite/innodb/r/records_in_range.result index e5a698f59cc7a..3ca1f241f0ae0 100644 --- a/mysql-test/suite/innodb/r/records_in_range.result +++ b/mysql-test/suite/innodb/r/records_in_range.result @@ -28,6 +28,8 @@ SET STATEMENT use_stat_tables=never FOR ANALYZE TABLE records_in_range_test; Table Op Msg_type Msg_text test.records_in_range_test analyze status OK +# n_leaf_pages changed because number of scanned leaf pages +# are assigned as leaf pages in case of multi level index SELECT index_name, stat_name, stat_value FROM mysql.innodb_index_stats WHERE diff --git a/mysql-test/suite/innodb/r/stats_external_pages.result b/mysql-test/suite/innodb/r/stats_external_pages.result new file mode 100644 index 0000000000000..8c01c5ad7e7f0 --- /dev/null +++ b/mysql-test/suite/innodb/r/stats_external_pages.result @@ -0,0 +1,58 @@ +CREATE TABLE t_blob ( +id INT AUTO_INCREMENT PRIMARY KEY, +col_int INT, +col_text TEXT, +col_blob BLOB, +col_longtext LONGTEXT, +INDEX idx_int (col_int) +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC; +INSERT INTO t_blob (col_int, col_text, col_blob, col_longtext) +SELECT +CASE WHEN seq <= 5000 THEN ((seq-1) % 500) + 1 ELSE NULL END, +REPEAT('A', 1000), +REPEAT('B', 5000), +REPEAT('C', 50000) +FROM seq_1_to_10000; +ANALYZE TABLE t_blob; +Table Op Msg_type Msg_text +test.t_blob analyze status Engine-independent statistics collected +test.t_blob analyze Warning Engine-independent statistics are not collected for column 'col_text' +test.t_blob analyze Warning Engine-independent statistics are not collected for column 'col_blob' +test.t_blob analyze Warning Engine-independent statistics are not collected for column 'col_longtext' +test.t_blob analyze status OK +SELECT @@global.innodb_stats_method AS stats_method, +index_name, +cardinality +FROM information_schema.STATISTICS +WHERE table_schema = 'test' + AND table_name = 't_blob' + AND index_name = 'idx_int'; +stats_method index_name cardinality +nulls_equal idx_int 1000 +DROP TABLE t_blob; +CREATE TABLE t_blob_compressed ( +id INT AUTO_INCREMENT PRIMARY KEY, +col_int INT, +col_longblob LONGBLOB, +INDEX idx_int (col_int) +) ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=4; +INSERT INTO t_blob_compressed (col_int, col_longblob) +SELECT +CASE WHEN seq <= 5000 THEN ((seq-1) % 500) + 1 ELSE NULL END, +REPEAT('X', 100000) +FROM seq_1_to_10000; +ANALYZE TABLE t_blob_compressed; +Table Op Msg_type Msg_text +test.t_blob_compressed analyze status Engine-independent statistics collected +test.t_blob_compressed analyze Warning Engine-independent statistics are not collected for column 'col_longblob' +test.t_blob_compressed analyze status OK +SELECT @@global.innodb_stats_method AS stats_method, +index_name, +cardinality +FROM information_schema.STATISTICS +WHERE table_schema = 'test' + AND table_name = 't_blob_compressed' + AND index_name = 'idx_int'; +stats_method index_name cardinality +nulls_equal idx_int 1000 +DROP TABLE t_blob_compressed; diff --git a/mysql-test/suite/innodb/r/stats_method,NULLS_IGNORED.rdiff b/mysql-test/suite/innodb/r/stats_method,NULLS_IGNORED.rdiff new file mode 100644 index 0000000000000..6a9f41b1445cb --- /dev/null +++ b/mysql-test/suite/innodb/r/stats_method,NULLS_IGNORED.rdiff @@ -0,0 +1,75 @@ +--- stats_method.result ++++ stats_method,NULLS_IGNORED.result +@@ -26,9 +26,9 @@ + WHERE table_schema = 'test' + AND table_name = 't_stats_method'; + stats_method index_name est_rows +-nulls_equal PRIMARY 100 +-nulls_equal idx_col1 11 +-nulls_equal idx_col2 6 ++nulls_ignored PRIMARY 100 ++nulls_ignored idx_col1 20 ++nulls_ignored idx_col2 10 + SELECT index_name, stat_name, stat_description, stat_value + FROM mysql.innodb_index_stats + WHERE table_name='t_stats_method' +@@ -76,12 +76,12 @@ + where table_name="t1" and index_name="PRIMARY" + and stat_name="n_diff_pfx01"; + stat_name stat_description +-n_diff_pfx01 c01꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵ ++n_diff_pfx01 c01꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵ NULLS_IGNORED + SELECT stat_name, stat_description FROM mysql.innodb_index_stats + where table_name="t1" and index_name="idx_all" + and stat_name="n_diff_pfx07"; + stat_name stat_description +-n_diff_pfx07 c33꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵,c34꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵,c35꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵,c36꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵,c37꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵ ++n_diff_pfx07 c33꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵,c34꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵,c35꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵,c36꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵,c37꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵ NULLS_IGNORED + DROP TABLE t1; + CREATE TABLE t_multilevel ( + id INT AUTO_INCREMENT PRIMARY KEY, +@@ -111,15 +111,15 @@ + AND stat_name IN ('n_diff_pfx01', 'n_nonnull_fld01', 'n_leaf_pages') + ORDER BY index_name, stat_name; + stats_method index_name stat_name stat_value +-nulls_equal idx_col1 n_diff_pfx01 501 +-nulls_equal idx_col1 n_leaf_pages 8 +-nulls_equal idx_col1 n_nonnull_fld01 2000 +-nulls_equal idx_col2 n_diff_pfx01 4001 +-nulls_equal idx_col2 n_leaf_pages 14 +-nulls_equal idx_col2 n_nonnull_fld01 8000 +-nulls_equal idx_col3 n_diff_pfx01 2501 +-nulls_equal idx_col3 n_leaf_pages 11 +-nulls_equal idx_col3 n_nonnull_fld01 5000 ++nulls_ignored idx_col1 n_diff_pfx01 8500 ++nulls_ignored idx_col1 n_leaf_pages 8 ++nulls_ignored idx_col1 n_nonnull_fld01 2000 ++nulls_ignored idx_col2 n_diff_pfx01 6000 ++nulls_ignored idx_col2 n_leaf_pages 14 ++nulls_ignored idx_col2 n_nonnull_fld01 8000 ++nulls_ignored idx_col3 n_diff_pfx01 7500 ++nulls_ignored idx_col3 n_leaf_pages 11 ++nulls_ignored idx_col3 n_nonnull_fld01 5000 + SELECT @@global.innodb_stats_method AS stats_method, + index_name, + cardinality +@@ -129,9 +129,9 @@ + AND index_name IN ('idx_col1', 'idx_col2', 'idx_col3') + ORDER BY index_name; + stats_method index_name cardinality +-nulls_equal idx_col1 525 +-nulls_equal idx_col2 4990 +-nulls_equal idx_col3 3326 ++nulls_ignored idx_col1 3326 ++nulls_ignored idx_col2 9980 ++nulls_ignored idx_col3 9980 + DROP TABLE t_multilevel; + CREATE TABLE t_not_null ( + id INT AUTO_INCREMENT PRIMARY KEY, +@@ -156,5 +156,5 @@ + index_name stat_name stat_value + PRIMARY n_diff_pfx01 100 + idx_not_null n_diff_pfx01 10 +-idx_nullable n_diff_pfx01 11 ++idx_nullable n_diff_pfx01 60 + DROP TABLE t_not_null; diff --git a/mysql-test/suite/innodb/r/stats_method,NULLS_UNEQUAL.rdiff b/mysql-test/suite/innodb/r/stats_method,NULLS_UNEQUAL.rdiff new file mode 100644 index 0000000000000..2ce2d78aa2cd3 --- /dev/null +++ b/mysql-test/suite/innodb/r/stats_method,NULLS_UNEQUAL.rdiff @@ -0,0 +1,75 @@ +--- stats_method.result ++++ stats_method,NULL_UNEQUAL.result +@@ -26,9 +26,9 @@ + WHERE table_schema = 'test' + AND table_name = 't_stats_method'; + stats_method index_name est_rows +-nulls_equal PRIMARY 100 +-nulls_equal idx_col1 11 +-nulls_equal idx_col2 6 ++nulls_unequal PRIMARY 100 ++nulls_unequal idx_col1 100 ++nulls_unequal idx_col2 100 + SELECT index_name, stat_name, stat_description, stat_value + FROM mysql.innodb_index_stats + WHERE table_name='t_stats_method' +@@ -76,12 +76,12 @@ + where table_name="t1" and index_name="PRIMARY" + and stat_name="n_diff_pfx01"; + stat_name stat_description +-n_diff_pfx01 c01꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵ ++n_diff_pfx01 c01꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵ NULLS_UNEQUAL + SELECT stat_name, stat_description FROM mysql.innodb_index_stats + where table_name="t1" and index_name="idx_all" + and stat_name="n_diff_pfx07"; + stat_name stat_description +-n_diff_pfx07 c33꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵,c34꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵,c35꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵,c36꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵,c37꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵ ++n_diff_pfx07 c33꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵,c34꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵,c35꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵,c36꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵,c37꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵ NULLS_UNEQUAL + DROP TABLE t1; + CREATE TABLE t_multilevel ( + id INT AUTO_INCREMENT PRIMARY KEY, +@@ -111,15 +111,15 @@ + AND stat_name IN ('n_diff_pfx01', 'n_nonnull_fld01', 'n_leaf_pages') + ORDER BY index_name, stat_name; + stats_method index_name stat_name stat_value +-nulls_equal idx_col1 n_diff_pfx01 501 +-nulls_equal idx_col1 n_leaf_pages 8 +-nulls_equal idx_col1 n_nonnull_fld01 2000 +-nulls_equal idx_col2 n_diff_pfx01 4001 +-nulls_equal idx_col2 n_leaf_pages 14 +-nulls_equal idx_col2 n_nonnull_fld01 8000 +-nulls_equal idx_col3 n_diff_pfx01 2501 +-nulls_equal idx_col3 n_leaf_pages 11 +-nulls_equal idx_col3 n_nonnull_fld01 5000 ++nulls_unequal idx_col1 n_diff_pfx01 8500 ++nulls_unequal idx_col1 n_leaf_pages 8 ++nulls_unequal idx_col1 n_nonnull_fld01 2000 ++nulls_unequal idx_col2 n_diff_pfx01 6000 ++nulls_unequal idx_col2 n_leaf_pages 14 ++nulls_unequal idx_col2 n_nonnull_fld01 8000 ++nulls_unequal idx_col3 n_diff_pfx01 7500 ++nulls_unequal idx_col3 n_leaf_pages 11 ++nulls_unequal idx_col3 n_nonnull_fld01 5000 + SELECT @@global.innodb_stats_method AS stats_method, + index_name, + cardinality +@@ -129,9 +129,9 @@ + AND index_name IN ('idx_col1', 'idx_col2', 'idx_col3') + ORDER BY index_name; + stats_method index_name cardinality +-nulls_equal idx_col1 525 +-nulls_equal idx_col2 4990 +-nulls_equal idx_col3 3326 ++nulls_unequal idx_col1 9980 ++nulls_unequal idx_col2 9980 ++nulls_unequal idx_col3 9980 + DROP TABLE t_multilevel; + CREATE TABLE t_not_null ( + id INT AUTO_INCREMENT PRIMARY KEY, +@@ -156,5 +156,5 @@ + index_name stat_name stat_value + PRIMARY n_diff_pfx01 100 + idx_not_null n_diff_pfx01 10 +-idx_nullable n_diff_pfx01 11 ++idx_nullable n_diff_pfx01 60 + DROP TABLE t_not_null; diff --git a/mysql-test/suite/innodb/r/stats_method.result b/mysql-test/suite/innodb/r/stats_method.result new file mode 100644 index 0000000000000..e1d56ef4374ee --- /dev/null +++ b/mysql-test/suite/innodb/r/stats_method.result @@ -0,0 +1,160 @@ +CREATE TABLE t_stats_method ( +id INT AUTO_INCREMENT PRIMARY KEY, +col1 INT, +col2 INT, +INDEX idx_col1 (col1), +INDEX idx_col2 (col2) +) ENGINE=InnoDB; +INSERT INTO t_stats_method (col1, col2) +SELECT +CASE +WHEN seq <= 50 THEN ((seq-1) % 10) + 1 +ELSE NULL +END, +CASE +WHEN seq <= 50 THEN ((seq-1) % 5) + 1 +ELSE NULL +END +FROM seq_1_to_100; +ANALYZE TABLE t_stats_method; +Table Op Msg_type Msg_text +test.t_stats_method analyze status OK +SELECT @@global.innodb_stats_method AS stats_method, +index_name, +cardinality AS est_rows +FROM information_schema.STATISTICS +WHERE table_schema = 'test' + AND table_name = 't_stats_method'; +stats_method index_name est_rows +nulls_equal PRIMARY 100 +nulls_equal idx_col1 11 +nulls_equal idx_col2 6 +SELECT index_name, stat_name, stat_description, stat_value +FROM mysql.innodb_index_stats +WHERE table_name='t_stats_method' +AND stat_name like 'n_nonnull_%'; +index_name stat_name stat_description stat_value +PRIMARY n_nonnull_fld01 id 100 +idx_col1 n_nonnull_fld01 col1 50 +idx_col1 n_nonnull_fld02 id 100 +idx_col2 n_nonnull_fld01 col2 50 +idx_col2 n_nonnull_fld02 id 100 +DROP TABLE t_stats_method; +SET NAMES utf8mb4; +BEGIN NOT ATOMIC +DECLARE c TEXT CHARACTER SET utf8mb4 DEFAULT ( +SELECT CONCAT('CREATE TABLE t1(', +GROUP_CONCAT(CONCAT('`c', LPAD(seq, 2, '0'), '꠵', +REPEAT('꠵', 59), '` INT') SEPARATOR ', '), ', ', +'PRIMARY KEY(', +(SELECT GROUP_CONCAT(CONCAT('`c', LPAD(seq, 2, '0'), '꠵', +REPEAT('꠵', 59), '`') SEPARATOR ', ') +FROM seq_1_to_32), +'), KEY `idx_all`(', +(SELECT GROUP_CONCAT(CONCAT('`c', LPAD(seq, 2, '0'), '꠵', +REPEAT('꠵', 59), '`') SEPARATOR ', ') +FROM seq_33_to_64), +')) ENGINE=InnoDB STATS_PERSISTENT=1') +FROM seq_1_to_64); +EXECUTE IMMEDIATE c; +END; +$$ +INSERT INTO t1 +SELECT seq, seq, seq, seq, seq, seq, seq, seq, +seq, seq, seq, seq, seq, seq, seq, seq, +seq, seq, seq, seq, seq, seq, seq, seq, +seq, seq, seq, seq, seq, seq, seq, seq, +seq, seq, seq, seq, seq, seq, seq, seq, +seq, seq, seq, seq, seq, seq, seq, seq, +seq, seq, seq, seq, seq, seq, seq, seq, +seq, seq, seq, seq, seq, seq, seq, seq +FROM seq_1_to_320; +ANALYZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 analyze status OK +SELECT stat_name, stat_description FROM mysql.innodb_index_stats +where table_name="t1" and index_name="PRIMARY" +and stat_name="n_diff_pfx01"; +stat_name stat_description +n_diff_pfx01 c01꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵ +SELECT stat_name, stat_description FROM mysql.innodb_index_stats +where table_name="t1" and index_name="idx_all" +and stat_name="n_diff_pfx07"; +stat_name stat_description +n_diff_pfx07 c33꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵,c34꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵,c35꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵,c36꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵,c37꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵꠵ +DROP TABLE t1; +CREATE TABLE t_multilevel ( +id INT AUTO_INCREMENT PRIMARY KEY, +col1 INT, +col2 INT, +col3 INT, +INDEX idx_col1 (col1), +INDEX idx_col2 (col2), +INDEX idx_col3 (col3) +) ENGINE=InnoDB STATS_PERSISTENT=1 STATS_SAMPLE_PAGES=20; +INSERT INTO t_multilevel (col1, col2, col3) +SELECT +CASE WHEN seq <= 2000 THEN ((seq-1) % 500) + 1 ELSE NULL END, +CASE WHEN seq <= 8000 THEN ((seq-1) % 4000) + 1 ELSE NULL END, +CASE WHEN seq <= 5000 THEN ((seq-1) % 2500) + 1 ELSE NULL END +FROM seq_1_to_10000; +ANALYZE TABLE t_multilevel; +Table Op Msg_type Msg_text +test.t_multilevel analyze status OK +SELECT @@global.innodb_stats_method AS stats_method, +index_name, +stat_name, +stat_value +FROM mysql.innodb_index_stats +WHERE table_name='t_multilevel' + AND index_name IN ('idx_col1', 'idx_col2', 'idx_col3') +AND stat_name IN ('n_diff_pfx01', 'n_nonnull_fld01', 'n_leaf_pages') +ORDER BY index_name, stat_name; +stats_method index_name stat_name stat_value +nulls_equal idx_col1 n_diff_pfx01 501 +nulls_equal idx_col1 n_leaf_pages 8 +nulls_equal idx_col1 n_nonnull_fld01 2000 +nulls_equal idx_col2 n_diff_pfx01 4001 +nulls_equal idx_col2 n_leaf_pages 14 +nulls_equal idx_col2 n_nonnull_fld01 8000 +nulls_equal idx_col3 n_diff_pfx01 2501 +nulls_equal idx_col3 n_leaf_pages 11 +nulls_equal idx_col3 n_nonnull_fld01 5000 +SELECT @@global.innodb_stats_method AS stats_method, +index_name, +cardinality +FROM information_schema.STATISTICS +WHERE table_schema = 'test' + AND table_name = 't_multilevel' + AND index_name IN ('idx_col1', 'idx_col2', 'idx_col3') +ORDER BY index_name; +stats_method index_name cardinality +nulls_equal idx_col1 525 +nulls_equal idx_col2 4990 +nulls_equal idx_col3 3326 +DROP TABLE t_multilevel; +CREATE TABLE t_not_null ( +id INT AUTO_INCREMENT PRIMARY KEY, +col_nullable INT, +col_not_null INT NOT NULL, +INDEX idx_nullable (col_nullable), +INDEX idx_not_null (col_not_null) +) ENGINE=InnoDB STATS_PERSISTENT=1; +INSERT INTO t_not_null (col_nullable, col_not_null) +SELECT +CASE WHEN seq <= 50 THEN ((seq-1) % 10) + 1 ELSE NULL END, +((seq-1) % 10) + 1 +FROM seq_1_to_100; +ANALYZE TABLE t_not_null; +Table Op Msg_type Msg_text +test.t_not_null analyze status OK +SELECT index_name, stat_name, stat_value +FROM mysql.innodb_index_stats +WHERE table_name='t_not_null' +AND stat_name IN ('n_diff_pfx01', 'n_nonnull_pfx01') +ORDER BY index_name, stat_name; +index_name stat_name stat_value +PRIMARY n_diff_pfx01 100 +idx_not_null n_diff_pfx01 10 +idx_nullable n_diff_pfx01 11 +DROP TABLE t_not_null; diff --git a/mysql-test/suite/innodb/t/records_in_range.test b/mysql-test/suite/innodb/t/records_in_range.test index 697dbc1e8dddf..0fc87692a90dc 100644 --- a/mysql-test/suite/innodb/t/records_in_range.test +++ b/mysql-test/suite/innodb/t/records_in_range.test @@ -43,6 +43,8 @@ ANALYZE TABLE records_in_range_test; # 16k or bigger page size: 1 leaf page # 8k page size: 2 leaf pages # 4k page size: 4 leaf pages +--echo # n_leaf_pages changed because number of scanned leaf pages +--echo # are assigned as leaf pages in case of multi level index SELECT index_name, stat_name, stat_value FROM mysql.innodb_index_stats WHERE diff --git a/mysql-test/suite/innodb/t/stats_external_pages.test b/mysql-test/suite/innodb/t/stats_external_pages.test new file mode 100644 index 0000000000000..3489bcdf9f94f --- /dev/null +++ b/mysql-test/suite/innodb/t/stats_external_pages.test @@ -0,0 +1,60 @@ +--source include/have_innodb.inc +--source include/have_sequence.inc + +CREATE TABLE t_blob ( + id INT AUTO_INCREMENT PRIMARY KEY, + col_int INT, + col_text TEXT, + col_blob BLOB, + col_longtext LONGTEXT, + INDEX idx_int (col_int) +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC; + +# Insert data with large BLOBs that will be stored externally +INSERT INTO t_blob (col_int, col_text, col_blob, col_longtext) +SELECT + CASE WHEN seq <= 5000 THEN ((seq-1) % 500) + 1 ELSE NULL END, + REPEAT('A', 1000), + REPEAT('B', 5000), + REPEAT('C', 50000) +FROM seq_1_to_10000; + +ANALYZE TABLE t_blob; + +SELECT @@global.innodb_stats_method AS stats_method, + index_name, + cardinality +FROM information_schema.STATISTICS +WHERE table_schema = 'test' + AND table_name = 't_blob' + AND index_name = 'idx_int'; + +DROP TABLE t_blob; + +# Test with compressed table (different blob_part_size calculation) +CREATE TABLE t_blob_compressed ( + id INT AUTO_INCREMENT PRIMARY KEY, + col_int INT, + col_longblob LONGBLOB, + INDEX idx_int (col_int) +) ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=4; + +# Insert large BLOBs +INSERT INTO t_blob_compressed (col_int, col_longblob) +SELECT + CASE WHEN seq <= 5000 THEN ((seq-1) % 500) + 1 ELSE NULL END, + REPEAT('X', 100000) +FROM seq_1_to_10000; + +ANALYZE TABLE t_blob_compressed; + + +SELECT @@global.innodb_stats_method AS stats_method, + index_name, + cardinality +FROM information_schema.STATISTICS +WHERE table_schema = 'test' + AND table_name = 't_blob_compressed' + AND index_name = 'idx_int'; + +DROP TABLE t_blob_compressed; diff --git a/mysql-test/suite/innodb/t/stats_method.combinations b/mysql-test/suite/innodb/t/stats_method.combinations new file mode 100644 index 0000000000000..6974ad2a4b2f4 --- /dev/null +++ b/mysql-test/suite/innodb/t/stats_method.combinations @@ -0,0 +1,14 @@ +[NULLS_EQUAL] +--innodb_stats_method=nulls_equal +--use_stat_tables=never +--optimizer_adjust_secondary_key_costs=8 + +[NULLS_UNEQUAL] +--innodb_stats_method=nulls_unequal +--use_stat_tables=never +--optimizer_adjust_secondary_key_costs=8 + +[NULLS_IGNORED] +--innodb_stats_method=nulls_ignored +--use_stat_tables=never +--optimizer_adjust_secondary_key_costs=8 diff --git a/mysql-test/suite/innodb/t/stats_method.test b/mysql-test/suite/innodb/t/stats_method.test new file mode 100644 index 0000000000000..edd9f461ed30c --- /dev/null +++ b/mysql-test/suite/innodb/t/stats_method.test @@ -0,0 +1,192 @@ +--source include/have_innodb.inc +--source include/have_sequence.inc + +# Test case to demonstrate different innodb_stats_method settings +CREATE TABLE t_stats_method ( + id INT AUTO_INCREMENT PRIMARY KEY, + col1 INT, + col2 INT, + INDEX idx_col1 (col1), + INDEX idx_col2 (col2) +) ENGINE=InnoDB; + +# For col1: 50 NULLs, 10 distinct non-NULL values +# For col2: 50 NULLs, 5 distinct non-NULL values +INSERT INTO t_stats_method (col1, col2) +SELECT + CASE + WHEN seq <= 50 THEN ((seq-1) % 10) + 1 + ELSE NULL + END, + CASE + WHEN seq <= 50 THEN ((seq-1) % 5) + 1 + ELSE NULL + END +FROM seq_1_to_100; + +ANALYZE TABLE t_stats_method; + +# cardinality = number of distinct value +# Nulls_equal: +#============== +# idx_col1: 50 NULLs + 10 distinct value = 11 +# idx_col2: 50 NULLs + 5 distinct value = 6 + +# Nulls_unequal: +#============== +# idx_col1: 50 NULLs + 10 distinct value = 60 +# idx_col2: 50 NULLs + 5 distinct value = 55 + +# Nulls_ignored: +#=============== + +# innodb_rec_per_key = +# number of non-null records / number of distinct keys +# +# cardinality = number of records / innodb_rec_per_key; +# idx_col1: 50 NULLs + 10 distinct value + +# innodb_rec_per_key = 50/10 = 5 +# cardinality = 100 / 5 = 20 + +# idx_col2: 50 NULLs + 5 distinct value +# innodb_rec_per_key = 50 / 5 = 10 +# cardinality = 100 / 10 = 10 + +SELECT @@global.innodb_stats_method AS stats_method, + index_name, + cardinality AS est_rows +FROM information_schema.STATISTICS +WHERE table_schema = 'test' + AND table_name = 't_stats_method'; + +SELECT index_name, stat_name, stat_description, stat_value +FROM mysql.innodb_index_stats +WHERE table_name='t_stats_method' +AND stat_name like 'n_nonnull_%'; + +DROP TABLE t_stats_method; + +SET NAMES utf8mb4; +# Test maximum number of index columns (64) for statistics calculation +DELIMITER $$; +BEGIN NOT ATOMIC + DECLARE c TEXT CHARACTER SET utf8mb4 DEFAULT ( + SELECT CONCAT('CREATE TABLE t1(', + GROUP_CONCAT(CONCAT('`c', LPAD(seq, 2, '0'), '꠵', + REPEAT('꠵', 59), '` INT') SEPARATOR ', '), ', ', + 'PRIMARY KEY(', + (SELECT GROUP_CONCAT(CONCAT('`c', LPAD(seq, 2, '0'), '꠵', + REPEAT('꠵', 59), '`') SEPARATOR ', ') + FROM seq_1_to_32), + '), KEY `idx_all`(', + (SELECT GROUP_CONCAT(CONCAT('`c', LPAD(seq, 2, '0'), '꠵', + REPEAT('꠵', 59), '`') SEPARATOR ', ') + FROM seq_33_to_64), + ')) ENGINE=InnoDB STATS_PERSISTENT=1') + FROM seq_1_to_64); + EXECUTE IMMEDIATE c; +END; +$$ +DELIMITER ;$$ + + +INSERT INTO t1 +SELECT seq, seq, seq, seq, seq, seq, seq, seq, + seq, seq, seq, seq, seq, seq, seq, seq, + seq, seq, seq, seq, seq, seq, seq, seq, + seq, seq, seq, seq, seq, seq, seq, seq, + seq, seq, seq, seq, seq, seq, seq, seq, + seq, seq, seq, seq, seq, seq, seq, seq, + seq, seq, seq, seq, seq, seq, seq, seq, + seq, seq, seq, seq, seq, seq, seq, seq +FROM seq_1_to_320; + +ANALYZE TABLE t1; + +# stats_method appended at the end of stat_description in case +# of non-default value +SELECT stat_name, stat_description FROM mysql.innodb_index_stats +where table_name="t1" and index_name="PRIMARY" +and stat_name="n_diff_pfx01"; + +# Truncation of stat_description +SELECT stat_name, stat_description FROM mysql.innodb_index_stats +where table_name="t1" and index_name="idx_all" +and stat_name="n_diff_pfx07"; + +DROP TABLE t1; + +# Test cardinality calculation with multi-level tree +CREATE TABLE t_multilevel ( + id INT AUTO_INCREMENT PRIMARY KEY, + col1 INT, + col2 INT, + col3 INT, + INDEX idx_col1 (col1), + INDEX idx_col2 (col2), + INDEX idx_col3 (col3) +) ENGINE=InnoDB STATS_PERSISTENT=1 STATS_SAMPLE_PAGES=20; + +# col1: 8000 NULLs + 2000 non-NULLs with 500 distinct values +# col2: 2000 NULLs + 8000 non-NULLs with 4000 distinct values +# col3: 5000 NULLs + 5000 non-NULLs with 2500 distinct values +INSERT INTO t_multilevel (col1, col2, col3) +SELECT + CASE WHEN seq <= 2000 THEN ((seq-1) % 500) + 1 ELSE NULL END, + CASE WHEN seq <= 8000 THEN ((seq-1) % 4000) + 1 ELSE NULL END, + CASE WHEN seq <= 5000 THEN ((seq-1) % 2500) + 1 ELSE NULL END +FROM seq_1_to_10000; + +ANALYZE TABLE t_multilevel; + +# Show detailed statistics for all columns +SELECT @@global.innodb_stats_method AS stats_method, + index_name, + stat_name, + stat_value +FROM mysql.innodb_index_stats +WHERE table_name='t_multilevel' + AND index_name IN ('idx_col1', 'idx_col2', 'idx_col3') + AND stat_name IN ('n_diff_pfx01', 'n_nonnull_fld01', 'n_leaf_pages') +ORDER BY index_name, stat_name; + +# Show cardinality from information_schema +SELECT @@global.innodb_stats_method AS stats_method, + index_name, + cardinality +FROM information_schema.STATISTICS +WHERE table_schema = 'test' + AND table_name = 't_multilevel' + AND index_name IN ('idx_col1', 'idx_col2', 'idx_col3') +ORDER BY index_name; + +DROP TABLE t_multilevel; + +# Test with NOT NULL columns to verify n_non_null_key_vals = n_diff_key_vals +CREATE TABLE t_not_null ( + id INT AUTO_INCREMENT PRIMARY KEY, + col_nullable INT, + col_not_null INT NOT NULL, + INDEX idx_nullable (col_nullable), + INDEX idx_not_null (col_not_null) +) ENGINE=InnoDB STATS_PERSISTENT=1; + +# Insert data: nullable has NULLs, not_null has no NULLs +INSERT INTO t_not_null (col_nullable, col_not_null) +SELECT + CASE WHEN seq <= 50 THEN ((seq-1) % 10) + 1 ELSE NULL END, + ((seq-1) % 10) + 1 +FROM seq_1_to_100; + +ANALYZE TABLE t_not_null; + +# For NOT NULL column: n_nonnull should equal n_diff +# For nullable column: n_nonnull should be less than total records +SELECT index_name, stat_name, stat_value +FROM mysql.innodb_index_stats +WHERE table_name='t_not_null' +AND stat_name IN ('n_diff_pfx01', 'n_nonnull_pfx01') +ORDER BY index_name, stat_name; + +DROP TABLE t_not_null; diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index d7ce6ae996ef9..25c102141c199 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -121,19 +121,6 @@ uint btr_cur_limit_optimistic_insert_debug; can be released by page reorganize, then it is reorganized */ #define BTR_CUR_PAGE_REORGANIZE_LIMIT (srv_page_size / 32) -/** The structure of a BLOB part header */ -/* @{ */ -/*--------------------------------------*/ -#define BTR_BLOB_HDR_PART_LEN 0 /*!< BLOB part len on this - page */ -#define BTR_BLOB_HDR_NEXT_PAGE_NO 4 /*!< next BLOB part page no, - FIL_NULL if none */ -/*--------------------------------------*/ -#define BTR_BLOB_HDR_SIZE 8 /*!< Size of a BLOB - part header, in bytes */ - -/* @} */ - /*******************************************************************//** Marks all extern fields in a record as owned by the record. This function should be called if the delete mark of a record is removed: a not delete @@ -5748,69 +5735,19 @@ ha_rows btr_estimate_n_rows_in_range(dict_index_t *index, /*================== EXTERNAL STORAGE OF BIG FIELDS ===================*/ -/***********************************************************//** -Gets the offset of the pointer to the externally stored part of a field. +/** Gets the offset of the pointer to the externally stored part of a field. +@param offsets offsets of record +@param n index of the external field @return offset of the pointer to the externally stored part */ -static -ulint -btr_rec_get_field_ref_offs( -/*=======================*/ - const rec_offs* offsets,/*!< in: array returned by rec_get_offsets() */ - ulint n) /*!< in: index of the external field */ +size_t btr_rec_get_field_ref_offs(const rec_offs *offsets, size_t n) noexcept { - ulint field_ref_offs; - ulint local_len; - - ut_a(rec_offs_nth_extern(offsets, n)); - field_ref_offs = rec_get_nth_field_offs(offsets, n, &local_len); - ut_a(len_is_stored(local_len)); - ut_a(local_len >= BTR_EXTERN_FIELD_REF_SIZE); + size_t local_len; + ut_ad(rec_offs_nth_extern(offsets, n)); + rec_offs field_ref_offs= rec_get_nth_field_offs(offsets, n, &local_len); + ut_ad(len_is_stored(local_len)); + ut_ad(local_len >= BTR_EXTERN_FIELD_REF_SIZE); - return(field_ref_offs + local_len - BTR_EXTERN_FIELD_REF_SIZE); -} - -/** Gets a pointer to the externally stored part of a field. -@param rec record -@param offsets rec_get_offsets(rec) -@param n index of the externally stored field -@return pointer to the externally stored part */ -#define btr_rec_get_field_ref(rec, offsets, n) \ - ((rec) + btr_rec_get_field_ref_offs(offsets, n)) - -/** Gets the externally stored size of a record, in units of a database page. -@param[in] rec record -@param[in] offsets array returned by rec_get_offsets() -@return externally stored part, in units of a database page */ -ulint -btr_rec_get_externally_stored_len( - const rec_t* rec, - const rec_offs* offsets) -{ - ulint n_fields; - ulint total_extern_len = 0; - ulint i; - - ut_ad(!rec_offs_comp(offsets) || !rec_get_node_ptr_flag(rec)); - - if (!rec_offs_any_extern(offsets)) { - return(0); - } - - n_fields = rec_offs_n_fields(offsets); - - for (i = 0; i < n_fields; i++) { - if (rec_offs_nth_extern(offsets, i)) { - - ulint extern_len = mach_read_from_4( - btr_rec_get_field_ref(rec, offsets, i) - + BTR_EXTERN_LEN + 4); - - total_extern_len += ut_calc_align( - extern_len, ulint(srv_page_size)); - } - } - - return total_extern_len >> srv_page_size_shift; + return field_ref_offs + local_len - BTR_EXTERN_FIELD_REF_SIZE; } /*******************************************************************//** @@ -6184,7 +6121,6 @@ btr_store_big_rec_extern_fields( for (i = 0; i < big_rec_vec->n_fields; i++) { const ulint field_no = big_rec_vec->fields[i].field_no; - field_ref = btr_rec_get_field_ref(rec, offsets, field_no); #if defined UNIV_DEBUG || defined UNIV_BLOB_LIGHT_DEBUG /* A zero BLOB pointer should have been initially inserted. */ diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index ec52352f6147c..c04e34944eb09 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -35,6 +35,8 @@ Created Jan 06, 2010 Vasil Dimov #include "que0que.h" #include "scope.h" #include "debug_sync.h" +#include "btr0cur.h" +#include "small_vector.h" #ifdef WITH_WSREP # include #endif @@ -44,84 +46,61 @@ Created Jan 06, 2010 Vasil Dimov #include #include -/* Sampling algorithm description @{ - -The algorithm is controlled by one number - N_SAMPLE_PAGES(index), -let it be A, which is the number of leaf pages to analyze for a given index -for each n-prefix (if the index is on 3 columns, then 3*A leaf pages will be -analyzed). - -Let the total number of leaf pages in the table be T. -Level 0 - leaf pages, level H - root. - -Definition: N-prefix-boring record is a record on a non-leaf page that equals -the next (to the right, cross page boundaries, skipping the supremum and -infimum) record on the same level when looking at the fist n-prefix columns. -The last (user) record on a level is not boring (it does not match the -non-existent user record to the right). We call the records boring because all -the records on the page below a boring record are equal to that boring record. - -We avoid diving below boring records when searching for a leaf page to -estimate the number of distinct records because we know that such a leaf -page will have number of distinct records == 1. - -For each n-prefix: start from the root level and full scan subsequent lower -levels until a level that contains at least A*10 distinct records is found. -Lets call this level LA. -As an optimization the search is canceled if it has reached level 1 (never -descend to the level 0 (leaf)) and also if the next level to be scanned -would contain more than A pages. The latter is because the user has asked -to analyze A leaf pages and it does not make sense to scan much more than -A non-leaf pages with the sole purpose of finding a good sample of A leaf -pages. - -After finding the appropriate level LA with >A*10 distinct records (or less in -the exceptions described above), divide it into groups of equal records and -pick A such groups. Then pick the last record from each group. For example, -let the level be: - -index: 0,1,2,3,4,5,6,7,8,9,10 -record: 1,1,1,2,2,7,7,7,7,7,9 - -There are 4 groups of distinct records and if A=2 random ones are selected, -e.g. 1,1,1 and 7,7,7,7,7, then records with indexes 2 and 9 will be selected. - -After selecting A records as described above, dive below them to find A leaf -pages and analyze them, finding the total number of distinct records. The -dive to the leaf level is performed by selecting a non-boring record from -each page and diving below it. - -This way, a total of A leaf pages are analyzed for the given n-prefix. - -Let the number of different key values found in each leaf page i be Pi (i=1..A). -Let N_DIFF_AVG_LEAF be (P1 + P2 + ... + PA) / A. -Let the number of different key values on level LA be N_DIFF_LA. -Let the total number of records on level LA be TOTAL_LA. -Let R be N_DIFF_LA / TOTAL_LA, we assume this ratio is the same on the -leaf level. -Let the number of leaf pages be N. -Then the total number of different key values on the leaf level is: -N * R * N_DIFF_AVG_LEAF. -See REF01 for the implementation. - -The above describes how to calculate the cardinality of an index. -This algorithm is executed for each n-prefix of a multi-column index -where n=1..n_uniq. -@} */ +/** Gets the externally stored size of a record, in units of a database page. +@param rec record +@param offsets array returned by rec_get_offsets() +@param table table containing the record +@return externally stored size in units of a database page */ +static uint32_t rec_get_n_blob_pages(const rec_t *rec, + const rec_offs *offsets, + const dict_table_t *table) noexcept +{ + ut_ad(!rec_offs_comp(offsets) || !rec_get_node_ptr_flag(rec)); + if (!rec_offs_any_extern(offsets)) + return 0; + uint32_t n_blobs= 0; + const uint32_t blob_part_size= (DICT_TF_MASK_ZIP_SSIZE & table->flags) + ? uint32_t(dict_tf_get_zip_size(table->flags)) - FIL_PAGE_DATA + : uint32_t(srv_page_size) - FIL_PAGE_DATA - + BTR_BLOB_HDR_SIZE - FIL_PAGE_DATA_END; + for (ulint i= rec_offs_n_fields(offsets); i--; ) + { + if (rec_offs_nth_extern(offsets, i)) + { + uint32_t len= + mach_read_from_4(btr_rec_get_field_ref(rec, offsets, i) + + BTR_EXTERN_LEN + 4); + n_blobs+= (len + blob_part_size - 1) / blob_part_size; + } + } + return n_blobs; +} + +/* Persistent-statistics sampling algorithm (overview). + +Estimates each index's cardinality by sampling A = N_SAMPLE_PAGES leaf pages +per n-column prefix (n = 1..n_uniq), rather than scanning all L leaf pages. + +Symbols: A sampled pages/prefix; L index leaf pages (L_ord = non-BLOB leaves); +LA the level chosen to sample from; see dict_stats_index_set_n_diff() [REF01] +for the final estimator and the meaning of TOTAL_LA, N_DIFF_LA, Pi, R. + +Call graph: + dict_stats_analyze_index() driver; loops over n_prefix + IndexLevelStats::analyze_level() full-scan one level, collect n_diff[] + IndexLevelStats::sample_leaf_pages() pick A records on LA, dive below each + PageStats::scan_below() descend to a leaf via non-boring recs + PageStats::scan() full leaf scan: distinct/non-null/BLOB + dict_stats_index_set_n_diff() [REF01] turn samples into cardinality */ /* names of the tables from the persistent statistics storage */ #define TABLE_STATS_NAME_PRINT "mysql.innodb_table_stats" #define INDEX_STATS_NAME_PRINT "mysql.innodb_index_stats" - -#ifdef UNIV_STATS_DEBUG -#define DEBUG_PRINTF(fmt, ...) printf(fmt, ## __VA_ARGS__) -#else /* UNIV_STATS_DEBUG */ #define DEBUG_PRINTF(fmt, ...) /* noop */ -#endif /* UNIV_STATS_DEBUG */ /* Gets the number of leaf pages to sample in persistent stats estimation */ #define N_SAMPLE_PAGES(index) \ - static_cast( \ + static_cast( \ (index)->table->stats_sample_pages != 0 \ ? (index)->table->stats_sample_pages \ : srv_stats_persistent_sample_pages) @@ -136,7 +115,7 @@ of keys. For example if a btree level is: index: 0,1,2,3,4,5,6,7,8,9,10,11,12 data: b,b,b,b,b,b,g,g,j,j,j, x, y then we would store 5,7,10,11,12 in the array. */ -typedef std::vector > boundaries_t; +typedef small_vector boundaries_t; /** Allocator type used for index_map_t. */ typedef ut_allocator > @@ -176,8 +155,10 @@ dict_stats_should_ignore_index( } +namespace { + /** expected column definition */ -struct dict_col_meta_t +struct ColMeta { /** column name */ const char *name; @@ -190,7 +171,7 @@ struct dict_col_meta_t }; /** For checking whether a table exists and has a predefined schema */ -struct dict_table_schema_t +struct TableSchema { /** table name */ span table_name; @@ -199,10 +180,12 @@ struct dict_table_schema_t /** number of columns */ unsigned n_cols; /** columns */ - const dict_col_meta_t columns[8]; + const ColMeta columns[8]; }; -static const dict_table_schema_t table_stats_schema = +} // anonymous namespace + +static const TableSchema table_stats_schema = { {C_STRING_WITH_LEN(TABLE_STATS_NAME)}, TABLE_STATS_NAME_PRINT, 6, { @@ -225,7 +208,7 @@ static const dict_table_schema_t table_stats_schema = } }; -static const dict_table_schema_t index_stats_schema = +static const TableSchema index_stats_schema = { {C_STRING_WITH_LEN(INDEX_STATS_NAME)}, INDEX_STATS_NAME_PRINT, 8, { @@ -347,7 +330,7 @@ static dberr_t dict_table_schema_check( /*====================*/ - const dict_table_schema_t* req_schema, /*!< in: required table + const TableSchema* req_schema, /*!< in: required table schema */ char* errstr, /*!< out: human readable error message if != DB_SUCCESS is @@ -702,15 +685,16 @@ dict_stats_assert_initialized( && strcmp((i1)->name, (i2)->name) == 0) /** Statistics for one field of an index. */ -struct index_field_stats_t +namespace { +struct IndexFieldStats { - ib_uint64_t n_diff_key_vals; - ib_uint64_t n_sample_sizes; - ib_uint64_t n_non_null_key_vals; + uint64_t n_diff_key_vals; + uint64_t n_sample_sizes; + uint64_t n_non_null_key_vals; - index_field_stats_t(ib_uint64_t n_diff_key_vals= 0, - ib_uint64_t n_sample_sizes= 0, - ib_uint64_t n_non_null_key_vals= 0) + IndexFieldStats(uint64_t n_diff_key_vals= 0, + uint64_t n_sample_sizes= 0, + uint64_t n_non_null_key_vals= 0) : n_diff_key_vals(n_diff_key_vals), n_sample_sizes(n_sample_sizes), n_non_null_key_vals(n_non_null_key_vals) { @@ -722,40 +706,23 @@ struct index_field_stats_t n_sample_sizes == UINT64_MAX && n_non_null_key_vals == UINT64_MAX; } }; +} // anonymous namespace -/*******************************************************************//** -Record the number of non_null key values in a given index for +/** Record the number of non_null key values in a given index for each n-column prefix of the index where 1 <= n <= dict_index_get_n_unique(index). The estimates are eventually stored in the array: -index->stat_n_non_null_key_vals[], which is indexed from 0 to n-1. */ -static -void -btr_record_not_null_field_in_rec( -/*=============================*/ - ulint n_unique, /*!< in: dict_index_get_n_unique(index), - number of columns uniquely determine - an index entry */ - const rec_offs* offsets, /*!< in: rec_get_offsets(rec, index), - its size could be for all fields or - that of "n_unique" */ - ib_uint64_t* n_not_null) /*!< in/out: array to record number of - not null rows for n-column prefix */ +index->stat_n_non_null_key_vals[], which is indexed from 0 to n-1. +@param n_unique number of unique column for an index +@param offsets offsets for all fields that of n_unique +@param n_non_null array to record number of non null rows for each individual column */ +__attribute__((nonnull)) +static void btr_record_not_null_field_in_rec(ulint n_unique, + const rec_offs *offsets, + uint64_t *n_not_null) noexcept { - ulint i; - - ut_ad(rec_offs_n_fields(offsets) >= n_unique); - - if (n_not_null == NULL) { - return; - } - - for (i = 0; i < n_unique; i++) { - if (rec_offs_nth_sql_null(offsets, i)) { - break; - } - - n_not_null[i]++; - } + ut_ad(rec_offs_n_fields(offsets) >= n_unique); + for (ulint i = 0; i < n_unique; i++) + n_not_null[i] += !rec_offs_nth_sql_null(offsets, i); } inline dberr_t @@ -820,7 +787,7 @@ btr_cur_t::open_random_leaf(rec_offs *&offsets, mem_heap_t *&heap, mtr_t &mtr) @param not_empty table not empty @return estimated table wide stats from sampled value */ #define BTR_TABLE_STATS_FROM_SAMPLE(value, index, sample, ext_size, not_empty) \ - (((value) * static_cast(index->stat_n_leaf_pages) \ + (((value) * static_cast(index->stat_n_leaf_pages) \ + (sample) - 1 + (ext_size) + (not_empty)) / ((sample) + (ext_size))) /** Estimates the number of different key values in a given index, for @@ -836,26 +803,26 @@ array result.n_non_null_key_vals. @return vector with statistics information empty vector if the index is unavailable. */ static -std::vector +std::vector btr_estimate_number_of_different_key_vals(dict_index_t* index, trx_id_t bulk_trx_id) { page_t* page; rec_t* rec; ulint n_cols; - ib_uint64_t* n_diff; - ib_uint64_t* n_not_null; + uint64_t* n_diff; + uint64_t* n_not_null; bool stats_null_not_equal; uint32_t n_sample_pages=1; /* number of pages to sample */ ulint not_empty_flag = 0; - ulint total_external_size = 0; + uint32_t n_blob_pages = 0; uintmax_t add_on; mtr_t mtr; mem_heap_t* heap = NULL; rec_offs* offsets_rec = NULL; rec_offs* offsets_next_rec = NULL; - std::vector result; + std::vector result; ut_ad(index->is_btree()); @@ -867,20 +834,17 @@ btr_estimate_number_of_different_key_vals(dict_index_t* index, * (sizeof *offsets_rec + sizeof *offsets_next_rec)); - n_diff = (ib_uint64_t*) mem_heap_zalloc( - heap, n_cols * sizeof(n_diff[0])); + n_diff = static_cast(mem_heap_zalloc( + heap, n_cols * sizeof(n_diff[0]))); - n_not_null = NULL; + n_not_null = static_cast(mem_heap_zalloc( + heap, n_cols * sizeof *n_not_null)); /* Check srv_innodb_stats_method setting, and decide whether we need to record non-null value and also decide if NULL is considered equal (by setting stats_null_not_equal value) */ switch (srv_innodb_stats_method) { case SRV_STATS_NULLS_IGNORED: - n_not_null = (ib_uint64_t*) mem_heap_zalloc( - heap, n_cols * sizeof *n_not_null); - /* fall through */ - case SRV_STATS_NULLS_UNEQUAL: /* for both SRV_STATS_NULLS_IGNORED and SRV_STATS_NULLS_UNEQUAL case, we will treat NULLs as unequal value */ @@ -986,10 +950,8 @@ btr_estimate_number_of_different_key_vals(dict_index_t* index, n_core, ULINT_UNDEFINED, &heap); - if (n_not_null != NULL) { - btr_record_not_null_field_in_rec( - n_cols, offsets_rec, n_not_null); - } + btr_record_not_null_field_in_rec( + n_cols, offsets_rec, n_not_null); } while (rec != page_get_supremum_rec(page)) { @@ -997,9 +959,8 @@ btr_estimate_number_of_different_key_vals(dict_index_t* index, rec_t* next_rec = page_rec_get_next(rec); if (!next_rec || next_rec == page_get_supremum_rec(page)) { - total_external_size += - btr_rec_get_externally_stored_len( - rec, offsets_rec); + n_blob_pages += rec_get_n_blob_pages( + rec, offsets_rec, index->table); break; } @@ -1021,14 +982,11 @@ btr_estimate_number_of_different_key_vals(dict_index_t* index, n_diff[j]++; } - if (n_not_null != NULL) { - btr_record_not_null_field_in_rec( - n_cols, offsets_next_rec, n_not_null); - } + btr_record_not_null_field_in_rec( + n_cols, offsets_next_rec, n_not_null); - total_external_size - += btr_rec_get_externally_stored_len( - rec, offsets_rec); + n_blob_pages += rec_get_n_blob_pages( + rec, offsets_rec, index->table); rec = next_rec; /* Initialize offsets_rec for the next round @@ -1071,12 +1029,12 @@ btr_estimate_number_of_different_key_vals(dict_index_t* index, result.reserve(n_cols); for (ulint j = 0; j < n_cols; j++) { - index_field_stats_t stat; + IndexFieldStats stat; stat.n_diff_key_vals = BTR_TABLE_STATS_FROM_SAMPLE( n_diff[j], index, n_sample_pages, - total_external_size, not_empty_flag); + n_blob_pages, not_empty_flag); /* If the tree is small, smaller than 10 * n_sample_pages + total_external_size, then @@ -1088,7 +1046,7 @@ btr_estimate_number_of_different_key_vals(dict_index_t* index, add_on = index->stat_n_leaf_pages / (10 * (n_sample_pages - + total_external_size)); + + n_blob_pages)); if (add_on > n_sample_pages) { add_on = n_sample_pages; @@ -1098,12 +1056,10 @@ btr_estimate_number_of_different_key_vals(dict_index_t* index, stat.n_sample_sizes = n_sample_pages; - if (n_not_null != NULL) { - stat.n_non_null_key_vals = - BTR_TABLE_STATS_FROM_SAMPLE( - n_not_null[j], index, n_sample_pages, - total_external_size, not_empty_flag); - } + stat.n_non_null_key_vals = + BTR_TABLE_STATS_FROM_SAMPLE( + n_not_null[j], index, n_sample_pages, + n_blob_pages, not_empty_flag); result.push_back(stat); } @@ -1191,7 +1147,7 @@ dict_stats_update_transient_for_index( /* Do not continue if table decryption has failed or table is already marked as corrupted. */ if (index->is_readable()) { - std::vector stats + std::vector stats = btr_estimate_number_of_different_key_vals( index, bulk_trx_id); @@ -1371,47 +1327,170 @@ static dberr_t btr_pcur_open_level(btr_pcur_t *pcur, ulint level, mtr_t *mtr, } -/* @{ Pseudo code about the relation between the following functions +namespace { -let N = N_SAMPLE_PAGES(index) +/** Input data that is used to calculate dict_index_t::stat_n_diff_key_vals[] +for each n-columns prefix (n from 1 to n_uniq). */ +struct NDiffData { + /** Index of the level on which the descent through the btree + stopped. level 0 is the leaf level. This is >= 1 because we + avoid scanning the leaf level because it may contain too many + pages and doing so is useless when combined with the random dives - + if we are to scan the leaf level, this means a full scan and we can + simply do that instead of fiddling with picking random records higher + in the tree and to dive below them. At the start of the analyzing + we may decide to do full scan of the leaf level, but then this + structure is not used in that code path. */ + ulint level; -dict_stats_analyze_index() - for each n_prefix - search for good enough level: - dict_stats_analyze_index_level() // only called if level has <= N pages - // full scan of the level in one mtr - collect statistics about the given level - if we are not satisfied with the level, search next lower level - we have found a good enough level here - dict_stats_analyze_index_for_n_prefix(that level, stats collected above) - // full scan of the level in one mtr - dive below some records and analyze the leaf page there: - dict_stats_analyze_index_below_cur() -@} */ + /** Number of records on the level where the descend through the btree + stopped. When we scan the btree from the root, we stop at some mid + level, choose some records from it and dive below them towards a leaf + page to analyze. */ + uint64_t n_recs_on_level; -/*********************************************************************//** -Find the total number and the number of distinct keys on a given level in -an index. Each of the 1..n_uniq prefixes are looked up and the results are -saved in the array n_diff[0] .. n_diff[n_uniq - 1]. The total number of -records on the level is saved in total_recs. -Also, the index of the last record in each group of equal records is saved -in n_diff_boundaries[0..n_uniq - 1], records indexing starts from the leftmost -record on the level and continues cross pages boundaries, counting from 0. */ -static -void -dict_stats_analyze_index_level( -/*===========================*/ - dict_index_t* index, /*!< in: index */ - ulint level, /*!< in: level */ - ib_uint64_t* n_diff, /*!< out: array for number of - distinct keys for all prefixes */ - ib_uint64_t* total_recs, /*!< out: total number of records */ - ib_uint64_t* total_pages, /*!< out: total number of pages */ - boundaries_t* n_diff_boundaries,/*!< out: boundaries of the groups - of distinct keys */ - mtr_t* mtr) /*!< in/out: mini-transaction */ + /** Number of different key values that were found on the mid level. */ + uint64_t n_diff_on_level; + + /** Number of leaf pages that are analyzed. This is also the same as + the number of records that we pick from the mid level and dive below + them. */ + uint64_t n_leaf_pages_to_analyze; + + /** Cumulative sum of the number of different key values that were + found on all analyzed pages. */ + uint64_t n_diff_all_analyzed_pages; + + /** Cumulative sum of the number of non-null key values that + were found on all analyzed pages. */ + uint64_t n_non_null_all_analyzed_pages; + + /** Cumulative sum of the number of external pages (stored outside of + the btree but in the same file segment). */ + uint64_t n_external_pages_sum; +}; + +/** Statistics collected for a single B-tree level +during index analysis. Used to track distinct key values, +record counts, and page information when analyzing index +levels for the query optimizer's statistics. */ +struct IndexLevelStats +{ +private: + /** Whether nulls are considered unequal for statistics */ + const bool nulls_unequal; + /** distance from the leaf level (0). */ + uint16_t level; +public: + /** Total number of pages */ + uint32_t n_pages; + /** Array for number of distinct keys */ + uint64_t *const n_diff; + /** Array of number of non-null keys for all prefixes */ + uint64_t *const n_non_null; + /** Boundaries of groups of distinct keys */ + boundaries_t *const bounds; + /** Total number of records */ + uint64_t n_recs; + /** Index */ + dict_index_t *const index; + /** Mini-transaction */ + mtr_t *const mtr; + + IndexLevelStats(bool nulls_unequal_, uint16_t level_, + uint64_t *n_diff_, uint64_t *n_non_null_, + boundaries_t *bounds_, dict_index_t *index_, + mtr_t *mtr_) : + nulls_unequal(nulls_unequal_), level(level_), n_pages(0), + n_diff(n_diff_), n_non_null(n_non_null_), bounds(bounds_), + n_recs(level_ ? 1 : 0), index(index_), mtr(mtr_) + { + } + + bool nulls_distinct() const { return nulls_unequal; } + + /** Get the current level number + @return level number */ + uint16_t get_level() const { return level; } + + /** Reset statistics for scanning a new level + @param[in] level_ new level number to scan */ + void reset_for_level(uint16_t level_) + { + level= level_; + n_recs= 0; + n_pages= 0; + for (ulint i= 0; i < index->n_uniq; i++) + bounds[i].clear(); + } + + void analyze_level() noexcept; + + void sample_leaf_pages(uint16_t n_prefix, + NDiffData *n_diff_data)const noexcept; +}; + +/** Statistics collected for a leaf level during index +analysis. Used to calculate the number of different key values, +number of blob pages, number of non-null values in an index +when looking at the first n_prefix columns. This will be +initialized in dict_stats_scan_page() */ +struct PageStats +{ + /** Collecting statistics for this Index */ + const dict_index_t *index; + /** Number of columns to consider for prefix statistics */ + const uint8_t n_prefix; + /** Whether NULL values are considered unequal for statistics */ + const bool nulls_unequal; + /** Number of externally stored blob pages encountered in scan */ + uint32_t n_blob= 0; + /** Number of distinct key values found for the first n_prefix columns. + On leaf pages this is the full count; on non-leaf pages it is capped at 2 + (0 = empty, 1 = boring/all-equal, 2 = non-boring). see scan(). */ + uint64_t n_diff= 0; + /** Number of non-null key values found for the prefix in scan + for the prefix */ + uint64_t n_non_null= 0; + + rec_offs *offsets1= nullptr; + + rec_offs *offsets2= nullptr; + + mem_heap_t *heap= nullptr; + + PageStats(dict_index_t *ind, uint8_t n_pfx, bool nulls) : + index(ind), n_prefix(n_pfx), nulls_unequal(nulls) + { + ulint size= (1 + REC_OFFS_HEADER_SIZE) + 1 + index->n_fields; + heap= mem_heap_create(size * (2 * sizeof (rec_offs))); + offsets1= static_cast( + mem_heap_alloc(heap, size * sizeof (*offsets1))); + offsets2= static_cast( + mem_heap_alloc(heap, size * sizeof (*offsets2))); + rec_offs_set_n_alloc(offsets1, size); + rec_offs_set_n_alloc(offsets2, size); + } + + void scan_below(const btr_cur_t *cur) noexcept; + + template + uint32_t scan(const page_t *page) noexcept; + + ~PageStats() { mem_heap_free(heap); } +}; + +} // anonymous namespace + +/** Full-scan one B-tree level in a single mtr. For every n-prefix (1..n_uniq) +compute n_diff[] (= N_DIFF_LA, distinct keys on the level) and n_recs +(= TOTAL_LA, records on the level), and record in bounds[] the index of the +last record of each group of equal keys (0-based, counted continuously across +pages). These boundaries let sample_leaf_pages() split the level into groups +and pick one representative record per group. +*/ +void IndexLevelStats::analyze_level() noexcept { - ulint n_uniq; mem_heap_t* heap; btr_pcur_t pcur; const page_t* page; @@ -1422,24 +1501,14 @@ dict_stats_analyze_index_level( ulint prev_rec_buf_size = 0; rec_offs* rec_offsets; rec_offs* prev_rec_offsets; - ulint i; - - DEBUG_PRINTF(" %s(table=%s, index=%s, level=" ULINTPF ")\n", - __func__, index->table->name, index->name, level); - - *total_recs = 0; - *total_pages = 0; - - n_uniq = dict_index_get_n_unique(index); - - /* elements in the n_diff array are 0..n_uniq-1 (inclusive) */ - memset(n_diff, 0x0, n_uniq * sizeof(n_diff[0])); + bool* col_nullable = nullptr; + ulint n_uniq = dict_index_get_n_unique(index); /* Allocate space for the offsets header (the allocation size at offsets[0] and the REC_OFFS_HEADER_SIZE bytes), and n_uniq + 1, so that this will never be less than the size calculated in rec_get_offsets_func(). */ - i = (REC_OFFS_HEADER_SIZE + 1 + 1) + n_uniq; + ulint i = (REC_OFFS_HEADER_SIZE + 1 + 1) + n_uniq; heap = mem_heap_create((2 * sizeof *rec_offsets) * i); rec_offsets = static_cast( @@ -1449,19 +1518,20 @@ dict_stats_analyze_index_level( rec_offs_set_n_alloc(rec_offsets, i); rec_offs_set_n_alloc(prev_rec_offsets, i); - /* reset the dynamic arrays n_diff_boundaries[0..n_uniq-1] */ - if (n_diff_boundaries != NULL) { - for (i = 0; i < n_uniq; i++) { - n_diff_boundaries[i].erase( - n_diff_boundaries[i].begin(), - n_diff_boundaries[i].end()); + uint16_t level= get_level(); + if (level == 0) { + col_nullable = static_cast( + mem_heap_alloc(heap, n_uniq * sizeof(bool))); + for (ulint i = 0; i < n_uniq; i++) { + col_nullable[i] = index->fields[i].col->is_nullable(); } } /* Position pcur on the leftmost record on the leftmost page on the desired level. */ - if (btr_pcur_open_level(&pcur, level, mtr, index) != DB_SUCCESS + if (btr_pcur_open_level(&pcur, level, mtr, + const_cast(index)) || !btr_pcur_move_to_next_on_page(&pcur)) { goto func_exit; } @@ -1512,8 +1582,7 @@ dict_stats_analyze_index_level( /* increment the pages counter at the end of each page */ if (rec_is_last_on_page) { - - (*total_pages)++; + n_pages++; } /* Skip delete-marked records on the leaf level. If we @@ -1547,10 +1616,22 @@ dict_stats_analyze_index_level( continue; } rec_offsets = rec_get_offsets(rec, index, rec_offsets, - level ? 0 : index->n_core_fields, + level + ? 0 + : index->n_core_fields, n_uniq, &heap); - (*total_recs)++; + n_recs++; + + if (n_non_null) { + ut_ad(col_nullable); + ut_ad(level == 0); + for (ulint i = 0; i < n_uniq; i++) { + n_non_null[i] += + col_nullable[i] && + !rec_offs_nth_sql_null(rec_offsets, i); + } + } if (prev_rec != NULL) { ulint matched_fields; @@ -1562,17 +1643,16 @@ dict_stats_analyze_index_level( cmp_rec_rec(prev_rec, rec, prev_rec_offsets, rec_offsets, index, - false, &matched_fields); + nulls_distinct(), + &matched_fields); for (i = matched_fields; i < n_uniq; i++) { - if (n_diff_boundaries != NULL) { + if (bounds != NULL) { /* push the index of the previous record, that is - the last one from a group of equal keys */ - ib_uint64_t idx; - /* the index of the current record is total_recs - 1, the index of the previous record is total_recs - 2; @@ -1581,9 +1661,7 @@ dict_stats_analyze_index_level( are in this branch then there is a previous record and thus total_recs >= 2 */ - idx = *total_recs - 2; - - n_diff_boundaries[i].push_back(idx); + bounds[i].emplace_back(n_recs - 2); } /* increment the number of different keys @@ -1608,9 +1686,9 @@ dict_stats_analyze_index_level( be on different pages and btr_cur_move_to_next_user_rec() will release the latch on the page that prev_rec is on */ - prev_rec = rec_copy_prefix_to_buf( - rec, index, n_uniq, - &prev_rec_buf, &prev_rec_buf_size); + prev_rec = rec_copy_prefix_to_buf(rec, index, n_uniq, + &prev_rec_buf, + &prev_rec_buf_size); prev_rec_is_copied = true; } else { @@ -1624,68 +1702,35 @@ dict_stats_analyze_index_level( } } - /* if *total_pages is left untouched then the above loop was not - entered at all and there is one page in the whole tree which is - empty or the loop was entered but this is level 0, contains one page - and all records are delete-marked */ - if (*total_pages == 0) { - + /* if n_pages is left untouched then + above loop was not entered at all and there is one page + in the whole tree which is empty or the loop was entered + but this is level 0, contains one page and all records + are delete-marked */ + if (n_pages == 0) { ut_ad(level == 0); - ut_ad(*total_recs == 0); - - *total_pages = 1; + ut_ad(n_recs == 0); + n_pages = 1; } /* if there are records on this level and boundaries should be saved */ - if (*total_recs > 0 && n_diff_boundaries != NULL) { - + if (n_recs > 0 && bounds != NULL) { /* remember the index of the last record on the level as the last one from the last group of equal keys; this holds for all possible prefixes */ - for (i = 0; i < n_uniq; i++) { - ib_uint64_t idx; - - idx = *total_recs - 1; - - n_diff_boundaries[i].push_back(idx); + for (ulint i = 0; i < n_uniq; i++) { + bounds[i].emplace_back(n_recs - 1); } } - /* now in n_diff_boundaries[i] there are exactly n_diff[i] integers, - for i=0..n_uniq-1 */ - -#ifdef UNIV_STATS_DEBUG - for (i = 0; i < n_uniq; i++) { - - DEBUG_PRINTF(" %s(): total recs: " UINT64PF - ", total pages: " UINT64PF - ", n_diff[" ULINTPF "]: " UINT64PF "\n", - __func__, *total_recs, - *total_pages, - i, n_diff[i]); - -#if 0 - if (n_diff_boundaries != NULL) { - ib_uint64_t j; - - DEBUG_PRINTF(" %s(): boundaries[%lu]: ", - __func__, i); - - for (j = 0; j < n_diff[i]; j++) { - ib_uint64_t idx; - - idx = n_diff_boundaries[i][j]; - - DEBUG_PRINTF(UINT64PF "=" UINT64PF ", ", - j, idx); + if (level == 0) { + for (ulint i = 0; i < n_uniq; i++) { + if (!col_nullable[i]) { + n_non_null[i]= n_recs; } - DEBUG_PRINTF("\n"); } -#endif } -#endif /* UNIV_STATS_DEBUG */ - func_exit: ut_free(prev_rec_buf); mem_heap_free(heap); @@ -1710,166 +1755,107 @@ page_rec_get_next_non_del_marked(const page_t *page, const rec_t *rec) return rec ? rec : page + (comp ? PAGE_NEW_SUPREMUM : PAGE_OLD_SUPREMUM); } -/** Scan a page, reading records from left to right and counting the number -of distinct records (looking only at the first n_prefix -columns) and the number of external pages pointed by records from this page. -If scan_method is QUIT_ON_FIRST_NON_BORING then the function -will return as soon as it finds a record that does not match its neighbor -to the right, which means that in the case of QUIT_ON_FIRST_NON_BORING the -returned n_diff can either be 0 (empty page), 1 (the whole page has all keys -equal) or 2 (the function found a non-boring record and returned). -@param[out] out_rec record, or NULL -@param[out] offsets1 rec_get_offsets() working space (must -be big enough) -@param[out] offsets2 rec_get_offsets() working space (must -be big enough) -@param[in] index index of the page -@param[in] page the page to scan -@param[in] n_prefix look at the first n_prefix columns -@param[in] n_core 0, or index->n_core_fields for leaf -@param[out] n_diff number of distinct records encountered -@param[out] n_external_pages if this is non-NULL then it will be set -to the number of externally stored pages which were encountered -@return offsets1 or offsets2 (the offsets of *out_rec), -or NULL if the page is empty and does not contain user records. */ -UNIV_INLINE -rec_offs* -dict_stats_scan_page( - const rec_t** out_rec, - rec_offs* offsets1, - rec_offs* offsets2, - const dict_index_t* index, - const page_t* page, - ulint n_prefix, - ulint n_core, - ib_uint64_t* n_diff, - ib_uint64_t* n_external_pages) +/** Scan a page left to right, counting distinct records for the first n_prefix +columns (and, on leaf pages, non-null values and external/BLOB pages). + +An "n-prefix-boring" record is a non-leaf record equal to the next record to +its right on the same level (crossing pages, skipping infimum/supremum) in the +first n_prefix columns; the last user record on a level is never boring. Every +record in the subtree below a boring record is equal to it, so that subtree +contributes exactly 1 distinct key which is why scan_below() never descends +through boring records. +@tparam leaf true = leaf scan: count ALL distinct keys (per nulls_unequal), + non-null values and BLOB page counts; + false = non-leaf scan: quick boring-detection that stops at the + first non-boring record, so n_diff ends as 0 (empty), + 1 (all keys equal / boring page) or 2 (non-boring found) +@param page index page to scan +@retval 0 if leaf=true OR if page is empty +@retval child_page_no if leaf=false and page contains records */ +template +uint32_t PageStats::scan(const page_t *page)noexcept { rec_offs* offsets_rec = offsets1; rec_offs* offsets_next_rec = offsets2; - const rec_t* rec; const rec_t* next_rec; - /* A dummy heap, to be passed to rec_get_offsets(). - Because offsets1,offsets2 should be big enough, - this memory heap should never be used. */ - mem_heap_t* heap = NULL; + const ulint n_core = leaf ? index->n_core_fields : 0; ut_ad(!!n_core == page_is_leaf(page)); + ulint n_recs = 1; const rec_t* (*get_next)(const page_t*, const rec_t*) = !n_core || srv_stats_include_delete_marked ? (page_is_comp(page) ? page_rec_next_get : page_rec_next_get) : page_is_comp(page) - ? page_rec_get_next_non_del_marked - : page_rec_get_next_non_del_marked; - - const bool should_count_external_pages = n_external_pages != NULL; - - if (should_count_external_pages) { - *n_external_pages = 0; - } - - rec = get_next(page, page_get_infimum_rec(page)); - + ? page_rec_get_next_non_del_marked + : page_rec_get_next_non_del_marked; + bool nullable = index->fields[n_prefix - 1].col->is_nullable(); + const rec_t *rec = get_next(page, page_get_infimum_rec(page)); + ut_ad(!n_blob); if (!rec || rec == page_get_supremum_rec(page)) { /* the page is empty or contains only delete-marked records */ - *n_diff = 0; - *out_rec = NULL; - return(NULL); + n_diff = 0; + return 0; } - offsets_rec = rec_get_offsets(rec, index, offsets_rec, n_core, ULINT_UNDEFINED, &heap); - - if (should_count_external_pages) { - *n_external_pages += btr_rec_get_externally_stored_len( - rec, offsets_rec); + if (leaf) { + n_blob += rec_get_n_blob_pages(rec, offsets_rec, index->table); + n_non_null += static_cast( + nullable && !rec_offs_nth_sql_null(offsets_rec, + n_prefix - 1)); } next_rec = get_next(page, rec); - - *n_diff = 1; - + n_diff = 1; while (next_rec && next_rec != page_get_supremum_rec(page)) { - ulint matched_fields; + n_recs++; + offsets_next_rec = rec_get_offsets(next_rec, index, offsets_next_rec, + n_core, ULINT_UNDEFINED, &heap); - offsets_next_rec = rec_get_offsets(next_rec, index, - offsets_next_rec, n_core, - ULINT_UNDEFINED, - &heap); - - /* check whether rec != next_rec when looking at - the first n_prefix fields */ + /* check whether rec != next_rec when looking at the first + n_prefix fields */ cmp_rec_rec(rec, next_rec, offsets_rec, offsets_next_rec, - index, false, &matched_fields); - + index, nulls_unequal, &matched_fields); if (matched_fields < n_prefix) { /* rec != next_rec, => rec is non-boring */ - - (*n_diff)++; - + n_diff++; if (!n_core) { break; } } rec = next_rec; - /* Assign offsets_rec = offsets_next_rec so that - offsets_rec matches with rec which was just assigned - rec = next_rec above. Also need to point - offsets_next_rec to the place where offsets_rec was - pointing before because we have just 2 placeholders - where data is actually stored: offsets1 and offsets2 - and we are using them in circular fashion - (offsets[_next]_rec are just pointers to those - placeholders). */ + /* Assign offsets_rec = offsets_next_rec so that offsets_rec matches + with rec which was just assigned rec = next_rec above. Also need to + point offsets_next_rec to the place where offsets_rec was pointing + before because we have just 2 placeholders where data is actually + stored: offsets1 and offsets2 and we are using them in circular fashion + (offsets[_next]_rec are just pointers to those placeholders). */ std::swap(offsets_rec, offsets_next_rec); - if (should_count_external_pages) { - *n_external_pages += btr_rec_get_externally_stored_len( - rec, offsets_rec); + if (leaf) { + n_blob += rec_get_n_blob_pages(rec, offsets_rec, + index->table); + n_non_null += nullable && + !rec_offs_nth_sql_null(offsets_rec, + n_prefix - 1); } - next_rec = get_next(page, next_rec); } - /* offsets1,offsets2 should have been big enough */ - ut_a(heap == NULL); - *out_rec = rec; - return(offsets_rec); + n_non_null = nullable ? n_non_null : n_recs; + return leaf ? 0 : btr_node_ptr_get_child_page_no(rec, offsets_rec); } -/** Dive below the current position of a cursor and calculate the number of -distinct records on the leaf page, when looking at the fist n_prefix -columns. Also calculate the number of external pages pointed by records -on the leaf page. -@param[in] cur cursor -@param[in] n_prefix look at the first n_prefix columns -when comparing records -@param[out] n_diff number of distinct records -@param[out] n_external_pages number of external pages -@return number of distinct records on the leaf page */ -static -void -dict_stats_analyze_index_below_cur( - const btr_cur_t* cur, - ulint n_prefix, - ib_uint64_t* n_diff, - ib_uint64_t* n_external_pages) +/** Descend from a record on level LA to a leaf page and scan it. +At each non-leaf level follow a non-boring child (a boring subtree is all-equal +and would only add 1 distinct key); stop early if the page is boring +(n_diff == 1), since no dive is needed. On the reached leaf, scan() +accumulates Pi (distinct records for the first n_prefix columns), non-null +counts, and the external/BLOB pages pointed to by records on that leaf. */ +void PageStats::scan_below(const btr_cur_t *cur) noexcept { - dict_index_t* index; - buf_block_t* block; - const page_t* page; - mem_heap_t* heap; - const rec_t* rec; - rec_offs* offsets1; - rec_offs* offsets2; - rec_offs* offsets_rec; - ulint size; - mtr_t mtr; - - index = btr_cur_get_index(cur); - /* Allocate offsets for the record and the node pointer, for node pointer records. In a secondary index, the node pointer record will consist of all index fields followed by a child @@ -1878,48 +1864,28 @@ dict_stats_analyze_index_below_cur( offsets[0] and the REC_OFFS_HEADER_SIZE bytes), and n_fields + 1, so that this will never be less than the size calculated in rec_get_offsets_func(). */ - size = (1 + REC_OFFS_HEADER_SIZE) + 1 + dict_index_get_n_fields(index); - - heap = mem_heap_create(size * (sizeof *offsets1 + sizeof *offsets2)); - - offsets1 = static_cast(mem_heap_alloc( - heap, size * sizeof *offsets1)); - - offsets2 = static_cast(mem_heap_alloc( - heap, size * sizeof *offsets2)); - - rec_offs_set_n_alloc(offsets1, size); - rec_offs_set_n_alloc(offsets2, size); - - rec = btr_cur_get_rec(cur); - page = btr_cur_get_page(cur); + rec_t* rec = btr_cur_get_rec(cur); + const page_t* page = btr_cur_get_page(cur); ut_ad(!page_is_leaf(page)); - - offsets_rec = rec_get_offsets(rec, index, offsets1, 0, - ULINT_UNDEFINED, &heap); - - page_id_t page_id(index->table->space_id, - btr_node_ptr_get_child_page_no( - rec, offsets_rec)); - const ulint zip_size = index->table->space->zip_size(); - - /* assume no external pages by default - in case we quit from this - function without analyzing any leaf pages */ - *n_external_pages = 0; - + rec_offs* offsets_rec = rec_get_offsets(rec, index, offsets1, 0, + ULINT_UNDEFINED, &heap); + page_id_t page_id(index->table->space_id, + btr_node_ptr_get_child_page_no(rec, offsets_rec)); + const ulint zip_size = index->table->space->zip_size(); + const bool may_ibuf_exist = !index->is_clust() && !index->is_unique(); + mtr_t mtr; mtr_start(&mtr); /* descend to the leaf level on the B-tree */ for (;;) { - dberr_t err; - - block = buf_page_get_gen(page_id, zip_size, - RW_S_LATCH, NULL, BUF_GET, - &mtr, &err, - !index->is_clust() - && 1 == btr_page_get_level(page)); + dberr_t err; + buf_block_t* block = buf_page_get_gen( + page_id, zip_size, RW_S_LATCH, nullptr, + BUF_GET, &mtr, &err, + may_ibuf_exist && 1 == btr_page_get_level(page)); if (!block) { - goto func_exit; + mtr_commit(&mtr); + return; } page = block->page.frame; @@ -1928,153 +1894,75 @@ dict_stats_analyze_index_below_cur( /* leaf level */ break; } - /* else */ /* search for the first non-boring record on the page */ - offsets_rec = dict_stats_scan_page( - &rec, offsets1, offsets2, index, page, n_prefix, - 0, n_diff, NULL); + uint32_t child_page_no = scan(page); - /* pages on level > 0 are not allowed to be empty */ - ut_a(offsets_rec != NULL); + /* level > 0, so child page number shouldn't be 0 */ + ut_a(child_page_no != 0); /* if page is not empty (offsets_rec != NULL) then n_diff must be > 0, otherwise there is a bug in dict_stats_scan_page() */ - ut_a(*n_diff > 0); + ut_a(n_diff > 0); - if (*n_diff == 1) { + if (n_diff == 1) { mtr_commit(&mtr); - - /* page has all keys equal and the end of the page - was reached by dict_stats_scan_page(), no need to - descend to the leaf level */ - mem_heap_free(heap); - /* can't get an estimate for n_external_pages here - because we do not dive to the leaf level, assume no - external pages (*n_external_pages was assigned to 0 - above). */ + /* page has all keys equal and the end of the page was reached by + dict_stats_scan_page(), no need to descend to the leaf level */ + /* can't get an estimate for n_external_pages here because we + do not dive to the leaf level, assume no external pages + (*n_external_pages was assigned to 0 above). */ return; } - /* else */ /* when we instruct dict_stats_scan_page() to quit on the first non-boring record it finds, then the returned n_diff can either be 0 (empty page), 1 (page has all keys equal) or 2 (non-boring record was found) */ - ut_a(*n_diff == 2); - + ut_a(n_diff == 2); /* we have a non-boring record in rec, descend below it */ - - page_id.set_page_no( - btr_node_ptr_get_child_page_no(rec, offsets_rec)); + page_id.set_page_no(child_page_no); } /* make sure we got a leaf page as a result from the above loop */ ut_ad(page_is_leaf(page)); + /* scan the leaf page and find the number of distinct keys, when looking + only at the first n_prefix columns; also estimate the number of externally + stored pages pointed by records on this page */ + scan(page); - /* scan the leaf page and find the number of distinct keys, - when looking only at the first n_prefix columns; also estimate - the number of externally stored pages pointed by records on this - page */ - - offsets_rec = dict_stats_scan_page( - &rec, offsets1, offsets2, index, page, n_prefix, - index->n_core_fields, n_diff, - n_external_pages); - -#if 0 - DEBUG_PRINTF(" %s(): n_diff below page_no=%lu: " UINT64PF "\n", - __func__, page_no, n_diff); -#endif - -func_exit: mtr_commit(&mtr); - mem_heap_free(heap); } -/** Input data that is used to calculate dict_index_t::stat_n_diff_key_vals[] -for each n-columns prefix (n from 1 to n_uniq). */ -struct n_diff_data_t { - /** Index of the level on which the descent through the btree - stopped. level 0 is the leaf level. This is >= 1 because we - avoid scanning the leaf level because it may contain too many - pages and doing so is useless when combined with the random dives - - if we are to scan the leaf level, this means a full scan and we can - simply do that instead of fiddling with picking random records higher - in the tree and to dive below them. At the start of the analyzing - we may decide to do full scan of the leaf level, but then this - structure is not used in that code path. */ - ulint level; - - /** Number of records on the level where the descend through the btree - stopped. When we scan the btree from the root, we stop at some mid - level, choose some records from it and dive below them towards a leaf - page to analyze. */ - ib_uint64_t n_recs_on_level; - - /** Number of different key values that were found on the mid level. */ - ib_uint64_t n_diff_on_level; - - /** Number of leaf pages that are analyzed. This is also the same as - the number of records that we pick from the mid level and dive below - them. */ - ib_uint64_t n_leaf_pages_to_analyze; - - /** Cumulative sum of the number of different key values that were - found on all analyzed pages. */ - ib_uint64_t n_diff_all_analyzed_pages; - - /** Cumulative sum of the number of external pages (stored outside of - the btree but in the same file segment). */ - ib_uint64_t n_external_pages_sum; -}; - /** Estimate the number of different key values in an index when looking at -the first n_prefix columns. For a given level in an index select -n_diff_data->n_leaf_pages_to_analyze records from that level and dive below -them to the corresponding leaf pages, then scan those leaf pages and save the -sampling results in n_diff_data->n_diff_all_analyzed_pages. -@param[in] index index +the first n_prefix columns. Using the group boundaries from analyze_level(), +split level LA into n_leaf_pages_to_analyze (<= A) groups of equal keys, pick +the last record of each group and dive below it (scan_below()). Accumulate over +the samples: n_diff_all_analyzed_pages (= sum of Pi), n_non_null_all_analyzed_pages +and n_external_pages_sum (= E, external/BLOB pages). @param[in] n_prefix look at first 'n_prefix' columns when comparing records -@param[in] boundaries a vector that contains -n_diff_data->n_diff_on_level integers each of which represents the index (on -level 'level', counting from left/smallest to right/biggest from 0) of the -last record from each group of distinct keys @param[in,out] n_diff_data n_diff_all_analyzed_pages and -n_external_pages_sum in this structure will be set by this function. The -members level, n_diff_on_level and n_leaf_pages_to_analyze must be set by the -caller in advance - they are used by some calculations inside this function -@param[in,out] mtr mini-transaction */ -static +n_external_pages_sum in this structure will be set by this function. +The members level, n_diff_on_level and n_leaf_pages_to_analyze +must be set by the caller in advance - they are used by +some calculations inside this function */ void -dict_stats_analyze_index_for_n_prefix( - dict_index_t* index, - ulint n_prefix, - const boundaries_t* boundaries, - n_diff_data_t* n_diff_data, - mtr_t* mtr) +IndexLevelStats::sample_leaf_pages( + uint16_t n_prefix, + NDiffData* n_diff_data)const noexcept { btr_pcur_t pcur; const page_t* page; - ib_uint64_t rec_idx; - ib_uint64_t i; - -#if 0 - DEBUG_PRINTF(" %s(table=%s, index=%s, level=%lu, n_prefix=%lu," - " n_diff_on_level=" UINT64PF ")\n", - __func__, index->table->name, index->name, level, - n_prefix, n_diff_data->n_diff_on_level); -#endif + uint64_t rec_idx; + uint64_t i; ut_ad(n_diff_data->level); /* Position pcur on the leftmost record on the leftmost page on the desired level. */ - n_diff_data->n_diff_all_analyzed_pages = 0; - n_diff_data->n_external_pages_sum = 0; - - if (btr_pcur_open_level(&pcur, n_diff_data->level, mtr, index) + if (btr_pcur_open_level(&pcur, n_diff_data->level, mtr, + index) != DB_SUCCESS || !btr_pcur_move_to_next_on_page(&pcur)) { return; @@ -2095,8 +1983,8 @@ dict_stats_analyze_index_for_n_prefix( return; } - const ib_uint64_t last_idx_on_level = boundaries->at( - static_cast(n_diff_data->n_diff_on_level - 1)); + const uint64_t last_idx_on_level = bounds[n_prefix - 1][ + unsigned(n_diff_data->n_diff_on_level - 1)]; rec_idx = 0; @@ -2129,12 +2017,12 @@ dict_stats_analyze_index_for_n_prefix( then we select a random record from each segment and dive below it */ - const ib_uint64_t n_diff = n_diff_data->n_diff_on_level; - const ib_uint64_t n_pick + const uint64_t n_diff = n_diff_data->n_diff_on_level; + const uint64_t n_pick = n_diff_data->n_leaf_pages_to_analyze; - const ib_uint64_t left = n_diff * i / n_pick; - const ib_uint64_t right = n_diff * (i + 1) / n_pick - 1; + const uint64_t left = n_diff * i / n_pick; + const uint64_t right = n_diff * (i + 1) / n_pick - 1; ut_a(left <= right); ut_a(right <= last_idx_on_level); @@ -2142,8 +2030,8 @@ dict_stats_analyze_index_for_n_prefix( const ulint rnd = ut_rnd_interval( static_cast(right - left)); - const ib_uint64_t dive_below_idx - = boundaries->at(static_cast(left + rnd)); + const uint64_t dive_below_idx + = bounds[n_prefix - 1][unsigned(left + rnd)]; #if 0 DEBUG_PRINTF(" %s(): dive below record with index=" @@ -2178,13 +2066,10 @@ dict_stats_analyze_index_for_n_prefix( ut_a(rec_idx == dive_below_idx); - ib_uint64_t n_diff_on_leaf_page; - ib_uint64_t n_external_pages; + PageStats *leaf_stats = new PageStats(index, (uint8_t)n_prefix, + nulls_distinct()); - dict_stats_analyze_index_below_cur(btr_pcur_get_btr_cur(&pcur), - n_prefix, - &n_diff_on_leaf_page, - &n_external_pages); + leaf_stats->scan_below(btr_pcur_get_btr_cur(&pcur)); /* We adjust n_diff_on_leaf_page here to avoid counting one value twice - once as the last on some page and once @@ -2200,28 +2085,38 @@ dict_stats_analyze_index_for_n_prefix( 2 distinct records per page (average). Having 4 pages below non-boring records, it would (wrongly) estimate the number of distinct records to 8. */ - if (n_diff_on_leaf_page > 0) { - n_diff_on_leaf_page--; + if (leaf_stats->n_diff > 0) { + leaf_stats->n_diff--; } - n_diff_data->n_diff_all_analyzed_pages += n_diff_on_leaf_page; + n_diff_data->n_diff_all_analyzed_pages += + leaf_stats->n_diff; + + /* n_non_null is a per-record count, so no boundary + double-count adjustment (unlike n_diff above). */ + n_diff_data->n_non_null_all_analyzed_pages += + leaf_stats->n_non_null; - n_diff_data->n_external_pages_sum += n_external_pages; + n_diff_data->n_external_pages_sum += + leaf_stats->n_blob; + + delete leaf_stats; } } /** statistics for an index */ -struct index_stats_t +namespace { +struct IndexStats { - std::vector stats; + std::vector stats; uint32_t index_size; uint32_t n_leaf_pages; - index_stats_t(ulint n_uniq) : index_size(1), n_leaf_pages(1) + IndexStats(ulint n_uniq) : index_size(1), n_leaf_pages(1) { stats.reserve(n_uniq); for (ulint i= 0; i < n_uniq; ++i) - stats.push_back(index_field_stats_t{0, 1, 0}); + stats.push_back(IndexFieldStats{0, 1, 0}); } void set_bulk_operation() @@ -2237,15 +2132,47 @@ struct index_stats_t return true; } }; +} // anonymous namespace -/** Set dict_index_t::stat_n_diff_key_vals[] and stat_n_sample_sizes[]. +/** Save the persistent statistics of a table or an index. +@param table table whose stats to save +@param stats_method innodb_stats_method variable value +@param index_id Index ID to save statistics for (0=all) +@return DB_SUCCESS or error code */ +static +dberr_t dict_stats_save(dict_table_t *table, unsigned stats_method, + index_id_t index_id= 0) noexcept; + +/** [REF01] Derive stat_n_diff_key_vals[] / stat_n_non_null_key_vals[] and +stat_n_sample_sizes[] from the sampled data. + +Per n-prefix, on the sampled level LA (see dict_stats_analyze_index()): + TOTAL_LA = records on LA (n_recs_on_level) + N_DIFF_LA= distinct keys on LA (n_diff_on_level) + Pi = distinct keys on the i-th sampled leaf page (i = 1..A) + D = ordinary leaves analyzed (n_leaf_pages_to_analyze) + E = external/BLOB pages linked from those D leaves (n_external_pages_sum) + L = total index leaf pages (index_stats.n_leaf_pages) + + R = N_DIFF_LA / TOTAL_LA distinctness ratio on LA, assumed + to also hold at the leaf level + avg(Pi) = n_diff_all_analyzed_pages / D + L_ord = L * D/(D+E) ordinary (non-BLOB) leaf pages + n_diff = L_ord * R * avg(Pi) + +BLOB pages live in BTR_SEG_LEAF, so they are included in L; the D/(D+E) factor +removes them, giving an estimate over data-bearing leaves only. n_non_null uses +the same L_ord scaling applied to the per-sample non-null counts, so for a +NOT NULL column it yields the estimated record count. +@param[in] index index @param[in] n_diff_data input data to use to derive the results @param[in,out] index_stats index stats to set */ UNIV_INLINE void dict_stats_index_set_n_diff( - const n_diff_data_t* n_diff_data, - index_stats_t& index_stats) + const dict_index_t* index, + const NDiffData* n_diff_data, + IndexStats& index_stats) { for (ulint n_prefix = index_stats.stats.size(); n_prefix >= 1; @@ -2256,12 +2183,12 @@ dict_stats_index_set_n_diff( 0 to index->stat_n_diff_key_vals[n_prefix - 1], which the formula below does. */ - const n_diff_data_t* data = &n_diff_data[n_prefix - 1]; + const NDiffData* data = &n_diff_data[n_prefix - 1]; ut_ad(data->n_leaf_pages_to_analyze > 0); ut_ad(data->n_recs_on_level > 0); - ib_uint64_t n_ordinary_leaf_pages; + uint64_t n_ordinary_leaf_pages; if (data->level == 1) { /* If we know the number of records on level 1, then @@ -2273,10 +2200,10 @@ dict_stats_index_set_n_diff( external pages in total linked from those D ordinary leaf pages, then this means that the ratio ordinary/external is D/E. Then the ratio ordinary/total - is D / (D + E). Knowing that the total number of pages - is T (including ordinary and external) then we estimate - that the total number of ordinary leaf pages is - T * D / (D + E). */ + is D / (D + E). Knowing that the total number of leaf + pages is L (including ordinary and external) then we + estimate that the number of ordinary leaf pages is + L_ord = L * D / (D + E). */ n_ordinary_leaf_pages = index_stats.n_leaf_pages * data->n_leaf_pages_to_analyze @@ -2284,7 +2211,7 @@ dict_stats_index_set_n_diff( + data->n_external_pages_sum); } - /* See REF01 for an explanation of the algorithm */ + /* n_diff = L_ord * R * avg(Pi); see [REF01] above */ index_stats.stats[n_prefix - 1].n_diff_key_vals = n_ordinary_leaf_pages @@ -2294,6 +2221,21 @@ dict_stats_index_set_n_diff( * data->n_diff_all_analyzed_pages / data->n_leaf_pages_to_analyze; + /* Calculate n_non_null_key_vals from sampled leaf pages. + Non-NULL values are counted only at the leaf level, because + non-leaf pages hold node pointers rather than data records and + NULL tracking is only meaningful for actual records. The ratio + of non-NULL values is assumed consistent across leaf pages, so + sampling gives an accurate estimate for the whole index. We use + the same formula as n_diff: multiply the average non-NULL count + per sampled leaf by the number of ordinary leaf pages. For a + NOT NULL column every record is counted, so this yields the + estimated record count (i.e. no NULLs). */ + index_stats.stats[n_prefix - 1].n_non_null_key_vals + = n_ordinary_leaf_pages + * data->n_non_null_all_analyzed_pages + / data->n_leaf_pages_to_analyze; + index_stats.stats[n_prefix - 1].n_sample_sizes = data->n_leaf_pages_to_analyze; @@ -2313,20 +2255,34 @@ dict_stats_index_set_n_diff( } } -/** Calculates new statistics for a given index and saves them to the index -members stat_n_diff_key_vals[], stat_n_sample_sizes[], stat_index_size and -stat_n_leaf_pages. This function can be slow. -@param[in] index index to analyze +/** Calculate and persist statistics for one index by sampling. Saves the +results into stat_n_diff_key_vals[], stat_n_sample_sizes[], stat_index_size +and stat_n_leaf_pages. This function can be slow. + +For each n-prefix (n = 1..n_uniq): + 1. Descend from the root, full-scanning each level (analyze_level()) until a + level LA has >= 10*A distinct keys for this prefix. The search stops early + at level 1 (never scan the leaf level) or if the next level would exceed A + pages then scanning many more than A non-leaf pages to sample A leaf pages is + not worthwhile. + 2. Divide LA into groups of equal keys, pick A representative records, and + dive below each to a leaf page (sample_leaf_pages()). + 3. Derive the estimate from the samples (dict_stats_index_set_n_diff()). + +Shortcut: tiny indexes (root_level == 0, or A*n_uniq > L) are scanned in full +instead of sampled, which is both faster and exact. +@param[in] index index to analyze +@param[in] stats_method innodb_stats_method variable @return index stats */ -static index_stats_t dict_stats_analyze_index(dict_index_t* index) +static IndexStats dict_stats_analyze_index(dict_index_t *index, + unsigned stats_method) { bool level_is_analyzed; - ulint n_uniq; - ulint n_prefix; - ib_uint64_t total_recs; - ib_uint64_t total_pages; mtr_t mtr; - index_stats_t result(index->n_uniq); + IndexStats result(index->n_uniq); + uint64_t n_sample_pages = N_SAMPLE_PAGES(index); + uint64_t n_diff_required = N_DIFF_REQUIRED(index); + DBUG_ENTER("dict_stats_analyze_index"); DBUG_PRINT("info", ("index: %s, online status: %d", index->name(), @@ -2373,7 +2329,7 @@ static index_stats_t dict_stats_analyze_index(dict_index_t* index) mtr.start(); mtr_sx_lock_index(index, &mtr); - n_uniq = dict_index_get_n_unique(index); + uint16_t n_uniq = dict_index_get_n_unique(index); /* If the tree has just one level (and one page) or if the user has requested to sample too many pages then do full scan. @@ -2385,7 +2341,7 @@ static index_stats_t dict_stats_analyze_index(dict_index_t* index) since it will be faster and will give better results. */ if (root_level == 0 - || N_SAMPLE_PAGES(index) * n_uniq > result.n_leaf_pages) { + || n_sample_pages * n_uniq > result.n_leaf_pages) { if (root_level == 0) { DEBUG_PRINTF(" %s(): just one page," @@ -2398,46 +2354,51 @@ static index_stats_t dict_stats_analyze_index(dict_index_t* index) /* do full scan of level 0; save results directly into the index */ - dict_stats_analyze_index_level(index, - 0 /* leaf level */, - index->stat_n_diff_key_vals, - &total_recs, - &total_pages, - NULL /* boundaries not needed */, - &mtr); + IndexLevelStats leaf_stats( + stats_method > SRV_STATS_NULLS_EQUAL, 0, + index->stat_n_diff_key_vals, + index->stat_n_non_null_key_vals, + nullptr, index, &mtr); + + memset(index->stat_n_diff_key_vals, 0x0, + n_uniq * sizeof(uint64_t)); + if (index->stat_n_non_null_key_vals) { + memset(index->stat_n_non_null_key_vals, + 0x0, n_uniq * sizeof(uint64_t)); + } + + leaf_stats.analyze_level(); mtr.commit(); index->table->stats_mutex_lock(); for (ulint i = 0; i < n_uniq; i++) { - result.stats[i].n_diff_key_vals = index->stat_n_diff_key_vals[i]; - result.stats[i].n_sample_sizes = total_pages; - result.stats[i].n_non_null_key_vals = index->stat_n_non_null_key_vals[i]; + result.stats[i].n_diff_key_vals = + index->stat_n_diff_key_vals[i]; + result.stats[i].n_sample_sizes = + leaf_stats.n_pages; + result.stats[i].n_non_null_key_vals = + index->stat_n_non_null_key_vals[i]; } - result.n_leaf_pages = index->stat_n_leaf_pages; + /* For multi-level indexes, use the actual number + of leaf pages counted during the full scan. + For single-page indexes (root_level == 0), + use the pre-calculated stat_n_leaf_pages which + is always 1. */ + result.n_leaf_pages = (root_level != 0) + ? leaf_stats.n_pages + : index->stat_n_leaf_pages; index->table->stats_mutex_unlock(); DBUG_RETURN(result); } - /* For each level that is being scanned in the btree, this contains the - number of different key values for all possible n-column prefixes. */ - ib_uint64_t* n_diff_on_level = UT_NEW_ARRAY( - ib_uint64_t, n_uniq, mem_key_dict_stats_n_diff_on_level); - - /* For each level that is being scanned in the btree, this contains the - index of the last record from each group of equal records (when - comparing only the first n columns, n=1..n_uniq). */ - boundaries_t* n_diff_boundaries = UT_NEW_ARRAY_NOKEY(boundaries_t, - n_uniq); - - /* For each n-column prefix this array contains the input data that is - used to calculate dict_index_t::stat_n_diff_key_vals[]. */ - n_diff_data_t* n_diff_data = UT_NEW_ARRAY_NOKEY(n_diff_data_t, n_uniq); - - /* total_recs is also used to estimate the number of pages on one - level below, so at the start we have 1 page (the root) */ - total_recs = 1; + /* For each n-column prefix this array contains the + input data that is used to calculate + dict_index_t::stat_n_diff_key_vals[]. */ + NDiffData* n_diff_data = + static_cast( + ut_zalloc_nokey(n_uniq * sizeof(NDiffData))); /* Here we use the following optimization: If we find that level L is the first one (searching from the @@ -2450,13 +2411,21 @@ static index_stats_t dict_stats_analyze_index(dict_index_t* index) keys (on n_prefix columns) is L, we continue from L when searching for D distinct keys on n_prefix-1 columns. */ auto level = root_level; + uint16_t n_prefix; level_is_analyzed = false; - for (n_prefix = n_uniq; n_prefix >= 1; n_prefix--) { + /* Stack-allocated buffers for level statistics. + Since there can be at most 64 fields in an index, + stack allocation is safe and efficient. */ + uint64_t n_diff_buf[64]; + boundaries_t bounds[64]; + ut_ad(n_uniq <= 64); + + IndexLevelStats level_stats(stats_method > SRV_STATS_NULLS_EQUAL, + level, n_diff_buf, nullptr, + bounds, index, &mtr); - DEBUG_PRINTF(" %s(): searching level with >=%llu " - "distinct records, n_prefix=" ULINTPF "\n", - __func__, N_DIFF_REQUIRED(index), n_prefix); + for (n_prefix = n_uniq; n_prefix >= 1; n_prefix--) { /* Commit the mtr to release the tree S lock to allow other threads to do some work too. */ @@ -2485,21 +2454,19 @@ static index_stats_t dict_stats_analyze_index(dict_index_t* index) we pick level 1 even if it does not have enough distinct records because we do not want to scan the leaf level because it may contain too many records */ - if (level_is_analyzed - && (n_diff_on_level[n_prefix - 1] >= N_DIFF_REQUIRED(index) - || level == 1)) { - + if (level_is_analyzed && + (level_stats.n_diff[n_prefix - 1] >= + n_diff_required || level == 1)) { goto found_level; } /* search for a level that contains enough distinct records */ - if (level_is_analyzed && level > 1) { /* if this does not hold we should be on "found_level" instead of here */ - ut_ad(n_diff_on_level[n_prefix - 1] - < N_DIFF_REQUIRED(index)); + ut_ad(level_stats.n_diff[n_prefix - 1] + < n_diff_required); level--; level_is_analyzed = false; @@ -2524,7 +2491,7 @@ static index_stats_t dict_stats_analyze_index(dict_index_t* index) total_recs is left from the previous iteration when we scanned one level upper or we have not scanned any levels yet in which case total_recs is 1. */ - if (total_recs > N_SAMPLE_PAGES(index)) { + if (level_stats.n_recs > n_sample_pages) { /* if the above cond is true then we are not at the root level since on the root @@ -2542,19 +2509,17 @@ static index_stats_t dict_stats_analyze_index(dict_index_t* index) } mtr.rollback_to_savepoint(1); - dict_stats_analyze_index_level(index, - level, - n_diff_on_level, - &total_recs, - &total_pages, - n_diff_boundaries, - &mtr); + + level_stats.reset_for_level(level); + + level_stats.analyze_level(); + mtr.rollback_to_savepoint(1); level_is_analyzed = true; if (level == 1 - || n_diff_on_level[n_prefix - 1] - >= N_DIFF_REQUIRED(index)) { + || level_stats.n_diff[n_prefix - 1] + >= n_diff_required) { /* we have reached the last level we could scan or we found a good level with many distinct records */ @@ -2565,12 +2530,6 @@ static index_stats_t dict_stats_analyze_index(dict_index_t* index) level_is_analyzed = false; } found_level: - - DEBUG_PRINTF(" %s(): found level " ULINTPF - " that has " UINT64PF - " distinct records for n_prefix=" ULINTPF "\n", - __func__, level, n_diff_on_level[n_prefix - 1], - n_prefix); /* here we are either on level 1 or the level that we are on contains >= N_DIFF_REQUIRED distinct keys or we did not scan deeper levels because they would contain too many pages */ @@ -2582,44 +2541,38 @@ static index_stats_t dict_stats_analyze_index(dict_index_t* index) /* if any of these is 0 then there is exactly one page in the B-tree and it is empty and we should have done full scan and should not be here */ - ut_ad(total_recs > 0); - ut_ad(n_diff_on_level[n_prefix - 1] > 0); + ut_ad(level_stats.n_recs > 0); + ut_ad(level_stats.n_diff[n_prefix - 1] > 0); - ut_ad(N_SAMPLE_PAGES(index) > 0); + ut_ad(n_sample_pages > 0); - n_diff_data_t* data = &n_diff_data[n_prefix - 1]; + NDiffData* data = &n_diff_data[n_prefix - 1]; data->level = level; - data->n_recs_on_level = total_recs; + data->n_recs_on_level = level_stats.n_recs; - data->n_diff_on_level = n_diff_on_level[n_prefix - 1]; + data->n_diff_on_level = level_stats.n_diff[n_prefix - 1]; data->n_leaf_pages_to_analyze = std::min( - N_SAMPLE_PAGES(index), - n_diff_on_level[n_prefix - 1]); + n_sample_pages, + level_stats.n_diff[n_prefix - 1]); - /* pick some records from this level and dive below them for - the given n_prefix */ - - dict_stats_analyze_index_for_n_prefix( - index, n_prefix, &n_diff_boundaries[n_prefix - 1], - data, &mtr); + ut_ad(level_stats.n_non_null == nullptr); + /* pick some records from this level and dive below + them for the given n_prefix */ + level_stats.sample_leaf_pages(n_prefix, data); } mtr.commit(); - UT_DELETE_ARRAY(n_diff_boundaries); - - UT_DELETE_ARRAY(n_diff_on_level); - /* n_prefix == 0 means that the above loop did not end up prematurely due to tree being changed and so n_diff_data[] is set up. */ if (n_prefix == 0) { - dict_stats_index_set_n_diff(n_diff_data, result); + dict_stats_index_set_n_diff(index, n_diff_data, result); } - UT_DELETE_ARRAY(n_diff_data); + ut_free(n_diff_data); DBUG_RETURN(result); } @@ -2628,10 +2581,9 @@ dberr_t dict_stats_update_persistent(dict_table_t *table) noexcept { dict_index_t* index; - DEBUG_PRINTF("%s(table=%s)\n", __func__, table->name); - DEBUG_SYNC_C("dict_stats_update_persistent"); + const unsigned stats_method = unsigned(srv_innodb_stats_method); if (trx_sys.is_registered(nullptr, table->bulk_trx_id)) { dict_stats_empty_table(table, false); return DB_SUCCESS_LOCKED_REC; @@ -2656,7 +2608,7 @@ dberr_t dict_stats_update_persistent(dict_table_t *table) noexcept dict_stats_empty_index(index, false); table->stats_mutex_unlock(); - index_stats_t stats = dict_stats_analyze_index(index); + IndexStats stats = dict_stats_analyze_index(index, stats_method); if (stats.is_bulk_operation()) { dict_stats_empty_table(table, false); @@ -2669,7 +2621,8 @@ dberr_t dict_stats_update_persistent(dict_table_t *table) noexcept for (size_t i = 0; i < stats.stats.size(); ++i) { index->stat_n_diff_key_vals[i] = stats.stats[i].n_diff_key_vals; index->stat_n_sample_sizes[i] = stats.stats[i].n_sample_sizes; - index->stat_n_non_null_key_vals[i] = stats.stats[i].n_non_null_key_vals; + index->stat_n_non_null_key_vals[i] = + stats.stats[i].n_non_null_key_vals; } ulint n_unique = dict_index_get_n_unique(index); @@ -2697,7 +2650,7 @@ dberr_t dict_stats_update_persistent(dict_table_t *table) noexcept } table->stats_mutex_unlock(); - stats = dict_stats_analyze_index(index); + stats = dict_stats_analyze_index(index, stats_method); table->stats_mutex_lock(); if (stats.is_bulk_operation()) { @@ -2732,18 +2685,14 @@ dberr_t dict_stats_update_persistent(dict_table_t *table) noexcept table->stats_mutex_unlock(); - return(DB_SUCCESS); + return dict_stats_save(table, stats_method); } dberr_t dict_stats_update_persistent_try(dict_table_t *table) { if (table->stats_is_persistent() && dict_stats_persistent_storage_check(false) == SCHEMA_OK) - { - if (dberr_t err= dict_stats_update_persistent(table)) - return err; - return dict_stats_save(table); - } + return dict_stats_update_persistent(table); return DB_SUCCESS; } @@ -2763,8 +2712,8 @@ dict_stats_save_index_stat( dict_index_t* index, time_t last_update, const char* stat_name, - ib_uint64_t stat_value, - ib_uint64_t* sample_size, + uint64_t stat_value, + uint64_t* sample_size, const char* stat_description, trx_t* trx) { @@ -2874,11 +2823,25 @@ dict_stats_report_error(dict_table_t* table, bool defragment) return err; } -/** Save the persistent statistics of a table or an index. -@param table table whose stats to save -@param only_for_index the index ID to save statistics for (0=all) -@return DB_SUCCESS or error code */ -dberr_t dict_stats_save(dict_table_t* table, index_id_t index_id) +static const char *stats_method_name[]= { + nullptr, " NULLS_UNEQUAL", " NULLS_IGNORED" +}; + +/** Return a display name for the innodb_stats_method +@param stat_method innodb_stats_method during index statistics +@return innodb_stats_method name */ +static const char *get_innodb_stats_method(unsigned stat_method) +{ + ut_ad(stat_method < 3); + static_assert(SRV_STATS_NULLS_EQUAL == 0, ""); + static_assert(SRV_STATS_NULLS_UNEQUAL == 1, ""); + static_assert(SRV_STATS_NULLS_IGNORED == 2, ""); + return stat_method < 3 ? stats_method_name[stat_method] : nullptr; +} + +static +dberr_t dict_stats_save(dict_table_t* table, unsigned stats_method, + index_id_t index_id) noexcept { pars_info_t* pinfo; char db_utf8[MAX_DB_UTF8_LEN]; @@ -3040,13 +3003,31 @@ dberr_t dict_stats_save(dict_table_t* table, index_id_t index_id) snprintf(stat_description, sizeof(stat_description), "%s", index->fields[0].name()); for (unsigned j = 1; j <= i; j++) { - size_t len; - - len = strlen(stat_description); - + size_t len = strlen(stat_description); + size_t remaining = + sizeof(stat_description) - len; + /* Ensure we have enough space for + "," + field_name + null terminator */ + size_t field_name_len = + strlen(index->fields[j].name()); + if (remaining < field_name_len + 2) { + break; + } snprintf(stat_description + len, - sizeof(stat_description) - len, - ",%s", index->fields[j].name()); + remaining, ",%s", + index->fields[j].name()); + } + + if (const char *stats_method_name = + get_innodb_stats_method(stats_method)) { + size_t desc_len = strlen(stat_description); + size_t method_len = strlen(stats_method_name); + size_t remaining = sizeof(stat_description) + - desc_len - 1; + if (method_len < remaining) { + strncat(stat_description, + stats_method_name, remaining); + } } ret = dict_stats_save_index_stat( @@ -3058,6 +3039,24 @@ dberr_t dict_stats_save(dict_table_t* table, index_id_t index_id) if (ret != DB_SUCCESS) { goto rollback_and_exit; } + + /* Update n_nonnull_fld */ + snprintf(stat_name, sizeof(stat_name), + "n_nonnull_fld%02u", i + 1); + + snprintf(stat_description, + sizeof(stat_description), + "%s", index->fields[i].name()); + + ret = dict_stats_save_index_stat( + index, now, stat_name, + index->stat_n_non_null_key_vals[i], + &index->stat_n_sample_sizes[i], + stat_description, trx); + + if (ret != DB_SUCCESS) { + goto rollback_and_exit; + } } ret = dict_stats_save_index_stat(index, now, "n_leaf_pages", @@ -3088,6 +3087,11 @@ dberr_t dict_stats_save(dict_table_t* table, index_id_t index_id) goto free_and_exit; } +static dberr_t dict_stats_save(dict_table_t *table) +{ + return dict_stats_save(table, unsigned(srv_innodb_stats_method)); +} + void dict_stats_empty_table_and_save(dict_table_t *table) { dict_stats_empty_table(table, true); @@ -3178,13 +3182,41 @@ dict_stats_fetch_table_stats_step( return(TRUE); } +/** Report that a row with a malformed or out-of-range stat_name was found +in mysql.innodb_index_stats and is being ignored. +@param table table the statistics belong to +@param index index the statistics belong to +@param stat_name value of the stat_name column (not NUL-terminated) +@param stat_name_len length of stat_name */ +static void dict_stats_report_bad_index_stat( + const dict_table_t* table, + const dict_index_t* index, + const char* stat_name, + size_t stat_name_len) +{ + char db_utf8[MAX_DB_UTF8_LEN]; + char table_utf8[MAX_TABLE_UTF8_LEN]; + + dict_fs2utf8(table->name.m_name, db_utf8, sizeof(db_utf8), + table_utf8, sizeof(table_utf8)); + + sql_print_information("InnoDB: Malformed stat_name='%.*s' in " + INDEX_STATS_NAME_PRINT " WHERE " + "database_name=%`s AND table_name=%`s AND " + "index_name=%`s", + (int) stat_name_len, stat_name, + db_utf8, table_utf8, index->name()); +} + /** Aux struct used to pass a table and a boolean to dict_stats_fetch_index_stats_step(). */ -struct index_fetch_t { +namespace { +struct IndexFetch { dict_table_t* table; /*!< table whose indexes are to be modified */ bool stats_were_modified; /*!< will be set to true if at least one index stats were modified */ }; +} // anonymous namespace /*********************************************************************//** Called for the rows that are selected by @@ -3212,14 +3244,14 @@ dict_stats_fetch_index_stats_step( modified anything */ { sel_node_t* node = (sel_node_t*) node_void; - index_fetch_t* arg = (index_fetch_t*) arg_void; + IndexFetch* arg = (IndexFetch*) arg_void; dict_table_t* table = arg->table; dict_index_t* index = NULL; que_common_t* cnode; const char* stat_name = NULL; ulint stat_name_len = ULINT_UNDEFINED; - ib_uint64_t stat_value = UINT64_UNDEFINED; - ib_uint64_t sample_size = UINT64_UNDEFINED; + uint64_t stat_value = UINT64_UNDEFINED; + uint64_t sample_size = UINT64_UNDEFINED; int i; /* this should loop exactly 4 times - for the columns that @@ -3372,22 +3404,8 @@ dict_stats_fetch_index_stats_step( || num_ptr[0] < '0' || num_ptr[0] > '9' || num_ptr[1] < '0' || num_ptr[1] > '9') { - char db_utf8[MAX_DB_UTF8_LEN]; - char table_utf8[MAX_TABLE_UTF8_LEN]; - - dict_fs2utf8(table->name.m_name, - db_utf8, sizeof(db_utf8), - table_utf8, sizeof(table_utf8)); - - ib::info out; - out << "Ignoring strange row from " - << INDEX_STATS_NAME_PRINT << " WHERE" - " database_name = '" << db_utf8 - << "' AND table_name = '" << table_utf8 - << "' AND index_name = '" << index->name() - << "' AND stat_name = '"; - out.write(stat_name, stat_name_len); - out << "'; because stat_name is malformed"; + dict_stats_report_bad_index_stat( + table, index, stat_name, stat_name_len); return(TRUE); } /* else */ @@ -3400,24 +3418,8 @@ dict_stats_fetch_index_stats_step( if (n_pfx == 0 || n_pfx > n_uniq) { - char db_utf8[MAX_DB_UTF8_LEN]; - char table_utf8[MAX_TABLE_UTF8_LEN]; - - dict_fs2utf8(table->name.m_name, - db_utf8, sizeof(db_utf8), - table_utf8, sizeof(table_utf8)); - - ib::info out; - out << "Ignoring strange row from " - << INDEX_STATS_NAME_PRINT << " WHERE" - " database_name = '" << db_utf8 - << "' AND table_name = '" << table_utf8 - << "' AND index_name = '" << index->name() - << "' AND stat_name = '"; - out.write(stat_name, stat_name_len); - out << "'; because stat_name is out of range, the index" - " has " << n_uniq << " unique columns"; - + dict_stats_report_bad_index_stat( + table, index, stat_name, stat_name_len); return(TRUE); } /* else */ @@ -3426,7 +3428,7 @@ dict_stats_fetch_index_stats_step( if (sample_size != UINT64_UNDEFINED) { index->stat_n_sample_sizes[n_pfx - 1] = - std::max(sample_size, 1); + std::max(sample_size, 1); } else { /* hmm, strange... the user must have UPDATEd the table manually and SET sample_size = NULL */ @@ -3435,6 +3437,43 @@ dict_stats_fetch_index_stats_step( index->stat_n_non_null_key_vals[n_pfx - 1] = 0; + arg->stats_were_modified = true; + } else if (stat_name_len >= sizeof "n_nonnull_fld" + && strncasecmp("n_nonnull_fld", stat_name, + sizeof "n_nonnull_fld" - 1) == 0) { + + const char* num_ptr; + unsigned n_field; + + /* point num_ptr into "01" from "n_nonnull_fld01..." */ + num_ptr = stat_name + (sizeof "n_nonnull_fld" - 1); + + /* stat_name should have exactly 2 chars appended to "n_nonnull_fld" + and they should be digits */ + if (stat_name_len != sizeof "n_nonnull_fld01" - 1 + || num_ptr[0] < '0' || num_ptr[0] > '9' + || num_ptr[1] < '0' || num_ptr[1] > '9') { + + dict_stats_report_bad_index_stat( + table, index, stat_name, stat_name_len); + return(TRUE); + } + /* else */ + + /* extract 01 from "n_nonnull_fld01..." into n_col + note that stat_name does not have a terminating '\0' */ + n_field = (num_ptr[0] - '0') * 10 + (num_ptr[1] - '0'); + + if (n_field == 0 || n_field > index->n_uniq) { + + dict_stats_report_bad_index_stat( + table, index, stat_name, stat_name_len); + return(TRUE); + } + /* else */ + + index->stat_n_non_null_key_vals[n_field - 1] = stat_value; + arg->stats_were_modified = true; } else { /* silently ignore rows with unknown stat_name, the @@ -3448,7 +3487,7 @@ dict_stats_fetch_index_stats_step( /** Read the stored persistent statistics of a table. */ dberr_t dict_stats_fetch_from_ps(dict_table_t *table) { - index_fetch_t index_fetch_arg; + IndexFetch index_fetch_arg; pars_info_t* pinfo; char db_utf8[MAX_DB_UTF8_LEN]; char table_utf8[MAX_TABLE_UTF8_LEN]; @@ -3570,6 +3609,7 @@ dict_stats_update_for_index( dict_index_t* index) /*!< in/out: index */ { dict_table_t *const table= index->table; + unsigned stats_method = unsigned(srv_innodb_stats_method); ut_ad(table->stat_initialized()); if (table->stats_is_persistent()) @@ -3592,7 +3632,7 @@ dict_stats_update_for_index( table->name.basename(), index->name()); break; case SCHEMA_OK: - index_stats_t stats{dict_stats_analyze_index(index)}; + IndexStats stats{dict_stats_analyze_index(index, stats_method)}; table->stats_mutex_lock(); index->stat_index_size = stats.index_size; index->stat_n_leaf_pages = stats.n_leaf_pages; @@ -3604,7 +3644,7 @@ dict_stats_update_for_index( } table->stat_sum_of_other_index_sizes+= index->stat_index_size; table->stats_mutex_unlock(); - dict_stats_save(table, index->id); + dict_stats_save(table, stats_method, index->id); return; } @@ -3837,12 +3877,12 @@ test_dict_stats_save() dict_table_t table; dict_index_t index1; dict_field_t index1_fields[1]; - ib_uint64_t index1_stat_n_diff_key_vals[1]; - ib_uint64_t index1_stat_n_sample_sizes[1]; + uint64_t index1_stat_n_diff_key_vals[1]; + uint64_t index1_stat_n_sample_sizes[1]; dict_index_t index2; dict_field_t index2_fields[4]; - ib_uint64_t index2_stat_n_diff_key_vals[4]; - ib_uint64_t index2_stat_n_sample_sizes[4]; + uint64_t index2_stat_n_diff_key_vals[4]; + uint64_t index2_stat_n_sample_sizes[4]; dberr_t ret; /* craft a dummy dict_table_t */ @@ -3993,11 +4033,11 @@ test_dict_stats_fetch_from_ps() { dict_table_t table; dict_index_t index1; - ib_uint64_t index1_stat_n_diff_key_vals[1]; - ib_uint64_t index1_stat_n_sample_sizes[1]; + uint64_t index1_stat_n_diff_key_vals[1]; + uint64_t index1_stat_n_sample_sizes[1]; dict_index_t index2; - ib_uint64_t index2_stat_n_diff_key_vals[4]; - ib_uint64_t index2_stat_n_sample_sizes[4]; + uint64_t index2_stat_n_diff_key_vals[4]; + uint64_t index2_stat_n_sample_sizes[4]; dberr_t ret; /* craft a dummy dict_table_t */ diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 9ff636aca2ad6..1d2abcb46b46f 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -5754,12 +5754,8 @@ dberr_t ha_innobase::statistics_init(dict_table_t *table, bool recalc) switch (dict_stats_persistent_storage_check(false)) { case SCHEMA_OK: if (recalc) - { recalc: err= dict_stats_update_persistent(table); - if (err == DB_SUCCESS) - err= dict_stats_save(table); - } else { err= dict_stats_fetch_from_ps(table); @@ -21421,8 +21417,6 @@ void alter_stats_rebuild(dict_table_t *table, THD *thd, bool copy) DBUG_VOID_RETURN; dberr_t ret= dict_stats_update_persistent(table); - if (ret == DB_SUCCESS) - ret= dict_stats_save(table); if (ret != DB_SUCCESS) push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, ER_ALTER_INFO, "Error updating stats for table after" diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index 935be50543677..7a2c8128e3733 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -70,6 +70,19 @@ enum { #define btr_cur_get_block(cursor) ((cursor)->page_cur.block) #define btr_cur_get_rec(cursor) ((cursor)->page_cur.rec) +/** The structure of a BLOB part header */ +/* @{ */ +/*--------------------------------------*/ +#define BTR_BLOB_HDR_PART_LEN 0 /*!< BLOB part len on this + page */ +#define BTR_BLOB_HDR_NEXT_PAGE_NO 4 /*!< next BLOB part page no, + FIL_NULL if none */ +/*--------------------------------------*/ +#define BTR_BLOB_HDR_SIZE 8 /*!< Size of a BLOB + part header, in bytes */ + +/* @} */ + /*********************************************************//** Returns the compressed page on which the tree cursor is positioned. @return pointer to compressed page, or NULL if the page is not compressed */ @@ -468,14 +481,19 @@ ha_rows btr_estimate_n_rows_in_range(dict_index_t *index, btr_pos_t *range_start, btr_pos_t *range_end); -/** Gets the externally stored size of a record, in units of a database page. -@param[in] rec record -@param[in] offsets array returned by rec_get_offsets() -@return externally stored part, in units of a database page */ -ulint -btr_rec_get_externally_stored_len( - const rec_t* rec, - const rec_offs* offsets); +/** Gets the offset of the pointer to the externally stored part of a field. +@param offsets offsets of record +@param n index of the external field +@return offset of the pointer to the externally stored part */ +size_t btr_rec_get_field_ref_offs(const rec_offs *offsets, size_t n) noexcept; + +/** Gets a pointer to the externally stored part of a field. +@param rec record +@param offsets rec_get_offsets(rec) +@param n index of the externally stored field +@return pointer to the externally stored part */ +#define btr_rec_get_field_ref(rec, offsets, n) \ + ((rec) + btr_rec_get_field_ref_offs(offsets, n)) /*******************************************************************//** Marks non-updated off-page fields as disowned by this record. The ownership diff --git a/storage/innobase/include/data0type.h b/storage/innobase/include/data0type.h index 2c5d38996d14f..8eaf9272a9bb8 100644 --- a/storage/innobase/include/data0type.h +++ b/storage/innobase/include/data0type.h @@ -31,7 +31,7 @@ Created 1/16/1996 Heikki Tuuri #define UNIV_SQL_DEFAULT (UNIV_SQL_NULL - 1) /** @return whether a length is actually stored in a field */ -#define len_is_stored(len) (len != UNIV_SQL_NULL && len != UNIV_SQL_DEFAULT) +#define len_is_stored(len) (len < UNIV_SQL_DEFAULT) extern ulint data_mysql_default_charset_coll; #define DATA_MYSQL_BINARY_CHARSET_COLL 63 diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index 30064666b9169..684af5afee440 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -1086,7 +1086,7 @@ struct dict_index_t { /*----------------------*/ /** Statistics for query optimization */ /* @{ */ - ib_uint64_t* stat_n_diff_key_vals; + uint64_t* stat_n_diff_key_vals; /*!< approximate number of different key values for this index, for each n-column prefix where 1 <= n <= @@ -1099,7 +1099,7 @@ struct dict_index_t { to calculate each of stat_n_diff_key_vals[], e.g. stat_n_sample_sizes[3] pages were sampled to get the number stat_n_diff_key_vals[3]. */ - ib_uint64_t* stat_n_non_null_key_vals; + uint64_t* stat_n_non_null_key_vals; /* approximate number of non-null key values for this index, for each column where 1 <= n <= dict_get_n_unique(index) (the array diff --git a/storage/innobase/include/dict0stats.h b/storage/innobase/include/dict0stats.h index 91ec49535998d..09911fb86e2e6 100644 --- a/storage/innobase/include/dict0stats.h +++ b/storage/innobase/include/dict0stats.h @@ -95,12 +95,6 @@ enum dict_stats_schema_check { dict_stats_schema_check dict_stats_persistent_storage_check(bool dict_already_locked= false) noexcept; -/** Save the persistent statistics of a table or an index. -@param table table whose stats to save -@param only_for_index the index ID to save statistics for (0=all) -@return DB_SUCCESS or error code */ -dberr_t dict_stats_save(dict_table_t* table, index_id_t index_id= 0); - /** Read the stored persistent statistics of a table. */ dberr_t dict_stats_fetch_from_ps(dict_table_t *table); @@ -112,12 +106,13 @@ is relatively quick and is used to calculate non-persistent statistics. @retval DB_SUCCESS_LOCKED REC if the table under bulk insert operation */ dberr_t dict_stats_update_transient(dict_table_t *table) noexcept; -/** -Calculate new estimates for table and index statistics. This function -is slower than dict_stats_update_transient(). -@param table table for which the persistent statistics are being updated -@return DB_SUCCESS or error code -@retval DB_SUCCESS_LOCKED_REC if the table under bulk insert operation */ +/** Updates the persistent statistics for a table. +This function calculates and stores index statistics +for all indexes of the table. The statistics are stored +in the InnoDB data dictionary tables via dict_stats_save() +@param table table whose stats to update +@retval DB_SUCCESS_LOCKED_REC if the table under bulk insert operation +@return DB_SUCCESS or error code */ dberr_t dict_stats_update_persistent(dict_table_t *table) noexcept; /** diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index 46a8a9a3fdb17..9f5f18ac72dc3 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -149,9 +149,6 @@ using the call command. */ #define UNIV_AIO_DEBUG /* prints info about submitted and reaped AIO requests to the log. */ -#define UNIV_STATS_DEBUG /* prints various stats - related debug info from - dict0stats.c */ #define FTS_INTERNAL_DIAG_PRINT /* FTS internal debugging info output */ #endif