MDEV-40332 InnoDB: Defragmentation of BASE_IDX in SYS_VIRTUAL failed: Data structure corruption

mysql-test/suite/innodb/r/sys_defragment_root.result

@@ -0,0 +1,44 @@
+#
+# MDEV-40332 InnoDB: Defragmentation of BASE_IDX in
+#	SYS_VIRTUAL failed: Data structure corruption
+#
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/root_datadir --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/root_datadir --innodb_undo_directory=MYSQLTEST_VARDIR/tmp/root_datadir --debug_dbug=+d,defer_sys_virtual
+# SYS_VIRTUAL is absent for now
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES
+WHERE SPACE=0 AND NAME='SYS_VIRTUAL';
+NAME
+SET GLOBAL innodb_file_per_table= 0;
+Warnings:
+Warning	1287	'@@innodb_file_per_table' is deprecated and will be removed in a future release
+SET GLOBAL innodb_limit_optimistic_insert_debug= 2;
+CREATE TABLE t1(f1 INT NOT NULL, f2 INT NOT NULL, f3 INT NOT NULL,
+INDEX(f1), INDEX(f2), INDEX(f3))
+STATS_PERSISTENT=0 ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t1 SELECT seq, seq, seq FROM seq_1_to_16384;
+INSERT INTO t1 SELECT seq, seq, seq FROM seq_1_to_16384;
+COMMIT;
+# Restart without the flag: SYS_VIRTUAL is now created
+# into the grown tablespace, so its BASE_IDX root page
+# lands in a high extent.
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/root_datadir --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/root_datadir --innodb_undo_directory=MYSQLTEST_VARDIR/tmp/root_datadir
+# SYS_VIRTUAL now exists, and its BASE_IDX root landed
+# above the 10M (640-page) shrink floor
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES
+WHERE SPACE=0 AND NAME='SYS_VIRTUAL';
+NAME
+SYS_VIRTUAL
+SELECT PAGE_NO > 640 AS root_above_shrink_floor
+FROM INFORMATION_SCHEMA.INNODB_SYS_INDEXES WHERE SPACE=0 AND NAME='BASE_IDX';
+root_above_shrink_floor
+1
+SET GLOBAL innodb_limit_optimistic_insert_debug= 2;
+INSERT INTO t1 SELECT seq, seq, seq FROM seq_1_to_32;
+DROP TABLE t1;
+InnoDB		0 transactions not purged
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/root_datadir --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/root_datadir --innodb_undo_directory=MYSQLTEST_VARDIR/tmp/root_datadir --innodb_data_file_path=ibdata1:10M:autoextend:autoshrink --innodb_fast_shutdown=0
+SELECT FILE_SIZE < $fsize AS shrunk
+FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESPACES WHERE SPACE=0;
+shrunk
+1
+# restart

mysql-test/suite/innodb/t/sys_defragment_root.opt

@@ -0,0 +1,2 @@
+--innodb_page_size=16k
+--innodb_sys_tablespaces

mysql-test/suite/innodb/t/sys_defragment_root.test

@@ -0,0 +1,66 @@
+--source include/have_innodb.inc
+--source include/have_debug.inc
+--source include/have_sequence.inc
+--source include/not_embedded.inc
+--source include/not_windows.inc
+--source include/not_valgrind.inc
+
+--echo #
+--echo # MDEV-40332 InnoDB: Defragmentation of BASE_IDX in
+--echo #	SYS_VIRTUAL failed: Data structure corruption
+--echo #
+
+let bugdir= $MYSQLTEST_VARDIR/tmp/root_datadir;
+--mkdir $bugdir
+let $dirs= --innodb-data-home-dir=$bugdir --innodb-log-group-home-dir=$bugdir --innodb_undo_directory=$bugdir;
+
+let $restart_parameters=$dirs --debug_dbug=+d,defer_sys_virtual;
+--source include/restart_mysqld.inc
+
+--echo # SYS_VIRTUAL is absent for now
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES
+WHERE SPACE=0 AND NAME='SYS_VIRTUAL';
+
+SET GLOBAL innodb_file_per_table= 0;
+SET GLOBAL innodb_limit_optimistic_insert_debug= 2;
+CREATE TABLE t1(f1 INT NOT NULL, f2 INT NOT NULL, f3 INT NOT NULL,
+                INDEX(f1), INDEX(f2), INDEX(f3))
+  STATS_PERSISTENT=0 ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t1 SELECT seq, seq, seq FROM seq_1_to_16384;
+INSERT INTO t1 SELECT seq, seq, seq FROM seq_1_to_16384;
+COMMIT;
+
+--echo # Restart without the flag: SYS_VIRTUAL is now created
+--echo # into the grown tablespace, so its BASE_IDX root page
+--echo # lands in a high extent.
+let $restart_parameters=$dirs;
+--source include/restart_mysqld.inc
+
+--echo # SYS_VIRTUAL now exists, and its BASE_IDX root landed
+--echo # above the 10M (640-page) shrink floor
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES
+WHERE SPACE=0 AND NAME='SYS_VIRTUAL';
+
+SELECT PAGE_NO > 640 AS root_above_shrink_floor
+FROM INFORMATION_SCHEMA.INNODB_SYS_INDEXES WHERE SPACE=0 AND NAME='BASE_IDX';
+
+SET GLOBAL innodb_limit_optimistic_insert_debug= 2;
+INSERT INTO t1 SELECT seq, seq, seq FROM seq_1_to_32;
+DROP TABLE t1;
+--source include/wait_all_purged.inc
+
+let $fsize=`SELECT FILE_SIZE FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESPACES
+            WHERE SPACE=0`;
+
+let $restart_parameters=$dirs --innodb_data_file_path=ibdata1:10M:autoextend:autoshrink --innodb_fast_shutdown=0;
+--source include/restart_mysqld.inc
+
+let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err;
+
+evalp SELECT FILE_SIZE < $fsize AS shrunk
+FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESPACES WHERE SPACE=0;
+
+let $restart_parameters=;
+--source include/restart_mysqld.inc
+--rmdir $bugdir

storage/innobase/dict/dict0crea.cc

@@ -1445,6 +1445,7 @@ dberr_t dict_sys_t::create_or_check_sys_tables() noexcept
   }
   if (!sys_virtual)
   {
+    DBUG_EXECUTE_IF("defer_sys_virtual", goto commit_trx;);
     error= que_eval_sql(nullptr, "PROCEDURE CREATE_VIRTUAL() IS\n"
                         "BEGIN\n"
                         "CREATE TABLE\n"
@@ -1480,7 +1481,9 @@ dberr_t dict_sys_t::create_or_check_sys_tables() noexcept
                       goto err_exit;
                     }
                   });
-
+#ifndef DBUG_OFF
+commit_trx:
+#endif /* !DBUG_OFF */
   trx->commit();
   row_mysql_unlock_data_dictionary(trx);
   trx->clear_and_free();
@@ -1509,6 +1512,12 @@ dberr_t dict_sys_t::create_or_check_sys_tables() noexcept
   else
     prevent_eviction(sys_foreign_cols);
 
+  DBUG_EXECUTE_IF("defer_sys_virtual",
+                  {
+                    unlock();
+                    return DB_SUCCESS;
+                  });
+
   if (sys_virtual);
   else if (!(sys_virtual= load_table(SYS_TABLE[SYS_VIRTUAL])))
   {

storage/innobase/fsp/fsp0fsp.cc

@@ -5317,6 +5317,32 @@ class SpaceDefragmenter final
     return FIL_NULL;
   }
 
+  /** A B-tree root page cannot be relocated: its page number
+  is recorded in SYS_INDEXES and the dictionary cache, and it
+  stores the FSEG segment headers(PAGE_BTR_SEG_LEAF and
+  PAGE_BTR_SEG_TOP).  The system tablespace cannot shrink below
+  the highest root page, and relocating any page at or below
+  it would not reduce the tablespace size.
+  @return the highest extent that holds such a root page, or 0 */
+  uint32_t max_root_extent() noexcept
+  {
+    uint32_t max_extent= 0;
+    for (dict_table_t *table :
+         {dict_sys.sys_tables, dict_sys.sys_columns, dict_sys.sys_indexes,
+          dict_sys.sys_fields, dict_sys.sys_foreign,
+          dict_sys.sys_foreign_cols, dict_sys.sys_virtual})
+    {
+      if (!table)
+        continue;
+      for (dict_index_t *index= dict_table_get_first_index(table);
+           index; index= dict_table_get_next_index(index))
+        if (index->page != FIL_NULL)
+          max_extent= std::max(max_extent,
+                               (index->page / m_extent_size) * m_extent_size);
+    }
+    return max_extent;
+  }
+
   /** Defragment the indexes */
   dberr_t defragment_index(dict_index_t &index) noexcept
   {
@@ -5327,6 +5353,9 @@ class SpaceDefragmenter final
   /** Defragment the table */
   dberr_t defragment_table(const dict_table_t *table) noexcept
   {
+    if (!table)
+      return DB_SUCCESS;
+
     for (dict_index_t *index= dict_table_get_first_index(table);
          index; index= dict_table_get_next_index(index))
     {
@@ -5358,7 +5387,11 @@ class SpaceDefragmenter final
 
     uint32_t free_limit= fil_system.sys_space->free_limit;
     uint32_t fixed_size= srv_sys_space.get_min_size();
-    while (free_limit > fixed_size)
+    /* A root page cannot be relocated, so the tablespace cannot
+    shrink below the highest root.  Never map an extent at or
+    below that floor */
+    uint32_t floor= std::max(fixed_size, max_root_extent());
+    while (free_limit > floor)
     {
       uint32_t state= m_extent_info[free_limit];
 
@@ -5566,6 +5599,11 @@ dberr_t IndexDefragmenter::defragment_level(
     }
 
     uint32_t new_extent= space_defrag->get_new_extent(cur_extent);
+    /* The root page has no parent node pointer cannot be relocated.
+    find_new_extents() never maps an extent at or below the highest
+    root, so a root is never scheduled for relocation. */
+    ut_ad(cur_page_no != m_index.page || new_extent == cur_extent);
+
     /* There is no need for extent to be changed */
     if (new_extent == cur_extent)
     {

storage/innobase/include/dict0dict.h

@@ -1332,7 +1332,11 @@ class dict_sys_t
 
   /** @return whether all non-hard-coded system tables exist */
   bool sys_tables_exist() const
-  { return UNIV_LIKELY(sys_foreign && sys_foreign_cols && sys_virtual); }
+  {
+    DBUG_EXECUTE_IF("defer_sys_virtual",
+                    return sys_foreign && sys_foreign_cols;);
+    return UNIV_LIKELY(sys_foreign && sys_foreign_cols && sys_virtual);
+  }
 
   /** list of persistent tables that can be evicted */
   UT_LIST_BASE_NODE_T(dict_table_t) table_LRU;
@@ -1508,9 +1512,9 @@ class dict_sys_t
   bool is_sys_table(table_id_t table_id) const noexcept
   {
     return (table_id > 0 && table_id <= 4) ||
-      table_id == sys_foreign->id ||
-      table_id == sys_foreign_cols->id ||
-      table_id == sys_virtual->id;
+      (sys_foreign && table_id == sys_foreign->id) ||
+      (sys_foreign_cols && table_id == sys_foreign_cols->id) ||
+      (sys_virtual && table_id == sys_virtual->id);
   }
 };