MDEV-39092 Copy Aria data and logs as part of backup

mysql-test/main/backup_server_restore.result

@@ -1,6 +1,4 @@
 Prepare database
-CREATE TABLE tinno (i INTEGER) ENGINE=InnoDB;
-INSERT INTO tinno VALUES (1), (2), (3), (4);
 CREATE TABLE tariatr (i INTEGER) ENGINE=Aria TRANSACTIONAL=1;
 INSERT INTO tariatr VALUES (2), (3), (5), (7);
 CREATE TABLE tariant (i INTEGER) ENGINE=Aria TRANSACTIONAL=0;
@@ -10,12 +8,6 @@ BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory';
 Restore the database
 # restart: --datadir=MYSQLTEST_VARDIR/some_directory
 Check contents after restore
-SELECT * FROM tinno;
-i
-1
-2
-3
-4
 SELECT * FROM tariatr;
 i
 2
@@ -36,6 +28,5 @@ Note	1034	Table is fixed
 Restart database in original data directory 
 # restart
 Clean up
-DROP TABLE tinno;
 DROP TABLE tariatr;
 DROP TABLE tariant;

mysql-test/main/backup_server_restore.test

@@ -2,8 +2,6 @@
 --source include/have_innodb.inc
 
 --echo Prepare database
-CREATE TABLE tinno (i INTEGER) ENGINE=InnoDB;
-INSERT INTO tinno VALUES (1), (2), (3), (4);
 CREATE TABLE tariatr (i INTEGER) ENGINE=Aria TRANSACTIONAL=1;
 INSERT INTO tariatr VALUES (2), (3), (5), (7);
 CREATE TABLE tariant (i INTEGER) ENGINE=Aria TRANSACTIONAL=0;
@@ -23,7 +21,6 @@ call mtr.add_suppression("Checking table: ");
 --source include/restart_mysqld.inc
 
 --echo Check contents after restore
-SELECT * FROM tinno;
 SELECT * FROM tariatr;
 SELECT * FROM tariant;
 
@@ -32,7 +29,6 @@ SELECT * FROM tariant;
 --source include/restart_mysqld.inc
 
 --echo Clean up
-DROP TABLE tinno;
 DROP TABLE tariatr;
 DROP TABLE tariant;
 --rmdir $MYSQLTEST_VARDIR/some_directory

sql/handler.h

@@ -1517,6 +1517,17 @@ struct backup_target
 #endif
 };
 
+/** BACKUP SERVER processing phase. Phases must occur in this order. */
+enum backup_proc_phase {
+  BACKUP_PROC_PHASE_BACKUP_LOCKED = 0,  /* concurrent backup guaranteed not to run */
+  BACKUP_PROC_PHASE_DDL_LOCKED,         /* DDL can't happen concurrently */
+  BACKUP_PROC_PHASE_COMMIT_LOCKED};     /* Commits are blocked */
+
+/** BACKUP SERVER ending phase. Phases must occur in this order. */
+enum backup_end_phase {
+  BACKUP_END_PHASE_COMMIT_LOCKED,  /* Commits are blocked */
+  BACKUP_END_PHASE_BACKUP_LOCKED};  /* Only concurrent backups blocked */
+
 /*
   handlerton is a singleton structure - one instance per storage engine -
   to provide access to storage engine functionality that works on the
@@ -1920,28 +1931,41 @@ struct handlerton : public transaction_participant
   void (*end_backup)(void);
 
   /**
-     Start of BACKUP SERVER: collect all files to be backed up
+     Start of BACKUP SERVER
      @param thd     current session
      @param target  backup target
      @return error code
      @retval 0 on success
   */
   int (*backup_start)(THD *thd, backup_target target);
   /**
-     Process a file that was collected in backup_start().
+     Called when a given phase of backup processing starts.
+     @param thd   current session
+     @param phase the processing phase which started
+     @return error code
+     @retval 0 on completion
+     @note All subsequent calls to backup_step until next call to
+           backup_start_proc_phase or backup_end are in that phase
+  */
+  int (*backup_start_proc_phase)(THD *thd, backup_proc_phase phase);
+  /**
+     Process a unit (file, table etc.) in current phase. Engines should
+     define their units in such a way that they may be processed concurrently.
      @param thd   current session
-     @return number of files remaining, or negative on error
+     @param phase the processing phase for this step
+     @return number of units remaining in processing phase, or negative on error
      @retval 0 on completion
   */
-  int (*backup_step)(THD *thd);
+  int (*backup_step)(THD *thd, backup_proc_phase phase);
   /**
      Finish copying and determine the logical time of the backup snapshot.
      @param thd   current sesssion
+     @param phase which phase of the end stage to run
      @param abort whether BACKUP SERVER was aborted
      @return error code
      @retval 0 on success
   */
-  int (*backup_end)(THD *thd, bool abort);
+  int (*backup_end)(THD *thd, backup_end_phase phase, bool abort);
   /**
      Clean up after any backup_end().
      @param thd     the parameter on which backup_end() was invoked

sql/sql_backup.cc

@@ -194,20 +194,35 @@ static my_bool backup_start(THD *thd, plugin_ref plugin, void *dst) noexcept
   return false;
 }
 
+static my_bool backup_start_proc_phase(THD *thd, plugin_ref plugin, void *arg) noexcept
+{
+  handlerton *hton= plugin_hton(plugin);
+  if (hton->backup_start_proc_phase)
+    return hton->backup_start_proc_phase(thd, *static_cast<backup_proc_phase*>(arg));
+  return false;
+}
+
+struct backup_end_args
+{
+  backup_end_phase phase;
+  bool abort;
+};
+
 static my_bool backup_end(THD *thd, plugin_ref plugin, void *arg) noexcept
 {
+  backup_end_args *args= static_cast<backup_end_args*>(arg);
   handlerton *hton= plugin_hton(plugin);
   if (hton->backup_end)
-    return hton->backup_end(thd, arg != nullptr);
+    return hton->backup_end(thd, args->phase, args->abort);
   return false;
 }
 
-static my_bool backup_step(THD *thd, plugin_ref plugin, void *) noexcept
+static my_bool backup_step(THD *thd, plugin_ref plugin, void *arg) noexcept
 {
   handlerton *hton= plugin_hton(plugin);
   int res= 0;
   if (hton->backup_step)
-    while ((res= hton->backup_step(thd)))
+    while ((res= hton->backup_step(thd, *static_cast<backup_proc_phase*>(arg))))
       if (res < 0)
         break;
   return res != 0;
@@ -221,6 +236,45 @@ static my_bool backup_finalize(THD *thd, plugin_ref plugin, void *dst) noexcept
   return 0;
 }
 
+static bool start_proc_phase(THD *thd, backup_proc_phase phase)
+{
+  return plugin_foreach_with_mask(thd, backup_start_proc_phase,
+                                  MYSQL_STORAGE_ENGINE_PLUGIN,
+                                  PLUGIN_IS_DELETED|PLUGIN_IS_READY,
+                                  &phase);
+}
+
+static bool run_backup_processing(THD *thd, backup_proc_phase phase)
+{
+  /* The backup_step may be invoked in multiple concurrent threads.
+  At the time backup_end is invoked, all backup_step will have to complete. */
+  return plugin_foreach_with_mask(thd, backup_step,
+                                  MYSQL_STORAGE_ENGINE_PLUGIN,
+                                  PLUGIN_IS_DELETED|PLUGIN_IS_READY,
+                                  &phase);
+}
+
+static bool run_backup_end_phase(THD *thd, backup_end_phase phase, bool failed)
+{
+  backup_end_args args {phase, failed};
+  return plugin_foreach_with_mask(thd, backup_end, MYSQL_STORAGE_ENGINE_PLUGIN,
+                             PLUGIN_IS_DELETED|PLUGIN_IS_READY,
+                             &args) ||
+                             failed;
+}
+
+static bool upgrade_and_process(THD *thd,
+                                MDL_request& mdl_request,
+                                enum_mdl_type lock_level,
+                                backup_proc_phase phase)
+{
+  bool fail= thd->mdl_context.upgrade_shared_lock(mdl_request.ticket,
+                                                  lock_level,
+                                                  thd->variables.lock_wait_timeout) ||
+             start_proc_phase(thd, phase);
+  return fail || run_backup_processing(thd, phase);
+}
+
 bool Sql_cmd_backup::execute(THD *thd)
 {
   if (check_global_access(thd, RELOAD_ACL) ||
@@ -267,20 +321,21 @@ bool Sql_cmd_backup::execute(THD *thd)
                                       MYSQL_STORAGE_ENGINE_PLUGIN,
                                       PLUGIN_IS_DELETED|PLUGIN_IS_READY, &dir);
 
-  /* The backup_step may be invoked in multiple concurrent threads.
-  At the time backup_end is invoked, all backup_step will have to complete. */
   if (!fail)
-    fail= plugin_foreach_with_mask(thd, backup_step,
-                                   MYSQL_STORAGE_ENGINE_PLUGIN,
-                                   PLUGIN_IS_DELETED|PLUGIN_IS_READY, nullptr);
+  {
+    fail= start_proc_phase(thd, BACKUP_PROC_PHASE_BACKUP_LOCKED) ||
+          run_backup_processing(thd, BACKUP_PROC_PHASE_BACKUP_LOCKED) ||
+          upgrade_and_process(thd, mdl_request, MDL_BACKUP_WAIT_DDL,
+                              BACKUP_PROC_PHASE_DDL_LOCKED) ||
+          upgrade_and_process(thd, mdl_request, MDL_BACKUP_WAIT_COMMIT,
+                              BACKUP_PROC_PHASE_COMMIT_LOCKED);
+  }
 
-  fail=
-    thd->mdl_context.upgrade_shared_lock(mdl_request.ticket,
-                                         MDL_BACKUP_WAIT_COMMIT,
-                                         thd->variables.lock_wait_timeout) ||
-    plugin_foreach_with_mask(thd, backup_end, MYSQL_STORAGE_ENGINE_PLUGIN,
-                             PLUGIN_IS_DELETED|PLUGIN_IS_READY,
-                             reinterpret_cast<void*>(fail)) || fail;
+  fail= run_backup_end_phase(thd, BACKUP_END_PHASE_COMMIT_LOCKED, fail) || fail;
+
+  mdl_request.ticket->downgrade_lock(MDL_BACKUP_START);
+
+  fail= run_backup_end_phase(thd, BACKUP_END_PHASE_BACKUP_LOCKED, fail) || fail;
 
   /* The final part must not interfere with the use of the server datadir.
   Release the locks. */

storage/innobase/handler/backup_innodb.cc

@@ -772,14 +772,18 @@ int innodb_backup_start(THD *thd, backup_target target) noexcept
   return innodb_backup.init(thd, target);
 }
 
-int innodb_backup_step(THD *thd) noexcept
+int innodb_backup_step(THD *thd, backup_proc_phase phase) noexcept
 {
-  return innodb_backup.step(thd);
+  if (phase==BACKUP_PROC_PHASE_BACKUP_LOCKED)
+    return innodb_backup.step(thd);
+  return 0;
 }
 
-int innodb_backup_end(THD *thd, bool abort) noexcept
+int innodb_backup_end(THD *thd, backup_end_phase phase, bool abort) noexcept
 {
-  return innodb_backup.end(thd, abort);
+  if (phase==BACKUP_END_PHASE_BACKUP_LOCKED)
+    return innodb_backup.end(thd, abort);
+  return 0;
 }
 
 int innodb_backup_finalize(THD *thd, backup_target target) noexcept

storage/innobase/handler/backup_innodb.h

@@ -25,10 +25,11 @@ int innodb_backup_start(THD *thd, backup_target target) noexcept;
 /**
    Process a file that was collected in backup_start().
    @param thd   current session
+   @param phase current processing phase
    @return number of files remaining, or negative on error
    @retval 0 on completion
 */
-int innodb_backup_step(THD *thd) noexcept;
+int innodb_backup_step(THD *thd, backup_proc_phase phase) noexcept;
 
 /**
    Finish copying and determine the logical time of the backup snapshot.
@@ -37,7 +38,7 @@ int innodb_backup_step(THD *thd) noexcept;
    @return error code
    @retval 0 on success
 */
-int innodb_backup_end(THD *thd, bool abort) noexcept;
+int innodb_backup_end(THD *thd, backup_end_phase phase, bool abort) noexcept;
 
 /**
    Clean up after innodb_backup_end().

storage/maria/ha_maria.cc

@@ -3944,6 +3944,7 @@ static int ha_maria_init(void *p)
   maria_hton->end_backup= maria_end_backup;
   maria_hton->update_optimizer_costs= aria_update_optimizer_costs;
   maria_hton->backup_start= aria_backup_start;
+  maria_hton->backup_start_proc_phase= aria_backup_start_proc_phase;
   maria_hton->backup_step= aria_backup_step;
   maria_hton->backup_end= aria_backup_end;