MDEV-14992 BACKUP SERVER

client/CMakeLists.txt

@@ -93,7 +93,10 @@ TARGET_LINK_LIBRARIES(mariadb-conv mysys strings)
 
 IF(WIN32)
   MYSQL_ADD_EXECUTABLE(echo echo.c COMPONENT Test)
-ENDIF(WIN32)
+ELSE()
+  # just a code example, do not install it
+  CONFIGURE_FILE(mariadb-backup-cat mariadb-backup-cat COPYONLY)
+ENDIF()
 
 # async_example is just a code example, do not install it.
 ADD_EXECUTABLE(async_example async_example.c)

client/mariadb-backup-cat

@@ -0,0 +1,3 @@
+#!/bin/sh
+exec cat > "$MARIADB_DATADIR/$1.tar"
+# This is for use by mysql-test/mtr backup.backup_stream

libmysqld/CMakeLists.txt

@@ -168,6 +168,7 @@ SET(SQL_EMBEDDED_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc
            ../sql/opt_context_store_replay.cc ../sql/opt_context_store_replay.h
            ../sql/sql_path.cc
            ../sql/sql_json_lib.cc ../sql/sql_json_lib.h
+           ../sql/sql_backup.cc
            ${GEN_SOURCES}
            ${MYSYS_LIBWRAP_SOURCE}
 )

mysql-test/collections/buildbot_suites.bat

@@ -6,6 +6,7 @@ innodb,^
 versioning,^
 plugins,^
 mariabackup,^
+backup,^
 roles,^
 auth_gssapi,^
 mysql_sha2,^

mysql-test/main/backup_server.result

@@ -0,0 +1,6 @@
+BACKUP SERVER TO '$datadir/some_directory';
+ERROR HY000: Incorrect arguments to BACKUP SERVER TO
+BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory';
+BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory';
+ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists")
+BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory';

mysql-test/main/backup_server.test

@@ -0,0 +1,12 @@
+--let $datadir=`select @@datadir`
+--error ER_WRONG_ARGUMENTS
+evalp BACKUP SERVER TO '$datadir/some_directory';
+--error 0,1
+--rmdir $MYSQLTEST_VARDIR/some_directory
+evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory';
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--error 21
+evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory';
+--rmdir $MYSQLTEST_VARDIR/some_directory
+evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory';
+--rmdir $MYSQLTEST_VARDIR/some_directory

mysql-test/main/backup_server_locking.result

@@ -0,0 +1,17 @@
+BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory';
+ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists")
+BACKUP STAGE START;
+connect  backup,localhost,root;
+SET STATEMENT max_statement_time=0.1 FOR
+BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory';
+ERROR 70100: Query was interrupted: execution time limit 0.1 sec exceeded
+connection default;
+BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory';
+ERROR HY000: Can't execute the command as you have a BACKUP STAGE active
+BACKUP STAGE END;
+connection backup;
+SET STATEMENT max_statement_time=0.1 FOR
+BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory';
+ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists")
+disconnect backup;
+connection default;

mysql-test/main/backup_server_locking.test

@@ -0,0 +1,31 @@
+--source include/not_embedded.inc
+--source include/count_sessions.inc
+
+--mkdir $MYSQLTEST_VARDIR/some_directory
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--error 21
+evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory';
+
+BACKUP STAGE START;
+--connect (backup,localhost,root)
+--error ER_STATEMENT_TIMEOUT
+evalp SET STATEMENT max_statement_time=0.1 FOR
+BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory';
+
+--connection default
+
+--error ER_BACKUP_LOCK_IS_ACTIVE
+evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory';
+
+BACKUP STAGE END;
+--connection backup
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--error 21
+evalp SET STATEMENT max_statement_time=0.1 FOR
+BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory';
+--disconnect backup
+--connection default
+
+--rmdir $MYSQLTEST_VARDIR/some_directory
+
+--source include/wait_until_count_sessions.inc

mysql-test/main/grant_backup_server.result

@@ -0,0 +1,27 @@
+CREATE USER user1@localhost IDENTIFIED BY '';
+connect con1,localhost,user1;
+BACKUP SERVER TO 'some_directory';
+ERROR 42000: Access denied; you need (at least one of) the RELOAD privilege(s) for this operation
+disconnect con1;
+connection default;
+GRANT SELECT ON test.* TO user1@localhost;
+connect con1,localhost,user1;
+BACKUP SERVER TO 'some_directory';
+ERROR 42000: Access denied; you need (at least one of) the RELOAD privilege(s) for this operation
+disconnect con1;
+connection default;
+GRANT RELOAD ON test.* TO user1@localhost;
+ERROR HY000: Incorrect usage of DB GRANT and GLOBAL PRIVILEGES
+GRANT RELOAD ON *.* TO user1@localhost;
+connect con1,localhost,user1;
+BACKUP SERVER TO 'some_directory';
+ERROR 42000: Access denied; you need (at least one of) the SELECT privilege(s) for this operation
+disconnect con1;
+connection default;
+GRANT SELECT ON *.* TO user1@localhost;
+connect con1,localhost,user1;
+BACKUP SERVER TO '$datadir/some_directory';
+ERROR HY000: Incorrect arguments to BACKUP SERVER TO
+disconnect con1;
+connection default;
+DROP USER user1@localhost;

mysql-test/main/grant_backup_server.test

@@ -0,0 +1,29 @@
+--source include/not_embedded.inc
+CREATE USER user1@localhost IDENTIFIED BY '';
+--connect con1,localhost,user1
+--error ER_SPECIFIC_ACCESS_DENIED_ERROR
+BACKUP SERVER TO 'some_directory';
+--disconnect con1
+--connection default
+GRANT SELECT ON test.* TO user1@localhost;
+--connect con1,localhost,user1
+--error ER_SPECIFIC_ACCESS_DENIED_ERROR
+BACKUP SERVER TO 'some_directory';
+--disconnect con1
+--connection default
+--error ER_WRONG_USAGE
+GRANT RELOAD ON test.* TO user1@localhost;
+GRANT RELOAD ON *.* TO user1@localhost;
+--connect con1,localhost,user1
+--error ER_SPECIFIC_ACCESS_DENIED_ERROR
+BACKUP SERVER TO 'some_directory';
+--disconnect con1
+--connection default
+GRANT SELECT ON *.* TO user1@localhost;
+--connect con1,localhost,user1
+--let $datadir=`select @@datadir`
+--error ER_WRONG_ARGUMENTS
+evalp BACKUP SERVER TO '$datadir/some_directory';
+--disconnect con1
+--connection default
+DROP USER user1@localhost;

mysql-test/main/mysqld--help.result

@@ -2044,7 +2044,7 @@ performance-schema-max-socket-classes 10
 performance-schema-max-socket-instances -1
 performance-schema-max-sql-text-length 1024
 performance-schema-max-stage-classes 170
-performance-schema-max-statement-classes 227
+performance-schema-max-statement-classes 228
 performance-schema-max-statement-stack 10
 performance-schema-max-table-handles -1
 performance-schema-max-table-instances -1

mysql-test/mariadb-test-run.pl

@@ -180,6 +180,7 @@ END
     main-
     archive-
     atomic-
+    backup-
     binlog-
     binlog_encryption-
     binlog_in_engine-

mysql-test/suite/backup/backup_ddl_concurrent_verify.result

@@ -0,0 +1,53 @@
+CREATE TABLE t_myisam (a INT) ENGINE=MyISAM;
+INSERT INTO t_myisam VALUES (1), (2), (3);
+CREATE TABLE t_aria (a INT) ENGINE=Aria TRANSACTIONAL=0;
+INSERT INTO t_aria VALUES (1), (2), (3);
+CREATE TABLE t_arch (a INT) ENGINE=ARCHIVE;
+INSERT INTO t_arch VALUES (1), (2), (3);
+CREATE PROCEDURE reader_churn(IN iterations INT)
+BEGIN
+DECLARE i INT DEFAULT 0;
+WHILE i < iterations DO
+SELECT SLEEP(0.002) INTO @discard
+FROM t_myisam, t_aria, t_arch LIMIT 1;
+SET i = i + 1;
+END WHILE;
+END|
+CREATE PROCEDURE ddl_churn(IN iterations INT)
+BEGIN
+DECLARE i INT DEFAULT 0;
+WHILE i < iterations DO
+ALTER TABLE t_myisam ADD INDEX ix (a);
+ALTER TABLE t_myisam DROP INDEX ix;
+ALTER TABLE t_myisam FORCE, ALGORITHM=COPY;
+SET i = i + 1;
+END WHILE;
+END|
+connect  r1,localhost,root,,;
+CALL reader_churn(800);
+connect  r2,localhost,root,,;
+CALL reader_churn(800);
+connect  d1,localhost,root,,;
+CALL ddl_churn(400);
+connection default;
+connection r1;
+connection r2;
+connection d1;
+connection default;
+disconnect r1;
+disconnect r2;
+disconnect d1;
+# restart: --datadir=MYSQLTEST_VARDIR/backup_ddl_concurrent_verify
+SELECT COUNT(*) FROM t_myisam;
+COUNT(*)
+3
+SELECT COUNT(*) FROM t_aria;
+COUNT(*)
+3
+SELECT COUNT(*) FROM t_arch;
+COUNT(*)
+3
+# restart
+DROP PROCEDURE reader_churn;
+DROP PROCEDURE ddl_churn;
+DROP TABLE t_myisam, t_aria, t_arch;

mysql-test/suite/backup/backup_ddl_concurrent_verify.test

@@ -0,0 +1,137 @@
+--source include/have_archive.inc
+--source include/have_aria.inc
+
+#
+# Verify a backup that nominally succeeded while DDL was running.
+#
+# BACKUP SERVER can fail with ER_LOCK_DEADLOCK next to concurrent DDL, which is
+# what backup_ddl_deadlock reproduces. Because of that, the other DDL tests
+# never get as far as checking the backup they produced. This one tolerates the
+# deadlock and retries until a backup completes, so that a backup taken while
+# DDL was in progress is actually read back.
+#
+# Two things could make such a backup wrong:
+#
+#  - flush_tables() skips a table it cannot lock. The deadlock is reported, but
+#    a plain lock wait timeout is swallowed by its error handler, and the
+#    fallback that opens an instance of its own asks for the metadata lock with
+#    a zero timeout. A DDL merely waiting for an exclusive lock is enough to
+#    make that fail, and in that state the table has not been closed yet, so it
+#    can be copied while still marked open.
+#
+#  - MDL_BACKUP_ALTER_COPY is not blocked by any backup phase, so an
+#    ALTER TABLE ... ALGORITHM=COPY keeps writing its #sql- intermediate files
+#    throughout BACKUP_PHASE_NO_DDL. copy_misc_files() selects files by
+#    extension only and does not exclude them.
+#
+
+CREATE TABLE t_myisam (a INT) ENGINE=MyISAM;
+INSERT INTO t_myisam VALUES (1), (2), (3);
+
+CREATE TABLE t_aria (a INT) ENGINE=Aria TRANSACTIONAL=0;
+INSERT INTO t_aria VALUES (1), (2), (3);
+
+CREATE TABLE t_arch (a INT) ENGINE=ARCHIVE;
+INSERT INTO t_arch VALUES (1), (2), (3);
+
+DELIMITER |;
+# Keeps the tables open, so that flush_tables() has no free TABLE instance and
+# has to take a metadata lock of its own.
+CREATE PROCEDURE reader_churn(IN iterations INT)
+BEGIN
+  DECLARE i INT DEFAULT 0;
+  WHILE i < iterations DO
+    SELECT SLEEP(0.002) INTO @discard
+      FROM t_myisam, t_aria, t_arch LIMIT 1;
+    SET i = i + 1;
+  END WHILE;
+END|
+
+# ADD/DROP INDEX rewrites the index file in place; FORCE with ALGORITHM=COPY
+# rebuilds through a #sql- intermediate table. Row counts are unaffected.
+CREATE PROCEDURE ddl_churn(IN iterations INT)
+BEGIN
+  DECLARE i INT DEFAULT 0;
+  WHILE i < iterations DO
+    ALTER TABLE t_myisam ADD INDEX ix (a);
+    ALTER TABLE t_myisam DROP INDEX ix;
+    ALTER TABLE t_myisam FORCE, ALGORITHM=COPY;
+    SET i = i + 1;
+  END WHILE;
+END|
+DELIMITER ;|
+
+--let $target_directory=$MYSQLTEST_VARDIR/backup_ddl_concurrent_verify
+--error 0,1
+--rmdir $target_directory
+
+--connect (r1,localhost,root,,)
+--send CALL reader_churn(800)
+--connect (r2,localhost,root,,)
+--send CALL reader_churn(800)
+--connect (d1,localhost,root,,)
+--send CALL ddl_churn(400)
+
+--connection default
+
+# Do not start before the DDL is actually running, so that the backup that is
+# kept is one that overlapped it.
+--let $wait_condition= SELECT COUNT(*) FROM information_schema.processlist WHERE info LIKE 'ALTER TABLE t_myisam%'
+--source include/wait_condition.inc
+
+# Retry until one backup gets through, keeping that one.
+--let $attempts= 40
+--let $have_backup= 0
+--disable_query_log
+while ($attempts)
+{
+  --error 0,1
+  --rmdir $target_directory
+  --error 0,ER_LOCK_DEADLOCK
+  --eval BACKUP SERVER TO '$target_directory'
+  if ($mysql_errno == 0)
+  {
+    --let $have_backup= 1
+    --let $attempts= 1
+  }
+  dec $attempts;
+}
+--enable_query_log
+
+--connection r1
+--reap
+--connection r2
+--reap
+--connection d1
+--reap
+
+--connection default
+--disconnect r1
+--disconnect r2
+--disconnect d1
+
+if (!$have_backup)
+{
+  --skip every BACKUP SERVER attempt hit ER_LOCK_DEADLOCK
+}
+
+# An ALTER TABLE ... ALGORITHM=COPY that was running during the backup must not
+# have left its intermediate table in it.
+--list_files $target_directory/test #sql*
+
+# The tables have to come back without needing repair, so any message about a
+# table being crashed or not closed properly fails the test here.
+--let $restart_parameters=--datadir=$target_directory
+--source include/restart_mysqld.inc
+
+SELECT COUNT(*) FROM t_myisam;
+SELECT COUNT(*) FROM t_aria;
+SELECT COUNT(*) FROM t_arch;
+
+--let $restart_parameters=
+--source include/restart_mysqld.inc
+
+DROP PROCEDURE reader_churn;
+DROP PROCEDURE ddl_churn;
+DROP TABLE t_myisam, t_aria, t_arch;
+--rmdir $target_directory

mysql-test/suite/backup/backup_innodb,debug.rdiff

@@ -0,0 +1,19 @@
+--- backup_innodb.result
++++ backup_innodb,debug.result
+@@ -24,13 +24,16 @@
+ BEGIN;
+ DELETE FROM t;
+ connect backup,localhost,root;
++SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume';
+ BACKUP SERVER TO 'target_directory' 4 CONCURRENT;
+ connection default;
++SET DEBUG_SYNC='now WAIT_FOR start';
+ connect rename,localhost,root;
+ RENAME TABLE rt TO rtt;
+ disconnect rename;
+ connection default;
+ INSERT INTO t(a) SELECT * FROM seq_1_to_30000;
++SET DEBUG_SYNC='now SIGNAL resume';
+ connection backup;
+ disconnect backup;
+ connection default;

mysql-test/suite/backup/backup_innodb.combinations

@@ -0,0 +1,4 @@
+[archived]
+innodb_log_archive=ON
+[circular]
+innodb_log_archive=OFF