From 89853ab7bee0bace4a813b98be6ec05379d43f5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 18 Aug 2026 11:14:26 +0300 Subject: [PATCH 1/5] MDEV-40756 Incorrect multi-batch recovery of file size file_name_t::page0_lsn: Keep track of the last applied recv_sys_t::parse_page0() so that a multi-batch recovery will not reset the file to a smaller size. Reviewed by: Thirunarayanan Balathandayuthapani (cherry picked from commit 8f00e6caca633c783140db86d3a48a96de67cf38) --- storage/innobase/log/log0recv.cc | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 736c226cac4fc..690128f5685c9 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -642,6 +642,9 @@ struct file_name_t { /** Log sequence number of a FILE_CREATE record, or 0 */ lsn_t create_lsn = 0; + /** Last FSP_SIZE or FSP_SPACE_FLAGS change, or 0 */ + lsn_t page0_lsn = 0; + /** FSP_SIZE of tablespace */ uint32_t size = 0; @@ -3098,12 +3101,19 @@ void recv_sys_t::parse_page0(const page_id_t id, const byte *b, f{flags ? mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + b) : file_name_t::initial_flags}; recv_spaces_t::iterator it= recv_spaces.find(space_id); - if (it != recv_spaces.end() && !it->second.space) + if (it != recv_spaces.end()) { - if (size) - it->second.size= s; - if (flags) - it->second.flags= f; + if (it->second.page0_lsn > lsn) + return; + it->second.page0_lsn= lsn; + + if (!it->second.space) + { + if (size) + it->second.size= s; + if (flags) + it->second.flags= f; + } } fil_space_set_recv_size_and_flags(space_id, s, f); } From c46764b9b7e59e7e11bcf5b93310addfb794beee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 31 Aug 2026 13:32:58 +0300 Subject: [PATCH 2/5] MDEV-40932 SET GLOBAL innodb_log_archive=OFF may still break recovery log_t::set_archive(archive=false): Ensure that both the latest checkpoint and the latest log record (which has possibly not been written out yet) will carry the log_sys.get_sequence_bit(lsn)==1, to guarantee a successful recovery after the switch to the innodb_log_archive=OFF format. Tested by: Matthias Leich Reviewed by: Thirunarayanan Balathandayuthapani (cherry picked from commit 16373161744c2ebb62aa0d493b96191c62dd35ec) --- storage/innobase/log/log0log.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index f1834bd8f670c..ae23635932886 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -836,7 +836,9 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept */ circular_recovery_from_sequence_bit_0= !get_sequence_bit(wait_lsn); } - else if (circular_recovery_from_sequence_bit_0 || checkpoint < first_lsn) + else if (circular_recovery_from_sequence_bit_0 || + checkpoint < first_lsn || + get_sequence_bit(checkpoint) != get_sequence_bit(wait_lsn)) { /* Some records after the latest checkpoint may have been written From 760fdc300cacac7bef564fc5709f7ed6197c212e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 4 Sep 2026 08:06:01 +0300 Subject: [PATCH 3/5] MDEV-41021 SET GLOBAL innodb_log_archive ignores read-only mode log_t::set_archive(): Prevent a crash in log_t::header_rewrite() by refusing the operation if the log is read-only. --- mysql-test/suite/innodb/r/log_archive.result | 2 ++ mysql-test/suite/innodb/t/log_archive.test | 2 ++ storage/innobase/log/log0log.cc | 6 ++++++ 3 files changed, 10 insertions(+) diff --git a/mysql-test/suite/innodb/r/log_archive.result b/mysql-test/suite/innodb/r/log_archive.result index 3bc20f3c2fc39..6f12acda34b13 100644 --- a/mysql-test/suite/innodb/r/log_archive.result +++ b/mysql-test/suite/innodb/r/log_archive.result @@ -50,6 +50,8 @@ ALTER TABLE mysql.innodb_table_stats ROW_FORMAT=COMPACT, ALGORITHM=INPLACE; ERROR 0A000: ALGORITHM=INPLACE is not supported. Reason: Running in read-only mode. Try ALGORITHM=COPY TRUNCATE TABLE mysql.innodb_table_stats; ERROR HY000: Table 'innodb_table_stats' is read only +SET GLOBAL innodb_log_archive=!@@GLOBAL.innodb_log_archive; +ERROR HY000: InnoDB is in read only mode # restart: --innodb-log-recovery-start=0 --innodb-log-recovery-target=18446744073705357290 SELECT variable_name, variable_value FROM information_schema.global_status WHERE variable_name LIKE 'INNODB_LSN%'; diff --git a/mysql-test/suite/innodb/t/log_archive.test b/mysql-test/suite/innodb/t/log_archive.test index 282a0676c1e9a..925108880ba8a 100644 --- a/mysql-test/suite/innodb/t/log_archive.test +++ b/mysql-test/suite/innodb/t/log_archive.test @@ -148,6 +148,8 @@ DROP TABLE mysql.innodb_index_stats; ALTER TABLE mysql.innodb_table_stats ROW_FORMAT=COMPACT, ALGORITHM=INPLACE; --error ER_OPEN_AS_READONLY TRUNCATE TABLE mysql.innodb_table_stats; +--error ER_INNODB_READ_ONLY +SET GLOBAL innodb_log_archive=!@@GLOBAL.innodb_log_archive; --source include/shutdown_mysqld.inc --move_file $DATADIR/ib_logfile0 $DATADIR/ib_ffffffffff803000.log diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index ae23635932886..5f4a59b41fc3a 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -767,6 +767,12 @@ void log_t::header_rewrite(my_bool archive) noexcept @param thd SQL connection */ void log_t::set_archive(my_bool archive, THD *thd) noexcept { + if (UNIV_UNLIKELY(recv_sys.rpo != 0)) + { + my_error(ER_INNODB_READ_ONLY, MYF(0)); + return; + } + thd_wait_begin(thd, THD_WAIT_DISKIO); tpool::tpool_wait_begin(); lsn_t wait_lsn; From b0944c9a38c3a3d5d7b9c87d3b137f67236bd744 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 4 Sep 2026 16:47:43 +0300 Subject: [PATCH 4/5] MDEV-14992/MDEV-38362/MDEV-39092 BACKUP SERVER MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The following SQL statements will be introduced: BACKUP SERVER TO '/path/to/directory' [ 1 CONCURRENT ]; BACKUP SERVER WITH [ 1 CONCURRENT ] 'command'; In place of the 1, any positive number of threads may be specified. For the first variant, '/path/to' must exist and '/path/to/directory' must be compatible with secure_file_priv and not exist; that is where the backup will be written to. For the second variant, 'command' must be the name of a script or command that will be executed in a child process. The standard input of that command will be in a format that is compatible with GNU tar --format=oldgnu (and also BSD tar variants that are also part of Microsoft Windows and Apple macOS). The command is expected to optionally compress and encrypt the stream and redirect it to a file on a local or a remote server. The BACKUP SERVER WITH will append an additional argument, a positive base-ten number in ASCII, starting with 1, to identify the current thread. In this way, each concurrent stream can write a separate file. The InnoDB write-ahead log will be streamed near the end, because the stream format requires the file sizes to be declared upfront. TODO: implement the following: In BACKUP SERVER TO ... 2 CONCURRENT or more, one thread will be responsible for copying the InnoDB write-ahead log from the beginning of the backup. Other files will be copied by other threads. Note: In single-threaded BACKUP SERVER TO, the parameter innodb_log_recovery_start that is written out to backup.cnf is STRICTLY NECESSARY TO AVOID CORRUPTION during recovery! By default, InnoDB crash recovery starts from the latest available log checkpoint. However, for restoring a backup, recovery must start from the checkpoint that was the latest when the backup was started. Starting recovery from a possible later checkpoint will result in a corrupted database! The backup or the first stream will contain a file backup.cnf, which includes parameters needed for restoring the backup. Currently, these are innodb_log_recovery_start and innodb_log_recovery_target. If innodb_log_recovery_target>0, InnoDB will be in read-only mode, not allowing any writes to persistent files other than via the log application. To restore a streaming backup made with BACKUP SERVER WITH, an empty directory needs to be created and all streams be extracted there using the standard tar utility of the operating system, optionally after undoing any encryption or compression that had been added by the backup command. Then, the backup is prepared or MariaDB server started up on the extracted directory, similar to as if the BACKUP SERVER TO statement had been used. The following will be implemented separately: MDEV-39061 mariadb-backup compatible wrapper script for BACKUP SERVER MDEV-40163 Partial backup and restore MDEV-39091 Back up ENGINE=RocksDB MDEV-40333 Concurrent DDL for Aria tables in BACKUP SERVER The implementation introduces a basic multi-threaded driver Sql_cmd_backup, storage engine interfaces, and basic copying of the storage engines InnoDB, Aria, MyISAM, MERGE (MyISAM), Archive, CSV. backup_target: A structured data type to represent a target directory. On Microsoft Windows, we must use directory paths because there is no variant of CopyFileEx() that would work on file handles. backup_sink: Wraps a per-thread output stream as well as storage engine specific context. handlerton::backup_start(), handlerton::backup_end(): Invoked at the start or end of a backup phase, in the thread that executes a BACKUP SERVER statement. handlerton::backup_step(): A backup step that can be invoked from multiple threads concurrently, between the execution of the corresponding handlerton::backup_start() and handlerton::backup_end() of the same phase. copy_entire_file(): A file copying service for POSIX systems. copy_mmap(): A zero-copy alternative to backup::copy(), to copy from a memory-mapped buffer. copy_file_range_try(): A wrapper for Linux copy_file_range(2), which may fail with EOPNOTSUPP or EXDEV and thus require a fallback to copy_mmap() or backup::copy(). backup::copy(): A partial or sparse file-copying service. On other platforms than FreeBSD or Microsoft Windows, there are shortcut alternatives to this. Note: On Linux we never invoke sendfile(2) for copying between files, because can be much slower than the alternatives. backup_stream_append_plain(): A wrapper of backup::append(), which is the streaming equivalent of backup::copy(). backup_stream_zeropad(): Zero-pad the last tar block if needed. backup_stream_append_async(): A variant of backup_stream_append_plain() where the source file region is guaranteed to be immutable after the call returns. Zero-copy mmap(2) or Linux sendfile(2) are inherently risky for copying data files that may be modified in place, because it could introduce a race condition between a page write that runs concurrently with a child process that is reading the data from the pipe. backup::append(): On systems where we can determine the size of the pipe buffer, invoke backup_stream_append_async() for the initial write, and pread_write() for the last part, to guarantee that the data written by the zero-copy shortcut will have been consumed before the call returns and the caller is able to resume writes to the source region. pread_write(): On 64-bit systems, allocate a buffer of up to 1 MiB. This is the "slow path" of copying or streaming files. struct Aria_backup: Context for multi-threaded backup, comprising directory handles, a mutex and enum Aria_backup_status. aria_backup_start(): Prepare the context for aria_backup_step(). Most files are copied in BACKUP_PHASE_NO_DDL after flush_tables(thd, FLUSH_NON_TRANS_TABLES) has been invoked. All ENGINE=Aria files (including TRANSACTIONAL=0) are copied in BACKUP_PHASE_NO_COMMIT. Thanks to Andrzej Jarząbek for writing test cases and suggesting this logic. aria_backup_step(): Copy one non-ACID file. Acquires Aria_backup::mutex, traverses directories to construct one file name, releases the mutex, and copies the file if one was found. aria_backup_data(): Copy one data file. On Microsoft Windows, this assumes that the current directory is the datadir. This assumption would not hold in the Embedded Server library, which is not supported on Microsoft Windows. aria_backup_log(): Copy one ENGINE=Aria log file. aria_backup_end(): Finish a copying phase and clean up the context. InnoDB_backup::init(): Wait for a possible previous BACKUP SERVER operation to reach the very end of InnoDB_backup::context::cleanup() so that the context can be safely reused. InnoDB_backup::queue: Collection of tablespace IDs and payload sizes at the start of the backup, and the log_sys.first_lsn of log files that have to be included in the backup. If any data file is created or extended while the backup is executing, we must have the corresponding write-ahead-log entries that we are copying since the latest checkpoint that was completed when the backup started. If any tablespaces are deleted during the backup, we may or may not copy them, and the application of a FILE_DELETE record will remove them. Similarly, applying FILE_RENAME or FILE_CREATE records will rename or create files during recovery as needed. log_sys.backup: Whether BACKUP SERVER is in progress. The purpose of this is to make BACKUP SERVER prevent the concurrent execution of SET GLOBAL innodb_log_archive=OFF or SET GLOBAL innodb_log_file_size when innodb_log_archive=OFF. log_sys.archived_checkpoint: Keep track of the earliest available checkpoint, corresponding to log_sys.archived_lsn. This reflects SET GLOBAL innodb_log_recovery_start (which is settable now), for incremental backup. fil_system.have_all_spaces: Whether all tablespace metadata is guaranteed to be known. To speed up startup, InnoDB does not normally open all tablespace files. fil_space_t::create_lsn: Change to Atomic_relaxed and use this to indicate tablespace creation LSN, in addition to indicate undo tablespace rebuild LSN. fil_space_t::backup_end: The first page number that is not being backed up (by default 0, to indicate that no backup is in progress). fil_space_t::BACKUP_BATCH_SIZE: The number of preceding pages that will be covered by fil_space_t::backup_end. This is the unit of "page range locking" during InnoDB backup. buf_page_t::write_fix_try(), buf_page_t::write_unfix_try(): Try to set or unset a fake "write fix" on a page, to prevent concurrent flush() during a backup batch. The atomic operations may run concurrently with set_reinit() and set_freed(). The fake "write fix" does not prevent any concurrent read or write of the page data in the buffer pool; it only blocks writes to the underlying data file. buf_page_t::flush(): Atomically test and set write fix, and skip the operation if the fake "write fix" was set. buf_page_t::set_freed(), buf_page_t::set_reinit(): Employ a compare-and-exchange loop to accommodate for the "write fix". innodb_backup_batch_wait(): Look up any pages that we are about to back up. For any dirty pages, invoke buf_page_t::write_fix_try() to try to set a fake "write fix" lock-free. If the page is currently write-fixed between buf_page_t::flush() and buf_page_t::write_complete(), acquire and release a page U-latch to wait for the conflicting write to complete. InnoDB_backup::backup_batch_start(), InnoDB_backup::backup_batch_stop(): Adjust fil_space_t::backup_end and fake "write fix" of dirty pages to protect the copying of a range of pages from the underlying file. InnoDB_backup::commit(): Enqueue the remaining log to be copied. InnoDB_backup::checkpoint_complete(): If backup is running and commit() has not been called, add each completed innodb_archive_log=ON file to the queue. Else, skip or delete, as appropriate. log_t::backup_start(): If we were running with innodb_log_archive=ON, ensure that the latest file is a valid recovery starting point. That is, wait for the latest log checkpoint to be within the file. buf_flush_list_space(): Check for concurrent backup before writing each page. This is inefficient, but this function may be invoked from multiple threads concurrently, and it cannot be changed easily, especially for fil_crypt_thread(). fil_ibd_create(): Set fil_space_t::create_lsn after the file has been created. dict_load_tablespaces(): Determine the size of each file if upgrade==true. Backup depends on that. buf_dblwr_t::begin(), buf_dblwr_t::end(), buf_dblwr_t::size(): Accessors to allow BACKUP SERVER to skip the contents of the doublewrite buffer in the system tablespace. It is only useful for crash recovery in case a data page had been incompletely written by the time the server was killed. If the server is killed during a backup, the backup will be incomplete and unusable anyway. Furthermore, the page range locking makes page writes and backup mutually exclusive. --- libmysqld/CMakeLists.txt | 1 + mysql-test/collections/buildbot_suites.bat | 1 + mysql-test/main/backup_server.result | 6 + mysql-test/main/backup_server.test | 12 + mysql-test/main/backup_server_locking.result | 17 + mysql-test/main/backup_server_locking.test | 31 + mysql-test/main/grant_backup_server.result | 27 + mysql-test/main/grant_backup_server.test | 29 + mysql-test/main/mysqld--help.result | 2 +- mysql-test/mariadb-test-run.pl | 1 + .../backup_ddl_concurrent_verify.result | 53 + .../backup/backup_ddl_concurrent_verify.test | 137 ++ .../suite/backup/backup_innodb,debug.rdiff | 16 + .../suite/backup/backup_innodb.combinations | 4 + mysql-test/suite/backup/backup_innodb.opt | 1 + mysql-test/suite/backup/backup_innodb.result | 65 + mysql-test/suite/backup/backup_innodb.test | 107 + .../backup/backup_nonacid_flush_stress.result | 51 + .../backup/backup_nonacid_flush_stress.test | 86 + .../suite/backup/backup_stream,debug.rdiff | 16 + .../suite/backup/backup_stream.combinations | 4 + mysql-test/suite/backup/backup_stream.result | 38 + mysql-test/suite/backup/backup_stream.test | 84 + .../backup_sys_stats_not_flushed.result | 34 + .../backup/backup_sys_stats_not_flushed.test | 81 + mysql-test/suite/backup/locks.result | 26 + mysql-test/suite/backup/locks.test | 37 + mysql-test/suite/backup/suite.pm | 19 + .../encryption/r/backup_encrypted,debug.rdiff | 16 + .../encryption/r/backup_encrypted.result | 65 + .../r/backup_encrypted_debug.result | 68 + .../suite/encryption/t/backup_encrypted.opt | 2 + .../suite/encryption/t/backup_encrypted.test | 4 + .../encryption/t/backup_encrypted_debug.opt | 6 + .../encryption/t/backup_encrypted_debug.test | 4 + mysql-test/suite/innodb/r/log_archive.result | 3 + mysql-test/suite/innodb/t/log_archive.test | 8 + .../perfschema/r/max_program_zero.result | 2 +- .../suite/perfschema/r/ortho_iter.result | 2 +- .../perfschema/r/privilege_table_io.result | 2 +- .../r/start_server_disable_idle.result | 2 +- .../r/start_server_disable_stages.result | 2 +- .../r/start_server_disable_statements.result | 2 +- .../start_server_disable_transactions.result | 2 +- .../r/start_server_disable_waits.result | 2 +- .../perfschema/r/start_server_innodb.result | 2 +- .../r/start_server_low_index.result | 2 +- .../r/start_server_low_table_lock.result | 2 +- .../r/start_server_no_account.result | 2 +- .../r/start_server_no_cond_class.result | 2 +- .../r/start_server_no_cond_inst.result | 2 +- .../r/start_server_no_file_class.result | 2 +- .../r/start_server_no_file_inst.result | 2 +- .../perfschema/r/start_server_no_host.result | 2 +- .../perfschema/r/start_server_no_index.result | 2 +- .../perfschema/r/start_server_no_mdl.result | 2 +- .../r/start_server_no_memory_class.result | 2 +- .../r/start_server_no_mutex_class.result | 2 +- .../r/start_server_no_mutex_inst.result | 2 +- ..._server_no_prepared_stmts_instances.result | 2 +- .../r/start_server_no_rwlock_class.result | 2 +- .../r/start_server_no_rwlock_inst.result | 2 +- .../r/start_server_no_setup_actors.result | 2 +- .../r/start_server_no_setup_objects.result | 2 +- .../r/start_server_no_socket_class.result | 2 +- .../r/start_server_no_socket_inst.result | 2 +- .../r/start_server_no_stage_class.result | 2 +- .../r/start_server_no_stages_history.result | 2 +- ...start_server_no_stages_history_long.result | 2 +- .../start_server_no_statements_history.result | 2 +- ...t_server_no_statements_history_long.result | 2 +- .../r/start_server_no_table_hdl.result | 2 +- .../r/start_server_no_table_inst.result | 2 +- .../r/start_server_no_table_lock.result | 2 +- .../r/start_server_no_thread_class.result | 2 +- .../r/start_server_no_thread_inst.result | 2 +- ...tart_server_no_transactions_history.result | 2 +- ...server_no_transactions_history_long.result | 2 +- .../perfschema/r/start_server_no_user.result | 2 +- .../r/start_server_no_waits_history.result | 2 +- .../start_server_no_waits_history_long.result | 2 +- .../perfschema/r/start_server_off.result | 2 +- .../suite/perfschema/r/start_server_on.result | 2 +- .../r/start_server_variables.result | 2 +- .../r/statement_program_lost_inst.result | 2 +- .../suite/sys_vars/r/sysvars_innodb.result | 2 +- sql/CMakeLists.txt | 1 + sql/handler.h | 40 + sql/mysqld.cc | 3 +- sql/mysqld.h | 2 +- sql/sql_backup.cc | 1087 +++++++++++ sql/sql_backup.h | 53 + sql/sql_backup_interface.h | 305 +++ sql/sql_command.h | 1 + sql/sql_parse.cc | 4 +- sql/sql_yacc.yy | 28 +- sql/sys_vars.inl | 2 + storage/innobase/CMakeLists.txt | 2 + storage/innobase/btr/btr0sea.cc | 1 - storage/innobase/buf/buf0buf.cc | 126 +- storage/innobase/buf/buf0dblwr.cc | 2 +- storage/innobase/buf/buf0flu.cc | 152 +- storage/innobase/buf/buf0lru.cc | 11 +- storage/innobase/dict/dict0load.cc | 30 +- storage/innobase/fil/fil0fil.cc | 5 +- storage/innobase/handler/backup_innodb.cc | 1713 +++++++++++++++++ storage/innobase/handler/backup_innodb.h | 58 + storage/innobase/handler/ha_innodb.cc | 54 +- storage/innobase/include/buf0buf.h | 27 +- storage/innobase/include/buf0dblwr.h | 7 + storage/innobase/include/fil0fil.h | 30 +- storage/innobase/include/log0log.h | 62 +- storage/innobase/log/log0log.cc | 80 +- storage/innobase/log/log0recv.cc | 11 +- storage/innobase/mtr/mtr0mtr.cc | 69 +- storage/innobase/os/os0file.cc | 10 +- storage/maria/CMakeLists.txt | 1 + storage/maria/ha_maria.cc | 26 + storage/maria/ma_backup_server.c | 878 +++++++++ storage/maria/ma_backup_server.h | 68 + 120 files changed, 5974 insertions(+), 235 deletions(-) create mode 100644 mysql-test/main/backup_server.result create mode 100644 mysql-test/main/backup_server.test create mode 100644 mysql-test/main/backup_server_locking.result create mode 100644 mysql-test/main/backup_server_locking.test create mode 100644 mysql-test/main/grant_backup_server.result create mode 100644 mysql-test/main/grant_backup_server.test create mode 100644 mysql-test/suite/backup/backup_ddl_concurrent_verify.result create mode 100644 mysql-test/suite/backup/backup_ddl_concurrent_verify.test create mode 100644 mysql-test/suite/backup/backup_innodb,debug.rdiff create mode 100644 mysql-test/suite/backup/backup_innodb.combinations create mode 100644 mysql-test/suite/backup/backup_innodb.opt create mode 100644 mysql-test/suite/backup/backup_innodb.result create mode 100644 mysql-test/suite/backup/backup_innodb.test create mode 100644 mysql-test/suite/backup/backup_nonacid_flush_stress.result create mode 100644 mysql-test/suite/backup/backup_nonacid_flush_stress.test create mode 100644 mysql-test/suite/backup/backup_stream,debug.rdiff create mode 100644 mysql-test/suite/backup/backup_stream.combinations create mode 100644 mysql-test/suite/backup/backup_stream.result create mode 100644 mysql-test/suite/backup/backup_stream.test create mode 100644 mysql-test/suite/backup/backup_sys_stats_not_flushed.result create mode 100644 mysql-test/suite/backup/backup_sys_stats_not_flushed.test create mode 100644 mysql-test/suite/backup/locks.result create mode 100644 mysql-test/suite/backup/locks.test create mode 100644 mysql-test/suite/backup/suite.pm create mode 100644 mysql-test/suite/encryption/r/backup_encrypted,debug.rdiff create mode 100644 mysql-test/suite/encryption/r/backup_encrypted.result create mode 100644 mysql-test/suite/encryption/r/backup_encrypted_debug.result create mode 100644 mysql-test/suite/encryption/t/backup_encrypted.opt create mode 100644 mysql-test/suite/encryption/t/backup_encrypted.test create mode 100644 mysql-test/suite/encryption/t/backup_encrypted_debug.opt create mode 100644 mysql-test/suite/encryption/t/backup_encrypted_debug.test create mode 100644 sql/sql_backup.cc create mode 100644 sql/sql_backup.h create mode 100644 sql/sql_backup_interface.h create mode 100644 storage/innobase/handler/backup_innodb.cc create mode 100644 storage/innobase/handler/backup_innodb.h create mode 100644 storage/maria/ma_backup_server.c create mode 100644 storage/maria/ma_backup_server.h diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index 80b202aa4c03e..59f386a86edae 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -166,6 +166,7 @@ SET(SQL_EMBEDDED_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc ../sql/opt_hints.cc ../sql/opt_hints.h ../sql/opt_trace_ddl_info.cc ../sql/opt_trace_ddl_info.h ../sql/sql_path.cc + ../sql/sql_backup.cc ${GEN_SOURCES} ${MYSYS_LIBWRAP_SOURCE} ) diff --git a/mysql-test/collections/buildbot_suites.bat b/mysql-test/collections/buildbot_suites.bat index e673afd50c926..cde563ac97877 100644 --- a/mysql-test/collections/buildbot_suites.bat +++ b/mysql-test/collections/buildbot_suites.bat @@ -6,6 +6,7 @@ innodb,^ versioning,^ plugins,^ mariabackup,^ +backup,^ roles,^ auth_gssapi,^ mysql_sha2,^ diff --git a/mysql-test/main/backup_server.result b/mysql-test/main/backup_server.result new file mode 100644 index 0000000000000..7a064d7e5fe11 --- /dev/null +++ b/mysql-test/main/backup_server.result @@ -0,0 +1,6 @@ +BACKUP SERVER TO '$datadir/some_directory'; +ERROR HY000: Incorrect arguments to BACKUP SERVER TO +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists") +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; diff --git a/mysql-test/main/backup_server.test b/mysql-test/main/backup_server.test new file mode 100644 index 0000000000000..f9985c96b075f --- /dev/null +++ b/mysql-test/main/backup_server.test @@ -0,0 +1,12 @@ +--let $datadir=`select @@datadir` +--error ER_WRONG_ARGUMENTS +evalp BACKUP SERVER TO '$datadir/some_directory'; +--error 0,1 +--rmdir $MYSQLTEST_VARDIR/some_directory +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 21 +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--rmdir $MYSQLTEST_VARDIR/some_directory +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--rmdir $MYSQLTEST_VARDIR/some_directory diff --git a/mysql-test/main/backup_server_locking.result b/mysql-test/main/backup_server_locking.result new file mode 100644 index 0000000000000..6e49a09e3e873 --- /dev/null +++ b/mysql-test/main/backup_server_locking.result @@ -0,0 +1,17 @@ +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists") +BACKUP STAGE START; +connect backup,localhost,root; +SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR 70100: Query was interrupted: execution time limit 0.1 sec exceeded +connection default; +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't execute the command as you have a BACKUP STAGE active +BACKUP STAGE END; +connection backup; +SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists") +disconnect backup; +connection default; diff --git a/mysql-test/main/backup_server_locking.test b/mysql-test/main/backup_server_locking.test new file mode 100644 index 0000000000000..4e769b61b2f20 --- /dev/null +++ b/mysql-test/main/backup_server_locking.test @@ -0,0 +1,31 @@ +--source include/not_embedded.inc +--source include/count_sessions.inc + +--mkdir $MYSQLTEST_VARDIR/some_directory +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 21 +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; + +BACKUP STAGE START; +--connect (backup,localhost,root) +--error ER_STATEMENT_TIMEOUT +evalp SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; + +--connection default + +--error ER_BACKUP_LOCK_IS_ACTIVE +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; + +BACKUP STAGE END; +--connection backup +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 21 +evalp SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--disconnect backup +--connection default + +--rmdir $MYSQLTEST_VARDIR/some_directory + +--source include/wait_until_count_sessions.inc diff --git a/mysql-test/main/grant_backup_server.result b/mysql-test/main/grant_backup_server.result new file mode 100644 index 0000000000000..a4d85292dee45 --- /dev/null +++ b/mysql-test/main/grant_backup_server.result @@ -0,0 +1,27 @@ +CREATE USER user1@localhost IDENTIFIED BY ''; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the RELOAD privilege(s) for this operation +disconnect con1; +connection default; +GRANT SELECT ON test.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the RELOAD privilege(s) for this operation +disconnect con1; +connection default; +GRANT RELOAD ON test.* TO user1@localhost; +ERROR HY000: Incorrect usage of DB GRANT and GLOBAL PRIVILEGES +GRANT RELOAD ON *.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the SELECT privilege(s) for this operation +disconnect con1; +connection default; +GRANT SELECT ON *.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO '$datadir/some_directory'; +ERROR HY000: Incorrect arguments to BACKUP SERVER TO +disconnect con1; +connection default; +DROP USER user1@localhost; diff --git a/mysql-test/main/grant_backup_server.test b/mysql-test/main/grant_backup_server.test new file mode 100644 index 0000000000000..726abb19908bb --- /dev/null +++ b/mysql-test/main/grant_backup_server.test @@ -0,0 +1,29 @@ +--source include/not_embedded.inc +CREATE USER user1@localhost IDENTIFIED BY ''; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +GRANT SELECT ON test.* TO user1@localhost; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +--error ER_WRONG_USAGE +GRANT RELOAD ON test.* TO user1@localhost; +GRANT RELOAD ON *.* TO user1@localhost; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +GRANT SELECT ON *.* TO user1@localhost; +--connect con1,localhost,user1 +--let $datadir=`select @@datadir` +--error ER_WRONG_ARGUMENTS +evalp BACKUP SERVER TO '$datadir/some_directory'; +--disconnect con1 +--connection default +DROP USER user1@localhost; diff --git a/mysql-test/main/mysqld--help.result b/mysql-test/main/mysqld--help.result index 4e6593e67af42..93a89adcfef96 100644 --- a/mysql-test/main/mysqld--help.result +++ b/mysql-test/main/mysqld--help.result @@ -2039,7 +2039,7 @@ performance-schema-max-socket-classes 10 performance-schema-max-socket-instances -1 performance-schema-max-sql-text-length 1024 performance-schema-max-stage-classes 170 -performance-schema-max-statement-classes 227 +performance-schema-max-statement-classes 228 performance-schema-max-statement-stack 10 performance-schema-max-table-handles -1 performance-schema-max-table-instances -1 diff --git a/mysql-test/mariadb-test-run.pl b/mysql-test/mariadb-test-run.pl index 9aadfa5456d61..a0e751d5434d1 100755 --- a/mysql-test/mariadb-test-run.pl +++ b/mysql-test/mariadb-test-run.pl @@ -180,6 +180,7 @@ END main- archive- atomic- + backup- binlog- binlog_encryption- binlog_in_engine- diff --git a/mysql-test/suite/backup/backup_ddl_concurrent_verify.result b/mysql-test/suite/backup/backup_ddl_concurrent_verify.result new file mode 100644 index 0000000000000..6c9f12392a430 --- /dev/null +++ b/mysql-test/suite/backup/backup_ddl_concurrent_verify.result @@ -0,0 +1,53 @@ +CREATE TABLE t_myisam (a INT) ENGINE=MyISAM; +INSERT INTO t_myisam VALUES (1), (2), (3); +CREATE TABLE t_aria (a INT) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO t_aria VALUES (1), (2), (3); +CREATE TABLE t_arch (a INT) ENGINE=ARCHIVE; +INSERT INTO t_arch VALUES (1), (2), (3); +CREATE PROCEDURE reader_churn(IN iterations INT) +BEGIN +DECLARE i INT DEFAULT 0; +WHILE i < iterations DO +SELECT SLEEP(0.002) INTO @discard +FROM t_myisam, t_aria, t_arch LIMIT 1; +SET i = i + 1; +END WHILE; +END| +CREATE PROCEDURE ddl_churn(IN iterations INT) +BEGIN +DECLARE i INT DEFAULT 0; +WHILE i < iterations DO +ALTER TABLE t_myisam ADD INDEX ix (a); +ALTER TABLE t_myisam DROP INDEX ix; +ALTER TABLE t_myisam FORCE, ALGORITHM=COPY; +SET i = i + 1; +END WHILE; +END| +connect r1,localhost,root,,; +CALL reader_churn(800); +connect r2,localhost,root,,; +CALL reader_churn(800); +connect d1,localhost,root,,; +CALL ddl_churn(400); +connection default; +connection r1; +connection r2; +connection d1; +connection default; +disconnect r1; +disconnect r2; +disconnect d1; +# restart: --datadir=MYSQLTEST_VARDIR/backup_ddl_concurrent_verify +SELECT COUNT(*) FROM t_myisam; +COUNT(*) +3 +SELECT COUNT(*) FROM t_aria; +COUNT(*) +3 +SELECT COUNT(*) FROM t_arch; +COUNT(*) +3 +# restart +DROP PROCEDURE reader_churn; +DROP PROCEDURE ddl_churn; +DROP TABLE t_myisam, t_aria, t_arch; diff --git a/mysql-test/suite/backup/backup_ddl_concurrent_verify.test b/mysql-test/suite/backup/backup_ddl_concurrent_verify.test new file mode 100644 index 0000000000000..b9ce93ff1147d --- /dev/null +++ b/mysql-test/suite/backup/backup_ddl_concurrent_verify.test @@ -0,0 +1,137 @@ +--source include/have_archive.inc +--source include/have_aria.inc + +# +# Verify a backup that nominally succeeded while DDL was running. +# +# BACKUP SERVER can fail with ER_LOCK_DEADLOCK next to concurrent DDL, which is +# what backup_ddl_deadlock reproduces. Because of that, the other DDL tests +# never get as far as checking the backup they produced. This one tolerates the +# deadlock and retries until a backup completes, so that a backup taken while +# DDL was in progress is actually read back. +# +# Two things could make such a backup wrong: +# +# - flush_tables() skips a table it cannot lock. The deadlock is reported, but +# a plain lock wait timeout is swallowed by its error handler, and the +# fallback that opens an instance of its own asks for the metadata lock with +# a zero timeout. A DDL merely waiting for an exclusive lock is enough to +# make that fail, and in that state the table has not been closed yet, so it +# can be copied while still marked open. +# +# - MDL_BACKUP_ALTER_COPY is not blocked by any backup phase, so an +# ALTER TABLE ... ALGORITHM=COPY keeps writing its #sql- intermediate files +# throughout BACKUP_PHASE_NO_DDL. copy_misc_files() selects files by +# extension only and does not exclude them. +# + +CREATE TABLE t_myisam (a INT) ENGINE=MyISAM; +INSERT INTO t_myisam VALUES (1), (2), (3); + +CREATE TABLE t_aria (a INT) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO t_aria VALUES (1), (2), (3); + +CREATE TABLE t_arch (a INT) ENGINE=ARCHIVE; +INSERT INTO t_arch VALUES (1), (2), (3); + +DELIMITER |; +# Keeps the tables open, so that flush_tables() has no free TABLE instance and +# has to take a metadata lock of its own. +CREATE PROCEDURE reader_churn(IN iterations INT) +BEGIN + DECLARE i INT DEFAULT 0; + WHILE i < iterations DO + SELECT SLEEP(0.002) INTO @discard + FROM t_myisam, t_aria, t_arch LIMIT 1; + SET i = i + 1; + END WHILE; +END| + +# ADD/DROP INDEX rewrites the index file in place; FORCE with ALGORITHM=COPY +# rebuilds through a #sql- intermediate table. Row counts are unaffected. +CREATE PROCEDURE ddl_churn(IN iterations INT) +BEGIN + DECLARE i INT DEFAULT 0; + WHILE i < iterations DO + ALTER TABLE t_myisam ADD INDEX ix (a); + ALTER TABLE t_myisam DROP INDEX ix; + ALTER TABLE t_myisam FORCE, ALGORITHM=COPY; + SET i = i + 1; + END WHILE; +END| +DELIMITER ;| + +--let $target_directory=$MYSQLTEST_VARDIR/backup_ddl_concurrent_verify +--error 0,1 +--rmdir $target_directory + +--connect (r1,localhost,root,,) +--send CALL reader_churn(800) +--connect (r2,localhost,root,,) +--send CALL reader_churn(800) +--connect (d1,localhost,root,,) +--send CALL ddl_churn(400) + +--connection default + +# Do not start before the DDL is actually running, so that the backup that is +# kept is one that overlapped it. +--let $wait_condition= SELECT COUNT(*) FROM information_schema.processlist WHERE info LIKE 'ALTER TABLE t_myisam%' +--source include/wait_condition.inc + +# Retry until one backup gets through, keeping that one. +--let $attempts= 40 +--let $have_backup= 0 +--disable_query_log +while ($attempts) +{ + --error 0,1 + --rmdir $target_directory + --error 0,ER_LOCK_DEADLOCK + --eval BACKUP SERVER TO '$target_directory' + if ($mysql_errno == 0) + { + --let $have_backup= 1 + --let $attempts= 1 + } + dec $attempts; +} +--enable_query_log + +--connection r1 +--reap +--connection r2 +--reap +--connection d1 +--reap + +--connection default +--disconnect r1 +--disconnect r2 +--disconnect d1 + +if (!$have_backup) +{ + --skip every BACKUP SERVER attempt hit ER_LOCK_DEADLOCK +} + +# An ALTER TABLE ... ALGORITHM=COPY that was running during the backup must not +# have left its intermediate table in it. +--list_files $target_directory/test #sql* + +# The tables have to come back without needing repair, so any message about a +# table being crashed or not closed properly fails the test here. +--let $restart_parameters=--datadir=$target_directory +--source include/restart_mysqld.inc + +SELECT COUNT(*) FROM t_myisam; +SELECT COUNT(*) FROM t_aria; +SELECT COUNT(*) FROM t_arch; + +--let $restart_parameters= +--source include/restart_mysqld.inc + +DROP PROCEDURE reader_churn; +DROP PROCEDURE ddl_churn; +DROP TABLE t_myisam, t_aria, t_arch; +--rmdir $target_directory diff --git a/mysql-test/suite/backup/backup_innodb,debug.rdiff b/mysql-test/suite/backup/backup_innodb,debug.rdiff new file mode 100644 index 0000000000000..c16f904e2ad20 --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb,debug.rdiff @@ -0,0 +1,16 @@ +--- backup_innodb.result ++++ backup_innodb,debug.result +@@ -20,10 +20,13 @@ + BEGIN; + DELETE FROM t; + connect backup,localhost,root; ++SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; + BACKUP SERVER TO 'target_directory' 4 CONCURRENT; + connection default; ++SET DEBUG_SYNC='now WAIT_FOR start'; + INSERT INTO t(a) SELECT * FROM seq_1_to_30000; ++SET DEBUG_SYNC='now SIGNAL resume'; + connection backup; + disconnect backup; + connection default; + ROLLBACK; diff --git a/mysql-test/suite/backup/backup_innodb.combinations b/mysql-test/suite/backup/backup_innodb.combinations new file mode 100644 index 0000000000000..7fd419bba7692 --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.combinations @@ -0,0 +1,4 @@ +[archived] +innodb_log_archive=ON +[circular] +innodb_log_archive=OFF diff --git a/mysql-test/suite/backup/backup_innodb.opt b/mysql-test/suite/backup/backup_innodb.opt new file mode 100644 index 0000000000000..0902441fde5cb --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.opt @@ -0,0 +1 @@ +--secure-file-priv=$MYSQLTEST_VARDIR diff --git a/mysql-test/suite/backup/backup_innodb.result b/mysql-test/suite/backup/backup_innodb.result new file mode 100644 index 0000000000000..2fafd6cf6574f --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.result @@ -0,0 +1,65 @@ +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (2), (3), (5), (7); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (1), (1), (2), (3), (5); +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +BEGIN; +INSERT INTO t SET a=1; +BACKUP SERVER TO '$target_directory'; +ERROR HY000: The MariaDB server is running with the --secure-file-priv option so it cannot execute this statement +BACKUP SERVER TO '$target_directory'; +ROLLBACK; +SELECT * FROM t; +a b +1 +BACKUP SERVER TO '$target_directory' WITH '/bin/false'; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH '/bin/false'' at line 1 +BACKUP SERVER TO '$target_directory' WITH 4 CONCURRENT '/bin/false'; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH 4 CONCURRENT '/bin/false'' at line 1 +BACKUP SERVER WITH '/bin/false'; +ERROR HY000: IO Write error: (...) BACKUP SERVER +BEGIN; +DELETE FROM t; +connect backup,localhost,root; +BACKUP SERVER TO 'target_directory' 4 CONCURRENT; +connection default; +INSERT INTO t(a) SELECT * FROM seq_1_to_30000; +connection backup; +disconnect backup; +connection default; +ROLLBACK; +SELECT * FROM t; +a b +1 +DELETE FROM t; +DROP TABLE at0, at1; +# restart: --defaults-file=MYSQLTEST_VARDIR/some_directory/backup.cnf --datadir=MYSQLTEST_VARDIR/some_directory +SELECT * FROM t; +a b +1 +DELETE FROM t; +ERROR HY000: Table 't' is read only +SELECT * FROM at0; +i +1 +1 +2 +3 +5 +SELECT * FROM at1; +i +2 +3 +5 +7 +DROP TABLE t, at0, at1; +ERROR HY000: Table 't' is read only +SELECT * FROM at0; +ERROR 42S02: Table 'test.at0' doesn't exist +SELECT * FROM at1; +ERROR 42S02: Table 'test.at1' doesn't exist +# restart +SELECT * FROM t; +a b +DROP TABLE t; diff --git a/mysql-test/suite/backup/backup_innodb.test b/mysql-test/suite/backup/backup_innodb.test new file mode 100644 index 0000000000000..8dd670134a52b --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.test @@ -0,0 +1,107 @@ +--source include/have_sequence.inc +--source include/have_innodb.inc +--source include/maybe_debug.inc + +--disable_query_log +call mtr.add_suppression("mariadbd.*: IO Write error:"); +--enable_query_log + +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (2), (3), (5), (7); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (1), (1), (2), (3), (5); + +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +BEGIN; +INSERT INTO t SET a=1; + +--let $target_directory=/tmp/some_directory$MTR_COMBINATION_ARCHIVED +--error ER_OPTION_PREVENTS_STATEMENT +evalp BACKUP SERVER TO '$target_directory'; + +# comment out the following line +# (and replace all "rmdir" with "exec rm -fr" and remove backup_innodb.opt) +# to test cross-filesystem copy +--let $target_directory=$MYSQLTEST_VARDIR/some_directory + +# Clean up after a previous failed test, in case we are retrying. +--error 0,1 +--rmdir $target_directory + +evalp BACKUP SERVER TO '$target_directory'; +--rmdir $target_directory +ROLLBACK; +# BACKUP SERVER will implicitly commit the current transaction +SELECT * FROM t; + +--error ER_PARSE_ERROR +evalp BACKUP SERVER TO '$target_directory' WITH '/bin/false'; +--error ER_PARSE_ERROR +evalp BACKUP SERVER TO '$target_directory' WITH 4 CONCURRENT '/bin/false'; +--replace_regex /\(.*\)/(...)/ +--error ER_IO_WRITE_ERROR +BACKUP SERVER WITH '/bin/false'; + +BEGIN; +DELETE FROM t; + +--connect backup,localhost,root +if ($have_debug) { +SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; +} +--replace_result $target_directory target_directory +send_eval BACKUP SERVER TO '$target_directory' 4 CONCURRENT; +--connection default +if ($have_debug) { +SET DEBUG_SYNC='now WAIT_FOR start'; +} +INSERT INTO t(a) SELECT * FROM seq_1_to_30000; +if ($have_debug) { +SET DEBUG_SYNC='now SIGNAL resume'; +} +--connection backup +--reap +--disconnect backup +--connection default +ROLLBACK; +SELECT * FROM t; +DELETE FROM t; +DROP TABLE at0, at1; + +if ($MARIADB_UPGRADE_EXE) { +let target_directory=$target_directory; +perl; +open IN, "<", "$ENV{MYSQLTEST_VARDIR}/my.cnf"; +open OUT, ">>", "$ENV{target_directory}/backup.cnf"; +print OUT while (); +close(IN); +close(OUT); +EOF +} +if (!$MARIADB_UPGRADE_EXE) { + --exec cat $MYSQLTEST_VARDIR/my.cnf >> $target_directory/backup.cnf +} +--let $restart_parameters=--defaults-file=$target_directory/backup.cnf --datadir=$target_directory +--source include/restart_mysqld.inc + +SELECT * FROM t; +# A nonzero innodb_log_recovery_target makes InnoDB read-only. +--error ER_OPEN_AS_READONLY +DELETE FROM t; +# Non-InnoDB tables are read-write. +SELECT * FROM at0; +SELECT * FROM at1; +--error ER_OPEN_AS_READONLY +DROP TABLE t, at0, at1; +--error ER_NO_SUCH_TABLE +SELECT * FROM at0; +--error ER_NO_SUCH_TABLE +SELECT * FROM at1; + +--let $restart_parameters= +--source include/restart_mysqld.inc +SELECT * FROM t; +DROP TABLE t; + +--rmdir $target_directory diff --git a/mysql-test/suite/backup/backup_nonacid_flush_stress.result b/mysql-test/suite/backup/backup_nonacid_flush_stress.result new file mode 100644 index 0000000000000..de15304712338 --- /dev/null +++ b/mysql-test/suite/backup/backup_nonacid_flush_stress.result @@ -0,0 +1,51 @@ +CREATE TABLE t_myisam (a INT) ENGINE=MyISAM; +INSERT INTO t_myisam VALUES (1), (2), (3); +CREATE TABLE t_aria (a INT) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO t_aria VALUES (1), (2), (3); +CREATE TABLE t_arch (a INT) ENGINE=ARCHIVE; +INSERT INTO t_arch VALUES (1), (2), (3); +CREATE PROCEDURE churn(IN i INT) +BEGIN +WHILE i > 0 DO +SELECT SLEEP(0.002) INTO @discard +FROM t_myisam, t_aria, t_arch LIMIT 1; +SET i = i - 1; +END WHILE; +END| +connect r1,localhost,root,,; +CALL churn(400); +connect r2,localhost,root,,; +CALL churn(400); +connect r3,localhost,root,,; +CALL churn(400); +connect r4,localhost,root,,; +CALL churn(400); +connection default; +connection r1; +connection r2; +connection r3; +connection r4; +connection default; +disconnect r1; +disconnect r2; +disconnect r3; +disconnect r4; +# restart: --datadir=MYSQLTEST_VARDIR/backup_nonacid_purge_stress +SELECT * FROM t_myisam ORDER BY a; +a +1 +2 +3 +SELECT * FROM t_aria ORDER BY a; +a +1 +2 +3 +SELECT * FROM t_arch ORDER BY a; +a +1 +2 +3 +# restart +DROP PROCEDURE churn; +DROP TABLE t_myisam, t_aria, t_arch; diff --git a/mysql-test/suite/backup/backup_nonacid_flush_stress.test b/mysql-test/suite/backup/backup_nonacid_flush_stress.test new file mode 100644 index 0000000000000..ecf258785cfcf --- /dev/null +++ b/mysql-test/suite/backup/backup_nonacid_flush_stress.test @@ -0,0 +1,86 @@ +--source include/have_archive.inc +--source include/have_aria.inc + +# +# Stress test: several readers open and close tables as fast as +# they can while backups run back to back. This exercises the arrival of a +# reader at every point of the flush. + +CREATE TABLE t_myisam (a INT) ENGINE=MyISAM; +INSERT INTO t_myisam VALUES (1), (2), (3); + +CREATE TABLE t_aria (a INT) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO t_aria VALUES (1), (2), (3); + +CREATE TABLE t_arch (a INT) ENGINE=ARCHIVE; +INSERT INTO t_arch VALUES (1), (2), (3); + +# Each iteration opens all three tables, holds them for a few milliseconds and +# closes them again, so the readers cycle through open and closed states many +# times over the course of a backup. +DELIMITER |; +CREATE PROCEDURE churn(IN i INT) +BEGIN + WHILE i > 0 DO + SELECT SLEEP(0.002) INTO @discard + FROM t_myisam, t_aria, t_arch LIMIT 1; + SET i = i - 1; + END WHILE; +END| +DELIMITER ;| + +--let $target_directory=$MYSQLTEST_VARDIR/backup_nonacid_purge_stress +--error 0,1 +--rmdir $target_directory + +--connect (r1,localhost,root,,) +--send CALL churn(400) +--connect (r2,localhost,root,,) +--send CALL churn(400) +--connect (r3,localhost,root,,) +--send CALL churn(400) +--connect (r4,localhost,root,,) +--send CALL churn(400) + +--connection default +--let $i= 6 +while ($i) +{ + --error 0,1 + --rmdir $target_directory + --disable_query_log + --eval BACKUP SERVER TO '$target_directory' + --enable_query_log + dec $i; +} + +--connection r1 +--reap +--connection r2 +--reap +--connection r3 +--reap +--connection r4 +--reap + +--connection default +--disconnect r1 +--disconnect r2 +--disconnect r3 +--disconnect r4 + +# The last backup was taken with the readers still running, so it is only +# complete if every table was closed before its files were copied. +--let $restart_parameters=--datadir=$target_directory +--source include/restart_mysqld.inc + +SELECT * FROM t_myisam ORDER BY a; +SELECT * FROM t_aria ORDER BY a; +SELECT * FROM t_arch ORDER BY a; + +--let $restart_parameters= +--source include/restart_mysqld.inc + +DROP PROCEDURE churn; +DROP TABLE t_myisam, t_aria, t_arch; +--rmdir $target_directory diff --git a/mysql-test/suite/backup/backup_stream,debug.rdiff b/mysql-test/suite/backup/backup_stream,debug.rdiff new file mode 100644 index 0000000000000..4900994abacba --- /dev/null +++ b/mysql-test/suite/backup/backup_stream,debug.rdiff @@ -0,0 +1,16 @@ +--- backup_stream.result ++++ backup_stream,debug.result +@@ -8,10 +8,13 @@ + BEGIN; + DELETE FROM t; + connect backup,localhost,root; ++SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; + BACKUP SERVER WITH 2 CONCURRENT 'stream.bat'; + connection default; ++SET DEBUG_SYNC='now WAIT_FOR start'; + INSERT INTO t(a) SELECT * FROM seq_1_to_30000; ++SET DEBUG_SYNC='now SIGNAL resume'; + connection backup; + disconnect backup; + connection default; + ROLLBACK; diff --git a/mysql-test/suite/backup/backup_stream.combinations b/mysql-test/suite/backup/backup_stream.combinations new file mode 100644 index 0000000000000..7fd419bba7692 --- /dev/null +++ b/mysql-test/suite/backup/backup_stream.combinations @@ -0,0 +1,4 @@ +[archived] +innodb_log_archive=ON +[circular] +innodb_log_archive=OFF diff --git a/mysql-test/suite/backup/backup_stream.result b/mysql-test/suite/backup/backup_stream.result new file mode 100644 index 0000000000000..e9913f7d82e87 --- /dev/null +++ b/mysql-test/suite/backup/backup_stream.result @@ -0,0 +1,38 @@ +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +INSERT INTO t SET a=1; +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (1); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (0); +BEGIN; +DELETE FROM t; +connect backup,localhost,root; +BACKUP SERVER WITH 2 CONCURRENT 'stream.bat'; +connection default; +INSERT INTO t(a) SELECT * FROM seq_1_to_30000; +connection backup; +disconnect backup; +connection default; +ROLLBACK; +SELECT * FROM t; +a b +1 +DELETE FROM t; +DROP TABLE at0, at1; +# restart: --defaults-file=MYSQLTEST_VARDIR/backup_stream/backup.cnf --datadir=MYSQLTEST_VARDIR/backup_stream +SELECT * FROM at0; +i +0 +SELECT * FROM at1; +i +1 +SELECT * FROM t; +a b +1 +DELETE FROM t; +ERROR HY000: Table 't' is read only +# restart +SELECT * FROM t; +a b +DROP TABLE t; diff --git a/mysql-test/suite/backup/backup_stream.test b/mysql-test/suite/backup/backup_stream.test new file mode 100644 index 0000000000000..ff1d98d3f1766 --- /dev/null +++ b/mysql-test/suite/backup/backup_stream.test @@ -0,0 +1,84 @@ +--source include/have_sequence.inc +--source include/have_innodb.inc +--source include/maybe_debug.inc + +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +INSERT INTO t SET a=1; + +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (1); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (0); + +--remove_files_wildcard $MYSQL_TMP_DIR/. stream.bat +--write_file $MYSQL_TMP_DIR/stream.bat +#!/bin/sh +exec cat > $MYSQL_TMP_DIR/$1.tar +EOF + +--let $script=$MYSQL_TMP_DIR/stream.bat +if (!$MARIADB_UPGRADE_EXE) { +# Because we may run on Linux /dev/shm which may be mounted as noexec, +# we cannot rely on chmod +x, but must explicitly invoke a shell on the script. +--let $script=/bin/sh $script +} +if ($MARIADB_UPGRADE_EXE) +{ +--exec echo "@cat > %MYSQL_TMP_DIR%\%1.tar" > $MYSQL_TMP_DIR/stream.bat +} + +BEGIN; +DELETE FROM t; + +--connect backup,localhost,root +if ($have_debug) { +SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; +} +--replace_result $script stream.bat +send_eval BACKUP SERVER WITH 2 CONCURRENT '$script'; +--connection default +if ($have_debug) { +SET DEBUG_SYNC='now WAIT_FOR start'; +} +INSERT INTO t(a) SELECT * FROM seq_1_to_30000; +if ($have_debug) { +SET DEBUG_SYNC='now SIGNAL resume'; +} +--connection backup +--reap +--remove_file $MYSQL_TMP_DIR/stream.bat +--disconnect backup +--connection default +ROLLBACK; +SELECT * FROM t; +DELETE FROM t; +DROP TABLE at0, at1; + +--let $target_directory=$MYSQLTEST_VARDIR/backup_stream +# Clean up after a previous failed test, in case we are retrying. +--error 0,1 +--rmdir $target_directory +--mkdir $target_directory +--exec tar xf $MYSQL_TMP_DIR/1.tar -C $target_directory +--exec tar xf $MYSQL_TMP_DIR/2.tar -C $target_directory + +--exec cat $MYSQLTEST_VARDIR/my.cnf >> $target_directory/backup.cnf + +--let $restart_parameters=--defaults-file=$target_directory/backup.cnf --datadir=$target_directory +--source include/restart_mysqld.inc +SELECT * FROM at0; +SELECT * FROM at1; +SELECT * FROM t; +--error ER_OPEN_AS_READONLY +DELETE FROM t; + +--remove_file $MYSQL_TMP_DIR/1.tar +--remove_file $MYSQL_TMP_DIR/2.tar + +--let $restart_parameters= +--source include/restart_mysqld.inc +SELECT * FROM t; +DROP TABLE t; + +--rmdir $target_directory diff --git a/mysql-test/suite/backup/backup_sys_stats_not_flushed.result b/mysql-test/suite/backup/backup_sys_stats_not_flushed.result new file mode 100644 index 0000000000000..43d1297c9d4de --- /dev/null +++ b/mysql-test/suite/backup/backup_sys_stats_not_flushed.result @@ -0,0 +1,34 @@ +CREATE TABLE t1 (a INT, KEY(a)) ENGINE=MyISAM; +INSERT INTO t1 VALUES (1), (2), (3); +CREATE PROCEDURE fill_stats(IN rows_wanted INT) +BEGIN +DECLARE i INT DEFAULT 0; +WHILE i < rows_wanted DO +INSERT INTO mysql.table_stats (db_name, table_name, cardinality) +VALUES ('test', CONCAT('tbl', i), i); +SET i = i + 1; +END WHILE; +END| +CALL fill_stats(600); +connect backup,localhost,root,,; +SET DEBUG_SYNC='after_purge_tables SIGNAL purged WAIT_FOR go'; +BACKUP SERVER TO 'target_directory'; +connection default; +SET DEBUG_SYNC='now WAIT_FOR purged'; +connect writer,localhost,root,,; +DELETE FROM mysql.table_stats WHERE SLEEP(0.005) = 0; +connection default; +SET DEBUG_SYNC='now SIGNAL go'; +connection writer; +connection backup; +connection default; +disconnect backup; +disconnect writer; +SET DEBUG_SYNC='RESET'; +# restart: --datadir=MYSQLTEST_VARDIR/backup_sys_stats_not_flushed +CHECK TABLE mysql.table_stats; +Table Op Msg_type Msg_text +mysql.table_stats check status OK +# restart +DROP PROCEDURE fill_stats; +DROP TABLE t1; diff --git a/mysql-test/suite/backup/backup_sys_stats_not_flushed.test b/mysql-test/suite/backup/backup_sys_stats_not_flushed.test new file mode 100644 index 0000000000000..9e701ede81fda --- /dev/null +++ b/mysql-test/suite/backup/backup_sys_stats_not_flushed.test @@ -0,0 +1,81 @@ +--source include/have_aria.inc +--source include/have_debug_sync.inc + +# +# Test that non-transactional Aria system tables (stats tables) get copied +# correctly by BACKUP SERVER. The risk point is that as opposed to user tables +# writes to these tables are not blocked in the "no DDL" phase, so they need +# to be flushed and copied in the "no commit" phase. +# BACKUP SERVER succeeds, but a statistics table in the backup was copied while +# it was being written, even though the phase begins by flushing it. +# +# A scenario where mysql.table_stats is written in BACKUP_PHASE_NO_DDL: +# DELETE asks for MDL_SHARED_WRITE, which is below the MDL_SHARED_UPGRADABLE +# threshold in lock_table_names(), so it does not take that function's +# statement-wide MDL_BACKUP_DDL, and the per-table lock its category earns it, +# MDL_BACKUP_SYS_DML, does not conflict with the backup lock until +# MDL_BACKUP_WAIT_COMMIT. +# + +CREATE TABLE t1 (a INT, KEY(a)) ENGINE=MyISAM; +INSERT INTO t1 VALUES (1), (2), (3); + +# Enough rows that the DELETE keeps writing for the whole of the phase. The +# rows have to go in one at a time: a system table cannot be write-locked +# together with another table, so INSERT ... SELECT is rejected. +DELIMITER |; +CREATE PROCEDURE fill_stats(IN rows_wanted INT) +BEGIN + DECLARE i INT DEFAULT 0; + WHILE i < rows_wanted DO + INSERT INTO mysql.table_stats (db_name, table_name, cardinality) + VALUES ('test', CONCAT('tbl', i), i); + SET i = i + 1; + END WHILE; +END| +DELIMITER ;| +CALL fill_stats(600); + +--let $target_directory=$MYSQLTEST_VARDIR/backup_sys_stats_not_flushed +--error 0,1 +--rmdir $target_directory + +--connect (backup,localhost,root,,) +SET DEBUG_SYNC='after_purge_tables SIGNAL purged WAIT_FOR go'; +--replace_result $target_directory target_directory +--send_eval BACKUP SERVER TO '$target_directory' + +--connection default +SET DEBUG_SYNC='now WAIT_FOR purged'; + +--connect (writer,localhost,root,,) +--send DELETE FROM mysql.table_stats WHERE SLEEP(0.005) = 0 + +--connection default +--let $wait_condition= SELECT COUNT(*) FROM information_schema.processlist WHERE state = 'User sleep' AND info LIKE 'DELETE FROM mysql.table_stats%' +--source include/wait_condition.inc + +SET DEBUG_SYNC='now SIGNAL go'; + +--connection writer +--reap +--connection backup +--reap + +--connection default +--disconnect backup +--disconnect writer +SET DEBUG_SYNC='RESET'; + +# The statistics table in the backup has to be usable without repair. +--let $restart_parameters=--datadir=$target_directory +--source include/restart_mysqld.inc + +CHECK TABLE mysql.table_stats; + +--let $restart_parameters= +--source include/restart_mysqld.inc + +DROP PROCEDURE fill_stats; +DROP TABLE t1; +--rmdir $target_directory diff --git a/mysql-test/suite/backup/locks.result b/mysql-test/suite/backup/locks.result new file mode 100644 index 0000000000000..e3032b1ab4164 --- /dev/null +++ b/mysql-test/suite/backup/locks.result @@ -0,0 +1,26 @@ +CREATE TABLE t(a SERIAL)ENGINE=InnoDB; +BACKUP STAGE START; +BACKUP STAGE BLOCK_DDL; +BACKUP STAGE BLOCK_COMMIT; +connect backup,localhost,root; +FLUSH TABLES WITH READ LOCK; +FLUSH TABLES t FOR EXPORT; +SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/backup_locks'; +ERROR 70100: Query was interrupted: execution time limit 0.1 sec exceeded +connection default; +BACKUP STAGE END; +connection backup; +UNLOCK TABLES; +FLUSH TABLES t FOR EXPORT; +BACKUP SERVER TO '$MYSQLTEST_VARDIR/backup_locks'; +FLUSH TABLES t FOR EXPORT; +ERROR HY000: Can't execute the given command because you have active locked tables or an active transaction +FLUSH TABLES t FOR EXPORT; +ERROR HY000: Can't execute the given command because you have active locked tables or an active transaction +UNLOCK TABLES; +FLUSH TABLES t FOR EXPORT; +connection default; +BACKUP SERVER TO '$MYSQLTEST_VARDIR/backup_locks'; +disconnect backup; +DROP TABLE t; diff --git a/mysql-test/suite/backup/locks.test b/mysql-test/suite/backup/locks.test new file mode 100644 index 0000000000000..3f0ce35717d87 --- /dev/null +++ b/mysql-test/suite/backup/locks.test @@ -0,0 +1,37 @@ +--source include/have_innodb.inc + +CREATE TABLE t(a SERIAL)ENGINE=InnoDB; +BACKUP STAGE START; +BACKUP STAGE BLOCK_DDL; +BACKUP STAGE BLOCK_COMMIT; + +--connect (backup,localhost,root) +FLUSH TABLES WITH READ LOCK; +FLUSH TABLES t FOR EXPORT; + +--error 0,1 +--rmdir $MYSQLTEST_VARDIR/backup_locks + +--error ER_STATEMENT_TIMEOUT +evalp SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/backup_locks'; +--connection default +BACKUP STAGE END; +--connection backup +UNLOCK TABLES; +FLUSH TABLES t FOR EXPORT; +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/backup_locks'; +--rmdir $MYSQLTEST_VARDIR/backup_locks +--error ER_LOCK_OR_ACTIVE_TRANSACTION +FLUSH TABLES t FOR EXPORT; +--error ER_LOCK_OR_ACTIVE_TRANSACTION +FLUSH TABLES t FOR EXPORT; +UNLOCK TABLES; +FLUSH TABLES t FOR EXPORT; + +--connection default +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/backup_locks'; +--disconnect backup +--rmdir $MYSQLTEST_VARDIR/backup_locks + +DROP TABLE t; diff --git a/mysql-test/suite/backup/suite.pm b/mysql-test/suite/backup/suite.pm new file mode 100644 index 0000000000000..7d36a22655efa --- /dev/null +++ b/mysql-test/suite/backup/suite.pm @@ -0,0 +1,19 @@ +package My::Suite::Backup; + +@ISA = qw(My::Suite); +use My::Find; +use File::Basename; +use strict; + +return "Not run for embedded server" if $::opt_embedded_server; + +my $have_cat = index(`echo meowl|cat 2>&1`,"meowl") >= 0; +my $have_tar = `tar --version 2>&1` =~ /tar .*\d\.\d/; + +sub skip_combinations { + my %skip; + $skip{'backup_stream.test'} = 'needs cat,tar' unless $have_cat && $have_tar; + %skip; +} + +bless { }; diff --git a/mysql-test/suite/encryption/r/backup_encrypted,debug.rdiff b/mysql-test/suite/encryption/r/backup_encrypted,debug.rdiff new file mode 100644 index 0000000000000..c16f904e2ad20 --- /dev/null +++ b/mysql-test/suite/encryption/r/backup_encrypted,debug.rdiff @@ -0,0 +1,16 @@ +--- backup_innodb.result ++++ backup_innodb,debug.result +@@ -20,10 +20,13 @@ + BEGIN; + DELETE FROM t; + connect backup,localhost,root; ++SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; + BACKUP SERVER TO 'target_directory' 4 CONCURRENT; + connection default; ++SET DEBUG_SYNC='now WAIT_FOR start'; + INSERT INTO t(a) SELECT * FROM seq_1_to_30000; ++SET DEBUG_SYNC='now SIGNAL resume'; + connection backup; + disconnect backup; + connection default; + ROLLBACK; diff --git a/mysql-test/suite/encryption/r/backup_encrypted.result b/mysql-test/suite/encryption/r/backup_encrypted.result new file mode 100644 index 0000000000000..2fafd6cf6574f --- /dev/null +++ b/mysql-test/suite/encryption/r/backup_encrypted.result @@ -0,0 +1,65 @@ +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (2), (3), (5), (7); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (1), (1), (2), (3), (5); +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +BEGIN; +INSERT INTO t SET a=1; +BACKUP SERVER TO '$target_directory'; +ERROR HY000: The MariaDB server is running with the --secure-file-priv option so it cannot execute this statement +BACKUP SERVER TO '$target_directory'; +ROLLBACK; +SELECT * FROM t; +a b +1 +BACKUP SERVER TO '$target_directory' WITH '/bin/false'; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH '/bin/false'' at line 1 +BACKUP SERVER TO '$target_directory' WITH 4 CONCURRENT '/bin/false'; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH 4 CONCURRENT '/bin/false'' at line 1 +BACKUP SERVER WITH '/bin/false'; +ERROR HY000: IO Write error: (...) BACKUP SERVER +BEGIN; +DELETE FROM t; +connect backup,localhost,root; +BACKUP SERVER TO 'target_directory' 4 CONCURRENT; +connection default; +INSERT INTO t(a) SELECT * FROM seq_1_to_30000; +connection backup; +disconnect backup; +connection default; +ROLLBACK; +SELECT * FROM t; +a b +1 +DELETE FROM t; +DROP TABLE at0, at1; +# restart: --defaults-file=MYSQLTEST_VARDIR/some_directory/backup.cnf --datadir=MYSQLTEST_VARDIR/some_directory +SELECT * FROM t; +a b +1 +DELETE FROM t; +ERROR HY000: Table 't' is read only +SELECT * FROM at0; +i +1 +1 +2 +3 +5 +SELECT * FROM at1; +i +2 +3 +5 +7 +DROP TABLE t, at0, at1; +ERROR HY000: Table 't' is read only +SELECT * FROM at0; +ERROR 42S02: Table 'test.at0' doesn't exist +SELECT * FROM at1; +ERROR 42S02: Table 'test.at1' doesn't exist +# restart +SELECT * FROM t; +a b +DROP TABLE t; diff --git a/mysql-test/suite/encryption/r/backup_encrypted_debug.result b/mysql-test/suite/encryption/r/backup_encrypted_debug.result new file mode 100644 index 0000000000000..a5a792d13b301 --- /dev/null +++ b/mysql-test/suite/encryption/r/backup_encrypted_debug.result @@ -0,0 +1,68 @@ +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (2), (3), (5), (7); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (1), (1), (2), (3), (5); +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +BEGIN; +INSERT INTO t SET a=1; +BACKUP SERVER TO '$target_directory'; +ERROR HY000: The MariaDB server is running with the --secure-file-priv option so it cannot execute this statement +BACKUP SERVER TO '$target_directory'; +ROLLBACK; +SELECT * FROM t; +a b +1 +BACKUP SERVER TO '$target_directory' WITH '/bin/false'; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH '/bin/false'' at line 1 +BACKUP SERVER TO '$target_directory' WITH 4 CONCURRENT '/bin/false'; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH 4 CONCURRENT '/bin/false'' at line 1 +BACKUP SERVER WITH '/bin/false'; +ERROR HY000: IO Write error: (...) BACKUP SERVER +BEGIN; +DELETE FROM t; +connect backup,localhost,root; +SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; +BACKUP SERVER TO 'target_directory' 4 CONCURRENT; +connection default; +SET DEBUG_SYNC='now WAIT_FOR start'; +INSERT INTO t(a) SELECT * FROM seq_1_to_30000; +SET DEBUG_SYNC='now SIGNAL resume'; +connection backup; +disconnect backup; +connection default; +ROLLBACK; +SELECT * FROM t; +a b +1 +DELETE FROM t; +DROP TABLE at0, at1; +# restart: --defaults-file=MYSQLTEST_VARDIR/some_directory/backup.cnf --datadir=MYSQLTEST_VARDIR/some_directory +SELECT * FROM t; +a b +1 +DELETE FROM t; +ERROR HY000: Table 't' is read only +SELECT * FROM at0; +i +1 +1 +2 +3 +5 +SELECT * FROM at1; +i +2 +3 +5 +7 +DROP TABLE t, at0, at1; +ERROR HY000: Table 't' is read only +SELECT * FROM at0; +ERROR 42S02: Table 'test.at0' doesn't exist +SELECT * FROM at1; +ERROR 42S02: Table 'test.at1' doesn't exist +# restart +SELECT * FROM t; +a b +DROP TABLE t; diff --git a/mysql-test/suite/encryption/t/backup_encrypted.opt b/mysql-test/suite/encryption/t/backup_encrypted.opt new file mode 100644 index 0000000000000..1a9099a649ddb --- /dev/null +++ b/mysql-test/suite/encryption/t/backup_encrypted.opt @@ -0,0 +1,2 @@ +--innodb-encrypt-tables +--innodb-encrypt-log diff --git a/mysql-test/suite/encryption/t/backup_encrypted.test b/mysql-test/suite/encryption/t/backup_encrypted.test new file mode 100644 index 0000000000000..3ef5ecf675d82 --- /dev/null +++ b/mysql-test/suite/encryption/t/backup_encrypted.test @@ -0,0 +1,4 @@ +--source include/have_file_key_management_plugin.inc +--source include/have_innodb.inc +--source include/skip_innodb_log_archive.inc # innodb_encrypt_log +--source ../../backup/backup_innodb.test diff --git a/mysql-test/suite/encryption/t/backup_encrypted_debug.opt b/mysql-test/suite/encryption/t/backup_encrypted_debug.opt new file mode 100644 index 0000000000000..1f02d810e3ccf --- /dev/null +++ b/mysql-test/suite/encryption/t/backup_encrypted_debug.opt @@ -0,0 +1,6 @@ +--innodb-encrypt-tables +--innodb-encrypt-log +--innodb-encryption-rotate-key-age=2 +--innodb-encryption-threads=4 +--innodb-tablespaces-encryption +--plugin-load-add=$DEBUG_KEY_MANAGEMENT_SO diff --git a/mysql-test/suite/encryption/t/backup_encrypted_debug.test b/mysql-test/suite/encryption/t/backup_encrypted_debug.test new file mode 100644 index 0000000000000..93f9a660ba26b --- /dev/null +++ b/mysql-test/suite/encryption/t/backup_encrypted_debug.test @@ -0,0 +1,4 @@ +--source include/have_innodb.inc +--source include/skip_innodb_log_archive.inc # innodb_encrypt_log +--source include/have_debug.inc +--source ../../backup/backup_innodb.test diff --git a/mysql-test/suite/innodb/r/log_archive.result b/mysql-test/suite/innodb/r/log_archive.result index 6f12acda34b13..4ec69a83b6e3b 100644 --- a/mysql-test/suite/innodb/r/log_archive.result +++ b/mysql-test/suite/innodb/r/log_archive.result @@ -7,6 +7,7 @@ call mtr.add_suppression("InnoDB: Renaming ib_ffffffffff803000\\.log to ib_logfi call mtr.add_suppression("InnoDB: ignoring .*ib_ffffffffffc00000\\.log"); call mtr.add_suppression("InnoDB: Plugin initialization aborted"); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed"); +call mtr.add_suppression("mariadbd.*: Out of memory"); # restart: --innodb-read-only --innodb-log-recovery-start=12288 SELECT variable_name, variable_value FROM information_schema.global_status WHERE variable_name LIKE 'INNODB_LSN%'; @@ -52,6 +53,8 @@ TRUNCATE TABLE mysql.innodb_table_stats; ERROR HY000: Table 'innodb_table_stats' is read only SET GLOBAL innodb_log_archive=!@@GLOBAL.innodb_log_archive; ERROR HY000: InnoDB is in read only mode +BACKUP SERVER TO '$target_directory'; +ERROR HY000: InnoDB is in read only mode # restart: --innodb-log-recovery-start=0 --innodb-log-recovery-target=18446744073705357290 SELECT variable_name, variable_value FROM information_schema.global_status WHERE variable_name LIKE 'INNODB_LSN%'; diff --git a/mysql-test/suite/innodb/t/log_archive.test b/mysql-test/suite/innodb/t/log_archive.test index 925108880ba8a..a250e95aa9f98 100644 --- a/mysql-test/suite/innodb/t/log_archive.test +++ b/mysql-test/suite/innodb/t/log_archive.test @@ -10,6 +10,7 @@ call mtr.add_suppression("InnoDB: Renaming ib_ffffffffff803000\\.log to ib_logfi call mtr.add_suppression("InnoDB: ignoring .*ib_ffffffffffc00000\\.log"); call mtr.add_suppression("InnoDB: Plugin initialization aborted"); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed"); +call mtr.add_suppression("mariadbd.*: Out of memory"); let DATADIR= `select @@datadir`; --source include/shutdown_mysqld.inc @@ -150,6 +151,13 @@ ALTER TABLE mysql.innodb_table_stats ROW_FORMAT=COMPACT, ALGORITHM=INPLACE; TRUNCATE TABLE mysql.innodb_table_stats; --error ER_INNODB_READ_ONLY SET GLOBAL innodb_log_archive=!@@GLOBAL.innodb_log_archive; +--let $target_directory=$MYSQLTEST_VARDIR/log_archive_backup +# Clean up after a previous failed test, in case we are retrying. +--error 0,1 +--rmdir $target_directory +--error ER_INNODB_READ_ONLY +evalp BACKUP SERVER TO '$target_directory'; +--rmdir $target_directory --source include/shutdown_mysqld.inc --move_file $DATADIR/ib_logfile0 $DATADIR/ib_ffffffffff803000.log diff --git a/mysql-test/suite/perfschema/r/max_program_zero.result b/mysql-test/suite/perfschema/r/max_program_zero.result index 047643e06988d..a0e486b2af9c0 100644 --- a/mysql-test/suite/perfschema/r/max_program_zero.result +++ b/mysql-test/suite/perfschema/r/max_program_zero.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 1 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/ortho_iter.result b/mysql-test/suite/perfschema/r/ortho_iter.result index 56c22c8453d8e..589704f4056de 100644 --- a/mysql-test/suite/perfschema/r/ortho_iter.result +++ b/mysql-test/suite/perfschema/r/ortho_iter.result @@ -251,7 +251,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/privilege_table_io.result b/mysql-test/suite/perfschema/r/privilege_table_io.result index 0c82be9f05810..b518052613308 100644 --- a/mysql-test/suite/perfschema/r/privilege_table_io.result +++ b/mysql-test/suite/perfschema/r/privilege_table_io.result @@ -57,7 +57,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_idle.result b/mysql-test/suite/perfschema/r/start_server_disable_idle.result index d0665e3bf4c65..12b956d90769c 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_idle.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_idle.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_stages.result b/mysql-test/suite/perfschema/r/start_server_disable_stages.result index 2ef68328144ff..30ce9d12e56a0 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_stages.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_stages.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_statements.result b/mysql-test/suite/perfschema/r/start_server_disable_statements.result index 0ece2a0c52ed1..4bacdbdfedf68 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_statements.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_statements.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_transactions.result b/mysql-test/suite/perfschema/r/start_server_disable_transactions.result index ededc09aac95d..3a6831eb2c7ff 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_transactions.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_transactions.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_waits.result b/mysql-test/suite/perfschema/r/start_server_disable_waits.result index 23db9362161e4..a864576dfa979 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_waits.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_waits.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_innodb.result b/mysql-test/suite/perfschema/r/start_server_innodb.result index cff87457b3bef..09cba65c57ce0 100644 --- a/mysql-test/suite/perfschema/r/start_server_innodb.result +++ b/mysql-test/suite/perfschema/r/start_server_innodb.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_low_index.result b/mysql-test/suite/perfschema/r/start_server_low_index.result index 11cade0a2132f..dbd36d2eaa92d 100644 --- a/mysql-test/suite/perfschema/r/start_server_low_index.result +++ b/mysql-test/suite/perfschema/r/start_server_low_index.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_low_table_lock.result b/mysql-test/suite/perfschema/r/start_server_low_table_lock.result index 484550095202e..fab64f49d45e6 100644 --- a/mysql-test/suite/perfschema/r/start_server_low_table_lock.result +++ b/mysql-test/suite/perfschema/r/start_server_low_table_lock.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_account.result b/mysql-test/suite/perfschema/r/start_server_no_account.result index aab8d3eba9caa..e2cccdba19b43 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_account.result +++ b/mysql-test/suite/perfschema/r/start_server_no_account.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_cond_class.result b/mysql-test/suite/perfschema/r/start_server_no_cond_class.result index 4dfdab9de9f30..44b114013ace2 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_cond_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_cond_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result b/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result index a8d0cbeca3855..27ecb59a40f17 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_file_class.result b/mysql-test/suite/perfschema/r/start_server_no_file_class.result index fcc01880a7107..5189d36618b05 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_file_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_file_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_file_inst.result b/mysql-test/suite/perfschema/r/start_server_no_file_inst.result index c56201e7d0a80..533c02383c7b8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_file_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_file_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_host.result b/mysql-test/suite/perfschema/r/start_server_no_host.result index 662beb3b88a49..e328ea3d3c26a 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_host.result +++ b/mysql-test/suite/perfschema/r/start_server_no_host.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_index.result b/mysql-test/suite/perfschema/r/start_server_no_index.result index ccff0cb113faa..686f430cdc440 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_index.result +++ b/mysql-test/suite/perfschema/r/start_server_no_index.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mdl.result b/mysql-test/suite/perfschema/r/start_server_no_mdl.result index ebe64409deb0c..c2b6dc3d9eb74 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mdl.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mdl.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_memory_class.result b/mysql-test/suite/perfschema/r/start_server_no_memory_class.result index 01a217c534bfd..2a4cdcf8ddd37 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_memory_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_memory_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result b/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result index 1b3efda5210a9..9b77c7b897c47 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result b/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result index 599498915f3f1..3a62653efc18f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result b/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result index 73ac1acb9f145..7dd5842809135 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result +++ b/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result b/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result index 04aa037b960e3..3598c722b9453 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result b/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result index e8156711eb3f2..7a60805d2d03f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result b/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result index 2d17bd6f49203..76da2883ba6b2 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result +++ b/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result b/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result index 16afee28ee70b..58ac763394a96 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result +++ b/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_socket_class.result b/mysql-test/suite/perfschema/r/start_server_no_socket_class.result index 9de0006aa7abc..5a86d51a06231 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_socket_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_socket_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 0 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result b/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result index bcef0e5c01b05..75ab84f4b3cb8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 0 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stage_class.result b/mysql-test/suite/perfschema/r/start_server_no_stage_class.result index 1dda39dc79e92..bb750ebd34a26 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stage_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stage_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 0 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stages_history.result b/mysql-test/suite/perfschema/r/start_server_no_stages_history.result index 95584521eceb2..6cf3f740fb1be 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stages_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stages_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result index da6c54b6bbafa..1f2716410a9bc 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_statements_history.result b/mysql-test/suite/perfschema/r/start_server_no_statements_history.result index 09a9a544f5b13..f1078542f69ef 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_statements_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_statements_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result index 5ce9874799e8b..a6fc523df2c60 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result b/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result index a170fe097fd23..304732cb37927 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 0 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_inst.result b/mysql-test/suite/perfschema/r/start_server_no_table_inst.result index 3f009de021d1a..3ef43495d0f22 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 0 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_lock.result b/mysql-test/suite/perfschema/r/start_server_no_table_lock.result index db4fe3413106b..20cf1a531ca51 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_lock.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_lock.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_thread_class.result b/mysql-test/suite/perfschema/r/start_server_no_thread_class.result index b1b6e614c43ac..e8eb4589fce56 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_thread_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_thread_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result b/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result index e540f3ce78cbd..4dbf292fc3c10 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result b/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result index 80da238ba0ed7..09168c68f9fe8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result index f5d32f0100968..9ec63e991deda 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_user.result b/mysql-test/suite/perfschema/r/start_server_no_user.result index cb249b4e242d3..861b7d897c9c5 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_user.result +++ b/mysql-test/suite/perfschema/r/start_server_no_user.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_waits_history.result b/mysql-test/suite/perfschema/r/start_server_no_waits_history.result index c419aad2995f3..710cef232da58 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_waits_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_waits_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result index 0b2bf39ff874f..df4351a8f358f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_off.result b/mysql-test/suite/perfschema/r/start_server_off.result index 17db0395d894b..f3bf7bf7fb15e 100644 --- a/mysql-test/suite/perfschema/r/start_server_off.result +++ b/mysql-test/suite/perfschema/r/start_server_off.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_on.result b/mysql-test/suite/perfschema/r/start_server_on.result index cff87457b3bef..09cba65c57ce0 100644 --- a/mysql-test/suite/perfschema/r/start_server_on.result +++ b/mysql-test/suite/perfschema/r/start_server_on.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_variables.result b/mysql-test/suite/perfschema/r/start_server_variables.result index f25f9ab69d1c3..b86116e099cdd 100644 --- a/mysql-test/suite/perfschema/r/start_server_variables.result +++ b/mysql-test/suite/perfschema/r/start_server_variables.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/statement_program_lost_inst.result b/mysql-test/suite/perfschema/r/statement_program_lost_inst.result index 442e212557b55..3e8469b881801 100644 --- a/mysql-test/suite/perfschema/r/statement_program_lost_inst.result +++ b/mysql-test/suite/perfschema/r/statement_program_lost_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 2 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 97fd9a7c0ef61..f0834ea3a1e12 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -1064,7 +1064,7 @@ NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 18446744073709551615 NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL -READ_ONLY YES +READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME INNODB_LOG_RECOVERY_TARGET SESSION_VALUE NULL diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index e54e894e1d0fc..fd16d901d64fb 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -163,6 +163,7 @@ SET (SQL_SOURCE grant.cc sql_explain.cc sql_analyze_stmt.cc + sql_backup.cc sql_join_cache.cc create_options.cc multi_range_read.cc opt_histogram_json.cc diff --git a/sql/handler.h b/sql/handler.h index 3ab9e0bcd1a8e..2474b6c52f1ce 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -23,6 +23,7 @@ #include "sql_const.h" #include "sql_basic_types.h" +#include "sql_backup_interface.h" #include "mysqld.h" /* server_id */ #include "optimizer_costs.h" #include "sql_plugin.h" /* plugin_ref, st_plugin_int, plugin */ @@ -1892,9 +1893,48 @@ struct handlerton : public transaction_participant /********************************************************************* backup **********************************************************************/ + + /** BACKUP STAGE START */ void (*prepare_for_backup)(void); + /** BACKUP STAGE END */ void (*end_backup)(void); + /** + Start of a BACKUP SERVER phase, + when no backup_step() or backup_end() is pending. + @param thd current session + @param target backup target + @param phase BACKUP_PHASE_PREPARE_START, ... (not BACKUP_PHASE_ABORT) + @param sink worker context + @return backup context object to be attached to sink, or nullptr + @retval -1 on failure + */ + void *(*backup_start)(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink); + /** + Process a file that was collected in backup_start(). + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked + @param sink worker context + @return number of files remaining, or negative on error + @retval 0 on completion + */ + int (*backup_step)(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink); + /** + Finish a phase, once all calls for the current phase are completed. + @param thd current sesssion + @param target backup target + @param phase last phase on which backup_start() was successfully + invoked, or BACKUP_PHASE_ABORT or BACKUP_PHASE_FINISH + @param sink worker context + @return error code + @retval 0 on success + */ + int (*backup_end)(THD *thd, const backup_target *target, backup_phase phase, + const backup_sink *sink); + /********************************************************************** WSREP specific **********************************************************************/ diff --git a/sql/mysqld.cc b/sql/mysqld.cc index ead3f926f8cdf..696353cefaa26 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -3537,6 +3537,7 @@ SHOW_VAR com_status_vars[]= { {"assign_to_keycache", STMT_STATUS(SQLCOM_ASSIGN_TO_KEYCACHE)}, {"backup", STMT_STATUS(SQLCOM_BACKUP)}, {"backup_lock", STMT_STATUS(SQLCOM_BACKUP_LOCK)}, + {"backup_server", STMT_STATUS(SQLCOM_BACKUP_SERVER)}, {"begin", STMT_STATUS(SQLCOM_BEGIN)}, {"binlog", STMT_STATUS(SQLCOM_BINLOG_BASE64_EVENT)}, {"call_procedure", STMT_STATUS(SQLCOM_CALL)}, @@ -9451,7 +9452,7 @@ fn_format_relative_to_data_home(char * to, const char *name, @retval FALSE The path isn't secure */ -bool is_secure_file_path(char *path) +bool is_secure_file_path(const char *path) { char buff1[FN_REFLEN], buff2[FN_REFLEN]; size_t opt_secure_file_priv_len; diff --git a/sql/mysqld.h b/sql/mysqld.h index f04d2bacaa5bb..66745327af711 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -83,7 +83,7 @@ void unlink_thd(THD *thd); void refresh_status_legacy(THD *thd); void refresh_session_status(THD *thd); void refresh_global_status(); -bool is_secure_file_path(char *path); +bool is_secure_file_path(const char *path); extern void init_net_server_extension(THD *thd); extern void handle_accepted_socket(MYSQL_SOCKET new_sock, MYSQL_SOCKET sock); extern void create_new_thread(CONNECT *connect); diff --git a/sql/sql_backup.cc b/sql/sql_backup.cc new file mode 100644 index 0000000000000..4c4caf32b8493 --- /dev/null +++ b/sql/sql_backup.cc @@ -0,0 +1,1087 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "my_global.h" +#include "mdl.h" +#include "mysys_err.h" +#include "sql_class.h" +#include "sql_base.h" +#include "sql_backup.h" +#include "sql_backup_interface.h" +#include "sql_parse.h" +#include "my_atomic_wrapper.h" +#include "tpool.h" +#include "aligned.h" + +static bool backup_execute(THD *thd, const char *target, const char *command, + int threads); + +/** + @brief BACKUP SERVER command + + BACKUP SERVER copies the storage of a running server optionally + using multiple CONCURRENT threads, each one writing to the target + directory or its own tar stream. + + BACKUP SERVER TO '/path/to/directory' [ 1 CONCURRENT ]; + BACKUP SERVER WITH [ 1 CONCURRENT ] 'command'; + + The operation is divided into multiple phases; @see backup_phase + + Each phase is divided into "start" and "end", which are invoked + by this thread, and a number of "step" in between that may be + invoked by multiple CONCURRENT threads. + @see handlerton::backup_start + @see handlerton::backup_step + @see handlerton::backup_end + + Most phases are tied with BACKUP STAGE locks; the actual first work phase: + @see BACKUP_PHASE_START + @see MDL_BACKUP_START + + The snapshot for transactional storage engines is determined in: + @see BACKUP_PHASE_NO_COMMIT + @see MDL_BACKUP_WAIT_COMMIT + + Some backup_phase for preparation and clean-up are executed while + not holding any MDL. + + @param thd client connection + @return whether the operation failed +*/ +bool Sql_cmd_backup::execute(THD *thd) +{ + return backup_execute(thd, target, command, threads); +} + +#ifdef _WIN32 +using tpool::pread; +using tpool::pwrite; +#else +# include +/** + Copy a file using a memory mapping. + @tparam stream true=write to a stream, false=pwrite to a file + @param p map to source file + @param out_fd destination + @param o start offset + @param end last offset (exclusive) + @return error code + @retval 0 on success + @retval 1 if a memory mapping failed +*/ +template +static ssize_t mmap_copy(const void *p, int out_fd, uint64_t o, uint64_t end) +{ + size_t c= size_t(end - o); + ssize_t ret; + for (const char *b= static_cast(p) + o;; + b+= ret, o+= uint64_t(ret)) + { + const size_t size{std::min(c, size_t(INT_MAX >> 20 << 20))}; + if (stream) + if ((ret= backup_stream_write(out_fd, b, size))) + break; + ret= stream ? size : pwrite(out_fd, b, size, off_t(o)); + if (ret < 0) + break; + c-= ret; + if (!c) + { + ret= 0; + break; + } + if (!ret) + { + ret= -1; + break; + } + } + return ret; +} +#endif + +/** + Copy a file using positioned reads. + @tparam stream true=write to a stream, false=pwrite to a file + @param in_fd source file + @param out_fd destination + @param o start offset + @param end last offset (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +template +static ssize_t pread_write(backup::handle in_fd, backup_fd out_fd, + uint64_t o, uint64_t end) noexcept +{ + assert(end >= o); + uint64_t count{end - o}; +#if SIZEOF_SIZE_T < 8 + size_t size{65536}; +#else + constexpr size_t max_size{1 << 20}; + /* An integer multiple of 4096 bytes, up to max_size */ + size_t size{count >= max_size ? max_size : (size_t(count) + 4095) & ~4095}; +#endif + char *b= static_cast(aligned_malloc(size, 4096)); + if (!b) + return -1; + ssize_t ret; + for (;; o+= ret) + { + if (count < size) + size= count; + ret= pread(in_fd, b, ssize_t(size), o); + if (ret > 0) + { + if (!stream) + ret= pwrite(out_fd, b, size_t(ret), o); + else if (backup_stream_write(out_fd, b, size_t(ret))) + { + ret= -1; + break; + } + } + if (ret < 0) + break; + count-= uint64_t(ret); + if (!count) + { + ret= 0; + break; + } + if (!ret) + { + ret= -1; + break; + } + } + aligned_free(b); + return ret; +} + +#ifdef __APPLE__ +/* The inline copy_entire_file() invokes fcopyfile() */ +#elif defined _WIN32 +/* CopyFileEx() should be used */ +#else +/** Copy a file (whole content). +@param src source file descriptor +@param dst target to append src to +@return error code (non-positive) +@retval 0 on success */ +extern "C" int copy_entire_file(int src, int dst) +{ + uint64_t end(lseek(src, 0, SEEK_END)); +#ifdef POSIX_FADV_SEQUENTIAL + std::ignore= posix_fadvise(src, 0, 0, POSIX_FADV_SEQUENTIAL); +#endif + int ret; +# ifdef copy_file_shortcut + ret= int(copy_file_shortcut(src, dst, 0, end)); + if (ret == 1) +# endif + { +# ifdef copy_file_mmap + void *p= mmap(nullptr, size_t{end}, PROT_READ, MAP_SHARED, src, 0); + if (p != MAP_FAILED) + { + ret= int(copy_file_mmap(p, dst, 0, end)); + munmap(p, size_t{end}); + } + else +# endif + { + ret= backup::copy(src, dst, 0, end); + } + } +#ifdef POSIX_FADV_DONTNEED + std::ignore= posix_fadvise(src, 0, 0, POSIX_FADV_DONTNEED); +#endif + return ret; +} +#endif + +#if defined __linux__ || defined __FreeBSD__ +using copying_step= ssize_t(int,int,size_t,off_t*); +template +static ssize_t stepwise(int in_fd, int out_fd, off_t offset, off_t end) + noexcept +{ + for (;;) + { + const size_t c{size_t(std::min(end - offset, INT_MAX >> 20 << 20))}; + ssize_t ret= step(in_fd, out_fd, c, &offset); + if (ret < 0) + { + if (nonblocking && errno == EAGAIN) + continue; + return ret; + } + if (offset == end) + return 0; + if (!ret) + return -1; + } +} + +/* Copy between files in a single (type of) file system */ +static inline ssize_t +copy_step(int in_fd, int out_fd, size_t count, off_t *offset) noexcept +{ + return copy_file_range(in_fd, offset, out_fd, offset, count, 0); +} +# define cfr(src,dst,start,end) stepwise(src, dst, start, end) +#endif + +namespace backup { +/** Copy a portion of a file. +@param src source file descriptor +@param dst target to append src to +@param start first offset to copy +@param end last offset to copy (exclusive) +@return error code (non-positive) +@retval 0 on success */ +int copy(handle src, backup_fd dst, uint64_t start, uint64_t end) noexcept +{ + assert(end >= start); +#ifdef __FreeBSD__ + /* On FreeBSD, copy_file_range() without flags just works */ + return int(cfr(src, dst, off_t(start), off_t(end))); +#else + ssize_t ret{pread_write(src, dst, start, end)}; + assert(ret <= 0); + return int(ret); +#endif +} + +/** + Append a file snippet to stream, + after a corresponding call to backup_stream_start(). + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_zeropad() must be invoked. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +int append(handle src, backup_fd stream, uint64_t start, uint64_t end) noexcept +{ + assert(stream != backup_sink::NO_STREAM); + assert(end >= start); +#if 0 /* On Linux, this turned out to be slower in our tests! */ + /* + It is unsafe to zero-copy send from src unless we can guarantee + that the receiving end of the pipe has consumed everything before + our caller re-enables writes to this src region. + + On Linux, even for MAP_PRIVATE, the following has been documented: + It is unspecified whether changes made to the file after the + mmap() call are visible in the mapped region. + */ +# ifdef __linux__ + const int pipe_size{fcntl(stream, F_GETPIPE_SZ)}; +# elif defined __FreeBSD__ || defined __APPLE__ + // https://unix.stackexchange.com/questions/11946/how-big-is-the-pipe-buffer + constexpr int pipe_size{65536}; +# else + constexpr int pipe_size{0}; +# endif + if (pipe_size > 0) + { + /* + It should be safe to invoke the zero-copy path if we can + guarantee that the recipient will drain the buffer before the + final nonzero-copy path completes. We assume that this is + guaranteed by writing at least pipe_size bytes via the + nonzero-copy path after this shortcut. + */ + const uint64_t fast_end{start + uint64_t(pipe_size)}; + if (fast_end < end) + { + if (int ret= backup_stream_append_async(src, stream, start, fast_end)) + return ret; + start= fast_end; + } + } +#endif + return int(pread_write(src, stream, start, end)); +} +} + +extern "C" int backup_stream_append_plain(backup_fd src, backup_fd stream, + uint64_t start, uint64_t end) +{ + /* + On Windows, this invokes native_file_handle::native_file_handle(HANDLE). + + Elsewhere, this should be a tail-call, for example JMP rel32 + (0xe9) on IA-32 or AMD64. This non-inline wrapper exists only + because the basic API target is C, not C++, which is required + because of Windows. + */ + return backup::append(src, stream, start, end); +} + +/** + Zero-pad a the stream to a multiple of 512 bytes. + + @param stream backup stream + @param written least significant bits of the number of payload appended + @return error code (non-positive) + @retval 0 on success +*/ +extern "C" int backup_stream_zeropad(backup_fd stream, size_t written) +{ + static constexpr const char zerobuf[511]{'\0'}; + written&= 511; + return written ? backup_stream_write(stream, zerobuf, 512 - written) : 0; +} + +#ifdef __linux__ +/** + Try to copy a portion of a file via copy_file_range(2). + @param src source file descriptor + @param dst target to append src to + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success + @retval 1 if a fallback to copy_mmap() or backup::copy() is needed +*/ +extern "C" +int copy_file_range_try(int src, int dst, uint64_t start, uint64_t end) +{ + assert(end >= start); + ssize_t ret{cfr(src, dst, off_t(start), off_t(end))}; + assert(ret <= 0); + if (ret && (errno == EOPNOTSUPP || errno == EXDEV)) + return 1; + return int(ret); +} +#endif + +#ifdef copy_file_mmap +/** + Copy from a memory mapping to a file. + @param map source file mapping + @param dst target to append map to + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +extern "C" int +copy_mmap(const void *map, int dst, uint64_t start, uint64_t end) +{ + return int(mmap_copy(map, dst, start, end)); +} +#endif + +/** Append to the configuration file. +@param target backup target directory +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +extern "C" int backup_config_append(IF_WIN(const char*, int) target, + const char *config, size_t size) +{ + /* FIXME: append to a pre-created configuration file */ +#ifdef _WIN32 + HANDLE dst; + { + std::string path{target}; + path.append("/backup.cnf"); + dst= CreateFile(path.c_str(), GENERIC_WRITE, 0, + my_win_file_secattr(), CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (dst != INVALID_HANDLE_VALUE) + { + BOOL ok; + for (;;) + { + DWORD written; + ok= WriteFile(dst, config, DWORD(size), &written, nullptr); + if (ok || GetLastError() != ERROR_IO_PENDING) + break; + assert(written < DWORD(size)); + config+= written; + size-= size_t(written); + } + if (CloseHandle(dst) & ok) + return 0; + } + } +#else + int dst= openat(target, "backup.cnf", + O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (dst < 0) + return dst; + ssize_t ret; + for (; (ret= write(dst, config, size)) >= 0; config+= ret, size -= ret) + { + assert(size_t(ret) <= size); + if (!(size-= size_t(ret))) + { + ret= 0; + break; + } + } + if (!(close(dst) | ret)) + return 0; +#endif + my_error(ER_CANT_CREATE_FILE, MYF(0), "backup.cnf", errno); + return -1; +} + +/** backup context */ +struct backup_target_phase +{ + /** target directory or stream */ + backup_target target; + /** current phase of backup */ + backup_phase phase; + /** backup worker state (output stream) */ + backup_sink sink; + /** stream object for streaming backup */ + FILE *stream; + /** handlerton::backup_step return value in multi-threaded operation */ + int ret; + /** engine-specific backup context */ + std::unordered_map &ha_data; +}; + +/** + Inform a storage engine of an upcoming backup by invoking + handlerton::backup_start(BACKUP_PHASE_PREPARE_START) before + acquiring any locks. + @param thd current session + @param plugin storage engine + @return whether the operation failed +*/ +static my_bool backup_preparation(THD *thd, plugin_ref plugin, void*) noexcept +{ + const auto bs= plugin_hton(plugin)->backup_start; + return bs && bs(thd, nullptr, BACKUP_PHASE_PREPARE_START, nullptr); +} + +/** + Invoke handlerton::backup_start() on a storage engine, + when there are no pending handlerton::backup_step() in any thread. + @param thd current session + @param plugin storage engine + @param arg the backup_target_phase context + @return whether the operation failed +*/ +static my_bool backup_start(THD *thd, plugin_ref plugin, void *arg) noexcept +{ + const handlerton *hton= plugin_hton(plugin); + backup_target_phase &t{*static_cast(arg)}; + assert(int{t.phase} >= 0 || t.phase == BACKUP_PHASE_FINISH); + if (hton->backup_start) + { + t.sink.ha_data= t.ha_data[hton]; + void *data= hton->backup_start(thd, &t.target, t.phase, &t.sink); + if (data == reinterpret_cast(-1)) + return true; + assert(!t.ha_data[hton] || t.ha_data[hton] == data); + t.ha_data[hton]= data; + } + return false; +} + +/** + Invoke handlerton::backup_end() on a storage engine, + when there are no pending handlerton::backup_step() in any thread. + @param thd current session + @param plugin storage engine + @param arg the backup_target_phase context + @return whether the operation failed +*/ +static my_bool backup_end(THD *thd, plugin_ref plugin, void *arg) noexcept +{ + const handlerton *hton= plugin_hton(plugin); + backup_target_phase &t{*static_cast(arg)}; + if (hton->backup_end) + { + t.sink.ha_data= t.ha_data[hton]; + return hton->backup_end(thd, &t.target, t.phase, &t.sink); + } + return false; +} + +/** + Invoke handlerton::backup_step() on a storage engine in a thread + that may or may not be associated with a BACKUP SERVER connection, + between handlerton::backup_start() and handlerton::backup_end() + of the same backup_phase. + @param thd the BACKUP SERVER session + @param plugin storage engine + @param arg the backup_target_phase context + @return whether the operation failed +*/ +static my_bool backup_step(THD *thd, plugin_ref plugin, void *arg) noexcept +{ + const handlerton *hton= plugin_hton(plugin); + backup_target_phase &t{*static_cast(arg)}; + assert(int{t.phase} >= 0 || t.phase == BACKUP_PHASE_FINISH); + int res= 0; + if (hton->backup_step) + { + t.sink.ha_data= t.ha_data[hton]; + while ((res= hton->backup_step(thd, &t.target, t.phase, &t.sink))) + if (res < 0) + break; + } + return res != 0; +} + +/** Number of background tasks executing backup_step_callback */ +static Atomic_counter backup_step_callback_pending{0}; + +/** Invoke backup_step() in a background task */ +static void backup_step_callback(void *arg) noexcept +{ + backup_target_phase &t{*static_cast(arg)}; + assert(!t.ret); + t.ret= plugin_foreach_with_mask(nullptr, backup_step, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, &t); +#ifndef NDEBUG + auto was_pending= +#endif + backup_step_callback_pending--; + assert(was_pending); +} + +/** + Execute all handlerton::backup_step() until completion or failure. + @param thd current connection + @param target_phase backup target and phase + @param threads number of execution threads + @param tp thread pool +*/ +static bool backup_steps(THD *thd, backup_target_phase *target_phase, + int threads, tpool::thread_pool *tp) +{ + assert(!backup_step_callback_pending); + if (threads == 1) + return plugin_foreach_with_mask(thd, backup_step, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase); + tpool::task *const tasks= + static_cast(alloca(threads * sizeof *tasks)); + backup_step_callback_pending= threads - 1; + for (int n{threads}; --n; ) + { + target_phase[n].phase= target_phase->phase; + tp->submit_task(new (&tasks[n]) tpool::task{backup_step_callback, + &target_phase[n]}); + } + bool fail= plugin_foreach_with_mask(thd, backup_step, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase); + while (backup_step_callback_pending) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + if (fail) + return fail; + + for (int n{threads}; --n; ) + if (target_phase[n].ret) + { + my_error(ER_UNKNOWN_ERROR, MYF(0)); + return true; + } + + return false; +} + +/** + BACKUP SERVER driver + + @param thd client connection + @param target BACKUP SERVER TO directory to be created + @param command BACKUP SERVER WITH command + @param threads number of CONCURRENT threads to employ + @return whether the operation failed +*/ +static bool backup_execute(THD *thd, const char *target, const char *command, + int threads) +{ + assert(!!target == !command); + assert(threads > 0); + + if (check_global_access(thd, RELOAD_ACL) || + check_global_access(thd, SELECT_ACL)) + return true; + + if (!target); + else if (error_if_data_home_dir(target, "BACKUP SERVER TO")) + return true; + else if (!is_secure_file_path(target)) + { + my_error(ER_OPTION_PREVENTS_STATEMENT, MYF(0), "--secure-file-priv"); + return true; + } + + if (thd->current_backup_stage != BACKUP_FINISHED) + { + my_error(ER_BACKUP_LOCK_IS_ACTIVE, MYF(0)); + return true; + } + + bool fail{plugin_foreach_with_mask(thd, backup_preparation, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + nullptr)}; + if (fail) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return true; + } + + /* Block concurrent BACKUP SERVER and BACKUP STAGE */ + MDL_request mdl_request; + MDL_REQUEST_INIT(&mdl_request, MDL_key::BACKUP, "", "", MDL_BACKUP_START, + MDL_EXPLICIT); + + if (thd->mdl_context.acquire_lock(&mdl_request, + thd->variables.lock_wait_timeout)) + return true; + + tpool::thread_pool *tp= nullptr; + std::unordered_map ha_data{}; + backup_target_phase *target_phase= static_cast + (alloca(threads * sizeof *target_phase)); + if (threads > 1 && !(tp= tpool::create_thread_pool_generic())) + { + oor: + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + err_exit: + thd->mdl_context.release_lock(mdl_request.ticket); + delete tp; + return true; + } + + if (command) + { + char cmd[1024]; + for (int t{threads}; t; ) + { + if (snprintf(cmd, sizeof cmd, "%s %d", command, t) >= int(sizeof cmd)) + goto oor; + FILE *f= my_popen(cmd, "w"); + if (!f) + { + while (t < threads) + my_pclose(target_phase[t++].stream); + goto oor; + } +#ifdef _WIN32 + HANDLE sink= (HANDLE) _get_osfhandle(_fileno(f)); +#else + int sink= fileno(f); +# ifdef __linux__ + /* Request a larger pipe buffer; the default appears to be 64 KiB. */ + fcntl(sink, F_SETPIPE_SZ, 1 << 20/* 1 MiB */); +# endif +#endif + new (&target_phase[--t]) + backup_target_phase{backup_target{IF_WIN(nullptr, -1)}, + BACKUP_PHASE_START, backup_sink{sink, nullptr}, f, 0, ha_data}; + } + } + else if (my_mkdir(target, 0755, MYF(MY_WME))) + goto err_exit; + else + { +#ifndef _WIN32 + const int dir{open(target, O_DIRECTORY)}; + if (dir < 0) + { + my_error(EE_CANT_MKDIR, MYF(ME_BELL), target, errno); + goto err_exit; + } +#endif + for (int t{threads}; t; ) + { + new (&target_phase[--t]) + backup_target_phase{backup_target{IF_WIN(target, dir)}, + BACKUP_PHASE_START, + backup_sink{backup_sink::NO_STREAM, nullptr}, nullptr, 0, ha_data}; + } + } + + static_assert(int{MDL_BACKUP_START} + 1 == int{MDL_BACKUP_FLUSH}, ""); + static_assert(int{MDL_BACKUP_START} + 2 == int{MDL_BACKUP_WAIT_FLUSH}, ""); + static_assert(int{MDL_BACKUP_START} + 3 == int{MDL_BACKUP_WAIT_DDL}, ""); + static_assert(int{MDL_BACKUP_START} + 4 == int{MDL_BACKUP_WAIT_COMMIT}, ""); + static_assert(int{BACKUP_PHASE_START} + 1 == + int{BACKUP_PHASE_NO_BEGIN_NON_TRANS}, ""); + static_assert(int{BACKUP_PHASE_START} + 2 == + int{BACKUP_PHASE_NO_DML_NON_TRANS}, ""); + static_assert(int{BACKUP_PHASE_START} + 3 == int{BACKUP_PHASE_NO_DDL}, ""); + static_assert(int{BACKUP_PHASE_START} + 4 == + int{BACKUP_PHASE_NO_COMMIT}, ""); + int phase= int{BACKUP_PHASE_START}; + goto backup_phase_start; + + for (; phase <= int{BACKUP_PHASE_NO_COMMIT}; phase++) + { + assert(!fail); + { + const enum_mdl_type mdl= + enum_mdl_type(phase - int{BACKUP_PHASE_START} + int{MDL_BACKUP_START}); + fail= + thd->mdl_context.upgrade_shared_lock(mdl_request.ticket, mdl, + thd->variables.lock_wait_timeout); + if (fail) + break; + } + + if ((phase == BACKUP_PHASE_NO_DDL || phase == BACKUP_PHASE_NO_COMMIT) && + /* Invoke handler::extra(HA_EXTRA_FLUSH) */ + (fail= flush_tables(thd, phase == BACKUP_PHASE_NO_DDL + ? FLUSH_NON_TRANS_TABLES + : FLUSH_SYS_TABLES))) + break; + + backup_phase_start: + target_phase->phase= backup_phase(phase); + fail= plugin_foreach_with_mask(thd, backup_start, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase); + if (fail) + break; + fail= backup_steps(thd, target_phase, threads, tp) || + plugin_foreach_with_mask(thd, backup_end, MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase); + if (fail) + break; + } + + /* The final part must not interfere with the use of the server datadir. + Release the locks. */ + thd->mdl_context.release_lock(mdl_request.ticket); + if (!fail) + { + target_phase->phase= BACKUP_PHASE_FINISH; + fail= plugin_foreach_with_mask(thd, backup_start, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase) || + backup_steps(thd, target_phase, threads, tp); + } + else + { + target_phase->phase= BACKUP_PHASE_ABORT; + plugin_foreach_with_mask(thd, backup_end, MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, target_phase); + target_phase->phase= BACKUP_PHASE_FINISH; + } + + fail= + plugin_foreach_with_mask(thd, backup_end, MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase) || fail; + delete tp; + + if (command) + { + int pfail{0}; + for (int t= threads; t--; ) + pfail|= my_pclose(target_phase[t].stream); + if (!fail && (fail= !!pfail)) + my_error(ER_NET_ERROR_ON_WRITE, MYF(0)); + } +#ifndef _WIN32 + else + std::ignore= close(target_phase->target.fd); +#endif + + if (!fail) + my_ok(thd); + return fail; +} + +/** + Encode an octal string. + @param start first byte of buffer + @param end first byte after buffer + @param n number to encode +*/ +static void ustar_write_octal(char *start, char *end, uint64_t n) noexcept +{ + for (*--end= '\0'; --end >= start; n>>= 3) + *end= char('0' + (n & 7)); +} + +/** + Encode a quantity in 12 bytes. + @param start first byte of the buffer + @param n number to encode +*/ +static void ustar_write_dozen(char *start, uint64_t n) noexcept +{ + if (n < 1ULL << 33) + ustar_write_octal(start, start + 12, n); + else + { + const uint32_t head{my_htobe32(1U << 31)}; + n= my_htobe64(n); + memcpy(start, &head, 4); + memcpy(start + 4, &n, 8); + } +} + +/** Initialize a ustar block +@param buf GNU tape archiver --format=oldgnu header block +@param name name of the block +@param mode file access mode +@param size physical size of the following block */ +static void ustar_block_init(char *buf, const char *name, mode_t mode, + uint64_t size) noexcept +{ + strncpy(buf, name, 100); + ustar_write_octal(buf + 100, buf + 108, uint64_t(mode)); + ustar_write_octal(buf + 108, buf + 116, 0/* POSIX uid */); + ustar_write_octal(buf + 116, buf + 124, 0/* POSIX gid */); + ustar_write_dozen(buf + 124, size); + /* last modification time */ + ustar_write_octal(buf + 136, buf + 148, 0); + memset(buf + 148, ' ', 9); /* initial block checksum and dummy type */ + memset(buf + 157, '\0', 100); /* name of linked file (unused) */ + memcpy(buf + 257, "ustar ", 8); + strncpy(buf + 265, "root" /* POSIX owner name */, 32); + strncpy(buf + 297, "root" /* POSIX group name */, 512 - 297); + /* The caller will fill in the rest and invoke ustar_block_checksum() */ +} + +/** + Compute and write the POSIX tar block checksum. + @param buf POSIX tar block +*/ +static void ustar_block_checksum(char *buf) noexcept +{ + uint16_t sum{0}; + for (int i{0}; i < 512; i++) + sum+= uint16_t{uint8_t(buf[i])}; + ustar_write_octal(buf + 148, buf + 155, sum); +} + +/** + Write data to a stream. + @param stream backup stream + @param buf source buffer + @param size length of the buffer (usually an integer multiple of 512) + @return error code (non-positive) + @retval 0 on success +*/ +extern "C" int backup_stream_write(backup_fd stream, const void *buf, + size_t size) +{ +#ifdef _WIN32 + for (DWORD sz= DWORD(size);;) + { + DWORD wrote; + if (WriteFile(stream, buf, sz, &wrote, nullptr)) + { + assert(wrote == sz); + return 0; + } + else if (GetLastError() != ERROR_IO_PENDING) + return -1; + buf= static_cast(buf) + wrote; + sz-= wrote; + } +#else + do + { + ssize_t wrote= write(stream, buf, size); + assert(wrote <= ssize_t(size)); + if (wrote < 0) + { + if (errno == EAGAIN) + continue; + return -1; + } + buf= static_cast(buf) + wrote; + size-= size_t(wrote); + } + while (size); +#endif + return 0; +} + +/** + Copy a prefix of a NUL terminated string to a buffer, NUL-padded. + @param b output buffer + @param s NUL terminated string + @param size size of buf, in bytes +*/ +static inline char *ustar_zeropad(char *b, const char *s, size_t size) noexcept +{ +#if defined __GNUC__ && __GNUC__ >= 8 +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wstringop-truncation" +#endif + return strncpy(b, s, size); +#if defined __GNUC__ && __GNUC__ >= 8 +# pragma GCC diagnostic pop +#endif +} + +/** Start streaming a file. +@param stream backup stream +@param name file name +@param mode file access mode +@param size physical length of the file, in bytes +@param chunks payload chunks of a sparse file, or nullptr +@param n_chunks number of chunks; 0 unless sparse file +@return error code (non-positive) +@retval 0 on success */ +extern "C" +int backup_stream_start(backup_fd stream, + const char *name, mode_t mode, uint64_t size, + const struct backup_chunk *chunks, size_t n_chunks) +{ + assert(stream != backup_sink::NO_STREAM); + char buf[512]; + size_t s= strlen(name); + if (s > 100) + { + /* Write a block that contains the full name length, + followed by blocks that contain the full name, in + tar --format=oldgnu */ + ustar_block_init(buf, "././@LongLink", 0644, s); + ustar_block_checksum(buf); + if (int err= backup_stream_write(stream, buf, sizeof buf)) + return err; + const size_t whole{s & ~(sizeof buf)}; + if (whole) + if (int err= backup_stream_write(stream, name, whole)) + return err; + if (s - whole) + { + ustar_zeropad(buf, name + whole, sizeof buf); + if (int err= backup_stream_write(stream, buf, sizeof buf)) + return err; + } + } + + ustar_block_init(buf, name, mode, size); + if (!n_chunks) + buf[156]= '0'; + else + { + buf[156]= 'S'; + char *h= &buf[386]; + if (n_chunks > 4) + return -1; // FIXME; support more chunks + for (size_t i= 0; i < n_chunks; i++, h+= 24) + { + ustar_write_dozen(h, chunks[i].offset); + ustar_write_dozen(h + 12, chunks[i].length); + } + ustar_write_dozen(&buf[0x1e3], chunks[n_chunks - 1].offset + + chunks[n_chunks - 1].length); + } + ustar_block_checksum(buf); + return backup_stream_write(stream, buf, sizeof buf); +} + +/** Append to the configuration file. +@param target backup stream +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +extern "C" int backup_stream_config(backup_fd stream, + const char *config, size_t size) +{ + /* FIXME: append to a pre-created configuration file */ + if (int ret= + backup_stream_start(stream, "backup.cnf", 0644, size, nullptr, 0)) + return ret; + char buf[512]; + const size_t whole{size & ~((sizeof buf) - 1)}; + if (whole) + if (int err= backup_stream_write(stream, config, whole)) + return err; + if (size == whole) + return 0; + ustar_zeropad(buf, config + whole, sizeof buf); + return backup_stream_write(stream, buf, sizeof buf); +} + +#ifndef _WIN32 +# ifdef __linux__ +# include +/** Copy a file to a stream or to a regular file. */ +static inline ssize_t +send_step(int in_fd, int out_fd, size_t count, off_t *offset) noexcept +{ + return sendfile(out_fd, in_fd, offset, count); +} +# endif + +/** + Zero-copy append an immutable file snippet to a stream. + + The caller guarantees that this section of the file will remain + intact until the stream is closed. This guarantee is needed + because the receiving end of the stream pipe might delay consuming + the data, and the operating system might point the pipe buffer + to the block cache for a long time. + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_zeropad() must be invoked. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +extern "C" int backup_stream_append_async(int src, int stream, + uint64_t start, uint64_t end) +{ + assert(stream != backup_sink::NO_STREAM); +# ifdef __linux__ + return int(stepwise(src, stream, off_t(start), off_t(end))); +# else +# if SIZEOF_SIZE_T > 4 + void *p= mmap(nullptr, size_t{end}, PROT_READ, MAP_SHARED, src, 0); + if (p != MAP_FAILED) + { + ssize_t ret= mmap_copy(p, stream, start, end); + munmap(p, size_t{end}); + return int(ret); + } +# endif + return int(pread_write(src, stream, start, end)); +# endif +} +#endif diff --git a/sql/sql_backup.h b/sql/sql_backup.h new file mode 100644 index 0000000000000..2759e0d64b43d --- /dev/null +++ b/sql/sql_backup.h @@ -0,0 +1,53 @@ +/***************************************************************************** +Copyright (c) 2026 MariaDB plc. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +#pragma once + +/** BACKUP SERVER */ +class Sql_cmd_backup : public Sql_cmd +{ + /** target directory, or nullptr when streaming */ + const char *const target{nullptr}; + /** argument of my_popen() for streaming backup, or nullptr */ + const char *const command{nullptr}; + /** number of concurrent threads to use */ + const int threads; + +public: + /** + Constructor. + @param target name of target or scratch directory + @param threads number of concurrent threads to use + */ + Sql_cmd_backup(LEX_CSTRING target, int threads) : + target(target.str), threads(threads) {} + /** + Constructor. + @param threads number of concurrent threads to use + @param command nullptr, or a shell command for handling a backup stream + */ + Sql_cmd_backup(int threads, LEX_CSTRING command) : + command(command.str), threads(threads) {} + ~Sql_cmd_backup()= default; + + bool execute(THD *thd) override; + + enum_sql_command sql_command_code() const override + { + return SQLCOM_BACKUP_SERVER; + } +}; diff --git a/sql/sql_backup_interface.h b/sql/sql_backup_interface.h new file mode 100644 index 0000000000000..229b944ac9092 --- /dev/null +++ b/sql/sql_backup_interface.h @@ -0,0 +1,305 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#pragma once +#include + +/** BACKUP SERVER target */ +struct backup_target +{ +#ifdef _WIN32 + /** Target directory path name, or nullptr if streaming */ + const char *path; +#else + /** Target directory descriptor, or -1 if streaming */ + int fd; +#endif +}; + +/** BACKUP SERVER worker specific context */ +struct backup_sink +{ +#ifdef _WIN32 +# ifdef __cplusplus + /** A value indicating an invalid stream */ + static constexpr HANDLE NO_STREAM{INVALID_HANDLE_VALUE}; +# endif + /** Target pipe, or NO_STREAM if path!=nullptr */ + HANDLE stream; +#else +# ifdef __cplusplus + /** A value indicating an invalid file descriptor or stream */ + static constexpr int NO_STREAM{-1}; +# endif + /** Target pipe, or NO_STREAM if copying to a directory */ + int stream; +#endif + /** storage engine context returned by handlerton::backup_start() */ + void *ha_data; +}; + +/** BACKUP SERVER execution phase; @see Sql_cmd_backup::execute() */ +enum backup_phase +{ + /** finish backup, possibly after BACKUP_PHASE_ABORT */ + BACKUP_PHASE_FINISH= -2, + /** abort any operation */ + BACKUP_PHASE_ABORT= -1, + /** preparatory phase executed while holding no locks */ + BACKUP_PHASE_PREPARE_START= 0, + /** initial actual work phase; @see MDL_BACKUP_START */ + BACKUP_PHASE_START, + /** copy while new writes to non-transactional tables are blocked; + @see MDL_BACKUP_FLUSH */ + BACKUP_PHASE_NO_BEGIN_NON_TRANS, + /** copy while any writes to non-transactional tables are blocked; + @see MDL_BACKUP_WAIT_FLUSH */ + BACKUP_PHASE_NO_DML_NON_TRANS, + /** copy files while DDL is blocked; @see MDL_BACKUP_WAIT_DDL */ + BACKUP_PHASE_NO_DDL, + /** determine the logical time of the backup and copy any + remaining files while MDL_BACKUP_WAIT_COMMIT is active; + this is followed by BACKUP_PHASE_FINISH */ + BACKUP_PHASE_NO_COMMIT +}; + +/** A payload chunk in a sparse file that is being streamed */ +struct backup_chunk +{ + /** byte offset of the start of the payload, from the start of the file */ + uint64_t offset; + /** length of the hole */ + uint64_t length; +}; + +/** File descriptor */ +typedef IF_WIN(HANDLE, int) backup_fd; + +#ifdef _WIN32 +/* Use CopyFileEx() to copy entire files */ +#elif defined __APPLE__ +/* You should invoke fclonefileat(2) manually before attempting +copy_entire_file() or backup::copy() */ +# include +# include +# include +/** Copy an entire file. +@param src source file descriptor +@param dst target to append src to +@return error code (negative) +@retval 0 on success */ +inline int copy_entire_file(int src, int dst) +{ + return fcopyfile(src, dst, NULL, COPYFILE_ALL | COPYFILE_CLONE); +} +#else +# ifdef __cplusplus +extern "C" +# endif +/** Copy an entire file. +@param src source file descriptor +@param dst target to append src to +@return error code (non-positive) +@retval 0 on success */ +int copy_entire_file(int src, int dst); +#endif + +#ifdef __cplusplus +# ifdef _WIN32 +struct native_file_handle; +# endif +namespace backup { + +typedef IF_WIN(native_file_handle, int) handle; + +/** + Copy a portion of a file. + @param src source file descriptor + @param dst target to append src to + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +int copy(handle src, backup_fd dst, uint64_t start, uint64_t end) noexcept; + +/** + Append a file snippet to the stream, + after a corresponding call to backup_stream_start(). + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_write() must be invoked to zero-pad the output. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +int append(handle src, backup_fd stream, uint64_t start, uint64_t end) + noexcept; +} +#endif + +#if defined _WIN32 || defined __FreeBSD__ +/* There is no special variant of backup::copy(). */ +#else +# if SIZEOF_SIZE_T > 4 +# ifdef __cplusplus +extern "C" +# endif +/** + Copy from a memory mapping to a file. + @param map source file mapping + @param dst target to append map to + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +int copy_mmap(const void *map, int dst, uint64_t start, uint64_t end); +# define copy_file_mmap copy_mmap +# endif + +# ifdef __linux__ +# ifdef __cplusplus +extern "C" +# endif +/** + Try to copy a portion of a file via copy_file_range(2). + @param src source file descriptor + @param dst target to append src to + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success + @retval 1 if a fallback to copy_mmap() or backup::copy() is needed +*/ +int copy_file_range_try(int src, int dst, uint64_t start, uint64_t end); +# define copy_file_shortcut copy_file_range_try +# endif +#endif + +#ifdef __cplusplus +extern "C" +#endif +/** Append to the configuration file. +@param target backup target directory +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +int backup_config_append(IF_WIN(const char*, int) target, + const char *config, size_t size); + +#ifdef __cplusplus +extern "C" +#endif +/** Append to the configuration file. +@param target backup stream +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +int backup_stream_config(backup_fd stream, const char *config, size_t size); + +#ifdef __cplusplus +extern "C" +#endif +/** Start streaming a file. +@param target backup target +@param name file name +@param mode file access mode +@param size physical length of the file, in bytes +@param chunks payload chunks of a sparse file, or nullptr +@param n_chunks number of chunks; 0 unless sparse file +@return error code (non-positive) +@retval 0 on success */ +int backup_stream_start(backup_fd stream, + const char *name, mode_t mode, uint64_t size, + const struct backup_chunk *chunks, size_t n_chunks); + +#ifdef __cplusplus +extern "C" +#endif +/** + Write data to a stream. + @param stream backup stream + @param buf source buffer + @param size length of the buffer (usually an integer multiple of 512) + @return error code (non-positive) + @retval 0 on success +*/ +int backup_stream_write(backup_fd stream, const void *buf, size_t size); + +#ifdef __cplusplus +extern "C" +#endif +/** + Append a file snippet to the stream, + after a corresponding call to backup_stream_start(). + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_zeropad() must be invoked. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +int backup_stream_append_plain(backup_fd src, backup_fd stream, + uint64_t start, uint64_t end); + +#ifdef __cplusplus +extern "C" +#endif +/** + Zero-pad a the stream to a multiple of 512 bytes. + + @param stream backup stream + @param written least significant bits of the number of payload appended + @return error code (non-positive) + @retval 0 on success +*/ +int backup_stream_zeropad(backup_fd stream, size_t written); + +#ifdef _WIN32 +# define backup_stream_append_async backup_stream_append_plain +#else +# ifdef __cplusplus +extern "C" +# endif +/** + Zero-copy append an immutable file snippet to a stream. + + The caller guarantees that this section of the file will remain + intact until the stream is closed. This guarantee is needed + because the receiving end of the stream pipe might delay consuming + the data, and the operating system might point the pipe buffer + to the block cache for a long time. + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_zeropad() must be invoked. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +int backup_stream_append_async(int src, int stream, + uint64_t start, uint64_t end); +#endif diff --git a/sql/sql_command.h b/sql/sql_command.h index 9c9166706a034..b8903399711f0 100644 --- a/sql/sql_command.h +++ b/sql/sql_command.h @@ -103,6 +103,7 @@ enum enum_sql_command { SQLCOM_SHOW_PACKAGE_BODY_CODE, SQLCOM_BACKUP, SQLCOM_BACKUP_LOCK, SQLCOM_SHOW_CREATE_SERVER, + SQLCOM_BACKUP_SERVER, /* When a command is added here, be sure it's also added in mysqld.cc diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index dc069695b23b6..4b67aed0f6475 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -781,6 +781,7 @@ void init_update_queries(void) sql_command_flags[SQLCOM_DROP_SERVER]|= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_BACKUP]= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_BACKUP_LOCK]= CF_AUTO_COMMIT_TRANS; + sql_command_flags[SQLCOM_BACKUP_SERVER]= CF_AUTO_COMMIT_TRANS; /* The following statements can deal with temporary tables, @@ -5899,6 +5900,7 @@ mysql_execute_command(THD *thd, bool is_called_from_prepared_stmt) case SQLCOM_CALL: case SQLCOM_REVOKE: case SQLCOM_GRANT: + case SQLCOM_BACKUP_SERVER: if (thd->variables.option_bits & OPTION_IF_EXISTS) lex->create_info.set(DDL_options_st::OPT_IF_EXISTS); DBUG_ASSERT(lex->m_sql_cmd != NULL); @@ -10249,7 +10251,7 @@ int test_if_data_home_dir(const char *dir) if (!dir) DBUG_RETURN(0); - (void) fn_format(path, dir, "", "", MY_RETURN_REAL_PATH); + (void) fn_format(path, dir, "", "", MY_RETURN_REAL_PATH|MY_RESOLVE_SYMLINKS); DBUG_RETURN(path_starts_from_data_home_dir(path)); } diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index b2071b1c50b7f..08bf1fb7d84c8 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -50,6 +50,7 @@ #include "sql_alter.h" // Sql_cmd_alter_table* #include "sql_truncate.h" // Sql_cmd_truncate_table #include "sql_admin.h" // Sql_cmd_analyze/Check..._table +#include "sql_backup.h" #include "sql_partition_admin.h" // Sql_cmd_alter_table_*_part. #include "sql_handler.h" // Sql_cmd_handler_* #include "sql_signal.h" @@ -1473,7 +1474,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); %type json_on_response %type json_type_constraint -%type json_key_unique_constraint +%type json_key_unique_constraint opt_concurrent %type json_predicate %type field_type field_type_all field_type_all_builtin @@ -15563,6 +15564,31 @@ backup_statements: /* Table list is empty for unlock */ Lex->sql_command= SQLCOM_BACKUP_LOCK; } + | SERVER_SYM TO_SYM TEXT_STRING_sys opt_concurrent + { + Lex->sql_command= SQLCOM_BACKUP_SERVER; + Lex->m_sql_cmd= new (thd->mem_root) Sql_cmd_backup($3, $4); + } + | SERVER_SYM WITH opt_concurrent TEXT_STRING_sys + { + Lex->sql_command= SQLCOM_BACKUP_SERVER; + Lex->m_sql_cmd= new (thd->mem_root) Sql_cmd_backup($3, $4); + } + ; + +opt_concurrent: + /* empty */ + { $$= 1; } + | ulonglong_num CONCURRENT + { + $$= int($1); + if ($1 < 1 || $1 > 256) + { + my_error(ER_DATA_OUT_OF_RANGE, myf(0), "CONCURRENT", + "BACKUP SERVER"); + MYSQL_YYABORT; + } + } ; opt_delete_gtid_domain: diff --git a/sql/sys_vars.inl b/sql/sys_vars.inl index 0f8aa1eb63bf6..8e5f0b984e81e 100644 --- a/sql/sys_vars.inl +++ b/sql/sys_vars.inl @@ -2506,6 +2506,7 @@ public: bool session_update(THD *thd, set_var *var) override; }; +#ifdef HAVE_REPLICATION /* Class for replicate_events_marked_for_skip. We need a custom update function that ensures the slave is stopped when @@ -2647,6 +2648,7 @@ public: } const uchar *global_value_ptr(THD *thd, const LEX_CSTRING *base) const override; }; +#endif /* HAVE_REPLICATION */ /** diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 9e3a23b34ab46..d63751a16af08 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -185,6 +185,8 @@ SET(INNOBASE_SOURCES handler/handler0alter.cc handler/innodb_binlog.cc handler/i_s.cc + handler/backup_innodb.h + handler/backup_innodb.cc ibuf/ibuf0ibuf.cc include/btr0btr.h include/btr0btr.inl diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 49f9a2a96a0ab..8174c3ac9d66a 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -1204,7 +1204,6 @@ btr_search_guess_on_hash( } ut_ad(!block->page.is_read_fixed(state)); - ut_ad(!block->page.is_write_fixed(state) || latch_mode == BTR_SEARCH_LEAF); const dict_index_t *block_index= block->index; if (index != block_index && index_id == block_index->id) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 577973db456b0..17be16e2de657 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -2298,7 +2298,7 @@ void buf_page_free(fil_space_t *space, uint32_t page, mtr_t *mtr) if (block->index) btr_search_drop_page_hash_index(block, nullptr); #endif /* BTR_CUR_HASH_ADAPT */ - block->page.set_freed(block->page.state()); + block->page.set_freed(); mtr->memo_push(block, MTR_MEMO_PAGE_X_MODIFY); } @@ -2367,7 +2367,7 @@ buf_page_t *buf_page_get_zip(const page_id_t page_id) noexcept exclusive latch on this block and either in progress or invoking buf_pool_t::corrupted_evict(). - Let us aqcuire and release buf_pool.mutex to ensure that any + Let us acquire and release buf_pool.mutex to ensure that any buf_pool_t::corrupted_evict() will proceed before we reacquire the hash_lock that it could be waiting for. @@ -2501,7 +2501,12 @@ buf_block_t *buf_pool_t::unzip(buf_page_t *b, buf_pool_t::hash_chain &chain) case buf_page_t::REINIT + 1: break; default: - ut_ad(state < buf_page_t::READ_FIX); + /* + There may be a fake "write fix" if a thread is executing + between InnoDB_backup::backup_batch_start() and + InnoDB_backup::backup_batch_stop() on this page. + */ + ut_ad(!buf_page_t::is_read_fixed(state)); if (state < buf_page_t::UNFIXED + 1) { @@ -2520,7 +2525,7 @@ buf_block_t *buf_pool_t::unzip(buf_page_t *b, buf_pool_t::hash_chain &chain) goto wait_for_unfix; } - /* Ensure that another buf_page_get_low() or buf_page_t::page_fix() + /* Ensure that another buf_page_get_low() or buf_pool_t::page_fix() will wait for block->page.lock.x_unlock(). buf_relocate() will copy the state from b to block and replace b with block in page_hash. */ b->set_state(buf_page_t::READ_FIX); @@ -2879,7 +2884,17 @@ buf_page_get_gen( if (!nowait) { goto latch_waited; } else { - ut_ad(state < buf_page_t::READ_FIX); + /* If a thread is executing between + InnoDB_backup::backup_batch_start() and + InnoDB_backup::backup_batch_stop() + for this page, a fake "write + fix" may exist. We are free to + modify the page in the buffer + pool, but buf_page_t::flush() + will refuse to write it to the + file system. */ + ut_ad(!buf_page_t:: + is_read_fixed(state)); } /* fall through */ case RW_S_LATCH: @@ -2893,8 +2908,7 @@ buf_page_get_gen( } else { not_read_fixed: ut_ad(state > buf_page_t::FREED); - ut_ad(state < buf_page_t::READ_FIX - || state > buf_page_t::WRITE_FIX); + ut_ad(!buf_page_t::is_read_fixed(state)); if (UNIV_UNLIKELY(!block->page.frame && mode == BUF_PEEK_IF_IN_POOL)) { /* The BUF_PEEK_IF_IN_POOL mode is mainly used @@ -2956,7 +2970,8 @@ buf_page_get_gen( break; case RW_SX_LATCH: block->page.lock.u_lock(); - ut_ad(!block->page.is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist */ + ut_ad(!block->page.is_read_fixed()); break; default: ut_ad(rw_latch == RW_X_LATCH); @@ -3039,7 +3054,8 @@ buf_block_t *buf_page_optimistic_get(buf_block_t *block, goto fail; else { - ut_ad(!block->page.is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist */ + ut_ad(!block->page.is_read_fixed()); if (modify_clock != block->modify_clock || block->page.is_freed()) { @@ -3151,6 +3167,37 @@ buf_pool_t::page_hash_table::replace(buf_pool_t::hash_chain &chain, *prev= bpage; } +/** Mark the block as reinitialized in the file. @see set_freed() */ +void buf_page_t::set_reinit() noexcept +{ + /* + Ensure that lock.x_lock() is being held (hopefully, by us). + This blocks a concurrent flush(), protected by lock.u_lock(). + */ + ut_ad(lock.is_write_locked()); + uint32_t s{state()}; + do + { + /* + InnoDB_backup::backup_batch_start() may set fix() and + write_fix_try() on any dirty block to prevent flush() from + writing changes back to the file system. As long as buf_relocate() + will not be invoked, we may safely mark such blocks as REINIT; + write_unfix_try() will account for that. + */ + ut_ad(s < READ_FIX || (s > WRITE_FIX && frame)); + if (!((REINIT ^ s) & LRU_MASK)) + break; + } + /* + fetch_add() or fetch_sub() are not safe here, because this may run + concurrently with write_fix_try() or write_unfix_try(). + */ + while (!zip.fix.compare_exchange_weak(s, REINIT + buf_fix_count(s), + std::memory_order_acquire, + std::memory_order_relaxed)); +} + static buf_block_t *buf_page_create_low(page_id_t page_id, ulint zip_size, mtr_t *mtr, buf_block_t *free_block) noexcept @@ -3198,10 +3245,9 @@ static buf_block_t *buf_page_create_low(page_id_t page_id, ulint zip_size, mysql_mutex_unlock(&buf_pool.mutex); bpage->lock.x_lock(); - const page_id_t id{bpage->id()}; - if (UNIV_UNLIKELY(id != page_id)) + if (UNIV_UNLIKELY(bpage->id() != page_id)) { - ut_ad(id.is_corrupted()); + ut_ad(bpage->id().is_corrupted()); ut_ad(bpage->is_freed()); bpage->unfix(); bpage->lock.x_unlock(); @@ -3209,26 +3255,23 @@ static buf_block_t *buf_page_create_low(page_id_t page_id, ulint zip_size, } mysql_mutex_lock(&buf_pool.mutex); state= bpage->state(); - ut_ad(!bpage->is_io_fixed(state)); ut_ad(bpage->buf_fix_count(state)); } else state= bpage->state(); - ut_ad(state > buf_page_t::FREED); - ut_ad(state < buf_page_t::READ_FIX); /* In addition to our buffer-fix, there may be another that is held by a concurrent IORequest::read_complete() that had - released the bpage->lock in bpage->read_complete(...) but not + released the bpage->lock in bpage->read_complete(...) but not yet invoked bpage->unfix(). This should only be due to an - asynchronous read-ahead for a page that was actually marked as - freed in the underlying data file. */ - ut_ad(bpage->buf_fix_count(state) <= 2); + unnecessary asynchronous read-ahead for a page that was actually + marked as freed in the underlying data file. - if (state < buf_page_t::UNFIXED) - bpage->set_reinit(buf_page_t::FREED); - else - bpage->set_reinit(state & buf_page_t::LRU_MASK); + Alternatively, another thread may be executing between + InnoDB_backup::backup_batch_start() and + InnoDB_backup::backup_batch_stop() on this page. */ + ut_ad(bpage->buf_fix_count(state) <= 2); + ut_ad(state < buf_page_t::READ_FIX || state > buf_page_t::WRITE_FIX); if (UNIV_LIKELY(bpage->frame != nullptr)) { @@ -3242,23 +3285,40 @@ static buf_block_t *buf_page_create_low(page_id_t page_id, ulint zip_size, } else { + /* + Augment the compressed-only ROW_FORMAT=COMPRESSED page with + an uncompressed page frame. + */ page_hash_latch &hash_lock= buf_pool.page_hash.lock_get(chain); for (;;) { hash_lock.lock(); state= bpage->state(); - if ((state & ~buf_page_t::LRU_MASK) == 1) + if (state == buf_page_t::FREED + 1 || + state == buf_page_t::UNFIXED + 1 || + state == buf_page_t::REINIT + 1) break; - /* Wait for a concurrent IORequest::read_complete() to - invoke bpage->unfix(), for an unnecessary read-ahead of - a freed page. */ - ut_ad((state & ~buf_page_t::LRU_MASK) == 2); + ut_ad(state > buf_page_t::FREED); + /* + Wait for the competing thread to invoke buf_page_t::unfix() + and possibly buf_page_t::write_unfix_try() before that. + */ + ut_ad(bpage->buf_fix_count(state) == 2); hash_lock.unlock(); + if (state >= buf_page_t::READ_FIX) + { + ut_ad(state > buf_page_t::WRITE_FIX); + /* InnoDB_backup::step() may take a long time. */ + mysql_mutex_unlock(&buf_pool.mutex); + std::this_thread::yield(); + mysql_mutex_lock(&buf_pool.mutex); + } } mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_relocate(bpage, &free_block->page); free_block->page.lock.x_lock(); + free_block->page.set_state(buf_page_t::REINIT + 1); buf_flush_relocate_on_flush_list(bpage, &free_block->page); mysql_mutex_unlock(&buf_pool.flush_list_mutex); @@ -3272,7 +3332,7 @@ static buf_block_t *buf_page_create_low(page_id_t page_id, ulint zip_size, #endif ut_free(bpage); mtr->memo_push(free_block, MTR_MEMO_PAGE_X_FIX); - bpage= &free_block->page; + return free_block; } } else @@ -3283,12 +3343,9 @@ static buf_block_t *buf_page_create_low(page_id_t page_id, ulint zip_size, #ifdef BTR_CUR_HASH_ADAPT ut_ad(!reinterpret_cast(bpage)->index); #endif - const auto state= bpage->state(); - ut_ad(state >= buf_page_t::FREED); - bpage->set_reinit(state < buf_page_t::UNFIXED ? buf_page_t::FREED - : state & buf_page_t::LRU_MASK); } + bpage->set_reinit(); #ifdef BTR_CUR_HASH_ADAPT if (drop_hash_entry) btr_search_drop_page_hash_index(reinterpret_cast(bpage), @@ -3831,8 +3888,7 @@ void buf_pool_t::validate() noexcept /* do nothing */ break; default: - if (f >= buf_page_t::READ_FIX - && f < buf_page_t::WRITE_FIX) { + if (buf_page_t::is_read_fixed(f)) { /* A read-fixed block is not necessarily in the page_hash yet. */ break; diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index fb4a7bc5d9958..b9afbb158250f 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -752,7 +752,7 @@ void buf_dblwr_t::flush_buffered_writes_completed(const IORequest &request) static_cast(frame))); ut_ad(lsn); ut_ad(lsn >= bpage->oldest_modification()); - if (lsn < e.request.node->space->get_create_lsn()) + if (lsn < e.request.node->space->create_lsn) { /* mtr_t::commit_shrink() must have been invoked between buf_dblwr_t::flush_buffered_writes() and diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 299da45582cd7..7a30f78c479b8 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -35,6 +35,7 @@ Created 11/11/1995 Heikki Tuuri #include "buf0buf.h" #include "buf0checksum.h" #include "buf0dblwr.h" +#include "backup_innodb.h" #include "srv0start.h" #include "page0zip.h" #include "fil0fil.h" @@ -244,7 +245,7 @@ void buf_flush_remove_pages(uint32_t id) noexcept { const auto s= bpage->state(); ut_ad(s >= buf_page_t::REMOVE_HASH); - ut_ad(s < buf_page_t::READ_FIX || s >= buf_page_t::WRITE_FIX); + ut_ad(!buf_page_t::is_read_fixed(s)); buf_page_t *prev= UT_LIST_GET_PREV(list, bpage); const page_id_t bpage_id(bpage->id()); @@ -786,7 +787,7 @@ bool buf_page_t::flush(fil_space_t *space) noexcept ut_ad((space->is_temporary()) == (space == fil_system.temp_space)); ut_ad(space->referenced()); - const auto s= state(); + uint32_t s{state()}; const lsn_t lsn= mach_read_from_8(my_assume_aligned<8> @@ -812,16 +813,40 @@ bool buf_page_t::flush(fil_space_t *space) noexcept return false; } - if (UNIV_UNLIKELY(lsn < space->get_create_lsn())) + if (UNIV_UNLIKELY(lsn < space->create_lsn)) { ut_ad(!space->is_temporary()); ut_ad(!space->is_being_imported()); goto freed; } - ut_d(const auto f=) zip.fix.fetch_add(WRITE_FIX - UNFIXED); - ut_ad(f >= UNFIXED); - ut_ad(f < READ_FIX); + do + { + ut_ad(s >= UNFIXED); + if (UNIV_UNLIKELY(s >= READ_FIX)) + { + /* + A thread must have successfully executed write_fix_try() after + fix(), executing between InnoDB_backup::backup_batch_start() + and InnoDB_backup::backup_batch_stop(). + */ + ut_ad(s > WRITE_FIX); + /* Backup skips the temporary tablespace */ + ut_ad(oldest_modification() > 2); + lock.u_unlock(true); + return false; + } + } + /* + compare_exchange_strong() ensures that the write-fix was set by us + and not a concurrent write_fix_try(). + */ + while (!zip.fix.compare_exchange_strong(s, s + (WRITE_FIX - UNFIXED), + std::memory_order_acquire, + std::memory_order_relaxed)); + + ut_ad(s >= UNFIXED); + ut_ad(s < READ_FIX); ut_ad((space == fil_system.temp_space) ? oldest_modification() == 2 : oldest_modification() > 2); @@ -1002,6 +1027,14 @@ uint32_t fil_space_t::flush_freed(bool writable) noexcept mysql_mutex_assert_not_owner(&buf_pool.flush_list_mutex); mysql_mutex_assert_not_owner(&buf_pool.mutex); + /* Note: There is no need to check if another thread is executing + between InnoDB_backup::backup_batch_start() and + InnoDB_backup::backup_batch_end(), because we are only overwriting + freed (garbage) pages. If backup reads a torn page, it will also + have copied a corresponding FREE_PAGE record, which would be applied + on recovery. Besides, the freed page should never be reachable from + other pages that are part of the snapshot. */ + const bool punch_hole= chain.start->punch_hole == 1; if (!punch_hole && !srv_immediate_scrub_data_uncompressed) return 0; @@ -1084,7 +1117,7 @@ static ulint buf_flush_try_neighbors(fil_space_t *space, (FIL_PAGE_LSN + (bpage->zip.data ? bpage->zip.data : bpage->frame))); ut_ad(lsn >= bpage->oldest_modification()); - if (UNIV_UNLIKELY(lsn < space->get_create_lsn())) + if (UNIV_UNLIKELY(lsn < space->create_lsn)) { ut_a(!bpage->flush(space)); mysql_mutex_unlock(&buf_pool.mutex); @@ -1226,15 +1259,20 @@ fil_space_t *fil_space_t::get_for_write(uint32_t id) noexcept return space; } -/** Start writing out pages for a tablespace. -@param id tablespace identifier -@return tablespace and number of pages written */ -static std::pair buf_flush_space(const uint32_t id) - noexcept +/** + Start writing out pages for a tablespace. + @param id tablespace identifier + @param end fil_space_t::backup_page_end() + @return tablespace and number of pages written at this point +*/ +static std::pair +buf_flush_space(const uint32_t id, uint32_t *end) noexcept { - if (fil_space_t *space= fil_space_t::get_for_write(id)) - return {space, space->flush_freed(true)}; - return {nullptr, 0}; + fil_space_t *space= fil_space_t::get_for_write(id); + if (!space) + return {nullptr, 0}; + *end= space->backup_page_end(); + return {space, space->flush_freed(true)}; } struct flush_counters_t @@ -1294,6 +1332,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, ? 0 : buf_pool.flush_neighbors; fil_space_t *space= nullptr; uint32_t last_space_id= FIL_NULL; + uint32_t backup_page_end= 0; static_assert(FIL_NULL > SRV_TMP_SPACE_ID, "consistency"); static_assert(FIL_NULL > SRV_SPACE_ID_UPPER_BOUND, "consistency"); @@ -1343,11 +1382,12 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, if (state < buf_page_t::READ_FIX && bpage->lock.u_lock_try(true)) { - ut_ad(!bpage->is_io_fixed()); + ut_ad(!bpage->is_read_fixed()); /* tolerate write_fix_try() */ switch (bpage->oldest_modification()) { case 2: /* LRU flushing will always evict pages of the temporary tablespace, in buf_page_write_complete(). */ + ut_ad(!bpage->is_io_fixed()); ++n->evicted; break; case 1: @@ -1361,6 +1401,8 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, /* fall through */ case 0: bpage->lock.u_unlock(true); + /* Reload in case write_fix_try() had been undone meanwhile. */ + state= bpage->state(); goto evict; } /* Block is ready for flush. Dispatch an IO request. */ @@ -1374,9 +1416,15 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, mysql_mutex_unlock(&buf_pool.mutex); if (space) space->release(); - auto p= buf_flush_space(space_id); - space= p.first; last_space_id= space_id; + auto p= buf_flush_space(space_id, &backup_page_end); + /* + In case our backup_page_end just missed a write in + backup_batch_start(), innodb_backup_batch_wait() will wait + until our bpage->lock.u_unlock() will be released in + buf_page_t::write_complete(). + */ + space= p.first; if (!space) { mysql_mutex_lock(&buf_pool.mutex); @@ -1411,7 +1459,12 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, break; } - if (neighbors && space->is_rotational() && UNIV_LIKELY(!to_withdraw) && + /* + Skip innodb_flush_neighbors if InnoDB_backup is in progress + and we may be badly I/O bound. + */ + if (neighbors && UNIV_LIKELY(!(to_withdraw | backup_page_end)) && + space->is_rotational() && /* Skip neighbourhood flush from LRU list if we haven't yet reached half of the free page target. */ UT_LIST_GET_LEN(buf_pool.free) * 2 >= free_limit) @@ -1423,10 +1476,17 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, flush: if (UNIV_UNLIKELY(to_withdraw != 0)) to_withdraw= buf_flush_LRU_to_withdraw(to_withdraw, *bpage); - if (bpage->flush(space)) + const uint32_t page{bpage->id().page_no()}; + if (page < backup_page_end && + page >= backup_page_end - space->BACKUP_BATCH_SIZE) + bpage->lock.u_unlock(true); + else if (bpage->flush(space)) + { ++n->flushed; - else - continue; + goto reacquire_mutex; + } + + continue; } goto reacquire_mutex; @@ -1486,6 +1546,7 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept ? 0 : buf_pool.flush_neighbors; fil_space_t *space= nullptr; uint32_t last_space_id= FIL_NULL; + uint32_t backup_page_end= 0; static_assert(FIL_NULL > SRV_TMP_SPACE_ID, "consistency"); static_assert(FIL_NULL > SRV_SPACE_ID_UPPER_BOUND, "consistency"); @@ -1527,7 +1588,9 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept if (!bpage->lock.u_lock_try(true)) goto skip; - ut_ad(!bpage->is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist; + we will check it in buf_page_t::flush() */ + ut_ad(!bpage->is_read_fixed()); if (bpage->oldest_modification() == 1) { @@ -1558,9 +1621,15 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept mysql_mutex_unlock(&buf_pool.mutex); if (space) space->release(); - auto p= buf_flush_space(space_id); - space= p.first; last_space_id= space_id; + auto p= buf_flush_space(space_id, &backup_page_end); + /* + In case our backup_page_end just missed a write in + backup_batch_start(), innodb_backup_batch_wait() will wait + until our bpage->lock.u_unlock() will be released in + buf_page_t::write_complete(). + */ + space= p.first; mysql_mutex_lock(&buf_pool.mutex); buf_pool.stat.n_pages_written+= p.second; mysql_mutex_lock(&buf_pool.flush_list_mutex); @@ -1581,9 +1650,17 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept mysql_mutex_unlock(&buf_pool.flush_list_mutex); do { - if (neighbors && space->is_rotational()) + if (neighbors && UNIV_LIKELY(!backup_page_end) && + space->is_rotational()) count+= buf_flush_try_neighbors(space, page_id, bpage, neighbors == 1, count, max_n); + else if (page_id.page_no() < backup_page_end && + page_id.page_no() >= + backup_page_end - space->BACKUP_BATCH_SIZE) + { + bpage->lock.u_unlock(true); + continue; + } else if (bpage->flush(space)) ++count; else @@ -1736,7 +1813,21 @@ bool buf_flush_list_space(fil_space_t *space, ulint *n_flushed) noexcept goto was_freed; } mysql_mutex_unlock(&buf_pool.flush_list_mutex); - if (bpage->flush(space)) + const uint32_t page{bpage->id().page_no()}; + const uint32_t backup_page_end{space->backup_page_end()}; + /* + In case our backup_page_end just missed a write in + backup_batch_start(), innodb_backup_batch_wait() will wait + until our bpage->lock.u_unlock() will be released in + buf_page_t::write_complete(). + */ + if (UNIV_UNLIKELY(page < backup_page_end) && + page >= backup_page_end - space->BACKUP_BATCH_SIZE) + { + bpage->lock.u_unlock(true); + may_have_skipped= true; + } + else if (bpage->flush(space)) { ++n_flush; if (!--max_n_flush) @@ -2071,6 +2162,7 @@ inline lsn_t log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept this->end_lsn= end_lsn; if (!archive) { + archived_checkpoint= checkpoint; archived_lsn= end_lsn; checkpoint_completed: if (resize_log.m_file == log.m_file) @@ -2088,9 +2180,11 @@ inline lsn_t log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept /* Make the previous archived log file read-only */ #ifdef _WIN32 resize_log.close(); - SetFileAttributesA(get_archive_path().c_str(), + SetFileAttributesA(get_archive_path(get_first_lsn() - capacity()).c_str(), FILE_ATTRIBUTE_READONLY | FILE_ATTRIBUTE_ARCHIVE); + innodb_backup_checkpoint(); #else + innodb_backup_checkpoint(); struct stat st; if (!fstat(resize_log.m_file, &st)) st.st_mode&= 0444; @@ -2098,7 +2192,7 @@ inline lsn_t log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept st.st_mode= 0444; if (fchmod(resize_log.m_file, st.st_mode)) my_error(ER_ERROR_ON_CLOSE, MYF(ME_ERROR_LOG), - get_archive_path().c_str(), errno); + get_archive_path(get_first_lsn() - capacity()).c_str(), errno); resize_log.close(); #endif } diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 95e2a6cbe0972..7b8bf9a3e06b9 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -1271,20 +1271,15 @@ void buf_LRU_truncate_temp(uint32_t threshold) /* Set the extent descriptor page state as FREED */ for (uint32_t cur_xdes_page= xdes_calc_descriptor_page( 0, fil_system.temp_space->free_limit); - cur_xdes_page >= threshold;) + cur_xdes_page >= threshold; + cur_xdes_page-= uint32_t(srv_page_size)) { mtr_t mtr{nullptr}; mtr.start(); if (buf_block_t* block= buf_page_get_gen( page_id_t(SRV_TMP_SPACE_ID, cur_xdes_page), 0, RW_X_LATCH, nullptr, BUF_PEEK_IF_IN_POOL, &mtr)) - { - uint32_t state= block->page.state(); - ut_ad(state > buf_page_t::UNFIXED); - ut_ad(state < buf_page_t::READ_FIX); - block->page.set_freed(state); - } - cur_xdes_page-= uint32_t(srv_page_size); + block->page.set_freed(); mtr.commit(); } diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index d8976dec66974..49ffe82fcd49c 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -889,6 +889,10 @@ void dict_load_tablespaces(const std::set *spaces, bool upgrade) dict_sys.lock(SRW_LOCK_CALL); + if (fil_system.have_all_spaces) { + goto done; + } + if (!spaces && !upgrade && !encryption_key_id_exists(FIL_DEFAULT_ENCRYPTION_KEY)) { max_space_id = dict_find_max_space_id(&pcur, &mtr); @@ -934,7 +938,15 @@ void dict_load_tablespaces(const std::set *spaces, bool upgrade) newly created or rebuilt tables or partitions, but will otherwise ignore the flag. */ - if (fil_space_for_table_exists_in_mem(space_id, flags)) { + fil_space_t* space + = fil_space_for_table_exists_in_mem(space_id, flags); + + if (space) { + if (upgrade) { + space->get_size(); + } + next: + max_space_id = ut_max(max_space_id, space_id); continue; } @@ -959,11 +971,13 @@ void dict_load_tablespaces(const std::set *spaces, bool upgrade) const bool not_dropped{!rec_get_deleted_flag(rec, 0)}; /* Check that the .ibd file exists. */ - if (fil_ibd_open(space_id, dict_tf_to_fsp_flags(flags), - not_dropped - ? fil_space_t::VALIDATE_NOTHING - : fil_space_t::MAYBE_MISSING, - name, filepath)) { + space = fil_ibd_open( + space_id, dict_tf_to_fsp_flags(flags), + not_dropped + ? fil_space_t::VALIDATE_NOTHING + : fil_space_t::MAYBE_MISSING, + name, filepath); + if (space && (!upgrade || space->get_size())) { } else if (!not_dropped) { } else if (srv_operation == SRV_OPERATION_NORMAL && srv_start_after_restore @@ -980,11 +994,11 @@ void dict_load_tablespaces(const std::set *spaces, bool upgrade) static_cast(len), field); } - max_space_id = ut_max(max_space_id, space_id); - ut_free(filepath); + goto next; } + fil_system.have_all_spaces = upgrade; done: mtr.commit(); diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 47bfde7ebd23d..7973837028ebd 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -2035,6 +2035,7 @@ fil_ibd_create( fil_node_t* node = space->add(path, OS_FILE_CLOSED, size, false, true); space->set_stopped(); mysql_mutex_unlock(&fil_system.mutex); + lsn_t create_lsn; buf_block_t *header[2]; { @@ -2053,9 +2054,10 @@ fil_ibd_create( header[0]->page.lock.x_lock(); header[1]->page.lock.x_lock(); mtr.commit(); + create_lsn = mtr.commit_lsn(); /* Durably write the FILE_CREATE record (as well as records for the fsp_header_init() above before creating the file. */ - log_write_up_to(mtr.commit_lsn(), true); + log_write_up_to(create_lsn, true); } DEBUG_SYNC_C("fil_ibd_create_logged"); @@ -2136,6 +2138,7 @@ fil_ibd_create( mysql_mutex_lock(&fil_system.mutex); space->clear_stopped(); + space->create_lsn = create_lsn; node->handle = file; node->find_metadata(IF_WIN(,true)); if (++fil_system.n_open >= srv_max_n_open_files) { diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc new file mode 100644 index 0000000000000..62847a4763eec --- /dev/null +++ b/storage/innobase/handler/backup_innodb.cc @@ -0,0 +1,1713 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "my_global.h" +#include "sql_class.h" +#include "backup_innodb.h" +#include "trx0trx.h" +#include "buf0flu.h" +#include "log0crypt.h" +#include "dict0load.h" +#include +#ifdef __linux__ +# include +# include +#endif + +/** Associate a transaction with the current session +@param thd session +@return InnoDB transaction */ +trx_t *check_trx_exists(THD *thd) noexcept; + +/** Try to write-fix a block. +@param s expected state() +@return new s (right before potentially setting the write-fix); +a write-fix was acquired if !is_freed(s) && !is_io_fixed(s) holds */ +inline uint32_t buf_page_t::write_fix_try(uint32_t s) noexcept +{ + /* The calling thread must hold a fix() */ + ut_ad(s > FREED); + /* + set_freed(), set_reinit() or flush() may run concurrently. + compare_exchange_strong() ensures that the write-fix was set by us. + */ + while (!is_freed(s) && !is_io_fixed(s) && + !zip.fix.compare_exchange_strong(s, s + (WRITE_FIX - UNFIXED), + std::memory_order_acquire, + std::memory_order_relaxed)); + return s; +} + +/** Try to undo a successful write_fix_try(). */ +inline void buf_page_t::write_unfix_try() noexcept +{ + uint32_t s{state()}; + /* set_freed() or set_reinit() may clear the write-fix before or + during this loop. We must use a compare-and-exchange loop. */ + while (is_write_fixed(s) && + !zip.fix.compare_exchange_weak(s, s - (WRITE_FIX - UNFIXED), + std::memory_order_relaxed, + std::memory_order_relaxed)); +} + +/** + Ensure that there are no page writes in progress. + @param end array of fil_space_t::BACKUP_BATCH_SIZE block descriptors + @param space_id tablespace identifier + @param end_page last page number that is being copied + @return pointer to the new end of the array, of write-fixed blocks +*/ +static buf_page_t **innodb_backup_batch_wait(buf_page_t **end, + uint32_t space_id, + uint32_t end_page) noexcept +{ + const page_id_t start + {space_id, end_page & ~(fil_space_t::BACKUP_BATCH_SIZE - 1)}; + ut_ad(end_page - 1 > start.page_no()); + for (page_id_t id{space_id, end_page};; --id) + { + auto &chain= buf_pool.page_hash.cell_get(id.fold()); + page_hash_latch &hash_lock{buf_pool.page_hash.lock_get(chain)}; + hash_lock.lock_shared(); + buf_page_t *const b= *end= buf_pool.page_hash.get(id, chain); + if (b && b->oldest_modification_acquire() > 2) + { + uint32_t state{b->fix()}; + /* The above buffer-fix froze b in buf_pool.page_hash */ + hash_lock.unlock_shared(); + /* + We found out that this page was dirty, and eventually such pages + must be either freed or written back to the file system. + + The lock-free buf_page_t::write_fix_try() aims to block + concurrent asynchronous buf_page_t::flush(). It will not + block any access to the page in the buffer pool; we are + not holding any page latch. + */ + state= b->write_fix_try(state + 1); + if (UNIV_LIKELY(!b->is_io_fixed(state))) + { + if (UNIV_LIKELY(!b->is_freed(state))) + { + /* Schedule a call of b->write_unfix_try() and b->unfix(). */ + end++; + goto next; + } + /* + Freed blocks will not be written back to the file system. + As noted in fil_space_t::flush_freed(), we do allow + concurrent FALLOC_FL_PUNCH_HOLE of PAGE_COMPRESSED pages + and NUL writes by immediate_scrub_data_uncompressed=ON + while the data is being backed up. This should be fine, + because those operations are only overwriting freed + (garbage) data. + */ + } + else if (b->is_write_fixed(state)) + { + /* Wait for buf_page_t::write_complete() */ + b->lock.u_lock(); + ut_ad(!b->is_io_fixed()); + b->lock.u_unlock(); + } + + /* + Any subsequent write to this page will be held up by + fil_space_t::backup_page_end() until + fil_space_t::backup_stop() is invoked by + InnoDB_backup::backup_batch_stop(). + */ + b->unfix(); + } + else + hash_lock.unlock_shared(); + next: + if (id == start) + return end; + } +} + +namespace +{ +/** Backup state and context; mostly protected by log_sys.latch */ +class InnoDB_backup +{ +public: + InnoDB_backup() { mutex.init(); } + ~InnoDB_backup() { mutex.destroy(); } + +private: + enum State { + /** no BACKUP SERVER is in progress */ + IDLE, + /** BACKUP SERVER is active; checkpoint_complete() will fill the queue */ + PROCESSING, + /** BACKUP SERVER is cleaning up, not protected by any MDL */ + CLEANUP + }; + + /** Backup context */ + struct context + { + /** Start LSN of the first backed up log file */ + const lsn_t first_lsn{}; + /** Start LSN of the last log file, or LSN_MAX if not determined yet */ + lsn_t max_first_lsn{}; + /** Final LSN of the backup, or LSN_MAX if not determined yet; + on error, assigned to 0 while holding InnoDB_backup::mutex and + exclusive log_sys.latch */ + lsn_t last_lsn{}; + /** size of the first log file */ + const uint64_t first_size{}; + /** Checkpoint at the start of the backup */ + const lsn_t checkpoint{}; + /** Log record pointing to the checkpoint */ + const lsn_t checkpoint_end_lsn{}; + /** the original innodb_log_file_size; 0 if innodb_log_archive was enabled */ + const uint64_t old_size{}; + /** state of the operation; protected by log_sys.latch */ + Atomic_relaxed state{IDLE}; + /** the start LSN of the last hard-linked file, or 0 */ + std::atomic last_hardlink{}; + + /** + Note that a log file was hard-linked. + @param lsn start LSN of a hard-linked file + */ + void note_hardlink(lsn_t lsn) noexcept + { + for (lsn_t last= last_hardlink.load(std::memory_order_relaxed); + last < lsn && !last_hardlink. + compare_exchange_weak(last, lsn, + std::memory_order_relaxed, + std::memory_order_relaxed); ) {} + } + + /** Ensure that the last, hard-linked log file is not shared with + the server data directory, by copying it until the final LSN + @param target backup target directory + @param hl last_hardlink + @return error code + @retval 0 on success + */ + ATTRIBUTE_COLD int de_hardlink(const backup_target &target, lsn_t hl) + noexcept + { +#ifdef _WIN32 + std::string src{target.path}; + src.push_back('/'); + std::string dst{src}; + src.append("ib_logfile101"); + log_sys.append_archive_name(dst, hl); + const char *const s_{src.c_str()}, *const d_{dst.c_str()}; + if (!MoveFileEx(d_, s_, 0)) + { + my_osmaperr(GetLastError()); + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), d_, s_, errno); + return 1; + } + HANDLE s, d; + for (;;) + { + s= CreateFile(s_, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + my_win_file_secattr(), OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN, + nullptr); + if (s != INVALID_HANDLE_VALUE) + break; + switch (GetLastError()) { + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + Sleep(10); + continue; + } + my_osmaperr(GetLastError()); + my_error(ER_FILE_NOT_FOUND, MYF(ME_ERROR_LOG), s_, errno); + return 1; + } + d= CreateFile(d_, GENERIC_WRITE, 0, my_win_file_secattr(), + CREATE_NEW, FILE_ATTRIBUTE_NORMAL, nullptr); + if (d == INVALID_HANDLE_VALUE) + { + error_return: + my_osmaperr(GetLastError()); + std::ignore= CloseHandle(s); + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), s_, d_, errno); + return 1; + } +#else + std::string dst; + log_sys.append_archive_name(dst, hl); + const char *const d_{dst.c_str()}; + int d{-1}; + int err= ER_FILE_NOT_FOUND; + int s= openat(target.fd, d_, O_RDONLY); + if (s == -1) + { + error_return: + my_error(err, MYF(ME_ERROR_LOG), d_, errno); + if (s != -1) + std::ignore= close(s); + return 1; + } + err= ER_CANT_DELETE_FILE; + if (unlinkat(target.fd, d_, 0)) + goto error_return; + err= ER_CANT_CREATE_FILE; + d= openat(target.fd, d_, O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (d < 0) + goto error_return; +#endif + const uint64_t end{log_sys.START_OFFSET + last_lsn - hl}; + /* First, extend the file to a valid size. */ +#ifdef _WIN32 + int f; + { + LARGE_INTEGER li; + li.QuadPart= std::max(log_sys.FILE_SIZE_MIN, + (end + 4095) & ~4095ULL); + f= !SetFilePointerEx(d, li, nullptr, FILE_BEGIN) || !SetEndOfFile(d); + } +#else + int f= + ftruncate(d, std::max(log_sys.FILE_SIZE_MIN, + (end + 4095) & ~4095LL)); +#endif + if (!f) + { + const uint64_t begin= log_sys.START_OFFSET + + (hl == first_lsn) * (checkpoint - hl); +#ifdef POSIX_FADV_SEQUENTIAL + std::ignore= posix_fadvise(s, begin, end - begin, + POSIX_FADV_SEQUENTIAL); +#endif +#ifdef copy_file_shortcut + if (1 == (f= copy_file_shortcut(s, d, begin, end))) +#endif + f= backup::copy(s, d, begin, end); +#ifdef POSIX_FADV_DONTNEED + std::ignore= posix_fadvise(s, 0, 0, POSIX_FADV_DONTNEED); +#endif + if (!f && hl == first_lsn) + { + uint64_t cp_buf[8]{}; + write_checkpoint_buf(cp_buf, + checkpoint_end_lsn - hl + log_sys.START_OFFSET); + f= write_checkpoint(d, cp_buf); + } + } + if (IF_WIN(!CloseHandle(d), close(d)) | f) + goto error_return; + std::ignore= IF_WIN(CloseHandle(s), close(s)); +#ifdef _WIN32 + if (!DeleteFile(s_)) + { + my_osmaperr(GetLastError()); + my_error(ER_CANT_DELETE_FILE, MYF(ME_ERROR_LOG), s_, errno); + return 1; + } +#endif + return 0; + } + + /** + Finish a backup. + @param target backup target + @param sink backup worker context + @return error code + @retval 0 on success + */ + int cleanup(const backup_target &target, const backup_sink &sink) noexcept + { + int fail{0}; + ut_ad(state == CLEANUP); + const lsn_t hl{last_hardlink.load(std::memory_order_relaxed)}; + if (hl != LSN_MAX) + { + /* abort() had not been invoked for this backup; finish it */ + log_sys.latch.rd_lock(); + const lsn_t current_first_lsn{log_sys.get_first_lsn()}; + log_sys.latch.rd_unlock(); + + if (hl == current_first_lsn) + { + ut_ad(sink.stream == sink.NO_STREAM); + fail= de_hardlink(target, hl); + } + if (!fail) + fail= write_config(target, sink); + } + state= IDLE; /* unblock init() */ + return fail; + } + }; + + /** backup context */ + context ctx; + + /** mutex protecting queue, non_log */ + srw_mutex mutex; + /** collection of files and sizes, followed by any log files to be copied */ + std::vector queue; + /** number of non-log files at the start of the queue */ + size_t non_log; + +public: + /** + Start of BACKUP SERVER: collect all files to be backed up + @param thd current session + @return ctx + @retval -1 on failure + */ + void *init(THD *thd) noexcept + { + log_sys.latch.wr_lock(); + while (ctx.state != IDLE) + { + /* A previous BACKUP SERVER has not reached context::cleanup(). */ + log_sys.latch.wr_unlock(); + std::this_thread::sleep_for(std::chrono::seconds(1)); + log_sys.latch.wr_lock(); + } + + ut_d(mutex.wr_lock()); + ut_ad(!non_log); + ut_ad(queue.empty()); + ut_d(mutex.wr_unlock()); + + uint64_t old_size; + + if (log_sys.backup_start(&old_size, thd)) + { + log_sys.latch.wr_unlock(); + fail: + my_error(ER_OUT_OF_RESOURCES, MYF(ME_ERROR_LOG)); + return reinterpret_cast(-1); + } + + mutex.wr_lock(); + ut_ad(!non_log); + ut_ad(queue.empty()); + ut_ad(ctx.state == IDLE); + + try + { + lsn_t start_end; + const lsn_t start= +#if 1 /* TODO: for incremental backup, allow the start to be specified */ + log_sys.get_latest_checkpoint(start_end); +#else + log_sys.archived_checkpoint; + start_end= log_sys.archived_lsn; +#endif + ut_ad(start_end >= start); + ut_ad(start >= log_sys.get_first_lsn()); + + new (&ctx) context{ + log_sys.get_first_lsn(), LSN_MAX, LSN_MAX, log_sys.file_size, + start, start_end, old_size, PROCESSING, 0 + }; + + /* Collect all tablespaces that have been created before our + start checkpoint. Newer tablespaces will be recovered by the + innodb_log_archive=ON recovery. + + If a tablespace is deleted before step() is invoked, the file + will not be copied, and a FILE_DELETE record in the log will + ensure correct recovery. + + If a tablespace is renamed between this and end(), the recovery + of a FILE_RENAME record will ensure the correct file name, + no matter which name was used by step(). */ + mysql_mutex_lock(&fil_system.mutex); + for (fil_space_t &space : fil_system.space_list) + if (space.id < SRV_SPACE_ID_UPPER_BOUND && + !space.is_being_imported() && !space.is_stopping() && + space.create_lsn <= start) try + { + /* FIXME: how to initialize create_lsn for old files, to + have efficient incremental backup? + fil_node_t::read_page0() cannot assign it from + FIL_PAGE_LSN because that would not reflect the file + creation but for example allocating or freeing a page. + Perhaps we can read it from page 1 (change buffer bitmap)? + + The easy parts of initializing space->create_lsn are + as follows: + (1) In log_parse_file() when processing FILE_CREATE + (2) In deferred_spaces.create() + (3) In fil_ibd_create() outside recovery */ + queue.emplace_back + (uint64_t{space.id} | + uint64_t{std::min(space.size, space.free_limit)} << 32); + } catch(...) { mysql_mutex_unlock(&fil_system.mutex); throw; } + mysql_mutex_unlock(&fil_system.mutex); + non_log= queue.size(); + } + catch (std::bad_alloc&) { + queue.clear(); + mutex.wr_unlock(); + log_sys.backup_stop(old_size, thd); + mutex.wr_lock(); + ctx.state= IDLE; + mutex.wr_unlock(); + goto fail; + } + + mutex.wr_unlock(); + log_sys.latch.wr_unlock(); + DEBUG_SYNC(thd, "innodb_backup_start"); + return &ctx; + } + + /** + Process a file that was collected at init(). + This may be invoked from multiple concurrent threads. + @param target backup target + @param phase backup phase + @param sink backup worker context + @return number of files remaining, or negative on error + @retval 0 on completion + */ + int step(const backup_target &target, backup_phase phase, + const backup_sink &sink) noexcept + { + uint64_t id_limit{0}; + mutex.wr_lock(); + ut_ad(&ctx == sink.ha_data); + ut_ad(ctx.state != IDLE); + ut_ad(ctx.last_lsn != LSN_MAX || phase == BACKUP_PHASE_START); + size_t size{queue.size()}, non_log_files{non_log}; + ut_ad(size >= non_log_files); + + if (UNIV_UNLIKELY(!size)) + { + done: + mutex.wr_unlock(); + return int(size); + } + + if (UNIV_UNLIKELY(ctx.last_lsn == 0)) + { + /* An error was flagged. */ + size= size_t(-1); + goto done; + } + + non_log-= size == non_log_files; + id_limit= queue.back(); + queue.pop_back(); + mutex.wr_unlock(); + + if (size > non_log_files) + { + log_sys.latch.rd_lock(); + const lsn_t first{log_sys.get_first_lsn()}; + log_sys.latch.rd_unlock(); + if (UNIV_UNLIKELY(id_limit > first)) + /* Wait for checkpoint_complete(). */ + buf_flush_sync_batch(id_limit, true); + if (replicate(id_limit, target, sink, id_limit < first)) + return -1; + } + else if (fil_space_t *space= fil_space_t::get(uint32_t(id_limit))) + { + ut_ad(ctx.state == PROCESSING); + ut_ad(phase == BACKUP_PHASE_START); + int res= -1; + uint32_t start{0}, limit{uint32_t(id_limit >> 32)}; +#ifdef _WIN32 + if (sink.stream == sink.NO_STREAM) + { + for (fil_node_t *node= UT_LIST_GET_FIRST(space->chain);;) + { + if ((res= backup(target.path, node, start, limit))) + break; + fil_node_t *next= UT_LIST_GET_NEXT(chain, node); + if (!next) + break; + const uint32_t size{node->size}; + start+= size; + if (limit >= size) + limit-= size; + else + limit= 0; + node= next; + } + } + else + { + for (fil_node_t *node= UT_LIST_GET_FIRST(space->chain);;) + { + if ((res= stream(sink.stream, node, start, limit))) + break; + fil_node_t *next= UT_LIST_GET_NEXT(chain, node); + if (!next) + break; + const uint32_t size{node->size}; + start+= size; + if (limit >= size) + limit-= size; + else + limit= 0; + node= next; + } + } +#else + int fd; + int (*method)(int, fil_node_t *, uint32_t, uint32_t); + if (sink.stream == sink.NO_STREAM) + { + fd= target.fd; + method= backup; + } + else + { + fd= sink.stream; + method= stream; + } + for (fil_node_t *node= UT_LIST_GET_FIRST(space->chain);;) + { +# ifdef HAVE_POSIX_FALLOCATE + if (limit & 3 && !UT_LIST_GET_NEXT(chain, node)) + { + const uint32_t page_size{space->physical_size()}; + if ((limit * page_size) & 4095) + /* os_file_set_size() extends ROW_FORMAT=COMPRESSED files to + multiples of 4096 bytes. There may be up to 3 pages + (of 1024 bytes) that have not been written out yet. + We must cap the limit to the actual file size. */ + limit= + std::min(limit, + uint32_t(os_file_get_size(node->handle) / page_size)); + } +# endif + res= (*method)(fd, node, start, limit); +#ifdef POSIX_FADV_DONTNEED + std::ignore= posix_fadvise(node->handle, 0, 0, POSIX_FADV_DONTNEED); +#endif + if (res) + break; + fil_node_t *next= UT_LIST_GET_NEXT(chain, node); + if (!next) + break; + const uint32_t size{node->size}; + start+= size; + if (limit >= size) + limit-= size; + else + limit= 0; + node= next; + } +#endif + space->release(); + if (res) + return res; + } + + ut_ad(size > 0); + return int(std::min(size_t{std::numeric_limits::max()}, size - 1)); + } + + /** + Determine the logical time of the backup snapshot. + @return whether the operation failed + */ + bool commit() noexcept + { + log_sys.latch.wr_lock(); + mutex.wr_lock(); + ut_ad(!non_log); + ut_ad(ctx.state == PROCESSING); + ut_ad(ctx.max_first_lsn == LSN_MAX); + if (ctx.last_lsn == 0) + { + log_sys.latch.wr_unlock(); + mutex.wr_unlock(); + return true; + } + ut_ad(ctx.last_lsn == LSN_MAX); + const lsn_t last_lsn{log_sys.get_lsn()}; + lsn_t lsn{log_sys.get_first_lsn()}; + try { + /* Schedule the remaining log for copying */ + queue.emplace_back(lsn); + const lsn_t next_lsn{lsn + log_sys.capacity()}; + if (next_lsn < last_lsn) + queue.emplace_back(lsn= next_lsn); + ctx.max_first_lsn= lsn; + ctx.last_lsn= last_lsn; + } + catch (std::bad_alloc&) { + ctx.last_lsn= 0; + } + log_sys.latch.wr_unlock(); + mutex.wr_unlock(); + /* + Ensure that all data will be available to replicate(). Some + might only reside in log_sys.buf. A durable write is not + necessary, because a system crash will make the backup unusable. + */ + log_write_up_to(last_lsn, false); + return false; + } + + /** + Enable SET GLOBAL innodb_log_archive, innodb_log_file_size. + @param thd current session + */ + ATTRIBUTE_NOINLINE void backup_stop(THD *thd) noexcept + { + ut_ad(!non_log); + ut_ad(ctx.state == PROCESSING); + ctx.state= CLEANUP; /* Unsubscribe from checkpoint_complete() */ + const uint64_t old_size{ctx.old_size}; + mutex.wr_unlock(); + /* Enable SET GLOBAL innodb_log_archive, innodb_log_file_size */ + log_sys.backup_stop(old_size, thd); + } + + /** + Abort the backup. + @param thd current session + @param sink backup worker context + @return 0 (always) + */ + ATTRIBUTE_COLD int abort(THD *thd, const backup_sink &sink) noexcept + { + if (!sink.ha_data) + /* init() must have failed; we have nothing to do */ + return 0; + ut_ad(&ctx == static_cast(sink.ha_data)); + log_sys.latch.wr_lock(); + mutex.wr_lock(); + ut_ad(ctx.state == PROCESSING); + ut_ad(!log_sys.resize_in_progress()); + ut_ad(log_sys.archive); + + /* inform cleanup() that we will clean up */ + ctx.last_hardlink.store(LSN_MAX, std::memory_order_relaxed); + + if (!ctx.old_size) + { + /* + The server was running with innodb_log_archive=ON. + There is nothing to clean up other than the incomplete backup, + which the operator might want to check before deleting. + */ + non_log= 0; + queue.clear(); + } + else + { + log_sys.latch.wr_unlock(); + mutex.wr_unlock(); + std::ignore= log_sys.backup_stop_archiving(nullptr); + log_sys.latch.wr_lock(); + mutex.wr_lock(); + delete_logs(); + } + + backup_stop(thd); + return 0; + } + + /** + Clean up after releasing locks. + @param thd current session + @param sink backup worker context + @return error code + @retval 0 on success + */ + void *finish_start(THD *thd, const backup_sink &sink) noexcept + { + if (!sink.ha_data) + return nullptr; /* init() must have failed; nothing to do */ + void *ret{&ctx}; + ut_ad(ret == static_cast(sink.ha_data)); + mutex.wr_lock(); + ut_ad(!non_log); + if (ctx.state == CLEANUP) + { + /* abort() must have been called already */ + ut_ad(queue.empty()); + mutex.wr_unlock(); + return &ctx; + } + + ut_ad(!log_sys.resize_in_progress()); + ut_ad(log_sys.archive); + ut_ad(ctx.state == PROCESSING); + + const uint64_t old_size{ctx.old_size}; + mutex.wr_unlock(); + + /* Restore innodb_log_archive as it was before init() */ + if (old_size != 0 && log_sys.backup_stop_archiving(thd)) + ret= reinterpret_cast(-1); + + log_sys.latch.wr_lock(); + mutex.wr_lock(); + ut_ad(log_sys.get_first_lsn() >= ctx.max_first_lsn); + + backup_stop(thd); + return ret; + } + + /** + Clean up after finish_start(). + @param target backup target + @param sink backup worker context + @return error code + @retval 0 on success + */ + int finish_end(const backup_target &target, const backup_sink &sink) noexcept + { + ut_ad(!sink.ha_data || &ctx == static_cast(sink.ha_data)); + return sink.ha_data ? ctx.cleanup(target, sink) : 0; + } + + /** + Complete the first checkpoint in a new archive log file. + */ + void checkpoint_complete() noexcept + { + ut_ad(log_sys.latch_have_wr()); + if (ctx.state == PROCESSING) + { + const lsn_t lsn{log_sys.get_first_lsn() - log_sys.capacity()}; + mutex.wr_lock(); + if (ctx.state != PROCESSING); + else if (ctx.last_lsn == LSN_MAX) + { + /* commit() was not invoked yet */ + try { + queue.emplace_back(lsn); + } + catch (std::bad_alloc&) { + ctx.last_lsn= 0; + } + } + else if (lsn > ctx.last_lsn && ctx.old_size) + /* + The server was running with innodb_log_archive=OFF, and this + log file covers some changes after the end of the backup. + Let us delete the file straight away, to keep step() and + delete_logs() simple. + */ + IF_WIN(DeleteFile,unlink)(log_sys.get_archive_path(lsn).c_str()); + mutex.wr_unlock(); + } + } + +private: + /** + Safely start backing up a tablespace file. + @param end array of fil_space_t::BACKUP_BATCH_SIZE block descriptors + @param space tablespace that is being backed up + @param end_page first page not to copy + @return pointer to the new end of the array, of write-fixed blocks + */ + static buf_page_t **backup_batch_start(buf_page_t **end, + fil_space_t *space, uint32_t end_page) + noexcept + { + ut_ad(end_page); + space->backup_start(end_page); + /* Block any writes that might be posted after checking + fil_space_t::backup_page_end(). */ + return innodb_backup_batch_wait(end, space->id, end_page - 1); + } + /** + Stop backing up a tablespace. + @param space tablespace + @param begin first write-fixed block descriptor + @param end end of write-fixed block descriptors + */ + static void backup_batch_stop(fil_space_t *space, + buf_page_t **begin, buf_page_t **end) noexcept + { + space->backup_stop(); + while (begin != end) + { + buf_page_t *b= *begin++; + b->write_unfix_try(); + b->unfix(); + } + } + + /** + Delete unnecessary logs that had been created for backup. + */ + void delete_logs() noexcept + { + ut_ad(log_sys.latch_have_wr()); + ut_ad(ctx.old_size); + ut_ad(non_log <= queue.size()); + + const lsn_t first_lsn{log_sys.get_first_lsn()}; + size_t i{non_log}; + non_log= 0; + while (i < queue.size()) + { + const lsn_t lsn{queue[i++]}; + if (lsn != first_lsn) + IF_WIN(DeleteFile,unlink)(log_sys.get_archive_path(lsn).c_str()); + } + queue.clear(); + } + +#ifdef copy_file_shortcut + /** + Try copy_file_shortcut() to back up a persistent InnoDB data file. + @param dst target file handle + @param node InnoDB data file + @param start the page number at the start of the file + @param limit the size of the file before the doublewrite buffer + @param page_size node->space->physical_size() + @param final_limit the size of the file at init(), or 0 if no dblwr + @param blocks descriptor array of fil_space_t::BACKUP_BATCH_SIZE + @return error code (non-positive) + @retval 0 on success + */ + static int copy_file_shortcut_try(int dst, fil_node_t *node, + uint32_t start, uint32_t limit, + uint32_t page_size, uint32_t final_limit, + buf_page_t **blocks) + noexcept + { +# if 0 + return 1; // work around https://github.com/rr-debugger/rr/issues/4059 +# endif + for (uint32_t page{0};;) + { + while (page < limit) + { + start+= fil_space_t::BACKUP_BATCH_SIZE; + buf_page_t **end= backup_batch_start(blocks, node->space, start); + const uint64_t o{uint64_t{page} * page_size}; + page= std::min(limit, page + fil_space_t::BACKUP_BATCH_SIZE); + /* TODO: avoid copying freed page ranges, or pages that were + allocated after the backup started */ + int err{copy_file_shortcut(node->handle, dst, + o, uint64_t{page} * page_size)}; + backup_batch_stop(node->space, blocks, end); + if (err) + return err; + } + + if (final_limit != 0 && page == buf_dblwr.begin()) + { + /* Copy the rest after the doublewrite buffer. */ + ut_ad(start == page); + limit= final_limit; + start= page+= buf_dblwr.size(); + } + else + return 0; + } + } +#endif + + /** + Back up a persistent InnoDB data file. + @param target backup target directory + @param node InnoDB data file + @param start the page number at the start of the file + @param limit the size of the file at init() + @return error code (non-positive) + @retval 0 on success + */ + static int backup(IF_WIN(const char *,int) target, fil_node_t *node, + uint32_t start, uint32_t limit) noexcept + { + for (bool tried_mkdir{false};;) + { +#ifdef _WIN32 + std::string path{target}; + path.push_back('/'); + path.append(node->name); + HANDLE f= CreateFile(path.c_str(), GENERIC_WRITE, 0, + my_win_file_secattr(), CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (f == INVALID_HANDLE_VALUE) + { + unsigned long err= GetLastError(); + if (err == ERROR_PATH_NOT_FOUND && !tried_mkdir && + node->space->id && !srv_is_undo_tablespace(node->space->id)) + { + tried_mkdir= true; + path.erase(path.rfind('/')); + if (CreateDirectory(path.c_str(), + my_dir_security_attributes.lpSecurityDescriptor + ? &my_dir_security_attributes : nullptr) || + (err= GetLastError()) == ERROR_ALREADY_EXISTS) + continue; + } + + my_osmaperr(err); + goto fail; + } +#else + int f; +# ifdef __APPLE__ + /* aio::synchronous() in another thread may concurrently invoke + pwrite(2) on node->handle. We assume that both pwrite(2) and + fclonefileat(2) are atomic with respect to each other. Should + this assumption be invalid, some data files in the backup may be + corrupted. This corruption can be fixed by either removing this + special handling, or by implementing file-level locking. */ + f= fclonefileat(node->handle, target, node->name, 0); + if (!f) + break; + switch (errno) { + case ENOENT: + goto try_mkdir; + case ENOTSUP: + break; + default: + goto fail; + } +# endif + f= openat(target, node->name, + O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (f < 0) + { + if (errno == ENOENT) + { +# ifdef __APPLE__ + try_mkdir: +# endif + if (!tried_mkdir && node->space->id && + !srv_is_undo_tablespace(node->space->id)) + { + tried_mkdir= true; + const char *sep= strchr(node->name, '/'); + ut_ad(sep); + sep= strchr(sep + 1, '/'); + ut_ad(sep); + std::string dir{node->name, size_t(sep - node->name)}; + if (!mkdirat(target, dir.c_str(), 0777) || errno == EEXIST) + continue; + } + } + goto fail; + } +#endif + const uint32_t page_size{node->space->physical_size()}; + int err{0}; + if (node->size < limit) + limit= node->size; +#ifdef POSIX_FADV_SEQUENTIAL + std::ignore= posix_fadvise(node->handle, 0, off_t(limit) * page_size, + POSIX_FADV_SEQUENTIAL); +#endif + /* + For the system tablespace, a minimum size has been configured + which may be larger than the currently used size. Preserve the + original size. + + For other persistent data files, fil_node_t::read_page0() + expects at least 4 * innodb_page_size bytes. Small + ROW_FORMAT=COMPRESSED files may be zero-filled to this size. + */ + const uint64_t min_size= + std::max(uint64_t{FIL_IBD_FILE_INITIAL_SIZE} << srv_page_size_shift, + uint64_t{node->size} * page_size); + if (uint64_t{limit} * page_size < min_size) + { + /* Expand the target file to the minimum size. */ +#ifdef _WIN32 + LARGE_INTEGER li; + li.QuadPart= min_size; + err= !SetFilePointerEx(f, li, nullptr, FILE_BEGIN) || !SetEndOfFile(f); +#else + err= ftruncate(f, min_size); +#endif + if (err) + limit= 0; + } + + const uint32_t final_limit= + node == fil_system.sys_space->chain.start && + buf_dblwr.begin() + buf_dblwr.size() == buf_dblwr.end() && + limit > buf_dblwr.end() + ? limit : 0; + if (final_limit) + limit= buf_dblwr.begin(); + + buf_page_t *blocks[fil_space_t::BACKUP_BATCH_SIZE]; +#ifdef copy_file_shortcut + err= copy_file_shortcut_try(f, node, start, limit, + page_size, final_limit, blocks); + if (err == 1) +#endif + { +#ifdef copy_file_mmap + const size_t c{size_t{final_limit ? final_limit : limit} * page_size}; + void *p= mmap(nullptr, c, PROT_READ, MAP_SHARED, node->handle, 0); + if (p != MAP_FAILED) + { + for (uint32_t page{0};;) + { + while (page < limit) + { + start+= fil_space_t::BACKUP_BATCH_SIZE; + buf_page_t **end= backup_batch_start(blocks, node->space, start); + const uint64_t o{uint64_t{page} * page_size}; + page= std::min(limit, page + fil_space_t::BACKUP_BATCH_SIZE); + /* TODO: avoid copying freed page ranges, or pages that were + allocated after the backup started */ + err= copy_file_mmap(p, f, o, uint64_t{page} * page_size); + backup_batch_stop(node->space, blocks, end); + if (err) + break; + } + + if (final_limit != 0 && !err && page == buf_dblwr.begin()) + { + /* Copy the rest after the doublewrite buffer. */ + ut_ad(start == page); + limit= final_limit; + start= page+= buf_dblwr.size(); + } + else + break; + } + munmap(p, c); + } + else +#endif + for (uint32_t page{0};;) + { + while (page < limit) + { + start+= fil_space_t::BACKUP_BATCH_SIZE; + buf_page_t **end= backup_batch_start(blocks, node->space, start); + const uint64_t o{uint64_t{page} * page_size}; + page= std::min(limit, page + fil_space_t::BACKUP_BATCH_SIZE); + /* TODO: avoid copying freed page ranges, or pages that were + allocated after the backup started */ + err= + backup::copy(node->handle, f, o, uint64_t{page} * page_size); + backup_batch_stop(node->space, blocks, end); + if (err) + break; + } + + if (final_limit != 0 && !err && page == buf_dblwr.begin()) + { + /* Copy the rest after the doublewrite buffer. */ + ut_ad(start == page); + limit= final_limit; + start= page+= buf_dblwr.size(); + } + else + break; + } + } + + if (IF_WIN(!CloseHandle(f), close(f)) | err) + goto fail; + break; + } + return 0; + fail: + my_error(ER_CANT_CREATE_FILE, MYF(0), node->name, errno); + return -1; + } + + /** + Stream a persistent InnoDB data file. + @param stream backup target stream + @param node InnoDB data file + @param start the page number at the start of the file + @param limit the size of the file at init() + @return error code (non-positive) + @retval 0 on success + */ + static int stream(backup_fd stream, fil_node_t *node, + uint32_t start, uint32_t limit) noexcept + { + const uint32_t page_size{node->space->physical_size()}, + file_size= std::max(std::max(limit, node->size), + (FIL_IBD_FILE_INITIAL_SIZE << srv_page_size_shift) / + page_size); + uint64_t physical_size{uint64_t{limit} * page_size}; + backup_chunk chunk[3]{ + {0, physical_size}, + {uint64_t{file_size} * page_size, 0} + }; + size_t n_chunk= (file_size > limit) * 2; + + if (file_size < limit) + { + limit= file_size; + chunk[0].length= chunk[1].offset; + } + +#ifdef POSIX_FADV_SEQUENTIAL + std::ignore= posix_fadvise(node->handle, 0, chunk[0].length, + POSIX_FADV_SEQUENTIAL); +#endif + + if (node == fil_system.sys_space->chain.start && + buf_dblwr.end() && + buf_dblwr.begin() + buf_dblwr.size() == buf_dblwr.end() && + limit > buf_dblwr.end()) + { + n_chunk= 2 + !!n_chunk; + limit= buf_dblwr.begin(); + physical_size-= uint64_t{buf_dblwr.size()} * page_size; + memmove(chunk + 1, chunk, 2 * sizeof *chunk); + chunk[0].length= uint64_t{limit} * page_size; + chunk[1].offset= uint64_t{buf_dblwr.end()} * page_size; + chunk[1].length-= chunk[1].offset; + } + + int err= backup_stream_start(stream, node->name, 0644, + physical_size, chunk, n_chunk); + if (err) + limit= 0; + + uint32_t page{0}; + + loop: + while (page < limit) + { + buf_page_t *blocks[fil_space_t::BACKUP_BATCH_SIZE], **end= blocks; + { + const uint32_t end_page{start + fil_space_t::BACKUP_BATCH_SIZE}; + end= backup_batch_start(end, node->space, end_page); + start= end_page; + } + uint32_t last{std::min(limit, page + fil_space_t::BACKUP_BATCH_SIZE)}; + /* TODO: avoid copying freed page ranges, or pages that were + allocated after the backup started */ + err= backup::append(node->handle, stream, + uint64_t{page} * page_size, + uint64_t{last} * page_size); + page= last; + backup_batch_stop(node->space, blocks, end); + if (err) + goto fail; + } + + if (limit == buf_dblwr.begin() && n_chunk == 3) + { + /* Copy the rest after the doublewrite buffer. */ + ut_ad(start == page); + start= page+= buf_dblwr.size(); + limit= page + uint32_t(chunk[1].length >> srv_page_size_shift); + goto loop; + } + + if (err) + fail: + my_error(ER_IO_WRITE_ERROR, MYF(0), errno, strerror(errno), + "BACKUP SERVER"); + return err; + } + +private: + /** + Initialize a checkpoint header buffer pointing to the start of the backup. + @param buf checkpoint buffer + @param c offset of the FILE_CHECKPOINT mini-transaction + */ + static void write_checkpoint_buf(uint64_t *buf, uint64_t c) noexcept + { + ut_ad(c >= log_sys.START_OFFSET); + if (log_sys.is_encrypted()) + log_crypt_write_header(reinterpret_cast(buf), true); + buf[4 * log_sys.is_encrypted()]= my_htobe64(c); + } + + /** Write a checkpoint header pointing to the start of the backup. + @param dst target file + @param buf checkpoint header + @return error code + @retval 0 on success */ + static int write_checkpoint(backup_fd dst, const void *buf) noexcept + { +#ifdef _WIN32 + using tpool::pwrite; +#endif + for (ssize_t o= 0, count= 64; count;) + { + ssize_t ret= + pwrite(dst, static_cast(buf) + o, count, o); + if (ret <= 0 || ret > count) + return -1; + o+= ret; + count-= ret; + } + return 0; + } + +public: + /** Maximum length of the configuration string */ + static constexpr size_t CONFIG_SIZE= + sizeof "[server]\n# checkpoint=" + + sizeof "innodb_log_recovery_start=" + + sizeof "innodb_log_recovery_target=\n" + 45 * 3; + + /** Write the configuration parameters for restoring the backup + @param config buffer for configuration string + @param ctx backup context + @return size of the configuration string */ + static size_t write_config_buf(char *config, const context &ctx) + noexcept + { + ut_ad(ctx.last_lsn != LSN_MAX); + return size_t(snprintf(config, CONFIG_SIZE, + "[server]\n# checkpoint=" LSN_PF "\n" + "innodb_log_recovery_start=" LSN_PF "\n" + "innodb_log_recovery_target=" LSN_PF "\n", + ctx.checkpoint, ctx.checkpoint_end_lsn, + ctx.last_lsn)); + } + + /** Write the configuration parameters for restoring the backup + @param target backup target + @param sink backup worker context + @param ctx backup context + @return error code (non-positive) + @retval 0 on success */ + static int write_config(const backup_target &target, + const backup_sink &sink) noexcept + { + char config[CONFIG_SIZE]; + const size_t size + {write_config_buf(config, *static_cast(sink.ha_data))}; + return sink.stream == sink.NO_STREAM + ? backup_config_append(IF_WIN(target.path, target.fd), config, size) + : backup_stream_config(sink.stream, config, size); + } + + /** + Hard-link (copy) or rename (move) or stream an archive log file. + @param lsn The first LSN in the file + @param target backup target + @param sink backup context + @param old lsn < log_sys.get_first_lsn() + @return error code + @retval 0 on success + */ + static int replicate(lsn_t lsn, + const backup_target &target, + const backup_sink &sink, bool old) noexcept + { + ut_ad(log_get_lsn() >= lsn); + const std::string p{log_sys.get_archive_path(lsn)}; + const char *const path= p.c_str(), *basename= strrchr(path, '/'); + if (!basename) + basename= path; + else + basename++; + context &ctx{*static_cast(sink.ha_data)}; + const bool move{old && ctx.old_size}; + uint64_t cp_buf[8]{}; +#ifdef _WIN32 + ut_ad(!target.path == (sink.stream != sink.NO_STREAM)); + std::string b; + const char *destname= nullptr; + if (!target.path) + goto send_file; + b= target.path; + b.push_back('/'); + b.append(basename); + destname= b.c_str(); + unsigned long err; + if (move) + { + if (!MoveFileEx(path, destname, MOVEFILE_COPY_ALLOWED)) + { + fail: + err= GetLastError(); + got_err: + my_osmaperr(err); + if (target.path) + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), path, basename, + errno); + else + my_error(ER_IO_WRITE_ERROR, MYF(ME_ERROR_LOG), + errno, strerror(errno), "BACKUP SERVER"); + return -1; + } + + if (lsn < ctx.checkpoint) + { + if (!SetFileAttributes(destname, FILE_ATTRIBUTE_NORMAL)) + goto fail; + HANDLE dh= CreateFile(destname, GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (dh == INVALID_HANDLE_VALUE) + goto fail; + if (os_file_set_sparse_win32(dh)) + std::ignore= + os_file_punch_hole(dh, 0, log_sys.START_OFFSET + + ((ctx.checkpoint - lsn) & ~4095ULL)); + write_checkpoint_buf(cp_buf, ctx.checkpoint_end_lsn - lsn + + log_sys.START_OFFSET); + int fail= write_checkpoint(dh, cp_buf); + std::ignore= CloseHandle(dh); + if (fail) + goto fail; + } + return 0; + } + else if (CreateHardLink(destname, path, nullptr)) + { + ctx.note_hardlink(lsn); + return 0; + } + + if ((err= GetLastError()) != ERROR_NOT_SAME_DEVICE) + goto got_err; + /* Hard-linking failed. Try copying with the final name. */ + if (target.path) + { + b= target.path; + b.push_back('/'); + b.append(basename); + destname= b.c_str(); + + if (lsn >= ctx.checkpoint && lsn < ctx.max_first_lsn) + { + /* Copy a middle log file entirely. */ + while (!CopyFileEx(path, basename, nullptr, nullptr, nullptr, + COPY_FILE_NO_BUFFERING)) + switch (GetLastError()) { + default: + goto fail; + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + Sleep(10); + } + return 0; + } + } + + send_file: + HANDLE src; + for (;;) + { + src= CreateFile(path, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + my_win_file_secattr(), OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN, + nullptr); + if (src != INVALID_HANDLE_VALUE) + break; + switch (GetLastError()) { + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + Sleep(10); + continue; + } + goto fail; + } + HANDLE dst{sink.stream}; + if (dst == INVALID_HANDLE_VALUE) + { + dst= CreateFile(destname, GENERIC_WRITE, 0, my_win_file_secattr(), + CREATE_NEW, FILE_ATTRIBUTE_NORMAL, nullptr); + if (dst == INVALID_HANDLE_VALUE) + { + std::ignore= CloseHandle(src); + goto fail; + } + } +#else + if (sink.stream != sink.NO_STREAM); + else if (move + ? !renameat(AT_FDCWD, path, target.fd, basename) + : !linkat(AT_FDCWD, path, target.fd, basename, AT_SYMLINK_FOLLOW)) + { + if (!move) + ctx.note_hardlink(lsn); +# ifdef __linux__ + else if (lsn != ctx.first_lsn); + else if (off_t garbage= (ctx.checkpoint - lsn) & ~4095ULL) + /* Best effort to punch a hole to free up some garbage in + the first file. We do not care about failures. */ + if (!fchmodat(target.fd, basename, 0644, 0)) + { + int dst= openat(target.fd, basename, O_RDWR); + if (dst >= 0) + fallocate(dst, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + log_sys.START_OFFSET, garbage); + close(dst); + std::ignore= fchmodat(target.fd, basename, 0444, 0); + } +# endif + return 0; + } + else if (errno != EXDEV) + { + fail: + if (sink.stream == sink.NO_STREAM) + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), path, basename, errno); + else + my_error(ER_IO_WRITE_ERROR, MYF(ME_ERROR_LOG), errno, strerror(errno), + "BACKUP SERVER"); + return -1; + } + + const int src{open(path, O_RDONLY)}; + if (src < 0) + goto fail; + if (move && unlink(path)) + { + close_and_fail: + std::ignore= close(src); + goto fail; + } + int dst{sink.stream}; + if (dst < 0) + { + dst= openat(target.fd, basename, + O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (dst < 0) + goto close_and_fail; + } + int err; +#endif + backup_chunk chunks[3], *chunk{chunks}; + *chunk++= {log_sys.START_OFFSET, ctx.last_lsn - lsn}; + if (lsn < ctx.checkpoint) + { + /* Copy the necessary part of the first log file. */ + ut_ad(lsn == ctx.first_lsn); + write_checkpoint_buf(cp_buf, ctx.checkpoint_end_lsn - lsn + + log_sys.START_OFFSET); + chunk[-1]= {0, 512}; + const lsn_t end= + std::min(ctx.last_lsn, lsn + ctx.first_size - log_sys.START_OFFSET); + *chunk++= + {log_sys.START_OFFSET + ctx.checkpoint - lsn, end - ctx.checkpoint}; + chunk->offset= end - lsn; + goto pad_size; + } + else if (lsn < ctx.max_first_lsn) + { + /* Copy a middle log file entirely. */ +#ifdef _WIN32 + ut_ad(dst == sink.stream); + chunk->offset= os_file_get_size(src); +#else + chunk->offset= uint64_t(lseek(src, 0, SEEK_END)); +# ifdef POSIX_FADV_SEQUENTIAL + std::ignore= posix_fadvise(src, 0, chunk->offset, POSIX_FADV_SEQUENTIAL); +# endif + if (dst != sink.stream) + { + err= copy_entire_file(src, dst); + goto close_dst; + } +#endif + /* Omit the checkpoint header from the stream. */ + chunk[-1].length= chunk->offset - log_sys.START_OFFSET; + goto stream_file; + } + else + { + ut_ad(ctx.max_first_lsn == lsn); + ut_ad(ctx.last_lsn > lsn); + ut_ad(ctx.last_lsn != LSN_MAX); + ut_ad(chunk[-1].length == ctx.last_lsn - lsn); + chunk->offset= chunk[-1].length; + pad_size: +#ifdef POSIX_FADV_SEQUENTIAL + std::ignore= posix_fadvise(src, chunk[-1].offset, chunk[-1].length, + POSIX_FADV_SEQUENTIAL); +#endif + /* Set the logical size of the file. */ + chunk->offset= + std::max(log_sys.FILE_SIZE_MIN, + (chunk->offset + (log_sys.START_OFFSET + 4095)) & + ~4095ULL); + } + + if (dst == sink.stream) + { + stream_file: + chunk++->length= 0; + const backup_chunk &end{chunk[-2]}; + ut_ad(chunk - chunks == 2 || chunk - chunks == 3); + const size_t cp_size{(size_t(chunk - chunks) & 1) << 9}; + err= backup_stream_start(dst, basename, + 0444 | int{lsn == ctx.max_first_lsn} << 7, + end.length + cp_size, chunks, chunk - chunks); + if (!err && cp_size) + err= backup_stream_write(dst, cp_buf, sizeof cp_buf) || + backup_stream_write(dst, field_ref_zero, cp_size - sizeof cp_buf); + if (!err) + { + err= backup_stream_append_async(src, dst, end.offset, + end.offset + end.length) || + backup_stream_zeropad(dst, size_t(end.length)); + } + } + else + { + /* First, extend the file to a valid size. */ +#ifdef _WIN32 + LARGE_INTEGER li; + li.QuadPart= chunk->offset; + err= !SetFilePointerEx(dst, li, nullptr, FILE_BEGIN) || + !SetEndOfFile(dst) || +#else + err= ftruncate(dst, chunk->offset) || +#endif + backup::copy(src, dst, chunk[-1].offset, chunk[-1].offset + + chunk[-1].length) || + (lsn < ctx.checkpoint && write_checkpoint(dst, cp_buf)); +#ifdef _WIN32 + err|= !CloseHandle(dst); +#else + close_dst: + err|= close(dst); +#endif + } + + if (err | IF_WIN(!CloseHandle(src), close(src))) + goto fail; + + return 0; + } +}; + +/** The backup context; protected by log_sys.latch */ +static InnoDB_backup innodb_backup; +} + +bool log_t::backup_start(uint64_t *old_size, THD *thd) noexcept +{ + ut_ad(latch_have_wr()); + ut_ad(!backup); + ut_ad(end_lsn >= last_checkpoint_lsn); + backup= true; + *old_size= 0; + if (archive) + { + if (first_lsn > last_checkpoint_lsn) + { + /* Wait for recovery to be independent from the previous log. */ + mysql_mutex_lock(&buf_pool.flush_list_mutex); + buf_flush_wait(end_lsn, false); + ut_ad(first_lsn <= last_checkpoint_lsn); + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + } + return false; + } + const uint64_t old_file_size{file_size}; + latch.wr_unlock(); + const bool fail{set_archive(true, thd, true)}; + latch.wr_lock(); + if (!fail) + { + *old_size= old_file_size; + return false; + } + ut_ad(backup); + backup= false; + const uint64_t new_file_size{file_size}; + latch.wr_unlock(); + if (old_file_size != new_file_size && old_file_size && + resize_start(old_file_size, thd) == RESIZE_STARTED) + resize_finish(thd); + latch.wr_lock(); + return true; +} + +void log_t::backup_stop(uint64_t old_size, THD *thd) noexcept +{ + ut_ad(latch_have_wr()); + /* We will be invoked with old_size=0 after a failed backup_start(), + or if innodb_log_archive=ON held during a successful backup_start(). */ + ut_ad(!old_size || !resize_in_progress()); + ut_ad(!old_size || backup); + backup= false; + const uint64_t new_size{file_size}; + latch.wr_unlock(); + if (old_size && old_size != new_size && + resize_start(old_size, thd) == RESIZE_STARTED) + resize_finish(thd); +} + +void *innodb_backup_start(THD *thd, const backup_target *, + backup_phase phase, const backup_sink *sink) noexcept +{ + switch (phase) { + case BACKUP_PHASE_PREPARE_START: + if (!fil_system.have_all_spaces) + { + /* To speed up startup, InnoDB does not normally open all + tablespace files that are pointed to by SYS_TABLES. + InnoDB_backup::init() assumes that the information of all + tablespaces is available, including files that had been created + before the server was started, and never opened in the course of + the current server execution. */ + dict_load_tablespaces(nullptr, true); + ut_ad(fil_system.have_all_spaces); + } + return 0; + case BACKUP_PHASE_START: + return innodb_backup.init(thd); + case BACKUP_PHASE_NO_COMMIT: + if (innodb_backup.commit()) + return reinterpret_cast(-1); + /* fall through */ + default: + return sink->ha_data; + case BACKUP_PHASE_FINISH: + return innodb_backup.finish_start(thd, *sink); + } +} + +int innodb_backup_step(THD *, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept +{ + switch (phase) { + case BACKUP_PHASE_START: + case BACKUP_PHASE_NO_COMMIT: + case BACKUP_PHASE_FINISH: + return innodb_backup.step(*target, phase, *sink); + default: + return 0; + } +} + +int innodb_backup_end(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept +{ + switch (phase) { + default: + return 0; + case BACKUP_PHASE_FINISH: + return innodb_backup.finish_end(*target, *sink); + case BACKUP_PHASE_ABORT: + return innodb_backup.abort(thd, *sink); + } +} + +void innodb_backup_checkpoint() noexcept +{ + innodb_backup.checkpoint_complete(); +} diff --git a/storage/innobase/handler/backup_innodb.h b/storage/innobase/handler/backup_innodb.h new file mode 100644 index 0000000000000..e9393038e5a21 --- /dev/null +++ b/storage/innobase/handler/backup_innodb.h @@ -0,0 +1,58 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/** + Start of a BACKUP SERVER phase, + when no innodb_backup_step() or innodb_backup_end() is pending. + @param thd current session + @param target backup target + @param phase BACKUP_PHASE_START, ... (not BACKUP_PHASE_ABORT) + @param sink worker context + @return backup context object to be attached to sink, or nullptr + @retval -1 on failure +*/ +void *innodb_backup_start(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) + noexcept; + +/** + Process a file that was collected in innodb_backup_start(). + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked + @param sink worker context + @return number of files remaining, or negative on error + @retval 0 on completion +*/ +int innodb_backup_step(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept; + +/** + Finish a phase, once all calls for the current phase are completed. + @param thd current sesssion + @param target backup target + @param phase last phase on which backup_start() was successfully invoked, + or BACKUP_PHASE_ABORT or BACKUP_PHASE_FINISH + @param sink worker context + @return error code + @retval 0 on success +*/ +int innodb_backup_end(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept; + +/** + Complete the first checkpoint in a new archive log file. +*/ +void innodb_backup_checkpoint() noexcept; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index d09250515aa27..e4f03b4e8521d 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -152,6 +152,7 @@ MDL_ticket *get_mdl_ticket(TABLE *table) noexcept; #include "ha_innodb.h" #include "i_s.h" +#include "backup_innodb.h" #include #include @@ -4185,6 +4186,9 @@ static int innodb_init(void* p) = innodb_prepare_commit_versioned; innobase_hton->update_optimizer_costs= innobase_update_optimizer_costs; + innobase_hton->backup_start = innodb_backup_start; + innobase_hton->backup_step = innodb_backup_step; + innobase_hton->backup_end = innodb_backup_end; innobase_hton->binlog_init= innodb_binlog_init; innobase_hton->set_binlog_max_size= ibb_set_max_size; innobase_hton->binlog_write_direct_ordered= @@ -18885,39 +18889,7 @@ static void innodb_log_file_size_update(THD *thd, st_mysql_sys_var*, ib_senderrf(thd, IB_LOG_LEVEL_ERROR, ER_CANT_CREATE_HANDLER_FILE); break; case log_t::RESIZE_STARTED: - for (timespec abstime;;) - { - if (thd_kill_level(thd)) - { - log_sys.resize_abort(thd); - break; - } - - set_timespec(abstime, 5); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - lsn_t resizing= log_sys.resize_in_progress(); - if (resizing > buf_pool.get_oldest_modification(0)) - { - buf_pool.page_cleaner_wakeup(true); - my_cond_timedwait(&buf_pool.done_flush_list, - &buf_pool.flush_list_mutex.m_mutex, &abstime); - resizing= log_sys.resize_in_progress(); - } - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - if (!resizing || !log_sys.resize_running(thd)) - break; - log_sys.latch.wr_lock(); - while (resizing > log_sys.get_lsn()) - { - ut_ad(!log_sys.is_mmap()); - /* The server is almost idle. Write dummy FILE_CHECKPOINT records - to ensure that the log resizing will complete. */ - mtr_t mtr{nullptr}; - mtr.start(); - mtr.commit_files(log_sys.last_checkpoint_lsn); - } - log_sys.latch.wr_unlock(); - } + log_sys.resize_finish(thd); } } mysql_mutex_lock(&LOCK_global_system_variables); @@ -19774,7 +19746,9 @@ static void innodb_log_archive_update(THD *thd, st_mysql_sys_var*, { /* MDEV-36828 TODO: On failure, report which other setting conflicted with the request */ + mysql_mutex_unlock(&LOCK_global_system_variables); log_sys.set_archive(*static_cast(save), thd); + mysql_mutex_lock(&LOCK_global_system_variables); } static MYSQL_SYSVAR_BOOL(log_archive, log_sys.archive, @@ -19787,10 +19761,20 @@ static MYSQL_SYSVAR_UINT64_T(log_archive_start, innodb_log_archive_start, "initial value of innodb_lsn_archived; 0=auto-detect", nullptr, nullptr, 0, 0, std::numeric_limits::max(), 0); +static void innodb_log_recovery_start_update(THD *, st_mysql_sys_var*, + void *, const void *save) noexcept +{ + const lsn_t lsn{*static_cast(save)}; + recv_sys.recovery_start= lsn; + if (lsn && log_sys.archive) + log_sys.archived_checkpoint= lsn; +} + static MYSQL_SYSVAR_UINT64_T(log_recovery_start, recv_sys.recovery_start, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + PLUGIN_VAR_RQCMDARG, "LSN to start recovery from (0=automatic)", - nullptr, nullptr, 0, 0, std::numeric_limits::max(), 0); + nullptr, innodb_log_recovery_start_update, + 0, 0, std::numeric_limits::max(), 0); static MYSQL_SYSVAR_UINT64_T(log_recovery_target, recv_sys.rpo, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 3328319fd82b4..33ba10d7acaa3 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -644,6 +644,14 @@ class buf_page_t /** @return whether this block is read fixed */ bool is_read_fixed() const noexcept { return is_read_fixed(state()); } + /** Try to write-fix a block. + @param s expected state() + @return new s (right before potentially setting the write-fix); + a write-fix was acquired if !is_freed(s) && !is_io_fixed(s) holds */ + inline uint32_t write_fix_try(uint32_t s) noexcept; + /** Try to undo a successful write_fix_try(). */ + inline void write_unfix_try() noexcept; + /** @return if this belongs to buf_pool.unzip_LRU */ bool belongs_to_unzip_LRU() const noexcept { return UNIV_LIKELY_NULL(zip.data) && frame; } @@ -653,13 +661,8 @@ class buf_page_t bool is_freed() const noexcept { return is_freed(state()); } bool is_reinit() const { return !(~state() & REINIT); } - void set_reinit(uint32_t prev_state) noexcept - { - ut_ad(prev_state < READ_FIX); - ut_d(const auto s=) zip.fix.fetch_add(REINIT - prev_state); - ut_ad(s > prev_state); - ut_ad(s < prev_state + UNFIXED); - } + /** Mark the block as reinitialized in the file. @see set_freed() */ + void set_reinit() noexcept; uint32_t read_unfix(uint32_t s) noexcept { @@ -671,14 +674,8 @@ class buf_page_t return old_state + (s - READ_FIX); } - void set_freed(uint32_t prev_state, uint32_t count= 0) noexcept - { - ut_ad(lock.is_write_locked()); - ut_ad(prev_state >= UNFIXED); - ut_ad(prev_state < READ_FIX); - ut_d(auto s=) zip.fix.fetch_sub((prev_state & LRU_MASK) - FREED - count); - ut_ad(!((prev_state ^ s) & LRU_MASK)); - } + /** Mark an X-latched block as freed in the tablespace. @see set_reinit() */ + void set_freed() noexcept; inline void set_state(uint32_t s) noexcept; inline void set_corrupt_id() noexcept; diff --git a/storage/innobase/include/buf0dblwr.h b/storage/innobase/include/buf0dblwr.h index 04d9b5485b165..7f70ccd59b887 100644 --- a/storage/innobase/include/buf0dblwr.h +++ b/storage/innobase/include/buf0dblwr.h @@ -177,6 +177,13 @@ class buf_dblwr_t (id >= block2 && id < block2 + block_size); } + /** the first page number */ + uint32_t begin() const noexcept { return block1.page_no(); } + /** @return the first page number after the doublewrite buffer */ + uint32_t end() const noexcept { return block2.page_no() + block_size; } + /** the size of the doublewrite buffer, in pages */ + uint32_t size() const noexcept { return 2 * block_size; } + /** Wait for flush_buffered_writes() to be fully completed */ void wait_flush_buffered_writes() noexcept { diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 67fa1bfd026f0..7d5467ecaa077 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -408,6 +408,9 @@ struct fil_space_t final /** Whether any corruption of this tablespace has been reported */ mutable std::atomic_flag is_corrupted= ATOMIC_FLAG_INIT; + /** first page number that is not yet being backed up, or 0 */ + std::atomic backup_end{0}; + public: /** mutex to protect freed_ranges and last_freed_lsn */ std::mutex freed_range_mutex; @@ -418,9 +421,11 @@ struct fil_space_t final /** LSN of freeing last page; protected by freed_range_mutex */ lsn_t last_freed_lsn= 0; - /** LSN of undo tablespace creation or 0; protected by latch */ - lsn_t create_lsn= 0; public: + /** LSN of tablespace creation or undo tablespace reinitialization; + protected by fil_system.mutex and (is_stopped() or log_sys.latch) */ + Atomic_relaxed create_lsn{0}; + /** @return whether this is the temporary tablespace */ bool is_temporary() const noexcept { return UNIV_UNLIKELY(id == SRV_TMP_SPACE_ID); } @@ -434,12 +439,6 @@ struct fil_space_t final /** @return whether a page has been freed */ inline bool is_freed(uint32_t page) noexcept; - /** Set create_lsn. */ - inline void set_create_lsn(lsn_t lsn) noexcept; - - /** @return the latest tablespace rebuild LSN, or 0 */ - lsn_t get_create_lsn() const noexcept { return create_lsn; } - /** Apply freed_ranges to the file. @param writable whether the file is writable @return number of pages written or hole-punched */ @@ -1058,6 +1057,19 @@ struct fil_space_t final VALIDATE_IMPORT }; + /** Note that we backing up some pages of the underlying files. + @param last_page the last page that is being backed up (0=stop backup) */ + void backup_start(uint32_t last_page) noexcept + { backup_end.store(last_page, std::memory_order_release); } + /** Note that we are not currently backing up the underlying files. */ + void backup_stop() noexcept { backup_start(0); } + /** @return the first page number that is not being backed up */ + uint32_t backup_page_end() const noexcept + { return backup_end.load(std::memory_order_acquire); } + + /** The size of a backup::copy() batch in pages */ + static constexpr uint32_t BACKUP_BATCH_SIZE{64}; + /** Update the data structures on write completion */ void complete_write() noexcept; @@ -1463,6 +1475,8 @@ struct fil_system_t my_bool buffered; /** whether fdatasync() is needed on data files */ Atomic_relaxed need_unflushed_spaces; + /** whether dict_load_tablespaces(nullptr, true) is unnecessary */ + Atomic_relaxed have_all_spaces; /** Try to enable or disable write-through of data files */ void set_write_through(bool write_through); diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index bb5456f056e2e..eada4ddf19df6 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -221,6 +221,8 @@ struct log_t /** whether !archive log records may have been written with get_sequence_bit()==0 */ bool circular_recovery_from_sequence_bit_0:1; + /** whether we are between backup_start() and backup_stop() */ + bool backup:1; public: /** the default value of log_mmap */ static constexpr bool log_mmap_default= @@ -288,6 +290,8 @@ struct log_t Atomic_relaxed last_checkpoint_lsn; /** The log writer (protected by latch.wr_lock()) */ lsn_t (*writer)() noexcept; + /** the earliest available checkpoint; protected by latch.wr_lock() */ + lsn_t archived_checkpoint; /** end_lsn of the first available checkpoint, or 0; protected by latch.wr_lock() */ lsn_t archived_lsn; @@ -369,11 +373,24 @@ struct log_t RESIZE_NO_CHANGE, RESIZE_IN_PROGRESS, RESIZE_STARTED, RESIZE_FAILED }; +private: /** Start resizing the log and release the exclusive latch. + @param size requested new file_size + @param thd the current thread identifier + @param backup whether the caller is backup_start() or backup_stop() + @return whether the resizing was started successfully */ + resize_start_status resize_start(uint64_t size, void *thd, bool backup) + noexcept; +public: + /** Start resizing the log. @param size requested new file_size @param thd the current thread identifier @return whether the resizing was started successfully */ - resize_start_status resize_start(os_offset_t size, void *thd) noexcept; + resize_start_status resize_start(uint64_t size, void *thd) noexcept + { return resize_start(size, thd, false); } + + /** Wait for the completion of resize_start() == RESIZE_STARTED */ + void resize_finish(THD *thd) noexcept; /** Abort a resize_start() that we started. @param thd thread identifier that had been passed to resize_start() */ @@ -397,10 +414,37 @@ struct log_t resize_write_low(lsn, end, len, seq); } +private: + /** SET GLOBAL innodb_log_archive, or start/stop BACKUP SERVER + @param archive the new value of innodb_log_archive + @param thd SQL connection + @param backup whether the caller is backup_start() or backup_stop() + @return whether the operation failed */ + bool set_archive(my_bool archive, THD *thd, bool backup) noexcept; +public: /** SET GLOBAL innodb_log_archive @param archive the new value of innodb_log_archive - @param thd SQL connection */ - void set_archive(my_bool archive, THD *thd) noexcept; + @param thd SQL connection + @return whether the operation failed */ + bool set_archive(my_bool archive, THD *thd) noexcept + { return set_archive(archive, thd, false); } + + /** Start BACKUP SERVER. + @param old_size the old file_size, or 0 on failure or when + already running innodb_log_archive=ON + @param thd SQL connection + @return whether the operation failed */ + bool backup_start(uint64_t *old_size, THD *thd) noexcept; + /** Stop log archiving in BACKUP SERVER clean-up + @param thd SQL connection + @return whether the operation failed */ + bool backup_stop_archiving(THD *thd) noexcept + { return set_archive(false, thd, true); } + + /** Stop BACKUP SERVER. + @param old_size the value returned by backup_start() + @param thd SQL connection */ + void backup_stop(uint64_t old_size, THD *thd) noexcept; private: /** Replicate a write to the log. @@ -700,6 +744,18 @@ struct log_t /** @return the first LSN of the log file */ lsn_t get_first_lsn() const noexcept { return first_lsn; } + /** + Determine the latest checkpoint. + @param end LSN leading to the FILE_CHECKPOINT record + @return the latest checkpoint LSN + */ + lsn_t get_latest_checkpoint(lsn_t &end) const noexcept + { + ut_ad(latch_have_any()); + end= end_lsn; + return last_checkpoint_lsn; + } + /** Set the recovered checkpoint. @param lsn log sequence number of the checkpoint @param end_lsn LSN passed to write_checkpoint() diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 5f4a59b41fc3a..7831b6b1283bf 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -635,7 +635,7 @@ void log_t::set_buffered(bool buffered) noexcept } #endif - /** Try to enable or disable durable writes (update log_write_through) */ +/** Try to enable or disable durable writes (update log_write_through) */ void log_t::set_write_through(bool write_through) { if (is_mmap() || high_level_read_only || recv_sys.rpo) @@ -764,15 +764,18 @@ void log_t::header_rewrite(my_bool archive) noexcept /** SET GLOBAL innodb_log_archive @param archive the new value of innodb_log_archive -@param thd SQL connection */ -void log_t::set_archive(my_bool archive, THD *thd) noexcept +@param thd SQL connection +@param backup whether the caller is backup_start() or backup_stop() +@return whether the operation failed */ +bool log_t::set_archive(my_bool archive, THD *thd, bool backup) noexcept { if (UNIV_UNLIKELY(recv_sys.rpo != 0)) { my_error(ER_INNODB_READ_ONLY, MYF(0)); - return; + return true; } + bool fail= false; thd_wait_begin(thd, THD_WAIT_DISKIO); tpool::tpool_wait_begin(); lsn_t wait_lsn; @@ -786,12 +789,20 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept my_printf_error(ER_WRONG_USAGE, "SET GLOBAL innodb_log_file_size is in progress", MYF(0)); + fail: + fail= true; + wait_lsn= 0; break; } if (archive == this->archive) break; - if (thd_kill_level(thd)) - break; + if ((!backup || archive) && thd_kill_level(thd)) + goto fail; + if (!backup && this->backup) + { + my_printf_error(ER_WRONG_USAGE, "BACKUP SERVER is in progress", MYF(0)); + goto fail; + } if (resize_log.is_opened()) { @@ -928,7 +939,7 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept if (!log.is_opened()) { my_error(ER_ERROR_ON_READ, MYF(0), old_name, errno); - break; + goto fail; } } #endif @@ -953,7 +964,7 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept { my_error(ER_ERROR_ON_RENAME, MYF(0), old_name, new_name, my_errno); first_lsn= old_first_lsn; - break; + goto fail; } if (archive) @@ -985,14 +996,16 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept thd_wait_end(thd); if (wait_lsn) mtr_flush_ahead(wait_lsn); + return fail; } -/** Start resizing the log and release the exclusive latch. -@param size requested new file_size -@param thd the current thread identifier +/** Start resizing the log. +@param size requested new file_size +@param thd the current thread identifier +@param backup whether the caller is backup_start() or backup_stop() @return whether the resizing was started successfully */ -log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) - noexcept +log_t::resize_start_status log_t::resize_start(uint64_t size, void *thd, + bool backup) noexcept { ut_ad(size >= 4U << 20); ut_ad(!(size & 4095)); @@ -1023,6 +1036,9 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) resize_target= size; } } + else if (!backup && this->backup) + /* backup_start() or backup_stop() is running */ + status= RESIZE_FAILED; else { lsn_t start_lsn; @@ -1124,6 +1140,44 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) return status; } +/** Wait for the completion of resize_start() == RESIZE_STARTED */ +void log_t::resize_finish(THD *thd) noexcept +{ + for (timespec abstime;;) + { + if (thd_kill_level(thd)) + { + resize_abort(thd); + break; + } + + set_timespec(abstime, 5); + mysql_mutex_lock(&buf_pool.flush_list_mutex); + lsn_t resizing= resize_in_progress(); + if (resizing > buf_pool.get_oldest_modification(0)) + { + buf_pool.page_cleaner_wakeup(true); + my_cond_timedwait(&buf_pool.done_flush_list, + &buf_pool.flush_list_mutex.m_mutex, &abstime); + resizing= resize_in_progress(); + } + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + if (!resizing || !resize_running(thd)) + break; + latch.wr_lock(); + while (resizing > get_lsn()) + { + ut_ad(!is_mmap()); + /* The server is almost idle. Write dummy FILE_CHECKPOINT records + to ensure that the log resizing will complete. */ + mtr_t mtr{nullptr}; + mtr.start(); + mtr.commit_files(last_checkpoint_lsn); + } + latch.wr_unlock(); + } +} + /** Abort a resize_start() that we started. */ void log_t::resize_abort(void *thd) noexcept { diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 690128f5685c9..b1136c8fd5758 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1260,6 +1260,7 @@ ATTRIBUTE_COLD static void fil_delete_apply(uint32_t id, const char *name) if (fil_ibd_load(id, name, space) == FIL_LOAD_OK) { ut_ad(space); + deferred_spaces.remove(id); fil_delete_apply(space); return; } @@ -1295,10 +1296,6 @@ static void fil_name_process(const char *name, ulint len, uint32_t space_id, file_name_t& f = p.first->second; ut_ad(!f.space || f.space->id == space_id); auto d = deferred_spaces.find(space_id); - /* deferred_spaces must be a proper subset of recv_spaces. - Both here and in recv_validate_tablespace(), deferred_spaces.add() - may only be invoked for entries that exist in recv_spaces. */ - ut_ad(!d || !p.second); if (deleted) { /* Got FILE_DELETE */ @@ -2159,6 +2156,7 @@ dberr_t recv_sys_t::find_checkpoint() memset_aligned<4096>(const_cast(field_ref_zero), 0, 4096); /* Mark the redo log for upgrading. */ lsn= file_checkpoint= log_sys.last_checkpoint_lsn; + log_sys.archived_checkpoint= lsn; log_sys.set_recovered_lsn(lsn); if (rpo && rpo != lsn) { @@ -2237,7 +2235,8 @@ dberr_t recv_sys_t::find_checkpoint() log_sys.set_recovered_checkpoint(checkpoint_lsn, lsn= end_lsn, field == log_t::CHECKPOINT_1); } - if (!log_sys.last_checkpoint_lsn) + log_sys.archived_checkpoint= log_sys.last_checkpoint_lsn; + if (!log_sys.archived_checkpoint) goto got_no_checkpoint; else if (!log_sys.archived_lsn) log_sys.archived_lsn= lsn; @@ -4228,7 +4227,7 @@ static buf_block_t *recv_recover_page(buf_block_t *block, mtr_t &mtr, /* There have been no operations that modify the page. Any buffered changes will be merged in ibuf_upgrade(). */ ut_ad(!mtr.has_modifications()); - block->page.set_freed(block->page.state()); + block->page.set_freed(); } /* Make sure that committing mtr does not change the modification diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 21e4d1ae1c9ce..8afeacd1f3752 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -313,7 +313,8 @@ void mtr_t::release_unlogged() buf_block_t *block= static_cast(slot.object); ut_d(const auto s=) block->page.unfix(); ut_ad(s >= buf_page_t::FREED); - ut_ad(s < buf_page_t::READ_FIX); + /* A fake "write fix" of InnoDB_backup may exist for a latched page */ + ut_ad(!buf_page_t::is_read_fixed(s)); if (slot.type & MTR_MEMO_MODIFY) { @@ -440,7 +441,8 @@ void mtr_t::commit_log(mtr_t *mtr, std::pair lsns) noexcept ut_ad(b->page.id() < end_page_id); ut_d(const auto s= b->page.state()); ut_ad(s > buf_page_t::FREED); - ut_ad(s < buf_page_t::READ_FIX); + /* A fake "write fix" of InnoDB_backup may exist for a latched page */ + ut_ad(!buf_page_t::is_read_fixed(s)); ut_ad(mach_read_from_8(b->page.frame + FIL_PAGE_LSN) <= mtr->m_commit_lsn); mach_write_to_8(b->page.frame + FIL_PAGE_LSN, mtr->m_commit_lsn); @@ -492,7 +494,13 @@ void mtr_t::commit_log(mtr_t *mtr, std::pair lsns) noexcept ut_ad(bpage->oldest_modification() < mtr->m_commit_lsn); ut_ad(bpage->id() < end_page_id); ut_ad(s >= buf_page_t::FREED); - ut_ad(s < buf_page_t::READ_FIX); + /* If a thread is executing between + InnoDB_backup::backup_batch_start() and + InnoDB_backup::backup_batch_stop() for this page, a fake + "write fix" may exist. We are free to modify the page in the + buffer pool, but buf_page_t::flush() will refuse to write it + to the file system. */ + ut_ad(!buf_page_t::is_read_fixed(s)); ut_ad(mach_read_from_8(bpage->frame + FIL_PAGE_LSN) <= mtr->m_commit_lsn); mach_write_to_8(bpage->frame + FIL_PAGE_LSN, mtr->m_commit_lsn); @@ -595,12 +603,34 @@ void mtr_t::rollback_to_savepoint(ulint begin, ulint end) m_memo.erase(m_memo.begin() + begin, m_memo.begin() + end); } -/** Set create_lsn. */ -inline void fil_space_t::set_create_lsn(lsn_t lsn) noexcept +/** Mark an X-latched block as freed in the tablespace. @see set_reinit() */ +void buf_page_t::set_freed() noexcept { - /* Concurrent log_checkpoint_low() must be impossible. */ - ut_ad(latch.have_wr()); - create_lsn= lsn; + /* + Ensure that lock.x_lock() is being held (hopefully, by us). + This blocks a concurrent flush(), protected by lock.u_lock(). + */ + ut_ad(lock.is_write_locked()); + uint32_t s{state()}; + do + { + ut_ad(s >= UNFIXED); + /* + InnoDB_backup::backup_batch_start() may set fix() and + write_fix_try() on any dirty block to prevent flush() from + writing changes back to the file system. We may safely mark such + blocks as FREED; write_unfix_try() will account for that. + */ + ut_ad(s > WRITE_FIX || s < READ_FIX); + } + /* + fetch_sub() is not safe here, because this may run concurrently + with write_fix_try() or write_unfix_try(). + */ + while (!zip.fix.compare_exchange_weak(s, FREED + (s & ~LRU_MASK), + std::memory_order_acquire, + std::memory_order_relaxed) && + !is_freed(s)); } /** Commit a mini-transaction that is shrinking a tablespace. @@ -635,7 +665,7 @@ void mtr_t::commit_shrink(fil_space_t &space, uint32_t size) if (space.id == TRX_SYS_SPACE) srv_sys_space.set_last_file_size(file->size); else - space.set_create_lsn(m_commit_lsn); + space.create_lsn= m_commit_lsn; mysql_mutex_unlock(&fil_system.mutex); @@ -674,7 +704,8 @@ void mtr_t::commit_shrink(fil_space_t &space, uint32_t size) const page_id_t id{b->page.id()}; const auto s= b->page.state(); ut_ad(s > buf_page_t::FREED); - ut_ad(s < buf_page_t::READ_FIX); + /* A fake "write fix" of InnoDB_backup may exist for a latched page */ + ut_ad(!buf_page_t::is_read_fixed(s)); ut_ad(b->page.frame); ut_ad(mach_read_from_8(b->page.frame + FIL_PAGE_LSN) <= m_commit_lsn); ut_ad(!b->page.zip.data); // we no not shrink ROW_FORMAT=COMPRESSED @@ -695,7 +726,7 @@ void mtr_t::commit_shrink(fil_space_t &space, uint32_t size) { ut_ad(id.space() == high.space()); if (s >= buf_page_t::UNFIXED) - b->page.set_freed(s); + b->page.set_freed(); if (b->page.oldest_modification() > 1) b->page.reset_oldest_modification(); slot.type= mtr_memo_type_t(slot.type & ~MTR_MEMO_MODIFY); @@ -1542,7 +1573,8 @@ buf_block_t *mtr_t::page_lock(buf_block_t *block, ulint rw_latch) noexcept case RW_SX_LATCH: fix_type= MTR_MEMO_PAGE_SX_FIX; block->page.lock.u_lock(); - ut_ad(!block->page.is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist */ + ut_ad(!block->page.is_read_fixed()); break; default: ut_ad(rw_latch == RW_X_LATCH); @@ -1553,7 +1585,8 @@ buf_block_t *mtr_t::page_lock(buf_block_t *block, ulint rw_latch) noexcept page_lock_upgrade(*block); return block; } - ut_ad(!block->page.is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist */ + ut_ad(!block->page.is_read_fixed()); } done: @@ -1588,11 +1621,13 @@ void mtr_t::upgrade_buffer_fix(ulint savepoint, rw_lock_type_t rw_latch) break; case RW_SX_LATCH: block->page.lock.u_lock(); - ut_ad(!block->page.is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist */ + ut_ad(!block->page.is_read_fixed()); break; case RW_X_LATCH: block->page.lock.x_lock(); - ut_ad(!block->page.is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist */ + ut_ad(!block->page.is_read_fixed()); } ut_ad(page_id_t(page_get_space_id(block->page.frame), @@ -1784,7 +1819,7 @@ void mtr_t::init(buf_block_t *b) m_freed_space= nullptr; } - b->page.set_reinit(b->page.state() & buf_page_t::LRU_MASK); + b->page.set_reinit(); if (!is_logged()) return; @@ -1849,7 +1884,7 @@ void mtr_t::free(const fil_space_t &space, uint32_t offset) if (block->index) btr_search_drop_page_hash_index(block, nullptr); #endif /* BTR_CUR_HASH_ADAPT */ - block->page.set_freed(block->page.state()); + block->page.set_freed(); } } diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 6494c5e21b96e..4ef86095f78a3 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -2017,7 +2017,15 @@ os_file_create_func( ); DWORD create_flag = OPEN_EXISTING; - DWORD share_mode = read_only + /* BACKUP SERVER may invoke CreateHardLink() on a log file that + may concurrently be written to. This is why we must allow + FILE_SHARE_WRITE. This has the side effect that multiple InnoDB + instances may be concurrently started on the same log file. + However, InnoDB will not write any log before it has successfully + opened data files. As long as the multiple instances are also + opening the same InnoDB data files (such as the system tablespace), + they should fail to start up concurrently. */ + DWORD share_mode = read_only || type == OS_LOG_FILE ? FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE : FILE_SHARE_READ | FILE_SHARE_DELETE; diff --git a/storage/maria/CMakeLists.txt b/storage/maria/CMakeLists.txt index 9d528f980dc5b..43b950f05d99a 100644 --- a/storage/maria/CMakeLists.txt +++ b/storage/maria/CMakeLists.txt @@ -45,6 +45,7 @@ SET(ARIA_SOURCES ma_init.c ma_open.c ma_extra.c ma_info.c ma_rkey.c ha_maria.h maria_def.h ma_recovery_util.c ma_servicethread.c ma_norec.c ma_crypt.c ma_backup.c + ma_backup_server.c ma_backup_server.h ) IF(APPLE) diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 8f5e47daea728..558fc83d8c3f0 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -23,6 +23,7 @@ #include #include #include "ha_maria.h" +#include "ma_backup_server.h" #include "trnman_public.h" #include "trnman.h" @@ -3728,7 +3729,29 @@ void maria_end_backup() translog_enable_purge(); } +/* A C++ function pointer compatible wrapper of a C function */ +static void * +aria_backup_start_wrap(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) +{ + return aria_backup_start(thd, target, phase, sink); +} +/* A C++ function pointer compatible wrapper of a C function */ +static int +aria_backup_step_wrap(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) +{ + return aria_backup_step(thd, target, phase, sink); +} + +/* A C++ function pointer compatible wrapper of a C function */ +static int +aria_backup_end_wrap(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) +{ + return aria_backup_end(thd, target, phase, sink); +} #define SHOW_MSG_LEN (FN_REFLEN + 20) /** @@ -3942,6 +3965,9 @@ static int ha_maria_init(void *p) maria_hton->prepare_for_backup= maria_prepare_for_backup; maria_hton->end_backup= maria_end_backup; maria_hton->update_optimizer_costs= aria_update_optimizer_costs; + maria_hton->backup_start= aria_backup_start_wrap; + maria_hton->backup_step= aria_backup_step_wrap; + maria_hton->backup_end= aria_backup_end_wrap; /* TODO: decide if we support Maria being used for log tables */ maria_hton->flags= (HTON_CAN_RECREATE | HTON_SUPPORT_LOG_TABLES | diff --git a/storage/maria/ma_backup_server.c b/storage/maria/ma_backup_server.c new file mode 100644 index 0000000000000..2fccf58e87be8 --- /dev/null +++ b/storage/maria/ma_backup_server.c @@ -0,0 +1,878 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "maria_def.h" +#include "ma_backup_server.h" +#include "mysqld_error.h" + +#if 1 /* can't #include "sql/table.h" because it is C++ */ +# define tmp_file_prefix "#sql" +# define tmp_file_prefix_length 4 +#endif + +#ifndef _WIN32 +# include +# include +#endif + +/** + Report that a directory cannot be read. + @param name directory name +*/ +ATTRIBUTE_COLD ATTRIBUTE_NOINLINE static void dir_error(const char *name) +{ +#ifdef _WIN32 + my_osmaperr(GetLastError()); +#endif + my_error(ER_CANT_READ_DIR, MYF(0), name, errno); +} + +/** + Predicate for checking if a file must be included in the backup. + @param file_name file base name to check + @param len strlen(file_name) +*/ +typedef int (*name_predicate)(const char *file_name, size_t len); + +/** + Determine if a non-Aria file may be backed up. + @param file_name candidate file name + @param len strlen(file_name) + @retval FALSE if the file must be excluded + @retval TRUE if the file may be included +*/ +static int is_db_file(const char *file_name, size_t len) +{ + uint32_t suffix; + assert(len >= 4); + memcpy(&suffix, file_name + len - 4, 4); + switch (suffix) { +#ifdef WORDS_BIGENDIAN + case 0x2e41524d: /* .ARM ENGINE=ARCHIVE metadata */ + case 0x2e41525a: /* .ARZ ENGINE=ARCHIVE compressed data */ + case 0x2e43534d: /* .CSM ENGINE=CSV metadata */ + case 0x2e435356: /* .CSV ENGINE=CSV data ("comma separated values") */ + case 0x2e4d5247: /* .MRG ENGINE=MRG_MyISAM */ + case 0x2e4d5944: /* .MYD ENGINE=MyISAM data heap */ + case 0x2e4d5949: /* .MYI ENGINE=MyISAM indexes */ + case 0x2e545247: /* .TRG trigger definition */ + case 0x2e54524e: /* .TRN trigger name */ + case 0x2e66726d: /* .frm form (SHOW CREATE TABLE) */ + case 0x2e706172: /* .par PARTITION metadata */ +#else + case 0x4d52412e: /* .ARM ENGINE=ARCHIVE metadata */ + case 0x5a52412e: /* .ARZ ENGINE=ARCHIVE compressed data */ + case 0x4d53432e: /* .CSM ENGINE=CSV metadata */ + case 0x5653432e: /* .CSV ENGINE=CSV data ("comma separated values") */ + case 0x47524d2e: /* .MRG ENGINE=MRG_MyISAM */ + case 0x44594d2e: /* .MYD ENGINE=MyISAM data heap */ + case 0x49594d2e: /* .MYI ENGINE=MyISAM indexes */ + case 0x4752542e: /* .TRG trigger definition */ + case 0x4e52542e: /* .TRN trigger name */ + case 0x6d72662e: /* .frm form (SHOW CREATE TABLE) */ + case 0x7261702e: /* .par PARTITION metadata */ +#endif + return TRUE; + } + return len == 6 && !memcmp(file_name, C_STRING_WITH_LEN("db.opt")); +} + +/** + Determine if a file is an ENGINE=Aria file. + @param file_name candidate file name + @param len strlen(file_name) + @retval FALSE if the file must be excluded + @retval TRUE if the file may be included +*/ +static int is_ma_file(const char *file_name, size_t len) +{ + uint32_t suffix; + assert(len >= 4); + memcpy(&suffix, file_name + len - 4, 4); + switch (suffix) { +#ifdef WORDS_BIGENDIAN + case 0x2e4d4144: /* .MAD ENGINE=Aria data heap */ + case 0x2e4d4149: /* .MAI ENGINE=Aria indexes */ +#else + case 0x44414d2e: /* .MAD ENGINE=Aria data heap */ + case 0x49414d2e: /* .MAI ENGINE=Aria indexes */ +#endif + return TRUE; + } + return FALSE; +} + +/** Backup status */ +enum Aria_backup_status { + /** No error yet */ + BACKUP_OK, + /** A failure has occurred */ + BACKUP_FAIL, + /** translog_lock() has been called */ + TRANSLOG_LOCKED, + /** translog_lock() and translog_disable_purge() have been called */ + TRANSLOG_PURGE_DISABLED +}; + +/** Backup state */ +struct Aria_backup +{ +#ifndef _WIN32 + /** directory stream */ + DIR *dir; + /** the readdir(dir) result for which subdir was opened */ + const struct dirent *d; + /** subdirectory stream, or NULL if iterating to next entry in dir */ + DIR *subdir; +#else + /** directory iterator */ + HANDLE dir; + /** subdirectory iterator, or INVALID_HANDLE_VALUE */ + HANDLE subdir; +#endif + /** status */ + enum Aria_backup_status status; + /** mutex protecting d, subdir, status in concurrent aria_backup_step() */ + pthread_mutex_t mutex; +#ifdef _WIN32 + /** FindFirstFileA()/FindNextFile() buffer for dir */ + WIN32_FIND_DATAA d; + /** FindFirstFileA()/FindNextFile() buffer for subdir */ + WIN32_FIND_DATAA sd; +#endif +}; + +/** + Create a subdirectory unless we are streaming or it pre-exists. + @param target BACKUP SERVER target + @param name base name of subdirectory to create + @return error code (also errno will be set) + @retval 0 on success (errno might not be touched) +*/ +static int aria_backup_mkdir(const struct backup_target *target, + const char *name) +{ +#ifdef _WIN32 + if (!target->path) + return 0; + int ret= snprintf(NULL, 0, "%s/%s", target->path, name) + 1; + char *path= malloc(ret); + if (!path) + { + my_error(ER_TOO_LONG_IDENT, MYF(0), name); + return 1; + } + snprintf(path, ret, "%s/%s", target->path, name); + if (CreateDirectory(path, NULL)) + ret= 0; + else + { + const DWORD err= GetLastError(); + ret= err != ERROR_ALREADY_EXISTS; + if (ret) + { + my_osmaperr(err); + my_error(ER_CANT_CREATE_FILE, MYF(0), path, errno); + } + } + free(path); + return ret; +#else + if (target->fd == -1 || likely(!mkdirat(target->fd, name, 0777)) || + errno == EEXIST) + return 0; + my_error(ER_CANT_CREATE_FILE, MYF(0), name, errno); + return 1; +#endif +} + +#ifndef _WIN32 +/** + Copy a file. + @param target BACKUP SERVER target (possibly, a directory) + @param sink per-thread context (possibly, a stream to write to) + @param dirfd source directory + @param path file name + @return error code (also errno will be set) + @retval 0 on success (errno might not be touched) +*/ +static int aria_backup_file(const struct backup_target *target, + const struct backup_sink *sink, + int dirfd, const char *path) +{ + int ret= -1, src= openat(dirfd, path, O_RDONLY), dst= sink->stream; + if (src < 0) + { + my_error(ER_CANT_OPEN_FILE, MYF(0), path, errno); + return ret; + } + if (dst < 0) + { + dst= openat(target->fd, path, O_CREAT | O_EXCL | O_WRONLY, 0666); + if (dst < 0) + my_error(ER_CANT_CREATE_FILE, MYF(0), path, errno); + else + { + ret= copy_entire_file(src, dst) | close(dst); + if (ret) + write_error: + my_error(ER_ERROR_ON_WRITE, MYF(0), path, errno); + } + } + else + { + uint64_t end= (uint64_t) lseek(src, 0, SEEK_END); + ret= backup_stream_start(dst, path, 0644, end, NULL, 0) || + backup_stream_append_plain(src, dst, 0, end) || + backup_stream_zeropad(dst, (size_t) end); + if (ret) + goto write_error; + } + close(src); + return ret; +} +#else +/** + Copy a file. + @param target BACKUP SERVER target (possibly, a directory) + @param sink per-thread context (possibly, a stream to write to) + @param path file name + @param dir_prefix length of the path prefix to omit from the backup + @return error code (also errno will be set) + @retval 0 on success (errno might not be touched) +*/ +static int aria_backup_file(const struct backup_target *target, + const struct backup_sink *sink, + const char *path, size_t dir_prefix) +{ + int ret= -1; + if (sink->stream == INVALID_HANDLE_VALUE) + { + int len= snprintf(NULL, 0, "%s/%s", target->path, path + dir_prefix) + 1; + char *dstpath= malloc(len); + if (!dstpath) + my_error(ER_TOO_LONG_IDENT, MYF(0), path + dir_prefix); + else + { + snprintf(dstpath, len, "%s/%s", target->path, path + dir_prefix); + while (!CopyFileEx(path, dstpath, NULL, NULL, NULL, + COPY_FILE_NO_BUFFERING)) + switch (GetLastError()) { + default: + my_osmaperr(GetLastError()); + my_error(ER_CANT_CREATE_FILE, MYF(0), dstpath, errno); + goto done; + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + Sleep(10); + } + ret= 0; + done: + free(dstpath); + } + } + else + { + LARGE_INTEGER li; + HANDLE src, dst= sink->stream; + for (;;) + { + src= CreateFile(path, GENERIC_READ, + FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, + my_win_file_secattr(), OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, NULL); + if (src != INVALID_HANDLE_VALUE) + break; + switch (GetLastError()) { + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + Sleep(10); + continue; + } + + my_osmaperr(GetLastError()); + my_error(ER_FILE_NOT_FOUND, MYF(ME_ERROR_LOG), path, errno); + return ret; + } + + ret= !GetFileSizeEx(src, &li) || + backup_stream_start(dst, path + dir_prefix, 0644, li.QuadPart, + NULL, 0) || + backup_stream_append_plain(src, dst, 0, li.QuadPart) || + backup_stream_zeropad(dst, (size_t) li.QuadPart); + (void) CloseHandle(src); + + if (ret) + { + my_osmaperr(GetLastError()); + my_error(ER_ERROR_ON_WRITE, MYF(0), path + dir_prefix, errno); + } + } + return ret; +} +#endif + +/** + Back up a data file. + @param target BACKUP SERVER target (possibly, a directory) + @param sink per-thread context (possibly, a stream to write to) + @param include_p predicate for files to include + @retval 1 on success if some work remains + @retval 0 on successful completion + @retval -1 on error +*/ +static int aria_backup_data(const struct backup_target *target, + const struct backup_sink *sink, + name_predicate include_p) +{ + const char *filename= NULL; + char path[FN_REFLEN + 2]; + struct Aria_backup *const ab= sink->ha_data; + int left= 0; +#ifndef _WIN32 + struct dirent *d; + struct stat sb; + assert(ab->dir); +#else + assert(ab->dir != INVALID_HANDLE_VALUE); +#endif + pthread_mutex_lock(&ab->mutex); + + if (ab->status != BACKUP_OK) + { + assert(ab->status == BACKUP_FAIL); + err_exit: + left= -1; + ab->status= BACKUP_FAIL; + my_error(ER_UNKNOWN_ERROR, MYF(0)); + } +#ifndef _WIN32 + else if (!ab->subdir) + { + DIR *const dir= ab->dir; + while ((d= readdir(dir)) != NULL) + { + switch (d->d_type) { + default: + continue; + case DT_DIR: + break; + case DT_UNKNOWN: + if (fstatat(dirfd(dir), d->d_name, &sb, 0) || + (sb.st_mode & S_IFMT) != S_IFDIR) + continue; + } + if (!aria_backup_mkdir(target, d->d_name)) + { + int dfd= openat(dirfd(dir), d->d_name, O_DIRECTORY); + ab->d= d; + if (dfd >= 0) + { + if ((ab->subdir= fdopendir(dfd))) + goto consume_subdir; + close(dfd); + } + dir_error(d->d_name); + } + goto err_exit; + } + } + else + { + consume_subdir: + assert(ab->d); + assert(ab->d->d_type == DT_DIR || ab->d->d_type == DT_UNKNOWN); + while ((d= readdir(ab->subdir)) != NULL) + { + const char *const name= d->d_name; + size_t len; + switch (d->d_type) { + default: + continue; + case DT_REG: + case DT_LNK: + break; + case DT_UNKNOWN: + if (fstatat(dirfd(ab->subdir), name, &sb, 0) || + (sb.st_mode & S_IFMT) != S_IFREG) + continue; + } + if ((len= strlen(name)) < 4 || + /* + As noted in MDEV-25854, file names that start with #sql + must be excluded from the backup. For example, a call to + MDL_context::upgrade_shared_lock() in + mysql_inplace_alter_table() could time out, resulting in + cleanup_table_after_inplace_alter() deleting a + #sql-alter*.frm file before we get a chance to copy it. + */ + !memcmp(name, tmp_file_prefix, tmp_file_prefix_length) || + !(*include_p)(name, len)) + continue; + + /* Consume a file name */ + if ((int) sizeof path <= + snprintf(path, sizeof path, "%s/%s", ab->d->d_name, name)) + { + path[(sizeof path) - 1]= '\0'; + my_error(ER_TOO_LONG_IDENT, MYF(0), path); + goto err_exit; + } + filename= path; + break; + } + + left= 1; + if (!d) + { + closedir(ab->subdir); + ab->d= NULL; + ab->subdir= NULL; + } + } +#else + else if (ab->subdir == INVALID_HANDLE_VALUE) + { + do + { + if (!(ab->d.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) + continue; + + if ((int) sizeof path <= + snprintf(path, sizeof path, "%s/*.*", ab->d.cFileName)) + name_too_long: + my_error(ER_TOO_LONG_IDENT, MYF(0), path); + else if (aria_backup_mkdir(target, ab->d.cFileName)); + else if ((ab->subdir= FindFirstFileA(path, &ab->sd)) != + INVALID_HANDLE_VALUE) + goto consume_subdir; + else + dir_error(path); + goto err_exit; + } + while (FindNextFile(ab->dir, &ab->d)); + } + else + { + consume_subdir: + do + { + const char *const name= ab->sd.cFileName; + size_t len; + if (ab->sd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + continue; + if ((len= strlen(name)) < 4 || + /* + As noted in MDEV-25854, file names that start with #sql + must be excluded from the backup. For example, a call to + MDL_context::upgrade_shared_lock() in + mysql_inplace_alter_table() could time out, resulting in + cleanup_table_after_inplace_alter() deleting a + #sql-alter*.frm file before we get a chance to copy it. + */ + !memcmp(name, tmp_file_prefix, tmp_file_prefix_length) || + !(*include_p)(name, len)) + continue; + + /* Consume a file name */ + if ((int) sizeof path <= + snprintf(path, sizeof path, "%s/%s", ab->d.cFileName, name)) + goto name_too_long; + + filename= path; + } + while ((left= FindNextFile(ab->subdir, &ab->sd)) && !filename); + + if (!left) + { + FindClose(ab->subdir); + ab->subdir= INVALID_HANDLE_VALUE; + left= FindNextFile(ab->dir, &ab->d); + } + } +#endif + + pthread_mutex_unlock(&ab->mutex); + assert(!filename || left >= 0); + + if (filename && +#ifndef _WIN32 + aria_backup_file(target, sink, dirfd(ab->dir), filename) && +#else + aria_backup_file(target, sink, filename, 0) && +#endif + TRUE) + return -1; + return left; +} + +/** + Back up an ENGINE=Aria log file. + @param target BACKUP SERVER target (possibly, a directory) + @param sink per-thread context (possibly, a stream to write to) + @retval 1 on success if some work remains + @retval 0 on successful completion + @retval -1 on error +*/ +static int aria_backup_log(const struct backup_target *target, + const struct backup_sink *sink) +{ + struct Aria_backup *const ab= sink->ha_data; + int left; + const char *filename= NULL; + char path[IF_WIN(FN_REFLEN, 0) + sizeof "aria_log.00000001"]; + pthread_mutex_lock(&ab->mutex); +#ifndef _WIN32 + assert(!ab->d); + assert(ab->dir); + assert(!ab->subdir); +#else + assert(ab->dir != INVALID_HANDLE_VALUE); + assert(ab->subdir == INVALID_HANDLE_VALUE); +#endif + + if (ab->status != TRANSLOG_PURGE_DISABLED) + { + assert(ab->status == BACKUP_FAIL); +#ifdef _WIN32 + err_exit: +#endif + left= -1; + ab->status= BACKUP_FAIL; + } + else + { +#ifndef _WIN32 + struct dirent *d; + while ((d= readdir(ab->dir)) != NULL) + { + struct stat sb; + switch (d->d_type) { + default: + continue; + case DT_REG: + case DT_LNK: + break; + case DT_UNKNOWN: + if (fstatat(dirfd(ab->dir), d->d_name, &sb, 0) || + (sb.st_mode & S_IFMT) != S_IFREG) + continue; + } + if ((strncmp(d->d_name, C_STRING_WITH_LEN("aria_log.")) || + strlen(d->d_name) != (sizeof path) - 1) && + strcmp(d->d_name, "aria_log_control")) + continue; + compile_time_assert(sizeof "aria_log_control" <= sizeof path); + filename= strcpy(path, d->d_name); + break; + } + left= d != NULL; +#else + do + { + const char *const name= ab->d.cFileName; + if (ab->d.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + continue; + if ((strncmp(name, C_STRING_WITH_LEN("aria_log.")) || + strlen(name) != (sizeof "aria_log.00000001") - 1) && + strcmp(name, "aria_log_control")) + continue; + else if ((int) sizeof path <= + snprintf(path, sizeof path, "%s/%s", maria_data_root, name)) + { + path[(sizeof path) - 1]= '\0'; + my_error(ER_TOO_LONG_IDENT, MYF(0), path); + goto err_exit; + } + filename= path; + } + while ((left= FindNextFile(ab->dir, &ab->d)) && !filename); +#endif + } + + pthread_mutex_unlock(&ab->mutex); + assert(!filename || left >= 0); + + if (filename && +#ifndef _WIN32 + aria_backup_file(target, sink, dirfd(ab->dir), filename) && +#else + aria_backup_file(target, sink, filename, + strlen(maria_data_root) + 1) && +#endif + TRUE) + return -1; + return left; +} + +/** + Start of a BACKUP SERVER phase, + when no aria_backup_step() or aria_backup_end() is pending. + @param thd current session + @param target backup target + @param phase BACKUP_PHASE_START, ... (not BACKUP_PHASE_ABORT) + @param sink worker context + @return backup context object to be attached to backup_target + @retval NULL if no context needs to be created + @retval -1 on failure +*/ +void *aria_backup_start(THD *thd, const struct backup_target *target, + enum backup_phase phase, + const struct backup_sink *sink) +{ + switch (phase) { + struct Aria_backup* ab; + case BACKUP_PHASE_PREPARE_START: + assert(!sink); + return NULL; + case BACKUP_PHASE_START: + case BACKUP_PHASE_NO_BEGIN_NON_TRANS: + case BACKUP_PHASE_NO_DML_NON_TRANS: + assert(!sink->ha_data); + break; + case BACKUP_PHASE_NO_DDL: + assert(!sink->ha_data); + if (!(ab= calloc(1, sizeof *ab))) + return (void*) -1; + pthread_mutex_init(&ab->mutex, NULL); +#ifdef _WIN32 + ab->subdir= INVALID_HANDLE_VALUE; + ab->dir= FindFirstFileA("*.*", &ab->d); + if (ab->dir != INVALID_HANDLE_VALUE) + return ab; +#else + { + int dfd= open(mysql_data_home, O_DIRECTORY); + if (dfd >= 0) + { + if ((ab->dir= fdopendir(dfd))) + return ab; + close(dfd); + } + } +#endif + dir_error(mysql_data_home); + pthread_mutex_destroy(&ab->mutex); + free(ab); + return (void*) -1; + case BACKUP_PHASE_NO_COMMIT: +#ifndef NDEBUG + ab= sink->ha_data; +#endif +#ifndef _WIN32 + assert(ab->dir); + assert(!ab->subdir); + assert(!ab->d); +#else + assert(ab->dir != INVALID_HANDLE_VALUE); + assert(ab->subdir == INVALID_HANDLE_VALUE); +#endif + assert(ab->status == BACKUP_OK); + translog_flush(translog_get_horizon()); + break; + case BACKUP_PHASE_FINISH: + if (!sink) + break; + ab= sink->ha_data; + if (!ab) + break; + if (ab->status != TRANSLOG_LOCKED) + { + assert(ab->status == BACKUP_FAIL); + break; + } +#ifndef _WIN32 + assert(!ab->dir); + assert(!ab->subdir); + assert(!ab->d); + { + int dfd= open(maria_data_root, O_DIRECTORY); + if (dfd >= 0) + { + if ((ab->dir= fdopendir(dfd))) + { + ab->status= TRANSLOG_PURGE_DISABLED; + translog_disable_purge(); + break; + } + close(dfd); + } + } +#else + assert(ab->dir == INVALID_HANDLE_VALUE); + assert(ab->subdir == INVALID_HANDLE_VALUE); + { + char path[FN_REFLEN * 2 + 2]; + if ((int) sizeof path > + snprintf(path, sizeof path, "%s/aria_log*", maria_data_root) && + (ab->dir= FindFirstFileA(path, &ab->d)) != + INVALID_HANDLE_VALUE) + { + ab->status= TRANSLOG_PURGE_DISABLED; + translog_disable_purge(); + break; + } + } +#endif + dir_error(maria_data_root); + ab->status= BACKUP_FAIL; + translog_unlock(); + return (void*) -1; + case BACKUP_PHASE_ABORT: + break; + } + return sink->ha_data; +} + +/** + Process a file that was collected in aria_backup_start(). + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked + @param sink worker context + @retval 1 on success if some work remains + @retval 0 on completion + @retval -1 on error +*/ +int aria_backup_step(THD *thd, const struct backup_target *target, + enum backup_phase phase, const struct backup_sink *sink) +{ + switch (phase) { + case BACKUP_PHASE_PREPARE_START: + assert(!sink); + break; + case BACKUP_PHASE_START: + case BACKUP_PHASE_NO_BEGIN_NON_TRANS: + case BACKUP_PHASE_NO_DML_NON_TRANS: + assert(!sink->ha_data); + break; + case BACKUP_PHASE_NO_DDL: + return aria_backup_data(target, sink, is_db_file); + case BACKUP_PHASE_NO_COMMIT: + return aria_backup_data(target, sink, is_ma_file); + case BACKUP_PHASE_FINISH: + return aria_backup_log(target, sink); + case BACKUP_PHASE_ABORT: + break; + } + return 0; +} + +/** + Finish a phase, once all calls for the current phase are completed. + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked, + or BACKUP_PHASE_ABORT or BACKUP_PHASE_FINISH + @param sink worker context + @return error code + @retval 0 on success +*/ +int aria_backup_end(THD *thd, const struct backup_target *target, + enum backup_phase phase, const struct backup_sink *sink) +{ + switch (phase) { + struct Aria_backup* ab; + case BACKUP_PHASE_PREPARE_START: + assert(!sink); + break; + case BACKUP_PHASE_START: + case BACKUP_PHASE_NO_BEGIN_NON_TRANS: + case BACKUP_PHASE_NO_DML_NON_TRANS: + assert(!sink->ha_data); + break; + case BACKUP_PHASE_NO_DDL: + ab= sink->ha_data; + if (!ab) + break; + /* Rewind the directory for BACKUP_PHASE_NO_COMMIT */ +#ifndef _WIN32 + assert(!ab->d); + assert(!ab->subdir); + assert(ab->dir); + rewinddir(ab->dir); +#else + assert(ab->subdir == INVALID_HANDLE_VALUE); + assert(ab->dir != INVALID_HANDLE_VALUE); + FindClose(ab->dir); + ab->dir= FindFirstFileA("*.*", &ab->d); + if (ab->dir == INVALID_HANDLE_VALUE) + { + dir_error(mysql_data_home); + return -1; + } +#endif + break; + case BACKUP_PHASE_NO_COMMIT: + ab= sink->ha_data; + if (!ab) + break; + assert(ab->status == BACKUP_OK); +#ifndef _WIN32 + assert(ab->dir); + assert(!ab->subdir); + assert(!ab->d); + closedir(ab->dir); + ab->dir= NULL; +#else + assert(ab->dir != INVALID_HANDLE_VALUE); + assert(ab->subdir == INVALID_HANDLE_VALUE); + FindClose(ab->dir); + ab->dir= INVALID_HANDLE_VALUE; +#endif + /* + Block further writes to the ENGINE=Aria log until all + aria_backup_log() have completed in phase=BACKUP_PHASE_FINISH. + This guarantees a consistent snapshot with respect to the + translog_get_horizon() that we had determined at the + aria_backup_start() of this phase. + */ + ab->status= TRANSLOG_LOCKED; + translog_lock(); + break; + case BACKUP_PHASE_ABORT: + break; + case BACKUP_PHASE_FINISH: + ab= sink->ha_data; + if (!ab) + break; + switch (ab->status) { + case BACKUP_OK: + case BACKUP_FAIL: + break; + case TRANSLOG_PURGE_DISABLED: + translog_enable_purge(); + /* fall through */ + case TRANSLOG_LOCKED: + translog_unlock(); + break; + } +#ifndef _WIN32 + if (ab->dir) + closedir(ab->dir); + if (ab->subdir) + closedir(ab->subdir); +#else + if (ab->dir != INVALID_HANDLE_VALUE) + FindClose(ab->dir); + if (ab->subdir != INVALID_HANDLE_VALUE) + FindClose(ab->subdir); +#endif + pthread_mutex_destroy(&ab->mutex); + free(ab); + } + + return 0; +} diff --git a/storage/maria/ma_backup_server.h b/storage/maria/ma_backup_server.h new file mode 100644 index 0000000000000..9ddceccc10919 --- /dev/null +++ b/storage/maria/ma_backup_server.h @@ -0,0 +1,68 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#pragma once + +#include "sql_backup_interface.h" + +#ifdef __cplusplus +extern "C" +#endif +/** + Start of a BACKUP SERVER phase, + when no aria_backup_step() or aria_backup_end() is pending. + @param thd current session + @param target backup target + @param phase BACKUP_PHASE_START, ... (not BACKUP_PHASE_ABORT) + @param sink worker context + @return backup context object to be attached to backup_target + @retval NULL if no context needs to be created + @retval -1 on failure +*/ +void *aria_backup_start(THD *thd, const struct backup_target *target, + enum backup_phase phase, + const struct backup_sink *sink); + +#ifdef __cplusplus +extern "C" +#endif +/** + Process a file that was collected in aria_backup_start(). + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked + @param sink worker context + @retval 1 on success if some work remains + @retval 0 on successful completion + @retval -1 on error +*/ +int aria_backup_step(THD *thd, const struct backup_target *target, + enum backup_phase phase, const struct backup_sink *sink); + +#ifdef __cplusplus +extern "C" +#endif +/** + Finish a phase, once all calls for the current phase are completed. + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked, + or BACKUP_PHASE_ABORT or BACKUP_PHASE_FINISH + @param sink worker context + @return error code + @retval 0 on success +*/ +int aria_backup_end(THD *thd, const struct backup_target *target, + enum backup_phase phase, const struct backup_sink *sink); From 0a23ebb6b17d1783fd9f54475b269197793b7482 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 29 Jun 2026 23:44:43 +0530 Subject: [PATCH 5/5] MDEV-39061 mariadb-backup compatible wrapper for BACKUP SERVER This adds a shell script that lets users keep using their existing mariadb-backup commands while the real work is done by the new server-side BACKUP SERVER command. The goal is "drop-in": users should not have to change their backup scripts. extra/mariabackup/scripts/mariadb-backup-server.sh (plain POSIX sh) understands the usual mariadb-backup modes and translates each one. A companion helper, extra/mariabackup/scripts/mbstream-server.sh, lets streamed backups be unpacked by pipelines that expect the mbstream CLI. Both are documented in extra/mariabackup/scripts/README.md. --backup ======== Connects with the mariadb client and runs "BACKUP SERVER TO ''". Connection options (--user, --host, --port, --socket, --defaults-file, ssl, ...) are passed through to the client; --parallel=N becomes the " CONCURRENT": mariadb-backup runs a dedicated log_copying_thread() besides its N-data copy threads. Minimum value is 2 After the backup it writes backup-prepare.cnf into the backup directory, recording what --prepare needs later: where mariadbd lives, the InnoDB parameters (page size, data file path, undo tablespaces, checksum algorithm, log file size), and if the server is encrypted then how to reload the encryption key plugin (the file_key_management variables), so an encrypted backup can be prepared without extra input. --backup --stream ================= Runs "BACKUP SERVER WITH ''", where writes the tar into a fifo that the wrapper drains to its own stdout, so the backup reaches the consumer as it is produced and never lands on local disk. backup-prepare.cnf is appended as a final tar afterwards. The server's tar carries no end-of-archive marker; only that trailing archive adds one, so the whole stream extracts with a plain "tar -x". --parallel is ignored here, with a warning. CONCURRENT selects the number of output sinks as well as workers, and N self-contained tars cannot be interleaved on one stdout, so a stream needs one writer. For parallel streaming use BACKUP SERVER WITH N CONCURRENT directly, giving each stream its own destination via the appended index. Three properties follow from how BACKUP SERVER streams, all differing from mariadb-backup: - local: the stream command runs inside the server, so the wrapper must share its filesystem; ---target-dir is optional in stream mode (scratch for the per-stream. - tar only: any --stream= (including xbstream) yields tar; - single-threaded: one worker, so no parallel read either. --target-dir is optional in stream mode; nothing is spooled there, it only holds the helper script, the fifo and backup-prepare.cnf (a mktemp dir is used otherwise). mbstream-server.sh maps the mbstream CLI onto a plain "tar -x"/"tar -c", so existing "mbstream -x"/"-c" pipelines keep working on the wrapper's stream. mbstream-only flags (-p/--parallel, ...) are accepted and ignored; any other unknown option is rejected. Environment overrides (mainly for testing): MARIADB (client), MARIADBD (the --prepare bootstrap server) and TAR (the tar implementation, e.g. TAR=bsdtar) can each be overridden. To run the bootstrap under rr, put it in MARIADBD and let rr's own _RR_TRACE_DIR choose the trace location, e.g. _RR_TRACE_DIR=/dev/shm/rr MARIADBD='rr record mariadbd' --prepare ========= Starts "mariadbd --bootstrap" on the backup directory using backup-prepare.cnf as its defaults file, replays the archived redo log between the start and target LSN read from backup.cnf, then builds a fresh ib_logfile0 so a normal server can start on the directory. mariadbd is taken from the path recorded in backup-prepare.cnf if that binary exists, otherwise by searching /libexec, /sbin, /bin and the configured install directories. PATH is not searched; set MARIADBD to point elsewhere. User --defaults-file/-extra-file and encryption options are layered onto the bootstrap. --copy-back / --move-back ========================= Copy or move a prepared backup into the datadir. The datadir is created if missing, a non-empty datadir is refused unless --force-non-empty-directories is given, and a chown reminder is printed. If --aria-log-dir-path is given, the Aria logs (aria_log_control, aria_log.*) are relocated into that directory. Packaging ========= The wrapper is not installed by default and never replaces the real mariadb-backup / mbstream binaries. 1. cmake -DWITH_MARIABACKUP_WRAPPER=ON (default OFF) controls it. 2. When ON, the scripts install as /usr/bin/mariadb-backup-server and /usr/bin/mbstream-server, tagged COMPONENT Backup so they ship in the mariadb-backup package. 3. RPM: nothing extra to do. the component handles it. 4. DEB: not wired. debian/rules uses --fail-missing and does not enable the option, so the -server binaries are not listed. To ship via DEB, make a paired change: add -DWITH_MARIABACKUP_WRAPPER=ON in debian/rules and list both usr/bin/mariadb-backup-server and usr/bin/mbstream-server in debian/mariadb-backup.install together. 5. The real mariadb-backup/mbstream binaries and the mariabackup symlink are left untouched; opt in via an alias or a symlink early in PATH. Limitations (not supported yet) =============================== 1) Incremental backup & prepare (--incremental-basedir, --incremental-dir, --apply-log-only) 2) --rollback-xa 3) Partial backup (--databases, --tables, --tables-file) 4) Output compression and encryption (--compress, --encrypt) 5) --export is accepted but only warns and runs a plain recovery 6) --extra-lsndir is ignored 7) --parallel is ignored with --stream 8) Windows: POSIX sh only, not installed on Windows Behaviour differences from native mariadb-backup ================================================ - The wrapper needs the mariadb client on PATH for --backup, and mariadbd on PATH (or recorded in backup-prepare.cnf) --backup; --prepare needs mariadbd recorded in backup-prepare.cnf, in a standard install directory, or named by MARIADBD - BACKUP SERVER refuses an already-existing target directory - BACKUP SERVER does copy the data file as raw pages without checksum validation, so a corrupted table is not detected at backup time - --prepare only works on a wrapper-made backup. It needs backup-prepare.cnf) - --stream is tar, not xbstream, local-only and single-threaded Tests ===== include/have_mariabackup_wrapper.inc redirects $XTRABACKUP to mariadb-backup-server.sh and $XBSTREAM to mbstream-server.sh, skipping when a wrapper or the mariadb client is unavailable. include/have_mariabackup_combination.inc runs a test under both the [CLIENT] mariadb-backup binary and the [SERVER] wrapper. --- extra/mariabackup/CMakeLists.txt | 30 ++ extra/mariabackup/scripts/README.md | 263 ++++++++++++ .../scripts/mariadb-backup-server.sh | 384 ++++++++++++++++++ extra/mariabackup/scripts/mbstream-server.sh | 63 +++ .../have_mariabackup_combination.combinations | 3 + .../include/have_mariabackup_combination.inc | 5 + .../include/have_mariabackup_wrapper.inc | 43 ++ .../suite/mariabackup/aria_encrypted.test | 2 + .../suite/mariabackup/aria_log_dir_path.inc | 104 +++++ .../suite/mariabackup/aria_log_dir_path.test | 107 +---- .../mariabackup/aria_log_dir_path_rel.test | 2 +- .../mariabackup/defer_space,SERVER.rdiff | 10 + mysql-test/suite/mariabackup/defer_space.test | 8 +- .../mariabackup/full_backup,SERVER.rdiff | 30 ++ mysql-test/suite/mariabackup/full_backup.test | 16 +- mysql-test/suite/mariabackup/huge_lsn.test | 2 +- .../suite/mariabackup/log_tables,SERVER.rdiff | 16 + mysql-test/suite/mariabackup/log_tables.test | 4 + mysql-test/suite/mariabackup/mdev-18438.test | 6 + .../mariabackup/partition_notwin,SERVER.rdiff | 8 + .../suite/mariabackup/partition_notwin.test | 13 +- .../suite/mariabackup/relative_path.test | 1 + .../mariabackup/row_format_redundant.test | 1 + mysql-test/suite/mariabackup/small_ibd.test | 1 + .../mariabackup/undo_space_id,SERVER.rdiff | 13 + .../suite/mariabackup/undo_space_id.test | 4 + .../suite/mariabackup/undo_truncate.test | 1 + .../unencrypted_page_compressed,SERVER.rdiff | 8 + .../unencrypted_page_compressed.test | 6 +- mysql-test/suite/mariabackup/vector.test | 1 + 30 files changed, 1043 insertions(+), 112 deletions(-) create mode 100644 extra/mariabackup/scripts/README.md create mode 100755 extra/mariabackup/scripts/mariadb-backup-server.sh create mode 100755 extra/mariabackup/scripts/mbstream-server.sh create mode 100644 mysql-test/include/have_mariabackup_combination.combinations create mode 100644 mysql-test/include/have_mariabackup_combination.inc create mode 100644 mysql-test/include/have_mariabackup_wrapper.inc create mode 100644 mysql-test/suite/mariabackup/aria_log_dir_path.inc create mode 100644 mysql-test/suite/mariabackup/defer_space,SERVER.rdiff create mode 100644 mysql-test/suite/mariabackup/full_backup,SERVER.rdiff create mode 100644 mysql-test/suite/mariabackup/log_tables,SERVER.rdiff create mode 100644 mysql-test/suite/mariabackup/partition_notwin,SERVER.rdiff create mode 100644 mysql-test/suite/mariabackup/undo_space_id,SERVER.rdiff create mode 100644 mysql-test/suite/mariabackup/unencrypted_page_compressed,SERVER.rdiff diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt index a71030887c43f..5fad252fbf5a0 100644 --- a/extra/mariabackup/CMakeLists.txt +++ b/extra/mariabackup/CMakeLists.txt @@ -114,3 +114,33 @@ ADD_DEPENDENCIES(mbstream GenError) IF(MSVC) SET_TARGET_PROPERTIES(mbstream PROPERTIES LINK_FLAGS setargv.obj) ENDIF() + + +######################################################################## +# mariadb-backup-server: BACKUP SERVER-compatible shell wrapper +######################################################################## +# A drop-in mariadb-backup-compatible POSIX-sh wrapper that translates the +# CLI into server-side BACKUP SERVER SQL. Experimental; OFF by default +# Installed as a mariadb-backup-server; clients opt in via symlink/alias +# (see extra/mariabackup/scripts/README.md). +OPTION(WITH_MARIABACKUP_WRAPPER + "Install the BACKUP SERVER shell wrapper (mariadb-backup-server)" OFF) +ADD_FEATURE_INFO(MARIABACKUP_WRAPPER WITH_MARIABACKUP_WRAPPER + "BACKUP SERVER-compatible mariadb-backup shell wrapper") + +IF(WITH_MARIABACKUP_WRAPPER AND NOT WIN32) + # --prepare runs an offline mariadbd bootstrap, so the wrapper + # has to be able to find the server binary without a running + # server to ask. + SET(bindir ${INSTALL_BINDIRABS}) + SET(sbindir ${INSTALL_SBINDIRABS}) + CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/scripts/mariadb-backup-server.sh + ${CMAKE_CURRENT_BINARY_DIR}/mariadb-backup-server ESCAPE_QUOTES @ONLY) + INSTALL_SCRIPT(${CMAKE_CURRENT_BINARY_DIR}/mariadb-backup-server + DESTINATION ${INSTALL_BINDIR} COMPONENT Backup) + + CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/scripts/mbstream-server.sh + ${CMAKE_CURRENT_BINARY_DIR}/mbstream-server COPYONLY) + INSTALL_SCRIPT(${CMAKE_CURRENT_BINARY_DIR}/mbstream-server + DESTINATION ${INSTALL_BINDIR} COMPONENT Backup) +ENDIF() diff --git a/extra/mariabackup/scripts/README.md b/extra/mariabackup/scripts/README.md new file mode 100644 index 0000000000000..f9253022a36a0 --- /dev/null +++ b/extra/mariabackup/scripts/README.md @@ -0,0 +1,263 @@ +# mariadb-backup-server.sh — a BACKUP SERVER wrapper + +`mariadb-backup-server.sh` makes the server-side `BACKUP SERVER` command look +like the old `mariadb-backup` tool. You call it with the familiar +`--backup` / `--prepare` / `--copy-back` options; under the hood +it just runs `BACKUP SERVER TO ''` over a normal `mariadb` +connection and lets the server do the work. It's plain POSIX `sh`, +so it runs anywhere `mariadb-backup` does. + +You need a server that supports `BACKUP SERVER`, the `mariadb` client on +`PATH`, and an account allowed to run `BACKUP SERVER`. The parent of the +target directory has to exist and be writable. + +## Installing / enabling it + +The wrapper is experimental and off by default. Build with +`-DWITH_MARIABACKUP_WRAPPER=ON` and it installs next to the real +binaries, under its own names so it never shadows them: + +``` +/usr/bin/mariadb-backup # the C++ binary, unchanged +/usr/bin/mariadb-backup-server # this wrapper +/usr/bin/mbstream # the C++ binary, unchanged +/usr/bin/mbstream-server # the tar-based mbstream shim (for --stream) +``` + +To send your existing `mariadb-backup` (and `mbstream`) calls through the +wrapper instead, point the names at them yourself — an alias, or a symlink +earlier in `PATH`: + +```sh +alias mariadb-backup=mariadb-backup-server +alias mbstream=mbstream-server +# or +ln -s /usr/bin/mariadb-backup-server ~/bin/mariadb-backup +ln -s /usr/bin/mbstream-server ~/bin/mbstream +``` + +`mbstream-server` is only needed for `--stream` backups (it extracts the +wrapper's tar stream). + +## Backing up + +```sh +mariadb-backup-server --backup --target-dir=/backup/full +``` + +That runs `BACKUP SERVER TO '/backup/full'`. Use `--parallel=N` to ask for N +concurrent streams (`... N CONCURRENT`; N=1 is the default and changes +nothing). + +Connection options: +`--user`, `--password`, `--host`, `--port`, `--socket`, `--defaults-file`, +`--defaults-extra-file` and their short forms are passed straight to +the `mariadb` client. + +`--throttle`, `--no-lock` and `--safe-slave-backup` are accepted and ignored; + +When the backup finishes the wrapper drops a `backup-prepare.cnf` into the +target dir, next to the server's own `backup.cnf`. It records where `mariadbd` +lives and the InnoDB layout, so `--prepare` can recover the backup later +without you respelling all of that. + +### Streaming to stdout + +```sh +mariadb-backup-server --backup --stream=tar > /backup/full.tar +``` + +`--stream` runs `BACKUP SERVER WITH [N CONCURRENT] ''` instead of +`... TO ''`. The server hands each stream's tar to ``, the wrapper +collects the parts and writes them to its stdout, so you can redirect to a file +or pipe onwards. + +Two things follow from how `BACKUP SERVER` streams, and both differ from +`mariadb-backup`: + +- **Local only.** The stream command runs *inside the server*, so its output +lands on the server's filesystem. The wrapper can only pick it up when it runs +on the same host (a shared filesystem), as the user the server writes as +(typically `mysql`), or with a `--target-dir` the server can write. A remote +`--host` cannot stream this way. + +- **tar only.** The server emits tar, never `xbstream`. Any `--stream=` +value (including `xbstream`) is accepted but the output is always tar. + +`--target-dir` is optional here; when given it is just the scratch directory for +the per-stream parts (otherwise a `mktemp` dir is used). + +The output is the per-stream tar entries concatenated, with +`backup-prepare.cnf` appended as the trailing archive. + +```sh +mkdir restore && tar -xf /backup/full.tar -C restore +``` + +After extraction the directory holds the data files, the server's `backup.cnf` +*and* the wrapper's `backup-prepare.cnf`, so it can be prepared exactly like a +directory backup: + +```sh +mariadb-backup-server --prepare --target-dir=restore +``` + +### `mbstream.sh` — the extraction shim + +The real `mbstream`/`xbstream` binary cannot read the wrapper's stream, because +that stream is plain tar, not the `xbstream` format. So a companion shim, +`mbstream.sh`, ships next to `mariadb-backup-server.sh` and maps the `mbstream` CLI onto +`tar`, letting existing pipelines (and tests) that call `mbstream` keep working +unchanged: + +```sh +mbstream-server -x -C restore < /backup/full.tar # tar -x -C restore +mbstream-server -c -C dir file1 file2 # tar -c -C dir file1 file2 +``` + +Notes: + +- **Extraction is a plain `tar -x`** — the stream has a single +end-of-archive marker + +- **mbstream-only options are accepted and ignored** — +`-p` / `--parallel`, compression flags, `-v` : So legacy +invocations don't break. + +- It understands `-x` (extract, from stdin) and `-c` (create, to stdout), +with `-C `; anything else is treated as a file operand or ignored. + +- It is a thin tar wrapper, so it only understands the wrapper's tar streams — +do not point it at a real `xbstream` archive, and do not feed the wrapper's +output to the real `mbstream`. + +## Preparing + +```sh +mariadb-backup-server --prepare --target-dir=/backup/full +``` + +Prepare makes the backup bootable: it starts `mariadbd --bootstrap` on the +target directory, replays the archived redo up to the backup's end LSN, then +replaces the archived log with a fresh `ib_logfile0` so a normal server can +start on it. Both `backup.cnf` and `backup-prepare.cnf` have to be there (they +are, if this wrapper took the backup). + +Options: + +- `--use-memory=N` — buffer pool size during recovery. +- `--innodb-*` and `--tmpdir` are forwarded to the bootstrap server. +- `--defaults-file` / `--defaults-extra-file`, and encryption options such as + `--file-key-management*` / `--loose-file-key-management*` / `--plugin-load-add`, + are layered onto the bootstrap (as an extra defaults file / extra options) so + you can supply anything `backup-prepare.cnf` did not record. + +The `mariadbd` used for the bootstrap is the path recorded in +`backup-prepare.cnf` at backup time if that binary exists; otherwise `mariadbd` +from `PATH`. (Recorded-first matters: it is the same version that took the +backup, so it can always parse the backed-up tablespace format.) + +## Restoring + +With the backup prepared and the server stopped, put it into the datadir: + +```sh +mariadb-backup-server --copy-back --target-dir=/backup/full --datadir=/var/lib/mysql +mariadb-backup-server --move-back --target-dir=/backup/full --datadir=/var/lib/mysql +``` + +`--copy-back` leaves the backup where it is; `--move-back` renames the files +across, which is quicker on the same filesystem but consumes the backup. Either +way the datadir is created if missing, and the wrapper won't write into a +non-empty datadir unless you add `--force-non-empty-directories`. + +If the source server kept its Aria logs outside the datadir, pass the same +`--aria-log-dir-path=` you use on the server. The wrapper creates that +directory and moves the restored `aria_log_control` / `aria_log.*` files into +it, so the server finds them on restart: + +```sh +mariadb-backup-server --copy-back --target-dir=/backup/full --datadir=/var/lib/mysql \ + --aria-log-dir-path=/var/lib/aria_logs +``` + +Neither fixes ownership, so finish up with: + +```sh +chown -R mysql:mysql /var/lib/mysql +systemctl start mariadb +``` + +## What it doesn't do + +These stop with an error instead of quietly producing an incomplete backup: + +- incremental backup/prepare: `--incremental-basedir`, `--incremental-dir`, `--apply-log-only` +- partial backup: `--databases`, `--tables`, `--tables-file`. This needs server-side +`backup_include`/`backup_exclude`, which don't exist yet +- compression and encryption of the output: `--compress`, `--encrypt` (error out) +- `--rollback-xa`: not supported (errors out) + +`--stream` is supported with the caveats above (local only, tar only, extract +with a plain `tar -x`); + +`--export` is accepted but not implemented: it warns and does a plain recovery +(no per-table `.cfg` files for IMPORT TABLESPACE). + +## Environment overrides + +The wrapped commands can be overridden via environment variables, +mainly for testing: + +- `MARIADB`: the client used to talk to the server (default `mariadb`), +e.g. `MARIADB='mariadb --protocol=tcp'`. + +- `MARIADBD`: the server binary the `--prepare` bootstrap runs (by default the +path recorded in `backup-prepare.cnf`, else `mariadbd` on `PATH`). When set it +overrides that resolution. To run it under `rr`, put `rr` in `MARIADBD` and let +`rr`'s own `_RR_TRACE_DIR` choose where the trace goes, e.g. +`_RR_TRACE_DIR=/dev/shm/rr MARIADBD='rr record mariadbd' ...`. + +- `TAR`: the tar implementation (default `tar`), +e.g. `TAR=bsdtar` (libarchive-tools). +Used by `--stream` and by `mbstream-server`. + +## The two .cnf files + +`backup.cnf` is written by the server and tells `--prepare` what parts of redo to replay: + +```ini +[server] +# checkpoint=54088 +innodb_log_recovery_start=54088 # recovery starts scanning here +innodb_log_recovery_target=56337 # the backup's end LSN; recovery stops here +``` + +`backup-prepare.cnf` is written by the wrapper and handed to the prepare +bootstrap as its defaults file: + +```ini +# mariadbd=/usr/sbin/mariadbd +[mariadbd] +innodb_page_size=16384 +innodb_data_file_path=ibdata1:12M:autoextend +innodb_undo_tablespaces=3 +innodb_checksum_algorithm=full_crc32 +innodb_log_file_size=100663296 +``` + +If the server is encrypted it also records how to load the key-management +plugin again, so an encrypted backup can be prepared without extra input. +For `file_key_management` it captures every plugin variable (the same way +`mariadb-backup` writes them into `backup-my.cnf`) + +```ini +plugin-load-add=file_key_management +innodb_encrypt_log=ON +loose-file-key-management +loose-file_key_management_filename=/etc/mysql/keys.enc +loose-file_key_management_filekey=FILE:/etc/mysql/keyfile.key +loose-file_key_management_encryption_algorithm=aes_cbc +loose-file_key_management_digest=sha1 +loose-file_key_management_use_pbkdf2=1 +``` diff --git a/extra/mariabackup/scripts/mariadb-backup-server.sh b/extra/mariabackup/scripts/mariadb-backup-server.sh new file mode 100755 index 0000000000000..d4209fc71e8d3 --- /dev/null +++ b/extra/mariabackup/scripts/mariadb-backup-server.sh @@ -0,0 +1,384 @@ +#!/bin/sh +me=${0##*/} +die() { + echo "$me: $*" >&2 + exit 1 +} + +# Overridable for testing: +# MARIADB='mariadb --protocol=tcp', TAR=bsdtar, +# MARIADBD='rr record mariadbd'. +: "${MARIADB:=mariadb}" +: "${TAR:=tar}" + +MODE= +TARGET_DIR= +DATADIR= +PARALLEL= +USE_MEMORY= +FORCE_NON_EMPTY= +EXPORT= +ROLLBACK_XA= +MARIADB_OPTS= +INNODB_OPTS= +MYSQLD_EXTRA= +PREPARE_DEFAULTS= +ARIA_LOG_DIR= +STREAM= + +while [ $# -gt 0 ]; do + case $1 in + --backup) MODE=backup ;; + --prepare|--apply-log) MODE=prepare ;; + --copy-back) MODE=copy-back ;; + --move-back) MODE=move-back ;; + + --target-dir=*) TARGET_DIR=${1#*=} ;; + --datadir=*) DATADIR=${1#*=} ;; + --aria-log-dir-path=*) ARIA_LOG_DIR=${1#*=} ;; + --use-memory=*) USE_MEMORY=${1#*=} ;; + --parallel=*) PARALLEL=${1#*=} ;; + + --export) EXPORT=1 ;; + --rollback-xa) ROLLBACK_XA=1 ;; + --force-non-empty-directories) FORCE_NON_EMPTY=1 ;; + + --innodb|--innodb=*|--innodb-*|--innodb_*|--skip-innodb-*|--skip_innodb_*) + INNODB_OPTS="$INNODB_OPTS $1" ;; + --tmpdir=*) MYSQLD_EXTRA="$MYSQLD_EXTRA $1" ;; + -t) MYSQLD_EXTRA="$MYSQLD_EXTRA --tmpdir=$2"; shift ;; + -t*) MYSQLD_EXTRA="$MYSQLD_EXTRA --tmpdir=${1#-t}" ;; + --incremental-basedir=*|--incremental-dir=*) + die "incremental backup/prepare is not supported" ;; + --apply-log-only) + die "--apply-log-only is not supported" ;; + --databases=*|--databases-exclude=*|--tables=*|--tables-exclude=*|--tables-file=*) + die "partial backup needs server-side backup_include/backup_exclude, which doesn't exist yet" ;; + # BACKUP SERVER only ever produces tar + --stream|--stream=*) STREAM=1 ;; + --compress|--compress=*|--compress-threads=*) die "--compress is not supported" ;; + --encrypt|--encrypt=*) die "--encrypt is not supported" ;; + --innobackupex) die "innobackupex mode is not supported" ;; + + --defaults-file=*|--defaults-extra-file=*) + MARIADB_OPTS="$MARIADB_OPTS $1" + PREPARE_DEFAULTS="$PREPARE_DEFAULTS --defaults-extra-file=${1#*=}" ;; + --file-key-management*|--plugin-load-add=*|--loose-file-key-management*|\ + --aria-encrypt-tables*|--encrypt-tmp-disk-tables*) + PREPARE_DEFAULTS="$PREPARE_DEFAULTS $1" ;; + + --user=*|--password=*|--host=*|--port=*|--socket=*|\ + --defaults-group=*|\ + --secure-auth|--skip-secure-auth|--ssl|--ssl-verify-server-cert|\ + --ssl-ca=*|--ssl-capath=*|--ssl-cert=*|--ssl-cipher=*|\ + --ssl-crl=*|--ssl-crlpath=*|--ssl-key=*|--tls-version=*) + MARIADB_OPTS="$MARIADB_OPTS $1" ;; + -p) + if [ -n "${2-}" ] && case $2 in -*) false ;; *) true ;; esac; then + MARIADB_OPTS="$MARIADB_OPTS -p$2"; shift + else + MARIADB_OPTS="$MARIADB_OPTS -p" + fi ;; + -u|-P|-S) MARIADB_OPTS="$MARIADB_OPTS $1 $2"; shift ;; + -H) MARIADB_OPTS="$MARIADB_OPTS --host=$2"; shift ;; + -p*|-u*|-P*|-S*) MARIADB_OPTS="$MARIADB_OPTS $1" ;; + -H*) MARIADB_OPTS="$MARIADB_OPTS --host=${1#-H}" ;; + + -h) DATADIR=$2; shift ;; + -h*) DATADIR=${1#-h} ;; + + # Everything else is accepted and ignored: + *) ;; + esac + shift +done + +# Streaming spools nothing, so --target-dir is optional there. +[ -n "$STREAM" ] || [ -n "$TARGET_DIR" ] || die "--target-dir required" + +ask() { $MARIADB $MARIADB_OPTS -BN -e "$1" 2>/dev/null; } + +# mariadb-backup runs a dedicated log_copying_thread() +# besides the --parallel data-copy threads, +# so --parallel=N is N+1 workers, at least 2, at most 256. +concurrent_clause() { + _p=$PARALLEL + while :; do + case $_p in 0?*) _p=${_p#0} ;; *) break ;; esac + done + case $_p in + ''|*[!0-9]*) _c=2 ;; + ????*) _c=256 ;; + *) _c=$((_p + 1)) + [ "$_c" -ge 2 ] || _c=2 + [ "$_c" -le 256 ] || _c=256 ;; + esac + printf ' %s CONCURRENT' "$_c" +} + +# Print the first mariadbd/mysqld found under basedir $1, +# else return 1. CMake substitutes @sbindir@/@bindir@; +# unsubstituted they are not absolute, so the case skips them. +find_mariadbd() { + for _d in "${1-}/libexec" "${1-}/sbin" "${1-}/bin" '@sbindir@' '@bindir@' + do + case $_d in /*) ;; *) continue ;; esac + for _n in mariadbd mysqld; do + [ -x "$_d/$_n" ] && { printf '%s\n' "$_d/$_n"; return 0; } + done + done + return 1 +} + +# Print what --prepare's offline bootstrap needs: +# the mariadbd path, the InnoDB parameters, and +# how to reload the encryption key plugin. +write_prepare_cnf() { + _mariadbd= + _pidfile=$(ask "SELECT @@global.pid_file") + if [ -n "$_pidfile" ] && [ -r "$_pidfile" ]; then + _pid=$(cat "$_pidfile" 2>/dev/null) + [ -n "$_pid" ] && _mariadbd=$(readlink -f "/proc/$_pid/exe" 2>/dev/null) + fi + [ -n "$_mariadbd" ] && [ -x "$_mariadbd" ] || _mariadbd= + [ -n "$_mariadbd" ] || + _mariadbd=$(find_mariadbd "$(ask "SELECT @@global.basedir")") || + _mariadbd= + + _page_size=$(ask "SELECT @@global.innodb_page_size") + _data_file_path=$(ask "SELECT @@global.innodb_data_file_path") + _undo_ts=$(ask "SELECT @@global.innodb_undo_tablespaces") + _checksum=$(ask "SELECT @@global.innodb_checksum_algorithm") + _log_file_size=$(ask "SELECT @@global.innodb_log_file_size") + + _enc=$(ask "SELECT LOWER(plugin_name) FROM information_schema.PLUGINS + WHERE plugin_type='ENCRYPTION' AND plugin_status='ACTIVE' LIMIT 1") + + [ -n "$_mariadbd" ] && echo "# mariadbd=$_mariadbd" + echo "[mariadbd]" + [ -n "$_page_size" ] && echo "innodb_page_size=$_page_size" + [ -n "$_data_file_path" ] && echo "innodb_data_file_path=$_data_file_path" + [ -n "$_undo_ts" ] && echo "innodb_undo_tablespaces=$_undo_ts" + [ -n "$_checksum" ] && echo "innodb_checksum_algorithm=$_checksum" + [ -n "$_log_file_size" ] && echo "innodb_log_file_size=$_log_file_size" + + if [ -n "$_enc" ]; then + echo "plugin-load-add=$_enc" + _plugin_dir=$(ask "SELECT @@global.plugin_dir") + [ -n "$_plugin_dir" ] && echo "plugin-dir=$_plugin_dir" + case $(ask "SELECT @@global.innodb_encrypt_log") in + 1|ON) echo "innodb_encrypt_log=ON" ;; + esac + if [ "$_enc" = file_key_management ]; then + echo "loose-file-key-management" + ask "SHOW VARIABLES LIKE 'file_key_management%'" | + while read -r _fkm_name _fkm_value; do + [ -n "$_fkm_name" ] && echo "loose-$_fkm_name=$_fkm_value" + done + fi + fi +} + + +# prepare +if [ "$MODE" = prepare ]; then + [ -z "$ROLLBACK_XA" ] || die "--rollback-xa is not supported" + [ -d "$TARGET_DIR" ] || die "no such directory: $TARGET_DIR" + [ -f "$TARGET_DIR/backup.cnf" ] || die "backup.cnf not found in $TARGET_DIR" + + cnf=$TARGET_DIR/backup-prepare.cnf + [ -f "$cnf" ] || die "$cnf missing - was this backup made by the wrapper?" + [ -z "$EXPORT" ] || echo "$me: --export not implemented, doing a plain recovery" >&2 + + if [ -n "${MARIADBD-}" ]; then + mariadbd=$MARIADBD + else + mariadbd=$(sed -n 's/^# *mariadbd=//p' "$cnf" | tail -n1) + if [ -z "$mariadbd" ] || [ ! -x "$mariadbd" ]; then + mariadbd=$(find_mariadbd "") || + die "cannot locate the server binary for recovery;\ + set MARIADBD=/path/to/mariadbd" + fi + fi + + # the LSN window recovery must replay + start=$(grep '^innodb_log_recovery_start' "$TARGET_DIR/backup.cnf" | cut -d= -f2 | tr -d ' ') + target=$(grep '^innodb_log_recovery_target' "$TARGET_DIR/backup.cnf" | cut -d= -f2 | tr -d ' ') + + opts="--datadir=$TARGET_DIR --innodb=FORCE" + [ -n "$start" ] && opts="$opts --innodb-log-recovery-start=$start" + [ -n "$target" ] && opts="$opts --innodb-log-recovery-target=$target" + [ -n "$USE_MEMORY" ] && opts="$opts --innodb-buffer-pool-size=$USE_MEMORY" + opts="$opts$INNODB_OPTS$MYSQLD_EXTRA" + + # Recovery leaves the archived log (ib_.log) behind, + # which a normal server won't boot from, so build a + # fresh ib_logfile0 (header plus a checkpoint at the end LSN) + # to replace it. + input=/dev/null + newlog=$TARGET_DIR/ib_logfile0.new + if [ -n "$target" ]; then + lsn=$(printf '%016x' "$target") + rm -f "$newlog" + input=$(mktemp) + cat > "$input" <&2 + exit 0 +fi + + +# copy-back / move-back +if [ "$MODE" = copy-back ] || [ "$MODE" = move-back ]; then + [ -d "$TARGET_DIR" ] || die "no such directory: $TARGET_DIR" + [ -f "$TARGET_DIR/backup.cnf" ] || die "backup.cnf not found in $TARGET_DIR" + [ -n "$DATADIR" ] || die "--datadir required for --$MODE" + + # mariadb-backup creates the datadir if it's missing; do the same. + [ -d "$DATADIR" ] || mkdir -p "$DATADIR" || die "cannot create datadir: $DATADIR" + + # dotfiles included + if [ -z "$FORCE_NON_EMPTY" ]; then + for f in "$DATADIR"/* "$DATADIR"/.[!.]* "$DATADIR"/..?*; do + { [ -e "$f" ] || [ -L "$f" ]; } && + die "datadir not empty: $DATADIR (pass --force-non-empty-directories)" + done + fi + + if [ "$MODE" = copy-back ]; then + echo "$me: copying $TARGET_DIR -> $DATADIR" >&2 + cp -R "$TARGET_DIR"/. "$DATADIR"/ || die "copy-back failed" + else + echo "$me: moving $TARGET_DIR -> $DATADIR" >&2 + for f in "$TARGET_DIR"/* "$TARGET_DIR"/.[!.]* "$TARGET_DIR"/..?*; do + { [ -e "$f" ] || [ -L "$f" ]; } || continue + mv "$f" "$DATADIR"/ || die "move-back failed" + done + fi + + # backup put the Aria logs at the datadir root; + # --aria-log-dir-path means the server will look + # elsewhere on restart. + if [ -n "$ARIA_LOG_DIR" ] && [ "$ARIA_LOG_DIR" != "$DATADIR" ]; then + mkdir -p "$ARIA_LOG_DIR" || die "cannot create aria-log-dir-path: $ARIA_LOG_DIR" + for f in "$DATADIR"/aria_log_control "$DATADIR"/aria_log.*; do + [ -e "$f" ] && { mv "$f" "$ARIA_LOG_DIR"/ || die "aria log relocate failed"; } + done + fi + + echo "Restore completed: $DATADIR" >&2 + echo "$me: now run: chown -R mysql:mysql $DATADIR && start the server" >&2 + exit 0 +fi + + +# backup (streaming) +# The server feeds this stream's tar to a helper, +# which writes it into a fifo that we drain to stdout, +# so nothing lands on local disk. +# +# CONCURRENT picks the number of sinks as well as workers, +# and N separate tars cannot be interleaved on one stdout, +# so --parallel cannot be honoured here. +if [ -n "$STREAM" ]; then + case $PARALLEL in + ''|*[!0-9]*|0|1) ;; + *) echo "$me: ignoring --parallel=$PARALLEL:" \ + "a single stream needs a single writer" >&2 ;; + esac + + if [ -n "$TARGET_DIR" ]; then + scratch=$TARGET_DIR + mkdir -p "$scratch" || die "cannot create scratch dir: $scratch" + keep_scratch=1 + else + scratch=$(mktemp -d "${TMPDIR:-/tmp}/mariabackup-stream.XXXXXX") \ + || die "cannot create scratch directory" + keep_scratch= + fi + + helper=$scratch/.mariabackup-stream.sh + fifo=$scratch/.mariabackup-stream.fifo + drain= + + cleanup_stream() { + # unlinking does not release a reader still blocked on open(2) + [ -z "$drain" ] || kill "$drain" 2>/dev/null + rm -f "$helper" "$fifo" "$scratch/backup-prepare.cnf" + [ -n "$keep_scratch" ] || rmdir "$scratch" 2>/dev/null + } + + mkfifo -m 600 "$fifo" || { cleanup_stream; die "cannot create fifo: $fifo"; } + + # the stream index the server appends is ignored: there is + # only one stream + cat > "$helper" < "$fifo" +EOF + + # before the writer: opening a fifo for writing + # blocks until a reader does + cat "$fifo" & + drain=$! + + # "/bin/sh " needs no execute bit, which /dev/shm + # may forbid. Client output goes to stderr to keep stdout + # clean for the tar. + $MARIADB $MARIADB_OPTS -e "BACKUP SERVER WITH '/bin/sh $helper'" >&2 \ + || { cleanup_stream; die "BACKUP SERVER failed"; } + + # the tar must be complete on stdout before anything + # is appended to it + wait "$drain" || { drain=; cleanup_stream; die "streaming to stdout failed"; } + drain= + + # Append backup-prepare.cnf as one more tar so it extracts + # alongside backup.cnf. The server writes no end-of-archive + # marker; this trailing archive supplies the only one, + # so plain "tar -x" reads the whole stream. + echo "$me: appending backup-prepare.cnf to the stream" >&2 + write_prepare_cnf > "$scratch/backup-prepare.cnf" \ + || { cleanup_stream; die "could not build backup-prepare.cnf"; } + $TAR -C "$scratch" -cf - backup-prepare.cnf \ + || { cleanup_stream; die "could not append backup-prepare.cnf to the stream"; } + + cleanup_stream + echo "$me: streamed the backup plus backup-prepare.cnf to stdout" >&2 + exit 0 +fi + + +# backup (directory) +parent=$(dirname "$TARGET_DIR") +[ -d "$parent" ] || die "parent directory does not exist: $parent" +[ -w "$parent" ] || die "parent directory is not writable: $parent" + +sql="BACKUP SERVER TO '$TARGET_DIR'$(concurrent_clause)" +$MARIADB $MARIADB_OPTS -e "$sql" || die "BACKUP SERVER failed" + +[ -f "$TARGET_DIR/backup.cnf" ] || exit 0 + +echo "$me: writing backup-prepare.cnf" >&2 +write_prepare_cnf > "$TARGET_DIR/backup-prepare.cnf" diff --git a/extra/mariabackup/scripts/mbstream-server.sh b/extra/mariabackup/scripts/mbstream-server.sh new file mode 100755 index 0000000000000..f5719e69f06b1 --- /dev/null +++ b/extra/mariabackup/scripts/mbstream-server.sh @@ -0,0 +1,63 @@ +#!/bin/sh +# mbstream-compatible for the BACKUP SERVER wrapper. +# +# This maps the mbstream CLI to `tar`: +# +# mbstream -x -C < archive -> tar -x -C +# mbstream -c > archive -> tar -c +# +# mbstream-only options are accepted and ignored so +# existing invocations keep working unchanged. +me=${0##*/} +die() { + echo "$me: $*" >&2 + exit 1 +} + +# The tar implementation can be overridden for testing. +# e.g. TAR=bsdtar. +: "${TAR:=tar}" + +mode= +dir=. +files= + +while [ $# -gt 0 ]; do + case $1 in + -x|--extract) mode=x ;; + -c|--create) mode=c ;; + -C|--directory) dir=$2; shift ;; + -C*) dir=${1#-C} ;; + --directory=*) dir=${1#*=} ;; + + # mbstream-only flags that have no tar equivalent: drop them. + -p|--parallel) shift ;; + -p*|--parallel=*) ;; + --decompress|--compress) ;; + -v|--verbose) ;; + + --) shift; break ;; + # Reject anything we do not recognise rather than + # silently ignoring it (e.g. GNU tar's -b takes an argument + # that would otherwise be mistaken for a file operand). + # This is an mbstream-on-tar shim, not tar. + -*) die "unsupported option: $1" ;; + *) files="$files $1" ;; + esac + shift +done + +while [ $# -gt 0 ]; do + files="$files $1" + shift +done + +case $mode in + # The server's tar carries no end-of-archive marker; only the + # trailing backup-prepare.cnf adds the single end marker + x) exec $TAR -x -f - -C "$dir" ;; + c) + [ -n "$files" ] || files=. + exec $TAR -c -f - -C "$dir" $files ;; + *) die "expected -x (extract) or -c (create)" ;; +esac diff --git a/mysql-test/include/have_mariabackup_combination.combinations b/mysql-test/include/have_mariabackup_combination.combinations new file mode 100644 index 0000000000000..12bc42487d520 --- /dev/null +++ b/mysql-test/include/have_mariabackup_combination.combinations @@ -0,0 +1,3 @@ +[CLIENT] + +[SERVER] diff --git a/mysql-test/include/have_mariabackup_combination.inc b/mysql-test/include/have_mariabackup_combination.inc new file mode 100644 index 0000000000000..a2b79d6310efc --- /dev/null +++ b/mysql-test/include/have_mariabackup_combination.inc @@ -0,0 +1,5 @@ +if ($MTR_COMBINATION_SERVER) +{ + --source include/have_mariabackup_wrapper.inc +} + diff --git a/mysql-test/include/have_mariabackup_wrapper.inc b/mysql-test/include/have_mariabackup_wrapper.inc new file mode 100644 index 0000000000000..dbe6a061a2f7a --- /dev/null +++ b/mysql-test/include/have_mariabackup_wrapper.inc @@ -0,0 +1,43 @@ +# Redirect `$XTRABACKUP` so existing test invocations like +# +# --exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf \ +# --backup --target-dir=$targetdir +# +# run through extra/mariabackup/scripts/mariadb-backup-server.sh — the BACKUP +# SERVER compatibility wrapper — without any change to the test body. +# +# --source include/have_mariabackup_wrapper.inc +# # ... rest of the test, using $XTRABACKUP as usual ... +# +# $XTRABACKUP : now points at mariadb-backup-server.sh + +--source include/not_windows.inc + +--let MARIABACKUP_WRAPPER=$MYSQL_TEST_DIR/../extra/mariabackup/scripts/mariadb-backup-server.sh +--let MBSTREAM_WRAPPER=$MYSQL_TEST_DIR/../extra/mariabackup/scripts/mbstream-server.sh + +# The wrapper shells out to the bare `mariadb` client, which mtr does not put +# on PATH. Prepend the build's client directories so it resolves. A `let` with +# no leading $ is exported to the environment of later --exec commands. +--let PATH=$MYSQL_BINDIR/client:$MYSQL_BINDIR/client_release:$MYSQL_BINDIR/client_debug:$MYSQL_BINDIR/bin:$PATH + +# --prepare needs the server binary, and @@basedir is the +# source tree in an out-of-source build. mtr already found it, +# so just hand it over. +--let MARIADBD=$MYSQLD + +--error 0,1 +perl; +my $w = $ENV{MARIABACKUP_WRAPPER}; +my $m = $ENV{MBSTREAM_WRAPPER}; +exit 1 unless $w && -x $w && $m && -x $m; +exit 0; +EOF + +if ($errno) +{ + --skip mariadb-backup-server.sh wrapper unavailable (script or sh missing) +} + +--let XTRABACKUP=$MARIABACKUP_WRAPPER +--let XBSTREAM=$MBSTREAM_WRAPPER diff --git a/mysql-test/suite/mariabackup/aria_encrypted.test b/mysql-test/suite/mariabackup/aria_encrypted.test index 6ed3e0d08eff3..017a0b249dcfe 100644 --- a/mysql-test/suite/mariabackup/aria_encrypted.test +++ b/mysql-test/suite/mariabackup/aria_encrypted.test @@ -1,6 +1,7 @@ --source include/have_file_key_management.inc --source include/have_innodb.inc --source include/have_sequence.inc +--source include/have_mariabackup_combination.inc --echo # --echo # MDEV-38246 aria_read index failed on encrypted database during backup @@ -16,3 +17,4 @@ let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir; drop table if exists t1,t2; +rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path.inc b/mysql-test/suite/mariabackup/aria_log_dir_path.inc new file mode 100644 index 0000000000000..1c7d8d12679d8 --- /dev/null +++ b/mysql-test/suite/mariabackup/aria_log_dir_path.inc @@ -0,0 +1,104 @@ +--source include/have_maria.inc + +--echo # +--echo # MDEV-30968 mariadb-backup does not copy Aria logs if aria_log_dir_path is used +--echo # + +--let $datadir=`SELECT @@datadir` +--let $targetdir=$MYSQLTEST_VARDIR/tmp/backup + +if ($ARIA_LOGDIR_MARIADB == '') +{ + --let $ARIA_LOGDIR_MARIADB=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path +} + +if ($ARIA_LOGDIR_FS == '') +{ + --let $ARIA_LOGDIR_FS=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path +} + +--let $server_parameters=--aria-log-file-size=8388608 --aria-log-purge-type=external --loose-aria-log-dir-path=$ARIA_LOGDIR_MARIADB + + +--echo # Restart mariadbd with the test specific parameters +--mkdir $ARIA_LOGDIR_FS +--let $restart_parameters=$server_parameters +--source include/restart_mysqld.inc + + +--echo # Create and populate an Aria table (and Aria logs) +CREATE TABLE t1 (id INT, txt LONGTEXT) ENGINE=Aria; +DELIMITER $$; +BEGIN NOT ATOMIC + FOR id IN 0..9 DO + INSERT INTO test.t1 (id, txt) VALUES (id, REPEAT(id,1024*1024)); + END FOR; +END; +$$ +DELIMITER ;$$ + + +--echo # Testing aria log files before --backup +SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; +--file_exists $ARIA_LOGDIR_FS/aria_log_control +--file_exists $ARIA_LOGDIR_FS/aria_log.00000001 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000002 +--error 1 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000003 +--replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ +SHOW ENGINE aria logs; + +--echo # mariadb-backup --backup +--disable_result_log +--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir +--enable_result_log + + +--echo # mariadb-backup --prepare +--disable_result_log +--exec $XTRABACKUP --prepare --target-dir=$targetdir +--enable_result_log + +--echo # shutdown server +--disable_result_log +--source include/shutdown_mysqld.inc +--echo # remove datadir +--rmdir $datadir +--echo # remove aria-log-dir-path +--rmdir $ARIA_LOGDIR_FS + + +--echo # mariadb-backup --copy-back +--let $mariadb_backup_parameters=--defaults-file=$MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=$datadir --target-dir=$targetdir --parallel=2 --throttle=1 --aria-log-dir-path=$ARIA_LOGDIR_MARIADB +--replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--exec echo "# with parameters: $mariadb_backup_parameters" +--exec $XTRABACKUP $mariadb_backup_parameters + + +--echo # starting server +--let $restart_parameters=$server_parameters +--source include/start_mysqld.inc +--enable_result_log +--rmdir $targetdir + + +--echo # Check that the table is there after --copy-back +SELECT COUNT(*) from t1; +DROP TABLE t1; + + +--echo # Testing aria log files after --copy-back +SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; +--file_exists $ARIA_LOGDIR_FS/aria_log_control +#--file_exists $ARIA_LOGDIR_FS/aria_log.00000001 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000002 +--error 1 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000003 +--replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ +SHOW ENGINE aria logs; + + +--echo # Restarting mariadbd with default parameters +--let $restart_parameters= +--source include/restart_mysqld.inc +--rmdir $ARIA_LOGDIR_FS diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path.test b/mysql-test/suite/mariabackup/aria_log_dir_path.test index 40bc39446bf00..e479458edfddb 100644 --- a/mysql-test/suite/mariabackup/aria_log_dir_path.test +++ b/mysql-test/suite/mariabackup/aria_log_dir_path.test @@ -1,105 +1,2 @@ ---source include/have_maria.inc - ---echo # ---echo # MDEV-30968 mariadb-backup does not copy Aria logs if aria_log_dir_path is used ---echo # - ---let $datadir=`SELECT @@datadir` ---let $targetdir=$MYSQLTEST_VARDIR/tmp/backup - -if ($ARIA_LOGDIR_MARIADB == '') -{ - --let $ARIA_LOGDIR_MARIADB=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path -} - -if ($ARIA_LOGDIR_FS == '') -{ - --let $ARIA_LOGDIR_FS=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path -} - ---let $server_parameters=--aria-log-file-size=8388608 --aria-log-purge-type=external --loose-aria-log-dir-path=$ARIA_LOGDIR_MARIADB - - ---echo # Restart mariadbd with the test specific parameters ---mkdir $ARIA_LOGDIR_FS ---let $restart_parameters=$server_parameters ---source include/restart_mysqld.inc - - ---echo # Create and populate an Aria table (and Aria logs) -CREATE TABLE t1 (id INT, txt LONGTEXT) ENGINE=Aria; -DELIMITER $$; -BEGIN NOT ATOMIC - FOR id IN 0..9 DO - INSERT INTO test.t1 (id, txt) VALUES (id, REPEAT(id,1024*1024)); - END FOR; -END; -$$ -DELIMITER ;$$ - - ---echo # Testing aria log files before --backup -SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; ---file_exists $ARIA_LOGDIR_FS/aria_log_control ---file_exists $ARIA_LOGDIR_FS/aria_log.00000001 ---file_exists $ARIA_LOGDIR_FS/aria_log.00000002 ---error 1 ---file_exists $ARIA_LOGDIR_FS/aria_log.00000003 ---replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ -SHOW ENGINE aria logs; - ---echo # mariadb-backup --backup ---disable_result_log ---mkdir $targetdir ---exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir ---enable_result_log - - ---echo # mariadb-backup --prepare ---disable_result_log ---exec $XTRABACKUP --prepare --target-dir=$targetdir ---enable_result_log - ---echo # shutdown server ---disable_result_log ---source include/shutdown_mysqld.inc ---echo # remove datadir ---rmdir $datadir ---echo # remove aria-log-dir-path ---rmdir $ARIA_LOGDIR_FS - - ---echo # mariadb-backup --copy-back ---let $mariadb_backup_parameters=--defaults-file=$MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=$datadir --target-dir=$targetdir --parallel=2 --throttle=1 --aria-log-dir-path=$ARIA_LOGDIR_MARIADB ---replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR $MYSQLTEST_VARDIR MYSQLTEST_VARDIR ---exec echo "# with parameters: $mariadb_backup_parameters" ---exec $XTRABACKUP $mariadb_backup_parameters - - ---echo # starting server ---let $restart_parameters=$server_parameters ---source include/start_mysqld.inc ---enable_result_log ---rmdir $targetdir - - ---echo # Check that the table is there after --copy-back -SELECT COUNT(*) from t1; -DROP TABLE t1; - - ---echo # Testing aria log files after --copy-back -SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; ---file_exists $ARIA_LOGDIR_FS/aria_log_control -#--file_exists $ARIA_LOGDIR_FS/aria_log.00000001 ---file_exists $ARIA_LOGDIR_FS/aria_log.00000002 ---error 1 ---file_exists $ARIA_LOGDIR_FS/aria_log.00000003 ---replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ -SHOW ENGINE aria logs; - - ---echo # Restarting mariadbd with default parameters ---let $restart_parameters= ---source include/restart_mysqld.inc ---rmdir $ARIA_LOGDIR_FS +--source include/have_mariabackup_combination.inc +--source aria_log_dir_path.inc diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test b/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test index c8169959929b9..b2875f496daac 100644 --- a/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test +++ b/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test @@ -1,4 +1,4 @@ --let $ARIA_LOGDIR_MARIADB=../../tmp/backup_aria_log_dir_path_rel --let $ARIA_LOGDIR_FS=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path_rel ---source aria_log_dir_path.test +--source aria_log_dir_path.inc diff --git a/mysql-test/suite/mariabackup/defer_space,SERVER.rdiff b/mysql-test/suite/mariabackup/defer_space,SERVER.rdiff new file mode 100644 index 0000000000000..f71d98dc11c82 --- /dev/null +++ b/mysql-test/suite/mariabackup/defer_space,SERVER.rdiff @@ -0,0 +1,10 @@ +--- defer_space.result ++++ defer_space,SERVER.result +@@ -21,7 +21,5 @@ + CREATE TABLE t1(c INT) ENGINE=INNODB; + # Corrupt the table + # restart +-# xtrabackup backup +-FOUND 10 /Header page consists of zero bytes*/ in backup.log + UNLOCK TABLES; + DROP TABLE t1; diff --git a/mysql-test/suite/mariabackup/defer_space.test b/mysql-test/suite/mariabackup/defer_space.test index 397a1ff5dc23c..bad0ed259600c 100644 --- a/mysql-test/suite/mariabackup/defer_space.test +++ b/mysql-test/suite/mariabackup/defer_space.test @@ -2,6 +2,7 @@ --source include/have_debug.inc --source include/not_embedded.inc --source include/no_valgrind_without_big.inc +--source include/have_mariabackup_combination.inc call mtr.add_suppression("InnoDB: Expected tablespace id .*"); --echo # Mariabackup --backup with page0 INIT_PAGE redo record @@ -51,6 +52,8 @@ close FILE or die "close"; EOF --source include/start_mysqld.inc +if (!$MTR_COMBINATION_SERVER) +{ echo # xtrabackup backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log; @@ -62,7 +65,8 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir= --let SEARCH_PATTERN=Header page consists of zero bytes* --let SEARCH_FILE=$backuplog --source include/search_pattern_in_file.inc -UNLOCK TABLES; -DROP TABLE t1; rmdir $targetdir; remove_file $backuplog; +} +UNLOCK TABLES; +DROP TABLE t1; diff --git a/mysql-test/suite/mariabackup/full_backup,SERVER.rdiff b/mysql-test/suite/mariabackup/full_backup,SERVER.rdiff new file mode 100644 index 0000000000000..4e2798c8a3b3d --- /dev/null +++ b/mysql-test/suite/mariabackup/full_backup,SERVER.rdiff @@ -0,0 +1,30 @@ +--- full_backup.result ++++ full_backup,SERVER.result +@@ -3,10 +3,6 @@ + SET GLOBAL innodb_max_purge_lag_wait=0; + # xtrabackup backup + NOT FOUND /InnoDB: Allocated tablespace ID/ in backup.log +-SELECT variable_value FROM information_schema.global_status +-WHERE variable_name = 'INNODB_BUFFER_POOL_PAGES_DIRTY'; +-variable_value +-0 + INSERT INTO t VALUES(2); + # xtrabackup prepare + # shutdown server +@@ -44,16 +40,3 @@ + # + # MDEV-34713: mariadb_upgrade_info should be backed up and restored + # +-CREATE TABLE t2(i INT) ENGINE INNODB; +-INSERT INTO t2 VALUES(100); +-# xtrabackup backup +-# xtrabackup prepare +-# shutdown server +-# remove datadir +-# xtrabackup move back +-# restart: --innodb_undo_tablespaces=0 +-FOUND 1 /^[0-9]+\.[0-9]+\.[0-9]+/ in mariadb_upgrade_info +-SELECT * FROM t2; +-i +-100 +-DROP TABLE t2; diff --git a/mysql-test/suite/mariabackup/full_backup.test b/mysql-test/suite/mariabackup/full_backup.test index 8a1ce756eeb17..1b75e4d2c051a 100644 --- a/mysql-test/suite/mariabackup/full_backup.test +++ b/mysql-test/suite/mariabackup/full_backup.test @@ -1,3 +1,4 @@ +--source include/have_mariabackup_combination.inc --source include/innodb_page_size.inc CREATE TABLE t(i INT) ENGINE INNODB; @@ -8,8 +9,13 @@ let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --let $backup_log=$MYSQLTEST_VARDIR/tmp/backup.log --disable_result_log +# The new BACKUP SERVER wrapper ignores --innodb-log-write-ahead-size, so the +# invalid-value invocation that is expected to fail does not apply to it. +if (!$MTR_COMBINATION_SERVER) +{ --error 1 exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --parallel=10 --innodb-log-write-ahead-size=4095 > $backup_log 2>&1; +} exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --parallel=10 --innodb-log-write-ahead-size=10000 --innodb_log_checkpoint_now=1 > $backup_log 2>&1; --enable_result_log @@ -19,8 +25,12 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir= --source include/search_pattern_in_file.inc --remove_file $backup_log +# Only the native mariabackup leaves dirty pages worth checking here. +if (!$MTR_COMBINATION_SERVER) +{ SELECT variable_value FROM information_schema.global_status WHERE variable_name = 'INNODB_BUFFER_POOL_PAGES_DIRTY'; +} INSERT INTO t VALUES(2); @@ -38,6 +48,8 @@ rmdir $targetdir; --echo # MDEV-27121 mariabackup incompatible with disabled dedicated --echo # undo log tablespaces --echo # +# The new BACKUP SERVER wrapper's prepare does not preserve prepared XA +# transactions across backup/restore, so this scenario does not apply to it. call mtr.add_suppression("InnoDB: innodb_undo_tablespaces=0 disables dedicated undo log tablespaces"); call mtr.add_suppression("InnoDB: Cannot change innodb_undo_tablespaces=0 because previous shutdown was not with innodb_fast_shutdown=0"); call mtr.add_suppression("Found 1 prepared XA transactions"); @@ -72,7 +84,8 @@ rmdir $targetdir; --echo # --echo # MDEV-34713: mariadb_upgrade_info should be backed up and restored --echo # - +if (!$MTR_COMBINATION_SERVER) +{ CREATE TABLE t2(i INT) ENGINE INNODB; INSERT INTO t2 VALUES(100); @@ -106,3 +119,4 @@ SELECT * FROM t2; DROP TABLE t2; --remove_file $_datadir/mariadb_upgrade_info rmdir $targetdir; +} diff --git a/mysql-test/suite/mariabackup/huge_lsn.test b/mysql-test/suite/mariabackup/huge_lsn.test index a7c9b0f3f0d3e..d50866d83e5ae 100644 --- a/mysql-test/suite/mariabackup/huge_lsn.test +++ b/mysql-test/suite/mariabackup/huge_lsn.test @@ -1,6 +1,6 @@ --source include/not_embedded.inc --source include/have_file_key_management.inc - +--source include/have_mariabackup_combination.inc --echo # --echo # MDEV-13416 mariabackup fails with EFAULT "Bad Address" --echo # diff --git a/mysql-test/suite/mariabackup/log_tables,SERVER.rdiff b/mysql-test/suite/mariabackup/log_tables,SERVER.rdiff new file mode 100644 index 0000000000000..921b39ff6cd58 --- /dev/null +++ b/mysql-test/suite/mariabackup/log_tables,SERVER.rdiff @@ -0,0 +1,16 @@ +--- mysql-test/suite/mariabackup/log_tables.result ++++ mysql-test/suite/mariabackup/log_tables,SERVER.result +@@ -10,7 +10,6 @@ + (command_type = "Query" OR command_type = "Execute") ; + event_time user_host thread_id server_id command_type argument + TIMESTAMP USER_HOST THREAD_ID 1 Query INSERT INTO t VALUES (1) +-# Insert new row into general_log table after it has been copied on BLOCK_DDL. + # Backup to dir. + # Xtrabackup prepare. + # shutdown server +@@ -22,5 +21,4 @@ + (command_type = "Query" OR command_type = "Execute") ; + event_time user_host thread_id server_id command_type argument + TIMESTAMP USER_HOST THREAD_ID 1 Query INSERT INTO t VALUES (1) +-TIMESTAMP USER_HOST THREAD_ID 1 Query INSERT INTO test.t VALUES (2) + DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/log_tables.test b/mysql-test/suite/mariabackup/log_tables.test index 707057a80e642..41c8581e2659f 100644 --- a/mysql-test/suite/mariabackup/log_tables.test +++ b/mysql-test/suite/mariabackup/log_tables.test @@ -1,6 +1,7 @@ # Test for copying log tables tail --source include/have_aria.inc --source include/have_debug.inc +--source include/have_mariabackup_combination.inc --let $targetdir=$MYSQLTEST_VARDIR/tmp/backup @@ -21,8 +22,11 @@ SELECT * FROM mysql.general_log WHERE argument LIKE "INSERT INTO %" AND (command_type = "Query" OR command_type = "Execute") ; +if (!$MTR_COMBINATION_SERVER) +{ --echo # Insert new row into general_log table after it has been copied on BLOCK_DDL. --let after_stage_block_ddl=INSERT INTO test.t VALUES (2) +} --echo # Backup to dir. --disable_result_log diff --git a/mysql-test/suite/mariabackup/mdev-18438.test b/mysql-test/suite/mariabackup/mdev-18438.test index a6ec45476ff2c..f05b06b147384 100644 --- a/mysql-test/suite/mariabackup/mdev-18438.test +++ b/mysql-test/suite/mariabackup/mdev-18438.test @@ -1,8 +1,14 @@ +--source include/have_mariabackup_combination.inc let $basedir=$MYSQLTEST_VARDIR/tmp/mdev-18438; mkdir $basedir; exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --extra-lsndir=$basedir/extra_lsndir --stream=xbstream > $basedir/stream.xb; mkdir $basedir/backup; +# The BACKUP SERVER wrapper ignores --extra-lsndir, so it never creates +# extra_lsndir; only the native tool does. +if (!$MTR_COMBINATION_SERVER) +{ rmdir $basedir/extra_lsndir; +} --disable_result_log exec $XBSTREAM -x -C $basedir/backup < $basedir/stream.xb; --enable_result_log diff --git a/mysql-test/suite/mariabackup/partition_notwin,SERVER.rdiff b/mysql-test/suite/mariabackup/partition_notwin,SERVER.rdiff new file mode 100644 index 0000000000000..dba5eb260296d --- /dev/null +++ b/mysql-test/suite/mariabackup/partition_notwin,SERVER.rdiff @@ -0,0 +1,8 @@ +--- partition_notwin.result ++++ partition_notwin,SERVER.result +@@ -7,5 +7,4 @@ + ) engine=myisam + partition by hash (id) + partitions 600; +-FOUND 1 /Error 24 on file ./test/t1#P#p\d+\.MY[DI] open during `test`.`t1` table copy: Too many open files/ in backup.log + drop table t1; diff --git a/mysql-test/suite/mariabackup/partition_notwin.test b/mysql-test/suite/mariabackup/partition_notwin.test index 10687e19935e6..85bafcfb5b83d 100644 --- a/mysql-test/suite/mariabackup/partition_notwin.test +++ b/mysql-test/suite/mariabackup/partition_notwin.test @@ -1,5 +1,6 @@ source include/not_windows.inc; source include/have_partition.inc; +source include/have_mariabackup_combination.inc; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; let $log=$MYSQL_TMP_DIR/backup.log; @@ -14,11 +15,21 @@ create table t1 ( partition by hash (id) partitions 600; -error 1; +let $errno = 1; +if ($MTR_COMBINATION_SERVER) +{ +let $errno = 0; +} +--error $errno exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir > $log 2>&1; + +# The "Too many open files" diagnostic only applies to native mariabackup. +if (!$MTR_COMBINATION_SERVER) +{ let SEARCH_FILE=$log; let SEARCH_PATTERN=Error 24 on file ./test/t1#P#p\d+\.MY[DI] open during `test`.`t1` table copy: Too many open files; source include/search_pattern_in_file.inc; +} rmdir $targetdir; #remove_file $log; diff --git a/mysql-test/suite/mariabackup/relative_path.test b/mysql-test/suite/mariabackup/relative_path.test index bd25a217e711e..60c287403165c 100644 --- a/mysql-test/suite/mariabackup/relative_path.test +++ b/mysql-test/suite/mariabackup/relative_path.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/have_mariabackup_combination.inc CREATE TABLE t(i INT) ENGINE INNODB; INSERT INTO t VALUES(1); diff --git a/mysql-test/suite/mariabackup/row_format_redundant.test b/mysql-test/suite/mariabackup/row_format_redundant.test index 5bae9218d840c..1820fec9ebfaa 100644 --- a/mysql-test/suite/mariabackup/row_format_redundant.test +++ b/mysql-test/suite/mariabackup/row_format_redundant.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/have_mariabackup_combination.inc --let $targetdir=$MYSQLTEST_VARDIR/tmp/backup diff --git a/mysql-test/suite/mariabackup/small_ibd.test b/mysql-test/suite/mariabackup/small_ibd.test index bb476b8771e98..9076abff6c4cf 100644 --- a/mysql-test/suite/mariabackup/small_ibd.test +++ b/mysql-test/suite/mariabackup/small_ibd.test @@ -1,4 +1,5 @@ --source include/innodb_page_size.inc +--source include/have_mariabackup_combination.inc # Check if ibd smaller than page size are skipped # It is possible, due to race conditions that new file diff --git a/mysql-test/suite/mariabackup/undo_space_id,SERVER.rdiff b/mysql-test/suite/mariabackup/undo_space_id,SERVER.rdiff new file mode 100644 index 0000000000000..d886111b06fb7 --- /dev/null +++ b/mysql-test/suite/mariabackup/undo_space_id,SERVER.rdiff @@ -0,0 +1,13 @@ +--- undo_space_id.result ++++ undo_space_id,SERVER.result +@@ -11,10 +11,3 @@ + undo001 + undo002 + DROP TABLE t1; +-# +-# MDEV-33980 mariadb-backup --backup is missing +-# retry logic for undo tablespaces +-# +-# xtrabackup backup +-# Display undo log files from target directory +-FOUND 5 /Retrying to read undo tablespace*/ in backup.log diff --git a/mysql-test/suite/mariabackup/undo_space_id.test b/mysql-test/suite/mariabackup/undo_space_id.test index 168740fc528ce..e239189040c89 100644 --- a/mysql-test/suite/mariabackup/undo_space_id.test +++ b/mysql-test/suite/mariabackup/undo_space_id.test @@ -1,5 +1,6 @@ --source include/have_innodb.inc --source include/have_debug.inc +--source include/have_mariabackup_combination.inc --echo # Create 2 UNDO TABLESPACE(UNDO001(space_id =3), UNDO002(space_id =4)) @@ -24,6 +25,8 @@ list_files $basedir undo*; DROP TABLE t1; rmdir $basedir; +if (!$MTR_COMBINATION_SERVER) +{ --echo # --echo # MDEV-33980 mariadb-backup --backup is missing --echo # retry logic for undo tablespaces @@ -42,3 +45,4 @@ list_files $basedir undo*; --source include/search_pattern_in_file.inc rmdir $basedir; remove_file $backuplog; +} diff --git a/mysql-test/suite/mariabackup/undo_truncate.test b/mysql-test/suite/mariabackup/undo_truncate.test index a23c9cf64ff6b..901bf73920a41 100644 --- a/mysql-test/suite/mariabackup/undo_truncate.test +++ b/mysql-test/suite/mariabackup/undo_truncate.test @@ -2,6 +2,7 @@ --source include/not_embedded.inc --source include/have_sequence.inc --source include/have_file_key_management.inc +--source include/have_mariabackup_combination.inc SET GLOBAL innodb_undo_log_truncate = 0; diff --git a/mysql-test/suite/mariabackup/unencrypted_page_compressed,SERVER.rdiff b/mysql-test/suite/mariabackup/unencrypted_page_compressed,SERVER.rdiff new file mode 100644 index 0000000000000..e785a0601fe94 --- /dev/null +++ b/mysql-test/suite/mariabackup/unencrypted_page_compressed,SERVER.rdiff @@ -0,0 +1,8 @@ +--- unencrypted_page_compressed.result ++++ unencrypted_page_compressed,SERVER.result +@@ -8,5 +8,4 @@ + # Corrupt the table + # restart: --skip-innodb-buffer-pool-load-at-startup + # xtrabackup backup +-FOUND 1 /Database page corruption detected.*/ in backup.log + drop table t1; diff --git a/mysql-test/suite/mariabackup/unencrypted_page_compressed.test b/mysql-test/suite/mariabackup/unencrypted_page_compressed.test index 68f22e69e0325..fc79a0a211e89 100644 --- a/mysql-test/suite/mariabackup/unencrypted_page_compressed.test +++ b/mysql-test/suite/mariabackup/unencrypted_page_compressed.test @@ -1,3 +1,4 @@ +--source include/have_mariabackup_combination.inc call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`.`t1` has an unreadable root page"); @@ -39,6 +40,8 @@ echo # xtrabackup backup; --disable_result_log let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log; +if (!$MTR_COMBINATION_SERVER) +{ --error 1 exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --core-file > $backuplog; --enable_result_log @@ -47,6 +50,7 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --let SEARCH_FILE=$backuplog --source include/search_pattern_in_file.inc remove_file $backuplog; +rmdir $targetdir; +} drop table t1; -rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/vector.test b/mysql-test/suite/mariabackup/vector.test index 4f60ddce10e9c..1889d8c412ec3 100644 --- a/mysql-test/suite/mariabackup/vector.test +++ b/mysql-test/suite/mariabackup/vector.test @@ -1,3 +1,4 @@ +--source include/have_mariabackup_combination.inc create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)) engine=innodb; insert t1 (v) values (Vec_Fromtext('[0.418,0.809,0.823,0.598,0.033]')), (Vec_Fromtext('[0.687,0.789,0.496,0.574,0.917]')),