From 8596f51e83e5d90b2717f62e6b7928b098c4e52a Mon Sep 17 00:00:00 2001 From: Andrzej Jarzabek Date: Thu, 28 May 2026 14:31:32 +0200 Subject: [PATCH 1/7] Implement table and file copy using backup_step --- storage/maria/ha_maria.cc | 2 +- storage/maria/ma_backup_server.cc | 515 ++++++++++++++++++++++++------ 2 files changed, 415 insertions(+), 102 deletions(-) diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 802ad0f9fcc19..991ce6f7611eb 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -3944,7 +3944,7 @@ static int ha_maria_init(void *p) maria_hton->end_backup= maria_end_backup; maria_hton->update_optimizer_costs= aria_update_optimizer_costs; maria_hton->backup_start= aria_backup_start; - //maria_hton->backup_step= aria_backup_step; + maria_hton->backup_step= aria_backup_step; maria_hton->backup_end= aria_backup_end; /* TODO: decide if we support Maria being used for log tables */ diff --git a/storage/maria/ma_backup_server.cc b/storage/maria/ma_backup_server.cc index 25777dff3c19a..417a9e01699ec 100644 --- a/storage/maria/ma_backup_server.cc +++ b/storage/maria/ma_backup_server.cc @@ -20,11 +20,17 @@ # include "sql_class.h" # include "table_cache.h" #endif +#include +#include +#include #include #include #include #include #include "span.h" +#include +#include +#include /* Implementation of functions declatred in ma_backup.h: @@ -33,7 +39,58 @@ namespace { - /** Backup state; protected by log_sys.latch */ + /* Utility class to implement the "backup step" interface when + processing several lists. It implements the logic where an item + is processed (copied) from the first list which has available + items, and a "remaining" counter accumulates the number of + items remaining to be processed on all lists, regardless of + whether an item from that list was processed or not. */ + class Copy_from_list + { + int m_remaining {0}; + bool m_copy_done; + public: + Copy_from_list(bool copy_done= false) noexcept + : m_copy_done(copy_done) + { + } + + bool copy_done() const noexcept + { + return m_copy_done; + } + + int remaining() const noexcept + { + return m_remaining; + } + + template + bool operator()(const T &list, std::atomic &copied, + Fn copy_action) noexcept + { + if(!m_copy_done) + { + size_t idx= copied.fetch_add(1, std::memory_order_relaxed); + if (idx < list.size()) + { + if (copy_action(list[idx]) != 0) + return 1; + m_copy_done= true; + m_remaining+= list.size() - idx - 1U; + } + } + else + { + size_t current_copied= copied.load(std::memory_order_relaxed); + if (current_copied < list.size()) + m_remaining+= list.size() - current_copied; + } + return 0; + } + }; + + class Aria_backup { public: @@ -76,13 +133,93 @@ namespace return false; } - int end(const backup_target &target, const backup_sink &sink) noexcept + bool start_copy_dml_safe(const backup_target &target, const backup_sink &sink) noexcept + { + assert(translog_purge_disabled); + if (scan_dbdirs()) + return true; + flatten_table_lists(); + if (sink.stream == sink.NO_STREAM) + return ensure_target_dirs(target); + return false; + } + + bool start_copy_unsafe() noexcept + { + if (scan_logs()) + return true; + return false; + } + + /* Copy an Aria table that is safe to be copied while concurrent DML + is in progress. */ + int dml_safe_copy_step(const backup_target &target, const backup_sink &sink) noexcept + { + Copy_from_list copy_from_list; + auto copy_table_action= [this, target, sink](const table_ref &table) noexcept + { + return copy_table(target, sink, table); + }; + if (copy_from_list(dml_safe_table_list, dml_safe_tables_copied, + copy_table_action) != 0) + return -1; + if (copy_from_list(unsafe_tables_list, unsafe_tables_copied, + copy_table_action) != 0) + return -1; + if (copy_from_list(misc_files, misc_files_copied, + [this, target, sink](const std::string &path) noexcept + { + return copy_file(target, sink, path, false); + }) != 0) + return -1; + return copy_from_list.remaining(); + } + + /* Copy an entity that is not safe to copy if there are concurrent + writes to it. One entity is copied, of the first category that has + any remaning entities to be copied. Returns the total number of + entities to be copied in all categories. Categories in order: + - log control file + - log files + - Aria tables + - other ("miscellaneous") files + */ + int unsafe_copy_step(const backup_target &target, const backup_sink &sink) noexcept + { + bool copy_done= false; + + /* If control file is always the first file copied and there is only + one, it is never included in the "steps remaining" calculation. + Should the order be changed, the calculation needs to be updated for + the control file as well. */ + if (have_control_file) + { + bool already_copied= control_file_copied.exchange(true); + if (!already_copied) + { + if (copy_control_file(target, sink) != 0) + return -1; + copy_done= true; + } + } + + Copy_from_list copy_from_list(copy_done); + if (copy_from_list(log_files, log_files_copied, + [this, target, sink](const std::string &path) noexcept + { + return copy_file(target, sink, path, false); + }) != 0) + return -1; + + return copy_from_list.remaining(); + } + + int end(bool /*abort*/) noexcept { - int ret_val= perform_backup(target, sink); assert(translog_purge_disabled); translog_purge_disabled= false; translog_enable_purge(); - return ret_val; + return 0; } private: #ifndef _WIN32 @@ -94,20 +231,44 @@ namespace /** whether the Aria translog_disable_purge() is in effect */ bool translog_purge_disabled{false}; static constexpr const char zerobuf[511]{}; + /* All file suffixes are 4 characters long (dot and 3 letter extension) */ + static constexpr size_t suffix_len= 4; + static const char* data_ext; + static const char* index_ext; + static const LEX_CSTRING log_file_prefix; + static const LEX_CSTRING tmp_prefix; + static const char* control_file_name; + using dir_name = std::string; using dir_contents = std::vector; - using database_dir = std::pair; - std::vector database_dirs; + using database_dir = std::pair; + using database_dirs = std::vector; + /* Transactional tables with checksum */ + database_dirs dml_safe_tables; + /* All other Aria tables */ + database_dirs unsafe_tables; + /* Aria log files */ std::vector log_files; + std::vector misc_files; + /* directories in which misc files are */ + std::vector misc_dirs; + bool have_control_file = false; + bool safe_files_copied = false; - int perform_backup(const backup_target &target, const backup_sink &sink) - noexcept - { - return scan_datadir() || copy_databases(target, sink) || - copy_control_file(target, sink) || - translog_flush(translog_get_horizon()) || - copy_logs(target, sink); - } + /* Refer to a string stored elsewhere */ + using dir_ref= std::string_view; + using tablename_ref= std::string_view; + using table_ref= std::pair; + using table_list= std::vector; + + /* Flattened versions of dml_safe_tables and unsafe_tables. */ + table_list dml_safe_table_list; + table_list unsafe_tables_list; + std::atomic dml_safe_tables_copied {0}; + std::atomic unsafe_tables_copied {0}; + std::atomic log_files_copied {0}; + std::atomic misc_files_copied {0}; + std::atomic control_file_copied {false}; ATTRIBUTE_COLD ATTRIBUTE_NOINLINE static int dir_error(const char *name) noexcept @@ -116,7 +277,7 @@ namespace return 1; } - int scan_datadir() noexcept + int scan_dbdirs() noexcept { /* Scan the server data directory for Aria table files. */ MY_DIR *data_dir= my_dir(mysql_real_data_home, MYF(MY_WANT_STAT)); @@ -127,27 +288,14 @@ namespace st_::span{data_dir->dir_entry, data_dir->number_of_files}) if ((fi.mystat->st_mode & S_IFMT) == S_IFDIR) - if ((fail= scan_database_dir(fi.name)) != 0) - break; + { + fail= scan_database_dir(fi.name); + if (fail != 0) + goto func_exit; + } + func_exit: my_dirend(data_dir); - if (fail) - return fail; - - /* Scan aria_log_dir_path for the transaction logs and control file. */ - MY_DIR *log_dir= my_dir(maria_data_root, MYF(MY_WANT_STAT)); - if (!log_dir) - return dir_error(maria_data_root); - for (const fileinfo &fi : - st_::span{log_dir->dir_entry, - log_dir->number_of_files}) - { - if (!strncmp(fi.name, C_STRING_WITH_LEN("aria_log."))) - log_files.emplace_back(fi.name); - else if (!strcmp(fi.name, "aria_log_control")) - have_control_file = true; - } - my_dirend(log_dir); - return 0; + return fail; } int scan_database_dir(const char* dir_name) noexcept @@ -156,30 +304,115 @@ namespace MY_DIR *dir_info= my_dir(dir_path.c_str(), MYF(MY_WANT_STAT)); if (!dir_info) return dir_error(dir_path.c_str()); - std::vector files_to_backup; + int fail= 0; + dir_contents safe; + dir_contents unsafe; + for (const fileinfo &fi : + st_::span{dir_info->dir_entry, + dir_info->number_of_files}) + { + const char* filename= fi.name; + size_t filename_len = strlen(filename); + if (filename_len >= suffix_len) + { + const char* suffix = filename + filename_len - suffix_len; + if(match_suffix(suffix, index_ext)) + { + if (!is_tmp_table(filename)) + { + auto is_safe = is_safe_table(dir_name, filename); + if (std::holds_alternative(is_safe)) + { + std::string table_name(filename, filename_len - suffix_len); + if (std::get(is_safe)) + safe.push_back(std::move(table_name)); + else + unsafe.push_back(std::move(table_name)); + } + else + { + fail= std::get(is_safe); + goto finish; + } + } + } + else if (match_misc_ext(suffix) || !strcmp(filename, "db.opt")) + { + if(misc_dirs.empty() || misc_dirs.back() != dir_name) + misc_dirs.emplace_back(dir_name); + misc_files.push_back(std::string(dir_name) + "/" + filename); + } + } + } + if(!fail) + { + if (!safe.empty()) + dml_safe_tables.emplace_back(dir_name, std::move(safe)); + if (!unsafe.empty()) + unsafe_tables.emplace_back(dir_name, std::move(unsafe)); + } + finish: + my_dirend(dir_info); + return fail; + } + + static bool is_tmp_table(const char* filename) noexcept + { + return begins_with(filename, tmp_prefix); + } + + void flatten_table_lists() noexcept + { + flatten_table_list(dml_safe_tables, dml_safe_table_list); + flatten_table_list(unsafe_tables, unsafe_tables_list); + } + + static void flatten_table_list(const database_dirs& dirs, table_list& list) noexcept + { + for (const database_dir& dir : dirs) + { + for (const std::string& table : dir.second) + list.emplace_back(dir.first, table); + } + } + + int scan_logs() noexcept + { + const char *base_dir= maria_data_root; + MY_DIR *dir_info= my_dir(base_dir, MYF(MY_WANT_STAT)); + if (!dir_info) + return dir_error(base_dir); for (const fileinfo &fi : st_::span{dir_info->dir_entry, dir_info->number_of_files}) - if (is_db_file(fi.name)) - files_to_backup.emplace_back(fi.name); - if (!files_to_backup.empty()) - database_dirs.emplace_back(dir_name, std::move(files_to_backup)); + if (begins_with(fi.name, log_file_prefix)) + log_files.emplace_back(fi.name); + else if (strcmp(fi.name, "aria_log_control") == 0) + have_control_file = true; my_dirend(dir_info); return 0; } - int copy_databases(const backup_target &target, const backup_sink &sink) - noexcept + bool ensure_target_dirs(const backup_target &target) noexcept { - for (const database_dir &dir : database_dirs) + using string = std::string; + std::vector dirs; + for (const database_dir &dir : dml_safe_tables) + dirs.push_back(&dir.first); + for (const database_dir &dir : unsafe_tables) + dirs.push_back(&dir.first); + for (const string &dir: misc_dirs) + dirs.push_back(&dir); + std::sort(dirs.begin(), dirs.end(), + [](const string *a, const string *b) { return *a < *b; }); + auto dirs_end = std::unique(dirs.begin(), dirs.end(), + [](const string *a, const string *b) { return *a == *b; }); + for (auto it = dirs.begin(); it != dirs_end; ++it) { - if (sink.stream != sink.NO_STREAM); - else if (int fail= ensure_target_subdir(target, dir.first.c_str())) - return fail; - if (int fail= copy_database(target, sink, dir)) - return fail; + if (ensure_target_subdir(target, (*it)->c_str())) + return true; } - return 0; + return false; } /* @@ -204,19 +437,64 @@ namespace return 1; } - int copy_database(const backup_target &target, const backup_sink &sink, - const database_dir& dir) noexcept + /* Returns result or error code. */ + std::variant is_safe_table(const char* dir_name, const char* myi_file_name) { - std::string file_path; - for (const std::string &file : dir.second) + ARIA_TABLE_CAPABILITIES cap; +#ifndef _WIN32 + std::string path= std::string(dir_name) + "/" + myi_file_name; + File fd= openat(datadir_fd, path.c_str(), O_RDONLY); + if (fd < 0) { - file_path= dir.first; - file_path.push_back('/'); - file_path.append(file); - if (int fail= copy_file(target, sink, file_path.c_str(), false)) - return fail; + my_errno= errno; + my_error(ER_CANT_OPEN_FILE, MYF(0), path.c_str(), errno); } - return 0; +#else + std::string path= std::string(maria_data_root) + "/" + + dir_name + "/" + myi_file_name; + File fd= my_open(path.c_str(), O_RDONLY, MYF(MY_WME)); +#endif + if (fd < 0) + { + return my_errno; + } + std::variant result; + mysql_mutex_lock(&THR_LOCK_maria); + int fail = aria_get_capabilities(fd, myi_file_name, &cap); + if (fail) + { + my_error(ER_FILE_CORRUPT, MYF(0), path.c_str()); + result= fail; + goto end; + } + result = cap.transactional && cap.checksum; + aria_free_capabilities(&cap); +end: + mysql_mutex_unlock(&THR_LOCK_maria); + close(fd); + return result; + } + + int copy_table(const backup_target &target, const backup_sink &sink, + const table_ref& table) noexcept + { + dir_ref dir_name = table.first; + tablename_ref table_name = table.second; + std::string index_path; + index_path.reserve(dir_name.size() + table_name.size() + 5); + index_path= dir_name; + index_path += "/"; + size_t dir_part_length = index_path.size(); + index_path.resize(index_path.size() + table_name.size()); + std::copy(table_name.begin(), table_name.end(), + index_path.begin() + dir_part_length); + std::string data_path; + data_path.reserve(dir_name.size() + table_name.size() + 5); + data_path= index_path; + index_path+= index_ext; + data_path+= data_ext; + return copy_file(target, sink, index_path, false) || + copy_file(target, sink, data_path, false); } int copy_control_file(const backup_target &target, const backup_sink &sink) @@ -224,16 +502,13 @@ namespace { if (!have_control_file) return 0; - return copy_file(target, sink, "aria_log_control", true); + return copy_file(target, sink, control_file_name, true); } - int copy_logs(const backup_target &target, const backup_sink &sink) - noexcept + int copy_file(const backup_target &target, const backup_sink &sink, + const std::string &path, bool is_log) const noexcept { - for (const std::string &file : log_files) - if (int fail= copy_file(target, sink, file.c_str(), true)) - return fail; - return 0; + return copy_file(target, sink, path.c_str(), is_log); } int copy_file(const backup_target &target, const backup_sink &sink, @@ -348,24 +623,23 @@ namespace #endif } + static bool match_suffix(const char* suffix1, const char* suffix2) noexcept + { + return !memcmp(suffix1, suffix2, suffix_len); + } - static bool is_db_file(const char* file_name) noexcept + /* Match if suffix is one of the "other" extensions we need to copy */ + static bool match_misc_ext(const char* suffix_str) noexcept { - size_t len= strlen(file_name); - if (len < 4) - return false; uint32_t suffix; - memcpy(&suffix, file_name + len - 4, 4); + static_assert (suffix_len == sizeof(suffix)); + memcpy(&suffix, suffix_str, suffix_len); switch (suffix) { - default: - return len == 6 && !memcmp(file_name, C_STRING_WITH_LEN("db.opt")); #ifdef WORDS_BIGENDIAN case 0x2e41524d: /* .ARM ENGINE=ARCHIVE metadata */ case 0x2e41525a: /* .ARZ ENGINE=ARCHIVE compressed data */ case 0x2e43534d: /* .CSM ENGINE=CSV metadata */ case 0x2e435356: /* .CSV ENGINE=CSV data ("comma separated values") */ - case 0x2e4d4144: /* .MAD ENGINE=Aria data heap */ - case 0x2e4d4149: /* .MAI ENGINE=Aria indexes */ case 0x2e4d5247: /* .MRG ENGINE=MRG_MyISAM */ case 0x2e4d5944: /* .MYD ENGINE=MyISAM data heap */ case 0x2e4d5949: /* .MYI ENGINE=MyISAM indexes */ @@ -376,8 +650,6 @@ namespace case 0x5a52412e: /* .ARZ ENGINE=ARCHIVE compressed data */ case 0x4d53432e: /* .CSM ENGINE=CSV metadata */ case 0x5653432e: /* .CSV ENGINE=CSV data ("comma separated values") */ - case 0x44414d2e: /* .MAD ENGINE=Aria data heap */ - case 0x49414d2e: /* .MAI ENGINE=Aria indexes */ case 0x47524d2e: /* .MRG ENGINE=MRG_MyISAM */ case 0x44594d2e: /* .MYD ENGINE=MyISAM data heap */ case 0x49594d2e: /* .MYI ENGINE=MyISAM indexes */ @@ -385,15 +657,17 @@ namespace case 0x7261702e: /* .par PARTITION metadata */ #endif return true; + default: + return false; } } - /** - Construct a file path. - @param dir directory name - @param name file name - @return dir/name - */ + static bool begins_with(const char* str, const LEX_CSTRING &prefix) noexcept + { + return strncmp(str, prefix.str, prefix.length) == 0; + } + + /** @return the target directory path */ static std::string make_path(const char *dir, const char *name) { std::string path{dir}; @@ -402,52 +676,91 @@ namespace return path; } }; + + + const LEX_CSTRING Aria_backup::log_file_prefix {C_STRING_WITH_LEN("aria_log.")}; + const LEX_CSTRING Aria_backup::tmp_prefix {C_STRING_WITH_LEN("#sql")}; + const char* Aria_backup::data_ext (MARIA_NAME_DEXT); + const char* Aria_backup::index_ext (MARIA_NAME_IEXT); + const char* Aria_backup::control_file_name {"aria_log_control"}; } void *aria_backup_start(THD *thd, const backup_target *target, backup_phase phase, const backup_sink *sink) noexcept { - switch (phase) { - case BACKUP_PHASE_PREPARE_START: + Aria_backup *aria_backup {}; + if (phase == BACKUP_PHASE_PREPARE_START) + { return 0; - default: - return sink->ha_data; - case BACKUP_PHASE_NO_COMMIT: + } + else if (phase == BACKUP_PHASE_START) + { assert(!sink->ha_data); - Aria_backup *aria_backup{new Aria_backup}; + aria_backup= new Aria_backup(); if (aria_backup->initialize()) { delete aria_backup; - return reinterpret_cast(-1); + goto error; } return aria_backup; } + + assert (sink->ha_data != reinterpret_cast(-1)); + aria_backup= static_cast(sink->ha_data); + assert(aria_backup); + switch(phase) + { + case BACKUP_PHASE_NO_DDL: +#if 1 // FIXME: invoke these only for Aria, MyISAM, CSV but not others + tc_purge(); + tdc_purge(true); +#endif + if (aria_backup->start_copy_dml_safe(*target, *sink)) + goto error; + break; + case BACKUP_PHASE_NO_COMMIT: + if (aria_backup->start_copy_unsafe()) + goto error; + break; + default: + break; + } + return sink->ha_data; +error: + return reinterpret_cast(-1); } -#if 0 // FIXME: implement the actual copying here -int aria_backup_step(THD*, const backup_target*, backup_phase, - const backup_sink*) noexcept + +int aria_backup_step(THD*, const backup_target *target, backup_phase phase, + const backup_sink *sink) noexcept { - return 0; + assert (sink->ha_data != reinterpret_cast(-1)); + Aria_backup *aria_backup= static_cast(sink->ha_data); + assert(aria_backup); + switch (phase) + { + case BACKUP_PHASE_NO_DDL: + return aria_backup->dml_safe_copy_step(*target, *sink); + case BACKUP_PHASE_NO_COMMIT: + return aria_backup->unsafe_copy_step(*target, *sink); + default: + return 0; + } } -#endif int aria_backup_end(THD *thd, const backup_target *target, backup_phase phase, const backup_sink *sink) noexcept { + assert (sink->ha_data != reinterpret_cast(-1)); Aria_backup *aria_backup= static_cast(sink->ha_data); switch (phase) { case BACKUP_PHASE_NO_COMMIT: assert(aria_backup); -#if 1 // FIXME: invoke these only for Aria, MyISAM, CSV but not others - tc_purge(); - tdc_purge(true); -#endif - return aria_backup->end(*target, *sink); + return aria_backup->end(thd); case BACKUP_PHASE_FINISH: delete aria_backup; /* fall through */ default: return 0; } -} +} \ No newline at end of file From 7bf8289fcf097b908a54a2ab28ee96b8e79544b4 Mon Sep 17 00:00:00 2001 From: Andrzej Jarzabek Date: Tue, 9 Jun 2026 16:28:07 +0200 Subject: [PATCH 2/7] Fix Windows build issue. --- storage/maria/ma_backup_server.cc | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/storage/maria/ma_backup_server.cc b/storage/maria/ma_backup_server.cc index 417a9e01699ec..0628cbea3a444 100644 --- a/storage/maria/ma_backup_server.cc +++ b/storage/maria/ma_backup_server.cc @@ -75,18 +75,18 @@ namespace if (idx < list.size()) { if (copy_action(list[idx]) != 0) - return 1; + return true; m_copy_done= true; - m_remaining+= list.size() - idx - 1U; + m_remaining+= static_cast(list.size() - idx - 1U); } } else { size_t current_copied= copied.load(std::memory_order_relaxed); if (current_copied < list.size()) - m_remaining+= list.size() - current_copied; + m_remaining+= static_cast(list.size() - current_copied); } - return 0; + return false; } }; @@ -679,7 +679,7 @@ namespace const LEX_CSTRING Aria_backup::log_file_prefix {C_STRING_WITH_LEN("aria_log.")}; - const LEX_CSTRING Aria_backup::tmp_prefix {C_STRING_WITH_LEN("#sql")}; + const LEX_CSTRING Aria_backup::tmp_prefix {C_STRING_WITH_LEN(tmp_file_prefix)}; const char* Aria_backup::data_ext (MARIA_NAME_DEXT); const char* Aria_backup::index_ext (MARIA_NAME_IEXT); const char* Aria_backup::control_file_name {"aria_log_control"}; @@ -710,6 +710,12 @@ void *aria_backup_start(THD *thd, const backup_target *target, assert(aria_backup); switch(phase) { +#if 1 // FIXME: invoke these only for Aria, MyISAM, CSV but not others + case BACKUP_PHASE_NO_DML_NON_TRANS: + tc_purge(); + tdc_purge(true); + break; +#endif case BACKUP_PHASE_NO_DDL: #if 1 // FIXME: invoke these only for Aria, MyISAM, CSV but not others tc_purge(); From 7fb3d05b23f108ad8cd64531c946f8faeedd0b63 Mon Sep 17 00:00:00 2001 From: Andrzej Jarzabek Date: Tue, 9 Jun 2026 20:34:50 +0200 Subject: [PATCH 3/7] Incorporate changes from code review. --- storage/maria/ma_backup_server.cc | 71 ++++++++++++++++--------------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/storage/maria/ma_backup_server.cc b/storage/maria/ma_backup_server.cc index 0628cbea3a444..73a44a94db1b7 100644 --- a/storage/maria/ma_backup_server.cc +++ b/storage/maria/ma_backup_server.cc @@ -233,11 +233,14 @@ namespace static constexpr const char zerobuf[511]{}; /* All file suffixes are 4 characters long (dot and 3 letter extension) */ static constexpr size_t suffix_len= 4; - static const char* data_ext; - static const char* index_ext; - static const LEX_CSTRING log_file_prefix; - static const LEX_CSTRING tmp_prefix; - static const char* control_file_name; + static constexpr const char* data_ext {MARIA_NAME_DEXT}; + static constexpr const char* index_ext {MARIA_NAME_IEXT}; + static constexpr LEX_CSTRING log_file_prefix {C_STRING_WITH_LEN("aria_log.")}; + static constexpr LEX_CSTRING tmp_prefix {C_STRING_WITH_LEN(tmp_file_prefix)}; + /* TODO: .frm failes are not Aria-specific, .MYD and .MYI are MyISAM files; + they are copied here as a stop-gap */ + static constexpr const char* misc_exts[] {".MYD", ".MYI", ".frm"}; + static constexpr const char* control_file_name {"aria_log_control"}; using dir_name = std::string; using dir_contents = std::vector; using database_dir = std::pair; @@ -300,7 +303,8 @@ namespace int scan_database_dir(const char* dir_name) noexcept { - const std::string dir_path{make_path(mysql_real_data_home, dir_name)}; + const char* base_dir = maria_data_root; + const std::string dir_path= build_path(base_dir, dir_name); MY_DIR *dir_info= my_dir(dir_path.c_str(), MYF(MY_WANT_STAT)); if (!dir_info) return dir_error(dir_path.c_str()); @@ -340,7 +344,7 @@ namespace { if(misc_dirs.empty() || misc_dirs.back() != dir_name) misc_dirs.emplace_back(dir_name); - misc_files.push_back(std::string(dir_name) + "/" + filename); + misc_files.push_back(build_path(dir_name, filename)); } } } @@ -423,12 +427,16 @@ namespace noexcept { #ifdef _WIN32 - if (CreateDirectory(make_path(target.path, name).c_str(), nullptr)) - return 0; - DWORD err= GetLastError(); - if (err == ERROR_ALREADY_EXISTS) - return 0; - my_osmaperr(err); + const std::string dir_path= build_path(target.path, name); + if (!CreateDirectory(dir_path.c_str(), nullptr)) + { + DWORD err = GetLastError(); + if (err != ERROR_ALREADY_EXISTS) + { + my_osmaperr(err); + return 1; + } + } #else if (likely(!mkdirat(target.fd, name, 0777) || errno == EEXIST)) return 0; @@ -483,11 +491,8 @@ namespace std::string index_path; index_path.reserve(dir_name.size() + table_name.size() + 5); index_path= dir_name; - index_path += "/"; - size_t dir_part_length = index_path.size(); - index_path.resize(index_path.size() + table_name.size()); - std::copy(table_name.begin(), table_name.end(), - index_path.begin() + dir_part_length); + index_path += '/'; + index_path.append(table_name.begin(), table_name.end()); std::string data_path; data_path.reserve(dir_name.size() + table_name.size() + 5); data_path= index_path; @@ -557,14 +562,13 @@ namespace close(src_fd); return ret_val; #else - const std::string src_path - {make_path(is_log ? maria_data_root : mysql_real_data_home, path)}; + const std::string src_path= build_path(is_log ? maria_data_root : mysql_real_data_home, path); if (sink.stream == sink.NO_STREAM) { - std::string dest_path{make_path(target.path, path)}; - if (!CopyFileEx(src_path.c_str(), dest_path.c_str(), - nullptr, nullptr, nullptr, COPY_FILE_NO_BUFFERING)) + const std::string dest_path= build_path(target.path, path); + if (!CopyFileEx(src_path.c_str(), dest_path.c_str(), nullptr, nullptr, nullptr, + COPY_FILE_NO_BUFFERING)) { my_osmaperr(GetLastError()); my_error(ER_CANT_CREATE_FILE, MYF(0), dest_path.c_str(), errno); @@ -667,22 +671,18 @@ namespace return strncmp(str, prefix.str, prefix.length) == 0; } - /** @return the target directory path */ - static std::string make_path(const char *dir, const char *name) + static std::string build_path(const char *base_path, const char *filename) noexcept { - std::string path{dir}; - path.push_back('/'); - path.append(name); + std::string path; + const size_t base_len= strlen(base_path); + const size_t filename_len= strlen(filename); + path.reserve(base_len + filename_len + 1); + path.append(base_path, base_len); + path+= '/'; + path.append(filename, filename_len); return path; } }; - - - const LEX_CSTRING Aria_backup::log_file_prefix {C_STRING_WITH_LEN("aria_log.")}; - const LEX_CSTRING Aria_backup::tmp_prefix {C_STRING_WITH_LEN(tmp_file_prefix)}; - const char* Aria_backup::data_ext (MARIA_NAME_DEXT); - const char* Aria_backup::index_ext (MARIA_NAME_IEXT); - const char* Aria_backup::control_file_name {"aria_log_control"}; } void *aria_backup_start(THD *thd, const backup_target *target, @@ -712,6 +712,7 @@ void *aria_backup_start(THD *thd, const backup_target *target, { #if 1 // FIXME: invoke these only for Aria, MyISAM, CSV but not others case BACKUP_PHASE_NO_DML_NON_TRANS: + /* FIXME: Would be better to selectively purge only the tables we need. */ tc_purge(); tdc_purge(true); break; From b5b9b834ab0805b460040eda5937aa41c4183375 Mon Sep 17 00:00:00 2001 From: Andrzej Jarzabek Date: Wed, 10 Jun 2026 09:58:27 +0200 Subject: [PATCH 4/7] Fix Windows crash. Enable backup_server_restore for Windows. --- storage/maria/ma_backup_server.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/storage/maria/ma_backup_server.cc b/storage/maria/ma_backup_server.cc index 73a44a94db1b7..7e3a778faa6d2 100644 --- a/storage/maria/ma_backup_server.cc +++ b/storage/maria/ma_backup_server.cc @@ -479,7 +479,11 @@ namespace aria_free_capabilities(&cap); end: mysql_mutex_unlock(&THR_LOCK_maria); +#ifndef _WIN32 close(fd); +#else + my_close(fd, MYF(0)); +#endif return result; } From a99a4e3fc5d779b634401ea0ca11935d1a8e32c5 Mon Sep 17 00:00:00 2001 From: Andrzej Jarzabek Date: Wed, 10 Jun 2026 12:02:32 +0200 Subject: [PATCH 5/7] Fix concurrent BACKUP_PHASE_FINISH issue. --- storage/maria/ma_backup_server.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/maria/ma_backup_server.cc b/storage/maria/ma_backup_server.cc index 7e3a778faa6d2..86372a96e6fcf 100644 --- a/storage/maria/ma_backup_server.cc +++ b/storage/maria/ma_backup_server.cc @@ -30,6 +30,7 @@ #include "span.h" #include #include +#include #include /* @@ -774,4 +775,4 @@ int aria_backup_end(THD *thd, const backup_target *target, backup_phase phase, default: return 0; } -} \ No newline at end of file +} From ce74e993d3616f1f8ee44be55a84ce2c686be1b8 Mon Sep 17 00:00:00 2001 From: Andrzej Jarzabek Date: Fri, 12 Jun 2026 08:09:48 +0200 Subject: [PATCH 6/7] Move copying of non-Aria out of Aria engine plugin. Add a test for Aria concurrent backup with many tables. Add a test for Aria backup with non-standard log directory. --- .../backup/backup_aria_concurrent.result | 20 ++ .../suite/backup/backup_aria_concurrent.test | 143 ++++++++ .../suite/backup/backup_aria_log_dir.result | 53 +++ .../suite/backup/backup_aria_log_dir.test | 74 ++++ mysql-test/suite/backup/backup_nonacid.result | 56 +++ mysql-test/suite/backup/backup_nonacid.test | 45 +++ sql/sql_backup.cc | 289 ++++++++++++++- sql/sql_backup_interface.h | 146 ++++++-- storage/maria/ma_backup_server.cc | 330 ++++-------------- 9 files changed, 869 insertions(+), 287 deletions(-) create mode 100644 mysql-test/suite/backup/backup_aria_concurrent.result create mode 100644 mysql-test/suite/backup/backup_aria_concurrent.test create mode 100644 mysql-test/suite/backup/backup_aria_log_dir.result create mode 100644 mysql-test/suite/backup/backup_aria_log_dir.test create mode 100644 mysql-test/suite/backup/backup_nonacid.result create mode 100644 mysql-test/suite/backup/backup_nonacid.test diff --git a/mysql-test/suite/backup/backup_aria_concurrent.result b/mysql-test/suite/backup/backup_aria_concurrent.result new file mode 100644 index 0000000000000..c7862eaa8600e --- /dev/null +++ b/mysql-test/suite/backup/backup_aria_concurrent.result @@ -0,0 +1,20 @@ +15 30 7500 +Back up the database +BACKUP SERVER TO '$target_directory' 4 CONCURRENT; +Restore the database +# restart: --datadir=MYSQLTEST_VARDIR/some_directory +Check contents after restore +SELECT COUNT(*) FROM table_checks; +COUNT(*) +400 +SELECT * FROM table_checks WHERE sum_id <> 15; +tbl_name sum_id str_len blob_len num_rows +SELECT * FROM table_checks WHERE str_len <> 30; +tbl_name sum_id str_len blob_len num_rows +SELECT * FROM table_checks WHERE blob_len <> 7500; +tbl_name sum_id str_len blob_len num_rows +SELECT * FROM table_checks WHERE num_rows <> 5; +tbl_name sum_id str_len blob_len num_rows +Restart database in original data directory +# restart +Clean up diff --git a/mysql-test/suite/backup/backup_aria_concurrent.test b/mysql-test/suite/backup/backup_aria_concurrent.test new file mode 100644 index 0000000000000..390d28b6a3efb --- /dev/null +++ b/mysql-test/suite/backup/backup_aria_concurrent.test @@ -0,0 +1,143 @@ + +--source include/have_aria.inc + +--disable_query_log + + +DELIMITER //; +CREATE PROCEDURE populate_data(IN t_name VARCHAR(64), IN num_rows INT) +BEGIN + DECLARE i INT DEFAULT 1; + SET @query = CONCAT('INSERT INTO ', t_name, ' (id, str_val, blob_val) VALUES (?, ?, ?)'); + PREPARE stmt FROM @query; + + WHILE i <= num_rows DO + SET @str = CONCAT('_row_', i); + # Generate a predictable but repeating blob based on the row index + SET @blb = REPEAT(CHAR(97 + (i % 26)), 1500); + EXECUTE stmt USING i, @str, @blb; + SET i = i + 1; + END WHILE; + + DEALLOCATE PREPARE stmt; +END// +DELIMITER ;// + +# Create this many tables transactional and non-transactional each +let $tab_num= 200; +let $num_rows= 5; + +let $i = 1; +while ($i <= $tab_num) { + + let $tr=0; + while ($tr <= 1) { + + let $suff= _$i; + let $table_name= ta_tr$tr$suff; + + eval CREATE TABLE $table_name ( + id INT PRIMARY KEY, + str_val VARCHAR(255), + blob_val BLOB, + INDEX idx_str (str_val) + ) ENGINE=Aria TRANSACTIONAL=$tr; + + eval CALL populate_data('$table_name', $num_rows); + + inc $tr; + } + + inc $i; +} + +--enable_query_log + +# All tables have the same data, so we query only one for reference + +let $sum_id= `SELECT SUM(id) FROM ta_tr0_1`; +let $str_len= `SELECT SUM(LENGTH(str_val)) FROM ta_tr0_1`; +let $blob_len= `SELECT SUM(LENGTH(blob_val)) FROM ta_tr0_1`; + +echo $sum_id $str_len $blob_len; + +--let $target_directory=$MYSQLTEST_VARDIR/some_directory + +# Clean up after a previous failed test, in case we are retrying. +--error 0,1 +--rmdir $target_directory + +--echo Back up the database +evalp BACKUP SERVER TO '$target_directory' 4 CONCURRENT; + +--echo Restore the database +--let $restart_parameters=--datadir=$target_directory +--source include/restart_mysqld.inc + +--echo Check contents after restore + +--disable_query_log +CREATE TEMPORARY TABLE table_checks ( + tbl_name VARCHAR(64), + sum_id INT, + str_len INT, + blob_len INT, + num_rows INT +) ENGINE=MEMORY; + +let $i = 1; +while ($i <= $tab_num) { + let $tr=0; + while ($tr <= 1) { + + let $suff= _$i; + let $table_name= ta_tr$tr$suff; + + let $r_sum_id= `SELECT SUM(id) FROM $table_name`; + let $r_str_len= `SELECT SUM(LENGTH(str_val)) FROM $table_name`; + let $r_blob_len= `SELECT SUM(LENGTH(blob_val)) FROM $table_name`; + let $r_num_rows= `SELECT COUNT(*) FROM $table_name`; + + eval INSERT INTO table_checks VALUES ('$table_name', $r_sum_id, $r_str_len, $r_blob_len, $r_num_rows); + + inc $tr; + } + inc $i; +} + +--enable_query_log + +SELECT COUNT(*) FROM table_checks; + +# We expect results in the table to always match the results captured before the BACKUP +# Returned rowsets should be empty +eval SELECT * FROM table_checks WHERE sum_id <> $sum_id; +eval SELECT * FROM table_checks WHERE str_len <> $str_len; +eval SELECT * FROM table_checks WHERE blob_len <> $blob_len; +eval SELECT * FROM table_checks WHERE num_rows <> $num_rows; + +--echo Restart database in original data directory +--let $restart_parameters= +--source include/restart_mysqld.inc + +--echo Clean up + +--disable_query_log + +let $i = 1; +while ($i <= $tab_num) { + let $tr=0; + while ($tr <= 1) { + let $suff= _$i; + let $table_name= ta_tr$tr$suff; + eval DROP TABLE $table_name; + inc $tr; + } + inc $i; +} + +DROP PROCEDURE populate_data; + +--enable_query_log + +--rmdir $target_directory diff --git a/mysql-test/suite/backup/backup_aria_log_dir.result b/mysql-test/suite/backup/backup_aria_log_dir.result new file mode 100644 index 0000000000000..d16ca8158cb45 --- /dev/null +++ b/mysql-test/suite/backup/backup_aria_log_dir.result @@ -0,0 +1,53 @@ +# restart: --aria-log-dir-path=MYSQLTEST_VARDIR/log_directory +CREATE TABLE t ( +id INT PRIMARY KEY, +str_val VARCHAR(255), +blob_val BLOB, +INDEX idx_str (str_val) +) ENGINE=Aria TRANSACTIONAL=1; +CREATE PROCEDURE populate_data(IN num_rows INT) +BEGIN +DECLARE i INT DEFAULT 0; +WHILE i < num_rows DO +SET @str = CONCAT('_row_', i); +SET @blb = REPEAT(CHAR(97 + (i % 26)), 1500); +INSERT INTO t (id, str_val, blob_val) VALUES (i, @str, @blb); +SET i = i + 1; +END WHILE; +END// +CALL populate_data(10000); +SELECT COUNT(*) from t; +COUNT(*) +10000 +SELECT SUM(id) FROM t; +SUM(id) +49995000 +SELECT SUM(LENGTH(str_val)) FROM t; +SUM(LENGTH(str_val)) +88890 +SELECT SUM(LENGTH(blob_val)) FROM t; +SUM(LENGTH(blob_val)) +15000000 +Back up the database +BACKUP SERVER TO '$target_directory'; +Restore the database +# restart: --datadir=MYSQLTEST_VARDIR/some_directory +Check contents after restore +SELECT COUNT(*) from t; +COUNT(*) +10000 +SELECT SUM(id) FROM t; +SUM(id) +49995000 +SELECT SUM(LENGTH(str_val)) FROM t; +SUM(LENGTH(str_val)) +88890 +SELECT SUM(LENGTH(blob_val)) FROM t; +SUM(LENGTH(blob_val)) +15000000 +Restart database in original log and data directories +# restart: --aria-log-dir-path=MYSQLTEST_VARDIR/log_directory +Clean up +DROP PROCEDURE populate_data; +DROP TABLE t; +# restart diff --git a/mysql-test/suite/backup/backup_aria_log_dir.test b/mysql-test/suite/backup/backup_aria_log_dir.test new file mode 100644 index 0000000000000..7b8123dcd37b6 --- /dev/null +++ b/mysql-test/suite/backup/backup_aria_log_dir.test @@ -0,0 +1,74 @@ +--source include/have_aria.inc + +--let $log_directory=$MYSQLTEST_VARDIR/log_directory +--let $target_directory=$MYSQLTEST_VARDIR/some_directory + +# Clean up after a previous failed test, in case we are retrying. +--error 0,1 +--rmdir $log_directory +--error 0,1 +--rmdir $target_directory + +--mkdir $log_directory + +--let $orig_restart_parameters=--aria-log-dir-path=$log_directory +--let $restart_parameters=$orig_restart_parameters +--source include/restart_mysqld.inc + +CREATE TABLE t ( + id INT PRIMARY KEY, + str_val VARCHAR(255), + blob_val BLOB, + INDEX idx_str (str_val) +) ENGINE=Aria TRANSACTIONAL=1; + +--disable_warnings +DELIMITER //; +CREATE PROCEDURE populate_data(IN num_rows INT) +BEGIN + DECLARE i INT DEFAULT 0; + WHILE i < num_rows DO + SET @str = CONCAT('_row_', i); + SET @blb = REPEAT(CHAR(97 + (i % 26)), 1500); + INSERT INTO t (id, str_val, blob_val) VALUES (i, @str, @blb); + SET i = i + 1; + END WHILE; +END// +DELIMITER ;// +--enable_warnings + +CALL populate_data(10000); + +SELECT COUNT(*) from t; +SELECT SUM(id) FROM t; +SELECT SUM(LENGTH(str_val)) FROM t; +SELECT SUM(LENGTH(blob_val)) FROM t; + +--echo Back up the database +evalp BACKUP SERVER TO '$target_directory'; + +--echo Restore the database +--let $restart_parameters=--datadir=$target_directory +--source include/restart_mysqld.inc + +--echo Check contents after restore + +SELECT COUNT(*) from t; +SELECT SUM(id) FROM t; +SELECT SUM(LENGTH(str_val)) FROM t; +SELECT SUM(LENGTH(blob_val)) FROM t; + +--echo Restart database in original log and data directories +--let $restart_parameters=$orig_restart_parameters +--source include/restart_mysqld.inc + +--echo Clean up + +DROP PROCEDURE populate_data; +DROP TABLE t; + +--let $restart_parameters= +--source include/restart_mysqld.inc + +--rmdir $target_directory +--rmdir $log_directory diff --git a/mysql-test/suite/backup/backup_nonacid.result b/mysql-test/suite/backup/backup_nonacid.result new file mode 100644 index 0000000000000..9027dd735bedf --- /dev/null +++ b/mysql-test/suite/backup/backup_nonacid.result @@ -0,0 +1,56 @@ +CREATE TABLE t_archive (id int unsigned) ENGINE=ARCHIVE; +INSERT INTO t_archive VALUES (2), (3), (5), (7), (11); +CREATE DATABASE d; +CREATE TABLE d.t_csv (id int unsigned NOT NULL) ENGINE=CSV; +INSERT INTO d.t_csv VALUES (4), (26), (41), (60), (83), (109); +CREATE TABLE t_myisam1 (id int unsigned) ENGINE=MyISAM; +INSERT INTO t_myisam1 VALUES (1), (1), (2), (3), (5), (8); +CREATE TABLE t_myisam2 (id int unsigned) ENGINE=MyISAM; +INSERT INTO t_myisam2 VALUES (13), (21), (34), (55), (89), (144); +CREATE TABLE t_mrg (id int unsigned) ENGINE=MRG_MyISAM UNION=(t_myisam1, t_myisam2); +BACKUP SERVER TO '$target_directory'; +# restart: --datadir=MYSQLTEST_VARDIR/some_directory +SELECT * FROM t_archive ORDER BY id; +id +2 +3 +5 +7 +11 +SELECT * FROM d.t_csv ORDER BY id; +id +4 +26 +41 +60 +83 +109 +SELECT * FROM t_myisam1 ORDER BY id; +id +1 +1 +2 +3 +5 +8 +SELECT * FROM t_mrg ORDER BY id; +id +1 +1 +2 +3 +5 +8 +13 +21 +34 +55 +89 +144 +# restart +DROP TABLE t_archive; +DROP TABLE d.t_csv; +DROP TABLE t_myisam1; +DROP TABLE t_myisam2; +DROP TABLE t_mrg; +DROP DATABASE d; diff --git a/mysql-test/suite/backup/backup_nonacid.test b/mysql-test/suite/backup/backup_nonacid.test new file mode 100644 index 0000000000000..eac863a191685 --- /dev/null +++ b/mysql-test/suite/backup/backup_nonacid.test @@ -0,0 +1,45 @@ +--source include/have_csv.inc +--source include/have_archive.inc + +CREATE TABLE t_archive (id int unsigned) ENGINE=ARCHIVE; +INSERT INTO t_archive VALUES (2), (3), (5), (7), (11); + +CREATE DATABASE d; +CREATE TABLE d.t_csv (id int unsigned NOT NULL) ENGINE=CSV; +INSERT INTO d.t_csv VALUES (4), (26), (41), (60), (83), (109); + +CREATE TABLE t_myisam1 (id int unsigned) ENGINE=MyISAM; +INSERT INTO t_myisam1 VALUES (1), (1), (2), (3), (5), (8); + +CREATE TABLE t_myisam2 (id int unsigned) ENGINE=MyISAM; +INSERT INTO t_myisam2 VALUES (13), (21), (34), (55), (89), (144); + +CREATE TABLE t_mrg (id int unsigned) ENGINE=MRG_MyISAM UNION=(t_myisam1, t_myisam2); + +--let $target_directory=$MYSQLTEST_VARDIR/some_directory + +# Clean up after a previous failed test, in case we are retrying. +--error 0,1 +--rmdir $target_directory + +evalp BACKUP SERVER TO '$target_directory'; + +--let $restart_parameters=--datadir=$target_directory +--source include/restart_mysqld.inc + +SELECT * FROM t_archive ORDER BY id; +SELECT * FROM d.t_csv ORDER BY id; +SELECT * FROM t_myisam1 ORDER BY id; +SELECT * FROM t_mrg ORDER BY id; + +--let $restart_parameters= +--source include/restart_mysqld.inc + +DROP TABLE t_archive; +DROP TABLE d.t_csv; +DROP TABLE t_myisam1; +DROP TABLE t_myisam2; +DROP TABLE t_mrg; +DROP DATABASE d; + +--rmdir $target_directory diff --git a/sql/sql_backup.cc b/sql/sql_backup.cc index f4ad618b4694a..256a469bf9337 100644 --- a/sql/sql_backup.cc +++ b/sql/sql_backup.cc @@ -24,6 +24,12 @@ #include "tpool.h" #include "aligned.h" +#include +#include +#include + +static constexpr const char zerobuf[511]{}; + #if defined __linux__ || defined __FreeBSD__ using copying_step= ssize_t(int,int,size_t,off_t*); template @@ -168,7 +174,80 @@ static ssize_t pread_write(IF_WIN(const native_file_handle&,int) in_fd, #ifdef __APPLE__ /* The inline copy_entire_file() invokes fcopyfile() */ #elif defined _WIN32 -/* CopyFileEx() should be used */ +/** Copy entire file. + @param src_path path file file to copy + @param dst_path path of file to copy to + @param target backup target + @param sink worker context + @return error code (non-positive) + @retval 0 on success + @note Wrapper for CopyFileExA, will report error using my_error */ +extern "C" +int copy_entire_file(const char *src_path, const char *dst_path, + const struct backup_target *target, + const struct backup_sink *sink) +{ + if (sink->stream == sink->NO_STREAM) + { + std::string full_dst_path= build_path(target->path, dst_path); + if (!CopyFileEx(src_path, full_dst_path.c_str(), nullptr, nullptr, nullptr, + COPY_FILE_NO_BUFFERING)) + { + my_osmaperr(GetLastError()); + my_error(ER_CANT_CREATE_FILE, MYF(0), full_dst_path.c_str(), errno); + return 1; + } + } + else + { + HANDLE src, dst{sink->stream}; + for (;;) + { + src= CreateFile(src_path, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + my_win_file_secattr(), OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (src != INVALID_HANDLE_VALUE) + break; + + switch (GetLastError()) { + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + + my_osmaperr(GetLastError()); + my_error(ER_FILE_NOT_FOUND, MYF(ME_ERROR_LOG), src_path, errno); + return -1; + } + + LARGE_INTEGER li; + if (!GetFileSizeEx(src, &li)) + { + write_error: + my_osmaperr(GetLastError()); + my_error(ER_ERROR_ON_WRITE, MYF(0), dst_path, errno); + if (src != INVALID_HANDLE_VALUE) + CloseHandle(src); + return -1; + } + + if (backup_stream_start(dst, dst_path, 0644, li.QuadPart, nullptr, 0) || + backup_stream_append_plain(src, dst, 0, li.QuadPart)) + goto write_error; + + if (size_t pad= size_t(li.LowPart) & 511) + if (backup_stream_write(dst, zerobuf, 512 - pad)) + goto write_error; + if (!CloseHandle(src)) + { + src= INVALID_HANDLE_VALUE; + goto write_error; + } + } + return 0; +} #else /** Copy a file (whole content). @param src source file descriptor @@ -181,6 +260,59 @@ extern "C" int copy_entire_file(int src, int dst) } #endif +#ifndef _WIN32 +/** Copy an entire file to target. +@param src_fd source file descriptor +@param target backup target +@return error code (non-positive) +@retval 0 on success +@note Any intermediate directories must already exist in the target. */ +# ifdef __cplusplus +extern "C" +# endif +int copy_fd_to_target(int src_fd, + const struct backup_target *target, + const char *path, + const struct backup_sink *sink) +{ + int ret_val= 0; + int tgt_fd{sink->stream}; + if (tgt_fd == sink->NO_STREAM) + { + tgt_fd= openat(target->fd, path, + O_CREAT | O_EXCL | O_WRONLY, 0666); + + if (tgt_fd < 0) + { + my_error(ER_CANT_CREATE_FILE, MYF(0), path, errno); + ret_val= 1; + } + else + { + ret_val= copy_entire_file(src_fd, tgt_fd); + if (ret_val | close(tgt_fd)) + { + write_error: + my_error(ER_ERROR_ON_WRITE, MYF(0), path, errno); + ret_val= 1; + } + } + } + else + { + uint64_t end= uint64_t(lseek(src_fd, 0, SEEK_END)); + if (backup_stream_start(tgt_fd, path, 0644, end, nullptr, 0) || + backup_stream_append(src_fd, tgt_fd, 0, end)) + goto write_error; + if (size_t pad= size_t(end) & 511) + if (backup_stream_write(tgt_fd, zerobuf, 512 - pad)) + goto write_error; + } + + return ret_val; +} +#endif + /** Copy a portion of a file. @param src source file descriptor @param dst target to append src to @@ -215,6 +347,155 @@ extern "C" int copy_file(IF_WIN(const native_file_handle&,int) src, return int(ret); } +/** Ensure a file can be copied to a subdirectory in target. +May create the subdirectory. +@param target backup target +@param name subdirectory name +@return error code (non-positive) +@retval 0 on success +@note If the directory is created, the directory containing it must + already exist: nested directory creation is not supported. */ +extern "C" int ensure_target_subdir(const struct backup_target *target, + const char* name) +{ + +#ifdef _WIN32 + const std::string dir_path= build_path(target->path, name); + if (!CreateDirectory(dir_path.c_str(), nullptr)) + { + DWORD err = GetLastError(); + if (err != ERROR_ALREADY_EXISTS) + { + my_osmaperr(err); + return 1; + } + } +#else + if (likely(!mkdirat(target->fd, name, 0777) || errno == EEXIST)) + return 0; +#endif + my_error(ER_CANT_CREATE_FILE, MYF(0), name, errno); + return 1; +} + +/** Copy entire file from data directory target, preserving path. +@param path relative path of file +@param target backup target +@param sink worker context +@return error code (non-positive) +@retval 0 on success +@note The file will be copied to the same path relative to + target directory. Any intermediate directories must + already exist in the target. */ +extern "C" int copy_datafile_to_target(const char *path, + const struct backup_target *target, + const backup_sink *sink) +{ +#ifndef _WIN32 + int src_fd = open(path, O_RDONLY); + if (src_fd < 0) + { + my_error(ER_CANT_OPEN_FILE, MYF(0), path, errno); + return 1; + } + int ret_val= copy_fd_to_target(src_fd, target, path, sink); + close(src_fd); + return ret_val; +#else + return copy_entire_file(path, path, target, sink); +#endif +} + +/* all extensions have the same length, adjust if that changes */ +static constexpr size_t ext_len= 4; + +/* Files not copied by plugin backup implementations: files managed by +SQL layer and miscellaneous engine files to be copied bunde DDL lock */ +static constexpr const char* misc_exts[] {".frm", ".par", ".MYD", ".MYI", ".MRG", + ".ARM", ".ARZ", ".CSM", ".CSV"}; +static constexpr const char db_opt_name[] {"db.opt"}; +static constexpr size_t db_opt_len= sizeof(db_opt_name) - 1; + +static bool match_ext(const char* ext1, const char* ext2) noexcept +{ + return memcmp(ext1, + ext2, + ext_len) == 0; +} + +static bool match_misc_ext(const char* file_ext) noexcept +{ + return std::find_if(std::begin(misc_exts), std::end(misc_exts), + [file_ext](const char* misc_ext) { + return match_ext(file_ext, misc_ext); + }) != std::end(misc_exts); +} + +static bool is_db_opt(const char* filename, size_t filename_len) +{ + return filename_len == db_opt_len && + memcmp(filename, db_opt_name, db_opt_len) == 0; +} + +static bool is_misc_file(const char* filename) +{ + size_t filename_len= strlen(filename); + if (filename_len < ext_len) + return false; + const char *file_ext = filename + filename_len - ext_len; + return match_misc_ext(file_ext) || is_db_opt(filename, filename_len); +} + +std::string build_path(const char *base_path, const char *filename) noexcept +{ + std::string path; + const size_t base_len= strlen(base_path); + const size_t filename_len= strlen(filename); + path.reserve(base_len + filename_len + 1); + path.append(base_path, base_len); + path+= '/'; + path.append(filename, filename_len); + return path; +} + +static bool copy_misc_files(const backup_target *target, + const backup_sink *sink) +{ + Dir_scan datadir(".", MYF(MY_WANT_STAT)); + if (datadir.is_error()) + return true; + std::unordered_set ensured_dirs; + int error= datadir.for_each([target, sink, &ensured_dirs](const fileinfo &fi) + { + if ((fi.mystat->st_mode & S_IFMT) == S_IFDIR) + { + const char* dir_name= fi.name; + if(sink->stream == sink->NO_STREAM && + ensured_dirs.insert(dir_name).second) + { + int fail= ensure_target_subdir(target, dir_name); + if (fail) + return fail; + } + Dir_scan dbdir(dir_name, MYF(0)); + if (dbdir.is_error()) + return 1; + return dbdir.for_each([target, sink, dir_name](const fileinfo &fi) + { + if (is_misc_file(fi.name)) + { + const std::string path= build_path(dir_name, fi.name); + return copy_datafile_to_target(path.c_str(), target, sink); + } + return 0; + }); + } + return 0; + }); + return error != 0; +} + + /** Append to the configuration file. @param target backup target directory @param config the configuration file snippet to append @@ -558,6 +839,12 @@ bool Sql_cmd_backup::execute(THD *thd) } backup_phase_start: target_phase->phase= backup_phase(phase); + + if (phase == BACKUP_PHASE_NO_DDL) + fail= copy_misc_files(&target_phase->target, &target_phase->sink); + if (fail) + break; + fail= plugin_foreach_with_mask(thd, backup_start, MYSQL_STORAGE_ENGINE_PLUGIN, PLUGIN_IS_DELETED|PLUGIN_IS_READY, diff --git a/sql/sql_backup_interface.h b/sql/sql_backup_interface.h index 94892ba7591ab..07209b8c7af20 100644 --- a/sql/sql_backup_interface.h +++ b/sql/sql_backup_interface.h @@ -13,7 +13,16 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ +#include +#include +#include + +#ifdef __cplusplus +# include +#endif + struct backup_target; +struct backup_sink; /** A payload chunk in a sparse file that is being streamed */ struct backup_chunk @@ -25,14 +34,30 @@ struct backup_chunk }; #ifdef _WIN32 -/* Use CopyFileEx() to copy entire files */ +/** Copy entire file. + @param src_path path file file to copy + @param dst_path path of file to copy to + @param target backup target + @param sink worker context + @return error code (non-positive) + @retval 0 on success + @note Wrapper for CopyFileExA, will report error using my_error */ +# ifdef __cplusplus +extern "C" +# endif +int copy_entire_file(const char *src_path, + const char *dst_path, + const struct backup_target *target, + const struct backup_sink *sink); + struct native_file_handle; -#elif defined __APPLE__ +#else +# if defined __APPLE__ /* You should invoke fclonefileat(2) manually before attempting copy_entire_file() or copy_file() */ -# include -# include -# include +# include +# include +# include /** Copy an entire file. @param src source file descriptor @param dst target to append src to @@ -42,20 +67,39 @@ inline int copy_entire_file(int src, int dst) { return fcopyfile(src, dst, NULL, COPYFILE_ALL | COPYFILE_CLONE); } -#else -# ifdef __cplusplus +# else +# ifdef __cplusplus extern "C" -# endif +# endif /** Copy an entire file. @param src source file descriptor @param dst target to append src to @return error code (non-positive) @retval 0 on success */ int copy_entire_file(int src, int dst); +# endif + +/** Copy an entire file to target. +@param src_fd source file descriptor +@param target backup target +@param path target file path +@param sink worker context +@return error code (non-positive) +@retval 0 on success +@note Any intermediate directories must already exist in the target. */ +# ifdef __cplusplus +extern "C" +# endif +int copy_fd_to_target(int src_fd, + const struct backup_target *target, + const char *path, + const struct backup_sink *sink); + #endif #ifdef __cplusplus extern "C" +{ #endif /** Copy a portion of a file. @param src source file descriptor @@ -68,9 +112,29 @@ int copy_file(IF_WIN(const native_file_handle&,int) src, IF_WIN(const native_file_handle&,int) dst, uint64_t start, uint64_t end); -#ifdef __cplusplus -extern "C" -#endif +/** Ensure a file can be copied to a subdirectory in target. +May create the subdirectory. +@param target backup target +@param name subdirectory name +@return error code (non-positive) +@retval 0 on success +@note If the directory is created, the directory containing it must + already exist: nested directory creation is not supported. */ +int ensure_target_subdir(const struct backup_target *target, const char* name); + +/** Copy entire file from data directory target, preserving path. +@param path relative path of file +@param target backup target +@param sink worker context +@return error code (non-positive) +@retval 0 on success +@note The file will be copied to the same path relative to + target directory. Any intermediate directories must + already exist in the target. */ +int copy_datafile_to_target(const char *path, + const struct backup_target *target, + const struct backup_sink *sink); + /** Append to the configuration file. @param target backup target directory @param config the configuration file snippet to append @@ -80,9 +144,7 @@ extern "C" int backup_config_append(IF_WIN(const char*, int) target, const char *config, size_t size); -#ifdef __cplusplus -extern "C" -#endif + /** Append to the configuration file. @param target backup stream @param config the configuration file snippet to append @@ -92,9 +154,6 @@ extern "C" int backup_stream_config(IF_WIN(HANDLE, int) stream, const char *config, size_t size); -#ifdef __cplusplus -extern "C" -#endif /** Start streaming a file. @param target backup target @param name file name @@ -108,9 +167,6 @@ int backup_stream_start(IF_WIN(HANDLE, int) stream, const char *name, mode_t mode, uint64_t size, const struct backup_chunk *chunks, size_t n_chunks); -#ifdef __cplusplus -extern "C" -#endif /** Write data to a stream. @param stream backup stream @@ -122,9 +178,6 @@ extern "C" int backup_stream_write(IF_WIN(HANDLE, int) stream, const void *buf, size_t size); -#ifdef __cplusplus -extern "C" -#endif /** Append a file snippet to the stream, after a corresponding call to backup_stream_start(). @@ -143,9 +196,6 @@ int backup_stream_append(IF_WIN(const native_file_handle&,int) src, uint64_t start, uint64_t end); #ifdef __linux__ -# ifdef __cplusplus -extern "C" -# endif /** Append an immutable snippet of a file to the stream, allowing Linux sendfile(2) to be invoked. @@ -168,11 +218,49 @@ int backup_stream_append_async(int src, int stream, #endif #ifdef _WIN32 -# ifdef __cplusplus -extern "C" -# endif int backup_stream_append_plain(HANDLE src, HANDLE stream, uint64_t start, uint64_t end); #else # define backup_stream_append_plain backup_stream_append #endif + +#ifdef __cplusplus +} // extern "C" + + /* RAII wrapper for my_dir() */ +class Dir_scan +{ +public: + explicit Dir_scan(const char* path, myf flags) noexcept + { + dir_info= my_dir(path, flags); + if (!dir_info) + { + my_error(ER_CANT_READ_DIR, MYF(0), path, my_errno); + } + } + ~Dir_scan() noexcept + { + my_dirend(dir_info); + } + bool is_error() const noexcept + { + return !dir_info; + } + template + int for_each(Fn fn) const noexcept + { + for (size_t i= 0; i < dir_info->number_of_files; i++) + { + if (fn(dir_info->dir_entry[i]) != 0) + return 1; + } + return 0; + } +private: + MY_DIR *dir_info {nullptr}; +}; + +std::string build_path(const char *base_path, const char *filename) noexcept; + +#endif diff --git a/storage/maria/ma_backup_server.cc b/storage/maria/ma_backup_server.cc index 86372a96e6fcf..34c0ede49c812 100644 --- a/storage/maria/ma_backup_server.cc +++ b/storage/maria/ma_backup_server.cc @@ -99,8 +99,6 @@ namespace ~Aria_backup() { #ifndef _WIN32 - if (datadir_fd >= 0) - std::ignore= close(datadir_fd); if (logdir_fd >= 0) std::ignore= close(logdir_fd); #endif @@ -112,15 +110,9 @@ namespace { #ifndef _WIN32 /* Aria table files live under the server data directory - (mysql_real_data_home), while the transaction logs and control file + (current directory), while the transaction logs and control file live under aria_log_dir_path (maria_data_root). These differ when aria_log_dir_path is set, so open and scan them separately. */ - datadir_fd= open(mysql_real_data_home, O_DIRECTORY); - if (datadir_fd < 0) - { - my_error(ER_CANT_READ_DIR, MYF(0), mysql_real_data_home, errno); - return true; - } logdir_fd= open(maria_data_root, O_DIRECTORY); if (logdir_fd < 0) { @@ -134,13 +126,13 @@ namespace return false; } - bool start_copy_dml_safe(const backup_target &target, const backup_sink &sink) noexcept + bool start_copy_dml_safe(const backup_target *target, const backup_sink *sink) noexcept { assert(translog_purge_disabled); if (scan_dbdirs()) return true; flatten_table_lists(); - if (sink.stream == sink.NO_STREAM) + if (sink->stream == sink->NO_STREAM) return ensure_target_dirs(target); return false; } @@ -154,7 +146,7 @@ namespace /* Copy an Aria table that is safe to be copied while concurrent DML is in progress. */ - int dml_safe_copy_step(const backup_target &target, const backup_sink &sink) noexcept + int dml_safe_copy_step(const backup_target *target, const backup_sink *sink) noexcept { Copy_from_list copy_from_list; auto copy_table_action= [this, target, sink](const table_ref &table) noexcept @@ -167,12 +159,6 @@ namespace if (copy_from_list(unsafe_tables_list, unsafe_tables_copied, copy_table_action) != 0) return -1; - if (copy_from_list(misc_files, misc_files_copied, - [this, target, sink](const std::string &path) noexcept - { - return copy_file(target, sink, path, false); - }) != 0) - return -1; return copy_from_list.remaining(); } @@ -185,7 +171,7 @@ namespace - Aria tables - other ("miscellaneous") files */ - int unsafe_copy_step(const backup_target &target, const backup_sink &sink) noexcept + int unsafe_copy_step(const backup_target *target, const backup_sink *sink) noexcept { bool copy_done= false; @@ -208,7 +194,7 @@ namespace if (copy_from_list(log_files, log_files_copied, [this, target, sink](const std::string &path) noexcept { - return copy_file(target, sink, path, false); + return copy_log_file(target, sink, path.c_str()); }) != 0) return -1; @@ -224,24 +210,19 @@ namespace } private: #ifndef _WIN32 - /** The server data directory (Aria table files) */ - int datadir_fd{-1}; - /** The Aria log directory aria_log_dir_path (logs, control file) */ + /** The server data directory */ int logdir_fd{-1}; #endif /** whether the Aria translog_disable_purge() is in effect */ bool translog_purge_disabled{false}; - static constexpr const char zerobuf[511]{}; - /* All file suffixes are 4 characters long (dot and 3 letter extension) */ - static constexpr size_t suffix_len= 4; + + /* File extensions are 4 characters long (dot and 3 letter extension) */ + static constexpr size_t ext_len= 4; static constexpr const char* data_ext {MARIA_NAME_DEXT}; static constexpr const char* index_ext {MARIA_NAME_IEXT}; static constexpr LEX_CSTRING log_file_prefix {C_STRING_WITH_LEN("aria_log.")}; static constexpr LEX_CSTRING tmp_prefix {C_STRING_WITH_LEN(tmp_file_prefix)}; - /* TODO: .frm failes are not Aria-specific, .MYD and .MYI are MyISAM files; - they are copied here as a stop-gap */ - static constexpr const char* misc_exts[] {".MYD", ".MYI", ".frm"}; - static constexpr const char* control_file_name {"aria_log_control"}; + static constexpr LEX_CSTRING control_file_name {C_STRING_WITH_LEN("aria_log_control")}; using dir_name = std::string; using dir_contents = std::vector; using database_dir = std::pair; @@ -252,9 +233,6 @@ namespace database_dirs unsafe_tables; /* Aria log files */ std::vector log_files; - std::vector misc_files; - /* directories in which misc files are */ - std::vector misc_dirs; bool have_control_file = false; bool safe_files_copied = false; @@ -271,7 +249,6 @@ namespace std::atomic dml_safe_tables_copied {0}; std::atomic unsafe_tables_copied {0}; std::atomic log_files_copied {0}; - std::atomic misc_files_copied {0}; std::atomic control_file_copied {false}; ATTRIBUTE_COLD ATTRIBUTE_NOINLINE @@ -283,10 +260,9 @@ namespace int scan_dbdirs() noexcept { - /* Scan the server data directory for Aria table files. */ - MY_DIR *data_dir= my_dir(mysql_real_data_home, MYF(MY_WANT_STAT)); + MY_DIR *data_dir= my_dir(".", MYF(MY_WANT_STAT)); if (!data_dir) - return dir_error(mysql_real_data_home); + return dir_error("."); int fail= 0; for (const fileinfo &fi : st_::span{data_dir->dir_entry, @@ -304,11 +280,9 @@ namespace int scan_database_dir(const char* dir_name) noexcept { - const char* base_dir = maria_data_root; - const std::string dir_path= build_path(base_dir, dir_name); - MY_DIR *dir_info= my_dir(dir_path.c_str(), MYF(MY_WANT_STAT)); + MY_DIR *dir_info= my_dir(dir_name, MYF(MY_WANT_STAT)); if (!dir_info) - return dir_error(dir_path.c_str()); + return dir_error(dir_name); int fail= 0; dir_contents safe; dir_contents unsafe; @@ -316,19 +290,20 @@ namespace st_::span{dir_info->dir_entry, dir_info->number_of_files}) { - const char* filename= fi.name; - size_t filename_len = strlen(filename); - if (filename_len >= suffix_len) + const LEX_CSTRING filename {fi.name, strlen(fi.name)}; + if (filename.length >= ext_len) { - const char* suffix = filename + filename_len - suffix_len; - if(match_suffix(suffix, index_ext)) + /* Length of filename without extension. */ + size_t base_filename_len= filename.length - ext_len; + const char* suffix = filename.str + base_filename_len; + if(match_ext(suffix, index_ext)) { if (!is_tmp_table(filename)) { - auto is_safe = is_safe_table(dir_name, filename); + auto is_safe = is_safe_table(dir_name, filename.str); if (std::holds_alternative(is_safe)) { - std::string table_name(filename, filename_len - suffix_len); + std::string table_name(filename.str, base_filename_len); if (std::get(is_safe)) safe.push_back(std::move(table_name)); else @@ -341,12 +316,6 @@ namespace } } } - else if (match_misc_ext(suffix) || !strcmp(filename, "db.opt")) - { - if(misc_dirs.empty() || misc_dirs.back() != dir_name) - misc_dirs.emplace_back(dir_name); - misc_files.push_back(build_path(dir_name, filename)); - } } } if(!fail) @@ -361,7 +330,7 @@ namespace return fail; } - static bool is_tmp_table(const char* filename) noexcept + static bool is_tmp_table(const LEX_CSTRING &filename) noexcept { return begins_with(filename, tmp_prefix); } @@ -390,15 +359,18 @@ namespace for (const fileinfo &fi : st_::span{dir_info->dir_entry, dir_info->number_of_files}) - if (begins_with(fi.name, log_file_prefix)) - log_files.emplace_back(fi.name); - else if (strcmp(fi.name, "aria_log_control") == 0) + { + const LEX_CSTRING filename {fi.name, strlen(fi.name)}; + if (begins_with(filename, log_file_prefix)) + log_files.emplace_back(LEX_STRING_WITH_LEN(filename)); + else if (is_control_file_name(filename)) have_control_file = true; + } my_dirend(dir_info); return 0; } - bool ensure_target_dirs(const backup_target &target) noexcept + bool ensure_target_dirs(const backup_target *target) noexcept { using string = std::string; std::vector dirs; @@ -406,8 +378,6 @@ namespace dirs.push_back(&dir.first); for (const database_dir &dir : unsafe_tables) dirs.push_back(&dir.first); - for (const string &dir: misc_dirs) - dirs.push_back(&dir); std::sort(dirs.begin(), dirs.end(), [](const string *a, const string *b) { return *a < *b; }); auto dirs_end = std::unique(dirs.begin(), dirs.end(), @@ -424,47 +394,21 @@ namespace Create directory in the target directory if it does not exist. Return 0 on success, non-0 on failure. Set errno in case of failure */ - int ensure_target_subdir(const backup_target &target, const char *name) + int ensure_target_subdir(const backup_target *target, const char *name) noexcept { -#ifdef _WIN32 - const std::string dir_path= build_path(target.path, name); - if (!CreateDirectory(dir_path.c_str(), nullptr)) - { - DWORD err = GetLastError(); - if (err != ERROR_ALREADY_EXISTS) - { - my_osmaperr(err); - return 1; - } - } -#else - if (likely(!mkdirat(target.fd, name, 0777) || errno == EEXIST)) - return 0; -#endif - my_error(ER_CANT_CREATE_FILE, MYF(0), name, errno); - return 1; + return ::ensure_target_subdir(target, name); } /* Returns result or error code. */ std::variant is_safe_table(const char* dir_name, const char* myi_file_name) { ARIA_TABLE_CAPABILITIES cap; -#ifndef _WIN32 - std::string path= std::string(dir_name) + "/" + myi_file_name; - File fd= openat(datadir_fd, path.c_str(), O_RDONLY); - if (fd < 0) - { - my_errno= errno; - my_error(ER_CANT_OPEN_FILE, MYF(0), path.c_str(), errno); - } -#else - std::string path= std::string(maria_data_root) + "/" + - dir_name + "/" + myi_file_name; + std::string path= build_path(dir_name, myi_file_name); File fd= my_open(path.c_str(), O_RDONLY, MYF(MY_WME)); -#endif if (fd < 0) { + my_error(ER_CANT_OPEN_FILE, MYF(0), path.c_str(), my_errno); return my_errno; } std::variant result; @@ -480,15 +424,11 @@ namespace aria_free_capabilities(&cap); end: mysql_mutex_unlock(&THR_LOCK_maria); -#ifndef _WIN32 - close(fd); -#else my_close(fd, MYF(0)); -#endif return result; } - int copy_table(const backup_target &target, const backup_sink &sink, + int copy_table(const backup_target *target, const backup_sink *sink, const table_ref& table) noexcept { dir_ref dir_name = table.first; @@ -503,189 +443,70 @@ namespace data_path= index_path; index_path+= index_ext; data_path+= data_ext; - return copy_file(target, sink, index_path, false) || - copy_file(target, sink, data_path, false); + + return copy_table_file(target, sink, index_path) || + copy_table_file(target, sink, data_path); } - int copy_control_file(const backup_target &target, const backup_sink &sink) - noexcept + int copy_control_file(const backup_target *target, const backup_sink *sink) noexcept { if (!have_control_file) return 0; - return copy_file(target, sink, control_file_name, true); + return copy_log_file(target, sink, control_file_name.str); } - int copy_file(const backup_target &target, const backup_sink &sink, - const std::string &path, bool is_log) const noexcept + int copy_table_file(const backup_target *target, + const backup_sink *sink, + const std::string &path) const noexcept { - return copy_file(target, sink, path.c_str(), is_log); + return copy_table_file(target, sink, path.c_str()); } - int copy_file(const backup_target &target, const backup_sink &sink, - const char *path, bool is_log) const noexcept + int copy_table_file(const backup_target *target, + const backup_sink *sink, + const char *path) const noexcept + { + return ::copy_datafile_to_target(path, target, sink); + } + + int copy_log_file(const backup_target *target, + const backup_sink *sink, + const char *filename) { #ifndef _WIN32 - int ret_val{0}; - int src_fd{openat(is_log ? logdir_fd : datadir_fd, path, O_RDONLY)}; + int src_fd = openat(logdir_fd, filename, O_RDONLY); if (src_fd < 0) { - my_error(ER_CANT_OPEN_FILE, MYF(0), path, errno); + my_error(ER_CANT_OPEN_FILE, MYF(0), + build_path(maria_data_root, filename).c_str(), + errno); return 1; } - int tgt_fd{sink.stream}; - if (tgt_fd == sink.NO_STREAM) - { - tgt_fd= openat(target.fd, path, - O_CREAT | O_EXCL | O_WRONLY, 0666); - if (tgt_fd < 0) - { - my_error(ER_CANT_CREATE_FILE, MYF(0), path, errno); - ret_val= 1; - } - else - { - ret_val= copy_entire_file(src_fd, tgt_fd); - if (ret_val | close(tgt_fd)) - { - write_error: - my_error(ER_ERROR_ON_WRITE, MYF(0), path, errno); - ret_val= 1; - } - } - } - else - { - uint64_t end= uint64_t(lseek(src_fd, 0, SEEK_END)); - if (backup_stream_start(tgt_fd, path, 0644, end, nullptr, 0) || - backup_stream_append(src_fd, tgt_fd, 0, end)) - goto write_error; - if (size_t pad= size_t(end) & 511) - if (backup_stream_write(tgt_fd, zerobuf, 512 - pad)) - goto write_error; - } - + int ret_val= copy_fd_to_target(src_fd, target, filename, sink); close(src_fd); return ret_val; #else - const std::string src_path= build_path(is_log ? maria_data_root : mysql_real_data_home, path); - - if (sink.stream == sink.NO_STREAM) - { - const std::string dest_path= build_path(target.path, path); - if (!CopyFileEx(src_path.c_str(), dest_path.c_str(), nullptr, nullptr, nullptr, - COPY_FILE_NO_BUFFERING)) - { - my_osmaperr(GetLastError()); - my_error(ER_CANT_CREATE_FILE, MYF(0), dest_path.c_str(), errno); - return 1; - } - } - else - { - HANDLE src, dst{sink.stream}; - for (;;) - { - src= CreateFile(src_path.c_str(), GENERIC_READ, - FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, - my_win_file_secattr(), OPEN_EXISTING, - FILE_ATTRIBUTE_NORMAL, nullptr); - if (src != INVALID_HANDLE_VALUE) - break; - switch (GetLastError()) { - case ERROR_SHARING_VIOLATION: - case ERROR_LOCK_VIOLATION: - std::this_thread::sleep_for(std::chrono::seconds(1)); - continue; - } - - my_osmaperr(GetLastError()); - my_error(ER_FILE_NOT_FOUND, MYF(ME_ERROR_LOG), src_path.c_str(), - errno); - return -1; - } - - LARGE_INTEGER li; - if (!GetFileSizeEx(src, &li)) - { - write_error: - my_osmaperr(GetLastError()); - my_error(ER_ERROR_ON_WRITE, MYF(0), path, errno); - if (src != INVALID_HANDLE_VALUE) - CloseHandle(src); - return -1; - } - - if (backup_stream_start(dst, path, 0644, li.QuadPart, nullptr, 0) || - backup_stream_append_plain(src, dst, 0, li.QuadPart)) - goto write_error; - - if (size_t pad= size_t(li.LowPart) & 511) - if (backup_stream_write(dst, zerobuf, 512 - pad)) - goto write_error; - if (!CloseHandle(src)) - { - src= INVALID_HANDLE_VALUE; - goto write_error; - } - } - return 0; + return copy_entire_file(build_path(maria_data_root, filename).c_str(), + filename, target, sink); #endif } - static bool match_suffix(const char* suffix1, const char* suffix2) noexcept + static bool match_ext(const char* ext1, const char* ext2) noexcept { - return !memcmp(suffix1, suffix2, suffix_len); + return memcmp(ext1, ext2, ext_len) == 0; } - /* Match if suffix is one of the "other" extensions we need to copy */ - static bool match_misc_ext(const char* suffix_str) noexcept + static bool begins_with(const LEX_CSTRING &str, const LEX_CSTRING &prefix) noexcept { - uint32_t suffix; - static_assert (suffix_len == sizeof(suffix)); - memcpy(&suffix, suffix_str, suffix_len); - switch (suffix) { -#ifdef WORDS_BIGENDIAN - case 0x2e41524d: /* .ARM ENGINE=ARCHIVE metadata */ - case 0x2e41525a: /* .ARZ ENGINE=ARCHIVE compressed data */ - case 0x2e43534d: /* .CSM ENGINE=CSV metadata */ - case 0x2e435356: /* .CSV ENGINE=CSV data ("comma separated values") */ - case 0x2e4d5247: /* .MRG ENGINE=MRG_MyISAM */ - case 0x2e4d5944: /* .MYD ENGINE=MyISAM data heap */ - case 0x2e4d5949: /* .MYI ENGINE=MyISAM indexes */ - case 0x2e66726d: /* .frm form (SHOW CREATE TABLE) */ - case 0x2e706172: /* .par PARTITION metadata */ -#else - case 0x4d52412e: /* .ARM ENGINE=ARCHIVE metadata */ - case 0x5a52412e: /* .ARZ ENGINE=ARCHIVE compressed data */ - case 0x4d53432e: /* .CSM ENGINE=CSV metadata */ - case 0x5653432e: /* .CSV ENGINE=CSV data ("comma separated values") */ - case 0x47524d2e: /* .MRG ENGINE=MRG_MyISAM */ - case 0x44594d2e: /* .MYD ENGINE=MyISAM data heap */ - case 0x49594d2e: /* .MYI ENGINE=MyISAM indexes */ - case 0x6d72662e: /* .frm form (SHOW CREATE TABLE) */ - case 0x7261702e: /* .par PARTITION metadata */ -#endif - return true; - default: + if (str.length < prefix.length) return false; - } - } - - static bool begins_with(const char* str, const LEX_CSTRING &prefix) noexcept - { - return strncmp(str, prefix.str, prefix.length) == 0; + return memcmp(str.str, prefix.str, prefix.length) == 0; } - static std::string build_path(const char *base_path, const char *filename) noexcept + static bool is_control_file_name(const LEX_CSTRING &str) { - std::string path; - const size_t base_len= strlen(base_path); - const size_t filename_len= strlen(filename); - path.reserve(base_len + filename_len + 1); - path.append(base_path, base_len); - path+= '/'; - path.append(filename, filename_len); - return path; + return str.length == control_file_name.length && + memcmp(str.str, control_file_name.str, control_file_name.length) == 0; } }; } @@ -717,17 +538,12 @@ void *aria_backup_start(THD *thd, const backup_target *target, { #if 1 // FIXME: invoke these only for Aria, MyISAM, CSV but not others case BACKUP_PHASE_NO_DML_NON_TRANS: - /* FIXME: Would be better to selectively purge only the tables we need. */ tc_purge(); tdc_purge(true); break; #endif case BACKUP_PHASE_NO_DDL: -#if 1 // FIXME: invoke these only for Aria, MyISAM, CSV but not others - tc_purge(); - tdc_purge(true); -#endif - if (aria_backup->start_copy_dml_safe(*target, *sink)) + if (aria_backup->start_copy_dml_safe(target, sink)) goto error; break; case BACKUP_PHASE_NO_COMMIT: @@ -752,9 +568,9 @@ int aria_backup_step(THD*, const backup_target *target, backup_phase phase, switch (phase) { case BACKUP_PHASE_NO_DDL: - return aria_backup->dml_safe_copy_step(*target, *sink); + return aria_backup->dml_safe_copy_step(target, sink); case BACKUP_PHASE_NO_COMMIT: - return aria_backup->unsafe_copy_step(*target, *sink); + return aria_backup->unsafe_copy_step(target, sink); default: return 0; } From 733a4b79b26b56645d5e1fcafe85f23e68a938ca Mon Sep 17 00:00:00 2001 From: Andrzej Jarzabek Date: Wed, 24 Jun 2026 11:49:26 +0200 Subject: [PATCH 7/7] Refactor and simplify code given all Aria tables files are now copied in the same phase. --- storage/maria/ma_backup_server.cc | 198 ++++++------------------------ 1 file changed, 37 insertions(+), 161 deletions(-) diff --git a/storage/maria/ma_backup_server.cc b/storage/maria/ma_backup_server.cc index 34c0ede49c812..f4c4d20ec0187 100644 --- a/storage/maria/ma_backup_server.cc +++ b/storage/maria/ma_backup_server.cc @@ -31,7 +31,6 @@ #include #include #include -#include /* Implementation of functions declatred in ma_backup.h: @@ -40,58 +39,6 @@ namespace { - /* Utility class to implement the "backup step" interface when - processing several lists. It implements the logic where an item - is processed (copied) from the first list which has available - items, and a "remaining" counter accumulates the number of - items remaining to be processed on all lists, regardless of - whether an item from that list was processed or not. */ - class Copy_from_list - { - int m_remaining {0}; - bool m_copy_done; - public: - Copy_from_list(bool copy_done= false) noexcept - : m_copy_done(copy_done) - { - } - - bool copy_done() const noexcept - { - return m_copy_done; - } - - int remaining() const noexcept - { - return m_remaining; - } - - template - bool operator()(const T &list, std::atomic &copied, - Fn copy_action) noexcept - { - if(!m_copy_done) - { - size_t idx= copied.fetch_add(1, std::memory_order_relaxed); - if (idx < list.size()) - { - if (copy_action(list[idx]) != 0) - return true; - m_copy_done= true; - m_remaining+= static_cast(list.size() - idx - 1U); - } - } - else - { - size_t current_copied= copied.load(std::memory_order_relaxed); - if (current_copied < list.size()) - m_remaining+= static_cast(list.size() - current_copied); - } - return false; - } - }; - - class Aria_backup { public: @@ -148,18 +95,11 @@ namespace is in progress. */ int dml_safe_copy_step(const backup_target *target, const backup_sink *sink) noexcept { - Copy_from_list copy_from_list; - auto copy_table_action= [this, target, sink](const table_ref &table) noexcept - { - return copy_table(target, sink, table); - }; - if (copy_from_list(dml_safe_table_list, dml_safe_tables_copied, - copy_table_action) != 0) - return -1; - if (copy_from_list(unsafe_tables_list, unsafe_tables_copied, - copy_table_action) != 0) - return -1; - return copy_from_list.remaining(); + return copy_from_list_step(flat_table_list, tables_copied, + [this, target, sink](const table_ref &table) noexcept + { + return copy_table(target, sink, table); + }); } /* Copy an entity that is not safe to copy if there are concurrent @@ -173,8 +113,6 @@ namespace */ int unsafe_copy_step(const backup_target *target, const backup_sink *sink) noexcept { - bool copy_done= false; - /* If control file is always the first file copied and there is only one, it is never included in the "steps remaining" calculation. Should the order be changed, the calculation needs to be updated for @@ -186,19 +124,18 @@ namespace { if (copy_control_file(target, sink) != 0) return -1; - copy_done= true; + size_t current_copied= log_files_copied.load(std::memory_order_relaxed); + return (current_copied < log_files.size()) ? + static_cast(log_files.size() - current_copied) : + 0; } } - Copy_from_list copy_from_list(copy_done); - if (copy_from_list(log_files, log_files_copied, - [this, target, sink](const std::string &path) noexcept - { - return copy_log_file(target, sink, path.c_str()); - }) != 0) - return -1; - - return copy_from_list.remaining(); + return copy_from_list_step(log_files, log_files_copied, + [this, target, sink](const std::string &path) noexcept + { + return copy_log_file(target, sink, path.c_str()); + }); } int end(bool /*abort*/) noexcept @@ -227,10 +164,8 @@ namespace using dir_contents = std::vector; using database_dir = std::pair; using database_dirs = std::vector; - /* Transactional tables with checksum */ - database_dirs dml_safe_tables; - /* All other Aria tables */ - database_dirs unsafe_tables; + /* Collection of tables to be backed up. */ + database_dirs tables; /* Aria log files */ std::vector log_files; @@ -243,11 +178,8 @@ namespace using table_ref= std::pair; using table_list= std::vector; - /* Flattened versions of dml_safe_tables and unsafe_tables. */ - table_list dml_safe_table_list; - table_list unsafe_tables_list; - std::atomic dml_safe_tables_copied {0}; - std::atomic unsafe_tables_copied {0}; + table_list flat_table_list; + std::atomic tables_copied {0}; std::atomic log_files_copied {0}; std::atomic control_file_copied {false}; @@ -283,9 +215,7 @@ namespace MY_DIR *dir_info= my_dir(dir_name, MYF(MY_WANT_STAT)); if (!dir_info) return dir_error(dir_name); - int fail= 0; - dir_contents safe; - dir_contents unsafe; + dir_contents dir_tables; for (const fileinfo &fi : st_::span{dir_info->dir_entry, dir_info->number_of_files}) @@ -300,34 +230,15 @@ namespace { if (!is_tmp_table(filename)) { - auto is_safe = is_safe_table(dir_name, filename.str); - if (std::holds_alternative(is_safe)) - { - std::string table_name(filename.str, base_filename_len); - if (std::get(is_safe)) - safe.push_back(std::move(table_name)); - else - unsafe.push_back(std::move(table_name)); - } - else - { - fail= std::get(is_safe); - goto finish; - } + dir_tables.emplace_back(filename.str, base_filename_len); } } } } - if(!fail) - { - if (!safe.empty()) - dml_safe_tables.emplace_back(dir_name, std::move(safe)); - if (!unsafe.empty()) - unsafe_tables.emplace_back(dir_name, std::move(unsafe)); - } - finish: + if (!dir_tables.empty()) + tables.emplace_back(dir_name, std::move(dir_tables)); my_dirend(dir_info); - return fail; + return 0; } static bool is_tmp_table(const LEX_CSTRING &filename) noexcept @@ -337,8 +248,7 @@ namespace void flatten_table_lists() noexcept { - flatten_table_list(dml_safe_tables, dml_safe_table_list); - flatten_table_list(unsafe_tables, unsafe_tables_list); + flatten_table_list(tables, flat_table_list); } static void flatten_table_list(const database_dirs& dirs, table_list& list) noexcept @@ -372,60 +282,26 @@ namespace bool ensure_target_dirs(const backup_target *target) noexcept { - using string = std::string; - std::vector dirs; - for (const database_dir &dir : dml_safe_tables) - dirs.push_back(&dir.first); - for (const database_dir &dir : unsafe_tables) - dirs.push_back(&dir.first); - std::sort(dirs.begin(), dirs.end(), - [](const string *a, const string *b) { return *a < *b; }); - auto dirs_end = std::unique(dirs.begin(), dirs.end(), - [](const string *a, const string *b) { return *a == *b; }); - for (auto it = dirs.begin(); it != dirs_end; ++it) - { - if (ensure_target_subdir(target, (*it)->c_str())) + for (const database_dir &dir : tables) + if(::ensure_target_subdir(target, dir.first.c_str()) != 0) return true; - } return false; } - /* - Create directory in the target directory if it does not exist. - Return 0 on success, non-0 on failure. Set errno in case of failure - */ - int ensure_target_subdir(const backup_target *target, const char *name) - noexcept - { - return ::ensure_target_subdir(target, name); - } - - /* Returns result or error code. */ - std::variant is_safe_table(const char* dir_name, const char* myi_file_name) + template + static int copy_from_list_step(const std::vector &list, + std::atomic &copied, + Fn copy_action) { - ARIA_TABLE_CAPABILITIES cap; - std::string path= build_path(dir_name, myi_file_name); - File fd= my_open(path.c_str(), O_RDONLY, MYF(MY_WME)); - if (fd < 0) + size_t idx= copied.fetch_add(1, std::memory_order_relaxed); + if (idx < list.size()) { - my_error(ER_CANT_OPEN_FILE, MYF(0), path.c_str(), my_errno); - return my_errno; - } - std::variant result; - mysql_mutex_lock(&THR_LOCK_maria); - int fail = aria_get_capabilities(fd, myi_file_name, &cap); - if (fail) - { - my_error(ER_FILE_CORRUPT, MYF(0), path.c_str()); - result= fail; - goto end; + if (copy_action(list[idx]) != 0) + return -1; + return static_cast(list.size() - idx - 1U); } - result = cap.transactional && cap.checksum; - aria_free_capabilities(&cap); -end: - mysql_mutex_unlock(&THR_LOCK_maria); - my_close(fd, MYF(0)); - return result; + + return 0; } int copy_table(const backup_target *target, const backup_sink *sink,