From 71f911c45982136d525e21497dbf2b7c5cb60caa Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Fri, 15 May 2026 18:27:21 +0530 Subject: [PATCH 1/2] MDEV-28730 Remove internal parser usage from InnoDB FTS InnoDB FTS performed all reads and DML on its auxiliary, common and CONFIG tables through the InnoDB internal SQL graph parser. Replace that path with direct B-tree access via a new query-executor layer, and delete the parser-era helpers. row0query.h, row/row0query.cc - new QueryExecutor: - General MVCC-aware record traversal and basic DML on the clustered index. Sits on a btr_pcur and a transaction-owned mtr; record processing goes through a RecordCallback that bundles two std::functions: compare_record() returns SKIP / PROCESS / STOP for a record process_record() handles each PROCESS-ed (MVCC-visible) row Public API: read() scan a clustered index with a search key read_all() full clustered scan (optional start tuple) read_by_index() scan a secondary index, fetch the matching clustered record, deliver it to the callback insert_record() insert a tuple into the clustered index delete_record() delete a row identified by tuple delete_all() delete every row in the clustered index select_for_update() position+X-lock the matching clustered row update_record() update the row select_for_update() locked, falling back to optimistic/pessimistic and external storage paths as needed replace_record() upsert: select_for_update()+update_record(), else insert_record() lock_table(), handle_wait(), commit_mtr() fts0exec.h, fts/fts0exec.cc - new FTSQueryExecutor: Thin wrapper over QueryExecutor specialised for FTS tables. Opens and locks the required tables once and exposes typed helpers keyed by table family. Auxiliary INDEX_[1..6]: open_all_aux_tables() insert_aux_record(aux_index, fts_aux_data_t) delete_aux_record(aux_index, fts_aux_data_t) read_aux() range scan from a given word read_from_range() paginated read that absorbs DB_FTS_EXCEED_RESULT_CACHE_LIMIT internally and resumes from the last word seen Common deletion tables (DELETED, DELETED_CACHE, BEING_DELETED, BEING_DELETED_CACHE): open_all_deletion_tables() insert_common_record(), delete_common_record(), delete_all_common_records(), read_all_common() CONFIG table (): open_config_table() / set_config_table() insert_config_record(), update_config_record() (upsert), delete_config_record(), read_config_with_lock() fts_aux_data_t carries the auxiliary row payload. RecordCallback specialisations live alongside the executor: CommonTableReader collects doc_ids from common tables that share the schema. ConfigReader extracts and provides compare_config_key() for fast key matching. AuxRecordReader scans auxiliary indexes with an AuxCompareMode (GREATER_EQUAL / GREATER / LIKE / EQUAL) driving the comparator; tracks the last word seen so a paginated scan can resume. fts_query() walks index and common tables via QueryExecutor::read_by_index() with RecordCallback; fts_write_node() writes auxiliary rows through FTSQueryExecutor::insert_aux_record() / delete_aux_record() with fts_aux_data_t. fts_optimize_write_word() now goes through the same insert/delete path. fts_select_index{,_by_range,_by_hash} return uint8_t (was ulint) with a simpler control flow. fts_optimize_table() binds a thd to its transaction whether invoked from a user thread or the FTS optimize thread. fts_optimize_t drops its fts_index_table and fts_common_table fts_table_t fields; fts_query_t drops fts_common_table. storage/innobase/fts/fts0sql.cc is deleted along with the commented-out and unreferenced parser-era helpers it held. dict_sys.latch is now acquired once per fts_sync_table(), fts_optimize_table() and fts_query() call to open every auxiliary and common table in one pass, instead of being re-acquired per table. --- .../suite/innodb_fts/r/index_table.result | 2 +- mysql-test/suite/innodb_fts/r/sync.result | 12 +- .../suite/innodb_fts/t/index_table.test | 2 +- mysql-test/suite/innodb_fts/t/sync.test | 6 +- storage/innobase/CMakeLists.txt | 5 +- storage/innobase/fts/fts0config.cc | 309 +-- storage/innobase/fts/fts0exec.cc | 886 ++++++++ storage/innobase/fts/fts0fts.cc | 1965 ++++++----------- storage/innobase/fts/fts0opt.cc | 1465 ++++-------- storage/innobase/fts/fts0que.cc | 1337 ++++------- storage/innobase/fts/fts0sql.cc | 208 -- storage/innobase/handler/ha_innodb.cc | 96 +- storage/innobase/handler/i_s.cc | 268 ++- storage/innobase/include/btr0cur.h | 2 +- storage/innobase/include/fts0exec.h | 422 ++++ storage/innobase/include/fts0fts.h | 115 +- storage/innobase/include/fts0priv.h | 184 +- storage/innobase/include/fts0types.h | 2 +- storage/innobase/include/fts0types.inl | 18 +- storage/innobase/include/page0cur.h | 15 + storage/innobase/include/row0mysql.h | 4 + storage/innobase/include/row0query.h | 244 ++ storage/innobase/include/row0sel.h | 57 + storage/innobase/include/ut0new.h | 1 + storage/innobase/page/page0cur.cc | 6 +- storage/innobase/row/row0merge.cc | 22 +- storage/innobase/row/row0mysql.cc | 17 +- storage/innobase/row/row0query.cc | 445 ++++ storage/innobase/row/row0sel.cc | 498 +++-- 29 files changed, 4468 insertions(+), 4145 deletions(-) create mode 100644 storage/innobase/fts/fts0exec.cc delete mode 100644 storage/innobase/fts/fts0sql.cc create mode 100644 storage/innobase/include/fts0exec.h create mode 100644 storage/innobase/include/row0query.h create mode 100644 storage/innobase/row/row0query.cc diff --git a/mysql-test/suite/innodb_fts/r/index_table.result b/mysql-test/suite/innodb_fts/r/index_table.result index 909a889db4230..78832669cdbca 100644 --- a/mysql-test/suite/innodb_fts/r/index_table.result +++ b/mysql-test/suite/innodb_fts/r/index_table.result @@ -5,7 +5,7 @@ id INT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY, title VARCHAR(200), content TEXT ) ENGINE= InnoDB; -SET STATEMENT debug_dbug='+d,innodb_report_deadlock' FOR +SET STATEMENT debug_dbug='+d,fts_load_stopword_fail' FOR CREATE FULLTEXT INDEX idx ON articles (title, content); ERROR HY000: Got error 11 "Resource temporarily unavailable" from storage engine InnoDB CREATE FULLTEXT INDEX idx ON articles (title, content); diff --git a/mysql-test/suite/innodb_fts/r/sync.result b/mysql-test/suite/innodb_fts/r/sync.result index 928efffdb21d9..862df4cb4b3e4 100644 --- a/mysql-test/suite/innodb_fts/r/sync.result +++ b/mysql-test/suite/innodb_fts/r/sync.result @@ -25,7 +25,7 @@ mysql 1 3 2 3 0 SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE; WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION SET GLOBAL innodb_ft_aux_table=default; -SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); +SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database') ORDER BY FTS_DOC_ID; FTS_DOC_ID title 1 mysql 2 database @@ -42,11 +42,11 @@ database 2 3 2 3 6 mysql 1 3 2 1 0 mysql 1 3 2 3 0 SET GLOBAL innodb_ft_aux_table=default; -SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); +SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database') ORDER BY FTS_DOC_ID; FTS_DOC_ID title -3 mysql database 1 mysql 2 database +3 mysql database connection default; DROP TABLE t1; # Case 2: Test insert and insert(sync) @@ -79,12 +79,12 @@ mysql 1 4 3 1 0 mysql 1 4 3 3 0 mysql 1 4 3 4 0 SET GLOBAL innodb_ft_aux_table=default; -SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); +SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database') ORDER BY FTS_DOC_ID; FTS_DOC_ID title -3 mysql database -4 mysql database 1 mysql 2 database +3 mysql database +4 mysql database connection default; disconnect con1; DROP TABLE t1; diff --git a/mysql-test/suite/innodb_fts/t/index_table.test b/mysql-test/suite/innodb_fts/t/index_table.test index 89c0905323083..cfe27b4226848 100644 --- a/mysql-test/suite/innodb_fts/t/index_table.test +++ b/mysql-test/suite/innodb_fts/t/index_table.test @@ -18,7 +18,7 @@ CREATE TABLE articles ( ) ENGINE= InnoDB; --error ER_GET_ERRNO -SET STATEMENT debug_dbug='+d,innodb_report_deadlock' FOR +SET STATEMENT debug_dbug='+d,fts_load_stopword_fail' FOR CREATE FULLTEXT INDEX idx ON articles (title, content); CREATE FULLTEXT INDEX idx ON articles (title, content); diff --git a/mysql-test/suite/innodb_fts/t/sync.test b/mysql-test/suite/innodb_fts/t/sync.test index 56b9052a47ab8..348c37d7389d8 100644 --- a/mysql-test/suite/innodb_fts/t/sync.test +++ b/mysql-test/suite/innodb_fts/t/sync.test @@ -40,7 +40,7 @@ SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE; SET GLOBAL innodb_ft_aux_table=default; -SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); +SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database') ORDER BY FTS_DOC_ID; SET DEBUG_SYNC= 'now SIGNAL selected'; @@ -54,7 +54,7 @@ SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE; SET GLOBAL innodb_ft_aux_table=default; -SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); +SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database') ORDER BY FTS_DOC_ID; connection default; @@ -96,7 +96,7 @@ SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE; SET GLOBAL innodb_ft_aux_table=default; -SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); +SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database') ORDER BY FTS_DOC_ID; connection default; disconnect con1; diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 9e3a23b34ab46..c1b4a5c9559fc 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -172,10 +172,10 @@ SET(INNOBASE_SOURCES fts/fts0ast.cc fts/fts0blex.cc fts/fts0config.cc + fts/fts0exec.cc fts/fts0opt.cc fts/fts0pars.cc fts/fts0que.cc - fts/fts0sql.cc fts/fts0tlex.cc gis/gis0geo.cc gis/gis0rtree.cc @@ -242,6 +242,7 @@ SET(INNOBASE_SOURCES include/fsp_binlog.h include/fts0ast.h include/fts0blex.h + include/fts0exec.h include/fts0fts.h include/fts0opt.h include/fts0pars.h @@ -315,6 +316,7 @@ SET(INNOBASE_SOURCES include/row0mysql.h include/row0purge.h include/row0quiesce.h + include/row0query.h include/row0row.h include/row0row.inl include/row0sel.h @@ -403,6 +405,7 @@ SET(INNOBASE_SOURCES row/row0undo.cc row/row0upd.cc row/row0quiesce.cc + row/row0query.cc row/row0vers.cc srv/srv0mon.cc srv/srv0srv.cc diff --git a/storage/innobase/fts/fts0config.cc b/storage/innobase/fts/fts0config.cc index 4ff14edf5d016..fa127c27b058a 100644 --- a/storage/innobase/fts/fts0config.cc +++ b/storage/innobase/fts/fts0config.cc @@ -27,100 +27,33 @@ Created 2007/5/9 Sunny Bains #include "trx0roll.h" #include "row0sel.h" +#include "fts0exec.h" #include "fts0priv.h" +#include "log.h" -/******************************************************************//** -Callback function for fetching the config value. -@return always returns TRUE */ -static -ibool -fts_config_fetch_value( -/*===================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to - ib_vector_t */ -{ - sel_node_t* node = static_cast(row); - fts_string_t* value = static_cast(user_arg); - - dfield_t* dfield = que_node_get_val(node->select_list); - dtype_t* type = dfield_get_type(dfield); - ulint len = dfield_get_len(dfield); - void* data = dfield_get_data(dfield); - - ut_a(dtype_get_mtype(type) == DATA_VARCHAR); - - if (len != UNIV_SQL_NULL) { - ulint max_len = ut_min(value->f_len - 1, len); - - memcpy(value->f_str, data, max_len); - value->f_len = max_len; - value->f_str[value->f_len] = '\0'; - } - - return(TRUE); -} - -/******************************************************************//** -Get value from the config table. The caller must ensure that enough +/** Get value from the config table. The caller must ensure that enough space is allocated for value to hold the column contents. +@param trx transaction +@param table Indexed fts table +@param name name of the key +@param value value of the key @return DB_SUCCESS or error code */ -dberr_t -fts_config_get_value( -/*=================*/ - trx_t* trx, /*!< transaction */ - fts_table_t* fts_table, /*!< in: the indexed - FTS table */ - const char* name, /*!< in: get config value for - this parameter name */ - fts_string_t* value) /*!< out: value read from - config table */ +dberr_t fts_config_get_value(FTSQueryExecutor *executor, const dict_table_t *table, + const char *name, fts_string_t *value) noexcept { - pars_info_t* info; - que_t* graph; - dberr_t error; - ulint name_len = strlen(name); - char table_name[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - *value->f_str = '\0'; - ut_a(value->f_len > 0); - - pars_info_bind_function(info, "my_func", fts_config_fetch_value, - value); - - /* The len field of value must be set to the max bytes that - it can hold. On a successful read, the len field will be set - to the actual number of bytes copied to value. */ - pars_info_bind_varchar_literal(info, "name", (byte*) name, name_len); - - fts_table->suffix = "CONFIG"; - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS SELECT value FROM $table_name" - " WHERE key = :name;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - trx->op_info = "getting FTS config value"; - - error = fts_eval_sql(trx, graph); - que_graph_free(graph); - return(error); + executor->trx()->op_info= "getting FTS config value"; + ConfigReader reader; + dberr_t err= executor->read_config_with_lock(name, reader); + if (err == DB_SUCCESS) + { + ulint max_len= ut_min(value->f_len - 1, reader.value_span.size()); + memcpy(value->f_str, reader.value_span.data(), max_len); + value->f_len= max_len; + value->f_str[value->f_len]= '\0'; + executor->release_lock(); + } + else value->f_str[0]= '\0'; + return err; } /*********************************************************************//** @@ -157,105 +90,38 @@ column contents. dberr_t fts_config_get_index_value( /*=======================*/ - trx_t* trx, /*!< transaction */ + FTSQueryExecutor* executor, /*!< in: query executor */ dict_index_t* index, /*!< in: index */ const char* param, /*!< in: get config value for this parameter name */ - fts_string_t* value) /*!< out: value read from + fts_string_t* value) noexcept /*!< out: value read from config table */ { - char* name; - dberr_t error; - fts_table_t fts_table; - - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, - index->table); - /* We are responsible for free'ing name. */ - name = fts_config_create_index_param_name(param, index); + char *name = fts_config_create_index_param_name(param, index); - error = fts_config_get_value(trx, &fts_table, name, value); + dberr_t error = fts_config_get_value(executor, index->table, name, value); ut_free(name); return(error); } -/******************************************************************//** -Set the value in the config table for name. +/** Set the value in the config table for name. +@param executor query executor +@param table indexed fulltext table +@param name key for the config +@param value value of the key @return DB_SUCCESS or error code */ dberr_t -fts_config_set_value( -/*=================*/ - trx_t* trx, /*!< transaction */ - fts_table_t* fts_table, /*!< in: the indexed - FTS table */ - const char* name, /*!< in: get config value for - this parameter name */ - const fts_string_t* - value) /*!< in: value to update */ +fts_config_set_value(FTSQueryExecutor *executor, const dict_table_t *table, + const char *name, const fts_string_t *value) noexcept { - pars_info_t* info; - que_t* graph; - dberr_t error; - undo_no_t undo_no; - undo_no_t n_rows_updated; - ulint name_len = strlen(name); - char table_name[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - pars_info_bind_varchar_literal(info, "name", (byte*) name, name_len); - pars_info_bind_varchar_literal(info, "value", - value->f_str, value->f_len); - - const bool dict_locked = fts_table->table->fts->dict_locked; - - fts_table->suffix = "CONFIG"; - fts_get_table_name(fts_table, table_name, dict_locked); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, info, - "BEGIN UPDATE $table_name SET value = :value" - " WHERE key = :name;"); - - trx->op_info = "setting FTS config value"; - - undo_no = trx->undo_no; - - error = fts_eval_sql(trx, graph); - - que_graph_free(graph); - - n_rows_updated = trx->undo_no - undo_no; - - /* Check if we need to do an insert. */ - if (error == DB_SUCCESS && n_rows_updated == 0) { - info = pars_info_create(); - - pars_info_bind_varchar_literal( - info, "name", (byte*) name, name_len); - - pars_info_bind_varchar_literal( - info, "value", value->f_str, value->f_len); - - fts_get_table_name(fts_table, table_name, dict_locked); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, info, - "BEGIN\n" - "INSERT INTO $table_name VALUES(:name, :value);"); - - trx->op_info = "inserting FTS config value"; - - error = fts_eval_sql(trx, graph); - - que_graph_free(graph); - } - - return(error); + executor->trx()->op_info= "setting FTS config value"; + char value_str[FTS_MAX_CONFIG_VALUE_LEN + 1]; + memcpy(value_str, value->f_str, value->f_len); + value_str[value->f_len]= '\0'; + return executor->update_config_record(name, value_str); } /******************************************************************//** @@ -264,95 +130,44 @@ Set the value specific to an FTS index in the config table. dberr_t fts_config_set_index_value( /*=======================*/ - trx_t* trx, /*!< transaction */ + FTSQueryExecutor* executor, /*!< in: query executor */ dict_index_t* index, /*!< in: index */ const char* param, /*!< in: get config value for this parameter name */ - fts_string_t* value) /*!< out: value read from + fts_string_t* value) noexcept /*!< out: value read from config table */ { - char* name; - dberr_t error; - fts_table_t fts_table; - - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, - index->table); - /* We are responsible for free'ing name. */ - name = fts_config_create_index_param_name(param, index); + char *name = fts_config_create_index_param_name(param, index); - error = fts_config_set_value(trx, &fts_table, name, value); + dberr_t error = fts_config_set_value(executor, index->table, name, value); ut_free(name); return(error); } -/******************************************************************//** -Get an ulint value from the config table. +/** Set an ulint value in the config table. +@param trx transaction +@param table user table +@param name name of the key +@param int_value value of the key to be set @return DB_SUCCESS if all OK else error code */ dberr_t -fts_config_get_ulint( -/*=================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: the indexed - FTS table */ - const char* name, /*!< in: param name */ - ulint* int_value) /*!< out: value */ +fts_config_set_ulint(FTSQueryExecutor *executor, const dict_table_t *table, + const char *name, ulint int_value) noexcept { - dberr_t error; - fts_string_t value; - - /* We set the length of value to the max bytes it can hold. This - information is used by the callback that reads the value.*/ - value.f_len = FTS_MAX_CONFIG_VALUE_LEN; - value.f_str = static_cast(ut_malloc_nokey(value.f_len + 1)); - - error = fts_config_get_value(trx, fts_table, name, &value); - - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ") reading `" << name << "'"; - } else { - *int_value = strtoul((char*) value.f_str, NULL, 10); - } - - ut_free(value.f_str); - - return(error); -} - -/******************************************************************//** -Set an ulint value in the config table. -@return DB_SUCCESS if all OK else error code */ -dberr_t -fts_config_set_ulint( -/*=================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: the indexed - FTS table */ - const char* name, /*!< in: param name */ - ulint int_value) /*!< in: value */ -{ - dberr_t error; - fts_string_t value; - - /* We set the length of value to the max bytes it can hold. This - information is used by the callback that reads the value.*/ - value.f_len = FTS_MAX_CONFIG_VALUE_LEN; - value.f_str = static_cast(ut_malloc_nokey(value.f_len + 1)); - - ut_a(FTS_MAX_INT_LEN < FTS_MAX_CONFIG_VALUE_LEN); - - value.f_len = (ulint) snprintf( - (char*) value.f_str, FTS_MAX_INT_LEN, ULINTPF, int_value); - - error = fts_config_set_value(trx, fts_table, name, &value); - - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ") writing `" << name << "'"; - } - - ut_free(value.f_str); - - return(error); + fts_string_t value; + /* We set the length of value to the max bytes it can hold. This + information is used by the callback that reads the value.*/ + value.f_len= FTS_MAX_CONFIG_VALUE_LEN; + value.f_str= static_cast(ut_malloc_nokey(value.f_len + 1)); + ut_a(FTS_MAX_INT_LEN < FTS_MAX_CONFIG_VALUE_LEN); + value.f_len= (ulint) snprintf((char*) value.f_str, FTS_MAX_INT_LEN, + ULINTPF, int_value); + dberr_t error= fts_config_set_value(executor, table, name, &value); + if (UNIV_UNLIKELY(error != DB_SUCCESS)) + sql_print_error("InnoDB: (%s) writing `%s'", ut_strerr(error), name); + ut_free(value.f_str); + return error; } diff --git a/storage/innobase/fts/fts0exec.cc b/storage/innobase/fts/fts0exec.cc new file mode 100644 index 0000000000000..210ae28dc2788 --- /dev/null +++ b/storage/innobase/fts/fts0exec.cc @@ -0,0 +1,886 @@ +/***************************************************************************** +Copyright (c) 2025, MariaDB PLC. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA +*****************************************************************************/ + +/**************************************************//** +@file fts/fts0exec.cc + +Created 2025/11/05 +*******************************************************/ + +#include "fts0exec.h" +#include "row0query.h" +#include "fts0fts.h" +#include "fts0types.h" +#include "fts0vlc.h" +#include "fts0priv.h" +#include "btr0btr.h" +#include "btr0cur.h" +#include "dict0dict.h" +#include "row0ins.h" +#include "row0upd.h" +#include "row0sel.h" +#include "eval0eval.h" +#include "que0que.h" +#include "trx0trx.h" +#include "lock0lock.h" +#include "rem0cmp.h" +#include "page0cur.h" +#include "ha_prototypes.h" + + +FTSQueryExecutor::FTSQueryExecutor( + trx_t *trx, const dict_table_t *fts_table) + : m_executor(trx), m_table(fts_table) +{} + +FTSQueryExecutor::~FTSQueryExecutor() +{ + for (uint8_t i= 0; i < FTS_NUM_AUX_INDEX; i++) + if (m_aux_tables[i]) m_aux_tables[i]->release(); + + for (uint8_t i= 0; i < NUM_DELETION_TABLES; i++) + if (m_common_tables[i]) m_common_tables[i]->release(); + + if (m_config_table) m_config_table->release(); +} + +dberr_t FTSQueryExecutor::open_aux_table(uint8_t aux_index) noexcept +{ + if (m_aux_tables[aux_index]) return DB_SUCCESS; + + char table_name[MAX_FULL_NAME_LEN]; + construct_table_name(table_name, fts_get_suffix(aux_index), false); + + m_aux_tables[aux_index]= dict_table_open_on_name( + table_name, false, DICT_ERR_IGNORE_TABLESPACE); + return m_aux_tables[aux_index] ? DB_SUCCESS : DB_TABLE_NOT_FOUND; +} + +dberr_t FTSQueryExecutor::open_all_aux_tables(dict_index_t *fts_index) noexcept +{ + for (uint8_t idx= 0; idx < FTS_NUM_AUX_INDEX; idx++) + { + dict_table_t *table= m_aux_tables[idx]; + if (table) + { + table->release(); + m_aux_tables[idx]= nullptr; + } + } + m_index= fts_index; + for (uint8_t idx= 0; idx < FTS_NUM_AUX_INDEX; idx++) + { + dberr_t err= open_aux_table(idx); + if (err) return err; + } + return DB_SUCCESS; +} + +const char* FTSQueryExecutor::get_deletion_table_name( + FTSDeletionTable table_type) noexcept +{ + switch (table_type) + { + case FTSDeletionTable::DELETED: return "DELETED"; + case FTSDeletionTable::DELETED_CACHE: return "DELETED_CACHE"; + case FTSDeletionTable::BEING_DELETED: return "BEING_DELETED"; + case FTSDeletionTable::BEING_DELETED_CACHE: return "BEING_DELETED_CACHE"; + default: return nullptr; + } +} + +/** Helper to convert table name to deletion table enum */ +static FTSDeletionTable get_deletion_table_type(const char *tbl_name) noexcept +{ + if (!strcmp(tbl_name, "DELETED")) return FTSDeletionTable::DELETED; + if (!strcmp(tbl_name, "DELETED_CACHE")) return FTSDeletionTable::DELETED_CACHE; + if (!strcmp(tbl_name, "BEING_DELETED")) return FTSDeletionTable::BEING_DELETED; + if (!strcmp(tbl_name, "BEING_DELETED_CACHE")) return FTSDeletionTable::BEING_DELETED_CACHE; + return FTSDeletionTable::MAX_DELETION_TABLES; +} + +dberr_t FTSQueryExecutor::open_deletion_table(FTSDeletionTable table_type) noexcept +{ + uint8_t index= to_index(table_type); + if (index >= NUM_DELETION_TABLES) + return DB_ERROR; + + if (m_common_tables[index]) return DB_SUCCESS; + + const char *suffix_name= get_deletion_table_name(table_type); + if (!suffix_name) return DB_ERROR; + + char table_name[MAX_FULL_NAME_LEN]; + construct_table_name(table_name, suffix_name, true); + + m_common_tables[index]= dict_table_open_on_name( + table_name, false, DICT_ERR_IGNORE_TABLESPACE); + return m_common_tables[index] ? DB_SUCCESS : DB_TABLE_NOT_FOUND; +} + +dberr_t FTSQueryExecutor::open_config_table() noexcept +{ + if (m_config_table) return DB_SUCCESS; + char table_name[MAX_FULL_NAME_LEN]; + construct_table_name(table_name, "CONFIG", true); + + m_config_table= dict_table_open_on_name( + table_name, false, DICT_ERR_IGNORE_TABLESPACE); + return m_config_table ? DB_SUCCESS : DB_TABLE_NOT_FOUND; +} + +dberr_t FTSQueryExecutor::open_all_deletion_tables() noexcept +{ + for (uint8_t i= 0; i < NUM_DELETION_TABLES; i++) + { + FTSDeletionTable table_type= static_cast(i); + dberr_t err= open_deletion_table(table_type); + if (err) return err; + } + return DB_SUCCESS; +} + +dberr_t FTSQueryExecutor::lock_aux_tables(uint8_t aux_index, + lock_mode mode) noexcept +{ + dict_table_t *table= m_aux_tables[aux_index]; + if (table == nullptr) return DB_TABLE_NOT_FOUND; + dberr_t err= m_executor.lock_table(table, mode); + if (err == DB_LOCK_WAIT) err= m_executor.handle_wait(err, true); + return err; +} + +dberr_t FTSQueryExecutor::lock_common_tables(uint8_t index, + lock_mode mode) noexcept +{ + dict_table_t *table= m_common_tables[index]; + if (table == nullptr) return DB_TABLE_NOT_FOUND; + dberr_t err= m_executor.lock_table(table, mode); + if (err == DB_LOCK_WAIT) err= m_executor.handle_wait(err, true); + return err; +} + +dberr_t FTSQueryExecutor::insert_aux_record( + uint8_t aux_index, const fts_aux_data_t *aux_data) noexcept +{ + ut_ad(!dict_sys.locked()); + if (aux_index >= FTS_NUM_AUX_INDEX) return DB_ERROR; + + dberr_t err= open_aux_table(aux_index); + if (err != DB_SUCCESS) return err; + err= lock_aux_tables(aux_index, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t *table= m_aux_tables[aux_index]; + dict_index_t* index= dict_table_get_first_index(table); + + if (index->n_fields != 7 || index->n_uniq != 2) + return DB_ERROR; + + byte sys_buf[DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN]= {0}; + dfield_t fields[7]; + doc_id_t first_doc_id, last_doc_id; + + dtuple_t tuple{0, 7, 2, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 7); + /* Field 0: word (VARCHAR) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, aux_data->word, aux_data->word_len); + + /* Field 1: first_doc_id (INT) */ + field= dtuple_get_nth_field(&tuple, 1); + fts_write_doc_id(&first_doc_id, aux_data->first_doc_id); + dfield_set_data(field, &first_doc_id, sizeof(doc_id_t)); + + /* Field 2: trx_id (DB_TRX_ID) */ + field= dtuple_get_nth_field(&tuple, 2); + dfield_set_data(field, sys_buf, DATA_TRX_ID_LEN); + + /* Field 3: roll_ptr (DB_ROLL_PTR) */ + field= dtuple_get_nth_field(&tuple, 3); + dfield_set_data(field, sys_buf + DATA_TRX_ID_LEN, DATA_ROLL_PTR_LEN); + + /* Field 4: last_doc_id (UNSIGNED INT) */ + field= dtuple_get_nth_field(&tuple, 4); + fts_write_doc_id(&last_doc_id, aux_data->last_doc_id); + dfield_set_data(field, &last_doc_id, sizeof(doc_id_t)); + + /* Field 5: doc_count (UINT32_T) */ + byte doc_count[4]; + mach_write_to_4(doc_count, aux_data->doc_count); + field= dtuple_get_nth_field(&tuple, 5); + dfield_set_data(field, doc_count, sizeof(doc_count)); + + /* Field 6: ilist (VARBINARY) */ + field= dtuple_get_nth_field(&tuple, 6); + dfield_set_data(field, aux_data->ilist, aux_data->ilist_len); + + return m_executor.insert_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::insert_common_record( + const char *tbl_name, doc_id_t doc_id) noexcept +{ + ut_ad(!dict_sys.locked()); + FTSDeletionTable table_type= get_deletion_table_type(tbl_name); + if (table_type == FTSDeletionTable::MAX_DELETION_TABLES) return DB_ERROR; + + dberr_t err= open_deletion_table(table_type); + if (err != DB_SUCCESS) return err; + + uint8_t index_no= to_index(table_type); + err= lock_common_tables(index_no, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[index_no]; + dict_index_t* index= dict_table_get_first_index(table); + + if (index->n_fields != 3 || index->n_uniq != 1) + return DB_ERROR; + + byte sys_buf[DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN]= {0}; + dfield_t fields[3]; + + dtuple_t tuple{0, 3, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 3); + /* Field 0: doc_id (INT) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id(&write_doc_id, doc_id); + dfield_set_data(field, &write_doc_id, sizeof(doc_id_t)); + + /* Field 1: trx_id (DB_TRX_ID) */ + field= dtuple_get_nth_field(&tuple, 1); + dfield_set_data(field, sys_buf, DATA_TRX_ID_LEN); + + /* Field 2: roll_ptr (DB_ROLL_PTR) */ + field= dtuple_get_nth_field(&tuple, 2); + dfield_set_data(field, sys_buf + DATA_TRX_ID_LEN, DATA_ROLL_PTR_LEN); + + return m_executor.insert_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::insert_config_record( + const char *key, const char *value) noexcept +{ + dberr_t err= open_config_table(); + if (err != DB_SUCCESS) return err; + + err= m_executor.lock_table(m_config_table, LOCK_IX); + if (err == DB_LOCK_WAIT) err= m_executor.handle_wait(err, true); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_config_table; + dict_index_t* index= dict_table_get_first_index(table); + + if (index->n_fields != 4 || index->n_uniq != 1) + return DB_ERROR; + + byte sys_buf[DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN]= {0}; + dfield_t fields[4]; + + dtuple_t tuple{0, 4, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 4); + /* Field 0: key (CHAR(50)) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, key, strlen(key)); + + /* Field 1: trx_id (DB_TRX_ID) */ + field= dtuple_get_nth_field(&tuple, 1); + dfield_set_data(field, sys_buf, DATA_TRX_ID_LEN); + + /* Field 2: roll_ptr (DB_ROLL_PTR) */ + field= dtuple_get_nth_field(&tuple, 2); + dfield_set_data(field, sys_buf + DATA_TRX_ID_LEN, DATA_ROLL_PTR_LEN); + + /* Field 3: value (CHAR(200)) */ + field= dtuple_get_nth_field(&tuple, 3); + dfield_set_data(field, value, strlen(value)); + + return m_executor.insert_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::update_config_record( + const char *key, const char *value) noexcept +{ + dberr_t err= open_config_table(); + if (err != DB_SUCCESS) return err; + + err= m_executor.lock_table(m_config_table, LOCK_IX); + if (err == DB_LOCK_WAIT) err= m_executor.handle_wait(err, true); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_config_table; + dict_index_t* index= dict_table_get_first_index(table); + + if (index->n_fields != 4 || index->n_uniq != 1) + return DB_ERROR; + + byte sys_buf[DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN]= {0}; + dfield_t search_fields[1]; + dfield_t insert_fields[4]; + + dtuple_t search_tuple{0, 1, 1, 0, search_fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&search_tuple, index, 1); + dfield_t *field= dtuple_get_nth_field(&search_tuple, 0); + dfield_set_data(field, key, strlen(key)); + + dtuple_t insert_tuple{0, 4, 1, 0, insert_fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&insert_tuple, index, 4); + + /* Field 0: key (CHAR(50)) */ + field= dtuple_get_nth_field(&insert_tuple, 0); + dfield_set_data(field, key, strlen(key)); + + /* Field 1: trx_id (DB_TRX_ID) */ + field= dtuple_get_nth_field(&insert_tuple, 1); + dfield_set_data(field, sys_buf, DATA_TRX_ID_LEN); + + /* Field 2: roll_ptr (DB_ROLL_PTR) */ + field= dtuple_get_nth_field(&insert_tuple, 2); + dfield_set_data(field, sys_buf + DATA_TRX_ID_LEN, DATA_ROLL_PTR_LEN); + + /* Field 3: value (CHAR(200)) */ + field= dtuple_get_nth_field(&insert_tuple, 3); + dfield_set_data(field, value, strlen(value)); + + upd_field_t upd_field; + upd_field.field_no= 3; + upd_field.orig_len= 0; + upd_field.exp= nullptr; + dfield_set_data(&upd_field.new_val, value, strlen(value)); + dict_col_copy_type(dict_index_get_nth_col(index, 3), + dfield_get_type(&upd_field.new_val)); + + upd_t update; + update.heap= nullptr; + update.info_bits= 0; + update.old_vrow= nullptr; + update.n_fields= 1; + update.fields= &upd_field; + + return m_executor.replace_record(table, &search_tuple, &update, + &insert_tuple); +} + +dberr_t FTSQueryExecutor::delete_aux_record( + uint8_t aux_index, const fts_aux_data_t *aux_data) noexcept +{ + ut_ad(!dict_sys.locked()); + if (aux_index >= FTS_NUM_AUX_INDEX) return DB_ERROR; + + dberr_t err= open_aux_table(aux_index); + if (err != DB_SUCCESS) return err; + err= lock_aux_tables(aux_index, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t *table= m_aux_tables[aux_index]; + dict_index_t *index= dict_table_get_first_index(table); + + if (dict_table_get_next_index(index) != nullptr) + return DB_ERROR; + + dfield_t fields[1]; + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: word (VARCHAR) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, aux_data->word, aux_data->word_len); + + return m_executor.delete_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::delete_common_record( + const char *table_name, doc_id_t doc_id) noexcept +{ + ut_ad(!dict_sys.locked()); + FTSDeletionTable table_type= get_deletion_table_type(table_name); + if (table_type == FTSDeletionTable::MAX_DELETION_TABLES) + return DB_ERROR; + + dberr_t err= open_deletion_table(table_type); + if (err != DB_SUCCESS) return err; + + uint8_t cached_index= to_index(table_type); + err= lock_common_tables(cached_index, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[cached_index]; + dict_index_t* index= dict_table_get_first_index(table); + + dfield_t fields[1]; + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: doc_id */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id(&write_doc_id, doc_id); + dfield_set_data(field, &write_doc_id, sizeof(doc_id_t)); + + return m_executor.delete_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::delete_all_common_records( + const char *table_name) noexcept +{ + ut_ad(!dict_sys.locked()); + FTSDeletionTable table_type= get_deletion_table_type(table_name); + if (table_type == FTSDeletionTable::MAX_DELETION_TABLES) return DB_ERROR; + + dberr_t err= open_deletion_table(table_type); + if (err != DB_SUCCESS) return err; + + uint8_t cached_index= to_index(table_type); + err= lock_common_tables(cached_index, LOCK_X); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[cached_index]; + return m_executor.delete_all(table); +} + +dberr_t FTSQueryExecutor::delete_config_record(const char *key) noexcept +{ + ut_ad(!dict_sys.locked()); + dberr_t err= open_config_table(); + if (err != DB_SUCCESS) return err; + + err= m_executor.lock_table(m_config_table, LOCK_IX); + if (err == DB_LOCK_WAIT) err= m_executor.handle_wait(err, true); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_config_table; + dict_index_t* index= dict_table_get_first_index(table); + + dfield_t fields[1]; + + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: key (CHAR(50)) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, key, strlen(key)); + + return m_executor.delete_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::read_config_with_lock(const char *key, + RecordCallback &callback) noexcept +{ + ut_ad(!dict_sys.locked()); + dberr_t err= open_config_table(); + if (err != DB_SUCCESS) return err; + + err= m_executor.lock_table(m_config_table, LOCK_IX); + if (err == DB_LOCK_WAIT) err= m_executor.handle_wait(err, true); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_config_table; + dict_index_t* index= dict_table_get_first_index(table); + + dfield_t fields[1]; + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: key (CHAR(50)) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, key, strlen(key)); + + err= m_executor.select_for_update(table, &tuple, &callback); + return (err == DB_SUCCESS_LOCKED_REC) ? DB_SUCCESS : err; +} + +dtuple_t* FTSQueryExecutor::create_word_search_tuple( + dict_table_t *table, const fts_string_t *word, + mem_heap_t *heap) noexcept +{ + if (!word || word->f_len == 0) + return nullptr; + + dtuple_t* tuple= dtuple_create(heap, 1); + dict_table_copy_types(tuple, table); + dfield_t* field= dtuple_get_nth_field(tuple, 0); + dfield_set_data(field, word->f_str, word->f_len); + dtuple_set_n_fields_cmp(tuple, 1); + + return tuple; +} + +dberr_t FTSQueryExecutor::read_all_common(const char *tbl_name, + RecordCallback &callback) noexcept +{ + ut_ad(!dict_sys.locked()); + FTSDeletionTable table_type= get_deletion_table_type(tbl_name); + if (table_type == FTSDeletionTable::MAX_DELETION_TABLES) return DB_ERROR; + + dberr_t err= open_deletion_table(table_type); + if (err != DB_SUCCESS) return err; + + uint8_t index_no= to_index(table_type); + err= lock_common_tables(index_no, LOCK_IS); + if (err != DB_SUCCESS) return err; + + dict_table_t *table= m_common_tables[index_no]; + dict_index_t *index= dict_table_get_first_index(table); + err= m_executor.read_by_index(table, index, nullptr, PAGE_CUR_G, + callback, true); + return (err == DB_SUCCESS_LOCKED_REC) ? DB_SUCCESS : err; +} + +CommonTableReader::CommonTableReader() : RecordCallback( + [this](const rec_t *rec, const dict_index_t *index, + const rec_offs *offsets) -> dberr_t + { + if (!dict_table_is_comp(index->table)) + { + ulint doc_id_len; + ulint offset= rec_get_nth_field_offs_old(rec, 0, &doc_id_len); + if (offset != 0 || doc_id_len == UNIV_SQL_NULL || doc_id_len != 8) + return DB_CORRUPTION; + } + + doc_ids.push_back(mach_read_from_8(rec)); + return DB_SUCCESS; + }, + [](const dtuple_t *search_tuple, const rec_t *rec, + const dict_index_t *index, const rec_offs *offsets) -> RecordCompareAction + { return RecordCompareAction::PROCESS; }) {} + + +ConfigReader::ConfigReader() : RecordCallback( + [this](const rec_t *rec, const dict_index_t *index, + const rec_offs *offsets) -> dberr_t + { + ulint value_len; + const byte *value_data= rec_get_nth_field(rec, offsets, 3, &value_len); + + if (value_data && value_len != UNIV_SQL_NULL && value_len > 0) + value_span= std::string_view(reinterpret_cast(value_data), + value_len); + + return DB_SUCCESS; + }, + [](const dtuple_t *search_tuple, const rec_t *rec, + const dict_index_t *index, const rec_offs *offsets) -> RecordCompareAction + { + return compare_config_key(search_tuple, rec, index, offsets); + }) {} + +/** Initial size of nodes in fts_word_t. */ +static const ulint FTS_WORD_NODES_INIT_SIZE= 64; + +/** Initialize fts_word_t structure */ +static void init_fts_word(fts_word_t *word, const byte *utf8, ulint len) +{ + mem_heap_t* heap= mem_heap_create(sizeof(fts_node_t)); + memset(word, 0, sizeof(*word)); + word->text.f_len= len; + word->text.f_str= static_cast(mem_heap_alloc(heap, len + 1)); + memcpy(word->text.f_str, utf8, len); + word->text.f_str[len]= 0; + word->heap_alloc= ib_heap_allocator_create(heap); + word->nodes= ib_vector_create(word->heap_alloc, sizeof(fts_node_t), + FTS_WORD_NODES_INIT_SIZE); +} + +/** AuxRecordReader default word processor implementation */ +dberr_t AuxRecordReader::default_word_processor( + const rec_t *rec, const dict_index_t *index, + const rec_offs *offsets, void *user_arg) +{ + ib_vector_t *words= static_cast(user_arg); + + /* Extract fields using rec_get_nth_field() */ + ulint word_len; + const byte* word_data= rec_get_nth_field(rec, offsets, 0, &word_len); + if (!word_data || word_len == UNIV_SQL_NULL || word_len > FTS_MAX_WORD_LEN) + return DB_SUCCESS; + + ulint first_doc_id_len; + const byte* first_doc_id_data= rec_get_nth_field(rec, offsets, 1, &first_doc_id_len); + doc_id_t first_doc_id= fts_read_doc_id(first_doc_id_data); + + ulint last_doc_id_len; + const byte* last_doc_id_data= rec_get_nth_field(rec, offsets, 4, &last_doc_id_len); + doc_id_t last_doc_id= fts_read_doc_id(last_doc_id_data); + + ulint doc_count_len; + const byte* doc_count_data= rec_get_nth_field(rec, offsets, 5, &doc_count_len); + ulint doc_count= mach_read_from_4(doc_count_data); + + ulint ilist_len; + const byte* ilist_data= rec_get_nth_field(rec, offsets, 6, &ilist_len); + + if (word_data && word_len > 0 && word_len != UNIV_SQL_NULL) + { + if (!m_word_heap) + m_word_heap= mem_heap_create(256); + + if (m_last_word.f_str) + mem_heap_empty(m_word_heap); + + m_last_word.f_str= static_cast( + mem_heap_dup(m_word_heap, word_data, word_len)); + m_last_word.f_len= word_len; + m_last_word.f_n_char= 0; + } + + fts_word_t *word; + bool is_word_init= false; + + ut_ad(word_len <= FTS_MAX_WORD_LEN); + + if (ib_vector_size(words) == 0) + { + /* First word - push and initialize */ + word= static_cast(ib_vector_push(words, nullptr)); + init_fts_word(word, word_data, word_len); + is_word_init= true; + } + else + { + /* Check if this word is different from the last word */ + word= static_cast(ib_vector_last(words)); + if (word_len != word->text.f_len || + memcmp(word->text.f_str, word_data, word_len)) + { + /* Different word - push new word and initialize */ + word= static_cast(ib_vector_push(words, nullptr)); + init_fts_word(word, word_data, word_len); + is_word_init= true; + } + } + fts_node_t *node= static_cast( + ib_vector_push(word->nodes, nullptr)); + + /* Use extracted field values */ + node->first_doc_id= first_doc_id; + node->last_doc_id= last_doc_id; + node->doc_count= doc_count; + + node->ilist_size_alloc= node->ilist_size= 0; + node->ilist= nullptr; + + if (ilist_data && ilist_len != UNIV_SQL_NULL && ilist_len > 0) + { + node->ilist_size_alloc= node->ilist_size= ilist_len; + if (ilist_len) + { + node->ilist= static_cast(ut_malloc_nokey(ilist_len)); + memcpy(node->ilist, ilist_data, ilist_len); + } + if (ilist_len == 0) return DB_SUCCESS_LOCKED_REC; + } + + if (this->total_memory) + { + if (is_word_init) + { + *this->total_memory+= + sizeof(fts_word_t) + sizeof(ib_alloc_t) + + sizeof(ib_vector_t) + word_len + + sizeof(fts_node_t) * FTS_WORD_NODES_INIT_SIZE; + } + *this->total_memory+= node->ilist_size; + if (*this->total_memory >= fts_result_cache_limit) + return DB_FTS_EXCEED_RESULT_CACHE_LIMIT; + } + return DB_SUCCESS; +} + +/** AuxRecordReader comparison logic implementation */ +RecordCompareAction AuxRecordReader::compare_record( + const dtuple_t *search_tuple, const rec_t *rec, + const dict_index_t *index, const rec_offs *offsets) noexcept +{ + if (!search_tuple) return RecordCompareAction::PROCESS; + int cmp_result; + switch (compare_mode) + { + case AuxCompareMode::GREATER_EQUAL: + case AuxCompareMode::GREATER: + { + int match= 0; + cmp_result= cmp_dtuple_rec_bytes(rec, *index, *search_tuple, &match, + index->table->not_redundant()); + if (compare_mode == AuxCompareMode::GREATER_EQUAL) + return (cmp_result <= 0) ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; + else + return (cmp_result < 0) ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; + } + case AuxCompareMode::LIKE: + case AuxCompareMode::EQUAL: + { + const dfield_t* search_field= dtuple_get_nth_field(search_tuple, 0); + const void* search_data= dfield_get_data(search_field); + ulint search_len= dfield_get_len(search_field); + + if (!search_data || search_len == UNIV_SQL_NULL) + return RecordCompareAction::PROCESS; + + ulint word_len; + const byte* word_data= rec_get_nth_field(rec, offsets, 0, &word_len); + if (!word_data || word_len == UNIV_SQL_NULL) + return RecordCompareAction::SKIP; + + const dtype_t* type= dfield_get_type(search_field); + cmp_result= cmp_data(type->mtype, type->prtype, false, + static_cast(search_data), + search_len, word_data, word_len); + if (compare_mode == AuxCompareMode::EQUAL) + return cmp_result == 0 + ? RecordCompareAction::PROCESS + : RecordCompareAction::STOP; + else /* AuxCompareMode::LIKE */ + { + int prefix_cmp= cmp_data(type->mtype, type->prtype, false, + static_cast(search_data), + search_len, word_data, + search_len <= word_len ? search_len : word_len); + + if (prefix_cmp != 0) return RecordCompareAction::STOP; + return (search_len <= word_len) ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; + } + } + } + return RecordCompareAction::PROCESS; +} + +/** Direct config key comparison implementation */ +RecordCompareAction ConfigReader::compare_config_key( + const dtuple_t *search_tuple, const rec_t *rec, + const dict_index_t *index, const rec_offs *offsets) +{ + if (!search_tuple) return RecordCompareAction::PROCESS; + const dfield_t *search_field= dtuple_get_nth_field(search_tuple, 0); + const void *search_data= dfield_get_data(search_field); + ulint search_len= dfield_get_len(search_field); + if (!search_data || search_len == UNIV_SQL_NULL) + return RecordCompareAction::PROCESS; + + ulint rec_key_len; + const byte *rec_key_data= rec_get_nth_field(rec, offsets, 0, &rec_key_len); + + if (!rec_key_data || rec_key_len == UNIV_SQL_NULL) + return RecordCompareAction::SKIP; + + const dtype_t *type= dfield_get_type(search_field); + int cmp_result= cmp_data(type->mtype, type->prtype, false, + static_cast(search_data), + search_len, rec_key_data, rec_key_len); + + return (cmp_result == 0) ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; +} + +void FTSQueryExecutor::construct_table_name( + char *table_name, const char *suffix, bool common_table) noexcept +{ + ut_ad(m_table); + ut_ad(common_table || m_index); + const size_t dbname_len= m_table->name.dblen() + 1; + ut_ad(dbname_len > 1); + memcpy(table_name, m_table->name.m_name, dbname_len); + memcpy(table_name+= dbname_len, "FTS_", 4); + table_name+= 4; + + int len= fts_write_object_id(m_table->id, table_name); + if (!common_table) + { + table_name[len]= '_'; + ++len; + len+= fts_write_object_id(m_index->id, table_name + len); + } + ut_a(len >= 16); + ut_a(len < FTS_AUX_MIN_TABLE_ID_LENGTH); + table_name+= len; + *table_name++= '_'; + strcpy(table_name, suffix); +} + +dberr_t FTSQueryExecutor::read_aux(uint8_t aux_index, + const fts_string_t *start_word, + RecordCallback &callback) noexcept +{ + ut_ad(!dict_sys.locked()); + + if (aux_index >= FTS_NUM_AUX_INDEX) + return DB_ERROR; + + dberr_t error= open_aux_table(aux_index); + if (error != DB_SUCCESS) + return error; + + error= lock_aux_tables(aux_index, LOCK_IS); + if (error != DB_SUCCESS) + return error; + + dict_table_t *table= m_aux_tables[aux_index]; + dict_index_t *index= dict_table_get_first_index(table); + + mem_heap_t* heap= nullptr; + dtuple_t* search_tuple= nullptr; + page_cur_mode_t mode= PAGE_CUR_G; + + if (start_word && start_word->f_len > 0) + { + if (heap == nullptr) + heap= mem_heap_create(256); + search_tuple= create_word_search_tuple(table, start_word, heap); + mode= PAGE_CUR_GE; + } + error= m_executor.read_by_index(table, index, search_tuple, + mode, callback, true); + if (heap) + mem_heap_free(heap); + return error; +} diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index afc380e367cf9..954c0d161fe93 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -30,6 +30,7 @@ Full Text Search interface #include "dict0stats_bg.h" #include "row0sel.h" #include "fts0fts.h" +#include "fts0exec.h" #include "fts0priv.h" #include "fts0types.h" #include "fts0types.inl" @@ -44,6 +45,37 @@ static const ulint FTS_MAX_ID_LEN = 32; /** Column name from the FTS config table */ #define FTS_MAX_CACHE_SIZE_IN_MB "cache_size_in_mb" +/** Compare function to check if record's doc_id > search tuple's doc_id +@param[in] search_tuple Search tuple containing target doc_id +@param[in] rec Record to check +@param[in] index Index containing the record +@return true if record's doc_id > search tuple's doc_id */ +static +RecordCompareAction doc_id_comparator( + const dtuple_t* search_tuple, + const rec_t* rec, + const dict_index_t* index, + const rec_offs *offsets) +{ + /* Get target doc_id from search tuple */ + const dfield_t* search_field= dtuple_get_nth_field(search_tuple, 0); + const byte* search_data= static_cast(dfield_get_data(search_field)); + doc_id_t target_doc_id= fts_read_doc_id(search_data); + ulint doc_col_pos= dict_col_get_clust_pos( + &index->table->cols[index->table->fts->doc_col], index); + + ulint len; + const byte* doc_id_data= rec_get_nth_field(rec, offsets, doc_col_pos, &len); + + if (len != sizeof(doc_id_t)) + return RecordCompareAction::SKIP; + + doc_id_t rec_doc_id= fts_read_doc_id(doc_id_data); + return (rec_doc_id > target_doc_id) + ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; +} + /** Verify if a aux table name is a obsolete table by looking up the key word in the obsolete table names */ #define FTS_IS_OBSOLETE_AUX_TABLE(table_name) \ @@ -148,27 +180,6 @@ const fts_index_selector_t fts_index_selector[] = { { 0 , NULL } }; -/** Default config values for FTS indexes on a table. */ -static const char* fts_config_table_insert_values_sql = - "PROCEDURE P() IS\n" - "BEGIN\n" - "\n" - "INSERT INTO $config_table VALUES('" - FTS_MAX_CACHE_SIZE_IN_MB "', '256');\n" - "" - "INSERT INTO $config_table VALUES('" - FTS_OPTIMIZE_LIMIT_IN_SECS "', '180');\n" - "" - "INSERT INTO $config_table VALUES ('" - FTS_SYNCED_DOC_ID "', '0');\n" - "" - "INSERT INTO $config_table VALUES ('" - FTS_TOTAL_DELETED_COUNT "', '0');\n" - "" /* Note: 0 == FTS_TABLE_STATE_RUNNING */ - "INSERT INTO $config_table VALUES ('" - FTS_TABLE_STATE "', '0');\n" - "END;\n"; - /** FTS tokenize parameter for plugin parser */ struct fts_tokenize_param_t { fts_doc_t* result_doc; /*!< Result doc for tokens */ @@ -208,30 +219,6 @@ fts_add_doc_by_id( fts_trx_table_t*ftt, /*!< in: FTS trx table */ doc_id_t doc_id); /*!< in: doc id */ -/** Tokenize a document. -@param[in,out] doc document to tokenize -@param[out] result tokenization result -@param[in] parser pluggable parser */ -static -void -fts_tokenize_document( - fts_doc_t* doc, - fts_doc_t* result, - st_mysql_ftparser* parser); - -/** Continue to tokenize a document. -@param[in,out] doc document to tokenize -@param[in] add_pos add this position to all tokens from this tokenization -@param[out] result tokenization result -@param[in] parser pluggable parser */ -static -void -fts_tokenize_document_next( - fts_doc_t* doc, - ulint add_pos, - fts_doc_t* result, - st_mysql_ftparser* parser); - /** Create the vector of fts_get_doc_t instances. @param[in,out] cache fts cache @return vector of fts_get_doc_t instances */ @@ -266,7 +253,6 @@ fts_cache_destroy(fts_cache_t* cache) /** Get a character set based on precise type. @param prtype precise type @return the corresponding character set */ -UNIV_INLINE CHARSET_INFO* fts_get_charset(ulint prtype) { @@ -298,7 +284,6 @@ fts_get_charset(ulint prtype) /****************************************************************//** This function loads the default InnoDB stopword list */ -static void fts_load_default_stopword( /*======================*/ @@ -343,187 +328,162 @@ fts_load_default_stopword( stopword_info->status = STOPWORD_FROM_DEFAULT; } -/****************************************************************//** -Callback function to read a single stopword value. -@return Always return TRUE */ -static -ibool -fts_read_stopword( -/*==============*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ib_vector_t */ -{ - ib_alloc_t* allocator; - fts_stopword_t* stopword_info; - sel_node_t* sel_node; - que_node_t* exp; - ib_rbt_t* stop_words; - dfield_t* dfield; - fts_string_t str; - mem_heap_t* heap; - ib_rbt_bound_t parent; - dict_table_t* table; - - sel_node = static_cast(row); - table = sel_node->table_list->table; - stopword_info = static_cast(user_arg); - - stop_words = stopword_info->cached_stopword; - allocator = static_cast(stopword_info->heap); - heap = static_cast(allocator->arg); - - exp = sel_node->select_list; - - /* We only need to read the first column */ - dfield = que_node_get_val(exp); - - str.f_n_char = 0; - str.f_str = static_cast(dfield_get_data(dfield)); - str.f_len = dfield_get_len(dfield); - exp = que_node_get_next(exp); - ut_ad(exp); - - if (table->versioned()) { - dfield = que_node_get_val(exp); - ut_ad(dfield_get_type(dfield)->vers_sys_end()); - void* data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - if (table->versioned_by_id()) { - ut_ad(len == sizeof trx_id_max_bytes); - if (0 != memcmp(data, trx_id_max_bytes, len)) { - return true; - } - } else { - ut_ad(len == sizeof timestamp_max_bytes); - if (!IS_MAX_TIMESTAMP(data)) { - return true; - } - } - } - ut_ad(!que_node_get_next(exp)); - - /* Only create new node if it is a value not already existed */ - if (str.f_len != UNIV_SQL_NULL - && rbt_search(stop_words, &parent, &str) != 0) { - - fts_tokenizer_word_t new_word; - - new_word.nodes = ib_vector_create( - allocator, sizeof(fts_node_t), 4); - - new_word.text.f_str = static_cast( - mem_heap_alloc(heap, str.f_len + 1)); - - memcpy(new_word.text.f_str, str.f_str, str.f_len); - - new_word.text.f_n_char = 0; - new_word.text.f_len = str.f_len; - new_word.text.f_str[str.f_len] = 0; - - rbt_insert(stop_words, &new_word, &new_word); - } - - return(TRUE); -} - -/******************************************************************//** -Load user defined stopword from designated user table +/** Load user defined stopword from designated user table +@param fts fulltext structure +@param stopword_table stopword table +@param stopword_info stopword information @return whether the operation is successful */ -static -bool -fts_load_user_stopword( -/*===================*/ - fts_t* fts, /*!< in: FTS struct */ - const char* stopword_table_name, /*!< in: Stopword table - name */ - fts_stopword_t* stopword_info) /*!< in: Stopword info */ +bool fts_load_user_stopword(FTSQueryExecutor *executor, fts_t *fts, + const char *stopword_table, + fts_stopword_t *stopword_info) noexcept { - if (!fts->dict_locked) { - dict_sys.lock(SRW_LOCK_CALL); - } - - /* Validate the user table existence in the right format */ - bool ret= false; - const char* row_end; - stopword_info->charset = fts_valid_stopword_table(stopword_table_name, - &row_end); - if (!stopword_info->charset) { + trx_t* trx= executor->trx(); + if (!fts->dict_locked) dict_sys.lock(SRW_LOCK_CALL); + /* Validate the user table existence in the right format */ + bool ret= false; + const char *row_end; + stopword_info->charset= fts_valid_stopword_table( + stopword_table, &row_end); + if (!stopword_info->charset) + { cleanup: - if (!fts->dict_locked) { - dict_sys.unlock(); - } + if (!fts->dict_locked) dict_sys.unlock(); + return ret; + } - return ret; - } + if (!stopword_info->cached_stopword) + { + /* Create the stopword RB tree with the stopword column + charset. All comparison will use this charset */ + stopword_info->cached_stopword= rbt_create_arg_cmp( + sizeof(fts_tokenizer_word_t), innobase_fts_text_cmp, + (void*)stopword_info->charset); + } - trx_t* trx = trx_create(); - trx->op_info = "Load user stopword table into FTS cache"; + /* Load the stopword table */ + dict_table_t* table= + dict_sys.load_table({stopword_table, strlen(stopword_table)}); + if (!table) + goto cleanup; + + /* Use the passed executor's transaction */ + trx->op_info= "Load user stopword table into FTS cache"; + ib_rbt_t* stop_words= stopword_info->cached_stopword; + ib_alloc_t* allocator= static_cast(stopword_info->heap); + mem_heap_t* heap= static_cast(allocator->arg); + + /* Find the field number for 'value' column */ + dict_index_t* clust_index= dict_table_get_first_index(table); + ulint value_field_no= ULINT_UNDEFINED; + for (ulint i= 0; i < dict_index_get_n_fields(clust_index); i++) + { + const dict_field_t* field= dict_index_get_nth_field(clust_index, i); + if (strcmp(field->name, "value") == 0) + { + value_field_no= i; + break; + } + } + if (value_field_no == ULINT_UNDEFINED) + { + sql_print_error("InnoDB: Could not find 'value' column in " + "stopword table %s", stopword_table); + goto cleanup; + } - if (!stopword_info->cached_stopword) { - /* Create the stopword RB tree with the stopword column - charset. All comparison will use this charset */ - stopword_info->cached_stopword = rbt_create_arg_cmp( - sizeof(fts_tokenizer_word_t), innobase_fts_text_cmp, - (void*)stopword_info->charset); + auto process_stopword= [&](const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets) -> dberr_t + { + mem_heap_t *offsets_heap= nullptr; + if (offsets == nullptr) + offsets= rec_get_offsets(rec, index, nullptr, index->n_core_fields, + ULINT_UNDEFINED, &offsets_heap); - } + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, value_field_no, + &field_len); - pars_info_t* info = pars_info_create(); - - pars_info_bind_id(info, "table_stopword", stopword_table_name); - pars_info_bind_id(info, "row_end", row_end); - - pars_info_bind_function(info, "my_func", fts_read_stopword, - stopword_info); - - que_t* graph = pars_sql( - info, - "PROCEDURE P() IS\n" - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT value, $row_end" - " FROM $table_stopword;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;" - "END;\n"); - - for (;;) { - dberr_t error = fts_eval_sql(trx, graph); - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - stopword_info->status = STOPWORD_USER_TABLE; - break; - } else { - fts_sql_rollback(trx); + if (field_len == UNIV_SQL_NULL) return DB_SUCCESS; - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "Lock wait timeout reading user" - " stopword table. Retrying!"; + ib_rbt_bound_t parent; + fts_string_t str; + str.f_n_char= 0; + str.f_str= const_cast(field_data); + str.f_len= field_len; - trx->error_state = DB_SUCCESS; - } else { - ib::error() << "Error '" << error - << "' while reading user stopword" - " table."; - ret = FALSE; - break; - } - } - } + /* Handle system versioning - check row_end column if versioned */ + if (table->versioned()) + { + ulint end_len; + const byte* end_data= rec_get_nth_field( + rec, offsets, table->vers_end + index->n_uniq + 2, &end_len); - que_graph_free(graph); - trx->free(); - ret = true; - goto cleanup; + if (table->versioned_by_id()) + { + ut_ad(end_len == sizeof trx_id_max_bytes); + if (0 != memcmp(end_data, trx_id_max_bytes, end_len)) + goto func_exit; + } + else + { + ut_ad(end_len == sizeof timestamp_max_bytes); + if (!IS_MAX_TIMESTAMP(end_data)) + goto func_exit; + } + } + + if (str.f_len != UNIV_SQL_NULL && + rbt_search(stop_words, &parent, &str) != 0) + { + fts_tokenizer_word_t new_word; + new_word.nodes= ib_vector_create(allocator, sizeof(fts_node_t), 4); + new_word.text.f_str= static_cast( + mem_heap_alloc(heap, str.f_len + 1)); + memcpy(new_word.text.f_str, str.f_str, str.f_len); + new_word.text.f_n_char= 0; + new_word.text.f_len= str.f_len; + new_word.text.f_str[str.f_len]= 0; + rbt_insert(stop_words, &new_word, &new_word); + } +func_exit: + if (offsets_heap) mem_heap_free(offsets_heap); + return DB_SUCCESS; /* Continue processing */ + }; + + RecordCallback callback(process_stopword, + [](const dtuple_t*, const rec_t*, + const dict_index_t*, const rec_offs*) { + return RecordCompareAction::PROCESS; + }); + /* Read all records from the stopword table */ + for (;;) + { + dberr_t error= executor->read_by_index( + table, clust_index, nullptr, PAGE_CUR_G, callback); + if (UNIV_LIKELY(error == DB_SUCCESS)) + { + stopword_info->status= STOPWORD_USER_TABLE; + ret= true; + break; + } + else + { + if (error == DB_LOCK_WAIT_TIMEOUT) + { + sql_print_warning("InnoDB: Lock wait timeout reading " + "user stopword table. Retrying!"); + trx->error_state= DB_SUCCESS; + } + else + { + sql_print_error("InnoDB: Error '%s' while reading user " + "stopword table.", ut_strerr(error)); + ret= false; + break; + } + } + } + goto cleanup; } /******************************************************************//** @@ -587,6 +547,46 @@ fts_cache_init( } } +/** Construct the name of an internal FTS table for the given table. +@param[in] fts_table metadata on fulltext-indexed table +@param[out] table_name a name up to MAX_FULL_NAME_LEN +@param[in] dict_locked whether dict_sys.latch is being held */ +void fts_get_table_name(const fts_table_t* fts_table, char* table_name, + bool dict_locked) +{ + if (!dict_locked) dict_sys.freeze(SRW_LOCK_CALL); + ut_ad(dict_sys.frozen()); + /* Include the separator as well. */ + const size_t dbname_len= fts_table->table->name.dblen() + 1; + ut_ad(dbname_len > 1); + memcpy(table_name, fts_table->table->name.m_name, dbname_len); + if (!dict_locked) dict_sys.unfreeze(); + + memcpy(table_name += dbname_len, "FTS_", 4); + table_name += 4; + int len; + switch (fts_table->type) + { + case FTS_COMMON_TABLE: + len= fts_write_object_id(fts_table->table_id, table_name); + break; + + case FTS_INDEX_TABLE: + len= fts_write_object_id(fts_table->table_id, table_name); + table_name[len]= '_'; + ++len; + len+= fts_write_object_id(fts_table->index_id, table_name + len); + break; + + default: ut_error; + } + ut_a(len >= 16); + ut_a(len < FTS_AUX_MIN_TABLE_ID_LENGTH); + table_name+= len; + *table_name++= '_'; + strcpy(table_name, fts_table->suffix); +} + /****************************************************************//** Create a FTS cache. */ fts_cache_t* @@ -1815,88 +1815,78 @@ CREATE TABLE $FTS_PREFIX_CONFIG @param[in] skip_doc_id_index Skip index on doc id @return DB_SUCCESS if succeed */ dberr_t -fts_create_common_tables( - trx_t* trx, - dict_table_t* table, - bool skip_doc_id_index) +fts_create_common_tables(trx_t *trx, dict_table_t *table, + bool skip_doc_id_index) { - dberr_t error; - que_t* graph; - fts_table_t fts_table; - mem_heap_t* heap = mem_heap_create(1024); - pars_info_t* info; - char fts_name[MAX_FULL_NAME_LEN]; - char full_name[sizeof(fts_common_tables) / sizeof(char*)] - [MAX_FULL_NAME_LEN]; - - dict_index_t* index = NULL; - - FTS_INIT_FTS_TABLE(&fts_table, NULL, FTS_COMMON_TABLE, table); - - error = fts_drop_common_tables(trx, &fts_table, true); - - if (error != DB_SUCCESS) { - - goto func_exit; - } - - /* Create the FTS tables that are common to an FTS index. */ - for (ulint i = 0; fts_common_tables[i] != NULL; ++i) { - - fts_table.suffix = fts_common_tables[i]; - fts_get_table_name(&fts_table, full_name[i], true); - dict_table_t* common_table = fts_create_one_common_table( - trx, table, full_name[i], fts_table.suffix, heap); - - if (!common_table) { - trx->error_state = DB_SUCCESS; - error = DB_ERROR; - goto func_exit; - } - - mem_heap_empty(heap); - } + dberr_t error= DB_SUCCESS; + char full_name[sizeof(fts_common_tables) / sizeof(char*)][MAX_FULL_NAME_LEN]; + dict_index_t *index= nullptr; + fts_table_t fts_table; + FTS_INIT_FTS_TABLE(&fts_table, NULL, FTS_COMMON_TABLE, table); + error = fts_drop_common_tables(trx, &fts_table, true); + if (error != DB_SUCCESS) return error; + mem_heap_t *heap= mem_heap_create(1024); - /* Write the default settings to the config table. */ - info = pars_info_create(); + FTSQueryExecutor executor(trx, table); + /* Create the FTS tables that are common to an FTS index. */ + for (ulint i= 0; fts_common_tables[i]; ++i) + { + fts_table.suffix= fts_common_tables[i]; + fts_get_table_name(&fts_table, full_name[i], true); + dict_table_t *common_table= fts_create_one_common_table( + trx, table, full_name[i], fts_table.suffix, heap); + if (!common_table) + { + trx->error_state= DB_SUCCESS; + error= DB_ERROR; + mem_heap_free(heap); + return error; + } - fts_table.suffix = "CONFIG"; - fts_get_table_name(&fts_table, fts_name, true); - pars_info_bind_id(info, "config_table", fts_name); + if (i == 2) + executor.set_config_table(common_table); - graph = pars_sql( - info, fts_config_table_insert_values_sql); + mem_heap_empty(heap); + } - error = fts_eval_sql(trx, graph); + /** Does the following insert operation: + INSERT INTO $config_table VALUES('"FTS_MAX_CACHE_SIZE_IN_MB"', '256');" + INSERT INTO $config_table VALUES('"FTS_OPTIMIZE_LIMIT_IN_SECS"', '180');" + INSERT INTO $config_table VALUES ('"FTS_SYNCED_DOC_ID "', '0');" + INSERT INTO $config_table VALUES ('"FTS_TOTAL_DELETED_COUNT "', '0');" + INSERT INTO $config_table VALUES ('"FTS_TABLE_STATE "', '0');" */ + error= executor.insert_config_record(FTS_MAX_CACHE_SIZE_IN_MB, "256"); + if (error == DB_SUCCESS) + error= executor.insert_config_record(FTS_OPTIMIZE_LIMIT_IN_SECS, "180"); - que_graph_free(graph); + if (error == DB_SUCCESS) + error= executor.insert_config_record(FTS_SYNCED_DOC_ID, "0"); - if (error != DB_SUCCESS || skip_doc_id_index) { + if (error == DB_SUCCESS) + error= executor.insert_config_record(FTS_TOTAL_DELETED_COUNT, "0"); - goto func_exit; - } + if (error == DB_SUCCESS) + error= executor.insert_config_record(FTS_TABLE_STATE, "0"); - if (table->versioned()) { - index = dict_mem_index_create(table, - FTS_DOC_ID_INDEX.str, - DICT_UNIQUE, 2); - dict_mem_index_add_field(index, FTS_DOC_ID.str, 0); - dict_mem_index_add_field(index, table->cols[table->vers_end].name(*table).str, 0); - } else { - index = dict_mem_index_create(table, - FTS_DOC_ID_INDEX.str, - DICT_UNIQUE, 1); - dict_mem_index_add_field(index, FTS_DOC_ID.str, 0); - } + if (error != DB_SUCCESS || skip_doc_id_index) goto func_exit; - error = row_create_index_for_mysql(index, trx, NULL, - FIL_ENCRYPTION_DEFAULT, - FIL_DEFAULT_ENCRYPTION_KEY); + if (table->versioned()) + { + index= dict_mem_index_create(table, FTS_DOC_ID_INDEX.str, DICT_UNIQUE, 2); + dict_mem_index_add_field(index, FTS_DOC_ID.str, 0); + dict_mem_index_add_field(index, table->cols[table->vers_end].name(*table).str, 0); + } + else + { + index= dict_mem_index_create(table, FTS_DOC_ID_INDEX.str, DICT_UNIQUE, 1); + dict_mem_index_add_field(index, FTS_DOC_ID.str, 0); + } + error= row_create_index_for_mysql(index, trx, NULL, FIL_ENCRYPTION_DEFAULT, + FIL_DEFAULT_ENCRYPTION_KEY); func_exit: - mem_heap_free(heap); - - return(error); + mem_heap_free(heap); + return error; } /** Create one FTS auxiliary index table for an FTS index. @@ -2361,39 +2351,6 @@ fts_trx_add_op( fts_trx_table_add_op(stmt_ftt, doc_id, state, fts_indexes); } -/******************************************************************//** -Fetch callback that converts a textual document id to a binary value and -stores it in the given place. -@return always returns NULL */ -static -ibool -fts_fetch_store_doc_id( -/*===================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: doc_id_t* to store - doc_id in */ -{ - int n_parsed; - sel_node_t* node = static_cast(row); - doc_id_t* doc_id = static_cast(user_arg); - dfield_t* dfield = que_node_get_val(node->select_list); - dtype_t* type = dfield_get_type(dfield); - ulint len = dfield_get_len(dfield); - - char buf[32]; - - ut_a(dtype_get_mtype(type) == DATA_VARCHAR); - ut_a(len > 0 && len < sizeof(buf)); - - memcpy(buf, dfield_get_data(dfield), len); - buf[len] = '\0'; - - n_parsed = sscanf(buf, FTS_DOC_ID_FORMAT, doc_id); - ut_a(n_parsed == 1); - - return(FALSE); -} - /*********************************************************************//** Get the next available document id. @return DB_SUCCESS if OK */ @@ -2426,70 +2383,48 @@ fts_get_next_doc_id( } /** Read the synced document id from the fts configuration table -@param table fts table -@param doc_id document id to be read -@param trx transaction to read from config table +@param executor query executor +@param table fts table +@param doc_id document id to be read @return DB_SUCCESS in case of success */ static -dberr_t fts_read_synced_doc_id(const dict_table_t *table, - doc_id_t *doc_id, - trx_t *trx) +dberr_t fts_read_synced_doc_id(FTSQueryExecutor *executor, + const dict_table_t *table, + doc_id_t *doc_id) noexcept { - dberr_t error; - char table_name[MAX_FULL_NAME_LEN]; - - fts_table_t fts_table; - fts_table.suffix= "CONFIG"; - fts_table.table_id= table->id; - fts_table.type= FTS_COMMON_TABLE; - fts_table.table= table; ut_a(table->fts->doc_col != ULINT_UNDEFINED); - - trx->op_info = "update the next FTS document id"; - pars_info_t *info= pars_info_create(); - pars_info_bind_function(info, "my_func", fts_fetch_store_doc_id, - doc_id); - - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "config_table", table_name); - - que_t *graph= fts_parse_sql( - &fts_table, info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS SELECT value FROM $config_table" - " WHERE key = 'synced_doc_id' FOR UPDATE;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - + executor->trx()->op_info= "reading synced FTS document id"; + ConfigReader reader; *doc_id= 0; - error = fts_eval_sql(trx, graph); - que_graph_free(graph); + dberr_t error= executor->read_config_with_lock("synced_doc_id", reader); + if (error == DB_SUCCESS) + { + char value_buf[32]; + size_t copy_len= std::min(reader.value_span.size(), sizeof(value_buf) - 1); + memcpy(value_buf, reader.value_span.data(), copy_len); + value_buf[copy_len]= '\0'; + int n_parsed= sscanf(value_buf, FTS_DOC_ID_FORMAT, doc_id); + if (n_parsed != 1) error= DB_ERROR; + executor->release_lock(); + } return error; } /** This function fetch the Doc ID from CONFIG table, and compare with the Doc ID supplied. And store the larger one to the CONFIG table. -@param table fts table -@param cmp_doc_id Doc ID to compare -@param doc_id larger document id after comparing "cmp_doc_id" to - the one stored in CONFIG table -@param trx transaction +@param executor query executor +@param table fts table +@param cmp_doc_id Doc ID to compare +@param doc_id larger document id after comparing "cmp_doc_id" to + the one stored in CONFIG table @return DB_SUCCESS if OK */ static dberr_t fts_cmp_set_sync_doc_id( + FTSQueryExecutor *executor, const dict_table_t *table, doc_id_t cmp_doc_id, - doc_id_t *doc_id, - trx_t *trx=nullptr) + doc_id_t *doc_id) noexcept { ut_ad(!srv_read_only_mode || recv_sys.rpo); @@ -2499,14 +2434,8 @@ fts_cmp_set_sync_doc_id( fts_cache_t* cache= table->fts->cache; dberr_t error = DB_SUCCESS; - const trx_t* const caller_trx = trx; - - if (trx == nullptr) { - trx = trx_create(); - trx_start_internal_read_only(trx); - } retry: - error = fts_read_synced_doc_id(table, doc_id, trx); + error = fts_read_synced_doc_id(executor, table, doc_id); if (error != DB_SUCCESS) goto func_exit; @@ -2526,27 +2455,19 @@ fts_cmp_set_sync_doc_id( if (cmp_doc_id && cmp_doc_id >= *doc_id) { error = fts_update_sync_doc_id( - table, cache->synced_doc_id, trx); + executor, table, cache->synced_doc_id); } *doc_id = cache->next_doc_id; func_exit: - if (caller_trx) { - return error; - } - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - } else { + if (error != DB_SUCCESS) { *doc_id = 0; ib::error() << "(" << error << ") while getting next doc id " "for table " << table->name; - fts_sql_rollback(trx); - if (error == DB_DEADLOCK || error == DB_LOCK_WAIT_TIMEOUT) { DEBUG_SYNC_C("fts_cmp_set_sync_doc_id_retry"); std::this_thread::sleep_for(FTS_DEADLOCK_RETRY_WAIT); @@ -2554,88 +2475,24 @@ fts_cmp_set_sync_doc_id( } } - trx->clear_and_free(); - return(error); } /** Update the last document id. This function could create a new transaction to update the last document id. -@param table table to be updated -@param doc_id last document id -@param trx update trx or null +@param executor query executor +@param table table to be updated +@param doc_id last document id @retval DB_SUCCESS if OK */ dberr_t -fts_update_sync_doc_id( - const dict_table_t* table, - doc_id_t doc_id, - trx_t* trx) +fts_update_sync_doc_id(FTSQueryExecutor *executor, + const dict_table_t *table, + doc_id_t doc_id) noexcept { - byte id[FTS_MAX_ID_LEN]; - pars_info_t* info; - fts_table_t fts_table; - ulint id_len; - que_t* graph = NULL; - dberr_t error; - ibool local_trx = FALSE; - fts_cache_t* cache = table->fts->cache; - char fts_name[MAX_FULL_NAME_LEN]; - - ut_ad(!srv_read_only_mode || recv_sys.rpo); - - if (recv_sys.rpo) { - return DB_READ_ONLY; - } - - fts_table.suffix = "CONFIG"; - fts_table.table_id = table->id; - fts_table.type = FTS_COMMON_TABLE; - fts_table.table = table; - - if (!trx) { - trx = trx_create(); - trx_start_internal(trx); - - trx->op_info = "setting last FTS document id"; - local_trx = TRUE; - } - - info = pars_info_create(); - - id_len = (ulint) snprintf( - (char*) id, sizeof(id), FTS_DOC_ID_FORMAT, doc_id + 1); - - pars_info_bind_varchar_literal(info, "doc_id", id, id_len); - - fts_get_table_name(&fts_table, fts_name, - table->fts->dict_locked); - pars_info_bind_id(info, "table_name", fts_name); - - graph = fts_parse_sql( - &fts_table, info, - "BEGIN" - " UPDATE $table_name SET value = :doc_id" - " WHERE key = 'synced_doc_id';"); - - error = fts_eval_sql(trx, graph); - - que_graph_free(graph); - - if (local_trx) { - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - cache->synced_doc_id = doc_id; - } else { - ib::error() << "(" << error << ") while" - " updating last doc id for table" - << table->name; - - fts_sql_rollback(trx); - } - trx->clear_and_free(); - } - - return(error); + if (srv_read_only_mode) return DB_READ_ONLY; + char id[FTS_MAX_ID_LEN]; + snprintf(id, sizeof(id), FTS_DOC_ID_FORMAT, doc_id + 1); + return executor->update_config_record("synced_doc_id", id); } /*********************************************************************//** @@ -2694,13 +2551,9 @@ fts_delete( fts_trx_table_t*ftt, /*!< in: FTS trx table */ fts_trx_row_t* row) /*!< in: row */ { - que_t* graph; - fts_table_t fts_table; - doc_id_t write_doc_id; dict_table_t* table = ftt->table; doc_id_t doc_id = row->doc_id; trx_t* trx = ftt->fts_trx->trx; - pars_info_t* info = pars_info_create(); fts_cache_t* cache = table->fts->cache; /* we do not index Documents whose Doc ID value is 0 */ @@ -2711,12 +2564,6 @@ fts_delete( ut_a(row->state == FTS_DELETE || row->state == FTS_MODIFY); - FTS_INIT_FTS_TABLE(&fts_table, "DELETED", FTS_COMMON_TABLE, table); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, doc_id); - fts_bind_doc_id(info, "doc_id", &write_doc_id); - /* It is possible we update a record that has not yet been sync-ed into cache from last crash (delete Doc will not initialize the sync). Avoid any added counter accounting until the FTS cache @@ -2741,20 +2588,9 @@ fts_delete( } /* Note the deleted document for OPTIMIZE to purge. */ - char table_name[MAX_FULL_NAME_LEN]; - trx->op_info = "adding doc id to FTS DELETED"; - - fts_table.suffix = "DELETED"; - - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "deleted", table_name); - - graph = fts_parse_sql(&fts_table, info, - "BEGIN INSERT INTO $deleted VALUES (:doc_id);"); - - dberr_t error = fts_eval_sql(trx, graph); - que_graph_free(graph); + FTSQueryExecutor executor(trx, table); + dberr_t error= executor.insert_common_record("DELETED", doc_id); /* Increment the total deleted count, this is used to calculate the number of documents indexed. */ @@ -2924,105 +2760,18 @@ fts_doc_free( } /*********************************************************************//** -Callback function for fetch that stores the text of an FTS document, -converting each column to UTF-16. -@return always FALSE */ -ibool -fts_query_expansion_fetch_doc( -/*==========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts_doc_t* */ -{ - que_node_t* exp; - sel_node_t* node = static_cast(row); - fts_doc_t* result_doc = static_cast(user_arg); - dfield_t* dfield; - ulint len; - ulint doc_len; - fts_doc_t doc; - CHARSET_INFO* doc_charset = NULL; - ulint field_no = 0; - - len = 0; - - fts_doc_init(&doc); - doc.found = TRUE; - - exp = node->select_list; - doc_len = 0; - - doc_charset = result_doc->charset; - - /* Copy each indexed column content into doc->text.f_str */ - while (exp) { - dfield = que_node_get_val(exp); - len = dfield_get_len(dfield); - - /* NULL column */ - if (len == UNIV_SQL_NULL) { - exp = que_node_get_next(exp); - continue; - } - - if (!doc_charset) { - doc_charset = fts_get_charset(dfield->type.prtype); - } - - doc.charset = doc_charset; - - if (dfield_is_ext(dfield)) { - /* We ignore columns that are stored externally, this - could result in too many words to search */ - exp = que_node_get_next(exp); - continue; - } else { - doc.text.f_n_char = 0; - - doc.text.f_str = static_cast( - dfield_get_data(dfield)); - - doc.text.f_len = len; - } - - if (field_no == 0) { - fts_tokenize_document(&doc, result_doc, - result_doc->parser); - } else { - fts_tokenize_document_next(&doc, doc_len, result_doc, - result_doc->parser); - } - - exp = que_node_get_next(exp); - - doc_len += (exp) ? len + 1 : len; - - field_no++; - } - - ut_ad(doc_charset); - - if (!result_doc->charset) { - result_doc->charset = doc_charset; - } - - fts_doc_free(&doc); - - return(FALSE); -} - -/*********************************************************************//** -fetch and tokenize the document. */ -static -void -fts_fetch_doc_from_rec( -/*===================*/ - fts_get_doc_t* get_doc, /*!< in: FTS index's get_doc struct */ - dict_index_t* clust_index, /*!< in: cluster index */ - btr_pcur_t* pcur, /*!< in: cursor whose position - has been stored */ - rec_offs* offsets, /*!< in: offsets */ - fts_doc_t* doc) /*!< out: fts doc to hold parsed - documents */ +fetch and tokenize the document. */ +static +void +fts_fetch_doc_from_rec( +/*===================*/ + fts_get_doc_t* get_doc, /*!< in: FTS index's get_doc struct */ + dict_index_t* clust_index, /*!< in: cluster index */ + btr_pcur_t* pcur, /*!< in: cursor whose position + has been stored */ + rec_offs* offsets, /*!< in: offsets */ + fts_doc_t* doc) /*!< out: fts doc to hold parsed + documents */ { dict_index_t* index; const rec_t* clust_rec; @@ -3173,6 +2922,7 @@ fts_add_doc_from_tuple( mtr_start(&mtr); + FTSQueryExecutor executor(ftt->fts_trx->trx, ftt->table); ulint num_idx = ib_vector_size(cache->get_docs); for (ulint i = 0; i < num_idx; ++i) { @@ -3194,8 +2944,7 @@ fts_add_doc_from_tuple( if (table->fts->cache->stopword_info.status & STOPWORD_NOT_INIT) { - fts_load_stopword(table, NULL, NULL, - true, true); + fts_load_stopword(&executor, table); } fts_cache_add_doc( @@ -3297,6 +3046,7 @@ fts_add_doc_by_id( } } + FTSQueryExecutor executor(ftt->fts_trx->trx, table); /* If we have a match, add the data to doc structure */ if (btr_pcur_open_with_no_init(tuple, PAGE_CUR_LE, BTR_SEARCH_LEAF, &pcur, &mtr) @@ -3372,8 +3122,7 @@ fts_add_doc_by_id( if (table->fts->cache->stopword_info.status & STOPWORD_NOT_INIT) { - fts_load_stopword(table, NULL, - NULL, true, true); + fts_load_stopword(&executor, table); } fts_cache_add_doc( @@ -3446,28 +3195,6 @@ fts_add_doc_by_id( mem_heap_free(heap); } - -/*********************************************************************//** -Callback function to read a single ulint column. -return always returns TRUE */ -static -ibool -fts_read_ulint( -/*===========*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ulint */ -{ - sel_node_t* sel_node = static_cast(row); - ulint* value = static_cast(user_arg); - que_node_t* exp = sel_node->select_list; - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - - *value = mach_read_from_4(static_cast(data)); - - return(TRUE); -} - /*********************************************************************//** Get maximum Doc ID in a table if index "FTS_DOC_ID_INDEX" exists @return max Doc ID or 0 if index "FTS_DOC_ID_INDEX" does not exist */ @@ -3539,200 +3266,19 @@ fts_get_max_doc_id( return(doc_id); } -/*********************************************************************//** -Fetch document with the given document id. -@return DB_SUCCESS if OK else error */ -dberr_t -fts_doc_fetch_by_doc_id( -/*====================*/ - fts_get_doc_t* get_doc, /*!< in: state */ - doc_id_t doc_id, /*!< in: id of document to - fetch */ - dict_index_t* index_to_use, /*!< in: caller supplied FTS index, - or NULL */ - ulint option, /*!< in: search option, if it is - greater than doc_id or equal */ - fts_sql_callback - callback, /*!< in: callback to read */ - void* arg) /*!< in: callback arg */ -{ - pars_info_t* info; - dberr_t error; - const char* select_str; - doc_id_t write_doc_id; - dict_index_t* index; - trx_t* trx = trx_create(); - que_t* graph; - - trx->op_info = "fetching indexed FTS document"; - - /* The FTS index can be supplied by caller directly with - "index_to_use", otherwise, get it from "get_doc" */ - index = (index_to_use) ? index_to_use : get_doc->index_cache->index; - - if (get_doc && get_doc->get_document_graph) { - info = get_doc->get_document_graph->info; - } else { - info = pars_info_create(); - } - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, doc_id); - fts_bind_doc_id(info, "doc_id", &write_doc_id); - pars_info_bind_function(info, "my_func", callback, arg); - - select_str = fts_get_select_columns_str(index, info, info->heap); - pars_info_bind_id(info, "table_name", index->table->name.m_name); - - if (!get_doc || !get_doc->get_document_graph) { - if (option == FTS_FETCH_DOC_BY_ID_EQUAL) { - graph = fts_parse_sql( - NULL, - info, - mem_heap_printf(info->heap, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT %s FROM $table_name" - " WHERE %s = :doc_id;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c %% NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;", - select_str, - FTS_DOC_ID.str)); - } else { - ut_ad(option == FTS_FETCH_DOC_BY_ID_LARGE); - - /* This is used for crash recovery of table with - hidden DOC ID or FTS indexes. We will scan the table - to re-processing user table rows whose DOC ID or - FTS indexed documents have not been sync-ed to disc - during recent crash. - In the case that all fulltext indexes are dropped - for a table, we will keep the "hidden" FTS_DOC_ID - column, and this scan is to retreive the largest - DOC ID being used in the table to determine the - appropriate next DOC ID. - In the case of there exists fulltext index(es), this - operation will re-tokenize any docs that have not - been sync-ed to the disk, and re-prime the FTS - cached */ - graph = fts_parse_sql( - NULL, - info, - mem_heap_printf(info->heap, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT %s, %s FROM $table_name" - " WHERE %s > :doc_id;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c %% NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;", - FTS_DOC_ID.str, - select_str, - FTS_DOC_ID.str)); - } - if (get_doc) { - get_doc->get_document_graph = graph; - } - } else { - graph = get_doc->get_document_graph; - } - - error = fts_eval_sql(trx, graph); - fts_sql_commit(trx); - trx->free(); - - if (!get_doc) { - que_graph_free(graph); - } - - return(error); -} - -/*********************************************************************//** -Write out a single word's data as new entry/entries in the INDEX table. -@return DB_SUCCESS if all OK. */ -dberr_t -fts_write_node( -/*===========*/ - trx_t* trx, /*!< in: transaction */ - que_t** graph, /*!< in: query graph */ - fts_table_t* fts_table, /*!< in: aux table */ - fts_string_t* word, /*!< in: word in UTF-8 */ - fts_node_t* node) /*!< in: node columns */ +/** Write out a single word's data as new entry/entries in the INDEX table. +@param executor FTS Query Executor +@param selected auxiliary index number +@param aux_data auxiliary table data +@return DB_SUCCESS if all OK or error code */ +dberr_t fts_write_node(FTSQueryExecutor *executor, uint8_t selected, + const fts_aux_data_t *aux_data) noexcept { - pars_info_t* info; - dberr_t error; - ib_uint32_t doc_count; - time_t start_time; - doc_id_t last_doc_id; - doc_id_t first_doc_id; - char table_name[MAX_FULL_NAME_LEN]; - - ut_a(node->ilist != NULL); - - if (*graph) { - info = (*graph)->info; - } else { - info = pars_info_create(); - - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "index_table_name", table_name); - } - - pars_info_bind_varchar_literal(info, "token", word->f_str, word->f_len); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &first_doc_id, node->first_doc_id); - fts_bind_doc_id(info, "first_doc_id", &first_doc_id); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &last_doc_id, node->last_doc_id); - fts_bind_doc_id(info, "last_doc_id", &last_doc_id); - - ut_a(node->last_doc_id >= node->first_doc_id); - - /* Convert to "storage" byte order. */ - mach_write_to_4((byte*) &doc_count, node->doc_count); - pars_info_bind_int4_literal( - info, "doc_count", (const ib_uint32_t*) &doc_count); - - /* Set copy_name to FALSE since it's a static. */ - pars_info_bind_literal( - info, "ilist", node->ilist, node->ilist_size, - DATA_BLOB, DATA_BINARY_TYPE); - - if (!*graph) { - - *graph = fts_parse_sql( - fts_table, - info, - "BEGIN\n" - "INSERT INTO $index_table_name VALUES" - " (:token, :first_doc_id," - " :last_doc_id, :doc_count, :ilist);"); - } - - start_time = time(NULL); - error = fts_eval_sql(trx, *graph); - elapsed_time += time(NULL) - start_time; - ++n_nodes; - - return(error); + time_t start_time= time(NULL); + dberr_t error= executor->insert_aux_record(selected, aux_data); + elapsed_time+= time(NULL) - start_time; + ++n_nodes; + return error; } /** Sort an array of doc_id */ @@ -3742,157 +3288,85 @@ void fts_doc_ids_sort(ib_vector_t *doc_ids) std::sort(data, data + doc_ids->used); } -/*********************************************************************//** -Add rows to the DELETED_CACHE table. +/** Add rows to the DELETED_CACHE table. @return DB_SUCCESS if all went well else error code*/ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t -fts_sync_add_deleted_cache( -/*=======================*/ - fts_sync_t* sync, /*!< in: sync state */ - ib_vector_t* doc_ids) /*!< in: doc ids to add */ +fts_sync_add_deleted_cache(FTSQueryExecutor *executor, fts_sync_t *sync, + ib_vector_t *doc_ids) noexcept { - ulint i; - pars_info_t* info; - que_t* graph; - fts_table_t fts_table; - char table_name[MAX_FULL_NAME_LEN]; - doc_id_t dummy = 0; - dberr_t error = DB_SUCCESS; - ulint n_elems = ib_vector_size(doc_ids); - - ut_a(ib_vector_size(doc_ids) > 0); - - fts_doc_ids_sort(doc_ids); - - info = pars_info_create(); - - fts_bind_doc_id(info, "doc_id", &dummy); - - FTS_INIT_FTS_TABLE( - &fts_table, "DELETED_CACHE", FTS_COMMON_TABLE, sync->table); - - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - &fts_table, - info, - "BEGIN INSERT INTO $table_name VALUES (:doc_id);"); - - for (i = 0; i < n_elems && error == DB_SUCCESS; ++i) { - doc_id_t* update; - doc_id_t write_doc_id; - - update = static_cast(ib_vector_get(doc_ids, i)); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, *update); - fts_bind_doc_id(info, "doc_id", &write_doc_id); - - error = fts_eval_sql(sync->trx, graph); - } - - que_graph_free(graph); - - return(error); + ulint n_elems= ib_vector_size(doc_ids); + ut_a(n_elems > 0); + fts_doc_ids_sort(doc_ids); + dberr_t error= DB_SUCCESS; + for (uint32_t i= 0; i < n_elems && error == DB_SUCCESS; ++i) + { + doc_id_t *update= + static_cast(ib_vector_get(doc_ids, i)); + error= executor->insert_common_record("DELETED_CACHE", *update); + } + return error; } -/** Write the words and ilist to disk. -@param[in,out] trx transaction -@param[in] index_cache index cache -@param[in] unlock_cache whether unlock cache when write node +/** Write the words and ilist to disk +@param[in,out] executor query executor +@param[in] index_cache index cache +@param[in] unlock_cache whether unlock cache when write node @return DB_SUCCESS if all went well else error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_sync_write_words( - trx_t* trx, - fts_index_cache_t* index_cache, - bool unlock_cache) +dberr_t fts_sync_write_words(FTSQueryExecutor *executor, + fts_index_cache_t *index_cache, + bool unlock_cache) noexcept { - fts_table_t fts_table; - const ib_rbt_node_t* rbt_node; - dberr_t error = DB_SUCCESS; - ibool print_error = FALSE; - dict_table_t* table = index_cache->index->table; - - FTS_INIT_INDEX_TABLE( - &fts_table, NULL, FTS_INDEX_TABLE, index_cache->index); - - /* We iterate over the entire tree, even if there is an error, - since we want to free the memory used during caching. */ - for (rbt_node = rbt_first(index_cache->words); - rbt_node; - rbt_node = rbt_next(index_cache->words, rbt_node)) { - - ulint i; - ulint selected; - fts_tokenizer_word_t* word; - - word = rbt_value(fts_tokenizer_word_t, rbt_node); - - DBUG_EXECUTE_IF( - "fts_instrument_write_words_before_select_index", - std::this_thread::sleep_for( - std::chrono::milliseconds(300));); - - selected = fts_select_index( - index_cache->charset, word->text.f_str, - word->text.f_len); - - fts_table.suffix = fts_get_suffix(selected); - - /* We iterate over all the nodes even if there was an error */ - for (i = 0; i < ib_vector_size(word->nodes); ++i) { - - fts_node_t* fts_node = static_cast( - ib_vector_get(word->nodes, i)); - - if (fts_node->synced) { - continue; - } else { - fts_node->synced = true; - } - - /*FIXME: we need to handle the error properly. */ - if (error == DB_SUCCESS) { - if (unlock_cache) { - mysql_mutex_unlock( - &table->fts->cache->lock); - } - - error = fts_write_node( - trx, - &index_cache->ins_graph[selected], - &fts_table, &word->text, fts_node); - - DEBUG_SYNC_C("fts_write_node"); - DBUG_EXECUTE_IF("fts_write_node_crash", - DBUG_SUICIDE();); - - DBUG_EXECUTE_IF( - "fts_instrument_sync_sleep", - std::this_thread::sleep_for( - std::chrono::seconds(1));); - - if (unlock_cache) { - mysql_mutex_lock( - &table->fts->cache->lock); - } - } - } + dict_table_t *table= index_cache->index->table; + bool print_error= false; + dberr_t error= DB_SUCCESS; + for (const ib_rbt_node_t *rbt_node= rbt_first(index_cache->words); + rbt_node; rbt_node= rbt_next(index_cache->words, rbt_node)) + { + fts_tokenizer_word_t *word= rbt_value(fts_tokenizer_word_t, rbt_node); + DBUG_EXECUTE_IF("fts_instrument_write_words_before_select_index", + std::this_thread::sleep_for( + std::chrono::milliseconds(300));); + uint8_t selected= fts_select_index( + index_cache->charset, word->text.f_str, word->text.f_len); + + for (ulint i = 0; i < ib_vector_size(word->nodes); ++i) + { + fts_node_t* fts_node= + static_cast(ib_vector_get(word->nodes, i)); + if (fts_node->synced) continue; + else fts_node->synced= true; + /* FIXME: we need to handle the error properly. */ + if (error == DB_SUCCESS) + { + if (unlock_cache) mysql_mutex_unlock(&table->fts->cache->lock); + fts_aux_data_t aux_data((const char*)word->text.f_str, word->text.f_len, + fts_node->first_doc_id, fts_node->last_doc_id, + static_cast(fts_node->doc_count), fts_node->ilist, + fts_node->ilist_size); + error= fts_write_node(executor, selected, &aux_data); + DEBUG_SYNC_C("fts_write_node"); + DBUG_EXECUTE_IF("fts_write_node_crash", DBUG_SUICIDE();); + DBUG_EXECUTE_IF("fts_instrument_sync_sleep", + std::this_thread::sleep_for(std::chrono::seconds(1));); + + if (unlock_cache) mysql_mutex_lock(&table->fts->cache->lock); + } - n_nodes += ib_vector_size(word->nodes); + n_nodes+= ib_vector_size(word->nodes); - if (UNIV_UNLIKELY(error != DB_SUCCESS) && !print_error) { - ib::error() << "(" << error << ") writing" - " word node to FTS auxiliary index table " - << table->name; - print_error = TRUE; - } - } + if (UNIV_UNLIKELY(error != DB_SUCCESS) && !print_error) + { + sql_print_error("InnoDB: ( %s ) writing word node to FTS auxiliary " + "index table %s", ut_strerr(error), + table->name.m_name); + print_error= true; + } + } + } - return(error); + return error; } /*********************************************************************//** @@ -3907,24 +3381,22 @@ fts_sync_begin( trx_start_internal(sync->trx); } -/*********************************************************************//** -Run SYNC on the table, i.e., write out data from the index specific -cache to the FTS aux INDEX table and FTS aux doc id stats table. +/** Run SYNC on the table, i.e., write out data from the index +specific cache to the FTS aux INDEX table and FTS aux doc id +stats table. +@param executor query executor +@param sync sync state +@param index_cache index cache @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t -fts_sync_index( -/*===========*/ - fts_sync_t* sync, /*!< in: sync state */ - fts_index_cache_t* index_cache) /*!< in: index cache */ +fts_sync_index(FTSQueryExecutor *executor, fts_sync_t *sync, + fts_index_cache_t *index_cache) noexcept { - trx_t* trx = sync->trx; - - trx->op_info = "doing SYNC index"; - - ut_ad(rbt_validate(index_cache->words)); - - return(fts_sync_write_words(trx, index_cache, sync->unlock_cache)); + trx_t *trx= sync->trx; + trx->op_info= "doing SYNC index"; + ut_ad(rbt_validate(index_cache->words)); + return fts_sync_write_words(executor, index_cache, sync->unlock_cache); } /** Check if index cache has been synced completely @@ -3978,13 +3450,10 @@ fts_sync_index_reset( } } -/** Commit the SYNC, change state of processed doc ids etc. -@param[in,out] sync sync state -@return DB_SUCCESS if all OK */ +/** Commit the SYNC, change state of processed doc ids etc.\n@param[in,out]\texecutor\t\tquery executor\n@param[in,out]\tsync\t\t\tsync state\n@return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t -fts_sync_commit( - fts_sync_t* sync) +fts_sync_commit(FTSQueryExecutor *executor, fts_sync_t *sync) noexcept { dberr_t error; trx_t* trx = sync->trx; @@ -3995,8 +3464,8 @@ fts_sync_commit( /* After each Sync, update the CONFIG table about the max doc id we just sync-ed to index table */ - error = fts_cmp_set_sync_doc_id(sync->table, sync->max_doc_id, - &last_doc_id, trx); + error = fts_cmp_set_sync_doc_id(executor, sync->table, sync->max_doc_id, + &last_doc_id); /* Get the list of deleted documents that are either in the cache or were headed there but were deleted before the add @@ -4005,7 +3474,7 @@ fts_sync_commit( if (error == DB_SUCCESS && ib_vector_size(cache->deleted_doc_ids) > 0) { error = fts_sync_add_deleted_cache( - sync, cache->deleted_doc_ids); + executor, sync, cache->deleted_doc_ids); } /* We need to do this within the deleted lock since fts_delete() can @@ -4125,11 +3594,13 @@ fts_sync( sync->unlock_cache = unlock_cache; sync->in_progress = true; + const size_t fts_cache_size= fts_max_cache_size; DEBUG_SYNC_C("fts_sync_begin"); fts_sync_begin(sync); - + FTSQueryExecutor executor(sync->trx, sync->table); + error = executor.open_config_table(); + if (error) goto end_sync; begin_sync: - const size_t fts_cache_size= fts_max_cache_size; if (cache->total_size > fts_cache_size) { /* Avoid the case: sync never finish when insert/update keeps comming. */ @@ -4143,19 +3614,21 @@ fts_sync( } for (i = 0; i < ib_vector_size(cache->indexes); ++i) { - fts_index_cache_t* index_cache; - - index_cache = static_cast( + fts_index_cache_t *index_cache= + static_cast( ib_vector_get(cache->indexes, i)); if (index_cache->index->to_be_dropped) { continue; } + error= executor.open_all_aux_tables(index_cache->index); + + if (error) goto end_sync; DBUG_EXECUTE_IF("fts_instrument_sync_before_syncing", std::this_thread::sleep_for( std::chrono::milliseconds(300));); - error = fts_sync_index(sync, index_cache); + error = fts_sync_index(&executor, sync, index_cache); if (error != DB_SUCCESS) { goto end_sync; @@ -4192,7 +3665,7 @@ fts_sync( end_sync: if (error == DB_SUCCESS && !sync->interrupted) { - error = fts_sync_commit(sync); + error = fts_sync_commit(&executor, sync); } else { fts_sync_rollback(sync); } @@ -4514,7 +3987,6 @@ fts_tokenize_by_parser( @param[in,out] doc document to tokenize @param[out] result tokenization result @param[in] parser pluggable parser */ -static void fts_tokenize_document( fts_doc_t* doc, @@ -4544,12 +4016,6 @@ fts_tokenize_document( } } -/** Continue to tokenize a document. -@param[in,out] doc document to tokenize -@param[in] add_pos add this position to all tokens from this tokenization -@param[out] result tokenization result -@param[in] parser pluggable parser */ -static void fts_tokenize_document_next( fts_doc_t* doc, @@ -4664,7 +4130,12 @@ fts_init_doc_id( /* Then compare this value with the ID value stored in the CONFIG table. The larger one will be our new initial Doc ID */ - fts_cmp_set_sync_doc_id(table, 0, &max_doc_id); + trx_t* trx = trx_create(); + trx_start_internal_read_only(trx); + FTSQueryExecutor executor(trx, table); + fts_cmp_set_sync_doc_id(&executor, table, 0, &max_doc_id); + fts_sql_commit(trx); + trx->free(); /* If DICT_TF2_FTS_ADD_DOC_ID is set, we are in the process of creating index (and add doc id column. No need to recovery @@ -4684,81 +4155,6 @@ fts_init_doc_id( return(max_doc_id); } -/*********************************************************************//** -Fetch COUNT(*) from specified table. -@return the number of rows in the table */ -ulint -fts_get_rows_count( -/*===============*/ - fts_table_t* fts_table) /*!< in: fts table to read */ -{ - trx_t* trx; - pars_info_t* info; - que_t* graph; - dberr_t error; - ulint count = 0; - char table_name[MAX_FULL_NAME_LEN]; - - trx = trx_create(); - trx->op_info = "fetching FT table rows count"; - - info = pars_info_create(); - - pars_info_bind_function(info, "my_func", fts_read_ulint, &count); - - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT COUNT(*)" - " FROM $table_name;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - for (;;) { - error = fts_eval_sql(trx, graph); - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - - break; /* Exit the loop. */ - } else { - fts_sql_rollback(trx); - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading" - " FTS table. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << "(" << error - << ") while reading FTS table " - << table_name; - - break; /* Exit the loop. */ - } - } - } - - que_graph_free(graph); - - trx->free(); - - return(count); -} - /*********************************************************************//** Free the modified rows of a table. */ UNIV_INLINE @@ -5584,288 +4980,267 @@ FTS. bool fts_load_stopword( /*==============*/ - const dict_table_t* - table, /*!< in: Table with FTS */ - trx_t* trx, /*!< in: Transactions */ - const char* session_stopword_table, /*!< in: Session stopword table - name */ - bool stopword_is_on, /*!< in: Whether stopword - option is turned on/off */ - bool reload) /*!< in: Whether it is - for reloading FTS table */ + FTSQueryExecutor* executor, /*!< in: FTSQueryExecutor */ + const dict_table_t* table) noexcept /*!< in: Table with FTS */ { - fts_table_t fts_table; + fts_cache_t* cache = table->fts->cache; fts_string_t str; dberr_t error = DB_SUCCESS; - ulint use_stopword; - fts_cache_t* cache; const char* stopword_to_use = NULL; - ibool new_trx = FALSE; byte str_buffer[MAX_FULL_NAME_LEN + 1]; - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, table); - - cache = table->fts->cache; - - if (!reload && !(cache->stopword_info.status & STOPWORD_NOT_INIT)) { + /* If stopword is already initialized, return success */ + if (!(cache->stopword_info.status & STOPWORD_NOT_INIT)) { return true; } - if (!trx) { - trx = trx_create(); -#ifdef UNIV_DEBUG - trx->start_line = __LINE__; - trx->start_file = __FILE__; -#endif - trx_start_internal_low(trx, !high_level_read_only - && !recv_sys.rpo); - trx->op_info = "upload FTS stopword"; - new_trx = TRUE; - } - - /* First check whether stopword filtering is turned off */ - if (reload) { - error = fts_config_get_ulint( - trx, &fts_table, FTS_USE_STOPWORD, &use_stopword); - } else { - use_stopword = (ulint) stopword_is_on; - - error = fts_config_set_ulint( - trx, &fts_table, FTS_USE_STOPWORD, use_stopword); + /* First check if stopwords are enabled */ + ulint use_stopword= 0; + fts_string_t value; + value.f_len= FTS_MAX_CONFIG_VALUE_LEN; + value.f_str= static_cast(ut_malloc_nokey(value.f_len + 1)); + error= fts_config_get_value(executor, table, FTS_USE_STOPWORD, &value); + if (error == DB_SUCCESS) { + use_stopword= strtoul(reinterpret_cast(value.f_str), + nullptr, 10); } + ut_free(value.f_str); + if (error != DB_SUCCESS) { - goto cleanup; + /* Failed to read config */ + return false; } - - /* If stopword is turned off, no need to continue to load the - stopword into cache, but still need to do initialization */ + if (!use_stopword) { + /* Stopwords are disabled, mark as initialized but off */ cache->stopword_info.status = STOPWORD_OFF; - goto cleanup; + return true; } - if (reload) { - /* Fetch the stopword table name from FTS config - table */ - str.f_n_char = 0; - str.f_str = str_buffer; - str.f_len = sizeof(str_buffer) - 1; - - error = fts_config_get_value( - trx, &fts_table, FTS_STOPWORD_TABLE_NAME, &str); + /* Read FTS_STOPWORD_TABLE_NAME from CONFIG table */ + str.f_n_char = 0; + str.f_str = str_buffer; + str.f_len = sizeof(str_buffer) - 1; - if (error != DB_SUCCESS) { - goto cleanup; - } + error = fts_config_get_value(executor, table, FTS_STOPWORD_TABLE_NAME, &str); - if (*str.f_str) { - stopword_to_use = (const char*) str.f_str; - } - } else { - stopword_to_use = session_stopword_table; + if (error == DB_SUCCESS && *str.f_str) { + stopword_to_use = (const char*) str.f_str; } + /* Load user stopword table if specified, otherwise load default */ if (stopword_to_use - && fts_load_user_stopword(table->fts, stopword_to_use, - &cache->stopword_info)) { - /* Save the stopword table name to the configure - table */ - if (!reload) { - str.f_n_char = 0; - str.f_str = (byte*) stopword_to_use; - str.f_len = strlen(stopword_to_use); - - error = fts_config_set_value( - trx, &fts_table, FTS_STOPWORD_TABLE_NAME, &str); - } + && fts_load_user_stopword(executor, table->fts, stopword_to_use, + &cache->stopword_info)) { + /* Successfully loaded user stopword table */ } else { /* Load system default stopword list */ fts_load_default_stopword(&cache->stopword_info); } -cleanup: - if (new_trx) { - if (error == DB_SUCCESS) { - fts_sql_commit(trx); - } else { - fts_sql_rollback(trx); - } - - trx->clear_and_free(); - } - + /* Initialize cached_stopword RB-tree if not already created */ if (!cache->stopword_info.cached_stopword) { cache->stopword_info.cached_stopword = rbt_create_arg_cmp( sizeof(fts_tokenizer_word_t), innobase_fts_text_cmp, &my_charset_latin1); } - return error == DB_SUCCESS; + return true; } -/**********************************************************************//** -Callback function when we initialize the FTS at the start up -time. It recovers the maximum Doc IDs presented in the current table. -Tested by innodb_fts.crash_recovery -@return: always returns TRUE */ -static -ibool -fts_init_get_doc_id( -/*================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: table with fts */ -{ - doc_id_t doc_id = FTS_NULL_DOC_ID; - sel_node_t* node = static_cast(row); - que_node_t* exp = node->select_list; - dict_table_t* table = static_cast(user_arg); - fts_cache_t* cache = table->fts->cache; - - ut_ad(ib_vector_is_empty(cache->get_docs)); - - /* Copy each indexed column content into doc->text.f_str */ - if (exp) { - dfield_t* dfield = que_node_get_val(exp); - dtype_t* type = dfield_get_type(dfield); - void* data = dfield_get_data(dfield); - - ut_a(dtype_get_mtype(type) == DATA_INT); - - doc_id = static_cast(mach_read_from_8( - static_cast(data))); - - exp = que_node_get_next(que_node_get_next(exp)); - if (exp) { - ut_ad(table->versioned()); - dfield = que_node_get_val(exp); - type = dfield_get_type(dfield); - ut_ad(type->vers_sys_end()); - data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - if (table->versioned_by_id()) { - ut_ad(len == sizeof trx_id_max_bytes); - if (0 != memcmp(data, trx_id_max_bytes, len)) { - return true; - } - } else { - ut_ad(len == sizeof timestamp_max_bytes); - if (!IS_MAX_TIMESTAMP(data)) { - return true; - } - } - ut_ad(!(exp = que_node_get_next(exp))); - } - ut_ad(!exp); - - if (doc_id >= cache->next_doc_id) { - cache->next_doc_id = doc_id + 1; - } - } - - return(TRUE); -} - -/**********************************************************************//** -Callback function when we initialize the FTS at the start up +/** Callback function when we initialize the FTS at the start up time. It recovers Doc IDs that have not sync-ed to the auxiliary table, and require to bring them back into FTS index. +@param executor query executor +@param get_doc Document +@param doc_id document id to be fetched @return: always returns TRUE */ -static -ibool -fts_init_recover_doc( -/*=================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts cache */ +static void fts_init_recover_all_docs(FTSQueryExecutor *executor, + fts_get_doc_t *get_doc, + doc_id_t doc_id) noexcept { + executor->trx()->op_info= "fetching indexed FTS document"; + dict_index_t *fts_index= get_doc->index_cache->index; + dict_table_t *user_table= fts_index->table; + dict_index_t *fts_doc_id_index= user_table->fts_doc_id_index; + dict_index_t *clust_index= dict_table_get_first_index(user_table); + fts_cache_t *cache= get_doc->cache; + ut_a(user_table->fts->doc_col != ULINT_UNDEFINED); + ut_a(fts_doc_id_index); + /* Map FTS index columns to clustered index field positions */ + ulint *clust_field_nos= static_cast( + mem_heap_alloc(executor->get_heap(), + fts_index->n_user_defined_cols * sizeof(ulint))); + + for (unsigned i= 0; i < fts_index->n_user_defined_cols; i++) + { + dict_field_t* fts_field= dict_index_get_nth_field(fts_index, i); + clust_field_nos[i]= dict_col_get_index_pos(fts_field->col, clust_index); + } - fts_doc_t doc; - ulint doc_len = 0; - ulint field_no = 0; - fts_get_doc_t* get_doc = static_cast(user_arg); - doc_id_t doc_id = FTS_NULL_DOC_ID; - sel_node_t* node = static_cast(row); - que_node_t* exp = node->select_list; - fts_cache_t* cache = get_doc->cache; - st_mysql_ftparser* parser = get_doc->index_cache->index->parser; - - fts_doc_init(&doc); - doc.found = TRUE; - - ut_ad(cache); - - /* Copy each indexed column content into doc->text.f_str */ - while (exp) { - dfield_t* dfield = que_node_get_val(exp); - ulint len = dfield_get_len(dfield); - - if (field_no == 0) { - dtype_t* type = dfield_get_type(dfield); - void* data = dfield_get_data(dfield); - - ut_a(dtype_get_mtype(type) == DATA_INT); - - doc_id = static_cast(mach_read_from_8( - static_cast(data))); + dfield_t fields[1]; + dtuple_t search_tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&search_tuple, fts_doc_id_index, 1); + dfield_t* dfield= dtuple_get_nth_field(&search_tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id((byte*) &write_doc_id, doc_id); + dfield_set_data(dfield, &write_doc_id, sizeof(write_doc_id)); + + auto process_doc_recovery= [get_doc, cache, user_table, fts_index, + clust_field_nos](const rec_t* rec, + const dict_index_t* index, + const rec_offs* offsets) -> dberr_t + { + fts_doc_t doc; + ulint doc_len= 0; + doc_id_t doc_id= FTS_NULL_DOC_ID; + st_mysql_ftparser* parser= fts_index->parser; - field_no++; - exp = que_node_get_next(exp); - continue; - } + fts_doc_init(&doc); + doc.found= TRUE; - if (len == UNIV_SQL_NULL) { - exp = que_node_get_next(exp); - continue; - } + /* Extract doc_id from the clustered index record */ + ulint doc_col_pos= dict_col_get_index_pos( + &user_table->cols[user_table->fts->doc_col], index); - ut_ad(get_doc); + ulint processed_field= 0; + ulint len; + const byte* doc_id_data= rec_get_nth_field(rec, offsets, doc_col_pos, &len); - if (!get_doc->index_cache->charset) { - get_doc->index_cache->charset = fts_get_charset( - dfield->type.prtype); - } + if (len == sizeof(doc_id_t)) + { + doc_id= fts_read_doc_id(doc_id_data); - doc.charset = get_doc->index_cache->charset; + /* Process each indexed column content */ + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, col_pos, + &field_len); + if (field_len == UNIV_SQL_NULL) + continue; + if (!get_doc->index_cache->charset) + { + dict_field_t* fts_field= dict_index_get_nth_field(fts_index, i); + get_doc->index_cache->charset= fts_get_charset( + fts_field->col->prtype); + } + doc.charset= get_doc->index_cache->charset; + + /* Handle externally stored fields */ + if (rec_offs_nth_extern(offsets, col_pos)) + doc.text.f_str= btr_copy_externally_stored_field( + &doc.text.f_len, const_cast(field_data), + user_table->space->zip_size(), field_len, + static_cast(doc.self_heap->arg)); + else + { + doc.text.f_str= const_cast(field_data); + doc.text.f_len= field_len; + } - if (dfield_is_ext(dfield)) { - dict_table_t* table = cache->sync->table; + if (processed_field == 0) + fts_tokenize_document(&doc, NULL, parser); + else + fts_tokenize_document_next(&doc, doc_len, NULL, parser); - doc.text.f_str = btr_copy_externally_stored_field( - &doc.text.f_len, - static_cast(dfield_get_data(dfield)), - table->space->zip_size(), len, - static_cast(doc.self_heap->arg)); - } else { - doc.text.f_str = static_cast( - dfield_get_data(dfield)); + processed_field++; + doc_len+= + (i < (unsigned) get_doc->index_cache->index->n_user_defined_cols - 1) + ? field_len + 1 + : field_len; + } - doc.text.f_len = len; - } + fts_cache_add_doc(cache, get_doc->index_cache, doc_id, doc.tokens); + fts_doc_free(&doc); + cache->added++; - if (field_no == 1) { - fts_tokenize_document(&doc, NULL, parser); - } else { - fts_tokenize_document_next(&doc, doc_len, NULL, parser); - } + if (doc_id >= cache->next_doc_id) + cache->next_doc_id= doc_id + 1; + } - exp = que_node_get_next(exp); + return DB_SUCCESS; + }; + RecordCallback reader(process_doc_recovery, doc_id_comparator); + executor->read_by_index(user_table, fts_doc_id_index, &search_tuple, + PAGE_CUR_G, reader); +} - doc_len += (exp) ? len + 1 : len; +/** Get the next large document id and update it in fulltext cache +@param executor query executor +@param doc_id document id to be updated +@param index fulltext index */ +static void fts_init_get_doc_id(FTSQueryExecutor *executor, + doc_id_t doc_id, dict_index_t *index) noexcept +{ + executor->trx()->op_info= "fetching indexed FTS document"; + dict_table_t* user_table= index->table; + fts_cache_t* cache= user_table->fts->cache; + ut_a(user_table->fts->doc_col != ULINT_UNDEFINED); - field_no++; - } + dfield_t fields[1]; + dtuple_t search_tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&search_tuple, index, 1); + dfield_t* dfield= dtuple_get_nth_field(&search_tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id((byte*) &write_doc_id, doc_id); + dfield_set_data(dfield, &write_doc_id, sizeof(write_doc_id)); + + auto process_doc_id= [cache, user_table](const rec_t* rec, + const dict_index_t* index, + const rec_offs* offsets) -> dberr_t + { + ulint doc_col_pos= dict_col_get_index_pos( + &user_table->cols[user_table->fts->doc_col], index); - fts_cache_add_doc(cache, get_doc->index_cache, doc_id, doc.tokens); + ulint len; + const byte* doc_id_data= rec_get_nth_field(rec, offsets, doc_col_pos, &len); - fts_doc_free(&doc); + if (len == sizeof(doc_id_t)) + { + doc_id_t found_doc_id= mach_read_from_8(doc_id_data); + if (user_table->versioned()) + { + ulint vers_end_pos= dict_col_get_index_pos( + &user_table->cols[user_table->vers_end], index); + ulint vers_len; + const byte* vers_data= rec_get_nth_field(rec, offsets, + vers_end_pos, &vers_len); - cache->added++; + if (user_table->versioned_by_id()) + { + if (vers_len == sizeof(trx_id_max_bytes) && + memcmp(vers_data, trx_id_max_bytes, vers_len) != 0) + return DB_SUCCESS; + } + else + { + if (vers_len == sizeof(timestamp_max_bytes) && + !IS_MAX_TIMESTAMP(vers_data)) + return DB_SUCCESS; + } + } - if (doc_id >= cache->next_doc_id) { - cache->next_doc_id = doc_id + 1; - } + /* Update cache->next_doc_id if this doc_id is larger */ + if (found_doc_id >= cache->next_doc_id) + cache->next_doc_id= found_doc_id + 1; + } + return DB_SUCCESS; + }; - return(TRUE); + RecordCallback reader(process_doc_id, doc_id_comparator); + executor->read_by_index(user_table, index, &search_tuple, PAGE_CUR_G, + reader); } /**********************************************************************//** @@ -5885,6 +5260,9 @@ fts_init_index( fts_get_doc_t* get_doc = NULL; fts_cache_t* cache = table->fts->cache; bool need_init = false; + /* Declare variables before any goto to avoid initialization bypass */ + trx_t *trx = nullptr; + FTSQueryExecutor *executor= nullptr; /* First check cache->get_docs is initialized */ if (!has_cache_lock) { @@ -5905,13 +5283,19 @@ fts_init_index( start_doc = cache->synced_doc_id; + /* Create single FTSQueryExecutor for all operations */ + trx = trx_create(); + trx_start_internal_read_only(trx); + executor= new FTSQueryExecutor(trx, table); + if (!start_doc) { - trx_t *trx = trx_create(); - trx_start_internal_read_only(trx); - dberr_t err= fts_read_synced_doc_id(table, &start_doc, trx); - fts_sql_commit(trx); - trx->free(); + dberr_t err= fts_read_synced_doc_id( + executor, table, &start_doc); if (err != DB_SUCCESS) { + fts_sql_commit(trx); + trx->free(); + delete executor; + executor = nullptr; goto func_exit; } if (start_doc) { @@ -5928,13 +5312,11 @@ fts_init_index( ut_a(index); - fts_doc_fetch_by_doc_id(NULL, start_doc, index, - FTS_FETCH_DOC_BY_ID_LARGE, - fts_init_get_doc_id, table); + fts_init_get_doc_id(executor, start_doc, index); } else { if (table->fts->cache->stopword_info.status & STOPWORD_NOT_INIT) { - fts_load_stopword(table, NULL, NULL, true, true); + fts_load_stopword(executor, table); } for (ulint i = 0; i < ib_vector_size(cache->get_docs); ++i) { @@ -5943,9 +5325,7 @@ fts_init_index( index = get_doc->index_cache->index; - fts_doc_fetch_by_doc_id(NULL, start_doc, index, - FTS_FETCH_DOC_BY_ID_LARGE, - fts_init_recover_doc, get_doc); + fts_init_recover_all_docs(executor, get_doc, start_doc); } } @@ -5953,6 +5333,15 @@ fts_init_index( fts_get_docs_clear(cache->get_docs); + /* Commit transaction and cleanup */ + if (trx) { + fts_sql_commit(trx); + trx->free(); + } + if (executor) { + delete executor; + } + func_exit: if (!has_cache_lock) { mysql_mutex_unlock(&cache->lock); diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index ea6ddab1b2458..1902b34ce2a48 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -27,6 +27,7 @@ Completed 2011/7/10 Sunny and Jimmy Yang ***********************************************************************/ #include "fts0fts.h" +#include "fts0exec.h" #include "row0sel.h" #include "que0types.h" #include "fts0priv.h" @@ -70,9 +71,6 @@ static bool fts_opt_start_shutdown = false; Protected by fts_optimize_wq->mutex. */ static pthread_cond_t fts_opt_shutdown_cond; -/** Initial size of nodes in fts_word_t. */ -static const ulint FTS_WORD_NODES_INIT_SIZE = 64; - /** Last time we did check whether system need a sync */ static time_t last_check_sync_time; @@ -140,11 +138,6 @@ struct fts_optimize_t { char* name_prefix; /*!< FTS table name prefix */ - fts_table_t fts_index_table;/*!< Common table definition */ - - /*!< Common table definition */ - fts_table_t fts_common_table; - dict_table_t* table; /*!< Table that has to be queried */ dict_index_t* index; /*!< The FTS index to be optimized */ @@ -235,28 +228,6 @@ static ulint fts_optimize_time_limit; /** It's defined in fts0fts.cc */ extern const char* fts_common_tables[]; -/** SQL Statement for changing state of rows to be deleted from FTS Index. */ -static const char* fts_init_delete_sql = - "BEGIN\n" - "\n" - "INSERT INTO $BEING_DELETED\n" - "SELECT doc_id FROM $DELETED;\n" - "\n" - "INSERT INTO $BEING_DELETED_CACHE\n" - "SELECT doc_id FROM $DELETED_CACHE;\n"; - -static const char* fts_delete_doc_ids_sql = - "BEGIN\n" - "\n" - "DELETE FROM $DELETED WHERE doc_id = :doc_id1;\n" - "DELETE FROM $DELETED_CACHE WHERE doc_id = :doc_id2;\n"; - -static const char* fts_end_delete_sql = - "BEGIN\n" - "\n" - "DELETE FROM $BEING_DELETED;\n" - "DELETE FROM $BEING_DELETED_CACHE;\n"; - /**********************************************************************//** Initialize fts_zip_t. */ static @@ -327,238 +298,47 @@ fts_zip_init( *zip->word.f_str = '\0'; } -/**********************************************************************//** -Create a fts_optimizer_word_t instance. -@return new instance */ -static -fts_word_t* -fts_word_init( -/*==========*/ - fts_word_t* word, /*!< in: word to initialize */ - byte* utf8, /*!< in: UTF-8 string */ - ulint len) /*!< in: length of string in bytes */ -{ - mem_heap_t* heap = mem_heap_create(sizeof(fts_node_t)); - - memset(word, 0, sizeof(*word)); - - word->text.f_len = len; - word->text.f_str = static_cast(mem_heap_alloc(heap, len + 1)); - - /* Need to copy the NUL character too. */ - memcpy(word->text.f_str, utf8, word->text.f_len); - word->text.f_str[word->text.f_len] = 0; - - word->heap_alloc = ib_heap_allocator_create(heap); - - word->nodes = ib_vector_create( - word->heap_alloc, sizeof(fts_node_t), FTS_WORD_NODES_INIT_SIZE); - - return(word); -} - -/**********************************************************************//** -Read the FTS INDEX row. -@return fts_node_t instance */ -static -fts_node_t* -fts_optimize_read_node( -/*===================*/ - fts_word_t* word, /*!< in: */ - que_node_t* exp) /*!< in: */ -{ - int i; - fts_node_t* node = static_cast( - ib_vector_push(word->nodes, NULL)); - - /* Start from 1 since the first node has been read by the caller */ - for (i = 1; exp; exp = que_node_get_next(exp), ++i) { - - dfield_t* dfield = que_node_get_val(exp); - byte* data = static_cast( - dfield_get_data(dfield)); - ulint len = dfield_get_len(dfield); - - ut_a(len != UNIV_SQL_NULL); - - /* Note: The column numbers below must match the SELECT */ - switch (i) { - case 1: /* DOC_COUNT */ - node->doc_count = mach_read_from_4(data); - break; - - case 2: /* FIRST_DOC_ID */ - node->first_doc_id = fts_read_doc_id(data); - break; - - case 3: /* LAST_DOC_ID */ - node->last_doc_id = fts_read_doc_id(data); - break; - - case 4: /* ILIST */ - node->ilist_size_alloc = node->ilist_size = len; - node->ilist = static_cast(ut_malloc_nokey(len)); - memcpy(node->ilist, data, len); - break; - - default: - ut_error; - } - } - - /* Make sure all columns were read. */ - ut_a(i == 5); - - return(node); -} - -/**********************************************************************//** -Callback function to fetch the rows in an FTS INDEX record. -@return always returns non-NULL */ -ibool -fts_optimize_index_fetch_node( -/*==========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ib_vector_t */ -{ - fts_word_t* word; - sel_node_t* sel_node = static_cast(row); - fts_fetch_t* fetch = static_cast(user_arg); - ib_vector_t* words = static_cast(fetch->read_arg); - que_node_t* exp = sel_node->select_list; - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint dfield_len = dfield_get_len(dfield); - fts_node_t* node; - bool is_word_init = false; - - ut_a(dfield_len <= FTS_MAX_WORD_LEN); - - if (ib_vector_size(words) == 0) { - - word = static_cast(ib_vector_push(words, NULL)); - fts_word_init(word, (byte*) data, dfield_len); - is_word_init = true; - } - - word = static_cast(ib_vector_last(words)); - - if (dfield_len != word->text.f_len - || memcmp(word->text.f_str, data, dfield_len)) { - - word = static_cast(ib_vector_push(words, NULL)); - fts_word_init(word, (byte*) data, dfield_len); - is_word_init = true; - } - - node = fts_optimize_read_node(word, que_node_get_next(exp)); - - fetch->total_memory += node->ilist_size; - if (is_word_init) { - fetch->total_memory += sizeof(fts_word_t) - + sizeof(ib_alloc_t) + sizeof(ib_vector_t) + dfield_len - + sizeof(fts_node_t) * FTS_WORD_NODES_INIT_SIZE; - } else if (ib_vector_size(words) > FTS_WORD_NODES_INIT_SIZE) { - fetch->total_memory += sizeof(fts_node_t); - } - - if (fetch->total_memory >= fts_result_cache_limit) { - return(FALSE); - } - - return(TRUE); -} - -/**********************************************************************//** -Read the rows from the FTS inde. -@return DB_SUCCESS or error code */ -dberr_t -fts_index_fetch_nodes( -/*==================*/ - trx_t* trx, /*!< in: transaction */ - que_t** graph, /*!< in: prepared statement */ - fts_table_t* fts_table, /*!< in: table of the FTS INDEX */ - const fts_string_t* - word, /*!< in: the word to fetch */ - fts_fetch_t* fetch) /*!< in: fetch callback.*/ +dberr_t fts_index_fetch_nodes(FTSQueryExecutor *executor, dict_index_t *index, + const fts_string_t *word, void *user_arg, + FTSRecordProcessor processor, + AuxCompareMode compare_mode) noexcept { - pars_info_t* info; - dberr_t error; - char table_name[MAX_FULL_NAME_LEN]; - - trx->op_info = "fetching FTS index nodes"; - - if (*graph) { - info = (*graph)->info; - } else { - ulint selected; - - info = pars_info_create(); - - ut_a(fts_table->type == FTS_INDEX_TABLE); - - selected = fts_select_index(fts_table->charset, - word->f_str, word->f_len); - - fts_table->suffix = fts_get_suffix(selected); - - fts_get_table_name(fts_table, table_name); - - pars_info_bind_id(info, "table_name", table_name); - } - - pars_info_bind_function(info, "my_func", fetch->read_record, fetch); - pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len); - - if (!*graph) { - - *graph = fts_parse_sql( - fts_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT word, doc_count, first_doc_id, last_doc_id," - " ilist\n" - " FROM $table_name\n" - " WHERE word LIKE :word\n" - " ORDER BY first_doc_id;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - } - - for (;;) { - error = fts_eval_sql(trx, *graph); - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - - break; /* Exit the loop. */ - } else { - fts_sql_rollback(trx); - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading" - " FTS index. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << "(" << error - << ") while reading FTS index."; - - break; /* Exit the loop. */ - } - } - } - - return(error); + dberr_t error= DB_SUCCESS; + trx_t *trx= executor->trx(); + trx->op_info= "fetching FTS index nodes"; + CHARSET_INFO *cs= fts_index_get_charset(index); + uint8_t selected= fts_select_index(cs, word->f_str, word->f_len); + ulint total_memory= 0; + for (;;) + { + AuxRecordReader reader= processor + ? AuxRecordReader(user_arg, processor, compare_mode) + : AuxRecordReader(user_arg, &total_memory, compare_mode); + error= executor->read_aux(selected, word, reader); + if (UNIV_LIKELY(error == DB_SUCCESS || error == DB_RECORD_NOT_FOUND)) + { + if (error == DB_RECORD_NOT_FOUND) error = DB_SUCCESS; + fts_sql_commit(trx); + break; + } + else + { + fts_sql_rollback(trx); + if (error == DB_LOCK_WAIT_TIMEOUT) + { + sql_print_warning("InnoDB: Lock wait timeout reading FTS index." + "Retrying!"); + trx->error_state= DB_SUCCESS; + } + else + { + sql_print_error("InnoDB: (%s) while reading FTS index.", + ut_strerr(error)); + break; + } + } + } + return error; } /**********************************************************************//** @@ -662,88 +442,6 @@ fts_zip_read_word( return(zip->status == Z_OK || zip->status == Z_STREAM_END ? ptr : NULL); } -/**********************************************************************//** -Callback function to fetch and compress the word in an FTS -INDEX record. -@return FALSE on EOF */ -static -ibool -fts_fetch_index_words( -/*==================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ib_vector_t */ -{ - sel_node_t* sel_node = static_cast(row); - fts_zip_t* zip = static_cast(user_arg); - que_node_t* exp = sel_node->select_list; - dfield_t* dfield = que_node_get_val(exp); - - ut_a(dfield_get_len(dfield) <= FTS_MAX_WORD_LEN); - - uint16 len = uint16(dfield_get_len(dfield)); - void* data = dfield_get_data(dfield); - - /* Skip the duplicate words. */ - if (zip->word.f_len == len && !memcmp(zip->word.f_str, data, len)) { - return(TRUE); - } - - memcpy(zip->word.f_str, data, len); - zip->word.f_len = len; - - ut_a(zip->zp->avail_in == 0); - ut_a(zip->zp->next_in == NULL); - - /* The string is prefixed by len. */ - /* FIXME: This is not byte order agnostic (InnoDB data files - with FULLTEXT INDEX are not portable between little-endian and - big-endian systems!) */ - zip->zp->next_in = reinterpret_cast(&len); - zip->zp->avail_in = sizeof(len); - - /* Compress the word, create output blocks as necessary. */ - while (zip->zp->avail_in > 0) { - - /* No space left in output buffer, create a new one. */ - if (zip->zp->avail_out == 0) { - byte* block; - - block = static_cast( - ut_malloc_nokey(zip->block_sz)); - - ib_vector_push(zip->blocks, &block); - - zip->zp->next_out = block; - zip->zp->avail_out = static_cast(zip->block_sz); - } - - switch (zip->status = deflate(zip->zp, Z_NO_FLUSH)) { - case Z_OK: - if (zip->zp->avail_in == 0) { - zip->zp->next_in = static_cast(data); - zip->zp->avail_in = uInt(len); - ut_a(len <= FTS_MAX_WORD_LEN); - len = 0; - } - continue; - - case Z_STREAM_END: - case Z_BUF_ERROR: - case Z_STREAM_ERROR: - default: - ut_error; - } - } - - /* All data should have been compressed. */ - ut_a(zip->zp->avail_in == 0); - zip->zp->next_in = NULL; - - ++zip->n_words; - - return(zip->n_words >= zip->max_words ? FALSE : TRUE); -} - /**********************************************************************//** Finish Zip deflate. */ static @@ -786,241 +484,181 @@ fts_zip_deflate_end( memset(zip->zp, 0, sizeof(*zip->zp)); } -/**********************************************************************//** -Read the words from the FTS INDEX. +/** Read the words from the FTS INDEX. +@param executor query executor +@param optim optimize scratch pad +@param word get words gerater than this +@param n_words max words to read @return DB_SUCCESS if all OK, DB_TABLE_NOT_FOUND if no more indexes to search else error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_index_fetch_words( -/*==================*/ - fts_optimize_t* optim, /*!< in: optimize scratch pad */ - const fts_string_t* word, /*!< in: get words greater than this - word */ - ulint n_words)/*!< in: max words to read */ +dberr_t fts_index_fetch_words(FTSQueryExecutor *executor, + fts_optimize_t *optim, + const fts_string_t *word, + ulint n_words) noexcept { - pars_info_t* info; - que_t* graph; - ulint selected; - fts_zip_t* zip = NULL; - dberr_t error = DB_SUCCESS; - mem_heap_t* heap = static_cast(optim->self_heap->arg); - ibool inited = FALSE; + dberr_t error= DB_SUCCESS; + mem_heap_t *heap= static_cast(optim->self_heap->arg); + optim->trx->op_info= "fetching FTS index words"; - optim->trx->op_info = "fetching FTS index words"; + if (optim->zip == NULL) + optim->zip = fts_zip_create(heap, FTS_ZIP_BLOCK_SIZE, n_words); + else fts_zip_initialize(optim->zip); - if (optim->zip == NULL) { - optim->zip = fts_zip_create(heap, FTS_ZIP_BLOCK_SIZE, n_words); - } else { - fts_zip_initialize(optim->zip); - } + CHARSET_INFO *cs= fts_index_get_charset(optim->index); - for (selected = fts_select_index( - optim->fts_index_table.charset, word->f_str, word->f_len); - selected < FTS_NUM_AUX_INDEX; - selected++) { - - char table_name[MAX_FULL_NAME_LEN]; - - optim->fts_index_table.suffix = fts_get_suffix(selected); - - info = pars_info_create(); - - pars_info_bind_function( - info, "my_func", fts_fetch_index_words, optim->zip); - - pars_info_bind_varchar_literal( - info, "word", word->f_str, word->f_len); - - fts_get_table_name(&optim->fts_index_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - &optim->fts_index_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT word\n" - " FROM $table_name\n" - " WHERE word > :word\n" - " ORDER BY word;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - zip = optim->zip; - - for (;;) { - int err; - - if (!inited && ((err = deflateInit(zip->zp, 9)) - != Z_OK)) { - ib::error() << "ZLib deflateInit() failed: " - << err; - - error = DB_ERROR; - break; - } else { - inited = TRUE; - error = fts_eval_sql(optim->trx, graph); - } + /* Create compression processor with state */ + bool compress_inited = false; + auto compress_processor= [&compress_inited]( + const rec_t *rec, const dict_index_t *index, + const rec_offs *offsets, void *user_arg) -> dberr_t + { + fts_zip_t* zip= static_cast(user_arg); - if (UNIV_LIKELY(error == DB_SUCCESS)) { - //FIXME fts_sql_commit(optim->trx); - break; - } else { - //FIXME fts_sql_rollback(optim->trx); + /* Extract word field using rec_get_nth_field() */ + ulint word_len; + const byte* word_data= rec_get_nth_field(rec, offsets, 0, &word_len); - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "Lock wait timeout" - " reading document. Retrying!"; + if (!word_data || word_len == UNIV_SQL_NULL || word_len > FTS_MAX_WORD_LEN) + return DB_SUCCESS; - /* We need to reset the ZLib state. */ - inited = FALSE; - deflateEnd(zip->zp); - fts_zip_init(zip); + /* Skip duplicate words */ + if (zip->word.f_len == word_len && + !memcmp(zip->word.f_str, word_data, word_len)) + return DB_SUCCESS; - optim->trx->error_state = DB_SUCCESS; - } else { - ib::error() << "(" << error - << ") while reading document."; + /* Initialize deflate if not done yet */ + if (!compress_inited) + { + int err = deflateInit(zip->zp, 9); + if (err != Z_OK) + { + sql_print_error("InnoDB: ZLib deflateInit() failed: %d", err); + return DB_ERROR; + } + compress_inited = true; + } - break; /* Exit the loop. */ - } - } - } + /* Update current word */ + memcpy(zip->word.f_str, word_data, word_len); + zip->word.f_len = word_len; + ut_a(zip->zp->avail_in == 0); + ut_a(zip->zp->next_in == NULL); - que_graph_free(graph); + /* Compress the word with length prefix */ + uint16_t len = static_cast(word_len); + zip->zp->next_in = reinterpret_cast(&len); + zip->zp->avail_in = sizeof(len); - /* Check if max word to fetch is exceeded */ - if (optim->zip->n_words >= n_words) { - break; - } - } + /* Compress the word, create output blocks as necessary */ + while (zip->zp->avail_in > 0) + { + /* No space left in output buffer, create a new one */ + if (zip->zp->avail_out == 0) + { + byte* block= static_cast(ut_malloc_nokey(zip->block_sz)); + ib_vector_push(zip->blocks, &block); + zip->zp->next_out= block; + zip->zp->avail_out= static_cast(zip->block_sz); + } + + switch (zip->status = deflate(zip->zp, Z_NO_FLUSH)) + { + case Z_OK: + if (zip->zp->avail_in == 0) + { + zip->zp->next_in= const_cast(word_data); + zip->zp->avail_in = static_cast(len); + ut_a(len <= FTS_MAX_WORD_LEN); + len = 0; + } + continue; + case Z_STREAM_END: + case Z_BUF_ERROR: + case Z_STREAM_ERROR: + default: + ut_error; + } + } - if (error == DB_SUCCESS && zip->status == Z_OK && zip->n_words > 0) { + /* All data should have been compressed */ + ut_a(zip->zp->avail_in == 0); + zip->zp->next_in = NULL; - /* All data should have been read. */ - ut_a(zip->zp->avail_in == 0); + ++zip->n_words; - fts_zip_deflate_end(zip); - } else { - deflateEnd(zip->zp); - } + /* Continue until we reach max words */ + return zip->n_words < zip->max_words ? DB_SUCCESS : DB_SUCCESS_LOCKED_REC; + }; - return(error); -} + for (uint8_t selected= fts_select_index(cs, word->f_str, word->f_len); + selected < FTS_NUM_AUX_INDEX; selected++) + { + for (;;) + { + AuxRecordReader aux_reader(optim->zip, compress_processor, + AuxCompareMode::GREATER); + + error= executor->read_aux(selected, word, aux_reader); + if (UNIV_LIKELY(error == DB_SUCCESS || error == DB_RECORD_NOT_FOUND)) { + if (error == DB_RECORD_NOT_FOUND) error = DB_SUCCESS; + break; + } + else + { + if (error == DB_LOCK_WAIT_TIMEOUT) + { + sql_print_warning("InnoDB: Lock wait timeout reading " + "words. Retrying!"); + if (compress_inited) + { + deflateEnd(optim->zip->zp); + fts_zip_init(optim->zip); + compress_inited= false; + } + optim->trx->error_state = DB_SUCCESS; + } + else + { + sql_print_error("InnoDB: (%s) while reading words.", + ut_strerr(error)); + break; + } + } + } -/**********************************************************************//** -Callback function to fetch the doc id from the record. -@return always returns TRUE */ -static -ibool -fts_fetch_doc_ids( -/*==============*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ib_vector_t */ -{ - que_node_t* exp; - int i = 0; - sel_node_t* sel_node = static_cast(row); - fts_doc_ids_t* fts_doc_ids = static_cast(user_arg); - doc_id_t* update = static_cast( - ib_vector_push(fts_doc_ids->doc_ids, NULL)); - - for (exp = sel_node->select_list; - exp; - exp = que_node_get_next(exp), ++i) { - - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - - ut_a(len != UNIV_SQL_NULL); - - /* Note: The column numbers below must match the SELECT. */ - switch (i) { - case 0: /* DOC_ID */ - *update = fts_read_doc_id( - static_cast(data)); - break; + if (optim->zip->n_words >= n_words) break; + } - default: - ut_error; - } - } + fts_zip_t *zip = optim->zip; + if (error == DB_SUCCESS && zip->status == Z_OK && zip->n_words > 0) { + /* All data should have been read */ + ut_a(zip->zp->avail_in == 0); + fts_zip_deflate_end(zip); + } + else deflateEnd(zip->zp); - return(TRUE); + return error; } -/**********************************************************************//** -Read the rows from a FTS common auxiliary table. -@return DB_SUCCESS or error code */ -dberr_t -fts_table_fetch_doc_ids( -/*====================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: table */ - fts_doc_ids_t* doc_ids) /*!< in: For collecting doc ids */ +dberr_t fts_table_fetch_doc_ids(FTSQueryExecutor *executor, + dict_table_t *table, const char *tbl_name, + fts_doc_ids_t *doc_ids) noexcept { - dberr_t error; - que_t* graph; - pars_info_t* info = pars_info_create(); - ibool alloc_bk_trx = FALSE; - char table_name[MAX_FULL_NAME_LEN]; - - ut_a(fts_table->suffix != NULL); - ut_a(fts_table->type == FTS_COMMON_TABLE); - - if (!trx) { - trx = trx_create(); - alloc_bk_trx = TRUE; - } - - trx->op_info = "fetching FTS doc ids"; - - pars_info_bind_function(info, "my_func", fts_fetch_doc_ids, doc_ids); - - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT doc_id FROM $table_name;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - error = fts_eval_sql(trx, graph); - fts_sql_commit(trx); - que_graph_free(graph); - - if (error == DB_SUCCESS) { - fts_doc_ids_sort(doc_ids->doc_ids); - } - - if (alloc_bk_trx) { - trx->free(); - } + ut_ad(executor != nullptr); + executor->trx()->op_info = "fetching FTS doc ids"; + CommonTableReader reader; + dberr_t err= executor->read_all_common(tbl_name, reader); - return(error); + if (err == DB_SUCCESS) + { + const auto& doc_id_vector= reader.get_doc_ids(); + for (doc_id_t doc_id : doc_id_vector) + ib_vector_push(doc_ids->doc_ids, &doc_id); + fts_doc_ids_sort(doc_ids->doc_ids); + } + + return err; } /**********************************************************************//** @@ -1419,87 +1057,53 @@ fts_optimize_word( return(nodes); } -/**********************************************************************//** -Update the FTS index table. This is a delete followed by an insert. +/** Write the words and ilist to disk. +@param executor query executor +@param index fulltext index +@param word word to update +@param nodes nodes to update @return DB_SUCCESS or error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_optimize_write_word( -/*====================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: table of FTS index */ - fts_string_t* word, /*!< in: word data to write */ - ib_vector_t* nodes) /*!< in: the nodes to write */ +dberr_t fts_optimize_write_word(FTSQueryExecutor *executor, dict_index_t *index, + fts_string_t *word, ib_vector_t *nodes) noexcept { - ulint i; - pars_info_t* info; - que_t* graph; - ulint selected; - dberr_t error = DB_SUCCESS; - char table_name[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - ut_ad(fts_table->charset); - - pars_info_bind_varchar_literal( - info, "word", word->f_str, word->f_len); - - selected = fts_select_index(fts_table->charset, - word->f_str, word->f_len); - - fts_table->suffix = fts_get_suffix(selected); - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, - info, - "BEGIN DELETE FROM $table_name WHERE word = :word;"); - - error = fts_eval_sql(trx, graph); - - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ") during optimize," - " when deleting a word from the FTS index."; - } - - que_graph_free(graph); - graph = NULL; - - /* Even if the operation needs to be rolled back and redone, - we iterate over the nodes in order to free the ilist. */ - for (i = 0; i < ib_vector_size(nodes); ++i) { - - fts_node_t* node = (fts_node_t*) ib_vector_get(nodes, i); - - if (error == DB_SUCCESS) { - /* Skip empty node. */ - if (node->ilist == NULL) { - ut_ad(node->ilist_size == 0); - continue; - } - - error = fts_write_node( - trx, &graph, fts_table, word, node); + CHARSET_INFO *cs= fts_index_get_charset(index); + uint8_t selected= fts_select_index(cs, word->f_str, word->f_len); + fts_aux_data_t aux_data((const char*)word->f_str, word->f_len); + dberr_t err= executor->delete_aux_record(selected, &aux_data); + if (err != DB_SUCCESS) + { + sql_print_error("InnoDB: (%s) during optimize, when " + "deleting a word from the FTS index.", + ut_strerr(err)); + return err; + } - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ")" - " during optimize, while adding a" - " word to the FTS index."; - } - } + for (ulint i = 0; i < ib_vector_size(nodes); ++i) + { + fts_node_t* node = (fts_node_t*) ib_vector_get(nodes, i); + if (!node->ilist || node->ilist_size == 0) continue; - ut_free(node->ilist); - node->ilist = NULL; - node->ilist_size = node->ilist_size_alloc = 0; - } + fts_aux_data_t insert_data( + (const char*)word->f_str, word->f_len, + node->first_doc_id, node->last_doc_id, + static_cast(node->doc_count), node->ilist, + node->ilist_size); - if (graph != NULL) { - que_graph_free(graph); - } + err = executor->insert_aux_record(selected, &insert_data); + if (err != DB_SUCCESS) + { + sql_print_error("InnoDB: (%s) during optimize, when " + "inserting a word to the FTS index.", + ut_strerr(err)); + return err; + } + ut_free(node->ilist); + node->ilist= nullptr; + node->ilist_size= node->ilist_size_alloc= 0; + } - return(error); + return DB_SUCCESS; } /**********************************************************************//** @@ -1519,15 +1123,17 @@ fts_word_free( } /**********************************************************************//** -Optimize the word ilist and rewrite data to the FTS index. +Compact the nodes for a given word, the nodes passed in are +already optimized. @return status one of RESTART, EXIT, ERROR */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_optimize_compact( /*=================*/ + FTSQueryExecutor* executor, /*!< in: query executor */ fts_optimize_t* optim, /*!< in: optimize state data */ dict_index_t* index, /*!< in: current FTS being optimized */ - time_t start_time) /*!< in: optimize start time */ + time_t start_time) noexcept /*!< in: optimize start time */ { ulint i; dberr_t error = DB_SUCCESS; @@ -1536,7 +1142,6 @@ fts_optimize_compact( for (i = 0; i < size && error == DB_SUCCESS && !optim->done; ++i) { fts_word_t* word; ib_vector_t* nodes; - trx_t* trx = optim->trx; word = (fts_word_t*) ib_vector_get(optim->words, i); @@ -1547,13 +1152,13 @@ fts_optimize_compact( /* Update the data on disk. */ error = fts_optimize_write_word( - trx, &optim->fts_index_table, &word->text, nodes); + executor, index, &word->text, nodes); if (error == DB_SUCCESS) { /* Write the last word optimized to the config table, we use this value for restarting optimize. */ error = fts_config_set_index_value( - optim->trx, index, + executor, index, FTS_LAST_OPTIMIZED_WORD, &word->text); } @@ -1599,18 +1204,10 @@ fts_optimize_create( optim->trx = trx_create(); trx_start_internal(optim->trx); - optim->fts_common_table.table_id = table->id; - optim->fts_common_table.type = FTS_COMMON_TABLE; - optim->fts_common_table.table = table; - - optim->fts_index_table.table_id = table->id; - optim->fts_index_table.type = FTS_INDEX_TABLE; - optim->fts_index_table.table = table; - /* The common prefix for all this parent table's aux tables. */ char table_id[FTS_AUX_MIN_TABLE_ID_LENGTH]; const size_t table_id_len = 1 - + size_t(fts_get_table_id(&optim->fts_common_table, table_id)); + + size_t(fts_write_object_id(table->id, table_id)); dict_sys.freeze(SRW_LOCK_CALL); /* Include the separator as well. */ const size_t dbname_len = table->name.dblen() + 1; @@ -1680,175 +1277,136 @@ fts_optimize_free( mem_heap_free(heap); } -/**********************************************************************//** -Get the max time optimize should run in millisecs. +/** Get the max time optimize should run in millisecs. +@param executor query executor +@param table user table to be optimized @return max optimize time limit in millisecs. */ static -ulint -fts_optimize_get_time_limit( -/*========================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table) /*!< in: aux table */ +ulint fts_optimize_get_time_limit(FTSQueryExecutor *executor, + const dict_table_t *table) noexcept { - ulint time_limit = 0; - - fts_config_get_ulint( - trx, fts_table, - FTS_OPTIMIZE_LIMIT_IN_SECS, &time_limit); - - /* FIXME: This is returning milliseconds, while the variable - is being stored and interpreted as seconds! */ - return(time_limit * 1000); + ulint time_limit= 0; + fts_string_t value; + value.f_len= FTS_MAX_CONFIG_VALUE_LEN; + value.f_str= static_cast(ut_malloc_nokey(value.f_len + 1)); + dberr_t error= fts_config_get_value(executor, table, + FTS_OPTIMIZE_LIMIT_IN_SECS, &value); + if (error == DB_SUCCESS) + time_limit= strtoul(reinterpret_cast(value.f_str), nullptr, 10); + ut_free(value.f_str); + /* FIXME: This is returning milliseconds, while the variable + is being stored and interpreted as seconds! */ + return(time_limit * 1000); } -/**********************************************************************//** -Run OPTIMIZE on the given table. Note: this can take a very long time -(hours). */ +/** Run OPTIMIZE on the given table. Note: this can take a very +long time (hours). +@param executor query executor +@param optim optimize instance +@param index current fts being optimized +@param word starting word to optimize */ static -void -fts_optimize_words( -/*===============*/ - fts_optimize_t* optim, /*!< in: optimize instance */ - dict_index_t* index, /*!< in: current FTS being optimized */ - fts_string_t* word) /*!< in: the starting word to optimize */ +void fts_optimize_words(FTSQueryExecutor *executor, fts_optimize_t *optim, + dict_index_t *index, fts_string_t *word) noexcept { - fts_fetch_t fetch; - que_t* graph = NULL; - CHARSET_INFO* charset = optim->fts_index_table.charset; - - ut_a(!optim->done); - - /* Get the time limit from the config table. */ - fts_optimize_time_limit = fts_optimize_get_time_limit( - optim->trx, &optim->fts_common_table); - - const time_t start_time = time(NULL); - - /* Setup the callback to use for fetching the word ilist etc. */ - fetch.read_arg = optim->words; - fetch.read_record = fts_optimize_index_fetch_node; - - while (!optim->done) { - dberr_t error; - trx_t* trx = optim->trx; - ulint selected; - - ut_a(ib_vector_size(optim->words) == 0); - - selected = fts_select_index(charset, word->f_str, word->f_len); - - /* Read the index records to optimize. */ - fetch.total_memory = 0; - error = fts_index_fetch_nodes( - trx, &graph, &optim->fts_index_table, word, - &fetch); - ut_ad(fetch.total_memory < fts_result_cache_limit); - - if (error == DB_SUCCESS) { - /* There must be some nodes to read. */ - ut_a(ib_vector_size(optim->words) > 0); - - /* Optimize the nodes that were read and write - back to DB. */ - error = fts_optimize_compact(optim, index, start_time); - - if (error == DB_SUCCESS) { - fts_sql_commit(optim->trx); - } else { - fts_sql_rollback(optim->trx); - } - } - - ib_vector_reset(optim->words); - - if (error == DB_SUCCESS) { - if (!optim->done) { - if (!fts_zip_read_word(optim->zip, word)) { - optim->done = TRUE; - } else if (selected - != fts_select_index( - charset, word->f_str, - word->f_len) - && graph) { - que_graph_free(graph); - graph = NULL; - } - } - } else if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "Lock wait timeout during optimize." - " Retrying!"; + ut_a(!optim->done); + /* Get the time limit from the config table. */ + fts_optimize_time_limit= + fts_optimize_get_time_limit(executor, index->table); + const time_t start_time= time(NULL); - trx->error_state = DB_SUCCESS; - } else if (error == DB_DEADLOCK) { - ib::warn() << "Deadlock during optimize. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - optim->done = TRUE; /* Exit the loop. */ - } - } + while (!optim->done) + { + trx_t *trx= optim->trx; + ut_a(ib_vector_size(optim->words) == 0); + /* Read the index records to optimize. */ + dberr_t error= fts_index_fetch_nodes( + executor, index, word, optim->words, nullptr, AuxCompareMode::EQUAL); + if (error == DB_SUCCESS) + { + /* There must be some nodes to read. */ + ut_a(ib_vector_size(optim->words) > 0); + /* Optimize the nodes that were read and write back to DB. */ + error = fts_optimize_compact(executor, optim, index, start_time); + if (error == DB_SUCCESS) fts_sql_commit(optim->trx); + else fts_sql_rollback(optim->trx); + } + ib_vector_reset(optim->words); - if (graph != NULL) { - que_graph_free(graph); - } + if (error == DB_SUCCESS) + { + if (!optim->done && !fts_zip_read_word(optim->zip, word)) + optim->done= TRUE; + } + else if (error == DB_LOCK_WAIT_TIMEOUT) + { + sql_print_warning("InnoDB: Lock wait timeout during optimize. " + "Retrying!"); + trx->error_state= DB_SUCCESS; + } + else if (error == DB_DEADLOCK) + { + sql_print_warning("InnoDB: Deadlock during optimize. Retrying!"); + trx->error_state = DB_SUCCESS; + } + else optim->done = TRUE; + } } -/**********************************************************************//** -Optimize is complete. Set the completion time, and reset the optimize -start string for this FTS index to "". +/** Optimize is complete. Set the completion time, and reset the +optimize start string for this FTS index to "". +@param executor query executor +@param optim optimize instance +@param index table with one FTS index @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t -fts_optimize_index_completed( -/*=========================*/ - fts_optimize_t* optim, /*!< in: optimize instance */ - dict_index_t* index) /*!< in: table with one FTS index */ +fts_optimize_index_completed(FTSQueryExecutor *executor, + fts_optimize_t *optim, + dict_index_t *index) noexcept { - fts_string_t word; - dberr_t error; - byte buf[sizeof(ulint)]; - - /* If we've reached the end of the index then set the start - word to the empty string. */ - - word.f_len = 0; - word.f_str = buf; - *word.f_str = '\0'; - - error = fts_config_set_index_value( - optim->trx, index, FTS_LAST_OPTIMIZED_WORD, &word); - - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ") while updating" - " last optimized word!"; - } - - return(error); + fts_string_t word; + dberr_t error= DB_SUCCESS; + byte buf[sizeof(ulint)]; + /* If we've reached the end of the index then set the start + word to the empty string. */ + word.f_len= 0; + word.f_str= buf; + *word.f_str= '\0'; + + error= fts_config_set_index_value( + executor, index, FTS_LAST_OPTIMIZED_WORD, &word); + + if (UNIV_UNLIKELY(error != DB_SUCCESS)) + sql_print_error("InnoDB: (%s) while updating last optimized word!", + ut_strerr(error)); + return error; } -/**********************************************************************//** -Read the list of words from the FTS auxiliary index that will be -optimized in this pass. +/** Read the words that will be optimized in this pass. +@param executor query executor +@param optim optimize instance +@param index table with one FTS index +@param word buffer to use @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_optimize_index_read_words( -/*==========================*/ - fts_optimize_t* optim, /*!< in: optimize instance */ - dict_index_t* index, /*!< in: table with one FTS index */ - fts_string_t* word) /*!< in: buffer to use */ + FTSQueryExecutor* executor, + fts_optimize_t* optim, + dict_index_t* index, + fts_string_t* word) { dberr_t error = DB_SUCCESS; if (optim->del_list_regenerated) { word->f_len = 0; } else { - /* Get the last word that was optimized from the config table. */ error = fts_config_get_index_value( - optim->trx, index, FTS_LAST_OPTIMIZED_WORD, word); + executor, index, FTS_LAST_OPTIMIZED_WORD, word); } /* If record not found then we start from the top. */ @@ -1857,10 +1415,11 @@ fts_optimize_index_read_words( error = DB_SUCCESS; } + optim->index = index; while (error == DB_SUCCESS) { error = fts_index_fetch_words( - optim, word, fts_num_word_optimize); + executor, optim, word, fts_num_word_optimize); if (error == DB_SUCCESS) { /* Reset the last optimized word to '' if no @@ -1885,17 +1444,14 @@ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_optimize_index( /*===============*/ - fts_optimize_t* optim, /*!< in: optimize instance */ - dict_index_t* index) /*!< in: table with one FTS index */ + FTSQueryExecutor* executor, /*!< in: query executor */ + fts_optimize_t* optim, /*!< in: optimize instance */ + dict_index_t* index) /*!< in: table with one FTS index */ { fts_string_t word; dberr_t error; byte str[FTS_MAX_WORD_LEN + 1]; - /* Set the current index that we have to optimize. */ - optim->fts_index_table.index_id = index->id; - optim->fts_index_table.charset = fts_index_get_charset(index); - optim->done = FALSE; /* Optimize until !done */ /* We need to read the last word optimized so that we start from @@ -1909,7 +1465,7 @@ fts_optimize_index( memset(word.f_str, 0x0, word.f_len); /* Read the words that will be optimized in this pass. */ - error = fts_optimize_index_read_words(optim, index, &word); + error = fts_optimize_index_read_words(executor, optim, index, &word); if (error == DB_SUCCESS) { int zip_error; @@ -1929,7 +1485,7 @@ fts_optimize_index( optim->done = TRUE; } else { - fts_optimize_words(optim, index, &word); + fts_optimize_words(executor, optim, index, &word); } /* If we couldn't read any records then optimize is @@ -1938,7 +1494,8 @@ fts_optimize_index( completed. */ if (error == DB_SUCCESS && optim->zip->n_words == 0) { - error = fts_optimize_index_completed(optim, index); + error = fts_optimize_index_completed( + executor, optim, index); if (error == DB_SUCCESS) { ++optim->n_completed; @@ -1949,190 +1506,99 @@ fts_optimize_index( return(error); } -/**********************************************************************//** -Delete the document ids in the delete, and delete cache tables. +/** Purge the doc ids that are in the snapshot from +the master deleted table. +@param executor query executor +@param optim optimize instance @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_optimize_purge_deleted_doc_ids( -/*===============================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ +dberr_t fts_optimize_purge_deleted_doc_ids(FTSQueryExecutor *executor, + fts_optimize_t *optim) noexcept { - ulint i; - pars_info_t* info; - que_t* graph; - doc_id_t* update; - doc_id_t write_doc_id; - dberr_t error = DB_SUCCESS; - char deleted[MAX_FULL_NAME_LEN]; - char deleted_cache[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - ut_a(ib_vector_size(optim->to_delete->doc_ids) > 0); - - update = static_cast( - ib_vector_get(optim->to_delete->doc_ids, 0)); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, *update); - - /* This is required for the SQL parser to work. It must be able - to find the following variables. So we do it twice. */ - fts_bind_doc_id(info, "doc_id1", &write_doc_id); - fts_bind_doc_id(info, "doc_id2", &write_doc_id); - - /* Make sure the following two names are consistent with the name - used in the fts_delete_doc_ids_sql */ - optim->fts_common_table.suffix = fts_common_tables[3]; - fts_get_table_name(&optim->fts_common_table, deleted); - pars_info_bind_id(info, fts_common_tables[3], deleted); - - optim->fts_common_table.suffix = fts_common_tables[4]; - fts_get_table_name(&optim->fts_common_table, deleted_cache); - pars_info_bind_id(info, fts_common_tables[4], deleted_cache); - - graph = fts_parse_sql(NULL, info, fts_delete_doc_ids_sql); - - /* Delete the doc ids that were copied at the start. */ - for (i = 0; i < ib_vector_size(optim->to_delete->doc_ids); ++i) { - - update = static_cast(ib_vector_get( - optim->to_delete->doc_ids, i)); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, *update); - - fts_bind_doc_id(info, "doc_id1", &write_doc_id); - - fts_bind_doc_id(info, "doc_id2", &write_doc_id); - - error = fts_eval_sql(optim->trx, graph); - - // FIXME: Check whether delete actually succeeded! - if (error != DB_SUCCESS) { - - fts_sql_rollback(optim->trx); - break; - } - } - - que_graph_free(graph); + dberr_t error= DB_SUCCESS; + ut_a(ib_vector_size(optim->to_delete->doc_ids) > 0); + for (ulint i= 0; + i < ib_vector_size(optim->to_delete->doc_ids) && error != DB_SUCCESS; + ++i) + { + doc_id_t *update= + static_cast(ib_vector_get(optim->to_delete->doc_ids, i)); + error= executor->delete_common_record("DELETED", *update); + if (error == DB_SUCCESS) + error= executor->delete_common_record("DELETED_CACHE", *update); + } - return(error); + if (error != DB_SUCCESS) + fts_sql_rollback(optim->trx); + return error; } -/**********************************************************************//** -Delete the document ids in the pending delete, and delete tables. +/** Delete the document ids in the pending delete, and delete tables. +@param executor query executor +@param optim optimize instance @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_optimize_purge_deleted_doc_id_snapshot( -/*=======================================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ +dberr_t fts_optimize_purge_deleted_doc_id_snapshot(FTSQueryExecutor *executor, + fts_optimize_t *optim) noexcept { - dberr_t error; - que_t* graph; - pars_info_t* info; - char being_deleted[MAX_FULL_NAME_LEN]; - char being_deleted_cache[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - /* Make sure the following two names are consistent with the name - used in the fts_end_delete_sql */ - optim->fts_common_table.suffix = fts_common_tables[0]; - fts_get_table_name(&optim->fts_common_table, being_deleted); - pars_info_bind_id(info, fts_common_tables[0], being_deleted); - - optim->fts_common_table.suffix = fts_common_tables[1]; - fts_get_table_name(&optim->fts_common_table, being_deleted_cache); - pars_info_bind_id(info, fts_common_tables[1], being_deleted_cache); - - /* Delete the doc ids that were copied to delete pending state at - the start of optimize. */ - graph = fts_parse_sql(NULL, info, fts_end_delete_sql); - - error = fts_eval_sql(optim->trx, graph); - que_graph_free(graph); - - return(error); + dberr_t error= executor->delete_all_common_records("BEING_DELETED"); + if (error == DB_SUCCESS) + error= executor->delete_all_common_records("BEING_DELETED_CACHE"); + return error; } -/**********************************************************************//** -Copy the deleted doc ids that will be purged during this optimize run -to the being deleted FTS auxiliary tables. The transaction is committed -upon successfull copy and rolled back on DB_DUPLICATE_KEY error. +/** Check if there are records in BEING_DELETED table +@param executor query executor +@param optim optimize fts instance +@param n_rows number of rows exist in being_deleted table @return DB_SUCCESS if all OK */ static -ulint -fts_optimize_being_deleted_count( -/*=============================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ +dberr_t fts_optimize_being_deleted_count(FTSQueryExecutor *executor, + fts_optimize_t *optim, + ulint *n_rows) noexcept { - fts_table_t fts_table; - - FTS_INIT_FTS_TABLE(&fts_table, "BEING_DELETED", FTS_COMMON_TABLE, - optim->table); - - return(fts_get_rows_count(&fts_table)); + CommonTableReader reader; + dberr_t err= executor->read_all_common("BEING_DELETED", reader); + if (err == DB_SUCCESS) *n_rows= reader.get_doc_ids().size(); + return err; } -/*********************************************************************//** -Copy the deleted doc ids that will be purged during this optimize run -to the being deleted FTS auxiliary tables. The transaction is committed -upon successfull copy and rolled back on DB_DUPLICATE_KEY error. -@return DB_SUCCESS if all OK */ +/** Create a snapshot of deleted document IDs by moving them from +DELETED to BEING_DELETED and from DELETED_CACHE to +BEING_DELETED_CACHE. +@param executor query executor +@param optim optimize fts instance +@return DB_SUCCESS or error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_optimize_create_deleted_doc_id_snapshot( -/*========================================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ +dberr_t fts_optimize_create_deleted_doc_id_snapshot(FTSQueryExecutor *executor, + fts_optimize_t *optim) noexcept { - dberr_t error; - que_t* graph; - pars_info_t* info; - char being_deleted[MAX_FULL_NAME_LEN]; - char deleted[MAX_FULL_NAME_LEN]; - char being_deleted_cache[MAX_FULL_NAME_LEN]; - char deleted_cache[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); + dberr_t err= DB_SUCCESS; + CommonTableReader reader; - /* Make sure the following four names are consistent with the name - used in the fts_init_delete_sql */ - optim->fts_common_table.suffix = fts_common_tables[0]; - fts_get_table_name(&optim->fts_common_table, being_deleted); - pars_info_bind_id(info, fts_common_tables[0], being_deleted); + err= executor->read_all_common("DELETED", reader); + if (err != DB_SUCCESS && err != DB_RECORD_NOT_FOUND) return err; - optim->fts_common_table.suffix = fts_common_tables[3]; - fts_get_table_name(&optim->fts_common_table, deleted); - pars_info_bind_id(info, fts_common_tables[3], deleted); - - optim->fts_common_table.suffix = fts_common_tables[1]; - fts_get_table_name(&optim->fts_common_table, being_deleted_cache); - pars_info_bind_id(info, fts_common_tables[1], being_deleted_cache); - - optim->fts_common_table.suffix = fts_common_tables[4]; - fts_get_table_name(&optim->fts_common_table, deleted_cache); - pars_info_bind_id(info, fts_common_tables[4], deleted_cache); - - /* Move doc_ids that are to be deleted to state being deleted. */ - graph = fts_parse_sql(NULL, info, fts_init_delete_sql); - - error = fts_eval_sql(optim->trx, graph); - - que_graph_free(graph); + const auto& deleted_doc_ids = reader.get_doc_ids(); + for (doc_id_t doc_id : deleted_doc_ids) + { + err= executor->insert_common_record("BEING_DELETED", doc_id); + if (err != DB_SUCCESS) return err; + } - if (error != DB_SUCCESS) { - fts_sql_rollback(optim->trx); - } else { - fts_sql_commit(optim->trx); - } + reader.clear(); + err= executor->read_all_common("DELETED_CACHE", reader); + if (err != DB_SUCCESS && err != DB_RECORD_NOT_FOUND) return err; - optim->del_list_regenerated = TRUE; + const auto& deleted_cache_doc_ids= reader.get_doc_ids(); + for (doc_id_t doc_id : deleted_cache_doc_ids) + { + err= executor->insert_common_record("BEING_DELETED_CACHE", doc_id); + if (err != DB_SUCCESS) return err; + } - return(error); + optim->del_list_regenerated= TRUE; + return err; } /*********************************************************************//** @@ -2143,44 +1609,38 @@ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_optimize_read_deleted_doc_id_snapshot( /*======================================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ + FTSQueryExecutor* executor, /*!< in: FTS query executor */ + fts_optimize_t* optim) noexcept /*!< in: optimize instance */ { - dberr_t error; - - optim->fts_common_table.suffix = "BEING_DELETED"; - /* Read the doc_ids to delete. */ - error = fts_table_fetch_doc_ids( - optim->trx, &optim->fts_common_table, optim->to_delete); + dberr_t error = fts_table_fetch_doc_ids( + executor, optim->table, "BEING_DELETED", + optim->to_delete); if (error == DB_SUCCESS) { - optim->fts_common_table.suffix = "BEING_DELETED_CACHE"; - /* Read additional doc_ids to delete. */ error = fts_table_fetch_doc_ids( - optim->trx, &optim->fts_common_table, optim->to_delete); + executor, optim->table, "BEING_DELETED_CACHE", + optim->to_delete); } if (error != DB_SUCCESS) { - fts_doc_ids_free(optim->to_delete); optim->to_delete = NULL; } - return(error); } /*********************************************************************//** -Optimize all the FTS indexes, skipping those that have already been -optimized, since the FTS auxiliary indexes are not guaranteed to be -of the same cardinality. +Optimize the FTS indexes of a table. @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_optimize_indexes( /*=================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ + FTSQueryExecutor* executor, /*!< in: query executor */ + fts_optimize_t* optim) noexcept /*!< in: optimize instance */ { ulint i; dberr_t error = DB_SUCCESS; @@ -2192,7 +1652,17 @@ fts_optimize_indexes( index = static_cast( ib_vector_getp(fts->indexes, i)); - error = fts_optimize_index(optim, index); + + /* Open auxiliary tables for this index */ + error = executor->open_all_aux_tables(index); + if (error != DB_SUCCESS) { + break; + } + + error = fts_optimize_index(executor, optim, index); + if (error != DB_SUCCESS) { + break; + } } if (error == DB_SUCCESS) { @@ -2211,17 +1681,19 @@ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_optimize_purge_snapshot( /*========================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ + FTSQueryExecutor* executor, /*!< in: query executor */ + fts_optimize_t* optim) noexcept /*!< in: optimize instance */ { dberr_t error; /* Delete the doc ids from the master deleted tables, that were in the snapshot that was taken at the start of optimize. */ - error = fts_optimize_purge_deleted_doc_ids(optim); + error = fts_optimize_purge_deleted_doc_ids(executor, optim); if (error == DB_SUCCESS) { /* Destroy the deleted doc id snapshot. */ - error = fts_optimize_purge_deleted_doc_id_snapshot(optim); + error = fts_optimize_purge_deleted_doc_id_snapshot( + executor, optim); } if (error == DB_SUCCESS) { @@ -2233,6 +1705,7 @@ fts_optimize_purge_snapshot( return(error); } + /*********************************************************************//** Run OPTIMIZE on the given table by a background thread. @return DB_SUCCESS if all OK */ @@ -2259,7 +1732,7 @@ fts_optimize_table_bk( if (table->is_accessible() && table->fts && table->fts->cache && table->fts->cache->deleted >= FTS_OPTIMIZE_THRESHOLD) { - error = fts_optimize_table(table); + error = fts_optimize_table(table, fts_opt_thd); slot->last_run = time(NULL); @@ -2275,13 +1748,12 @@ fts_optimize_table_bk( return(error); } -/*********************************************************************//** -Run OPTIMIZE on the given table. +/** Run OPTIMIZE on the given table. +@param table table to be optimized +@param thd thread which executes optimize table @return DB_SUCCESS if all OK */ dberr_t -fts_optimize_table( -/*===============*/ - dict_table_t* table) /*!< in: table to optimiza */ +fts_optimize_table(dict_table_t *table, THD *thd) { ut_ad(!srv_read_only_mode || recv_sys.rpo); @@ -2295,14 +1767,32 @@ fts_optimize_table( optim = fts_optimize_create(table); + optim->trx->mysql_thd = thd; + + /* Create FTSQueryExecutor and open common tables */ + FTSQueryExecutor executor(optim->trx, table); + error = executor.open_all_deletion_tables(); + if (error != DB_SUCCESS) { +err_exit: + fts_optimize_free(optim); + return error; + } + + error = executor.open_config_table(); + if (error) { goto err_exit; } + // FIXME: Call this only at the start of optimize, currently we // rely on DB_DUPLICATE_KEY to handle corrupting the snapshot. /* Check whether there are still records in BEING_DELETED table */ - if (fts_optimize_being_deleted_count(optim) == 0) { + ulint n_rows = 0; + error= fts_optimize_being_deleted_count(&executor, optim, &n_rows); + + if (error == DB_SUCCESS && n_rows == 0) { /* Take a snapshot of the deleted document ids, they are copied to the BEING_ tables. */ - error = fts_optimize_create_deleted_doc_id_snapshot(optim); + error = fts_optimize_create_deleted_doc_id_snapshot( + &executor, optim); } /* A duplicate error is OK, since we don't erase the @@ -2317,7 +1807,7 @@ fts_optimize_table( /* These document ids will be filtered out during the index optimization phase. They are in the snapshot that we took above, at the start of the optimize. */ - error = fts_optimize_read_deleted_doc_id_snapshot(optim); + error = fts_optimize_read_deleted_doc_id_snapshot(&executor, optim); if (error == DB_SUCCESS) { @@ -2328,7 +1818,7 @@ fts_optimize_table( /* We would do optimization only if there are deleted records to be cleaned up */ if (ib_vector_size(optim->to_delete->doc_ids) > 0) { - error = fts_optimize_indexes(optim); + error = fts_optimize_indexes(&executor, optim); } } else { @@ -2346,8 +1836,10 @@ fts_optimize_table( /* Purge the doc ids that were in the snapshot from the snapshot tables and the master deleted table. */ - error = fts_optimize_purge_snapshot(optim); + error = fts_optimize_purge_snapshot( + &executor, optim); } + } } @@ -2432,8 +1924,8 @@ fts_optimize_remove_table( if (fts_opt_start_shutdown) { - ib::info() << "Try to remove table " << table->name - << " after FTS optimize thread exiting."; + sql_print_information("InnoDB: Try to remove table %s after FTS optimize " + "thread exiting.", table->name.m_name); while (fts_optimize_wq) std::this_thread::sleep_for(std::chrono::milliseconds(10)); return; @@ -2472,8 +1964,9 @@ fts_optimize_request_sync_table( /* FTS optimizer thread is already exited */ if (fts_opt_start_shutdown) { - ib::info() << "Try to sync table " << table->name - << " after FTS optimize thread exiting."; + sql_print_information("InnoDB: Try to sync table %s " + "after FTS optimize thread exiting.", + table->name.m_name); } else if (table->fts->sync_message) { /* If the table already has SYNC message in fts_optimize_wq queue then ignore it */ @@ -2770,7 +2263,7 @@ static void fts_optimize_callback(void *) pthread_cond_broadcast(&fts_opt_shutdown_cond); mysql_mutex_unlock(&fts_optimize_wq->mutex); - ib::info() << "FTS optimize thread exiting."; + sql_print_information("InnoDB: FTS optimize thread exiting."); } /**********************************************************************//** diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc index 0faf23aa6e97b..d921b34a9ce13 100644 --- a/storage/innobase/fts/fts0que.cc +++ b/storage/innobase/fts/fts0que.cc @@ -35,6 +35,7 @@ Completed 2011/7/10 Sunny and Jimmy Yang #include "fts0types.h" #include "fts0plugin.h" #include "fts0vlc.h" +#include "fts0exec.h" #include #include @@ -67,9 +68,6 @@ struct fts_query_t { trx_t* trx; /*!< The query transaction */ dict_index_t* index; /*!< The FTS index to search */ - /*!< FTS auxiliary common table def */ - - fts_table_t fts_common_table; fts_table_t fts_index_table;/*!< FTS auxiliary index table def */ @@ -149,6 +147,8 @@ struct fts_query_t { fts_ast_visit_sub_exp() */ st_mysql_ftparser* parser; /*!< fts plugin parser */ + + FTSQueryExecutor* executor; /*!< shared FTS query executor */ }; /** For phrase matching, first we collect the documents and the positions @@ -270,16 +270,6 @@ struct fts_word_freq_t { double idf; /*!< Inverse document frequency */ }; -/******************************************************************** -Callback function to fetch the rows in an FTS INDEX record. -@return always TRUE */ -static -ibool -fts_query_index_fetch_nodes( -/*========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg); /*!< in: pointer to ib_vector_t */ - /******************************************************************** Read and filter nodes. @return fts_node_t instance */ @@ -310,19 +300,155 @@ fts_ast_visit_sub_exp( fts_ast_callback visitor, void* arg); -#if 0 -/*****************************************************************//*** -Find a doc_id in a word's ilist. -@return TRUE if found. */ -static -ibool -fts_query_find_doc_id( -/*==================*/ - fts_select_t* select, /*!< in/out: search the doc id selected, - update the frequency if found. */ - void* data, /*!< in: doc id ilist */ - ulint len); /*!< in: doc id ilist size */ -#endif +/** Process query records for FTS queries. +@param rec record +@param index index +@param offsets record offsets +@param user_arg user argument +@return DB_SUCCESS to continue processing, DB_SUCCESS_LOCKED_REC to stop, or error code */ +static dberr_t node_query_processor( + const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets, void* user_arg) +{ + fts_query_t* query= static_cast(user_arg); + + /* Extract fields using rec_get_nth_field() */ + ulint word_len; + const byte* word_data= rec_get_nth_field(rec, offsets, 0, &word_len); + + if (!word_data || word_len == UNIV_SQL_NULL || word_len > FTS_MAX_WORD_LEN) + return DB_SUCCESS; + + ulint first_doc_id_len; + const byte* first_doc_id_data= rec_get_nth_field(rec, offsets, 1, &first_doc_id_len); + doc_id_t first_doc_id= fts_read_doc_id(first_doc_id_data); + + ulint last_doc_id_len; + const byte* last_doc_id_data= rec_get_nth_field(rec, offsets, 4, &last_doc_id_len); + doc_id_t last_doc_id= fts_read_doc_id(last_doc_id_data); + + ulint doc_count_len; + const byte* doc_count_data= rec_get_nth_field(rec, offsets, 5, &doc_count_len); + ulint doc_count= mach_read_from_4(doc_count_data); + + ulint ilist_len; + const byte* ilist_data= rec_get_nth_field(rec, offsets, 6, &ilist_len); + + /* Check if ilist is stored externally */ + bool ilist_external= rec_offs_nth_extern(offsets, 6); + mem_heap_t* ilist_heap= nullptr; + byte* external_ilist= nullptr; + + if (ilist_external && ilist_data && ilist_len != UNIV_SQL_NULL) + { + ilist_heap= mem_heap_create(ilist_len + 1000); + ulint external_len; + external_ilist= btr_copy_externally_stored_field( + &external_len, ilist_data, index->table->space->zip_size(), + ilist_len, ilist_heap); + if (external_ilist) + { + ilist_data= external_ilist; + ilist_len= external_len; + } + else + { + if (ilist_heap) mem_heap_free(ilist_heap); + return DB_SUCCESS; + } + } + + ut_a(query->cur_node->type == FTS_AST_TERM + || query->cur_node->type == FTS_AST_TEXT + || query->cur_node->type == FTS_AST_PARSER_PHRASE_LIST); + + fts_node_t node; + memset(&node, 0, sizeof(node)); + + fts_string_t term; + byte buf[FTS_MAX_WORD_LEN + 1]; + term.f_str= buf; + + /* Need to consider the wildcard search case, the word frequency + is created on the search string not the actual word. So we need + to assign the frequency on search string behalf. */ + if (query->cur_node->type == FTS_AST_TERM && query->cur_node->term.wildcard) + { + term.f_len = query->cur_node->term.ptr->len; + ut_ad(FTS_MAX_WORD_LEN >= term.f_len); + memcpy(term.f_str, query->cur_node->term.ptr->str, term.f_len); + } + else + { + term.f_len = word_len; + ut_ad(FTS_MAX_WORD_LEN >= word_len); + memcpy(term.f_str, word_data, word_len); + } + + /* Lookup the word in our rb tree, it must exist. */ + ib_rbt_bound_t parent; + int ret= rbt_search(query->word_freqs, &parent, &term); + + ut_a(ret == 0); + fts_word_freq_t* word_freq= rbt_value(fts_word_freq_t, parent.last); + bool skip = false; + + /* Use extracted field values */ + node.first_doc_id = first_doc_id; + skip= (query->oper == FTS_EXIST && query->upper_doc_id > 0 && + node.first_doc_id > query->upper_doc_id); + + node.last_doc_id = last_doc_id; + skip= (query->oper == FTS_EXIST && query->lower_doc_id > 0 && + node.last_doc_id < query->lower_doc_id); + + word_freq->doc_count += doc_count; + + dberr_t err= DB_SUCCESS; + + if (!skip) + { + if (ilist_data && ilist_len != UNIV_SQL_NULL && ilist_len > 0) + { + /* Process the ilist data (either inline or external) */ + query->error= fts_query_filter_doc_ids( + query, &word_freq->word, word_freq, &node, + const_cast(ilist_data), ilist_len, FALSE); + + if (query->error != DB_FTS_EXCEED_RESULT_CACHE_LIMIT) + err= query->error; + } + } + + if (ilist_heap) + mem_heap_free(ilist_heap); + return err; +} + +/* Comparator that signals how to treat the current record */ +RecordCompareAction doc_id_exact_match_comparator( + const dtuple_t *search_tuple, const rec_t *rec, + const dict_index_t *index, const rec_offs *offsets) +{ + const dfield_t* search_field= dtuple_get_nth_field(search_tuple, 0); + const byte* search_data= + static_cast(dfield_get_data(search_field)); + doc_id_t target_doc_id= fts_read_doc_id(search_data); + + ulint doc_col_pos= dict_col_get_clust_pos( + &index->table->cols[index->table->fts->doc_col], index); + + ulint len; + const byte* doc_id_data= rec_get_nth_field(rec, offsets, doc_col_pos, &len); + + if (len != sizeof(doc_id_t)) + return RecordCompareAction::SKIP; + + doc_id_t rec_doc_id= fts_read_doc_id(doc_id_data); + return rec_doc_id == target_doc_id + ? RecordCompareAction::PROCESS + : RecordCompareAction::STOP; +} /*************************************************************//** This function implements a simple "blind" query expansion search: @@ -371,107 +497,6 @@ fts_proximity_get_positions( fts_proximity_t* qualified_pos); /*!< out: the position info records ranges containing all matching words. */ -#if 0 -/******************************************************************** -Get the total number of words in a documents. */ -static -ulint -fts_query_terms_in_document( -/*========================*/ - /*!< out: DB_SUCCESS if all go well - else error code */ - fts_query_t* query, /*!< in: FTS query state */ - doc_id_t doc_id, /*!< in: the word to check */ - ulint* total); /*!< out: total words in document */ -#endif - -#if 0 -/*******************************************************************//** -Print the table used for calculating LCS. */ -static -void -fts_print_lcs_table( -/*================*/ - const ulint* table, /*!< in: array to print */ - ulint n_rows, /*!< in: total no. of rows */ - ulint n_cols) /*!< in: total no. of cols */ -{ - ulint i; - - for (i = 0; i < n_rows; ++i) { - ulint j; - - printf("\n"); - - for (j = 0; j < n_cols; ++j) { - - printf("%2lu ", FTS_ELEM(table, n_cols, i, j)); - } - } -} - -/******************************************************************** -Find the longest common subsequence between the query string and -the document. */ -static -ulint -fts_query_lcs( -/*==========*/ - /*!< out: LCS (length) between - two ilists */ - const ulint* p1, /*!< in: word positions of query */ - ulint len_p1, /*!< in: no. of elements in p1 */ - const ulint* p2, /*!< in: word positions within document */ - ulint len_p2) /*!< in: no. of elements in p2 */ -{ - int i; - ulint len = 0; - ulint r = len_p1; - ulint c = len_p2; - ulint size = (r + 1) * (c + 1) * sizeof(ulint); - ulint* table = (ulint*) ut_malloc_nokey(size); - - /* Traverse the table backwards, from the last row to the first and - also from the last column to the first. We compute the smaller - common subsequences first, then use the calculated values to determine - the longest common subsequence. The result will be in TABLE[0][0]. */ - for (i = r; i >= 0; --i) { - int j; - - for (j = c; j >= 0; --j) { - - if (p1[i] == (ulint) -1 || p2[j] == (ulint) -1) { - - FTS_ELEM(table, c, i, j) = 0; - - } else if (p1[i] == p2[j]) { - - FTS_ELEM(table, c, i, j) = FTS_ELEM( - table, c, i + 1, j + 1) + 1; - - } else { - - ulint value; - - value = ut_max( - FTS_ELEM(table, c, i + 1, j), - FTS_ELEM(table, c, i, j + 1)); - - FTS_ELEM(table, c, i, j) = value; - } - } - } - - len = FTS_ELEM(table, c, 0, 0); - - fts_print_lcs_table(table, r, c); - printf("\nLen=" ULINTPF "\n", len); - - ut_free(table); - - return(len); -} -#endif /*******************************************************************//** Compare two fts_ranking_t instance on their rank value and doc ids in @@ -1100,7 +1125,6 @@ fts_query_difference( const fts_string_t* token) /*!< in: token to search */ { ulint n_doc_ids= 0; - trx_t* trx = query->trx; dict_table_t* table = query->index->table; ut_a(query->oper == FTS_IGNORE); @@ -1112,10 +1136,8 @@ fts_query_difference( /* There is nothing we can substract from an empty set. */ if (query->doc_ids && !rbt_empty(query->doc_ids)) { ulint i; - fts_fetch_t fetch; const ib_vector_t* nodes; const fts_index_cache_t*index_cache; - que_t* graph = NULL; fts_cache_t* cache = table->fts->cache; dberr_t error; @@ -1153,21 +1175,21 @@ fts_query_difference( return(query->error); } - /* Setup the callback args for filtering and - consolidating the ilist. */ - fetch.read_arg = query; - fetch.read_record = fts_query_index_fetch_nodes; + AuxCompareMode compare_mode = AuxCompareMode::EQUAL; + if (query->cur_node->type == FTS_AST_TERM && + query->cur_node->term.wildcard) { + compare_mode = AuxCompareMode::LIKE; + } error = fts_index_fetch_nodes( - trx, &graph, &query->fts_index_table, token, &fetch); + query->executor, query->index, token, query, + node_query_processor, compare_mode); /* DB_FTS_EXCEED_RESULT_CACHE_LIMIT passed by 'query->error' */ ut_ad(!(query->error != DB_SUCCESS && error != DB_SUCCESS)); if (error != DB_SUCCESS) { query->error = error; } - - que_graph_free(graph); } /* The size can't increase. */ @@ -1194,7 +1216,6 @@ fts_query_intersect( fts_query_t* query, /*!< in: query instance */ const fts_string_t* token) /*!< in: the token to search */ { - trx_t* trx = query->trx; dict_table_t* table = query->index->table; ut_a(query->oper == FTS_EXIST); @@ -1204,10 +1225,8 @@ fts_query_intersect( if (!(rbt_empty(query->doc_ids) && query->multi_exist)) { ulint n_doc_ids = 0; ulint i; - fts_fetch_t fetch; const ib_vector_t* nodes; const fts_index_cache_t*index_cache; - que_t* graph = NULL; fts_cache_t* cache = table->fts->cache; dberr_t error; @@ -1278,13 +1297,15 @@ fts_query_intersect( return(query->error); } - /* Setup the callback args for filtering and - consolidating the ilist. */ - fetch.read_arg = query; - fetch.read_record = fts_query_index_fetch_nodes; + AuxCompareMode compare_mode = AuxCompareMode::EQUAL; + if (query->cur_node->type == FTS_AST_TERM && + query->cur_node->term.wildcard) { + compare_mode = AuxCompareMode::LIKE; + } error = fts_index_fetch_nodes( - trx, &graph, &query->fts_index_table, token, &fetch); + query->executor, query->index, token, query, + node_query_processor, compare_mode); /* DB_FTS_EXCEED_RESULT_CACHE_LIMIT passed by 'query->error' */ ut_ad(!(query->error != DB_SUCCESS && error != DB_SUCCESS)); @@ -1292,8 +1313,6 @@ fts_query_intersect( query->error = error; } - que_graph_free(graph); - if (query->error == DB_SUCCESS) { /* Make the intesection (rb tree) the current doc id set and free the old set. */ @@ -1371,10 +1390,7 @@ fts_query_union( fts_query_t* query, /*!< in: query instance */ fts_string_t* token) /*!< in: token to search */ { - fts_fetch_t fetch; ulint n_doc_ids = 0; - trx_t* trx = query->trx; - que_t* graph = NULL; dberr_t error; ut_a(query->oper == FTS_NONE || query->oper == FTS_DECR_RATING || @@ -1390,14 +1406,16 @@ fts_query_union( fts_query_cache(query, token); - /* Setup the callback args for filtering and - consolidating the ilist. */ - fetch.read_arg = query; - fetch.read_record = fts_query_index_fetch_nodes; + AuxCompareMode compare_mode = AuxCompareMode::EQUAL; + if (query->cur_node->type == FTS_AST_TERM && + query->cur_node->term.wildcard) { + compare_mode = AuxCompareMode::LIKE; + } /* Read the nodes from disk. */ error = fts_index_fetch_nodes( - trx, &graph, &query->fts_index_table, token, &fetch); + query->executor, query->index, token, query, node_query_processor, + compare_mode); /* DB_FTS_EXCEED_RESULT_CACHE_LIMIT passed by 'query->error' */ ut_ad(!(query->error != DB_SUCCESS && error != DB_SUCCESS)); @@ -1405,8 +1423,6 @@ fts_query_union( query->error = error; } - que_graph_free(graph); - if (query->error == DB_SUCCESS) { /* The size can't decrease. */ @@ -1923,476 +1939,229 @@ fts_query_match_phrase( return(phrase->found); } -/*****************************************************************//** -Callback function to fetch and search the document. +/** Callback function to fetch and search the document. +@param fts_index fulltext index +@param doc_id document id +@param arg user argument +@param expansion Expansion document @return whether the phrase is found */ static -ibool -fts_query_fetch_document( -/*=====================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts_doc_t* */ +dberr_t fts_query_fetch_document(dict_index_t *fts_index, + doc_id_t doc_id, + void *arg, bool expansion= false) { - - que_node_t* exp; - sel_node_t* node = static_cast(row); - fts_phrase_t* phrase = static_cast(user_arg); - ulint prev_len = 0; - ulint total_len = 0; - byte* document_text = NULL; - - exp = node->select_list; - - phrase->found = FALSE; - - /* For proximity search, we will need to get the whole document - from all fields, so first count the total length of the document - from all the fields */ - if (phrase->proximity_pos) { - while (exp) { - ulint field_len; - dfield_t* dfield = que_node_get_val(exp); - byte* data = static_cast( - dfield_get_data(dfield)); - - if (dfield_is_ext(dfield)) { - ulint local_len = dfield_get_len(dfield); - - local_len -= BTR_EXTERN_FIELD_REF_SIZE; - - field_len = mach_read_from_4( - data + local_len + BTR_EXTERN_LEN + 4); - } else { - field_len = dfield_get_len(dfield); - } - - if (field_len != UNIV_SQL_NULL) { - total_len += field_len + 1; - } - - exp = que_node_get_next(exp); - } - - document_text = static_cast(mem_heap_zalloc( - phrase->heap, total_len)); - - if (!document_text) { - return(FALSE); - } - } - - exp = node->select_list; - - while (exp) { - dfield_t* dfield = que_node_get_val(exp); - byte* data = static_cast( - dfield_get_data(dfield)); - ulint cur_len; - - if (dfield_is_ext(dfield)) { - data = btr_copy_externally_stored_field( - &cur_len, data, phrase->zip_size, - dfield_get_len(dfield), phrase->heap); - } else { - cur_len = dfield_get_len(dfield); - } - - if (cur_len != UNIV_SQL_NULL && cur_len != 0) { - if (phrase->proximity_pos) { - ut_ad(prev_len + cur_len <= total_len); - memcpy(document_text + prev_len, data, cur_len); - } else { - /* For phrase search */ - phrase->found = - fts_query_match_phrase( - phrase, - static_cast(data), - cur_len, prev_len, - phrase->heap); - } - - /* Document positions are calculated from the beginning - of the first field, need to save the length for each - searched field to adjust the doc position when search - phrases. */ - prev_len += cur_len + 1; - } - - if (phrase->found) { - break; - } - - exp = que_node_get_next(exp); - } - - if (phrase->proximity_pos) { - ut_ad(prev_len <= total_len); - - phrase->found = fts_proximity_is_word_in_range( - phrase, document_text, total_len); - } - - return(phrase->found); -} - -#if 0 -/******************************************************************** -Callback function to check whether a record was found or not. */ -static -ibool -fts_query_select( -/*=============*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts_doc_t* */ -{ - int i; - que_node_t* exp; - sel_node_t* node = row; - fts_select_t* select = user_arg; - - ut_a(select->word_freq); - ut_a(select->word_freq->doc_freqs); - - exp = node->select_list; - - for (i = 0; exp && !select->found; ++i) { - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - - switch (i) { - case 0: /* DOC_COUNT */ - if (len != UNIV_SQL_NULL && len != 0) { - - select->word_freq->doc_count += - mach_read_from_4(data); - } - break; - - case 1: /* ILIST */ - if (len != UNIV_SQL_NULL && len != 0) { - - fts_query_find_doc_id(select, data, len); - } - break; - - default: - ut_error; - } - - exp = que_node_get_next(exp); - } - - return(FALSE); -} - -/******************************************************************** -Read the rows from the FTS index, that match word and where the -doc id is between first and last doc id. -@return DB_SUCCESS if all go well else error code */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_query_find_term( -/*================*/ - fts_query_t* query, /*!< in: FTS query state */ - que_t** graph, /*!< in: prepared statement */ - const fts_string_t* word, /*!< in: the word to fetch */ - doc_id_t doc_id, /*!< in: doc id to match */ - ulint* min_pos,/*!< in/out: pos found must be - greater than this minimum value. */ - ibool* found) /*!< out: TRUE if found else FALSE */ -{ - pars_info_t* info; - dberr_t error; - fts_select_t select; - doc_id_t match_doc_id; - trx_t* trx = query->trx; - char table_name[MAX_FULL_NAME_LEN]; - - trx->op_info = "fetching FTS index matching nodes"; - - if (*graph) { - info = (*graph)->info; - } else { - ulint selected; - - info = pars_info_create(); - - selected = fts_select_index(*word->f_str); - query->fts_index_table.suffix = fts_get_suffix(selected); - - fts_get_table_name(&query->fts_index_table, table_name); - pars_info_bind_id(info, "index_table_name", table_name); - } - - select.found = FALSE; - select.doc_id = doc_id; - select.min_pos = *min_pos; - select.word_freq = fts_query_add_word_freq(query, word->f_str); - - pars_info_bind_function(info, "my_func", fts_query_select, &select); - pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &match_doc_id, doc_id); - - fts_bind_doc_id(info, "min_doc_id", &match_doc_id); - - fts_bind_doc_id(info, "max_doc_id", &match_doc_id); - - if (!*graph) { - - *graph = fts_parse_sql( - &query->fts_index_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT doc_count, ilist\n" - " FROM $index_table_name\n" - " WHERE word LIKE :word AND" - " first_doc_id <= :min_doc_id AND" - " last_doc_id >= :max_doc_id\n" - " ORDER BY first_doc_id;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - } - - for (;;) { - error = fts_eval_sql(trx, *graph); - - if (error == DB_SUCCESS) { - - break; /* Exit the loop. */ - } else { - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading FTS" - " index. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << error - << " while reading FTS index."; - - break; /* Exit the loop. */ - } - } - } - - /* Value to return */ - *found = select.found; - - if (*found) { - *min_pos = select.min_pos; - } - - return(error); -} - -/******************************************************************** -Callback aggregator for int columns. */ -static -ibool -fts_query_sum( -/*==========*/ - /*!< out: always returns TRUE */ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: ulint* */ -{ - - que_node_t* exp; - sel_node_t* node = row; - ulint* total = user_arg; - - exp = node->select_list; - - while (exp) { - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - - if (len != UNIV_SQL_NULL && len != 0) { - *total += mach_read_from_4(data); - } - - exp = que_node_get_next(exp); - } - - return(TRUE); -} - -/******************************************************************** -Calculate the total documents that contain a particular word (term). -@return DB_SUCCESS if all go well else error code */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_query_total_docs_containing_term( -/*=================================*/ - fts_query_t* query, /*!< in: FTS query state */ - const fts_string_t* word, /*!< in: the word to check */ - ulint* total) /*!< out: documents containing word */ -{ - pars_info_t* info; - dberr_t error; - que_t* graph; - ulint selected; - trx_t* trx = query->trx; - char table_name[MAX_FULL_NAME_LEN] - - trx->op_info = "fetching FTS index document count"; - - *total = 0; - - info = pars_info_create(); - - pars_info_bind_function(info, "my_func", fts_query_sum, total); - pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len); - - selected = fts_select_index(*word->f_str); - - query->fts_index_table.suffix = fts_get_suffix(selected); - - fts_get_table_name(&query->fts_index_table, table_name); - - pars_info_bind_id(info, "index_table_name", table_name); - - graph = fts_parse_sql( - &query->fts_index_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT doc_count\n" - " FROM $index_table_name\n" - " WHERE word = :word" - " ORDER BY first_doc_id;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - for (;;) { - error = fts_eval_sql(trx, graph); - - if (error == DB_SUCCESS) { - - break; /* Exit the loop. */ - } else { - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading FTS" - " index. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << error - << " while reading FTS index."; - - break; /* Exit the loop. */ - } - } - } - - que_graph_free(graph); - - return(error); -} - -/******************************************************************** -Get the total number of words in a documents. -@return DB_SUCCESS if all go well else error code */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_query_terms_in_document( -/*========================*/ - fts_query_t* query, /*!< in: FTS query state */ - doc_id_t doc_id, /*!< in: the word to check */ - ulint* total) /*!< out: total words in document */ -{ - pars_info_t* info; - dberr_t error; - que_t* graph; - doc_id_t read_doc_id; - trx_t* trx = query->trx; - char table_name[MAX_FULL_NAME_LEN]; - - trx->op_info = "fetching FTS document term count"; - - *total = 0; - - info = pars_info_create(); - - pars_info_bind_function(info, "my_func", fts_query_sum, total); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &read_doc_id, doc_id); - fts_bind_doc_id(info, "doc_id", &read_doc_id); - - query->fts_index_table.suffix = "DOC_ID"; - - fts_get_table_name(&query->fts_index_table, table_name); - - pars_info_bind_id(info, "index_table_name", table_name); - - graph = fts_parse_sql( - &query->fts_index_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT count\n" - " FROM $index_table_name\n" - " WHERE doc_id = :doc_id" - " BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - for (;;) { - error = fts_eval_sql(trx, graph); - - if (error == DB_SUCCESS) { - - break; /* Exit the loop. */ - } else { - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading FTS" - " doc id table. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << error << " while reading FTS" - " doc id table."; - - break; /* Exit the loop. */ - } - } - } - - que_graph_free(graph); - - return(error); -} + trx_t *trx= trx_create(); + trx->op_info= "fetching FTS document for query"; + dict_table_t *user_table= fts_index->table; + dict_index_t *fts_doc_id_index= user_table->fts_doc_id_index; + dict_index_t *clust_index= dict_table_get_first_index(user_table); + ut_a(user_table->fts->doc_col != ULINT_UNDEFINED); + ut_a(fts_doc_id_index); + + QueryExecutor executor(trx); + + /* Map FTS index columns to clustered index field positions */ + ulint *clust_field_nos= static_cast( + mem_heap_alloc(executor.get_heap(), + fts_index->n_user_defined_cols * sizeof(ulint))); + + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + dict_field_t* fts_field= dict_index_get_nth_field(fts_index, i); + clust_field_nos[i]= dict_col_get_index_pos(fts_field->col, clust_index); + } + dfield_t fields[1]; + dtuple_t search_tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N #endif + }; + dict_index_copy_types(&search_tuple, fts_doc_id_index, 1); + dfield_t* dfield= dtuple_get_nth_field(&search_tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id((byte*) &write_doc_id, doc_id); + dfield_set_data(dfield, &write_doc_id, sizeof(write_doc_id)); + + auto process_expansion_doc= [arg, fts_index, + clust_field_nos](const rec_t* rec, + const dict_index_t *index, + const rec_offs *offsets)-> dberr_t + { + fts_doc_t *result_doc= static_cast(arg); + fts_doc_t doc; + CHARSET_INFO *doc_charset= result_doc->charset; + fts_doc_init(&doc); + doc.found= TRUE; + + ulint doc_len= 0; + ulint field_no= 0; + + /* Process each indexed column content */ + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, + col_pos, &field_len); + + /* NULL column */ + if (field_len == UNIV_SQL_NULL) { + continue; + } + + /* Determine document charset from column if not provided */ + if (!doc_charset) + { + const dict_field_t* ifield= dict_index_get_nth_field(fts_index, i); + doc_charset= fts_get_charset(ifield->col->prtype); + } + + doc.charset= doc_charset; + /* Skip columns stored externally, as in fts_query_expansion_fetch_doc */ + if (rec_offs_nth_extern(offsets, col_pos)) { + continue; + } + + /* Use inline field data */ + doc.text.f_n_char= 0; + doc.text.f_str= const_cast(field_data); + doc.text.f_len= field_len; + + if (field_no == 0) + fts_tokenize_document(&doc, result_doc, result_doc->parser); + else + fts_tokenize_document_next(&doc, doc_len, result_doc, + result_doc->parser); + + /* Next field offset: add 1 for separator if more fields follow */ + doc_len+= ((i + 1) < fts_index->n_user_defined_cols) + ? field_len + 1 + : field_len; + + field_no++; + } + + ut_ad(doc_charset); + if (!result_doc->charset) { + result_doc->charset= doc_charset; + } + + fts_doc_free(&doc); + + return DB_SUCCESS; /* continue */ + }; + + auto process_doc_query= [arg, user_table, fts_index, + clust_field_nos](const rec_t* rec, + const dict_index_t* index, + const rec_offs* offsets) -> dberr_t + { + ulint prev_len= 0; + ulint total_len= 0; + byte *document_text= nullptr; + + fts_phrase_t *phrase= static_cast(arg); + phrase->found= FALSE; + + /* Extract doc_id from the clustered index record */ + ulint doc_col_pos= dict_col_get_index_pos( + &user_table->cols[user_table->fts->doc_col], index); + + ulint len; + rec_get_nth_field_offs(offsets, doc_col_pos, &len); + if (len != sizeof(doc_id_t)) + return DB_ERROR; + + /* For proximity search, first count total document length */ + if (phrase->proximity_pos) + { + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, + col_pos, &field_len); + if (rec_offs_nth_extern(offsets, col_pos)) + { + ulint local_len= field_len; + local_len-= BTR_EXTERN_FIELD_REF_SIZE; + field_len= mach_read_from_4( + field_data + local_len + BTR_EXTERN_LEN + 4); + } + if (field_len != UNIV_SQL_NULL) + total_len+= field_len + 1; + } + + document_text= + static_cast(mem_heap_zalloc(phrase->heap, total_len)); + if (!document_text) + return DB_ERROR; + } + + /* Process each indexed column content */ + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, + col_pos, &field_len); + byte* data= const_cast(field_data); + ulint cur_len; + + if (rec_offs_nth_extern(offsets, col_pos)) + { + data= btr_copy_externally_stored_field( + &cur_len, const_cast(field_data), phrase->zip_size, + field_len, phrase->heap); + } + else cur_len= field_len; + if (cur_len != UNIV_SQL_NULL && cur_len != 0) + { + if (phrase->proximity_pos) + { + ut_ad(prev_len + cur_len <= total_len); + memcpy(document_text + prev_len, data, cur_len); + } + else + { + /* For phrase search */ + phrase->found= fts_query_match_phrase( + phrase, data, cur_len, prev_len, phrase->heap); + } + + /* Document positions are calculated from the beginning + of the first field, need to save the length for each + searched field to adjust the doc position when search + phrases. */ + prev_len+= cur_len + 1; + } + + if (phrase->found) + break; + } + + if (phrase->proximity_pos) + { + ut_ad(prev_len <= total_len); + phrase->found= fts_proximity_is_word_in_range( + phrase, document_text, total_len); + } + + return phrase->found ? DB_SUCCESS_LOCKED_REC : DB_SUCCESS; /* Stop if found, continue if not found */ + }; + + RecordProcessor proc= expansion + ? RecordProcessor(process_expansion_doc) + : RecordProcessor(process_doc_query); + RecordCallback reader(proc, doc_id_exact_match_comparator); + dberr_t err= DB_SUCCESS; + err= executor.read_by_index(user_table, fts_doc_id_index, + &search_tuple, PAGE_CUR_GE, reader, true); + trx_commit_for_mysql(trx); + trx->free(); + if (err == DB_RECORD_NOT_FOUND) err= DB_SUCCESS; + return err; +} /*****************************************************************//** Retrieve the document and match the phrase tokens. @@ -2421,9 +2190,8 @@ fts_query_match_document( *found = phrase.found = FALSE; - error = fts_doc_fetch_by_doc_id( - get_doc, match->doc_id, NULL, FTS_FETCH_DOC_BY_ID_EQUAL, - fts_query_fetch_document, &phrase); + error = fts_query_fetch_document( + get_doc->index_cache->index, match->doc_id, &phrase); if (UNIV_UNLIKELY(error != DB_SUCCESS)) { ib::error() << "(" << error << ") matching document."; @@ -2468,21 +2236,14 @@ fts_query_is_in_proximity_range( phrase.proximity_pos = qualified_pos; phrase.found = FALSE; - err = fts_doc_fetch_by_doc_id( - &get_doc, match[0]->doc_id, NULL, FTS_FETCH_DOC_BY_ID_EQUAL, - fts_query_fetch_document, &phrase); + err = fts_query_fetch_document( + get_doc.index_cache->index, match[0]->doc_id, &phrase); if (UNIV_UNLIKELY(err != DB_SUCCESS)) { ib::error() << "(" << err << ") in verification" " phase of proximity search"; } - /* Free the prepared statement. */ - if (get_doc.get_document_graph) { - que_graph_free(get_doc.get_document_graph); - get_doc.get_document_graph = NULL; - } - mem_heap_free(phrase.heap); return(err == DB_SUCCESS && phrase.found); @@ -2591,6 +2352,7 @@ fts_query_phrase_split( ulint len = 0; ulint cur_pos = 0; fts_ast_node_t* term_node = NULL; + CHARSET_INFO* cs = fts_index_get_charset(query->index); if (node->type == FTS_AST_TEXT) { phrase.f_str = node->text.ptr->str; @@ -2614,7 +2376,7 @@ fts_query_phrase_split( } cur_len = innobase_mysql_fts_get_token( - query->fts_index_table.charset, + cs, reinterpret_cast(phrase.f_str) + cur_pos, reinterpret_cast(phrase.f_str) @@ -2637,7 +2399,7 @@ fts_query_phrase_split( result_str.f_str = term_node->term.ptr->str; result_str.f_len = term_node->term.ptr->len; result_str.f_n_char = fts_get_token_size( - query->fts_index_table.charset, + cs, reinterpret_cast(result_str.f_str), result_str.f_len); @@ -2654,8 +2416,7 @@ fts_query_phrase_split( if (fts_check_token( &result_str, - cache->stopword_info.cached_stopword, - query->fts_index_table.charset)) { + cache->stopword_info.cached_stopword, cs)) { /* Add the word to the RB tree so that we can calculate its frequency within a document. */ fts_query_add_word_freq(query, token); @@ -2717,10 +2478,7 @@ fts_query_phrase_search( /* Ignore empty strings. */ if (num_token > 0) { fts_string_t* token = NULL; - fts_fetch_t fetch; - trx_t* trx = query->trx; fts_ast_oper_t oper = query->oper; - que_t* graph = NULL; ulint i; dberr_t error; @@ -2755,11 +2513,6 @@ fts_query_phrase_search( } } - /* Setup the callback args for filtering and consolidating - the ilist. */ - fetch.read_arg = query; - fetch.read_record = fts_query_index_fetch_nodes; - for (i = 0; i < num_token; i++) { /* Search for the first word from the phrase. */ token = static_cast( @@ -2770,9 +2523,15 @@ fts_query_phrase_search( query->matched = query->match_array[i]; } + AuxCompareMode compare_mode = AuxCompareMode::EQUAL; + if (query->cur_node->type == FTS_AST_TERM && + query->cur_node->term.wildcard) { + compare_mode = AuxCompareMode::LIKE; + } + error = fts_index_fetch_nodes( - trx, &graph, &query->fts_index_table, - token, &fetch); + query->executor, query->index, token, query, + node_query_processor, compare_mode); /* DB_FTS_EXCEED_RESULT_CACHE_LIMIT passed by 'query->error' */ ut_ad(!(query->error != DB_SUCCESS && error != DB_SUCCESS)); @@ -2780,9 +2539,6 @@ fts_query_phrase_search( query->error = error; } - que_graph_free(graph); - graph = NULL; - fts_query_cache(query, token); if (!(query->flags & FTS_PHRASE) @@ -2923,12 +2679,11 @@ fts_query_get_token( if (node->term.wildcard) { - token->f_str = static_cast(ut_malloc_nokey(str_len + 2)); - token->f_len = str_len + 1; + token->f_str = static_cast(ut_malloc_nokey(str_len + 1)); + token->f_len = str_len; memcpy(token->f_str, node->term.ptr->str, str_len); - token->f_str[str_len] = '%'; token->f_str[token->f_len] = 0; new_ptr = token->f_str; @@ -3109,78 +2864,6 @@ fts_ast_visit_sub_exp( DBUG_RETURN(error); } -#if 0 -/*****************************************************************//*** -Check if the doc id exists in the ilist. -@return TRUE if doc id found */ -static -ulint -fts_query_find_doc_id( -/*==================*/ - fts_select_t* select, /*!< in/out: contains the doc id to - find, we update the word freq if - document found */ - void* data, /*!< in: doc id ilist */ - ulint len) /*!< in: doc id ilist size */ -{ - byte* ptr = data; - doc_id_t doc_id = 0; - ulint decoded = 0; - - /* Decode the ilist and search for selected doc_id. We also - calculate the frequency of the word in the document if found. */ - while (decoded < len && !select->found) { - ulint freq = 0; - ulint min_pos = 0; - ulint last_pos = 0; - ulint pos = fts_decode_vlc(&ptr); - - /* Add the delta. */ - doc_id += pos; - - while (*ptr) { - ++freq; - last_pos += fts_decode_vlc(&ptr); - - /* Only if min_pos is not set and the current - term exists in a position greater than the - min_pos of the previous term. */ - if (min_pos == 0 && last_pos > select->min_pos) { - min_pos = last_pos; - } - } - - /* Skip the end of word position marker. */ - ++ptr; - - /* Bytes decoded so far. */ - decoded = ptr - (byte*) data; - - /* A word may exist in the document but we only consider a - match if it exists in a position that is greater than the - position of the previous term. */ - if (doc_id == select->doc_id && min_pos > 0) { - fts_doc_freq_t* doc_freq; - - /* Add the doc id to the doc freq rb tree, if - the doc id doesn't exist it will be created. */ - doc_freq = fts_query_add_doc_freq( - select->word_freq->doc_freqs, doc_id); - - /* Avoid duplicating the frequency tally */ - if (doc_freq->freq == 0) { - doc_freq->freq = freq; - } - - select->found = TRUE; - select->min_pos = min_pos; - } - } - - return(select->found); -} -#endif - /*****************************************************************//** Read and filter nodes. @return DB_SUCCESS if all go well, @@ -3301,156 +2984,6 @@ fts_query_filter_doc_ids( } } -/*****************************************************************//** -Read the FTS INDEX row. -@return DB_SUCCESS if all go well. */ -static -dberr_t -fts_query_read_node( -/*================*/ - fts_query_t* query, /*!< in: query instance */ - const fts_string_t* word, /*!< in: current word */ - que_node_t* exp) /*!< in: query graph node */ -{ - int i; - int ret; - fts_node_t node; - ib_rbt_bound_t parent; - fts_word_freq_t* word_freq; - ibool skip = FALSE; - fts_string_t term; - byte buf[FTS_MAX_WORD_LEN + 1]; - dberr_t error = DB_SUCCESS; - - ut_a(query->cur_node->type == FTS_AST_TERM - || query->cur_node->type == FTS_AST_TEXT - || query->cur_node->type == FTS_AST_PARSER_PHRASE_LIST); - - memset(&node, 0, sizeof(node)); - term.f_str = buf; - - /* Need to consider the wildcard search case, the word frequency - is created on the search string not the actual word. So we need - to assign the frequency on search string behalf. */ - if (query->cur_node->type == FTS_AST_TERM - && query->cur_node->term.wildcard) { - - term.f_len = query->cur_node->term.ptr->len; - ut_ad(FTS_MAX_WORD_LEN >= term.f_len); - memcpy(term.f_str, query->cur_node->term.ptr->str, term.f_len); - } else { - term.f_len = word->f_len; - ut_ad(FTS_MAX_WORD_LEN >= word->f_len); - memcpy(term.f_str, word->f_str, word->f_len); - } - - /* Lookup the word in our rb tree, it must exist. */ - ret = rbt_search(query->word_freqs, &parent, &term); - - ut_a(ret == 0); - - word_freq = rbt_value(fts_word_freq_t, parent.last); - - /* Start from 1 since the first column has been read by the caller. - Also, we rely on the order of the columns projected, to filter - out ilists that are out of range and we always want to read - the doc_count irrespective of the suitability of the row. */ - - for (i = 1; exp && !skip; exp = que_node_get_next(exp), ++i) { - - dfield_t* dfield = que_node_get_val(exp); - byte* data = static_cast( - dfield_get_data(dfield)); - ulint len = dfield_get_len(dfield); - - ut_a(len != UNIV_SQL_NULL); - - /* Note: The column numbers below must match the SELECT. */ - - switch (i) { - case 1: /* DOC_COUNT */ - word_freq->doc_count += mach_read_from_4(data); - break; - - case 2: /* FIRST_DOC_ID */ - node.first_doc_id = fts_read_doc_id(data); - - /* Skip nodes whose doc ids are out range. */ - if (query->oper == FTS_EXIST - && query->upper_doc_id > 0 - && node.first_doc_id > query->upper_doc_id) { - skip = TRUE; - } - break; - - case 3: /* LAST_DOC_ID */ - node.last_doc_id = fts_read_doc_id(data); - - /* Skip nodes whose doc ids are out range. */ - if (query->oper == FTS_EXIST - && query->lower_doc_id > 0 - && node.last_doc_id < query->lower_doc_id) { - skip = TRUE; - } - break; - - case 4: /* ILIST */ - - error = fts_query_filter_doc_ids( - query, &word_freq->word, word_freq, - &node, data, len, FALSE); - - break; - - default: - ut_error; - } - } - - if (!skip) { - /* Make sure all columns were read. */ - - ut_a(i == 5); - } - - return error; -} - -/*****************************************************************//** -Callback function to fetch the rows in an FTS INDEX record. -@return always returns TRUE */ -static -ibool -fts_query_index_fetch_nodes( -/*========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to fts_fetch_t */ -{ - fts_string_t key; - sel_node_t* sel_node = static_cast(row); - fts_fetch_t* fetch = static_cast(user_arg); - fts_query_t* query = static_cast(fetch->read_arg); - que_node_t* exp = sel_node->select_list; - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint dfield_len = dfield_get_len(dfield); - - key.f_str = static_cast(data); - key.f_len = dfield_len; - - ut_a(dfield_len <= FTS_MAX_WORD_LEN); - - /* Note: we pass error out by 'query->error' */ - query->error = fts_query_read_node(query, &key, que_node_get_next(exp)); - - if (query->error != DB_SUCCESS) { - ut_ad(query->error == DB_FTS_EXCEED_RESULT_CACHE_LIMIT); - return(FALSE); - } else { - return(TRUE); - } -} - /*****************************************************************//** Calculate the inverse document frequency (IDF) for all the terms. */ static @@ -3837,7 +3370,7 @@ fts_query_parse( memset(&state, 0x0, sizeof(state)); - state.charset = query->fts_index_table.charset; + state.charset = fts_index_get_charset(query->index); DBUG_EXECUTE_IF("fts_instrument_query_disable_parser", query->parser = NULL;); @@ -3850,7 +3383,7 @@ fts_query_parse( } else { /* Setup the scanner to use, this depends on the mode flag. */ state.lexer = fts_lexer_create(mode, query_str, query_len); - state.charset = query->fts_index_table.charset; + state.charset = fts_index_get_charset(query->index); error = fts_parse(&state); fts_lexer_free(state.lexer); state.lexer = NULL; @@ -3931,10 +3464,6 @@ fts_query( query.deleted = fts_doc_ids_create(); query.cur_node = NULL; - query.fts_common_table.type = FTS_COMMON_TABLE; - query.fts_common_table.table_id = index->table->id; - query.fts_common_table.table = index->table; - charset = fts_index_get_charset(index); query.fts_index_table.type = FTS_INDEX_TABLE; @@ -3963,25 +3492,35 @@ fts_query( query.total_docs = dict_table_get_n_rows(index->table); - query.fts_common_table.suffix = "DELETED"; + /* Create single FTSQueryExecutor for entire query lifecycle */ + query.executor = new FTSQueryExecutor(query_trx, index->table); - /* Read the deleted doc_ids, we need these for filtering. */ - error = fts_table_fetch_doc_ids( - NULL, &query.fts_common_table, query.deleted); - - if (error != DB_SUCCESS) { - goto func_exit; + /* Prefetch all auxiliary and common tables to avoid repeated dict_sys.latch acquisitions */ + error = query.executor->open_all_aux_tables(index); + if (error == DB_SUCCESS) { + error = query.executor->open_all_deletion_tables(); } - query.fts_common_table.suffix = "DELETED_CACHE"; + if (error == DB_SUCCESS) { + /* Read the deleted doc_ids, we need these for filtering. */ + error = fts_table_fetch_doc_ids( + query.executor, index->table, "DELETED", + query.deleted); + } - error = fts_table_fetch_doc_ids( - NULL, &query.fts_common_table, query.deleted); + if (error == DB_SUCCESS) { + error = fts_table_fetch_doc_ids( + query.executor, index->table, "DELETED_CACHE", + query.deleted); + } if (error != DB_SUCCESS) { + query_trx->rollback(); goto func_exit; } + trx_commit_for_mysql(query_trx); + /* Get the deleted doc ids that are in the cache. */ fts_cache_append_deleted_doc_ids( index->table->fts->cache, query.deleted->doc_ids); @@ -4080,6 +3619,12 @@ fts_query( ut_free(lc_query_str); func_exit: + /* Clean up the dynamically allocated executor */ + if (query.executor) { + delete query.executor; + query.executor = nullptr; + } + fts_query_free(&query); query_trx->free(); @@ -4206,10 +3751,8 @@ fts_expand_query( fetch the original document and parse them. Future optimization could be done here if we support some forms of document-to-word mapping */ - fts_doc_fetch_by_doc_id(NULL, ranking->doc_id, index, - FTS_FETCH_DOC_BY_ID_EQUAL, - fts_query_expansion_fetch_doc, - &result_doc); + fts_query_fetch_document(index, ranking->doc_id, + &result_doc, true); /* Estimate memory used, see fts_process_token and fts_token_t. We ignore token size here. */ diff --git a/storage/innobase/fts/fts0sql.cc b/storage/innobase/fts/fts0sql.cc deleted file mode 100644 index 781d15f2befb0..0000000000000 --- a/storage/innobase/fts/fts0sql.cc +++ /dev/null @@ -1,208 +0,0 @@ -/***************************************************************************** - -Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2019, 2021, MariaDB Corporation. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/**************************************************//** -@file fts/fts0sql.cc -Full Text Search functionality. - -Created 2007-03-27 Sunny Bains -*******************************************************/ - -#include "que0que.h" -#include "trx0roll.h" -#include "pars0pars.h" -#include "dict0dict.h" -#include "fts0types.h" -#include "fts0priv.h" - -/** SQL statements for creating the ancillary FTS tables. */ - -/** Preamble to all SQL statements. */ -static const char* fts_sql_begin= - "PROCEDURE P() IS\n"; - -/** Postamble to non-committing SQL statements. */ -static const char* fts_sql_end= - "\n" - "END;\n"; - -/******************************************************************//** -Get the table id. -@return number of bytes written */ -int -fts_get_table_id( -/*=============*/ - const fts_table_t* - fts_table, /*!< in: FTS Auxiliary table */ - char* table_id) /*!< out: table id, must be at least - FTS_AUX_MIN_TABLE_ID_LENGTH bytes - long */ -{ - int len; - - ut_a(fts_table->table != NULL); - - switch (fts_table->type) { - case FTS_COMMON_TABLE: - len = fts_write_object_id(fts_table->table_id, table_id); - break; - - case FTS_INDEX_TABLE: - - len = fts_write_object_id(fts_table->table_id, table_id); - - table_id[len] = '_'; - ++len; - table_id += len; - - len += fts_write_object_id(fts_table->index_id, table_id); - break; - - default: - ut_error; - } - - ut_a(len >= 16); - ut_a(len < FTS_AUX_MIN_TABLE_ID_LENGTH); - - return(len); -} - -/** Construct the name of an internal FTS table for the given table. -@param[in] fts_table metadata on fulltext-indexed table -@param[out] table_name a name up to MAX_FULL_NAME_LEN -@param[in] dict_locked whether dict_sys.latch is being held */ -void fts_get_table_name(const fts_table_t* fts_table, char* table_name, - bool dict_locked) -{ - if (!dict_locked) { - dict_sys.freeze(SRW_LOCK_CALL); - } - ut_ad(dict_sys.frozen()); - /* Include the separator as well. */ - const size_t dbname_len = fts_table->table->name.dblen() + 1; - ut_ad(dbname_len > 1); - memcpy(table_name, fts_table->table->name.m_name, dbname_len); - if (!dict_locked) { - dict_sys.unfreeze(); - } - memcpy(table_name += dbname_len, "FTS_", 4); - table_name += 4; - table_name += fts_get_table_id(fts_table, table_name); - *table_name++ = '_'; - strcpy(table_name, fts_table->suffix); -} - -/******************************************************************//** -Parse an SQL string. -@return query graph */ -que_t* -fts_parse_sql( -/*==========*/ - fts_table_t* fts_table, /*!< in: FTS auxiliary table info */ - pars_info_t* info, /*!< in: info struct, or NULL */ - const char* sql) /*!< in: SQL string to evaluate */ -{ - char* str; - que_t* graph; - ibool dict_locked; - - str = ut_str3cat(fts_sql_begin, sql, fts_sql_end); - - dict_locked = (fts_table && fts_table->table->fts - && fts_table->table->fts->dict_locked); - - if (!dict_locked) { - /* The InnoDB SQL parser is not re-entrant. */ - dict_sys.lock(SRW_LOCK_CALL); - } - - graph = pars_sql(info, str); - ut_a(graph); - - if (!dict_locked) { - dict_sys.unlock(); - } - - ut_free(str); - - return(graph); -} - -/******************************************************************//** -Evaluate an SQL query graph. -@return DB_SUCCESS or error code */ -dberr_t -fts_eval_sql( -/*=========*/ - trx_t* trx, /*!< in: transaction */ - que_t* graph) /*!< in: Query graph to evaluate */ -{ - que_thr_t* thr; - - graph->trx = trx; - - ut_a(thr = que_fork_start_command(graph)); - - que_run_threads(thr); - - return(trx->error_state); -} - -/******************************************************************//** -Construct the column specification part of the SQL string for selecting the -indexed FTS columns for the given table. Adds the necessary bound -ids to the given 'info' and returns the SQL string. Examples: - -One indexed column named "text": - - "$sel0", - info/ids: sel0 -> "text" - -Two indexed columns named "subject" and "content": - - "$sel0, $sel1", - info/ids: sel0 -> "subject", sel1 -> "content", -@return heap-allocated WHERE string */ -const char* -fts_get_select_columns_str( -/*=======================*/ - dict_index_t* index, /*!< in: index */ - pars_info_t* info, /*!< in/out: parser info */ - mem_heap_t* heap) /*!< in: memory heap */ -{ - ulint i; - const char* str = ""; - - for (i = 0; i < index->n_user_defined_cols; i++) { - char* sel_str; - - dict_field_t* field = dict_index_get_nth_field(index, i); - - sel_str = mem_heap_printf(heap, "sel%lu", (ulong) i); - - /* Set copy_name to TRUE since it's dynamic. */ - pars_info_bind_id(info, sel_str, field->name); - - str = mem_heap_printf( - heap, "%s%s$%s", str, (*str) ? ", " : "", sel_str); - } - - return(str); -} diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 5c2ea50657521..3ff699b289994 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -11465,6 +11465,81 @@ ha_innobase::update_create_info( } } +/** Update FTS stopword configuration when reload is enabled +@param table FTS table +@param trx transaction +@param thd current thread +@param new_stopword_table stopword table name +@param new_stopword_is_on stopword enable setting +@return TRUE if success */ +static +bool innobase_fts_update_stopword_config(dict_table_t *table, trx_t *trx, + THD *thd, + const char *new_stopword_table, + bool new_stopword_is_on) +{ + ut_ad(dict_sys.locked()); + /* Create FTSQueryExecutor for config operations */ + FTSQueryExecutor executor(trx, table); + dberr_t error= DB_SUCCESS; + fts_string_t str; + + /* Load CONFIG table directly since we already hold dict_sys.lock() */ + char config_table_name[MAX_FULL_NAME_LEN]; + fts_table_t fts_table; + FTS_INIT_FTS_TABLE(&fts_table, nullptr, FTS_COMMON_TABLE, table); + fts_table.suffix = "CONFIG"; + fts_get_table_name(&fts_table, config_table_name, true); + const span config_name{ + config_table_name, strlen(config_table_name)}; + dict_table_t *config_table= dict_sys.load_table(config_name); + + if (config_table == nullptr) + return false; + + /* Assign CONFIG table directly to executor for optimized access */ + executor.set_config_table(config_table); + + /* Update FTS_USE_STOPWORD setting in CONFIG table */ + ulint use_stopword = (ulint) new_stopword_is_on; + fts_cache_t *cache= table->fts->cache; + error= fts_config_set_ulint( + &executor, table, FTS_USE_STOPWORD, use_stopword); + + if (error != DB_SUCCESS) + return false; + + if (!use_stopword) + { + cache->stopword_info.status = STOPWORD_OFF; + goto cleanup; + } + + /* Validate and update FTS_STOPWORD_TABLE_NAME in CONFIG + table if provided */ + if (new_stopword_table && + fts_load_user_stopword( + &executor, table->fts, new_stopword_table, + &table->fts->cache->stopword_info)) + { + /* Stopword table is valid, update CONFIG table */ + str.f_n_char = 0; + str.f_str = (byte*) new_stopword_table; + str.f_len = strlen(new_stopword_table); + + error = fts_config_set_value(&executor, table, FTS_STOPWORD_TABLE_NAME, &str); + } + else fts_load_default_stopword(&cache->stopword_info); +cleanup: + if (!cache->stopword_info.cached_stopword) + cache->stopword_info.cached_stopword= + rbt_create_arg_cmp( + sizeof(fts_tokenizer_word_t), innobase_fts_text_cmp, + &my_charset_latin1); + + return true; +} + /*****************************************************************//** Initialize the table FTS stopword list @return TRUE if success */ @@ -11487,9 +11562,24 @@ innobase_fts_load_stopword( } table->fts->dict_locked= true; - bool success= fts_load_stopword(table, trx, stopword_table, - THDVAR(thd, ft_enable_stopword), false); + bool own_trx= (trx == nullptr); + if (own_trx) + { + trx= trx_create(); + trx_start_internal(trx); + } + bool success= innobase_fts_update_stopword_config( + table, trx, thd, stopword_table, + THDVAR(thd, ft_enable_stopword)); + if (own_trx) + { + if (success) + trx_commit_for_mysql(trx); + else trx->rollback(); + trx->free(); + } table->fts->dict_locked= false; + DBUG_EXECUTE_IF("fts_load_stopword_fail", success= false;); return success; } @@ -15246,7 +15336,7 @@ ha_innobase::optimize( if (m_prebuilt->table->fts && m_prebuilt->table->fts->cache && m_prebuilt->table->space) { fts_sync_table(m_prebuilt->table); - fts_optimize_table(m_prebuilt->table); + fts_optimize_table(m_prebuilt->table, thd); } try_alter = false; } diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 5d260c7f0b072..9f68928c735b4 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -49,6 +49,7 @@ Created July 18, 2007 Vasil Dimov #include "fts0types.h" #include "fts0opt.h" #include "fts0priv.h" +#include "fts0exec.h" #include "btr0btr.h" #include "page0zip.h" #include "fil0fil.h" @@ -2204,7 +2205,6 @@ i_s_fts_deleted_generic_fill( Field** fields; TABLE* table = (TABLE*) tables->table; trx_t* trx; - fts_table_t fts_table; fts_doc_ids_t* deleted; dict_table_t* user_table; @@ -2235,12 +2235,12 @@ i_s_fts_deleted_generic_fill( trx = trx_create(); trx->op_info = "Select for FTS DELETE TABLE"; - FTS_INIT_FTS_TABLE(&fts_table, - (being_deleted) ? "BEING_DELETED" : "DELETED", - FTS_COMMON_TABLE, user_table); - - fts_table_fetch_doc_ids(trx, &fts_table, deleted); + FTSQueryExecutor executor(trx, user_table); + fts_table_fetch_doc_ids( + &executor, user_table, + being_deleted ? "BEING_DELETED" : "DELETED", deleted); + trx_commit_for_mysql(trx); dict_table_close(user_table, thd, mdl_ticket); trx->free(); @@ -2660,100 +2660,26 @@ struct st_maria_plugin i_s_innodb_ft_index_cache = MariaDB_PLUGIN_MATURITY_STABLE }; -/*******************************************************************//** -Go through a FTS index auxiliary table, fetch its rows and fill +/** Go through a FTS index auxiliary table, fetch its rows and fill FTS word cache structure. +@param executor FTS query executor +@param reader record reader for processing auxiliary table records +@param selected auxiliary index +@param word word to select @return DB_SUCCESS on success, otherwise error code */ static -dberr_t -i_s_fts_index_table_fill_selected( -/*==============================*/ - dict_index_t* index, /*!< in: FTS index */ - ib_vector_t* words, /*!< in/out: vector to hold - fetched words */ - ulint selected, /*!< in: selected FTS index */ - fts_string_t* word) /*!< in: word to select */ +dberr_t i_s_fts_index_table_fill_selected( + FTSQueryExecutor *executor, AuxRecordReader &reader, + uint8_t selected, fts_string_t *word) noexcept { - pars_info_t* info; - fts_table_t fts_table; - trx_t* trx; - que_t* graph; - dberr_t error; - fts_fetch_t fetch; - char table_name[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - fetch.read_arg = words; - fetch.read_record = fts_optimize_index_fetch_node; - fetch.total_memory = 0; - - DBUG_EXECUTE_IF("fts_instrument_result_cache_limit", - fts_result_cache_limit = 8192; - ); + DBUG_EXECUTE_IF("fts_instrument_result_cache_limit", + fts_result_cache_limit = 8192;); - trx = trx_create(); - - trx->op_info = "fetching FTS index nodes"; - - pars_info_bind_function(info, "my_func", fetch.read_record, &fetch); - pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len); - - FTS_INIT_INDEX_TABLE(&fts_table, fts_get_suffix(selected), - FTS_INDEX_TABLE, index); - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - &fts_table, info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT word, doc_count, first_doc_id, last_doc_id," - " ilist\n" - " FROM $table_name WHERE word >= :word;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - for (;;) { - error = fts_eval_sql(trx, graph); - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - - break; - } else { - fts_sql_rollback(trx); - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "Lock wait timeout reading" - " FTS index. Retrying!"; + dberr_t error= executor->read_aux(selected, word, reader); - trx->error_state = DB_SUCCESS; - } else { - ib::error() << "Error occurred while reading" - " FTS index: " << error; - break; - } - } - } - - que_graph_free(graph); - - trx->free(); - - if (fetch.total_memory >= fts_result_cache_limit) { - error = DB_FTS_EXCEED_RESULT_CACHE_LIMIT; - } - - return(error); + if (error == DB_RECORD_NOT_FOUND) + error = DB_SUCCESS; + return error; } /*******************************************************************//** @@ -2890,6 +2816,90 @@ i_s_fts_index_table_fill_one_fetch( DBUG_RETURN(ret); } +/** Read words from a single auxiliary index with retry logic +@param executor FTS query executor +@param reader record reader +@param selected auxiliary index number +@param thd thread +@param tables tables to fill +@param conv_str conversion buffer +@param heap memory heap +@param words words vector +@return 0 on success, 1 on failure */ +static +int i_s_fts_read_aux_index_words( + FTSQueryExecutor *executor, AuxRecordReader &reader, + uint8_t selected, THD *thd, TABLE_LIST *tables, + fts_string_t *conv_str, mem_heap_t *heap, + ib_vector_t *words) noexcept +{ + fts_string_t word; + bool has_more= false; + int ret= 0; + + word.f_str= NULL; + word.f_len= 0; + word.f_n_char= 0; + + do + { + /* Fetch from index with retry logic for lock timeouts */ + for (;;) + { + trx_t *trx= executor->trx(); + dberr_t error= i_s_fts_index_table_fill_selected( + executor, reader, selected, &word); + + if (UNIV_LIKELY(error == DB_SUCCESS || + error == DB_FTS_EXCEED_RESULT_CACHE_LIMIT)) + { + fts_sql_commit(trx); + has_more= error == DB_FTS_EXCEED_RESULT_CACHE_LIMIT; + break; + } + else + { + fts_sql_rollback(trx); + if (error == DB_LOCK_WAIT_TIMEOUT) + { + sql_print_warning("InnoDB: Lock wait timeout" + " while reading FTS index. Retrying!"); + trx->error_state= DB_SUCCESS; + /* Clear words and retry */ + i_s_fts_index_table_free_one_fetch(words); + reader.reset_total_memory(); /* Reset memory counter on retry */ + } + else + { + sql_print_error("InnoDB: Error occurred while reading" + " FTS index: %s", ut_strerr(error)); + i_s_fts_index_table_free_one_fetch(words); + return 1; + } + } + } + + if (has_more) + { + /* Prepare start point for next fetch */ + fts_word_t *last_word= static_cast(ib_vector_last(words)); + ut_ad(last_word != NULL); + fts_string_dup(&word, &last_word->text, heap); + reader.reset_total_memory(); /* Reset memory counter for next fetch */ + } + + /* Fill into tables */ + ret= i_s_fts_index_table_fill_one_fetch( + fts_index_get_charset(const_cast(executor->index())), + thd, tables, words,conv_str, has_more); + i_s_fts_index_table_free_one_fetch(words); + if (ret != 0) + return ret; + } while (has_more); + + return 0; +} + /*******************************************************************//** Go through a FTS index and its auxiliary tables, fetch rows in each table and fill INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE. @@ -2905,8 +2915,6 @@ i_s_fts_index_table_fill_one_index( { ib_vector_t* words; mem_heap_t* heap; - CHARSET_INFO* index_charset; - dberr_t error; int ret = 0; DBUG_ENTER("i_s_fts_index_table_fill_one_index"); @@ -2917,59 +2925,34 @@ i_s_fts_index_table_fill_one_index( words = ib_vector_create(ib_heap_allocator_create(heap), sizeof(fts_word_t), 256); - index_charset = fts_index_get_charset(index); - - /* Iterate through each auxiliary table as described in - fts_index_selector */ - for (ulint selected = 0; selected < FTS_NUM_AUX_INDEX; selected++) { - fts_string_t word; - bool has_more = false; - - word.f_str = NULL; - word.f_len = 0; - word.f_n_char = 0; + trx_t* trx= trx_create(); + trx->op_info= "fetching FTS index nodes"; + FTSQueryExecutor executor(trx, index->table); + dberr_t error= executor.open_all_aux_tables(index); - do { - /* Fetch from index */ - error = i_s_fts_index_table_fill_selected( - index, words, selected, &word); + if (error) return 1; + ulint total_memory= 0; + AuxRecordReader reader(words, &total_memory); - if (error == DB_SUCCESS) { - has_more = false; - } else if (error == DB_FTS_EXCEED_RESULT_CACHE_LIMIT) { - has_more = true; - } else { - i_s_fts_index_table_free_one_fetch(words); - ret = 1; - goto func_exit; - } - - if (has_more) { - fts_word_t* last_word; + for (uint8_t selected = 0; selected < FTS_NUM_AUX_INDEX; selected++) { + reader.reset_total_memory(); + ret = i_s_fts_read_aux_index_words( + &executor, reader, selected, + thd, tables, conv_str, heap, words); - /* Prepare start point for next fetch */ - last_word = static_cast(ib_vector_last(words)); - ut_ad(last_word != NULL); - fts_string_dup(&word, &last_word->text, heap); - } - - /* Fill into tables */ - ret = i_s_fts_index_table_fill_one_fetch( - index_charset, thd, tables, words, conv_str, - has_more); - i_s_fts_index_table_free_one_fetch(words); - - if (ret != 0) { - goto func_exit; - } - } while (has_more); + if (ret != 0) { + goto func_exit; + } } func_exit: + trx->free(); mem_heap_free(heap); DBUG_RETURN(ret); } + + /*******************************************************************//** Fill the dynamic table INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE @return 0 on success, 1 on failure */ @@ -3116,7 +3099,6 @@ i_s_fts_config_fill( Field** fields; TABLE* table = (TABLE*) tables->table; trx_t* trx; - fts_table_t fts_table; dict_table_t* user_table; ulint i = 0; dict_index_t* index = NULL; @@ -3149,9 +3131,7 @@ i_s_fts_config_fill( trx = trx_create(); trx->op_info = "Select for FTS CONFIG TABLE"; - - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, user_table); - + FTSQueryExecutor executor(trx, user_table); if (!ib_vector_is_empty(user_table->fts->indexes)) { index = (dict_index_t*) ib_vector_getp_const( user_table->fts->indexes, 0); @@ -3178,7 +3158,7 @@ i_s_fts_config_fill( key_name = (char*) fts_config_key[i]; } - fts_config_get_value(trx, &fts_table, key_name, &value); + fts_config_get_value(&executor, user_table, key_name, &value); if (allocated) { ut_free(key_name); diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index 53f88cc8ca1f5..9387adb234f24 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -346,7 +346,7 @@ btr_cur_del_mark_set_clust_rec( que_thr_t* thr, /*!< in: query thread */ const dtuple_t* entry, /*!< in: dtuple for the deleting record */ mtr_t* mtr) /*!< in/out: mini-transaction */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); + MY_ATTRIBUTE((nonnull(1,2,3,4,5,7), warn_unused_result)); /*************************************************************//** Tries to compress a page of the tree if it seems useful. It is assumed that mtr holds an x-latch on the tree and on the cursor page. To avoid diff --git a/storage/innobase/include/fts0exec.h b/storage/innobase/include/fts0exec.h new file mode 100644 index 0000000000000..b8defbdcd26f7 --- /dev/null +++ b/storage/innobase/include/fts0exec.h @@ -0,0 +1,422 @@ +/***************************************************************************** + +Copyright (c) 2025, MariaDB PLC. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file include/fts0exec.h +FTS Query Builder - Abstraction layer for FTS operations + +Created 2025/10/30 +*******************************************************/ + +#pragma once + +#include "row0query.h" +#include "fts0fts.h" +#include "fts0types.h" +#include "fts0opt.h" +#include "fts0ast.h" +#include + +/** Structure to represent FTS auxiliary table data for insertion */ +struct fts_aux_data_t +{ + /** + CREATE TABLE $FTS_PREFIX_INDEX_[1-6]( + word VARCHAR(FTS_MAX_WORD_LEN), + first_doc_id INT NOT NULL, + last_doc_id UNSIGNED NOT NULL, + doc_count UNSIGNED INT NOT NULL, + ilist VARBINARY NOT NULL, + UNIQUE CLUSTERED INDEX ON (word, first_doc_id)); + */ + const char *word; + ulint word_len; + doc_id_t first_doc_id; + doc_id_t last_doc_id; + uint32_t doc_count; + const byte *ilist; + ulint ilist_len; + + fts_aux_data_t(const char *w, ulint w_len) + : word(w), word_len(w_len) + { + first_doc_id= last_doc_id= doc_count= 0; + ilist= nullptr; + ilist_len= 0; + } + + fts_aux_data_t(const char* w, ulint w_len, doc_id_t first_id, + doc_id_t last_id, uint32_t d_count, const byte *il, + ulint il_len) + : word(w), word_len(w_len), first_doc_id(first_id), + last_doc_id(last_id), doc_count(d_count), ilist(il), + ilist_len(il_len) {} +}; + +/** FTS deletion table types for m_common_tables array indexing */ +enum class FTSDeletionTable : uint8_t +{ + DELETED= 0, + DELETED_CACHE= 1, + BEING_DELETED= 2, + BEING_DELETED_CACHE= 3, + MAX_DELETION_TABLES= 4 +}; + +/** Helper to convert FTSDeletionTable to array index */ +constexpr uint8_t to_index(FTSDeletionTable table_type) noexcept +{ + return static_cast(table_type); +} + +/** Number of deletion tables */ +constexpr uint8_t NUM_DELETION_TABLES = to_index(FTSDeletionTable::MAX_DELETION_TABLES); + +/** Abstraction over QueryExecutor for FTS auxiliary/common tables. +Handles table open/lock and provides typed helpers to insert, +delete and read records in FTS INDEX_1..INDEX_6 +and deletion tables (DELETED, BEING_DELETED, etc.) */ +class FTSQueryExecutor +{ +private: + QueryExecutor m_executor; + const dict_index_t *m_index; + const dict_table_t *const m_table; + /* FTS Auxiliary table pointers */ + dict_table_t *m_aux_tables[6]={nullptr}; + /* FTS deletion table pointers (DELETED, BEING_DELETED, etc.) */ + dict_table_t *m_common_tables[NUM_DELETION_TABLES]={nullptr}; + /* FTS CONFIG table pointer */ + dict_table_t *m_config_table={nullptr}; + + /** Table preparation methods */ + + /** Open FTS INDEX_[1..6] table for the given auxiliary index. + @return DB_SUCCESS or error code */ + dberr_t open_aux_table(uint8_t aux_index) noexcept; + + /** Open a deletion table (DELETED, BEING_DELETED, etc.). + @param table_type deletion table type + @return DB_SUCCESS or error code */ + dberr_t open_deletion_table(FTSDeletionTable table_type) noexcept; + + /** Helper to convert deletion table enum to string name */ + static const char* get_deletion_table_name( + FTSDeletionTable table_type) noexcept; + + /** Table lock operation */ + + /** Acquire a lock on an opened INDEX_[1..6] table. + Retries lock wait once via QueryExecutor. + @return DB_SUCCESS or error code */ + dberr_t lock_aux_tables(uint8_t aux_index, lock_mode mode) noexcept; + + /** Acquire a lock on an opened common FTS table. + Retries lock wait once via QueryExecutor. + @return DB_SUCCESS or error code */ + dberr_t lock_common_tables(uint8_t index, lock_mode mode) noexcept; + + /** Create search tuple for a word + @param table tuple to be created for this table + @param word word to create tuple for + @param heap memory heap + @return search tuple or nullptr if word is empty */ + dtuple_t* create_word_search_tuple( + dict_table_t *table, const fts_string_t *word, + mem_heap_t *heap) noexcept; +public: + /** Open all auxiliary tables + @return DB_SUCCESS or error code */ + dberr_t open_all_aux_tables(dict_index_t *fts_index) noexcept; + + /** Open all deletion tables (DELETED, BEING_DELETED, etc.) + @return DB_SUCCESS or error code */ + dberr_t open_all_deletion_tables() noexcept; + + /** Open FTS CONFIG table for configuration operations. + @return DB_SUCCESS or error code */ + dberr_t open_config_table() noexcept; + + /** Set CONFIG table directly (for cases where table is already opened) + @param config_table CONFIG table pointer */ + void set_config_table(dict_table_t *config_table) noexcept + { + m_config_table= config_table; + m_config_table->acquire(); + } + + /** Create executor bound to trx and FTS table/index. + @param trx transaction + @param fts_table FTS table */ + FTSQueryExecutor(trx_t *trx, const dict_table_t *fts_table); + + /** Release any opened table handles and executor resources. */ + ~FTSQueryExecutor(); + + /** High level DML operation on FTS TABLE */ + + /** Insert a row into auxiliary INDEX_[1..6] table. + Expects (word, first_doc_id, trx_id, roll_ptr, last_doc_id, + doc_count, ilist). + @param aux_index auxiliary table index + @param aux_data data to be inserted + @return DB_SUCCESS or error code */ + dberr_t insert_aux_record(uint8_t aux_index, + const fts_aux_data_t *aux_data) noexcept; + + /** Insert a single doc_id into a common table (e.g. DELETED, ...) + @param tbl_name common table name + @param doc_id document id to be inserted + @return DB_SUCCESS or error code */ + dberr_t insert_common_record(const char *tbl_name, doc_id_t doc_id) noexcept; + + /** Insert a key/value into CONFIG table. + @param key key for the config table + @param value value for the key + @return DB_SUCCESS or error code */ + dberr_t insert_config_record(const char *key, const char *value) noexcept; + + /** Delete one word row from INDEX_[1..6] by (word). + @param aux_index auxiliary table index + @param aux_data auxiliary table record + @return DB_SUCCESS or error code */ + dberr_t delete_aux_record(uint8_t aux_index, + const fts_aux_data_t *aux_data) noexcept; + + /** Delete a single doc_id row from a common table by (doc_id). + @param tbl_name common table name + @param doc_id document id to be deleted + @return DB_SUCCESS or error code */ + dberr_t delete_common_record(const char *tbl_name, doc_id_t doc_id) noexcept; + + /** Delete all rows from a common table. + @return DB_SUCCESS or error code */ + dberr_t delete_all_common_records(const char *tbl_name) noexcept; + + /** Delete a key from CONFIG table by (key). + @return DB_SUCCESS or error code */ + dberr_t delete_config_record(const char *key) noexcept; + + /** Upsert a key/value in CONFIG table. + Replaces 'value' if key exists, inserts otherwise. + @return DB_SUCCESS or error code */ + dberr_t update_config_record(const char *key, const char *value) noexcept; + + /** Select-for-update CONFIG row by 'key' + @return DB_SUCCESS or error code */ + dberr_t read_config_with_lock( + const char *key, RecordCallback &callback) noexcept; + + /** Read all rows from given COMMON table + Callback is invoked for processing the record */ + dberr_t read_all_common(const char *tbl_name, + RecordCallback &callback) noexcept; + mem_heap_t *get_heap() const noexcept + { return m_executor.get_heap(); } + + trx_t *trx() const noexcept + { return m_executor.get_trx(); } + + const dict_index_t* index() const noexcept + { return m_index; } + + void release_lock() + { + m_executor.commit_mtr(); + } + + /** Read records by index using underlying QueryExecutor + @param table table to read + @param sec_index secondary index + @param search_tuple search tuple + @param mode cursor mode + @param callback record callback + @return DB_SUCCESS or error code */ + dberr_t read_by_index(dict_table_t *table, dict_index_t *index, + const dtuple_t *search_tuple, page_cur_mode_t mode, + RecordCallback& callback) noexcept + { + dberr_t err= m_executor.read_by_index(table, index, search_tuple, + mode, callback, true); + return (err == DB_SUCCESS_LOCKED_REC) ? DB_SUCCESS : err; + } + + /** Construct FTS auxiliary table name + @param table_name output buffer for table name + @param suffix table suffix (e.g., "CONFIG", "INDEX_1") + @param common_table true for common tables, false for index tables */ + void construct_table_name(char *table_name, const char *suffix, + bool common_table) noexcept; + + dberr_t read_aux(uint8_t aux_index, + const fts_string_t* start_word, + RecordCallback& callback) noexcept; +}; + +/** Callback class for reading common table records +(DELETED, BEING_DELETED, DELETED_CACHE, BEING_DELETED_CACHE) */ +class CommonTableReader : public RecordCallback +{ +private: + std::vector doc_ids; + +public: + CommonTableReader(); + + const std::vector& get_doc_ids() const { return doc_ids; } + void clear() { doc_ids.clear(); } + + /** Fast common table field extraction for known table format. + Structure: (doc_id BIGINT UNSIGNED) - always known schema + @param rec record to extract from + @param index common table index + @param doc_id output doc_id value + @return true if extraction successful */ + static bool extract_common_fields( + const rec_t *rec, const dict_index_t *index, + doc_id_t *doc_id); +}; + +/** Callback class for reading FTS config table records */ +class ConfigReader : public RecordCallback +{ +public: + span value_span; + ConfigReader(); + + /** Extract the config table record. + Structure: (key VARCHAR, db_trx_id, db_roll_ptr, value TEXT) + @param rec record to extract from + @param index config table index + @param key_data output pointer to key data + @param key_len output key length + @param value_data output pointer to value data (optional) + @param value_len output value length (optional) + @return true if extraction successful */ + static bool extract_config_fields( + const rec_t *rec, const dict_index_t *index, + const byte **key_data, ulint *key_len, + const byte **value_data = nullptr, ulint *value_len = nullptr); + + /** Direct config key comparison - compares first field with tuple value. + @param search_tuple search tuple containing target key + @param rec record to compare + @param index config table index + @return comparison action */ + static RecordCompareAction compare_config_key( + const dtuple_t *search_tuple, const rec_t *rec, + const dict_index_t *index, const rec_offs *offsets); +}; + +/** Type alias for FTS record processor function */ +using FTSRecordProcessor= std::function< + dberr_t(const rec_t*, const dict_index_t*, const rec_offs*, void*)>; + +/** Comparison modes for AuxRecordReader */ +enum class AuxCompareMode +{ + /** >= comparison (range scan from word) */ + GREATER_EQUAL, + /** > comparison (exclude exact match) */ + GREATER, + /** LIKE pattern matching (prefix match) */ + LIKE, + /** = comparison (exact match) */ + EQUAL +}; + +/** Callback class for reading FTS auxiliary index table records */ +class AuxRecordReader : public RecordCallback +{ +private: + void *user_arg; + ulint *total_memory; + AuxCompareMode compare_mode; + fts_string_t m_last_word; + mem_heap_t* m_word_heap= nullptr; + +private: + /** FTS-specific record comparison logic */ + RecordCompareAction compare_record( + const dtuple_t *search_tuple, const rec_t *rec, + const dict_index_t *index, const rec_offs *offsets) noexcept; + + RecordComparator make_comparator() + { + return [this](const dtuple_t* search_tuple, const rec_t* rec, + const dict_index_t* index, const rec_offs* offsets) + -> RecordCompareAction + { + return this->compare_record(search_tuple, rec, index, offsets); + }; + } + +public: + /** Default word processor for FTS auxiliary table records */ + dberr_t default_word_processor(const rec_t *rec, const dict_index_t *index, + const rec_offs *offsets, void *user_arg); + + /* Constructor with custom processor */ + template + AuxRecordReader(void* user_data, + ProcessorFunc proc_func, + AuxCompareMode mode= AuxCompareMode::GREATER_EQUAL) + : RecordCallback( + [this, proc_func](const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets) -> dberr_t + { + return proc_func(rec, index, offsets, this->user_arg); + }, + make_comparator() + ), + user_arg(user_data), total_memory(nullptr), + compare_mode(mode), m_last_word{} {} + + /* Different constructor with default word processing */ + AuxRecordReader(void *user_data, ulint *memory_counter, + AuxCompareMode mode= AuxCompareMode::GREATER_EQUAL) + : RecordCallback( + [this](const rec_t *rec, const dict_index_t* index, + const rec_offs *offsets) -> dberr_t + { + return this->default_word_processor(rec, index, offsets, + this->user_arg); + }, + make_comparator() + ), + user_arg(user_data), total_memory(memory_counter), + compare_mode(mode), m_last_word{} {} + + ~AuxRecordReader() + { + if (m_word_heap) + { + mem_heap_free(m_word_heap); + m_word_heap = nullptr; + } + } + /** Reset total memory counter */ + void reset_total_memory() { if (total_memory) *total_memory = 0; } + + /** Get the last processed word (for pagination) + @return reference to last processed word */ + const fts_string_t &get_last_word() const noexcept + { + return m_last_word; + } +}; diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 9dd48e525194d..6b00de10c8ad5 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -40,6 +40,9 @@ Created 2011/09/02 Sunny Bains #include "mysql/plugin_ftparser.h" #include "lex_string.h" +/* Forward declarations */ +class FTSQueryExecutor; + /** "NULL" value of a document id. */ #define FTS_NULL_DOC_ID 0 @@ -611,13 +614,12 @@ fts_create( dict_table_t* table); /*!< out: table with FTS indexes */ -/*********************************************************************//** -Run OPTIMIZE on the given table. -@return DB_SUCCESS if all OK */ +/** Run OPTIMIZE on the given table. +@param table table to be optimized +@param thd thread +@return DB_SUCCESS if all ok */ dberr_t -fts_optimize_table( -/*===============*/ - dict_table_t* table); /*!< in: table to optimiza */ +fts_optimize_table(dict_table_t *table, THD *thd); /**********************************************************************//** Startup the optimize thread and create the work queue. */ @@ -780,14 +782,6 @@ fts_tokenize_document_internal( const char* doc, /*!< in: document to tokenize */ int len); /*!< in: document length */ -/*********************************************************************//** -Fetch COUNT(*) from specified table. -@return the number of rows in the table */ -ulint -fts_get_rows_count( -/*===============*/ - fts_table_t* fts_table); /*!< in: fts table to read */ - /*************************************************************//** Get maximum Doc ID in a table if index "FTS_DOC_ID_INDEX" exists @return max Doc ID or 0 if index "FTS_DOC_ID_INDEX" does not exist */ @@ -804,33 +798,24 @@ fts_get_max_doc_id( CHARSET_INFO *fts_valid_stopword_table(const char *stopword_table_name, const char **row_end= NULL); -/****************************************************************//** -This function loads specified stopword into FTS cache +/** This function loads specified stopword into FTS cache +@param executor FTSQueryExecutor instance +@param table table which has fts index @return true if success */ -bool -fts_load_stopword( -/*==============*/ - const dict_table_t* - table, /*!< in: Table with FTS */ - trx_t* trx, /*!< in: Transaction */ - const char* session_stopword_table, /*!< in: Session stopword table - name */ - bool stopword_is_on, /*!< in: Whether stopword - option is turned on/off */ - bool reload); /*!< in: Whether it is during - reload of FTS table */ +bool fts_load_stopword(FTSQueryExecutor *executor, + const dict_table_t *table) noexcept; + +/** Read the rows from the fulltext index +@param executor FTSQueryExecutor instance +@param table Fulltext table +@param tbl_name table name +@param doc_ids collecting doc ids +@return DB_SUCCESS or error code */ +dberr_t fts_table_fetch_doc_ids(FTSQueryExecutor *executor, + dict_table_t *table, const char *tbl_name, + fts_doc_ids_t *doc_ids) noexcept; /****************************************************************//** -Read the rows from the FTS index -@return DB_SUCCESS if OK */ -dberr_t -fts_table_fetch_doc_ids( -/*====================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: aux table */ - fts_doc_ids_t* doc_ids); /*!< in: For collecting - doc ids */ -/****************************************************************//** This function brings FTS index in sync when FTS index is first used. There are documents that have not yet sync-ed to auxiliary tables from last server abnormally shutdown, we will need to bring @@ -918,16 +903,56 @@ bool fts_check_aux_table(const char *name, /** Update the last document id. This function could create a new transaction to update the last document id. -@param table table to be updated -@param doc_id last document id -@param trx update trx or null +@param executor query executor +@param table table to be updated +@param doc_id last document id @retval DB_SUCCESS if OK */ dberr_t -fts_update_sync_doc_id(const dict_table_t *table, - doc_id_t doc_id, - trx_t *trx) - MY_ATTRIBUTE((nonnull(1))); +fts_update_sync_doc_id(FTSQueryExecutor *executor, + const dict_table_t *table, + doc_id_t doc_id) noexcept; /** Sync the table during commit phase @param[in] table table to be synced */ void fts_sync_during_ddl(dict_table_t* table); + +/** Tokenize a document. +@param[in,out] doc document to tokenize +@param[out] result tokenization result +@param[in] parser pluggable parser */ +void fts_tokenize_document( + fts_doc_t* doc, + fts_doc_t* result, + st_mysql_ftparser* parser); + +/** Continue to tokenize a document. +@param[in,out] doc document to tokenize +@param[in] add_pos add this position to all tokens from this tokenization +@param[out] result tokenization result +@param[in] parser pluggable parser */ +void fts_tokenize_document_next( + fts_doc_t* doc, + ulint add_pos, + fts_doc_t* result, + st_mysql_ftparser* parser); + +/** Get a character set based on precise type. +@param prtype precise type +@return the corresponding character set */ +CHARSET_INFO* fts_get_charset(ulint prtype); + +/** Load user defined stopword from designated user table +@param fts fulltext structure +@param stopword_table stopword table +@param stopword_info stopword information +@return whether the operation is successful */ +bool fts_load_user_stopword(FTSQueryExecutor *executor, fts_t *fts, + const char *stopword_table, + fts_stopword_t *stopword_info) noexcept; + +/****************************************************************//** +This function loads the default InnoDB stopword list */ +void +fts_load_default_stopword( +/*======================*/ + fts_stopword_t* stopword_info); /*!< in: stopword info */ diff --git a/storage/innobase/include/fts0priv.h b/storage/innobase/include/fts0priv.h index 1d3bc323841b9..237f7acda674d 100644 --- a/storage/innobase/include/fts0priv.h +++ b/storage/innobase/include/fts0priv.h @@ -28,10 +28,10 @@ Created 2011/09/02 Sunny Bains #define INNOBASE_FTS0PRIV_H #include "dict0dict.h" -#include "pars0pars.h" #include "que0que.h" #include "que0types.h" #include "fts0types.h" +#include "fts0exec.h" /* The various states of the FTS sub system pertaining to a table with FTS indexes defined on it. */ @@ -106,26 +106,6 @@ component. /** Maximum length of an integer stored in the config table value column. */ #define FTS_MAX_INT_LEN 32 -/******************************************************************//** -Parse an SQL string. %s is replaced with the table's id. -@return query graph */ -que_t* -fts_parse_sql( -/*==========*/ - fts_table_t* fts_table, /*!< in: FTS aux table */ - pars_info_t* info, /*!< in: info struct, or NULL */ - const char* sql) /*!< in: SQL string to evaluate */ - MY_ATTRIBUTE((nonnull(3), malloc, warn_unused_result)); -/******************************************************************//** -Evaluate a parsed SQL statement -@return DB_SUCCESS or error code */ -dberr_t -fts_eval_sql( -/*=========*/ - trx_t* trx, /*!< in: transaction */ - que_t* graph) /*!< in: Parsed statement */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); - /** Construct the name of an internal FTS table for the given table. @param[in] fts_table metadata on fulltext-indexed table @param[out] table_name a name up to MAX_FULL_NAME_LEN @@ -133,77 +113,15 @@ fts_eval_sql( void fts_get_table_name(const fts_table_t* fts_table, char* table_name, bool dict_locked = false) MY_ATTRIBUTE((nonnull)); -/******************************************************************//** -Construct the column specification part of the SQL string for selecting the -indexed FTS columns for the given table. Adds the necessary bound -ids to the given 'info' and returns the SQL string. Examples: -One indexed column named "text": - - "$sel0", - info/ids: sel0 -> "text" - -Two indexed columns named "subject" and "content": - - "$sel0, $sel1", - info/ids: sel0 -> "subject", sel1 -> "content", -@return heap-allocated WHERE string */ -const char* -fts_get_select_columns_str( -/*=======================*/ - dict_index_t* index, /*!< in: FTS index */ - pars_info_t* info, /*!< in/out: parser info */ - mem_heap_t* heap) /*!< in: memory heap */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); - -/** define for fts_doc_fetch_by_doc_id() "option" value, defines whether -we want to get Doc whose ID is equal to or greater or smaller than supplied -ID */ -#define FTS_FETCH_DOC_BY_ID_EQUAL 1 -#define FTS_FETCH_DOC_BY_ID_LARGE 2 -#define FTS_FETCH_DOC_BY_ID_SMALL 3 - -/*************************************************************//** -Fetch document (= a single row's indexed text) with the given -document id. -@return: DB_SUCCESS if fetch is successful, else error */ -dberr_t -fts_doc_fetch_by_doc_id( -/*====================*/ - fts_get_doc_t* get_doc, /*!< in: state */ - doc_id_t doc_id, /*!< in: id of document to fetch */ - dict_index_t* index_to_use, /*!< in: caller supplied FTS index, - or NULL */ - ulint option, /*!< in: search option, if it is - greater than doc_id or equal */ - fts_sql_callback - callback, /*!< in: callback to read - records */ - void* arg) /*!< in: callback arg */ - MY_ATTRIBUTE((nonnull(6))); - -/*******************************************************************//** -Callback function for fetch that stores the text of an FTS document, -converting each column to UTF-16. -@return always FALSE */ -ibool -fts_query_expansion_fetch_doc( -/*==========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts_doc_t* */ - MY_ATTRIBUTE((nonnull)); -/******************************************************************** -Write out a single word's data as new entry/entries in the INDEX table. -@return DB_SUCCESS if all OK. */ -dberr_t -fts_write_node( -/*===========*/ - trx_t* trx, /*!< in: transaction */ - que_t** graph, /*!< in: query graph */ - fts_table_t* fts_table, /*!< in: the FTS aux index */ - fts_string_t* word, /*!< in: word in UTF-8 */ - fts_node_t* node) /*!< in: node columns */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); +/** Write out a single word's data as new entry/entries in the INDEX table. +@param executor FTS Query Executor +@param selected auxiliary index number +@param aux_data auxiliary table data +@return DB_SUCCESS if all OK or error code */ +dberr_t fts_write_node(FTSQueryExecutor *executor, uint8_t selected, + const fts_aux_data_t *aux_data) noexcept + MY_ATTRIBUTE((nonnull, warn_unused_result)); /** Check if a fts token is a stopword or less than fts_min_token_size or greater than fts_max_token_size. @@ -252,35 +170,36 @@ fts_word_free( /*==========*/ fts_word_t* word) /*!< in: instance to free.*/ MY_ATTRIBUTE((nonnull)); -/******************************************************************//** -Read the rows from the FTS inde -@return DB_SUCCESS or error code */ -dberr_t -fts_index_fetch_nodes( -/*==================*/ - trx_t* trx, /*!< in: transaction */ - que_t** graph, /*!< in: prepared statement */ - fts_table_t* fts_table, /*!< in: FTS aux table */ - const fts_string_t* - word, /*!< in: the word to fetch */ - fts_fetch_t* fetch) /*!< in: fetch callback.*/ - MY_ATTRIBUTE((nonnull)); + +/** Read the rows from FTS index +@param trx transaction +@param index fulltext index +@param word word to fetch +@param user_arg user argument +@param processor custom processor to filter the word from record +@param compare_mode comparison mode for record matching +@return error code or DB_SUCCESS */ +dberr_t fts_index_fetch_nodes(FTSQueryExecutor *executor, dict_index_t *index, + const fts_string_t *word, + void *user_arg, + FTSRecordProcessor processor, + AuxCompareMode compare_mode) noexcept + MY_ATTRIBUTE((nonnull)); + #define fts_sql_commit(trx) trx_commit_for_mysql(trx) #define fts_sql_rollback(trx) (trx)->rollback() -/******************************************************************//** -Get value from config table. The caller must ensure that enough -space is allocated for value to hold the column contents + +/** Get value from the config table. The caller must ensure that enough +space is allocated for value to hold the column contents. +@param trx transaction +@param table Indexed fts table +@param name name of the key +@param value value of the key @return DB_SUCCESS or error code */ -dberr_t -fts_config_get_value( -/*=================*/ - trx_t* trx, /* transaction */ - fts_table_t* fts_table, /*!< in: the indexed FTS table */ - const char* name, /*!< in: get config value for - this parameter name */ - fts_string_t* value) /*!< out: value read from - config table */ - MY_ATTRIBUTE((nonnull)); +dberr_t fts_config_get_value(FTSQueryExecutor *executor, const dict_table_t *table, + const char *name, fts_string_t *value) noexcept + MY_ATTRIBUTE((nonnull)); + /******************************************************************//** Get value specific to an FTS index from the config table. The caller must ensure that enough space is allocated for value to hold the @@ -289,36 +208,36 @@ column contents. dberr_t fts_config_get_index_value( /*=======================*/ - trx_t* trx, /*!< transaction */ + FTSQueryExecutor* executor, /*!< in: query executor */ dict_index_t* index, /*!< in: index */ const char* param, /*!< in: get config value for this parameter name */ fts_string_t* value) /*!< out: value read from config table */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); + noexcept MY_ATTRIBUTE((nonnull, warn_unused_result)); /******************************************************************//** Set the value in the config table for name. @return DB_SUCCESS or error code */ dberr_t fts_config_set_value( /*=================*/ - trx_t* trx, /*!< transaction */ - fts_table_t* fts_table, /*!< in: the indexed FTS table */ + FTSQueryExecutor* executor, /*!< in: query executor */ + const dict_table_t* table, /*!< in: the indexed FTS table */ const char* name, /*!< in: get config value for this parameter name */ const fts_string_t* value) /*!< in: value to update */ - MY_ATTRIBUTE((nonnull)); + noexcept MY_ATTRIBUTE((nonnull)); /****************************************************************//** Set an ulint value in the config table. @return DB_SUCCESS if all OK else error code */ dberr_t fts_config_set_ulint( /*=================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: the indexed FTS table */ + FTSQueryExecutor* executor, /*!< in: query executor */ + const dict_table_t* table, /*!< in: the indexed FTS table */ const char* name, /*!< in: param name */ - ulint int_value) /*!< in: value */ + ulint int_value)noexcept /*!< in: value */ MY_ATTRIBUTE((nonnull, warn_unused_result)); /******************************************************************//** Set the value specific to an FTS index in the config table. @@ -326,26 +245,15 @@ Set the value specific to an FTS index in the config table. dberr_t fts_config_set_index_value( /*=======================*/ - trx_t* trx, /*!< transaction */ + FTSQueryExecutor* executor, /*!< in: query executor */ dict_index_t* index, /*!< in: index */ const char* param, /*!< in: get config value for this parameter name */ - fts_string_t* value) /*!< out: value read from + fts_string_t* value)noexcept /*!< out: value read from config table */ MY_ATTRIBUTE((nonnull, warn_unused_result)); /******************************************************************//** -Get an ulint value from the config table. -@return DB_SUCCESS or error code */ -dberr_t -fts_config_get_ulint( -/*=================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: the indexed FTS table */ - const char* name, /*!< in: param name */ - ulint* int_value) /*!< out: value */ - MY_ATTRIBUTE((nonnull)); -/******************************************************************//** Search cache for word. @return the word node vector if found else NULL */ const ib_vector_t* diff --git a/storage/innobase/include/fts0types.h b/storage/innobase/include/fts0types.h index 17b0f947de277..fc9b5c8c5321c 100644 --- a/storage/innobase/include/fts0types.h +++ b/storage/innobase/include/fts0types.h @@ -313,7 +313,7 @@ fts_get_suffix( @param[in] len string length in bytes @return the index to use for the string */ UNIV_INLINE -ulint +uint8_t fts_select_index( const CHARSET_INFO* cs, const byte* str, diff --git a/storage/innobase/include/fts0types.inl b/storage/innobase/include/fts0types.inl index 622d1337bad60..d172ab38ea42c 100644 --- a/storage/innobase/include/fts0types.inl +++ b/storage/innobase/include/fts0types.inl @@ -102,13 +102,13 @@ inline bool fts_is_charset_cjk(const CHARSET_INFO* cs) @param[in] len string length @retval the index to use for the string */ UNIV_INLINE -ulint +uint8_t fts_select_index_by_range( const CHARSET_INFO* cs, const byte* str, ulint len) { - ulint selected = 0; + uint8_t selected = 0; ulint value = innobase_strnxfrm(cs, str, len); while (fts_index_selector[selected].value != 0) { @@ -136,7 +136,7 @@ fts_select_index_by_range( @param[in] len string length @retval the index to use for the string */ UNIV_INLINE -ulint +uint8_t fts_select_index_by_hash( const CHARSET_INFO* cs, const byte* str, @@ -162,7 +162,7 @@ fts_select_index_by_hash( /* Get collation hash code */ my_ci_hash_sort(&hasher, cs, str, char_len); - return(hasher.m_nr1 % FTS_NUM_AUX_INDEX); + return static_cast(hasher.m_nr1 % FTS_NUM_AUX_INDEX); } /** Select the FTS auxiliary index for the given character. @@ -171,21 +171,17 @@ fts_select_index_by_hash( @param[in] len string length in bytes @retval the index to use for the string */ UNIV_INLINE -ulint +uint8_t fts_select_index( const CHARSET_INFO* cs, const byte* str, ulint len) { - ulint selected; - if (fts_is_charset_cjk(cs)) { - selected = fts_select_index_by_hash(cs, str, len); - } else { - selected = fts_select_index_by_range(cs, str, len); + return fts_select_index_by_hash(cs, str, len); } - return(selected); + return fts_select_index_by_range(cs, str, len); } /******************************************************************//** diff --git a/storage/innobase/include/page0cur.h b/storage/innobase/include/page0cur.h index 14a3ce9ba038c..ab871d7ef8511 100644 --- a/storage/innobase/include/page0cur.h +++ b/storage/innobase/include/page0cur.h @@ -244,6 +244,21 @@ bool page_cur_search_with_match_bytes(const dtuple_t &tuple, uint16_t *ilow_bytes) noexcept; +/** Compare a data tuple to a physical record. +@param rec B-tree index record +@param index index B-tree +@param tuple search key +@param match matched fields << 16 | bytes +@param comp nonzero if ROW_FORMAT=REDUNDANT is not being used +@return the comparison result of dtuple and rec +@retval 0 if dtuple is equal to rec +@retval negative if dtuple is less than rec +@retval positive if dtuple is greater than rec */ +int cmp_dtuple_rec_bytes(const rec_t *rec, + const dict_index_t &index, + const dtuple_t &tuple, int *match, ulint comp) + noexcept; + /***********************************************************//** Positions a page cursor on a randomly chosen user record on a page. If there are no user records, sets the cursor on the infimum record. */ diff --git a/storage/innobase/include/row0mysql.h b/storage/innobase/include/row0mysql.h index 2f50aa1560eb2..6887e14da4bab 100644 --- a/storage/innobase/include/row0mysql.h +++ b/storage/innobase/include/row0mysql.h @@ -185,6 +185,7 @@ row_create_prebuilt( the MySQL format */ /** Free a prebuilt struct for a TABLE handle. */ void row_prebuilt_free(row_prebuilt_t *prebuilt); + /*********************************************************************//** Updates the transaction pointers in query graphs stored in the prebuilt struct. */ @@ -556,6 +557,9 @@ struct row_prebuilt_t { and updates */ btr_pcur_t* clust_pcur; /*!< persistent cursor used in some selects and updates */ + mtr_t* mtr; /*!< external mini-transaction for + DML operations; when set, row_search_mvcc + will use this instead of creating its own */ que_fork_t* sel_graph; /*!< dummy query graph used in selects */ dtuple_t* search_tuple; /*!< prebuilt dtuple used in selects */ diff --git a/storage/innobase/include/row0query.h b/storage/innobase/include/row0query.h new file mode 100644 index 0000000000000..625533b3af105 --- /dev/null +++ b/storage/innobase/include/row0query.h @@ -0,0 +1,244 @@ +/***************************************************************************** + +Copyright (c) 2025, MariaDB PLC. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file include/row0query.h +General Query Executor + +Created 2025/10/30 +*******************************************************/ + +#pragma once + +#include "btr0pcur.h" +#include +#include "dict0types.h" +#include "data0types.h" +#include "db0err.h" +#include "lock0types.h" +#include "rem0rec.h" + +struct row_prebuilt_t; + +/** Comparator action for deciding how to treat a record */ +enum class RecordCompareAction +{ + /** Process this record via process_record */ + PROCESS, + /** Do not process this record, continue traversal */ + SKIP, + /** Stop traversal immediately */ + STOP +}; + +using RecordProcessor= std::function; + +using RecordComparator= std::function< + RecordCompareAction(const dtuple_t*, const rec_t*, + const dict_index_t*, const rec_offs*)>; + +/** Record processing callback interface using std::function. +Can be used by FTS, stats infrastructure, and other components +that need to process database records with custom logic. */ +class RecordCallback +{ +public: + /** Constructor with processor function and optional comparator + @param processor Function to process each record + @param comparator Optional function to filter records */ + RecordCallback( + RecordProcessor processor, + RecordComparator comparator= nullptr) + : process_record(processor), compare_record(comparator) {} + + virtual ~RecordCallback()= default; + + /** Called for each matching record */ + const RecordProcessor process_record; + + /** Comparison function for custom filtering */ + const RecordComparator compare_record; +}; + +/** General-purpose MVCC-aware record traversal and basic +DML executor. Provides a thin abstraction over B-tree cursors for +reading and mutating records with consistent-read (MVCC) handling, +and callback API. +- Open and iterate clustered/secondary indexes with page cursors. +- Build consistent-read versions when needed via transaction +read views. +- Filter and process records using RecordCallback: + - compare_record: decide SKIP/PROCESS/STOP for each record + - process_record: handle visible records; return DB_SUCCESS to continue, DB_SUCCESS_LOCKED_REC to stop +- Basic DML helpers (insert/delete/replace) and table locking. */ +class QueryExecutor +{ +private: + que_thr_t *m_thr; + btr_pcur_t m_pcur; + btr_pcur_t *m_clust_pcur; + trx_t *m_trx; + mtr_t *m_mtr; + mem_heap_t *m_heap; + + /* Prebuilt for row_search_mvcc call */ + row_prebuilt_t *m_prebuilt; + + /* Setup prebuilt structure for row_search_mvcc usage + @param table table to search + @param index index to use + @param tuple search tuple + @param mode search mode */ + void setup_prebuilt(dict_table_t *table, dict_index_t *index, + const dtuple_t *tuple, page_cur_mode_t mode) noexcept; + +public: + QueryExecutor(trx_t *trx); + ~QueryExecutor(); + + /** Insert a record in clustered index of the table + @param table table to be inserted + @param tuple tuple to be inserted + @return DB_SUCCESS on success, error code on failure */ + dberr_t insert_record(dict_table_t *table, dtuple_t *tuple) noexcept; + + /** Delete a record from the clustered index of the table + @param table table to be inserted + @param tuple tuple to be inserted + @return DB_SUCCESS on success, error code on failure */ + dberr_t delete_record(dict_table_t *table, dtuple_t *tuple) noexcept; + + /** Delete all records from the clustered index of the table + @param table table be be deleted + @return DB_SUCCESS on success, error code on failure */ + dberr_t delete_all(dict_table_t *table) noexcept; + + /** Acquire and lock a single clustered record for update + Performs a keyed lookup on the clustered index, validates MVCC visibility, + and acquires an X lock on the matching record. + @param[in] table Table containing the record + @param[in] search_tuple Exact key for clustered index lookup + @param[in] callback Optional record callback + @return DB_SUCCESS on successful lock + DB_RECORD_NOT_FOUND if no visible matching record + DB_LOCK_WAIT if waiting was required + error code on failure */ + dberr_t select_for_update(dict_table_t *table, dtuple_t *search_tuple, + RecordCallback *callback= nullptr) noexcept; + + /** Update the currently selected clustered record within an active mtr. + Attempts in-place update; falls back to optimistic/pessimistic update if needed, + including external field storage when required. + select_for_update() has positioned and locked m_pcur on the target row. + + The caller is responsible for committing or rolling back m_mtr after this call + @param[in] table target table + @param[in] update update descriptor (fields, new values) + @return DB_SUCCESS on success + DB_OVERFLOW/DB_UNDERFLOW during size-changing paths + error_code on failures */ + dberr_t update_record(dict_table_t *table, const upd_t *update) noexcept; + + + /** Try to update a record by key or insert if not found. + Performs a SELECT ... FOR UPDATE using search_tuple; + if found, updates the row; otherwise inserts a new record. + Note: + On update path, commits or rolls back the active mtr as needed. + On insert path, no active mtr remains upon return + @param[in] table target table + @param[in] search_tuple key identifying the target row + @param[in] update update descriptor (applied when found) + @param[in] insert_tuple tuple to insert when not found + @return DB_SUCCESS on successful update or insert + @retval DB_LOCK_WAIT to be retried, + @return error code on failure */ + dberr_t replace_record(dict_table_t *table, dtuple_t *search_tuple, + const upd_t *update, dtuple_t *insert_tuple) noexcept; + + /** Iterate clustered index records and process via callback. + Handles full table scan and index scan for range/select queries + Calls callback.compare_record() to decide SKIP/PROCESS/STOP for + each matching record. On PROCESS, invokes + callback.process_record() on an MVCC-visible version. + @param table table to read + @param tuple optional search key (range/point). nullptr => full scan + @param mode B-tree search mode (e.g., PAGE_CUR_GE) + @param callback record comparator/processor + @return DB_SUCCESS if at least one record was processed + @retval DB_RECORD_NOT_FOUND if no record matched + @return error code on failure */ + dberr_t read(dict_table_t *table, const dtuple_t *tuple, + page_cur_mode_t mode, + RecordCallback& callback) noexcept; + + /** Iterate all records in clustered index and process via callback. + Performs a full table scan without any search key. + Calls callback.compare_record() to decide SKIP/PROCESS/STOP for + each record. On PROCESS, invokes callback.process_record() on an + MVCC-visible version. + + @param table table to scan + @param callback record comparator/processor + @return DB_SUCCESS if at least one record was processed + @retval DB_RECORD_NOT_FOUND if no record matched + @return error code on failure */ + dberr_t read_all(dict_table_t *table, RecordCallback &callback, + const dtuple_t *start_tuple= nullptr) noexcept; + + /** Read records via a secondary index and process corresponding + clustered rows. Performs a range or point scan on the given secondary index, + filters secondary records with callback.compare_record(), then looks up + the matching clustered record and invokes callback.process_record() + on a MVCC-visible version. + + @param table Table to read + @param sec_index Secondary index used for traversal + @param search_tuple search key or nullptr for full scan + @param mode Cursor search mode + @param callback RecordCallback with comparator+processor + @return DB_SUCCESS on success + DB_RECORD_NOT_FOUND if no matching record was processed + error code on failure */ + dberr_t read_by_index(dict_table_t *table, dict_index_t *sec_index, + const dtuple_t *search_tuple, + page_cur_mode_t mode, + RecordCallback &callback, + bool all_read) noexcept; + + /** Acquire a table lock in the given mode for transaction. + @param table table to lock + @param mode lock mode + @return DB_SUCCESS, DB_LOCK_WAIT or error code */ + dberr_t lock_table(dict_table_t *table, lock_mode mode) noexcept; + + /** Handle a lock wait for the current transaction and thread context. + @param err the lock-related error to handle (e.g., DB_LOCK_WAIT) + @param table_lock true if the wait originated from table lock, else row lock + @return DB_SUCCESS if the wait completed successfully and lock was granted + @retval DB_LOCK_WAIT_TIMEOUT if timed out */ + dberr_t handle_wait(dberr_t err, bool table_lock) noexcept; + mem_heap_t *get_heap() const { return m_heap; } + trx_t *get_trx() const { return m_trx; } + void commit_mtr() noexcept + { + if (m_mtr) + m_mtr->commit(); + } +}; diff --git a/storage/innobase/include/row0sel.h b/storage/innobase/include/row0sel.h index 35e3cbe66315c..952a6b747af30 100644 --- a/storage/innobase/include/row0sel.h +++ b/storage/innobase/include/row0sel.h @@ -35,6 +35,7 @@ Created 12/19/1997 Heikki Tuuri #include "pars0sym.h" #include "btr0pcur.h" #include "row0mysql.h" +#include "row0query.h" /*********************************************************************//** Creates a select node struct. @@ -118,6 +119,18 @@ row_sel_convert_mysql_key_to_innobase( ulint key_len) /*!< in: MySQL key value length */ MY_ATTRIBUTE((nonnull(1,4,5))); +/* Forward declarations for callback policy classes */ +template +struct InnoDBPolicy; + +/* Type aliases for different row_search_mvcc use cases */ +/** MySQL format conversion and manages its own mtr */ +using MySQLRowCallback= InnoDBPolicy; +/** Used for internal read purpose */ +using InnoDBReadPolicy= InnoDBPolicy; +/** Used for internal update/delete purpose */ +using InnoDBDMLPolicy= InnoDBPolicy; + /** Search for rows in the database using cursor. Function is mainly used for tables that are shared across connections and so it employs technique that can help re-construct the rows that @@ -138,6 +151,7 @@ It also has optimization such as pre-caching the rows, using AHI, etc. cursor 'direction' should be 0. @return DB_SUCCESS, DB_RECORD_NOT_FOUND, DB_END_OF_INDEX, DB_DEADLOCK, DB_LOCK_TABLE_FULL, DB_CORRUPTION, or DB_TOO_BIG_RECORD */ +template dberr_t row_search_mvcc( byte* buf, @@ -147,6 +161,49 @@ row_search_mvcc( ulint direction) MY_ATTRIBUTE((warn_unused_result)); +/** Convenience wrapper for InnoDB callback (read-only, manages own mtr). +Used for FTS queries and other read-only operations where row_search_mvcc +manages its own mini-transaction lifecycle. +@param callback RecordCallback instance +@param mode search mode +@param prebuilt prebuilt structure +@param match_mode match mode +@param direction search direction +@return DB_SUCCESS or error code */ +inline dberr_t row_search_mvcc_callback( + RecordCallback* callback, + page_cur_mode_t mode, + row_prebuilt_t* prebuilt, + ulint match_mode, + ulint direction) noexcept +{ + return row_search_mvcc( + reinterpret_cast(callback), mode, prebuilt, + match_mode, direction); +} + +/** Convenience wrapper for InnoDB callback (DML, external mtr). +Used for SELECT FOR UPDATE and other DML operations where the caller +manages the mini-transaction lifecycle (e.g., needs to keep mtr open +across select_for_update() and update_record()). +@param callback RecordCallback instance +@param mode search mode +@param prebuilt prebuilt structure with external mtr set +@param match_mode match mode +@param direction search direction +@return DB_SUCCESS or error code */ +inline dberr_t row_search_mvcc_callback_dml( + RecordCallback* callback, + page_cur_mode_t mode, + row_prebuilt_t* prebuilt, + ulint match_mode, + ulint direction) noexcept +{ + return row_search_mvcc( + reinterpret_cast(callback), mode, prebuilt, + match_mode, direction); +} + /********************************************************************//** Count rows in a R-Tree leaf level. @return DB_SUCCESS if successful */ diff --git a/storage/innobase/include/ut0new.h b/storage/innobase/include/ut0new.h index bcc129601d11f..a974a093da1af 100644 --- a/storage/innobase/include/ut0new.h +++ b/storage/innobase/include/ut0new.h @@ -854,6 +854,7 @@ constexpr const char* const auto_event_names[] = "fts0ast", "fts0blex", "fts0config", + "fts0exec", "fts0file", "fts0fts", "fts0opt", diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc index d1d11b50de8a1..15319bf5b8866 100644 --- a/storage/innobase/page/page0cur.cc +++ b/storage/innobase/page/page0cur.cc @@ -78,9 +78,9 @@ static ulint cmp_get_pad_char(const dtype_t &type) noexcept @retval 0 if dtuple is equal to rec @retval negative if dtuple is less than rec @retval positive if dtuple is greater than rec */ -static int cmp_dtuple_rec_bytes(const rec_t *rec, - const dict_index_t &index, - const dtuple_t &tuple, int *match, ulint comp) +int cmp_dtuple_rec_bytes(const rec_t *rec, + const dict_index_t &index, + const dtuple_t &tuple, int *match, ulint comp) noexcept { ut_ad(dtuple_check_typed(&tuple)); diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 2dc11e4503e18..801b801f3f51f 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -42,6 +42,7 @@ Completed by Sunny Bains and Marko Makela #include "pars0pars.h" #include "ut0sort.h" #include "row0ftsort.h" +#include "fts0exec.h" #include "row0import.h" #include "row0vers.h" #include "handler0alter.h" @@ -3035,10 +3036,27 @@ row_merge_read_clustered_index( new_table->fts->cache->first_doc_id = new_table->fts->cache->next_doc_id; + trx_t *fts_trx = trx_create(); + trx_start_internal(fts_trx); + fts_trx->op_info= "setting last FTS document id"; + FTSQueryExecutor executor( + fts_trx, new_table); err= fts_update_sync_doc_id( + &executor, new_table, - new_table->fts->cache->synced_doc_id, - NULL); + new_table->fts->cache->synced_doc_id); + if (err == DB_SUCCESS) { + trx_commit_for_mysql(fts_trx); + new_table->fts->cache->synced_doc_id++; + } else { + sql_print_error( + "InnoDB: ( %s ) while updating " + "last doc id for table %s", + ut_strerr(err), + new_table->name.m_name); + fts_trx->rollback(); + } + fts_trx->free(); } } diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index b4c9fb8c6dfd4..2a926b41c1928 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -885,11 +885,11 @@ row_create_prebuilt( } prebuilt->pcur = static_cast( - mem_heap_zalloc(prebuilt->heap, - sizeof(btr_pcur_t))); + mem_heap_zalloc(prebuilt->heap, + sizeof(btr_pcur_t))); prebuilt->clust_pcur = static_cast( - mem_heap_zalloc(prebuilt->heap, - sizeof(btr_pcur_t))); + mem_heap_zalloc(prebuilt->heap, + sizeof(btr_pcur_t))); btr_pcur_reset(prebuilt->pcur); btr_pcur_reset(prebuilt->clust_pcur); @@ -935,8 +935,13 @@ void row_prebuilt_free(row_prebuilt_t *prebuilt) prebuilt->magic_n = ROW_PREBUILT_FREED; prebuilt->magic_n2 = ROW_PREBUILT_FREED; - btr_pcur_reset(prebuilt->pcur); - btr_pcur_reset(prebuilt->clust_pcur); + if (prebuilt->pcur) { + btr_pcur_reset(prebuilt->pcur); + } + + if (prebuilt->clust_pcur) { + btr_pcur_reset(prebuilt->clust_pcur); + } ut_free(prebuilt->mysql_template); diff --git a/storage/innobase/row/row0query.cc b/storage/innobase/row/row0query.cc new file mode 100644 index 0000000000000..571d21c0d9edf --- /dev/null +++ b/storage/innobase/row/row0query.cc @@ -0,0 +1,445 @@ +/***************************************************************************** + +Copyright (c) 2025, MariaDB PLC. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file row/row0query.cc +General Query Executor + +Created 2025/10/30 +*******************************************************/ + +#include "row0query.h" +#include "pars0pars.h" +#include "dict0dict.h" +#include "row0ins.h" +#include "row0upd.h" +#include "row0row.h" +#include "row0vers.h" +#include "row0sel.h" +#include "mem0mem.h" +#include "que0que.h" +#include "lock0lock.h" +#include "rem0rec.h" +#include "btr0pcur.h" +#include "btr0cur.h" + +QueryExecutor::QueryExecutor(trx_t *trx) : m_trx(trx), m_mtr(nullptr) +{ + m_heap= mem_heap_create(256); + m_thr= pars_complete_graph_for_exec(nullptr, trx, m_heap, nullptr); + btr_pcur_init(&m_pcur); + m_clust_pcur= nullptr; + m_prebuilt= nullptr; +} + +QueryExecutor::~QueryExecutor() +{ + btr_pcur_close(&m_pcur); + if (m_clust_pcur) + btr_pcur_close(m_clust_pcur); + + if (m_prebuilt) + { + if (m_prebuilt->old_vers_heap) + mem_heap_free(m_prebuilt->old_vers_heap); + + ut_ad(!m_prebuilt->blob_heap); + ut_ad(!m_prebuilt->fetch_cache[0]); + ut_ad(!m_prebuilt->rtr_info); + ut_ad(!m_prebuilt->mysql_template); + + m_prebuilt= nullptr; + } + + if (m_heap) mem_heap_free(m_heap); +} + +void QueryExecutor::setup_prebuilt( + dict_table_t *table, dict_index_t *index, + const dtuple_t *tuple, page_cur_mode_t mode) noexcept +{ + dict_index_t *clust_index= dict_table_get_first_index(table); + ulint ref_len= dict_index_get_n_unique(clust_index); + + if (!m_prebuilt) + { + m_prebuilt= static_cast( + mem_heap_zalloc(m_heap, sizeof(*m_prebuilt))); + + m_prebuilt->magic_n= ROW_PREBUILT_ALLOCATED; + m_prebuilt->magic_n2= ROW_PREBUILT_ALLOCATED; + + m_prebuilt->table= table; + m_prebuilt->heap= m_heap; + + m_prebuilt->sel_graph= static_cast( + que_node_get_parent(m_thr)); + m_prebuilt->sql_stat_start= TRUE; + m_prebuilt->in_fts_query= true; + ulint search_tuple_n_fields= 2 * (dict_table_get_n_cols(table) + + dict_table_get_n_v_cols(table)); + m_prebuilt->search_tuple= + dtuple_create(m_heap, search_tuple_n_fields); + m_prebuilt->clust_ref= dtuple_create(m_heap, ref_len); + dict_index_copy_types(m_prebuilt->clust_ref, clust_index, ref_len); + } + else if (m_prebuilt->table != table) + { + m_prebuilt->table= table; + dtuple_t* ref= dtuple_create(m_heap, ref_len); + dict_index_copy_types(ref, clust_index, ref_len); + m_prebuilt->clust_ref= ref; + btr_pcur_reset(&m_pcur); + if (m_clust_pcur) + btr_pcur_reset(m_clust_pcur); + m_prebuilt->fts_doc_id= 0; + } + + /* Configure prebuilt for search */ + m_prebuilt->trx= m_trx; + m_prebuilt->index= index; + if (!tuple) + m_prebuilt->search_tuple= nullptr; + else + m_prebuilt->search_tuple= const_cast(tuple); + + m_prebuilt->pcur= &m_pcur; + if (!index->is_clust()) + { + if (!m_clust_pcur) + { + m_clust_pcur= static_cast( + mem_heap_zalloc(m_prebuilt->heap, sizeof(btr_pcur_t))); + btr_pcur_init(m_clust_pcur); + } + } + m_prebuilt->clust_pcur= m_clust_pcur; + + m_prebuilt->mtr= nullptr; + m_prebuilt->template_type= ROW_MYSQL_NO_TEMPLATE; + m_prebuilt->select_lock_type= LOCK_NONE; + m_prebuilt->row_read_type= 0; + m_prebuilt->n_fetch_cached= 0; + m_prebuilt->fetch_cache_first= 0; + m_prebuilt->index_usable= 1; + m_prebuilt->need_to_access_clustered= index->is_clust() ? 0 : 1; + m_prebuilt->in_fts_query= true; +} + +dberr_t QueryExecutor::insert_record(dict_table_t *table, + dtuple_t *tuple) noexcept +{ + dict_index_t* index= dict_table_get_first_index(table); + return row_ins_clust_index_entry(index, tuple, m_thr, 0); +} + +dberr_t QueryExecutor::lock_table(dict_table_t *table, lock_mode mode) noexcept +{ + ut_ad(m_trx); + trx_start_if_not_started(m_trx, true); + return ::lock_table(table, nullptr, mode, m_thr); +} + +dberr_t QueryExecutor::handle_wait(dberr_t err, bool table_lock) noexcept +{ + ut_ad(m_trx); + m_trx->error_state= err; + if (table_lock) m_thr->lock_state= QUE_THR_LOCK_TABLE; + else m_thr->lock_state= QUE_THR_LOCK_ROW; + if (m_trx->lock.wait_thr) + { + dberr_t wait_err= lock_wait(m_thr); + if (wait_err == DB_LOCK_WAIT_TIMEOUT) err= wait_err; + if (wait_err == DB_SUCCESS) + { + m_thr->lock_state= QUE_THR_LOCK_NOLOCK; + return DB_SUCCESS; + } + } + return err; +} + +dberr_t QueryExecutor::delete_record(dict_table_t *table, + dtuple_t *tuple) noexcept +{ + dict_index_t *clust_index= dict_table_get_first_index(table); + ut_ad(m_mtr); + + /* Use select_for_update to find and lock the record with proper MVCC handling */ + dberr_t err= select_for_update(table, tuple, nullptr); + if (err != DB_SUCCESS) + return err; + + /* Record is now locked at m_pcur, mark it as deleted */ + rec_t *rec= btr_pcur_get_rec(&m_pcur); + rec_offs *offsets= rec_get_offsets(rec, clust_index, nullptr, + clust_index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + + err= btr_cur_del_mark_set_clust_rec(btr_pcur_get_block(&m_pcur), + rec, clust_index, offsets, m_thr, + nullptr, m_mtr); + + m_mtr->commit(); + return err; +} + +dberr_t QueryExecutor::delete_all(dict_table_t *table) noexcept +{ + dict_index_t *clust_index= dict_table_get_first_index(table); + dberr_t err= DB_SUCCESS; + + if (!m_mtr) + m_mtr= new (mem_heap_alloc(m_heap, sizeof(mtr_t))) mtr_t(m_trx); +retry: + m_mtr->start(); + m_mtr->set_named_space(table->space); + + err= m_pcur.open_leaf(true, clust_index, BTR_MODIFY_LEAF, m_mtr); + if (err != DB_SUCCESS || !btr_pcur_move_to_next(&m_pcur, m_mtr)) + { + m_mtr->commit(); + return err; + } + + while (!btr_pcur_is_after_last_on_page(&m_pcur) && + !btr_pcur_is_after_last_in_tree(&m_pcur)) + { + rec_t* rec= btr_pcur_get_rec(&m_pcur); + rec_offs *offsets= nullptr; + if (rec_get_deleted_flag(rec, dict_table_is_comp(table))) + goto next_rec; + if (rec_get_info_bits( + rec, dict_table_is_comp(table)) & REC_INFO_MIN_REC_FLAG) + goto next_rec; + + offsets= rec_get_offsets(rec, clust_index, nullptr, + clust_index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + err= lock_clust_rec_read_check_and_lock( + 0, btr_pcur_get_block(&m_pcur), rec, clust_index, offsets, LOCK_X, + LOCK_REC_NOT_GAP, m_thr); + + if (err == DB_LOCK_WAIT) + { + m_mtr->commit(); + err= handle_wait(err, false); + if (err != DB_SUCCESS) + return err; + goto retry; + } + else if (err != DB_SUCCESS && err != DB_SUCCESS_LOCKED_REC) + { + m_mtr->commit(); + return err; + } + + err= btr_cur_del_mark_set_clust_rec(btr_pcur_get_block(&m_pcur), + const_cast(rec), clust_index, + offsets, m_thr, nullptr, m_mtr); + if (err) + break; +next_rec: + if (!btr_pcur_move_to_next(&m_pcur, m_mtr)) + break; + } + + m_mtr->commit(); + return err; +} + +dberr_t QueryExecutor::select_for_update(dict_table_t *table, + dtuple_t *search_tuple, + RecordCallback *callback) noexcept +{ + ut_ad(m_trx); + dict_index_t *clust_index= dict_table_get_first_index(table); + setup_prebuilt(table, clust_index, search_tuple, PAGE_CUR_GE); + m_prebuilt->select_lock_type= LOCK_X; + + if (!m_mtr) + m_mtr= new (mem_heap_alloc(m_heap, sizeof(mtr_t))) mtr_t(m_trx); + + m_mtr->start(); + m_mtr->set_named_space(table->space); + + if (m_trx && !m_trx->read_view.is_open()) + { + trx_start_if_not_started(m_trx, false); + m_trx->read_view.open(m_trx); + } + + /* Provide external mtr for DML operation */ + m_prebuilt->mtr= m_mtr; + + dberr_t err= DB_SUCCESS; + + if (callback) + err= row_search_mvcc_callback_dml( + callback, PAGE_CUR_GE, m_prebuilt, 0, 0); + else + err= row_search_mvcc(nullptr, PAGE_CUR_GE, + m_prebuilt, 1, 0); + + if (err == DB_LOCK_WAIT) + { + m_mtr->commit(); + err= handle_wait(err, false); + if (err != DB_SUCCESS) + return err; + return DB_LOCK_WAIT; + } + + if (err == DB_END_OF_INDEX || err == DB_RECORD_NOT_FOUND) + err= DB_RECORD_NOT_FOUND; + else if (err == DB_SUCCESS_LOCKED_REC) + err= DB_SUCCESS; + + if (err != DB_SUCCESS) + m_mtr->commit(); + return err; +} + +dberr_t QueryExecutor::update_record(dict_table_t *table, + const upd_t *update) noexcept +{ + ut_ad(m_trx); + ut_ad(m_mtr); + dict_index_t *clust_index= dict_table_get_first_index(table); + rec_t *rec= btr_pcur_get_rec(&m_pcur); + mtr_x_lock_index(clust_index, m_mtr); + rec_offs *offsets= rec_get_offsets(rec, clust_index, nullptr, + clust_index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + + dberr_t err= DB_SUCCESS; + ulint cmpl_info= UPD_NODE_NO_ORD_CHANGE | UPD_NODE_NO_SIZE_CHANGE; + for (ulint i= 0; i < update->n_fields; i++) + { + const upd_field_t *upd_field= &update->fields[i]; + ulint field_no= upd_field->field_no; + if (field_no < rec_offs_n_fields(offsets)) + { + ulint old_len= rec_offs_nth_size(offsets, field_no); + ulint new_len= upd_field->new_val.len; + if (new_len != UNIV_SQL_NULL && new_len != old_len) + { + cmpl_info &= ~UPD_NODE_NO_SIZE_CHANGE; + err= DB_OVERFLOW; + break; + } + } + } + + if (cmpl_info & UPD_NODE_NO_SIZE_CHANGE) + err= btr_cur_update_in_place(BTR_NO_LOCKING_FLAG, + btr_pcur_get_btr_cur(&m_pcur), + offsets, const_cast(update), 0, + m_thr, m_trx->id, m_mtr); + if (err == DB_OVERFLOW) + { + big_rec_t *big_rec= nullptr; + err= btr_cur_optimistic_update(BTR_NO_LOCKING_FLAG, + btr_pcur_get_btr_cur(&m_pcur), + &offsets, &m_heap, + const_cast(update), + cmpl_info, m_thr, m_trx->id, m_mtr); + + if (err == DB_OVERFLOW || err == DB_UNDERFLOW) + { + mem_heap_t* offsets_heap= nullptr; + err= btr_cur_pessimistic_update(BTR_NO_LOCKING_FLAG, + btr_pcur_get_btr_cur(&m_pcur), + &offsets, &offsets_heap, m_heap, + &big_rec, const_cast(update), + cmpl_info, m_thr, m_trx->id, m_mtr); + + if (err == DB_SUCCESS && big_rec) + { + err= btr_store_big_rec_extern_fields(&m_pcur, offsets, big_rec, m_mtr, + BTR_STORE_UPDATE); + dtuple_big_rec_free(big_rec); + } + if (offsets_heap) mem_heap_free(offsets_heap); + } + } + return err; +} + +dberr_t QueryExecutor::replace_record( + dict_table_t *table, dtuple_t *search_tuple, + const upd_t *update, dtuple_t *insert_tuple) noexcept +{ +retry_again: + dberr_t err= select_for_update(table, search_tuple); + if (err == DB_SUCCESS) + { + err= update_record(table, update); + m_mtr->commit(); + return err; + } + else if (err == DB_RECORD_NOT_FOUND) + { + err= insert_record(table, insert_tuple); + return err; + } + else if (err == DB_LOCK_WAIT) + goto retry_again; + return err; +} + +dberr_t QueryExecutor::read_by_index(dict_table_t *table, + dict_index_t *index, + const dtuple_t *search_tuple, + page_cur_mode_t mode, + RecordCallback& callback, + bool read_all) noexcept +{ + ut_ad(table); + ut_ad(index); + ut_ad(callback.compare_record); + + setup_prebuilt(table, index, search_tuple, mode); + + if (m_trx && !m_trx->read_view.is_open()) + { + trx_start_if_not_started(m_trx, false); + m_trx->read_view.open(m_trx); + } + dberr_t err= row_search_mvcc_callback(&callback, mode, m_prebuilt, 0, 0); + if (read_all) + { + while (err == DB_SUCCESS) + err= row_search_mvcc_callback(&callback, mode, m_prebuilt, 0, + ROW_SEL_NEXT); + } + if (err == DB_END_OF_INDEX || err == DB_SUCCESS_LOCKED_REC) + err= DB_SUCCESS; + return err; +} + +dberr_t QueryExecutor::read(dict_table_t *table, const dtuple_t *tuple, + page_cur_mode_t mode, + RecordCallback& callback) noexcept +{ + ut_ad(table); + ut_ad(callback.compare_record); + + dict_index_t *clust_index= dict_table_get_first_index(table); + return read_by_index(table, clust_index, tuple, mode, callback, false); +} diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index c063c3b34b810..c8db9e477d88b 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -3341,6 +3341,7 @@ class Row_sel_get_clust_rec_for_mysql dtuple_t **vrow, mtr_t *mtr); }; + /*********************************************************************//** Retrieves the clustered index record corresponding to a record in a non-clustered index. Does the necessary locking. Used in the MySQL @@ -3349,7 +3350,7 @@ interface. dberr_t Row_sel_get_clust_rec_for_mysql::operator()( /*============================*/ - row_prebuilt_t* prebuilt,/*!< in: prebuilt struct in the handle */ + row_prebuilt_t* prebuilt,/*!< in: prebuilt struct */ dict_index_t* sec_index,/*!< in: secondary index where rec resides */ const rec_t* rec, /*!< in: record in a non-clustered index; if this is a locking read, then rec is not @@ -3371,38 +3372,42 @@ Row_sel_get_clust_rec_for_mysql::operator()( non-clustered record; the same mtr is used to access the clustered index */ { + /* Extract values from prebuilt */ + dtuple_t* clust_ref= prebuilt->clust_ref; + btr_pcur_t* clust_pcur= prebuilt->clust_pcur; + lock_mode select_lock_type= prebuilt->select_lock_type; + trx_t* trx= prebuilt->trx; + dict_index_t* clust_index; rec_t* old_vers; - trx_t* trx; - prebuilt->clust_pcur->old_rec = nullptr; + clust_pcur->old_rec = nullptr; *out_rec = NULL; - trx = thr_get_trx(thr); - row_build_row_ref_in_tuple(prebuilt->clust_ref, rec, + row_build_row_ref_in_tuple(clust_ref, rec, sec_index, *offsets); clust_index = dict_table_get_first_index(sec_index->table); - prebuilt->clust_pcur->btr_cur.page_cur.index = clust_index; + clust_pcur->btr_cur.page_cur.index = clust_index; - dberr_t err = btr_pcur_open_with_no_init(prebuilt->clust_ref, + dberr_t err = btr_pcur_open_with_no_init(clust_ref, PAGE_CUR_LE, BTR_SEARCH_LEAF, - prebuilt->clust_pcur, mtr); + clust_pcur, mtr); if (UNIV_UNLIKELY(err != DB_SUCCESS)) { return err; } - const rec_t* clust_rec = btr_pcur_get_rec(prebuilt->clust_pcur); + const rec_t* clust_rec = btr_pcur_get_rec(clust_pcur); - prebuilt->clust_pcur->trx_if_known = trx; + clust_pcur->trx_if_known = trx; /* Note: only if the search ends up on a non-infimum record is the low_match value the real match to the search tuple */ if (!page_rec_is_user_rec(clust_rec) - || btr_pcur_get_low_match(prebuilt->clust_pcur) + || btr_pcur_get_low_match(clust_pcur) < dict_index_get_n_unique(clust_index)) { - btr_cur_t* btr_cur = btr_pcur_get_btr_cur(prebuilt->pcur); + btr_cur_t* btr_cur = btr_pcur_get_btr_cur(clust_pcur); /* If this is a spatial index scan, and we are reading from a shadow buffer, the record could be already @@ -3413,7 +3418,7 @@ Row_sel_get_clust_rec_for_mysql::operator()( && (!(ulint(rec - btr_cur->rtr_info->matches->block->page.frame) >> srv_page_size_shift) - || rec != btr_pcur_get_rec(prebuilt->pcur))) { + || rec != btr_pcur_get_rec(clust_pcur))) { #ifdef UNIV_DEBUG rtr_info_t* rtr_info = btr_cur->rtr_info; mysql_mutex_lock(&rtr_info->matches->rtr_match_mutex); @@ -3427,23 +3432,23 @@ Row_sel_get_clust_rec_for_mysql::operator()( if (rec_get_deleted_flag(rec, dict_table_is_comp(sec_index->table)) - && prebuilt->select_lock_type == LOCK_NONE) { + && select_lock_type == LOCK_NONE) { clust_rec = NULL; goto func_exit; } - if (rec != btr_pcur_get_rec(prebuilt->pcur)) { + if (rec != btr_pcur_get_rec(clust_pcur)) { clust_rec = NULL; goto func_exit; } /* FIXME: Why is this block not the - same as btr_pcur_get_block(prebuilt->pcur), + same as btr_pcur_get_block(clust_pcur), and is it not unsafe to use RW_NO_LATCH here? */ buf_block_t* block = buf_page_get_gen( - btr_pcur_get_block(prebuilt->pcur)->page.id(), - btr_pcur_get_block(prebuilt->pcur)->zip_size(), + btr_pcur_get_block(clust_pcur)->page.id(), + btr_pcur_get_block(clust_pcur)->zip_size(), RW_NO_LATCH, NULL, BUF_GET, mtr, &err); ut_ad(block); // FIXME: avoid crash mem_heap_t* heap = mem_heap_create(256); @@ -3465,7 +3470,7 @@ Row_sel_get_clust_rec_for_mysql::operator()( #endif /* UNIV_DEBUG */ } else if (!rec_get_deleted_flag(rec, dict_table_is_comp(sec_index->table)) - || prebuilt->select_lock_type != LOCK_NONE) { + || select_lock_type != LOCK_NONE) { /* In a rare case it is possible that no clust rec is found for a delete-marked secondary index record: if row_undo_mod_clust() has already removed @@ -3494,15 +3499,15 @@ Row_sel_get_clust_rec_for_mysql::operator()( clust_index->n_core_fields, ULINT_UNDEFINED, offset_heap); - if (prebuilt->select_lock_type != LOCK_NONE) { + if (select_lock_type != LOCK_NONE) { /* Try to place a lock on the index record; we are searching the clust rec with a unique condition, hence we set a LOCK_REC_NOT_GAP type lock */ err = lock_clust_rec_read_check_and_lock( - 0, btr_pcur_get_block(prebuilt->clust_pcur), + 0, btr_pcur_get_block(clust_pcur), clust_rec, clust_index, *offsets, - prebuilt->select_lock_type, + select_lock_type, LOCK_REC_NOT_GAP, thr); @@ -3536,7 +3541,7 @@ Row_sel_get_clust_rec_for_mysql::operator()( break; case DB_SUCCESS_LOCKED_REC: const buf_page_t& bpage = btr_pcur_get_block( - prebuilt->clust_pcur)->page; + clust_pcur)->page; const lsn_t lsn = mach_read_from_8( bpage.frame + FIL_PAGE_LSN); @@ -3548,8 +3553,8 @@ Row_sel_get_clust_rec_for_mysql::operator()( 'old_vers' */ err = row_sel_build_prev_vers_for_mysql( prebuilt, clust_index, - clust_rec, offsets, offset_heap, &old_vers, - vrow, mtr); + clust_rec, offsets, offset_heap, + &old_vers, vrow, mtr); if (UNIV_UNLIKELY(err != DB_SUCCESS)) { return err; @@ -3622,11 +3627,11 @@ Row_sel_get_clust_rec_for_mysql::operator()( func_exit: *out_rec = clust_rec; - if (prebuilt->select_lock_type != LOCK_NONE) { + if (select_lock_type != LOCK_NONE) { /* We may use the cursor in update or in unlock_row(): store its position */ - btr_pcur_store_position(prebuilt->clust_pcur, mtr); + btr_pcur_store_position(clust_pcur, mtr); } return err; @@ -4320,6 +4325,75 @@ bool row_search_with_covering_prefix( return true; } +/** Templated policy for row_search_mvcc behavior. +@tparam NeedsConversion true if records need MySQL format conversion +@tparam ManagesMtr true if row_search_mvcc manages its own mtr */ +template +struct InnoDBPolicy +{ + static constexpr bool needs_conversion= NeedsConversion; + static constexpr bool manages_mtr= ManagesMtr; + + /** Convert a record to MySQL format + (only when NeedsConversion=true). + @param[out] mysql_buf MySQL row buffer + @param[in] prebuilt prebuilt structure + @param[in] rec record to convert + @param[in] vrow virtual column values (can be NULL) + @param[in] rec_clust true if rec is from clustered index + @param[in] index index of the record + @param[in] offsets record offsets + @return DB_SUCCESS on success + @retval DB_ERROR if conversion fails */ + static dberr_t output_record(byte* mysql_buf, + row_prebuilt_t* prebuilt, + const rec_t* rec, + const dtuple_t* vrow, + bool rec_clust, + dict_index_t* index, + rec_offs* offsets) noexcept + { + static_assert(NeedsConversion, "MySQL format conversion only"); + if (!row_sel_store_mysql_rec(mysql_buf, prebuilt, rec, vrow, + rec_clust, index, offsets)) + return DB_ERROR; + return DB_SUCCESS; + } + + /** Process a record via RecordCallback interface + @param[in] callback RecordCallback instance (can be NULL) + @param[in] prebuilt prebuilt structure with search tuple + @param[in] rec record to process + @param[in] index index of the record + @param[in] offsets record offsets + @return DB_SUCCESS on success + @retval DB_SUCCESS_LOCKED_REC to stop iteration after this record + @retval DB_RECORD_NOT_FOUND to skip this record + @retval error code on failure */ + static dberr_t output_record(RecordCallback* callback, + row_prebuilt_t* prebuilt, + const rec_t* rec, + dict_index_t* index, + rec_offs* offsets) noexcept + { + static_assert(!NeedsConversion, "InnoDB callback only"); + if (!callback) + return DB_SUCCESS; + + RecordCompareAction action= callback->compare_record( + prebuilt->search_tuple, rec, index, offsets); + + if (action == RecordCompareAction::PROCESS) + return callback->process_record(rec, index, offsets); + else if (action == RecordCompareAction::SKIP) + return DB_RECORD_NOT_FOUND; + else if (action == RecordCompareAction::STOP) + return DB_SUCCESS_LOCKED_REC; + + return DB_SUCCESS; + } +}; + /** Searches for rows in the database using cursor. Function is mainly used for tables that are shared across connections and so it employs technique that can help re-construct the rows that @@ -4339,6 +4413,7 @@ It also has optimization such as pre-caching the rows, using AHI, etc. pcur with stored position! In opening of a cursor 'direction' should be 0. @return DB_SUCCESS or error code */ +template dberr_t row_search_mvcc( byte* buf, @@ -4374,7 +4449,7 @@ row_search_mvcc( byte* next_buf = 0; bool spatial_search = false; - ut_ad(index && pcur && search_tuple); + ut_ad(index && pcur); ut_a(prebuilt->magic_n == ROW_PREBUILT_ALLOCATED); ut_a(prebuilt->magic_n2 == ROW_PREBUILT_ALLOCATED); @@ -4521,8 +4596,21 @@ row_search_mvcc( /* if the query is a plain locking SELECT, and the isolation level is <= TRX_ISO_READ_COMMITTED, then this is set to FALSE */ bool did_semi_consistent_read = false; - mtr_t mtr{trx}; - mtr.start(); + + /* Conditionally manage mtr based on policy */ + mtr_t local_mtr{trx}; + mtr_t* mtr; + + if constexpr (Callback::manages_mtr) { + /* row_search_mvcc manages its own mtr */ + mtr = &local_mtr; + mtr->start(); + } else { + /* Use external mtr from prebuilt (internal parser) */ + mtr = prebuilt->mtr; + ut_ad(mtr); + ut_ad(mtr->is_active()); + } mem_heap_t* heap = NULL; rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; @@ -4562,7 +4650,7 @@ row_search_mvcc( dberr_t err = DB_SUCCESS; switch (row_sel_try_search_shortcut_for_mysql( &rec, prebuilt, &offsets, &heap, - &mtr)) { + mtr)) { case SEL_FOUND: /* At this point, rec is protected by a page latch that was acquired by @@ -4616,7 +4704,9 @@ row_search_mvcc( case SEL_EXHAUSTED: err = DB_RECORD_NOT_FOUND; shortcut_done: - mtr.commit(); + if constexpr (Callback::manages_mtr) { + mtr->commit(); + } /* NOTE that we do NOT store the cursor position */ @@ -4634,8 +4724,10 @@ row_search_mvcc( ut_ad(0); } - mtr.commit(); - mtr.start(); + if constexpr (Callback::manages_mtr) { + mtr->commit(); + mtr->start(); + } } } #endif /* BTR_CUR_HASH_ADAPT */ @@ -4739,11 +4831,13 @@ row_search_mvcc( bool need_to_process = sel_restore_position_for_mysql( &same_user_rec, BTR_SEARCH_LEAF, - pcur, moves_up, &mtr); + pcur, moves_up, mtr); if (UNIV_UNLIKELY(need_to_process)) { if (UNIV_UNLIKELY(!btr_pcur_get_rec(pcur))) { - mtr.commit(); + if constexpr (Callback::manages_mtr) { + mtr->commit(); + } trx->op_info = ""; if (UNIV_LIKELY_NULL(heap)) { mem_heap_free(heap); @@ -4771,7 +4865,7 @@ row_search_mvcc( goto next_rec_after_check; } - } else if (dtuple_get_n_fields(search_tuple) > 0) { + } else if (search_tuple && dtuple_get_n_fields(search_tuple) > 0) { pcur->old_rec = nullptr; if (index->is_spatial()) { @@ -4792,11 +4886,15 @@ row_search_mvcc( } err = rtr_search_leaf(pcur, thr, search_tuple, mode, - &mtr); + mtr); } else { + /* For external mtr (DML operations), use BTR_MODIFY_LEAF + to get X-latch on page for subsequent update */ + btr_latch_mode latch_mode = Callback::manages_mtr + ? BTR_SEARCH_LEAF : BTR_MODIFY_LEAF; err = btr_pcur_open_with_no_init(search_tuple, mode, - BTR_SEARCH_LEAF, - pcur, &mtr); + latch_mode, + pcur, mtr); } if (err != DB_SUCCESS) { @@ -4829,7 +4927,7 @@ row_search_mvcc( err = sel_set_rec_lock(pcur, next_rec, index, offsets, prebuilt->select_lock_type, - LOCK_GAP, thr, &mtr); + LOCK_GAP, thr, mtr); switch (err) { case DB_SUCCESS_LOCKED_REC: @@ -4842,8 +4940,10 @@ row_search_mvcc( } } } else if (mode == PAGE_CUR_G || mode == PAGE_CUR_L) { + btr_latch_mode latch_mode = Callback::manages_mtr + ? BTR_SEARCH_LEAF : BTR_MODIFY_LEAF; err = pcur->open_leaf(mode == PAGE_CUR_G, index, - BTR_SEARCH_LEAF, &mtr); + latch_mode, mtr); if (err != DB_SUCCESS) { if (err == DB_DECRYPTION_FAILED) { @@ -4907,7 +5007,7 @@ row_search_mvcc( DEBUG_SYNC_C("row_search_rec_loop"); if (trx_is_interrupted(trx)) { if (!spatial_search) { - btr_pcur_store_position(pcur, &mtr); + btr_pcur_store_position(pcur, mtr); } err = DB_INTERRUPTED; goto normal_return; @@ -4948,7 +5048,7 @@ row_search_mvcc( err = sel_set_rec_lock(pcur, rec, index, offsets, prebuilt->select_lock_type, - LOCK_ORDINARY, thr, &mtr); + LOCK_ORDINARY, thr, mtr); switch (err) { case DB_SUCCESS_LOCKED_REC: @@ -5087,7 +5187,7 @@ row_search_mvcc( pcur, rec, index, offsets, prebuilt->select_lock_type, LOCK_GAP, - thr, &mtr); + thr, mtr); switch (err) { case DB_SUCCESS_LOCKED_REC: @@ -5098,15 +5198,18 @@ row_search_mvcc( } } - btr_pcur_store_position(pcur, &mtr); - - /* The found record was not a match, but may be used - as NEXT record (index_next). Set the relative position - to BTR_PCUR_BEFORE, to reflect that the position of - the persistent cursor is before the found/stored row - (pcur->old_rec). */ - ut_ad(pcur->rel_pos == BTR_PCUR_ON); - pcur->rel_pos = BTR_PCUR_BEFORE; + if constexpr (Callback::manages_mtr) { + btr_pcur_store_position(pcur, mtr); + + /* The found record was not a match, + but may be used as NEXT record (index_next). + Set the relative position to BTR_PCUR_BEFORE, + to reflect that the position of the persistent + cursor is before the found/stored row + (pcur->old_rec). */ + ut_ad(pcur->rel_pos == BTR_PCUR_ON); + pcur->rel_pos = BTR_PCUR_BEFORE; + } err = DB_RECORD_NOT_FOUND; goto normal_return; @@ -5123,7 +5226,7 @@ row_search_mvcc( pcur, rec, index, offsets, prebuilt->select_lock_type, LOCK_GAP, - thr, &mtr); + thr, mtr); switch (err) { case DB_SUCCESS_LOCKED_REC: @@ -5134,15 +5237,17 @@ row_search_mvcc( } } - btr_pcur_store_position(pcur, &mtr); - - /* The found record was not a match, but may be used - as NEXT record (index_next). Set the relative position - to BTR_PCUR_BEFORE, to reflect that the position of - the persistent cursor is before the found/stored row - (pcur->old_rec). */ - ut_ad(pcur->rel_pos == BTR_PCUR_ON); - pcur->rel_pos = BTR_PCUR_BEFORE; + if constexpr (Callback::manages_mtr) { + btr_pcur_store_position(pcur, mtr); + /* The found record was not a match, + but may be used as NEXT record (index_next). + Set the relative position to BTR_PCUR_BEFORE, + to reflect that the position of the persistent + cursor is before the found/stored row + (pcur->old_rec). */ + ut_ad(pcur->rel_pos == BTR_PCUR_ON); + pcur->rel_pos = BTR_PCUR_BEFORE; + } err = DB_RECORD_NOT_FOUND; goto normal_return; @@ -5245,7 +5350,7 @@ row_search_mvcc( err = sel_set_rec_lock(pcur, rec, index, offsets, prebuilt->select_lock_type, - lock_type, thr, &mtr); + lock_type, thr, mtr); switch (err) { const rec_t* old_vers; @@ -5292,7 +5397,7 @@ row_search_mvcc( row_sel_build_committed_vers_for_mysql( clust_index, prebuilt, rec, &offsets, &heap, &old_vers, - need_vrow ? &vrow : NULL, &mtr); + need_vrow ? &vrow : NULL, mtr); } /* Check whether it was a deadlock or not, if not @@ -5393,8 +5498,9 @@ row_search_mvcc( associated with 'old_vers' */ err = row_sel_build_prev_vers_for_mysql( prebuilt, clust_index, - rec, &offsets, &heap, &old_vers, - need_vrow ? &vrow : nullptr, &mtr); + rec, &offsets, &heap, + &old_vers, + need_vrow ? &vrow : nullptr, mtr); if (err != DB_SUCCESS) { @@ -5523,7 +5629,7 @@ row_search_mvcc( /* It was a non-clustered index and we must fetch also the clustered index record */ - mtr_extra_clust_savepoint = mtr.get_savepoint(); + mtr_extra_clust_savepoint = mtr->get_savepoint(); ut_ad(!vrow); /* The following call returns 'offsets' associated with @@ -5534,7 +5640,8 @@ row_search_mvcc( thr, &clust_rec, &offsets, &heap, need_vrow ? &vrow : NULL, - &mtr); + mtr); + if (err == DB_LOCK_WAIT && prebuilt->skip_locked) { err = lock_trx_handle_wait(trx); } @@ -5608,10 +5715,30 @@ row_search_mvcc( index may be in the wrong case, and the authoritative case is in result_rec, the appropriate version of the clustered index record. */ - if (!row_sel_store_mysql_rec( - buf, prebuilt, result_rec, vrow, - true, clust_index, offsets)) { - goto next_rec; + if constexpr (Callback::needs_conversion) { + /* MySQL format path */ + err= MySQLRowCallback::output_record( + buf, prebuilt, result_rec, vrow, + true, clust_index, offsets); + + if (err != DB_SUCCESS) + goto next_rec; + } else { + /* InnoDB callback path */ + err= InnoDBReadPolicy::output_record( + reinterpret_cast(buf), + prebuilt, result_rec, clust_index, offsets); + + if (err == DB_FTS_EXCEED_RESULT_CACHE_LIMIT) { + goto normal_return; + } else if (err == DB_SUCCESS_LOCKED_REC) { + err= DB_SUCCESS; + goto idx_cond_failed; + } else if (err == DB_RECORD_NOT_FOUND) { + goto next_rec; + } else if (err != DB_SUCCESS) { + goto lock_wait_or_error; + } } } } else { @@ -5653,89 +5780,132 @@ row_search_mvcc( /* We only convert from InnoDB row format to MySQL row format when ICP is disabled. */ - if (!prebuilt->pk_filter && !prebuilt->idx_cond) { - /* We use next_buf to track the allocation of buffers - where we store and enqueue the buffers for our - pre-fetch optimisation. + if constexpr (Callback::needs_conversion) { + /* MySQL format path with prefetch */ + if (!prebuilt->pk_filter && !prebuilt->idx_cond) { + /* We use next_buf to track the allocation of buffers + where we store and enqueue the buffers for our + pre-fetch optimisation. - If next_buf == 0 then we store the converted record - directly into the MySQL record buffer (buf). If it is - != 0 then we allocate a pre-fetch buffer and store the - converted record there. + If next_buf == 0 then we store the converted record + directly into the MySQL record buffer (buf). If it is + != 0 then we allocate a pre-fetch buffer and store the + converted record there. - If the conversion fails and the MySQL record buffer - was not written to then we reset next_buf so that - we can re-use the MySQL record buffer in the next - iteration. */ + If the conversion fails and the MySQL record buffer + was not written to then we reset next_buf so that + we can re-use the MySQL record buffer in the next + iteration. */ - next_buf = next_buf - ? row_sel_fetch_last_buf(prebuilt) : buf; + next_buf = next_buf + ? row_sel_fetch_last_buf(prebuilt) : buf; - if (!row_sel_store_mysql_rec( - next_buf, prebuilt, result_rec, vrow, - result_rec != rec, - result_rec != rec ? clust_index : index, - offsets)) { + err= MySQLRowCallback::output_record( + next_buf, prebuilt, result_rec, vrow, + result_rec != rec, + result_rec != rec ? clust_index : index, + offsets); - if (next_buf == buf) { - ut_a(prebuilt->n_fetch_cached == 0); - next_buf = 0; - } + if (err != DB_SUCCESS) { + if (next_buf == buf) { + ut_a(prebuilt->n_fetch_cached == 0); + next_buf = 0; + } - /* Only fresh inserts may contain incomplete - externally stored columns. Pretend that such - records do not exist. Such records may only be - accessed at the READ UNCOMMITTED isolation - level or when rolling back a recovered - transaction. Rollback happens at a lower - level, not here. */ - goto next_rec; - } + /* Only fresh inserts may contain incomplete + externally stored columns. Pretend that such + records do not exist. Such records may only be + accessed at the READ UNCOMMITTED isolation + level or when rolling back a recovered + transaction. Rollback happens at a lower + level, not here. */ + goto next_rec; + } - if (next_buf != buf) { - row_sel_enqueue_cache_row_for_mysql( - next_buf, prebuilt); + if (next_buf != buf) { + row_sel_enqueue_cache_row_for_mysql( + next_buf, prebuilt); + } + } else { + row_sel_enqueue_cache_row_for_mysql(buf, prebuilt); } } else { - row_sel_enqueue_cache_row_for_mysql(buf, prebuilt); + /* InnoDB callback path - no prefetch, direct processing */ + err= InnoDBReadPolicy::output_record( + reinterpret_cast(buf), + prebuilt, result_rec, + result_rec != rec ? clust_index : index, + offsets); + + if (err == DB_FTS_EXCEED_RESULT_CACHE_LIMIT) { + goto normal_return; + } else if (err == DB_RECORD_NOT_FOUND) { + goto next_rec; + } else if (err != DB_SUCCESS && + err != DB_SUCCESS_LOCKED_REC) { + goto lock_wait_or_error; + } } if (prebuilt->n_fetch_cached < MYSQL_FETCH_CACHE_SIZE) { goto next_rec; } } else { - if (!prebuilt->pk_filter && !prebuilt->idx_cond) { - /* The record was not yet converted to MySQL format. */ - if (!row_sel_store_mysql_rec( - buf, prebuilt, result_rec, vrow, - result_rec != rec, - result_rec != rec ? clust_index : index, - offsets)) { - /* Only fresh inserts may contain - incomplete externally stored - columns. Pretend that such records do - not exist. Such records may only be - accessed at the READ UNCOMMITTED - isolation level or when rolling back a - recovered transaction. Rollback - happens at a lower level, not here. */ - goto next_rec; + if constexpr (Callback::needs_conversion) { + /* MySQL format path */ + if (!prebuilt->pk_filter && !prebuilt->idx_cond) { + /* The record was not yet converted to MySQL format. */ + err= MySQLRowCallback::output_record( + buf, prebuilt, result_rec, vrow, + result_rec != rec, + result_rec != rec ? clust_index : index, + offsets); + + if (err != DB_SUCCESS) { + /* Only fresh inserts may contain + incomplete externally stored + columns. Pretend that such records do + not exist. Such records may only be + accessed at the READ UNCOMMITTED + isolation level or when rolling back a + recovered transaction. Rollback + happens at a lower level, not here. */ + goto next_rec; + } } - } - if (!prebuilt->clust_index_was_generated) { - } else if (result_rec != rec || index->is_primary()) { - memcpy(prebuilt->row_id, result_rec, DATA_ROW_ID_LEN); + if (!prebuilt->clust_index_was_generated) { + } else if (result_rec != rec || index->is_primary()) { + memcpy(prebuilt->row_id, result_rec, DATA_ROW_ID_LEN); + } else { + ulint len; + const byte* data = rec_get_nth_field( + result_rec, offsets, index->n_fields - 1, + &len); + ut_ad(dict_index_get_nth_col(index, + index->n_fields - 1) + ->prtype == (DATA_ROW_ID | DATA_NOT_NULL)); + ut_ad(len == DATA_ROW_ID_LEN); + memcpy(prebuilt->row_id, data, DATA_ROW_ID_LEN); + } } else { - ulint len; - const byte* data = rec_get_nth_field( - result_rec, offsets, index->n_fields - 1, - &len); - ut_ad(dict_index_get_nth_col(index, - index->n_fields - 1) - ->prtype == (DATA_ROW_ID | DATA_NOT_NULL)); - ut_ad(len == DATA_ROW_ID_LEN); - memcpy(prebuilt->row_id, data, DATA_ROW_ID_LEN); + /* InnoDB callback path - no MySQL format conversion */ + err= InnoDBReadPolicy::output_record( + reinterpret_cast(buf), + prebuilt, result_rec, + result_rec != rec ? clust_index : index, + offsets); + + if (err == DB_FTS_EXCEED_RESULT_CACHE_LIMIT) { + goto normal_return; + } else if (err == DB_SUCCESS_LOCKED_REC) { + err= DB_SUCCESS; + goto idx_cond_failed; + } else if (err == DB_RECORD_NOT_FOUND) { + goto next_rec; + } else if (err != DB_SUCCESS) { + goto lock_wait_or_error; + } } } @@ -5757,11 +5927,16 @@ row_search_mvcc( || prebuilt->select_lock_type != LOCK_NONE || prebuilt->used_in_HANDLER) { - /* Inside an update always store the cursor position */ + /* Inside an update always store the cursor position. + Exception: for external mtr (DML operations), keep the + page latched for the subsequent update operation. */ - if (!spatial_search) { - btr_pcur_store_position(pcur, &mtr); + if constexpr (Callback::manages_mtr) { + if (!spatial_search) { + btr_pcur_store_position(pcur, mtr); + } } + /* else: external mtr, keep page latched */ } goto normal_return; @@ -5796,14 +5971,14 @@ row_search_mvcc( if (spatial_search) { /* No need to do store restore for R-tree */ - mtr.rollback_to_savepoint(0); + mtr->rollback_to_savepoint(0); } else if (mtr_extra_clust_savepoint) { /* We must release any clustered index latches if we are moving to the next non-clustered index record, because we could break the latching order if we would access a different clustered index page right away without releasing the previous. */ - mtr.rollback_to_savepoint(mtr_extra_clust_savepoint); + mtr->rollback_to_savepoint(mtr_extra_clust_savepoint); } mtr_extra_clust_savepoint = 0; @@ -5811,7 +5986,7 @@ row_search_mvcc( if (moves_up) { if (UNIV_UNLIKELY(spatial_search)) { if (rtr_pcur_move_to_next( - search_tuple, mode, pcur, 0, &mtr)) { + search_tuple, mode, pcur, 0, mtr)) { goto rec_loop; } } else { @@ -5823,7 +5998,7 @@ row_search_mvcc( if (btr_pcur_is_after_last_in_tree(pcur)) { goto not_moved; } - err = btr_pcur_move_to_next_page(pcur, &mtr); + err = btr_pcur_move_to_next_page(pcur, mtr); if (err != DB_SUCCESS) { goto lock_wait_or_error; } @@ -5834,7 +6009,7 @@ row_search_mvcc( goto rec_loop; } } else { - if (btr_pcur_move_to_prev(pcur, &mtr)) { + if (btr_pcur_move_to_prev(pcur, mtr)) { goto rec_loop; } if (UNIV_UNLIKELY(!btr_pcur_get_rec(pcur))) { @@ -5846,7 +6021,7 @@ row_search_mvcc( not_moved: if (!spatial_search) { - btr_pcur_store_position(pcur, &mtr); + btr_pcur_store_position(pcur, mtr); } err = match_mode ? DB_RECORD_NOT_FOUND : DB_END_OF_INDEX; @@ -5854,7 +6029,7 @@ row_search_mvcc( lock_wait_or_error: if (!dict_index_is_spatial(index)) { - btr_pcur_store_position(pcur, &mtr); + btr_pcur_store_position(pcur, mtr); } page_read_error: /* Reset the old and new "did semi-consistent read" flags. */ @@ -5865,7 +6040,9 @@ row_search_mvcc( did_semi_consistent_read = false; lock_table_wait: - mtr.commit(); + if constexpr (Callback::manages_mtr) { + mtr->commit(); + } mtr_extra_clust_savepoint = 0; trx->error_state = err; @@ -5875,7 +6052,9 @@ row_search_mvcc( /* It was a lock wait, and it ended */ thr->lock_state = QUE_THR_LOCK_NOLOCK; - mtr.start(); + if constexpr (Callback::manages_mtr) { + mtr->start(); + } /* Table lock waited, go try to obtain table lock again */ @@ -5888,7 +6067,7 @@ row_search_mvcc( if (!dict_index_is_spatial(index)) { sel_restore_position_for_mysql( &same_user_rec, BTR_SEARCH_LEAF, pcur, - moves_up, &mtr); + moves_up, mtr); } if (trx->isolation_level <= TRX_ISO_READ_COMMITTED @@ -5923,7 +6102,10 @@ row_search_mvcc( goto func_exit; normal_return: - mtr.commit(); + if constexpr (Callback::manages_mtr) { + mtr->commit(); + } + /* else: caller manages mtr, don't commit */ DEBUG_SYNC_C("row_search_for_mysql_before_return"); @@ -6847,3 +7029,13 @@ dberr_t row_check_index(row_prebuilt_t *prebuilt, ulint *n_rows) goto rec_loop; } + +/* Explicit template instantiations for row_search_mvcc */ +template dberr_t row_search_mvcc( + byte*, page_cur_mode_t, row_prebuilt_t*, ulint, ulint); + +template dberr_t row_search_mvcc( + byte*, page_cur_mode_t, row_prebuilt_t*, ulint, ulint); + +template dberr_t row_search_mvcc( + byte*, page_cur_mode_t, row_prebuilt_t*, ulint, ulint); From a9665958948732b39b5a71f2b66672a97883837b Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Fri, 15 May 2026 18:28:05 +0530 Subject: [PATCH 2/2] Serialize concurrent fulltext optimization dict_acquire_mdl(): function now supports both MDL_SHARED (default) and MDL_EXCLUSIVE via the exclusive template parameter. Updated FTS optimize to acquire MDL_EXCLUSIVE first then downgrade to MDL_SHARED_UPGRADABLE to serialize concurrent fulltext optimizations. --- storage/innobase/dict/dict0dict.cc | 60 +++++++++++++++++----------- storage/innobase/fts/fts0opt.cc | 34 +++++++++++++++- storage/innobase/include/dict0dict.h | 31 +++++++------- storage/innobase/lock/lock0lock.cc | 4 +- 4 files changed, 87 insertions(+), 42 deletions(-) diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 1d3e5c0e0543a..a148547766218 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -612,20 +612,22 @@ dict_table_t *dict_sys_t::find_table(const span &name) }); } -/** Acquire MDL shared for the table name. -@tparam trylock whether to use non-blocking operation +/** Acquire MDL for the table name. +By default, acquires MDL_SHARED lock. Use exclusive=true for MDL_EXCLUSIVE. +@tparam trylock whether to use non-blocking operation +@tparam exclusive MDL Exclusive lock (default: false, MDL_SHARED) @param[in,out] table table object @param[in,out] mdl_context MDL context @param[out] mdl MDL ticket @param[in] table_op operation to perform when opening -@return table object after locking MDL shared +@return table object after locking MDL @retval nullptr if the table is not readable, or if trylock && MDL blocked */ -template +template __attribute__((nonnull, warn_unused_result)) dict_table_t* -dict_acquire_mdl_shared(dict_table_t *table, - MDL_context *mdl_context, MDL_ticket **mdl, - dict_table_op_t table_op) +dict_acquire_mdl(dict_table_t *table, + MDL_context *mdl_context, MDL_ticket **mdl, + dict_table_op_t table_op) { char db_buf[NAME_LEN + 1], db_buf1[NAME_LEN + 1]; char tbl_buf[NAME_LEN + 1], tbl_buf1[NAME_LEN + 1]; @@ -655,7 +657,8 @@ dict_acquire_mdl_shared(dict_table_t *table, { MDL_request request; - MDL_REQUEST_INIT(&request,MDL_key::TABLE, db_buf, tbl_buf, MDL_SHARED, + MDL_REQUEST_INIT(&request,MDL_key::TABLE, db_buf, tbl_buf, + exclusive ? MDL_EXCLUSIVE : MDL_SHARED, MDL_EXPLICIT); if (trylock ? mdl_context->try_acquire_lock(&request) @@ -755,23 +758,27 @@ dict_acquire_mdl_shared(dict_table_t *table, goto retry; } -template dict_table_t* dict_acquire_mdl_shared -(dict_table_t*,MDL_context*,MDL_ticket**,dict_table_op_t); +template dict_table_t* dict_acquire_mdl + (dict_table_t*,MDL_context*,MDL_ticket**,dict_table_op_t); +template dict_table_t* dict_acquire_mdl + (dict_table_t*,MDL_context*,MDL_ticket**,dict_table_op_t); -/** Acquire MDL shared for the table name. +/** Acquire MDL for the table name. +By default, acquires MDL_SHARED lock. Use exclusive=true for MDL_EXCLUSIVE. @tparam trylock whether to use non-blocking operation +@tparam exclusive Used to take MDL_EXCLUSIVE lock (default: false, MDL_SHARED) @param[in,out] table table object @param[in,out] thd background thread @param[out] mdl mdl ticket @param[in] table_op operation to perform when opening -@return table object after locking MDL shared +@return table object after locking MDL @retval nullptr if the table is not readable, or if trylock && MDL blocked */ -template +template dict_table_t* -dict_acquire_mdl_shared(dict_table_t *table, - THD *thd, - MDL_ticket **mdl, - dict_table_op_t table_op) +dict_acquire_mdl(dict_table_t *table, + THD *thd, + MDL_ticket **mdl, + dict_table_op_t table_op) { if (!table || !mdl) return table; @@ -793,13 +800,18 @@ dict_acquire_mdl_shared(dict_table_t *table, if (db_len == 0) return table; /* InnoDB system tables are not covered by MDL */ - return dict_acquire_mdl_shared(table, &thd->mdl_context, mdl, table_op); + return dict_acquire_mdl( + table, &thd->mdl_context, mdl, table_op); } -template dict_table_t* dict_acquire_mdl_shared -(dict_table_t*,THD*,MDL_ticket**,dict_table_op_t); -template dict_table_t* dict_acquire_mdl_shared -(dict_table_t*,THD*,MDL_ticket**,dict_table_op_t); +template dict_table_t* dict_acquire_mdl + (dict_table_t*,THD*,MDL_ticket**,dict_table_op_t); +template dict_table_t* dict_acquire_mdl + (dict_table_t*,THD*,MDL_ticket**,dict_table_op_t); +template dict_table_t* dict_acquire_mdl + (dict_table_t*,THD*,MDL_ticket**,dict_table_op_t); +template dict_table_t* dict_acquire_mdl + (dict_table_t*,THD*,MDL_ticket**,dict_table_op_t); /** Look up a table by numeric identifier. @param[in] table_id table identifier @@ -824,7 +836,7 @@ dict_table_t *dict_table_open_on_id(table_id_t table_id, bool dict_locked, { if (thd) { - table= dict_acquire_mdl_shared(table, thd, mdl, table_op); + table= dict_acquire_mdl(table, thd, mdl, table_op); if (table) goto acquire; } @@ -1526,7 +1538,7 @@ dict_table_rename_in_cache( if (keep_mdl_name) { /* Preserve the original table name for - dict_table_t::parse_name() and dict_acquire_mdl_shared(). */ + dict_table_t::parse_name() and dict_acquire_mdl(). */ table->mdl_name.m_name = mem_heap_strdup(table->heap, table->name.m_name); } diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index 1902b34ce2a48..130e62a03e540 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -26,6 +26,8 @@ Completed 2011/7/10 Sunny and Jimmy Yang ***********************************************************************/ +#define MYSQL_SERVER + #include "fts0fts.h" #include "fts0exec.h" #include "row0sel.h" @@ -39,6 +41,7 @@ Completed 2011/7/10 Sunny and Jimmy Yang #include "fts0opt.h" #include "fts0vlc.h" #include "wsrep.h" +#include "sql_class.h" #ifdef WITH_WSREP extern Atomic_relaxed wsrep_sst_disable_writes; @@ -1761,6 +1764,26 @@ fts_optimize_table(dict_table_t *table, THD *thd) return DB_READ_ONLY; } + /* Serialize concurrent fts_optimize_table() on the same table: + acquire MDL_EXCLUSIVE first so a second caller blocks here, then + downgrade to MDL_SHARED_UPGRADABLE so other operations can proceed + while optimization is in progress. */ + MDL_ticket* mdl_ticket = nullptr; + dict_sys.freeze(SRW_LOCK_CALL); + if (dict_acquire_mdl(table, thd, &mdl_ticket) + != table) { + dict_sys.unfreeze(); + if (mdl_ticket) { + thd->mdl_context.release_lock(mdl_ticket); + } + return DB_TABLE_NOT_FOUND; + } + dict_sys.unfreeze(); + + if (mdl_ticket) { + mdl_ticket->downgrade_lock(MDL_SHARED_UPGRADABLE); + } + dberr_t error = DB_SUCCESS; fts_optimize_t* optim = NULL; fts_t* fts = table->fts; @@ -1775,6 +1798,9 @@ fts_optimize_table(dict_table_t *table, THD *thd) if (error != DB_SUCCESS) { err_exit: fts_optimize_free(optim); + if (mdl_ticket) { + thd->mdl_context.release_lock(mdl_ticket); + } return error; } @@ -1845,6 +1871,10 @@ fts_optimize_table(dict_table_t *table, THD *thd) fts_optimize_free(optim); + if (mdl_ticket) { + thd->mdl_context.release_lock(mdl_ticket); + } + return(error); } @@ -2108,8 +2138,8 @@ static void fts_optimize_sync_table(dict_table_t *table, bool process_message= false) { MDL_ticket* mdl_ticket= nullptr; - dict_table_t *sync_table= dict_acquire_mdl_shared(table, fts_opt_thd, - &mdl_ticket); + dict_table_t *sync_table= dict_acquire_mdl(table, fts_opt_thd, + &mdl_ticket); if (!sync_table) return; diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index 5bb79e286f0da..9fd24e3a4c069 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -93,35 +93,38 @@ enum dict_table_op_t { DICT_TABLE_OP_OPEN_ONLY_IF_CACHED }; -/** Acquire MDL shared for the table name. +/** Acquire MDL for the table name. +By default, acquires MDL_SHARED lock. Use exclusive=true for MDL_EXCLUSIVE. @tparam trylock whether to use non-blocking operation -@param[in,out] table table object +@tparam exclusive Used to take MDL_EXCLUSIVE lock (default: false, MDL_SHARED) @param[in,out] thd background thread @param[out] mdl mdl ticket @param[in] table_op operation to perform when opening -@return table object after locking MDL shared +@return table object after locking MDL @retval NULL if the table is not readable, or if trylock && MDL blocked */ -template +template dict_table_t* -dict_acquire_mdl_shared(dict_table_t *table, - THD *thd, - MDL_ticket **mdl, - dict_table_op_t table_op= DICT_TABLE_OP_NORMAL); +dict_acquire_mdl(dict_table_t *table, + THD *thd, + MDL_ticket **mdl, + dict_table_op_t table_op= DICT_TABLE_OP_NORMAL); -/** Acquire MDL shared for the table name. +/** Acquire MDL for the table name. +By default, acquires MDL_SHARED lock. Use exclusive=true for MDL_EXCLUSIVE. @tparam trylock whether to use non-blocking operation +@tparam exclusive Used to take MDL_EXCLUSIVE lock (default: false, MDL_SHARED) @param[in,out] table table object @param[in,out] mdl_context MDL context @param[out] mdl MDL ticket @param[in] table_op operation to perform when opening -@return table object after locking MDL shared +@return table object after locking MDL @retval nullptr if the table is not readable, or if trylock && MDL blocked */ -template +template __attribute__((nonnull, warn_unused_result)) dict_table_t* -dict_acquire_mdl_shared(dict_table_t *table, - MDL_context *mdl_context, MDL_ticket **mdl, - dict_table_op_t table_op); +dict_acquire_mdl(dict_table_t *table, + MDL_context *mdl_context, MDL_ticket **mdl, + dict_table_op_t table_op); /** Look up a table by numeric identifier. @param[in] table_id table identifier diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 0642ff58310e1..51c9e2db81f33 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -4328,8 +4328,8 @@ dberr_t lock_table_children(dict_table_t *table, trx_t *trx) children.end()) continue; /* We already acquired MDL on this child table. */ MDL_ticket *mdl= nullptr; - child= dict_acquire_mdl_shared(child, mdl_context, &mdl, - DICT_TABLE_OP_NORMAL); + child= dict_acquire_mdl(child, mdl_context, &mdl, + DICT_TABLE_OP_NORMAL); if (child) { if (mdl)