diff --git a/mysql-test/suite/innodb_fts/r/index_table.result b/mysql-test/suite/innodb_fts/r/index_table.result index 909a889db4230..78832669cdbca 100644 --- a/mysql-test/suite/innodb_fts/r/index_table.result +++ b/mysql-test/suite/innodb_fts/r/index_table.result @@ -5,7 +5,7 @@ id INT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY, title VARCHAR(200), content TEXT ) ENGINE= InnoDB; -SET STATEMENT debug_dbug='+d,innodb_report_deadlock' FOR +SET STATEMENT debug_dbug='+d,fts_load_stopword_fail' FOR CREATE FULLTEXT INDEX idx ON articles (title, content); ERROR HY000: Got error 11 "Resource temporarily unavailable" from storage engine InnoDB CREATE FULLTEXT INDEX idx ON articles (title, content); diff --git a/mysql-test/suite/innodb_fts/r/sync.result b/mysql-test/suite/innodb_fts/r/sync.result index 928efffdb21d9..862df4cb4b3e4 100644 --- a/mysql-test/suite/innodb_fts/r/sync.result +++ b/mysql-test/suite/innodb_fts/r/sync.result @@ -25,7 +25,7 @@ mysql 1 3 2 3 0 SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE; WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION SET GLOBAL innodb_ft_aux_table=default; -SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); +SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database') ORDER BY FTS_DOC_ID; FTS_DOC_ID title 1 mysql 2 database @@ -42,11 +42,11 @@ database 2 3 2 3 6 mysql 1 3 2 1 0 mysql 1 3 2 3 0 SET GLOBAL innodb_ft_aux_table=default; -SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); +SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database') ORDER BY FTS_DOC_ID; FTS_DOC_ID title -3 mysql database 1 mysql 2 database +3 mysql database connection default; DROP TABLE t1; # Case 2: Test insert and insert(sync) @@ -79,12 +79,12 @@ mysql 1 4 3 1 0 mysql 1 4 3 3 0 mysql 1 4 3 4 0 SET GLOBAL innodb_ft_aux_table=default; -SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); +SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database') ORDER BY FTS_DOC_ID; FTS_DOC_ID title -3 mysql database -4 mysql database 1 mysql 2 database +3 mysql database +4 mysql database connection default; disconnect con1; DROP TABLE t1; diff --git a/mysql-test/suite/innodb_fts/t/index_table.test b/mysql-test/suite/innodb_fts/t/index_table.test index 89c0905323083..cfe27b4226848 100644 --- a/mysql-test/suite/innodb_fts/t/index_table.test +++ b/mysql-test/suite/innodb_fts/t/index_table.test @@ -18,7 +18,7 @@ CREATE TABLE articles ( ) ENGINE= InnoDB; --error ER_GET_ERRNO -SET STATEMENT debug_dbug='+d,innodb_report_deadlock' FOR +SET STATEMENT debug_dbug='+d,fts_load_stopword_fail' FOR CREATE FULLTEXT INDEX idx ON articles (title, content); CREATE FULLTEXT INDEX idx ON articles (title, content); diff --git a/mysql-test/suite/innodb_fts/t/sync.test b/mysql-test/suite/innodb_fts/t/sync.test index 56b9052a47ab8..348c37d7389d8 100644 --- a/mysql-test/suite/innodb_fts/t/sync.test +++ b/mysql-test/suite/innodb_fts/t/sync.test @@ -40,7 +40,7 @@ SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE; SET GLOBAL innodb_ft_aux_table=default; -SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); +SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database') ORDER BY FTS_DOC_ID; SET DEBUG_SYNC= 'now SIGNAL selected'; @@ -54,7 +54,7 @@ SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE; SET GLOBAL innodb_ft_aux_table=default; -SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); +SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database') ORDER BY FTS_DOC_ID; connection default; @@ -96,7 +96,7 @@ SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE; SET GLOBAL innodb_ft_aux_table=default; -SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); +SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database') ORDER BY FTS_DOC_ID; connection default; disconnect con1; diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 9e3a23b34ab46..c1b4a5c9559fc 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -172,10 +172,10 @@ SET(INNOBASE_SOURCES fts/fts0ast.cc fts/fts0blex.cc fts/fts0config.cc + fts/fts0exec.cc fts/fts0opt.cc fts/fts0pars.cc fts/fts0que.cc - fts/fts0sql.cc fts/fts0tlex.cc gis/gis0geo.cc gis/gis0rtree.cc @@ -242,6 +242,7 @@ SET(INNOBASE_SOURCES include/fsp_binlog.h include/fts0ast.h include/fts0blex.h + include/fts0exec.h include/fts0fts.h include/fts0opt.h include/fts0pars.h @@ -315,6 +316,7 @@ SET(INNOBASE_SOURCES include/row0mysql.h include/row0purge.h include/row0quiesce.h + include/row0query.h include/row0row.h include/row0row.inl include/row0sel.h @@ -403,6 +405,7 @@ SET(INNOBASE_SOURCES row/row0undo.cc row/row0upd.cc row/row0quiesce.cc + row/row0query.cc row/row0vers.cc srv/srv0mon.cc srv/srv0srv.cc diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 1d3e5c0e0543a..a148547766218 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -612,20 +612,22 @@ dict_table_t *dict_sys_t::find_table(const span &name) }); } -/** Acquire MDL shared for the table name. -@tparam trylock whether to use non-blocking operation +/** Acquire MDL for the table name. +By default, acquires MDL_SHARED lock. Use exclusive=true for MDL_EXCLUSIVE. +@tparam trylock whether to use non-blocking operation +@tparam exclusive MDL Exclusive lock (default: false, MDL_SHARED) @param[in,out] table table object @param[in,out] mdl_context MDL context @param[out] mdl MDL ticket @param[in] table_op operation to perform when opening -@return table object after locking MDL shared +@return table object after locking MDL @retval nullptr if the table is not readable, or if trylock && MDL blocked */ -template +template __attribute__((nonnull, warn_unused_result)) dict_table_t* -dict_acquire_mdl_shared(dict_table_t *table, - MDL_context *mdl_context, MDL_ticket **mdl, - dict_table_op_t table_op) +dict_acquire_mdl(dict_table_t *table, + MDL_context *mdl_context, MDL_ticket **mdl, + dict_table_op_t table_op) { char db_buf[NAME_LEN + 1], db_buf1[NAME_LEN + 1]; char tbl_buf[NAME_LEN + 1], tbl_buf1[NAME_LEN + 1]; @@ -655,7 +657,8 @@ dict_acquire_mdl_shared(dict_table_t *table, { MDL_request request; - MDL_REQUEST_INIT(&request,MDL_key::TABLE, db_buf, tbl_buf, MDL_SHARED, + MDL_REQUEST_INIT(&request,MDL_key::TABLE, db_buf, tbl_buf, + exclusive ? MDL_EXCLUSIVE : MDL_SHARED, MDL_EXPLICIT); if (trylock ? mdl_context->try_acquire_lock(&request) @@ -755,23 +758,27 @@ dict_acquire_mdl_shared(dict_table_t *table, goto retry; } -template dict_table_t* dict_acquire_mdl_shared -(dict_table_t*,MDL_context*,MDL_ticket**,dict_table_op_t); +template dict_table_t* dict_acquire_mdl + (dict_table_t*,MDL_context*,MDL_ticket**,dict_table_op_t); +template dict_table_t* dict_acquire_mdl + (dict_table_t*,MDL_context*,MDL_ticket**,dict_table_op_t); -/** Acquire MDL shared for the table name. +/** Acquire MDL for the table name. +By default, acquires MDL_SHARED lock. Use exclusive=true for MDL_EXCLUSIVE. @tparam trylock whether to use non-blocking operation +@tparam exclusive Used to take MDL_EXCLUSIVE lock (default: false, MDL_SHARED) @param[in,out] table table object @param[in,out] thd background thread @param[out] mdl mdl ticket @param[in] table_op operation to perform when opening -@return table object after locking MDL shared +@return table object after locking MDL @retval nullptr if the table is not readable, or if trylock && MDL blocked */ -template +template dict_table_t* -dict_acquire_mdl_shared(dict_table_t *table, - THD *thd, - MDL_ticket **mdl, - dict_table_op_t table_op) +dict_acquire_mdl(dict_table_t *table, + THD *thd, + MDL_ticket **mdl, + dict_table_op_t table_op) { if (!table || !mdl) return table; @@ -793,13 +800,18 @@ dict_acquire_mdl_shared(dict_table_t *table, if (db_len == 0) return table; /* InnoDB system tables are not covered by MDL */ - return dict_acquire_mdl_shared(table, &thd->mdl_context, mdl, table_op); + return dict_acquire_mdl( + table, &thd->mdl_context, mdl, table_op); } -template dict_table_t* dict_acquire_mdl_shared -(dict_table_t*,THD*,MDL_ticket**,dict_table_op_t); -template dict_table_t* dict_acquire_mdl_shared -(dict_table_t*,THD*,MDL_ticket**,dict_table_op_t); +template dict_table_t* dict_acquire_mdl + (dict_table_t*,THD*,MDL_ticket**,dict_table_op_t); +template dict_table_t* dict_acquire_mdl + (dict_table_t*,THD*,MDL_ticket**,dict_table_op_t); +template dict_table_t* dict_acquire_mdl + (dict_table_t*,THD*,MDL_ticket**,dict_table_op_t); +template dict_table_t* dict_acquire_mdl + (dict_table_t*,THD*,MDL_ticket**,dict_table_op_t); /** Look up a table by numeric identifier. @param[in] table_id table identifier @@ -824,7 +836,7 @@ dict_table_t *dict_table_open_on_id(table_id_t table_id, bool dict_locked, { if (thd) { - table= dict_acquire_mdl_shared(table, thd, mdl, table_op); + table= dict_acquire_mdl(table, thd, mdl, table_op); if (table) goto acquire; } @@ -1526,7 +1538,7 @@ dict_table_rename_in_cache( if (keep_mdl_name) { /* Preserve the original table name for - dict_table_t::parse_name() and dict_acquire_mdl_shared(). */ + dict_table_t::parse_name() and dict_acquire_mdl(). */ table->mdl_name.m_name = mem_heap_strdup(table->heap, table->name.m_name); } diff --git a/storage/innobase/fts/fts0config.cc b/storage/innobase/fts/fts0config.cc index 4ff14edf5d016..fa127c27b058a 100644 --- a/storage/innobase/fts/fts0config.cc +++ b/storage/innobase/fts/fts0config.cc @@ -27,100 +27,33 @@ Created 2007/5/9 Sunny Bains #include "trx0roll.h" #include "row0sel.h" +#include "fts0exec.h" #include "fts0priv.h" +#include "log.h" -/******************************************************************//** -Callback function for fetching the config value. -@return always returns TRUE */ -static -ibool -fts_config_fetch_value( -/*===================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to - ib_vector_t */ -{ - sel_node_t* node = static_cast(row); - fts_string_t* value = static_cast(user_arg); - - dfield_t* dfield = que_node_get_val(node->select_list); - dtype_t* type = dfield_get_type(dfield); - ulint len = dfield_get_len(dfield); - void* data = dfield_get_data(dfield); - - ut_a(dtype_get_mtype(type) == DATA_VARCHAR); - - if (len != UNIV_SQL_NULL) { - ulint max_len = ut_min(value->f_len - 1, len); - - memcpy(value->f_str, data, max_len); - value->f_len = max_len; - value->f_str[value->f_len] = '\0'; - } - - return(TRUE); -} - -/******************************************************************//** -Get value from the config table. The caller must ensure that enough +/** Get value from the config table. The caller must ensure that enough space is allocated for value to hold the column contents. +@param trx transaction +@param table Indexed fts table +@param name name of the key +@param value value of the key @return DB_SUCCESS or error code */ -dberr_t -fts_config_get_value( -/*=================*/ - trx_t* trx, /*!< transaction */ - fts_table_t* fts_table, /*!< in: the indexed - FTS table */ - const char* name, /*!< in: get config value for - this parameter name */ - fts_string_t* value) /*!< out: value read from - config table */ +dberr_t fts_config_get_value(FTSQueryExecutor *executor, const dict_table_t *table, + const char *name, fts_string_t *value) noexcept { - pars_info_t* info; - que_t* graph; - dberr_t error; - ulint name_len = strlen(name); - char table_name[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - *value->f_str = '\0'; - ut_a(value->f_len > 0); - - pars_info_bind_function(info, "my_func", fts_config_fetch_value, - value); - - /* The len field of value must be set to the max bytes that - it can hold. On a successful read, the len field will be set - to the actual number of bytes copied to value. */ - pars_info_bind_varchar_literal(info, "name", (byte*) name, name_len); - - fts_table->suffix = "CONFIG"; - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS SELECT value FROM $table_name" - " WHERE key = :name;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - trx->op_info = "getting FTS config value"; - - error = fts_eval_sql(trx, graph); - que_graph_free(graph); - return(error); + executor->trx()->op_info= "getting FTS config value"; + ConfigReader reader; + dberr_t err= executor->read_config_with_lock(name, reader); + if (err == DB_SUCCESS) + { + ulint max_len= ut_min(value->f_len - 1, reader.value_span.size()); + memcpy(value->f_str, reader.value_span.data(), max_len); + value->f_len= max_len; + value->f_str[value->f_len]= '\0'; + executor->release_lock(); + } + else value->f_str[0]= '\0'; + return err; } /*********************************************************************//** @@ -157,105 +90,38 @@ column contents. dberr_t fts_config_get_index_value( /*=======================*/ - trx_t* trx, /*!< transaction */ + FTSQueryExecutor* executor, /*!< in: query executor */ dict_index_t* index, /*!< in: index */ const char* param, /*!< in: get config value for this parameter name */ - fts_string_t* value) /*!< out: value read from + fts_string_t* value) noexcept /*!< out: value read from config table */ { - char* name; - dberr_t error; - fts_table_t fts_table; - - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, - index->table); - /* We are responsible for free'ing name. */ - name = fts_config_create_index_param_name(param, index); + char *name = fts_config_create_index_param_name(param, index); - error = fts_config_get_value(trx, &fts_table, name, value); + dberr_t error = fts_config_get_value(executor, index->table, name, value); ut_free(name); return(error); } -/******************************************************************//** -Set the value in the config table for name. +/** Set the value in the config table for name. +@param executor query executor +@param table indexed fulltext table +@param name key for the config +@param value value of the key @return DB_SUCCESS or error code */ dberr_t -fts_config_set_value( -/*=================*/ - trx_t* trx, /*!< transaction */ - fts_table_t* fts_table, /*!< in: the indexed - FTS table */ - const char* name, /*!< in: get config value for - this parameter name */ - const fts_string_t* - value) /*!< in: value to update */ +fts_config_set_value(FTSQueryExecutor *executor, const dict_table_t *table, + const char *name, const fts_string_t *value) noexcept { - pars_info_t* info; - que_t* graph; - dberr_t error; - undo_no_t undo_no; - undo_no_t n_rows_updated; - ulint name_len = strlen(name); - char table_name[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - pars_info_bind_varchar_literal(info, "name", (byte*) name, name_len); - pars_info_bind_varchar_literal(info, "value", - value->f_str, value->f_len); - - const bool dict_locked = fts_table->table->fts->dict_locked; - - fts_table->suffix = "CONFIG"; - fts_get_table_name(fts_table, table_name, dict_locked); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, info, - "BEGIN UPDATE $table_name SET value = :value" - " WHERE key = :name;"); - - trx->op_info = "setting FTS config value"; - - undo_no = trx->undo_no; - - error = fts_eval_sql(trx, graph); - - que_graph_free(graph); - - n_rows_updated = trx->undo_no - undo_no; - - /* Check if we need to do an insert. */ - if (error == DB_SUCCESS && n_rows_updated == 0) { - info = pars_info_create(); - - pars_info_bind_varchar_literal( - info, "name", (byte*) name, name_len); - - pars_info_bind_varchar_literal( - info, "value", value->f_str, value->f_len); - - fts_get_table_name(fts_table, table_name, dict_locked); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, info, - "BEGIN\n" - "INSERT INTO $table_name VALUES(:name, :value);"); - - trx->op_info = "inserting FTS config value"; - - error = fts_eval_sql(trx, graph); - - que_graph_free(graph); - } - - return(error); + executor->trx()->op_info= "setting FTS config value"; + char value_str[FTS_MAX_CONFIG_VALUE_LEN + 1]; + memcpy(value_str, value->f_str, value->f_len); + value_str[value->f_len]= '\0'; + return executor->update_config_record(name, value_str); } /******************************************************************//** @@ -264,95 +130,44 @@ Set the value specific to an FTS index in the config table. dberr_t fts_config_set_index_value( /*=======================*/ - trx_t* trx, /*!< transaction */ + FTSQueryExecutor* executor, /*!< in: query executor */ dict_index_t* index, /*!< in: index */ const char* param, /*!< in: get config value for this parameter name */ - fts_string_t* value) /*!< out: value read from + fts_string_t* value) noexcept /*!< out: value read from config table */ { - char* name; - dberr_t error; - fts_table_t fts_table; - - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, - index->table); - /* We are responsible for free'ing name. */ - name = fts_config_create_index_param_name(param, index); + char *name = fts_config_create_index_param_name(param, index); - error = fts_config_set_value(trx, &fts_table, name, value); + dberr_t error = fts_config_set_value(executor, index->table, name, value); ut_free(name); return(error); } -/******************************************************************//** -Get an ulint value from the config table. +/** Set an ulint value in the config table. +@param trx transaction +@param table user table +@param name name of the key +@param int_value value of the key to be set @return DB_SUCCESS if all OK else error code */ dberr_t -fts_config_get_ulint( -/*=================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: the indexed - FTS table */ - const char* name, /*!< in: param name */ - ulint* int_value) /*!< out: value */ +fts_config_set_ulint(FTSQueryExecutor *executor, const dict_table_t *table, + const char *name, ulint int_value) noexcept { - dberr_t error; - fts_string_t value; - - /* We set the length of value to the max bytes it can hold. This - information is used by the callback that reads the value.*/ - value.f_len = FTS_MAX_CONFIG_VALUE_LEN; - value.f_str = static_cast(ut_malloc_nokey(value.f_len + 1)); - - error = fts_config_get_value(trx, fts_table, name, &value); - - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ") reading `" << name << "'"; - } else { - *int_value = strtoul((char*) value.f_str, NULL, 10); - } - - ut_free(value.f_str); - - return(error); -} - -/******************************************************************//** -Set an ulint value in the config table. -@return DB_SUCCESS if all OK else error code */ -dberr_t -fts_config_set_ulint( -/*=================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: the indexed - FTS table */ - const char* name, /*!< in: param name */ - ulint int_value) /*!< in: value */ -{ - dberr_t error; - fts_string_t value; - - /* We set the length of value to the max bytes it can hold. This - information is used by the callback that reads the value.*/ - value.f_len = FTS_MAX_CONFIG_VALUE_LEN; - value.f_str = static_cast(ut_malloc_nokey(value.f_len + 1)); - - ut_a(FTS_MAX_INT_LEN < FTS_MAX_CONFIG_VALUE_LEN); - - value.f_len = (ulint) snprintf( - (char*) value.f_str, FTS_MAX_INT_LEN, ULINTPF, int_value); - - error = fts_config_set_value(trx, fts_table, name, &value); - - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ") writing `" << name << "'"; - } - - ut_free(value.f_str); - - return(error); + fts_string_t value; + /* We set the length of value to the max bytes it can hold. This + information is used by the callback that reads the value.*/ + value.f_len= FTS_MAX_CONFIG_VALUE_LEN; + value.f_str= static_cast(ut_malloc_nokey(value.f_len + 1)); + ut_a(FTS_MAX_INT_LEN < FTS_MAX_CONFIG_VALUE_LEN); + value.f_len= (ulint) snprintf((char*) value.f_str, FTS_MAX_INT_LEN, + ULINTPF, int_value); + dberr_t error= fts_config_set_value(executor, table, name, &value); + if (UNIV_UNLIKELY(error != DB_SUCCESS)) + sql_print_error("InnoDB: (%s) writing `%s'", ut_strerr(error), name); + ut_free(value.f_str); + return error; } diff --git a/storage/innobase/fts/fts0exec.cc b/storage/innobase/fts/fts0exec.cc new file mode 100644 index 0000000000000..210ae28dc2788 --- /dev/null +++ b/storage/innobase/fts/fts0exec.cc @@ -0,0 +1,886 @@ +/***************************************************************************** +Copyright (c) 2025, MariaDB PLC. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA +*****************************************************************************/ + +/**************************************************//** +@file fts/fts0exec.cc + +Created 2025/11/05 +*******************************************************/ + +#include "fts0exec.h" +#include "row0query.h" +#include "fts0fts.h" +#include "fts0types.h" +#include "fts0vlc.h" +#include "fts0priv.h" +#include "btr0btr.h" +#include "btr0cur.h" +#include "dict0dict.h" +#include "row0ins.h" +#include "row0upd.h" +#include "row0sel.h" +#include "eval0eval.h" +#include "que0que.h" +#include "trx0trx.h" +#include "lock0lock.h" +#include "rem0cmp.h" +#include "page0cur.h" +#include "ha_prototypes.h" + + +FTSQueryExecutor::FTSQueryExecutor( + trx_t *trx, const dict_table_t *fts_table) + : m_executor(trx), m_table(fts_table) +{} + +FTSQueryExecutor::~FTSQueryExecutor() +{ + for (uint8_t i= 0; i < FTS_NUM_AUX_INDEX; i++) + if (m_aux_tables[i]) m_aux_tables[i]->release(); + + for (uint8_t i= 0; i < NUM_DELETION_TABLES; i++) + if (m_common_tables[i]) m_common_tables[i]->release(); + + if (m_config_table) m_config_table->release(); +} + +dberr_t FTSQueryExecutor::open_aux_table(uint8_t aux_index) noexcept +{ + if (m_aux_tables[aux_index]) return DB_SUCCESS; + + char table_name[MAX_FULL_NAME_LEN]; + construct_table_name(table_name, fts_get_suffix(aux_index), false); + + m_aux_tables[aux_index]= dict_table_open_on_name( + table_name, false, DICT_ERR_IGNORE_TABLESPACE); + return m_aux_tables[aux_index] ? DB_SUCCESS : DB_TABLE_NOT_FOUND; +} + +dberr_t FTSQueryExecutor::open_all_aux_tables(dict_index_t *fts_index) noexcept +{ + for (uint8_t idx= 0; idx < FTS_NUM_AUX_INDEX; idx++) + { + dict_table_t *table= m_aux_tables[idx]; + if (table) + { + table->release(); + m_aux_tables[idx]= nullptr; + } + } + m_index= fts_index; + for (uint8_t idx= 0; idx < FTS_NUM_AUX_INDEX; idx++) + { + dberr_t err= open_aux_table(idx); + if (err) return err; + } + return DB_SUCCESS; +} + +const char* FTSQueryExecutor::get_deletion_table_name( + FTSDeletionTable table_type) noexcept +{ + switch (table_type) + { + case FTSDeletionTable::DELETED: return "DELETED"; + case FTSDeletionTable::DELETED_CACHE: return "DELETED_CACHE"; + case FTSDeletionTable::BEING_DELETED: return "BEING_DELETED"; + case FTSDeletionTable::BEING_DELETED_CACHE: return "BEING_DELETED_CACHE"; + default: return nullptr; + } +} + +/** Helper to convert table name to deletion table enum */ +static FTSDeletionTable get_deletion_table_type(const char *tbl_name) noexcept +{ + if (!strcmp(tbl_name, "DELETED")) return FTSDeletionTable::DELETED; + if (!strcmp(tbl_name, "DELETED_CACHE")) return FTSDeletionTable::DELETED_CACHE; + if (!strcmp(tbl_name, "BEING_DELETED")) return FTSDeletionTable::BEING_DELETED; + if (!strcmp(tbl_name, "BEING_DELETED_CACHE")) return FTSDeletionTable::BEING_DELETED_CACHE; + return FTSDeletionTable::MAX_DELETION_TABLES; +} + +dberr_t FTSQueryExecutor::open_deletion_table(FTSDeletionTable table_type) noexcept +{ + uint8_t index= to_index(table_type); + if (index >= NUM_DELETION_TABLES) + return DB_ERROR; + + if (m_common_tables[index]) return DB_SUCCESS; + + const char *suffix_name= get_deletion_table_name(table_type); + if (!suffix_name) return DB_ERROR; + + char table_name[MAX_FULL_NAME_LEN]; + construct_table_name(table_name, suffix_name, true); + + m_common_tables[index]= dict_table_open_on_name( + table_name, false, DICT_ERR_IGNORE_TABLESPACE); + return m_common_tables[index] ? DB_SUCCESS : DB_TABLE_NOT_FOUND; +} + +dberr_t FTSQueryExecutor::open_config_table() noexcept +{ + if (m_config_table) return DB_SUCCESS; + char table_name[MAX_FULL_NAME_LEN]; + construct_table_name(table_name, "CONFIG", true); + + m_config_table= dict_table_open_on_name( + table_name, false, DICT_ERR_IGNORE_TABLESPACE); + return m_config_table ? DB_SUCCESS : DB_TABLE_NOT_FOUND; +} + +dberr_t FTSQueryExecutor::open_all_deletion_tables() noexcept +{ + for (uint8_t i= 0; i < NUM_DELETION_TABLES; i++) + { + FTSDeletionTable table_type= static_cast(i); + dberr_t err= open_deletion_table(table_type); + if (err) return err; + } + return DB_SUCCESS; +} + +dberr_t FTSQueryExecutor::lock_aux_tables(uint8_t aux_index, + lock_mode mode) noexcept +{ + dict_table_t *table= m_aux_tables[aux_index]; + if (table == nullptr) return DB_TABLE_NOT_FOUND; + dberr_t err= m_executor.lock_table(table, mode); + if (err == DB_LOCK_WAIT) err= m_executor.handle_wait(err, true); + return err; +} + +dberr_t FTSQueryExecutor::lock_common_tables(uint8_t index, + lock_mode mode) noexcept +{ + dict_table_t *table= m_common_tables[index]; + if (table == nullptr) return DB_TABLE_NOT_FOUND; + dberr_t err= m_executor.lock_table(table, mode); + if (err == DB_LOCK_WAIT) err= m_executor.handle_wait(err, true); + return err; +} + +dberr_t FTSQueryExecutor::insert_aux_record( + uint8_t aux_index, const fts_aux_data_t *aux_data) noexcept +{ + ut_ad(!dict_sys.locked()); + if (aux_index >= FTS_NUM_AUX_INDEX) return DB_ERROR; + + dberr_t err= open_aux_table(aux_index); + if (err != DB_SUCCESS) return err; + err= lock_aux_tables(aux_index, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t *table= m_aux_tables[aux_index]; + dict_index_t* index= dict_table_get_first_index(table); + + if (index->n_fields != 7 || index->n_uniq != 2) + return DB_ERROR; + + byte sys_buf[DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN]= {0}; + dfield_t fields[7]; + doc_id_t first_doc_id, last_doc_id; + + dtuple_t tuple{0, 7, 2, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 7); + /* Field 0: word (VARCHAR) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, aux_data->word, aux_data->word_len); + + /* Field 1: first_doc_id (INT) */ + field= dtuple_get_nth_field(&tuple, 1); + fts_write_doc_id(&first_doc_id, aux_data->first_doc_id); + dfield_set_data(field, &first_doc_id, sizeof(doc_id_t)); + + /* Field 2: trx_id (DB_TRX_ID) */ + field= dtuple_get_nth_field(&tuple, 2); + dfield_set_data(field, sys_buf, DATA_TRX_ID_LEN); + + /* Field 3: roll_ptr (DB_ROLL_PTR) */ + field= dtuple_get_nth_field(&tuple, 3); + dfield_set_data(field, sys_buf + DATA_TRX_ID_LEN, DATA_ROLL_PTR_LEN); + + /* Field 4: last_doc_id (UNSIGNED INT) */ + field= dtuple_get_nth_field(&tuple, 4); + fts_write_doc_id(&last_doc_id, aux_data->last_doc_id); + dfield_set_data(field, &last_doc_id, sizeof(doc_id_t)); + + /* Field 5: doc_count (UINT32_T) */ + byte doc_count[4]; + mach_write_to_4(doc_count, aux_data->doc_count); + field= dtuple_get_nth_field(&tuple, 5); + dfield_set_data(field, doc_count, sizeof(doc_count)); + + /* Field 6: ilist (VARBINARY) */ + field= dtuple_get_nth_field(&tuple, 6); + dfield_set_data(field, aux_data->ilist, aux_data->ilist_len); + + return m_executor.insert_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::insert_common_record( + const char *tbl_name, doc_id_t doc_id) noexcept +{ + ut_ad(!dict_sys.locked()); + FTSDeletionTable table_type= get_deletion_table_type(tbl_name); + if (table_type == FTSDeletionTable::MAX_DELETION_TABLES) return DB_ERROR; + + dberr_t err= open_deletion_table(table_type); + if (err != DB_SUCCESS) return err; + + uint8_t index_no= to_index(table_type); + err= lock_common_tables(index_no, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[index_no]; + dict_index_t* index= dict_table_get_first_index(table); + + if (index->n_fields != 3 || index->n_uniq != 1) + return DB_ERROR; + + byte sys_buf[DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN]= {0}; + dfield_t fields[3]; + + dtuple_t tuple{0, 3, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 3); + /* Field 0: doc_id (INT) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id(&write_doc_id, doc_id); + dfield_set_data(field, &write_doc_id, sizeof(doc_id_t)); + + /* Field 1: trx_id (DB_TRX_ID) */ + field= dtuple_get_nth_field(&tuple, 1); + dfield_set_data(field, sys_buf, DATA_TRX_ID_LEN); + + /* Field 2: roll_ptr (DB_ROLL_PTR) */ + field= dtuple_get_nth_field(&tuple, 2); + dfield_set_data(field, sys_buf + DATA_TRX_ID_LEN, DATA_ROLL_PTR_LEN); + + return m_executor.insert_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::insert_config_record( + const char *key, const char *value) noexcept +{ + dberr_t err= open_config_table(); + if (err != DB_SUCCESS) return err; + + err= m_executor.lock_table(m_config_table, LOCK_IX); + if (err == DB_LOCK_WAIT) err= m_executor.handle_wait(err, true); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_config_table; + dict_index_t* index= dict_table_get_first_index(table); + + if (index->n_fields != 4 || index->n_uniq != 1) + return DB_ERROR; + + byte sys_buf[DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN]= {0}; + dfield_t fields[4]; + + dtuple_t tuple{0, 4, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 4); + /* Field 0: key (CHAR(50)) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, key, strlen(key)); + + /* Field 1: trx_id (DB_TRX_ID) */ + field= dtuple_get_nth_field(&tuple, 1); + dfield_set_data(field, sys_buf, DATA_TRX_ID_LEN); + + /* Field 2: roll_ptr (DB_ROLL_PTR) */ + field= dtuple_get_nth_field(&tuple, 2); + dfield_set_data(field, sys_buf + DATA_TRX_ID_LEN, DATA_ROLL_PTR_LEN); + + /* Field 3: value (CHAR(200)) */ + field= dtuple_get_nth_field(&tuple, 3); + dfield_set_data(field, value, strlen(value)); + + return m_executor.insert_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::update_config_record( + const char *key, const char *value) noexcept +{ + dberr_t err= open_config_table(); + if (err != DB_SUCCESS) return err; + + err= m_executor.lock_table(m_config_table, LOCK_IX); + if (err == DB_LOCK_WAIT) err= m_executor.handle_wait(err, true); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_config_table; + dict_index_t* index= dict_table_get_first_index(table); + + if (index->n_fields != 4 || index->n_uniq != 1) + return DB_ERROR; + + byte sys_buf[DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN]= {0}; + dfield_t search_fields[1]; + dfield_t insert_fields[4]; + + dtuple_t search_tuple{0, 1, 1, 0, search_fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&search_tuple, index, 1); + dfield_t *field= dtuple_get_nth_field(&search_tuple, 0); + dfield_set_data(field, key, strlen(key)); + + dtuple_t insert_tuple{0, 4, 1, 0, insert_fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&insert_tuple, index, 4); + + /* Field 0: key (CHAR(50)) */ + field= dtuple_get_nth_field(&insert_tuple, 0); + dfield_set_data(field, key, strlen(key)); + + /* Field 1: trx_id (DB_TRX_ID) */ + field= dtuple_get_nth_field(&insert_tuple, 1); + dfield_set_data(field, sys_buf, DATA_TRX_ID_LEN); + + /* Field 2: roll_ptr (DB_ROLL_PTR) */ + field= dtuple_get_nth_field(&insert_tuple, 2); + dfield_set_data(field, sys_buf + DATA_TRX_ID_LEN, DATA_ROLL_PTR_LEN); + + /* Field 3: value (CHAR(200)) */ + field= dtuple_get_nth_field(&insert_tuple, 3); + dfield_set_data(field, value, strlen(value)); + + upd_field_t upd_field; + upd_field.field_no= 3; + upd_field.orig_len= 0; + upd_field.exp= nullptr; + dfield_set_data(&upd_field.new_val, value, strlen(value)); + dict_col_copy_type(dict_index_get_nth_col(index, 3), + dfield_get_type(&upd_field.new_val)); + + upd_t update; + update.heap= nullptr; + update.info_bits= 0; + update.old_vrow= nullptr; + update.n_fields= 1; + update.fields= &upd_field; + + return m_executor.replace_record(table, &search_tuple, &update, + &insert_tuple); +} + +dberr_t FTSQueryExecutor::delete_aux_record( + uint8_t aux_index, const fts_aux_data_t *aux_data) noexcept +{ + ut_ad(!dict_sys.locked()); + if (aux_index >= FTS_NUM_AUX_INDEX) return DB_ERROR; + + dberr_t err= open_aux_table(aux_index); + if (err != DB_SUCCESS) return err; + err= lock_aux_tables(aux_index, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t *table= m_aux_tables[aux_index]; + dict_index_t *index= dict_table_get_first_index(table); + + if (dict_table_get_next_index(index) != nullptr) + return DB_ERROR; + + dfield_t fields[1]; + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: word (VARCHAR) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, aux_data->word, aux_data->word_len); + + return m_executor.delete_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::delete_common_record( + const char *table_name, doc_id_t doc_id) noexcept +{ + ut_ad(!dict_sys.locked()); + FTSDeletionTable table_type= get_deletion_table_type(table_name); + if (table_type == FTSDeletionTable::MAX_DELETION_TABLES) + return DB_ERROR; + + dberr_t err= open_deletion_table(table_type); + if (err != DB_SUCCESS) return err; + + uint8_t cached_index= to_index(table_type); + err= lock_common_tables(cached_index, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[cached_index]; + dict_index_t* index= dict_table_get_first_index(table); + + dfield_t fields[1]; + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: doc_id */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id(&write_doc_id, doc_id); + dfield_set_data(field, &write_doc_id, sizeof(doc_id_t)); + + return m_executor.delete_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::delete_all_common_records( + const char *table_name) noexcept +{ + ut_ad(!dict_sys.locked()); + FTSDeletionTable table_type= get_deletion_table_type(table_name); + if (table_type == FTSDeletionTable::MAX_DELETION_TABLES) return DB_ERROR; + + dberr_t err= open_deletion_table(table_type); + if (err != DB_SUCCESS) return err; + + uint8_t cached_index= to_index(table_type); + err= lock_common_tables(cached_index, LOCK_X); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[cached_index]; + return m_executor.delete_all(table); +} + +dberr_t FTSQueryExecutor::delete_config_record(const char *key) noexcept +{ + ut_ad(!dict_sys.locked()); + dberr_t err= open_config_table(); + if (err != DB_SUCCESS) return err; + + err= m_executor.lock_table(m_config_table, LOCK_IX); + if (err == DB_LOCK_WAIT) err= m_executor.handle_wait(err, true); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_config_table; + dict_index_t* index= dict_table_get_first_index(table); + + dfield_t fields[1]; + + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: key (CHAR(50)) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, key, strlen(key)); + + return m_executor.delete_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::read_config_with_lock(const char *key, + RecordCallback &callback) noexcept +{ + ut_ad(!dict_sys.locked()); + dberr_t err= open_config_table(); + if (err != DB_SUCCESS) return err; + + err= m_executor.lock_table(m_config_table, LOCK_IX); + if (err == DB_LOCK_WAIT) err= m_executor.handle_wait(err, true); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_config_table; + dict_index_t* index= dict_table_get_first_index(table); + + dfield_t fields[1]; + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: key (CHAR(50)) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, key, strlen(key)); + + err= m_executor.select_for_update(table, &tuple, &callback); + return (err == DB_SUCCESS_LOCKED_REC) ? DB_SUCCESS : err; +} + +dtuple_t* FTSQueryExecutor::create_word_search_tuple( + dict_table_t *table, const fts_string_t *word, + mem_heap_t *heap) noexcept +{ + if (!word || word->f_len == 0) + return nullptr; + + dtuple_t* tuple= dtuple_create(heap, 1); + dict_table_copy_types(tuple, table); + dfield_t* field= dtuple_get_nth_field(tuple, 0); + dfield_set_data(field, word->f_str, word->f_len); + dtuple_set_n_fields_cmp(tuple, 1); + + return tuple; +} + +dberr_t FTSQueryExecutor::read_all_common(const char *tbl_name, + RecordCallback &callback) noexcept +{ + ut_ad(!dict_sys.locked()); + FTSDeletionTable table_type= get_deletion_table_type(tbl_name); + if (table_type == FTSDeletionTable::MAX_DELETION_TABLES) return DB_ERROR; + + dberr_t err= open_deletion_table(table_type); + if (err != DB_SUCCESS) return err; + + uint8_t index_no= to_index(table_type); + err= lock_common_tables(index_no, LOCK_IS); + if (err != DB_SUCCESS) return err; + + dict_table_t *table= m_common_tables[index_no]; + dict_index_t *index= dict_table_get_first_index(table); + err= m_executor.read_by_index(table, index, nullptr, PAGE_CUR_G, + callback, true); + return (err == DB_SUCCESS_LOCKED_REC) ? DB_SUCCESS : err; +} + +CommonTableReader::CommonTableReader() : RecordCallback( + [this](const rec_t *rec, const dict_index_t *index, + const rec_offs *offsets) -> dberr_t + { + if (!dict_table_is_comp(index->table)) + { + ulint doc_id_len; + ulint offset= rec_get_nth_field_offs_old(rec, 0, &doc_id_len); + if (offset != 0 || doc_id_len == UNIV_SQL_NULL || doc_id_len != 8) + return DB_CORRUPTION; + } + + doc_ids.push_back(mach_read_from_8(rec)); + return DB_SUCCESS; + }, + [](const dtuple_t *search_tuple, const rec_t *rec, + const dict_index_t *index, const rec_offs *offsets) -> RecordCompareAction + { return RecordCompareAction::PROCESS; }) {} + + +ConfigReader::ConfigReader() : RecordCallback( + [this](const rec_t *rec, const dict_index_t *index, + const rec_offs *offsets) -> dberr_t + { + ulint value_len; + const byte *value_data= rec_get_nth_field(rec, offsets, 3, &value_len); + + if (value_data && value_len != UNIV_SQL_NULL && value_len > 0) + value_span= std::string_view(reinterpret_cast(value_data), + value_len); + + return DB_SUCCESS; + }, + [](const dtuple_t *search_tuple, const rec_t *rec, + const dict_index_t *index, const rec_offs *offsets) -> RecordCompareAction + { + return compare_config_key(search_tuple, rec, index, offsets); + }) {} + +/** Initial size of nodes in fts_word_t. */ +static const ulint FTS_WORD_NODES_INIT_SIZE= 64; + +/** Initialize fts_word_t structure */ +static void init_fts_word(fts_word_t *word, const byte *utf8, ulint len) +{ + mem_heap_t* heap= mem_heap_create(sizeof(fts_node_t)); + memset(word, 0, sizeof(*word)); + word->text.f_len= len; + word->text.f_str= static_cast(mem_heap_alloc(heap, len + 1)); + memcpy(word->text.f_str, utf8, len); + word->text.f_str[len]= 0; + word->heap_alloc= ib_heap_allocator_create(heap); + word->nodes= ib_vector_create(word->heap_alloc, sizeof(fts_node_t), + FTS_WORD_NODES_INIT_SIZE); +} + +/** AuxRecordReader default word processor implementation */ +dberr_t AuxRecordReader::default_word_processor( + const rec_t *rec, const dict_index_t *index, + const rec_offs *offsets, void *user_arg) +{ + ib_vector_t *words= static_cast(user_arg); + + /* Extract fields using rec_get_nth_field() */ + ulint word_len; + const byte* word_data= rec_get_nth_field(rec, offsets, 0, &word_len); + if (!word_data || word_len == UNIV_SQL_NULL || word_len > FTS_MAX_WORD_LEN) + return DB_SUCCESS; + + ulint first_doc_id_len; + const byte* first_doc_id_data= rec_get_nth_field(rec, offsets, 1, &first_doc_id_len); + doc_id_t first_doc_id= fts_read_doc_id(first_doc_id_data); + + ulint last_doc_id_len; + const byte* last_doc_id_data= rec_get_nth_field(rec, offsets, 4, &last_doc_id_len); + doc_id_t last_doc_id= fts_read_doc_id(last_doc_id_data); + + ulint doc_count_len; + const byte* doc_count_data= rec_get_nth_field(rec, offsets, 5, &doc_count_len); + ulint doc_count= mach_read_from_4(doc_count_data); + + ulint ilist_len; + const byte* ilist_data= rec_get_nth_field(rec, offsets, 6, &ilist_len); + + if (word_data && word_len > 0 && word_len != UNIV_SQL_NULL) + { + if (!m_word_heap) + m_word_heap= mem_heap_create(256); + + if (m_last_word.f_str) + mem_heap_empty(m_word_heap); + + m_last_word.f_str= static_cast( + mem_heap_dup(m_word_heap, word_data, word_len)); + m_last_word.f_len= word_len; + m_last_word.f_n_char= 0; + } + + fts_word_t *word; + bool is_word_init= false; + + ut_ad(word_len <= FTS_MAX_WORD_LEN); + + if (ib_vector_size(words) == 0) + { + /* First word - push and initialize */ + word= static_cast(ib_vector_push(words, nullptr)); + init_fts_word(word, word_data, word_len); + is_word_init= true; + } + else + { + /* Check if this word is different from the last word */ + word= static_cast(ib_vector_last(words)); + if (word_len != word->text.f_len || + memcmp(word->text.f_str, word_data, word_len)) + { + /* Different word - push new word and initialize */ + word= static_cast(ib_vector_push(words, nullptr)); + init_fts_word(word, word_data, word_len); + is_word_init= true; + } + } + fts_node_t *node= static_cast( + ib_vector_push(word->nodes, nullptr)); + + /* Use extracted field values */ + node->first_doc_id= first_doc_id; + node->last_doc_id= last_doc_id; + node->doc_count= doc_count; + + node->ilist_size_alloc= node->ilist_size= 0; + node->ilist= nullptr; + + if (ilist_data && ilist_len != UNIV_SQL_NULL && ilist_len > 0) + { + node->ilist_size_alloc= node->ilist_size= ilist_len; + if (ilist_len) + { + node->ilist= static_cast(ut_malloc_nokey(ilist_len)); + memcpy(node->ilist, ilist_data, ilist_len); + } + if (ilist_len == 0) return DB_SUCCESS_LOCKED_REC; + } + + if (this->total_memory) + { + if (is_word_init) + { + *this->total_memory+= + sizeof(fts_word_t) + sizeof(ib_alloc_t) + + sizeof(ib_vector_t) + word_len + + sizeof(fts_node_t) * FTS_WORD_NODES_INIT_SIZE; + } + *this->total_memory+= node->ilist_size; + if (*this->total_memory >= fts_result_cache_limit) + return DB_FTS_EXCEED_RESULT_CACHE_LIMIT; + } + return DB_SUCCESS; +} + +/** AuxRecordReader comparison logic implementation */ +RecordCompareAction AuxRecordReader::compare_record( + const dtuple_t *search_tuple, const rec_t *rec, + const dict_index_t *index, const rec_offs *offsets) noexcept +{ + if (!search_tuple) return RecordCompareAction::PROCESS; + int cmp_result; + switch (compare_mode) + { + case AuxCompareMode::GREATER_EQUAL: + case AuxCompareMode::GREATER: + { + int match= 0; + cmp_result= cmp_dtuple_rec_bytes(rec, *index, *search_tuple, &match, + index->table->not_redundant()); + if (compare_mode == AuxCompareMode::GREATER_EQUAL) + return (cmp_result <= 0) ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; + else + return (cmp_result < 0) ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; + } + case AuxCompareMode::LIKE: + case AuxCompareMode::EQUAL: + { + const dfield_t* search_field= dtuple_get_nth_field(search_tuple, 0); + const void* search_data= dfield_get_data(search_field); + ulint search_len= dfield_get_len(search_field); + + if (!search_data || search_len == UNIV_SQL_NULL) + return RecordCompareAction::PROCESS; + + ulint word_len; + const byte* word_data= rec_get_nth_field(rec, offsets, 0, &word_len); + if (!word_data || word_len == UNIV_SQL_NULL) + return RecordCompareAction::SKIP; + + const dtype_t* type= dfield_get_type(search_field); + cmp_result= cmp_data(type->mtype, type->prtype, false, + static_cast(search_data), + search_len, word_data, word_len); + if (compare_mode == AuxCompareMode::EQUAL) + return cmp_result == 0 + ? RecordCompareAction::PROCESS + : RecordCompareAction::STOP; + else /* AuxCompareMode::LIKE */ + { + int prefix_cmp= cmp_data(type->mtype, type->prtype, false, + static_cast(search_data), + search_len, word_data, + search_len <= word_len ? search_len : word_len); + + if (prefix_cmp != 0) return RecordCompareAction::STOP; + return (search_len <= word_len) ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; + } + } + } + return RecordCompareAction::PROCESS; +} + +/** Direct config key comparison implementation */ +RecordCompareAction ConfigReader::compare_config_key( + const dtuple_t *search_tuple, const rec_t *rec, + const dict_index_t *index, const rec_offs *offsets) +{ + if (!search_tuple) return RecordCompareAction::PROCESS; + const dfield_t *search_field= dtuple_get_nth_field(search_tuple, 0); + const void *search_data= dfield_get_data(search_field); + ulint search_len= dfield_get_len(search_field); + if (!search_data || search_len == UNIV_SQL_NULL) + return RecordCompareAction::PROCESS; + + ulint rec_key_len; + const byte *rec_key_data= rec_get_nth_field(rec, offsets, 0, &rec_key_len); + + if (!rec_key_data || rec_key_len == UNIV_SQL_NULL) + return RecordCompareAction::SKIP; + + const dtype_t *type= dfield_get_type(search_field); + int cmp_result= cmp_data(type->mtype, type->prtype, false, + static_cast(search_data), + search_len, rec_key_data, rec_key_len); + + return (cmp_result == 0) ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; +} + +void FTSQueryExecutor::construct_table_name( + char *table_name, const char *suffix, bool common_table) noexcept +{ + ut_ad(m_table); + ut_ad(common_table || m_index); + const size_t dbname_len= m_table->name.dblen() + 1; + ut_ad(dbname_len > 1); + memcpy(table_name, m_table->name.m_name, dbname_len); + memcpy(table_name+= dbname_len, "FTS_", 4); + table_name+= 4; + + int len= fts_write_object_id(m_table->id, table_name); + if (!common_table) + { + table_name[len]= '_'; + ++len; + len+= fts_write_object_id(m_index->id, table_name + len); + } + ut_a(len >= 16); + ut_a(len < FTS_AUX_MIN_TABLE_ID_LENGTH); + table_name+= len; + *table_name++= '_'; + strcpy(table_name, suffix); +} + +dberr_t FTSQueryExecutor::read_aux(uint8_t aux_index, + const fts_string_t *start_word, + RecordCallback &callback) noexcept +{ + ut_ad(!dict_sys.locked()); + + if (aux_index >= FTS_NUM_AUX_INDEX) + return DB_ERROR; + + dberr_t error= open_aux_table(aux_index); + if (error != DB_SUCCESS) + return error; + + error= lock_aux_tables(aux_index, LOCK_IS); + if (error != DB_SUCCESS) + return error; + + dict_table_t *table= m_aux_tables[aux_index]; + dict_index_t *index= dict_table_get_first_index(table); + + mem_heap_t* heap= nullptr; + dtuple_t* search_tuple= nullptr; + page_cur_mode_t mode= PAGE_CUR_G; + + if (start_word && start_word->f_len > 0) + { + if (heap == nullptr) + heap= mem_heap_create(256); + search_tuple= create_word_search_tuple(table, start_word, heap); + mode= PAGE_CUR_GE; + } + error= m_executor.read_by_index(table, index, search_tuple, + mode, callback, true); + if (heap) + mem_heap_free(heap); + return error; +} diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index afc380e367cf9..954c0d161fe93 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -30,6 +30,7 @@ Full Text Search interface #include "dict0stats_bg.h" #include "row0sel.h" #include "fts0fts.h" +#include "fts0exec.h" #include "fts0priv.h" #include "fts0types.h" #include "fts0types.inl" @@ -44,6 +45,37 @@ static const ulint FTS_MAX_ID_LEN = 32; /** Column name from the FTS config table */ #define FTS_MAX_CACHE_SIZE_IN_MB "cache_size_in_mb" +/** Compare function to check if record's doc_id > search tuple's doc_id +@param[in] search_tuple Search tuple containing target doc_id +@param[in] rec Record to check +@param[in] index Index containing the record +@return true if record's doc_id > search tuple's doc_id */ +static +RecordCompareAction doc_id_comparator( + const dtuple_t* search_tuple, + const rec_t* rec, + const dict_index_t* index, + const rec_offs *offsets) +{ + /* Get target doc_id from search tuple */ + const dfield_t* search_field= dtuple_get_nth_field(search_tuple, 0); + const byte* search_data= static_cast(dfield_get_data(search_field)); + doc_id_t target_doc_id= fts_read_doc_id(search_data); + ulint doc_col_pos= dict_col_get_clust_pos( + &index->table->cols[index->table->fts->doc_col], index); + + ulint len; + const byte* doc_id_data= rec_get_nth_field(rec, offsets, doc_col_pos, &len); + + if (len != sizeof(doc_id_t)) + return RecordCompareAction::SKIP; + + doc_id_t rec_doc_id= fts_read_doc_id(doc_id_data); + return (rec_doc_id > target_doc_id) + ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; +} + /** Verify if a aux table name is a obsolete table by looking up the key word in the obsolete table names */ #define FTS_IS_OBSOLETE_AUX_TABLE(table_name) \ @@ -148,27 +180,6 @@ const fts_index_selector_t fts_index_selector[] = { { 0 , NULL } }; -/** Default config values for FTS indexes on a table. */ -static const char* fts_config_table_insert_values_sql = - "PROCEDURE P() IS\n" - "BEGIN\n" - "\n" - "INSERT INTO $config_table VALUES('" - FTS_MAX_CACHE_SIZE_IN_MB "', '256');\n" - "" - "INSERT INTO $config_table VALUES('" - FTS_OPTIMIZE_LIMIT_IN_SECS "', '180');\n" - "" - "INSERT INTO $config_table VALUES ('" - FTS_SYNCED_DOC_ID "', '0');\n" - "" - "INSERT INTO $config_table VALUES ('" - FTS_TOTAL_DELETED_COUNT "', '0');\n" - "" /* Note: 0 == FTS_TABLE_STATE_RUNNING */ - "INSERT INTO $config_table VALUES ('" - FTS_TABLE_STATE "', '0');\n" - "END;\n"; - /** FTS tokenize parameter for plugin parser */ struct fts_tokenize_param_t { fts_doc_t* result_doc; /*!< Result doc for tokens */ @@ -208,30 +219,6 @@ fts_add_doc_by_id( fts_trx_table_t*ftt, /*!< in: FTS trx table */ doc_id_t doc_id); /*!< in: doc id */ -/** Tokenize a document. -@param[in,out] doc document to tokenize -@param[out] result tokenization result -@param[in] parser pluggable parser */ -static -void -fts_tokenize_document( - fts_doc_t* doc, - fts_doc_t* result, - st_mysql_ftparser* parser); - -/** Continue to tokenize a document. -@param[in,out] doc document to tokenize -@param[in] add_pos add this position to all tokens from this tokenization -@param[out] result tokenization result -@param[in] parser pluggable parser */ -static -void -fts_tokenize_document_next( - fts_doc_t* doc, - ulint add_pos, - fts_doc_t* result, - st_mysql_ftparser* parser); - /** Create the vector of fts_get_doc_t instances. @param[in,out] cache fts cache @return vector of fts_get_doc_t instances */ @@ -266,7 +253,6 @@ fts_cache_destroy(fts_cache_t* cache) /** Get a character set based on precise type. @param prtype precise type @return the corresponding character set */ -UNIV_INLINE CHARSET_INFO* fts_get_charset(ulint prtype) { @@ -298,7 +284,6 @@ fts_get_charset(ulint prtype) /****************************************************************//** This function loads the default InnoDB stopword list */ -static void fts_load_default_stopword( /*======================*/ @@ -343,187 +328,162 @@ fts_load_default_stopword( stopword_info->status = STOPWORD_FROM_DEFAULT; } -/****************************************************************//** -Callback function to read a single stopword value. -@return Always return TRUE */ -static -ibool -fts_read_stopword( -/*==============*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ib_vector_t */ -{ - ib_alloc_t* allocator; - fts_stopword_t* stopword_info; - sel_node_t* sel_node; - que_node_t* exp; - ib_rbt_t* stop_words; - dfield_t* dfield; - fts_string_t str; - mem_heap_t* heap; - ib_rbt_bound_t parent; - dict_table_t* table; - - sel_node = static_cast(row); - table = sel_node->table_list->table; - stopword_info = static_cast(user_arg); - - stop_words = stopword_info->cached_stopword; - allocator = static_cast(stopword_info->heap); - heap = static_cast(allocator->arg); - - exp = sel_node->select_list; - - /* We only need to read the first column */ - dfield = que_node_get_val(exp); - - str.f_n_char = 0; - str.f_str = static_cast(dfield_get_data(dfield)); - str.f_len = dfield_get_len(dfield); - exp = que_node_get_next(exp); - ut_ad(exp); - - if (table->versioned()) { - dfield = que_node_get_val(exp); - ut_ad(dfield_get_type(dfield)->vers_sys_end()); - void* data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - if (table->versioned_by_id()) { - ut_ad(len == sizeof trx_id_max_bytes); - if (0 != memcmp(data, trx_id_max_bytes, len)) { - return true; - } - } else { - ut_ad(len == sizeof timestamp_max_bytes); - if (!IS_MAX_TIMESTAMP(data)) { - return true; - } - } - } - ut_ad(!que_node_get_next(exp)); - - /* Only create new node if it is a value not already existed */ - if (str.f_len != UNIV_SQL_NULL - && rbt_search(stop_words, &parent, &str) != 0) { - - fts_tokenizer_word_t new_word; - - new_word.nodes = ib_vector_create( - allocator, sizeof(fts_node_t), 4); - - new_word.text.f_str = static_cast( - mem_heap_alloc(heap, str.f_len + 1)); - - memcpy(new_word.text.f_str, str.f_str, str.f_len); - - new_word.text.f_n_char = 0; - new_word.text.f_len = str.f_len; - new_word.text.f_str[str.f_len] = 0; - - rbt_insert(stop_words, &new_word, &new_word); - } - - return(TRUE); -} - -/******************************************************************//** -Load user defined stopword from designated user table +/** Load user defined stopword from designated user table +@param fts fulltext structure +@param stopword_table stopword table +@param stopword_info stopword information @return whether the operation is successful */ -static -bool -fts_load_user_stopword( -/*===================*/ - fts_t* fts, /*!< in: FTS struct */ - const char* stopword_table_name, /*!< in: Stopword table - name */ - fts_stopword_t* stopword_info) /*!< in: Stopword info */ +bool fts_load_user_stopword(FTSQueryExecutor *executor, fts_t *fts, + const char *stopword_table, + fts_stopword_t *stopword_info) noexcept { - if (!fts->dict_locked) { - dict_sys.lock(SRW_LOCK_CALL); - } - - /* Validate the user table existence in the right format */ - bool ret= false; - const char* row_end; - stopword_info->charset = fts_valid_stopword_table(stopword_table_name, - &row_end); - if (!stopword_info->charset) { + trx_t* trx= executor->trx(); + if (!fts->dict_locked) dict_sys.lock(SRW_LOCK_CALL); + /* Validate the user table existence in the right format */ + bool ret= false; + const char *row_end; + stopword_info->charset= fts_valid_stopword_table( + stopword_table, &row_end); + if (!stopword_info->charset) + { cleanup: - if (!fts->dict_locked) { - dict_sys.unlock(); - } + if (!fts->dict_locked) dict_sys.unlock(); + return ret; + } - return ret; - } + if (!stopword_info->cached_stopword) + { + /* Create the stopword RB tree with the stopword column + charset. All comparison will use this charset */ + stopword_info->cached_stopword= rbt_create_arg_cmp( + sizeof(fts_tokenizer_word_t), innobase_fts_text_cmp, + (void*)stopword_info->charset); + } - trx_t* trx = trx_create(); - trx->op_info = "Load user stopword table into FTS cache"; + /* Load the stopword table */ + dict_table_t* table= + dict_sys.load_table({stopword_table, strlen(stopword_table)}); + if (!table) + goto cleanup; + + /* Use the passed executor's transaction */ + trx->op_info= "Load user stopword table into FTS cache"; + ib_rbt_t* stop_words= stopword_info->cached_stopword; + ib_alloc_t* allocator= static_cast(stopword_info->heap); + mem_heap_t* heap= static_cast(allocator->arg); + + /* Find the field number for 'value' column */ + dict_index_t* clust_index= dict_table_get_first_index(table); + ulint value_field_no= ULINT_UNDEFINED; + for (ulint i= 0; i < dict_index_get_n_fields(clust_index); i++) + { + const dict_field_t* field= dict_index_get_nth_field(clust_index, i); + if (strcmp(field->name, "value") == 0) + { + value_field_no= i; + break; + } + } + if (value_field_no == ULINT_UNDEFINED) + { + sql_print_error("InnoDB: Could not find 'value' column in " + "stopword table %s", stopword_table); + goto cleanup; + } - if (!stopword_info->cached_stopword) { - /* Create the stopword RB tree with the stopword column - charset. All comparison will use this charset */ - stopword_info->cached_stopword = rbt_create_arg_cmp( - sizeof(fts_tokenizer_word_t), innobase_fts_text_cmp, - (void*)stopword_info->charset); + auto process_stopword= [&](const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets) -> dberr_t + { + mem_heap_t *offsets_heap= nullptr; + if (offsets == nullptr) + offsets= rec_get_offsets(rec, index, nullptr, index->n_core_fields, + ULINT_UNDEFINED, &offsets_heap); - } + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, value_field_no, + &field_len); - pars_info_t* info = pars_info_create(); - - pars_info_bind_id(info, "table_stopword", stopword_table_name); - pars_info_bind_id(info, "row_end", row_end); - - pars_info_bind_function(info, "my_func", fts_read_stopword, - stopword_info); - - que_t* graph = pars_sql( - info, - "PROCEDURE P() IS\n" - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT value, $row_end" - " FROM $table_stopword;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;" - "END;\n"); - - for (;;) { - dberr_t error = fts_eval_sql(trx, graph); - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - stopword_info->status = STOPWORD_USER_TABLE; - break; - } else { - fts_sql_rollback(trx); + if (field_len == UNIV_SQL_NULL) return DB_SUCCESS; - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "Lock wait timeout reading user" - " stopword table. Retrying!"; + ib_rbt_bound_t parent; + fts_string_t str; + str.f_n_char= 0; + str.f_str= const_cast(field_data); + str.f_len= field_len; - trx->error_state = DB_SUCCESS; - } else { - ib::error() << "Error '" << error - << "' while reading user stopword" - " table."; - ret = FALSE; - break; - } - } - } + /* Handle system versioning - check row_end column if versioned */ + if (table->versioned()) + { + ulint end_len; + const byte* end_data= rec_get_nth_field( + rec, offsets, table->vers_end + index->n_uniq + 2, &end_len); - que_graph_free(graph); - trx->free(); - ret = true; - goto cleanup; + if (table->versioned_by_id()) + { + ut_ad(end_len == sizeof trx_id_max_bytes); + if (0 != memcmp(end_data, trx_id_max_bytes, end_len)) + goto func_exit; + } + else + { + ut_ad(end_len == sizeof timestamp_max_bytes); + if (!IS_MAX_TIMESTAMP(end_data)) + goto func_exit; + } + } + + if (str.f_len != UNIV_SQL_NULL && + rbt_search(stop_words, &parent, &str) != 0) + { + fts_tokenizer_word_t new_word; + new_word.nodes= ib_vector_create(allocator, sizeof(fts_node_t), 4); + new_word.text.f_str= static_cast( + mem_heap_alloc(heap, str.f_len + 1)); + memcpy(new_word.text.f_str, str.f_str, str.f_len); + new_word.text.f_n_char= 0; + new_word.text.f_len= str.f_len; + new_word.text.f_str[str.f_len]= 0; + rbt_insert(stop_words, &new_word, &new_word); + } +func_exit: + if (offsets_heap) mem_heap_free(offsets_heap); + return DB_SUCCESS; /* Continue processing */ + }; + + RecordCallback callback(process_stopword, + [](const dtuple_t*, const rec_t*, + const dict_index_t*, const rec_offs*) { + return RecordCompareAction::PROCESS; + }); + /* Read all records from the stopword table */ + for (;;) + { + dberr_t error= executor->read_by_index( + table, clust_index, nullptr, PAGE_CUR_G, callback); + if (UNIV_LIKELY(error == DB_SUCCESS)) + { + stopword_info->status= STOPWORD_USER_TABLE; + ret= true; + break; + } + else + { + if (error == DB_LOCK_WAIT_TIMEOUT) + { + sql_print_warning("InnoDB: Lock wait timeout reading " + "user stopword table. Retrying!"); + trx->error_state= DB_SUCCESS; + } + else + { + sql_print_error("InnoDB: Error '%s' while reading user " + "stopword table.", ut_strerr(error)); + ret= false; + break; + } + } + } + goto cleanup; } /******************************************************************//** @@ -587,6 +547,46 @@ fts_cache_init( } } +/** Construct the name of an internal FTS table for the given table. +@param[in] fts_table metadata on fulltext-indexed table +@param[out] table_name a name up to MAX_FULL_NAME_LEN +@param[in] dict_locked whether dict_sys.latch is being held */ +void fts_get_table_name(const fts_table_t* fts_table, char* table_name, + bool dict_locked) +{ + if (!dict_locked) dict_sys.freeze(SRW_LOCK_CALL); + ut_ad(dict_sys.frozen()); + /* Include the separator as well. */ + const size_t dbname_len= fts_table->table->name.dblen() + 1; + ut_ad(dbname_len > 1); + memcpy(table_name, fts_table->table->name.m_name, dbname_len); + if (!dict_locked) dict_sys.unfreeze(); + + memcpy(table_name += dbname_len, "FTS_", 4); + table_name += 4; + int len; + switch (fts_table->type) + { + case FTS_COMMON_TABLE: + len= fts_write_object_id(fts_table->table_id, table_name); + break; + + case FTS_INDEX_TABLE: + len= fts_write_object_id(fts_table->table_id, table_name); + table_name[len]= '_'; + ++len; + len+= fts_write_object_id(fts_table->index_id, table_name + len); + break; + + default: ut_error; + } + ut_a(len >= 16); + ut_a(len < FTS_AUX_MIN_TABLE_ID_LENGTH); + table_name+= len; + *table_name++= '_'; + strcpy(table_name, fts_table->suffix); +} + /****************************************************************//** Create a FTS cache. */ fts_cache_t* @@ -1815,88 +1815,78 @@ CREATE TABLE $FTS_PREFIX_CONFIG @param[in] skip_doc_id_index Skip index on doc id @return DB_SUCCESS if succeed */ dberr_t -fts_create_common_tables( - trx_t* trx, - dict_table_t* table, - bool skip_doc_id_index) +fts_create_common_tables(trx_t *trx, dict_table_t *table, + bool skip_doc_id_index) { - dberr_t error; - que_t* graph; - fts_table_t fts_table; - mem_heap_t* heap = mem_heap_create(1024); - pars_info_t* info; - char fts_name[MAX_FULL_NAME_LEN]; - char full_name[sizeof(fts_common_tables) / sizeof(char*)] - [MAX_FULL_NAME_LEN]; - - dict_index_t* index = NULL; - - FTS_INIT_FTS_TABLE(&fts_table, NULL, FTS_COMMON_TABLE, table); - - error = fts_drop_common_tables(trx, &fts_table, true); - - if (error != DB_SUCCESS) { - - goto func_exit; - } - - /* Create the FTS tables that are common to an FTS index. */ - for (ulint i = 0; fts_common_tables[i] != NULL; ++i) { - - fts_table.suffix = fts_common_tables[i]; - fts_get_table_name(&fts_table, full_name[i], true); - dict_table_t* common_table = fts_create_one_common_table( - trx, table, full_name[i], fts_table.suffix, heap); - - if (!common_table) { - trx->error_state = DB_SUCCESS; - error = DB_ERROR; - goto func_exit; - } - - mem_heap_empty(heap); - } + dberr_t error= DB_SUCCESS; + char full_name[sizeof(fts_common_tables) / sizeof(char*)][MAX_FULL_NAME_LEN]; + dict_index_t *index= nullptr; + fts_table_t fts_table; + FTS_INIT_FTS_TABLE(&fts_table, NULL, FTS_COMMON_TABLE, table); + error = fts_drop_common_tables(trx, &fts_table, true); + if (error != DB_SUCCESS) return error; + mem_heap_t *heap= mem_heap_create(1024); - /* Write the default settings to the config table. */ - info = pars_info_create(); + FTSQueryExecutor executor(trx, table); + /* Create the FTS tables that are common to an FTS index. */ + for (ulint i= 0; fts_common_tables[i]; ++i) + { + fts_table.suffix= fts_common_tables[i]; + fts_get_table_name(&fts_table, full_name[i], true); + dict_table_t *common_table= fts_create_one_common_table( + trx, table, full_name[i], fts_table.suffix, heap); + if (!common_table) + { + trx->error_state= DB_SUCCESS; + error= DB_ERROR; + mem_heap_free(heap); + return error; + } - fts_table.suffix = "CONFIG"; - fts_get_table_name(&fts_table, fts_name, true); - pars_info_bind_id(info, "config_table", fts_name); + if (i == 2) + executor.set_config_table(common_table); - graph = pars_sql( - info, fts_config_table_insert_values_sql); + mem_heap_empty(heap); + } - error = fts_eval_sql(trx, graph); + /** Does the following insert operation: + INSERT INTO $config_table VALUES('"FTS_MAX_CACHE_SIZE_IN_MB"', '256');" + INSERT INTO $config_table VALUES('"FTS_OPTIMIZE_LIMIT_IN_SECS"', '180');" + INSERT INTO $config_table VALUES ('"FTS_SYNCED_DOC_ID "', '0');" + INSERT INTO $config_table VALUES ('"FTS_TOTAL_DELETED_COUNT "', '0');" + INSERT INTO $config_table VALUES ('"FTS_TABLE_STATE "', '0');" */ + error= executor.insert_config_record(FTS_MAX_CACHE_SIZE_IN_MB, "256"); + if (error == DB_SUCCESS) + error= executor.insert_config_record(FTS_OPTIMIZE_LIMIT_IN_SECS, "180"); - que_graph_free(graph); + if (error == DB_SUCCESS) + error= executor.insert_config_record(FTS_SYNCED_DOC_ID, "0"); - if (error != DB_SUCCESS || skip_doc_id_index) { + if (error == DB_SUCCESS) + error= executor.insert_config_record(FTS_TOTAL_DELETED_COUNT, "0"); - goto func_exit; - } + if (error == DB_SUCCESS) + error= executor.insert_config_record(FTS_TABLE_STATE, "0"); - if (table->versioned()) { - index = dict_mem_index_create(table, - FTS_DOC_ID_INDEX.str, - DICT_UNIQUE, 2); - dict_mem_index_add_field(index, FTS_DOC_ID.str, 0); - dict_mem_index_add_field(index, table->cols[table->vers_end].name(*table).str, 0); - } else { - index = dict_mem_index_create(table, - FTS_DOC_ID_INDEX.str, - DICT_UNIQUE, 1); - dict_mem_index_add_field(index, FTS_DOC_ID.str, 0); - } + if (error != DB_SUCCESS || skip_doc_id_index) goto func_exit; - error = row_create_index_for_mysql(index, trx, NULL, - FIL_ENCRYPTION_DEFAULT, - FIL_DEFAULT_ENCRYPTION_KEY); + if (table->versioned()) + { + index= dict_mem_index_create(table, FTS_DOC_ID_INDEX.str, DICT_UNIQUE, 2); + dict_mem_index_add_field(index, FTS_DOC_ID.str, 0); + dict_mem_index_add_field(index, table->cols[table->vers_end].name(*table).str, 0); + } + else + { + index= dict_mem_index_create(table, FTS_DOC_ID_INDEX.str, DICT_UNIQUE, 1); + dict_mem_index_add_field(index, FTS_DOC_ID.str, 0); + } + error= row_create_index_for_mysql(index, trx, NULL, FIL_ENCRYPTION_DEFAULT, + FIL_DEFAULT_ENCRYPTION_KEY); func_exit: - mem_heap_free(heap); - - return(error); + mem_heap_free(heap); + return error; } /** Create one FTS auxiliary index table for an FTS index. @@ -2361,39 +2351,6 @@ fts_trx_add_op( fts_trx_table_add_op(stmt_ftt, doc_id, state, fts_indexes); } -/******************************************************************//** -Fetch callback that converts a textual document id to a binary value and -stores it in the given place. -@return always returns NULL */ -static -ibool -fts_fetch_store_doc_id( -/*===================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: doc_id_t* to store - doc_id in */ -{ - int n_parsed; - sel_node_t* node = static_cast(row); - doc_id_t* doc_id = static_cast(user_arg); - dfield_t* dfield = que_node_get_val(node->select_list); - dtype_t* type = dfield_get_type(dfield); - ulint len = dfield_get_len(dfield); - - char buf[32]; - - ut_a(dtype_get_mtype(type) == DATA_VARCHAR); - ut_a(len > 0 && len < sizeof(buf)); - - memcpy(buf, dfield_get_data(dfield), len); - buf[len] = '\0'; - - n_parsed = sscanf(buf, FTS_DOC_ID_FORMAT, doc_id); - ut_a(n_parsed == 1); - - return(FALSE); -} - /*********************************************************************//** Get the next available document id. @return DB_SUCCESS if OK */ @@ -2426,70 +2383,48 @@ fts_get_next_doc_id( } /** Read the synced document id from the fts configuration table -@param table fts table -@param doc_id document id to be read -@param trx transaction to read from config table +@param executor query executor +@param table fts table +@param doc_id document id to be read @return DB_SUCCESS in case of success */ static -dberr_t fts_read_synced_doc_id(const dict_table_t *table, - doc_id_t *doc_id, - trx_t *trx) +dberr_t fts_read_synced_doc_id(FTSQueryExecutor *executor, + const dict_table_t *table, + doc_id_t *doc_id) noexcept { - dberr_t error; - char table_name[MAX_FULL_NAME_LEN]; - - fts_table_t fts_table; - fts_table.suffix= "CONFIG"; - fts_table.table_id= table->id; - fts_table.type= FTS_COMMON_TABLE; - fts_table.table= table; ut_a(table->fts->doc_col != ULINT_UNDEFINED); - - trx->op_info = "update the next FTS document id"; - pars_info_t *info= pars_info_create(); - pars_info_bind_function(info, "my_func", fts_fetch_store_doc_id, - doc_id); - - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "config_table", table_name); - - que_t *graph= fts_parse_sql( - &fts_table, info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS SELECT value FROM $config_table" - " WHERE key = 'synced_doc_id' FOR UPDATE;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - + executor->trx()->op_info= "reading synced FTS document id"; + ConfigReader reader; *doc_id= 0; - error = fts_eval_sql(trx, graph); - que_graph_free(graph); + dberr_t error= executor->read_config_with_lock("synced_doc_id", reader); + if (error == DB_SUCCESS) + { + char value_buf[32]; + size_t copy_len= std::min(reader.value_span.size(), sizeof(value_buf) - 1); + memcpy(value_buf, reader.value_span.data(), copy_len); + value_buf[copy_len]= '\0'; + int n_parsed= sscanf(value_buf, FTS_DOC_ID_FORMAT, doc_id); + if (n_parsed != 1) error= DB_ERROR; + executor->release_lock(); + } return error; } /** This function fetch the Doc ID from CONFIG table, and compare with the Doc ID supplied. And store the larger one to the CONFIG table. -@param table fts table -@param cmp_doc_id Doc ID to compare -@param doc_id larger document id after comparing "cmp_doc_id" to - the one stored in CONFIG table -@param trx transaction +@param executor query executor +@param table fts table +@param cmp_doc_id Doc ID to compare +@param doc_id larger document id after comparing "cmp_doc_id" to + the one stored in CONFIG table @return DB_SUCCESS if OK */ static dberr_t fts_cmp_set_sync_doc_id( + FTSQueryExecutor *executor, const dict_table_t *table, doc_id_t cmp_doc_id, - doc_id_t *doc_id, - trx_t *trx=nullptr) + doc_id_t *doc_id) noexcept { ut_ad(!srv_read_only_mode || recv_sys.rpo); @@ -2499,14 +2434,8 @@ fts_cmp_set_sync_doc_id( fts_cache_t* cache= table->fts->cache; dberr_t error = DB_SUCCESS; - const trx_t* const caller_trx = trx; - - if (trx == nullptr) { - trx = trx_create(); - trx_start_internal_read_only(trx); - } retry: - error = fts_read_synced_doc_id(table, doc_id, trx); + error = fts_read_synced_doc_id(executor, table, doc_id); if (error != DB_SUCCESS) goto func_exit; @@ -2526,27 +2455,19 @@ fts_cmp_set_sync_doc_id( if (cmp_doc_id && cmp_doc_id >= *doc_id) { error = fts_update_sync_doc_id( - table, cache->synced_doc_id, trx); + executor, table, cache->synced_doc_id); } *doc_id = cache->next_doc_id; func_exit: - if (caller_trx) { - return error; - } - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - } else { + if (error != DB_SUCCESS) { *doc_id = 0; ib::error() << "(" << error << ") while getting next doc id " "for table " << table->name; - fts_sql_rollback(trx); - if (error == DB_DEADLOCK || error == DB_LOCK_WAIT_TIMEOUT) { DEBUG_SYNC_C("fts_cmp_set_sync_doc_id_retry"); std::this_thread::sleep_for(FTS_DEADLOCK_RETRY_WAIT); @@ -2554,88 +2475,24 @@ fts_cmp_set_sync_doc_id( } } - trx->clear_and_free(); - return(error); } /** Update the last document id. This function could create a new transaction to update the last document id. -@param table table to be updated -@param doc_id last document id -@param trx update trx or null +@param executor query executor +@param table table to be updated +@param doc_id last document id @retval DB_SUCCESS if OK */ dberr_t -fts_update_sync_doc_id( - const dict_table_t* table, - doc_id_t doc_id, - trx_t* trx) +fts_update_sync_doc_id(FTSQueryExecutor *executor, + const dict_table_t *table, + doc_id_t doc_id) noexcept { - byte id[FTS_MAX_ID_LEN]; - pars_info_t* info; - fts_table_t fts_table; - ulint id_len; - que_t* graph = NULL; - dberr_t error; - ibool local_trx = FALSE; - fts_cache_t* cache = table->fts->cache; - char fts_name[MAX_FULL_NAME_LEN]; - - ut_ad(!srv_read_only_mode || recv_sys.rpo); - - if (recv_sys.rpo) { - return DB_READ_ONLY; - } - - fts_table.suffix = "CONFIG"; - fts_table.table_id = table->id; - fts_table.type = FTS_COMMON_TABLE; - fts_table.table = table; - - if (!trx) { - trx = trx_create(); - trx_start_internal(trx); - - trx->op_info = "setting last FTS document id"; - local_trx = TRUE; - } - - info = pars_info_create(); - - id_len = (ulint) snprintf( - (char*) id, sizeof(id), FTS_DOC_ID_FORMAT, doc_id + 1); - - pars_info_bind_varchar_literal(info, "doc_id", id, id_len); - - fts_get_table_name(&fts_table, fts_name, - table->fts->dict_locked); - pars_info_bind_id(info, "table_name", fts_name); - - graph = fts_parse_sql( - &fts_table, info, - "BEGIN" - " UPDATE $table_name SET value = :doc_id" - " WHERE key = 'synced_doc_id';"); - - error = fts_eval_sql(trx, graph); - - que_graph_free(graph); - - if (local_trx) { - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - cache->synced_doc_id = doc_id; - } else { - ib::error() << "(" << error << ") while" - " updating last doc id for table" - << table->name; - - fts_sql_rollback(trx); - } - trx->clear_and_free(); - } - - return(error); + if (srv_read_only_mode) return DB_READ_ONLY; + char id[FTS_MAX_ID_LEN]; + snprintf(id, sizeof(id), FTS_DOC_ID_FORMAT, doc_id + 1); + return executor->update_config_record("synced_doc_id", id); } /*********************************************************************//** @@ -2694,13 +2551,9 @@ fts_delete( fts_trx_table_t*ftt, /*!< in: FTS trx table */ fts_trx_row_t* row) /*!< in: row */ { - que_t* graph; - fts_table_t fts_table; - doc_id_t write_doc_id; dict_table_t* table = ftt->table; doc_id_t doc_id = row->doc_id; trx_t* trx = ftt->fts_trx->trx; - pars_info_t* info = pars_info_create(); fts_cache_t* cache = table->fts->cache; /* we do not index Documents whose Doc ID value is 0 */ @@ -2711,12 +2564,6 @@ fts_delete( ut_a(row->state == FTS_DELETE || row->state == FTS_MODIFY); - FTS_INIT_FTS_TABLE(&fts_table, "DELETED", FTS_COMMON_TABLE, table); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, doc_id); - fts_bind_doc_id(info, "doc_id", &write_doc_id); - /* It is possible we update a record that has not yet been sync-ed into cache from last crash (delete Doc will not initialize the sync). Avoid any added counter accounting until the FTS cache @@ -2741,20 +2588,9 @@ fts_delete( } /* Note the deleted document for OPTIMIZE to purge. */ - char table_name[MAX_FULL_NAME_LEN]; - trx->op_info = "adding doc id to FTS DELETED"; - - fts_table.suffix = "DELETED"; - - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "deleted", table_name); - - graph = fts_parse_sql(&fts_table, info, - "BEGIN INSERT INTO $deleted VALUES (:doc_id);"); - - dberr_t error = fts_eval_sql(trx, graph); - que_graph_free(graph); + FTSQueryExecutor executor(trx, table); + dberr_t error= executor.insert_common_record("DELETED", doc_id); /* Increment the total deleted count, this is used to calculate the number of documents indexed. */ @@ -2924,105 +2760,18 @@ fts_doc_free( } /*********************************************************************//** -Callback function for fetch that stores the text of an FTS document, -converting each column to UTF-16. -@return always FALSE */ -ibool -fts_query_expansion_fetch_doc( -/*==========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts_doc_t* */ -{ - que_node_t* exp; - sel_node_t* node = static_cast(row); - fts_doc_t* result_doc = static_cast(user_arg); - dfield_t* dfield; - ulint len; - ulint doc_len; - fts_doc_t doc; - CHARSET_INFO* doc_charset = NULL; - ulint field_no = 0; - - len = 0; - - fts_doc_init(&doc); - doc.found = TRUE; - - exp = node->select_list; - doc_len = 0; - - doc_charset = result_doc->charset; - - /* Copy each indexed column content into doc->text.f_str */ - while (exp) { - dfield = que_node_get_val(exp); - len = dfield_get_len(dfield); - - /* NULL column */ - if (len == UNIV_SQL_NULL) { - exp = que_node_get_next(exp); - continue; - } - - if (!doc_charset) { - doc_charset = fts_get_charset(dfield->type.prtype); - } - - doc.charset = doc_charset; - - if (dfield_is_ext(dfield)) { - /* We ignore columns that are stored externally, this - could result in too many words to search */ - exp = que_node_get_next(exp); - continue; - } else { - doc.text.f_n_char = 0; - - doc.text.f_str = static_cast( - dfield_get_data(dfield)); - - doc.text.f_len = len; - } - - if (field_no == 0) { - fts_tokenize_document(&doc, result_doc, - result_doc->parser); - } else { - fts_tokenize_document_next(&doc, doc_len, result_doc, - result_doc->parser); - } - - exp = que_node_get_next(exp); - - doc_len += (exp) ? len + 1 : len; - - field_no++; - } - - ut_ad(doc_charset); - - if (!result_doc->charset) { - result_doc->charset = doc_charset; - } - - fts_doc_free(&doc); - - return(FALSE); -} - -/*********************************************************************//** -fetch and tokenize the document. */ -static -void -fts_fetch_doc_from_rec( -/*===================*/ - fts_get_doc_t* get_doc, /*!< in: FTS index's get_doc struct */ - dict_index_t* clust_index, /*!< in: cluster index */ - btr_pcur_t* pcur, /*!< in: cursor whose position - has been stored */ - rec_offs* offsets, /*!< in: offsets */ - fts_doc_t* doc) /*!< out: fts doc to hold parsed - documents */ +fetch and tokenize the document. */ +static +void +fts_fetch_doc_from_rec( +/*===================*/ + fts_get_doc_t* get_doc, /*!< in: FTS index's get_doc struct */ + dict_index_t* clust_index, /*!< in: cluster index */ + btr_pcur_t* pcur, /*!< in: cursor whose position + has been stored */ + rec_offs* offsets, /*!< in: offsets */ + fts_doc_t* doc) /*!< out: fts doc to hold parsed + documents */ { dict_index_t* index; const rec_t* clust_rec; @@ -3173,6 +2922,7 @@ fts_add_doc_from_tuple( mtr_start(&mtr); + FTSQueryExecutor executor(ftt->fts_trx->trx, ftt->table); ulint num_idx = ib_vector_size(cache->get_docs); for (ulint i = 0; i < num_idx; ++i) { @@ -3194,8 +2944,7 @@ fts_add_doc_from_tuple( if (table->fts->cache->stopword_info.status & STOPWORD_NOT_INIT) { - fts_load_stopword(table, NULL, NULL, - true, true); + fts_load_stopword(&executor, table); } fts_cache_add_doc( @@ -3297,6 +3046,7 @@ fts_add_doc_by_id( } } + FTSQueryExecutor executor(ftt->fts_trx->trx, table); /* If we have a match, add the data to doc structure */ if (btr_pcur_open_with_no_init(tuple, PAGE_CUR_LE, BTR_SEARCH_LEAF, &pcur, &mtr) @@ -3372,8 +3122,7 @@ fts_add_doc_by_id( if (table->fts->cache->stopword_info.status & STOPWORD_NOT_INIT) { - fts_load_stopword(table, NULL, - NULL, true, true); + fts_load_stopword(&executor, table); } fts_cache_add_doc( @@ -3446,28 +3195,6 @@ fts_add_doc_by_id( mem_heap_free(heap); } - -/*********************************************************************//** -Callback function to read a single ulint column. -return always returns TRUE */ -static -ibool -fts_read_ulint( -/*===========*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ulint */ -{ - sel_node_t* sel_node = static_cast(row); - ulint* value = static_cast(user_arg); - que_node_t* exp = sel_node->select_list; - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - - *value = mach_read_from_4(static_cast(data)); - - return(TRUE); -} - /*********************************************************************//** Get maximum Doc ID in a table if index "FTS_DOC_ID_INDEX" exists @return max Doc ID or 0 if index "FTS_DOC_ID_INDEX" does not exist */ @@ -3539,200 +3266,19 @@ fts_get_max_doc_id( return(doc_id); } -/*********************************************************************//** -Fetch document with the given document id. -@return DB_SUCCESS if OK else error */ -dberr_t -fts_doc_fetch_by_doc_id( -/*====================*/ - fts_get_doc_t* get_doc, /*!< in: state */ - doc_id_t doc_id, /*!< in: id of document to - fetch */ - dict_index_t* index_to_use, /*!< in: caller supplied FTS index, - or NULL */ - ulint option, /*!< in: search option, if it is - greater than doc_id or equal */ - fts_sql_callback - callback, /*!< in: callback to read */ - void* arg) /*!< in: callback arg */ -{ - pars_info_t* info; - dberr_t error; - const char* select_str; - doc_id_t write_doc_id; - dict_index_t* index; - trx_t* trx = trx_create(); - que_t* graph; - - trx->op_info = "fetching indexed FTS document"; - - /* The FTS index can be supplied by caller directly with - "index_to_use", otherwise, get it from "get_doc" */ - index = (index_to_use) ? index_to_use : get_doc->index_cache->index; - - if (get_doc && get_doc->get_document_graph) { - info = get_doc->get_document_graph->info; - } else { - info = pars_info_create(); - } - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, doc_id); - fts_bind_doc_id(info, "doc_id", &write_doc_id); - pars_info_bind_function(info, "my_func", callback, arg); - - select_str = fts_get_select_columns_str(index, info, info->heap); - pars_info_bind_id(info, "table_name", index->table->name.m_name); - - if (!get_doc || !get_doc->get_document_graph) { - if (option == FTS_FETCH_DOC_BY_ID_EQUAL) { - graph = fts_parse_sql( - NULL, - info, - mem_heap_printf(info->heap, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT %s FROM $table_name" - " WHERE %s = :doc_id;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c %% NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;", - select_str, - FTS_DOC_ID.str)); - } else { - ut_ad(option == FTS_FETCH_DOC_BY_ID_LARGE); - - /* This is used for crash recovery of table with - hidden DOC ID or FTS indexes. We will scan the table - to re-processing user table rows whose DOC ID or - FTS indexed documents have not been sync-ed to disc - during recent crash. - In the case that all fulltext indexes are dropped - for a table, we will keep the "hidden" FTS_DOC_ID - column, and this scan is to retreive the largest - DOC ID being used in the table to determine the - appropriate next DOC ID. - In the case of there exists fulltext index(es), this - operation will re-tokenize any docs that have not - been sync-ed to the disk, and re-prime the FTS - cached */ - graph = fts_parse_sql( - NULL, - info, - mem_heap_printf(info->heap, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT %s, %s FROM $table_name" - " WHERE %s > :doc_id;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c %% NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;", - FTS_DOC_ID.str, - select_str, - FTS_DOC_ID.str)); - } - if (get_doc) { - get_doc->get_document_graph = graph; - } - } else { - graph = get_doc->get_document_graph; - } - - error = fts_eval_sql(trx, graph); - fts_sql_commit(trx); - trx->free(); - - if (!get_doc) { - que_graph_free(graph); - } - - return(error); -} - -/*********************************************************************//** -Write out a single word's data as new entry/entries in the INDEX table. -@return DB_SUCCESS if all OK. */ -dberr_t -fts_write_node( -/*===========*/ - trx_t* trx, /*!< in: transaction */ - que_t** graph, /*!< in: query graph */ - fts_table_t* fts_table, /*!< in: aux table */ - fts_string_t* word, /*!< in: word in UTF-8 */ - fts_node_t* node) /*!< in: node columns */ +/** Write out a single word's data as new entry/entries in the INDEX table. +@param executor FTS Query Executor +@param selected auxiliary index number +@param aux_data auxiliary table data +@return DB_SUCCESS if all OK or error code */ +dberr_t fts_write_node(FTSQueryExecutor *executor, uint8_t selected, + const fts_aux_data_t *aux_data) noexcept { - pars_info_t* info; - dberr_t error; - ib_uint32_t doc_count; - time_t start_time; - doc_id_t last_doc_id; - doc_id_t first_doc_id; - char table_name[MAX_FULL_NAME_LEN]; - - ut_a(node->ilist != NULL); - - if (*graph) { - info = (*graph)->info; - } else { - info = pars_info_create(); - - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "index_table_name", table_name); - } - - pars_info_bind_varchar_literal(info, "token", word->f_str, word->f_len); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &first_doc_id, node->first_doc_id); - fts_bind_doc_id(info, "first_doc_id", &first_doc_id); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &last_doc_id, node->last_doc_id); - fts_bind_doc_id(info, "last_doc_id", &last_doc_id); - - ut_a(node->last_doc_id >= node->first_doc_id); - - /* Convert to "storage" byte order. */ - mach_write_to_4((byte*) &doc_count, node->doc_count); - pars_info_bind_int4_literal( - info, "doc_count", (const ib_uint32_t*) &doc_count); - - /* Set copy_name to FALSE since it's a static. */ - pars_info_bind_literal( - info, "ilist", node->ilist, node->ilist_size, - DATA_BLOB, DATA_BINARY_TYPE); - - if (!*graph) { - - *graph = fts_parse_sql( - fts_table, - info, - "BEGIN\n" - "INSERT INTO $index_table_name VALUES" - " (:token, :first_doc_id," - " :last_doc_id, :doc_count, :ilist);"); - } - - start_time = time(NULL); - error = fts_eval_sql(trx, *graph); - elapsed_time += time(NULL) - start_time; - ++n_nodes; - - return(error); + time_t start_time= time(NULL); + dberr_t error= executor->insert_aux_record(selected, aux_data); + elapsed_time+= time(NULL) - start_time; + ++n_nodes; + return error; } /** Sort an array of doc_id */ @@ -3742,157 +3288,85 @@ void fts_doc_ids_sort(ib_vector_t *doc_ids) std::sort(data, data + doc_ids->used); } -/*********************************************************************//** -Add rows to the DELETED_CACHE table. +/** Add rows to the DELETED_CACHE table. @return DB_SUCCESS if all went well else error code*/ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t -fts_sync_add_deleted_cache( -/*=======================*/ - fts_sync_t* sync, /*!< in: sync state */ - ib_vector_t* doc_ids) /*!< in: doc ids to add */ +fts_sync_add_deleted_cache(FTSQueryExecutor *executor, fts_sync_t *sync, + ib_vector_t *doc_ids) noexcept { - ulint i; - pars_info_t* info; - que_t* graph; - fts_table_t fts_table; - char table_name[MAX_FULL_NAME_LEN]; - doc_id_t dummy = 0; - dberr_t error = DB_SUCCESS; - ulint n_elems = ib_vector_size(doc_ids); - - ut_a(ib_vector_size(doc_ids) > 0); - - fts_doc_ids_sort(doc_ids); - - info = pars_info_create(); - - fts_bind_doc_id(info, "doc_id", &dummy); - - FTS_INIT_FTS_TABLE( - &fts_table, "DELETED_CACHE", FTS_COMMON_TABLE, sync->table); - - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - &fts_table, - info, - "BEGIN INSERT INTO $table_name VALUES (:doc_id);"); - - for (i = 0; i < n_elems && error == DB_SUCCESS; ++i) { - doc_id_t* update; - doc_id_t write_doc_id; - - update = static_cast(ib_vector_get(doc_ids, i)); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, *update); - fts_bind_doc_id(info, "doc_id", &write_doc_id); - - error = fts_eval_sql(sync->trx, graph); - } - - que_graph_free(graph); - - return(error); + ulint n_elems= ib_vector_size(doc_ids); + ut_a(n_elems > 0); + fts_doc_ids_sort(doc_ids); + dberr_t error= DB_SUCCESS; + for (uint32_t i= 0; i < n_elems && error == DB_SUCCESS; ++i) + { + doc_id_t *update= + static_cast(ib_vector_get(doc_ids, i)); + error= executor->insert_common_record("DELETED_CACHE", *update); + } + return error; } -/** Write the words and ilist to disk. -@param[in,out] trx transaction -@param[in] index_cache index cache -@param[in] unlock_cache whether unlock cache when write node +/** Write the words and ilist to disk +@param[in,out] executor query executor +@param[in] index_cache index cache +@param[in] unlock_cache whether unlock cache when write node @return DB_SUCCESS if all went well else error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_sync_write_words( - trx_t* trx, - fts_index_cache_t* index_cache, - bool unlock_cache) +dberr_t fts_sync_write_words(FTSQueryExecutor *executor, + fts_index_cache_t *index_cache, + bool unlock_cache) noexcept { - fts_table_t fts_table; - const ib_rbt_node_t* rbt_node; - dberr_t error = DB_SUCCESS; - ibool print_error = FALSE; - dict_table_t* table = index_cache->index->table; - - FTS_INIT_INDEX_TABLE( - &fts_table, NULL, FTS_INDEX_TABLE, index_cache->index); - - /* We iterate over the entire tree, even if there is an error, - since we want to free the memory used during caching. */ - for (rbt_node = rbt_first(index_cache->words); - rbt_node; - rbt_node = rbt_next(index_cache->words, rbt_node)) { - - ulint i; - ulint selected; - fts_tokenizer_word_t* word; - - word = rbt_value(fts_tokenizer_word_t, rbt_node); - - DBUG_EXECUTE_IF( - "fts_instrument_write_words_before_select_index", - std::this_thread::sleep_for( - std::chrono::milliseconds(300));); - - selected = fts_select_index( - index_cache->charset, word->text.f_str, - word->text.f_len); - - fts_table.suffix = fts_get_suffix(selected); - - /* We iterate over all the nodes even if there was an error */ - for (i = 0; i < ib_vector_size(word->nodes); ++i) { - - fts_node_t* fts_node = static_cast( - ib_vector_get(word->nodes, i)); - - if (fts_node->synced) { - continue; - } else { - fts_node->synced = true; - } - - /*FIXME: we need to handle the error properly. */ - if (error == DB_SUCCESS) { - if (unlock_cache) { - mysql_mutex_unlock( - &table->fts->cache->lock); - } - - error = fts_write_node( - trx, - &index_cache->ins_graph[selected], - &fts_table, &word->text, fts_node); - - DEBUG_SYNC_C("fts_write_node"); - DBUG_EXECUTE_IF("fts_write_node_crash", - DBUG_SUICIDE();); - - DBUG_EXECUTE_IF( - "fts_instrument_sync_sleep", - std::this_thread::sleep_for( - std::chrono::seconds(1));); - - if (unlock_cache) { - mysql_mutex_lock( - &table->fts->cache->lock); - } - } - } + dict_table_t *table= index_cache->index->table; + bool print_error= false; + dberr_t error= DB_SUCCESS; + for (const ib_rbt_node_t *rbt_node= rbt_first(index_cache->words); + rbt_node; rbt_node= rbt_next(index_cache->words, rbt_node)) + { + fts_tokenizer_word_t *word= rbt_value(fts_tokenizer_word_t, rbt_node); + DBUG_EXECUTE_IF("fts_instrument_write_words_before_select_index", + std::this_thread::sleep_for( + std::chrono::milliseconds(300));); + uint8_t selected= fts_select_index( + index_cache->charset, word->text.f_str, word->text.f_len); + + for (ulint i = 0; i < ib_vector_size(word->nodes); ++i) + { + fts_node_t* fts_node= + static_cast(ib_vector_get(word->nodes, i)); + if (fts_node->synced) continue; + else fts_node->synced= true; + /* FIXME: we need to handle the error properly. */ + if (error == DB_SUCCESS) + { + if (unlock_cache) mysql_mutex_unlock(&table->fts->cache->lock); + fts_aux_data_t aux_data((const char*)word->text.f_str, word->text.f_len, + fts_node->first_doc_id, fts_node->last_doc_id, + static_cast(fts_node->doc_count), fts_node->ilist, + fts_node->ilist_size); + error= fts_write_node(executor, selected, &aux_data); + DEBUG_SYNC_C("fts_write_node"); + DBUG_EXECUTE_IF("fts_write_node_crash", DBUG_SUICIDE();); + DBUG_EXECUTE_IF("fts_instrument_sync_sleep", + std::this_thread::sleep_for(std::chrono::seconds(1));); + + if (unlock_cache) mysql_mutex_lock(&table->fts->cache->lock); + } - n_nodes += ib_vector_size(word->nodes); + n_nodes+= ib_vector_size(word->nodes); - if (UNIV_UNLIKELY(error != DB_SUCCESS) && !print_error) { - ib::error() << "(" << error << ") writing" - " word node to FTS auxiliary index table " - << table->name; - print_error = TRUE; - } - } + if (UNIV_UNLIKELY(error != DB_SUCCESS) && !print_error) + { + sql_print_error("InnoDB: ( %s ) writing word node to FTS auxiliary " + "index table %s", ut_strerr(error), + table->name.m_name); + print_error= true; + } + } + } - return(error); + return error; } /*********************************************************************//** @@ -3907,24 +3381,22 @@ fts_sync_begin( trx_start_internal(sync->trx); } -/*********************************************************************//** -Run SYNC on the table, i.e., write out data from the index specific -cache to the FTS aux INDEX table and FTS aux doc id stats table. +/** Run SYNC on the table, i.e., write out data from the index +specific cache to the FTS aux INDEX table and FTS aux doc id +stats table. +@param executor query executor +@param sync sync state +@param index_cache index cache @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t -fts_sync_index( -/*===========*/ - fts_sync_t* sync, /*!< in: sync state */ - fts_index_cache_t* index_cache) /*!< in: index cache */ +fts_sync_index(FTSQueryExecutor *executor, fts_sync_t *sync, + fts_index_cache_t *index_cache) noexcept { - trx_t* trx = sync->trx; - - trx->op_info = "doing SYNC index"; - - ut_ad(rbt_validate(index_cache->words)); - - return(fts_sync_write_words(trx, index_cache, sync->unlock_cache)); + trx_t *trx= sync->trx; + trx->op_info= "doing SYNC index"; + ut_ad(rbt_validate(index_cache->words)); + return fts_sync_write_words(executor, index_cache, sync->unlock_cache); } /** Check if index cache has been synced completely @@ -3978,13 +3450,10 @@ fts_sync_index_reset( } } -/** Commit the SYNC, change state of processed doc ids etc. -@param[in,out] sync sync state -@return DB_SUCCESS if all OK */ +/** Commit the SYNC, change state of processed doc ids etc.\n@param[in,out]\texecutor\t\tquery executor\n@param[in,out]\tsync\t\t\tsync state\n@return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t -fts_sync_commit( - fts_sync_t* sync) +fts_sync_commit(FTSQueryExecutor *executor, fts_sync_t *sync) noexcept { dberr_t error; trx_t* trx = sync->trx; @@ -3995,8 +3464,8 @@ fts_sync_commit( /* After each Sync, update the CONFIG table about the max doc id we just sync-ed to index table */ - error = fts_cmp_set_sync_doc_id(sync->table, sync->max_doc_id, - &last_doc_id, trx); + error = fts_cmp_set_sync_doc_id(executor, sync->table, sync->max_doc_id, + &last_doc_id); /* Get the list of deleted documents that are either in the cache or were headed there but were deleted before the add @@ -4005,7 +3474,7 @@ fts_sync_commit( if (error == DB_SUCCESS && ib_vector_size(cache->deleted_doc_ids) > 0) { error = fts_sync_add_deleted_cache( - sync, cache->deleted_doc_ids); + executor, sync, cache->deleted_doc_ids); } /* We need to do this within the deleted lock since fts_delete() can @@ -4125,11 +3594,13 @@ fts_sync( sync->unlock_cache = unlock_cache; sync->in_progress = true; + const size_t fts_cache_size= fts_max_cache_size; DEBUG_SYNC_C("fts_sync_begin"); fts_sync_begin(sync); - + FTSQueryExecutor executor(sync->trx, sync->table); + error = executor.open_config_table(); + if (error) goto end_sync; begin_sync: - const size_t fts_cache_size= fts_max_cache_size; if (cache->total_size > fts_cache_size) { /* Avoid the case: sync never finish when insert/update keeps comming. */ @@ -4143,19 +3614,21 @@ fts_sync( } for (i = 0; i < ib_vector_size(cache->indexes); ++i) { - fts_index_cache_t* index_cache; - - index_cache = static_cast( + fts_index_cache_t *index_cache= + static_cast( ib_vector_get(cache->indexes, i)); if (index_cache->index->to_be_dropped) { continue; } + error= executor.open_all_aux_tables(index_cache->index); + + if (error) goto end_sync; DBUG_EXECUTE_IF("fts_instrument_sync_before_syncing", std::this_thread::sleep_for( std::chrono::milliseconds(300));); - error = fts_sync_index(sync, index_cache); + error = fts_sync_index(&executor, sync, index_cache); if (error != DB_SUCCESS) { goto end_sync; @@ -4192,7 +3665,7 @@ fts_sync( end_sync: if (error == DB_SUCCESS && !sync->interrupted) { - error = fts_sync_commit(sync); + error = fts_sync_commit(&executor, sync); } else { fts_sync_rollback(sync); } @@ -4514,7 +3987,6 @@ fts_tokenize_by_parser( @param[in,out] doc document to tokenize @param[out] result tokenization result @param[in] parser pluggable parser */ -static void fts_tokenize_document( fts_doc_t* doc, @@ -4544,12 +4016,6 @@ fts_tokenize_document( } } -/** Continue to tokenize a document. -@param[in,out] doc document to tokenize -@param[in] add_pos add this position to all tokens from this tokenization -@param[out] result tokenization result -@param[in] parser pluggable parser */ -static void fts_tokenize_document_next( fts_doc_t* doc, @@ -4664,7 +4130,12 @@ fts_init_doc_id( /* Then compare this value with the ID value stored in the CONFIG table. The larger one will be our new initial Doc ID */ - fts_cmp_set_sync_doc_id(table, 0, &max_doc_id); + trx_t* trx = trx_create(); + trx_start_internal_read_only(trx); + FTSQueryExecutor executor(trx, table); + fts_cmp_set_sync_doc_id(&executor, table, 0, &max_doc_id); + fts_sql_commit(trx); + trx->free(); /* If DICT_TF2_FTS_ADD_DOC_ID is set, we are in the process of creating index (and add doc id column. No need to recovery @@ -4684,81 +4155,6 @@ fts_init_doc_id( return(max_doc_id); } -/*********************************************************************//** -Fetch COUNT(*) from specified table. -@return the number of rows in the table */ -ulint -fts_get_rows_count( -/*===============*/ - fts_table_t* fts_table) /*!< in: fts table to read */ -{ - trx_t* trx; - pars_info_t* info; - que_t* graph; - dberr_t error; - ulint count = 0; - char table_name[MAX_FULL_NAME_LEN]; - - trx = trx_create(); - trx->op_info = "fetching FT table rows count"; - - info = pars_info_create(); - - pars_info_bind_function(info, "my_func", fts_read_ulint, &count); - - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT COUNT(*)" - " FROM $table_name;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - for (;;) { - error = fts_eval_sql(trx, graph); - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - - break; /* Exit the loop. */ - } else { - fts_sql_rollback(trx); - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading" - " FTS table. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << "(" << error - << ") while reading FTS table " - << table_name; - - break; /* Exit the loop. */ - } - } - } - - que_graph_free(graph); - - trx->free(); - - return(count); -} - /*********************************************************************//** Free the modified rows of a table. */ UNIV_INLINE @@ -5584,288 +4980,267 @@ FTS. bool fts_load_stopword( /*==============*/ - const dict_table_t* - table, /*!< in: Table with FTS */ - trx_t* trx, /*!< in: Transactions */ - const char* session_stopword_table, /*!< in: Session stopword table - name */ - bool stopword_is_on, /*!< in: Whether stopword - option is turned on/off */ - bool reload) /*!< in: Whether it is - for reloading FTS table */ + FTSQueryExecutor* executor, /*!< in: FTSQueryExecutor */ + const dict_table_t* table) noexcept /*!< in: Table with FTS */ { - fts_table_t fts_table; + fts_cache_t* cache = table->fts->cache; fts_string_t str; dberr_t error = DB_SUCCESS; - ulint use_stopword; - fts_cache_t* cache; const char* stopword_to_use = NULL; - ibool new_trx = FALSE; byte str_buffer[MAX_FULL_NAME_LEN + 1]; - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, table); - - cache = table->fts->cache; - - if (!reload && !(cache->stopword_info.status & STOPWORD_NOT_INIT)) { + /* If stopword is already initialized, return success */ + if (!(cache->stopword_info.status & STOPWORD_NOT_INIT)) { return true; } - if (!trx) { - trx = trx_create(); -#ifdef UNIV_DEBUG - trx->start_line = __LINE__; - trx->start_file = __FILE__; -#endif - trx_start_internal_low(trx, !high_level_read_only - && !recv_sys.rpo); - trx->op_info = "upload FTS stopword"; - new_trx = TRUE; - } - - /* First check whether stopword filtering is turned off */ - if (reload) { - error = fts_config_get_ulint( - trx, &fts_table, FTS_USE_STOPWORD, &use_stopword); - } else { - use_stopword = (ulint) stopword_is_on; - - error = fts_config_set_ulint( - trx, &fts_table, FTS_USE_STOPWORD, use_stopword); + /* First check if stopwords are enabled */ + ulint use_stopword= 0; + fts_string_t value; + value.f_len= FTS_MAX_CONFIG_VALUE_LEN; + value.f_str= static_cast(ut_malloc_nokey(value.f_len + 1)); + error= fts_config_get_value(executor, table, FTS_USE_STOPWORD, &value); + if (error == DB_SUCCESS) { + use_stopword= strtoul(reinterpret_cast(value.f_str), + nullptr, 10); } + ut_free(value.f_str); + if (error != DB_SUCCESS) { - goto cleanup; + /* Failed to read config */ + return false; } - - /* If stopword is turned off, no need to continue to load the - stopword into cache, but still need to do initialization */ + if (!use_stopword) { + /* Stopwords are disabled, mark as initialized but off */ cache->stopword_info.status = STOPWORD_OFF; - goto cleanup; + return true; } - if (reload) { - /* Fetch the stopword table name from FTS config - table */ - str.f_n_char = 0; - str.f_str = str_buffer; - str.f_len = sizeof(str_buffer) - 1; - - error = fts_config_get_value( - trx, &fts_table, FTS_STOPWORD_TABLE_NAME, &str); + /* Read FTS_STOPWORD_TABLE_NAME from CONFIG table */ + str.f_n_char = 0; + str.f_str = str_buffer; + str.f_len = sizeof(str_buffer) - 1; - if (error != DB_SUCCESS) { - goto cleanup; - } + error = fts_config_get_value(executor, table, FTS_STOPWORD_TABLE_NAME, &str); - if (*str.f_str) { - stopword_to_use = (const char*) str.f_str; - } - } else { - stopword_to_use = session_stopword_table; + if (error == DB_SUCCESS && *str.f_str) { + stopword_to_use = (const char*) str.f_str; } + /* Load user stopword table if specified, otherwise load default */ if (stopword_to_use - && fts_load_user_stopword(table->fts, stopword_to_use, - &cache->stopword_info)) { - /* Save the stopword table name to the configure - table */ - if (!reload) { - str.f_n_char = 0; - str.f_str = (byte*) stopword_to_use; - str.f_len = strlen(stopword_to_use); - - error = fts_config_set_value( - trx, &fts_table, FTS_STOPWORD_TABLE_NAME, &str); - } + && fts_load_user_stopword(executor, table->fts, stopword_to_use, + &cache->stopword_info)) { + /* Successfully loaded user stopword table */ } else { /* Load system default stopword list */ fts_load_default_stopword(&cache->stopword_info); } -cleanup: - if (new_trx) { - if (error == DB_SUCCESS) { - fts_sql_commit(trx); - } else { - fts_sql_rollback(trx); - } - - trx->clear_and_free(); - } - + /* Initialize cached_stopword RB-tree if not already created */ if (!cache->stopword_info.cached_stopword) { cache->stopword_info.cached_stopword = rbt_create_arg_cmp( sizeof(fts_tokenizer_word_t), innobase_fts_text_cmp, &my_charset_latin1); } - return error == DB_SUCCESS; + return true; } -/**********************************************************************//** -Callback function when we initialize the FTS at the start up -time. It recovers the maximum Doc IDs presented in the current table. -Tested by innodb_fts.crash_recovery -@return: always returns TRUE */ -static -ibool -fts_init_get_doc_id( -/*================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: table with fts */ -{ - doc_id_t doc_id = FTS_NULL_DOC_ID; - sel_node_t* node = static_cast(row); - que_node_t* exp = node->select_list; - dict_table_t* table = static_cast(user_arg); - fts_cache_t* cache = table->fts->cache; - - ut_ad(ib_vector_is_empty(cache->get_docs)); - - /* Copy each indexed column content into doc->text.f_str */ - if (exp) { - dfield_t* dfield = que_node_get_val(exp); - dtype_t* type = dfield_get_type(dfield); - void* data = dfield_get_data(dfield); - - ut_a(dtype_get_mtype(type) == DATA_INT); - - doc_id = static_cast(mach_read_from_8( - static_cast(data))); - - exp = que_node_get_next(que_node_get_next(exp)); - if (exp) { - ut_ad(table->versioned()); - dfield = que_node_get_val(exp); - type = dfield_get_type(dfield); - ut_ad(type->vers_sys_end()); - data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - if (table->versioned_by_id()) { - ut_ad(len == sizeof trx_id_max_bytes); - if (0 != memcmp(data, trx_id_max_bytes, len)) { - return true; - } - } else { - ut_ad(len == sizeof timestamp_max_bytes); - if (!IS_MAX_TIMESTAMP(data)) { - return true; - } - } - ut_ad(!(exp = que_node_get_next(exp))); - } - ut_ad(!exp); - - if (doc_id >= cache->next_doc_id) { - cache->next_doc_id = doc_id + 1; - } - } - - return(TRUE); -} - -/**********************************************************************//** -Callback function when we initialize the FTS at the start up +/** Callback function when we initialize the FTS at the start up time. It recovers Doc IDs that have not sync-ed to the auxiliary table, and require to bring them back into FTS index. +@param executor query executor +@param get_doc Document +@param doc_id document id to be fetched @return: always returns TRUE */ -static -ibool -fts_init_recover_doc( -/*=================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts cache */ +static void fts_init_recover_all_docs(FTSQueryExecutor *executor, + fts_get_doc_t *get_doc, + doc_id_t doc_id) noexcept { + executor->trx()->op_info= "fetching indexed FTS document"; + dict_index_t *fts_index= get_doc->index_cache->index; + dict_table_t *user_table= fts_index->table; + dict_index_t *fts_doc_id_index= user_table->fts_doc_id_index; + dict_index_t *clust_index= dict_table_get_first_index(user_table); + fts_cache_t *cache= get_doc->cache; + ut_a(user_table->fts->doc_col != ULINT_UNDEFINED); + ut_a(fts_doc_id_index); + /* Map FTS index columns to clustered index field positions */ + ulint *clust_field_nos= static_cast( + mem_heap_alloc(executor->get_heap(), + fts_index->n_user_defined_cols * sizeof(ulint))); + + for (unsigned i= 0; i < fts_index->n_user_defined_cols; i++) + { + dict_field_t* fts_field= dict_index_get_nth_field(fts_index, i); + clust_field_nos[i]= dict_col_get_index_pos(fts_field->col, clust_index); + } - fts_doc_t doc; - ulint doc_len = 0; - ulint field_no = 0; - fts_get_doc_t* get_doc = static_cast(user_arg); - doc_id_t doc_id = FTS_NULL_DOC_ID; - sel_node_t* node = static_cast(row); - que_node_t* exp = node->select_list; - fts_cache_t* cache = get_doc->cache; - st_mysql_ftparser* parser = get_doc->index_cache->index->parser; - - fts_doc_init(&doc); - doc.found = TRUE; - - ut_ad(cache); - - /* Copy each indexed column content into doc->text.f_str */ - while (exp) { - dfield_t* dfield = que_node_get_val(exp); - ulint len = dfield_get_len(dfield); - - if (field_no == 0) { - dtype_t* type = dfield_get_type(dfield); - void* data = dfield_get_data(dfield); - - ut_a(dtype_get_mtype(type) == DATA_INT); - - doc_id = static_cast(mach_read_from_8( - static_cast(data))); + dfield_t fields[1]; + dtuple_t search_tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&search_tuple, fts_doc_id_index, 1); + dfield_t* dfield= dtuple_get_nth_field(&search_tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id((byte*) &write_doc_id, doc_id); + dfield_set_data(dfield, &write_doc_id, sizeof(write_doc_id)); + + auto process_doc_recovery= [get_doc, cache, user_table, fts_index, + clust_field_nos](const rec_t* rec, + const dict_index_t* index, + const rec_offs* offsets) -> dberr_t + { + fts_doc_t doc; + ulint doc_len= 0; + doc_id_t doc_id= FTS_NULL_DOC_ID; + st_mysql_ftparser* parser= fts_index->parser; - field_no++; - exp = que_node_get_next(exp); - continue; - } + fts_doc_init(&doc); + doc.found= TRUE; - if (len == UNIV_SQL_NULL) { - exp = que_node_get_next(exp); - continue; - } + /* Extract doc_id from the clustered index record */ + ulint doc_col_pos= dict_col_get_index_pos( + &user_table->cols[user_table->fts->doc_col], index); - ut_ad(get_doc); + ulint processed_field= 0; + ulint len; + const byte* doc_id_data= rec_get_nth_field(rec, offsets, doc_col_pos, &len); - if (!get_doc->index_cache->charset) { - get_doc->index_cache->charset = fts_get_charset( - dfield->type.prtype); - } + if (len == sizeof(doc_id_t)) + { + doc_id= fts_read_doc_id(doc_id_data); - doc.charset = get_doc->index_cache->charset; + /* Process each indexed column content */ + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, col_pos, + &field_len); + if (field_len == UNIV_SQL_NULL) + continue; + if (!get_doc->index_cache->charset) + { + dict_field_t* fts_field= dict_index_get_nth_field(fts_index, i); + get_doc->index_cache->charset= fts_get_charset( + fts_field->col->prtype); + } + doc.charset= get_doc->index_cache->charset; + + /* Handle externally stored fields */ + if (rec_offs_nth_extern(offsets, col_pos)) + doc.text.f_str= btr_copy_externally_stored_field( + &doc.text.f_len, const_cast(field_data), + user_table->space->zip_size(), field_len, + static_cast(doc.self_heap->arg)); + else + { + doc.text.f_str= const_cast(field_data); + doc.text.f_len= field_len; + } - if (dfield_is_ext(dfield)) { - dict_table_t* table = cache->sync->table; + if (processed_field == 0) + fts_tokenize_document(&doc, NULL, parser); + else + fts_tokenize_document_next(&doc, doc_len, NULL, parser); - doc.text.f_str = btr_copy_externally_stored_field( - &doc.text.f_len, - static_cast(dfield_get_data(dfield)), - table->space->zip_size(), len, - static_cast(doc.self_heap->arg)); - } else { - doc.text.f_str = static_cast( - dfield_get_data(dfield)); + processed_field++; + doc_len+= + (i < (unsigned) get_doc->index_cache->index->n_user_defined_cols - 1) + ? field_len + 1 + : field_len; + } - doc.text.f_len = len; - } + fts_cache_add_doc(cache, get_doc->index_cache, doc_id, doc.tokens); + fts_doc_free(&doc); + cache->added++; - if (field_no == 1) { - fts_tokenize_document(&doc, NULL, parser); - } else { - fts_tokenize_document_next(&doc, doc_len, NULL, parser); - } + if (doc_id >= cache->next_doc_id) + cache->next_doc_id= doc_id + 1; + } - exp = que_node_get_next(exp); + return DB_SUCCESS; + }; + RecordCallback reader(process_doc_recovery, doc_id_comparator); + executor->read_by_index(user_table, fts_doc_id_index, &search_tuple, + PAGE_CUR_G, reader); +} - doc_len += (exp) ? len + 1 : len; +/** Get the next large document id and update it in fulltext cache +@param executor query executor +@param doc_id document id to be updated +@param index fulltext index */ +static void fts_init_get_doc_id(FTSQueryExecutor *executor, + doc_id_t doc_id, dict_index_t *index) noexcept +{ + executor->trx()->op_info= "fetching indexed FTS document"; + dict_table_t* user_table= index->table; + fts_cache_t* cache= user_table->fts->cache; + ut_a(user_table->fts->doc_col != ULINT_UNDEFINED); - field_no++; - } + dfield_t fields[1]; + dtuple_t search_tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&search_tuple, index, 1); + dfield_t* dfield= dtuple_get_nth_field(&search_tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id((byte*) &write_doc_id, doc_id); + dfield_set_data(dfield, &write_doc_id, sizeof(write_doc_id)); + + auto process_doc_id= [cache, user_table](const rec_t* rec, + const dict_index_t* index, + const rec_offs* offsets) -> dberr_t + { + ulint doc_col_pos= dict_col_get_index_pos( + &user_table->cols[user_table->fts->doc_col], index); - fts_cache_add_doc(cache, get_doc->index_cache, doc_id, doc.tokens); + ulint len; + const byte* doc_id_data= rec_get_nth_field(rec, offsets, doc_col_pos, &len); - fts_doc_free(&doc); + if (len == sizeof(doc_id_t)) + { + doc_id_t found_doc_id= mach_read_from_8(doc_id_data); + if (user_table->versioned()) + { + ulint vers_end_pos= dict_col_get_index_pos( + &user_table->cols[user_table->vers_end], index); + ulint vers_len; + const byte* vers_data= rec_get_nth_field(rec, offsets, + vers_end_pos, &vers_len); - cache->added++; + if (user_table->versioned_by_id()) + { + if (vers_len == sizeof(trx_id_max_bytes) && + memcmp(vers_data, trx_id_max_bytes, vers_len) != 0) + return DB_SUCCESS; + } + else + { + if (vers_len == sizeof(timestamp_max_bytes) && + !IS_MAX_TIMESTAMP(vers_data)) + return DB_SUCCESS; + } + } - if (doc_id >= cache->next_doc_id) { - cache->next_doc_id = doc_id + 1; - } + /* Update cache->next_doc_id if this doc_id is larger */ + if (found_doc_id >= cache->next_doc_id) + cache->next_doc_id= found_doc_id + 1; + } + return DB_SUCCESS; + }; - return(TRUE); + RecordCallback reader(process_doc_id, doc_id_comparator); + executor->read_by_index(user_table, index, &search_tuple, PAGE_CUR_G, + reader); } /**********************************************************************//** @@ -5885,6 +5260,9 @@ fts_init_index( fts_get_doc_t* get_doc = NULL; fts_cache_t* cache = table->fts->cache; bool need_init = false; + /* Declare variables before any goto to avoid initialization bypass */ + trx_t *trx = nullptr; + FTSQueryExecutor *executor= nullptr; /* First check cache->get_docs is initialized */ if (!has_cache_lock) { @@ -5905,13 +5283,19 @@ fts_init_index( start_doc = cache->synced_doc_id; + /* Create single FTSQueryExecutor for all operations */ + trx = trx_create(); + trx_start_internal_read_only(trx); + executor= new FTSQueryExecutor(trx, table); + if (!start_doc) { - trx_t *trx = trx_create(); - trx_start_internal_read_only(trx); - dberr_t err= fts_read_synced_doc_id(table, &start_doc, trx); - fts_sql_commit(trx); - trx->free(); + dberr_t err= fts_read_synced_doc_id( + executor, table, &start_doc); if (err != DB_SUCCESS) { + fts_sql_commit(trx); + trx->free(); + delete executor; + executor = nullptr; goto func_exit; } if (start_doc) { @@ -5928,13 +5312,11 @@ fts_init_index( ut_a(index); - fts_doc_fetch_by_doc_id(NULL, start_doc, index, - FTS_FETCH_DOC_BY_ID_LARGE, - fts_init_get_doc_id, table); + fts_init_get_doc_id(executor, start_doc, index); } else { if (table->fts->cache->stopword_info.status & STOPWORD_NOT_INIT) { - fts_load_stopword(table, NULL, NULL, true, true); + fts_load_stopword(executor, table); } for (ulint i = 0; i < ib_vector_size(cache->get_docs); ++i) { @@ -5943,9 +5325,7 @@ fts_init_index( index = get_doc->index_cache->index; - fts_doc_fetch_by_doc_id(NULL, start_doc, index, - FTS_FETCH_DOC_BY_ID_LARGE, - fts_init_recover_doc, get_doc); + fts_init_recover_all_docs(executor, get_doc, start_doc); } } @@ -5953,6 +5333,15 @@ fts_init_index( fts_get_docs_clear(cache->get_docs); + /* Commit transaction and cleanup */ + if (trx) { + fts_sql_commit(trx); + trx->free(); + } + if (executor) { + delete executor; + } + func_exit: if (!has_cache_lock) { mysql_mutex_unlock(&cache->lock); diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index ea6ddab1b2458..130e62a03e540 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -26,7 +26,10 @@ Completed 2011/7/10 Sunny and Jimmy Yang ***********************************************************************/ +#define MYSQL_SERVER + #include "fts0fts.h" +#include "fts0exec.h" #include "row0sel.h" #include "que0types.h" #include "fts0priv.h" @@ -38,6 +41,7 @@ Completed 2011/7/10 Sunny and Jimmy Yang #include "fts0opt.h" #include "fts0vlc.h" #include "wsrep.h" +#include "sql_class.h" #ifdef WITH_WSREP extern Atomic_relaxed wsrep_sst_disable_writes; @@ -70,9 +74,6 @@ static bool fts_opt_start_shutdown = false; Protected by fts_optimize_wq->mutex. */ static pthread_cond_t fts_opt_shutdown_cond; -/** Initial size of nodes in fts_word_t. */ -static const ulint FTS_WORD_NODES_INIT_SIZE = 64; - /** Last time we did check whether system need a sync */ static time_t last_check_sync_time; @@ -140,11 +141,6 @@ struct fts_optimize_t { char* name_prefix; /*!< FTS table name prefix */ - fts_table_t fts_index_table;/*!< Common table definition */ - - /*!< Common table definition */ - fts_table_t fts_common_table; - dict_table_t* table; /*!< Table that has to be queried */ dict_index_t* index; /*!< The FTS index to be optimized */ @@ -235,28 +231,6 @@ static ulint fts_optimize_time_limit; /** It's defined in fts0fts.cc */ extern const char* fts_common_tables[]; -/** SQL Statement for changing state of rows to be deleted from FTS Index. */ -static const char* fts_init_delete_sql = - "BEGIN\n" - "\n" - "INSERT INTO $BEING_DELETED\n" - "SELECT doc_id FROM $DELETED;\n" - "\n" - "INSERT INTO $BEING_DELETED_CACHE\n" - "SELECT doc_id FROM $DELETED_CACHE;\n"; - -static const char* fts_delete_doc_ids_sql = - "BEGIN\n" - "\n" - "DELETE FROM $DELETED WHERE doc_id = :doc_id1;\n" - "DELETE FROM $DELETED_CACHE WHERE doc_id = :doc_id2;\n"; - -static const char* fts_end_delete_sql = - "BEGIN\n" - "\n" - "DELETE FROM $BEING_DELETED;\n" - "DELETE FROM $BEING_DELETED_CACHE;\n"; - /**********************************************************************//** Initialize fts_zip_t. */ static @@ -327,238 +301,47 @@ fts_zip_init( *zip->word.f_str = '\0'; } -/**********************************************************************//** -Create a fts_optimizer_word_t instance. -@return new instance */ -static -fts_word_t* -fts_word_init( -/*==========*/ - fts_word_t* word, /*!< in: word to initialize */ - byte* utf8, /*!< in: UTF-8 string */ - ulint len) /*!< in: length of string in bytes */ -{ - mem_heap_t* heap = mem_heap_create(sizeof(fts_node_t)); - - memset(word, 0, sizeof(*word)); - - word->text.f_len = len; - word->text.f_str = static_cast(mem_heap_alloc(heap, len + 1)); - - /* Need to copy the NUL character too. */ - memcpy(word->text.f_str, utf8, word->text.f_len); - word->text.f_str[word->text.f_len] = 0; - - word->heap_alloc = ib_heap_allocator_create(heap); - - word->nodes = ib_vector_create( - word->heap_alloc, sizeof(fts_node_t), FTS_WORD_NODES_INIT_SIZE); - - return(word); -} - -/**********************************************************************//** -Read the FTS INDEX row. -@return fts_node_t instance */ -static -fts_node_t* -fts_optimize_read_node( -/*===================*/ - fts_word_t* word, /*!< in: */ - que_node_t* exp) /*!< in: */ -{ - int i; - fts_node_t* node = static_cast( - ib_vector_push(word->nodes, NULL)); - - /* Start from 1 since the first node has been read by the caller */ - for (i = 1; exp; exp = que_node_get_next(exp), ++i) { - - dfield_t* dfield = que_node_get_val(exp); - byte* data = static_cast( - dfield_get_data(dfield)); - ulint len = dfield_get_len(dfield); - - ut_a(len != UNIV_SQL_NULL); - - /* Note: The column numbers below must match the SELECT */ - switch (i) { - case 1: /* DOC_COUNT */ - node->doc_count = mach_read_from_4(data); - break; - - case 2: /* FIRST_DOC_ID */ - node->first_doc_id = fts_read_doc_id(data); - break; - - case 3: /* LAST_DOC_ID */ - node->last_doc_id = fts_read_doc_id(data); - break; - - case 4: /* ILIST */ - node->ilist_size_alloc = node->ilist_size = len; - node->ilist = static_cast(ut_malloc_nokey(len)); - memcpy(node->ilist, data, len); - break; - - default: - ut_error; - } - } - - /* Make sure all columns were read. */ - ut_a(i == 5); - - return(node); -} - -/**********************************************************************//** -Callback function to fetch the rows in an FTS INDEX record. -@return always returns non-NULL */ -ibool -fts_optimize_index_fetch_node( -/*==========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ib_vector_t */ +dberr_t fts_index_fetch_nodes(FTSQueryExecutor *executor, dict_index_t *index, + const fts_string_t *word, void *user_arg, + FTSRecordProcessor processor, + AuxCompareMode compare_mode) noexcept { - fts_word_t* word; - sel_node_t* sel_node = static_cast(row); - fts_fetch_t* fetch = static_cast(user_arg); - ib_vector_t* words = static_cast(fetch->read_arg); - que_node_t* exp = sel_node->select_list; - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint dfield_len = dfield_get_len(dfield); - fts_node_t* node; - bool is_word_init = false; - - ut_a(dfield_len <= FTS_MAX_WORD_LEN); - - if (ib_vector_size(words) == 0) { - - word = static_cast(ib_vector_push(words, NULL)); - fts_word_init(word, (byte*) data, dfield_len); - is_word_init = true; - } - - word = static_cast(ib_vector_last(words)); - - if (dfield_len != word->text.f_len - || memcmp(word->text.f_str, data, dfield_len)) { - - word = static_cast(ib_vector_push(words, NULL)); - fts_word_init(word, (byte*) data, dfield_len); - is_word_init = true; - } - - node = fts_optimize_read_node(word, que_node_get_next(exp)); - - fetch->total_memory += node->ilist_size; - if (is_word_init) { - fetch->total_memory += sizeof(fts_word_t) - + sizeof(ib_alloc_t) + sizeof(ib_vector_t) + dfield_len - + sizeof(fts_node_t) * FTS_WORD_NODES_INIT_SIZE; - } else if (ib_vector_size(words) > FTS_WORD_NODES_INIT_SIZE) { - fetch->total_memory += sizeof(fts_node_t); - } - - if (fetch->total_memory >= fts_result_cache_limit) { - return(FALSE); - } - - return(TRUE); -} - -/**********************************************************************//** -Read the rows from the FTS inde. -@return DB_SUCCESS or error code */ -dberr_t -fts_index_fetch_nodes( -/*==================*/ - trx_t* trx, /*!< in: transaction */ - que_t** graph, /*!< in: prepared statement */ - fts_table_t* fts_table, /*!< in: table of the FTS INDEX */ - const fts_string_t* - word, /*!< in: the word to fetch */ - fts_fetch_t* fetch) /*!< in: fetch callback.*/ -{ - pars_info_t* info; - dberr_t error; - char table_name[MAX_FULL_NAME_LEN]; - - trx->op_info = "fetching FTS index nodes"; - - if (*graph) { - info = (*graph)->info; - } else { - ulint selected; - - info = pars_info_create(); - - ut_a(fts_table->type == FTS_INDEX_TABLE); - - selected = fts_select_index(fts_table->charset, - word->f_str, word->f_len); - - fts_table->suffix = fts_get_suffix(selected); - - fts_get_table_name(fts_table, table_name); - - pars_info_bind_id(info, "table_name", table_name); - } - - pars_info_bind_function(info, "my_func", fetch->read_record, fetch); - pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len); - - if (!*graph) { - - *graph = fts_parse_sql( - fts_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT word, doc_count, first_doc_id, last_doc_id," - " ilist\n" - " FROM $table_name\n" - " WHERE word LIKE :word\n" - " ORDER BY first_doc_id;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - } - - for (;;) { - error = fts_eval_sql(trx, *graph); - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - - break; /* Exit the loop. */ - } else { - fts_sql_rollback(trx); - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading" - " FTS index. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << "(" << error - << ") while reading FTS index."; - - break; /* Exit the loop. */ - } - } - } - - return(error); + dberr_t error= DB_SUCCESS; + trx_t *trx= executor->trx(); + trx->op_info= "fetching FTS index nodes"; + CHARSET_INFO *cs= fts_index_get_charset(index); + uint8_t selected= fts_select_index(cs, word->f_str, word->f_len); + ulint total_memory= 0; + for (;;) + { + AuxRecordReader reader= processor + ? AuxRecordReader(user_arg, processor, compare_mode) + : AuxRecordReader(user_arg, &total_memory, compare_mode); + error= executor->read_aux(selected, word, reader); + if (UNIV_LIKELY(error == DB_SUCCESS || error == DB_RECORD_NOT_FOUND)) + { + if (error == DB_RECORD_NOT_FOUND) error = DB_SUCCESS; + fts_sql_commit(trx); + break; + } + else + { + fts_sql_rollback(trx); + if (error == DB_LOCK_WAIT_TIMEOUT) + { + sql_print_warning("InnoDB: Lock wait timeout reading FTS index." + "Retrying!"); + trx->error_state= DB_SUCCESS; + } + else + { + sql_print_error("InnoDB: (%s) while reading FTS index.", + ut_strerr(error)); + break; + } + } + } + return error; } /**********************************************************************//** @@ -662,88 +445,6 @@ fts_zip_read_word( return(zip->status == Z_OK || zip->status == Z_STREAM_END ? ptr : NULL); } -/**********************************************************************//** -Callback function to fetch and compress the word in an FTS -INDEX record. -@return FALSE on EOF */ -static -ibool -fts_fetch_index_words( -/*==================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ib_vector_t */ -{ - sel_node_t* sel_node = static_cast(row); - fts_zip_t* zip = static_cast(user_arg); - que_node_t* exp = sel_node->select_list; - dfield_t* dfield = que_node_get_val(exp); - - ut_a(dfield_get_len(dfield) <= FTS_MAX_WORD_LEN); - - uint16 len = uint16(dfield_get_len(dfield)); - void* data = dfield_get_data(dfield); - - /* Skip the duplicate words. */ - if (zip->word.f_len == len && !memcmp(zip->word.f_str, data, len)) { - return(TRUE); - } - - memcpy(zip->word.f_str, data, len); - zip->word.f_len = len; - - ut_a(zip->zp->avail_in == 0); - ut_a(zip->zp->next_in == NULL); - - /* The string is prefixed by len. */ - /* FIXME: This is not byte order agnostic (InnoDB data files - with FULLTEXT INDEX are not portable between little-endian and - big-endian systems!) */ - zip->zp->next_in = reinterpret_cast(&len); - zip->zp->avail_in = sizeof(len); - - /* Compress the word, create output blocks as necessary. */ - while (zip->zp->avail_in > 0) { - - /* No space left in output buffer, create a new one. */ - if (zip->zp->avail_out == 0) { - byte* block; - - block = static_cast( - ut_malloc_nokey(zip->block_sz)); - - ib_vector_push(zip->blocks, &block); - - zip->zp->next_out = block; - zip->zp->avail_out = static_cast(zip->block_sz); - } - - switch (zip->status = deflate(zip->zp, Z_NO_FLUSH)) { - case Z_OK: - if (zip->zp->avail_in == 0) { - zip->zp->next_in = static_cast(data); - zip->zp->avail_in = uInt(len); - ut_a(len <= FTS_MAX_WORD_LEN); - len = 0; - } - continue; - - case Z_STREAM_END: - case Z_BUF_ERROR: - case Z_STREAM_ERROR: - default: - ut_error; - } - } - - /* All data should have been compressed. */ - ut_a(zip->zp->avail_in == 0); - zip->zp->next_in = NULL; - - ++zip->n_words; - - return(zip->n_words >= zip->max_words ? FALSE : TRUE); -} - /**********************************************************************//** Finish Zip deflate. */ static @@ -786,241 +487,181 @@ fts_zip_deflate_end( memset(zip->zp, 0, sizeof(*zip->zp)); } -/**********************************************************************//** -Read the words from the FTS INDEX. +/** Read the words from the FTS INDEX. +@param executor query executor +@param optim optimize scratch pad +@param word get words gerater than this +@param n_words max words to read @return DB_SUCCESS if all OK, DB_TABLE_NOT_FOUND if no more indexes to search else error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_index_fetch_words( -/*==================*/ - fts_optimize_t* optim, /*!< in: optimize scratch pad */ - const fts_string_t* word, /*!< in: get words greater than this - word */ - ulint n_words)/*!< in: max words to read */ +dberr_t fts_index_fetch_words(FTSQueryExecutor *executor, + fts_optimize_t *optim, + const fts_string_t *word, + ulint n_words) noexcept { - pars_info_t* info; - que_t* graph; - ulint selected; - fts_zip_t* zip = NULL; - dberr_t error = DB_SUCCESS; - mem_heap_t* heap = static_cast(optim->self_heap->arg); - ibool inited = FALSE; + dberr_t error= DB_SUCCESS; + mem_heap_t *heap= static_cast(optim->self_heap->arg); + optim->trx->op_info= "fetching FTS index words"; - optim->trx->op_info = "fetching FTS index words"; + if (optim->zip == NULL) + optim->zip = fts_zip_create(heap, FTS_ZIP_BLOCK_SIZE, n_words); + else fts_zip_initialize(optim->zip); - if (optim->zip == NULL) { - optim->zip = fts_zip_create(heap, FTS_ZIP_BLOCK_SIZE, n_words); - } else { - fts_zip_initialize(optim->zip); - } + CHARSET_INFO *cs= fts_index_get_charset(optim->index); - for (selected = fts_select_index( - optim->fts_index_table.charset, word->f_str, word->f_len); - selected < FTS_NUM_AUX_INDEX; - selected++) { - - char table_name[MAX_FULL_NAME_LEN]; - - optim->fts_index_table.suffix = fts_get_suffix(selected); - - info = pars_info_create(); - - pars_info_bind_function( - info, "my_func", fts_fetch_index_words, optim->zip); - - pars_info_bind_varchar_literal( - info, "word", word->f_str, word->f_len); - - fts_get_table_name(&optim->fts_index_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - &optim->fts_index_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT word\n" - " FROM $table_name\n" - " WHERE word > :word\n" - " ORDER BY word;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - zip = optim->zip; - - for (;;) { - int err; - - if (!inited && ((err = deflateInit(zip->zp, 9)) - != Z_OK)) { - ib::error() << "ZLib deflateInit() failed: " - << err; - - error = DB_ERROR; - break; - } else { - inited = TRUE; - error = fts_eval_sql(optim->trx, graph); - } + /* Create compression processor with state */ + bool compress_inited = false; + auto compress_processor= [&compress_inited]( + const rec_t *rec, const dict_index_t *index, + const rec_offs *offsets, void *user_arg) -> dberr_t + { + fts_zip_t* zip= static_cast(user_arg); - if (UNIV_LIKELY(error == DB_SUCCESS)) { - //FIXME fts_sql_commit(optim->trx); - break; - } else { - //FIXME fts_sql_rollback(optim->trx); + /* Extract word field using rec_get_nth_field() */ + ulint word_len; + const byte* word_data= rec_get_nth_field(rec, offsets, 0, &word_len); - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "Lock wait timeout" - " reading document. Retrying!"; + if (!word_data || word_len == UNIV_SQL_NULL || word_len > FTS_MAX_WORD_LEN) + return DB_SUCCESS; - /* We need to reset the ZLib state. */ - inited = FALSE; - deflateEnd(zip->zp); - fts_zip_init(zip); + /* Skip duplicate words */ + if (zip->word.f_len == word_len && + !memcmp(zip->word.f_str, word_data, word_len)) + return DB_SUCCESS; - optim->trx->error_state = DB_SUCCESS; - } else { - ib::error() << "(" << error - << ") while reading document."; + /* Initialize deflate if not done yet */ + if (!compress_inited) + { + int err = deflateInit(zip->zp, 9); + if (err != Z_OK) + { + sql_print_error("InnoDB: ZLib deflateInit() failed: %d", err); + return DB_ERROR; + } + compress_inited = true; + } - break; /* Exit the loop. */ - } - } - } + /* Update current word */ + memcpy(zip->word.f_str, word_data, word_len); + zip->word.f_len = word_len; + ut_a(zip->zp->avail_in == 0); + ut_a(zip->zp->next_in == NULL); - que_graph_free(graph); + /* Compress the word with length prefix */ + uint16_t len = static_cast(word_len); + zip->zp->next_in = reinterpret_cast(&len); + zip->zp->avail_in = sizeof(len); - /* Check if max word to fetch is exceeded */ - if (optim->zip->n_words >= n_words) { - break; - } - } + /* Compress the word, create output blocks as necessary */ + while (zip->zp->avail_in > 0) + { + /* No space left in output buffer, create a new one */ + if (zip->zp->avail_out == 0) + { + byte* block= static_cast(ut_malloc_nokey(zip->block_sz)); + ib_vector_push(zip->blocks, &block); + zip->zp->next_out= block; + zip->zp->avail_out= static_cast(zip->block_sz); + } + + switch (zip->status = deflate(zip->zp, Z_NO_FLUSH)) + { + case Z_OK: + if (zip->zp->avail_in == 0) + { + zip->zp->next_in= const_cast(word_data); + zip->zp->avail_in = static_cast(len); + ut_a(len <= FTS_MAX_WORD_LEN); + len = 0; + } + continue; + case Z_STREAM_END: + case Z_BUF_ERROR: + case Z_STREAM_ERROR: + default: + ut_error; + } + } - if (error == DB_SUCCESS && zip->status == Z_OK && zip->n_words > 0) { + /* All data should have been compressed */ + ut_a(zip->zp->avail_in == 0); + zip->zp->next_in = NULL; - /* All data should have been read. */ - ut_a(zip->zp->avail_in == 0); + ++zip->n_words; - fts_zip_deflate_end(zip); - } else { - deflateEnd(zip->zp); - } + /* Continue until we reach max words */ + return zip->n_words < zip->max_words ? DB_SUCCESS : DB_SUCCESS_LOCKED_REC; + }; - return(error); -} + for (uint8_t selected= fts_select_index(cs, word->f_str, word->f_len); + selected < FTS_NUM_AUX_INDEX; selected++) + { + for (;;) + { + AuxRecordReader aux_reader(optim->zip, compress_processor, + AuxCompareMode::GREATER); + + error= executor->read_aux(selected, word, aux_reader); + if (UNIV_LIKELY(error == DB_SUCCESS || error == DB_RECORD_NOT_FOUND)) { + if (error == DB_RECORD_NOT_FOUND) error = DB_SUCCESS; + break; + } + else + { + if (error == DB_LOCK_WAIT_TIMEOUT) + { + sql_print_warning("InnoDB: Lock wait timeout reading " + "words. Retrying!"); + if (compress_inited) + { + deflateEnd(optim->zip->zp); + fts_zip_init(optim->zip); + compress_inited= false; + } + optim->trx->error_state = DB_SUCCESS; + } + else + { + sql_print_error("InnoDB: (%s) while reading words.", + ut_strerr(error)); + break; + } + } + } -/**********************************************************************//** -Callback function to fetch the doc id from the record. -@return always returns TRUE */ -static -ibool -fts_fetch_doc_ids( -/*==============*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ib_vector_t */ -{ - que_node_t* exp; - int i = 0; - sel_node_t* sel_node = static_cast(row); - fts_doc_ids_t* fts_doc_ids = static_cast(user_arg); - doc_id_t* update = static_cast( - ib_vector_push(fts_doc_ids->doc_ids, NULL)); - - for (exp = sel_node->select_list; - exp; - exp = que_node_get_next(exp), ++i) { - - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - - ut_a(len != UNIV_SQL_NULL); - - /* Note: The column numbers below must match the SELECT. */ - switch (i) { - case 0: /* DOC_ID */ - *update = fts_read_doc_id( - static_cast(data)); - break; + if (optim->zip->n_words >= n_words) break; + } - default: - ut_error; - } - } + fts_zip_t *zip = optim->zip; + if (error == DB_SUCCESS && zip->status == Z_OK && zip->n_words > 0) { + /* All data should have been read */ + ut_a(zip->zp->avail_in == 0); + fts_zip_deflate_end(zip); + } + else deflateEnd(zip->zp); - return(TRUE); + return error; } -/**********************************************************************//** -Read the rows from a FTS common auxiliary table. -@return DB_SUCCESS or error code */ -dberr_t -fts_table_fetch_doc_ids( -/*====================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: table */ - fts_doc_ids_t* doc_ids) /*!< in: For collecting doc ids */ +dberr_t fts_table_fetch_doc_ids(FTSQueryExecutor *executor, + dict_table_t *table, const char *tbl_name, + fts_doc_ids_t *doc_ids) noexcept { - dberr_t error; - que_t* graph; - pars_info_t* info = pars_info_create(); - ibool alloc_bk_trx = FALSE; - char table_name[MAX_FULL_NAME_LEN]; + ut_ad(executor != nullptr); + executor->trx()->op_info = "fetching FTS doc ids"; + CommonTableReader reader; + dberr_t err= executor->read_all_common(tbl_name, reader); - ut_a(fts_table->suffix != NULL); - ut_a(fts_table->type == FTS_COMMON_TABLE); - - if (!trx) { - trx = trx_create(); - alloc_bk_trx = TRUE; - } - - trx->op_info = "fetching FTS doc ids"; - - pars_info_bind_function(info, "my_func", fts_fetch_doc_ids, doc_ids); - - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT doc_id FROM $table_name;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - error = fts_eval_sql(trx, graph); - fts_sql_commit(trx); - que_graph_free(graph); - - if (error == DB_SUCCESS) { - fts_doc_ids_sort(doc_ids->doc_ids); - } - - if (alloc_bk_trx) { - trx->free(); - } - - return(error); + if (err == DB_SUCCESS) + { + const auto& doc_id_vector= reader.get_doc_ids(); + for (doc_id_t doc_id : doc_id_vector) + ib_vector_push(doc_ids->doc_ids, &doc_id); + fts_doc_ids_sort(doc_ids->doc_ids); + } + + return err; } /**********************************************************************//** @@ -1419,87 +1060,53 @@ fts_optimize_word( return(nodes); } -/**********************************************************************//** -Update the FTS index table. This is a delete followed by an insert. +/** Write the words and ilist to disk. +@param executor query executor +@param index fulltext index +@param word word to update +@param nodes nodes to update @return DB_SUCCESS or error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_optimize_write_word( -/*====================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: table of FTS index */ - fts_string_t* word, /*!< in: word data to write */ - ib_vector_t* nodes) /*!< in: the nodes to write */ +dberr_t fts_optimize_write_word(FTSQueryExecutor *executor, dict_index_t *index, + fts_string_t *word, ib_vector_t *nodes) noexcept { - ulint i; - pars_info_t* info; - que_t* graph; - ulint selected; - dberr_t error = DB_SUCCESS; - char table_name[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - ut_ad(fts_table->charset); - - pars_info_bind_varchar_literal( - info, "word", word->f_str, word->f_len); - - selected = fts_select_index(fts_table->charset, - word->f_str, word->f_len); - - fts_table->suffix = fts_get_suffix(selected); - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, - info, - "BEGIN DELETE FROM $table_name WHERE word = :word;"); - - error = fts_eval_sql(trx, graph); - - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ") during optimize," - " when deleting a word from the FTS index."; - } - - que_graph_free(graph); - graph = NULL; - - /* Even if the operation needs to be rolled back and redone, - we iterate over the nodes in order to free the ilist. */ - for (i = 0; i < ib_vector_size(nodes); ++i) { - - fts_node_t* node = (fts_node_t*) ib_vector_get(nodes, i); - - if (error == DB_SUCCESS) { - /* Skip empty node. */ - if (node->ilist == NULL) { - ut_ad(node->ilist_size == 0); - continue; - } - - error = fts_write_node( - trx, &graph, fts_table, word, node); + CHARSET_INFO *cs= fts_index_get_charset(index); + uint8_t selected= fts_select_index(cs, word->f_str, word->f_len); + fts_aux_data_t aux_data((const char*)word->f_str, word->f_len); + dberr_t err= executor->delete_aux_record(selected, &aux_data); + if (err != DB_SUCCESS) + { + sql_print_error("InnoDB: (%s) during optimize, when " + "deleting a word from the FTS index.", + ut_strerr(err)); + return err; + } - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ")" - " during optimize, while adding a" - " word to the FTS index."; - } - } + for (ulint i = 0; i < ib_vector_size(nodes); ++i) + { + fts_node_t* node = (fts_node_t*) ib_vector_get(nodes, i); + if (!node->ilist || node->ilist_size == 0) continue; - ut_free(node->ilist); - node->ilist = NULL; - node->ilist_size = node->ilist_size_alloc = 0; - } + fts_aux_data_t insert_data( + (const char*)word->f_str, word->f_len, + node->first_doc_id, node->last_doc_id, + static_cast(node->doc_count), node->ilist, + node->ilist_size); - if (graph != NULL) { - que_graph_free(graph); - } + err = executor->insert_aux_record(selected, &insert_data); + if (err != DB_SUCCESS) + { + sql_print_error("InnoDB: (%s) during optimize, when " + "inserting a word to the FTS index.", + ut_strerr(err)); + return err; + } + ut_free(node->ilist); + node->ilist= nullptr; + node->ilist_size= node->ilist_size_alloc= 0; + } - return(error); + return DB_SUCCESS; } /**********************************************************************//** @@ -1519,15 +1126,17 @@ fts_word_free( } /**********************************************************************//** -Optimize the word ilist and rewrite data to the FTS index. +Compact the nodes for a given word, the nodes passed in are +already optimized. @return status one of RESTART, EXIT, ERROR */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_optimize_compact( /*=================*/ + FTSQueryExecutor* executor, /*!< in: query executor */ fts_optimize_t* optim, /*!< in: optimize state data */ dict_index_t* index, /*!< in: current FTS being optimized */ - time_t start_time) /*!< in: optimize start time */ + time_t start_time) noexcept /*!< in: optimize start time */ { ulint i; dberr_t error = DB_SUCCESS; @@ -1536,7 +1145,6 @@ fts_optimize_compact( for (i = 0; i < size && error == DB_SUCCESS && !optim->done; ++i) { fts_word_t* word; ib_vector_t* nodes; - trx_t* trx = optim->trx; word = (fts_word_t*) ib_vector_get(optim->words, i); @@ -1547,13 +1155,13 @@ fts_optimize_compact( /* Update the data on disk. */ error = fts_optimize_write_word( - trx, &optim->fts_index_table, &word->text, nodes); + executor, index, &word->text, nodes); if (error == DB_SUCCESS) { /* Write the last word optimized to the config table, we use this value for restarting optimize. */ error = fts_config_set_index_value( - optim->trx, index, + executor, index, FTS_LAST_OPTIMIZED_WORD, &word->text); } @@ -1599,18 +1207,10 @@ fts_optimize_create( optim->trx = trx_create(); trx_start_internal(optim->trx); - optim->fts_common_table.table_id = table->id; - optim->fts_common_table.type = FTS_COMMON_TABLE; - optim->fts_common_table.table = table; - - optim->fts_index_table.table_id = table->id; - optim->fts_index_table.type = FTS_INDEX_TABLE; - optim->fts_index_table.table = table; - /* The common prefix for all this parent table's aux tables. */ char table_id[FTS_AUX_MIN_TABLE_ID_LENGTH]; const size_t table_id_len = 1 - + size_t(fts_get_table_id(&optim->fts_common_table, table_id)); + + size_t(fts_write_object_id(table->id, table_id)); dict_sys.freeze(SRW_LOCK_CALL); /* Include the separator as well. */ const size_t dbname_len = table->name.dblen() + 1; @@ -1680,175 +1280,136 @@ fts_optimize_free( mem_heap_free(heap); } -/**********************************************************************//** -Get the max time optimize should run in millisecs. +/** Get the max time optimize should run in millisecs. +@param executor query executor +@param table user table to be optimized @return max optimize time limit in millisecs. */ static -ulint -fts_optimize_get_time_limit( -/*========================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table) /*!< in: aux table */ +ulint fts_optimize_get_time_limit(FTSQueryExecutor *executor, + const dict_table_t *table) noexcept { - ulint time_limit = 0; - - fts_config_get_ulint( - trx, fts_table, - FTS_OPTIMIZE_LIMIT_IN_SECS, &time_limit); - - /* FIXME: This is returning milliseconds, while the variable - is being stored and interpreted as seconds! */ - return(time_limit * 1000); + ulint time_limit= 0; + fts_string_t value; + value.f_len= FTS_MAX_CONFIG_VALUE_LEN; + value.f_str= static_cast(ut_malloc_nokey(value.f_len + 1)); + dberr_t error= fts_config_get_value(executor, table, + FTS_OPTIMIZE_LIMIT_IN_SECS, &value); + if (error == DB_SUCCESS) + time_limit= strtoul(reinterpret_cast(value.f_str), nullptr, 10); + ut_free(value.f_str); + /* FIXME: This is returning milliseconds, while the variable + is being stored and interpreted as seconds! */ + return(time_limit * 1000); } -/**********************************************************************//** -Run OPTIMIZE on the given table. Note: this can take a very long time -(hours). */ +/** Run OPTIMIZE on the given table. Note: this can take a very +long time (hours). +@param executor query executor +@param optim optimize instance +@param index current fts being optimized +@param word starting word to optimize */ static -void -fts_optimize_words( -/*===============*/ - fts_optimize_t* optim, /*!< in: optimize instance */ - dict_index_t* index, /*!< in: current FTS being optimized */ - fts_string_t* word) /*!< in: the starting word to optimize */ +void fts_optimize_words(FTSQueryExecutor *executor, fts_optimize_t *optim, + dict_index_t *index, fts_string_t *word) noexcept { - fts_fetch_t fetch; - que_t* graph = NULL; - CHARSET_INFO* charset = optim->fts_index_table.charset; - - ut_a(!optim->done); - - /* Get the time limit from the config table. */ - fts_optimize_time_limit = fts_optimize_get_time_limit( - optim->trx, &optim->fts_common_table); - - const time_t start_time = time(NULL); - - /* Setup the callback to use for fetching the word ilist etc. */ - fetch.read_arg = optim->words; - fetch.read_record = fts_optimize_index_fetch_node; + ut_a(!optim->done); + /* Get the time limit from the config table. */ + fts_optimize_time_limit= + fts_optimize_get_time_limit(executor, index->table); + const time_t start_time= time(NULL); - while (!optim->done) { - dberr_t error; - trx_t* trx = optim->trx; - ulint selected; - - ut_a(ib_vector_size(optim->words) == 0); - - selected = fts_select_index(charset, word->f_str, word->f_len); - - /* Read the index records to optimize. */ - fetch.total_memory = 0; - error = fts_index_fetch_nodes( - trx, &graph, &optim->fts_index_table, word, - &fetch); - ut_ad(fetch.total_memory < fts_result_cache_limit); - - if (error == DB_SUCCESS) { - /* There must be some nodes to read. */ - ut_a(ib_vector_size(optim->words) > 0); - - /* Optimize the nodes that were read and write - back to DB. */ - error = fts_optimize_compact(optim, index, start_time); - - if (error == DB_SUCCESS) { - fts_sql_commit(optim->trx); - } else { - fts_sql_rollback(optim->trx); - } - } - - ib_vector_reset(optim->words); - - if (error == DB_SUCCESS) { - if (!optim->done) { - if (!fts_zip_read_word(optim->zip, word)) { - optim->done = TRUE; - } else if (selected - != fts_select_index( - charset, word->f_str, - word->f_len) - && graph) { - que_graph_free(graph); - graph = NULL; - } - } - } else if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "Lock wait timeout during optimize." - " Retrying!"; - - trx->error_state = DB_SUCCESS; - } else if (error == DB_DEADLOCK) { - ib::warn() << "Deadlock during optimize. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - optim->done = TRUE; /* Exit the loop. */ - } - } + while (!optim->done) + { + trx_t *trx= optim->trx; + ut_a(ib_vector_size(optim->words) == 0); + /* Read the index records to optimize. */ + dberr_t error= fts_index_fetch_nodes( + executor, index, word, optim->words, nullptr, AuxCompareMode::EQUAL); + if (error == DB_SUCCESS) + { + /* There must be some nodes to read. */ + ut_a(ib_vector_size(optim->words) > 0); + /* Optimize the nodes that were read and write back to DB. */ + error = fts_optimize_compact(executor, optim, index, start_time); + if (error == DB_SUCCESS) fts_sql_commit(optim->trx); + else fts_sql_rollback(optim->trx); + } + ib_vector_reset(optim->words); - if (graph != NULL) { - que_graph_free(graph); - } + if (error == DB_SUCCESS) + { + if (!optim->done && !fts_zip_read_word(optim->zip, word)) + optim->done= TRUE; + } + else if (error == DB_LOCK_WAIT_TIMEOUT) + { + sql_print_warning("InnoDB: Lock wait timeout during optimize. " + "Retrying!"); + trx->error_state= DB_SUCCESS; + } + else if (error == DB_DEADLOCK) + { + sql_print_warning("InnoDB: Deadlock during optimize. Retrying!"); + trx->error_state = DB_SUCCESS; + } + else optim->done = TRUE; + } } -/**********************************************************************//** -Optimize is complete. Set the completion time, and reset the optimize -start string for this FTS index to "". +/** Optimize is complete. Set the completion time, and reset the +optimize start string for this FTS index to "". +@param executor query executor +@param optim optimize instance +@param index table with one FTS index @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t -fts_optimize_index_completed( -/*=========================*/ - fts_optimize_t* optim, /*!< in: optimize instance */ - dict_index_t* index) /*!< in: table with one FTS index */ +fts_optimize_index_completed(FTSQueryExecutor *executor, + fts_optimize_t *optim, + dict_index_t *index) noexcept { - fts_string_t word; - dberr_t error; - byte buf[sizeof(ulint)]; - - /* If we've reached the end of the index then set the start - word to the empty string. */ - - word.f_len = 0; - word.f_str = buf; - *word.f_str = '\0'; - - error = fts_config_set_index_value( - optim->trx, index, FTS_LAST_OPTIMIZED_WORD, &word); - - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ") while updating" - " last optimized word!"; - } - - return(error); + fts_string_t word; + dberr_t error= DB_SUCCESS; + byte buf[sizeof(ulint)]; + /* If we've reached the end of the index then set the start + word to the empty string. */ + word.f_len= 0; + word.f_str= buf; + *word.f_str= '\0'; + + error= fts_config_set_index_value( + executor, index, FTS_LAST_OPTIMIZED_WORD, &word); + + if (UNIV_UNLIKELY(error != DB_SUCCESS)) + sql_print_error("InnoDB: (%s) while updating last optimized word!", + ut_strerr(error)); + return error; } -/**********************************************************************//** -Read the list of words from the FTS auxiliary index that will be -optimized in this pass. +/** Read the words that will be optimized in this pass. +@param executor query executor +@param optim optimize instance +@param index table with one FTS index +@param word buffer to use @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_optimize_index_read_words( -/*==========================*/ - fts_optimize_t* optim, /*!< in: optimize instance */ - dict_index_t* index, /*!< in: table with one FTS index */ - fts_string_t* word) /*!< in: buffer to use */ + FTSQueryExecutor* executor, + fts_optimize_t* optim, + dict_index_t* index, + fts_string_t* word) { dberr_t error = DB_SUCCESS; if (optim->del_list_regenerated) { word->f_len = 0; } else { - /* Get the last word that was optimized from the config table. */ error = fts_config_get_index_value( - optim->trx, index, FTS_LAST_OPTIMIZED_WORD, word); + executor, index, FTS_LAST_OPTIMIZED_WORD, word); } /* If record not found then we start from the top. */ @@ -1857,10 +1418,11 @@ fts_optimize_index_read_words( error = DB_SUCCESS; } + optim->index = index; while (error == DB_SUCCESS) { error = fts_index_fetch_words( - optim, word, fts_num_word_optimize); + executor, optim, word, fts_num_word_optimize); if (error == DB_SUCCESS) { /* Reset the last optimized word to '' if no @@ -1885,17 +1447,14 @@ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_optimize_index( /*===============*/ - fts_optimize_t* optim, /*!< in: optimize instance */ - dict_index_t* index) /*!< in: table with one FTS index */ + FTSQueryExecutor* executor, /*!< in: query executor */ + fts_optimize_t* optim, /*!< in: optimize instance */ + dict_index_t* index) /*!< in: table with one FTS index */ { fts_string_t word; dberr_t error; byte str[FTS_MAX_WORD_LEN + 1]; - /* Set the current index that we have to optimize. */ - optim->fts_index_table.index_id = index->id; - optim->fts_index_table.charset = fts_index_get_charset(index); - optim->done = FALSE; /* Optimize until !done */ /* We need to read the last word optimized so that we start from @@ -1909,7 +1468,7 @@ fts_optimize_index( memset(word.f_str, 0x0, word.f_len); /* Read the words that will be optimized in this pass. */ - error = fts_optimize_index_read_words(optim, index, &word); + error = fts_optimize_index_read_words(executor, optim, index, &word); if (error == DB_SUCCESS) { int zip_error; @@ -1929,7 +1488,7 @@ fts_optimize_index( optim->done = TRUE; } else { - fts_optimize_words(optim, index, &word); + fts_optimize_words(executor, optim, index, &word); } /* If we couldn't read any records then optimize is @@ -1938,7 +1497,8 @@ fts_optimize_index( completed. */ if (error == DB_SUCCESS && optim->zip->n_words == 0) { - error = fts_optimize_index_completed(optim, index); + error = fts_optimize_index_completed( + executor, optim, index); if (error == DB_SUCCESS) { ++optim->n_completed; @@ -1949,190 +1509,99 @@ fts_optimize_index( return(error); } -/**********************************************************************//** -Delete the document ids in the delete, and delete cache tables. +/** Purge the doc ids that are in the snapshot from +the master deleted table. +@param executor query executor +@param optim optimize instance @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_optimize_purge_deleted_doc_ids( -/*===============================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ +dberr_t fts_optimize_purge_deleted_doc_ids(FTSQueryExecutor *executor, + fts_optimize_t *optim) noexcept { - ulint i; - pars_info_t* info; - que_t* graph; - doc_id_t* update; - doc_id_t write_doc_id; - dberr_t error = DB_SUCCESS; - char deleted[MAX_FULL_NAME_LEN]; - char deleted_cache[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - ut_a(ib_vector_size(optim->to_delete->doc_ids) > 0); - - update = static_cast( - ib_vector_get(optim->to_delete->doc_ids, 0)); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, *update); - - /* This is required for the SQL parser to work. It must be able - to find the following variables. So we do it twice. */ - fts_bind_doc_id(info, "doc_id1", &write_doc_id); - fts_bind_doc_id(info, "doc_id2", &write_doc_id); - - /* Make sure the following two names are consistent with the name - used in the fts_delete_doc_ids_sql */ - optim->fts_common_table.suffix = fts_common_tables[3]; - fts_get_table_name(&optim->fts_common_table, deleted); - pars_info_bind_id(info, fts_common_tables[3], deleted); - - optim->fts_common_table.suffix = fts_common_tables[4]; - fts_get_table_name(&optim->fts_common_table, deleted_cache); - pars_info_bind_id(info, fts_common_tables[4], deleted_cache); - - graph = fts_parse_sql(NULL, info, fts_delete_doc_ids_sql); - - /* Delete the doc ids that were copied at the start. */ - for (i = 0; i < ib_vector_size(optim->to_delete->doc_ids); ++i) { - - update = static_cast(ib_vector_get( - optim->to_delete->doc_ids, i)); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, *update); - - fts_bind_doc_id(info, "doc_id1", &write_doc_id); - - fts_bind_doc_id(info, "doc_id2", &write_doc_id); - - error = fts_eval_sql(optim->trx, graph); - - // FIXME: Check whether delete actually succeeded! - if (error != DB_SUCCESS) { - - fts_sql_rollback(optim->trx); - break; - } - } - - que_graph_free(graph); + dberr_t error= DB_SUCCESS; + ut_a(ib_vector_size(optim->to_delete->doc_ids) > 0); + for (ulint i= 0; + i < ib_vector_size(optim->to_delete->doc_ids) && error != DB_SUCCESS; + ++i) + { + doc_id_t *update= + static_cast(ib_vector_get(optim->to_delete->doc_ids, i)); + error= executor->delete_common_record("DELETED", *update); + if (error == DB_SUCCESS) + error= executor->delete_common_record("DELETED_CACHE", *update); + } - return(error); + if (error != DB_SUCCESS) + fts_sql_rollback(optim->trx); + return error; } -/**********************************************************************//** -Delete the document ids in the pending delete, and delete tables. +/** Delete the document ids in the pending delete, and delete tables. +@param executor query executor +@param optim optimize instance @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_optimize_purge_deleted_doc_id_snapshot( -/*=======================================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ +dberr_t fts_optimize_purge_deleted_doc_id_snapshot(FTSQueryExecutor *executor, + fts_optimize_t *optim) noexcept { - dberr_t error; - que_t* graph; - pars_info_t* info; - char being_deleted[MAX_FULL_NAME_LEN]; - char being_deleted_cache[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - /* Make sure the following two names are consistent with the name - used in the fts_end_delete_sql */ - optim->fts_common_table.suffix = fts_common_tables[0]; - fts_get_table_name(&optim->fts_common_table, being_deleted); - pars_info_bind_id(info, fts_common_tables[0], being_deleted); - - optim->fts_common_table.suffix = fts_common_tables[1]; - fts_get_table_name(&optim->fts_common_table, being_deleted_cache); - pars_info_bind_id(info, fts_common_tables[1], being_deleted_cache); - - /* Delete the doc ids that were copied to delete pending state at - the start of optimize. */ - graph = fts_parse_sql(NULL, info, fts_end_delete_sql); - - error = fts_eval_sql(optim->trx, graph); - que_graph_free(graph); - - return(error); + dberr_t error= executor->delete_all_common_records("BEING_DELETED"); + if (error == DB_SUCCESS) + error= executor->delete_all_common_records("BEING_DELETED_CACHE"); + return error; } -/**********************************************************************//** -Copy the deleted doc ids that will be purged during this optimize run -to the being deleted FTS auxiliary tables. The transaction is committed -upon successfull copy and rolled back on DB_DUPLICATE_KEY error. +/** Check if there are records in BEING_DELETED table +@param executor query executor +@param optim optimize fts instance +@param n_rows number of rows exist in being_deleted table @return DB_SUCCESS if all OK */ static -ulint -fts_optimize_being_deleted_count( -/*=============================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ +dberr_t fts_optimize_being_deleted_count(FTSQueryExecutor *executor, + fts_optimize_t *optim, + ulint *n_rows) noexcept { - fts_table_t fts_table; - - FTS_INIT_FTS_TABLE(&fts_table, "BEING_DELETED", FTS_COMMON_TABLE, - optim->table); - - return(fts_get_rows_count(&fts_table)); + CommonTableReader reader; + dberr_t err= executor->read_all_common("BEING_DELETED", reader); + if (err == DB_SUCCESS) *n_rows= reader.get_doc_ids().size(); + return err; } -/*********************************************************************//** -Copy the deleted doc ids that will be purged during this optimize run -to the being deleted FTS auxiliary tables. The transaction is committed -upon successfull copy and rolled back on DB_DUPLICATE_KEY error. -@return DB_SUCCESS if all OK */ +/** Create a snapshot of deleted document IDs by moving them from +DELETED to BEING_DELETED and from DELETED_CACHE to +BEING_DELETED_CACHE. +@param executor query executor +@param optim optimize fts instance +@return DB_SUCCESS or error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_optimize_create_deleted_doc_id_snapshot( -/*========================================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ +dberr_t fts_optimize_create_deleted_doc_id_snapshot(FTSQueryExecutor *executor, + fts_optimize_t *optim) noexcept { - dberr_t error; - que_t* graph; - pars_info_t* info; - char being_deleted[MAX_FULL_NAME_LEN]; - char deleted[MAX_FULL_NAME_LEN]; - char being_deleted_cache[MAX_FULL_NAME_LEN]; - char deleted_cache[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - /* Make sure the following four names are consistent with the name - used in the fts_init_delete_sql */ - optim->fts_common_table.suffix = fts_common_tables[0]; - fts_get_table_name(&optim->fts_common_table, being_deleted); - pars_info_bind_id(info, fts_common_tables[0], being_deleted); - - optim->fts_common_table.suffix = fts_common_tables[3]; - fts_get_table_name(&optim->fts_common_table, deleted); - pars_info_bind_id(info, fts_common_tables[3], deleted); + dberr_t err= DB_SUCCESS; + CommonTableReader reader; - optim->fts_common_table.suffix = fts_common_tables[1]; - fts_get_table_name(&optim->fts_common_table, being_deleted_cache); - pars_info_bind_id(info, fts_common_tables[1], being_deleted_cache); + err= executor->read_all_common("DELETED", reader); + if (err != DB_SUCCESS && err != DB_RECORD_NOT_FOUND) return err; - optim->fts_common_table.suffix = fts_common_tables[4]; - fts_get_table_name(&optim->fts_common_table, deleted_cache); - pars_info_bind_id(info, fts_common_tables[4], deleted_cache); - - /* Move doc_ids that are to be deleted to state being deleted. */ - graph = fts_parse_sql(NULL, info, fts_init_delete_sql); - - error = fts_eval_sql(optim->trx, graph); - - que_graph_free(graph); + const auto& deleted_doc_ids = reader.get_doc_ids(); + for (doc_id_t doc_id : deleted_doc_ids) + { + err= executor->insert_common_record("BEING_DELETED", doc_id); + if (err != DB_SUCCESS) return err; + } - if (error != DB_SUCCESS) { - fts_sql_rollback(optim->trx); - } else { - fts_sql_commit(optim->trx); - } + reader.clear(); + err= executor->read_all_common("DELETED_CACHE", reader); + if (err != DB_SUCCESS && err != DB_RECORD_NOT_FOUND) return err; - optim->del_list_regenerated = TRUE; + const auto& deleted_cache_doc_ids= reader.get_doc_ids(); + for (doc_id_t doc_id : deleted_cache_doc_ids) + { + err= executor->insert_common_record("BEING_DELETED_CACHE", doc_id); + if (err != DB_SUCCESS) return err; + } - return(error); + optim->del_list_regenerated= TRUE; + return err; } /*********************************************************************//** @@ -2143,44 +1612,38 @@ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_optimize_read_deleted_doc_id_snapshot( /*======================================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ + FTSQueryExecutor* executor, /*!< in: FTS query executor */ + fts_optimize_t* optim) noexcept /*!< in: optimize instance */ { - dberr_t error; - - optim->fts_common_table.suffix = "BEING_DELETED"; - /* Read the doc_ids to delete. */ - error = fts_table_fetch_doc_ids( - optim->trx, &optim->fts_common_table, optim->to_delete); + dberr_t error = fts_table_fetch_doc_ids( + executor, optim->table, "BEING_DELETED", + optim->to_delete); if (error == DB_SUCCESS) { - optim->fts_common_table.suffix = "BEING_DELETED_CACHE"; - /* Read additional doc_ids to delete. */ error = fts_table_fetch_doc_ids( - optim->trx, &optim->fts_common_table, optim->to_delete); + executor, optim->table, "BEING_DELETED_CACHE", + optim->to_delete); } if (error != DB_SUCCESS) { - fts_doc_ids_free(optim->to_delete); optim->to_delete = NULL; } - return(error); } /*********************************************************************//** -Optimize all the FTS indexes, skipping those that have already been -optimized, since the FTS auxiliary indexes are not guaranteed to be -of the same cardinality. +Optimize the FTS indexes of a table. @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_optimize_indexes( /*=================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ + FTSQueryExecutor* executor, /*!< in: query executor */ + fts_optimize_t* optim) noexcept /*!< in: optimize instance */ { ulint i; dberr_t error = DB_SUCCESS; @@ -2192,7 +1655,17 @@ fts_optimize_indexes( index = static_cast( ib_vector_getp(fts->indexes, i)); - error = fts_optimize_index(optim, index); + + /* Open auxiliary tables for this index */ + error = executor->open_all_aux_tables(index); + if (error != DB_SUCCESS) { + break; + } + + error = fts_optimize_index(executor, optim, index); + if (error != DB_SUCCESS) { + break; + } } if (error == DB_SUCCESS) { @@ -2211,17 +1684,19 @@ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_optimize_purge_snapshot( /*========================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ + FTSQueryExecutor* executor, /*!< in: query executor */ + fts_optimize_t* optim) noexcept /*!< in: optimize instance */ { dberr_t error; /* Delete the doc ids from the master deleted tables, that were in the snapshot that was taken at the start of optimize. */ - error = fts_optimize_purge_deleted_doc_ids(optim); + error = fts_optimize_purge_deleted_doc_ids(executor, optim); if (error == DB_SUCCESS) { /* Destroy the deleted doc id snapshot. */ - error = fts_optimize_purge_deleted_doc_id_snapshot(optim); + error = fts_optimize_purge_deleted_doc_id_snapshot( + executor, optim); } if (error == DB_SUCCESS) { @@ -2233,6 +1708,7 @@ fts_optimize_purge_snapshot( return(error); } + /*********************************************************************//** Run OPTIMIZE on the given table by a background thread. @return DB_SUCCESS if all OK */ @@ -2259,7 +1735,7 @@ fts_optimize_table_bk( if (table->is_accessible() && table->fts && table->fts->cache && table->fts->cache->deleted >= FTS_OPTIMIZE_THRESHOLD) { - error = fts_optimize_table(table); + error = fts_optimize_table(table, fts_opt_thd); slot->last_run = time(NULL); @@ -2275,13 +1751,12 @@ fts_optimize_table_bk( return(error); } -/*********************************************************************//** -Run OPTIMIZE on the given table. +/** Run OPTIMIZE on the given table. +@param table table to be optimized +@param thd thread which executes optimize table @return DB_SUCCESS if all OK */ dberr_t -fts_optimize_table( -/*===============*/ - dict_table_t* table) /*!< in: table to optimiza */ +fts_optimize_table(dict_table_t *table, THD *thd) { ut_ad(!srv_read_only_mode || recv_sys.rpo); @@ -2289,20 +1764,61 @@ fts_optimize_table( return DB_READ_ONLY; } + /* Serialize concurrent fts_optimize_table() on the same table: + acquire MDL_EXCLUSIVE first so a second caller blocks here, then + downgrade to MDL_SHARED_UPGRADABLE so other operations can proceed + while optimization is in progress. */ + MDL_ticket* mdl_ticket = nullptr; + dict_sys.freeze(SRW_LOCK_CALL); + if (dict_acquire_mdl(table, thd, &mdl_ticket) + != table) { + dict_sys.unfreeze(); + if (mdl_ticket) { + thd->mdl_context.release_lock(mdl_ticket); + } + return DB_TABLE_NOT_FOUND; + } + dict_sys.unfreeze(); + + if (mdl_ticket) { + mdl_ticket->downgrade_lock(MDL_SHARED_UPGRADABLE); + } + dberr_t error = DB_SUCCESS; fts_optimize_t* optim = NULL; fts_t* fts = table->fts; optim = fts_optimize_create(table); + optim->trx->mysql_thd = thd; + + /* Create FTSQueryExecutor and open common tables */ + FTSQueryExecutor executor(optim->trx, table); + error = executor.open_all_deletion_tables(); + if (error != DB_SUCCESS) { +err_exit: + fts_optimize_free(optim); + if (mdl_ticket) { + thd->mdl_context.release_lock(mdl_ticket); + } + return error; + } + + error = executor.open_config_table(); + if (error) { goto err_exit; } + // FIXME: Call this only at the start of optimize, currently we // rely on DB_DUPLICATE_KEY to handle corrupting the snapshot. /* Check whether there are still records in BEING_DELETED table */ - if (fts_optimize_being_deleted_count(optim) == 0) { + ulint n_rows = 0; + error= fts_optimize_being_deleted_count(&executor, optim, &n_rows); + + if (error == DB_SUCCESS && n_rows == 0) { /* Take a snapshot of the deleted document ids, they are copied to the BEING_ tables. */ - error = fts_optimize_create_deleted_doc_id_snapshot(optim); + error = fts_optimize_create_deleted_doc_id_snapshot( + &executor, optim); } /* A duplicate error is OK, since we don't erase the @@ -2317,7 +1833,7 @@ fts_optimize_table( /* These document ids will be filtered out during the index optimization phase. They are in the snapshot that we took above, at the start of the optimize. */ - error = fts_optimize_read_deleted_doc_id_snapshot(optim); + error = fts_optimize_read_deleted_doc_id_snapshot(&executor, optim); if (error == DB_SUCCESS) { @@ -2328,7 +1844,7 @@ fts_optimize_table( /* We would do optimization only if there are deleted records to be cleaned up */ if (ib_vector_size(optim->to_delete->doc_ids) > 0) { - error = fts_optimize_indexes(optim); + error = fts_optimize_indexes(&executor, optim); } } else { @@ -2346,13 +1862,19 @@ fts_optimize_table( /* Purge the doc ids that were in the snapshot from the snapshot tables and the master deleted table. */ - error = fts_optimize_purge_snapshot(optim); + error = fts_optimize_purge_snapshot( + &executor, optim); } + } } fts_optimize_free(optim); + if (mdl_ticket) { + thd->mdl_context.release_lock(mdl_ticket); + } + return(error); } @@ -2432,8 +1954,8 @@ fts_optimize_remove_table( if (fts_opt_start_shutdown) { - ib::info() << "Try to remove table " << table->name - << " after FTS optimize thread exiting."; + sql_print_information("InnoDB: Try to remove table %s after FTS optimize " + "thread exiting.", table->name.m_name); while (fts_optimize_wq) std::this_thread::sleep_for(std::chrono::milliseconds(10)); return; @@ -2472,8 +1994,9 @@ fts_optimize_request_sync_table( /* FTS optimizer thread is already exited */ if (fts_opt_start_shutdown) { - ib::info() << "Try to sync table " << table->name - << " after FTS optimize thread exiting."; + sql_print_information("InnoDB: Try to sync table %s " + "after FTS optimize thread exiting.", + table->name.m_name); } else if (table->fts->sync_message) { /* If the table already has SYNC message in fts_optimize_wq queue then ignore it */ @@ -2615,8 +2138,8 @@ static void fts_optimize_sync_table(dict_table_t *table, bool process_message= false) { MDL_ticket* mdl_ticket= nullptr; - dict_table_t *sync_table= dict_acquire_mdl_shared(table, fts_opt_thd, - &mdl_ticket); + dict_table_t *sync_table= dict_acquire_mdl(table, fts_opt_thd, + &mdl_ticket); if (!sync_table) return; @@ -2770,7 +2293,7 @@ static void fts_optimize_callback(void *) pthread_cond_broadcast(&fts_opt_shutdown_cond); mysql_mutex_unlock(&fts_optimize_wq->mutex); - ib::info() << "FTS optimize thread exiting."; + sql_print_information("InnoDB: FTS optimize thread exiting."); } /**********************************************************************//** diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc index 0faf23aa6e97b..d921b34a9ce13 100644 --- a/storage/innobase/fts/fts0que.cc +++ b/storage/innobase/fts/fts0que.cc @@ -35,6 +35,7 @@ Completed 2011/7/10 Sunny and Jimmy Yang #include "fts0types.h" #include "fts0plugin.h" #include "fts0vlc.h" +#include "fts0exec.h" #include #include @@ -67,9 +68,6 @@ struct fts_query_t { trx_t* trx; /*!< The query transaction */ dict_index_t* index; /*!< The FTS index to search */ - /*!< FTS auxiliary common table def */ - - fts_table_t fts_common_table; fts_table_t fts_index_table;/*!< FTS auxiliary index table def */ @@ -149,6 +147,8 @@ struct fts_query_t { fts_ast_visit_sub_exp() */ st_mysql_ftparser* parser; /*!< fts plugin parser */ + + FTSQueryExecutor* executor; /*!< shared FTS query executor */ }; /** For phrase matching, first we collect the documents and the positions @@ -270,16 +270,6 @@ struct fts_word_freq_t { double idf; /*!< Inverse document frequency */ }; -/******************************************************************** -Callback function to fetch the rows in an FTS INDEX record. -@return always TRUE */ -static -ibool -fts_query_index_fetch_nodes( -/*========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg); /*!< in: pointer to ib_vector_t */ - /******************************************************************** Read and filter nodes. @return fts_node_t instance */ @@ -310,19 +300,155 @@ fts_ast_visit_sub_exp( fts_ast_callback visitor, void* arg); -#if 0 -/*****************************************************************//*** -Find a doc_id in a word's ilist. -@return TRUE if found. */ -static -ibool -fts_query_find_doc_id( -/*==================*/ - fts_select_t* select, /*!< in/out: search the doc id selected, - update the frequency if found. */ - void* data, /*!< in: doc id ilist */ - ulint len); /*!< in: doc id ilist size */ -#endif +/** Process query records for FTS queries. +@param rec record +@param index index +@param offsets record offsets +@param user_arg user argument +@return DB_SUCCESS to continue processing, DB_SUCCESS_LOCKED_REC to stop, or error code */ +static dberr_t node_query_processor( + const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets, void* user_arg) +{ + fts_query_t* query= static_cast(user_arg); + + /* Extract fields using rec_get_nth_field() */ + ulint word_len; + const byte* word_data= rec_get_nth_field(rec, offsets, 0, &word_len); + + if (!word_data || word_len == UNIV_SQL_NULL || word_len > FTS_MAX_WORD_LEN) + return DB_SUCCESS; + + ulint first_doc_id_len; + const byte* first_doc_id_data= rec_get_nth_field(rec, offsets, 1, &first_doc_id_len); + doc_id_t first_doc_id= fts_read_doc_id(first_doc_id_data); + + ulint last_doc_id_len; + const byte* last_doc_id_data= rec_get_nth_field(rec, offsets, 4, &last_doc_id_len); + doc_id_t last_doc_id= fts_read_doc_id(last_doc_id_data); + + ulint doc_count_len; + const byte* doc_count_data= rec_get_nth_field(rec, offsets, 5, &doc_count_len); + ulint doc_count= mach_read_from_4(doc_count_data); + + ulint ilist_len; + const byte* ilist_data= rec_get_nth_field(rec, offsets, 6, &ilist_len); + + /* Check if ilist is stored externally */ + bool ilist_external= rec_offs_nth_extern(offsets, 6); + mem_heap_t* ilist_heap= nullptr; + byte* external_ilist= nullptr; + + if (ilist_external && ilist_data && ilist_len != UNIV_SQL_NULL) + { + ilist_heap= mem_heap_create(ilist_len + 1000); + ulint external_len; + external_ilist= btr_copy_externally_stored_field( + &external_len, ilist_data, index->table->space->zip_size(), + ilist_len, ilist_heap); + if (external_ilist) + { + ilist_data= external_ilist; + ilist_len= external_len; + } + else + { + if (ilist_heap) mem_heap_free(ilist_heap); + return DB_SUCCESS; + } + } + + ut_a(query->cur_node->type == FTS_AST_TERM + || query->cur_node->type == FTS_AST_TEXT + || query->cur_node->type == FTS_AST_PARSER_PHRASE_LIST); + + fts_node_t node; + memset(&node, 0, sizeof(node)); + + fts_string_t term; + byte buf[FTS_MAX_WORD_LEN + 1]; + term.f_str= buf; + + /* Need to consider the wildcard search case, the word frequency + is created on the search string not the actual word. So we need + to assign the frequency on search string behalf. */ + if (query->cur_node->type == FTS_AST_TERM && query->cur_node->term.wildcard) + { + term.f_len = query->cur_node->term.ptr->len; + ut_ad(FTS_MAX_WORD_LEN >= term.f_len); + memcpy(term.f_str, query->cur_node->term.ptr->str, term.f_len); + } + else + { + term.f_len = word_len; + ut_ad(FTS_MAX_WORD_LEN >= word_len); + memcpy(term.f_str, word_data, word_len); + } + + /* Lookup the word in our rb tree, it must exist. */ + ib_rbt_bound_t parent; + int ret= rbt_search(query->word_freqs, &parent, &term); + + ut_a(ret == 0); + fts_word_freq_t* word_freq= rbt_value(fts_word_freq_t, parent.last); + bool skip = false; + + /* Use extracted field values */ + node.first_doc_id = first_doc_id; + skip= (query->oper == FTS_EXIST && query->upper_doc_id > 0 && + node.first_doc_id > query->upper_doc_id); + + node.last_doc_id = last_doc_id; + skip= (query->oper == FTS_EXIST && query->lower_doc_id > 0 && + node.last_doc_id < query->lower_doc_id); + + word_freq->doc_count += doc_count; + + dberr_t err= DB_SUCCESS; + + if (!skip) + { + if (ilist_data && ilist_len != UNIV_SQL_NULL && ilist_len > 0) + { + /* Process the ilist data (either inline or external) */ + query->error= fts_query_filter_doc_ids( + query, &word_freq->word, word_freq, &node, + const_cast(ilist_data), ilist_len, FALSE); + + if (query->error != DB_FTS_EXCEED_RESULT_CACHE_LIMIT) + err= query->error; + } + } + + if (ilist_heap) + mem_heap_free(ilist_heap); + return err; +} + +/* Comparator that signals how to treat the current record */ +RecordCompareAction doc_id_exact_match_comparator( + const dtuple_t *search_tuple, const rec_t *rec, + const dict_index_t *index, const rec_offs *offsets) +{ + const dfield_t* search_field= dtuple_get_nth_field(search_tuple, 0); + const byte* search_data= + static_cast(dfield_get_data(search_field)); + doc_id_t target_doc_id= fts_read_doc_id(search_data); + + ulint doc_col_pos= dict_col_get_clust_pos( + &index->table->cols[index->table->fts->doc_col], index); + + ulint len; + const byte* doc_id_data= rec_get_nth_field(rec, offsets, doc_col_pos, &len); + + if (len != sizeof(doc_id_t)) + return RecordCompareAction::SKIP; + + doc_id_t rec_doc_id= fts_read_doc_id(doc_id_data); + return rec_doc_id == target_doc_id + ? RecordCompareAction::PROCESS + : RecordCompareAction::STOP; +} /*************************************************************//** This function implements a simple "blind" query expansion search: @@ -371,107 +497,6 @@ fts_proximity_get_positions( fts_proximity_t* qualified_pos); /*!< out: the position info records ranges containing all matching words. */ -#if 0 -/******************************************************************** -Get the total number of words in a documents. */ -static -ulint -fts_query_terms_in_document( -/*========================*/ - /*!< out: DB_SUCCESS if all go well - else error code */ - fts_query_t* query, /*!< in: FTS query state */ - doc_id_t doc_id, /*!< in: the word to check */ - ulint* total); /*!< out: total words in document */ -#endif - -#if 0 -/*******************************************************************//** -Print the table used for calculating LCS. */ -static -void -fts_print_lcs_table( -/*================*/ - const ulint* table, /*!< in: array to print */ - ulint n_rows, /*!< in: total no. of rows */ - ulint n_cols) /*!< in: total no. of cols */ -{ - ulint i; - - for (i = 0; i < n_rows; ++i) { - ulint j; - - printf("\n"); - - for (j = 0; j < n_cols; ++j) { - - printf("%2lu ", FTS_ELEM(table, n_cols, i, j)); - } - } -} - -/******************************************************************** -Find the longest common subsequence between the query string and -the document. */ -static -ulint -fts_query_lcs( -/*==========*/ - /*!< out: LCS (length) between - two ilists */ - const ulint* p1, /*!< in: word positions of query */ - ulint len_p1, /*!< in: no. of elements in p1 */ - const ulint* p2, /*!< in: word positions within document */ - ulint len_p2) /*!< in: no. of elements in p2 */ -{ - int i; - ulint len = 0; - ulint r = len_p1; - ulint c = len_p2; - ulint size = (r + 1) * (c + 1) * sizeof(ulint); - ulint* table = (ulint*) ut_malloc_nokey(size); - - /* Traverse the table backwards, from the last row to the first and - also from the last column to the first. We compute the smaller - common subsequences first, then use the calculated values to determine - the longest common subsequence. The result will be in TABLE[0][0]. */ - for (i = r; i >= 0; --i) { - int j; - - for (j = c; j >= 0; --j) { - - if (p1[i] == (ulint) -1 || p2[j] == (ulint) -1) { - - FTS_ELEM(table, c, i, j) = 0; - - } else if (p1[i] == p2[j]) { - - FTS_ELEM(table, c, i, j) = FTS_ELEM( - table, c, i + 1, j + 1) + 1; - - } else { - - ulint value; - - value = ut_max( - FTS_ELEM(table, c, i + 1, j), - FTS_ELEM(table, c, i, j + 1)); - - FTS_ELEM(table, c, i, j) = value; - } - } - } - - len = FTS_ELEM(table, c, 0, 0); - - fts_print_lcs_table(table, r, c); - printf("\nLen=" ULINTPF "\n", len); - - ut_free(table); - - return(len); -} -#endif /*******************************************************************//** Compare two fts_ranking_t instance on their rank value and doc ids in @@ -1100,7 +1125,6 @@ fts_query_difference( const fts_string_t* token) /*!< in: token to search */ { ulint n_doc_ids= 0; - trx_t* trx = query->trx; dict_table_t* table = query->index->table; ut_a(query->oper == FTS_IGNORE); @@ -1112,10 +1136,8 @@ fts_query_difference( /* There is nothing we can substract from an empty set. */ if (query->doc_ids && !rbt_empty(query->doc_ids)) { ulint i; - fts_fetch_t fetch; const ib_vector_t* nodes; const fts_index_cache_t*index_cache; - que_t* graph = NULL; fts_cache_t* cache = table->fts->cache; dberr_t error; @@ -1153,21 +1175,21 @@ fts_query_difference( return(query->error); } - /* Setup the callback args for filtering and - consolidating the ilist. */ - fetch.read_arg = query; - fetch.read_record = fts_query_index_fetch_nodes; + AuxCompareMode compare_mode = AuxCompareMode::EQUAL; + if (query->cur_node->type == FTS_AST_TERM && + query->cur_node->term.wildcard) { + compare_mode = AuxCompareMode::LIKE; + } error = fts_index_fetch_nodes( - trx, &graph, &query->fts_index_table, token, &fetch); + query->executor, query->index, token, query, + node_query_processor, compare_mode); /* DB_FTS_EXCEED_RESULT_CACHE_LIMIT passed by 'query->error' */ ut_ad(!(query->error != DB_SUCCESS && error != DB_SUCCESS)); if (error != DB_SUCCESS) { query->error = error; } - - que_graph_free(graph); } /* The size can't increase. */ @@ -1194,7 +1216,6 @@ fts_query_intersect( fts_query_t* query, /*!< in: query instance */ const fts_string_t* token) /*!< in: the token to search */ { - trx_t* trx = query->trx; dict_table_t* table = query->index->table; ut_a(query->oper == FTS_EXIST); @@ -1204,10 +1225,8 @@ fts_query_intersect( if (!(rbt_empty(query->doc_ids) && query->multi_exist)) { ulint n_doc_ids = 0; ulint i; - fts_fetch_t fetch; const ib_vector_t* nodes; const fts_index_cache_t*index_cache; - que_t* graph = NULL; fts_cache_t* cache = table->fts->cache; dberr_t error; @@ -1278,13 +1297,15 @@ fts_query_intersect( return(query->error); } - /* Setup the callback args for filtering and - consolidating the ilist. */ - fetch.read_arg = query; - fetch.read_record = fts_query_index_fetch_nodes; + AuxCompareMode compare_mode = AuxCompareMode::EQUAL; + if (query->cur_node->type == FTS_AST_TERM && + query->cur_node->term.wildcard) { + compare_mode = AuxCompareMode::LIKE; + } error = fts_index_fetch_nodes( - trx, &graph, &query->fts_index_table, token, &fetch); + query->executor, query->index, token, query, + node_query_processor, compare_mode); /* DB_FTS_EXCEED_RESULT_CACHE_LIMIT passed by 'query->error' */ ut_ad(!(query->error != DB_SUCCESS && error != DB_SUCCESS)); @@ -1292,8 +1313,6 @@ fts_query_intersect( query->error = error; } - que_graph_free(graph); - if (query->error == DB_SUCCESS) { /* Make the intesection (rb tree) the current doc id set and free the old set. */ @@ -1371,10 +1390,7 @@ fts_query_union( fts_query_t* query, /*!< in: query instance */ fts_string_t* token) /*!< in: token to search */ { - fts_fetch_t fetch; ulint n_doc_ids = 0; - trx_t* trx = query->trx; - que_t* graph = NULL; dberr_t error; ut_a(query->oper == FTS_NONE || query->oper == FTS_DECR_RATING || @@ -1390,14 +1406,16 @@ fts_query_union( fts_query_cache(query, token); - /* Setup the callback args for filtering and - consolidating the ilist. */ - fetch.read_arg = query; - fetch.read_record = fts_query_index_fetch_nodes; + AuxCompareMode compare_mode = AuxCompareMode::EQUAL; + if (query->cur_node->type == FTS_AST_TERM && + query->cur_node->term.wildcard) { + compare_mode = AuxCompareMode::LIKE; + } /* Read the nodes from disk. */ error = fts_index_fetch_nodes( - trx, &graph, &query->fts_index_table, token, &fetch); + query->executor, query->index, token, query, node_query_processor, + compare_mode); /* DB_FTS_EXCEED_RESULT_CACHE_LIMIT passed by 'query->error' */ ut_ad(!(query->error != DB_SUCCESS && error != DB_SUCCESS)); @@ -1405,8 +1423,6 @@ fts_query_union( query->error = error; } - que_graph_free(graph); - if (query->error == DB_SUCCESS) { /* The size can't decrease. */ @@ -1923,476 +1939,229 @@ fts_query_match_phrase( return(phrase->found); } -/*****************************************************************//** -Callback function to fetch and search the document. +/** Callback function to fetch and search the document. +@param fts_index fulltext index +@param doc_id document id +@param arg user argument +@param expansion Expansion document @return whether the phrase is found */ static -ibool -fts_query_fetch_document( -/*=====================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts_doc_t* */ +dberr_t fts_query_fetch_document(dict_index_t *fts_index, + doc_id_t doc_id, + void *arg, bool expansion= false) { - - que_node_t* exp; - sel_node_t* node = static_cast(row); - fts_phrase_t* phrase = static_cast(user_arg); - ulint prev_len = 0; - ulint total_len = 0; - byte* document_text = NULL; - - exp = node->select_list; - - phrase->found = FALSE; - - /* For proximity search, we will need to get the whole document - from all fields, so first count the total length of the document - from all the fields */ - if (phrase->proximity_pos) { - while (exp) { - ulint field_len; - dfield_t* dfield = que_node_get_val(exp); - byte* data = static_cast( - dfield_get_data(dfield)); - - if (dfield_is_ext(dfield)) { - ulint local_len = dfield_get_len(dfield); - - local_len -= BTR_EXTERN_FIELD_REF_SIZE; - - field_len = mach_read_from_4( - data + local_len + BTR_EXTERN_LEN + 4); - } else { - field_len = dfield_get_len(dfield); - } - - if (field_len != UNIV_SQL_NULL) { - total_len += field_len + 1; - } - - exp = que_node_get_next(exp); - } - - document_text = static_cast(mem_heap_zalloc( - phrase->heap, total_len)); - - if (!document_text) { - return(FALSE); - } - } - - exp = node->select_list; - - while (exp) { - dfield_t* dfield = que_node_get_val(exp); - byte* data = static_cast( - dfield_get_data(dfield)); - ulint cur_len; - - if (dfield_is_ext(dfield)) { - data = btr_copy_externally_stored_field( - &cur_len, data, phrase->zip_size, - dfield_get_len(dfield), phrase->heap); - } else { - cur_len = dfield_get_len(dfield); - } - - if (cur_len != UNIV_SQL_NULL && cur_len != 0) { - if (phrase->proximity_pos) { - ut_ad(prev_len + cur_len <= total_len); - memcpy(document_text + prev_len, data, cur_len); - } else { - /* For phrase search */ - phrase->found = - fts_query_match_phrase( - phrase, - static_cast(data), - cur_len, prev_len, - phrase->heap); - } - - /* Document positions are calculated from the beginning - of the first field, need to save the length for each - searched field to adjust the doc position when search - phrases. */ - prev_len += cur_len + 1; - } - - if (phrase->found) { - break; - } - - exp = que_node_get_next(exp); - } - - if (phrase->proximity_pos) { - ut_ad(prev_len <= total_len); - - phrase->found = fts_proximity_is_word_in_range( - phrase, document_text, total_len); - } - - return(phrase->found); -} - -#if 0 -/******************************************************************** -Callback function to check whether a record was found or not. */ -static -ibool -fts_query_select( -/*=============*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts_doc_t* */ -{ - int i; - que_node_t* exp; - sel_node_t* node = row; - fts_select_t* select = user_arg; - - ut_a(select->word_freq); - ut_a(select->word_freq->doc_freqs); - - exp = node->select_list; - - for (i = 0; exp && !select->found; ++i) { - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - - switch (i) { - case 0: /* DOC_COUNT */ - if (len != UNIV_SQL_NULL && len != 0) { - - select->word_freq->doc_count += - mach_read_from_4(data); - } - break; - - case 1: /* ILIST */ - if (len != UNIV_SQL_NULL && len != 0) { - - fts_query_find_doc_id(select, data, len); - } - break; - - default: - ut_error; - } - - exp = que_node_get_next(exp); - } - - return(FALSE); -} - -/******************************************************************** -Read the rows from the FTS index, that match word and where the -doc id is between first and last doc id. -@return DB_SUCCESS if all go well else error code */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_query_find_term( -/*================*/ - fts_query_t* query, /*!< in: FTS query state */ - que_t** graph, /*!< in: prepared statement */ - const fts_string_t* word, /*!< in: the word to fetch */ - doc_id_t doc_id, /*!< in: doc id to match */ - ulint* min_pos,/*!< in/out: pos found must be - greater than this minimum value. */ - ibool* found) /*!< out: TRUE if found else FALSE */ -{ - pars_info_t* info; - dberr_t error; - fts_select_t select; - doc_id_t match_doc_id; - trx_t* trx = query->trx; - char table_name[MAX_FULL_NAME_LEN]; - - trx->op_info = "fetching FTS index matching nodes"; - - if (*graph) { - info = (*graph)->info; - } else { - ulint selected; - - info = pars_info_create(); - - selected = fts_select_index(*word->f_str); - query->fts_index_table.suffix = fts_get_suffix(selected); - - fts_get_table_name(&query->fts_index_table, table_name); - pars_info_bind_id(info, "index_table_name", table_name); - } - - select.found = FALSE; - select.doc_id = doc_id; - select.min_pos = *min_pos; - select.word_freq = fts_query_add_word_freq(query, word->f_str); - - pars_info_bind_function(info, "my_func", fts_query_select, &select); - pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &match_doc_id, doc_id); - - fts_bind_doc_id(info, "min_doc_id", &match_doc_id); - - fts_bind_doc_id(info, "max_doc_id", &match_doc_id); - - if (!*graph) { - - *graph = fts_parse_sql( - &query->fts_index_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT doc_count, ilist\n" - " FROM $index_table_name\n" - " WHERE word LIKE :word AND" - " first_doc_id <= :min_doc_id AND" - " last_doc_id >= :max_doc_id\n" - " ORDER BY first_doc_id;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - } - - for (;;) { - error = fts_eval_sql(trx, *graph); - - if (error == DB_SUCCESS) { - - break; /* Exit the loop. */ - } else { - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading FTS" - " index. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << error - << " while reading FTS index."; - - break; /* Exit the loop. */ - } - } - } - - /* Value to return */ - *found = select.found; - - if (*found) { - *min_pos = select.min_pos; - } - - return(error); -} - -/******************************************************************** -Callback aggregator for int columns. */ -static -ibool -fts_query_sum( -/*==========*/ - /*!< out: always returns TRUE */ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: ulint* */ -{ - - que_node_t* exp; - sel_node_t* node = row; - ulint* total = user_arg; - - exp = node->select_list; - - while (exp) { - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - - if (len != UNIV_SQL_NULL && len != 0) { - *total += mach_read_from_4(data); - } - - exp = que_node_get_next(exp); - } - - return(TRUE); -} - -/******************************************************************** -Calculate the total documents that contain a particular word (term). -@return DB_SUCCESS if all go well else error code */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_query_total_docs_containing_term( -/*=================================*/ - fts_query_t* query, /*!< in: FTS query state */ - const fts_string_t* word, /*!< in: the word to check */ - ulint* total) /*!< out: documents containing word */ -{ - pars_info_t* info; - dberr_t error; - que_t* graph; - ulint selected; - trx_t* trx = query->trx; - char table_name[MAX_FULL_NAME_LEN] - - trx->op_info = "fetching FTS index document count"; - - *total = 0; - - info = pars_info_create(); - - pars_info_bind_function(info, "my_func", fts_query_sum, total); - pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len); - - selected = fts_select_index(*word->f_str); - - query->fts_index_table.suffix = fts_get_suffix(selected); - - fts_get_table_name(&query->fts_index_table, table_name); - - pars_info_bind_id(info, "index_table_name", table_name); - - graph = fts_parse_sql( - &query->fts_index_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT doc_count\n" - " FROM $index_table_name\n" - " WHERE word = :word" - " ORDER BY first_doc_id;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - for (;;) { - error = fts_eval_sql(trx, graph); - - if (error == DB_SUCCESS) { - - break; /* Exit the loop. */ - } else { - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading FTS" - " index. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << error - << " while reading FTS index."; - - break; /* Exit the loop. */ - } - } - } - - que_graph_free(graph); - - return(error); -} - -/******************************************************************** -Get the total number of words in a documents. -@return DB_SUCCESS if all go well else error code */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_query_terms_in_document( -/*========================*/ - fts_query_t* query, /*!< in: FTS query state */ - doc_id_t doc_id, /*!< in: the word to check */ - ulint* total) /*!< out: total words in document */ -{ - pars_info_t* info; - dberr_t error; - que_t* graph; - doc_id_t read_doc_id; - trx_t* trx = query->trx; - char table_name[MAX_FULL_NAME_LEN]; - - trx->op_info = "fetching FTS document term count"; - - *total = 0; - - info = pars_info_create(); - - pars_info_bind_function(info, "my_func", fts_query_sum, total); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &read_doc_id, doc_id); - fts_bind_doc_id(info, "doc_id", &read_doc_id); - - query->fts_index_table.suffix = "DOC_ID"; - - fts_get_table_name(&query->fts_index_table, table_name); - - pars_info_bind_id(info, "index_table_name", table_name); - - graph = fts_parse_sql( - &query->fts_index_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT count\n" - " FROM $index_table_name\n" - " WHERE doc_id = :doc_id" - " BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - for (;;) { - error = fts_eval_sql(trx, graph); - - if (error == DB_SUCCESS) { - - break; /* Exit the loop. */ - } else { - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading FTS" - " doc id table. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << error << " while reading FTS" - " doc id table."; - - break; /* Exit the loop. */ - } - } - } - - que_graph_free(graph); - - return(error); -} + trx_t *trx= trx_create(); + trx->op_info= "fetching FTS document for query"; + dict_table_t *user_table= fts_index->table; + dict_index_t *fts_doc_id_index= user_table->fts_doc_id_index; + dict_index_t *clust_index= dict_table_get_first_index(user_table); + ut_a(user_table->fts->doc_col != ULINT_UNDEFINED); + ut_a(fts_doc_id_index); + + QueryExecutor executor(trx); + + /* Map FTS index columns to clustered index field positions */ + ulint *clust_field_nos= static_cast( + mem_heap_alloc(executor.get_heap(), + fts_index->n_user_defined_cols * sizeof(ulint))); + + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + dict_field_t* fts_field= dict_index_get_nth_field(fts_index, i); + clust_field_nos[i]= dict_col_get_index_pos(fts_field->col, clust_index); + } + dfield_t fields[1]; + dtuple_t search_tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N #endif + }; + dict_index_copy_types(&search_tuple, fts_doc_id_index, 1); + dfield_t* dfield= dtuple_get_nth_field(&search_tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id((byte*) &write_doc_id, doc_id); + dfield_set_data(dfield, &write_doc_id, sizeof(write_doc_id)); + + auto process_expansion_doc= [arg, fts_index, + clust_field_nos](const rec_t* rec, + const dict_index_t *index, + const rec_offs *offsets)-> dberr_t + { + fts_doc_t *result_doc= static_cast(arg); + fts_doc_t doc; + CHARSET_INFO *doc_charset= result_doc->charset; + fts_doc_init(&doc); + doc.found= TRUE; + + ulint doc_len= 0; + ulint field_no= 0; + + /* Process each indexed column content */ + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, + col_pos, &field_len); + + /* NULL column */ + if (field_len == UNIV_SQL_NULL) { + continue; + } + + /* Determine document charset from column if not provided */ + if (!doc_charset) + { + const dict_field_t* ifield= dict_index_get_nth_field(fts_index, i); + doc_charset= fts_get_charset(ifield->col->prtype); + } + + doc.charset= doc_charset; + /* Skip columns stored externally, as in fts_query_expansion_fetch_doc */ + if (rec_offs_nth_extern(offsets, col_pos)) { + continue; + } + + /* Use inline field data */ + doc.text.f_n_char= 0; + doc.text.f_str= const_cast(field_data); + doc.text.f_len= field_len; + + if (field_no == 0) + fts_tokenize_document(&doc, result_doc, result_doc->parser); + else + fts_tokenize_document_next(&doc, doc_len, result_doc, + result_doc->parser); + + /* Next field offset: add 1 for separator if more fields follow */ + doc_len+= ((i + 1) < fts_index->n_user_defined_cols) + ? field_len + 1 + : field_len; + + field_no++; + } + + ut_ad(doc_charset); + if (!result_doc->charset) { + result_doc->charset= doc_charset; + } + + fts_doc_free(&doc); + + return DB_SUCCESS; /* continue */ + }; + + auto process_doc_query= [arg, user_table, fts_index, + clust_field_nos](const rec_t* rec, + const dict_index_t* index, + const rec_offs* offsets) -> dberr_t + { + ulint prev_len= 0; + ulint total_len= 0; + byte *document_text= nullptr; + + fts_phrase_t *phrase= static_cast(arg); + phrase->found= FALSE; + + /* Extract doc_id from the clustered index record */ + ulint doc_col_pos= dict_col_get_index_pos( + &user_table->cols[user_table->fts->doc_col], index); + + ulint len; + rec_get_nth_field_offs(offsets, doc_col_pos, &len); + if (len != sizeof(doc_id_t)) + return DB_ERROR; + + /* For proximity search, first count total document length */ + if (phrase->proximity_pos) + { + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, + col_pos, &field_len); + if (rec_offs_nth_extern(offsets, col_pos)) + { + ulint local_len= field_len; + local_len-= BTR_EXTERN_FIELD_REF_SIZE; + field_len= mach_read_from_4( + field_data + local_len + BTR_EXTERN_LEN + 4); + } + if (field_len != UNIV_SQL_NULL) + total_len+= field_len + 1; + } + + document_text= + static_cast(mem_heap_zalloc(phrase->heap, total_len)); + if (!document_text) + return DB_ERROR; + } + + /* Process each indexed column content */ + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, + col_pos, &field_len); + byte* data= const_cast(field_data); + ulint cur_len; + + if (rec_offs_nth_extern(offsets, col_pos)) + { + data= btr_copy_externally_stored_field( + &cur_len, const_cast(field_data), phrase->zip_size, + field_len, phrase->heap); + } + else cur_len= field_len; + if (cur_len != UNIV_SQL_NULL && cur_len != 0) + { + if (phrase->proximity_pos) + { + ut_ad(prev_len + cur_len <= total_len); + memcpy(document_text + prev_len, data, cur_len); + } + else + { + /* For phrase search */ + phrase->found= fts_query_match_phrase( + phrase, data, cur_len, prev_len, phrase->heap); + } + + /* Document positions are calculated from the beginning + of the first field, need to save the length for each + searched field to adjust the doc position when search + phrases. */ + prev_len+= cur_len + 1; + } + + if (phrase->found) + break; + } + + if (phrase->proximity_pos) + { + ut_ad(prev_len <= total_len); + phrase->found= fts_proximity_is_word_in_range( + phrase, document_text, total_len); + } + + return phrase->found ? DB_SUCCESS_LOCKED_REC : DB_SUCCESS; /* Stop if found, continue if not found */ + }; + + RecordProcessor proc= expansion + ? RecordProcessor(process_expansion_doc) + : RecordProcessor(process_doc_query); + RecordCallback reader(proc, doc_id_exact_match_comparator); + dberr_t err= DB_SUCCESS; + err= executor.read_by_index(user_table, fts_doc_id_index, + &search_tuple, PAGE_CUR_GE, reader, true); + trx_commit_for_mysql(trx); + trx->free(); + if (err == DB_RECORD_NOT_FOUND) err= DB_SUCCESS; + return err; +} /*****************************************************************//** Retrieve the document and match the phrase tokens. @@ -2421,9 +2190,8 @@ fts_query_match_document( *found = phrase.found = FALSE; - error = fts_doc_fetch_by_doc_id( - get_doc, match->doc_id, NULL, FTS_FETCH_DOC_BY_ID_EQUAL, - fts_query_fetch_document, &phrase); + error = fts_query_fetch_document( + get_doc->index_cache->index, match->doc_id, &phrase); if (UNIV_UNLIKELY(error != DB_SUCCESS)) { ib::error() << "(" << error << ") matching document."; @@ -2468,21 +2236,14 @@ fts_query_is_in_proximity_range( phrase.proximity_pos = qualified_pos; phrase.found = FALSE; - err = fts_doc_fetch_by_doc_id( - &get_doc, match[0]->doc_id, NULL, FTS_FETCH_DOC_BY_ID_EQUAL, - fts_query_fetch_document, &phrase); + err = fts_query_fetch_document( + get_doc.index_cache->index, match[0]->doc_id, &phrase); if (UNIV_UNLIKELY(err != DB_SUCCESS)) { ib::error() << "(" << err << ") in verification" " phase of proximity search"; } - /* Free the prepared statement. */ - if (get_doc.get_document_graph) { - que_graph_free(get_doc.get_document_graph); - get_doc.get_document_graph = NULL; - } - mem_heap_free(phrase.heap); return(err == DB_SUCCESS && phrase.found); @@ -2591,6 +2352,7 @@ fts_query_phrase_split( ulint len = 0; ulint cur_pos = 0; fts_ast_node_t* term_node = NULL; + CHARSET_INFO* cs = fts_index_get_charset(query->index); if (node->type == FTS_AST_TEXT) { phrase.f_str = node->text.ptr->str; @@ -2614,7 +2376,7 @@ fts_query_phrase_split( } cur_len = innobase_mysql_fts_get_token( - query->fts_index_table.charset, + cs, reinterpret_cast(phrase.f_str) + cur_pos, reinterpret_cast(phrase.f_str) @@ -2637,7 +2399,7 @@ fts_query_phrase_split( result_str.f_str = term_node->term.ptr->str; result_str.f_len = term_node->term.ptr->len; result_str.f_n_char = fts_get_token_size( - query->fts_index_table.charset, + cs, reinterpret_cast(result_str.f_str), result_str.f_len); @@ -2654,8 +2416,7 @@ fts_query_phrase_split( if (fts_check_token( &result_str, - cache->stopword_info.cached_stopword, - query->fts_index_table.charset)) { + cache->stopword_info.cached_stopword, cs)) { /* Add the word to the RB tree so that we can calculate its frequency within a document. */ fts_query_add_word_freq(query, token); @@ -2717,10 +2478,7 @@ fts_query_phrase_search( /* Ignore empty strings. */ if (num_token > 0) { fts_string_t* token = NULL; - fts_fetch_t fetch; - trx_t* trx = query->trx; fts_ast_oper_t oper = query->oper; - que_t* graph = NULL; ulint i; dberr_t error; @@ -2755,11 +2513,6 @@ fts_query_phrase_search( } } - /* Setup the callback args for filtering and consolidating - the ilist. */ - fetch.read_arg = query; - fetch.read_record = fts_query_index_fetch_nodes; - for (i = 0; i < num_token; i++) { /* Search for the first word from the phrase. */ token = static_cast( @@ -2770,9 +2523,15 @@ fts_query_phrase_search( query->matched = query->match_array[i]; } + AuxCompareMode compare_mode = AuxCompareMode::EQUAL; + if (query->cur_node->type == FTS_AST_TERM && + query->cur_node->term.wildcard) { + compare_mode = AuxCompareMode::LIKE; + } + error = fts_index_fetch_nodes( - trx, &graph, &query->fts_index_table, - token, &fetch); + query->executor, query->index, token, query, + node_query_processor, compare_mode); /* DB_FTS_EXCEED_RESULT_CACHE_LIMIT passed by 'query->error' */ ut_ad(!(query->error != DB_SUCCESS && error != DB_SUCCESS)); @@ -2780,9 +2539,6 @@ fts_query_phrase_search( query->error = error; } - que_graph_free(graph); - graph = NULL; - fts_query_cache(query, token); if (!(query->flags & FTS_PHRASE) @@ -2923,12 +2679,11 @@ fts_query_get_token( if (node->term.wildcard) { - token->f_str = static_cast(ut_malloc_nokey(str_len + 2)); - token->f_len = str_len + 1; + token->f_str = static_cast(ut_malloc_nokey(str_len + 1)); + token->f_len = str_len; memcpy(token->f_str, node->term.ptr->str, str_len); - token->f_str[str_len] = '%'; token->f_str[token->f_len] = 0; new_ptr = token->f_str; @@ -3109,78 +2864,6 @@ fts_ast_visit_sub_exp( DBUG_RETURN(error); } -#if 0 -/*****************************************************************//*** -Check if the doc id exists in the ilist. -@return TRUE if doc id found */ -static -ulint -fts_query_find_doc_id( -/*==================*/ - fts_select_t* select, /*!< in/out: contains the doc id to - find, we update the word freq if - document found */ - void* data, /*!< in: doc id ilist */ - ulint len) /*!< in: doc id ilist size */ -{ - byte* ptr = data; - doc_id_t doc_id = 0; - ulint decoded = 0; - - /* Decode the ilist and search for selected doc_id. We also - calculate the frequency of the word in the document if found. */ - while (decoded < len && !select->found) { - ulint freq = 0; - ulint min_pos = 0; - ulint last_pos = 0; - ulint pos = fts_decode_vlc(&ptr); - - /* Add the delta. */ - doc_id += pos; - - while (*ptr) { - ++freq; - last_pos += fts_decode_vlc(&ptr); - - /* Only if min_pos is not set and the current - term exists in a position greater than the - min_pos of the previous term. */ - if (min_pos == 0 && last_pos > select->min_pos) { - min_pos = last_pos; - } - } - - /* Skip the end of word position marker. */ - ++ptr; - - /* Bytes decoded so far. */ - decoded = ptr - (byte*) data; - - /* A word may exist in the document but we only consider a - match if it exists in a position that is greater than the - position of the previous term. */ - if (doc_id == select->doc_id && min_pos > 0) { - fts_doc_freq_t* doc_freq; - - /* Add the doc id to the doc freq rb tree, if - the doc id doesn't exist it will be created. */ - doc_freq = fts_query_add_doc_freq( - select->word_freq->doc_freqs, doc_id); - - /* Avoid duplicating the frequency tally */ - if (doc_freq->freq == 0) { - doc_freq->freq = freq; - } - - select->found = TRUE; - select->min_pos = min_pos; - } - } - - return(select->found); -} -#endif - /*****************************************************************//** Read and filter nodes. @return DB_SUCCESS if all go well, @@ -3301,156 +2984,6 @@ fts_query_filter_doc_ids( } } -/*****************************************************************//** -Read the FTS INDEX row. -@return DB_SUCCESS if all go well. */ -static -dberr_t -fts_query_read_node( -/*================*/ - fts_query_t* query, /*!< in: query instance */ - const fts_string_t* word, /*!< in: current word */ - que_node_t* exp) /*!< in: query graph node */ -{ - int i; - int ret; - fts_node_t node; - ib_rbt_bound_t parent; - fts_word_freq_t* word_freq; - ibool skip = FALSE; - fts_string_t term; - byte buf[FTS_MAX_WORD_LEN + 1]; - dberr_t error = DB_SUCCESS; - - ut_a(query->cur_node->type == FTS_AST_TERM - || query->cur_node->type == FTS_AST_TEXT - || query->cur_node->type == FTS_AST_PARSER_PHRASE_LIST); - - memset(&node, 0, sizeof(node)); - term.f_str = buf; - - /* Need to consider the wildcard search case, the word frequency - is created on the search string not the actual word. So we need - to assign the frequency on search string behalf. */ - if (query->cur_node->type == FTS_AST_TERM - && query->cur_node->term.wildcard) { - - term.f_len = query->cur_node->term.ptr->len; - ut_ad(FTS_MAX_WORD_LEN >= term.f_len); - memcpy(term.f_str, query->cur_node->term.ptr->str, term.f_len); - } else { - term.f_len = word->f_len; - ut_ad(FTS_MAX_WORD_LEN >= word->f_len); - memcpy(term.f_str, word->f_str, word->f_len); - } - - /* Lookup the word in our rb tree, it must exist. */ - ret = rbt_search(query->word_freqs, &parent, &term); - - ut_a(ret == 0); - - word_freq = rbt_value(fts_word_freq_t, parent.last); - - /* Start from 1 since the first column has been read by the caller. - Also, we rely on the order of the columns projected, to filter - out ilists that are out of range and we always want to read - the doc_count irrespective of the suitability of the row. */ - - for (i = 1; exp && !skip; exp = que_node_get_next(exp), ++i) { - - dfield_t* dfield = que_node_get_val(exp); - byte* data = static_cast( - dfield_get_data(dfield)); - ulint len = dfield_get_len(dfield); - - ut_a(len != UNIV_SQL_NULL); - - /* Note: The column numbers below must match the SELECT. */ - - switch (i) { - case 1: /* DOC_COUNT */ - word_freq->doc_count += mach_read_from_4(data); - break; - - case 2: /* FIRST_DOC_ID */ - node.first_doc_id = fts_read_doc_id(data); - - /* Skip nodes whose doc ids are out range. */ - if (query->oper == FTS_EXIST - && query->upper_doc_id > 0 - && node.first_doc_id > query->upper_doc_id) { - skip = TRUE; - } - break; - - case 3: /* LAST_DOC_ID */ - node.last_doc_id = fts_read_doc_id(data); - - /* Skip nodes whose doc ids are out range. */ - if (query->oper == FTS_EXIST - && query->lower_doc_id > 0 - && node.last_doc_id < query->lower_doc_id) { - skip = TRUE; - } - break; - - case 4: /* ILIST */ - - error = fts_query_filter_doc_ids( - query, &word_freq->word, word_freq, - &node, data, len, FALSE); - - break; - - default: - ut_error; - } - } - - if (!skip) { - /* Make sure all columns were read. */ - - ut_a(i == 5); - } - - return error; -} - -/*****************************************************************//** -Callback function to fetch the rows in an FTS INDEX record. -@return always returns TRUE */ -static -ibool -fts_query_index_fetch_nodes( -/*========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to fts_fetch_t */ -{ - fts_string_t key; - sel_node_t* sel_node = static_cast(row); - fts_fetch_t* fetch = static_cast(user_arg); - fts_query_t* query = static_cast(fetch->read_arg); - que_node_t* exp = sel_node->select_list; - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint dfield_len = dfield_get_len(dfield); - - key.f_str = static_cast(data); - key.f_len = dfield_len; - - ut_a(dfield_len <= FTS_MAX_WORD_LEN); - - /* Note: we pass error out by 'query->error' */ - query->error = fts_query_read_node(query, &key, que_node_get_next(exp)); - - if (query->error != DB_SUCCESS) { - ut_ad(query->error == DB_FTS_EXCEED_RESULT_CACHE_LIMIT); - return(FALSE); - } else { - return(TRUE); - } -} - /*****************************************************************//** Calculate the inverse document frequency (IDF) for all the terms. */ static @@ -3837,7 +3370,7 @@ fts_query_parse( memset(&state, 0x0, sizeof(state)); - state.charset = query->fts_index_table.charset; + state.charset = fts_index_get_charset(query->index); DBUG_EXECUTE_IF("fts_instrument_query_disable_parser", query->parser = NULL;); @@ -3850,7 +3383,7 @@ fts_query_parse( } else { /* Setup the scanner to use, this depends on the mode flag. */ state.lexer = fts_lexer_create(mode, query_str, query_len); - state.charset = query->fts_index_table.charset; + state.charset = fts_index_get_charset(query->index); error = fts_parse(&state); fts_lexer_free(state.lexer); state.lexer = NULL; @@ -3931,10 +3464,6 @@ fts_query( query.deleted = fts_doc_ids_create(); query.cur_node = NULL; - query.fts_common_table.type = FTS_COMMON_TABLE; - query.fts_common_table.table_id = index->table->id; - query.fts_common_table.table = index->table; - charset = fts_index_get_charset(index); query.fts_index_table.type = FTS_INDEX_TABLE; @@ -3963,25 +3492,35 @@ fts_query( query.total_docs = dict_table_get_n_rows(index->table); - query.fts_common_table.suffix = "DELETED"; + /* Create single FTSQueryExecutor for entire query lifecycle */ + query.executor = new FTSQueryExecutor(query_trx, index->table); - /* Read the deleted doc_ids, we need these for filtering. */ - error = fts_table_fetch_doc_ids( - NULL, &query.fts_common_table, query.deleted); - - if (error != DB_SUCCESS) { - goto func_exit; + /* Prefetch all auxiliary and common tables to avoid repeated dict_sys.latch acquisitions */ + error = query.executor->open_all_aux_tables(index); + if (error == DB_SUCCESS) { + error = query.executor->open_all_deletion_tables(); } - query.fts_common_table.suffix = "DELETED_CACHE"; + if (error == DB_SUCCESS) { + /* Read the deleted doc_ids, we need these for filtering. */ + error = fts_table_fetch_doc_ids( + query.executor, index->table, "DELETED", + query.deleted); + } - error = fts_table_fetch_doc_ids( - NULL, &query.fts_common_table, query.deleted); + if (error == DB_SUCCESS) { + error = fts_table_fetch_doc_ids( + query.executor, index->table, "DELETED_CACHE", + query.deleted); + } if (error != DB_SUCCESS) { + query_trx->rollback(); goto func_exit; } + trx_commit_for_mysql(query_trx); + /* Get the deleted doc ids that are in the cache. */ fts_cache_append_deleted_doc_ids( index->table->fts->cache, query.deleted->doc_ids); @@ -4080,6 +3619,12 @@ fts_query( ut_free(lc_query_str); func_exit: + /* Clean up the dynamically allocated executor */ + if (query.executor) { + delete query.executor; + query.executor = nullptr; + } + fts_query_free(&query); query_trx->free(); @@ -4206,10 +3751,8 @@ fts_expand_query( fetch the original document and parse them. Future optimization could be done here if we support some forms of document-to-word mapping */ - fts_doc_fetch_by_doc_id(NULL, ranking->doc_id, index, - FTS_FETCH_DOC_BY_ID_EQUAL, - fts_query_expansion_fetch_doc, - &result_doc); + fts_query_fetch_document(index, ranking->doc_id, + &result_doc, true); /* Estimate memory used, see fts_process_token and fts_token_t. We ignore token size here. */ diff --git a/storage/innobase/fts/fts0sql.cc b/storage/innobase/fts/fts0sql.cc deleted file mode 100644 index 781d15f2befb0..0000000000000 --- a/storage/innobase/fts/fts0sql.cc +++ /dev/null @@ -1,208 +0,0 @@ -/***************************************************************************** - -Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2019, 2021, MariaDB Corporation. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/**************************************************//** -@file fts/fts0sql.cc -Full Text Search functionality. - -Created 2007-03-27 Sunny Bains -*******************************************************/ - -#include "que0que.h" -#include "trx0roll.h" -#include "pars0pars.h" -#include "dict0dict.h" -#include "fts0types.h" -#include "fts0priv.h" - -/** SQL statements for creating the ancillary FTS tables. */ - -/** Preamble to all SQL statements. */ -static const char* fts_sql_begin= - "PROCEDURE P() IS\n"; - -/** Postamble to non-committing SQL statements. */ -static const char* fts_sql_end= - "\n" - "END;\n"; - -/******************************************************************//** -Get the table id. -@return number of bytes written */ -int -fts_get_table_id( -/*=============*/ - const fts_table_t* - fts_table, /*!< in: FTS Auxiliary table */ - char* table_id) /*!< out: table id, must be at least - FTS_AUX_MIN_TABLE_ID_LENGTH bytes - long */ -{ - int len; - - ut_a(fts_table->table != NULL); - - switch (fts_table->type) { - case FTS_COMMON_TABLE: - len = fts_write_object_id(fts_table->table_id, table_id); - break; - - case FTS_INDEX_TABLE: - - len = fts_write_object_id(fts_table->table_id, table_id); - - table_id[len] = '_'; - ++len; - table_id += len; - - len += fts_write_object_id(fts_table->index_id, table_id); - break; - - default: - ut_error; - } - - ut_a(len >= 16); - ut_a(len < FTS_AUX_MIN_TABLE_ID_LENGTH); - - return(len); -} - -/** Construct the name of an internal FTS table for the given table. -@param[in] fts_table metadata on fulltext-indexed table -@param[out] table_name a name up to MAX_FULL_NAME_LEN -@param[in] dict_locked whether dict_sys.latch is being held */ -void fts_get_table_name(const fts_table_t* fts_table, char* table_name, - bool dict_locked) -{ - if (!dict_locked) { - dict_sys.freeze(SRW_LOCK_CALL); - } - ut_ad(dict_sys.frozen()); - /* Include the separator as well. */ - const size_t dbname_len = fts_table->table->name.dblen() + 1; - ut_ad(dbname_len > 1); - memcpy(table_name, fts_table->table->name.m_name, dbname_len); - if (!dict_locked) { - dict_sys.unfreeze(); - } - memcpy(table_name += dbname_len, "FTS_", 4); - table_name += 4; - table_name += fts_get_table_id(fts_table, table_name); - *table_name++ = '_'; - strcpy(table_name, fts_table->suffix); -} - -/******************************************************************//** -Parse an SQL string. -@return query graph */ -que_t* -fts_parse_sql( -/*==========*/ - fts_table_t* fts_table, /*!< in: FTS auxiliary table info */ - pars_info_t* info, /*!< in: info struct, or NULL */ - const char* sql) /*!< in: SQL string to evaluate */ -{ - char* str; - que_t* graph; - ibool dict_locked; - - str = ut_str3cat(fts_sql_begin, sql, fts_sql_end); - - dict_locked = (fts_table && fts_table->table->fts - && fts_table->table->fts->dict_locked); - - if (!dict_locked) { - /* The InnoDB SQL parser is not re-entrant. */ - dict_sys.lock(SRW_LOCK_CALL); - } - - graph = pars_sql(info, str); - ut_a(graph); - - if (!dict_locked) { - dict_sys.unlock(); - } - - ut_free(str); - - return(graph); -} - -/******************************************************************//** -Evaluate an SQL query graph. -@return DB_SUCCESS or error code */ -dberr_t -fts_eval_sql( -/*=========*/ - trx_t* trx, /*!< in: transaction */ - que_t* graph) /*!< in: Query graph to evaluate */ -{ - que_thr_t* thr; - - graph->trx = trx; - - ut_a(thr = que_fork_start_command(graph)); - - que_run_threads(thr); - - return(trx->error_state); -} - -/******************************************************************//** -Construct the column specification part of the SQL string for selecting the -indexed FTS columns for the given table. Adds the necessary bound -ids to the given 'info' and returns the SQL string. Examples: - -One indexed column named "text": - - "$sel0", - info/ids: sel0 -> "text" - -Two indexed columns named "subject" and "content": - - "$sel0, $sel1", - info/ids: sel0 -> "subject", sel1 -> "content", -@return heap-allocated WHERE string */ -const char* -fts_get_select_columns_str( -/*=======================*/ - dict_index_t* index, /*!< in: index */ - pars_info_t* info, /*!< in/out: parser info */ - mem_heap_t* heap) /*!< in: memory heap */ -{ - ulint i; - const char* str = ""; - - for (i = 0; i < index->n_user_defined_cols; i++) { - char* sel_str; - - dict_field_t* field = dict_index_get_nth_field(index, i); - - sel_str = mem_heap_printf(heap, "sel%lu", (ulong) i); - - /* Set copy_name to TRUE since it's dynamic. */ - pars_info_bind_id(info, sel_str, field->name); - - str = mem_heap_printf( - heap, "%s%s$%s", str, (*str) ? ", " : "", sel_str); - } - - return(str); -} diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 5c2ea50657521..3ff699b289994 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -11465,6 +11465,81 @@ ha_innobase::update_create_info( } } +/** Update FTS stopword configuration when reload is enabled +@param table FTS table +@param trx transaction +@param thd current thread +@param new_stopword_table stopword table name +@param new_stopword_is_on stopword enable setting +@return TRUE if success */ +static +bool innobase_fts_update_stopword_config(dict_table_t *table, trx_t *trx, + THD *thd, + const char *new_stopword_table, + bool new_stopword_is_on) +{ + ut_ad(dict_sys.locked()); + /* Create FTSQueryExecutor for config operations */ + FTSQueryExecutor executor(trx, table); + dberr_t error= DB_SUCCESS; + fts_string_t str; + + /* Load CONFIG table directly since we already hold dict_sys.lock() */ + char config_table_name[MAX_FULL_NAME_LEN]; + fts_table_t fts_table; + FTS_INIT_FTS_TABLE(&fts_table, nullptr, FTS_COMMON_TABLE, table); + fts_table.suffix = "CONFIG"; + fts_get_table_name(&fts_table, config_table_name, true); + const span config_name{ + config_table_name, strlen(config_table_name)}; + dict_table_t *config_table= dict_sys.load_table(config_name); + + if (config_table == nullptr) + return false; + + /* Assign CONFIG table directly to executor for optimized access */ + executor.set_config_table(config_table); + + /* Update FTS_USE_STOPWORD setting in CONFIG table */ + ulint use_stopword = (ulint) new_stopword_is_on; + fts_cache_t *cache= table->fts->cache; + error= fts_config_set_ulint( + &executor, table, FTS_USE_STOPWORD, use_stopword); + + if (error != DB_SUCCESS) + return false; + + if (!use_stopword) + { + cache->stopword_info.status = STOPWORD_OFF; + goto cleanup; + } + + /* Validate and update FTS_STOPWORD_TABLE_NAME in CONFIG + table if provided */ + if (new_stopword_table && + fts_load_user_stopword( + &executor, table->fts, new_stopword_table, + &table->fts->cache->stopword_info)) + { + /* Stopword table is valid, update CONFIG table */ + str.f_n_char = 0; + str.f_str = (byte*) new_stopword_table; + str.f_len = strlen(new_stopword_table); + + error = fts_config_set_value(&executor, table, FTS_STOPWORD_TABLE_NAME, &str); + } + else fts_load_default_stopword(&cache->stopword_info); +cleanup: + if (!cache->stopword_info.cached_stopword) + cache->stopword_info.cached_stopword= + rbt_create_arg_cmp( + sizeof(fts_tokenizer_word_t), innobase_fts_text_cmp, + &my_charset_latin1); + + return true; +} + /*****************************************************************//** Initialize the table FTS stopword list @return TRUE if success */ @@ -11487,9 +11562,24 @@ innobase_fts_load_stopword( } table->fts->dict_locked= true; - bool success= fts_load_stopword(table, trx, stopword_table, - THDVAR(thd, ft_enable_stopword), false); + bool own_trx= (trx == nullptr); + if (own_trx) + { + trx= trx_create(); + trx_start_internal(trx); + } + bool success= innobase_fts_update_stopword_config( + table, trx, thd, stopword_table, + THDVAR(thd, ft_enable_stopword)); + if (own_trx) + { + if (success) + trx_commit_for_mysql(trx); + else trx->rollback(); + trx->free(); + } table->fts->dict_locked= false; + DBUG_EXECUTE_IF("fts_load_stopword_fail", success= false;); return success; } @@ -15246,7 +15336,7 @@ ha_innobase::optimize( if (m_prebuilt->table->fts && m_prebuilt->table->fts->cache && m_prebuilt->table->space) { fts_sync_table(m_prebuilt->table); - fts_optimize_table(m_prebuilt->table); + fts_optimize_table(m_prebuilt->table, thd); } try_alter = false; } diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 5d260c7f0b072..9f68928c735b4 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -49,6 +49,7 @@ Created July 18, 2007 Vasil Dimov #include "fts0types.h" #include "fts0opt.h" #include "fts0priv.h" +#include "fts0exec.h" #include "btr0btr.h" #include "page0zip.h" #include "fil0fil.h" @@ -2204,7 +2205,6 @@ i_s_fts_deleted_generic_fill( Field** fields; TABLE* table = (TABLE*) tables->table; trx_t* trx; - fts_table_t fts_table; fts_doc_ids_t* deleted; dict_table_t* user_table; @@ -2235,12 +2235,12 @@ i_s_fts_deleted_generic_fill( trx = trx_create(); trx->op_info = "Select for FTS DELETE TABLE"; - FTS_INIT_FTS_TABLE(&fts_table, - (being_deleted) ? "BEING_DELETED" : "DELETED", - FTS_COMMON_TABLE, user_table); - - fts_table_fetch_doc_ids(trx, &fts_table, deleted); + FTSQueryExecutor executor(trx, user_table); + fts_table_fetch_doc_ids( + &executor, user_table, + being_deleted ? "BEING_DELETED" : "DELETED", deleted); + trx_commit_for_mysql(trx); dict_table_close(user_table, thd, mdl_ticket); trx->free(); @@ -2660,100 +2660,26 @@ struct st_maria_plugin i_s_innodb_ft_index_cache = MariaDB_PLUGIN_MATURITY_STABLE }; -/*******************************************************************//** -Go through a FTS index auxiliary table, fetch its rows and fill +/** Go through a FTS index auxiliary table, fetch its rows and fill FTS word cache structure. +@param executor FTS query executor +@param reader record reader for processing auxiliary table records +@param selected auxiliary index +@param word word to select @return DB_SUCCESS on success, otherwise error code */ static -dberr_t -i_s_fts_index_table_fill_selected( -/*==============================*/ - dict_index_t* index, /*!< in: FTS index */ - ib_vector_t* words, /*!< in/out: vector to hold - fetched words */ - ulint selected, /*!< in: selected FTS index */ - fts_string_t* word) /*!< in: word to select */ +dberr_t i_s_fts_index_table_fill_selected( + FTSQueryExecutor *executor, AuxRecordReader &reader, + uint8_t selected, fts_string_t *word) noexcept { - pars_info_t* info; - fts_table_t fts_table; - trx_t* trx; - que_t* graph; - dberr_t error; - fts_fetch_t fetch; - char table_name[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - fetch.read_arg = words; - fetch.read_record = fts_optimize_index_fetch_node; - fetch.total_memory = 0; - - DBUG_EXECUTE_IF("fts_instrument_result_cache_limit", - fts_result_cache_limit = 8192; - ); + DBUG_EXECUTE_IF("fts_instrument_result_cache_limit", + fts_result_cache_limit = 8192;); - trx = trx_create(); - - trx->op_info = "fetching FTS index nodes"; - - pars_info_bind_function(info, "my_func", fetch.read_record, &fetch); - pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len); - - FTS_INIT_INDEX_TABLE(&fts_table, fts_get_suffix(selected), - FTS_INDEX_TABLE, index); - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - &fts_table, info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT word, doc_count, first_doc_id, last_doc_id," - " ilist\n" - " FROM $table_name WHERE word >= :word;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - for (;;) { - error = fts_eval_sql(trx, graph); - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - - break; - } else { - fts_sql_rollback(trx); - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "Lock wait timeout reading" - " FTS index. Retrying!"; + dberr_t error= executor->read_aux(selected, word, reader); - trx->error_state = DB_SUCCESS; - } else { - ib::error() << "Error occurred while reading" - " FTS index: " << error; - break; - } - } - } - - que_graph_free(graph); - - trx->free(); - - if (fetch.total_memory >= fts_result_cache_limit) { - error = DB_FTS_EXCEED_RESULT_CACHE_LIMIT; - } - - return(error); + if (error == DB_RECORD_NOT_FOUND) + error = DB_SUCCESS; + return error; } /*******************************************************************//** @@ -2890,6 +2816,90 @@ i_s_fts_index_table_fill_one_fetch( DBUG_RETURN(ret); } +/** Read words from a single auxiliary index with retry logic +@param executor FTS query executor +@param reader record reader +@param selected auxiliary index number +@param thd thread +@param tables tables to fill +@param conv_str conversion buffer +@param heap memory heap +@param words words vector +@return 0 on success, 1 on failure */ +static +int i_s_fts_read_aux_index_words( + FTSQueryExecutor *executor, AuxRecordReader &reader, + uint8_t selected, THD *thd, TABLE_LIST *tables, + fts_string_t *conv_str, mem_heap_t *heap, + ib_vector_t *words) noexcept +{ + fts_string_t word; + bool has_more= false; + int ret= 0; + + word.f_str= NULL; + word.f_len= 0; + word.f_n_char= 0; + + do + { + /* Fetch from index with retry logic for lock timeouts */ + for (;;) + { + trx_t *trx= executor->trx(); + dberr_t error= i_s_fts_index_table_fill_selected( + executor, reader, selected, &word); + + if (UNIV_LIKELY(error == DB_SUCCESS || + error == DB_FTS_EXCEED_RESULT_CACHE_LIMIT)) + { + fts_sql_commit(trx); + has_more= error == DB_FTS_EXCEED_RESULT_CACHE_LIMIT; + break; + } + else + { + fts_sql_rollback(trx); + if (error == DB_LOCK_WAIT_TIMEOUT) + { + sql_print_warning("InnoDB: Lock wait timeout" + " while reading FTS index. Retrying!"); + trx->error_state= DB_SUCCESS; + /* Clear words and retry */ + i_s_fts_index_table_free_one_fetch(words); + reader.reset_total_memory(); /* Reset memory counter on retry */ + } + else + { + sql_print_error("InnoDB: Error occurred while reading" + " FTS index: %s", ut_strerr(error)); + i_s_fts_index_table_free_one_fetch(words); + return 1; + } + } + } + + if (has_more) + { + /* Prepare start point for next fetch */ + fts_word_t *last_word= static_cast(ib_vector_last(words)); + ut_ad(last_word != NULL); + fts_string_dup(&word, &last_word->text, heap); + reader.reset_total_memory(); /* Reset memory counter for next fetch */ + } + + /* Fill into tables */ + ret= i_s_fts_index_table_fill_one_fetch( + fts_index_get_charset(const_cast(executor->index())), + thd, tables, words,conv_str, has_more); + i_s_fts_index_table_free_one_fetch(words); + if (ret != 0) + return ret; + } while (has_more); + + return 0; +} + /*******************************************************************//** Go through a FTS index and its auxiliary tables, fetch rows in each table and fill INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE. @@ -2905,8 +2915,6 @@ i_s_fts_index_table_fill_one_index( { ib_vector_t* words; mem_heap_t* heap; - CHARSET_INFO* index_charset; - dberr_t error; int ret = 0; DBUG_ENTER("i_s_fts_index_table_fill_one_index"); @@ -2917,59 +2925,34 @@ i_s_fts_index_table_fill_one_index( words = ib_vector_create(ib_heap_allocator_create(heap), sizeof(fts_word_t), 256); - index_charset = fts_index_get_charset(index); - - /* Iterate through each auxiliary table as described in - fts_index_selector */ - for (ulint selected = 0; selected < FTS_NUM_AUX_INDEX; selected++) { - fts_string_t word; - bool has_more = false; - - word.f_str = NULL; - word.f_len = 0; - word.f_n_char = 0; + trx_t* trx= trx_create(); + trx->op_info= "fetching FTS index nodes"; + FTSQueryExecutor executor(trx, index->table); + dberr_t error= executor.open_all_aux_tables(index); - do { - /* Fetch from index */ - error = i_s_fts_index_table_fill_selected( - index, words, selected, &word); + if (error) return 1; + ulint total_memory= 0; + AuxRecordReader reader(words, &total_memory); - if (error == DB_SUCCESS) { - has_more = false; - } else if (error == DB_FTS_EXCEED_RESULT_CACHE_LIMIT) { - has_more = true; - } else { - i_s_fts_index_table_free_one_fetch(words); - ret = 1; - goto func_exit; - } - - if (has_more) { - fts_word_t* last_word; + for (uint8_t selected = 0; selected < FTS_NUM_AUX_INDEX; selected++) { + reader.reset_total_memory(); + ret = i_s_fts_read_aux_index_words( + &executor, reader, selected, + thd, tables, conv_str, heap, words); - /* Prepare start point for next fetch */ - last_word = static_cast(ib_vector_last(words)); - ut_ad(last_word != NULL); - fts_string_dup(&word, &last_word->text, heap); - } - - /* Fill into tables */ - ret = i_s_fts_index_table_fill_one_fetch( - index_charset, thd, tables, words, conv_str, - has_more); - i_s_fts_index_table_free_one_fetch(words); - - if (ret != 0) { - goto func_exit; - } - } while (has_more); + if (ret != 0) { + goto func_exit; + } } func_exit: + trx->free(); mem_heap_free(heap); DBUG_RETURN(ret); } + + /*******************************************************************//** Fill the dynamic table INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE @return 0 on success, 1 on failure */ @@ -3116,7 +3099,6 @@ i_s_fts_config_fill( Field** fields; TABLE* table = (TABLE*) tables->table; trx_t* trx; - fts_table_t fts_table; dict_table_t* user_table; ulint i = 0; dict_index_t* index = NULL; @@ -3149,9 +3131,7 @@ i_s_fts_config_fill( trx = trx_create(); trx->op_info = "Select for FTS CONFIG TABLE"; - - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, user_table); - + FTSQueryExecutor executor(trx, user_table); if (!ib_vector_is_empty(user_table->fts->indexes)) { index = (dict_index_t*) ib_vector_getp_const( user_table->fts->indexes, 0); @@ -3178,7 +3158,7 @@ i_s_fts_config_fill( key_name = (char*) fts_config_key[i]; } - fts_config_get_value(trx, &fts_table, key_name, &value); + fts_config_get_value(&executor, user_table, key_name, &value); if (allocated) { ut_free(key_name); diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index 53f88cc8ca1f5..9387adb234f24 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -346,7 +346,7 @@ btr_cur_del_mark_set_clust_rec( que_thr_t* thr, /*!< in: query thread */ const dtuple_t* entry, /*!< in: dtuple for the deleting record */ mtr_t* mtr) /*!< in/out: mini-transaction */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); + MY_ATTRIBUTE((nonnull(1,2,3,4,5,7), warn_unused_result)); /*************************************************************//** Tries to compress a page of the tree if it seems useful. It is assumed that mtr holds an x-latch on the tree and on the cursor page. To avoid diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index 5bb79e286f0da..9fd24e3a4c069 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -93,35 +93,38 @@ enum dict_table_op_t { DICT_TABLE_OP_OPEN_ONLY_IF_CACHED }; -/** Acquire MDL shared for the table name. +/** Acquire MDL for the table name. +By default, acquires MDL_SHARED lock. Use exclusive=true for MDL_EXCLUSIVE. @tparam trylock whether to use non-blocking operation -@param[in,out] table table object +@tparam exclusive Used to take MDL_EXCLUSIVE lock (default: false, MDL_SHARED) @param[in,out] thd background thread @param[out] mdl mdl ticket @param[in] table_op operation to perform when opening -@return table object after locking MDL shared +@return table object after locking MDL @retval NULL if the table is not readable, or if trylock && MDL blocked */ -template +template dict_table_t* -dict_acquire_mdl_shared(dict_table_t *table, - THD *thd, - MDL_ticket **mdl, - dict_table_op_t table_op= DICT_TABLE_OP_NORMAL); +dict_acquire_mdl(dict_table_t *table, + THD *thd, + MDL_ticket **mdl, + dict_table_op_t table_op= DICT_TABLE_OP_NORMAL); -/** Acquire MDL shared for the table name. +/** Acquire MDL for the table name. +By default, acquires MDL_SHARED lock. Use exclusive=true for MDL_EXCLUSIVE. @tparam trylock whether to use non-blocking operation +@tparam exclusive Used to take MDL_EXCLUSIVE lock (default: false, MDL_SHARED) @param[in,out] table table object @param[in,out] mdl_context MDL context @param[out] mdl MDL ticket @param[in] table_op operation to perform when opening -@return table object after locking MDL shared +@return table object after locking MDL @retval nullptr if the table is not readable, or if trylock && MDL blocked */ -template +template __attribute__((nonnull, warn_unused_result)) dict_table_t* -dict_acquire_mdl_shared(dict_table_t *table, - MDL_context *mdl_context, MDL_ticket **mdl, - dict_table_op_t table_op); +dict_acquire_mdl(dict_table_t *table, + MDL_context *mdl_context, MDL_ticket **mdl, + dict_table_op_t table_op); /** Look up a table by numeric identifier. @param[in] table_id table identifier diff --git a/storage/innobase/include/fts0exec.h b/storage/innobase/include/fts0exec.h new file mode 100644 index 0000000000000..b8defbdcd26f7 --- /dev/null +++ b/storage/innobase/include/fts0exec.h @@ -0,0 +1,422 @@ +/***************************************************************************** + +Copyright (c) 2025, MariaDB PLC. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file include/fts0exec.h +FTS Query Builder - Abstraction layer for FTS operations + +Created 2025/10/30 +*******************************************************/ + +#pragma once + +#include "row0query.h" +#include "fts0fts.h" +#include "fts0types.h" +#include "fts0opt.h" +#include "fts0ast.h" +#include + +/** Structure to represent FTS auxiliary table data for insertion */ +struct fts_aux_data_t +{ + /** + CREATE TABLE $FTS_PREFIX_INDEX_[1-6]( + word VARCHAR(FTS_MAX_WORD_LEN), + first_doc_id INT NOT NULL, + last_doc_id UNSIGNED NOT NULL, + doc_count UNSIGNED INT NOT NULL, + ilist VARBINARY NOT NULL, + UNIQUE CLUSTERED INDEX ON (word, first_doc_id)); + */ + const char *word; + ulint word_len; + doc_id_t first_doc_id; + doc_id_t last_doc_id; + uint32_t doc_count; + const byte *ilist; + ulint ilist_len; + + fts_aux_data_t(const char *w, ulint w_len) + : word(w), word_len(w_len) + { + first_doc_id= last_doc_id= doc_count= 0; + ilist= nullptr; + ilist_len= 0; + } + + fts_aux_data_t(const char* w, ulint w_len, doc_id_t first_id, + doc_id_t last_id, uint32_t d_count, const byte *il, + ulint il_len) + : word(w), word_len(w_len), first_doc_id(first_id), + last_doc_id(last_id), doc_count(d_count), ilist(il), + ilist_len(il_len) {} +}; + +/** FTS deletion table types for m_common_tables array indexing */ +enum class FTSDeletionTable : uint8_t +{ + DELETED= 0, + DELETED_CACHE= 1, + BEING_DELETED= 2, + BEING_DELETED_CACHE= 3, + MAX_DELETION_TABLES= 4 +}; + +/** Helper to convert FTSDeletionTable to array index */ +constexpr uint8_t to_index(FTSDeletionTable table_type) noexcept +{ + return static_cast(table_type); +} + +/** Number of deletion tables */ +constexpr uint8_t NUM_DELETION_TABLES = to_index(FTSDeletionTable::MAX_DELETION_TABLES); + +/** Abstraction over QueryExecutor for FTS auxiliary/common tables. +Handles table open/lock and provides typed helpers to insert, +delete and read records in FTS INDEX_1..INDEX_6 +and deletion tables (DELETED, BEING_DELETED, etc.) */ +class FTSQueryExecutor +{ +private: + QueryExecutor m_executor; + const dict_index_t *m_index; + const dict_table_t *const m_table; + /* FTS Auxiliary table pointers */ + dict_table_t *m_aux_tables[6]={nullptr}; + /* FTS deletion table pointers (DELETED, BEING_DELETED, etc.) */ + dict_table_t *m_common_tables[NUM_DELETION_TABLES]={nullptr}; + /* FTS CONFIG table pointer */ + dict_table_t *m_config_table={nullptr}; + + /** Table preparation methods */ + + /** Open FTS INDEX_[1..6] table for the given auxiliary index. + @return DB_SUCCESS or error code */ + dberr_t open_aux_table(uint8_t aux_index) noexcept; + + /** Open a deletion table (DELETED, BEING_DELETED, etc.). + @param table_type deletion table type + @return DB_SUCCESS or error code */ + dberr_t open_deletion_table(FTSDeletionTable table_type) noexcept; + + /** Helper to convert deletion table enum to string name */ + static const char* get_deletion_table_name( + FTSDeletionTable table_type) noexcept; + + /** Table lock operation */ + + /** Acquire a lock on an opened INDEX_[1..6] table. + Retries lock wait once via QueryExecutor. + @return DB_SUCCESS or error code */ + dberr_t lock_aux_tables(uint8_t aux_index, lock_mode mode) noexcept; + + /** Acquire a lock on an opened common FTS table. + Retries lock wait once via QueryExecutor. + @return DB_SUCCESS or error code */ + dberr_t lock_common_tables(uint8_t index, lock_mode mode) noexcept; + + /** Create search tuple for a word + @param table tuple to be created for this table + @param word word to create tuple for + @param heap memory heap + @return search tuple or nullptr if word is empty */ + dtuple_t* create_word_search_tuple( + dict_table_t *table, const fts_string_t *word, + mem_heap_t *heap) noexcept; +public: + /** Open all auxiliary tables + @return DB_SUCCESS or error code */ + dberr_t open_all_aux_tables(dict_index_t *fts_index) noexcept; + + /** Open all deletion tables (DELETED, BEING_DELETED, etc.) + @return DB_SUCCESS or error code */ + dberr_t open_all_deletion_tables() noexcept; + + /** Open FTS CONFIG table for configuration operations. + @return DB_SUCCESS or error code */ + dberr_t open_config_table() noexcept; + + /** Set CONFIG table directly (for cases where table is already opened) + @param config_table CONFIG table pointer */ + void set_config_table(dict_table_t *config_table) noexcept + { + m_config_table= config_table; + m_config_table->acquire(); + } + + /** Create executor bound to trx and FTS table/index. + @param trx transaction + @param fts_table FTS table */ + FTSQueryExecutor(trx_t *trx, const dict_table_t *fts_table); + + /** Release any opened table handles and executor resources. */ + ~FTSQueryExecutor(); + + /** High level DML operation on FTS TABLE */ + + /** Insert a row into auxiliary INDEX_[1..6] table. + Expects (word, first_doc_id, trx_id, roll_ptr, last_doc_id, + doc_count, ilist). + @param aux_index auxiliary table index + @param aux_data data to be inserted + @return DB_SUCCESS or error code */ + dberr_t insert_aux_record(uint8_t aux_index, + const fts_aux_data_t *aux_data) noexcept; + + /** Insert a single doc_id into a common table (e.g. DELETED, ...) + @param tbl_name common table name + @param doc_id document id to be inserted + @return DB_SUCCESS or error code */ + dberr_t insert_common_record(const char *tbl_name, doc_id_t doc_id) noexcept; + + /** Insert a key/value into CONFIG table. + @param key key for the config table + @param value value for the key + @return DB_SUCCESS or error code */ + dberr_t insert_config_record(const char *key, const char *value) noexcept; + + /** Delete one word row from INDEX_[1..6] by (word). + @param aux_index auxiliary table index + @param aux_data auxiliary table record + @return DB_SUCCESS or error code */ + dberr_t delete_aux_record(uint8_t aux_index, + const fts_aux_data_t *aux_data) noexcept; + + /** Delete a single doc_id row from a common table by (doc_id). + @param tbl_name common table name + @param doc_id document id to be deleted + @return DB_SUCCESS or error code */ + dberr_t delete_common_record(const char *tbl_name, doc_id_t doc_id) noexcept; + + /** Delete all rows from a common table. + @return DB_SUCCESS or error code */ + dberr_t delete_all_common_records(const char *tbl_name) noexcept; + + /** Delete a key from CONFIG table by (key). + @return DB_SUCCESS or error code */ + dberr_t delete_config_record(const char *key) noexcept; + + /** Upsert a key/value in CONFIG table. + Replaces 'value' if key exists, inserts otherwise. + @return DB_SUCCESS or error code */ + dberr_t update_config_record(const char *key, const char *value) noexcept; + + /** Select-for-update CONFIG row by 'key' + @return DB_SUCCESS or error code */ + dberr_t read_config_with_lock( + const char *key, RecordCallback &callback) noexcept; + + /** Read all rows from given COMMON table + Callback is invoked for processing the record */ + dberr_t read_all_common(const char *tbl_name, + RecordCallback &callback) noexcept; + mem_heap_t *get_heap() const noexcept + { return m_executor.get_heap(); } + + trx_t *trx() const noexcept + { return m_executor.get_trx(); } + + const dict_index_t* index() const noexcept + { return m_index; } + + void release_lock() + { + m_executor.commit_mtr(); + } + + /** Read records by index using underlying QueryExecutor + @param table table to read + @param sec_index secondary index + @param search_tuple search tuple + @param mode cursor mode + @param callback record callback + @return DB_SUCCESS or error code */ + dberr_t read_by_index(dict_table_t *table, dict_index_t *index, + const dtuple_t *search_tuple, page_cur_mode_t mode, + RecordCallback& callback) noexcept + { + dberr_t err= m_executor.read_by_index(table, index, search_tuple, + mode, callback, true); + return (err == DB_SUCCESS_LOCKED_REC) ? DB_SUCCESS : err; + } + + /** Construct FTS auxiliary table name + @param table_name output buffer for table name + @param suffix table suffix (e.g., "CONFIG", "INDEX_1") + @param common_table true for common tables, false for index tables */ + void construct_table_name(char *table_name, const char *suffix, + bool common_table) noexcept; + + dberr_t read_aux(uint8_t aux_index, + const fts_string_t* start_word, + RecordCallback& callback) noexcept; +}; + +/** Callback class for reading common table records +(DELETED, BEING_DELETED, DELETED_CACHE, BEING_DELETED_CACHE) */ +class CommonTableReader : public RecordCallback +{ +private: + std::vector doc_ids; + +public: + CommonTableReader(); + + const std::vector& get_doc_ids() const { return doc_ids; } + void clear() { doc_ids.clear(); } + + /** Fast common table field extraction for known table format. + Structure: (doc_id BIGINT UNSIGNED) - always known schema + @param rec record to extract from + @param index common table index + @param doc_id output doc_id value + @return true if extraction successful */ + static bool extract_common_fields( + const rec_t *rec, const dict_index_t *index, + doc_id_t *doc_id); +}; + +/** Callback class for reading FTS config table records */ +class ConfigReader : public RecordCallback +{ +public: + span value_span; + ConfigReader(); + + /** Extract the config table record. + Structure: (key VARCHAR, db_trx_id, db_roll_ptr, value TEXT) + @param rec record to extract from + @param index config table index + @param key_data output pointer to key data + @param key_len output key length + @param value_data output pointer to value data (optional) + @param value_len output value length (optional) + @return true if extraction successful */ + static bool extract_config_fields( + const rec_t *rec, const dict_index_t *index, + const byte **key_data, ulint *key_len, + const byte **value_data = nullptr, ulint *value_len = nullptr); + + /** Direct config key comparison - compares first field with tuple value. + @param search_tuple search tuple containing target key + @param rec record to compare + @param index config table index + @return comparison action */ + static RecordCompareAction compare_config_key( + const dtuple_t *search_tuple, const rec_t *rec, + const dict_index_t *index, const rec_offs *offsets); +}; + +/** Type alias for FTS record processor function */ +using FTSRecordProcessor= std::function< + dberr_t(const rec_t*, const dict_index_t*, const rec_offs*, void*)>; + +/** Comparison modes for AuxRecordReader */ +enum class AuxCompareMode +{ + /** >= comparison (range scan from word) */ + GREATER_EQUAL, + /** > comparison (exclude exact match) */ + GREATER, + /** LIKE pattern matching (prefix match) */ + LIKE, + /** = comparison (exact match) */ + EQUAL +}; + +/** Callback class for reading FTS auxiliary index table records */ +class AuxRecordReader : public RecordCallback +{ +private: + void *user_arg; + ulint *total_memory; + AuxCompareMode compare_mode; + fts_string_t m_last_word; + mem_heap_t* m_word_heap= nullptr; + +private: + /** FTS-specific record comparison logic */ + RecordCompareAction compare_record( + const dtuple_t *search_tuple, const rec_t *rec, + const dict_index_t *index, const rec_offs *offsets) noexcept; + + RecordComparator make_comparator() + { + return [this](const dtuple_t* search_tuple, const rec_t* rec, + const dict_index_t* index, const rec_offs* offsets) + -> RecordCompareAction + { + return this->compare_record(search_tuple, rec, index, offsets); + }; + } + +public: + /** Default word processor for FTS auxiliary table records */ + dberr_t default_word_processor(const rec_t *rec, const dict_index_t *index, + const rec_offs *offsets, void *user_arg); + + /* Constructor with custom processor */ + template + AuxRecordReader(void* user_data, + ProcessorFunc proc_func, + AuxCompareMode mode= AuxCompareMode::GREATER_EQUAL) + : RecordCallback( + [this, proc_func](const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets) -> dberr_t + { + return proc_func(rec, index, offsets, this->user_arg); + }, + make_comparator() + ), + user_arg(user_data), total_memory(nullptr), + compare_mode(mode), m_last_word{} {} + + /* Different constructor with default word processing */ + AuxRecordReader(void *user_data, ulint *memory_counter, + AuxCompareMode mode= AuxCompareMode::GREATER_EQUAL) + : RecordCallback( + [this](const rec_t *rec, const dict_index_t* index, + const rec_offs *offsets) -> dberr_t + { + return this->default_word_processor(rec, index, offsets, + this->user_arg); + }, + make_comparator() + ), + user_arg(user_data), total_memory(memory_counter), + compare_mode(mode), m_last_word{} {} + + ~AuxRecordReader() + { + if (m_word_heap) + { + mem_heap_free(m_word_heap); + m_word_heap = nullptr; + } + } + /** Reset total memory counter */ + void reset_total_memory() { if (total_memory) *total_memory = 0; } + + /** Get the last processed word (for pagination) + @return reference to last processed word */ + const fts_string_t &get_last_word() const noexcept + { + return m_last_word; + } +}; diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 9dd48e525194d..6b00de10c8ad5 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -40,6 +40,9 @@ Created 2011/09/02 Sunny Bains #include "mysql/plugin_ftparser.h" #include "lex_string.h" +/* Forward declarations */ +class FTSQueryExecutor; + /** "NULL" value of a document id. */ #define FTS_NULL_DOC_ID 0 @@ -611,13 +614,12 @@ fts_create( dict_table_t* table); /*!< out: table with FTS indexes */ -/*********************************************************************//** -Run OPTIMIZE on the given table. -@return DB_SUCCESS if all OK */ +/** Run OPTIMIZE on the given table. +@param table table to be optimized +@param thd thread +@return DB_SUCCESS if all ok */ dberr_t -fts_optimize_table( -/*===============*/ - dict_table_t* table); /*!< in: table to optimiza */ +fts_optimize_table(dict_table_t *table, THD *thd); /**********************************************************************//** Startup the optimize thread and create the work queue. */ @@ -780,14 +782,6 @@ fts_tokenize_document_internal( const char* doc, /*!< in: document to tokenize */ int len); /*!< in: document length */ -/*********************************************************************//** -Fetch COUNT(*) from specified table. -@return the number of rows in the table */ -ulint -fts_get_rows_count( -/*===============*/ - fts_table_t* fts_table); /*!< in: fts table to read */ - /*************************************************************//** Get maximum Doc ID in a table if index "FTS_DOC_ID_INDEX" exists @return max Doc ID or 0 if index "FTS_DOC_ID_INDEX" does not exist */ @@ -804,33 +798,24 @@ fts_get_max_doc_id( CHARSET_INFO *fts_valid_stopword_table(const char *stopword_table_name, const char **row_end= NULL); -/****************************************************************//** -This function loads specified stopword into FTS cache +/** This function loads specified stopword into FTS cache +@param executor FTSQueryExecutor instance +@param table table which has fts index @return true if success */ -bool -fts_load_stopword( -/*==============*/ - const dict_table_t* - table, /*!< in: Table with FTS */ - trx_t* trx, /*!< in: Transaction */ - const char* session_stopword_table, /*!< in: Session stopword table - name */ - bool stopword_is_on, /*!< in: Whether stopword - option is turned on/off */ - bool reload); /*!< in: Whether it is during - reload of FTS table */ +bool fts_load_stopword(FTSQueryExecutor *executor, + const dict_table_t *table) noexcept; + +/** Read the rows from the fulltext index +@param executor FTSQueryExecutor instance +@param table Fulltext table +@param tbl_name table name +@param doc_ids collecting doc ids +@return DB_SUCCESS or error code */ +dberr_t fts_table_fetch_doc_ids(FTSQueryExecutor *executor, + dict_table_t *table, const char *tbl_name, + fts_doc_ids_t *doc_ids) noexcept; /****************************************************************//** -Read the rows from the FTS index -@return DB_SUCCESS if OK */ -dberr_t -fts_table_fetch_doc_ids( -/*====================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: aux table */ - fts_doc_ids_t* doc_ids); /*!< in: For collecting - doc ids */ -/****************************************************************//** This function brings FTS index in sync when FTS index is first used. There are documents that have not yet sync-ed to auxiliary tables from last server abnormally shutdown, we will need to bring @@ -918,16 +903,56 @@ bool fts_check_aux_table(const char *name, /** Update the last document id. This function could create a new transaction to update the last document id. -@param table table to be updated -@param doc_id last document id -@param trx update trx or null +@param executor query executor +@param table table to be updated +@param doc_id last document id @retval DB_SUCCESS if OK */ dberr_t -fts_update_sync_doc_id(const dict_table_t *table, - doc_id_t doc_id, - trx_t *trx) - MY_ATTRIBUTE((nonnull(1))); +fts_update_sync_doc_id(FTSQueryExecutor *executor, + const dict_table_t *table, + doc_id_t doc_id) noexcept; /** Sync the table during commit phase @param[in] table table to be synced */ void fts_sync_during_ddl(dict_table_t* table); + +/** Tokenize a document. +@param[in,out] doc document to tokenize +@param[out] result tokenization result +@param[in] parser pluggable parser */ +void fts_tokenize_document( + fts_doc_t* doc, + fts_doc_t* result, + st_mysql_ftparser* parser); + +/** Continue to tokenize a document. +@param[in,out] doc document to tokenize +@param[in] add_pos add this position to all tokens from this tokenization +@param[out] result tokenization result +@param[in] parser pluggable parser */ +void fts_tokenize_document_next( + fts_doc_t* doc, + ulint add_pos, + fts_doc_t* result, + st_mysql_ftparser* parser); + +/** Get a character set based on precise type. +@param prtype precise type +@return the corresponding character set */ +CHARSET_INFO* fts_get_charset(ulint prtype); + +/** Load user defined stopword from designated user table +@param fts fulltext structure +@param stopword_table stopword table +@param stopword_info stopword information +@return whether the operation is successful */ +bool fts_load_user_stopword(FTSQueryExecutor *executor, fts_t *fts, + const char *stopword_table, + fts_stopword_t *stopword_info) noexcept; + +/****************************************************************//** +This function loads the default InnoDB stopword list */ +void +fts_load_default_stopword( +/*======================*/ + fts_stopword_t* stopword_info); /*!< in: stopword info */ diff --git a/storage/innobase/include/fts0priv.h b/storage/innobase/include/fts0priv.h index 1d3bc323841b9..237f7acda674d 100644 --- a/storage/innobase/include/fts0priv.h +++ b/storage/innobase/include/fts0priv.h @@ -28,10 +28,10 @@ Created 2011/09/02 Sunny Bains #define INNOBASE_FTS0PRIV_H #include "dict0dict.h" -#include "pars0pars.h" #include "que0que.h" #include "que0types.h" #include "fts0types.h" +#include "fts0exec.h" /* The various states of the FTS sub system pertaining to a table with FTS indexes defined on it. */ @@ -106,26 +106,6 @@ component. /** Maximum length of an integer stored in the config table value column. */ #define FTS_MAX_INT_LEN 32 -/******************************************************************//** -Parse an SQL string. %s is replaced with the table's id. -@return query graph */ -que_t* -fts_parse_sql( -/*==========*/ - fts_table_t* fts_table, /*!< in: FTS aux table */ - pars_info_t* info, /*!< in: info struct, or NULL */ - const char* sql) /*!< in: SQL string to evaluate */ - MY_ATTRIBUTE((nonnull(3), malloc, warn_unused_result)); -/******************************************************************//** -Evaluate a parsed SQL statement -@return DB_SUCCESS or error code */ -dberr_t -fts_eval_sql( -/*=========*/ - trx_t* trx, /*!< in: transaction */ - que_t* graph) /*!< in: Parsed statement */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); - /** Construct the name of an internal FTS table for the given table. @param[in] fts_table metadata on fulltext-indexed table @param[out] table_name a name up to MAX_FULL_NAME_LEN @@ -133,77 +113,15 @@ fts_eval_sql( void fts_get_table_name(const fts_table_t* fts_table, char* table_name, bool dict_locked = false) MY_ATTRIBUTE((nonnull)); -/******************************************************************//** -Construct the column specification part of the SQL string for selecting the -indexed FTS columns for the given table. Adds the necessary bound -ids to the given 'info' and returns the SQL string. Examples: -One indexed column named "text": - - "$sel0", - info/ids: sel0 -> "text" - -Two indexed columns named "subject" and "content": - - "$sel0, $sel1", - info/ids: sel0 -> "subject", sel1 -> "content", -@return heap-allocated WHERE string */ -const char* -fts_get_select_columns_str( -/*=======================*/ - dict_index_t* index, /*!< in: FTS index */ - pars_info_t* info, /*!< in/out: parser info */ - mem_heap_t* heap) /*!< in: memory heap */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); - -/** define for fts_doc_fetch_by_doc_id() "option" value, defines whether -we want to get Doc whose ID is equal to or greater or smaller than supplied -ID */ -#define FTS_FETCH_DOC_BY_ID_EQUAL 1 -#define FTS_FETCH_DOC_BY_ID_LARGE 2 -#define FTS_FETCH_DOC_BY_ID_SMALL 3 - -/*************************************************************//** -Fetch document (= a single row's indexed text) with the given -document id. -@return: DB_SUCCESS if fetch is successful, else error */ -dberr_t -fts_doc_fetch_by_doc_id( -/*====================*/ - fts_get_doc_t* get_doc, /*!< in: state */ - doc_id_t doc_id, /*!< in: id of document to fetch */ - dict_index_t* index_to_use, /*!< in: caller supplied FTS index, - or NULL */ - ulint option, /*!< in: search option, if it is - greater than doc_id or equal */ - fts_sql_callback - callback, /*!< in: callback to read - records */ - void* arg) /*!< in: callback arg */ - MY_ATTRIBUTE((nonnull(6))); - -/*******************************************************************//** -Callback function for fetch that stores the text of an FTS document, -converting each column to UTF-16. -@return always FALSE */ -ibool -fts_query_expansion_fetch_doc( -/*==========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts_doc_t* */ - MY_ATTRIBUTE((nonnull)); -/******************************************************************** -Write out a single word's data as new entry/entries in the INDEX table. -@return DB_SUCCESS if all OK. */ -dberr_t -fts_write_node( -/*===========*/ - trx_t* trx, /*!< in: transaction */ - que_t** graph, /*!< in: query graph */ - fts_table_t* fts_table, /*!< in: the FTS aux index */ - fts_string_t* word, /*!< in: word in UTF-8 */ - fts_node_t* node) /*!< in: node columns */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); +/** Write out a single word's data as new entry/entries in the INDEX table. +@param executor FTS Query Executor +@param selected auxiliary index number +@param aux_data auxiliary table data +@return DB_SUCCESS if all OK or error code */ +dberr_t fts_write_node(FTSQueryExecutor *executor, uint8_t selected, + const fts_aux_data_t *aux_data) noexcept + MY_ATTRIBUTE((nonnull, warn_unused_result)); /** Check if a fts token is a stopword or less than fts_min_token_size or greater than fts_max_token_size. @@ -252,35 +170,36 @@ fts_word_free( /*==========*/ fts_word_t* word) /*!< in: instance to free.*/ MY_ATTRIBUTE((nonnull)); -/******************************************************************//** -Read the rows from the FTS inde -@return DB_SUCCESS or error code */ -dberr_t -fts_index_fetch_nodes( -/*==================*/ - trx_t* trx, /*!< in: transaction */ - que_t** graph, /*!< in: prepared statement */ - fts_table_t* fts_table, /*!< in: FTS aux table */ - const fts_string_t* - word, /*!< in: the word to fetch */ - fts_fetch_t* fetch) /*!< in: fetch callback.*/ - MY_ATTRIBUTE((nonnull)); + +/** Read the rows from FTS index +@param trx transaction +@param index fulltext index +@param word word to fetch +@param user_arg user argument +@param processor custom processor to filter the word from record +@param compare_mode comparison mode for record matching +@return error code or DB_SUCCESS */ +dberr_t fts_index_fetch_nodes(FTSQueryExecutor *executor, dict_index_t *index, + const fts_string_t *word, + void *user_arg, + FTSRecordProcessor processor, + AuxCompareMode compare_mode) noexcept + MY_ATTRIBUTE((nonnull)); + #define fts_sql_commit(trx) trx_commit_for_mysql(trx) #define fts_sql_rollback(trx) (trx)->rollback() -/******************************************************************//** -Get value from config table. The caller must ensure that enough -space is allocated for value to hold the column contents + +/** Get value from the config table. The caller must ensure that enough +space is allocated for value to hold the column contents. +@param trx transaction +@param table Indexed fts table +@param name name of the key +@param value value of the key @return DB_SUCCESS or error code */ -dberr_t -fts_config_get_value( -/*=================*/ - trx_t* trx, /* transaction */ - fts_table_t* fts_table, /*!< in: the indexed FTS table */ - const char* name, /*!< in: get config value for - this parameter name */ - fts_string_t* value) /*!< out: value read from - config table */ - MY_ATTRIBUTE((nonnull)); +dberr_t fts_config_get_value(FTSQueryExecutor *executor, const dict_table_t *table, + const char *name, fts_string_t *value) noexcept + MY_ATTRIBUTE((nonnull)); + /******************************************************************//** Get value specific to an FTS index from the config table. The caller must ensure that enough space is allocated for value to hold the @@ -289,36 +208,36 @@ column contents. dberr_t fts_config_get_index_value( /*=======================*/ - trx_t* trx, /*!< transaction */ + FTSQueryExecutor* executor, /*!< in: query executor */ dict_index_t* index, /*!< in: index */ const char* param, /*!< in: get config value for this parameter name */ fts_string_t* value) /*!< out: value read from config table */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); + noexcept MY_ATTRIBUTE((nonnull, warn_unused_result)); /******************************************************************//** Set the value in the config table for name. @return DB_SUCCESS or error code */ dberr_t fts_config_set_value( /*=================*/ - trx_t* trx, /*!< transaction */ - fts_table_t* fts_table, /*!< in: the indexed FTS table */ + FTSQueryExecutor* executor, /*!< in: query executor */ + const dict_table_t* table, /*!< in: the indexed FTS table */ const char* name, /*!< in: get config value for this parameter name */ const fts_string_t* value) /*!< in: value to update */ - MY_ATTRIBUTE((nonnull)); + noexcept MY_ATTRIBUTE((nonnull)); /****************************************************************//** Set an ulint value in the config table. @return DB_SUCCESS if all OK else error code */ dberr_t fts_config_set_ulint( /*=================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: the indexed FTS table */ + FTSQueryExecutor* executor, /*!< in: query executor */ + const dict_table_t* table, /*!< in: the indexed FTS table */ const char* name, /*!< in: param name */ - ulint int_value) /*!< in: value */ + ulint int_value)noexcept /*!< in: value */ MY_ATTRIBUTE((nonnull, warn_unused_result)); /******************************************************************//** Set the value specific to an FTS index in the config table. @@ -326,26 +245,15 @@ Set the value specific to an FTS index in the config table. dberr_t fts_config_set_index_value( /*=======================*/ - trx_t* trx, /*!< transaction */ + FTSQueryExecutor* executor, /*!< in: query executor */ dict_index_t* index, /*!< in: index */ const char* param, /*!< in: get config value for this parameter name */ - fts_string_t* value) /*!< out: value read from + fts_string_t* value)noexcept /*!< out: value read from config table */ MY_ATTRIBUTE((nonnull, warn_unused_result)); /******************************************************************//** -Get an ulint value from the config table. -@return DB_SUCCESS or error code */ -dberr_t -fts_config_get_ulint( -/*=================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: the indexed FTS table */ - const char* name, /*!< in: param name */ - ulint* int_value) /*!< out: value */ - MY_ATTRIBUTE((nonnull)); -/******************************************************************//** Search cache for word. @return the word node vector if found else NULL */ const ib_vector_t* diff --git a/storage/innobase/include/fts0types.h b/storage/innobase/include/fts0types.h index 17b0f947de277..fc9b5c8c5321c 100644 --- a/storage/innobase/include/fts0types.h +++ b/storage/innobase/include/fts0types.h @@ -313,7 +313,7 @@ fts_get_suffix( @param[in] len string length in bytes @return the index to use for the string */ UNIV_INLINE -ulint +uint8_t fts_select_index( const CHARSET_INFO* cs, const byte* str, diff --git a/storage/innobase/include/fts0types.inl b/storage/innobase/include/fts0types.inl index 622d1337bad60..d172ab38ea42c 100644 --- a/storage/innobase/include/fts0types.inl +++ b/storage/innobase/include/fts0types.inl @@ -102,13 +102,13 @@ inline bool fts_is_charset_cjk(const CHARSET_INFO* cs) @param[in] len string length @retval the index to use for the string */ UNIV_INLINE -ulint +uint8_t fts_select_index_by_range( const CHARSET_INFO* cs, const byte* str, ulint len) { - ulint selected = 0; + uint8_t selected = 0; ulint value = innobase_strnxfrm(cs, str, len); while (fts_index_selector[selected].value != 0) { @@ -136,7 +136,7 @@ fts_select_index_by_range( @param[in] len string length @retval the index to use for the string */ UNIV_INLINE -ulint +uint8_t fts_select_index_by_hash( const CHARSET_INFO* cs, const byte* str, @@ -162,7 +162,7 @@ fts_select_index_by_hash( /* Get collation hash code */ my_ci_hash_sort(&hasher, cs, str, char_len); - return(hasher.m_nr1 % FTS_NUM_AUX_INDEX); + return static_cast(hasher.m_nr1 % FTS_NUM_AUX_INDEX); } /** Select the FTS auxiliary index for the given character. @@ -171,21 +171,17 @@ fts_select_index_by_hash( @param[in] len string length in bytes @retval the index to use for the string */ UNIV_INLINE -ulint +uint8_t fts_select_index( const CHARSET_INFO* cs, const byte* str, ulint len) { - ulint selected; - if (fts_is_charset_cjk(cs)) { - selected = fts_select_index_by_hash(cs, str, len); - } else { - selected = fts_select_index_by_range(cs, str, len); + return fts_select_index_by_hash(cs, str, len); } - return(selected); + return fts_select_index_by_range(cs, str, len); } /******************************************************************//** diff --git a/storage/innobase/include/page0cur.h b/storage/innobase/include/page0cur.h index 14a3ce9ba038c..ab871d7ef8511 100644 --- a/storage/innobase/include/page0cur.h +++ b/storage/innobase/include/page0cur.h @@ -244,6 +244,21 @@ bool page_cur_search_with_match_bytes(const dtuple_t &tuple, uint16_t *ilow_bytes) noexcept; +/** Compare a data tuple to a physical record. +@param rec B-tree index record +@param index index B-tree +@param tuple search key +@param match matched fields << 16 | bytes +@param comp nonzero if ROW_FORMAT=REDUNDANT is not being used +@return the comparison result of dtuple and rec +@retval 0 if dtuple is equal to rec +@retval negative if dtuple is less than rec +@retval positive if dtuple is greater than rec */ +int cmp_dtuple_rec_bytes(const rec_t *rec, + const dict_index_t &index, + const dtuple_t &tuple, int *match, ulint comp) + noexcept; + /***********************************************************//** Positions a page cursor on a randomly chosen user record on a page. If there are no user records, sets the cursor on the infimum record. */ diff --git a/storage/innobase/include/row0mysql.h b/storage/innobase/include/row0mysql.h index 2f50aa1560eb2..6887e14da4bab 100644 --- a/storage/innobase/include/row0mysql.h +++ b/storage/innobase/include/row0mysql.h @@ -185,6 +185,7 @@ row_create_prebuilt( the MySQL format */ /** Free a prebuilt struct for a TABLE handle. */ void row_prebuilt_free(row_prebuilt_t *prebuilt); + /*********************************************************************//** Updates the transaction pointers in query graphs stored in the prebuilt struct. */ @@ -556,6 +557,9 @@ struct row_prebuilt_t { and updates */ btr_pcur_t* clust_pcur; /*!< persistent cursor used in some selects and updates */ + mtr_t* mtr; /*!< external mini-transaction for + DML operations; when set, row_search_mvcc + will use this instead of creating its own */ que_fork_t* sel_graph; /*!< dummy query graph used in selects */ dtuple_t* search_tuple; /*!< prebuilt dtuple used in selects */ diff --git a/storage/innobase/include/row0query.h b/storage/innobase/include/row0query.h new file mode 100644 index 0000000000000..625533b3af105 --- /dev/null +++ b/storage/innobase/include/row0query.h @@ -0,0 +1,244 @@ +/***************************************************************************** + +Copyright (c) 2025, MariaDB PLC. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file include/row0query.h +General Query Executor + +Created 2025/10/30 +*******************************************************/ + +#pragma once + +#include "btr0pcur.h" +#include +#include "dict0types.h" +#include "data0types.h" +#include "db0err.h" +#include "lock0types.h" +#include "rem0rec.h" + +struct row_prebuilt_t; + +/** Comparator action for deciding how to treat a record */ +enum class RecordCompareAction +{ + /** Process this record via process_record */ + PROCESS, + /** Do not process this record, continue traversal */ + SKIP, + /** Stop traversal immediately */ + STOP +}; + +using RecordProcessor= std::function; + +using RecordComparator= std::function< + RecordCompareAction(const dtuple_t*, const rec_t*, + const dict_index_t*, const rec_offs*)>; + +/** Record processing callback interface using std::function. +Can be used by FTS, stats infrastructure, and other components +that need to process database records with custom logic. */ +class RecordCallback +{ +public: + /** Constructor with processor function and optional comparator + @param processor Function to process each record + @param comparator Optional function to filter records */ + RecordCallback( + RecordProcessor processor, + RecordComparator comparator= nullptr) + : process_record(processor), compare_record(comparator) {} + + virtual ~RecordCallback()= default; + + /** Called for each matching record */ + const RecordProcessor process_record; + + /** Comparison function for custom filtering */ + const RecordComparator compare_record; +}; + +/** General-purpose MVCC-aware record traversal and basic +DML executor. Provides a thin abstraction over B-tree cursors for +reading and mutating records with consistent-read (MVCC) handling, +and callback API. +- Open and iterate clustered/secondary indexes with page cursors. +- Build consistent-read versions when needed via transaction +read views. +- Filter and process records using RecordCallback: + - compare_record: decide SKIP/PROCESS/STOP for each record + - process_record: handle visible records; return DB_SUCCESS to continue, DB_SUCCESS_LOCKED_REC to stop +- Basic DML helpers (insert/delete/replace) and table locking. */ +class QueryExecutor +{ +private: + que_thr_t *m_thr; + btr_pcur_t m_pcur; + btr_pcur_t *m_clust_pcur; + trx_t *m_trx; + mtr_t *m_mtr; + mem_heap_t *m_heap; + + /* Prebuilt for row_search_mvcc call */ + row_prebuilt_t *m_prebuilt; + + /* Setup prebuilt structure for row_search_mvcc usage + @param table table to search + @param index index to use + @param tuple search tuple + @param mode search mode */ + void setup_prebuilt(dict_table_t *table, dict_index_t *index, + const dtuple_t *tuple, page_cur_mode_t mode) noexcept; + +public: + QueryExecutor(trx_t *trx); + ~QueryExecutor(); + + /** Insert a record in clustered index of the table + @param table table to be inserted + @param tuple tuple to be inserted + @return DB_SUCCESS on success, error code on failure */ + dberr_t insert_record(dict_table_t *table, dtuple_t *tuple) noexcept; + + /** Delete a record from the clustered index of the table + @param table table to be inserted + @param tuple tuple to be inserted + @return DB_SUCCESS on success, error code on failure */ + dberr_t delete_record(dict_table_t *table, dtuple_t *tuple) noexcept; + + /** Delete all records from the clustered index of the table + @param table table be be deleted + @return DB_SUCCESS on success, error code on failure */ + dberr_t delete_all(dict_table_t *table) noexcept; + + /** Acquire and lock a single clustered record for update + Performs a keyed lookup on the clustered index, validates MVCC visibility, + and acquires an X lock on the matching record. + @param[in] table Table containing the record + @param[in] search_tuple Exact key for clustered index lookup + @param[in] callback Optional record callback + @return DB_SUCCESS on successful lock + DB_RECORD_NOT_FOUND if no visible matching record + DB_LOCK_WAIT if waiting was required + error code on failure */ + dberr_t select_for_update(dict_table_t *table, dtuple_t *search_tuple, + RecordCallback *callback= nullptr) noexcept; + + /** Update the currently selected clustered record within an active mtr. + Attempts in-place update; falls back to optimistic/pessimistic update if needed, + including external field storage when required. + select_for_update() has positioned and locked m_pcur on the target row. + + The caller is responsible for committing or rolling back m_mtr after this call + @param[in] table target table + @param[in] update update descriptor (fields, new values) + @return DB_SUCCESS on success + DB_OVERFLOW/DB_UNDERFLOW during size-changing paths + error_code on failures */ + dberr_t update_record(dict_table_t *table, const upd_t *update) noexcept; + + + /** Try to update a record by key or insert if not found. + Performs a SELECT ... FOR UPDATE using search_tuple; + if found, updates the row; otherwise inserts a new record. + Note: + On update path, commits or rolls back the active mtr as needed. + On insert path, no active mtr remains upon return + @param[in] table target table + @param[in] search_tuple key identifying the target row + @param[in] update update descriptor (applied when found) + @param[in] insert_tuple tuple to insert when not found + @return DB_SUCCESS on successful update or insert + @retval DB_LOCK_WAIT to be retried, + @return error code on failure */ + dberr_t replace_record(dict_table_t *table, dtuple_t *search_tuple, + const upd_t *update, dtuple_t *insert_tuple) noexcept; + + /** Iterate clustered index records and process via callback. + Handles full table scan and index scan for range/select queries + Calls callback.compare_record() to decide SKIP/PROCESS/STOP for + each matching record. On PROCESS, invokes + callback.process_record() on an MVCC-visible version. + @param table table to read + @param tuple optional search key (range/point). nullptr => full scan + @param mode B-tree search mode (e.g., PAGE_CUR_GE) + @param callback record comparator/processor + @return DB_SUCCESS if at least one record was processed + @retval DB_RECORD_NOT_FOUND if no record matched + @return error code on failure */ + dberr_t read(dict_table_t *table, const dtuple_t *tuple, + page_cur_mode_t mode, + RecordCallback& callback) noexcept; + + /** Iterate all records in clustered index and process via callback. + Performs a full table scan without any search key. + Calls callback.compare_record() to decide SKIP/PROCESS/STOP for + each record. On PROCESS, invokes callback.process_record() on an + MVCC-visible version. + + @param table table to scan + @param callback record comparator/processor + @return DB_SUCCESS if at least one record was processed + @retval DB_RECORD_NOT_FOUND if no record matched + @return error code on failure */ + dberr_t read_all(dict_table_t *table, RecordCallback &callback, + const dtuple_t *start_tuple= nullptr) noexcept; + + /** Read records via a secondary index and process corresponding + clustered rows. Performs a range or point scan on the given secondary index, + filters secondary records with callback.compare_record(), then looks up + the matching clustered record and invokes callback.process_record() + on a MVCC-visible version. + + @param table Table to read + @param sec_index Secondary index used for traversal + @param search_tuple search key or nullptr for full scan + @param mode Cursor search mode + @param callback RecordCallback with comparator+processor + @return DB_SUCCESS on success + DB_RECORD_NOT_FOUND if no matching record was processed + error code on failure */ + dberr_t read_by_index(dict_table_t *table, dict_index_t *sec_index, + const dtuple_t *search_tuple, + page_cur_mode_t mode, + RecordCallback &callback, + bool all_read) noexcept; + + /** Acquire a table lock in the given mode for transaction. + @param table table to lock + @param mode lock mode + @return DB_SUCCESS, DB_LOCK_WAIT or error code */ + dberr_t lock_table(dict_table_t *table, lock_mode mode) noexcept; + + /** Handle a lock wait for the current transaction and thread context. + @param err the lock-related error to handle (e.g., DB_LOCK_WAIT) + @param table_lock true if the wait originated from table lock, else row lock + @return DB_SUCCESS if the wait completed successfully and lock was granted + @retval DB_LOCK_WAIT_TIMEOUT if timed out */ + dberr_t handle_wait(dberr_t err, bool table_lock) noexcept; + mem_heap_t *get_heap() const { return m_heap; } + trx_t *get_trx() const { return m_trx; } + void commit_mtr() noexcept + { + if (m_mtr) + m_mtr->commit(); + } +}; diff --git a/storage/innobase/include/row0sel.h b/storage/innobase/include/row0sel.h index 35e3cbe66315c..952a6b747af30 100644 --- a/storage/innobase/include/row0sel.h +++ b/storage/innobase/include/row0sel.h @@ -35,6 +35,7 @@ Created 12/19/1997 Heikki Tuuri #include "pars0sym.h" #include "btr0pcur.h" #include "row0mysql.h" +#include "row0query.h" /*********************************************************************//** Creates a select node struct. @@ -118,6 +119,18 @@ row_sel_convert_mysql_key_to_innobase( ulint key_len) /*!< in: MySQL key value length */ MY_ATTRIBUTE((nonnull(1,4,5))); +/* Forward declarations for callback policy classes */ +template +struct InnoDBPolicy; + +/* Type aliases for different row_search_mvcc use cases */ +/** MySQL format conversion and manages its own mtr */ +using MySQLRowCallback= InnoDBPolicy; +/** Used for internal read purpose */ +using InnoDBReadPolicy= InnoDBPolicy; +/** Used for internal update/delete purpose */ +using InnoDBDMLPolicy= InnoDBPolicy; + /** Search for rows in the database using cursor. Function is mainly used for tables that are shared across connections and so it employs technique that can help re-construct the rows that @@ -138,6 +151,7 @@ It also has optimization such as pre-caching the rows, using AHI, etc. cursor 'direction' should be 0. @return DB_SUCCESS, DB_RECORD_NOT_FOUND, DB_END_OF_INDEX, DB_DEADLOCK, DB_LOCK_TABLE_FULL, DB_CORRUPTION, or DB_TOO_BIG_RECORD */ +template dberr_t row_search_mvcc( byte* buf, @@ -147,6 +161,49 @@ row_search_mvcc( ulint direction) MY_ATTRIBUTE((warn_unused_result)); +/** Convenience wrapper for InnoDB callback (read-only, manages own mtr). +Used for FTS queries and other read-only operations where row_search_mvcc +manages its own mini-transaction lifecycle. +@param callback RecordCallback instance +@param mode search mode +@param prebuilt prebuilt structure +@param match_mode match mode +@param direction search direction +@return DB_SUCCESS or error code */ +inline dberr_t row_search_mvcc_callback( + RecordCallback* callback, + page_cur_mode_t mode, + row_prebuilt_t* prebuilt, + ulint match_mode, + ulint direction) noexcept +{ + return row_search_mvcc( + reinterpret_cast(callback), mode, prebuilt, + match_mode, direction); +} + +/** Convenience wrapper for InnoDB callback (DML, external mtr). +Used for SELECT FOR UPDATE and other DML operations where the caller +manages the mini-transaction lifecycle (e.g., needs to keep mtr open +across select_for_update() and update_record()). +@param callback RecordCallback instance +@param mode search mode +@param prebuilt prebuilt structure with external mtr set +@param match_mode match mode +@param direction search direction +@return DB_SUCCESS or error code */ +inline dberr_t row_search_mvcc_callback_dml( + RecordCallback* callback, + page_cur_mode_t mode, + row_prebuilt_t* prebuilt, + ulint match_mode, + ulint direction) noexcept +{ + return row_search_mvcc( + reinterpret_cast(callback), mode, prebuilt, + match_mode, direction); +} + /********************************************************************//** Count rows in a R-Tree leaf level. @return DB_SUCCESS if successful */ diff --git a/storage/innobase/include/ut0new.h b/storage/innobase/include/ut0new.h index bcc129601d11f..a974a093da1af 100644 --- a/storage/innobase/include/ut0new.h +++ b/storage/innobase/include/ut0new.h @@ -854,6 +854,7 @@ constexpr const char* const auto_event_names[] = "fts0ast", "fts0blex", "fts0config", + "fts0exec", "fts0file", "fts0fts", "fts0opt", diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 0642ff58310e1..51c9e2db81f33 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -4328,8 +4328,8 @@ dberr_t lock_table_children(dict_table_t *table, trx_t *trx) children.end()) continue; /* We already acquired MDL on this child table. */ MDL_ticket *mdl= nullptr; - child= dict_acquire_mdl_shared(child, mdl_context, &mdl, - DICT_TABLE_OP_NORMAL); + child= dict_acquire_mdl(child, mdl_context, &mdl, + DICT_TABLE_OP_NORMAL); if (child) { if (mdl) diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc index d1d11b50de8a1..15319bf5b8866 100644 --- a/storage/innobase/page/page0cur.cc +++ b/storage/innobase/page/page0cur.cc @@ -78,9 +78,9 @@ static ulint cmp_get_pad_char(const dtype_t &type) noexcept @retval 0 if dtuple is equal to rec @retval negative if dtuple is less than rec @retval positive if dtuple is greater than rec */ -static int cmp_dtuple_rec_bytes(const rec_t *rec, - const dict_index_t &index, - const dtuple_t &tuple, int *match, ulint comp) +int cmp_dtuple_rec_bytes(const rec_t *rec, + const dict_index_t &index, + const dtuple_t &tuple, int *match, ulint comp) noexcept { ut_ad(dtuple_check_typed(&tuple)); diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 2dc11e4503e18..801b801f3f51f 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -42,6 +42,7 @@ Completed by Sunny Bains and Marko Makela #include "pars0pars.h" #include "ut0sort.h" #include "row0ftsort.h" +#include "fts0exec.h" #include "row0import.h" #include "row0vers.h" #include "handler0alter.h" @@ -3035,10 +3036,27 @@ row_merge_read_clustered_index( new_table->fts->cache->first_doc_id = new_table->fts->cache->next_doc_id; + trx_t *fts_trx = trx_create(); + trx_start_internal(fts_trx); + fts_trx->op_info= "setting last FTS document id"; + FTSQueryExecutor executor( + fts_trx, new_table); err= fts_update_sync_doc_id( + &executor, new_table, - new_table->fts->cache->synced_doc_id, - NULL); + new_table->fts->cache->synced_doc_id); + if (err == DB_SUCCESS) { + trx_commit_for_mysql(fts_trx); + new_table->fts->cache->synced_doc_id++; + } else { + sql_print_error( + "InnoDB: ( %s ) while updating " + "last doc id for table %s", + ut_strerr(err), + new_table->name.m_name); + fts_trx->rollback(); + } + fts_trx->free(); } } diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index b4c9fb8c6dfd4..2a926b41c1928 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -885,11 +885,11 @@ row_create_prebuilt( } prebuilt->pcur = static_cast( - mem_heap_zalloc(prebuilt->heap, - sizeof(btr_pcur_t))); + mem_heap_zalloc(prebuilt->heap, + sizeof(btr_pcur_t))); prebuilt->clust_pcur = static_cast( - mem_heap_zalloc(prebuilt->heap, - sizeof(btr_pcur_t))); + mem_heap_zalloc(prebuilt->heap, + sizeof(btr_pcur_t))); btr_pcur_reset(prebuilt->pcur); btr_pcur_reset(prebuilt->clust_pcur); @@ -935,8 +935,13 @@ void row_prebuilt_free(row_prebuilt_t *prebuilt) prebuilt->magic_n = ROW_PREBUILT_FREED; prebuilt->magic_n2 = ROW_PREBUILT_FREED; - btr_pcur_reset(prebuilt->pcur); - btr_pcur_reset(prebuilt->clust_pcur); + if (prebuilt->pcur) { + btr_pcur_reset(prebuilt->pcur); + } + + if (prebuilt->clust_pcur) { + btr_pcur_reset(prebuilt->clust_pcur); + } ut_free(prebuilt->mysql_template); diff --git a/storage/innobase/row/row0query.cc b/storage/innobase/row/row0query.cc new file mode 100644 index 0000000000000..571d21c0d9edf --- /dev/null +++ b/storage/innobase/row/row0query.cc @@ -0,0 +1,445 @@ +/***************************************************************************** + +Copyright (c) 2025, MariaDB PLC. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file row/row0query.cc +General Query Executor + +Created 2025/10/30 +*******************************************************/ + +#include "row0query.h" +#include "pars0pars.h" +#include "dict0dict.h" +#include "row0ins.h" +#include "row0upd.h" +#include "row0row.h" +#include "row0vers.h" +#include "row0sel.h" +#include "mem0mem.h" +#include "que0que.h" +#include "lock0lock.h" +#include "rem0rec.h" +#include "btr0pcur.h" +#include "btr0cur.h" + +QueryExecutor::QueryExecutor(trx_t *trx) : m_trx(trx), m_mtr(nullptr) +{ + m_heap= mem_heap_create(256); + m_thr= pars_complete_graph_for_exec(nullptr, trx, m_heap, nullptr); + btr_pcur_init(&m_pcur); + m_clust_pcur= nullptr; + m_prebuilt= nullptr; +} + +QueryExecutor::~QueryExecutor() +{ + btr_pcur_close(&m_pcur); + if (m_clust_pcur) + btr_pcur_close(m_clust_pcur); + + if (m_prebuilt) + { + if (m_prebuilt->old_vers_heap) + mem_heap_free(m_prebuilt->old_vers_heap); + + ut_ad(!m_prebuilt->blob_heap); + ut_ad(!m_prebuilt->fetch_cache[0]); + ut_ad(!m_prebuilt->rtr_info); + ut_ad(!m_prebuilt->mysql_template); + + m_prebuilt= nullptr; + } + + if (m_heap) mem_heap_free(m_heap); +} + +void QueryExecutor::setup_prebuilt( + dict_table_t *table, dict_index_t *index, + const dtuple_t *tuple, page_cur_mode_t mode) noexcept +{ + dict_index_t *clust_index= dict_table_get_first_index(table); + ulint ref_len= dict_index_get_n_unique(clust_index); + + if (!m_prebuilt) + { + m_prebuilt= static_cast( + mem_heap_zalloc(m_heap, sizeof(*m_prebuilt))); + + m_prebuilt->magic_n= ROW_PREBUILT_ALLOCATED; + m_prebuilt->magic_n2= ROW_PREBUILT_ALLOCATED; + + m_prebuilt->table= table; + m_prebuilt->heap= m_heap; + + m_prebuilt->sel_graph= static_cast( + que_node_get_parent(m_thr)); + m_prebuilt->sql_stat_start= TRUE; + m_prebuilt->in_fts_query= true; + ulint search_tuple_n_fields= 2 * (dict_table_get_n_cols(table) + + dict_table_get_n_v_cols(table)); + m_prebuilt->search_tuple= + dtuple_create(m_heap, search_tuple_n_fields); + m_prebuilt->clust_ref= dtuple_create(m_heap, ref_len); + dict_index_copy_types(m_prebuilt->clust_ref, clust_index, ref_len); + } + else if (m_prebuilt->table != table) + { + m_prebuilt->table= table; + dtuple_t* ref= dtuple_create(m_heap, ref_len); + dict_index_copy_types(ref, clust_index, ref_len); + m_prebuilt->clust_ref= ref; + btr_pcur_reset(&m_pcur); + if (m_clust_pcur) + btr_pcur_reset(m_clust_pcur); + m_prebuilt->fts_doc_id= 0; + } + + /* Configure prebuilt for search */ + m_prebuilt->trx= m_trx; + m_prebuilt->index= index; + if (!tuple) + m_prebuilt->search_tuple= nullptr; + else + m_prebuilt->search_tuple= const_cast(tuple); + + m_prebuilt->pcur= &m_pcur; + if (!index->is_clust()) + { + if (!m_clust_pcur) + { + m_clust_pcur= static_cast( + mem_heap_zalloc(m_prebuilt->heap, sizeof(btr_pcur_t))); + btr_pcur_init(m_clust_pcur); + } + } + m_prebuilt->clust_pcur= m_clust_pcur; + + m_prebuilt->mtr= nullptr; + m_prebuilt->template_type= ROW_MYSQL_NO_TEMPLATE; + m_prebuilt->select_lock_type= LOCK_NONE; + m_prebuilt->row_read_type= 0; + m_prebuilt->n_fetch_cached= 0; + m_prebuilt->fetch_cache_first= 0; + m_prebuilt->index_usable= 1; + m_prebuilt->need_to_access_clustered= index->is_clust() ? 0 : 1; + m_prebuilt->in_fts_query= true; +} + +dberr_t QueryExecutor::insert_record(dict_table_t *table, + dtuple_t *tuple) noexcept +{ + dict_index_t* index= dict_table_get_first_index(table); + return row_ins_clust_index_entry(index, tuple, m_thr, 0); +} + +dberr_t QueryExecutor::lock_table(dict_table_t *table, lock_mode mode) noexcept +{ + ut_ad(m_trx); + trx_start_if_not_started(m_trx, true); + return ::lock_table(table, nullptr, mode, m_thr); +} + +dberr_t QueryExecutor::handle_wait(dberr_t err, bool table_lock) noexcept +{ + ut_ad(m_trx); + m_trx->error_state= err; + if (table_lock) m_thr->lock_state= QUE_THR_LOCK_TABLE; + else m_thr->lock_state= QUE_THR_LOCK_ROW; + if (m_trx->lock.wait_thr) + { + dberr_t wait_err= lock_wait(m_thr); + if (wait_err == DB_LOCK_WAIT_TIMEOUT) err= wait_err; + if (wait_err == DB_SUCCESS) + { + m_thr->lock_state= QUE_THR_LOCK_NOLOCK; + return DB_SUCCESS; + } + } + return err; +} + +dberr_t QueryExecutor::delete_record(dict_table_t *table, + dtuple_t *tuple) noexcept +{ + dict_index_t *clust_index= dict_table_get_first_index(table); + ut_ad(m_mtr); + + /* Use select_for_update to find and lock the record with proper MVCC handling */ + dberr_t err= select_for_update(table, tuple, nullptr); + if (err != DB_SUCCESS) + return err; + + /* Record is now locked at m_pcur, mark it as deleted */ + rec_t *rec= btr_pcur_get_rec(&m_pcur); + rec_offs *offsets= rec_get_offsets(rec, clust_index, nullptr, + clust_index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + + err= btr_cur_del_mark_set_clust_rec(btr_pcur_get_block(&m_pcur), + rec, clust_index, offsets, m_thr, + nullptr, m_mtr); + + m_mtr->commit(); + return err; +} + +dberr_t QueryExecutor::delete_all(dict_table_t *table) noexcept +{ + dict_index_t *clust_index= dict_table_get_first_index(table); + dberr_t err= DB_SUCCESS; + + if (!m_mtr) + m_mtr= new (mem_heap_alloc(m_heap, sizeof(mtr_t))) mtr_t(m_trx); +retry: + m_mtr->start(); + m_mtr->set_named_space(table->space); + + err= m_pcur.open_leaf(true, clust_index, BTR_MODIFY_LEAF, m_mtr); + if (err != DB_SUCCESS || !btr_pcur_move_to_next(&m_pcur, m_mtr)) + { + m_mtr->commit(); + return err; + } + + while (!btr_pcur_is_after_last_on_page(&m_pcur) && + !btr_pcur_is_after_last_in_tree(&m_pcur)) + { + rec_t* rec= btr_pcur_get_rec(&m_pcur); + rec_offs *offsets= nullptr; + if (rec_get_deleted_flag(rec, dict_table_is_comp(table))) + goto next_rec; + if (rec_get_info_bits( + rec, dict_table_is_comp(table)) & REC_INFO_MIN_REC_FLAG) + goto next_rec; + + offsets= rec_get_offsets(rec, clust_index, nullptr, + clust_index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + err= lock_clust_rec_read_check_and_lock( + 0, btr_pcur_get_block(&m_pcur), rec, clust_index, offsets, LOCK_X, + LOCK_REC_NOT_GAP, m_thr); + + if (err == DB_LOCK_WAIT) + { + m_mtr->commit(); + err= handle_wait(err, false); + if (err != DB_SUCCESS) + return err; + goto retry; + } + else if (err != DB_SUCCESS && err != DB_SUCCESS_LOCKED_REC) + { + m_mtr->commit(); + return err; + } + + err= btr_cur_del_mark_set_clust_rec(btr_pcur_get_block(&m_pcur), + const_cast(rec), clust_index, + offsets, m_thr, nullptr, m_mtr); + if (err) + break; +next_rec: + if (!btr_pcur_move_to_next(&m_pcur, m_mtr)) + break; + } + + m_mtr->commit(); + return err; +} + +dberr_t QueryExecutor::select_for_update(dict_table_t *table, + dtuple_t *search_tuple, + RecordCallback *callback) noexcept +{ + ut_ad(m_trx); + dict_index_t *clust_index= dict_table_get_first_index(table); + setup_prebuilt(table, clust_index, search_tuple, PAGE_CUR_GE); + m_prebuilt->select_lock_type= LOCK_X; + + if (!m_mtr) + m_mtr= new (mem_heap_alloc(m_heap, sizeof(mtr_t))) mtr_t(m_trx); + + m_mtr->start(); + m_mtr->set_named_space(table->space); + + if (m_trx && !m_trx->read_view.is_open()) + { + trx_start_if_not_started(m_trx, false); + m_trx->read_view.open(m_trx); + } + + /* Provide external mtr for DML operation */ + m_prebuilt->mtr= m_mtr; + + dberr_t err= DB_SUCCESS; + + if (callback) + err= row_search_mvcc_callback_dml( + callback, PAGE_CUR_GE, m_prebuilt, 0, 0); + else + err= row_search_mvcc(nullptr, PAGE_CUR_GE, + m_prebuilt, 1, 0); + + if (err == DB_LOCK_WAIT) + { + m_mtr->commit(); + err= handle_wait(err, false); + if (err != DB_SUCCESS) + return err; + return DB_LOCK_WAIT; + } + + if (err == DB_END_OF_INDEX || err == DB_RECORD_NOT_FOUND) + err= DB_RECORD_NOT_FOUND; + else if (err == DB_SUCCESS_LOCKED_REC) + err= DB_SUCCESS; + + if (err != DB_SUCCESS) + m_mtr->commit(); + return err; +} + +dberr_t QueryExecutor::update_record(dict_table_t *table, + const upd_t *update) noexcept +{ + ut_ad(m_trx); + ut_ad(m_mtr); + dict_index_t *clust_index= dict_table_get_first_index(table); + rec_t *rec= btr_pcur_get_rec(&m_pcur); + mtr_x_lock_index(clust_index, m_mtr); + rec_offs *offsets= rec_get_offsets(rec, clust_index, nullptr, + clust_index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + + dberr_t err= DB_SUCCESS; + ulint cmpl_info= UPD_NODE_NO_ORD_CHANGE | UPD_NODE_NO_SIZE_CHANGE; + for (ulint i= 0; i < update->n_fields; i++) + { + const upd_field_t *upd_field= &update->fields[i]; + ulint field_no= upd_field->field_no; + if (field_no < rec_offs_n_fields(offsets)) + { + ulint old_len= rec_offs_nth_size(offsets, field_no); + ulint new_len= upd_field->new_val.len; + if (new_len != UNIV_SQL_NULL && new_len != old_len) + { + cmpl_info &= ~UPD_NODE_NO_SIZE_CHANGE; + err= DB_OVERFLOW; + break; + } + } + } + + if (cmpl_info & UPD_NODE_NO_SIZE_CHANGE) + err= btr_cur_update_in_place(BTR_NO_LOCKING_FLAG, + btr_pcur_get_btr_cur(&m_pcur), + offsets, const_cast(update), 0, + m_thr, m_trx->id, m_mtr); + if (err == DB_OVERFLOW) + { + big_rec_t *big_rec= nullptr; + err= btr_cur_optimistic_update(BTR_NO_LOCKING_FLAG, + btr_pcur_get_btr_cur(&m_pcur), + &offsets, &m_heap, + const_cast(update), + cmpl_info, m_thr, m_trx->id, m_mtr); + + if (err == DB_OVERFLOW || err == DB_UNDERFLOW) + { + mem_heap_t* offsets_heap= nullptr; + err= btr_cur_pessimistic_update(BTR_NO_LOCKING_FLAG, + btr_pcur_get_btr_cur(&m_pcur), + &offsets, &offsets_heap, m_heap, + &big_rec, const_cast(update), + cmpl_info, m_thr, m_trx->id, m_mtr); + + if (err == DB_SUCCESS && big_rec) + { + err= btr_store_big_rec_extern_fields(&m_pcur, offsets, big_rec, m_mtr, + BTR_STORE_UPDATE); + dtuple_big_rec_free(big_rec); + } + if (offsets_heap) mem_heap_free(offsets_heap); + } + } + return err; +} + +dberr_t QueryExecutor::replace_record( + dict_table_t *table, dtuple_t *search_tuple, + const upd_t *update, dtuple_t *insert_tuple) noexcept +{ +retry_again: + dberr_t err= select_for_update(table, search_tuple); + if (err == DB_SUCCESS) + { + err= update_record(table, update); + m_mtr->commit(); + return err; + } + else if (err == DB_RECORD_NOT_FOUND) + { + err= insert_record(table, insert_tuple); + return err; + } + else if (err == DB_LOCK_WAIT) + goto retry_again; + return err; +} + +dberr_t QueryExecutor::read_by_index(dict_table_t *table, + dict_index_t *index, + const dtuple_t *search_tuple, + page_cur_mode_t mode, + RecordCallback& callback, + bool read_all) noexcept +{ + ut_ad(table); + ut_ad(index); + ut_ad(callback.compare_record); + + setup_prebuilt(table, index, search_tuple, mode); + + if (m_trx && !m_trx->read_view.is_open()) + { + trx_start_if_not_started(m_trx, false); + m_trx->read_view.open(m_trx); + } + dberr_t err= row_search_mvcc_callback(&callback, mode, m_prebuilt, 0, 0); + if (read_all) + { + while (err == DB_SUCCESS) + err= row_search_mvcc_callback(&callback, mode, m_prebuilt, 0, + ROW_SEL_NEXT); + } + if (err == DB_END_OF_INDEX || err == DB_SUCCESS_LOCKED_REC) + err= DB_SUCCESS; + return err; +} + +dberr_t QueryExecutor::read(dict_table_t *table, const dtuple_t *tuple, + page_cur_mode_t mode, + RecordCallback& callback) noexcept +{ + ut_ad(table); + ut_ad(callback.compare_record); + + dict_index_t *clust_index= dict_table_get_first_index(table); + return read_by_index(table, clust_index, tuple, mode, callback, false); +} diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index c063c3b34b810..c8db9e477d88b 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -3341,6 +3341,7 @@ class Row_sel_get_clust_rec_for_mysql dtuple_t **vrow, mtr_t *mtr); }; + /*********************************************************************//** Retrieves the clustered index record corresponding to a record in a non-clustered index. Does the necessary locking. Used in the MySQL @@ -3349,7 +3350,7 @@ interface. dberr_t Row_sel_get_clust_rec_for_mysql::operator()( /*============================*/ - row_prebuilt_t* prebuilt,/*!< in: prebuilt struct in the handle */ + row_prebuilt_t* prebuilt,/*!< in: prebuilt struct */ dict_index_t* sec_index,/*!< in: secondary index where rec resides */ const rec_t* rec, /*!< in: record in a non-clustered index; if this is a locking read, then rec is not @@ -3371,38 +3372,42 @@ Row_sel_get_clust_rec_for_mysql::operator()( non-clustered record; the same mtr is used to access the clustered index */ { + /* Extract values from prebuilt */ + dtuple_t* clust_ref= prebuilt->clust_ref; + btr_pcur_t* clust_pcur= prebuilt->clust_pcur; + lock_mode select_lock_type= prebuilt->select_lock_type; + trx_t* trx= prebuilt->trx; + dict_index_t* clust_index; rec_t* old_vers; - trx_t* trx; - prebuilt->clust_pcur->old_rec = nullptr; + clust_pcur->old_rec = nullptr; *out_rec = NULL; - trx = thr_get_trx(thr); - row_build_row_ref_in_tuple(prebuilt->clust_ref, rec, + row_build_row_ref_in_tuple(clust_ref, rec, sec_index, *offsets); clust_index = dict_table_get_first_index(sec_index->table); - prebuilt->clust_pcur->btr_cur.page_cur.index = clust_index; + clust_pcur->btr_cur.page_cur.index = clust_index; - dberr_t err = btr_pcur_open_with_no_init(prebuilt->clust_ref, + dberr_t err = btr_pcur_open_with_no_init(clust_ref, PAGE_CUR_LE, BTR_SEARCH_LEAF, - prebuilt->clust_pcur, mtr); + clust_pcur, mtr); if (UNIV_UNLIKELY(err != DB_SUCCESS)) { return err; } - const rec_t* clust_rec = btr_pcur_get_rec(prebuilt->clust_pcur); + const rec_t* clust_rec = btr_pcur_get_rec(clust_pcur); - prebuilt->clust_pcur->trx_if_known = trx; + clust_pcur->trx_if_known = trx; /* Note: only if the search ends up on a non-infimum record is the low_match value the real match to the search tuple */ if (!page_rec_is_user_rec(clust_rec) - || btr_pcur_get_low_match(prebuilt->clust_pcur) + || btr_pcur_get_low_match(clust_pcur) < dict_index_get_n_unique(clust_index)) { - btr_cur_t* btr_cur = btr_pcur_get_btr_cur(prebuilt->pcur); + btr_cur_t* btr_cur = btr_pcur_get_btr_cur(clust_pcur); /* If this is a spatial index scan, and we are reading from a shadow buffer, the record could be already @@ -3413,7 +3418,7 @@ Row_sel_get_clust_rec_for_mysql::operator()( && (!(ulint(rec - btr_cur->rtr_info->matches->block->page.frame) >> srv_page_size_shift) - || rec != btr_pcur_get_rec(prebuilt->pcur))) { + || rec != btr_pcur_get_rec(clust_pcur))) { #ifdef UNIV_DEBUG rtr_info_t* rtr_info = btr_cur->rtr_info; mysql_mutex_lock(&rtr_info->matches->rtr_match_mutex); @@ -3427,23 +3432,23 @@ Row_sel_get_clust_rec_for_mysql::operator()( if (rec_get_deleted_flag(rec, dict_table_is_comp(sec_index->table)) - && prebuilt->select_lock_type == LOCK_NONE) { + && select_lock_type == LOCK_NONE) { clust_rec = NULL; goto func_exit; } - if (rec != btr_pcur_get_rec(prebuilt->pcur)) { + if (rec != btr_pcur_get_rec(clust_pcur)) { clust_rec = NULL; goto func_exit; } /* FIXME: Why is this block not the - same as btr_pcur_get_block(prebuilt->pcur), + same as btr_pcur_get_block(clust_pcur), and is it not unsafe to use RW_NO_LATCH here? */ buf_block_t* block = buf_page_get_gen( - btr_pcur_get_block(prebuilt->pcur)->page.id(), - btr_pcur_get_block(prebuilt->pcur)->zip_size(), + btr_pcur_get_block(clust_pcur)->page.id(), + btr_pcur_get_block(clust_pcur)->zip_size(), RW_NO_LATCH, NULL, BUF_GET, mtr, &err); ut_ad(block); // FIXME: avoid crash mem_heap_t* heap = mem_heap_create(256); @@ -3465,7 +3470,7 @@ Row_sel_get_clust_rec_for_mysql::operator()( #endif /* UNIV_DEBUG */ } else if (!rec_get_deleted_flag(rec, dict_table_is_comp(sec_index->table)) - || prebuilt->select_lock_type != LOCK_NONE) { + || select_lock_type != LOCK_NONE) { /* In a rare case it is possible that no clust rec is found for a delete-marked secondary index record: if row_undo_mod_clust() has already removed @@ -3494,15 +3499,15 @@ Row_sel_get_clust_rec_for_mysql::operator()( clust_index->n_core_fields, ULINT_UNDEFINED, offset_heap); - if (prebuilt->select_lock_type != LOCK_NONE) { + if (select_lock_type != LOCK_NONE) { /* Try to place a lock on the index record; we are searching the clust rec with a unique condition, hence we set a LOCK_REC_NOT_GAP type lock */ err = lock_clust_rec_read_check_and_lock( - 0, btr_pcur_get_block(prebuilt->clust_pcur), + 0, btr_pcur_get_block(clust_pcur), clust_rec, clust_index, *offsets, - prebuilt->select_lock_type, + select_lock_type, LOCK_REC_NOT_GAP, thr); @@ -3536,7 +3541,7 @@ Row_sel_get_clust_rec_for_mysql::operator()( break; case DB_SUCCESS_LOCKED_REC: const buf_page_t& bpage = btr_pcur_get_block( - prebuilt->clust_pcur)->page; + clust_pcur)->page; const lsn_t lsn = mach_read_from_8( bpage.frame + FIL_PAGE_LSN); @@ -3548,8 +3553,8 @@ Row_sel_get_clust_rec_for_mysql::operator()( 'old_vers' */ err = row_sel_build_prev_vers_for_mysql( prebuilt, clust_index, - clust_rec, offsets, offset_heap, &old_vers, - vrow, mtr); + clust_rec, offsets, offset_heap, + &old_vers, vrow, mtr); if (UNIV_UNLIKELY(err != DB_SUCCESS)) { return err; @@ -3622,11 +3627,11 @@ Row_sel_get_clust_rec_for_mysql::operator()( func_exit: *out_rec = clust_rec; - if (prebuilt->select_lock_type != LOCK_NONE) { + if (select_lock_type != LOCK_NONE) { /* We may use the cursor in update or in unlock_row(): store its position */ - btr_pcur_store_position(prebuilt->clust_pcur, mtr); + btr_pcur_store_position(clust_pcur, mtr); } return err; @@ -4320,6 +4325,75 @@ bool row_search_with_covering_prefix( return true; } +/** Templated policy for row_search_mvcc behavior. +@tparam NeedsConversion true if records need MySQL format conversion +@tparam ManagesMtr true if row_search_mvcc manages its own mtr */ +template +struct InnoDBPolicy +{ + static constexpr bool needs_conversion= NeedsConversion; + static constexpr bool manages_mtr= ManagesMtr; + + /** Convert a record to MySQL format + (only when NeedsConversion=true). + @param[out] mysql_buf MySQL row buffer + @param[in] prebuilt prebuilt structure + @param[in] rec record to convert + @param[in] vrow virtual column values (can be NULL) + @param[in] rec_clust true if rec is from clustered index + @param[in] index index of the record + @param[in] offsets record offsets + @return DB_SUCCESS on success + @retval DB_ERROR if conversion fails */ + static dberr_t output_record(byte* mysql_buf, + row_prebuilt_t* prebuilt, + const rec_t* rec, + const dtuple_t* vrow, + bool rec_clust, + dict_index_t* index, + rec_offs* offsets) noexcept + { + static_assert(NeedsConversion, "MySQL format conversion only"); + if (!row_sel_store_mysql_rec(mysql_buf, prebuilt, rec, vrow, + rec_clust, index, offsets)) + return DB_ERROR; + return DB_SUCCESS; + } + + /** Process a record via RecordCallback interface + @param[in] callback RecordCallback instance (can be NULL) + @param[in] prebuilt prebuilt structure with search tuple + @param[in] rec record to process + @param[in] index index of the record + @param[in] offsets record offsets + @return DB_SUCCESS on success + @retval DB_SUCCESS_LOCKED_REC to stop iteration after this record + @retval DB_RECORD_NOT_FOUND to skip this record + @retval error code on failure */ + static dberr_t output_record(RecordCallback* callback, + row_prebuilt_t* prebuilt, + const rec_t* rec, + dict_index_t* index, + rec_offs* offsets) noexcept + { + static_assert(!NeedsConversion, "InnoDB callback only"); + if (!callback) + return DB_SUCCESS; + + RecordCompareAction action= callback->compare_record( + prebuilt->search_tuple, rec, index, offsets); + + if (action == RecordCompareAction::PROCESS) + return callback->process_record(rec, index, offsets); + else if (action == RecordCompareAction::SKIP) + return DB_RECORD_NOT_FOUND; + else if (action == RecordCompareAction::STOP) + return DB_SUCCESS_LOCKED_REC; + + return DB_SUCCESS; + } +}; + /** Searches for rows in the database using cursor. Function is mainly used for tables that are shared across connections and so it employs technique that can help re-construct the rows that @@ -4339,6 +4413,7 @@ It also has optimization such as pre-caching the rows, using AHI, etc. pcur with stored position! In opening of a cursor 'direction' should be 0. @return DB_SUCCESS or error code */ +template dberr_t row_search_mvcc( byte* buf, @@ -4374,7 +4449,7 @@ row_search_mvcc( byte* next_buf = 0; bool spatial_search = false; - ut_ad(index && pcur && search_tuple); + ut_ad(index && pcur); ut_a(prebuilt->magic_n == ROW_PREBUILT_ALLOCATED); ut_a(prebuilt->magic_n2 == ROW_PREBUILT_ALLOCATED); @@ -4521,8 +4596,21 @@ row_search_mvcc( /* if the query is a plain locking SELECT, and the isolation level is <= TRX_ISO_READ_COMMITTED, then this is set to FALSE */ bool did_semi_consistent_read = false; - mtr_t mtr{trx}; - mtr.start(); + + /* Conditionally manage mtr based on policy */ + mtr_t local_mtr{trx}; + mtr_t* mtr; + + if constexpr (Callback::manages_mtr) { + /* row_search_mvcc manages its own mtr */ + mtr = &local_mtr; + mtr->start(); + } else { + /* Use external mtr from prebuilt (internal parser) */ + mtr = prebuilt->mtr; + ut_ad(mtr); + ut_ad(mtr->is_active()); + } mem_heap_t* heap = NULL; rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; @@ -4562,7 +4650,7 @@ row_search_mvcc( dberr_t err = DB_SUCCESS; switch (row_sel_try_search_shortcut_for_mysql( &rec, prebuilt, &offsets, &heap, - &mtr)) { + mtr)) { case SEL_FOUND: /* At this point, rec is protected by a page latch that was acquired by @@ -4616,7 +4704,9 @@ row_search_mvcc( case SEL_EXHAUSTED: err = DB_RECORD_NOT_FOUND; shortcut_done: - mtr.commit(); + if constexpr (Callback::manages_mtr) { + mtr->commit(); + } /* NOTE that we do NOT store the cursor position */ @@ -4634,8 +4724,10 @@ row_search_mvcc( ut_ad(0); } - mtr.commit(); - mtr.start(); + if constexpr (Callback::manages_mtr) { + mtr->commit(); + mtr->start(); + } } } #endif /* BTR_CUR_HASH_ADAPT */ @@ -4739,11 +4831,13 @@ row_search_mvcc( bool need_to_process = sel_restore_position_for_mysql( &same_user_rec, BTR_SEARCH_LEAF, - pcur, moves_up, &mtr); + pcur, moves_up, mtr); if (UNIV_UNLIKELY(need_to_process)) { if (UNIV_UNLIKELY(!btr_pcur_get_rec(pcur))) { - mtr.commit(); + if constexpr (Callback::manages_mtr) { + mtr->commit(); + } trx->op_info = ""; if (UNIV_LIKELY_NULL(heap)) { mem_heap_free(heap); @@ -4771,7 +4865,7 @@ row_search_mvcc( goto next_rec_after_check; } - } else if (dtuple_get_n_fields(search_tuple) > 0) { + } else if (search_tuple && dtuple_get_n_fields(search_tuple) > 0) { pcur->old_rec = nullptr; if (index->is_spatial()) { @@ -4792,11 +4886,15 @@ row_search_mvcc( } err = rtr_search_leaf(pcur, thr, search_tuple, mode, - &mtr); + mtr); } else { + /* For external mtr (DML operations), use BTR_MODIFY_LEAF + to get X-latch on page for subsequent update */ + btr_latch_mode latch_mode = Callback::manages_mtr + ? BTR_SEARCH_LEAF : BTR_MODIFY_LEAF; err = btr_pcur_open_with_no_init(search_tuple, mode, - BTR_SEARCH_LEAF, - pcur, &mtr); + latch_mode, + pcur, mtr); } if (err != DB_SUCCESS) { @@ -4829,7 +4927,7 @@ row_search_mvcc( err = sel_set_rec_lock(pcur, next_rec, index, offsets, prebuilt->select_lock_type, - LOCK_GAP, thr, &mtr); + LOCK_GAP, thr, mtr); switch (err) { case DB_SUCCESS_LOCKED_REC: @@ -4842,8 +4940,10 @@ row_search_mvcc( } } } else if (mode == PAGE_CUR_G || mode == PAGE_CUR_L) { + btr_latch_mode latch_mode = Callback::manages_mtr + ? BTR_SEARCH_LEAF : BTR_MODIFY_LEAF; err = pcur->open_leaf(mode == PAGE_CUR_G, index, - BTR_SEARCH_LEAF, &mtr); + latch_mode, mtr); if (err != DB_SUCCESS) { if (err == DB_DECRYPTION_FAILED) { @@ -4907,7 +5007,7 @@ row_search_mvcc( DEBUG_SYNC_C("row_search_rec_loop"); if (trx_is_interrupted(trx)) { if (!spatial_search) { - btr_pcur_store_position(pcur, &mtr); + btr_pcur_store_position(pcur, mtr); } err = DB_INTERRUPTED; goto normal_return; @@ -4948,7 +5048,7 @@ row_search_mvcc( err = sel_set_rec_lock(pcur, rec, index, offsets, prebuilt->select_lock_type, - LOCK_ORDINARY, thr, &mtr); + LOCK_ORDINARY, thr, mtr); switch (err) { case DB_SUCCESS_LOCKED_REC: @@ -5087,7 +5187,7 @@ row_search_mvcc( pcur, rec, index, offsets, prebuilt->select_lock_type, LOCK_GAP, - thr, &mtr); + thr, mtr); switch (err) { case DB_SUCCESS_LOCKED_REC: @@ -5098,15 +5198,18 @@ row_search_mvcc( } } - btr_pcur_store_position(pcur, &mtr); - - /* The found record was not a match, but may be used - as NEXT record (index_next). Set the relative position - to BTR_PCUR_BEFORE, to reflect that the position of - the persistent cursor is before the found/stored row - (pcur->old_rec). */ - ut_ad(pcur->rel_pos == BTR_PCUR_ON); - pcur->rel_pos = BTR_PCUR_BEFORE; + if constexpr (Callback::manages_mtr) { + btr_pcur_store_position(pcur, mtr); + + /* The found record was not a match, + but may be used as NEXT record (index_next). + Set the relative position to BTR_PCUR_BEFORE, + to reflect that the position of the persistent + cursor is before the found/stored row + (pcur->old_rec). */ + ut_ad(pcur->rel_pos == BTR_PCUR_ON); + pcur->rel_pos = BTR_PCUR_BEFORE; + } err = DB_RECORD_NOT_FOUND; goto normal_return; @@ -5123,7 +5226,7 @@ row_search_mvcc( pcur, rec, index, offsets, prebuilt->select_lock_type, LOCK_GAP, - thr, &mtr); + thr, mtr); switch (err) { case DB_SUCCESS_LOCKED_REC: @@ -5134,15 +5237,17 @@ row_search_mvcc( } } - btr_pcur_store_position(pcur, &mtr); - - /* The found record was not a match, but may be used - as NEXT record (index_next). Set the relative position - to BTR_PCUR_BEFORE, to reflect that the position of - the persistent cursor is before the found/stored row - (pcur->old_rec). */ - ut_ad(pcur->rel_pos == BTR_PCUR_ON); - pcur->rel_pos = BTR_PCUR_BEFORE; + if constexpr (Callback::manages_mtr) { + btr_pcur_store_position(pcur, mtr); + /* The found record was not a match, + but may be used as NEXT record (index_next). + Set the relative position to BTR_PCUR_BEFORE, + to reflect that the position of the persistent + cursor is before the found/stored row + (pcur->old_rec). */ + ut_ad(pcur->rel_pos == BTR_PCUR_ON); + pcur->rel_pos = BTR_PCUR_BEFORE; + } err = DB_RECORD_NOT_FOUND; goto normal_return; @@ -5245,7 +5350,7 @@ row_search_mvcc( err = sel_set_rec_lock(pcur, rec, index, offsets, prebuilt->select_lock_type, - lock_type, thr, &mtr); + lock_type, thr, mtr); switch (err) { const rec_t* old_vers; @@ -5292,7 +5397,7 @@ row_search_mvcc( row_sel_build_committed_vers_for_mysql( clust_index, prebuilt, rec, &offsets, &heap, &old_vers, - need_vrow ? &vrow : NULL, &mtr); + need_vrow ? &vrow : NULL, mtr); } /* Check whether it was a deadlock or not, if not @@ -5393,8 +5498,9 @@ row_search_mvcc( associated with 'old_vers' */ err = row_sel_build_prev_vers_for_mysql( prebuilt, clust_index, - rec, &offsets, &heap, &old_vers, - need_vrow ? &vrow : nullptr, &mtr); + rec, &offsets, &heap, + &old_vers, + need_vrow ? &vrow : nullptr, mtr); if (err != DB_SUCCESS) { @@ -5523,7 +5629,7 @@ row_search_mvcc( /* It was a non-clustered index and we must fetch also the clustered index record */ - mtr_extra_clust_savepoint = mtr.get_savepoint(); + mtr_extra_clust_savepoint = mtr->get_savepoint(); ut_ad(!vrow); /* The following call returns 'offsets' associated with @@ -5534,7 +5640,8 @@ row_search_mvcc( thr, &clust_rec, &offsets, &heap, need_vrow ? &vrow : NULL, - &mtr); + mtr); + if (err == DB_LOCK_WAIT && prebuilt->skip_locked) { err = lock_trx_handle_wait(trx); } @@ -5608,10 +5715,30 @@ row_search_mvcc( index may be in the wrong case, and the authoritative case is in result_rec, the appropriate version of the clustered index record. */ - if (!row_sel_store_mysql_rec( - buf, prebuilt, result_rec, vrow, - true, clust_index, offsets)) { - goto next_rec; + if constexpr (Callback::needs_conversion) { + /* MySQL format path */ + err= MySQLRowCallback::output_record( + buf, prebuilt, result_rec, vrow, + true, clust_index, offsets); + + if (err != DB_SUCCESS) + goto next_rec; + } else { + /* InnoDB callback path */ + err= InnoDBReadPolicy::output_record( + reinterpret_cast(buf), + prebuilt, result_rec, clust_index, offsets); + + if (err == DB_FTS_EXCEED_RESULT_CACHE_LIMIT) { + goto normal_return; + } else if (err == DB_SUCCESS_LOCKED_REC) { + err= DB_SUCCESS; + goto idx_cond_failed; + } else if (err == DB_RECORD_NOT_FOUND) { + goto next_rec; + } else if (err != DB_SUCCESS) { + goto lock_wait_or_error; + } } } } else { @@ -5653,89 +5780,132 @@ row_search_mvcc( /* We only convert from InnoDB row format to MySQL row format when ICP is disabled. */ - if (!prebuilt->pk_filter && !prebuilt->idx_cond) { - /* We use next_buf to track the allocation of buffers - where we store and enqueue the buffers for our - pre-fetch optimisation. + if constexpr (Callback::needs_conversion) { + /* MySQL format path with prefetch */ + if (!prebuilt->pk_filter && !prebuilt->idx_cond) { + /* We use next_buf to track the allocation of buffers + where we store and enqueue the buffers for our + pre-fetch optimisation. - If next_buf == 0 then we store the converted record - directly into the MySQL record buffer (buf). If it is - != 0 then we allocate a pre-fetch buffer and store the - converted record there. + If next_buf == 0 then we store the converted record + directly into the MySQL record buffer (buf). If it is + != 0 then we allocate a pre-fetch buffer and store the + converted record there. - If the conversion fails and the MySQL record buffer - was not written to then we reset next_buf so that - we can re-use the MySQL record buffer in the next - iteration. */ + If the conversion fails and the MySQL record buffer + was not written to then we reset next_buf so that + we can re-use the MySQL record buffer in the next + iteration. */ - next_buf = next_buf - ? row_sel_fetch_last_buf(prebuilt) : buf; + next_buf = next_buf + ? row_sel_fetch_last_buf(prebuilt) : buf; - if (!row_sel_store_mysql_rec( - next_buf, prebuilt, result_rec, vrow, - result_rec != rec, - result_rec != rec ? clust_index : index, - offsets)) { + err= MySQLRowCallback::output_record( + next_buf, prebuilt, result_rec, vrow, + result_rec != rec, + result_rec != rec ? clust_index : index, + offsets); - if (next_buf == buf) { - ut_a(prebuilt->n_fetch_cached == 0); - next_buf = 0; - } + if (err != DB_SUCCESS) { + if (next_buf == buf) { + ut_a(prebuilt->n_fetch_cached == 0); + next_buf = 0; + } - /* Only fresh inserts may contain incomplete - externally stored columns. Pretend that such - records do not exist. Such records may only be - accessed at the READ UNCOMMITTED isolation - level or when rolling back a recovered - transaction. Rollback happens at a lower - level, not here. */ - goto next_rec; - } + /* Only fresh inserts may contain incomplete + externally stored columns. Pretend that such + records do not exist. Such records may only be + accessed at the READ UNCOMMITTED isolation + level or when rolling back a recovered + transaction. Rollback happens at a lower + level, not here. */ + goto next_rec; + } - if (next_buf != buf) { - row_sel_enqueue_cache_row_for_mysql( - next_buf, prebuilt); + if (next_buf != buf) { + row_sel_enqueue_cache_row_for_mysql( + next_buf, prebuilt); + } + } else { + row_sel_enqueue_cache_row_for_mysql(buf, prebuilt); } } else { - row_sel_enqueue_cache_row_for_mysql(buf, prebuilt); + /* InnoDB callback path - no prefetch, direct processing */ + err= InnoDBReadPolicy::output_record( + reinterpret_cast(buf), + prebuilt, result_rec, + result_rec != rec ? clust_index : index, + offsets); + + if (err == DB_FTS_EXCEED_RESULT_CACHE_LIMIT) { + goto normal_return; + } else if (err == DB_RECORD_NOT_FOUND) { + goto next_rec; + } else if (err != DB_SUCCESS && + err != DB_SUCCESS_LOCKED_REC) { + goto lock_wait_or_error; + } } if (prebuilt->n_fetch_cached < MYSQL_FETCH_CACHE_SIZE) { goto next_rec; } } else { - if (!prebuilt->pk_filter && !prebuilt->idx_cond) { - /* The record was not yet converted to MySQL format. */ - if (!row_sel_store_mysql_rec( - buf, prebuilt, result_rec, vrow, - result_rec != rec, - result_rec != rec ? clust_index : index, - offsets)) { - /* Only fresh inserts may contain - incomplete externally stored - columns. Pretend that such records do - not exist. Such records may only be - accessed at the READ UNCOMMITTED - isolation level or when rolling back a - recovered transaction. Rollback - happens at a lower level, not here. */ - goto next_rec; + if constexpr (Callback::needs_conversion) { + /* MySQL format path */ + if (!prebuilt->pk_filter && !prebuilt->idx_cond) { + /* The record was not yet converted to MySQL format. */ + err= MySQLRowCallback::output_record( + buf, prebuilt, result_rec, vrow, + result_rec != rec, + result_rec != rec ? clust_index : index, + offsets); + + if (err != DB_SUCCESS) { + /* Only fresh inserts may contain + incomplete externally stored + columns. Pretend that such records do + not exist. Such records may only be + accessed at the READ UNCOMMITTED + isolation level or when rolling back a + recovered transaction. Rollback + happens at a lower level, not here. */ + goto next_rec; + } } - } - if (!prebuilt->clust_index_was_generated) { - } else if (result_rec != rec || index->is_primary()) { - memcpy(prebuilt->row_id, result_rec, DATA_ROW_ID_LEN); + if (!prebuilt->clust_index_was_generated) { + } else if (result_rec != rec || index->is_primary()) { + memcpy(prebuilt->row_id, result_rec, DATA_ROW_ID_LEN); + } else { + ulint len; + const byte* data = rec_get_nth_field( + result_rec, offsets, index->n_fields - 1, + &len); + ut_ad(dict_index_get_nth_col(index, + index->n_fields - 1) + ->prtype == (DATA_ROW_ID | DATA_NOT_NULL)); + ut_ad(len == DATA_ROW_ID_LEN); + memcpy(prebuilt->row_id, data, DATA_ROW_ID_LEN); + } } else { - ulint len; - const byte* data = rec_get_nth_field( - result_rec, offsets, index->n_fields - 1, - &len); - ut_ad(dict_index_get_nth_col(index, - index->n_fields - 1) - ->prtype == (DATA_ROW_ID | DATA_NOT_NULL)); - ut_ad(len == DATA_ROW_ID_LEN); - memcpy(prebuilt->row_id, data, DATA_ROW_ID_LEN); + /* InnoDB callback path - no MySQL format conversion */ + err= InnoDBReadPolicy::output_record( + reinterpret_cast(buf), + prebuilt, result_rec, + result_rec != rec ? clust_index : index, + offsets); + + if (err == DB_FTS_EXCEED_RESULT_CACHE_LIMIT) { + goto normal_return; + } else if (err == DB_SUCCESS_LOCKED_REC) { + err= DB_SUCCESS; + goto idx_cond_failed; + } else if (err == DB_RECORD_NOT_FOUND) { + goto next_rec; + } else if (err != DB_SUCCESS) { + goto lock_wait_or_error; + } } } @@ -5757,11 +5927,16 @@ row_search_mvcc( || prebuilt->select_lock_type != LOCK_NONE || prebuilt->used_in_HANDLER) { - /* Inside an update always store the cursor position */ + /* Inside an update always store the cursor position. + Exception: for external mtr (DML operations), keep the + page latched for the subsequent update operation. */ - if (!spatial_search) { - btr_pcur_store_position(pcur, &mtr); + if constexpr (Callback::manages_mtr) { + if (!spatial_search) { + btr_pcur_store_position(pcur, mtr); + } } + /* else: external mtr, keep page latched */ } goto normal_return; @@ -5796,14 +5971,14 @@ row_search_mvcc( if (spatial_search) { /* No need to do store restore for R-tree */ - mtr.rollback_to_savepoint(0); + mtr->rollback_to_savepoint(0); } else if (mtr_extra_clust_savepoint) { /* We must release any clustered index latches if we are moving to the next non-clustered index record, because we could break the latching order if we would access a different clustered index page right away without releasing the previous. */ - mtr.rollback_to_savepoint(mtr_extra_clust_savepoint); + mtr->rollback_to_savepoint(mtr_extra_clust_savepoint); } mtr_extra_clust_savepoint = 0; @@ -5811,7 +5986,7 @@ row_search_mvcc( if (moves_up) { if (UNIV_UNLIKELY(spatial_search)) { if (rtr_pcur_move_to_next( - search_tuple, mode, pcur, 0, &mtr)) { + search_tuple, mode, pcur, 0, mtr)) { goto rec_loop; } } else { @@ -5823,7 +5998,7 @@ row_search_mvcc( if (btr_pcur_is_after_last_in_tree(pcur)) { goto not_moved; } - err = btr_pcur_move_to_next_page(pcur, &mtr); + err = btr_pcur_move_to_next_page(pcur, mtr); if (err != DB_SUCCESS) { goto lock_wait_or_error; } @@ -5834,7 +6009,7 @@ row_search_mvcc( goto rec_loop; } } else { - if (btr_pcur_move_to_prev(pcur, &mtr)) { + if (btr_pcur_move_to_prev(pcur, mtr)) { goto rec_loop; } if (UNIV_UNLIKELY(!btr_pcur_get_rec(pcur))) { @@ -5846,7 +6021,7 @@ row_search_mvcc( not_moved: if (!spatial_search) { - btr_pcur_store_position(pcur, &mtr); + btr_pcur_store_position(pcur, mtr); } err = match_mode ? DB_RECORD_NOT_FOUND : DB_END_OF_INDEX; @@ -5854,7 +6029,7 @@ row_search_mvcc( lock_wait_or_error: if (!dict_index_is_spatial(index)) { - btr_pcur_store_position(pcur, &mtr); + btr_pcur_store_position(pcur, mtr); } page_read_error: /* Reset the old and new "did semi-consistent read" flags. */ @@ -5865,7 +6040,9 @@ row_search_mvcc( did_semi_consistent_read = false; lock_table_wait: - mtr.commit(); + if constexpr (Callback::manages_mtr) { + mtr->commit(); + } mtr_extra_clust_savepoint = 0; trx->error_state = err; @@ -5875,7 +6052,9 @@ row_search_mvcc( /* It was a lock wait, and it ended */ thr->lock_state = QUE_THR_LOCK_NOLOCK; - mtr.start(); + if constexpr (Callback::manages_mtr) { + mtr->start(); + } /* Table lock waited, go try to obtain table lock again */ @@ -5888,7 +6067,7 @@ row_search_mvcc( if (!dict_index_is_spatial(index)) { sel_restore_position_for_mysql( &same_user_rec, BTR_SEARCH_LEAF, pcur, - moves_up, &mtr); + moves_up, mtr); } if (trx->isolation_level <= TRX_ISO_READ_COMMITTED @@ -5923,7 +6102,10 @@ row_search_mvcc( goto func_exit; normal_return: - mtr.commit(); + if constexpr (Callback::manages_mtr) { + mtr->commit(); + } + /* else: caller manages mtr, don't commit */ DEBUG_SYNC_C("row_search_for_mysql_before_return"); @@ -6847,3 +7029,13 @@ dberr_t row_check_index(row_prebuilt_t *prebuilt, ulint *n_rows) goto rec_loop; } + +/* Explicit template instantiations for row_search_mvcc */ +template dberr_t row_search_mvcc( + byte*, page_cur_mode_t, row_prebuilt_t*, ulint, ulint); + +template dberr_t row_search_mvcc( + byte*, page_cur_mode_t, row_prebuilt_t*, ulint, ulint); + +template dberr_t row_search_mvcc( + byte*, page_cur_mode_t, row_prebuilt_t*, ulint, ulint);