diff --git a/README.md b/README.md index 0a881c8..392f3a0 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,12 @@ $ wp sqlite import The path to the MySQL compatible dump file to import. When passing `-` as the file argument, the SQL commands are read from standard input. + [--refuse-overwrite] + Refuse to import when the SQLite database already contains user tables. By default the import merges into the existing database. + + [--verify-old-host=] + After loading, fail when any stored value still contains , or when any PHP-serialized value no longer unserializes. The check runs inside the import transaction, so a failure rolls everything back. A failed import into a new database file also deletes that file and its `-wal`, `-shm`, and `-journal` sidecars. + [--enable-ast-driver] Enables the AST driver for older integration plugin releases. The current driver API is selected automatically. @@ -27,6 +33,8 @@ $ wp sqlite import Exports an SQLite database to a MySQL compatible file. +Each `INSERT` statement is written on a single line: embedded newlines and other special bytes are backslash-escaped (including double quotes), so dumps are safe for line-oriented downstream rewriters as well as for quote-aware parsers. + ``` $ wp sqlite export [] [--tables=] [--exclude-tables] [--porcelain] ``` diff --git a/features/bootstrap/SQLiteFeatureContext.php b/features/bootstrap/SQLiteFeatureContext.php index d543dfa..48e3098 100644 --- a/features/bootstrap/SQLiteFeatureContext.php +++ b/features/bootstrap/SQLiteFeatureContext.php @@ -33,6 +33,18 @@ public function theSqliteDatabaseShouldContainATableNamed( $table_name ) { } } + /** + * @Then /^the SQLite database should not contain a table named "([^"]*)"$/ + */ + public function theSqliteDatabaseShouldNotContainATableNamed( $table_name ) { + $this->connectToDatabase(); + $stmt = $this->db->prepare( "SELECT name FROM sqlite_master WHERE type='table' AND name=?" ); + $stmt->execute( [ $table_name ] ); + $row = $stmt->fetch(); + if ( $row ) { + throw new Exception( "Table '$table_name' was found in the database but should not exist." ); + } + } /** * @Then /^the "([^"]*)" should contain a row with name "([^"]*)"$/ * @Then /^the "([^"]*)" should contain a row with name:$/ @@ -71,8 +83,15 @@ public function theFileShouldExist( $filename ) { private function connectToDatabase() { if ( ! $this->db ) { - $result = $this->proc( 'wp eval \'echo defined( "DB_PATH" ) ? DB_PATH : FQDB;\'' )->run_check(); + // Prefer the configured path without booting WordPress: the + // target may be a fresh import without WP tables installed. + $result = $this->proc( 'wp config get DB_PATH' )->run(); $db_file = trim( $result->stdout ); + if ( '' === $db_file || 0 === strpos( $db_file, 'Error:' ) ) { + // Fall back to a full boot so the drop-in resolves FQDB. + $result = $this->proc( 'wp eval \'echo defined( "FQDB" ) ? FQDB : ""; \'' )->run_check(); + $db_file = trim( $result->stdout ); + } if ( ! is_file( $db_file ) ) { throw new Exception( 'The WordPress SQLite database does not exist.' ); } @@ -345,4 +364,73 @@ public function iExportTheTableWithACommentInjectionName() { 'run' ); } + + /** + * @Given /^the SQLite database contains an option with a double quote$/ + */ + public function theSqliteDatabaseContainsAnOptionWithADoubleQuote() { + $this->connectToDatabase(); + + $this->db->exec( "DELETE FROM wp_options WHERE option_name = 'double-quote-option'" ); + + $stmt = $this->db->prepare( + ' + INSERT INTO wp_options (option_name, option_value, autoload) + VALUES (:option_name, :option_value, :autoload) + ' + ); + $stmt->execute( + array( + ':option_name' => 'double-quote-option', + ':option_value' => 'say "hello" backslash\\ end', + ':autoload' => 'yes', + ) + ); + } + + /** + * @Then /^the double quote option should be escaped and preserved$/ + */ + public function theDoubleQuoteOptionShouldBeEscapedAndPreserved() { + $this->connectToDatabase(); + + $stmt = $this->db->prepare( "SELECT option_value FROM wp_options WHERE option_name = 'double-quote-option'" ); + $stmt->execute(); + $value = $stmt->fetchColumn(); + + $expected = 'say "hello" backslash\\ end'; + if ( $expected !== $value ) { + throw new Exception( + sprintf( + 'Double quote option was not preserved. Expected %s, got %s.', + var_export( $expected, true ), + var_export( $value, true ) + ) + ); + } + } + + /** + * @Then /^every INSERT in the file "([^"]*)" should be on a single line$/ + */ + public function everyInsertInTheFileShouldBeOnASingleLine( $filename ) { + $full_path = $this->variables['RUN_DIR'] . '/' . $filename; + $lines = file( $full_path, FILE_IGNORE_NEW_LINES ); + if ( false === $lines ) { + throw new Exception( "Could not read file: $full_path" ); + } + $insert_count = 0; + foreach ( $lines as $line ) { + if ( 0 !== strpos( ltrim( $line ), 'INSERT INTO' ) ) { + continue; + } + ++$insert_count; + if ( ';' !== substr( rtrim( $line ), -1 ) ) { + throw new Exception( "INSERT statement spans multiple lines: $line" ); + } + } + if ( 0 === $insert_count ) { + throw new Exception( "No INSERT statements found in file: $full_path" ); + } + } } diff --git a/features/sqlite-export.feature b/features/sqlite-export.feature index 865dd09..44ed001 100644 --- a/features/sqlite-export.feature +++ b/features/sqlite-export.feature @@ -169,7 +169,7 @@ Feature: WP-CLI SQLite Export Command And the file "test_export_serialized_settings.sql" should exist And the file "test_export_serialized_settings.sql" should contain: """ - breadcrumb-separator";s:4:"\00bb + breadcrumb-separator\";s:4:\"\00bb """ And the file "test_export_serialized_settings.sql" should contain: """ @@ -201,3 +201,30 @@ Feature: WP-CLI SQLite Export Command Success: Imported from 'test_export_escape_bytes.sql'. """ And the SQL string escape bytes should be preserved + + @require-sqlite + Scenario: Export should escape double quotes with a backslash + Given the SQLite database contains an option with a double quote + When I run `wp sqlite export test_export_dquote.sql --tables=wp_options` + Then STDOUT should contain: + """ + Success: Export complete. File written to test_export_dquote.sql + """ + And the file "test_export_dquote.sql" should exist + And the file "test_export_dquote.sql" should contain: + """ + 'double-quote-option','say \"hello\" backslash\\ end','yes' + """ + When I run `wp sqlite --enable-ast-driver import test_export_dquote.sql` + Then STDOUT should contain: + """ + Success: Imported from 'test_export_dquote.sql'. + """ + And the double quote option should be escaped and preserved + + @require-sqlite + Scenario: Export keeps every INSERT on a single line + Given the SQLite database contains an option with SQL string escape bytes + When I run `wp sqlite export test_export_single_line.sql --tables=wp_options` + Then the file "test_export_single_line.sql" should exist + And every INSERT in the file "test_export_single_line.sql" should be on a single line diff --git a/features/sqlite-import.feature b/features/sqlite-import.feature index 9c7ae4b..f413e3f 100644 --- a/features/sqlite-import.feature +++ b/features/sqlite-import.feature @@ -211,3 +211,102 @@ Feature: WP-CLI SQLite Import Command """ And the SQLite database should contain a table named "wp_zz_parent" And the SQLite database should contain a table named "wp_aa_child" + + @require-sqlite + Scenario: Refuse to overwrite an existing database + Given a SQL dump file named "test_import.sql" with content: + """ + CREATE TABLE test_table (id INTEGER PRIMARY KEY, name TEXT); + INSERT INTO test_table (name) VALUES ('Test Name'); + """ + When I run `wp sqlite import test_import.sql` + Then STDOUT should contain: + """ + Success: Imported from 'test_import.sql'. + """ + When I try `wp sqlite import test_import.sql --refuse-overwrite` + Then the return code should be 1 + And STDERR should contain: + """ + already contains tables + """ + And the SQLite database should contain a table named "test_table" + + @require-sqlite + Scenario: Allow refuse-overwrite into a fresh database file + Given a SQL dump file named "test_import.sql" with content: + """ + CREATE TABLE test_table (id INTEGER PRIMARY KEY, name TEXT); + INSERT INTO test_table (name) VALUES ('Test Name'); + """ + When I run `wp config set DB_PATH '{RUN_DIR}/fresh.sqlite'` + And I run `wp sqlite import test_import.sql --refuse-overwrite` + Then STDOUT should contain: + """ + Success: Imported from 'test_import.sql'. + """ + And the SQLite database should contain a table named "test_table" + + @require-sqlite + Scenario: Failed import into a new database removes the new file + Given a SQL dump file named "test_import_bad.sql" with content: + """ + CREATE TABLE test_table (id INTEGER PRIMARY KEY, name TEXT); + INSERT INTO missing_table (name) VALUES ('Test Name'); + """ + When I run `wp config set DB_PATH '{RUN_DIR}/atomic.sqlite'` + And I try `wp sqlite import test_import_bad.sql` + Then the return code should be 1 + And STDERR should contain: + """ + Could not execute statement + """ + And the atomic.sqlite file should not exist + And the atomic.sqlite-wal file should not exist + And the atomic.sqlite-shm file should not exist + And the atomic.sqlite-journal file should not exist + + @require-sqlite + Scenario: Fail verification when the old host remains + Given a SQL dump file named "test_import.sql" with content: + """ + CREATE TABLE verify_host (id INTEGER PRIMARY KEY, name TEXT); + INSERT INTO verify_host (name) VALUES ('https://oldhost.invalid/page'); + """ + When I try `wp sqlite import test_import.sql --verify-old-host=oldhost.invalid` + Then the return code should be 1 + And STDERR should contain: + """ + still contain 'oldhost.invalid' + """ + And the SQLite database should not contain a table named "verify_host" + + @require-sqlite + Scenario: Pass verification when the old host is gone + Given a SQL dump file named "test_import.sql" with content: + """ + CREATE TABLE verify_host (id INTEGER PRIMARY KEY, name TEXT); + INSERT INTO verify_host (name) VALUES ('https://newhost.invalid/page'); + """ + When I run `wp sqlite import test_import.sql --verify-old-host=oldhost.invalid` + Then STDOUT should contain: + """ + Success: Imported from 'test_import.sql'. + """ + And the SQLite database should contain a table named "verify_host" + And the "verify_host" should contain a row with name "https://newhost.invalid/page" + + @require-sqlite + Scenario: Fail verification on corrupt serialized data + Given a SQL dump file named "test_import.sql" with content: + """ + CREATE TABLE verify_serialized (id INTEGER PRIMARY KEY, name TEXT); + INSERT INTO verify_serialized (name) VALUES ('a:1:{s:3:"foo";s:4:"broken}'); + """ + When I try `wp sqlite import test_import.sql --verify-old-host=never-present.invalid` + Then the return code should be 1 + And STDERR should contain: + """ + invalid serialized data + """ + And the SQLite database should not contain a table named "verify_serialized" diff --git a/src/Export.php b/src/Export.php index 648b38d..16c9496 100644 --- a/src/Export.php +++ b/src/Export.php @@ -191,6 +191,11 @@ protected function get_create_statement( $table_name ) { /** * Get the INSERT statements for a table. * + * Each yielded statement is a single line: embedded newlines and other + * special bytes are backslash-escaped by escape_string(), and the caller + * terminates every statement with PHP_EOL. Line-oriented downstream + * rewriters may therefore process the dump one statement per line. + * * @param string $table_name * * @return \Generator @@ -269,6 +274,11 @@ protected function escape_values( $values ) { /** * Escapes a string for use in an insert statement. * + * Uses full MySQL backslash escaping, including double quotes, so that + * each INSERT statement stays on a single line and remains safe for + * line-oriented downstream rewriters (e.g. go-search-replace) as well + * as for the quote-aware Import::parse_statements(). + * * @param $value * * @return string @@ -281,6 +291,7 @@ protected function escape_string( $value ) { "\n" => '\n', "\r" => '\r', "\x1a" => '\Z', + '"' => '\\"', '\\' => '\\\\', "'" => "\\'", ) diff --git a/src/Import.php b/src/Import.php index aaf6286..15fae25 100644 --- a/src/Import.php +++ b/src/Import.php @@ -3,6 +3,9 @@ use Exception; use Generator; +use PDO; +use RuntimeException; +use Throwable; use WP_CLI; use WP_MySQL_On_SQLite; use WP_SQLite_Driver; @@ -40,9 +43,15 @@ public function run( $sql_file_path, $args ) { WP_CLI::error( "Unable to open file: $import_file" ); } + $database_existed_before_import = $this->database_file_exists(); + // Create the driver after opening the input, so a missing file doesn't create a database. $this->driver = SQLiteDriverFactory::create_driver( true ); + if ( isset( $args['refuse-overwrite'] ) ) { + $this->refuse_when_tables_exist( $database_existed_before_import ); + } + /* * Set default SQL mode and other options as per the "mysqldump" command. * @@ -67,7 +76,24 @@ public function run( $sql_file_path, $args ) { */ $this->driver->execute_sqlite_query( 'PRAGMA foreign_keys = OFF' ); - $this->execute_statements( $handle ); + /* + * Load the dump inside a single transaction so a failed import leaves + * no partial data behind. Verification (when requested) runs before + * the COMMIT so a verification failure rolls back as well. + */ + $this->begin_transaction(); + try { + $this->execute_statements( $handle ); + $this->verify_after_import(); + $this->commit_transaction(); + } catch ( Throwable $exception ) { + $this->rollback_transaction(); + if ( ! $database_existed_before_import ) { + $this->delete_created_database_files(); + } + fclose( $handle ); + WP_CLI::error( sprintf( 'Import failed and was rolled back: %s', $exception->getMessage() ) ); + } fclose( $handle ); /* @@ -99,7 +125,8 @@ protected function execute_statements( $handle ) { foreach ( $this->parse_statements( $handle ) as $statement ) { try { $this->driver->query( $statement ); - } catch ( Exception $e ) { + } catch ( Throwable $statement_exception ) { + $retried = false; try { // Try converting encoding and retry $detected_encoding = mb_detect_encoding( $statement, mb_list_encodings(), true ); @@ -107,18 +134,309 @@ protected function execute_statements( $handle ) { $converted_statement = mb_convert_encoding( $statement, 'UTF-8', $detected_encoding ); echo 'Converted ecoding for statement: ' . $converted_statement . PHP_EOL; $this->driver->query( $converted_statement ); - } else { - // It's not an encoding issue, so rethrow the exception. - throw $e; + $retried = true; + } + } catch ( Throwable $retry_exception ) { + // The retry failed; the original failure is reported below. + unset( $retry_exception ); + } + if ( ! $retried ) { + throw new RuntimeException( + sprintf( + 'Could not execute statement: %s (%s)', + $statement, + $statement_exception->getMessage() + ) + ); + } + } + } + } + + /** + * Whether the SQLite database file already exists. + * + * Resolves the configured path without creating storage, so calling + * this before create_driver() tells whether the import will write into + * a new file or an existing one. + * + * @return bool + */ + protected function database_file_exists() { + try { + $database_path = SQLiteDriverFactory::get_database_path( false ); + } catch ( Throwable $exception ) { + return false; + } + return is_file( $database_path ); + } + + /** + * Refuse the import when the database already holds user tables. + * + * @param bool $database_existed_before_import Whether the database file existed before the driver was created. + * + * @return void + */ + protected function refuse_when_tables_exist( $database_existed_before_import ) { + $tables = $this->get_user_table_names(); + if ( empty( $tables ) ) { + return; + } + if ( ! $database_existed_before_import ) { + $this->delete_created_database_files(); + } + WP_CLI::error( 'Import refused: the SQLite database already contains tables. Run without --refuse-overwrite to import anyway.' ); + } + + /** + * Get the user table names in the target database. + * + * Internal SQLite bookkeeping tables are excluded, mirroring Export. + * + * @return string[] + */ + protected function get_user_table_names() { + $result = $this->driver->query( 'SHOW TABLES' ); + if ( $this->driver instanceof PDO ) { + $tables = $result->fetchAll( PDO::FETCH_COLUMN ); + } else { + $tables = array(); + foreach ( $result as $row ) { + $tables[] = array_values( (array) $row )[0]; + } + } + return array_values( + array_filter( + $tables, + function ( $table_name ) { + if ( in_array( $table_name, array( '_mysql_data_types_cache', 'sqlite_master', 'sqlite_sequence' ), true ) ) { + return false; } - } catch ( Exception $e ) { - WP_CLI::error( 'Could not execute statement: ' . $statement ); - echo $e->getMessage(); + return 0 !== strpos( $table_name, '_wp_sqlite_' ); + } + ) + ); + } + + /** + * Quote a MySQL identifier. + * + * @param string $identifier Identifier to quote. + * @return string + */ + protected function quote_identifier( $identifier ) { + return '`' . str_replace( '`', '``', $identifier ) . '`'; + } + + /** + * Start the import transaction. + * + * @return void + */ + protected function begin_transaction() { + try { + $this->driver->query( 'START TRANSACTION' ); + } catch ( Throwable $exception ) { + $this->driver->execute_sqlite_query( 'BEGIN' ); + } + } + + /** + * Commit the import transaction. + * + * @return void + */ + protected function commit_transaction() { + try { + $this->driver->query( 'COMMIT' ); + } catch ( Throwable $exception ) { + $this->driver->execute_sqlite_query( 'COMMIT' ); + } + } + + /** + * Roll back the import transaction. + * + * @return void + */ + protected function rollback_transaction() { + try { + $this->driver->query( 'ROLLBACK' ); + } catch ( Throwable $exception ) { + try { + $this->driver->execute_sqlite_query( 'ROLLBACK' ); + } catch ( Throwable $sqlite_exception ) { + // The connection may already be unusable; the import error below is what matters. + unset( $sqlite_exception ); + } + } + } + + /** + * Delete a database file created by a failed import. + * + * Removes the main file plus any -wal, -shm, and -journal sidecars so + * no partial database is left behind. + * + * @return void + */ + protected function delete_created_database_files() { + try { + $database_path = SQLiteDriverFactory::get_database_path( false ); + } catch ( Throwable $exception ) { + return; + } + if ( ! is_string( $database_path ) || '' === $database_path ) { + return; + } + foreach ( array( '', '-wal', '-shm', '-journal' ) as $suffix ) { + $file = $database_path . $suffix; + if ( is_file( $file ) ) { + unlink( $file ); + } + } + } + + /** + * Verify the imported data when --verify-old-host is given. + * + * Fails when any stored value still contains the old host, or when any + * value that looks like PHP-serialized data no longer unserializes. + * Runs inside the import transaction so a failure rolls everything back. + * + * @return void + */ + protected function verify_after_import() { + if ( ! isset( $this->args['verify-old-host'] ) ) { + return; + } + $old_host = $this->args['verify-old-host']; + if ( ! is_string( $old_host ) || '' === $old_host ) { + throw new RuntimeException( '--verify-old-host requires a non-empty value.' ); + } + + $host_hits = array(); + $serialized_hits = array(); + foreach ( $this->get_user_table_names() as $table_name ) { + $this->scan_table_for_verification( $table_name, $old_host, $host_hits, $serialized_hits ); + } + + if ( ! empty( $host_hits ) ) { + throw new RuntimeException( + sprintf( + "Verification failed: %d value(s) in the imported database still contain '%s' (e.g., table '%s').", + array_sum( $host_hits ), + $old_host, + array_keys( $host_hits )[0] + ) + ); + } + if ( ! empty( $serialized_hits ) ) { + throw new RuntimeException( + sprintf( + 'Verification failed: %d value(s) contain invalid serialized data (e.g., table \'%s\').', + array_sum( $serialized_hits ), + array_keys( $serialized_hits )[0] + ) + ); + } + WP_CLI::log( + sprintf( + "Verified: no occurrences of '%s' remain and serialized data is intact.", + $old_host + ) + ); + } + + /** + * Scan one table for verification failures. + * + * @param string $table_name Table to scan. + * @param string $old_host Host that must no longer occur. + * @param array $host_hits Hit counts per table, updated in place. + * @param array $serialized_hits Corrupt-serialized counts per table, updated in place. + * + * @return void + */ + protected function scan_table_for_verification( $table_name, $old_host, &$host_hits, &$serialized_hits ) { + $result = $this->driver->query( 'SELECT * FROM ' . $this->quote_identifier( $table_name ) ); + if ( $this->driver instanceof PDO ) { + $rows = $result->fetchAll( PDO::FETCH_ASSOC ); + } else { + $rows = array(); + foreach ( $result as $row ) { + $rows[] = (array) $row; + } + } + foreach ( $rows as $row ) { + foreach ( $row as $value ) { + if ( ! is_string( $value ) ) { + continue; + } + if ( false !== strpos( $value, $old_host ) ) { + $host_hits[ $table_name ] = isset( $host_hits[ $table_name ] ) ? $host_hits[ $table_name ] + 1 : 1; + } + if ( ! $this->is_valid_serialized_string( $value ) ) { + $serialized_hits[ $table_name ] = isset( $serialized_hits[ $table_name ] ) ? $serialized_hits[ $table_name ] + 1 : 1; } } } } + /** + * Check that a value holding PHP-serialized data still unserializes. + * + * Non-serialized values always pass; only values that look serialized + * but fail to unserialize are reported as corrupt. + * + * @param string $value Value to check. + * @return bool True when the value is not corrupt serialized data. + */ + protected function is_valid_serialized_string( $value ) { + if ( function_exists( 'is_serialized' ) ) { + $is_serialized = is_serialized( $value ); + } else { + $is_serialized = $this->looks_like_serialized( $value ); + } + if ( ! $is_serialized ) { + return true; + } + if ( 'b:0;' === trim( $value ) ) { + return true; + } + $unserialized = @unserialize( $value, array( 'allowed_classes' => false ) ); // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.serialize_unserialize -- Verification intentionally round-trips stored serialized values. + return false !== $unserialized; + } + + /** + * Fallback check for PHP-serialized strings when WordPress is unavailable. + * + * @param string $value Value to check. + * @return bool + */ + protected function looks_like_serialized( $value ) { + $value = trim( $value ); + if ( 'N;' === $value ) { + return true; + } + if ( strlen( $value ) < 4 || ':' !== $value[1] ) { + return false; + } + $last_character = substr( $value, -1 ); + if ( ';' !== $last_character && '}' !== $last_character ) { + return false; + } + $token = $value[0]; + if ( 's' === $token || 'a' === $token || 'O' === $token ) { + return (bool) preg_match( '/^' . $token . ':[0-9]+:/s', $value ); + } + if ( 'b' === $token || 'i' === $token || 'd' === $token ) { + return (bool) preg_match( '/^' . $token . ':[0-9.E+-]+;/', $value ); + } + return false; + } + /** * Parse SQL statements from an SQL dump file. * @param resource $handle The SQL dump file handle. diff --git a/src/SQLiteDriverFactory.php b/src/SQLiteDriverFactory.php index 7e8de01..5a7e1b8 100644 --- a/src/SQLiteDriverFactory.php +++ b/src/SQLiteDriverFactory.php @@ -38,12 +38,16 @@ public static function create_driver( $create_database_if_missing = false ) { } /** - * Resolve configuration without initializing existing storage. + * Resolve the configured SQLite database path. + * + * Does not check that the file exists and does not create storage + * unless $create_database_if_missing is true; callers use the false + * form to test whether an import would write into a new file. * * @param bool $create_database_if_missing Allow initialization when no secret path is recorded. * @return string */ - private static function get_database_path( $create_database_if_missing ) { + public static function get_database_path( $create_database_if_missing ) { if ( defined( 'DB_PATH' ) ) { $database_path = DB_PATH; } elseif ( defined( 'FQDB' ) ) { diff --git a/src/SQLite_Command.php b/src/SQLite_Command.php index 389c90b..099bad5 100644 --- a/src/SQLite_Command.php +++ b/src/SQLite_Command.php @@ -16,6 +16,12 @@ class SQLite_Command extends WP_CLI_Command { * * : The name of the SQL file to import. If '-', then reads from STDIN. If omitted, it will look for '{dbname}.sql'. * + * [--refuse-overwrite] + * : Refuse to import when the SQLite database already contains user tables. By default the import merges into the existing database. + * + * [--verify-old-host=] + * : After loading, fail when any stored value still contains , or when any PHP-serialized value no longer unserializes. The check runs inside the import transaction, so a failure rolls everything back. + * * [--enable-ast-driver] * : Enables the AST driver for older integration plugin releases. The current driver API is selected automatically. *