diff --git a/Firewall.php b/Firewall.php index 8ff4502..e19fad6 100644 --- a/Firewall.php +++ b/Firewall.php @@ -235,7 +235,7 @@ public function run() $result['ip'], $result['status'], isset($result['network']) ? $result['network'] : null, - isset($result['is_personal']) ? $result['is_personal'] : 'NULL' + !empty($result['is_personal']) ? 1 : 'NULL' ); } } @@ -256,6 +256,9 @@ public function run() $this->fw_modules[$module_name]->actionsForPassed($result); $this->fw_modules[$module_name]->diePage($result); } else { + if ( Get::get('sfw_test_ip') ) { + $this->fw_modules[$module_name]->diePage($result); + } $this->fw_modules[$module_name]->actionsForPassed($result); } } @@ -305,6 +308,7 @@ private function prioritize($results) $result['passed_ip'] = isset($fw_result['ip']) ? $fw_result['ip'] : $fw_result['passed_ip']; $result['blocked_ip'] = isset($fw_result['ip']) ? $fw_result['ip'] : $fw_result['blocked_ip']; $result['pattern'] = isset($fw_result['pattern']) ? $fw_result['pattern'] : array(); + $result['is_personal'] = !empty($fw_result['is_personal']) ? 1 : 0; } } } diff --git a/FirewallUpdater.php b/FirewallUpdater.php index 4c1158b..4b4bac4 100644 --- a/FirewallUpdater.php +++ b/FirewallUpdater.php @@ -283,44 +283,74 @@ public static function getMultifiles($api_key) /** @var \Cleantalk\Common\Helper\Helper $helper_class */ $helper_class = Mloader::get('Helper'); - // Getting remote file name + $urls = array(); - $result = $api_class::methodGet2sBlacklistsDb($api_key, 'multifiles', '3_1'); + // Getting common lists + $result = $api_class::methodGet2sBlacklistsDb($api_key, 'multifiles', '3_2', 1); if ( empty($result['error']) ) { if ( !empty($result['file_url']) ) { $file_urls = $helper_class::httpGetDataFromRemoteGzAndParseCsv($result['file_url']); if ( empty($file_urls['error']) ) { + // Extract common_lists_url_id from the file URL + preg_match('/bl_list_(.+)\.multifiles/m', $result['file_url'], $url_id_common); + if ( isset($url_id_common[1]) ) { + $fw_stats->common_lists_url_id = $url_id_common[1]; + } if ( !empty($result['file_ua_url']) ) { $file_urls[][0] = $result['file_ua_url']; } if ( !empty($result['file_ck_url']) ) { $file_urls[][0] = $result['file_ck_url']; } - $urls = array(); foreach ( $file_urls as $value ) { $urls[] = $value[0]; } - - $tries_for_download_again = 3 + (int)(count($urls) / 20); - $fw_stats->update_percent = round(100 / count($urls), 2); - Firewall::saveFwStats($fw_stats); - - return array( - 'next_stage' => array( - 'name' => [self::class, 'downloadFiles'], - 'args' => $urls, - 'accepted_tries' => $tries_for_download_again - ) - ); + } else { + throw new SfwUpdateException('getMultifiles: common: ' . $file_urls['error']); } - - throw new SfwUpdateException('getMultifiles: ' . $file_urls['error']); } } else { return $result; } - return null; + + // Getting personal lists + $result_personal = $api_class::methodGet2sBlacklistsDb($api_key, 'multifiles', '3_2', 0); + + if ( empty($result_personal['error']) ) { + if ( !empty($result_personal['file_url']) ) { + $file_urls_personal = $helper_class::httpGetDataFromRemoteGzAndParseCsv($result_personal['file_url']); + if ( empty($file_urls_personal['error']) ) { + // Extract personal_lists_url_id from the file URL + preg_match('/bl_list_(.+)\.multifiles/m', $result_personal['file_url'], $url_id); + if ( isset($url_id[1]) ) { + $fw_stats->personal_lists_url_id = $url_id[1]; + } + if ( !empty($result_personal['file_ck_url']) ) { + $file_urls_personal[][0] = $result_personal['file_ck_url']; + } + foreach ( $file_urls_personal as $value ) { + $urls[] = $value[0]; + } + } + } + } + + if ( empty($urls) ) { + throw new SfwUpdateException('getMultifiles: No URLs to download'); + } + + $tries_for_download_again = 3 + (int)(count($urls) / 20); + $fw_stats->update_percent = round(100 / count($urls), 2); + Firewall::saveFwStats($fw_stats); + + return array( + 'next_stage' => array( + 'name' => [self::class, 'downloadFiles'], + 'args' => $urls, + 'accepted_tries' => $tries_for_download_again + ) + ); } public static function downloadFiles($api_key, $urls) @@ -398,6 +428,8 @@ public static function createTables($_api_key) $db_tables_creator = new DbTablesCreator(); $table_name_sfw = $db_obj->prefix . Schema::getSchemaTablePrefix() . 'sfw'; $db_tables_creator->createTable($table_name_sfw); + $table_name_sfw_personal = $db_obj->prefix . Schema::getSchemaTablePrefix() . 'sfw_personal'; + $db_tables_creator->createTable($table_name_sfw_personal); $table_name_ua = $db_obj->prefix . Schema::getSchemaTablePrefix() . 'ua_bl'; $db_tables_creator->createTable($table_name_ua); @@ -414,7 +446,7 @@ public static function createTempTables($_api_key) $db_class = Mloader::get('Db'); $db_obj = $db_class::getInstance(); - // Preparing temporary tables + // Preparing temporary tables for common SFW $result = \Cleantalk\Common\Firewall\Modules\Sfw::createTempTables( $db_obj, $db_obj->prefix . APBCT_TBL_FIREWALL_DATA @@ -443,7 +475,21 @@ public static function processFiles($_api_key) $concrete_file = current($files); if ( strpos($concrete_file, 'bl_list') !== false ) { - $result = self::processFile($concrete_file); + // Determine direction: personal or common + if ( + !empty($fw_stats->personal_lists_url_id) + && strpos($concrete_file, $fw_stats->personal_lists_url_id) !== false + ) { + $direction = 'personal'; + } elseif ( + !empty($fw_stats->common_lists_url_id) + && strpos($concrete_file, $fw_stats->common_lists_url_id) !== false + ) { + $direction = 'common'; + } else { + $direction = 'common'; + } + $result = self::processFile($concrete_file, $direction); } if ( strpos($concrete_file, 'ua_list') !== false ) { @@ -476,7 +522,7 @@ public static function processFiles($_api_key) ); } - public static function processFile($file_path) + public static function processFile($file_path, $direction = 'common') { if ( !file_exists($file_path) ) { return array('error' => 'PROCESS FILE: ' . $file_path . ' is not exists.'); @@ -486,10 +532,17 @@ public static function processFile($file_path) $db_class = Mloader::get('Db'); $db_obj = $db_class::getInstance(); + $table_name = $direction === 'common' + ? $db_obj->prefix . APBCT_TBL_FIREWALL_DATA . '_temp' + : $db_obj->prefix . APBCT_TBL_FIREWALL_DATA_PERSONAL . '_temp'; + + $include_source = $direction === 'common'; + $result = \Cleantalk\Common\Firewall\Modules\Sfw::updateWriteToDb( $db_obj, - $db_obj->prefix . APBCT_TBL_FIREWALL_DATA . '_temp', - $file_path + $table_name, + $file_path, + $include_source ); if ( !empty($result['error']) ) { @@ -550,6 +603,12 @@ public static function processCk($file_path) $expected_networks_count = 0; $expected_ua_count = 0; + // Determine if this is a personal ck_list + $is_personal_ck = + !empty($fw_stats->personal_lists_url_id) && + strpos($file_path, $fw_stats->personal_lists_url_id) !== + false; + foreach ( $file_ck_url__data as $value ) { if ( trim($value[0], '"') === 'networks_count' ) { $expected_networks_count = $value[1]; @@ -559,8 +618,12 @@ public static function processCk($file_path) } } - $fw_stats->expected_networks_count = $expected_networks_count; - $fw_stats->expected_ua_count = $expected_ua_count; + if ( $is_personal_ck ) { + $fw_stats->expected_networks_count_personal = $expected_networks_count; + } else { + $fw_stats->expected_networks_count = $expected_networks_count; + $fw_stats->expected_ua_count = $expected_ua_count; + } Firewall::saveFwStats($fw_stats); if ( file_exists($file_path) ) { @@ -627,13 +690,28 @@ public static function endOfUpdateRenamingTables($_api_key) $db_obj->prefix . APBCT_TBL_FIREWALL_DATA ); - $fw_stats->update_mode = 0; - Firewall::saveFwStats($fw_stats); - if ( !empty($result['error']) ) { + $fw_stats->update_mode = 0; + Firewall::saveFwStats($fw_stats); throw new SfwUpdateException('endOfUpdateRenamingTables: ' . $result['error']); } + // ATOMIC REMOVE AND RENAME personal table + if ( $db_obj->isTableExists($db_obj->prefix . APBCT_TBL_FIREWALL_DATA_PERSONAL . '_temp') ) { + $result = \Cleantalk\Common\Firewall\Modules\Sfw::replaceDataTablesAtomically( + $db_obj, + $db_obj->prefix . APBCT_TBL_FIREWALL_DATA_PERSONAL + ); + if ( !empty($result['error']) ) { + $fw_stats->update_mode = 0; + Firewall::saveFwStats($fw_stats); + throw new SfwUpdateException('endOfUpdateRenamingTables personal: ' . $result['error']); + } + } + + $fw_stats->update_mode = 0; + Firewall::saveFwStats($fw_stats); + return array( 'next_stage' => array( 'name' => [self::class, 'endOfUpdateCheckingData'], @@ -670,6 +748,13 @@ public static function endOfUpdateCheckingData($_api_key) } $fw_stats->entries = $entries; + // Check personal table entries if exists + if ( $db_obj->isTableExists($db_obj->prefix . APBCT_TBL_FIREWALL_DATA_PERSONAL) ) { + $entries_personal = $db_obj->setQuery('')->getVar( + 'SELECT COUNT(*) FROM ' . $db_obj->prefix . APBCT_TBL_FIREWALL_DATA_PERSONAL + ); + $fw_stats->entries_personal = $entries_personal; + } Firewall::saveFwStats($fw_stats); return array( @@ -730,7 +815,10 @@ public static function endOfUpdate($_api_key, $is_first_updating = false) $fw_stats->update_percent = 0; $fw_stats->updating_id = null; $fw_stats->expected_networks_count = false; + $fw_stats->expected_networks_count_personal = false; $fw_stats->expected_ua_count = false; + $fw_stats->personal_lists_url_id = null; + $fw_stats->common_lists_url_id = null; Firewall::saveFwStats($fw_stats); return true; @@ -841,10 +929,11 @@ public function directUpdate() } // Preparing database infrastructure - // @ToDo need to implement returning result of the Activator::createTables work. $db_tables_creator = new DbTablesCreator(); $table_name = $db_obj->prefix . Schema::getSchemaTablePrefix() . 'sfw'; $db_tables_creator->createTable($table_name); + $table_name_personal = $db_obj->prefix . Schema::getSchemaTablePrefix() . 'sfw_personal'; + $db_tables_creator->createTable($table_name_personal); $result__creating_tmp_table = \Cleantalk\Common\Firewall\Modules\SFW::createTempTables( $db_obj, @@ -854,6 +943,15 @@ public function directUpdate() return array('error' => 'DIRECT UPDATING CREATE TMP TABLE: ' . $result__creating_tmp_table['error']); } + // Create personal temp table + $result__creating_tmp_table_personal = \Cleantalk\Common\Firewall\Modules\SFW::createTempTables( + $db_obj, + $db_obj->prefix . APBCT_TBL_FIREWALL_DATA_PERSONAL + ); + if ( !empty($result__creating_tmp_table_personal['error']) ) { + return array('error' => 'DIRECT UPDATING CREATE PERSONAL TMP TABLE: ' . $result__creating_tmp_table_personal['error']); + } + /** * UPDATING UA LIST */ @@ -886,6 +984,30 @@ public function directUpdate() return array('error' => 'DIRECT UPDATING BLACK LIST: WRONG RESPONSE FROM SFW::directUpdate'); } + /** + * UPDATING PERSONAL LIST + */ + $result_personal = \Cleantalk\Common\Firewall\Modules\Sfw::directUpdateGetBlackListsPersonal($this->api_key); + if ( empty($result_personal['error']) && !empty($result_personal['blacklist']) ) { + $personal_blacklists = $result_personal['blacklist']; + + $upd_result_personal = \Cleantalk\Common\Firewall\Modules\SFW::directUpdate( + $db_obj, + $db_obj->prefix . APBCT_TBL_FIREWALL_DATA_PERSONAL . '_temp', + $personal_blacklists + ); + + if ( !empty($upd_result_personal['error']) ) { + // Personal lists errors are not critical, continue update + } + + if ( is_int($upd_result_personal) ) { + $fw_stats = Firewall::getFwStats(); + $fw_stats->expected_networks_count_personal = $upd_result_personal; + Firewall::saveFwStats($fw_stats); + } + } + /** * UPDATING EXCLUSIONS LIST */ @@ -941,6 +1063,11 @@ public static function cleanData() $db_obj->prefix . APBCT_TBL_FIREWALL_DATA . '_temp' ); + \Cleantalk\Common\Firewall\Modules\SFW::dataTablesDelete( + $db_obj, + $db_obj->prefix . APBCT_TBL_FIREWALL_DATA_PERSONAL . '_temp' + ); + $fw_stats->firewall_update_percent = 0; $fw_stats->firewall_updating_id = null; Firewall::saveFwStats($fw_stats); diff --git a/FwStats.php b/FwStats.php index 4632802..043cd30 100644 --- a/FwStats.php +++ b/FwStats.php @@ -10,11 +10,17 @@ class FwStats public $update_period = 86400; public $updating_folder = 0; public $expected_networks_count = 0; + public $expected_networks_count_personal = 0; public $expected_ua_count = 0; + public $personal_lists_url_id; + public $common_lists_url_id; public $calls = 0; public $update_mode; public $last_update_time; public $last_update_way; public $entries; + public $entries_personal; public $errors; + public $firewall_update_percent; + public $firewall_updating_id; } diff --git a/Modules/Sfw.php b/Modules/Sfw.php index 1205141..8114ba2 100644 --- a/Modules/Sfw.php +++ b/Modules/Sfw.php @@ -15,6 +15,7 @@ class Sfw extends \Cleantalk\Common\Firewall\FirewallModule public $module_name = 'SFW'; private $test_status; + private $test_entry; private $blocked_ips = array(); /** @@ -27,6 +28,11 @@ class Sfw extends \Cleantalk\Common\Firewall\FirewallModule */ private $db__table__data; + /** + * @var string|null + */ + private $db__table__data_personal; + /** * @var string|null */ @@ -52,6 +58,13 @@ public function __construct($log_table, $data_table, $params = array()) $this->db__table__data = $db->prefix . $data_table ?: null; $this->db__table__logs = $db->prefix . $log_table ?: null; + // Set personal table name from params or default + if ( !empty($params['data_table_personal']) ) { + $this->db__table__data_personal = $db->prefix . $params['data_table_personal']; + } elseif ( defined('APBCT_TBL_FIREWALL_DATA_PERSONAL') ) { + $this->db__table__data_personal = $db->prefix . APBCT_TBL_FIREWALL_DATA_PERSONAL; + } + foreach ( $params as $param_name => $param ) { $this->$param_name = isset($this->$param_name) ? $param : false; } @@ -157,36 +170,54 @@ public function check() } $needles = array_unique($needles); - $query = $this->db->sfwGetFromBlacklist($this->db__table__data, $needles, $current_ip_v4); + $query = $this->db->sfwGetFromBlacklist($this->db__table__data, $this->db__table__data_personal, $needles, $current_ip_v4); $db_results = $this->db->fetchAll($query); $test_status = 1; if ( !empty($db_results) ) { + // Personal lists have priority over common lists + // Sort: personal entries first + usort($db_results, function ($a, $b) { + return (int)$b['is_personal'] - (int)$a['is_personal']; + }); + + $result_entry = null; foreach ( $db_results as $db_result ) { - $result_entry = array( + $is_personal = !empty($db_result['is_personal']); + $entry = array( 'ip' => $current_ip, 'network' => $helper_class::ipLong2ip($db_result['network']) . '/' . $helper_class::ipMaskLongToNumber((int)$db_result['mask']), - 'is_personal' => $db_result['source'], + 'is_personal' => $is_personal, ); if ( (int)$db_result['status'] === 1 ) { - $result_entry['status'] = 'PASS_SFW__BY_WHITELIST'; - break; + $entry['status'] = 'PASS_SFW__BY_WHITELIST'; } if ( (int)$db_result['status'] === 0 ) { $this->blocked_ips[] = $helper_class::ipLong2ip($db_result['network']); - $result_entry['status'] = 'DENY_SFW'; + $entry['status'] = 'DENY_SFW'; + } + + // Personal entry is decisive - use it and stop + if ( $is_personal ) { + $result_entry = $entry; + $test_status = (int)$db_result['status']; + break; } - $test_status = (int)$db_result['status']; + // Common entry - use as fallback if no personal found + if ( $result_entry === null ) { + $result_entry = $entry; + $test_status = (int)$db_result['status']; + } } } else { $result_entry = array( 'ip' => $current_ip, - 'is_personal' => null, + 'is_personal' => false, 'status' => 'PASS_SFW', ); } @@ -195,6 +226,7 @@ public function check() if ( $this->test && $_origin === 'sfw_test' ) { $this->test_status = $test_status; + $this->test_entry = $result_entry; } } @@ -332,15 +364,32 @@ public function diePage($result) /** * Message about IP status */ - if ( $this->test ) { - $message_ip_status = $this->localize->translate('IP in the common blacklist'); + $message_ip_status = ''; + $message_ip_status_color = 'green'; + + // Determine entry to use: test_entry for test mode, $result for live mode + $status_entry = $this->test ? $this->test_entry : $result; + $entry_status = isset($status_entry['status']) ? $status_entry['status'] : null; + $is_personal = isset($status_entry['is_personal']) && (int)$status_entry['is_personal'] === 1; + + $common_text_passed = $this->localize->translate('This IP is passed'); + $common_text_blocked = $this->localize->translate('This IP is blocked'); + $global_text = $this->localize->translate('(in global lists)'); + $personal_text = $this->localize->translate('(in personal lists)'); + $lists_text = $is_personal ? $personal_text : $global_text; + + if ( $entry_status === 'PASS_SFW__BY_WHITELIST' ) { + $message_ip_status = $common_text_passed . ' ' . $lists_text; + $message_ip_status_color = 'green'; + } elseif ( $entry_status === 'DENY_SFW' ) { + $message_ip_status = $common_text_blocked . ' ' . $lists_text; $message_ip_status_color = 'red'; + } elseif ( $entry_status === 'PASS_SFW' || $entry_status === 'PASS_SFW__BY_COOKIE' ) { + $message_ip_status = $this->localize->translate('This IP is passed (not in any lists)'); + $message_ip_status_color = 'green'; + } - if ( $this->test_status === 1 ) { - $message_ip_status = $this->localize->translate('IP in the whitelist'); - $message_ip_status_color = 'green'; - } - + if ( !empty($message_ip_status) ) { $replaces['{MESSAGE_IP_STATUS}'] = "

$message_ip_status

"; } @@ -381,8 +430,6 @@ public function printDiePage($result) { parent::diePage($result); - http_response_code(403); - $localize_js = array( 'sfw__random_get' => '1', ); @@ -392,7 +439,7 @@ public function printDiePage($result) $replaces = array( '{JQUERY_SCRIPT_URL}' => '', - '{LOCALIZE_SCRIPT}' => 'var ctPublicFunctions = ' . json_encode($localize_js) . ';' . + '{LOCALIZE_SCRIPT}' => 'var ct_setcookie = 1; var ctPublicFunctions = ' . json_encode($localize_js) . ';' . 'var ctPublic = ' . json_encode($localize_js_public) . ';', ); @@ -512,20 +559,139 @@ public static function directUpdateGetBlackLists($api_key) { /** @var \Cleantalk\Common\Api\Api $api_class */ $api_class = Mloader::get('Api'); + /** @var \Cleantalk\Common\Helper\Helper $helper_class */ + $helper_class = Mloader::get('Helper'); - // Getting remote file name - $result = $api_class::methodGet2sBlacklistsDb($api_key, null, '3_1'); + // Getting common blacklists via multifiles v3_2 (common_lists=1 means ONLY common) + $result = $api_class::methodGet2sBlacklistsDb($api_key, 'multifiles', '3_2', 1); - if ( empty($result['error']) ) { - return array( - 'blacklist' => isset($result['data']) ? $result['data'] : null, - 'useragents' => isset($result['data_user_agents']) ? $result['data_user_agents'] : null, - 'bl_count' => isset($result['networks_count']) ? $result['networks_count'] : null, - 'ua_count' => isset($result['ua_count']) ? $result['ua_count'] : null, - ); + if ( !empty($result['error']) ) { + return $result; + } + + if ( empty($result['file_url']) ) { + return array('error' => 'COMMON_LISTS_FILE_URL_IS_EMPTY'); + } + + // Get the index of file URLs from the multifiles response + $file_urls = $helper_class::httpGetDataFromRemoteGzAndParseCsv($result['file_url']); + if ( !empty($file_urls['error']) ) { + return array('error' => 'COMMON_LISTS_GET_INDEX: ' . $file_urls['error']); + } + + // Download and parse each common list file from the index + $all_entries = array(); + + foreach ( $file_urls as $file_url_entry ) { + if ( empty($file_url_entry[0]) ) { + continue; + } + + $url = $file_url_entry[0]; + + $file_data = $helper_class::httpGetDataFromRemoteGz($url); + if ( !is_string($file_data) ) { + continue; + } + + $parsed = $helper_class::bufferParseCsv($file_data); + if ( !is_array($parsed) ) { + continue; + } + + foreach ( $parsed as $entry ) { + if ( !empty($entry[0]) && !empty($entry[1]) ) { + $all_entries[] = $entry; + } + } + } + + // Get useragents from separate API response field + $useragents = null; + if ( !empty($result['file_ua_url']) ) { + $ua_data = $helper_class::httpGetDataFromRemoteGz($result['file_ua_url']); + if ( is_string($ua_data) ) { + $useragents = $helper_class::bufferParseCsv($ua_data); + if ( !is_array($useragents) ) { + $useragents = null; + } + } + } + + if ( empty($all_entries) ) { + return array('error' => 'COMMON_LISTS_NO_ENTRIES'); } - return $result; + return array( + 'blacklist' => $all_entries, + 'useragents' => $useragents, + 'bl_count' => count($all_entries), + 'ua_count' => is_array($useragents) ? count($useragents) : 0, + ); + } + + public static function directUpdateGetBlackListsPersonal($api_key) + { + /** @var \Cleantalk\Common\Api\Api $api_class */ + $api_class = Mloader::get('Api'); + /** @var \Cleantalk\Common\Helper\Helper $helper_class */ + $helper_class = Mloader::get('Helper'); + + // Getting personal blacklists file URL (common_lists=0 means personal only) + $result = $api_class::methodGet2sBlacklistsDb($api_key, 'multifiles', '3_2', 0); + + if ( !empty($result['error']) ) { + return $result; + } + + if ( empty($result['file_url']) ) { + return array('error' => 'PERSONAL_LISTS_FILE_URL_IS_EMPTY'); + } + + // Get the index of file URLs from the multifiles response + $file_urls = $helper_class::httpGetDataFromRemoteGzAndParseCsv($result['file_url']); + if ( !empty($file_urls['error']) ) { + return array('error' => 'PERSONAL_LISTS_GET_INDEX: ' . $file_urls['error']); + } + + // Download and parse each personal list file + $all_entries = array(); + foreach ( $file_urls as $file_url_entry ) { + if ( empty($file_url_entry[0]) ) { + continue; + } + + $url = $file_url_entry[0]; + + // Skip non-blacklist files (ua_list, ck_list) + if ( strpos($url, 'bl_list') === false ) { + continue; + } + + $file_data = $helper_class::httpGetDataFromRemoteGz($url); + if ( !is_string($file_data) ) { + continue; + } + + $parsed = $helper_class::bufferParseCsv($file_data); + if ( !is_array($parsed) ) { + continue; + } + + foreach ( $parsed as $entry ) { + if ( !empty($entry[0]) && !empty($entry[1]) ) { + $all_entries[] = $entry; + } + } + } + + if ( empty($all_entries) ) { + return array('error' => 'PERSONAL_LISTS_NO_ENTRIES'); + } + + return array( + 'blacklist' => $all_entries, + ); } public static function directUpdate($db, $db__table__data, $blacklists) @@ -576,7 +742,7 @@ public static function directUpdate($db, $db__table__data, $blacklists) * * @return array|int array('error' => STRING) */ - public static function updateWriteToDb($db, $db__table__data, $file_url = null) + public static function updateWriteToDb($db, $db__table__data, $file_url = null, $include_source = true) { $file_content = file_get_contents($file_url); @@ -592,7 +758,10 @@ public static function updateWriteToDb($db, $db__table__data, $file_url = null) reset($data); for ( $count_result = 0; current($data) !== false; ) { - $query = "INSERT INTO " . $db__table__data . " (network, mask, status, source) VALUES "; + $columns = $include_source + ? '(network, mask, status, source)' + : '(network, mask, status)'; + $query = "INSERT INTO " . $db__table__data . " $columns VALUES "; for ( $i = 0, $values = array(); @@ -609,9 +778,12 @@ public static function updateWriteToDb($db, $db__table__data, $file_url = null) $ip = preg_replace('/[^\d]*/', '', $entry[0]); $mask = preg_replace('/[^\d]*/', '', $entry[1]); $status = isset($entry[2]) ? $entry[2] : 0; - $source = isset($entry[3]) ? (int)$entry[3] : 'NULL'; - - $values[] = "($ip, $mask, $status, $source)"; + if ( $include_source ) { + $source = isset($entry[3]) ? (int)$entry[3] : 'NULL'; + $values[] = "($ip, $mask, $status, $source)"; + } else { + $values[] = "($ip, $mask, $status)"; + } } if ( !empty($values) ) { @@ -842,4 +1014,114 @@ public static function replaceDataTablesAtomically($db, $table_names) return true; } + + /** + * Add records to the personal SFW table. + * + * @param \Cleantalk\Common\Db\Db $db + * @param string $db__table__data Personal table name + * @param array $metadata Array of records with 'network', 'mask', 'status' keys + * + * @return array Result with 'total', 'added', 'updated', 'ignored' counts + * @throws \RuntimeException + */ + public static function privateRecordsAdd($db, $db__table__data, $metadata) + { + $added_count = 0; + $updated_count = 0; + $ignored_count = 0; + + foreach ( $metadata as $_key => $row ) { + // Find duplicate to use it on updating + $has_duplicate = false; + $query = "SELECT id, status FROM " . $db__table__data . " WHERE " + . "network = '" . (int)$row['network'] . "' AND " + . "mask = '" . (int)$row['mask'] . "'"; + + $db_result = $db->fetch($query); + if ( $db_result === false ) { + throw new \RuntimeException($db->getLastError()); + } + + // If the record is same - pass + if ( isset($db_result['status']) && (int)$db_result['status'] === (int)$row['status'] ) { + $ignored_count++; + continue; + } + + // If duplicate found create a chunk + if ( isset($db_result['id']) ) { + $id_chunk = "id = '" . (int)$db_result['id'] . "',"; + $has_duplicate = true; + } else { + $id_chunk = ''; + } + + // Insertion + $query = "INSERT INTO " . $db__table__data . " SET " + . $id_chunk + . "network = '" . (int)$row['network'] . "'," + . "mask = '" . (int)$row['mask'] . "'," + . "status = '" . (int)$row['status'] . "' " + . "ON DUPLICATE KEY UPDATE " + . "id = id," + . "network = network," + . "mask = mask," + . "status = '" . (int)$row['status'] . "';"; + + $db_result = $db->execute($query); + if ( $db_result === false ) { + throw new \RuntimeException($db->getLastError()); + } + + $added_count = $has_duplicate ? $added_count : $added_count + 1; + $updated_count = $has_duplicate ? $updated_count + 1 : $updated_count; + } + + return array( + 'total' => $added_count + $updated_count + $ignored_count, + 'added' => $added_count, + 'updated' => $updated_count, + 'ignored' => $ignored_count, + ); + } + + /** + * Delete records from the personal SFW table. + * + * @param \Cleantalk\Common\Db\Db $db + * @param string $db__table__data Personal table name + * @param array $metadata Array of records with 'network', 'mask' keys + * + * @return array Result with 'total', 'deleted', 'ignored' counts + * @throws \Exception + */ + public static function privateRecordsDelete($db, $db__table__data, $metadata) + { + $success_count = 0; + $ignored_count = 0; + + foreach ( $metadata as $_key => $row ) { + $query = "DELETE FROM " . $db__table__data . " WHERE " + . "network = '" . (int)$row['network'] . "' AND " + . "mask = '" . (int)$row['mask'] . "';"; + $db_result = $db->execute($query); + if ( $db_result === false ) { + throw new \Exception($db->getLastError()); + } + + $affected = (int)$db->getAffectedRows(); + if ( $affected === 0 ) { + $ignored_count++; + } else { + $success_count += $affected; + } + } + + return array( + 'total' => $success_count + $ignored_count, + 'deleted' => $success_count, + 'ignored' => $ignored_count, + ); + } }