Skip to content

New. SFW. Edits for adding personal SFW lists #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AntonV1211 wants to merge 5 commits into
base: master
Choose a base branch
from
Open

New. SFW. Edits for adding personal SFW lists #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
56d2d8a
New. SFW. Edits for adding personal SFW lists
AntonV1211 Jun 26, 2026
e5c859f
Fix phpcs errors
AntonV1211 Jun 26, 2026
794dec9
Mod. SFW. Editing comments, editing sfw downloads, and personal lists
AntonV1211 Jun 30, 2026
3a9ed7a
Mod. SFW. Editing comments, editing sfw downloads, and personal lists
AntonV1211 Jun 30, 2026
4735426
Fix review copilot
AntonV1211 Jul 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion Firewall.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ public function run()
$result['ip'],
$result['status'],
isset($result['network']) ? $result['network'] : null,
isset($result['is_personal']) ? $result['is_personal'] : 'NULL'
!empty($result['is_personal']) ? 1 : 'NULL'
);
}
}
Expand All @@ -256,6 +256,9 @@ public function run()
$this->fw_modules[$module_name]->actionsForPassed($result);
$this->fw_modules[$module_name]->diePage($result);
} else {
if ( Get::get('sfw_test_ip') ) {
$this->fw_modules[$module_name]->diePage($result);
}
$this->fw_modules[$module_name]->actionsForPassed($result);
}
}
Expand Down Expand Up @@ -305,6 +308,7 @@ private function prioritize($results)
$result['passed_ip'] = isset($fw_result['ip']) ? $fw_result['ip'] : $fw_result['passed_ip'];
$result['blocked_ip'] = isset($fw_result['ip']) ? $fw_result['ip'] : $fw_result['blocked_ip'];
$result['pattern'] = isset($fw_result['pattern']) ? $fw_result['pattern'] : array();
$result['is_personal'] = !empty($fw_result['is_personal']) ? 1 : 0;
}
}
}
Expand Down
185 changes: 156 additions & 29 deletions FirewallUpdater.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,44 +283,74 @@ public static function getMultifiles($api_key)
/** @var \Cleantalk\Common\Helper\Helper $helper_class */
$helper_class = Mloader::get('Helper');

// Getting remote file name
$urls = array();

$result = $api_class::methodGet2sBlacklistsDb($api_key, 'multifiles', '3_1');
// Getting common lists
$result = $api_class::methodGet2sBlacklistsDb($api_key, 'multifiles', '3_2', 1);

if ( empty($result['error']) ) {
if ( !empty($result['file_url']) ) {
$file_urls = $helper_class::httpGetDataFromRemoteGzAndParseCsv($result['file_url']);
if ( empty($file_urls['error']) ) {
// Extract common_lists_url_id from the file URL
preg_match('/bl_list_(.+)\.multifiles/m', $result['file_url'], $url_id_common);
if ( isset($url_id_common[1]) ) {
$fw_stats->common_lists_url_id = $url_id_common[1];
}
if ( !empty($result['file_ua_url']) ) {
$file_urls[][0] = $result['file_ua_url'];
}
if ( !empty($result['file_ck_url']) ) {
$file_urls[][0] = $result['file_ck_url'];
}
$urls = array();
foreach ( $file_urls as $value ) {
$urls[] = $value[0];
}

$tries_for_download_again = 3 + (int)(count($urls) / 20);
$fw_stats->update_percent = round(100 / count($urls), 2);
Firewall::saveFwStats($fw_stats);

return array(
'next_stage' => array(
'name' => [self::class, 'downloadFiles'],
'args' => $urls,
'accepted_tries' => $tries_for_download_again
)
);
} else {
throw new SfwUpdateException('getMultifiles: common: ' . $file_urls['error']);
}

throw new SfwUpdateException('getMultifiles: ' . $file_urls['error']);
}
} else {
return $result;
}
return null;

// Getting personal lists
$result_personal = $api_class::methodGet2sBlacklistsDb($api_key, 'multifiles', '3_2', 0);

if ( empty($result_personal['error']) ) {
if ( !empty($result_personal['file_url']) ) {
$file_urls_personal = $helper_class::httpGetDataFromRemoteGzAndParseCsv($result_personal['file_url']);
if ( empty($file_urls_personal['error']) ) {
// Extract personal_lists_url_id from the file URL
preg_match('/bl_list_(.+)\.multifiles/m', $result_personal['file_url'], $url_id);
if ( isset($url_id[1]) ) {
$fw_stats->personal_lists_url_id = $url_id[1];
}
if ( !empty($result_personal['file_ck_url']) ) {
$file_urls_personal[][0] = $result_personal['file_ck_url'];
}
foreach ( $file_urls_personal as $value ) {
$urls[] = $value[0];
}
}
}
}

if ( empty($urls) ) {
throw new SfwUpdateException('getMultifiles: No URLs to download');
}

$tries_for_download_again = 3 + (int)(count($urls) / 20);
$fw_stats->update_percent = round(100 / count($urls), 2);
Firewall::saveFwStats($fw_stats);

return array(
'next_stage' => array(
'name' => [self::class, 'downloadFiles'],
'args' => $urls,
'accepted_tries' => $tries_for_download_again
)
);
}

public static function downloadFiles($api_key, $urls)
Expand Down Expand Up @@ -398,6 +428,8 @@ public static function createTables($_api_key)
$db_tables_creator = new DbTablesCreator();
$table_name_sfw = $db_obj->prefix . Schema::getSchemaTablePrefix() . 'sfw';
$db_tables_creator->createTable($table_name_sfw);
$table_name_sfw_personal = $db_obj->prefix . Schema::getSchemaTablePrefix() . 'sfw_personal';
$db_tables_creator->createTable($table_name_sfw_personal);
$table_name_ua = $db_obj->prefix . Schema::getSchemaTablePrefix() . 'ua_bl';
$db_tables_creator->createTable($table_name_ua);

Expand All @@ -414,7 +446,7 @@ public static function createTempTables($_api_key)
$db_class = Mloader::get('Db');
$db_obj = $db_class::getInstance();

// Preparing temporary tables
// Preparing temporary tables for common SFW
$result = \Cleantalk\Common\Firewall\Modules\Sfw::createTempTables(
$db_obj,
$db_obj->prefix . APBCT_TBL_FIREWALL_DATA
Expand Down Expand Up @@ -443,7 +475,21 @@ public static function processFiles($_api_key)
$concrete_file = current($files);

if ( strpos($concrete_file, 'bl_list') !== false ) {
$result = self::processFile($concrete_file);
// Determine direction: personal or common
if (
!empty($fw_stats->personal_lists_url_id)
&& strpos($concrete_file, $fw_stats->personal_lists_url_id) !== false
) {
$direction = 'personal';
} elseif (
!empty($fw_stats->common_lists_url_id)
&& strpos($concrete_file, $fw_stats->common_lists_url_id) !== false
) {
$direction = 'common';
} else {
$direction = 'common';
}
$result = self::processFile($concrete_file, $direction);
}

if ( strpos($concrete_file, 'ua_list') !== false ) {
Expand Down Expand Up @@ -476,7 +522,7 @@ public static function processFiles($_api_key)
);
}

public static function processFile($file_path)
public static function processFile($file_path, $direction = 'common')
{
if ( !file_exists($file_path) ) {
return array('error' => 'PROCESS FILE: ' . $file_path . ' is not exists.');
Expand All @@ -486,10 +532,17 @@ public static function processFile($file_path)
$db_class = Mloader::get('Db');
$db_obj = $db_class::getInstance();

$table_name = $direction === 'common'
? $db_obj->prefix . APBCT_TBL_FIREWALL_DATA . '_temp'
: $db_obj->prefix . APBCT_TBL_FIREWALL_DATA_PERSONAL . '_temp';

$include_source = $direction === 'common';

$result = \Cleantalk\Common\Firewall\Modules\Sfw::updateWriteToDb(
$db_obj,
$db_obj->prefix . APBCT_TBL_FIREWALL_DATA . '_temp',
$file_path
$table_name,
$file_path,
$include_source
);

if ( !empty($result['error']) ) {
Expand Down Expand Up @@ -550,6 +603,12 @@ public static function processCk($file_path)
$expected_networks_count = 0;
$expected_ua_count = 0;

// Determine if this is a personal ck_list
$is_personal_ck =
!empty($fw_stats->personal_lists_url_id) &&
strpos($file_path, $fw_stats->personal_lists_url_id) !==
false;

foreach ( $file_ck_url__data as $value ) {
if ( trim($value[0], '"') === 'networks_count' ) {
$expected_networks_count = $value[1];
Expand All @@ -559,8 +618,12 @@ public static function processCk($file_path)
}
}

$fw_stats->expected_networks_count = $expected_networks_count;
$fw_stats->expected_ua_count = $expected_ua_count;
if ( $is_personal_ck ) {
$fw_stats->expected_networks_count_personal = $expected_networks_count;
} else {
$fw_stats->expected_networks_count = $expected_networks_count;
$fw_stats->expected_ua_count = $expected_ua_count;
}
Firewall::saveFwStats($fw_stats);

if ( file_exists($file_path) ) {
Expand Down Expand Up @@ -627,13 +690,28 @@ public static function endOfUpdateRenamingTables($_api_key)
$db_obj->prefix . APBCT_TBL_FIREWALL_DATA
);

$fw_stats->update_mode = 0;
Firewall::saveFwStats($fw_stats);

if ( !empty($result['error']) ) {
$fw_stats->update_mode = 0;
Firewall::saveFwStats($fw_stats);
throw new SfwUpdateException('endOfUpdateRenamingTables: ' . $result['error']);
}

// ATOMIC REMOVE AND RENAME personal table
if ( $db_obj->isTableExists($db_obj->prefix . APBCT_TBL_FIREWALL_DATA_PERSONAL . '_temp') ) {
$result = \Cleantalk\Common\Firewall\Modules\Sfw::replaceDataTablesAtomically(
$db_obj,
$db_obj->prefix . APBCT_TBL_FIREWALL_DATA_PERSONAL
);
if ( !empty($result['error']) ) {
$fw_stats->update_mode = 0;
Firewall::saveFwStats($fw_stats);
throw new SfwUpdateException('endOfUpdateRenamingTables personal: ' . $result['error']);
}
}

$fw_stats->update_mode = 0;
Firewall::saveFwStats($fw_stats);

return array(
'next_stage' => array(
'name' => [self::class, 'endOfUpdateCheckingData'],
Expand Down Expand Up @@ -670,6 +748,13 @@ public static function endOfUpdateCheckingData($_api_key)
}

$fw_stats->entries = $entries;
// Check personal table entries if exists
if ( $db_obj->isTableExists($db_obj->prefix . APBCT_TBL_FIREWALL_DATA_PERSONAL) ) {
$entries_personal = $db_obj->setQuery('')->getVar(
'SELECT COUNT(*) FROM ' . $db_obj->prefix . APBCT_TBL_FIREWALL_DATA_PERSONAL
);
$fw_stats->entries_personal = $entries_personal;
}
Firewall::saveFwStats($fw_stats);

return array(
Expand Down Expand Up @@ -730,7 +815,10 @@ public static function endOfUpdate($_api_key, $is_first_updating = false)
$fw_stats->update_percent = 0;
$fw_stats->updating_id = null;
$fw_stats->expected_networks_count = false;
$fw_stats->expected_networks_count_personal = false;
$fw_stats->expected_ua_count = false;
$fw_stats->personal_lists_url_id = null;
$fw_stats->common_lists_url_id = null;
Firewall::saveFwStats($fw_stats);

return true;
Expand Down Expand Up @@ -841,10 +929,11 @@ public function directUpdate()
}

// Preparing database infrastructure
// @ToDo need to implement returning result of the Activator::createTables work.
$db_tables_creator = new DbTablesCreator();
$table_name = $db_obj->prefix . Schema::getSchemaTablePrefix() . 'sfw';
$db_tables_creator->createTable($table_name);
$table_name_personal = $db_obj->prefix . Schema::getSchemaTablePrefix() . 'sfw_personal';
$db_tables_creator->createTable($table_name_personal);

$result__creating_tmp_table = \Cleantalk\Common\Firewall\Modules\SFW::createTempTables(
$db_obj,
Expand All @@ -854,6 +943,15 @@ public function directUpdate()
return array('error' => 'DIRECT UPDATING CREATE TMP TABLE: ' . $result__creating_tmp_table['error']);
}

// Create personal temp table
$result__creating_tmp_table_personal = \Cleantalk\Common\Firewall\Modules\SFW::createTempTables(
$db_obj,
$db_obj->prefix . APBCT_TBL_FIREWALL_DATA_PERSONAL
);
if ( !empty($result__creating_tmp_table_personal['error']) ) {
return array('error' => 'DIRECT UPDATING CREATE PERSONAL TMP TABLE: ' . $result__creating_tmp_table_personal['error']);
}

/**
* UPDATING UA LIST
*/
Expand Down Expand Up @@ -886,6 +984,30 @@ public function directUpdate()
return array('error' => 'DIRECT UPDATING BLACK LIST: WRONG RESPONSE FROM SFW::directUpdate');
}

/**
* UPDATING PERSONAL LIST
*/
$result_personal = \Cleantalk\Common\Firewall\Modules\Sfw::directUpdateGetBlackListsPersonal($this->api_key);
if ( empty($result_personal['error']) && !empty($result_personal['blacklist']) ) {
$personal_blacklists = $result_personal['blacklist'];

$upd_result_personal = \Cleantalk\Common\Firewall\Modules\SFW::directUpdate(
$db_obj,
$db_obj->prefix . APBCT_TBL_FIREWALL_DATA_PERSONAL . '_temp',
$personal_blacklists
);

if ( !empty($upd_result_personal['error']) ) {
// Personal lists errors are not critical, continue update
}

if ( is_int($upd_result_personal) ) {
$fw_stats = Firewall::getFwStats();
$fw_stats->expected_networks_count_personal = $upd_result_personal;
Firewall::saveFwStats($fw_stats);
}
}

/**
* UPDATING EXCLUSIONS LIST
*/
Expand Down Expand Up @@ -941,6 +1063,11 @@ public static function cleanData()
$db_obj->prefix . APBCT_TBL_FIREWALL_DATA . '_temp'
);

\Cleantalk\Common\Firewall\Modules\SFW::dataTablesDelete(
$db_obj,
$db_obj->prefix . APBCT_TBL_FIREWALL_DATA_PERSONAL . '_temp'
);

$fw_stats->firewall_update_percent = 0;
$fw_stats->firewall_updating_id = null;
Firewall::saveFwStats($fw_stats);
Expand Down
6 changes: 6 additions & 0 deletions FwStats.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,17 @@ class FwStats
public $update_period = 86400;
public $updating_folder = 0;
public $expected_networks_count = 0;
public $expected_networks_count_personal = 0;
public $expected_ua_count = 0;
public $personal_lists_url_id;
public $common_lists_url_id;
public $calls = 0;
public $update_mode;
public $last_update_time;
public $last_update_way;
public $entries;
public $entries_personal;
public $errors;
public $firewall_update_percent;
public $firewall_updating_id;
}
Loading
Loading