From fe8aa9824d6ffa8f2b904cc728b2753c388b4c00 Mon Sep 17 00:00:00 2001 From: Giovanni Date: Thu, 13 Aug 2026 11:01:18 +0200 Subject: [PATCH 1/2] merge DecodeShortURLs and Redirectors plugins Co-Authored-By: Claude Sonnet 5 --- .../SpamAssassin/Plugin/DecodeShortURLs.pm | 1004 +---------------- lib/Mail/SpamAssassin/Plugin/Redirectors.pm | 936 +++++++++++++-- t/decodeshorturl.t | 6 +- 3 files changed, 861 insertions(+), 1085 deletions(-) diff --git a/lib/Mail/SpamAssassin/Plugin/DecodeShortURLs.pm b/lib/Mail/SpamAssassin/Plugin/DecodeShortURLs.pm index 73ac5b9aa9..c48b0a9efb 100644 --- a/lib/Mail/SpamAssassin/Plugin/DecodeShortURLs.pm +++ b/lib/Mail/SpamAssassin/Plugin/DecodeShortURLs.pm @@ -5,9 +5,9 @@ # The ASF licenses this file to you under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at: -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,996 +17,70 @@ =head1 NAME -DecodeShortURLs - Check for shortened URLs +DecodeShortURLs - deprecated, merged into Mail::SpamAssassin::Plugin::Redirectors =head1 SYNOPSIS loadplugin Mail::SpamAssassin::Plugin::DecodeShortURLs - url_shortener tinyurl.com - url_shortener_get bit.ly - url_shortener_custom_user_agent t.co curl/8.6.0 - - body HAS_SHORT_URL eval:short_url() - describe HAS_SHORT_URL Message has one or more shortened URLs - - body SHORT_URL_REDIR eval:short_url_redir() - describe SHORT_URL_REDIR Message has shortened URL that resulted in a valid redirection - - body SHORT_URL_CHAINED eval:short_url_chained() - describe SHORT_URL_CHAINED Message has shortened URL chained to other shorteners - - body SHORT_URL_MAXCHAIN eval:short_url_maxchain() - describe SHORT_URL_MAXCHAIN Message has shortened URL that causes too many redirections - - body SHORT_URL_LOOP eval:short_url_loop() - describe SHORT_URL_LOOP Message has short URL that loops back to itself - - body SHORT_URL_200 eval:short_url_code('200') # Can check any non-redirect HTTP code - describe SHORT_URL_200 Message has shortened URL returning HTTP 200 - - body SHORT_URL_404 eval:short_url_code('404') # Can check any non-redirect HTTP code - describe SHORT_URL_404 Message has shortened URL returning HTTP 404 - - uri URI_TINYURL_BLOCKED m,https://tinyurl\.com/app/nospam, - describe URI_TINYURL_BLOCKED Message contains a tinyurl that has been disabled due to abuse - - uri URI_BITLY_BLOCKED m,^https://bitly\.com/a/blocked, - describe URI_BITLY_BLOCKED Message contains a bit.ly URL that has been disabled due to abuse - =head1 DESCRIPTION -This plugin looks for URLs shortened by a list of URL shortening services. -Upon finding a matching URL, plugin will send a HTTP request to the -shortening service and retrieve the Location-header which points to the -actual shortened URL. It then adds this URL to the list of URIs extracted -by SpamAssassin which can then be accessed by uri rules and plugins such as -URIDNSBL. - -This plugin will follow chained redirections, where a short URL redirects to -another short URL. Redirection depth limit can be set with -C. +B All of its functionality (the C +settings and the C eval rules) has been merged into +L, which also gained the ability to +follow a redirect chain that mixes shorteners and redirectors in either +order -- there is no functional difference between the two, both are just +a host whose HTTP response redirects elsewhere. -Maximum of C short URLs are checked in a message (10 by -default). Setting it to 0 disables HTTP requests, allowing only short_url() -test to work and report found shorteners. +This module remains only as a compatibility shim so that existing +configuration files with C +keep working unchanged. It loads C and inherits all of its +behavior. New configurations should load +C directly instead. -All supported rule types for checking short URLs and redirection status are -documented in L section. - -=head1 NOTES +=head1 ACKNOWLEDGEMENTS -This plugin runs at priority -10 so that it may -modify the parsed URI list prior to normal uri rules or the URIDNSBL plugin -but after the Redirector plugin. +Original DecodeShortURLs plugin was developed by Steve Freegard. =cut package Mail::SpamAssassin::Plugin::DecodeShortURLs; -use Mail::SpamAssassin::Plugin; -use Mail::SpamAssassin::Util qw(idn_to_ascii is_fqdn_valid); +use Mail::SpamAssassin::Plugin::Redirectors; use strict; use warnings; use vars qw(@ISA); -@ISA = qw(Mail::SpamAssassin::Plugin); - -my $VERSION = 4.00; +@ISA = qw(Mail::SpamAssassin::Plugin::Redirectors); -use constant HAS_LWP_USERAGENT => eval { require LWP::UserAgent; require LWP::Protocol::https; }; - -sub dbg { my $msg = shift; return Mail::SpamAssassin::Logger::dbg("DecodeShortURLs: $msg", @_); } -sub info { my $msg = shift; return Mail::SpamAssassin::Logger::info("DecodeShortURLs: $msg", @_); } +# Published rulesets (e.g. rules/25_url_shortener.cf) probe for plugin +# features with "if can(Mail::SpamAssassin::Plugin::DecodeShortURLs::has_x)", +# which calls the fully-qualified sub directly rather than as a method, +# that bypasses @ISA, so a plain subclass wouldn't expose subs it merely +# inherits from Redirectors. Alias them into this package's symbol table +# so such checks keep working. +BEGIN { + no strict 'refs'; + for my $sub (qw( + has_short_url has_autoclean has_short_url_code has_user_agent + has_custom_user_agent has_get has_clear has_timeout + has_max_redirections has_short_url_redir + )) { + *{$sub} = \&{"Mail::SpamAssassin::Plugin::Redirectors::$sub"}; + } +} sub new { my $class = shift; my $mailsaobject = shift; - $class = ref($class) || $class; - my $self = $class->SUPER::new($mailsaobject); - bless ($self, $class); - - if ($mailsaobject->{local_tests_only}) { - dbg("local tests only, disabling HTTP requests"); - $self->{net_disabled} = 1; - } - elsif (!HAS_LWP_USERAGENT) { - dbg("module LWP::UserAgent not installed, disabling HTTP requests"); - $self->{net_disabled} = 1; - } - - $self->set_config($mailsaobject->{conf}); - $self->register_method_priority ('check_dnsbl', -10); - $self->register_eval_rule('short_url', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); - $self->register_eval_rule('short_url_redir', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); - $self->register_eval_rule('short_url_200', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); - $self->register_eval_rule('short_url_404', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); - $self->register_eval_rule('short_url_code', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); - $self->register_eval_rule('short_url_chained', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); - $self->register_eval_rule('short_url_maxchain', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); - $self->register_eval_rule('short_url_loop', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); - $self->register_eval_rule('short_url_tests'); # for legacy plugin compatibility warning - - return $self; -} - -=head1 USER SETTINGS - -=over 4 - -=item url_shortener domain [domain...] (default: none) - -Domains that should be considered as an URL shortener. If the domain begins -with a '.', 3rd level tld of the main domain will be checked. - -Example: - - url_shortener tinyurl.com - url_shortener .page.link - -=back - -=over 4 - -=item url_shortener_get domain [domain...] (default: none) - -Alias to C. HTTP request will be done with GET method, -instead of default HEAD. Required for some services like bit.ly to return -blocked URL correctly. - -Example: - - url_shortener_get bit.ly - -=back - -=cut - -sub set_config { - my($self, $conf) = @_; - my @cmds = (); - - push (@cmds, { - setting => 'url_shortener', - default => {}, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_HASH_KEY_VALUE, - code => sub { - my ($self, $key, $value, $line) = @_; - if ($value eq '') { - return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; - } - foreach my $domain (split(/\s+/, $value)) { - $self->{url_shortener}->{lc $domain} = 1; # 1 == head - } - } - }); - - push (@cmds, { - setting => 'url_shortener_get', - code => sub { - my ($self, $key, $value, $line) = @_; - if ($value eq '') { - return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; - } - foreach my $domain (split(/\s+/, $value)) { - $self->{url_shortener}->{lc $domain} = 2; # 2 == get - } - } - }); - -=over 4 - -=item url_shortener_custom_user_agent domain user-agent (default: none) - -Custom HTTP user-agent to be used for specific domains, -instead of the default specified in C. -Required for some services like t.co to return blocked URL correctly. - -Example: - - url_shortener_custom_user_agent t.co curl/8.6.0 - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_custom_user_agent', - code => sub { - my ($self, $key, $value, $line) = @_; - if ($value eq '') { - return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; - } - my @values = split(/\s+/, $value); - my $domain = shift(@values); - my $ua = join('', @values); - $self->{url_shortener}->{user_agent}->{lc $domain} = $ua; - } - }); - -=over 4 - -=item clear_url_shortener [domain] [domain...] - -Clear configured url_shortener and url_shortener_get domains, for example to -override default settings from an update channel. If domains are specified, -then only those are removed from list. - -=back - -=cut - - push (@cmds, { - setting => 'clear_url_shortener', - code => sub { - my ($self, $key, $value, $line) = @_; - if ($value eq '') { - $self->{url_shortener} = {}; - } else { - foreach my $domain (split(/\s+/, $value)) { - delete $self->{url_shortener}->{lc $domain}; - } - } - } - }); - -=head1 PRIVILEGED SETTINGS - -=over 4 - -=item url_shortener_cache_type (default: none) - -The cache type that is being utilized. Currently only supported value is -C that implies C is a DBI connect string. -DBI module is required. - -Example: -url_shortener_cache_type dbi - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_cache_type', - default => '', - is_priv => 1, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING - }); - -=over 4 - -=item url_shortener_cache_dsn (default: none) - -The DBI dsn of the database to use. - -For SQLite, the database will be created automatically if it does not -already exist, the supplied path and file must be read/writable by the -user running spamassassin or spamd. - -For MySQL/MariaDB or PostgreSQL, see sql-directory for database table -creation clauses. - -You will need to have the proper DBI module for your database. For example -DBD::SQLite, DBD::mysql, DBD::MariaDB or DBD::Pg. - -Minimum required SQLite version is 3.24.0 (available from DBD::SQLite 1.59_01). - -Examples: - - url_shortener_cache_dsn dbi:SQLite:dbname=/var/lib/spamassassin/DecodeShortURLs.db - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_cache_dsn', - default => '', - is_priv => 1, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING - }); - -=over 4 - -=item url_shortener_cache_username (default: none) - -The username that should be used to connect to the database. Not used for -SQLite. - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_cache_username', - default => '', - is_priv => 1, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING - }); - -=over 4 - -=item url_shortener_cache_password (default: none) - -The password that should be used to connect to the database. Not used for -SQLite. - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_cache_password', - default => '', - is_priv => 1, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING - }); - -=over 4 - -=item url_shortener_cache_ttl (default: 86400) - -The length of time a cache entry will be valid for in seconds. -Default is 86400 (1 day). - -See C for database cleaning. - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_cache_ttl', - is_admin => 1, - default => 86400, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC - }); - -=head1 ADMINISTRATOR SETTINGS - -=over 4 - -=item url_shortener_cache_autoclean (default: 1000) - -Automatically purge old entries from database. Value describes a random run -chance of 1/x. The default value of 1000 means that cleaning is run -approximately once for every 1000 messages processed. Value of 1 would mean -database is cleaned every time a message is processed. - -Set 0 to disable automatic cleaning and to do it manually. - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_cache_autoclean', - is_admin => 1, - default => 1000, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC - }); - -=over 4 - -=item url_shortener_loginfo (default: 0 (off)) - -If this option is enabled (set to 1), then short URLs and the decoded URLs will be logged with info priority. - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_loginfo', - is_admin => 1, - default => 0, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_BOOL - }); - -=over 4 - -=item url_shortener_timeout (default: 5) - -Maximum time a short URL HTTP request can take, in seconds. - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_timeout', - is_admin => 1, - default => 5, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC - }); - -=over 4 - -=item max_short_urls (default: 10) - -Maximum amount of short URLs that will be looked up per message. Chained -redirections are not counted, only initial short URLs found. - -Setting it to 0 disables HTTP requests, allowing only short_url() test to -work and report any found shortener URLs. - -=back - -=cut - - push (@cmds, { - setting => 'max_short_urls', - is_admin => 1, - default => 10, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC - }); - -=over 4 - -=item max_short_url_redirections (default: 10) - -Maximum depth of chained redirections that a short URL can generate. - -=back - -=cut - - push (@cmds, { - setting => 'max_short_url_redirections', - is_admin => 1, - default => 10, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC - }); - -=over 4 - -=item url_shortener_user_agent (default: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36) - -Set default User-Agent header for HTTP requests. Some services require it to look -like a common browser. User-Agent can be overridden on a per url_shortener basis using -the C setting. - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_user_agent', - is_admin => 1, - default => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36', - type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING - }); - - $conf->{parser}->register_commands(\@cmds); -} - -=head1 ACKNOWLEDGEMENTS - -Original DecodeShortURLs plugin was developed by Steve Freegard. - -=cut - -sub short_url_tests { - # Legacy compatibility warning done in finish_parsing_start - return 0; -} - -sub finish_parsing_start { - my ($self, $opts) = @_; - - if ($opts->{conf}->{eval_to_rule}->{short_url_tests}) { - warn "DecodeShortURLs: Legacy configuration format detected. ". - "Eval function short_url_tests() is no longer supported, ". - "please see documentation for the new rule format.\n"; - } -} - -sub initialise_url_shortener_cache { - my ($self, $conf) = @_; - - return if $self->{dbh} && $self->{dbh_pid} && $self->{dbh_pid} == $$; - return if !$conf->{url_shortener_cache_type}; - - if (!$conf->{url_shortener_cache_dsn}) { - warn "DecodeShortURLs: invalid cache configuration\n"; - return; - } - - ## - ## SQLite - ## - if ($conf->{url_shortener_cache_type} =~ /^(?:dbi|sqlite)$/i - && $conf->{url_shortener_cache_dsn} =~ /^dbi:SQLite/) - { - eval { - local $SIG{'__DIE__'}; - require DBI; - require DBD::SQLite; - DBD::SQLite->VERSION(1.59_01); # Required for ON CONFLICT - $self->{dbh} = DBI->connect_cached( - $conf->{url_shortener_cache_dsn}, '', '', - {RaiseError => 1, PrintError => 0, InactiveDestroy => 1, AutoCommit => 1} - ); - $self->{dbh}->do(" - CREATE TABLE IF NOT EXISTS short_url_cache ( - short_url TEXT PRIMARY KEY NOT NULL, - decoded_url TEXT NOT NULL, - hits INTEGER NOT NULL DEFAULT 1, - created INTEGER NOT NULL, - modified INTEGER NOT NULL - ) - "); - # Maintaining index for cleaning is likely more expensive than occasional full table scan - #$self->{dbh}->do(" - # CREATE INDEX IF NOT EXISTS short_url_modified - # ON short_url_cache(created) - #"); - $self->{sth_insert} = $self->{dbh}->prepare(" - INSERT INTO short_url_cache (short_url, decoded_url, created, modified) - VALUES (?,?,strftime('%s','now'),strftime('%s','now')) - ON CONFLICT(short_url) DO UPDATE - SET decoded_url = excluded.decoded_url, - modified = excluded.modified, - hits = hits + 1 - "); - $self->{sth_select} = $self->{dbh}->prepare(" - SELECT decoded_url FROM short_url_cache - WHERE short_url = ? - "); - $self->{sth_delete} = $self->{dbh}->prepare(" - DELETE FROM short_url_cache - WHERE short_url = ? AND created < strftime('%s','now') - $conf->{url_shortener_cache_ttl} - "); - $self->{sth_clean} = $self->{dbh}->prepare(" - DELETE FROM short_url_cache - WHERE created < strftime('%s','now') - $conf->{url_shortener_cache_ttl} - "); - }; - } - ## - ## MySQL/MariaDB - ## - elsif (lc $conf->{url_shortener_cache_type} eq 'dbi' - && $conf->{url_shortener_cache_dsn} =~ /^dbi:(?:mysql|MariaDB)/i) - { - eval { - local $SIG{'__DIE__'}; - require DBI; - $self->{dbh} = DBI->connect_cached( - $conf->{url_shortener_cache_dsn}, - $conf->{url_shortener_cache_username}, - $conf->{url_shortener_cache_password}, - {RaiseError => 1, PrintError => 0, InactiveDestroy => 1, AutoCommit => 1} - ); - $self->{sth_insert} = $self->{dbh}->prepare(" - INSERT INTO short_url_cache (short_url, decoded_url, created, modified) - VALUES (?,?,UNIX_TIMESTAMP(),UNIX_TIMESTAMP()) - ON DUPLICATE KEY UPDATE - decoded_url = VALUES(decoded_url), - modified = VALUES(modified), - hits = hits + 1 - "); - $self->{sth_select} = $self->{dbh}->prepare(" - SELECT decoded_url FROM short_url_cache - WHERE short_url = ? - "); - $self->{sth_delete} = $self->{dbh}->prepare(" - DELETE FROM short_url_cache - WHERE short_url = ? AND created < UNIX_TIMESTAMP() - $conf->{url_shortener_cache_ttl} - "); - $self->{sth_clean} = $self->{dbh}->prepare(" - DELETE FROM short_url_cache - WHERE created < UNIX_TIMESTAMP() - $conf->{url_shortener_cache_ttl} - "); - }; - } - ## - ## PostgreSQL - ## - elsif (lc $conf->{url_shortener_cache_type} eq 'dbi' - && $conf->{url_shortener_cache_dsn} =~ /^dbi:Pg/i) - { - eval { - local $SIG{'__DIE__'}; - require DBI; - $self->{dbh} = DBI->connect_cached( - $conf->{url_shortener_cache_dsn}, - $conf->{url_shortener_cache_username}, - $conf->{url_shortener_cache_password}, - {RaiseError => 1, PrintError => 0, InactiveDestroy => 1, AutoCommit => 1} - ); - $self->{sth_insert} = $self->{dbh}->prepare(" - INSERT INTO short_url_cache (short_url, decoded_url, created, modified) - VALUES (?,?,CAST(EXTRACT(epoch FROM NOW()) AS INT),CAST(EXTRACT(epoch FROM NOW()) AS INT)) - ON CONFLICT (short_url) DO UPDATE SET - decoded_url = EXCLUDED.decoded_url, - modified = EXCLUDED.modified, - hits = short_url_cache.hits + 1 - "); - $self->{sth_select} = $self->{dbh}->prepare(" - SELECT decoded_url FROM short_url_cache - WHERE short_url = ? - "); - $self->{sth_delete} = $self->{dbh}->prepare(" - DELETE FROM short_url_cache - WHERE short_url = ? AND created < CAST(EXTRACT(epoch FROM NOW()) AS INT) - $conf->{url_shortener_cache_ttl} - "); - $self->{sth_clean} = $self->{dbh}->prepare(" - DELETE FROM short_url_cache - WHERE created < CAST(EXTRACT(epoch FROM NOW()) AS INT) - $conf->{url_shortener_cache_ttl} - "); - }; - ## - ## ... - ## - } else { - warn "DecodeShortURLs: invalid cache configuration\n"; - return; - } - - if ($@ || !$self->{sth_clean}) { - warn "DecodeShortURLs: cache connect failed: $@\n"; - undef $self->{dbh}; - undef $self->{dbh_pid}; - undef $self->{sth_insert}; - undef $self->{sth_select}; - undef $self->{sth_delete}; - undef $self->{sth_clean}; - } else { - $self->{dbh_pid} = $$; - } -} - -sub short_url { - my ($self, $pms) = @_; - - # Make sure checks are run - $self->_check_short($pms); - - return $pms->{short_url} ? 1 : 0; -} - -sub short_url_redir { - my ($self, $pms) = @_; - - # Make sure checks are run - $self->_check_short($pms); - - return $pms->{short_url_redir} ? 1 : 0; -} - -sub short_url_200 { - my ($self, $pms) = @_; - - # Make sure checks are run - $self->_check_short($pms); - - return $pms->{short_url_200} ? 1 : 0; -} - -sub short_url_404 { - my ($self, $pms) = @_; - - # Make sure checks are run - $self->_check_short($pms); - - return $pms->{short_url_404} ? 1 : 0; -} - -sub short_url_code { - my ($self, $pms, undef, $code) = @_; - - # Make sure checks are run - $self->_check_short($pms); - - return 0 unless defined $code && $code =~ /^\d{3}$/; - return $pms->{"short_url_$code"} ? 1 : 0; -} - -sub short_url_chained { - my ($self, $pms) = @_; - - # Make sure checks are run - $self->_check_short($pms); - - return $pms->{short_url_chained} ? 1 : 0; -} - -sub short_url_maxchain { - my ($self, $pms) = @_; - - # Make sure checks are run - $self->_check_short($pms); - - return $pms->{short_url_maxchain} ? 1 : 0; -} - -sub short_url_loop { - my ($self, $pms) = @_; - - # Make sure checks are run - $self->_check_short($pms); - - return $pms->{short_url_loop} ? 1 : 0; -} - -sub _check_shortener_uri { - my ($uri, $conf) = @_; - - local($1,$2); - return 0 unless $uri =~ m{^ - https?:// # Only http - (?:[^\@/?#]*\@)? # Ignore user:pass@ - ([^/?#:]+) # (Capture hostname) - (?::\d+)? # Possible port - (.*?\w)? # Some path wanted - }ix; - my $host = lc $1; - if(is_fqdn_valid($host)) { - $host = idn_to_ascii($host); - } - my $has_path = defined $2; - my $levels = $host =~ tr/.//; - # No point looking at single level "xxx.yy" without a path - return if $levels == 1 && !$has_path; - - if (exists $conf->{url_shortener}->{$host}) { - return { - 'uri' => $uri, - 'method' => $conf->{url_shortener}->{$host} == 1 ? 'head' : 'get', - 'user_agent' => (defined $conf->{url_shortener}->{user_agent}->{$host}) ? $conf->{url_shortener}->{user_agent}->{$host} : $conf->{url_shortener_user_agent}, - }; - } - # if domain is a 3rd level domain check if there is a url shortener - # on the www domain - elsif($levels == 2 && $host =~ /^www\.([^.]+\.[^.]+)$/i) { - my $domain = $1; - if(($host eq "www.$domain") and exists $conf->{url_shortener}->{$domain}) { - dbg("Found internal www redirection for domain $domain"); - return { - 'uri' => $uri, - 'method' => $conf->{url_shortener}->{$domain} == 1 ? 'head' : 'get', - 'user_agent' => (defined $conf->{url_shortener}->{user_agent}->{$host}) ? $conf->{url_shortener}->{user_agent}->{$host} : $conf->{url_shortener_user_agent}, - }; - } - } - # if domain is a 3rd level domain check if there is a url shortener - # on the 2nd level tld - elsif ($levels == 2 && $host =~ /^(?!www)[^.]+(\.[^.]+\.[^.]+)$/i && - exists $conf->{url_shortener}->{$1}) { - return { - 'uri' => $uri, - 'method' => $conf->{url_shortener}->{$1} == 1 ? 'head' : 'get', - 'user_agent' => (defined $conf->{url_shortener}->{user_agent}->{$host}) ? $conf->{url_shortener}->{user_agent}->{$host} : $conf->{url_shortener_user_agent}, - }; - } - return; -} - -sub check_dnsbl { - my ($self, $opts) = @_; - - $self->_check_short($opts->{permsgstatus}); -} - -sub _check_short { - my ($self, $pms) = @_; - - return if $pms->{short_url_checked}++; - my $conf = $pms->{conf}; - - # Sort short URLs into hash to de-dup them - my %short_urls; - my $uris = $pms->get_uri_detail_list(); - foreach my $uri (keys %$uris) { - my $info = $uris->{$uri}; - next unless $info->{domains} && $info->{cleaned}; - # Remove anchors and parameters from shortened uris - $uri =~ s/\/?(?:\#|\?).*//g; - if (my $short_url_info = _check_shortener_uri($uri, $conf)) { - $short_urls{$uri} = $short_url_info; - last if scalar keys %short_urls >= $conf->{max_short_urls}; - } - } - - # Bail out if no shortener was found - return unless %short_urls; - - # Mark that a URL shortener was found - $pms->{short_url} = 1; - - # Bail out if network lookups not enabled or max_short_urls 0 - return if $self->{net_disabled}; - return if !$conf->{max_short_urls}; - - # Initialize cache - $self->initialise_url_shortener_cache($conf); - - # Initialize LWP - my $ua = LWP::UserAgent->new( - 'agent' => $conf->{url_shortener_user_agent}, - 'max_redirect' => 0, - 'timeout' => $conf->{url_shortener_timeout}, + Mail::SpamAssassin::Logger::dbg( + "DecodeShortURLs: this plugin is deprecated, its functionality has been ". + "merged into Redirectors; please switch to ". + "'loadplugin Mail::SpamAssassin::Plugin::Redirectors'" ); - $ua->env_proxy; - - # Launch HTTP requests - foreach my $uri (keys %short_urls) { - $self->recursive_lookup($short_urls{$uri}, $pms, $ua); - } - # Automatically purge old entries - if ($self->{dbh} && $conf->{url_shortener_cache_autoclean} - && rand() < 1/$conf->{url_shortener_cache_autoclean}) - { - dbg("cleaning stale cache entries"); - eval { $self->{sth_clean}->execute(); }; - if ($@) { dbg("cache cleaning failed: $@"); } - } + return $class->SUPER::new($mailsaobject); } -sub recursive_lookup { - my ($self, $short_url_info, $pms, $ua, %been_here) = @_; - my $conf = $pms->{conf}; - - my $count = scalar keys %been_here; - dbg("redirection count $count") if $count; - if ($count >= $conf->{max_short_url_redirections}) { - dbg("found more than $conf->{max_short_url_redirections} shortener redirections"); - # Fire test - $pms->{short_url_maxchain} = 1; - return; - } - - my $short_url = $short_url_info->{uri}; - my $location; - if (defined($location = $self->cache_get($short_url))) { - if ($conf->{url_shortener_loginfo}) { - info("found cached $short_url => $location"); - } else { - dbg("found cached $short_url => $location"); - } - # Cached http code? - if ($location =~ /^\d{3}$/) { - $pms->{"short_url_$location"} = 1; - # Update cache - $self->cache_add($short_url, $location); - return; - } - } else { - # Not cached; do lookup - my $method = $short_url_info->{method}; - my $useragent = $short_url_info->{user_agent}; - if(defined $useragent) { - $ua->agent($useragent); - } else { - $ua->agent($conf->{url_shortener_user_agent}); - } - my $response = $ua->$method($short_url); - if (!$response->is_redirect) { - dbg("URL is not redirect: $short_url = ".$response->status_line); - my $rcode = $response->code; - if ($rcode =~ /^\d{3}$/) { - $pms->{"short_url_$rcode"} = 1; - # Update cache - $self->cache_add($short_url, $rcode); - } - return; - } - $location = $response->headers->{location}; - if ($conf->{url_shortener_loginfo}) { - info("found $short_url => $location"); - } else { - dbg("found $short_url => $location"); - } - } - - # Update cache - $self->cache_add($short_url, $location); - - # Bail out if $short_url redirects to itself - if ($short_url eq $location) { - dbg("URL is redirect to itself"); - return; - } - - # At this point we have a valid redirection and new URL in $response - $pms->{short_url_redir} = 1; - - # Set chained here otherwise we might mark a disabled page or - # redirect back to the same host as chaining incorrectly. - $pms->{short_url_chained} = 1 if $count; - - # Check if it is a redirection to a relative URI - # Make it an absolute URI and chain to it in that case - if ($location !~ m{^[a-z]+://}i) { - my $orig_location = $location; - my $orig_short_url = $short_url; - # Strip to.. - if (index($location, '/') == 0) { - $short_url =~ s{^([a-z]+://.*?)[/?#].*}{$1}; # ..absolute path base is http://example.com - } else { - $short_url =~ s{^([a-z]+://.*/)}{$1}; # ..relative path base is http://example.com/a/b/ - } - $location = "$short_url$location"; - dbg("looks like a redirection to a relative URI: $orig_short_url => $location ($orig_location)"); - } - - if (exists $been_here{$location}) { - # Loop detected - dbg("error: loop detected: $location"); - $pms->{short_url_loop} = 1; - return; - } - $been_here{$location} = 1; - $pms->add_uri_detail_list($location) if !$pms->{uri_detail_list}->{$location}; - - # Check for recursion - if (my $short_url_info = _check_shortener_uri($location, $conf)) { - # Recurse... - $self->recursive_lookup($short_url_info, $pms, $ua, %been_here); - } -} - -sub cache_add { - my ($self, $short_url, $decoded_url) = @_; - - return if !$self->{dbh}; - return if length($short_url) > 256 || length($decoded_url) > 512; - - # Upsert - eval { $self->{sth_insert}->execute($short_url, $decoded_url); }; - if ($@) { - dbg("could not add to cache: $@"); - } - - return; -} - -sub cache_get { - my ($self, $key) = @_; - - return if !$self->{dbh}; - - # Make sure expired entries are gone. Just a quick check for primary key, - # not that expensive. - eval { $self->{sth_delete}->execute($key); }; - if ($@) { - dbg("cache delete failed: $@"); - return; - } - - # Now try to get it (don't bother parsing if something was deleted above, - # it would be rare event anyway) - eval { $self->{sth_select}->execute($key); }; - if ($@) { - dbg("cache get failed: $@"); - return; - } - - my @row = $self->{sth_select}->fetchrow_array(); - if (@row) { - return $row[0]; - } - - return; -} - -# Version features -sub has_short_url { 1 } -sub has_autoclean { 1 } -sub has_short_url_code { 1 } -sub has_user_agent { 1 } # url_shortener_user_agent -sub has_custom_user_agent { 1 } # url_shortener_custom_user_agent -sub has_get { 1 } # url_shortener_get -sub has_clear { 1 } # clear_url_shortener -sub has_timeout { 1 } # url_shortener_timeout -sub has_max_redirections { 1 } # max_short_url_redirections -# short_url() will always hit if matching url_shortener was found, even -# without HTTP requests. To check if a valid HTTP redirection response was -# seen, use short_url_redir(). -sub has_short_url_redir { 1 } - 1; diff --git a/lib/Mail/SpamAssassin/Plugin/Redirectors.pm b/lib/Mail/SpamAssassin/Plugin/Redirectors.pm index 32e9cfcd45..e8d9a47e55 100644 --- a/lib/Mail/SpamAssassin/Plugin/Redirectors.pm +++ b/lib/Mail/SpamAssassin/Plugin/Redirectors.pm @@ -5,9 +5,9 @@ # The ASF licenses this file to you under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at: -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,13 +17,14 @@ =head1 NAME -Redirectors - Check for redirected URLs +Redirectors - Check for redirected and shortened URLs =head1 SYNOPSIS loadplugin Mail::SpamAssassin::Plugin::Redirectors url_redirector bing.com + url_shortener tinyurl.com body HAS_REDIR_URL eval:redir_url() describe HAS_REDIR_URL Message has one or more redirected URLs @@ -46,32 +47,72 @@ Redirectors - Check for redirected URLs body REDIR_URL_404 eval:redir_url_code('404') # Can check any non-redirect HTTP code describe REDIR_URL_404 Message has redirected URL returning HTTP 404 + body HAS_SHORT_URL eval:short_url() + describe HAS_SHORT_URL Message has one or more shortened URLs + + body SHORT_URL_REDIR eval:short_url_redir() + describe SHORT_URL_REDIR Message has shortened URL that resulted in a valid redirection + + body SHORT_URL_CHAINED eval:short_url_chained() + describe SHORT_URL_CHAINED Message has shortened URL chained to other shorteners + + body SHORT_URL_MAXCHAIN eval:short_url_maxchain() + describe SHORT_URL_MAXCHAIN Message has shortened URL that causes too many redirections + + body SHORT_URL_LOOP eval:short_url_loop() + describe SHORT_URL_LOOP Message has short URL that loops back to itself + + body SHORT_URL_200 eval:short_url_code('200') # Can check any non-redirect HTTP code + describe SHORT_URL_200 Message has shortened URL returning HTTP 200 + + body SHORT_URL_404 eval:short_url_code('404') # Can check any non-redirect HTTP code + describe SHORT_URL_404 Message has shortened URL returning HTTP 404 + + uri URI_TINYURL_BLOCKED m,https://tinyurl\.com/app/nospam, + describe URI_TINYURL_BLOCKED Message contains a tinyurl that has been disabled due to abuse + + uri URI_BITLY_BLOCKED m,^https://bitly\.com/a/blocked, + describe URI_BITLY_BLOCKED Message contains a bit.ly URL that has been disabled due to abuse + =head1 DESCRIPTION -This plugin looks for URLs redirected by a list of URL redirector services. -Upon finding a matching URL, plugin will send a HTTP request to the -redirector service and retrieve the Location-header which points to the -actual redirected URL. It then adds this URL to the list of URIs extracted -by SpamAssassin which can then be accessed by uri rules and plugins such as -URIDNSBL. +This plugin looks for URLs redirected or shortened by a list of URL +redirector/shortener services. Upon finding a matching URL, plugin will +send a HTTP request to the service and retrieve the Location-header which +points to the actual destination URL. It then adds this URL to the list of +URIs extracted by SpamAssassin which can then be accessed by uri rules and +plugins such as URIDNSBL. -This plugin will follow chained redirections, where a redirected URL redirects to -another redirector. Redirection depth limit can be set with -C. +This plugin will follow chained redirections, where a redirected or +shortened URL leads to another redirector or shortener, in any combination +and order -- for example a redirector that unwraps into a shortener, or a +shortener that unwraps into a redirector. Redirection depth limits can be +set with C and C. -Maximum of C redirected URLs are checked in a message (10 by -default). Setting it to 0 disables HTTP requests, allowing only redir_url() -test to work and report found redirectors. +Maximum of C redirector URLs and C shortener +URLs are checked in a message (10 each by default). Setting either to 0 +disables HTTP requests for that category, allowing only the corresponding +C/C test to work and report found redirectors or +shorteners. -All supported rule types for checking redirector URLs and redirection status are -documented in L section. +All supported rule types for checking redirected/shortened URLs and +redirection status are documented in L section. =head1 NOTES -This plugin runs before priority 0 so that it may -modify the parsed URI list prior to normal uri rules or the URIDNSBL plugin. -It should run before DecodeShortURLs plugin so that redirected short uris are also -checked. +This plugin runs before priority 0 so that it may modify the parsed URI +list prior to normal uri rules or the URIDNSBL plugin. + +Redirector and shortener matching share a single recursive walk over each +URI's redirect chain, so every hop is checked against both C +and C configuration regardless of which kind started the +chain. + +=head1 ACKNOWLEDGEMENTS + +The url_shortener functionality was originally provided by a separate +DecodeShortURLs plugin, functionality has been merged into this one; +C is now a deprecated compatibility shim that loads this plugin. =cut @@ -85,7 +126,7 @@ use warnings; use vars qw(@ISA); @ISA = qw(Mail::SpamAssassin::Plugin); -my $VERSION = 4.02; +my $VERSION = 4.03; use constant HAS_LWP_USERAGENT => eval { require LWP::UserAgent; require LWP::Protocol::https; }; use constant HAS_SELENIUM => eval { require Selenium::Remote::Driver; }; @@ -111,7 +152,8 @@ sub new { } $self->set_config($mailsaobject->{conf}); - # run at priority -15 so that redirected short uris can also be checked + # run at priority -15 so that redirected/shortened uris are always + # checked in a single pass, regardless of which kind of hop starts a chain $self->register_method_priority ('check_dnsbl', -15); $self->register_eval_rule('redir_url', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); $self->register_eval_rule('redir_url_valid', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); @@ -121,6 +163,15 @@ sub new { $self->register_eval_rule('redir_url_chained_domain', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); $self->register_eval_rule('redir_url_maxchain', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); $self->register_eval_rule('redir_url_loop', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); + $self->register_eval_rule('short_url', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); + $self->register_eval_rule('short_url_redir', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); + $self->register_eval_rule('short_url_200', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); + $self->register_eval_rule('short_url_404', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); + $self->register_eval_rule('short_url_code', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); + $self->register_eval_rule('short_url_chained', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); + $self->register_eval_rule('short_url_maxchain', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); + $self->register_eval_rule('short_url_loop', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); + $self->register_eval_rule('short_url_tests'); # for legacy DecodeShortURLs compatibility warning return $self; } @@ -264,7 +315,8 @@ Set Selenium port to use. Clear configured url_redirector domains, for example to override default settings from an update channel. If no arguments are given, -all entries are cleared. If domains are specified, only those are removed. +all redirector entries are cleared (url_shortener entries are untouched). If +domains are specified, only those are removed. When an entry includes a C, only that path is removed from the domain's allowlist; the domain entry itself is dropped only when its path @@ -280,11 +332,10 @@ added by a bare-domain configuration. code => sub { my ($self, $key, $value, $line) = @_; if ($value eq '') { - $self->{url_redirector_exact} = {}; - $self->{url_redirector_suffix} = {}; + _clear_all_entries_by_class($self, 'redirector'); } else { foreach my $token (split(/\s+/, $value)) { - _clear_redirector_entry($self, $token); + _clear_redirector_entry($self, $token, 'redirector'); } } } @@ -441,6 +492,123 @@ The regexp must match only the redirected domain. }, }); +=over 4 + +=item url_shortener domain [domain...] (default: none) + +Domains that should be considered as an URL shortener. If the domain begins +with a '.', any subdomain of the domain will be checked (see the leading-dot +rule under C). + +Example: + + url_shortener tinyurl.com + url_shortener .page.link + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener', + default => {}, + type => $Mail::SpamAssassin::Conf::CONF_TYPE_HASH_KEY_VALUE, + code => sub { + my ($self, $key, $value, $line) = @_; + if ($value eq '') { + return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; + } + foreach my $token (split(/\s+/, $value)) { + _add_redirector_entry($self, lc($token), 'head', 'shortener'); + } + } + }); + +=over 4 + +=item url_shortener_get domain [domain...] (default: none) + +Alias to C. HTTP request will be done with GET method, +instead of default HEAD. Required for some services like bit.ly to return +blocked URL correctly. + +Example: + + url_shortener_get bit.ly + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener_get', + code => sub { + my ($self, $key, $value, $line) = @_; + if ($value eq '') { + return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; + } + foreach my $token (split(/\s+/, $value)) { + _add_redirector_entry($self, lc($token), 'get', 'shortener'); + } + } + }); + +=over 4 + +=item url_shortener_custom_user_agent domain user-agent (default: none) + +Custom HTTP user-agent to be used for specific domains, +instead of the default specified in C. +Required for some services like t.co to return blocked URL correctly. + +Example: + + url_shortener_custom_user_agent t.co curl/8.6.0 + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener_custom_user_agent', + code => sub { + my ($self, $key, $value, $line) = @_; + if ($value eq '') { + return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; + } + my @values = split(/\s+/, $value); + my $domain = shift(@values); + my $ua = join('', @values); + $self->{url_shortener_custom_ua}->{lc $domain} = $ua; + } + }); + +=over 4 + +=item clear_url_shortener [domain] [domain...] + +Clear configured url_shortener and url_shortener_get domains, for example to +override default settings from an update channel (url_redirector entries are +untouched). If domains are specified, then only those are removed from list. + +=back + +=cut + + push (@cmds, { + setting => 'clear_url_shortener', + code => sub { + my ($self, $key, $value, $line) = @_; + if ($value eq '') { + _clear_all_entries_by_class($self, 'shortener'); + } else { + foreach my $domain (split(/\s+/, $value)) { + _clear_redirector_entry($self, lc($domain), 'shortener'); + } + } + } + }); + =head1 PRIVILEGED SETTINGS =over 4 @@ -554,6 +722,103 @@ See C for database cleaning. type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC }); +=over 4 + +=item url_shortener_cache_type (default: none) + +Same as C, for the (independent) shortener cache. + +Example: +url_shortener_cache_type dbi + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener_cache_type', + default => '', + is_priv => 1, + type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING + }); + +=over 4 + +=item url_shortener_cache_dsn (default: none) + +Same as C, for the (independent) shortener cache. + +Examples: + + url_shortener_cache_dsn dbi:SQLite:dbname=/var/lib/spamassassin/DecodeShortURLs.db + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener_cache_dsn', + default => '', + is_priv => 1, + type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING + }); + +=over 4 + +=item url_shortener_cache_username (default: none) + +The username that should be used to connect to the shortener cache database. +Not used for SQLite. + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener_cache_username', + default => '', + is_priv => 1, + type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING + }); + +=over 4 + +=item url_shortener_cache_password (default: none) + +The password that should be used to connect to the shortener cache database. +Not used for SQLite. + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener_cache_password', + default => '', + is_priv => 1, + type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING + }); + +=over 4 + +=item url_shortener_cache_ttl (default: 86400) + +The length of time a shortener cache entry will be valid for in seconds. +Default is 86400 (1 day). + +See C for database cleaning. + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener_cache_ttl', + is_admin => 1, + default => 86400, + type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC + }); + =head1 ADMINISTRATOR SETTINGS =over 4 @@ -668,13 +933,121 @@ like a common browser. type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING }); +=over 4 + +=item url_shortener_cache_autoclean (default: 1000) + +Same as C, for the (independent) shortener cache. + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener_cache_autoclean', + is_admin => 1, + default => 1000, + type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC + }); + +=over 4 + +=item url_shortener_loginfo (default: 0 (off)) + +If this option is enabled (set to 1), then short URLs and the decoded URLs will be logged with info priority. + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener_loginfo', + is_admin => 1, + default => 0, + type => $Mail::SpamAssassin::Conf::CONF_TYPE_BOOL + }); + +=over 4 + +=item url_shortener_timeout (default: 5) + +Maximum time a short URL HTTP request can take, in seconds. + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener_timeout', + is_admin => 1, + default => 5, + type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC + }); + +=over 4 + +=item max_short_urls (default: 10) + +Maximum amount of short URLs that will be looked up per message. Chained +redirections are not counted, only initial short URLs found. + +Setting it to 0 disables HTTP requests, allowing only short_url() test to +work and report any found shortener URLs. + +=back + +=cut + + push (@cmds, { + setting => 'max_short_urls', + is_admin => 1, + default => 10, + type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC + }); + +=over 4 + +=item max_short_url_redirections (default: 10) + +Maximum depth of chained redirections that a short URL can generate. + +=back + +=cut + + push (@cmds, { + setting => 'max_short_url_redirections', + is_admin => 1, + default => 10, + type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC + }); + +=over 4 + +=item url_shortener_user_agent (default: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36) + +Set default User-Agent header for HTTP requests. Some services require it to look +like a common browser. User-Agent can be overriden on a per url_shortener basis using +the C setting. + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener_user_agent', + is_admin => 1, + default => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36', + type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING + }); + $conf->{parser}->register_commands(\@cmds); } sub initialise_url_redirector_cache { my ($self, $conf) = @_; - return if $self->{dbh} && $self->{dbh_pid} && $self->{dbh_pid} == $$; + return if $self->{dbh_redir} && $self->{dbh_redir_pid} && $self->{dbh_redir_pid} == $$; return if !$conf->{url_redirector_cache_type}; if (!$conf->{url_redirector_cache_dsn}) { @@ -684,7 +1057,7 @@ sub initialise_url_redirector_cache { ## ## SQLite - ## + ## if ($conf->{url_redirector_cache_type} =~ /^(?:dbi|sqlite)$/i && $conf->{url_redirector_cache_dsn} =~ /^dbi:SQLite/) { @@ -693,11 +1066,11 @@ sub initialise_url_redirector_cache { require DBI; require DBD::SQLite; DBD::SQLite->VERSION(1.59_01); # Required for ON CONFLICT - $self->{dbh} = DBI->connect_cached( + $self->{dbh_redir} = DBI->connect_cached( $conf->{url_redirector_cache_dsn}, '', '', {RaiseError => 1, PrintError => 0, InactiveDestroy => 1, AutoCommit => 1} ); - $self->{dbh}->do(" + $self->{dbh_redir}->do(" CREATE TABLE IF NOT EXISTS redir_url_cache ( redir_url TEXT PRIMARY KEY NOT NULL, target_url TEXT NOT NULL, @@ -706,12 +1079,7 @@ sub initialise_url_redirector_cache { modified INTEGER NOT NULL ) "); - # Maintaining index for cleaning is likely more expensive than occasional full table scan - #$self->{dbh}->do(" - # CREATE INDEX IF NOT EXISTS redir_url_modified - # ON redir_url_cache(created) - #"); - $self->{sth_insert} = $self->{dbh}->prepare(" + $self->{sth_insert_redir} = $self->{dbh_redir}->prepare(" INSERT INTO redir_url_cache (redir_url, target_url, created, modified) VALUES (?,?,strftime('%s','now'),strftime('%s','now')) ON CONFLICT(redir_url) DO UPDATE @@ -719,15 +1087,15 @@ sub initialise_url_redirector_cache { modified = excluded.modified, hits = hits + 1 "); - $self->{sth_select} = $self->{dbh}->prepare(" + $self->{sth_select_redir} = $self->{dbh_redir}->prepare(" SELECT target_url FROM redir_url_cache WHERE redir_url = ? "); - $self->{sth_delete} = $self->{dbh}->prepare(" + $self->{sth_delete_redir} = $self->{dbh_redir}->prepare(" DELETE FROM redir_url_cache WHERE redir_url = ? AND created < strftime('%s','now') - $conf->{url_redirector_cache_ttl} "); - $self->{sth_clean} = $self->{dbh}->prepare(" + $self->{sth_clean_redir} = $self->{dbh_redir}->prepare(" DELETE FROM redir_url_cache WHERE created < strftime('%s','now') - $conf->{url_redirector_cache_ttl} "); @@ -735,20 +1103,20 @@ sub initialise_url_redirector_cache { } ## ## MySQL/MariaDB - ## + ## elsif (lc $conf->{url_redirector_cache_type} eq 'dbi' && $conf->{url_redirector_cache_dsn} =~ /^dbi:(?:mysql|MariaDB)/i) { eval { local $SIG{'__DIE__'}; require DBI; - $self->{dbh} = DBI->connect_cached( + $self->{dbh_redir} = DBI->connect_cached( $conf->{url_redirector_cache_dsn}, $conf->{url_redirector_cache_username}, $conf->{url_redirector_cache_password}, {RaiseError => 1, PrintError => 0, InactiveDestroy => 1, AutoCommit => 1} ); - $self->{sth_insert} = $self->{dbh}->prepare(" + $self->{sth_insert_redir} = $self->{dbh_redir}->prepare(" INSERT INTO redir_url_cache (redir_url, target_url, created, modified) VALUES (?,?,UNIX_TIMESTAMP(),UNIX_TIMESTAMP()) ON DUPLICATE KEY UPDATE @@ -756,15 +1124,15 @@ sub initialise_url_redirector_cache { modified = VALUES(modified), hits = hits + 1 "); - $self->{sth_select} = $self->{dbh}->prepare(" + $self->{sth_select_redir} = $self->{dbh_redir}->prepare(" SELECT target_url FROM redir_url_cache WHERE redir_url = ? "); - $self->{sth_delete} = $self->{dbh}->prepare(" + $self->{sth_delete_redir} = $self->{dbh_redir}->prepare(" DELETE FROM redir_url_cache WHERE redir_url = ? AND created < UNIX_TIMESTAMP() - $conf->{url_redirector_cache_ttl} "); - $self->{sth_clean} = $self->{dbh}->prepare(" + $self->{sth_clean_redir} = $self->{dbh_redir}->prepare(" DELETE FROM redir_url_cache WHERE created < UNIX_TIMESTAMP() - $conf->{url_redirector_cache_ttl} "); @@ -772,20 +1140,20 @@ sub initialise_url_redirector_cache { } ## ## PostgreSQL - ## + ## elsif (lc $conf->{url_redirector_cache_type} eq 'dbi' && $conf->{url_redirector_cache_dsn} =~ /^dbi:Pg/i) { eval { local $SIG{'__DIE__'}; require DBI; - $self->{dbh} = DBI->connect_cached( + $self->{dbh_redir} = DBI->connect_cached( $conf->{url_redirector_cache_dsn}, $conf->{url_redirector_cache_username}, $conf->{url_redirector_cache_password}, {RaiseError => 1, PrintError => 0, InactiveDestroy => 1, AutoCommit => 1} ); - $self->{sth_insert} = $self->{dbh}->prepare(" + $self->{sth_insert_redir} = $self->{dbh_redir}->prepare(" INSERT INTO redir_url_cache (redir_url, target_url, created, modified) VALUES (?,?,CAST(EXTRACT(epoch FROM NOW()) AS INT),CAST(EXTRACT(epoch FROM NOW()) AS INT)) ON CONFLICT (redir_url) DO UPDATE SET @@ -793,15 +1161,15 @@ sub initialise_url_redirector_cache { modified = EXCLUDED.modified, hits = redir_url_cache.hits + 1 "); - $self->{sth_select} = $self->{dbh}->prepare(" + $self->{sth_select_redir} = $self->{dbh_redir}->prepare(" SELECT target_url FROM redir_url_cache WHERE redir_url = ? "); - $self->{sth_delete} = $self->{dbh}->prepare(" + $self->{sth_delete_redir} = $self->{dbh_redir}->prepare(" DELETE FROM redir_url_cache WHERE redir_url = ? AND created < CAST(EXTRACT(epoch FROM NOW()) AS INT) - $conf->{url_redirector_cache_ttl} "); - $self->{sth_clean} = $self->{dbh}->prepare(" + $self->{sth_clean_redir} = $self->{dbh_redir}->prepare(" DELETE FROM redir_url_cache WHERE created < CAST(EXTRACT(epoch FROM NOW()) AS INT) - $conf->{url_redirector_cache_ttl} "); @@ -814,23 +1182,173 @@ sub initialise_url_redirector_cache { return; } - if ($@ || !$self->{sth_clean}) { + if ($@ || !$self->{sth_clean_redir}) { warn "Redirectors: cache connect failed: $@\n"; - undef $self->{dbh}; - undef $self->{dbh_pid}; - undef $self->{sth_insert}; - undef $self->{sth_select}; - undef $self->{sth_delete}; - undef $self->{sth_clean}; + undef $self->{dbh_redir}; + undef $self->{dbh_redir_pid}; + undef $self->{sth_insert_redir}; + undef $self->{sth_select_redir}; + undef $self->{sth_delete_redir}; + undef $self->{sth_clean_redir}; } else { - $self->{dbh_pid} = $$; + $self->{dbh_redir_pid} = $$; + } +} + +sub initialise_url_shortener_cache { + my ($self, $conf) = @_; + + return if $self->{dbh_short} && $self->{dbh_short_pid} && $self->{dbh_short_pid} == $$; + return if !$conf->{url_shortener_cache_type}; + + if (!$conf->{url_shortener_cache_dsn}) { + warn "Redirectors: invalid shortener cache configuration\n"; + return; + } + + ## + ## SQLite + ## + if ($conf->{url_shortener_cache_type} =~ /^(?:dbi|sqlite)$/i + && $conf->{url_shortener_cache_dsn} =~ /^dbi:SQLite/) + { + eval { + local $SIG{'__DIE__'}; + require DBI; + require DBD::SQLite; + DBD::SQLite->VERSION(1.59_01); # Required for ON CONFLICT + $self->{dbh_short} = DBI->connect_cached( + $conf->{url_shortener_cache_dsn}, '', '', + {RaiseError => 1, PrintError => 0, InactiveDestroy => 1, AutoCommit => 1} + ); + $self->{dbh_short}->do(" + CREATE TABLE IF NOT EXISTS short_url_cache ( + short_url TEXT PRIMARY KEY NOT NULL, + decoded_url TEXT NOT NULL, + hits INTEGER NOT NULL DEFAULT 1, + created INTEGER NOT NULL, + modified INTEGER NOT NULL + ) + "); + $self->{sth_insert_short} = $self->{dbh_short}->prepare(" + INSERT INTO short_url_cache (short_url, decoded_url, created, modified) + VALUES (?,?,strftime('%s','now'),strftime('%s','now')) + ON CONFLICT(short_url) DO UPDATE + SET decoded_url = excluded.decoded_url, + modified = excluded.modified, + hits = hits + 1 + "); + $self->{sth_select_short} = $self->{dbh_short}->prepare(" + SELECT decoded_url FROM short_url_cache + WHERE short_url = ? + "); + $self->{sth_delete_short} = $self->{dbh_short}->prepare(" + DELETE FROM short_url_cache + WHERE short_url = ? AND created < strftime('%s','now') - $conf->{url_shortener_cache_ttl} + "); + $self->{sth_clean_short} = $self->{dbh_short}->prepare(" + DELETE FROM short_url_cache + WHERE created < strftime('%s','now') - $conf->{url_shortener_cache_ttl} + "); + }; + } + ## + ## MySQL/MariaDB + ## + elsif (lc $conf->{url_shortener_cache_type} eq 'dbi' + && $conf->{url_shortener_cache_dsn} =~ /^dbi:(?:mysql|MariaDB)/i) + { + eval { + local $SIG{'__DIE__'}; + require DBI; + $self->{dbh_short} = DBI->connect_cached( + $conf->{url_shortener_cache_dsn}, + $conf->{url_shortener_cache_username}, + $conf->{url_shortener_cache_password}, + {RaiseError => 1, PrintError => 0, InactiveDestroy => 1, AutoCommit => 1} + ); + $self->{sth_insert_short} = $self->{dbh_short}->prepare(" + INSERT INTO short_url_cache (short_url, decoded_url, created, modified) + VALUES (?,?,UNIX_TIMESTAMP(),UNIX_TIMESTAMP()) + ON DUPLICATE KEY UPDATE + decoded_url = VALUES(decoded_url), + modified = VALUES(modified), + hits = hits + 1 + "); + $self->{sth_select_short} = $self->{dbh_short}->prepare(" + SELECT decoded_url FROM short_url_cache + WHERE short_url = ? + "); + $self->{sth_delete_short} = $self->{dbh_short}->prepare(" + DELETE FROM short_url_cache + WHERE short_url = ? AND created < UNIX_TIMESTAMP() - $conf->{url_shortener_cache_ttl} + "); + $self->{sth_clean_short} = $self->{dbh_short}->prepare(" + DELETE FROM short_url_cache + WHERE created < UNIX_TIMESTAMP() - $conf->{url_shortener_cache_ttl} + "); + }; + } + ## + ## PostgreSQL + ## + elsif (lc $conf->{url_shortener_cache_type} eq 'dbi' + && $conf->{url_shortener_cache_dsn} =~ /^dbi:Pg/i) + { + eval { + local $SIG{'__DIE__'}; + require DBI; + $self->{dbh_short} = DBI->connect_cached( + $conf->{url_shortener_cache_dsn}, + $conf->{url_shortener_cache_username}, + $conf->{url_shortener_cache_password}, + {RaiseError => 1, PrintError => 0, InactiveDestroy => 1, AutoCommit => 1} + ); + $self->{sth_insert_short} = $self->{dbh_short}->prepare(" + INSERT INTO short_url_cache (short_url, decoded_url, created, modified) + VALUES (?,?,CAST(EXTRACT(epoch FROM NOW()) AS INT),CAST(EXTRACT(epoch FROM NOW()) AS INT)) + ON CONFLICT (short_url) DO UPDATE SET + decoded_url = EXCLUDED.decoded_url, + modified = EXCLUDED.modified, + hits = short_url_cache.hits + 1 + "); + $self->{sth_select_short} = $self->{dbh_short}->prepare(" + SELECT decoded_url FROM short_url_cache + WHERE short_url = ? + "); + $self->{sth_delete_short} = $self->{dbh_short}->prepare(" + DELETE FROM short_url_cache + WHERE short_url = ? AND created < CAST(EXTRACT(epoch FROM NOW()) AS INT) - $conf->{url_shortener_cache_ttl} + "); + $self->{sth_clean_short} = $self->{dbh_short}->prepare(" + DELETE FROM short_url_cache + WHERE created < CAST(EXTRACT(epoch FROM NOW()) AS INT) - $conf->{url_shortener_cache_ttl} + "); + }; + ## + ## ... + ## + } else { + warn "Redirectors: invalid shortener cache configuration\n"; + return; + } + + if ($@ || !$self->{sth_clean_short}) { + warn "Redirectors: shortener cache connect failed: $@\n"; + undef $self->{dbh_short}; + undef $self->{dbh_short_pid}; + undef $self->{sth_insert_short}; + undef $self->{sth_select_short}; + undef $self->{sth_delete_short}; + undef $self->{sth_clean_short}; + } else { + $self->{dbh_short_pid} = $$; } } sub redir_url { my ($self, $pms) = @_; - # Make sure checks are run $self->_check_redir($pms); return $pms->{redir_url} ? 1 : 0; @@ -839,7 +1357,6 @@ sub redir_url { sub redir_url_valid { my ($self, $pms) = @_; - # Make sure checks are run $self->_check_redir($pms); return $pms->{redir_url_valid} ? 1 : 0; @@ -848,7 +1365,6 @@ sub redir_url_valid { sub redir_url_404 { my ($self, $pms) = @_; - # Make sure checks are run $self->_check_redir($pms); return $pms->{redir_url_404} ? 1 : 0; @@ -857,7 +1373,6 @@ sub redir_url_404 { sub redir_url_code { my ($self, $pms, undef, $code) = @_; - # Make sure checks are run $self->_check_redir($pms); return 0 unless defined $code && $code =~ /^\d{3}$/; @@ -867,7 +1382,6 @@ sub redir_url_code { sub redir_url_chained { my ($self, $pms) = @_; - # Make sure checks are run $self->_check_redir($pms); return $pms->{redir_url_chained} ? 1 : 0; @@ -876,7 +1390,6 @@ sub redir_url_chained { sub redir_url_chained_domain { my ($self, $pms) = @_; - # Make sure checks are run $self->_check_redir($pms); return $pms->{redir_url_chained_domain} ? 1 : 0; @@ -885,7 +1398,6 @@ sub redir_url_chained_domain { sub redir_url_maxchain { my ($self, $pms) = @_; - # Make sure checks are run $self->_check_redir($pms); return $pms->{redir_url_maxchain} ? 1 : 0; @@ -894,14 +1406,98 @@ sub redir_url_maxchain { sub redir_url_loop { my ($self, $pms) = @_; - # Make sure checks are run $self->_check_redir($pms); return $pms->{redir_url_loop} ? 1 : 0; } +sub short_url { + my ($self, $pms) = @_; + + $self->_check_redir($pms); + + return $pms->{short_url} ? 1 : 0; +} + +sub short_url_redir { + my ($self, $pms) = @_; + + $self->_check_redir($pms); + + return $pms->{short_url_redir} ? 1 : 0; +} + +sub short_url_200 { + my ($self, $pms) = @_; + + $self->_check_redir($pms); + + return $pms->{short_url_200} ? 1 : 0; +} + +sub short_url_404 { + my ($self, $pms) = @_; + + $self->_check_redir($pms); + + return $pms->{short_url_404} ? 1 : 0; +} + +sub short_url_code { + my ($self, $pms, undef, $code) = @_; + + $self->_check_redir($pms); + + return 0 unless defined $code && $code =~ /^\d{3}$/; + return $pms->{"short_url_$code"} ? 1 : 0; +} + +sub short_url_chained { + my ($self, $pms) = @_; + + $self->_check_redir($pms); + + return $pms->{short_url_chained} ? 1 : 0; +} + +sub short_url_maxchain { + my ($self, $pms) = @_; + + $self->_check_redir($pms); + + return $pms->{short_url_maxchain} ? 1 : 0; +} + +sub short_url_loop { + my ($self, $pms) = @_; + + $self->_check_redir($pms); + + return $pms->{short_url_loop} ? 1 : 0; +} + +sub short_url_tests { + # Legacy DecodeShortURLs compatibility warning done in finish_parsing_start + return 0; +} + +sub finish_parsing_start { + my ($self, $opts) = @_; + + if ($opts->{conf}->{eval_to_rule}->{short_url_tests}) { + warn "Redirectors: Legacy configuration format detected. ". + "Eval function short_url_tests() is no longer supported, ". + "please see documentation for the new rule format.\n"; + } +} + +# Add a host[/path] entry to the shared exact/suffix lookup buckets. +# $kind is 'redirector' (default), 'selenium', or 'shortener'; it records +# which legacy rule family (redir_url_* vs short_url_*) a hop matching this +# entry belongs to. sub _add_redirector_entry { - my ($conf, $token, $method) = @_; + my ($conf, $token, $method, $kind) = @_; + $kind = 'redirector' unless defined $kind; my ($domspec, $path) = split(/\//, $token, 2); $path = defined $path ? '/' . $path : '/'; @@ -915,13 +1511,17 @@ sub _add_redirector_entry { my $display = ($is_suffix ? '.' : '') . $domspec; warn "redirectors: $display already registered with method '$existing->{method}'; overriding with '$method'\n"; } - my $entry = $conf->{$bucket}->{$domspec} ||= { method => $method, paths => [] }; + my $entry = $conf->{$bucket}->{$domspec} ||= { method => $method, paths => [], kind => $kind }; $entry->{method} = $method; + $entry->{kind} = $kind; push @{$entry->{paths}}, $path unless grep { $_ eq $path } @{$entry->{paths}}; } +# $class is 'redirector' (matches kind ne 'shortener', i.e. redirector or +# selenium) or 'shortener' (matches kind eq 'shortener'). Undef means no +# kind filtering (back-compat for direct callers/tests predating the merge). sub _clear_redirector_entry { - my ($conf, $token) = @_; + my ($conf, $token, $class) = @_; $token = lc $token; my $has_path = ($token =~ /\//) ? 1 : 0; @@ -931,15 +1531,36 @@ sub _clear_redirector_entry { my $bucket = ($domspec =~ s/^\.//) ? 'url_redirector_suffix' : 'url_redirector_exact'; return unless length $domspec; + my $entry = $conf->{$bucket}->{$domspec} or return; + if (defined $class) { + my $matches = $class eq 'shortener' ? ($entry->{kind} eq 'shortener') + : ($entry->{kind} ne 'shortener'); + return unless $matches; + } + if (!$has_path) { delete $conf->{$bucket}->{$domspec}; return; } - my $entry = $conf->{$bucket}->{$domspec} or return; @{$entry->{paths}} = grep { $_ ne $path } @{$entry->{paths}}; delete $conf->{$bucket}->{$domspec} unless @{$entry->{paths}}; } +# Remove every entry of the given class ('redirector' or 'shortener') from +# both lookup buckets. Used by clear_url_redirector/clear_url_shortener with +# no arguments. +sub _clear_all_entries_by_class { + my ($conf, $class) = @_; + for my $bucket (qw(url_redirector_exact url_redirector_suffix)) { + next unless $conf->{$bucket}; + foreach my $dom (keys %{$conf->{$bucket}}) { + my $kind = $conf->{$bucket}->{$dom}->{kind}; + my $matches = $class eq 'shortener' ? ($kind eq 'shortener') : ($kind ne 'shortener'); + delete $conf->{$bucket}->{$dom} if $matches; + } + } +} + sub _entry_match_path { my ($entry, $path) = @_; for my $p (@{$entry->{paths}}) { @@ -1016,8 +1637,9 @@ sub _parse_uri { return ($uri, $host, $path, $rest); } -# Returns the redirector entry ({method, paths}) if $uri's host+path matches -# a configured url_redirector / url_redirector_get, else undef. +# Returns the redirector entry ({method, paths, kind}) if $uri's host+path +# matches a configured url_redirector / url_redirector_get / url_shortener*, +# else undef. sub _is_configured_redirector { my ($uri, $conf) = @_; @@ -1063,9 +1685,12 @@ sub _extract_embedded_uri { return; } +# Lazily build (and cache on $pms) the LWP::UserAgent for $kind +# ('redirector' or 'shortener'), each with its own default UA/timeout. sub _get_lwp_ua { - my ($self, $pms) = @_; - return $pms->{redir_lwp_ua} if exists $pms->{redir_lwp_ua}; + my ($self, $pms, $kind) = @_; + my $slot = $kind eq 'shortener' ? 'short_lwp_ua' : 'redir_lwp_ua'; + return $pms->{$slot} if exists $pms->{$slot}; my $conf = $pms->{conf}; # prevent "500 Header line too long (limit is 8192)" error when accessing @@ -1074,13 +1699,16 @@ sub _get_lwp_ua { use LWP::Protocol::http; push(@LWP::Protocol::http::EXTRA_SOCK_OPTS, MaxLineLength => 16*1024); }; + my ($agent, $timeout) = $kind eq 'shortener' + ? ($conf->{url_shortener_user_agent}, $conf->{url_shortener_timeout}) + : ($conf->{url_redirector_user_agent}, $conf->{url_redirector_timeout}); my $ua = LWP::UserAgent->new( - 'agent' => $conf->{url_redirector_user_agent}, + 'agent' => $agent, 'max_redirect' => 0, - 'timeout' => $conf->{url_redirector_timeout}, + 'timeout' => $timeout, ); $ua->env_proxy; - return $pms->{redir_lwp_ua} = $ua; + return $pms->{$slot} = $ua; } sub _get_selenium_ua { @@ -1131,26 +1759,30 @@ sub _get_selenium_ua { return $pms->{redir_selenium_ua} = $ua; } -# Perform an HTTP request for $uri using $method (LWP) or Selenium. +# Perform an HTTP request for $uri using $method (LWP) or Selenium, for a +# hop of the given $kind ('redirector'/'selenium' or 'shortener') -- this +# selects the cache backend, UA/timeout settings, and status-code flag +# prefix (redir_url_* vs short_url_*). # Returns the absolute, normalized Location URL on a usable redirect, -# or undef otherwise. Sets redir_url_ flags on $pms and writes +# or undef otherwise. Sets _ flags on $pms and writes # the cache. sub _do_http { - my ($self, $uri, $method, $pms) = @_; + my ($self, $uri, $method, $pms, $kind) = @_; my $conf = $pms->{conf}; + my $flag_prefix = $kind eq 'shortener' ? 'short_url' : 'redir_url'; my $redir_url = $uri; my $location; - if (defined($location = $self->cache_get($redir_url))) { - if ($conf->{url_redirector_loginfo}) { + if (defined($location = $self->cache_get($kind, $redir_url))) { + if ($conf->{url_redirector_loginfo} || $conf->{url_shortener_loginfo}) { info("found cached $redir_url => $location"); } else { dbg("found cached $redir_url => $location"); } if ($location =~ /^\d{3}$/) { - $pms->{"redir_url_$location"} = 1; - $self->cache_add($redir_url, $location); + $pms->{"${flag_prefix}_$location"} = 1; + $self->cache_add($kind, $redir_url, $location); return; } } else { @@ -1200,13 +1832,18 @@ sub _do_http { } } $location = $newurl; - $pms->{"redir_url_$rcode"} = 1; - $self->cache_add($redir_url, $rcode); + $pms->{"${flag_prefix}_$rcode"} = 1; + $self->cache_add($kind, $redir_url, $rcode); if($rcode !~ /^30[12]/) { return; } } else { - my $ua = $self->_get_lwp_ua($pms); + my $ua = $self->_get_lwp_ua($pms, $kind); + if ($kind eq 'shortener') { + my (undef, $host) = _parse_uri($redir_url, $conf); + my $custom_ua = defined $host ? $conf->{url_shortener_custom_ua}->{$host} : undef; + $ua->agent(defined $custom_ua ? $custom_ua : $conf->{url_shortener_user_agent}); + } my $response = $ua->$method($redir_url); return if not defined $response; @@ -1236,8 +1873,8 @@ sub _do_http { dbg("Found a meta http-equiv redirector, changing http response code from " . $response->code . " to $rcode"); } } else { - $pms->{"redir_url_$rcode"} = 1; - $self->cache_add($redir_url, $rcode); + $pms->{"${flag_prefix}_$rcode"} = 1; + $self->cache_add($kind, $redir_url, $rcode); } } if($rcode !~ /^30[12]/) { @@ -1248,7 +1885,7 @@ sub _do_http { if((exists $response->headers->{location}) or $http_equiv) { $location = $response->headers->{location} if not $http_equiv; if($redir_url ne $location) { - if ($conf->{url_redirector_loginfo}) { + if ($conf->{url_redirector_loginfo} || $conf->{url_shortener_loginfo}) { info("found $redir_url => $location"); } else { dbg("found $redir_url => $location"); @@ -1260,7 +1897,7 @@ sub _do_http { return unless defined $location; - $self->cache_add($redir_url, $location); + $self->cache_add($kind, $redir_url, $location); # Resolve relative Location header to absolute. if ($location !~ m{^[a-z]+://}i) { @@ -1299,6 +1936,17 @@ sub _do_http { # Recursive chain walker. Stops cleanly when neither # _is_configured_redirector nor _extract_embedded_uri matches. HTTP # requests are gated on _is_configured_redirector returning truthy. +# +# Both redirectors and shorteners flow through this single walk, so a hop +# of either kind is checked at every depth, a redirector unwrapping into +# a shortener (or vice versa) is followed either way. Detection +# (redir_url/short_url) and HTTP status-code flags stay tied to the kind +# of the entry that actually matched at that hop; chain-level flags +# (chained/chained_domain/loop/valid-redirect) fire for both legacy rule +# families together since they describe the shape of the chain as a +# whole, not which kind produced it. Maxchain is checked against both +# configured depth limits independently against the single shared depth +# counter. sub _walk_redirects { my ($self, $uri, $src_info, $pms, $depth, $been_here) = @_; my $conf = $pms->{conf}; @@ -1306,11 +1954,15 @@ sub _walk_redirects { if (exists $been_here->{"uri:$uri"}) { dbg("error: loop detected: $uri"); $pms->{redir_url_loop} = 1; + $pms->{short_url_loop} = 1; return; } - if ($depth >= $conf->{max_redir_url_redirections}) { - dbg("found more than $conf->{max_redir_url_redirections} redirections"); - $pms->{redir_url_maxchain} = 1; + my $over_redir = $depth >= $conf->{max_redir_url_redirections}; + my $over_short = $depth >= $conf->{max_short_url_redirections}; + if ($over_redir || $over_short) { + dbg("found more than allowed chained redirections at depth $depth"); + $pms->{redir_url_maxchain} = 1 if $over_redir; + $pms->{short_url_maxchain} = 1 if $over_short; return; } $been_here->{"uri:$uri"} = 1; @@ -1322,15 +1974,20 @@ sub _walk_redirects { # detection rule fires (matches the max_redir_urls=0 semantics: "found a # redirector but didn't probe it"). Fall through to embedded-URI extraction # in case the URL also carries a querystring redirect. - if ($rentry && $rentry->{method} eq 'selenium' && !$conf->{url_redirector_use_selenium}) { + if ($rentry && $rentry->{kind} eq 'selenium' && !$conf->{url_redirector_use_selenium}) { dbg("$uri matches url_redirector_selenium but url_redirector_use_selenium=0, skipping http lookup"); $pms->{redir_url} = 1; $rentry = undef; } if ($rentry) { - $pms->{redir_url} = 1; - $pms->{redir_url_chained} = 1 if $depth > 0; + my $kind = $rentry->{kind}; + my $flag_prefix = $kind eq 'shortener' ? 'short_url' : 'redir_url'; + $pms->{$flag_prefix} = 1; + if ($depth > 0) { + $pms->{redir_url_chained} = 1; + $pms->{short_url_chained} = 1; + } my (undef, $host) = $self->{main}->{registryboundaries}->uri_to_domain($uri); if (defined $host) { @@ -1342,19 +1999,34 @@ sub _walk_redirects { } return if $self->{net_disabled}; - return if !$conf->{max_redir_urls}; - # Seed cap: max_redir_urls counts initial (depth 0) redirector URIs. - if ($depth == 0) { - return if ++$pms->{redir_seed_count} > $conf->{max_redir_urls}; + if ($kind eq 'shortener') { + return if !$conf->{max_short_urls}; + if ($depth == 0) { + return if ++$pms->{short_seed_count} > $conf->{max_short_urls}; + } + } else { + return if !$conf->{max_redir_urls}; + if ($depth == 0) { + return if ++$pms->{redir_seed_count} > $conf->{max_redir_urls}; + } } - my $location = $self->_do_http($uri, $rentry->{method}, $pms); + # Shorteners historically strip anchors/query strings before fetching + # (their whole identity is the path token, and services often don't + # care about trailing #fragment/?query on the short link itself). + # Redirectors keep the query string intact, url_redirector_params + # needs it to extract embedded destination URIs. + my $fetch_uri = $uri; + $fetch_uri =~ s{/?[?#].*}{} if $kind eq 'shortener'; + + my $location = $self->_do_http($fetch_uri, $rentry->{method}, $pms, $kind); return unless defined $location; - if ($uri eq $location) { + if ($fetch_uri eq $location) { dbg("URL redirects to itself"); $pms->{redir_url_loop} = 1; + $pms->{short_url_loop} = 1; return; } @@ -1366,6 +2038,7 @@ sub _walk_redirects { _add_redirect_uri($pms, $location, $src_info); $pms->{redir_url_valid} = 1; + $pms->{short_url_redir} = 1; return $self->_walk_redirects($location, $src_info, $pms, $depth + 1, $been_here); } @@ -1391,6 +2064,7 @@ sub _check_redir { my $conf = $pms->{conf}; $self->initialise_url_redirector_cache($conf); + $self->initialise_url_shortener_cache($conf); # UAs are built lazily inside _do_http and cached on $pms. No upfront # construction here -- a message with only embedded-URI matches and no @@ -1403,12 +2077,20 @@ sub _check_redir { $self->_walk_redirects($uri, $info, $pms, 0, {}); } - if ($self->{dbh} && $conf->{url_redirector_cache_autoclean} + if ($self->{dbh_redir} && $conf->{url_redirector_cache_autoclean} && rand() < 1/$conf->{url_redirector_cache_autoclean}) { - dbg("cleaning stale cache entries"); - eval { $self->{sth_clean}->execute(); }; - if ($@) { dbg("cache cleaning failed: $@"); } + dbg("cleaning stale redirector cache entries"); + eval { $self->{sth_clean_redir}->execute(); }; + if ($@) { dbg("redirector cache cleaning failed: $@"); } + } + + if ($self->{dbh_short} && $conf->{url_shortener_cache_autoclean} + && rand() < 1/$conf->{url_shortener_cache_autoclean}) + { + dbg("cleaning stale shortener cache entries"); + eval { $self->{sth_clean_short}->execute(); }; + if ($@) { dbg("shortener cache cleaning failed: $@"); } } } @@ -1433,13 +2115,15 @@ sub _add_redirect_uri { } sub cache_add { - my ($self, $redir_url, $target_url) = @_; + my ($self, $kind, $key, $value) = @_; - return if !$self->{dbh}; - return if length($redir_url) > 256 || length($target_url) > 512; + my $dbh = $kind eq 'shortener' ? $self->{dbh_short} : $self->{dbh_redir}; + return if !$dbh; + return if length($key) > 256 || length($value) > 512; + my $sth = $kind eq 'shortener' ? $self->{sth_insert_short} : $self->{sth_insert_redir}; # Upsert - eval { $self->{sth_insert}->execute($redir_url, $target_url); }; + eval { $sth->execute($key, $value); }; if ($@) { dbg("could not add to cache: $@"); } @@ -1448,13 +2132,15 @@ sub cache_add { } sub cache_get { - my ($self, $key) = @_; + my ($self, $kind, $key) = @_; - return if !$self->{dbh}; + my $dbh = $kind eq 'shortener' ? $self->{dbh_short} : $self->{dbh_redir}; + return if !$dbh; + my $sth_delete = $kind eq 'shortener' ? $self->{sth_delete_short} : $self->{sth_delete_redir}; # Make sure expired entries are gone. Just a quick check for primary key, # not that expensive. - eval { $self->{sth_delete}->execute($key); }; + eval { $sth_delete->execute($key); }; if ($@) { dbg("cache delete failed: $@"); return; @@ -1462,13 +2148,14 @@ sub cache_get { # Now try to get it (don't bother parsing if something was deleted above, # it would be rare event anyway) - eval { $self->{sth_select}->execute($key); }; + my $sth_select = $kind eq 'shortener' ? $self->{sth_select_short} : $self->{sth_select_redir}; + eval { $sth_select->execute($key); }; if ($@) { dbg("cache get failed: $@"); return; } - my @row = $self->{sth_select}->fetchrow_array(); + my @row = $sth_select->fetchrow_array(); if (@row) { return $row[0]; } @@ -1488,5 +2175,18 @@ sub has_selenium_support { 1 } sub has_url_redirector_selenium { 1 } sub has_url_skip_redirect_to { 1 } sub has_url_redirector_path { 1 } # path-prefix syntax in url_redirector / url_redirector_get +sub has_short_url { 1 } +sub has_autoclean { 1 } +sub has_short_url_code { 1 } +sub has_user_agent { 1 } # url_shortener_user_agent +sub has_custom_user_agent { 1 } # url_shortener_custom_user_agent +sub has_get { 1 } # url_shortener_get +sub has_clear { 1 } # clear_url_shortener +sub has_timeout { 1 } # url_shortener_timeout +sub has_max_redirections { 1 } # max_short_url_redirections +# short_url() will always hit if matching url_shortener was found, even +# without HTTP requests. To check if a valid HTTP redirection response was +# seen, use short_url_redir(). +sub has_short_url_redir { 1 } 1; diff --git a/t/decodeshorturl.t b/t/decodeshorturl.t index 1e92385dc9..0612500d38 100755 --- a/t/decodeshorturl.t +++ b/t/decodeshorturl.t @@ -77,10 +77,12 @@ ok_all_patterns(); 'https://tinyurl.com/jf8wv76t => https://spamassassin.apache.org/' ); -sarun ("-D DecodeShortURLs -t < data/spam/decodeshorturl/anchor.eml 2>&1", \&patterns_run_cb); +# DecodeShortURLs is now a thin subclass of Redirectors; all logging (and +# thus the -D facility name) happens under "Redirectors". +sarun ("-D Redirectors -t < data/spam/decodeshorturl/anchor.eml 2>&1", \&patterns_run_cb); ok_all_patterns(); -sarun ("-D DecodeShortURLs -t < data/spam/decodeshorturl/params.eml 2>&1", \&patterns_run_cb); +sarun ("-D Redirectors -t < data/spam/decodeshorturl/params.eml 2>&1", \&patterns_run_cb); ok_all_patterns(); ### From 128e3743dee6ae78a450caf931c21d75dc262a98 Mon Sep 17 00:00:00 2001 From: Giovanni Date: Thu, 13 Aug 2026 11:02:01 +0200 Subject: [PATCH 2/2] alias all short_url_* subs to equivalent redirector subs Co-Authored-By: Claude Sonnet 5 --- UPGRADE | 7 + .../SpamAssassin/Plugin/DecodeShortURLs.pm | 2 +- lib/Mail/SpamAssassin/Plugin/Redirectors.pm | 807 ++++++------------ t/decodeshorturl.t | 13 +- 4 files changed, 281 insertions(+), 548 deletions(-) diff --git a/UPGRADE b/UPGRADE index 8b05393739..b43f508863 100644 --- a/UPGRADE +++ b/UPGRADE @@ -1,6 +1,13 @@ Note for Users Upgrading to SpamAssassin 4.0.3 ---------------------------------------------- +- Mail::SpamAssassin::Plugin::DecodeShortURLs has been merged into + Mail::SpamAssassin::Plugin::Redirectors and is now a deprecated + compatibility shim; all url_shortener*/short_url* settings and rules + still work as aliases of url_redirector*/redir_url*. Redirect chains + mixing shorteners and redirectors in any order are now followed + correctly, and query strings are no longer stripped before fetching. + - Mail::SpamAssassin::Plugin::ExtractText now has the possibility to cache results in order to speedup attachment processing. diff --git a/lib/Mail/SpamAssassin/Plugin/DecodeShortURLs.pm b/lib/Mail/SpamAssassin/Plugin/DecodeShortURLs.pm index c48b0a9efb..214351895b 100644 --- a/lib/Mail/SpamAssassin/Plugin/DecodeShortURLs.pm +++ b/lib/Mail/SpamAssassin/Plugin/DecodeShortURLs.pm @@ -29,7 +29,7 @@ B All of its functionality (the C settings and the C eval rules) has been merged into L, which also gained the ability to follow a redirect chain that mixes shorteners and redirectors in either -order -- there is no functional difference between the two, both are just +order, there is no functional difference between the two, both are just a host whose HTTP response redirects elsewhere. This module remains only as a compatibility shim so that existing diff --git a/lib/Mail/SpamAssassin/Plugin/Redirectors.pm b/lib/Mail/SpamAssassin/Plugin/Redirectors.pm index e8d9a47e55..381859e668 100644 --- a/lib/Mail/SpamAssassin/Plugin/Redirectors.pm +++ b/lib/Mail/SpamAssassin/Plugin/Redirectors.pm @@ -83,36 +83,37 @@ points to the actual destination URL. It then adds this URL to the list of URIs extracted by SpamAssassin which can then be accessed by uri rules and plugins such as URIDNSBL. -This plugin will follow chained redirections, where a redirected or -shortened URL leads to another redirector or shortener, in any combination -and order -- for example a redirector that unwraps into a shortener, or a -shortener that unwraps into a redirector. Redirection depth limits can be -set with C and C. - -Maximum of C redirector URLs and C shortener -URLs are checked in a message (10 each by default). Setting either to 0 -disables HTTP requests for that category, allowing only the corresponding -C/C test to work and report found redirectors or -shorteners. - -All supported rule types for checking redirected/shortened URLs and -redirection status are documented in L section. +This plugin will follow chained redirections, where a redirected URL leads +to another redirector, in any combination and order -- for example a +redirector that unwraps into what used to be called a "shortener", or vice +versa. There is no functional difference between a "redirector" and a +"shortener": both are just a domain whose HTTP response redirects +somewhere else, and both are followed by the same code path. +C/C/C +and their C/C/etc. settings are kept +as deprecated aliases of C/C/etc. for +backwards compatibility, and are planned for removal in a future version. +Likewise, C and its sibling eval rules are aliases of +C and friends. Redirection depth is limited by +C, and C redirector URLs are +checked in a message (10 by default); setting it to 0 disables HTTP +requests, allowing only C to work and report found +redirectors. + +All supported rule types for checking redirected URLs and redirection +status are documented in L section. =head1 NOTES This plugin runs before priority 0 so that it may modify the parsed URI list prior to normal uri rules or the URIDNSBL plugin. -Redirector and shortener matching share a single recursive walk over each -URI's redirect chain, so every hop is checked against both C -and C configuration regardless of which kind started the -chain. - =head1 ACKNOWLEDGEMENTS The url_shortener functionality was originally provided by a separate -DecodeShortURLs plugin, functionality has been merged into this one; -C is now a deprecated compatibility shim that loads this plugin. +DecodeShortURLs plugin; that functionality has been merged into this one, +and C is now a deprecated +compatibility shim that loads this plugin. =cut @@ -126,7 +127,7 @@ use warnings; use vars qw(@ISA); @ISA = qw(Mail::SpamAssassin::Plugin); -my $VERSION = 4.03; +my $VERSION = 4.10; use constant HAS_LWP_USERAGENT => eval { require LWP::UserAgent; require LWP::Protocol::https; }; use constant HAS_SELENIUM => eval { require Selenium::Remote::Driver; }; @@ -153,7 +154,7 @@ sub new { $self->set_config($mailsaobject->{conf}); # run at priority -15 so that redirected/shortened uris are always - # checked in a single pass, regardless of which kind of hop starts a chain + # checked in a single pass, regardless of what type of hop starts a chain $self->register_method_priority ('check_dnsbl', -15); $self->register_eval_rule('redir_url', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); $self->register_eval_rule('redir_url_valid', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); @@ -221,6 +222,13 @@ Example: The last line follows C but not C. +C (and C, C) are +deprecated aliases of C (and C, +C) kept for backwards compatibility with configs +written for the old DecodeShortURLs plugin, there is no functional +difference between the two names, and the C spelling is +planned for removal in a future version. + =back =cut @@ -229,19 +237,30 @@ sub set_config { my($self, $conf) = @_; my @cmds = (); + # url_shortener is a pure alias of url_redirector, a "shortener" and a + # "redirector" are the same mechanism (an HTTP redirect), the terminology + # difference is not functional. Kept only for config backwards + # compatibility; planned for removal in a future version. + my $url_redirector_code = sub { + my ($self, $key, $value, $line) = @_; + if ($value eq '') { + return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; + } + foreach my $token (split(/\s+/, $value)) { + _add_redirector_entry($self, $token, 'head'); + } + }; push (@cmds, { setting => 'url_redirector', default => {}, type => $Mail::SpamAssassin::Conf::CONF_TYPE_HASH_KEY_VALUE, - code => sub { - my ($self, $key, $value, $line) = @_; - if ($value eq '') { - return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; - } - foreach my $token (split(/\s+/, $value)) { - _add_redirector_entry($self, $token, 'head'); - } - } + code => $url_redirector_code, + }); + push (@cmds, { + setting => 'url_shortener', + default => {}, + type => $Mail::SpamAssassin::Conf::CONF_TYPE_HASH_KEY_VALUE, + code => $url_redirector_code, }); =over 4 @@ -313,10 +332,9 @@ Set Selenium port to use. =item clear_url_redirector [domain[/path]] [domain[/path]...] -Clear configured url_redirector domains, for example to +Clear configured url_redirector/url_shortener domains, for example to override default settings from an update channel. If no arguments are given, -all redirector entries are cleared (url_shortener entries are untouched). If -domains are specified, only those are removed. +all entries are cleared. If domains are specified, only those are removed. When an entry includes a C, only that path is removed from the domain's allowlist; the domain entry itself is dropped only when its path @@ -327,18 +345,23 @@ added by a bare-domain configuration. =cut - push (@cmds, { - setting => 'clear_url_redirector', - code => sub { - my ($self, $key, $value, $line) = @_; - if ($value eq '') { - _clear_all_entries_by_class($self, 'redirector'); - } else { - foreach my $token (split(/\s+/, $value)) { - _clear_redirector_entry($self, $token, 'redirector'); - } + my $clear_url_redirector_code = sub { + my ($self, $key, $value, $line) = @_; + if ($value eq '') { + _clear_all_redirector_entries($self); + } else { + foreach my $token (split(/\s+/, $value)) { + _clear_redirector_entry($self, $token); } } + }; + push (@cmds, { + setting => 'clear_url_redirector', + code => $clear_url_redirector_code, + }); + push (@cmds, { + setting => 'clear_url_shortener', + code => $clear_url_redirector_code, }); =over 4 @@ -355,17 +378,22 @@ restricts the match. =cut + my $url_redirector_get_code = sub { + my ($self, $key, $value, $line) = @_; + if ($value eq '') { + return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; + } + foreach my $token (split(/\s+/, $value)) { + _add_redirector_entry($self, $token, 'get'); + } + }; push (@cmds, { setting => 'url_redirector_get', - code => sub { - my ($self, $key, $value, $line) = @_; - if ($value eq '') { - return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; - } - foreach my $token (split(/\s+/, $value)) { - _add_redirector_entry($self, $token, 'get'); - } - } + code => $url_redirector_get_code, + }); + push (@cmds, { + setting => 'url_shortener_get', + code => $url_redirector_get_code, }); =over 4 @@ -494,119 +522,37 @@ The regexp must match only the redirected domain. =over 4 -=item url_shortener domain [domain...] (default: none) - -Domains that should be considered as an URL shortener. If the domain begins -with a '.', any subdomain of the domain will be checked (see the leading-dot -rule under C). - -Example: - - url_shortener tinyurl.com - url_shortener .page.link - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener', - default => {}, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_HASH_KEY_VALUE, - code => sub { - my ($self, $key, $value, $line) = @_; - if ($value eq '') { - return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; - } - foreach my $token (split(/\s+/, $value)) { - _add_redirector_entry($self, lc($token), 'head', 'shortener'); - } - } - }); - -=over 4 - -=item url_shortener_get domain [domain...] (default: none) - -Alias to C. HTTP request will be done with GET method, -instead of default HEAD. Required for some services like bit.ly to return -blocked URL correctly. - -Example: - - url_shortener_get bit.ly - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_get', - code => sub { - my ($self, $key, $value, $line) = @_; - if ($value eq '') { - return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; - } - foreach my $token (split(/\s+/, $value)) { - _add_redirector_entry($self, lc($token), 'get', 'shortener'); - } - } - }); - -=over 4 - -=item url_shortener_custom_user_agent domain user-agent (default: none) +=item url_redirector_custom_user_agent domain user-agent (default: none) Custom HTTP user-agent to be used for specific domains, -instead of the default specified in C. +instead of the default specified in C. Required for some services like t.co to return blocked URL correctly. Example: - url_shortener_custom_user_agent t.co curl/8.6.0 + url_redirector_custom_user_agent t.co curl/8.6.0 =back =cut - push (@cmds, { - setting => 'url_shortener_custom_user_agent', - code => sub { - my ($self, $key, $value, $line) = @_; - if ($value eq '') { - return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; - } - my @values = split(/\s+/, $value); - my $domain = shift(@values); - my $ua = join('', @values); - $self->{url_shortener_custom_ua}->{lc $domain} = $ua; + my $url_redirector_custom_user_agent_code = sub { + my ($self, $key, $value, $line) = @_; + if ($value eq '') { + return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; } + my @values = split(/\s+/, $value); + my $domain = shift(@values); + my $ua = join('', @values); + $self->{url_redirector_custom_ua}->{lc $domain} = $ua; + }; + push (@cmds, { + setting => 'url_redirector_custom_user_agent', + code => $url_redirector_custom_user_agent_code, }); - -=over 4 - -=item clear_url_shortener [domain] [domain...] - -Clear configured url_shortener and url_shortener_get domains, for example to -override default settings from an update channel (url_redirector entries are -untouched). If domains are specified, then only those are removed from list. - -=back - -=cut - push (@cmds, { - setting => 'clear_url_shortener', - code => sub { - my ($self, $key, $value, $line) = @_; - if ($value eq '') { - _clear_all_entries_by_class($self, 'shortener'); - } else { - foreach my $domain (split(/\s+/, $value)) { - _clear_redirector_entry($self, lc($domain), 'shortener'); - } - } - } + setting => 'url_shortener_custom_user_agent', + code => $url_redirector_custom_user_agent_code, }); =head1 PRIVILEGED SETTINGS @@ -726,10 +672,9 @@ See C for database cleaning. =item url_shortener_cache_type (default: none) -Same as C, for the (independent) shortener cache. - -Example: -url_shortener_cache_type dbi +Deprecated alias of C -- there is only one +cache now, shared by everything this plugin fetches (redirectors and +shorteners alike). =back @@ -737,20 +682,18 @@ url_shortener_cache_type dbi push (@cmds, { setting => 'url_shortener_cache_type', - default => '', is_priv => 1, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING + code => sub { + my ($self, $key, $value, $line) = @_; + $self->{url_redirector_cache_type} = $value; + } }); =over 4 =item url_shortener_cache_dsn (default: none) -Same as C, for the (independent) shortener cache. - -Examples: - - url_shortener_cache_dsn dbi:SQLite:dbname=/var/lib/spamassassin/DecodeShortURLs.db +Deprecated alias of C. =back @@ -758,17 +701,18 @@ Examples: push (@cmds, { setting => 'url_shortener_cache_dsn', - default => '', is_priv => 1, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING + code => sub { + my ($self, $key, $value, $line) = @_; + $self->{url_redirector_cache_dsn} = $value; + } }); =over 4 =item url_shortener_cache_username (default: none) -The username that should be used to connect to the shortener cache database. -Not used for SQLite. +Deprecated alias of C. =back @@ -776,17 +720,18 @@ Not used for SQLite. push (@cmds, { setting => 'url_shortener_cache_username', - default => '', is_priv => 1, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING + code => sub { + my ($self, $key, $value, $line) = @_; + $self->{url_redirector_cache_username} = $value; + } }); =over 4 =item url_shortener_cache_password (default: none) -The password that should be used to connect to the shortener cache database. -Not used for SQLite. +Deprecated alias of C. =back @@ -794,19 +739,20 @@ Not used for SQLite. push (@cmds, { setting => 'url_shortener_cache_password', - default => '', is_priv => 1, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING + code => sub { + my ($self, $key, $value, $line) = @_; + $self->{url_redirector_cache_password} = $value; + } }); =over 4 =item url_shortener_cache_ttl (default: 86400) -The length of time a shortener cache entry will be valid for in seconds. -Default is 86400 (1 day). +Deprecated alias of C. -See C for database cleaning. +See C for database cleaning. =back @@ -815,8 +761,13 @@ See C for database cleaning. push (@cmds, { setting => 'url_shortener_cache_ttl', is_admin => 1, - default => 86400, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC + code => sub { + my ($self, $key, $value, $line) = @_; + unless ($value =~ /^\d+$/) { + return $Mail::SpamAssassin::Conf::INVALID_VALUE; + } + $self->{url_redirector_cache_ttl} = $value + 0; + } }); =head1 ADMINISTRATOR SETTINGS @@ -937,7 +888,7 @@ like a common browser. =item url_shortener_cache_autoclean (default: 1000) -Same as C, for the (independent) shortener cache. +Deprecated alias of C. =back @@ -946,15 +897,20 @@ Same as C, for the (independent) shortener cache push (@cmds, { setting => 'url_shortener_cache_autoclean', is_admin => 1, - default => 1000, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC + code => sub { + my ($self, $key, $value, $line) = @_; + unless ($value =~ /^\d+$/) { + return $Mail::SpamAssassin::Conf::INVALID_VALUE; + } + $self->{url_redirector_cache_autoclean} = $value + 0; + } }); =over 4 =item url_shortener_loginfo (default: 0 (off)) -If this option is enabled (set to 1), then short URLs and the decoded URLs will be logged with info priority. +Deprecated alias of C. =back @@ -963,15 +919,28 @@ If this option is enabled (set to 1), then short URLs and the decoded URLs will push (@cmds, { setting => 'url_shortener_loginfo', is_admin => 1, - default => 0, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_BOOL + code => sub { + my ($self, $key, $value, $line) = @_; + unless (defined $value && $value !~ /^$/) { + return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; + } + # bug 4462: allow yes/1 and no/0 for boolean values + my $lc = lc $value; + if ($lc eq 'yes' || $lc eq '1') { + $self->{url_redirector_loginfo} = 1; + } elsif ($lc eq 'no' || $lc eq '0') { + $self->{url_redirector_loginfo} = 0; + } else { + return $Mail::SpamAssassin::Conf::INVALID_VALUE; + } + } }); =over 4 =item url_shortener_timeout (default: 5) -Maximum time a short URL HTTP request can take, in seconds. +Deprecated alias of C. =back @@ -980,19 +949,22 @@ Maximum time a short URL HTTP request can take, in seconds. push (@cmds, { setting => 'url_shortener_timeout', is_admin => 1, - default => 5, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC + code => sub { + my ($self, $key, $value, $line) = @_; + unless ($value =~ /^\d+$/) { + return $Mail::SpamAssassin::Conf::INVALID_VALUE; + } + $self->{url_redirector_timeout} = $value + 0; + } }); =over 4 =item max_short_urls (default: 10) -Maximum amount of short URLs that will be looked up per message. Chained -redirections are not counted, only initial short URLs found. - -Setting it to 0 disables HTTP requests, allowing only short_url() test to -work and report any found shortener URLs. +Deprecated alias of C -- there is only one budget now, +shared by everything this plugin fetches (redirectors and shorteners +alike). =back @@ -1001,15 +973,20 @@ work and report any found shortener URLs. push (@cmds, { setting => 'max_short_urls', is_admin => 1, - default => 10, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC + code => sub { + my ($self, $key, $value, $line) = @_; + unless ($value =~ /^\d+$/) { + return $Mail::SpamAssassin::Conf::INVALID_VALUE; + } + $self->{max_redir_urls} = $value + 0; + } }); =over 4 =item max_short_url_redirections (default: 10) -Maximum depth of chained redirections that a short URL can generate. +Deprecated alias of C. =back @@ -1018,17 +995,22 @@ Maximum depth of chained redirections that a short URL can generate. push (@cmds, { setting => 'max_short_url_redirections', is_admin => 1, - default => 10, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC + code => sub { + my ($self, $key, $value, $line) = @_; + unless ($value =~ /^\d+$/) { + return $Mail::SpamAssassin::Conf::INVALID_VALUE; + } + $self->{max_redir_url_redirections} = $value + 0; + } }); =over 4 =item url_shortener_user_agent (default: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36) -Set default User-Agent header for HTTP requests. Some services require it to look -like a common browser. User-Agent can be overriden on a per url_shortener basis using -the C setting. +Deprecated alias of C. Per-domain overrides use +C (C is +itself a deprecated alias of that). =back @@ -1037,8 +1019,10 @@ the C setting. push (@cmds, { setting => 'url_shortener_user_agent', is_admin => 1, - default => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36', - type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING + code => sub { + my ($self, $key, $value, $line) = @_; + $self->{url_redirector_user_agent} = $value; + } }); $conf->{parser}->register_commands(\@cmds); @@ -1195,157 +1179,6 @@ sub initialise_url_redirector_cache { } } -sub initialise_url_shortener_cache { - my ($self, $conf) = @_; - - return if $self->{dbh_short} && $self->{dbh_short_pid} && $self->{dbh_short_pid} == $$; - return if !$conf->{url_shortener_cache_type}; - - if (!$conf->{url_shortener_cache_dsn}) { - warn "Redirectors: invalid shortener cache configuration\n"; - return; - } - - ## - ## SQLite - ## - if ($conf->{url_shortener_cache_type} =~ /^(?:dbi|sqlite)$/i - && $conf->{url_shortener_cache_dsn} =~ /^dbi:SQLite/) - { - eval { - local $SIG{'__DIE__'}; - require DBI; - require DBD::SQLite; - DBD::SQLite->VERSION(1.59_01); # Required for ON CONFLICT - $self->{dbh_short} = DBI->connect_cached( - $conf->{url_shortener_cache_dsn}, '', '', - {RaiseError => 1, PrintError => 0, InactiveDestroy => 1, AutoCommit => 1} - ); - $self->{dbh_short}->do(" - CREATE TABLE IF NOT EXISTS short_url_cache ( - short_url TEXT PRIMARY KEY NOT NULL, - decoded_url TEXT NOT NULL, - hits INTEGER NOT NULL DEFAULT 1, - created INTEGER NOT NULL, - modified INTEGER NOT NULL - ) - "); - $self->{sth_insert_short} = $self->{dbh_short}->prepare(" - INSERT INTO short_url_cache (short_url, decoded_url, created, modified) - VALUES (?,?,strftime('%s','now'),strftime('%s','now')) - ON CONFLICT(short_url) DO UPDATE - SET decoded_url = excluded.decoded_url, - modified = excluded.modified, - hits = hits + 1 - "); - $self->{sth_select_short} = $self->{dbh_short}->prepare(" - SELECT decoded_url FROM short_url_cache - WHERE short_url = ? - "); - $self->{sth_delete_short} = $self->{dbh_short}->prepare(" - DELETE FROM short_url_cache - WHERE short_url = ? AND created < strftime('%s','now') - $conf->{url_shortener_cache_ttl} - "); - $self->{sth_clean_short} = $self->{dbh_short}->prepare(" - DELETE FROM short_url_cache - WHERE created < strftime('%s','now') - $conf->{url_shortener_cache_ttl} - "); - }; - } - ## - ## MySQL/MariaDB - ## - elsif (lc $conf->{url_shortener_cache_type} eq 'dbi' - && $conf->{url_shortener_cache_dsn} =~ /^dbi:(?:mysql|MariaDB)/i) - { - eval { - local $SIG{'__DIE__'}; - require DBI; - $self->{dbh_short} = DBI->connect_cached( - $conf->{url_shortener_cache_dsn}, - $conf->{url_shortener_cache_username}, - $conf->{url_shortener_cache_password}, - {RaiseError => 1, PrintError => 0, InactiveDestroy => 1, AutoCommit => 1} - ); - $self->{sth_insert_short} = $self->{dbh_short}->prepare(" - INSERT INTO short_url_cache (short_url, decoded_url, created, modified) - VALUES (?,?,UNIX_TIMESTAMP(),UNIX_TIMESTAMP()) - ON DUPLICATE KEY UPDATE - decoded_url = VALUES(decoded_url), - modified = VALUES(modified), - hits = hits + 1 - "); - $self->{sth_select_short} = $self->{dbh_short}->prepare(" - SELECT decoded_url FROM short_url_cache - WHERE short_url = ? - "); - $self->{sth_delete_short} = $self->{dbh_short}->prepare(" - DELETE FROM short_url_cache - WHERE short_url = ? AND created < UNIX_TIMESTAMP() - $conf->{url_shortener_cache_ttl} - "); - $self->{sth_clean_short} = $self->{dbh_short}->prepare(" - DELETE FROM short_url_cache - WHERE created < UNIX_TIMESTAMP() - $conf->{url_shortener_cache_ttl} - "); - }; - } - ## - ## PostgreSQL - ## - elsif (lc $conf->{url_shortener_cache_type} eq 'dbi' - && $conf->{url_shortener_cache_dsn} =~ /^dbi:Pg/i) - { - eval { - local $SIG{'__DIE__'}; - require DBI; - $self->{dbh_short} = DBI->connect_cached( - $conf->{url_shortener_cache_dsn}, - $conf->{url_shortener_cache_username}, - $conf->{url_shortener_cache_password}, - {RaiseError => 1, PrintError => 0, InactiveDestroy => 1, AutoCommit => 1} - ); - $self->{sth_insert_short} = $self->{dbh_short}->prepare(" - INSERT INTO short_url_cache (short_url, decoded_url, created, modified) - VALUES (?,?,CAST(EXTRACT(epoch FROM NOW()) AS INT),CAST(EXTRACT(epoch FROM NOW()) AS INT)) - ON CONFLICT (short_url) DO UPDATE SET - decoded_url = EXCLUDED.decoded_url, - modified = EXCLUDED.modified, - hits = short_url_cache.hits + 1 - "); - $self->{sth_select_short} = $self->{dbh_short}->prepare(" - SELECT decoded_url FROM short_url_cache - WHERE short_url = ? - "); - $self->{sth_delete_short} = $self->{dbh_short}->prepare(" - DELETE FROM short_url_cache - WHERE short_url = ? AND created < CAST(EXTRACT(epoch FROM NOW()) AS INT) - $conf->{url_shortener_cache_ttl} - "); - $self->{sth_clean_short} = $self->{dbh_short}->prepare(" - DELETE FROM short_url_cache - WHERE created < CAST(EXTRACT(epoch FROM NOW()) AS INT) - $conf->{url_shortener_cache_ttl} - "); - }; - ## - ## ... - ## - } else { - warn "Redirectors: invalid shortener cache configuration\n"; - return; - } - - if ($@ || !$self->{sth_clean_short}) { - warn "Redirectors: shortener cache connect failed: $@\n"; - undef $self->{dbh_short}; - undef $self->{dbh_short_pid}; - undef $self->{sth_insert_short}; - undef $self->{sth_select_short}; - undef $self->{sth_delete_short}; - undef $self->{sth_clean_short}; - } else { - $self->{dbh_short_pid} = $$; - } -} - sub redir_url { my ($self, $pms) = @_; @@ -1411,69 +1244,26 @@ sub redir_url_loop { return $pms->{redir_url_loop} ? 1 : 0; } -sub short_url { - my ($self, $pms) = @_; - - $self->_check_redir($pms); - - return $pms->{short_url} ? 1 : 0; -} +# short_url() and friends are deprecated aliases of redir_url() and +# friends, a "shortener" and a "redirector" are the same mechanism, so +# there is nothing left for a separate short_url_* implementation to do. +# short_url_redir/short_url_200 are thin wrappers rather than glob aliases +# since their names don't line up 1:1 with a redir_url_* counterpart. +*short_url = \&redir_url; +*short_url_chained = \&redir_url_chained; +*short_url_maxchain = \&redir_url_maxchain; +*short_url_loop = \&redir_url_loop; +*short_url_404 = \&redir_url_404; +*short_url_code = \&redir_url_code; sub short_url_redir { my ($self, $pms) = @_; - - $self->_check_redir($pms); - - return $pms->{short_url_redir} ? 1 : 0; + return $self->redir_url_valid($pms); } sub short_url_200 { my ($self, $pms) = @_; - - $self->_check_redir($pms); - - return $pms->{short_url_200} ? 1 : 0; -} - -sub short_url_404 { - my ($self, $pms) = @_; - - $self->_check_redir($pms); - - return $pms->{short_url_404} ? 1 : 0; -} - -sub short_url_code { - my ($self, $pms, undef, $code) = @_; - - $self->_check_redir($pms); - - return 0 unless defined $code && $code =~ /^\d{3}$/; - return $pms->{"short_url_$code"} ? 1 : 0; -} - -sub short_url_chained { - my ($self, $pms) = @_; - - $self->_check_redir($pms); - - return $pms->{short_url_chained} ? 1 : 0; -} - -sub short_url_maxchain { - my ($self, $pms) = @_; - - $self->_check_redir($pms); - - return $pms->{short_url_maxchain} ? 1 : 0; -} - -sub short_url_loop { - my ($self, $pms) = @_; - - $self->_check_redir($pms); - - return $pms->{short_url_loop} ? 1 : 0; + return $self->redir_url_code($pms, undef, '200'); } sub short_url_tests { @@ -1492,12 +1282,8 @@ sub finish_parsing_start { } # Add a host[/path] entry to the shared exact/suffix lookup buckets. -# $kind is 'redirector' (default), 'selenium', or 'shortener'; it records -# which legacy rule family (redir_url_* vs short_url_*) a hop matching this -# entry belongs to. sub _add_redirector_entry { - my ($conf, $token, $method, $kind) = @_; - $kind = 'redirector' unless defined $kind; + my ($conf, $token, $method) = @_; my ($domspec, $path) = split(/\//, $token, 2); $path = defined $path ? '/' . $path : '/'; @@ -1511,17 +1297,13 @@ sub _add_redirector_entry { my $display = ($is_suffix ? '.' : '') . $domspec; warn "redirectors: $display already registered with method '$existing->{method}'; overriding with '$method'\n"; } - my $entry = $conf->{$bucket}->{$domspec} ||= { method => $method, paths => [], kind => $kind }; + my $entry = $conf->{$bucket}->{$domspec} ||= { method => $method, paths => [] }; $entry->{method} = $method; - $entry->{kind} = $kind; push @{$entry->{paths}}, $path unless grep { $_ eq $path } @{$entry->{paths}}; } -# $class is 'redirector' (matches kind ne 'shortener', i.e. redirector or -# selenium) or 'shortener' (matches kind eq 'shortener'). Undef means no -# kind filtering (back-compat for direct callers/tests predating the merge). sub _clear_redirector_entry { - my ($conf, $token, $class) = @_; + my ($conf, $token) = @_; $token = lc $token; my $has_path = ($token =~ /\//) ? 1 : 0; @@ -1532,11 +1314,6 @@ sub _clear_redirector_entry { return unless length $domspec; my $entry = $conf->{$bucket}->{$domspec} or return; - if (defined $class) { - my $matches = $class eq 'shortener' ? ($entry->{kind} eq 'shortener') - : ($entry->{kind} ne 'shortener'); - return unless $matches; - } if (!$has_path) { delete $conf->{$bucket}->{$domspec}; @@ -1546,19 +1323,12 @@ sub _clear_redirector_entry { delete $conf->{$bucket}->{$domspec} unless @{$entry->{paths}}; } -# Remove every entry of the given class ('redirector' or 'shortener') from -# both lookup buckets. Used by clear_url_redirector/clear_url_shortener with -# no arguments. -sub _clear_all_entries_by_class { - my ($conf, $class) = @_; - for my $bucket (qw(url_redirector_exact url_redirector_suffix)) { - next unless $conf->{$bucket}; - foreach my $dom (keys %{$conf->{$bucket}}) { - my $kind = $conf->{$bucket}->{$dom}->{kind}; - my $matches = $class eq 'shortener' ? ($kind eq 'shortener') : ($kind ne 'shortener'); - delete $conf->{$bucket}->{$dom} if $matches; - } - } +# Remove every entry from both lookup buckets. Used by +# clear_url_redirector/clear_url_shortener with no arguments. +sub _clear_all_redirector_entries { + my ($conf) = @_; + $conf->{url_redirector_exact} = {}; + $conf->{url_redirector_suffix} = {}; } sub _entry_match_path { @@ -1637,7 +1407,7 @@ sub _parse_uri { return ($uri, $host, $path, $rest); } -# Returns the redirector entry ({method, paths, kind}) if $uri's host+path +# Returns the redirector entry ({method, paths}) if $uri's host+path # matches a configured url_redirector / url_redirector_get / url_shortener*, # else undef. sub _is_configured_redirector { @@ -1685,12 +1455,11 @@ sub _extract_embedded_uri { return; } -# Lazily build (and cache on $pms) the LWP::UserAgent for $kind -# ('redirector' or 'shortener'), each with its own default UA/timeout. +# Lazily build (and cache on $pms) the single LWP::UserAgent used for +# every fetch this plugin makes. sub _get_lwp_ua { - my ($self, $pms, $kind) = @_; - my $slot = $kind eq 'shortener' ? 'short_lwp_ua' : 'redir_lwp_ua'; - return $pms->{$slot} if exists $pms->{$slot}; + my ($self, $pms) = @_; + return $pms->{redir_lwp_ua} if exists $pms->{redir_lwp_ua}; my $conf = $pms->{conf}; # prevent "500 Header line too long (limit is 8192)" error when accessing @@ -1699,16 +1468,13 @@ sub _get_lwp_ua { use LWP::Protocol::http; push(@LWP::Protocol::http::EXTRA_SOCK_OPTS, MaxLineLength => 16*1024); }; - my ($agent, $timeout) = $kind eq 'shortener' - ? ($conf->{url_shortener_user_agent}, $conf->{url_shortener_timeout}) - : ($conf->{url_redirector_user_agent}, $conf->{url_redirector_timeout}); my $ua = LWP::UserAgent->new( - 'agent' => $agent, + 'agent' => $conf->{url_redirector_user_agent}, 'max_redirect' => 0, - 'timeout' => $timeout, + 'timeout' => $conf->{url_redirector_timeout}, ); $ua->env_proxy; - return $pms->{$slot} = $ua; + return $pms->{redir_lwp_ua} = $ua; } sub _get_selenium_ua { @@ -1759,30 +1525,26 @@ sub _get_selenium_ua { return $pms->{redir_selenium_ua} = $ua; } -# Perform an HTTP request for $uri using $method (LWP) or Selenium, for a -# hop of the given $kind ('redirector'/'selenium' or 'shortener') -- this -# selects the cache backend, UA/timeout settings, and status-code flag -# prefix (redir_url_* vs short_url_*). +# Perform an HTTP request for $uri using $method (LWP) or Selenium. # Returns the absolute, normalized Location URL on a usable redirect, -# or undef otherwise. Sets _ flags on $pms and writes +# or undef otherwise. Sets redir_url_ flags on $pms and writes # the cache. sub _do_http { - my ($self, $uri, $method, $pms, $kind) = @_; + my ($self, $uri, $method, $pms) = @_; my $conf = $pms->{conf}; - my $flag_prefix = $kind eq 'shortener' ? 'short_url' : 'redir_url'; my $redir_url = $uri; my $location; - if (defined($location = $self->cache_get($kind, $redir_url))) { - if ($conf->{url_redirector_loginfo} || $conf->{url_shortener_loginfo}) { + if (defined($location = $self->cache_get($redir_url))) { + if ($conf->{url_redirector_loginfo}) { info("found cached $redir_url => $location"); } else { dbg("found cached $redir_url => $location"); } if ($location =~ /^\d{3}$/) { - $pms->{"${flag_prefix}_$location"} = 1; - $self->cache_add($kind, $redir_url, $location); + $pms->{"redir_url_$location"} = 1; + $self->cache_add($redir_url, $location); return; } } else { @@ -1832,18 +1594,16 @@ sub _do_http { } } $location = $newurl; - $pms->{"${flag_prefix}_$rcode"} = 1; - $self->cache_add($kind, $redir_url, $rcode); + $pms->{"redir_url_$rcode"} = 1; + $self->cache_add($redir_url, $rcode); if($rcode !~ /^30[12]/) { return; } } else { - my $ua = $self->_get_lwp_ua($pms, $kind); - if ($kind eq 'shortener') { - my (undef, $host) = _parse_uri($redir_url, $conf); - my $custom_ua = defined $host ? $conf->{url_shortener_custom_ua}->{$host} : undef; - $ua->agent(defined $custom_ua ? $custom_ua : $conf->{url_shortener_user_agent}); - } + my $ua = $self->_get_lwp_ua($pms); + my (undef, $host) = _parse_uri($redir_url, $conf); + my $custom_ua = defined $host ? $conf->{url_redirector_custom_ua}->{$host} : undef; + $ua->agent(defined $custom_ua ? $custom_ua : $conf->{url_redirector_user_agent}); my $response = $ua->$method($redir_url); return if not defined $response; @@ -1873,8 +1633,8 @@ sub _do_http { dbg("Found a meta http-equiv redirector, changing http response code from " . $response->code . " to $rcode"); } } else { - $pms->{"${flag_prefix}_$rcode"} = 1; - $self->cache_add($kind, $redir_url, $rcode); + $pms->{"redir_url_$rcode"} = 1; + $self->cache_add($redir_url, $rcode); } } if($rcode !~ /^30[12]/) { @@ -1885,7 +1645,7 @@ sub _do_http { if((exists $response->headers->{location}) or $http_equiv) { $location = $response->headers->{location} if not $http_equiv; if($redir_url ne $location) { - if ($conf->{url_redirector_loginfo} || $conf->{url_shortener_loginfo}) { + if ($conf->{url_redirector_loginfo}) { info("found $redir_url => $location"); } else { dbg("found $redir_url => $location"); @@ -1897,7 +1657,7 @@ sub _do_http { return unless defined $location; - $self->cache_add($kind, $redir_url, $location); + $self->cache_add($redir_url, $location); # Resolve relative Location header to absolute. if ($location !~ m{^[a-z]+://}i) { @@ -1937,16 +1697,10 @@ sub _do_http { # _is_configured_redirector nor _extract_embedded_uri matches. HTTP # requests are gated on _is_configured_redirector returning truthy. # -# Both redirectors and shorteners flow through this single walk, so a hop -# of either kind is checked at every depth, a redirector unwrapping into -# a shortener (or vice versa) is followed either way. Detection -# (redir_url/short_url) and HTTP status-code flags stay tied to the kind -# of the entry that actually matched at that hop; chain-level flags -# (chained/chained_domain/loop/valid-redirect) fire for both legacy rule -# families together since they describe the shape of the chain as a -# whole, not which kind produced it. Maxchain is checked against both -# configured depth limits independently against the single shared depth -# counter. +# There is no functional distinction between what used to be called a +# "redirector" and a "shortener" -- both are just a configured host whose +# response redirects elsewhere, and a hop of either origin is checked at +# every depth, so a chain can freely mix them in any order. sub _walk_redirects { my ($self, $uri, $src_info, $pms, $depth, $been_here) = @_; my $conf = $pms->{conf}; @@ -1954,15 +1708,11 @@ sub _walk_redirects { if (exists $been_here->{"uri:$uri"}) { dbg("error: loop detected: $uri"); $pms->{redir_url_loop} = 1; - $pms->{short_url_loop} = 1; return; } - my $over_redir = $depth >= $conf->{max_redir_url_redirections}; - my $over_short = $depth >= $conf->{max_short_url_redirections}; - if ($over_redir || $over_short) { - dbg("found more than allowed chained redirections at depth $depth"); - $pms->{redir_url_maxchain} = 1 if $over_redir; - $pms->{short_url_maxchain} = 1 if $over_short; + if ($depth >= $conf->{max_redir_url_redirections}) { + dbg("found more than $conf->{max_redir_url_redirections} chained redirections"); + $pms->{redir_url_maxchain} = 1; return; } $been_here->{"uri:$uri"} = 1; @@ -1974,20 +1724,15 @@ sub _walk_redirects { # detection rule fires (matches the max_redir_urls=0 semantics: "found a # redirector but didn't probe it"). Fall through to embedded-URI extraction # in case the URL also carries a querystring redirect. - if ($rentry && $rentry->{kind} eq 'selenium' && !$conf->{url_redirector_use_selenium}) { + if ($rentry && $rentry->{method} eq 'selenium' && !$conf->{url_redirector_use_selenium}) { dbg("$uri matches url_redirector_selenium but url_redirector_use_selenium=0, skipping http lookup"); $pms->{redir_url} = 1; $rentry = undef; } if ($rentry) { - my $kind = $rentry->{kind}; - my $flag_prefix = $kind eq 'shortener' ? 'short_url' : 'redir_url'; - $pms->{$flag_prefix} = 1; - if ($depth > 0) { - $pms->{redir_url_chained} = 1; - $pms->{short_url_chained} = 1; - } + $pms->{redir_url} = 1; + $pms->{redir_url_chained} = 1 if $depth > 0; my (undef, $host) = $self->{main}->{registryboundaries}->uri_to_domain($uri); if (defined $host) { @@ -1999,34 +1744,21 @@ sub _walk_redirects { } return if $self->{net_disabled}; - - if ($kind eq 'shortener') { - return if !$conf->{max_short_urls}; - if ($depth == 0) { - return if ++$pms->{short_seed_count} > $conf->{max_short_urls}; - } - } else { - return if !$conf->{max_redir_urls}; - if ($depth == 0) { - return if ++$pms->{redir_seed_count} > $conf->{max_redir_urls}; - } + return if !$conf->{max_redir_urls}; + if ($depth == 0) { + return if ++$pms->{redir_seed_count} > $conf->{max_redir_urls}; } - # Shorteners historically strip anchors/query strings before fetching - # (their whole identity is the path token, and services often don't - # care about trailing #fragment/?query on the short link itself). - # Redirectors keep the query string intact, url_redirector_params - # needs it to extract embedded destination URIs. - my $fetch_uri = $uri; - $fetch_uri =~ s{/?[?#].*}{} if $kind eq 'shortener'; + # Strip the fragment before fetching, RFC 3986 defines it as + # client-side-only. + (my $fetch_uri = $uri) =~ s/#.*//; - my $location = $self->_do_http($fetch_uri, $rentry->{method}, $pms, $kind); + my $location = $self->_do_http($fetch_uri, $rentry->{method}, $pms); return unless defined $location; if ($fetch_uri eq $location) { dbg("URL redirects to itself"); $pms->{redir_url_loop} = 1; - $pms->{short_url_loop} = 1; return; } @@ -2038,7 +1770,6 @@ sub _walk_redirects { _add_redirect_uri($pms, $location, $src_info); $pms->{redir_url_valid} = 1; - $pms->{short_url_redir} = 1; return $self->_walk_redirects($location, $src_info, $pms, $depth + 1, $been_here); } @@ -2064,9 +1795,8 @@ sub _check_redir { my $conf = $pms->{conf}; $self->initialise_url_redirector_cache($conf); - $self->initialise_url_shortener_cache($conf); - # UAs are built lazily inside _do_http and cached on $pms. No upfront + # The UA is built lazily inside _do_http and cached on $pms. No upfront # construction here -- a message with only embedded-URI matches and no # HTTP-eligible URIs will not create a UA at all. my $uris = $pms->get_uri_detail_list(); @@ -2080,17 +1810,9 @@ sub _check_redir { if ($self->{dbh_redir} && $conf->{url_redirector_cache_autoclean} && rand() < 1/$conf->{url_redirector_cache_autoclean}) { - dbg("cleaning stale redirector cache entries"); + dbg("cleaning stale cache entries"); eval { $self->{sth_clean_redir}->execute(); }; - if ($@) { dbg("redirector cache cleaning failed: $@"); } - } - - if ($self->{dbh_short} && $conf->{url_shortener_cache_autoclean} - && rand() < 1/$conf->{url_shortener_cache_autoclean}) - { - dbg("cleaning stale shortener cache entries"); - eval { $self->{sth_clean_short}->execute(); }; - if ($@) { dbg("shortener cache cleaning failed: $@"); } + if ($@) { dbg("cache cleaning failed: $@"); } } } @@ -2115,15 +1837,13 @@ sub _add_redirect_uri { } sub cache_add { - my ($self, $kind, $key, $value) = @_; + my ($self, $key, $value) = @_; - my $dbh = $kind eq 'shortener' ? $self->{dbh_short} : $self->{dbh_redir}; - return if !$dbh; + return if !$self->{dbh_redir}; return if length($key) > 256 || length($value) > 512; - my $sth = $kind eq 'shortener' ? $self->{sth_insert_short} : $self->{sth_insert_redir}; # Upsert - eval { $sth->execute($key, $value); }; + eval { $self->{sth_insert_redir}->execute($key, $value); }; if ($@) { dbg("could not add to cache: $@"); } @@ -2132,15 +1852,13 @@ sub cache_add { } sub cache_get { - my ($self, $kind, $key) = @_; + my ($self, $key) = @_; - my $dbh = $kind eq 'shortener' ? $self->{dbh_short} : $self->{dbh_redir}; - return if !$dbh; + return if !$self->{dbh_redir}; - my $sth_delete = $kind eq 'shortener' ? $self->{sth_delete_short} : $self->{sth_delete_redir}; # Make sure expired entries are gone. Just a quick check for primary key, # not that expensive. - eval { $sth_delete->execute($key); }; + eval { $self->{sth_delete_redir}->execute($key); }; if ($@) { dbg("cache delete failed: $@"); return; @@ -2148,14 +1866,13 @@ sub cache_get { # Now try to get it (don't bother parsing if something was deleted above, # it would be rare event anyway) - my $sth_select = $kind eq 'shortener' ? $self->{sth_select_short} : $self->{sth_select_redir}; - eval { $sth_select->execute($key); }; + eval { $self->{sth_select_redir}->execute($key); }; if ($@) { dbg("cache get failed: $@"); return; } - my @row = $sth_select->fetchrow_array(); + my @row = $self->{sth_select_redir}->fetchrow_array(); if (@row) { return $row[0]; } diff --git a/t/decodeshorturl.t b/t/decodeshorturl.t index 0612500d38..25fa8dfad7 100755 --- a/t/decodeshorturl.t +++ b/t/decodeshorturl.t @@ -82,6 +82,12 @@ ok_all_patterns(); sarun ("-D Redirectors -t < data/spam/decodeshorturl/anchor.eml 2>&1", \&patterns_run_cb); ok_all_patterns(); +# Query strings are not stripped before fetching a shortener (only +# fragments are, since RFC 3986 defines them as client-side-only). +%patterns = ( + 'https://tinyurl.com/jf8wv76t?p=1&q=2 => https://spamassassin.apache.org/?p=1&q=2' +); + sarun ("-D Redirectors -t < data/spam/decodeshorturl/params.eml 2>&1", \&patterns_run_cb); ok_all_patterns(); @@ -111,8 +117,11 @@ describe HAS_SHORT_URL Message contains one or more shortened URLs sarun ("-t < data/spam/decodeshorturl/base.eml", \&patterns_run_cb); ok_all_patterns(); +# url_shortener_cache_* is a deprecated alias of url_redirector_cache_*; +# there is only one cache table now (redir_url_cache), shared by +# redirectors and shorteners alike. my $dbh = DBI->connect("dbi:SQLite:dbname=$workdir/DecodeShortURLs.db","",""); -my @row = $dbh->selectrow_array("SELECT decoded_url FROM short_url_cache WHERE short_url = 'http://bit.ly/30yH6WK'"); +my @row = $dbh->selectrow_array("SELECT target_url FROM redir_url_cache WHERE redir_url = 'http://bit.ly/30yH6WK'"); is($row[0], 'http://spamassassin.apache.org/'); # Check another email to cleanup old entries from database @@ -120,7 +129,7 @@ sarun ("-t < data/spam/decodeshorturl/base2.eml", \&patterns_run_cb); ok_all_patterns(); $dbh = DBI->connect("dbi:SQLite:dbname=$workdir/DecodeShortURLs.db","",""); -@row = $dbh->selectrow_array("SELECT decoded_url FROM short_url_cache WHERE short_url = 'http://bit.ly/30yH6WK'"); +@row = $dbh->selectrow_array("SELECT target_url FROM redir_url_cache WHERE redir_url = 'http://bit.ly/30yH6WK'"); isnt($row[0], 'https://spamassassin.apache.org/'); }