diff --git a/UPGRADE b/UPGRADE index 8b05393739..b43f508863 100644 --- a/UPGRADE +++ b/UPGRADE @@ -1,6 +1,13 @@ Note for Users Upgrading to SpamAssassin 4.0.3 ---------------------------------------------- +- Mail::SpamAssassin::Plugin::DecodeShortURLs has been merged into + Mail::SpamAssassin::Plugin::Redirectors and is now a deprecated + compatibility shim; all url_shortener*/short_url* settings and rules + still work as aliases of url_redirector*/redir_url*. Redirect chains + mixing shorteners and redirectors in any order are now followed + correctly, and query strings are no longer stripped before fetching. + - Mail::SpamAssassin::Plugin::ExtractText now has the possibility to cache results in order to speedup attachment processing. diff --git a/lib/Mail/SpamAssassin/Plugin/DecodeShortURLs.pm b/lib/Mail/SpamAssassin/Plugin/DecodeShortURLs.pm index 73ac5b9aa9..214351895b 100644 --- a/lib/Mail/SpamAssassin/Plugin/DecodeShortURLs.pm +++ b/lib/Mail/SpamAssassin/Plugin/DecodeShortURLs.pm @@ -5,9 +5,9 @@ # The ASF licenses this file to you under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at: -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,996 +17,70 @@ =head1 NAME -DecodeShortURLs - Check for shortened URLs +DecodeShortURLs - deprecated, merged into Mail::SpamAssassin::Plugin::Redirectors =head1 SYNOPSIS loadplugin Mail::SpamAssassin::Plugin::DecodeShortURLs - url_shortener tinyurl.com - url_shortener_get bit.ly - url_shortener_custom_user_agent t.co curl/8.6.0 - - body HAS_SHORT_URL eval:short_url() - describe HAS_SHORT_URL Message has one or more shortened URLs - - body SHORT_URL_REDIR eval:short_url_redir() - describe SHORT_URL_REDIR Message has shortened URL that resulted in a valid redirection - - body SHORT_URL_CHAINED eval:short_url_chained() - describe SHORT_URL_CHAINED Message has shortened URL chained to other shorteners - - body SHORT_URL_MAXCHAIN eval:short_url_maxchain() - describe SHORT_URL_MAXCHAIN Message has shortened URL that causes too many redirections - - body SHORT_URL_LOOP eval:short_url_loop() - describe SHORT_URL_LOOP Message has short URL that loops back to itself - - body SHORT_URL_200 eval:short_url_code('200') # Can check any non-redirect HTTP code - describe SHORT_URL_200 Message has shortened URL returning HTTP 200 - - body SHORT_URL_404 eval:short_url_code('404') # Can check any non-redirect HTTP code - describe SHORT_URL_404 Message has shortened URL returning HTTP 404 - - uri URI_TINYURL_BLOCKED m,https://tinyurl\.com/app/nospam, - describe URI_TINYURL_BLOCKED Message contains a tinyurl that has been disabled due to abuse - - uri URI_BITLY_BLOCKED m,^https://bitly\.com/a/blocked, - describe URI_BITLY_BLOCKED Message contains a bit.ly URL that has been disabled due to abuse - =head1 DESCRIPTION -This plugin looks for URLs shortened by a list of URL shortening services. -Upon finding a matching URL, plugin will send a HTTP request to the -shortening service and retrieve the Location-header which points to the -actual shortened URL. It then adds this URL to the list of URIs extracted -by SpamAssassin which can then be accessed by uri rules and plugins such as -URIDNSBL. - -This plugin will follow chained redirections, where a short URL redirects to -another short URL. Redirection depth limit can be set with -C. +B All of its functionality (the C +settings and the C eval rules) has been merged into +L, which also gained the ability to +follow a redirect chain that mixes shorteners and redirectors in either +order, there is no functional difference between the two, both are just +a host whose HTTP response redirects elsewhere. -Maximum of C short URLs are checked in a message (10 by -default). Setting it to 0 disables HTTP requests, allowing only short_url() -test to work and report found shorteners. +This module remains only as a compatibility shim so that existing +configuration files with C +keep working unchanged. It loads C and inherits all of its +behavior. New configurations should load +C directly instead. -All supported rule types for checking short URLs and redirection status are -documented in L section. - -=head1 NOTES +=head1 ACKNOWLEDGEMENTS -This plugin runs at priority -10 so that it may -modify the parsed URI list prior to normal uri rules or the URIDNSBL plugin -but after the Redirector plugin. +Original DecodeShortURLs plugin was developed by Steve Freegard. =cut package Mail::SpamAssassin::Plugin::DecodeShortURLs; -use Mail::SpamAssassin::Plugin; -use Mail::SpamAssassin::Util qw(idn_to_ascii is_fqdn_valid); +use Mail::SpamAssassin::Plugin::Redirectors; use strict; use warnings; use vars qw(@ISA); -@ISA = qw(Mail::SpamAssassin::Plugin); - -my $VERSION = 4.00; +@ISA = qw(Mail::SpamAssassin::Plugin::Redirectors); -use constant HAS_LWP_USERAGENT => eval { require LWP::UserAgent; require LWP::Protocol::https; }; - -sub dbg { my $msg = shift; return Mail::SpamAssassin::Logger::dbg("DecodeShortURLs: $msg", @_); } -sub info { my $msg = shift; return Mail::SpamAssassin::Logger::info("DecodeShortURLs: $msg", @_); } +# Published rulesets (e.g. rules/25_url_shortener.cf) probe for plugin +# features with "if can(Mail::SpamAssassin::Plugin::DecodeShortURLs::has_x)", +# which calls the fully-qualified sub directly rather than as a method, +# that bypasses @ISA, so a plain subclass wouldn't expose subs it merely +# inherits from Redirectors. Alias them into this package's symbol table +# so such checks keep working. +BEGIN { + no strict 'refs'; + for my $sub (qw( + has_short_url has_autoclean has_short_url_code has_user_agent + has_custom_user_agent has_get has_clear has_timeout + has_max_redirections has_short_url_redir + )) { + *{$sub} = \&{"Mail::SpamAssassin::Plugin::Redirectors::$sub"}; + } +} sub new { my $class = shift; my $mailsaobject = shift; - $class = ref($class) || $class; - my $self = $class->SUPER::new($mailsaobject); - bless ($self, $class); - - if ($mailsaobject->{local_tests_only}) { - dbg("local tests only, disabling HTTP requests"); - $self->{net_disabled} = 1; - } - elsif (!HAS_LWP_USERAGENT) { - dbg("module LWP::UserAgent not installed, disabling HTTP requests"); - $self->{net_disabled} = 1; - } - - $self->set_config($mailsaobject->{conf}); - $self->register_method_priority ('check_dnsbl', -10); - $self->register_eval_rule('short_url', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); - $self->register_eval_rule('short_url_redir', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); - $self->register_eval_rule('short_url_200', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); - $self->register_eval_rule('short_url_404', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); - $self->register_eval_rule('short_url_code', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); - $self->register_eval_rule('short_url_chained', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); - $self->register_eval_rule('short_url_maxchain', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); - $self->register_eval_rule('short_url_loop', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); - $self->register_eval_rule('short_url_tests'); # for legacy plugin compatibility warning - - return $self; -} - -=head1 USER SETTINGS - -=over 4 - -=item url_shortener domain [domain...] (default: none) - -Domains that should be considered as an URL shortener. If the domain begins -with a '.', 3rd level tld of the main domain will be checked. - -Example: - - url_shortener tinyurl.com - url_shortener .page.link - -=back - -=over 4 - -=item url_shortener_get domain [domain...] (default: none) - -Alias to C. HTTP request will be done with GET method, -instead of default HEAD. Required for some services like bit.ly to return -blocked URL correctly. - -Example: - - url_shortener_get bit.ly - -=back - -=cut - -sub set_config { - my($self, $conf) = @_; - my @cmds = (); - - push (@cmds, { - setting => 'url_shortener', - default => {}, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_HASH_KEY_VALUE, - code => sub { - my ($self, $key, $value, $line) = @_; - if ($value eq '') { - return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; - } - foreach my $domain (split(/\s+/, $value)) { - $self->{url_shortener}->{lc $domain} = 1; # 1 == head - } - } - }); - - push (@cmds, { - setting => 'url_shortener_get', - code => sub { - my ($self, $key, $value, $line) = @_; - if ($value eq '') { - return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; - } - foreach my $domain (split(/\s+/, $value)) { - $self->{url_shortener}->{lc $domain} = 2; # 2 == get - } - } - }); - -=over 4 - -=item url_shortener_custom_user_agent domain user-agent (default: none) - -Custom HTTP user-agent to be used for specific domains, -instead of the default specified in C. -Required for some services like t.co to return blocked URL correctly. - -Example: - - url_shortener_custom_user_agent t.co curl/8.6.0 - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_custom_user_agent', - code => sub { - my ($self, $key, $value, $line) = @_; - if ($value eq '') { - return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; - } - my @values = split(/\s+/, $value); - my $domain = shift(@values); - my $ua = join('', @values); - $self->{url_shortener}->{user_agent}->{lc $domain} = $ua; - } - }); - -=over 4 - -=item clear_url_shortener [domain] [domain...] - -Clear configured url_shortener and url_shortener_get domains, for example to -override default settings from an update channel. If domains are specified, -then only those are removed from list. - -=back - -=cut - - push (@cmds, { - setting => 'clear_url_shortener', - code => sub { - my ($self, $key, $value, $line) = @_; - if ($value eq '') { - $self->{url_shortener} = {}; - } else { - foreach my $domain (split(/\s+/, $value)) { - delete $self->{url_shortener}->{lc $domain}; - } - } - } - }); - -=head1 PRIVILEGED SETTINGS - -=over 4 - -=item url_shortener_cache_type (default: none) - -The cache type that is being utilized. Currently only supported value is -C that implies C is a DBI connect string. -DBI module is required. - -Example: -url_shortener_cache_type dbi - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_cache_type', - default => '', - is_priv => 1, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING - }); - -=over 4 - -=item url_shortener_cache_dsn (default: none) - -The DBI dsn of the database to use. - -For SQLite, the database will be created automatically if it does not -already exist, the supplied path and file must be read/writable by the -user running spamassassin or spamd. - -For MySQL/MariaDB or PostgreSQL, see sql-directory for database table -creation clauses. - -You will need to have the proper DBI module for your database. For example -DBD::SQLite, DBD::mysql, DBD::MariaDB or DBD::Pg. - -Minimum required SQLite version is 3.24.0 (available from DBD::SQLite 1.59_01). - -Examples: - - url_shortener_cache_dsn dbi:SQLite:dbname=/var/lib/spamassassin/DecodeShortURLs.db - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_cache_dsn', - default => '', - is_priv => 1, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING - }); - -=over 4 - -=item url_shortener_cache_username (default: none) - -The username that should be used to connect to the database. Not used for -SQLite. - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_cache_username', - default => '', - is_priv => 1, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING - }); - -=over 4 - -=item url_shortener_cache_password (default: none) - -The password that should be used to connect to the database. Not used for -SQLite. - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_cache_password', - default => '', - is_priv => 1, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING - }); - -=over 4 - -=item url_shortener_cache_ttl (default: 86400) - -The length of time a cache entry will be valid for in seconds. -Default is 86400 (1 day). - -See C for database cleaning. - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_cache_ttl', - is_admin => 1, - default => 86400, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC - }); - -=head1 ADMINISTRATOR SETTINGS - -=over 4 - -=item url_shortener_cache_autoclean (default: 1000) - -Automatically purge old entries from database. Value describes a random run -chance of 1/x. The default value of 1000 means that cleaning is run -approximately once for every 1000 messages processed. Value of 1 would mean -database is cleaned every time a message is processed. - -Set 0 to disable automatic cleaning and to do it manually. - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_cache_autoclean', - is_admin => 1, - default => 1000, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC - }); - -=over 4 - -=item url_shortener_loginfo (default: 0 (off)) - -If this option is enabled (set to 1), then short URLs and the decoded URLs will be logged with info priority. - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_loginfo', - is_admin => 1, - default => 0, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_BOOL - }); - -=over 4 - -=item url_shortener_timeout (default: 5) - -Maximum time a short URL HTTP request can take, in seconds. - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_timeout', - is_admin => 1, - default => 5, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC - }); - -=over 4 - -=item max_short_urls (default: 10) - -Maximum amount of short URLs that will be looked up per message. Chained -redirections are not counted, only initial short URLs found. - -Setting it to 0 disables HTTP requests, allowing only short_url() test to -work and report any found shortener URLs. - -=back - -=cut - - push (@cmds, { - setting => 'max_short_urls', - is_admin => 1, - default => 10, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC - }); - -=over 4 - -=item max_short_url_redirections (default: 10) - -Maximum depth of chained redirections that a short URL can generate. - -=back - -=cut - - push (@cmds, { - setting => 'max_short_url_redirections', - is_admin => 1, - default => 10, - type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC - }); - -=over 4 - -=item url_shortener_user_agent (default: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36) - -Set default User-Agent header for HTTP requests. Some services require it to look -like a common browser. User-Agent can be overridden on a per url_shortener basis using -the C setting. - -=back - -=cut - - push (@cmds, { - setting => 'url_shortener_user_agent', - is_admin => 1, - default => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36', - type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING - }); - - $conf->{parser}->register_commands(\@cmds); -} - -=head1 ACKNOWLEDGEMENTS - -Original DecodeShortURLs plugin was developed by Steve Freegard. - -=cut - -sub short_url_tests { - # Legacy compatibility warning done in finish_parsing_start - return 0; -} - -sub finish_parsing_start { - my ($self, $opts) = @_; - - if ($opts->{conf}->{eval_to_rule}->{short_url_tests}) { - warn "DecodeShortURLs: Legacy configuration format detected. ". - "Eval function short_url_tests() is no longer supported, ". - "please see documentation for the new rule format.\n"; - } -} - -sub initialise_url_shortener_cache { - my ($self, $conf) = @_; - - return if $self->{dbh} && $self->{dbh_pid} && $self->{dbh_pid} == $$; - return if !$conf->{url_shortener_cache_type}; - - if (!$conf->{url_shortener_cache_dsn}) { - warn "DecodeShortURLs: invalid cache configuration\n"; - return; - } - - ## - ## SQLite - ## - if ($conf->{url_shortener_cache_type} =~ /^(?:dbi|sqlite)$/i - && $conf->{url_shortener_cache_dsn} =~ /^dbi:SQLite/) - { - eval { - local $SIG{'__DIE__'}; - require DBI; - require DBD::SQLite; - DBD::SQLite->VERSION(1.59_01); # Required for ON CONFLICT - $self->{dbh} = DBI->connect_cached( - $conf->{url_shortener_cache_dsn}, '', '', - {RaiseError => 1, PrintError => 0, InactiveDestroy => 1, AutoCommit => 1} - ); - $self->{dbh}->do(" - CREATE TABLE IF NOT EXISTS short_url_cache ( - short_url TEXT PRIMARY KEY NOT NULL, - decoded_url TEXT NOT NULL, - hits INTEGER NOT NULL DEFAULT 1, - created INTEGER NOT NULL, - modified INTEGER NOT NULL - ) - "); - # Maintaining index for cleaning is likely more expensive than occasional full table scan - #$self->{dbh}->do(" - # CREATE INDEX IF NOT EXISTS short_url_modified - # ON short_url_cache(created) - #"); - $self->{sth_insert} = $self->{dbh}->prepare(" - INSERT INTO short_url_cache (short_url, decoded_url, created, modified) - VALUES (?,?,strftime('%s','now'),strftime('%s','now')) - ON CONFLICT(short_url) DO UPDATE - SET decoded_url = excluded.decoded_url, - modified = excluded.modified, - hits = hits + 1 - "); - $self->{sth_select} = $self->{dbh}->prepare(" - SELECT decoded_url FROM short_url_cache - WHERE short_url = ? - "); - $self->{sth_delete} = $self->{dbh}->prepare(" - DELETE FROM short_url_cache - WHERE short_url = ? AND created < strftime('%s','now') - $conf->{url_shortener_cache_ttl} - "); - $self->{sth_clean} = $self->{dbh}->prepare(" - DELETE FROM short_url_cache - WHERE created < strftime('%s','now') - $conf->{url_shortener_cache_ttl} - "); - }; - } - ## - ## MySQL/MariaDB - ## - elsif (lc $conf->{url_shortener_cache_type} eq 'dbi' - && $conf->{url_shortener_cache_dsn} =~ /^dbi:(?:mysql|MariaDB)/i) - { - eval { - local $SIG{'__DIE__'}; - require DBI; - $self->{dbh} = DBI->connect_cached( - $conf->{url_shortener_cache_dsn}, - $conf->{url_shortener_cache_username}, - $conf->{url_shortener_cache_password}, - {RaiseError => 1, PrintError => 0, InactiveDestroy => 1, AutoCommit => 1} - ); - $self->{sth_insert} = $self->{dbh}->prepare(" - INSERT INTO short_url_cache (short_url, decoded_url, created, modified) - VALUES (?,?,UNIX_TIMESTAMP(),UNIX_TIMESTAMP()) - ON DUPLICATE KEY UPDATE - decoded_url = VALUES(decoded_url), - modified = VALUES(modified), - hits = hits + 1 - "); - $self->{sth_select} = $self->{dbh}->prepare(" - SELECT decoded_url FROM short_url_cache - WHERE short_url = ? - "); - $self->{sth_delete} = $self->{dbh}->prepare(" - DELETE FROM short_url_cache - WHERE short_url = ? AND created < UNIX_TIMESTAMP() - $conf->{url_shortener_cache_ttl} - "); - $self->{sth_clean} = $self->{dbh}->prepare(" - DELETE FROM short_url_cache - WHERE created < UNIX_TIMESTAMP() - $conf->{url_shortener_cache_ttl} - "); - }; - } - ## - ## PostgreSQL - ## - elsif (lc $conf->{url_shortener_cache_type} eq 'dbi' - && $conf->{url_shortener_cache_dsn} =~ /^dbi:Pg/i) - { - eval { - local $SIG{'__DIE__'}; - require DBI; - $self->{dbh} = DBI->connect_cached( - $conf->{url_shortener_cache_dsn}, - $conf->{url_shortener_cache_username}, - $conf->{url_shortener_cache_password}, - {RaiseError => 1, PrintError => 0, InactiveDestroy => 1, AutoCommit => 1} - ); - $self->{sth_insert} = $self->{dbh}->prepare(" - INSERT INTO short_url_cache (short_url, decoded_url, created, modified) - VALUES (?,?,CAST(EXTRACT(epoch FROM NOW()) AS INT),CAST(EXTRACT(epoch FROM NOW()) AS INT)) - ON CONFLICT (short_url) DO UPDATE SET - decoded_url = EXCLUDED.decoded_url, - modified = EXCLUDED.modified, - hits = short_url_cache.hits + 1 - "); - $self->{sth_select} = $self->{dbh}->prepare(" - SELECT decoded_url FROM short_url_cache - WHERE short_url = ? - "); - $self->{sth_delete} = $self->{dbh}->prepare(" - DELETE FROM short_url_cache - WHERE short_url = ? AND created < CAST(EXTRACT(epoch FROM NOW()) AS INT) - $conf->{url_shortener_cache_ttl} - "); - $self->{sth_clean} = $self->{dbh}->prepare(" - DELETE FROM short_url_cache - WHERE created < CAST(EXTRACT(epoch FROM NOW()) AS INT) - $conf->{url_shortener_cache_ttl} - "); - }; - ## - ## ... - ## - } else { - warn "DecodeShortURLs: invalid cache configuration\n"; - return; - } - - if ($@ || !$self->{sth_clean}) { - warn "DecodeShortURLs: cache connect failed: $@\n"; - undef $self->{dbh}; - undef $self->{dbh_pid}; - undef $self->{sth_insert}; - undef $self->{sth_select}; - undef $self->{sth_delete}; - undef $self->{sth_clean}; - } else { - $self->{dbh_pid} = $$; - } -} - -sub short_url { - my ($self, $pms) = @_; - - # Make sure checks are run - $self->_check_short($pms); - - return $pms->{short_url} ? 1 : 0; -} - -sub short_url_redir { - my ($self, $pms) = @_; - - # Make sure checks are run - $self->_check_short($pms); - - return $pms->{short_url_redir} ? 1 : 0; -} - -sub short_url_200 { - my ($self, $pms) = @_; - - # Make sure checks are run - $self->_check_short($pms); - - return $pms->{short_url_200} ? 1 : 0; -} - -sub short_url_404 { - my ($self, $pms) = @_; - - # Make sure checks are run - $self->_check_short($pms); - - return $pms->{short_url_404} ? 1 : 0; -} - -sub short_url_code { - my ($self, $pms, undef, $code) = @_; - - # Make sure checks are run - $self->_check_short($pms); - - return 0 unless defined $code && $code =~ /^\d{3}$/; - return $pms->{"short_url_$code"} ? 1 : 0; -} - -sub short_url_chained { - my ($self, $pms) = @_; - - # Make sure checks are run - $self->_check_short($pms); - - return $pms->{short_url_chained} ? 1 : 0; -} - -sub short_url_maxchain { - my ($self, $pms) = @_; - - # Make sure checks are run - $self->_check_short($pms); - - return $pms->{short_url_maxchain} ? 1 : 0; -} - -sub short_url_loop { - my ($self, $pms) = @_; - - # Make sure checks are run - $self->_check_short($pms); - - return $pms->{short_url_loop} ? 1 : 0; -} - -sub _check_shortener_uri { - my ($uri, $conf) = @_; - - local($1,$2); - return 0 unless $uri =~ m{^ - https?:// # Only http - (?:[^\@/?#]*\@)? # Ignore user:pass@ - ([^/?#:]+) # (Capture hostname) - (?::\d+)? # Possible port - (.*?\w)? # Some path wanted - }ix; - my $host = lc $1; - if(is_fqdn_valid($host)) { - $host = idn_to_ascii($host); - } - my $has_path = defined $2; - my $levels = $host =~ tr/.//; - # No point looking at single level "xxx.yy" without a path - return if $levels == 1 && !$has_path; - - if (exists $conf->{url_shortener}->{$host}) { - return { - 'uri' => $uri, - 'method' => $conf->{url_shortener}->{$host} == 1 ? 'head' : 'get', - 'user_agent' => (defined $conf->{url_shortener}->{user_agent}->{$host}) ? $conf->{url_shortener}->{user_agent}->{$host} : $conf->{url_shortener_user_agent}, - }; - } - # if domain is a 3rd level domain check if there is a url shortener - # on the www domain - elsif($levels == 2 && $host =~ /^www\.([^.]+\.[^.]+)$/i) { - my $domain = $1; - if(($host eq "www.$domain") and exists $conf->{url_shortener}->{$domain}) { - dbg("Found internal www redirection for domain $domain"); - return { - 'uri' => $uri, - 'method' => $conf->{url_shortener}->{$domain} == 1 ? 'head' : 'get', - 'user_agent' => (defined $conf->{url_shortener}->{user_agent}->{$host}) ? $conf->{url_shortener}->{user_agent}->{$host} : $conf->{url_shortener_user_agent}, - }; - } - } - # if domain is a 3rd level domain check if there is a url shortener - # on the 2nd level tld - elsif ($levels == 2 && $host =~ /^(?!www)[^.]+(\.[^.]+\.[^.]+)$/i && - exists $conf->{url_shortener}->{$1}) { - return { - 'uri' => $uri, - 'method' => $conf->{url_shortener}->{$1} == 1 ? 'head' : 'get', - 'user_agent' => (defined $conf->{url_shortener}->{user_agent}->{$host}) ? $conf->{url_shortener}->{user_agent}->{$host} : $conf->{url_shortener_user_agent}, - }; - } - return; -} - -sub check_dnsbl { - my ($self, $opts) = @_; - - $self->_check_short($opts->{permsgstatus}); -} - -sub _check_short { - my ($self, $pms) = @_; - - return if $pms->{short_url_checked}++; - my $conf = $pms->{conf}; - - # Sort short URLs into hash to de-dup them - my %short_urls; - my $uris = $pms->get_uri_detail_list(); - foreach my $uri (keys %$uris) { - my $info = $uris->{$uri}; - next unless $info->{domains} && $info->{cleaned}; - # Remove anchors and parameters from shortened uris - $uri =~ s/\/?(?:\#|\?).*//g; - if (my $short_url_info = _check_shortener_uri($uri, $conf)) { - $short_urls{$uri} = $short_url_info; - last if scalar keys %short_urls >= $conf->{max_short_urls}; - } - } - - # Bail out if no shortener was found - return unless %short_urls; - - # Mark that a URL shortener was found - $pms->{short_url} = 1; - - # Bail out if network lookups not enabled or max_short_urls 0 - return if $self->{net_disabled}; - return if !$conf->{max_short_urls}; - - # Initialize cache - $self->initialise_url_shortener_cache($conf); - - # Initialize LWP - my $ua = LWP::UserAgent->new( - 'agent' => $conf->{url_shortener_user_agent}, - 'max_redirect' => 0, - 'timeout' => $conf->{url_shortener_timeout}, + Mail::SpamAssassin::Logger::dbg( + "DecodeShortURLs: this plugin is deprecated, its functionality has been ". + "merged into Redirectors; please switch to ". + "'loadplugin Mail::SpamAssassin::Plugin::Redirectors'" ); - $ua->env_proxy; - - # Launch HTTP requests - foreach my $uri (keys %short_urls) { - $self->recursive_lookup($short_urls{$uri}, $pms, $ua); - } - # Automatically purge old entries - if ($self->{dbh} && $conf->{url_shortener_cache_autoclean} - && rand() < 1/$conf->{url_shortener_cache_autoclean}) - { - dbg("cleaning stale cache entries"); - eval { $self->{sth_clean}->execute(); }; - if ($@) { dbg("cache cleaning failed: $@"); } - } + return $class->SUPER::new($mailsaobject); } -sub recursive_lookup { - my ($self, $short_url_info, $pms, $ua, %been_here) = @_; - my $conf = $pms->{conf}; - - my $count = scalar keys %been_here; - dbg("redirection count $count") if $count; - if ($count >= $conf->{max_short_url_redirections}) { - dbg("found more than $conf->{max_short_url_redirections} shortener redirections"); - # Fire test - $pms->{short_url_maxchain} = 1; - return; - } - - my $short_url = $short_url_info->{uri}; - my $location; - if (defined($location = $self->cache_get($short_url))) { - if ($conf->{url_shortener_loginfo}) { - info("found cached $short_url => $location"); - } else { - dbg("found cached $short_url => $location"); - } - # Cached http code? - if ($location =~ /^\d{3}$/) { - $pms->{"short_url_$location"} = 1; - # Update cache - $self->cache_add($short_url, $location); - return; - } - } else { - # Not cached; do lookup - my $method = $short_url_info->{method}; - my $useragent = $short_url_info->{user_agent}; - if(defined $useragent) { - $ua->agent($useragent); - } else { - $ua->agent($conf->{url_shortener_user_agent}); - } - my $response = $ua->$method($short_url); - if (!$response->is_redirect) { - dbg("URL is not redirect: $short_url = ".$response->status_line); - my $rcode = $response->code; - if ($rcode =~ /^\d{3}$/) { - $pms->{"short_url_$rcode"} = 1; - # Update cache - $self->cache_add($short_url, $rcode); - } - return; - } - $location = $response->headers->{location}; - if ($conf->{url_shortener_loginfo}) { - info("found $short_url => $location"); - } else { - dbg("found $short_url => $location"); - } - } - - # Update cache - $self->cache_add($short_url, $location); - - # Bail out if $short_url redirects to itself - if ($short_url eq $location) { - dbg("URL is redirect to itself"); - return; - } - - # At this point we have a valid redirection and new URL in $response - $pms->{short_url_redir} = 1; - - # Set chained here otherwise we might mark a disabled page or - # redirect back to the same host as chaining incorrectly. - $pms->{short_url_chained} = 1 if $count; - - # Check if it is a redirection to a relative URI - # Make it an absolute URI and chain to it in that case - if ($location !~ m{^[a-z]+://}i) { - my $orig_location = $location; - my $orig_short_url = $short_url; - # Strip to.. - if (index($location, '/') == 0) { - $short_url =~ s{^([a-z]+://.*?)[/?#].*}{$1}; # ..absolute path base is http://example.com - } else { - $short_url =~ s{^([a-z]+://.*/)}{$1}; # ..relative path base is http://example.com/a/b/ - } - $location = "$short_url$location"; - dbg("looks like a redirection to a relative URI: $orig_short_url => $location ($orig_location)"); - } - - if (exists $been_here{$location}) { - # Loop detected - dbg("error: loop detected: $location"); - $pms->{short_url_loop} = 1; - return; - } - $been_here{$location} = 1; - $pms->add_uri_detail_list($location) if !$pms->{uri_detail_list}->{$location}; - - # Check for recursion - if (my $short_url_info = _check_shortener_uri($location, $conf)) { - # Recurse... - $self->recursive_lookup($short_url_info, $pms, $ua, %been_here); - } -} - -sub cache_add { - my ($self, $short_url, $decoded_url) = @_; - - return if !$self->{dbh}; - return if length($short_url) > 256 || length($decoded_url) > 512; - - # Upsert - eval { $self->{sth_insert}->execute($short_url, $decoded_url); }; - if ($@) { - dbg("could not add to cache: $@"); - } - - return; -} - -sub cache_get { - my ($self, $key) = @_; - - return if !$self->{dbh}; - - # Make sure expired entries are gone. Just a quick check for primary key, - # not that expensive. - eval { $self->{sth_delete}->execute($key); }; - if ($@) { - dbg("cache delete failed: $@"); - return; - } - - # Now try to get it (don't bother parsing if something was deleted above, - # it would be rare event anyway) - eval { $self->{sth_select}->execute($key); }; - if ($@) { - dbg("cache get failed: $@"); - return; - } - - my @row = $self->{sth_select}->fetchrow_array(); - if (@row) { - return $row[0]; - } - - return; -} - -# Version features -sub has_short_url { 1 } -sub has_autoclean { 1 } -sub has_short_url_code { 1 } -sub has_user_agent { 1 } # url_shortener_user_agent -sub has_custom_user_agent { 1 } # url_shortener_custom_user_agent -sub has_get { 1 } # url_shortener_get -sub has_clear { 1 } # clear_url_shortener -sub has_timeout { 1 } # url_shortener_timeout -sub has_max_redirections { 1 } # max_short_url_redirections -# short_url() will always hit if matching url_shortener was found, even -# without HTTP requests. To check if a valid HTTP redirection response was -# seen, use short_url_redir(). -sub has_short_url_redir { 1 } - 1; diff --git a/lib/Mail/SpamAssassin/Plugin/Redirectors.pm b/lib/Mail/SpamAssassin/Plugin/Redirectors.pm index 32e9cfcd45..381859e668 100644 --- a/lib/Mail/SpamAssassin/Plugin/Redirectors.pm +++ b/lib/Mail/SpamAssassin/Plugin/Redirectors.pm @@ -5,9 +5,9 @@ # The ASF licenses this file to you under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at: -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,13 +17,14 @@ =head1 NAME -Redirectors - Check for redirected URLs +Redirectors - Check for redirected and shortened URLs =head1 SYNOPSIS loadplugin Mail::SpamAssassin::Plugin::Redirectors url_redirector bing.com + url_shortener tinyurl.com body HAS_REDIR_URL eval:redir_url() describe HAS_REDIR_URL Message has one or more redirected URLs @@ -46,32 +47,73 @@ Redirectors - Check for redirected URLs body REDIR_URL_404 eval:redir_url_code('404') # Can check any non-redirect HTTP code describe REDIR_URL_404 Message has redirected URL returning HTTP 404 -=head1 DESCRIPTION + body HAS_SHORT_URL eval:short_url() + describe HAS_SHORT_URL Message has one or more shortened URLs + + body SHORT_URL_REDIR eval:short_url_redir() + describe SHORT_URL_REDIR Message has shortened URL that resulted in a valid redirection + + body SHORT_URL_CHAINED eval:short_url_chained() + describe SHORT_URL_CHAINED Message has shortened URL chained to other shorteners + + body SHORT_URL_MAXCHAIN eval:short_url_maxchain() + describe SHORT_URL_MAXCHAIN Message has shortened URL that causes too many redirections + + body SHORT_URL_LOOP eval:short_url_loop() + describe SHORT_URL_LOOP Message has short URL that loops back to itself -This plugin looks for URLs redirected by a list of URL redirector services. -Upon finding a matching URL, plugin will send a HTTP request to the -redirector service and retrieve the Location-header which points to the -actual redirected URL. It then adds this URL to the list of URIs extracted -by SpamAssassin which can then be accessed by uri rules and plugins such as -URIDNSBL. + body SHORT_URL_200 eval:short_url_code('200') # Can check any non-redirect HTTP code + describe SHORT_URL_200 Message has shortened URL returning HTTP 200 -This plugin will follow chained redirections, where a redirected URL redirects to -another redirector. Redirection depth limit can be set with -C. + body SHORT_URL_404 eval:short_url_code('404') # Can check any non-redirect HTTP code + describe SHORT_URL_404 Message has shortened URL returning HTTP 404 -Maximum of C redirected URLs are checked in a message (10 by -default). Setting it to 0 disables HTTP requests, allowing only redir_url() -test to work and report found redirectors. + uri URI_TINYURL_BLOCKED m,https://tinyurl\.com/app/nospam, + describe URI_TINYURL_BLOCKED Message contains a tinyurl that has been disabled due to abuse -All supported rule types for checking redirector URLs and redirection status are -documented in L section. + uri URI_BITLY_BLOCKED m,^https://bitly\.com/a/blocked, + describe URI_BITLY_BLOCKED Message contains a bit.ly URL that has been disabled due to abuse + +=head1 DESCRIPTION + +This plugin looks for URLs redirected or shortened by a list of URL +redirector/shortener services. Upon finding a matching URL, plugin will +send a HTTP request to the service and retrieve the Location-header which +points to the actual destination URL. It then adds this URL to the list of +URIs extracted by SpamAssassin which can then be accessed by uri rules and +plugins such as URIDNSBL. + +This plugin will follow chained redirections, where a redirected URL leads +to another redirector, in any combination and order -- for example a +redirector that unwraps into what used to be called a "shortener", or vice +versa. There is no functional difference between a "redirector" and a +"shortener": both are just a domain whose HTTP response redirects +somewhere else, and both are followed by the same code path. +C/C/C +and their C/C/etc. settings are kept +as deprecated aliases of C/C/etc. for +backwards compatibility, and are planned for removal in a future version. +Likewise, C and its sibling eval rules are aliases of +C and friends. Redirection depth is limited by +C, and C redirector URLs are +checked in a message (10 by default); setting it to 0 disables HTTP +requests, allowing only C to work and report found +redirectors. + +All supported rule types for checking redirected URLs and redirection +status are documented in L section. =head1 NOTES -This plugin runs before priority 0 so that it may -modify the parsed URI list prior to normal uri rules or the URIDNSBL plugin. -It should run before DecodeShortURLs plugin so that redirected short uris are also -checked. +This plugin runs before priority 0 so that it may modify the parsed URI +list prior to normal uri rules or the URIDNSBL plugin. + +=head1 ACKNOWLEDGEMENTS + +The url_shortener functionality was originally provided by a separate +DecodeShortURLs plugin; that functionality has been merged into this one, +and C is now a deprecated +compatibility shim that loads this plugin. =cut @@ -85,7 +127,7 @@ use warnings; use vars qw(@ISA); @ISA = qw(Mail::SpamAssassin::Plugin); -my $VERSION = 4.02; +my $VERSION = 4.10; use constant HAS_LWP_USERAGENT => eval { require LWP::UserAgent; require LWP::Protocol::https; }; use constant HAS_SELENIUM => eval { require Selenium::Remote::Driver; }; @@ -111,7 +153,8 @@ sub new { } $self->set_config($mailsaobject->{conf}); - # run at priority -15 so that redirected short uris can also be checked + # run at priority -15 so that redirected/shortened uris are always + # checked in a single pass, regardless of what type of hop starts a chain $self->register_method_priority ('check_dnsbl', -15); $self->register_eval_rule('redir_url', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); $self->register_eval_rule('redir_url_valid', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); @@ -121,6 +164,15 @@ sub new { $self->register_eval_rule('redir_url_chained_domain', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); $self->register_eval_rule('redir_url_maxchain', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); $self->register_eval_rule('redir_url_loop', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); + $self->register_eval_rule('short_url', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); + $self->register_eval_rule('short_url_redir', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); + $self->register_eval_rule('short_url_200', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); + $self->register_eval_rule('short_url_404', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); + $self->register_eval_rule('short_url_code', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); + $self->register_eval_rule('short_url_chained', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); + $self->register_eval_rule('short_url_maxchain', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); + $self->register_eval_rule('short_url_loop', $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS); + $self->register_eval_rule('short_url_tests'); # for legacy DecodeShortURLs compatibility warning return $self; } @@ -170,6 +222,13 @@ Example: The last line follows C but not C. +C (and C, C) are +deprecated aliases of C (and C, +C) kept for backwards compatibility with configs +written for the old DecodeShortURLs plugin, there is no functional +difference between the two names, and the C spelling is +planned for removal in a future version. + =back =cut @@ -178,19 +237,30 @@ sub set_config { my($self, $conf) = @_; my @cmds = (); + # url_shortener is a pure alias of url_redirector, a "shortener" and a + # "redirector" are the same mechanism (an HTTP redirect), the terminology + # difference is not functional. Kept only for config backwards + # compatibility; planned for removal in a future version. + my $url_redirector_code = sub { + my ($self, $key, $value, $line) = @_; + if ($value eq '') { + return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; + } + foreach my $token (split(/\s+/, $value)) { + _add_redirector_entry($self, $token, 'head'); + } + }; push (@cmds, { setting => 'url_redirector', default => {}, type => $Mail::SpamAssassin::Conf::CONF_TYPE_HASH_KEY_VALUE, - code => sub { - my ($self, $key, $value, $line) = @_; - if ($value eq '') { - return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; - } - foreach my $token (split(/\s+/, $value)) { - _add_redirector_entry($self, $token, 'head'); - } - } + code => $url_redirector_code, + }); + push (@cmds, { + setting => 'url_shortener', + default => {}, + type => $Mail::SpamAssassin::Conf::CONF_TYPE_HASH_KEY_VALUE, + code => $url_redirector_code, }); =over 4 @@ -262,7 +332,7 @@ Set Selenium port to use. =item clear_url_redirector [domain[/path]] [domain[/path]...] -Clear configured url_redirector domains, for example to +Clear configured url_redirector/url_shortener domains, for example to override default settings from an update channel. If no arguments are given, all entries are cleared. If domains are specified, only those are removed. @@ -275,19 +345,23 @@ added by a bare-domain configuration. =cut - push (@cmds, { - setting => 'clear_url_redirector', - code => sub { - my ($self, $key, $value, $line) = @_; - if ($value eq '') { - $self->{url_redirector_exact} = {}; - $self->{url_redirector_suffix} = {}; - } else { - foreach my $token (split(/\s+/, $value)) { - _clear_redirector_entry($self, $token); - } + my $clear_url_redirector_code = sub { + my ($self, $key, $value, $line) = @_; + if ($value eq '') { + _clear_all_redirector_entries($self); + } else { + foreach my $token (split(/\s+/, $value)) { + _clear_redirector_entry($self, $token); } } + }; + push (@cmds, { + setting => 'clear_url_redirector', + code => $clear_url_redirector_code, + }); + push (@cmds, { + setting => 'clear_url_shortener', + code => $clear_url_redirector_code, }); =over 4 @@ -304,17 +378,22 @@ restricts the match. =cut + my $url_redirector_get_code = sub { + my ($self, $key, $value, $line) = @_; + if ($value eq '') { + return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; + } + foreach my $token (split(/\s+/, $value)) { + _add_redirector_entry($self, $token, 'get'); + } + }; push (@cmds, { setting => 'url_redirector_get', - code => sub { - my ($self, $key, $value, $line) = @_; - if ($value eq '') { - return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; - } - foreach my $token (split(/\s+/, $value)) { - _add_redirector_entry($self, $token, 'get'); - } - } + code => $url_redirector_get_code, + }); + push (@cmds, { + setting => 'url_shortener_get', + code => $url_redirector_get_code, }); =over 4 @@ -441,6 +520,41 @@ The regexp must match only the redirected domain. }, }); +=over 4 + +=item url_redirector_custom_user_agent domain user-agent (default: none) + +Custom HTTP user-agent to be used for specific domains, +instead of the default specified in C. +Required for some services like t.co to return blocked URL correctly. + +Example: + + url_redirector_custom_user_agent t.co curl/8.6.0 + +=back + +=cut + + my $url_redirector_custom_user_agent_code = sub { + my ($self, $key, $value, $line) = @_; + if ($value eq '') { + return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; + } + my @values = split(/\s+/, $value); + my $domain = shift(@values); + my $ua = join('', @values); + $self->{url_redirector_custom_ua}->{lc $domain} = $ua; + }; + push (@cmds, { + setting => 'url_redirector_custom_user_agent', + code => $url_redirector_custom_user_agent_code, + }); + push (@cmds, { + setting => 'url_shortener_custom_user_agent', + code => $url_redirector_custom_user_agent_code, + }); + =head1 PRIVILEGED SETTINGS =over 4 @@ -554,6 +668,108 @@ See C for database cleaning. type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC }); +=over 4 + +=item url_shortener_cache_type (default: none) + +Deprecated alias of C -- there is only one +cache now, shared by everything this plugin fetches (redirectors and +shorteners alike). + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener_cache_type', + is_priv => 1, + code => sub { + my ($self, $key, $value, $line) = @_; + $self->{url_redirector_cache_type} = $value; + } + }); + +=over 4 + +=item url_shortener_cache_dsn (default: none) + +Deprecated alias of C. + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener_cache_dsn', + is_priv => 1, + code => sub { + my ($self, $key, $value, $line) = @_; + $self->{url_redirector_cache_dsn} = $value; + } + }); + +=over 4 + +=item url_shortener_cache_username (default: none) + +Deprecated alias of C. + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener_cache_username', + is_priv => 1, + code => sub { + my ($self, $key, $value, $line) = @_; + $self->{url_redirector_cache_username} = $value; + } + }); + +=over 4 + +=item url_shortener_cache_password (default: none) + +Deprecated alias of C. + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener_cache_password', + is_priv => 1, + code => sub { + my ($self, $key, $value, $line) = @_; + $self->{url_redirector_cache_password} = $value; + } + }); + +=over 4 + +=item url_shortener_cache_ttl (default: 86400) + +Deprecated alias of C. + +See C for database cleaning. + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener_cache_ttl', + is_admin => 1, + code => sub { + my ($self, $key, $value, $line) = @_; + unless ($value =~ /^\d+$/) { + return $Mail::SpamAssassin::Conf::INVALID_VALUE; + } + $self->{url_redirector_cache_ttl} = $value + 0; + } + }); + =head1 ADMINISTRATOR SETTINGS =over 4 @@ -668,13 +884,154 @@ like a common browser. type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING }); +=over 4 + +=item url_shortener_cache_autoclean (default: 1000) + +Deprecated alias of C. + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener_cache_autoclean', + is_admin => 1, + code => sub { + my ($self, $key, $value, $line) = @_; + unless ($value =~ /^\d+$/) { + return $Mail::SpamAssassin::Conf::INVALID_VALUE; + } + $self->{url_redirector_cache_autoclean} = $value + 0; + } + }); + +=over 4 + +=item url_shortener_loginfo (default: 0 (off)) + +Deprecated alias of C. + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener_loginfo', + is_admin => 1, + code => sub { + my ($self, $key, $value, $line) = @_; + unless (defined $value && $value !~ /^$/) { + return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE; + } + # bug 4462: allow yes/1 and no/0 for boolean values + my $lc = lc $value; + if ($lc eq 'yes' || $lc eq '1') { + $self->{url_redirector_loginfo} = 1; + } elsif ($lc eq 'no' || $lc eq '0') { + $self->{url_redirector_loginfo} = 0; + } else { + return $Mail::SpamAssassin::Conf::INVALID_VALUE; + } + } + }); + +=over 4 + +=item url_shortener_timeout (default: 5) + +Deprecated alias of C. + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener_timeout', + is_admin => 1, + code => sub { + my ($self, $key, $value, $line) = @_; + unless ($value =~ /^\d+$/) { + return $Mail::SpamAssassin::Conf::INVALID_VALUE; + } + $self->{url_redirector_timeout} = $value + 0; + } + }); + +=over 4 + +=item max_short_urls (default: 10) + +Deprecated alias of C -- there is only one budget now, +shared by everything this plugin fetches (redirectors and shorteners +alike). + +=back + +=cut + + push (@cmds, { + setting => 'max_short_urls', + is_admin => 1, + code => sub { + my ($self, $key, $value, $line) = @_; + unless ($value =~ /^\d+$/) { + return $Mail::SpamAssassin::Conf::INVALID_VALUE; + } + $self->{max_redir_urls} = $value + 0; + } + }); + +=over 4 + +=item max_short_url_redirections (default: 10) + +Deprecated alias of C. + +=back + +=cut + + push (@cmds, { + setting => 'max_short_url_redirections', + is_admin => 1, + code => sub { + my ($self, $key, $value, $line) = @_; + unless ($value =~ /^\d+$/) { + return $Mail::SpamAssassin::Conf::INVALID_VALUE; + } + $self->{max_redir_url_redirections} = $value + 0; + } + }); + +=over 4 + +=item url_shortener_user_agent (default: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36) + +Deprecated alias of C. Per-domain overrides use +C (C is +itself a deprecated alias of that). + +=back + +=cut + + push (@cmds, { + setting => 'url_shortener_user_agent', + is_admin => 1, + code => sub { + my ($self, $key, $value, $line) = @_; + $self->{url_redirector_user_agent} = $value; + } + }); + $conf->{parser}->register_commands(\@cmds); } sub initialise_url_redirector_cache { my ($self, $conf) = @_; - return if $self->{dbh} && $self->{dbh_pid} && $self->{dbh_pid} == $$; + return if $self->{dbh_redir} && $self->{dbh_redir_pid} && $self->{dbh_redir_pid} == $$; return if !$conf->{url_redirector_cache_type}; if (!$conf->{url_redirector_cache_dsn}) { @@ -684,7 +1041,7 @@ sub initialise_url_redirector_cache { ## ## SQLite - ## + ## if ($conf->{url_redirector_cache_type} =~ /^(?:dbi|sqlite)$/i && $conf->{url_redirector_cache_dsn} =~ /^dbi:SQLite/) { @@ -693,11 +1050,11 @@ sub initialise_url_redirector_cache { require DBI; require DBD::SQLite; DBD::SQLite->VERSION(1.59_01); # Required for ON CONFLICT - $self->{dbh} = DBI->connect_cached( + $self->{dbh_redir} = DBI->connect_cached( $conf->{url_redirector_cache_dsn}, '', '', {RaiseError => 1, PrintError => 0, InactiveDestroy => 1, AutoCommit => 1} ); - $self->{dbh}->do(" + $self->{dbh_redir}->do(" CREATE TABLE IF NOT EXISTS redir_url_cache ( redir_url TEXT PRIMARY KEY NOT NULL, target_url TEXT NOT NULL, @@ -706,12 +1063,7 @@ sub initialise_url_redirector_cache { modified INTEGER NOT NULL ) "); - # Maintaining index for cleaning is likely more expensive than occasional full table scan - #$self->{dbh}->do(" - # CREATE INDEX IF NOT EXISTS redir_url_modified - # ON redir_url_cache(created) - #"); - $self->{sth_insert} = $self->{dbh}->prepare(" + $self->{sth_insert_redir} = $self->{dbh_redir}->prepare(" INSERT INTO redir_url_cache (redir_url, target_url, created, modified) VALUES (?,?,strftime('%s','now'),strftime('%s','now')) ON CONFLICT(redir_url) DO UPDATE @@ -719,15 +1071,15 @@ sub initialise_url_redirector_cache { modified = excluded.modified, hits = hits + 1 "); - $self->{sth_select} = $self->{dbh}->prepare(" + $self->{sth_select_redir} = $self->{dbh_redir}->prepare(" SELECT target_url FROM redir_url_cache WHERE redir_url = ? "); - $self->{sth_delete} = $self->{dbh}->prepare(" + $self->{sth_delete_redir} = $self->{dbh_redir}->prepare(" DELETE FROM redir_url_cache WHERE redir_url = ? AND created < strftime('%s','now') - $conf->{url_redirector_cache_ttl} "); - $self->{sth_clean} = $self->{dbh}->prepare(" + $self->{sth_clean_redir} = $self->{dbh_redir}->prepare(" DELETE FROM redir_url_cache WHERE created < strftime('%s','now') - $conf->{url_redirector_cache_ttl} "); @@ -735,20 +1087,20 @@ sub initialise_url_redirector_cache { } ## ## MySQL/MariaDB - ## + ## elsif (lc $conf->{url_redirector_cache_type} eq 'dbi' && $conf->{url_redirector_cache_dsn} =~ /^dbi:(?:mysql|MariaDB)/i) { eval { local $SIG{'__DIE__'}; require DBI; - $self->{dbh} = DBI->connect_cached( + $self->{dbh_redir} = DBI->connect_cached( $conf->{url_redirector_cache_dsn}, $conf->{url_redirector_cache_username}, $conf->{url_redirector_cache_password}, {RaiseError => 1, PrintError => 0, InactiveDestroy => 1, AutoCommit => 1} ); - $self->{sth_insert} = $self->{dbh}->prepare(" + $self->{sth_insert_redir} = $self->{dbh_redir}->prepare(" INSERT INTO redir_url_cache (redir_url, target_url, created, modified) VALUES (?,?,UNIX_TIMESTAMP(),UNIX_TIMESTAMP()) ON DUPLICATE KEY UPDATE @@ -756,15 +1108,15 @@ sub initialise_url_redirector_cache { modified = VALUES(modified), hits = hits + 1 "); - $self->{sth_select} = $self->{dbh}->prepare(" + $self->{sth_select_redir} = $self->{dbh_redir}->prepare(" SELECT target_url FROM redir_url_cache WHERE redir_url = ? "); - $self->{sth_delete} = $self->{dbh}->prepare(" + $self->{sth_delete_redir} = $self->{dbh_redir}->prepare(" DELETE FROM redir_url_cache WHERE redir_url = ? AND created < UNIX_TIMESTAMP() - $conf->{url_redirector_cache_ttl} "); - $self->{sth_clean} = $self->{dbh}->prepare(" + $self->{sth_clean_redir} = $self->{dbh_redir}->prepare(" DELETE FROM redir_url_cache WHERE created < UNIX_TIMESTAMP() - $conf->{url_redirector_cache_ttl} "); @@ -772,20 +1124,20 @@ sub initialise_url_redirector_cache { } ## ## PostgreSQL - ## + ## elsif (lc $conf->{url_redirector_cache_type} eq 'dbi' && $conf->{url_redirector_cache_dsn} =~ /^dbi:Pg/i) { eval { local $SIG{'__DIE__'}; require DBI; - $self->{dbh} = DBI->connect_cached( + $self->{dbh_redir} = DBI->connect_cached( $conf->{url_redirector_cache_dsn}, $conf->{url_redirector_cache_username}, $conf->{url_redirector_cache_password}, {RaiseError => 1, PrintError => 0, InactiveDestroy => 1, AutoCommit => 1} ); - $self->{sth_insert} = $self->{dbh}->prepare(" + $self->{sth_insert_redir} = $self->{dbh_redir}->prepare(" INSERT INTO redir_url_cache (redir_url, target_url, created, modified) VALUES (?,?,CAST(EXTRACT(epoch FROM NOW()) AS INT),CAST(EXTRACT(epoch FROM NOW()) AS INT)) ON CONFLICT (redir_url) DO UPDATE SET @@ -793,15 +1145,15 @@ sub initialise_url_redirector_cache { modified = EXCLUDED.modified, hits = redir_url_cache.hits + 1 "); - $self->{sth_select} = $self->{dbh}->prepare(" + $self->{sth_select_redir} = $self->{dbh_redir}->prepare(" SELECT target_url FROM redir_url_cache WHERE redir_url = ? "); - $self->{sth_delete} = $self->{dbh}->prepare(" + $self->{sth_delete_redir} = $self->{dbh_redir}->prepare(" DELETE FROM redir_url_cache WHERE redir_url = ? AND created < CAST(EXTRACT(epoch FROM NOW()) AS INT) - $conf->{url_redirector_cache_ttl} "); - $self->{sth_clean} = $self->{dbh}->prepare(" + $self->{sth_clean_redir} = $self->{dbh_redir}->prepare(" DELETE FROM redir_url_cache WHERE created < CAST(EXTRACT(epoch FROM NOW()) AS INT) - $conf->{url_redirector_cache_ttl} "); @@ -814,23 +1166,22 @@ sub initialise_url_redirector_cache { return; } - if ($@ || !$self->{sth_clean}) { + if ($@ || !$self->{sth_clean_redir}) { warn "Redirectors: cache connect failed: $@\n"; - undef $self->{dbh}; - undef $self->{dbh_pid}; - undef $self->{sth_insert}; - undef $self->{sth_select}; - undef $self->{sth_delete}; - undef $self->{sth_clean}; + undef $self->{dbh_redir}; + undef $self->{dbh_redir_pid}; + undef $self->{sth_insert_redir}; + undef $self->{sth_select_redir}; + undef $self->{sth_delete_redir}; + undef $self->{sth_clean_redir}; } else { - $self->{dbh_pid} = $$; + $self->{dbh_redir_pid} = $$; } } sub redir_url { my ($self, $pms) = @_; - # Make sure checks are run $self->_check_redir($pms); return $pms->{redir_url} ? 1 : 0; @@ -839,7 +1190,6 @@ sub redir_url { sub redir_url_valid { my ($self, $pms) = @_; - # Make sure checks are run $self->_check_redir($pms); return $pms->{redir_url_valid} ? 1 : 0; @@ -848,7 +1198,6 @@ sub redir_url_valid { sub redir_url_404 { my ($self, $pms) = @_; - # Make sure checks are run $self->_check_redir($pms); return $pms->{redir_url_404} ? 1 : 0; @@ -857,7 +1206,6 @@ sub redir_url_404 { sub redir_url_code { my ($self, $pms, undef, $code) = @_; - # Make sure checks are run $self->_check_redir($pms); return 0 unless defined $code && $code =~ /^\d{3}$/; @@ -867,7 +1215,6 @@ sub redir_url_code { sub redir_url_chained { my ($self, $pms) = @_; - # Make sure checks are run $self->_check_redir($pms); return $pms->{redir_url_chained} ? 1 : 0; @@ -876,7 +1223,6 @@ sub redir_url_chained { sub redir_url_chained_domain { my ($self, $pms) = @_; - # Make sure checks are run $self->_check_redir($pms); return $pms->{redir_url_chained_domain} ? 1 : 0; @@ -885,7 +1231,6 @@ sub redir_url_chained_domain { sub redir_url_maxchain { my ($self, $pms) = @_; - # Make sure checks are run $self->_check_redir($pms); return $pms->{redir_url_maxchain} ? 1 : 0; @@ -894,12 +1239,49 @@ sub redir_url_maxchain { sub redir_url_loop { my ($self, $pms) = @_; - # Make sure checks are run $self->_check_redir($pms); return $pms->{redir_url_loop} ? 1 : 0; } +# short_url() and friends are deprecated aliases of redir_url() and +# friends, a "shortener" and a "redirector" are the same mechanism, so +# there is nothing left for a separate short_url_* implementation to do. +# short_url_redir/short_url_200 are thin wrappers rather than glob aliases +# since their names don't line up 1:1 with a redir_url_* counterpart. +*short_url = \&redir_url; +*short_url_chained = \&redir_url_chained; +*short_url_maxchain = \&redir_url_maxchain; +*short_url_loop = \&redir_url_loop; +*short_url_404 = \&redir_url_404; +*short_url_code = \&redir_url_code; + +sub short_url_redir { + my ($self, $pms) = @_; + return $self->redir_url_valid($pms); +} + +sub short_url_200 { + my ($self, $pms) = @_; + return $self->redir_url_code($pms, undef, '200'); +} + +sub short_url_tests { + # Legacy DecodeShortURLs compatibility warning done in finish_parsing_start + return 0; +} + +sub finish_parsing_start { + my ($self, $opts) = @_; + + if ($opts->{conf}->{eval_to_rule}->{short_url_tests}) { + warn "Redirectors: Legacy configuration format detected. ". + "Eval function short_url_tests() is no longer supported, ". + "please see documentation for the new rule format.\n"; + } +} + +# Add a host[/path] entry to the shared exact/suffix lookup buckets. sub _add_redirector_entry { my ($conf, $token, $method) = @_; @@ -931,15 +1313,24 @@ sub _clear_redirector_entry { my $bucket = ($domspec =~ s/^\.//) ? 'url_redirector_suffix' : 'url_redirector_exact'; return unless length $domspec; + my $entry = $conf->{$bucket}->{$domspec} or return; + if (!$has_path) { delete $conf->{$bucket}->{$domspec}; return; } - my $entry = $conf->{$bucket}->{$domspec} or return; @{$entry->{paths}} = grep { $_ ne $path } @{$entry->{paths}}; delete $conf->{$bucket}->{$domspec} unless @{$entry->{paths}}; } +# Remove every entry from both lookup buckets. Used by +# clear_url_redirector/clear_url_shortener with no arguments. +sub _clear_all_redirector_entries { + my ($conf) = @_; + $conf->{url_redirector_exact} = {}; + $conf->{url_redirector_suffix} = {}; +} + sub _entry_match_path { my ($entry, $path) = @_; for my $p (@{$entry->{paths}}) { @@ -1016,8 +1407,9 @@ sub _parse_uri { return ($uri, $host, $path, $rest); } -# Returns the redirector entry ({method, paths}) if $uri's host+path matches -# a configured url_redirector / url_redirector_get, else undef. +# Returns the redirector entry ({method, paths}) if $uri's host+path +# matches a configured url_redirector / url_redirector_get / url_shortener*, +# else undef. sub _is_configured_redirector { my ($uri, $conf) = @_; @@ -1063,6 +1455,8 @@ sub _extract_embedded_uri { return; } +# Lazily build (and cache on $pms) the single LWP::UserAgent used for +# every fetch this plugin makes. sub _get_lwp_ua { my ($self, $pms) = @_; return $pms->{redir_lwp_ua} if exists $pms->{redir_lwp_ua}; @@ -1207,6 +1601,9 @@ sub _do_http { } } else { my $ua = $self->_get_lwp_ua($pms); + my (undef, $host) = _parse_uri($redir_url, $conf); + my $custom_ua = defined $host ? $conf->{url_redirector_custom_ua}->{$host} : undef; + $ua->agent(defined $custom_ua ? $custom_ua : $conf->{url_redirector_user_agent}); my $response = $ua->$method($redir_url); return if not defined $response; @@ -1299,6 +1696,11 @@ sub _do_http { # Recursive chain walker. Stops cleanly when neither # _is_configured_redirector nor _extract_embedded_uri matches. HTTP # requests are gated on _is_configured_redirector returning truthy. +# +# There is no functional distinction between what used to be called a +# "redirector" and a "shortener" -- both are just a configured host whose +# response redirects elsewhere, and a hop of either origin is checked at +# every depth, so a chain can freely mix them in any order. sub _walk_redirects { my ($self, $uri, $src_info, $pms, $depth, $been_here) = @_; my $conf = $pms->{conf}; @@ -1309,7 +1711,7 @@ sub _walk_redirects { return; } if ($depth >= $conf->{max_redir_url_redirections}) { - dbg("found more than $conf->{max_redir_url_redirections} redirections"); + dbg("found more than $conf->{max_redir_url_redirections} chained redirections"); $pms->{redir_url_maxchain} = 1; return; } @@ -1343,16 +1745,18 @@ sub _walk_redirects { return if $self->{net_disabled}; return if !$conf->{max_redir_urls}; - - # Seed cap: max_redir_urls counts initial (depth 0) redirector URIs. if ($depth == 0) { return if ++$pms->{redir_seed_count} > $conf->{max_redir_urls}; } - my $location = $self->_do_http($uri, $rentry->{method}, $pms); + # Strip the fragment before fetching, RFC 3986 defines it as + # client-side-only. + (my $fetch_uri = $uri) =~ s/#.*//; + + my $location = $self->_do_http($fetch_uri, $rentry->{method}, $pms); return unless defined $location; - if ($uri eq $location) { + if ($fetch_uri eq $location) { dbg("URL redirects to itself"); $pms->{redir_url_loop} = 1; return; @@ -1392,7 +1796,7 @@ sub _check_redir { $self->initialise_url_redirector_cache($conf); - # UAs are built lazily inside _do_http and cached on $pms. No upfront + # The UA is built lazily inside _do_http and cached on $pms. No upfront # construction here -- a message with only embedded-URI matches and no # HTTP-eligible URIs will not create a UA at all. my $uris = $pms->get_uri_detail_list(); @@ -1403,11 +1807,11 @@ sub _check_redir { $self->_walk_redirects($uri, $info, $pms, 0, {}); } - if ($self->{dbh} && $conf->{url_redirector_cache_autoclean} + if ($self->{dbh_redir} && $conf->{url_redirector_cache_autoclean} && rand() < 1/$conf->{url_redirector_cache_autoclean}) { dbg("cleaning stale cache entries"); - eval { $self->{sth_clean}->execute(); }; + eval { $self->{sth_clean_redir}->execute(); }; if ($@) { dbg("cache cleaning failed: $@"); } } } @@ -1433,13 +1837,13 @@ sub _add_redirect_uri { } sub cache_add { - my ($self, $redir_url, $target_url) = @_; + my ($self, $key, $value) = @_; - return if !$self->{dbh}; - return if length($redir_url) > 256 || length($target_url) > 512; + return if !$self->{dbh_redir}; + return if length($key) > 256 || length($value) > 512; # Upsert - eval { $self->{sth_insert}->execute($redir_url, $target_url); }; + eval { $self->{sth_insert_redir}->execute($key, $value); }; if ($@) { dbg("could not add to cache: $@"); } @@ -1450,11 +1854,11 @@ sub cache_add { sub cache_get { my ($self, $key) = @_; - return if !$self->{dbh}; + return if !$self->{dbh_redir}; # Make sure expired entries are gone. Just a quick check for primary key, # not that expensive. - eval { $self->{sth_delete}->execute($key); }; + eval { $self->{sth_delete_redir}->execute($key); }; if ($@) { dbg("cache delete failed: $@"); return; @@ -1462,13 +1866,13 @@ sub cache_get { # Now try to get it (don't bother parsing if something was deleted above, # it would be rare event anyway) - eval { $self->{sth_select}->execute($key); }; + eval { $self->{sth_select_redir}->execute($key); }; if ($@) { dbg("cache get failed: $@"); return; } - my @row = $self->{sth_select}->fetchrow_array(); + my @row = $self->{sth_select_redir}->fetchrow_array(); if (@row) { return $row[0]; } @@ -1488,5 +1892,18 @@ sub has_selenium_support { 1 } sub has_url_redirector_selenium { 1 } sub has_url_skip_redirect_to { 1 } sub has_url_redirector_path { 1 } # path-prefix syntax in url_redirector / url_redirector_get +sub has_short_url { 1 } +sub has_autoclean { 1 } +sub has_short_url_code { 1 } +sub has_user_agent { 1 } # url_shortener_user_agent +sub has_custom_user_agent { 1 } # url_shortener_custom_user_agent +sub has_get { 1 } # url_shortener_get +sub has_clear { 1 } # clear_url_shortener +sub has_timeout { 1 } # url_shortener_timeout +sub has_max_redirections { 1 } # max_short_url_redirections +# short_url() will always hit if matching url_shortener was found, even +# without HTTP requests. To check if a valid HTTP redirection response was +# seen, use short_url_redir(). +sub has_short_url_redir { 1 } 1; diff --git a/t/decodeshorturl.t b/t/decodeshorturl.t index 1e92385dc9..25fa8dfad7 100755 --- a/t/decodeshorturl.t +++ b/t/decodeshorturl.t @@ -77,10 +77,18 @@ ok_all_patterns(); 'https://tinyurl.com/jf8wv76t => https://spamassassin.apache.org/' ); -sarun ("-D DecodeShortURLs -t < data/spam/decodeshorturl/anchor.eml 2>&1", \&patterns_run_cb); +# DecodeShortURLs is now a thin subclass of Redirectors; all logging (and +# thus the -D facility name) happens under "Redirectors". +sarun ("-D Redirectors -t < data/spam/decodeshorturl/anchor.eml 2>&1", \&patterns_run_cb); ok_all_patterns(); -sarun ("-D DecodeShortURLs -t < data/spam/decodeshorturl/params.eml 2>&1", \&patterns_run_cb); +# Query strings are not stripped before fetching a shortener (only +# fragments are, since RFC 3986 defines them as client-side-only). +%patterns = ( + 'https://tinyurl.com/jf8wv76t?p=1&q=2 => https://spamassassin.apache.org/?p=1&q=2' +); + +sarun ("-D Redirectors -t < data/spam/decodeshorturl/params.eml 2>&1", \&patterns_run_cb); ok_all_patterns(); ### @@ -109,8 +117,11 @@ describe HAS_SHORT_URL Message contains one or more shortened URLs sarun ("-t < data/spam/decodeshorturl/base.eml", \&patterns_run_cb); ok_all_patterns(); +# url_shortener_cache_* is a deprecated alias of url_redirector_cache_*; +# there is only one cache table now (redir_url_cache), shared by +# redirectors and shorteners alike. my $dbh = DBI->connect("dbi:SQLite:dbname=$workdir/DecodeShortURLs.db","",""); -my @row = $dbh->selectrow_array("SELECT decoded_url FROM short_url_cache WHERE short_url = 'http://bit.ly/30yH6WK'"); +my @row = $dbh->selectrow_array("SELECT target_url FROM redir_url_cache WHERE redir_url = 'http://bit.ly/30yH6WK'"); is($row[0], 'http://spamassassin.apache.org/'); # Check another email to cleanup old entries from database @@ -118,7 +129,7 @@ sarun ("-t < data/spam/decodeshorturl/base2.eml", \&patterns_run_cb); ok_all_patterns(); $dbh = DBI->connect("dbi:SQLite:dbname=$workdir/DecodeShortURLs.db","",""); -@row = $dbh->selectrow_array("SELECT decoded_url FROM short_url_cache WHERE short_url = 'http://bit.ly/30yH6WK'"); +@row = $dbh->selectrow_array("SELECT target_url FROM redir_url_cache WHERE redir_url = 'http://bit.ly/30yH6WK'"); isnt($row[0], 'https://spamassassin.apache.org/'); }