diff --git a/packages/spacecat-shared-utils/src/bot-blocker-detect/bot-blocker-detect.js b/packages/spacecat-shared-utils/src/bot-blocker-detect/bot-blocker-detect.js index 80444271d..66ea3bbfb 100644 --- a/packages/spacecat-shared-utils/src/bot-blocker-detect/bot-blocker-detect.js +++ b/packages/spacecat-shared-utils/src/bot-blocker-detect/bot-blocker-detect.js @@ -30,7 +30,85 @@ const CONFIDENCE_MEDIUM = 0.95; const CONFIDENCE_ABSOLUTE = 1.0; const DEFAULT_TIMEOUT = 5000; -function analyzeResponse(response) { +/** + * SpaceCat bot identification constants + */ +export const SPACECAT_BOT_USER_AGENT = 'Spacecat/1.0'; + +/** + * Gets SpaceCat bot IPs from environment variable + * @param {string} ipsString - Comma-separated IPs (from env/secrets) - REQUIRED + * @returns {Array} Array of IP addresses + * @throws {Error} If ipsString is not provided + */ +export function getSpacecatBotIps(ipsString) { + if (!ipsString) { + throw new Error('SPACECAT_BOT_IPS environment variable is required but not set'); + } + + return ipsString.split(',').map((ip) => ip.trim()).filter((ip) => ip); +} + +/** + * Formats allowlist message with current bot IPs + * @param {string} botIps - Comma-separated IPs from secrets - REQUIRED + * @returns {object} Formatted message with IPs and user-agent + * @throws {Error} If botIps is not provided + */ +export function formatAllowlistMessage(botIps) { + const ips = getSpacecatBotIps(botIps); + + return { + title: 'To allowlist SpaceCat bot:', + ips, + userAgent: SPACECAT_BOT_USER_AGENT, + }; +} + +/** + * HTML patterns for detecting challenge pages + */ +const CHALLENGE_PATTERNS = { + cloudflare: [ + /Checking your browser/i, + /Just a moment\.\.\./i, + /Verifying you are human/i, + /Please wait.*CloudFlare/i, + /cf-turnstile/i, + /challenge-platform/i, + /cf-chl-widget/i, // Cloudflare challenge widget + /ray\s*id.*cloudflare/i, // Cloudflare Ray ID in error pages + /__cf_chl_tk/i, // Cloudflare challenge token + /cloudflare.*security/i, + /attention required.*cloudflare/i, + ], + imperva: [ + /_Incapsula_Resource/i, + /Incapsula incident ID/i, + /incap_ses/i, // Imperva session cookie + /visid_incap/i, // Imperva visitor ID + ], + akamai: [ + /Access Denied.*Akamai/i, + /Reference.*Akamai/i, + ], + general: [ + /captcha/i, + /human verification/i, + /recaptcha/i, + /hcaptcha/i, + /datadome/i, + /dd-request-id/i, + ], +}; + +/** + * Analyzes response for bot protection indicators + * @param {Object} response - Response object with status and headers + * @param {string} [html] - Optional HTML content for deeper analysis + * @returns {Object} Detection result + */ +function analyzeResponse(response, html = null) { const { status, headers } = response; // Check for CDN/blocker infrastructure presence (lazy evaluation for performance) @@ -45,6 +123,12 @@ function analyzeResponse(response) { || headers.get('x-amz-cf-pop') || headers.get('via')?.includes('CloudFront'); + // Check HTML content for challenge page patterns (if HTML provided) + const htmlHasChallenge = (patterns) => { + if (!html) return false; + return patterns.some((pattern) => pattern.test(html)); + }; + // Active blocking (403 status with known blocker) if (status === 403 && hasCloudflare()) { return { @@ -88,6 +172,16 @@ function analyzeResponse(response) { // Success with known infrastructure present (infrastructure detected but allowing requests) if (status === 200 && hasCloudflare()) { + // Check if HTML contains challenge page (even though status is 200) + if (htmlHasChallenge(CHALLENGE_PATTERNS.cloudflare)) { + return { + crawlable: false, + type: 'cloudflare', + confidence: CONFIDENCE_HIGH, + reason: 'Challenge page detected despite 200 status', + }; + } + return { crawlable: true, type: 'cloudflare-allowed', @@ -96,6 +190,14 @@ function analyzeResponse(response) { } if (status === 200 && hasImperva()) { + if (htmlHasChallenge(CHALLENGE_PATTERNS.imperva)) { + return { + crawlable: false, + type: 'imperva', + confidence: CONFIDENCE_HIGH, + reason: 'Challenge page detected despite 200 status', + }; + } return { crawlable: true, type: 'imperva-allowed', @@ -104,6 +206,14 @@ function analyzeResponse(response) { } if (status === 200 && hasAkamai()) { + if (htmlHasChallenge(CHALLENGE_PATTERNS.akamai)) { + return { + crawlable: false, + type: 'akamai', + confidence: CONFIDENCE_HIGH, + reason: 'Challenge page detected despite 200 status', + }; + } return { crawlable: true, type: 'akamai-allowed', @@ -129,6 +239,15 @@ function analyzeResponse(response) { // Success with no known infrastructure if (status === 200) { + // Still check for generic challenge patterns + if (htmlHasChallenge(CHALLENGE_PATTERNS.general)) { + return { + crawlable: false, + type: 'unknown', + confidence: 0.7, + reason: 'Generic challenge patterns detected', + }; + } return { crawlable: true, type: 'none', @@ -136,6 +255,56 @@ function analyzeResponse(response) { }; } + // Generic 403 Forbidden - definitely blocked (unknown CDN/protection) + if (status === 403) { + return { + crawlable: false, + type: 'unknown', + confidence: 0.7, + reason: 'HTTP 403 Forbidden - access denied', + }; + } + + // Rate limiting + if (status === 429) { + return { + crawlable: false, + type: 'rate-limit', + confidence: CONFIDENCE_HIGH, + reason: 'HTTP 429 Too Many Requests - rate limit exceeded', + }; + } + + // 401 Unauthorized + if (status === 401) { + return { + crawlable: false, + type: 'auth-required', + confidence: CONFIDENCE_HIGH, + reason: 'HTTP 401 Unauthorized - authentication required', + }; + } + + // 406 Not Acceptable (often user-agent rejection) + if (status === 406) { + return { + crawlable: false, + type: 'user-agent-rejected', + confidence: 0.8, + reason: 'HTTP 406 Not Acceptable - likely user-agent rejection', + }; + } + + // Other 4xx client errors + if (status >= 400 && status < 500) { + return { + crawlable: false, + type: 'http-error', + confidence: 0.6, + reason: `HTTP ${status} - client error`, + }; + } + // Unknown status without known blocker signature return { crawlable: true, @@ -207,3 +376,27 @@ export async function detectBotBlocker({ baseUrl, timeout = DEFAULT_TIMEOUT }) { return analyzeError(error); } } + +/** + * Analyzes already-fetched response data for bot protection. + * Used by content scraper to analyze Puppeteer results without making another request. + * + * @param {Object} data - Response data to analyze + * @param {number} data.status - HTTP status code + * @param {Object} data.headers - Response headers (plain object or Headers object) + * @param {string} [data.html] - Optional HTML content for challenge page detection + * @returns {Object} Detection result (same format as detectBotBlocker) + */ +export function analyzeBotProtection({ status, headers, html }) { + // Convert headers to Headers object if plain object + const headersObj = headers instanceof Headers + ? headers + : new Headers(Object.entries(headers || {})); + + const mockResponse = { + status, + headers: headersObj, + }; + + return analyzeResponse(mockResponse, html); +} diff --git a/packages/spacecat-shared-utils/src/index.js b/packages/spacecat-shared-utils/src/index.js index 4ce594db2..eb03ffb0f 100644 --- a/packages/spacecat-shared-utils/src/index.js +++ b/packages/spacecat-shared-utils/src/index.js @@ -110,7 +110,13 @@ export * as llmoConfig from './llmo-config.js'; export * as schemas from './schemas.js'; export { detectLocale } from './locale-detect/locale-detect.js'; -export { detectBotBlocker } from './bot-blocker-detect/bot-blocker-detect.js'; +export { + detectBotBlocker, + analyzeBotProtection, + SPACECAT_BOT_USER_AGENT, + getSpacecatBotIps, + formatAllowlistMessage, +} from './bot-blocker-detect/bot-blocker-detect.js'; export { prettifyLogForwardingConfig } from './cdn-helpers.js'; export { diff --git a/packages/spacecat-shared-utils/test/bot-blocker-detect/bot-blocker-detect.test.js b/packages/spacecat-shared-utils/test/bot-blocker-detect/bot-blocker-detect.test.js index 5979121b0..573bfd164 100644 --- a/packages/spacecat-shared-utils/test/bot-blocker-detect/bot-blocker-detect.test.js +++ b/packages/spacecat-shared-utils/test/bot-blocker-detect/bot-blocker-detect.test.js @@ -15,7 +15,13 @@ import { expect } from 'chai'; import nock from 'nock'; -import { detectBotBlocker } from '../../src/bot-blocker-detect/bot-blocker-detect.js'; +import { + detectBotBlocker, + analyzeBotProtection, + getSpacecatBotIps, + formatAllowlistMessage, + SPACECAT_BOT_USER_AGENT, +} from '../../src/bot-blocker-detect/bot-blocker-detect.js'; describe('Bot Blocker Detection', () => { const baseUrl = 'https://www.example.com'; @@ -118,18 +124,6 @@ describe('Bot Blocker Detection', () => { expect(result.confidence).to.equal(1.0); }); - it('returns unknown for unrecognized status codes', async () => { - nock(baseUrl) - .head('/') - .reply(500); - - const result = await detectBotBlocker({ baseUrl }); - - expect(result.crawlable).to.be.true; - expect(result.type).to.equal('unknown'); - expect(result.confidence).to.equal(0.5); - }); - it('returns unknown for unrecognized errors', async () => { const error = new Error('Connection timeout'); error.code = 'ETIMEDOUT'; @@ -145,7 +139,7 @@ describe('Bot Blocker Detection', () => { expect(result.confidence).to.equal(0.3); }); - it('does not detect blocking for 403 without known headers', async () => { + it('detects 403 as blocked even without known CDN headers', async () => { nock(baseUrl) .head('/') .reply(403, '', { @@ -154,9 +148,10 @@ describe('Bot Blocker Detection', () => { const result = await detectBotBlocker({ baseUrl }); - expect(result.crawlable).to.be.true; + expect(result.crawlable).to.be.false; expect(result.type).to.equal('unknown'); - expect(result.confidence).to.equal(0.5); + expect(result.confidence).to.equal(0.7); + expect(result.reason).to.equal('HTTP 403 Forbidden - access denied'); }); // New CDN detection tests @@ -343,4 +338,512 @@ describe('Bot Blocker Detection', () => { expect(result.confidence).to.equal(1.0); }); }); + + describe('analyzeBotProtection', () => { + it('detects Cloudflare challenge page with 200 status', () => { + const html = 'Just a moment...Checking your browser before accessing example.com'; + const headers = { 'cf-ray': '123456789-CDG', server: 'cloudflare' }; + + const result = analyzeBotProtection({ + status: 200, + headers, + html, + }); + + expect(result.crawlable).to.be.false; + expect(result.type).to.equal('cloudflare'); + expect(result.confidence).to.equal(0.99); + expect(result.reason).to.equal('Challenge page detected despite 200 status'); + }); + + it('detects Cloudflare challenge page with "Verifying you are human"', () => { + const html = '

zepbound.lilly.com

Verifying you are human. This may take a few seconds.

'; + const headers = { 'cf-ray': '9a211d4cca225831' }; + + const result = analyzeBotProtection({ + status: 200, + headers, + html, + }); + + expect(result.crawlable).to.be.false; + expect(result.type).to.equal('cloudflare'); + expect(result.confidence).to.equal(0.99); + }); + + it('detects Cloudflare challenge page with turnstile', () => { + const html = ''; + const headers = { 'cf-ray': '123' }; + + const result = analyzeBotProtection({ + status: 200, + headers, + html, + }); + + expect(result.crawlable).to.be.false; + expect(result.type).to.equal('cloudflare'); + }); + + it('returns cloudflare-allowed when Cloudflare present but no challenge', () => { + // Create HTML > 10KB to ensure it's not flagged as "suspiciously short" + const realContent = 'This is real page content. '.repeat(400); // ~11KB + const html = `Real Page Title

Welcome to our site

${realContent}

`; + const headers = { 'cf-ray': '123456789-CDG' }; + + const result = analyzeBotProtection({ + status: 200, + headers, + html, + }); + + expect(result.crawlable).to.be.true; + expect(result.type).to.equal('cloudflare-allowed'); + expect(result.confidence).to.equal(1.0); + }); + + it('detects Imperva challenge page with 200 status', () => { + const html = '_Incapsula_Resource detected'; + const headers = { 'x-iinfo': 'some-value' }; + + const result = analyzeBotProtection({ + status: 200, + headers, + html, + }); + + expect(result.crawlable).to.be.false; + expect(result.type).to.equal('imperva'); + expect(result.confidence).to.equal(0.99); + }); + + it('returns imperva-allowed when Imperva present but no challenge', () => { + // Create HTML > 10KB to ensure it's not flagged as "suspiciously short" + const realContent = 'Real content with no Imperva patterns. '.repeat(300); // ~12KB + const html = `${realContent}`; + const headers = { 'x-iinfo': 'some-value' }; + + const result = analyzeBotProtection({ + status: 200, + headers, + html, + }); + + expect(result.crawlable).to.be.true; + expect(result.type).to.equal('imperva-allowed'); + }); + + it('detects generic CAPTCHA challenge', () => { + const html = '
'; + const headers = {}; + + const result = analyzeBotProtection({ + status: 200, + headers, + html, + }); + + expect(result.crawlable).to.be.false; + expect(result.type).to.equal('unknown'); + expect(result.confidence).to.equal(0.7); + }); + + it('works without HTML (backwards compatibility)', () => { + const headers = { 'cf-ray': '123456789-CDG' }; + + const result = analyzeBotProtection({ + status: 200, + headers, + html: null, + }); + + expect(result.crawlable).to.be.true; + expect(result.type).to.equal('cloudflare-allowed'); + }); + + it('works with Headers object', () => { + const html = 'Just a moment...'; + const headers = new Headers({ 'cf-ray': '123' }); + + const result = analyzeBotProtection({ + status: 200, + headers, + html, + }); + + expect(result.crawlable).to.be.false; + expect(result.type).to.equal('cloudflare'); + }); + + it('detects Cloudflare blocking with 403', () => { + const html = 'Access denied'; + const headers = { 'cf-ray': '123' }; + + const result = analyzeBotProtection({ + status: 403, + headers, + html, + }); + + expect(result.crawlable).to.be.false; + expect(result.type).to.equal('cloudflare'); + expect(result.confidence).to.equal(0.99); + }); + + it('returns none for clean 200 with no protection', () => { + const html = 'Normal page content'; + const headers = {}; + + const result = analyzeBotProtection({ + status: 200, + headers, + html, + }); + + expect(result.crawlable).to.be.true; + expect(result.type).to.equal('none'); + expect(result.confidence).to.equal(1.0); + }); + + // New tests for enhanced patterns + it('detects Cloudflare challenge widget (cf-chl-widget)', () => { + const html = '
'; + const headers = { 'cf-ray': '123' }; + + const result = analyzeBotProtection({ + status: 200, + headers, + html, + }); + + expect(result.crawlable).to.be.false; + expect(result.type).to.equal('cloudflare'); + }); + + it('detects Cloudflare challenge token (__cf_chl_tk)', () => { + const html = ''; + const headers = { 'cf-ray': '123' }; + + const result = analyzeBotProtection({ + status: 200, + headers, + html, + }); + + expect(result.crawlable).to.be.false; + expect(result.type).to.equal('cloudflare'); + }); + + it('detects Imperva session cookie pattern (incap_ses)', () => { + const html = 'incap_ses_123_456789 detected'; + const headers = { 'x-iinfo': 'some-value' }; + + const result = analyzeBotProtection({ + status: 200, + headers, + html, + }); + + expect(result.crawlable).to.be.false; + expect(result.type).to.equal('imperva'); + }); + + it('detects DataDome bot protection', () => { + const html = ''; + const headers = {}; + + const result = analyzeBotProtection({ + status: 200, + headers, + html, + }); + + expect(result.crawlable).to.be.false; + expect(result.type).to.equal('unknown'); + expect(result.confidence).to.equal(0.7); + }); + + it('detects reCAPTCHA', () => { + const html = '
'; + const headers = {}; + + const result = analyzeBotProtection({ + status: 200, + headers, + html, + }); + + expect(result.crawlable).to.be.false; + expect(result.type).to.equal('unknown'); + }); + + it('detects Akamai challenge page', () => { + const html = 'Access Denied by Akamai security policy'; + const headers = { 'x-akamai-request-id': 'abc123' }; + + const result = analyzeBotProtection({ + status: 200, + headers, + html, + }); + + expect(result.crawlable).to.be.false; + expect(result.type).to.equal('akamai'); + }); + + // Tests for generic HTTP error code handling + it('detects generic 403 Forbidden without known CDN', () => { + const html = 'Access denied'; + const headers = {}; + + const result = analyzeBotProtection({ + status: 403, + headers, + html, + }); + + expect(result.crawlable).to.be.false; + expect(result.type).to.equal('unknown'); + expect(result.confidence).to.equal(0.7); + expect(result.reason).to.equal('HTTP 403 Forbidden - access denied'); + }); + + it('detects 429 Too Many Requests (rate limiting)', () => { + const html = 'Too many requests'; + const headers = {}; + + const result = analyzeBotProtection({ + status: 429, + headers, + html, + }); + + expect(result.crawlable).to.be.false; + expect(result.type).to.equal('rate-limit'); + expect(result.confidence).to.equal(0.99); + expect(result.reason).to.equal('HTTP 429 Too Many Requests - rate limit exceeded'); + }); + + it('detects 401 Unauthorized', () => { + const html = 'Unauthorized'; + const headers = {}; + + const result = analyzeBotProtection({ + status: 401, + headers, + html, + }); + + expect(result.crawlable).to.be.false; + expect(result.type).to.equal('auth-required'); + expect(result.confidence).to.equal(0.99); + expect(result.reason).to.equal('HTTP 401 Unauthorized - authentication required'); + }); + + it('detects 406 Not Acceptable (user-agent rejection)', () => { + const html = 'Not acceptable'; + const headers = {}; + + const result = analyzeBotProtection({ + status: 406, + headers, + html, + }); + + expect(result.crawlable).to.be.false; + expect(result.type).to.equal('user-agent-rejected'); + expect(result.confidence).to.equal(0.8); + expect(result.reason).to.equal('HTTP 406 Not Acceptable - likely user-agent rejection'); + }); + + it('detects other 4xx client errors', () => { + const html = 'Error'; + const headers = {}; + + const result = analyzeBotProtection({ + status: 418, + headers, + html, + }); + + expect(result.crawlable).to.be.false; + expect(result.type).to.equal('http-error'); + expect(result.confidence).to.equal(0.6); + expect(result.reason).to.equal('HTTP 418 - client error'); + }); + + it('does not treat 5xx server errors as bot protection', () => { + const html = 'Internal server error'; + const headers = {}; + + const result = analyzeBotProtection({ + status: 500, + headers, + html, + }); + + // Server errors are NOT bot protection - they're infrastructure issues + expect(result.crawlable).to.be.true; + expect(result.type).to.equal('unknown'); + expect(result.confidence).to.equal(0.5); + }); + + it('prioritizes known CDN detection over generic 403', () => { + // This ensures that if we have both 403 AND Cloudflare headers, + // we return the more specific 'cloudflare' type, not generic 'unknown' + const html = 'Access denied'; + const headers = { 'cf-ray': '123456' }; + + const result = analyzeBotProtection({ + status: 403, + headers, + html, + }); + + expect(result.crawlable).to.be.false; + expect(result.type).to.equal('cloudflare'); + expect(result.confidence).to.equal(0.99); + }); + + // Edge case: 3xx redirects (not handled by specific rules) + it('returns unknown with crawlable true for 3xx redirect status codes', () => { + const html = 'Redirecting...'; + const headers = {}; + + const result = analyzeBotProtection({ + status: 301, + headers, + html, + }); + + expect(result.crawlable).to.be.true; + expect(result.type).to.equal('unknown'); + expect(result.confidence).to.equal(0.5); + }); + + // Edge case: 1xx informational responses + it('returns unknown with crawlable true for 1xx informational status codes', () => { + const html = ''; + const headers = {}; + + const result = analyzeBotProtection({ + status: 100, + headers, + html, + }); + + expect(result.crawlable).to.be.true; + expect(result.type).to.equal('unknown'); + expect(result.confidence).to.equal(0.5); + }); + + // Edge case: Unusual 5xx-range status codes are NOT bot protection + it('does not treat very unusual 5xx status codes as bot protection', () => { + const html = ''; + const headers = {}; + + const result = analyzeBotProtection({ + status: 999, + headers, + html, + }); + + // Server errors (even unusual ones) are NOT bot protection + expect(result.crawlable).to.be.true; + expect(result.type).to.equal('unknown'); + expect(result.confidence).to.equal(0.5); + }); + + // Edge case: No headers provided (null/undefined) + it('handles null headers gracefully', () => { + const html = 'Normal page'; + + const result = analyzeBotProtection({ + status: 200, + headers: null, + html, + }); + + expect(result.crawlable).to.be.true; + expect(result.type).to.equal('none'); + expect(result.confidence).to.equal(1.0); + }); + + // Edge case: Undefined headers + it('handles undefined headers gracefully', () => { + const html = 'Normal page'; + + const result = analyzeBotProtection({ + status: 200, + headers: undefined, + html, + }); + + expect(result.crawlable).to.be.true; + expect(result.type).to.equal('none'); + expect(result.confidence).to.equal(1.0); + }); + }); + + describe('IP Management Functions', () => { + describe('getSpacecatBotIps', () => { + it('should throw error when IPs not provided', () => { + expect(() => getSpacecatBotIps(null)).to.throw('SPACECAT_BOT_IPS environment variable is required but not set'); + expect(() => getSpacecatBotIps('')).to.throw('SPACECAT_BOT_IPS environment variable is required but not set'); + expect(() => getSpacecatBotIps(undefined)).to.throw('SPACECAT_BOT_IPS environment variable is required but not set'); + }); + + it('should parse comma-separated IPs', () => { + const botIps = '1.2.3.4,5.6.7.8,9.10.11.12'; + const ips = getSpacecatBotIps(botIps); + + expect(ips).to.deep.equal(['1.2.3.4', '5.6.7.8', '9.10.11.12']); + }); + + it('should trim whitespace from IP addresses', () => { + const botIps = ' 1.2.3.4 , 5.6.7.8 , 9.10.11.12 '; + const ips = getSpacecatBotIps(botIps); + + expect(ips).to.deep.equal(['1.2.3.4', '5.6.7.8', '9.10.11.12']); + }); + + it('should filter out empty IP entries', () => { + const botIps = '1.2.3.4,,5.6.7.8, ,9.10.11.12'; + const ips = getSpacecatBotIps(botIps); + + expect(ips).to.deep.equal(['1.2.3.4', '5.6.7.8', '9.10.11.12']); + }); + + it('should handle single IP', () => { + const botIps = '192.168.1.1'; + const ips = getSpacecatBotIps(botIps); + + expect(ips).to.deep.equal(['192.168.1.1']); + }); + }); + + describe('formatAllowlistMessage', () => { + it('should format allowlist message with IPs', () => { + const botIps = '1.2.3.4,5.6.7.8,9.10.11.12'; + const message = formatAllowlistMessage(botIps); + + expect(message).to.deep.equal({ + title: 'To allowlist SpaceCat bot:', + ips: ['1.2.3.4', '5.6.7.8', '9.10.11.12'], + userAgent: SPACECAT_BOT_USER_AGENT, + }); + }); + + it('should throw error when IPs not provided', () => { + expect(() => formatAllowlistMessage(null)).to.throw('SPACECAT_BOT_IPS environment variable is required but not set'); + }); + + it('should include correct user-agent', () => { + const botIps = '1.2.3.4'; + const message = formatAllowlistMessage(botIps); + + expect(message.userAgent).to.equal('Spacecat/1.0'); + }); + }); + }); }); diff --git a/packages/spacecat-shared-utils/test/index.test.js b/packages/spacecat-shared-utils/test/index.test.js index 8b6d107ef..ac6a8c0d0 100644 --- a/packages/spacecat-shared-utils/test/index.test.js +++ b/packages/spacecat-shared-utils/test/index.test.js @@ -17,6 +17,7 @@ import * as allExports from '../src/index.js'; describe('Index Exports', () => { const expectedExports = [ + 'analyzeBotProtection', 'arrayEquals', 'composeAuditURL', 'composeBaseURL', @@ -66,6 +67,9 @@ describe('Index Exports', () => { 'getHighPageViewsLowFormViewsMetrics', 'getHighPageViewsLowFormCtrMetrics', 'FORMS_AUDIT_INTERVAL', + 'getSpacecatBotIps', + 'formatAllowlistMessage', + 'SPACECAT_BOT_USER_AGENT', 'SPACECAT_USER_AGENT', 'isAWSLambda', 'instrumentAWSClient',