From 69b9d6e87ff711216a8559ace15b76df047b623e Mon Sep 17 00:00:00 2001 From: Dave Rogers Date: Fri, 7 Aug 2026 10:59:36 -0700 Subject: [PATCH 1/3] feat: support excluding anomalous dates from static analytics reports #4907 Co-Authored-By: Claude Fable 5 --- .../analytics/static_site/fetch.py | 15 +++++++++++++++ .../analytics/static_site/generator.py | 5 +++++ analytics/anvil-explorer-sheets/constants.py | 3 +++ .../anvil-explorer-sheets/generate_static_site.py | 3 ++- gh-pages/anvil-explorer/data/meta.json | 2 +- gh-pages/anvil-explorer/data/monthly_traffic.json | 4 ++-- 6 files changed, 28 insertions(+), 4 deletions(-) diff --git a/analytics/analytics_package/analytics/static_site/fetch.py b/analytics/analytics_package/analytics/static_site/fetch.py index ca47c15ec..89e4c378d 100644 --- a/analytics/analytics_package/analytics/static_site/fetch.py +++ b/analytics/analytics_package/analytics/static_site/fetch.py @@ -4,6 +4,7 @@ from urllib.parse import urlparse, parse_qs from .. import sheets_elements as elements +from ..api import parse_filter_expressions from .._sheets_utils import get_data_df_from_fields from ..entities import ( DIMENSION_YEAR_MONTH, @@ -337,6 +338,7 @@ def fetch_data( exclude_pages=None, base_dimension_filter=None, search_path=None, + exclude_dates=None, ): """Fetch all analytics data for the static site. @@ -350,6 +352,9 @@ def fetch_data( exclude_pages: Optional list of page paths to exclude from pageview data. base_dimension_filter: Optional GA4 dimension filter dict applied to all queries. search_path: Optional search page path to extract search queries from (e.g., "/search"). + exclude_dates: Optional list of dates (YYYY-MM-DD) to exclude from all queries, + e.g. days with known synthetic/bot traffic. Applies to GA4 queries only, + not to data merged from historic_data_path. Returns: Dict containing DataFrames and stats for each data type. @@ -357,6 +362,16 @@ def fetch_data( if custom_events is None: custom_events = [] + if exclude_dates: + base_dimension_filter = parse_filter_expressions( + [ + base_dimension_filter, + ";".join(f"date!={d.replace('-', '')}" for d in exclude_dates), + ], + False, + ) + print(f"Excluding dates from all queries: {', '.join(exclude_dates)}") + report_dates = elements.get_bounds_for_month_and_prev(current_month) start_date_current = report_dates["start_current"] end_date_current = report_dates["end_current"] diff --git a/analytics/analytics_package/analytics/static_site/generator.py b/analytics/analytics_package/analytics/static_site/generator.py index 751a17eac..eab09a44c 100644 --- a/analytics/analytics_package/analytics/static_site/generator.py +++ b/analytics/analytics_package/analytics/static_site/generator.py @@ -24,6 +24,7 @@ def generate_site( exclude_pages=None, base_dimension_filter=None, search_path=None, + exclude_dates=None, ): """Generate a static analytics site. @@ -48,6 +49,9 @@ def generate_site( base_dimension_filter: Optional GA4 dimension filter dict applied to all queries (e.g., an audience filter to exclude bot/tutorial traffic). search_path: Optional search page path to extract search queries from (e.g., "/search"). + exclude_dates: Optional list of dates (YYYY-MM-DD) to exclude from all queries, + e.g. days with known synthetic/bot traffic. Applies to GA4 queries only, + not to data merged from historic_data_path. """ if custom_events is None: custom_events = [] @@ -69,6 +73,7 @@ def generate_site( exclude_pages=exclude_pages, base_dimension_filter=base_dimension_filter, search_path=search_path, + exclude_dates=exclude_dates, ) if title_resolver: diff --git a/analytics/anvil-explorer-sheets/constants.py b/analytics/anvil-explorer-sheets/constants.py index 3324528f3..8be7adcbc 100644 --- a/analytics/anvil-explorer-sheets/constants.py +++ b/analytics/anvil-explorer-sheets/constants.py @@ -7,6 +7,9 @@ # The name of the spreadsheet with the report SHEET_NAME = "AnVIL Explorer" ANVIL_EXPLORER_ID = "383267328" +# Dates of known synthetic/bot traffic (headless-browser burst, likely a load test) +# to exclude from reports; see DataBiosphere/data-browser#4907. +EXCLUDE_BOT_TRAFFIC_DATES = ["2025-02-10", "2025-02-11"] SECRET_NAME = 'ANVIL_ANALYTICS_REPORTING_CLIENT_SECRET_PATH' GA_PROPERTY_PORTAL = "368678391" # AnVIL Explorer - GA4 ANALYTICS_START = "2024-01-01" diff --git a/analytics/anvil-explorer-sheets/generate_static_site.py b/analytics/anvil-explorer-sheets/generate_static_site.py index 9d356ae1a..fadbef274 100644 --- a/analytics/anvil-explorer-sheets/generate_static_site.py +++ b/analytics/anvil-explorer-sheets/generate_static_site.py @@ -5,7 +5,7 @@ import analytics.api as ga from analytics.static_site import generate_site, enrich_detail_records, make_event_charts -from constants import CURRENT_MONTH, ANVIL_EXPLORER_ID, SECRET_NAME, ANALYTICS_START, OAUTH_PORT +from constants import CURRENT_MONTH, ANVIL_EXPLORER_ID, SECRET_NAME, ANALYTICS_START, OAUTH_PORT, EXCLUDE_BOT_TRAFFIC_DATES from utils import fetch_dataset_title_map os.environ.setdefault(SECRET_NAME, "../../.credentials/anvil_ga4_credentials.json") @@ -95,4 +95,5 @@ def resolve_dataset_titles(data): event_charts=make_event_charts("Dataset", "/datasets", "2026-03-01"), title_resolver=resolve_dataset_titles, access_request_urls=["duos.org", "dbgap.ncbi.nlm.nih.gov"], + exclude_dates=EXCLUDE_BOT_TRAFFIC_DATES, ) diff --git a/gh-pages/anvil-explorer/data/meta.json b/gh-pages/anvil-explorer/data/meta.json index d5b61951a..64acfc9be 100644 --- a/gh-pages/anvil-explorer/data/meta.json +++ b/gh-pages/anvil-explorer/data/meta.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-08-06 22:50:04", + "generated_at": "2026-08-07 01:01:11", "current_month": "2026-07", "current_month_start": "2026-07-01", "current_month_end": "2026-07-31", diff --git a/gh-pages/anvil-explorer/data/monthly_traffic.json b/gh-pages/anvil-explorer/data/monthly_traffic.json index 4614d5545..f68bb3258 100644 --- a/gh-pages/anvil-explorer/data/monthly_traffic.json +++ b/gh-pages/anvil-explorer/data/monthly_traffic.json @@ -86,8 +86,8 @@ }, { "month": "2025-02", - "users": 9480, - "pageviews": 19951 + "users": 277, + "pageviews": 2239 }, { "month": "2025-01", From afe2b782fe866ed70a7d8d5cd5f953b0c0b2cc89 Mon Sep 17 00:00:00 2001 From: Dave Rogers Date: Fri, 7 Aug 2026 14:09:53 -0700 Subject: [PATCH 2/3] fix: validate exclude_dates format before building filter #4907 Co-Authored-By: Claude Fable 5 --- analytics/analytics_package/analytics/static_site/fetch.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/analytics/analytics_package/analytics/static_site/fetch.py b/analytics/analytics_package/analytics/static_site/fetch.py index 89e4c378d..0514183a8 100644 --- a/analytics/analytics_package/analytics/static_site/fetch.py +++ b/analytics/analytics_package/analytics/static_site/fetch.py @@ -363,6 +363,9 @@ def fetch_data( custom_events = [] if exclude_dates: + for d in exclude_dates: + if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", d): + raise ValueError(f"exclude_dates entries must be YYYY-MM-DD, got: {d!r}") base_dimension_filter = parse_filter_expressions( [ base_dimension_filter, From 085764d6c9f50b34ea68131468010d10473eff9f Mon Sep 17 00:00:00 2001 From: Dave Rogers Date: Fri, 7 Aug 2026 14:40:46 -0700 Subject: [PATCH 3/3] fix: reject non-existent calendar dates in exclude_dates #4907 Co-Authored-By: Claude Fable 5 --- analytics/analytics_package/analytics/static_site/fetch.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/analytics/analytics_package/analytics/static_site/fetch.py b/analytics/analytics_package/analytics/static_site/fetch.py index 0514183a8..57175e22b 100644 --- a/analytics/analytics_package/analytics/static_site/fetch.py +++ b/analytics/analytics_package/analytics/static_site/fetch.py @@ -1,6 +1,7 @@ """Fetch analytics data from GA4 for the static site.""" import re +from datetime import date from urllib.parse import urlparse, parse_qs from .. import sheets_elements as elements @@ -366,6 +367,7 @@ def fetch_data( for d in exclude_dates: if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", d): raise ValueError(f"exclude_dates entries must be YYYY-MM-DD, got: {d!r}") + date.fromisoformat(d) base_dimension_filter = parse_filter_expressions( [ base_dimension_filter,