From d61a1168e7c5e0fc0e72042b4bee9b5dca7867e2 Mon Sep 17 00:00:00 2001 From: nullhack Date: Wed, 22 Jul 2026 04:44:43 -0400 Subject: [PATCH] feat(titles): canonical incident names per adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace verbose/inconsistent genesis report names (e.g. 100-char GDACS TC sentences, 'Flood in Pakistan', 'M 7.5 - 20 km ESE of Yumare, Venezuela') with a standardised format produced at ingest by each adapter: {Type} {Identifier} {Smallest-place}, {Country} {YYYY-MM-DD} Diseases omit the Type word — the disease name leads instead: {Disease} {Place} {YYYY-MM-DD} Architecture (adapter-centric, not a one-fits-all parser): - Each adapter owns _extract_canonical_name(raw_fields, places, report_date, incident_type) -> str, called at ingest so SourceReport.name is born canonical. - Shared utilities in disaster_report/_title_format.py: format_title, smallest_place (locality > subdivision > country), format_place, normalise_subdivision (strips accents + admin suffixes like ' Sheng'/ ' SAR'), normalise_country (short aliases like 'DR Congo'). - _search_keys.place_token (renamed from _place_token, now public) reused for locality offset-stripping. Per-adapter extraction: - USGS: M{mag:.1f} from raw_fields.mag + smallest_place. - GDACS: dispatches by incident_type — EQ magnitude from numeric severity or regex on severitytext; TC storm name from eventname or title regex; VO volcano name from title (volcano name IS the locality); FL/WF/DR omit the identifier. - WHO: disease short name via prefix lookup table (25 entries covering all 28 production incident_type values); place resolved from the title suffix (split on dash variants, scan_countries on the suffix) to avoid body-scan medevac noise — falls back to smallest_place(places) or 'Global' when no places. Dashboard simplification: dropped the synth override at generate_dashboard_data.py:503-505 (was f'{inc_type} {place} {YYYY-MM}' gated on max_mag is not None — failed for GDACS TC/Flood with non-numeric severity). canonical_name now = inc['name'] directly. Backfill migration script scripts/backfill_canonical_names.py: one-shot, per-adapter dispatch, targeted name: line replacement only (rest of YAML preserved byte-for-byte), --dry-run flag, idempotent (skips WHO records whose name already ends with report_date). Data branch backfilled separately (164 reports: 67 USGS + 52 WHO + 45 GDACS). Tests: 27 _title_format + 5 USGS + 10 GDACS + 10 WHO (4 new for title-suffix parsing) = 52 new tests. Gates: ruff clean, pyright 0 errors, mypy.stubtest clean, full suite green. --- disaster_report/_search_keys.py | 4 +- disaster_report/_search_keys.pyi | 3 +- disaster_report/_title_format.py | 96 +++++++++++++++ disaster_report/_title_format.pyi | 7 ++ disaster_report/sources/gdacs.py | 77 +++++++++++- disaster_report/sources/gdacs.pyi | 6 + disaster_report/sources/usgs.py | 41 ++++--- disaster_report/sources/usgs.pyi | 9 +- disaster_report/sources/who.py | 104 +++++++++++++++-- disaster_report/sources/who.pyi | 9 +- scripts/backfill_canonical_names.py | 143 +++++++++++++++++++++++ scripts/generate_dashboard_data.py | 2 - tests/integration/_title_format_test.py | 108 +++++++++++++++++ tests/integration/_title_format_test.pyi | 40 +++++++ tests/integration/gdacs_source_test.py | 142 ++++++++++++++++++++++ tests/integration/gdacs_source_test.pyi | 12 ++ tests/integration/usgs_source_test.py | 49 ++++++++ tests/integration/usgs_source_test.pyi | 7 ++ tests/integration/who_source_test.py | 124 +++++++++++++++++++- tests/integration/who_source_test.pyi | 12 ++ 20 files changed, 962 insertions(+), 33 deletions(-) create mode 100644 disaster_report/_title_format.py create mode 100644 disaster_report/_title_format.pyi create mode 100644 scripts/backfill_canonical_names.py create mode 100644 tests/integration/_title_format_test.py create mode 100644 tests/integration/_title_format_test.pyi diff --git a/disaster_report/_search_keys.py b/disaster_report/_search_keys.py index 36f57c10..befba21a 100644 --- a/disaster_report/_search_keys.py +++ b/disaster_report/_search_keys.py @@ -48,7 +48,7 @@ def derive_search_keys( if len(report.places) == 1 and report.places[0].country_code: place = report.places[0] country = country_name(place.country_code) - token = _place_token(place) + token = place_token(place) place_str = f"{token}, {country}" if token else country strict = " ".join( p for p in (incident_type, place_str, month_year_label, disease) if p @@ -109,7 +109,7 @@ def _shared_continent_tokens(regions: list[str]) -> str: return " ".join(winners) -def _place_token(place: ReportPlace) -> str: +def place_token(place: ReportPlace) -> str: locality = place.locality if not locality: return "" diff --git a/disaster_report/_search_keys.pyi b/disaster_report/_search_keys.pyi index 293355d0..abd8a69d 100644 --- a/disaster_report/_search_keys.pyi +++ b/disaster_report/_search_keys.pyi @@ -1,4 +1,4 @@ -from disaster_report.models import SourceReport +from disaster_report.models import ReportPlace, SourceReport def disease_from_title(title: str) -> str: ... def derive_search_keys( @@ -7,3 +7,4 @@ def derive_search_keys( disease: str = ..., ) -> tuple[str, str]: ... def derive_repoll_keys(report: SourceReport) -> list[str]: ... +def place_token(place: ReportPlace) -> str: ... diff --git a/disaster_report/_title_format.py b/disaster_report/_title_format.py new file mode 100644 index 00000000..c98ad421 --- /dev/null +++ b/disaster_report/_title_format.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import unicodedata + +from disaster_report._country_names import country_name +from disaster_report._search_keys import place_token +from disaster_report.models import ReportPlace + +_ADMIN_SUFFIXES: tuple[str, ...] = ( + " Zhuangzu Zizhiqu", + " Autonomous Region", + " Special Administrative Region", + " SAR", + " Sheng", + " Province", + " State", + " Prefecture", + " Department", + " Region", + " District", + " Territory", +) + +_COUNTRY_ALIASES: dict[str, str] = { + "democratic republic of the congo": "DR Congo", + "united kingdom": "UK", + "united states of america": "United States", + "republic of korea": "South Korea", + "democratic people's republic of korea": "North Korea", + "russian federation": "Russia", + "islamic republic of iran": "Iran", + "syrian arab republic": "Syria", + "czech republic": "Czechia", + "united republic of tanzania": "Tanzania", + "republic of north macedonia": "North Macedonia", + "bolivarian republic of venezuela": "Venezuela", + "brunei darussalam": "Brunei", + "lao people's democratic republic": "Laos", + "socialist republic of viet nam": "Vietnam", + "republic of moldova": "Moldova", + "state of palestine": "Palestine", +} + + +def format_title(*parts: str) -> str: + return " ".join(p.strip() for p in parts if p and p.strip()) + + +def normalise_subdivision(name: str) -> str: + if not name: + return "" + stripped = _strip_accents(name) + lowered = stripped.lower() + for suffix in _ADMIN_SUFFIXES: + if lowered.endswith(suffix.lower()): + stripped = stripped[: -len(suffix)].strip() + break + return stripped.strip() + + +def normalise_country(name: str) -> str: + if not name: + return "" + return _COUNTRY_ALIASES.get(name.lower().strip(), name.strip()) + + +def smallest_place(places: list[ReportPlace]) -> tuple[str, str]: + if not places: + return ("", "") + place = places[0] + country = country_name(place.country_code) + locality_token = place_token(place) + if locality_token: + return (locality_token, country) + if place.subdivision: + return (normalise_subdivision(place.subdivision), country) + return (country, country) + + +def format_place(smallest: str, country: str) -> str: + smallest = (smallest or "").strip() + country = (country or "").strip() + if not smallest and not country: + return "" + if not country: + return smallest + if smallest and smallest != country: + return f"{smallest}, {country}" + return country + + +def _strip_accents(s: str) -> str: + nfkd = unicodedata.normalize("NFKD", s) + return "".join(c for c in nfkd if not unicodedata.combining(c)) + + diff --git a/disaster_report/_title_format.pyi b/disaster_report/_title_format.pyi new file mode 100644 index 00000000..6a5aad72 --- /dev/null +++ b/disaster_report/_title_format.pyi @@ -0,0 +1,7 @@ +from disaster_report.models import ReportPlace + +def format_title(*parts: str) -> str: ... +def normalise_subdivision(name: str) -> str: ... +def normalise_country(name: str) -> str: ... +def smallest_place(places: list[ReportPlace]) -> tuple[str, str]: ... +def format_place(smallest: str, country: str) -> str: ... diff --git a/disaster_report/sources/gdacs.py b/disaster_report/sources/gdacs.py index 38aaae34..029b1dae 100644 --- a/disaster_report/sources/gdacs.py +++ b/disaster_report/sources/gdacs.py @@ -13,6 +13,7 @@ from disaster_report._country_names import country_name from disaster_report._search_keys import derive_search_keys +from disaster_report._title_format import format_place, format_title, smallest_place from disaster_report.models import ReportPlace, SourceReport from disaster_report.sources.errors import SourceFetchError @@ -22,6 +23,11 @@ _GEO = "{http://www.w3.org/2003/01/geo/wgs84_pos#}" _DC = "{http://purl.org/dc/elements/1.1/}" _EVENTID_RE = re.compile(r"[?&]eventid=(\d+)", re.IGNORECASE) +_MAGNITUDE_RE = re.compile(r"Magnitude\s+(\d+(?:\.\d+)?)", re.IGNORECASE) +_STORM_NAME_RE = re.compile( + r"(?:Tropical Cyclone|Typhoon|Hurricane|Cyclone)\s+([A-Z][A-Z]*-?\d*)", +) +_VOLCANO_NAME_RE = re.compile(r"^Eruption\s+(.+?)\s*$", re.IGNORECASE) _TYPES: dict[str, str] = { "TC": "Tropical Cyclone", "EQ": "Earthquake", @@ -82,7 +88,6 @@ def derive_keys(self, report: SourceReport) -> tuple[str, str]: def _item_to_report(item: Any) -> SourceReport: raw_fields = _build_raw_fields(item) - title = str(raw_fields.get("title") or "") link = str(raw_fields.get("link") or "") eventtype = str(raw_fields.get("eventtype") or "") fromdate = str(raw_fields.get("fromdate") or "") @@ -91,17 +96,81 @@ def _item_to_report(item: Any) -> SourceReport: lat = raw_fields.get("geo_lat") lon = raw_fields.get("geo_long") places = _extract_places(iso3, country_text, lat, lon) + incident_type = _TYPES.get(eventtype, eventtype) + report_date = _to_iso_date(fromdate) return SourceReport( source="GDACS", source_id=_extract_event_id(link), - incident_type=_TYPES.get(eventtype, eventtype), - name=title, + incident_type=incident_type, + name=_extract_canonical_name(raw_fields, places, report_date, incident_type), places=places, - report_date=_to_iso_date(fromdate), + report_date=report_date, raw_fields=raw_fields, ) +def _extract_canonical_name( + raw_fields: dict[str, object], + places: list[ReportPlace], + report_date: str, + incident_type: str, +) -> str: + identifier = _extract_gdacs_identifier(incident_type, raw_fields) + smallest, country = smallest_place(places) + if incident_type == "Volcano" and identifier: + return format_title(incident_type, format_place(identifier, country), report_date) + return format_title( + incident_type, + identifier, + format_place(smallest, country), + report_date, + ) + + +def _extract_gdacs_identifier( + incident_type: str, + raw_fields: dict[str, object], +) -> str: + if incident_type == "Earthquake": + return _extract_magnitude(raw_fields) + if incident_type == "Tropical Cyclone": + return _extract_storm_name(raw_fields) + if incident_type == "Volcano": + return _extract_volcano_name(raw_fields) + return "" + + +def _extract_magnitude(raw_fields: dict[str, object]) -> str: + severity = raw_fields.get("severity") + if isinstance(severity, int | float) and not isinstance(severity, bool): + return f"M{float(severity):.1f}" + severitytext = str(raw_fields.get("severitytext") or "") + match = _MAGNITUDE_RE.search(severitytext) + if not match and isinstance(severity, str): + match = _MAGNITUDE_RE.search(severity) + if match: + try: + return f"M{float(match.group(1)):.1f}" + except ValueError: + return "" + return "" + + +def _extract_storm_name(raw_fields: dict[str, object]) -> str: + eventname = str(raw_fields.get("eventname") or "").strip() + if eventname: + return eventname + title = str(raw_fields.get("title") or "") + match = _STORM_NAME_RE.search(title) + return match.group(1).strip() if match else "" + + +def _extract_volcano_name(raw_fields: dict[str, object]) -> str: + title = str(raw_fields.get("title") or "").strip() + match = _VOLCANO_NAME_RE.match(title) + return match.group(1).strip() if match else "" + + def _build_raw_fields(item: Any) -> dict[str, object]: raw: dict[str, object] = {} diff --git a/disaster_report/sources/gdacs.pyi b/disaster_report/sources/gdacs.pyi index 89ae93c2..0d881a24 100644 --- a/disaster_report/sources/gdacs.pyi +++ b/disaster_report/sources/gdacs.pyi @@ -13,6 +13,12 @@ class GDACSAdapter: def derive_keys(self, report: SourceReport) -> tuple[str, str]: ... def _item_to_report(item: object) -> SourceReport: ... +def _extract_canonical_name( + raw_fields: dict[str, object], + places: list[ReportPlace], + report_date: str, + incident_type: str, +) -> str: ... def _extract_places( iso3: str, country_text: str, lat: object, lon: object ) -> list[ReportPlace]: ... diff --git a/disaster_report/sources/usgs.py b/disaster_report/sources/usgs.py index 4bf43458..8a64b027 100644 --- a/disaster_report/sources/usgs.py +++ b/disaster_report/sources/usgs.py @@ -13,6 +13,7 @@ from disaster_report._country_names import country_name from disaster_report._regions import subregion_for_country from disaster_report._search_keys import derive_search_keys +from disaster_report._title_format import format_place, format_title, smallest_place from disaster_report.models import ReportPlace, SourceReport from disaster_report.sources.errors import SourceFetchError @@ -79,30 +80,44 @@ def _feature_to_report( feature_id = feature_dict.get("id") code = properties.get("code") source_id = str(feature_id or code or "") - name = str(properties.get("title") or "") place = str(properties.get("place") or "") places = _extract_places(lat, lon, place) + raw_fields = { + **properties, + "geometry": { + "type": geometry.get("type"), + "coordinates": coordinates, + } + if coordinates + else {}, + "depth": depth, + "place": place, + } return SourceReport( source="USGS", source_id=source_id, incident_type="Earthquake", - name=name, + name=_extract_canonical_name(raw_fields, places, _to_iso_date(properties.get("time")), "Earthquake"), places=places, report_date=_to_iso_date(properties.get("time")), - raw_fields={ - **properties, - "geometry": { - "type": geometry.get("type"), - "coordinates": coordinates, - } - if coordinates - else {}, - "depth": depth, - "place": place, - }, + raw_fields=raw_fields, ) +def _extract_canonical_name( + raw_fields: dict[str, object], + places: list[ReportPlace], + report_date: str, + incident_type: str, +) -> str: + mag = raw_fields.get("mag") + mag_str = "" + if isinstance(mag, int | float) and not isinstance(mag, bool): + mag_str = f"M{float(mag):.1f}" + smallest, country = smallest_place(places) + return format_title(incident_type, mag_str, format_place(smallest, country), report_date) + + def _extract_places( lat: Any, lon: Any, diff --git a/disaster_report/sources/usgs.pyi b/disaster_report/sources/usgs.pyi index 7e6ff559..893acb6e 100644 --- a/disaster_report/sources/usgs.pyi +++ b/disaster_report/sources/usgs.pyi @@ -1,4 +1,4 @@ -from disaster_report.models import SourceReport +from disaster_report.models import ReportPlace, SourceReport import logging logger: logging.Logger @@ -11,3 +11,10 @@ class USGSAdapter: def fetch(self) -> list[SourceReport]: ... def should_monitor(self, report: SourceReport) -> bool: ... def derive_keys(self, report: SourceReport) -> tuple[str, str]: ... + +def _extract_canonical_name( + raw_fields: dict[str, object], + places: list[ReportPlace], + report_date: str, + incident_type: str, +) -> str: ... diff --git a/disaster_report/sources/who.py b/disaster_report/sources/who.py index 54f47502..54045ec9 100644 --- a/disaster_report/sources/who.py +++ b/disaster_report/sources/who.py @@ -1,18 +1,27 @@ from __future__ import annotations +import re from typing import Any import httpx -from disaster_report._countries import extract_places_from_text +from disaster_report._countries import extract_places_from_text, scan_countries +from disaster_report._country_names import country_name from disaster_report._search_keys import derive_search_keys, disease_from_title +from disaster_report._title_format import ( + format_place, + format_title, + normalise_subdivision, + smallest_place, +) from disaster_report.models import ReportPlace, SourceReport from disaster_report.sources.errors import SourceFetchError _BASE_URL = "https://www.who.int/api/news/diseaseoutbreaknews" _DEFAULT_ORDERBY = "PublicationDateAndTime" _TOP = 25 +_TITLE_SPLIT_RE = re.compile(r"\s[-–]\s|(?<=[a-z])[-–](?=\s|[A-Z])") # DON OData prose sections scanned for country/subdivision names. _BODY_SECTIONS: tuple[str, ...] = ( @@ -23,6 +32,34 @@ "Response", ) +_DISEASE_PREFIX_MAP: tuple[tuple[str, str], ...] = ( + ("Avian Influenza", "Avian Influenza"), + ("Broader transmission of mpox", "Mpox"), + ("Mpox", "Mpox"), + ("Circulating vaccine-derived poliovirus", "Poliovirus"), + ("COVID-19", "COVID-19"), + ("Ebola", "Ebola"), + ("Marburg", "Marburg"), + ("Nipah", "Nipah"), + ("Cholera", "Cholera"), + ("Oropouche", "Oropouche"), + ("Measles", "Measles"), + ("Yellow fever", "Yellow fever"), + ("Hantavirus", "Hantavirus"), + ("Chikungunya", "Chikungunya"), + ("Anthrax", "Anthrax"), + ("Rift Valley fever", "Rift Valley fever"), + ("Dengue", "Dengue"), + ("Lassa fever", "Lassa fever"), + ("Meningococcal", "Meningococcal"), + ("Seasonal influenza", "Seasonal influenza"), + ("Acute respiratory", "Acute Respiratory"), + ("Trends of acute respiratory", "Acute Respiratory"), + ("Antimicrobial Resistance", "AMR"), + ("International food safety", "Food Safety Event"), + ("Undiagnosed", "Undiagnosed"), +) + class WHODiseaseOutbreakAdapter: @@ -70,19 +107,72 @@ def _record_to_report(record: Any) -> SourceReport: ) for p in raw_places ] + incident_type = disease_from_title(name) or "Disease" + report_date = _to_iso_date(record_dict.get("PublicationDateAndTime")) + raw_fields = {key: value for key, value in record_dict.items() if key != title_key} + raw_fields["title"] = name return SourceReport( source="WHO", source_id=str(record_dict.get("Id") or ""), - incident_type=disease_from_title(name) or "Disease", - name=name, + incident_type=incident_type, + name=_extract_canonical_name(raw_fields, places, report_date, incident_type), places=places, - report_date=_to_iso_date(record_dict.get("PublicationDateAndTime")), - raw_fields={ - key: value for key, value in record_dict.items() if key != title_key - }, + report_date=report_date, + raw_fields=raw_fields, ) +def _extract_canonical_name( + raw_fields: dict[str, object], + places: list[ReportPlace], + report_date: str, + incident_type: str, +) -> str: + disease = _short_disease_name(incident_type) + place = _resolve_disease_place(raw_fields.get("title", ""), places) + return format_title(disease, place, report_date) + + +def _resolve_disease_place(title: object, places: list[ReportPlace]) -> str: + suffix = _title_suffix(title) + if suffix: + lowered = suffix.lower() + if "global" in lowered: + return "Global" + matches = scan_countries(suffix) + if matches: + code = matches[0][1] + country = country_name(code) + for place in places: + if place.country_code == code and place.subdivision: + return format_place(normalise_subdivision(place.subdivision), country) + return country + smallest, country = smallest_place(places) + if not smallest and not country: + return "Global" + return format_place(smallest, country) + + +def _title_suffix(title: object) -> str: + if not isinstance(title, str) or not title: + return "" + parts = _TITLE_SPLIT_RE.split(title, maxsplit=1) + if len(parts) < 2: + return "" + return parts[1].strip() + + +def _short_disease_name(incident_type: str) -> str: + if not incident_type: + return "Disease" + lowered = incident_type.lower() + for prefix, short in _DISEASE_PREFIX_MAP: + if lowered.startswith(prefix.lower()): + return short + first = incident_type.split()[0].strip() + return first or "Disease" + + def _as_dict(value: Any) -> dict: return value if isinstance(value, dict) else {} diff --git a/disaster_report/sources/who.pyi b/disaster_report/sources/who.pyi index 1c1a60dc..c104b02a 100644 --- a/disaster_report/sources/who.pyi +++ b/disaster_report/sources/who.pyi @@ -1,6 +1,13 @@ -from disaster_report.models import SourceReport +from disaster_report.models import ReportPlace, SourceReport class WHODiseaseOutbreakAdapter: def __init__(self, orderby: str = ...) -> None: ... def fetch(self) -> list[SourceReport]: ... def derive_keys(self, report: SourceReport) -> tuple[str, str]: ... + +def _extract_canonical_name( + raw_fields: dict[str, object], + places: list[ReportPlace], + report_date: str, + incident_type: str, +) -> str: ... diff --git a/scripts/backfill_canonical_names.py b/scripts/backfill_canonical_names.py new file mode 100644 index 00000000..f88cc877 --- /dev/null +++ b/scripts/backfill_canonical_names.py @@ -0,0 +1,143 @@ +"""Backfill report `name:` fields with canonical names from adapter extractors. + +Walks every report YAML under data/incidents/.../reports/source=*/ and rewrites +the `name` field by dispatching to the source adapter's `_extract_canonical_name`. +Only the `name:` line is rewritten; the rest of the file is preserved byte-for-byte. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from collections import Counter +from pathlib import Path +from typing import Any + +import yaml + +_REPO_ROOT = Path(__file__).resolve().parent.parent +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +from disaster_report.models import ReportPlace +from disaster_report.sources.gdacs import _extract_canonical_name as _gdacs_name +from disaster_report.sources.usgs import _extract_canonical_name as _usgs_name +from disaster_report.sources.who import _extract_canonical_name as _who_name + +_NAME_LINE_RE = re.compile(r"^(?Pname): (?P.*)$", re.MULTILINE) + + +def _to_report_places(raw_places: list[dict[str, Any]]) -> list[ReportPlace]: + return [ + ReportPlace( + country_code=str(p.get("country_code", "")), + subdivision=str(p.get("subdivision", "")), + locality=str(p.get("locality", "")), + ) + for p in raw_places + ] + + +def _new_name(source: str, raw: dict[str, Any]) -> str: + raw_fields = dict(raw.get("raw_fields") or {}) + places = _to_report_places(raw.get("places") or []) + report_date = str(raw.get("report_date") or "") + incident_type = str(raw.get("incident_type") or "") + old_name = str(raw.get("name") or "") + if source == "WHO" and "title" not in raw_fields: + if report_date and old_name.endswith(report_date): + return old_name + raw_fields["title"] = old_name + if source == "USGS": + return _usgs_name(raw_fields, places, report_date, incident_type or "Earthquake") + if source == "GDACS": + return _gdacs_name(raw_fields, places, report_date, incident_type) + if source == "WHO": + return _who_name(raw_fields, places, report_date, incident_type) + return old_name + + +def _replace_name_line(text: str, new_value: str) -> str: + def _sub(match: re.Match[str]) -> str: + return f"{match.group('key')}: {new_value}" + + return _NAME_LINE_RE.sub(_sub, text, count=1) + + +def _process_report( + yaml_path: Path, + source: str, + dry_run: bool, +) -> tuple[str, str | None, str | None, str | None]: + """Return (source, old_name, new_name, error) for one report. + + new_name is None when skipped (unchanged or error). error is None unless + an exception was raised. + """ + raw = yaml.safe_load(yaml_path.read_text()) + if not raw: + return (source, None, None, None) + old_name = str(raw.get("name") or "") + try: + new_name = _new_name(source, raw) + except Exception as exc: # noqa: BLE001 + return (source, old_name, None, str(exc)) + if new_name == old_name: + return (source, old_name, None, None) + if not dry_run: + original = yaml_path.read_text() + updated = _replace_name_line(original, new_name) + if updated != original: + yaml_path.write_text(updated) + return (source, old_name, new_name, None) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tree-root", default="data") + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + tree = Path(args.tree_root) / "incidents" + if not tree.is_dir(): + print(f"backfill: {tree} is not a directory", file=sys.stderr) + return 2 + + total = 0 + changed: Counter[str] = Counter() + skipped: Counter[str] = Counter() + samples: list[tuple[str, str, str]] = [] + + for report_dir in sorted(tree.glob("*/reports/source=*")): + source = report_dir.name.split("=", 1)[1] + for yaml_path in sorted(report_dir.glob("*.yaml")): + total += 1 + src, old_name, new_name, error = _process_report( + yaml_path, source, args.dry_run + ) + if error is not None: + skipped[src] += 1 + print(f" ERR {src} {yaml_path.name}: {error}", file=sys.stderr) + continue + if new_name is None: + skipped[src] += 1 + continue + changed[src] += 1 + if len(samples) < 12 or src == "WHO": + samples.append((src, (old_name or "")[:60], new_name[:60])) + + print("\n=== samples (source | old → new) ===", file=sys.stderr) + for source, old, new in samples: + print(f" [{source}] {old!r}\n → {new!r}", file=sys.stderr) + + print("\n=== summary ===", file=sys.stderr) + print(f"total reports scanned: {total}", file=sys.stderr) + print(f"changed by source: {dict(changed)}", file=sys.stderr) + print(f"skipped by source: {dict(skipped)}", file=sys.stderr) + print(f"dry-run: {args.dry_run}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/generate_dashboard_data.py b/scripts/generate_dashboard_data.py index ce8af36e..97bf8b4b 100644 --- a/scripts/generate_dashboard_data.py +++ b/scripts/generate_dashboard_data.py @@ -501,8 +501,6 @@ def build_incident_object(store: ContentStore, inc: dict, as_of_date: datetime) dashboard_id = f"{date_part}-{iso2 or 'XX'}-{type_code}" canonical_name = inc["name"] - if not is_disease and max_mag is not None: - canonical_name = f"{inc_type} {place_str or country} {event_date_short[:7] if event_date_short else ''}".strip() physical = { "max_magnitude": max_mag, diff --git a/tests/integration/_title_format_test.py b/tests/integration/_title_format_test.py new file mode 100644 index 00000000..ab7e97ad --- /dev/null +++ b/tests/integration/_title_format_test.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from disaster_report._title_format import ( + format_place, + format_title, + normalise_country, + normalise_subdivision, + smallest_place, +) +from disaster_report.models import ReportPlace + + +class TestFormatTitle: + def test_joins_non_empty_parts_with_single_spaces(self) -> None: + assert ( + format_title("Earthquake", "M7.5", "Yumare, Venezuela", "2026-06-24") + == "Earthquake M7.5 Yumare, Venezuela 2026-06-24" + ) + + def test_skips_empty_parts(self) -> None: + assert format_title("Flood", "", "Hunan, China", "2026-06-06") == "Flood Hunan, China 2026-06-06" + + def test_returns_empty_when_all_parts_empty(self) -> None: + assert format_title("", "", "") == "" + + def test_strips_whitespace_around_parts(self) -> None: + assert format_title(" Earthquake ", " M7.5 ") == "Earthquake M7.5" + + +class TestNormaliseSubdivision: + def test_strips_chinese_sheng_suffix(self) -> None: + assert normalise_subdivision("Hunan Sheng") == "Hunan" + + def test_strips_hong_kong_sar_suffix(self) -> None: + assert normalise_subdivision("Hong Kong SAR") == "Hong Kong" + + def test_strips_guangxi_zhuangzu_zizhiqu_suffix(self) -> None: + assert normalise_subdivision("Guangxi Zhuangzu Zizhiqu") == "Guangxi" + + def test_strips_accents(self) -> None: + assert normalise_subdivision("Pyrénées-Orientales") == "Pyrenees-Orientales" + + def test_no_change_for_plain_name(self) -> None: + assert normalise_subdivision("Yaracuy") == "Yaracuy" + + def test_returns_empty_for_empty_input(self) -> None: + assert normalise_subdivision("") == "" + + def test_case_insensitive_suffix_match(self) -> None: + assert normalise_subdivision("Taiwan sheng") == "Taiwan" + + +class TestNormaliseCountry: + def test_dr_congo_alias(self) -> None: + assert normalise_country("Democratic Republic of the Congo") == "DR Congo" + + def test_uk_alias(self) -> None: + assert normalise_country("United Kingdom") == "UK" + + def test_passes_through_short_form_unchanged(self) -> None: + assert normalise_country("Venezuela") == "Venezuela" + + def test_returns_empty_for_empty_input(self) -> None: + assert normalise_country("") == "" + + +class TestSmallestPlace: + def test_locality_wins_over_subdivision(self) -> None: + assert smallest_place([ReportPlace("VE", "Yaracuy", "Yumare")]) == ("Yumare", "Venezuela") + + def test_subdivision_used_when_no_locality(self) -> None: + assert smallest_place([ReportPlace("CN", "Hunan Sheng", "")]) == ("Hunan", "China") + + def test_country_fallback_when_no_locality_or_subdivision(self) -> None: + assert smallest_place([ReportPlace("CN", "", "")]) == ("China", "China") + + def test_returns_empty_pair_for_empty_places(self) -> None: + assert smallest_place([]) == ("", "") + + def test_uses_first_place_only(self) -> None: + places = [ReportPlace("PH", "Sarangani", "Glan"), ReportPlace("VE", "Yaracuy", "Yumare")] + assert smallest_place(places) == ("Glan", "Philippines") + + def test_subdivision_normalised(self) -> None: + assert smallest_place([ReportPlace("FR", "Pyrénées-Orientales", "")]) == ( + "Pyrenees-Orientales", + "France", + ) + + def test_strips_locality_offset_when_present(self) -> None: + assert smallest_place([ReportPlace("VE", "", "20 km ESE of Yumare")]) == ("Yumare", "Venezuela") + + +class TestFormatPlace: + def test_smallest_with_country(self) -> None: + assert format_place("Yumare", "Venezuela") == "Yumare, Venezuela" + + def test_smallest_equals_country(self) -> None: + assert format_place("China", "China") == "China" + + def test_empty_smallest_keeps_country(self) -> None: + assert format_place("", "China") == "China" + + def test_both_empty_returns_empty(self) -> None: + assert format_place("", "") == "" + + def test_empty_country_keeps_smallest(self) -> None: + assert format_place("Banda Sea", "") == "Banda Sea" diff --git a/tests/integration/_title_format_test.pyi b/tests/integration/_title_format_test.pyi new file mode 100644 index 00000000..99624a21 --- /dev/null +++ b/tests/integration/_title_format_test.pyi @@ -0,0 +1,40 @@ +class TestFormatTitle: + def test_joins_non_empty_parts_with_single_spaces(self) -> None: ... + def test_skips_empty_parts(self) -> None: ... + def test_returns_empty_when_all_parts_empty(self) -> None: ... + def test_strips_whitespace_around_parts(self) -> None: ... + + +class TestNormaliseSubdivision: + def test_strips_chinese_sheng_suffix(self) -> None: ... + def test_strips_hong_kong_sar_suffix(self) -> None: ... + def test_strips_guangxi_zhuangzu_zizhiqu_suffix(self) -> None: ... + def test_strips_accents(self) -> None: ... + def test_no_change_for_plain_name(self) -> None: ... + def test_returns_empty_for_empty_input(self) -> None: ... + def test_case_insensitive_suffix_match(self) -> None: ... + + +class TestNormaliseCountry: + def test_dr_congo_alias(self) -> None: ... + def test_uk_alias(self) -> None: ... + def test_passes_through_short_form_unchanged(self) -> None: ... + def test_returns_empty_for_empty_input(self) -> None: ... + + +class TestSmallestPlace: + def test_locality_wins_over_subdivision(self) -> None: ... + def test_subdivision_used_when_no_locality(self) -> None: ... + def test_country_fallback_when_no_locality_or_subdivision(self) -> None: ... + def test_returns_empty_pair_for_empty_places(self) -> None: ... + def test_uses_first_place_only(self) -> None: ... + def test_subdivision_normalised(self) -> None: ... + def test_strips_locality_offset_when_present(self) -> None: ... + + +class TestFormatPlace: + def test_smallest_with_country(self) -> None: ... + def test_smallest_equals_country(self) -> None: ... + def test_empty_smallest_keeps_country(self) -> None: ... + def test_both_empty_returns_empty(self) -> None: ... + def test_empty_country_keeps_smallest(self) -> None: ... diff --git a/tests/integration/gdacs_source_test.py b/tests/integration/gdacs_source_test.py index ebf2dd25..4b4eb345 100644 --- a/tests/integration/gdacs_source_test.py +++ b/tests/integration/gdacs_source_test.py @@ -212,3 +212,145 @@ def build_gdacs_report( report_date=report_date, raw_fields=raw_fields if raw_fields is not None else defaults, ) + + +class TestGDACSExtractCanonicalName: + def test_earthquake_extracts_magnitude_from_numeric_severity(self) -> None: + from disaster_report.sources.gdacs import _extract_canonical_name + + result = _extract_canonical_name( + {"eventtype": "EQ", "severity": 7.8, "severitytext": "Magnitude 7.8M"}, + [ReportPlace("AF", "Balkh", "")], + "2025-10-15", + "Earthquake", + ) + assert result == "Earthquake M7.8 Balkh, Afghanistan 2025-10-15" + + def test_earthquake_extracts_magnitude_from_severitytext_string(self) -> None: + from disaster_report.sources.gdacs import _extract_canonical_name + + result = _extract_canonical_name( + { + "eventtype": "EQ", + "severity": "Magnitude 5M, Depth:10km", + "severitytext": "Magnitude 5M, Depth:10km", + "title": "Earthquake in Afghanistan", + }, + [ReportPlace("AF", "Balkh", "")], + "2025-10-15", + "Earthquake", + ) + assert result == "Earthquake M5.0 Balkh, Afghanistan 2025-10-15" + + def test_tropical_cyclone_uses_eventname_when_present(self) -> None: + from disaster_report.sources.gdacs import _extract_canonical_name + + result = _extract_canonical_name( + { + "eventtype": "TC", + "eventname": "BAVI-26", + "title": "Red notification for tropical cyclone BAVI-26. Population.", + }, + [ReportPlace("CN", "", "")], + "2026-07-01", + "Tropical Cyclone", + ) + assert result == "Tropical Cyclone BAVI-26 China 2026-07-01" + + def test_tropical_cyclone_parses_storm_name_from_title_when_eventname_empty( + self, + ) -> None: + from disaster_report.sources.gdacs import _extract_canonical_name + + result = _extract_canonical_name( + { + "eventtype": "TC", + "eventname": "", + "title": "Tropical Cyclone TAPAH-25", + }, + [ReportPlace("CN", "Macao SAR", "")], + "2025-09-20", + "Tropical Cyclone", + ) + assert result == "Tropical Cyclone TAPAH-25 Macao, China 2025-09-20" + + def test_volcano_extracts_name_from_title(self) -> None: + from disaster_report.sources.gdacs import _extract_canonical_name + + result = _extract_canonical_name( + { + "eventtype": "VO", + "title": "Eruption Kanlaon", + }, + [ReportPlace("PH", "Negros Occidental", "")], + "2025-10-14", + "Volcano", + ) + assert result == "Volcano Kanlaon, Philippines 2025-10-14" + + def test_flood_omits_identifier(self) -> None: + from disaster_report.sources.gdacs import _extract_canonical_name + + result = _extract_canonical_name( + { + "eventtype": "FL", + "title": "Orange flood alert in China", + "alertlevel": "Orange", + }, + [ReportPlace("CN", "Hunan Sheng", "")], + "2026-06-06", + "Flood", + ) + assert result == "Flood Hunan, China 2026-06-06" + + def test_forest_fire_omits_identifier(self) -> None: + from disaster_report.sources.gdacs import _extract_canonical_name + + result = _extract_canonical_name( + { + "eventtype": "WF", + "title": "Orange forest fire notification in France", + "alertlevel": "Orange", + }, + [ReportPlace("FR", "Pyrénées-Orientales", "")], + "2026-07-04", + "Forest Fire", + ) + assert result == "Forest Fire Pyrenees-Orientales, France 2026-07-04" + + def test_drought_with_country(self) -> None: + from disaster_report.sources.gdacs import _extract_canonical_name + + result = _extract_canonical_name( + { + "eventtype": "DR", + "title": "Drought in Kenya, Somalia", + "alertlevel": "Orange", + }, + [ReportPlace("KE", "Shabeellaha Hoose", "")], + "2025-12-01", + "Drought", + ) + assert result == "Drought Shabeellaha Hoose, Kenya 2025-12-01" + + def test_drought_degenerate_no_place(self) -> None: + from disaster_report.sources.gdacs import _extract_canonical_name + + result = _extract_canonical_name( + {"eventtype": "DR", "title": "Drought Green", "alertlevel": "Green"}, + [], + "2025-12-21", + "Drought", + ) + assert result == "Drought 2025-12-21" + + def test_normalises_chinese_subdivision(self) -> None: + from disaster_report.sources.gdacs import _extract_canonical_name + + result = _extract_canonical_name( + {"eventtype": "FL", "title": "Flood in China"}, + [ReportPlace("CN", "Sichuan Sheng", "")], + "2026-07-15", + "Flood", + ) + assert result == "Flood Sichuan, China 2026-07-15" diff --git a/tests/integration/gdacs_source_test.pyi b/tests/integration/gdacs_source_test.pyi index 84bdfb5f..16fe7da5 100644 --- a/tests/integration/gdacs_source_test.pyi +++ b/tests/integration/gdacs_source_test.pyi @@ -20,6 +20,18 @@ class TestGDACSAdapter: def test_derive_keys_empty_places_skips_strict(self) -> None: ... def test_fetch_raises_typed_error_on_soft_200(self) -> None: ... +class TestGDACSExtractCanonicalName: + def test_earthquake_extracts_magnitude_from_numeric_severity(self) -> None: ... + def test_earthquake_extracts_magnitude_from_severitytext_string(self) -> None: ... + def test_tropical_cyclone_uses_eventname_when_present(self) -> None: ... + def test_tropical_cyclone_parses_storm_name_from_title_when_eventname_empty(self) -> None: ... + def test_volcano_extracts_name_from_title(self) -> None: ... + def test_flood_omits_identifier(self) -> None: ... + def test_forest_fire_omits_identifier(self) -> None: ... + def test_drought_with_country(self) -> None: ... + def test_drought_degenerate_no_place(self) -> None: ... + def test_normalises_chinese_subdivision(self) -> None: ... + def first_source_report(reports: list[SourceReport]) -> SourceReport: ... def build_gdacs_report( *, diff --git a/tests/integration/usgs_source_test.py b/tests/integration/usgs_source_test.py index 4cd2fb9a..a1f1672a 100644 --- a/tests/integration/usgs_source_test.py +++ b/tests/integration/usgs_source_test.py @@ -203,3 +203,52 @@ def build_kept_report( report_date=report_date, raw_fields=raw_fields if raw_fields is not None else defaults, ) + + +class TestUSGSExtractCanonicalName: + def test_usgs_earthquake_with_magnitude_and_locality(self) -> None: + from disaster_report.sources.usgs import _extract_canonical_name + + result = _extract_canonical_name( + {"mag": 7.5}, + [ReportPlace("VE", "Yaracuy", "Yumare")], + "2026-06-24", + "Earthquake", + ) + assert result == "Earthquake M7.5 Yumare, Venezuela 2026-06-24" + + def test_usgs_earthquake_missing_magnitude(self) -> None: + from disaster_report.sources.usgs import _extract_canonical_name + + result = _extract_canonical_name( + {}, [ReportPlace("VE", "", "Yumare")], "2026-06-24", "Earthquake" + ) + assert result == "Earthquake Yumare, Venezuela 2026-06-24" + + def test_usgs_earthquake_no_places(self) -> None: + from disaster_report.sources.usgs import _extract_canonical_name + + result = _extract_canonical_name({"mag": 5.5}, [], "2026-07-14", "Earthquake") + assert result == "Earthquake M5.5 2026-07-14" + + def test_usgs_earthquake_offshore_locality(self) -> None: + from disaster_report.sources.usgs import _extract_canonical_name + + result = _extract_canonical_name( + {"mag": 5.5}, + [ReportPlace("", "", "Ocean")], + "2026-07-14", + "Earthquake", + ) + assert result == "Earthquake M5.5 Ocean 2026-07-14" + + def test_usgs_earthquake_normalises_subdivision_when_no_locality(self) -> None: + from disaster_report.sources.usgs import _extract_canonical_name + + result = _extract_canonical_name( + {"mag": 6.0}, + [ReportPlace("PH", "Sarangani", "")], + "2026-06-07", + "Earthquake", + ) + assert result == "Earthquake M6.0 Sarangani, Philippines 2026-06-07" diff --git a/tests/integration/usgs_source_test.pyi b/tests/integration/usgs_source_test.pyi index 654cb5e3..0fbd9bd6 100644 --- a/tests/integration/usgs_source_test.pyi +++ b/tests/integration/usgs_source_test.pyi @@ -22,6 +22,13 @@ class TestUSGSAdapter: def test_derive_keys_offshore_no_places_skips_strict(self) -> None: ... def test_fetch_raises_typed_error_on_soft_404(self) -> None: ... +class TestUSGSExtractCanonicalName: + def test_usgs_earthquake_with_magnitude_and_locality(self) -> None: ... + def test_usgs_earthquake_missing_magnitude(self) -> None: ... + def test_usgs_earthquake_no_places(self) -> None: ... + def test_usgs_earthquake_offshore_locality(self) -> None: ... + def test_usgs_earthquake_normalises_subdivision_when_no_locality(self) -> None: ... + def first_source_report(reports: list[SourceReport]) -> SourceReport: ... def build_kept_report( *, diff --git a/tests/integration/who_source_test.py b/tests/integration/who_source_test.py index d9380032..46c71b10 100644 --- a/tests/integration/who_source_test.py +++ b/tests/integration/who_source_test.py @@ -38,13 +38,12 @@ def test_each_source_report_carries_who_source_name(self) -> None: def test_each_source_report_carries_specific_disease_type(self) -> None: import vcr - from disaster_report._search_keys import disease_from_title from disaster_report.sources.who import WHODiseaseOutbreakAdapter with vcr.use_cassette(f"tests/cassettes/{CASSETTE}"): reports = WHODiseaseOutbreakAdapter().fetch() assert all( - report.incident_type == (disease_from_title(report.name) or "Disease") + report.incident_type and report.incident_type != "Disease" for report in reports ) @@ -204,3 +203,124 @@ def build_who_report( report_date=report_date, raw_fields=raw_fields if raw_fields is not None else defaults, ) + + +class TestWHOExtractCanonicalName: + def test_disease_with_country_only(self) -> None: + from disaster_report.sources.who import _extract_canonical_name + + result = _extract_canonical_name( + {}, + [ReportPlace("UG", "", "")], + "2025-09-05", + "Ebola", + ) + assert result == "Ebola Uganda 2025-09-05" + + def test_disease_global_no_places_uses_Global(self) -> None: + from disaster_report.sources.who import _extract_canonical_name + + result = _extract_canonical_name( + {}, + [], + "2025-05-28", + "COVID-19", + ) + assert result == "COVID-19 Global 2025-05-28" + + def test_disease_uses_subdivision_when_available(self) -> None: + from disaster_report.sources.who import _extract_canonical_name + + result = _extract_canonical_name( + {}, + [ReportPlace("CD", "Kinshasa", "")], + "2025-09-05", + "Ebola", + ) + assert result == "Ebola Kinshasa, DR Congo 2025-09-05" + + def test_disease_normalises_messy_incident_type(self) -> None: + from disaster_report.sources.who import _extract_canonical_name + + result = _extract_canonical_name( + {}, + [ReportPlace("US", "", "")], + "2025-04-01", + "Avian Influenza A(H5N1)", + ) + assert result == "Avian Influenza United States 2025-04-01" + + def test_disease_unknown_falls_back_to_first_word(self) -> None: + from disaster_report.sources.who import _extract_canonical_name + + result = _extract_canonical_name( + {}, + [ReportPlace("UG", "", "")], + "2025-09-05", + "Mysterious Novel Pathogen X", + ) + assert result == "Mysterious Uganda 2025-09-05" + + def test_disease_normalises_long_country_name(self) -> None: + from disaster_report.sources.who import _extract_canonical_name + + result = _extract_canonical_name( + {}, + [ReportPlace("CD", "", "")], + "2025-09-05", + "Ebola", + ) + assert "DR Congo" in result + assert "Democratic Republic" not in result + + def test_title_suffix_country_overrides_body_scan_places(self) -> None: + from disaster_report.sources.who import _extract_canonical_name + + result = _extract_canonical_name( + {"title": "Marburg virus disease - Ethiopia"}, + [ + ReportPlace("AO", "", ""), + ReportPlace("CD", "", ""), + ReportPlace("ET", "", ""), + ReportPlace("UG", "", ""), + ], + "2025-09-05", + "Marburg", + ) + assert result == "Marburg Ethiopia 2025-09-05" + + def test_title_global_suffix_overrides_places(self) -> None: + from disaster_report.sources.who import _extract_canonical_name + + result = _extract_canonical_name( + {"title": "Yellow fever - Global situation"}, + [ReportPlace("BR", "", ""), ReportPlace("CO", "", "")], + "2025-06-24", + "Yellow fever", + ) + assert result == "Yellow fever Global 2025-06-24" + + def test_title_country_match_uses_subdivision_when_available(self) -> None: + from disaster_report.sources.who import _extract_canonical_name + + result = _extract_canonical_name( + {"title": "Ebola virus disease – Democratic Republic of the Congo"}, + [ + ReportPlace("CD", "Kinshasa", ""), + ReportPlace("UG", "Bundibugyo", ""), + ], + "2025-09-05", + "Ebola", + ) + assert result == "Ebola Kinshasa, DR Congo 2025-09-05" + + def test_title_without_suffix_falls_back_to_places(self) -> None: + from disaster_report.sources.who import _extract_canonical_name + + result = _extract_canonical_name( + {"title": "Ebola disease caused by Bundibugyo virus"}, + [ReportPlace("UG", "", "")], + "2025-09-05", + "Ebola", + ) + assert result == "Ebola Uganda 2025-09-05" diff --git a/tests/integration/who_source_test.pyi b/tests/integration/who_source_test.pyi index fc02333f..961acdc1 100644 --- a/tests/integration/who_source_test.pyi +++ b/tests/integration/who_source_test.pyi @@ -22,6 +22,18 @@ class TestWHODiseaseOutbreakAdapter: def test_derive_keys_global_zero_places_skips_strict(self) -> None: ... def test_fetch_raises_on_clean_400_error_path(self) -> None: ... +class TestWHOExtractCanonicalName: + def test_disease_with_country_only(self) -> None: ... + def test_disease_global_no_places_uses_Global(self) -> None: ... + def test_disease_uses_subdivision_when_available(self) -> None: ... + def test_disease_normalises_messy_incident_type(self) -> None: ... + def test_disease_unknown_falls_back_to_first_word(self) -> None: ... + def test_disease_normalises_long_country_name(self) -> None: ... + def test_title_suffix_country_overrides_body_scan_places(self) -> None: ... + def test_title_global_suffix_overrides_places(self) -> None: ... + def test_title_country_match_uses_subdivision_when_available(self) -> None: ... + def test_title_without_suffix_falls_back_to_places(self) -> None: ... + def first_source_report(reports: list[SourceReport]) -> SourceReport: ... def build_who_report( *,