diff --git a/changelog.d/541.fixed.md b/changelog.d/541.fixed.md new file mode 100644 index 00000000..eb4b7b4d --- /dev/null +++ b/changelog.d/541.fixed.md @@ -0,0 +1 @@ +Fixed UK local-authority metadata selection by matching LAD22 and LAD23 lookup assets to the managed dataset identity and verifying their integrity. diff --git a/docs/outputs.md b/docs/outputs.md index ff1708e1..502fa5d2 100644 --- a/docs/outputs.md +++ b/docs/outputs.md @@ -277,11 +277,17 @@ impacts = compute_uk_constituency_impacts( impacts.constituency_results ``` -`compute_uk_local_authority_impacts` follows the same pattern. Pass -`constituency_csv_path` or `local_authority_csv_path` to use a specific +`compute_uk_local_authority_impacts` follows the same pattern. Managed +callers should pass the bundle dataset name as `dataset_identity`. The bundle +maps named legacy datasets to the LAD22 lookup; LAD23 is the default for every +other dataset, including future Microcosm releases. Bundle-managed lookup files +are verified against their pinned SHA-256 hashes. + +Pass `constituency_csv_path` or `local_authority_csv_path` to use a specific metadata file; pass `download_missing_assets=False` to skip metadata downloads -and use code-only labels. Legacy matrix arguments are accepted for backward -compatibility but ignored. See [Regions](regions.md). +and use code-only labels. An explicit metadata path is treated as a caller-owned +override and is not checked against the bundle hash. Legacy matrix arguments are +accepted for backward compatibility but ignored. See [Regions](regions.md). ## Writing your own diff --git a/docs/regions.md b/docs/regions.md index 3bf64d73..6a2e4093 100644 --- a/docs/regions.md +++ b/docs/regions.md @@ -82,12 +82,15 @@ from policyengine.outputs import compute_uk_local_authority_impacts impacts = compute_uk_local_authority_impacts( baseline_simulation=baseline, reform_simulation=reform, + dataset_identity="enhanced_frs_2024_25", ) impacts.local_authority_results ``` Local-authority impacts follow the same longwise pattern using `la_code_oa`. -Pass `local_authority_csv_path` to use a specific metadata CSV, or +The bundle uses `dataset_identity` to select LAD22 metadata for explicitly +listed legacy datasets and LAD23 metadata for all other datasets. Pass +`local_authority_csv_path` to use a specific caller-owned metadata CSV, or `download_missing_assets=False` to skip metadata download and use code-only labels. The legacy `weight_matrix_path` and `year` arguments are accepted for backward compatibility but ignored. diff --git a/src/policyengine/countries/uk/regions.py b/src/policyengine/countries/uk/regions.py index e6ea8c14..251b8c9a 100644 --- a/src/policyengine/countries/uk/regions.py +++ b/src/policyengine/countries/uk/regions.py @@ -21,6 +21,7 @@ CONSTITUENCY_ASSET_SPEC, LOCAL_AUTHORITY_ASSET_SPEC, UK_GEOGRAPHY_BUCKET_URI, + verify_uk_geography_lookup_asset, ) from policyengine.provenance.manifest import resolve_region_dataset_path @@ -92,6 +93,7 @@ def _load_local_authorities_from_csv() -> list[dict]: gcs_bucket=LOCAL_AUTHORITY_ASSET_SPEC.bucket, gcs_key=LOCAL_AUTHORITY_ASSET_SPEC.lookup_csv_filename, ) + verify_uk_geography_lookup_asset(csv_path, LOCAL_AUTHORITY_ASSET_SPEC) import pandas as pd df = pd.read_csv(csv_path) diff --git a/src/policyengine/data/bundle/manifest.json b/src/policyengine/data/bundle/manifest.json index 4a8dcffc..2cfd32aa 100644 --- a/src/policyengine/data/bundle/manifest.json +++ b/src/policyengine/data/bundle/manifest.json @@ -277,6 +277,34 @@ "spm-calculator" ] }, + "geography_assets": { + "uk": { + "local_authority": { + "assets": { + "lad22": { + "bucket": "policyengine-uk-data-private", + "filename": "local_authorities_lad22.csv", + "sha256": "58057da44c3dcd44bef22136515222d281f133d9e77e9a15172232248be1f21e" + }, + "lad23": { + "bucket": "policyengine-uk-data-private", + "filename": "local_authorities_lad23.csv", + "sha256": "b9b7ed5a1072ba28c5bfd55fcc3fd974582f5bd82ee385601b65907bdee7ec2c" + } + }, + "default_vintage": "lad23", + "lad22_dataset_identities": [ + "enhanced_frs_2023_24", + "enhanced_frs_2024_25", + "enhanced_frs_2024_25_tiny", + "frs_2023_24", + "frs_2024_25", + "frs_2024_25_tiny", + "populace_uk_2023" + ] + } + } + }, "measurements": { "spm": { "as_of": null, diff --git a/src/policyengine/data/uk_geography_assets.py b/src/policyengine/data/uk_geography_assets.py index 14257314..91d6d425 100644 --- a/src/policyengine/data/uk_geography_assets.py +++ b/src/policyengine/data/uk_geography_assets.py @@ -1,14 +1,22 @@ """Canonical UK geography asset metadata and resolution helpers.""" +import hashlib +import json import os from dataclasses import dataclass +from enum import Enum +from functools import lru_cache +from importlib.resources import files from pathlib import Path from typing import Optional, Sequence, Union +from pydantic import BaseModel, Field, model_validator + UK_GEOGRAPHY_BUCKET = "policyengine-uk-data-private" UK_GEOGRAPHY_BUCKET_URI = f"gs://{UK_GEOGRAPHY_BUCKET}" UK_GEOGRAPHY_DATA_DIR_ENV = "POLICYENGINE_UK_GEOGRAPHY_DATA_DIR" POLICYENGINE_DATA_FOLDER_ENV = "POLICYENGINE_DATA_FOLDER" +PathLike = Union[os.PathLike, str] @dataclass(frozen=True) @@ -21,6 +29,7 @@ class UKGeographyAssetSpec: bucket: str = UK_GEOGRAPHY_BUCKET weight_matrix_bucket: Optional[str] = None lookup_csv_bucket: Optional[str] = None + lookup_csv_sha256: Optional[str] = None @property def resolved_weight_matrix_bucket(self) -> str: @@ -39,17 +48,128 @@ class UKGeographyAssetPaths: lookup_csv_path: str +class LocalAuthorityVintage(str, Enum): + """Supported UK local-authority code rosters.""" + + LAD22 = "lad22" + LAD23 = "lad23" + + +class UKLocalAuthorityLookupAsset(BaseModel): + """One immutable local-authority display-metadata asset.""" + + bucket: str + filename: str + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + + +class UKLocalAuthorityLookupConfiguration(BaseModel): + """Bundle-owned rules for choosing local-authority display metadata.""" + + default_vintage: LocalAuthorityVintage + lad22_dataset_identities: frozenset[str] = Field(default_factory=frozenset) + assets: dict[LocalAuthorityVintage, UKLocalAuthorityLookupAsset] + + @model_validator(mode="after") + def validate_required_assets(self) -> "UKLocalAuthorityLookupConfiguration": + missing = set(LocalAuthorityVintage) - set(self.assets) + if missing: + values = ", ".join(sorted(vintage.value for vintage in missing)) + raise ValueError(f"Missing UK local-authority lookup assets: {values}") + if self.default_vintage is not LocalAuthorityVintage.LAD23: + raise ValueError("The default UK local-authority vintage must be LAD23") + return self + + CONSTITUENCY_ASSET_SPEC = UKGeographyAssetSpec( geography_type="constituency", weight_matrix_filename="parliamentary_constituency_weights.h5", lookup_csv_filename="constituencies_2024.csv", ) -LOCAL_AUTHORITY_ASSET_SPEC = UKGeographyAssetSpec( - geography_type="local_authority", - weight_matrix_filename="local_authority_weights.h5", - lookup_csv_filename="local_authorities_2021.csv", + +@lru_cache +def get_uk_local_authority_lookup_configuration() -> ( + UKLocalAuthorityLookupConfiguration +): + """Load the bundle-owned dataset-to-local-authority mapping.""" + + manifest_path = files("policyengine").joinpath("data", "bundle", "manifest.json") + bundle = json.loads(manifest_path.read_text()) + try: + payload = bundle["geography_assets"]["uk"]["local_authority"] + except (KeyError, TypeError) as exc: + raise ValueError( + "The PolicyEngine bundle has no UK local-authority lookup configuration" + ) from exc + return UKLocalAuthorityLookupConfiguration.model_validate(payload) + + +def resolve_uk_local_authority_vintage( + dataset_identity: Optional[str], +) -> LocalAuthorityVintage: + """Resolve LAD22 only for explicitly mapped legacy datasets. + + LAD23 is the default for every dataset identity not named in the bundle's + compatibility set, including future datasets. + """ + + configuration = get_uk_local_authority_lookup_configuration() + if dataset_identity in configuration.lad22_dataset_identities: + return LocalAuthorityVintage.LAD22 + return configuration.default_vintage + + +def uk_local_authority_asset_spec( + vintage: LocalAuthorityVintage, +) -> UKGeographyAssetSpec: + """Build the asset specification for one local-authority code vintage.""" + + asset = get_uk_local_authority_lookup_configuration().assets[vintage] + return UKGeographyAssetSpec( + geography_type="local_authority", + weight_matrix_filename="local_authority_weights.h5", + lookup_csv_filename=asset.filename, + lookup_csv_bucket=asset.bucket, + lookup_csv_sha256=asset.sha256, + ) + + +def resolve_uk_local_authority_asset_spec( + dataset_identity: Optional[str], +) -> UKGeographyAssetSpec: + """Select the local-authority asset for a managed dataset identity.""" + + return uk_local_authority_asset_spec( + resolve_uk_local_authority_vintage(dataset_identity) + ) + + +LOCAL_AUTHORITY_LAD22_ASSET_SPEC = uk_local_authority_asset_spec( + LocalAuthorityVintage.LAD22 ) +LOCAL_AUTHORITY_LAD23_ASSET_SPEC = uk_local_authority_asset_spec( + LocalAuthorityVintage.LAD23 +) +# Backwards-compatible name for callers that do not provide a dataset identity. +LOCAL_AUTHORITY_ASSET_SPEC = LOCAL_AUTHORITY_LAD23_ASSET_SPEC + + +def verify_uk_geography_lookup_asset( + path: PathLike, + spec: UKGeographyAssetSpec, +) -> None: + """Verify a bundle-managed lookup file against its pinned content hash.""" + + if spec.lookup_csv_sha256 is None: + return + candidate = Path(path).expanduser() + actual_sha256 = hashlib.sha256(candidate.read_bytes()).hexdigest() + if actual_sha256 != spec.lookup_csv_sha256: + raise ValueError( + f"UK {spec.geography_type} lookup CSV failed its SHA-256 check: " + f"{candidate}; expected {spec.lookup_csv_sha256}, got {actual_sha256}" + ) def _env_path(name: str) -> Optional[Path]: @@ -121,9 +241,6 @@ def resolve( raise NotImplementedError -PathLike = Union[os.PathLike, str] - - class LocalUKGeographyAssetStrategy(UKGeographyAssetStrategy): """Resolve geography assets from explicit paths or local search dirs.""" @@ -170,6 +287,8 @@ def resolve( ) if weight_path is not None and csv_path is not None: + if lookup_csv_path is None: + verify_uk_geography_lookup_asset(csv_path, spec) self.last_error = None return UKGeographyAssetPaths( weight_matrix_path=str(weight_path), @@ -252,6 +371,9 @@ def resolve( self.last_error = f"GCS download failed for {spec.geography_type}: {exc}" return None + if lookup_csv_path is None: + verify_uk_geography_lookup_asset(resolved_lookup_csv_path, spec) + self.last_error = None return UKGeographyAssetPaths( weight_matrix_path=str(resolved_weight_matrix_path), diff --git a/src/policyengine/outputs/__init__.py b/src/policyengine/outputs/__init__.py index ca85f2e9..7f426e6a 100644 --- a/src/policyengine/outputs/__init__.py +++ b/src/policyengine/outputs/__init__.py @@ -70,13 +70,23 @@ from policyengine.outputs.uk_geography_assets import ( CONSTITUENCY_ASSET_SPEC, LOCAL_AUTHORITY_ASSET_SPEC, + LOCAL_AUTHORITY_LAD22_ASSET_SPEC, + LOCAL_AUTHORITY_LAD23_ASSET_SPEC, GCSUKGeographyAssetStrategy, + LocalAuthorityVintage, LocalUKGeographyAssetStrategy, UKGeographyAssetPaths, UKGeographyAssetSpec, UKGeographyAssetStrategy, + UKLocalAuthorityLookupAsset, + UKLocalAuthorityLookupConfiguration, default_uk_geography_asset_strategies, + get_uk_local_authority_lookup_configuration, resolve_uk_geography_asset_paths, + resolve_uk_local_authority_asset_spec, + resolve_uk_local_authority_vintage, + uk_local_authority_asset_spec, + verify_uk_geography_lookup_asset, ) __all__ = [ @@ -131,11 +141,21 @@ "compute_uk_local_authority_impacts", "CONSTITUENCY_ASSET_SPEC", "LOCAL_AUTHORITY_ASSET_SPEC", + "LOCAL_AUTHORITY_LAD22_ASSET_SPEC", + "LOCAL_AUTHORITY_LAD23_ASSET_SPEC", + "LocalAuthorityVintage", "GCSUKGeographyAssetStrategy", "LocalUKGeographyAssetStrategy", "UKGeographyAssetPaths", "UKGeographyAssetSpec", "UKGeographyAssetStrategy", + "UKLocalAuthorityLookupAsset", + "UKLocalAuthorityLookupConfiguration", "default_uk_geography_asset_strategies", + "get_uk_local_authority_lookup_configuration", + "resolve_uk_local_authority_asset_spec", + "resolve_uk_local_authority_vintage", "resolve_uk_geography_asset_paths", + "uk_local_authority_asset_spec", + "verify_uk_geography_lookup_asset", ] diff --git a/src/policyengine/outputs/local_authority_impact.py b/src/policyengine/outputs/local_authority_impact.py index 40d50e8f..0c0ae943 100644 --- a/src/policyengine/outputs/local_authority_impact.py +++ b/src/policyengine/outputs/local_authority_impact.py @@ -10,7 +10,9 @@ from pydantic import ConfigDict from policyengine.core import Output -from policyengine.data.uk_geography_assets import LOCAL_AUTHORITY_ASSET_SPEC +from policyengine.data.uk_geography_assets import ( + resolve_uk_local_authority_asset_spec, +) from policyengine.outputs.uk_geography_assets import ( UKGeographyAssetStrategy, ) @@ -63,6 +65,7 @@ def compute_uk_local_authority_impacts( year: str = "2025", asset_strategies: Optional[Sequence[UKGeographyAssetStrategy]] = None, download_missing_assets: bool = True, + dataset_identity: Optional[str] = None, ) -> LocalAuthorityImpact: """Compute per-local-authority income changes for UK. @@ -71,19 +74,22 @@ def compute_uk_local_authority_impacts( reform_simulation: Completed reform simulation. weight_matrix_path: Deprecated and ignored. Local-authority outputs now group by ``la_code_oa`` on the household output. - local_authority_csv_path: Optional path to local_authorities_2021.csv. + local_authority_csv_path: Optional path to a local-authority lookup CSV. If omitted, standard local paths are checked before downloading from GCS. If still unavailable, results use geography codes as names. year: Deprecated and ignored. asset_strategies: Deprecated and ignored. download_missing_assets: Whether to download the optional lookup CSV from GCS when no local CSV is found. + dataset_identity: Managed bundle dataset identity. Explicitly mapped + legacy datasets use LAD22 lookup metadata; all others use LAD23. Returns: LocalAuthorityImpact with local_authority_results populated. """ + asset_spec = resolve_uk_local_authority_asset_spec(dataset_identity) lookup_csv_path = resolve_uk_geography_lookup_csv_path( - LOCAL_AUTHORITY_ASSET_SPEC, + asset_spec, lookup_csv_path=local_authority_csv_path, download_missing_assets=download_missing_assets, ) diff --git a/src/policyengine/outputs/uk_geography_assets.py b/src/policyengine/outputs/uk_geography_assets.py index 55c41052..5f1302ef 100644 --- a/src/policyengine/outputs/uk_geography_assets.py +++ b/src/policyengine/outputs/uk_geography_assets.py @@ -3,25 +3,45 @@ from policyengine.data.uk_geography_assets import ( CONSTITUENCY_ASSET_SPEC, LOCAL_AUTHORITY_ASSET_SPEC, + LOCAL_AUTHORITY_LAD22_ASSET_SPEC, + LOCAL_AUTHORITY_LAD23_ASSET_SPEC, UK_GEOGRAPHY_BUCKET, GCSUKGeographyAssetStrategy, + LocalAuthorityVintage, LocalUKGeographyAssetStrategy, UKGeographyAssetPaths, UKGeographyAssetSpec, UKGeographyAssetStrategy, + UKLocalAuthorityLookupAsset, + UKLocalAuthorityLookupConfiguration, default_uk_geography_asset_strategies, + get_uk_local_authority_lookup_configuration, resolve_uk_geography_asset_paths, + resolve_uk_local_authority_asset_spec, + resolve_uk_local_authority_vintage, + uk_local_authority_asset_spec, + verify_uk_geography_lookup_asset, ) __all__ = [ "CONSTITUENCY_ASSET_SPEC", "LOCAL_AUTHORITY_ASSET_SPEC", + "LOCAL_AUTHORITY_LAD22_ASSET_SPEC", + "LOCAL_AUTHORITY_LAD23_ASSET_SPEC", + "LocalAuthorityVintage", "UK_GEOGRAPHY_BUCKET", "UKGeographyAssetPaths", "UKGeographyAssetSpec", "UKGeographyAssetStrategy", + "UKLocalAuthorityLookupAsset", + "UKLocalAuthorityLookupConfiguration", "LocalUKGeographyAssetStrategy", "GCSUKGeographyAssetStrategy", "default_uk_geography_asset_strategies", + "get_uk_local_authority_lookup_configuration", + "resolve_uk_local_authority_asset_spec", + "resolve_uk_local_authority_vintage", "resolve_uk_geography_asset_paths", + "uk_local_authority_asset_spec", + "verify_uk_geography_lookup_asset", ] diff --git a/src/policyengine/outputs/uk_geography_impact.py b/src/policyengine/outputs/uk_geography_impact.py index ff2528ea..bf5a97cc 100644 --- a/src/policyengine/outputs/uk_geography_impact.py +++ b/src/policyengine/outputs/uk_geography_impact.py @@ -9,6 +9,7 @@ UKGeographyAssetSpec, default_download_dir, default_local_search_dirs, + verify_uk_geography_lookup_asset, ) @@ -39,6 +40,7 @@ def resolve_uk_geography_lookup_csv_path( for search_dir in default_local_search_dirs(): path = search_dir / spec.lookup_csv_filename if path.is_file(): + verify_uk_geography_lookup_asset(path, spec) return str(path) if not download_missing_assets: @@ -49,16 +51,18 @@ def resolve_uk_geography_lookup_csv_path( except ImportError: return None + target_path = default_download_dir() / spec.lookup_csv_filename + target_path.parent.mkdir(parents=True, exist_ok=True) try: - target_path = default_download_dir() / spec.lookup_csv_filename - target_path.parent.mkdir(parents=True, exist_ok=True) - return download_gcs_file( + downloaded_path = download_gcs_file( bucket=spec.resolved_lookup_csv_bucket, file_path=spec.lookup_csv_filename, local_path=str(target_path), ) except Exception: return None + verify_uk_geography_lookup_asset(downloaded_path, spec) + return str(downloaded_path) def _load_lookup_metadata( diff --git a/tests/test_local_authority_impact.py b/tests/test_local_authority_impact.py index 8e54a50a..65529d1d 100644 --- a/tests/test_local_authority_impact.py +++ b/tests/test_local_authority_impact.py @@ -9,7 +9,11 @@ from policyengine.outputs.local_authority_impact import ( compute_uk_local_authority_impacts, ) -from policyengine.outputs.uk_geography_assets import LOCAL_AUTHORITY_ASSET_SPEC +from policyengine.outputs.uk_geography_assets import ( + LOCAL_AUTHORITY_ASSET_SPEC, + LOCAL_AUTHORITY_LAD22_ASSET_SPEC, + LOCAL_AUTHORITY_LAD23_ASSET_SPEC, +) def _make_sim(household_data: dict) -> MagicMock: @@ -176,19 +180,62 @@ def test_compute_uses_local_lookup_csv_without_matrix_or_gcs( ] ).to_csv(csv_path, index=False) + impact = compute_uk_local_authority_impacts( + baseline, + reform, + local_authority_csv_path=str(csv_path), + download_missing_assets=True, + ) + + assert impact.local_authority_csv_path == str(csv_path) + assert len(impact.local_authority_results) == 2 + assert impact.local_authority_results[0]["local_authority_name"] == "A" + + +@pytest.mark.parametrize( + ("dataset_identity", "expected_spec"), + [ + ("populace_uk_2023", LOCAL_AUTHORITY_LAD22_ASSET_SPEC), + ("future_microcosm_release", LOCAL_AUTHORITY_LAD23_ASSET_SPEC), + ], +) +def test_compute_selects_lookup_from_dataset_identity( + tmp_path, + dataset_identity, + expected_spec, +): + baseline = _make_sim( + { + "la_code_oa": ["LA001"], + "household_net_income": [100.0], + "household_weight": [1.0], + } + ) + reform = _make_sim( + { + "la_code_oa": ["LA001"], + "household_net_income": [110.0], + "household_weight": [1.0], + } + ) + csv_path = _write_lookup_csv( + tmp_path, + [{"code": "LA001", "name": "A", "x": 0, "y": 0}], + ) + with patch( - "policyengine.outputs.uk_geography_impact.default_local_search_dirs", - return_value=[tmp_path], - ): - impact = compute_uk_local_authority_impacts( + "policyengine.outputs.local_authority_impact." + "resolve_uk_geography_lookup_csv_path", + return_value=csv_path, + ) as resolve_lookup: + compute_uk_local_authority_impacts( baseline, reform, - download_missing_assets=True, + dataset_identity=dataset_identity, + download_missing_assets=False, ) - assert impact.local_authority_csv_path == str(csv_path) - assert len(impact.local_authority_results) == 2 - assert impact.local_authority_results[0]["local_authority_name"] == "A" + assert resolve_lookup.call_args.args[0] == expected_spec def test_compute_local_authority_impacts_does_not_require_lookup_csv_or_matrix( diff --git a/tests/test_uk_geography_assets.py b/tests/test_uk_geography_assets.py index 3668cd12..6e4c7540 100644 --- a/tests/test_uk_geography_assets.py +++ b/tests/test_uk_geography_assets.py @@ -1,5 +1,6 @@ """Tests for UK geography asset resolution.""" +import hashlib from pathlib import Path from unittest.mock import patch @@ -8,11 +9,20 @@ from policyengine.outputs.uk_geography_assets import ( CONSTITUENCY_ASSET_SPEC, LOCAL_AUTHORITY_ASSET_SPEC, + LOCAL_AUTHORITY_LAD22_ASSET_SPEC, + LOCAL_AUTHORITY_LAD23_ASSET_SPEC, GCSUKGeographyAssetStrategy, + LocalAuthorityVintage, LocalUKGeographyAssetStrategy, UKGeographyAssetSpec, UKGeographyAssetStrategy, + get_uk_local_authority_lookup_configuration, resolve_uk_geography_asset_paths, + resolve_uk_local_authority_asset_spec, + resolve_uk_local_authority_vintage, +) +from policyengine.outputs.uk_geography_impact import ( + resolve_uk_geography_lookup_csv_path, ) @@ -20,6 +30,92 @@ def _touch(path: Path) -> None: path.write_text("test asset") +EXPECTED_LAD22_DATASETS = { + "enhanced_frs_2023_24", + "enhanced_frs_2024_25", + "enhanced_frs_2024_25_tiny", + "frs_2023_24", + "frs_2024_25", + "frs_2024_25_tiny", + "populace_uk_2023", +} + + +def test_local_authority_configuration_maps_only_declared_datasets_to_lad22(): + configuration = get_uk_local_authority_lookup_configuration() + + assert configuration.default_vintage is LocalAuthorityVintage.LAD23 + assert configuration.lad22_dataset_identities == EXPECTED_LAD22_DATASETS + for dataset_identity in EXPECTED_LAD22_DATASETS: + assert ( + resolve_uk_local_authority_vintage(dataset_identity) + is LocalAuthorityVintage.LAD22 + ) + assert ( + resolve_uk_local_authority_asset_spec(dataset_identity) + == LOCAL_AUTHORITY_LAD22_ASSET_SPEC + ) + + +@pytest.mark.parametrize("dataset_identity", [None, "", "future_microcosm_release"]) +def test_local_authority_configuration_defaults_unknown_datasets_to_lad23( + dataset_identity, +): + assert ( + resolve_uk_local_authority_vintage(dataset_identity) + is LocalAuthorityVintage.LAD23 + ) + assert ( + resolve_uk_local_authority_asset_spec(dataset_identity) + == LOCAL_AUTHORITY_LAD23_ASSET_SPEC + ) + assert LOCAL_AUTHORITY_ASSET_SPEC == LOCAL_AUTHORITY_LAD23_ASSET_SPEC + + +def test_lookup_resolver_rejects_bundle_asset_with_wrong_hash(tmp_path): + lookup_path = tmp_path / "lookup.csv" + lookup_path.write_text("code,x,y,name\nLA001,0,0,Authority\n") + spec = UKGeographyAssetSpec( + geography_type="test", + weight_matrix_filename="unused.h5", + lookup_csv_filename=lookup_path.name, + lookup_csv_sha256="0" * 64, + ) + + with patch( + "policyengine.outputs.uk_geography_impact.default_local_search_dirs", + return_value=[tmp_path], + ): + with pytest.raises(ValueError, match="failed its SHA-256 check"): + resolve_uk_geography_lookup_csv_path( + spec, + download_missing_assets=False, + ) + + +def test_lookup_resolver_accepts_bundle_asset_with_matching_hash(tmp_path): + lookup_path = tmp_path / "lookup.csv" + contents = b"code,x,y,name\nLA001,0,0,Authority\n" + lookup_path.write_bytes(contents) + spec = UKGeographyAssetSpec( + geography_type="test", + weight_matrix_filename="unused.h5", + lookup_csv_filename=lookup_path.name, + lookup_csv_sha256=hashlib.sha256(contents).hexdigest(), + ) + + with patch( + "policyengine.outputs.uk_geography_impact.default_local_search_dirs", + return_value=[tmp_path], + ): + resolved = resolve_uk_geography_lookup_csv_path( + spec, + download_missing_assets=False, + ) + + assert resolved == str(lookup_path) + + def test_local_strategy_resolves_explicit_paths(tmp_path): weight_matrix_path = tmp_path / "custom_weights.h5" lookup_csv_path = tmp_path / "custom_lookup.csv" @@ -38,13 +134,18 @@ def test_local_strategy_resolves_explicit_paths(tmp_path): def test_local_strategy_resolves_standard_files_from_search_dir(tmp_path): - weight_matrix_path = tmp_path / LOCAL_AUTHORITY_ASSET_SPEC.weight_matrix_filename - lookup_csv_path = tmp_path / LOCAL_AUTHORITY_ASSET_SPEC.lookup_csv_filename + spec = UKGeographyAssetSpec( + geography_type="local_authority", + weight_matrix_filename="local_authority_weights.h5", + lookup_csv_filename="local_authorities.csv", + ) + weight_matrix_path = tmp_path / spec.weight_matrix_filename + lookup_csv_path = tmp_path / spec.lookup_csv_filename _touch(weight_matrix_path) _touch(lookup_csv_path) paths = resolve_uk_geography_asset_paths( - LOCAL_AUTHORITY_ASSET_SPEC, + spec, asset_strategies=[LocalUKGeographyAssetStrategy(search_dirs=[tmp_path])], )