Select UK local-authority lookup by dataset

changelog.d/541.fixed.md

@@ -0,0 +1 @@
+Fixed UK local-authority metadata selection by matching LAD22 and LAD23 lookup assets to the managed dataset identity and verifying their integrity.

docs/outputs.md

@@ -277,11 +277,17 @@ impacts = compute_uk_constituency_impacts(
 impacts.constituency_results
 ```
 
-`compute_uk_local_authority_impacts` follows the same pattern. Pass
-`constituency_csv_path` or `local_authority_csv_path` to use a specific
+`compute_uk_local_authority_impacts` follows the same pattern. Managed
+callers should pass the bundle dataset name as `dataset_identity`. The bundle
+maps named legacy datasets to the LAD22 lookup; LAD23 is the default for every
+other dataset, including future Microcosm releases. Bundle-managed lookup files
+are verified against their pinned SHA-256 hashes.
+
+Pass `constituency_csv_path` or `local_authority_csv_path` to use a specific
 metadata file; pass `download_missing_assets=False` to skip metadata downloads
-and use code-only labels. Legacy matrix arguments are accepted for backward
-compatibility but ignored. See [Regions](regions.md).
+and use code-only labels. An explicit metadata path is treated as a caller-owned
+override and is not checked against the bundle hash. Legacy matrix arguments are
+accepted for backward compatibility but ignored. See [Regions](regions.md).
 
 ## Writing your own
 

docs/regions.md

@@ -82,12 +82,15 @@ from policyengine.outputs import compute_uk_local_authority_impacts
 impacts = compute_uk_local_authority_impacts(
     baseline_simulation=baseline,
     reform_simulation=reform,
+    dataset_identity="enhanced_frs_2024_25",
 )
 impacts.local_authority_results
 ```
 
 Local-authority impacts follow the same longwise pattern using `la_code_oa`.
-Pass `local_authority_csv_path` to use a specific metadata CSV, or
+The bundle uses `dataset_identity` to select LAD22 metadata for explicitly
+listed legacy datasets and LAD23 metadata for all other datasets. Pass
+`local_authority_csv_path` to use a specific caller-owned metadata CSV, or
 `download_missing_assets=False` to skip metadata download and use code-only
 labels. The legacy `weight_matrix_path` and `year` arguments are accepted for
 backward compatibility but ignored.

src/policyengine/countries/uk/regions.py

@@ -21,6 +21,7 @@
     CONSTITUENCY_ASSET_SPEC,
     LOCAL_AUTHORITY_ASSET_SPEC,
     UK_GEOGRAPHY_BUCKET_URI,
+    verify_uk_geography_lookup_asset,
 )
 from policyengine.provenance.manifest import resolve_region_dataset_path
 
@@ -92,6 +93,7 @@ def _load_local_authorities_from_csv() -> list[dict]:
             gcs_bucket=LOCAL_AUTHORITY_ASSET_SPEC.bucket,
             gcs_key=LOCAL_AUTHORITY_ASSET_SPEC.lookup_csv_filename,
         )
+        verify_uk_geography_lookup_asset(csv_path, LOCAL_AUTHORITY_ASSET_SPEC)
         import pandas as pd
 
         df = pd.read_csv(csv_path)

src/policyengine/data/bundle/manifest.json

@@ -277,6 +277,34 @@
       "spm-calculator"
     ]
   },
+  "geography_assets": {
+    "uk": {
+      "local_authority": {
+        "assets": {
+          "lad22": {
+            "bucket": "policyengine-uk-data-private",
+            "filename": "local_authorities_lad22.csv",
+            "sha256": "58057da44c3dcd44bef22136515222d281f133d9e77e9a15172232248be1f21e"
+          },
+          "lad23": {
+            "bucket": "policyengine-uk-data-private",
+            "filename": "local_authorities_lad23.csv",
+            "sha256": "b9b7ed5a1072ba28c5bfd55fcc3fd974582f5bd82ee385601b65907bdee7ec2c"
+          }
+        },
+        "default_vintage": "lad23",
+        "lad22_dataset_identities": [
+          "enhanced_frs_2023_24",
+          "enhanced_frs_2024_25",
+          "enhanced_frs_2024_25_tiny",
+          "frs_2023_24",
+          "frs_2024_25",
+          "frs_2024_25_tiny",
+          "populace_uk_2023"
+        ]
+      }
+    }
+  },
   "measurements": {
     "spm": {
       "as_of": null,

src/policyengine/data/uk_geography_assets.py

@@ -1,14 +1,22 @@
 """Canonical UK geography asset metadata and resolution helpers."""
 
+import hashlib
+import json
 import os
 from dataclasses import dataclass
+from enum import Enum
+from functools import lru_cache
+from importlib.resources import files
 from pathlib import Path
 from typing import Optional, Sequence, Union
 
+from pydantic import BaseModel, Field, model_validator
+
 UK_GEOGRAPHY_BUCKET = "policyengine-uk-data-private"
 UK_GEOGRAPHY_BUCKET_URI = f"gs://{UK_GEOGRAPHY_BUCKET}"
 UK_GEOGRAPHY_DATA_DIR_ENV = "POLICYENGINE_UK_GEOGRAPHY_DATA_DIR"
 POLICYENGINE_DATA_FOLDER_ENV = "POLICYENGINE_DATA_FOLDER"
+PathLike = Union[os.PathLike, str]
 
 
 @dataclass(frozen=True)
@@ -21,6 +29,7 @@ class UKGeographyAssetSpec:
     bucket: str = UK_GEOGRAPHY_BUCKET
     weight_matrix_bucket: Optional[str] = None
     lookup_csv_bucket: Optional[str] = None
+    lookup_csv_sha256: Optional[str] = None
 
     @property
     def resolved_weight_matrix_bucket(self) -> str:
@@ -39,17 +48,128 @@ class UKGeographyAssetPaths:
     lookup_csv_path: str
 
 
+class LocalAuthorityVintage(str, Enum):
+    """Supported UK local-authority code rosters."""
+
+    LAD22 = "lad22"
+    LAD23 = "lad23"
+
+
+class UKLocalAuthorityLookupAsset(BaseModel):
+    """One immutable local-authority display-metadata asset."""
+
+    bucket: str
+    filename: str
+    sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
+
+
+class UKLocalAuthorityLookupConfiguration(BaseModel):
+    """Bundle-owned rules for choosing local-authority display metadata."""
+
+    default_vintage: LocalAuthorityVintage
+    lad22_dataset_identities: frozenset[str] = Field(default_factory=frozenset)
+    assets: dict[LocalAuthorityVintage, UKLocalAuthorityLookupAsset]
+
+    @model_validator(mode="after")
+    def validate_required_assets(self) -> "UKLocalAuthorityLookupConfiguration":
+        missing = set(LocalAuthorityVintage) - set(self.assets)
+        if missing:
+            values = ", ".join(sorted(vintage.value for vintage in missing))
+            raise ValueError(f"Missing UK local-authority lookup assets: {values}")
+        if self.default_vintage is not LocalAuthorityVintage.LAD23:
+            raise ValueError("The default UK local-authority vintage must be LAD23")
+        return self
+
+
 CONSTITUENCY_ASSET_SPEC = UKGeographyAssetSpec(
     geography_type="constituency",
     weight_matrix_filename="parliamentary_constituency_weights.h5",
     lookup_csv_filename="constituencies_2024.csv",
 )
 
-LOCAL_AUTHORITY_ASSET_SPEC = UKGeographyAssetSpec(
-    geography_type="local_authority",
-    weight_matrix_filename="local_authority_weights.h5",
-    lookup_csv_filename="local_authorities_2021.csv",
+
+@lru_cache
+def get_uk_local_authority_lookup_configuration() -> (
+    UKLocalAuthorityLookupConfiguration
+):
+    """Load the bundle-owned dataset-to-local-authority mapping."""
+
+    manifest_path = files("policyengine").joinpath("data", "bundle", "manifest.json")
+    bundle = json.loads(manifest_path.read_text())
+    try:
+        payload = bundle["geography_assets"]["uk"]["local_authority"]
+    except (KeyError, TypeError) as exc:
+        raise ValueError(
+            "The PolicyEngine bundle has no UK local-authority lookup configuration"
+        ) from exc
+    return UKLocalAuthorityLookupConfiguration.model_validate(payload)
+
+
+def resolve_uk_local_authority_vintage(
+    dataset_identity: Optional[str],
+) -> LocalAuthorityVintage:
+    """Resolve LAD22 only for explicitly mapped legacy datasets.
+
+    LAD23 is the default for every dataset identity not named in the bundle's
+    compatibility set, including future datasets.
+    """
+
+    configuration = get_uk_local_authority_lookup_configuration()
+    if dataset_identity in configuration.lad22_dataset_identities:
+        return LocalAuthorityVintage.LAD22
+    return configuration.default_vintage
+
+
+def uk_local_authority_asset_spec(
+    vintage: LocalAuthorityVintage,
+) -> UKGeographyAssetSpec:
+    """Build the asset specification for one local-authority code vintage."""
+
+    asset = get_uk_local_authority_lookup_configuration().assets[vintage]
+    return UKGeographyAssetSpec(
+        geography_type="local_authority",
+        weight_matrix_filename="local_authority_weights.h5",
+        lookup_csv_filename=asset.filename,
+        lookup_csv_bucket=asset.bucket,
+        lookup_csv_sha256=asset.sha256,
+    )
+
+
+def resolve_uk_local_authority_asset_spec(
+    dataset_identity: Optional[str],
+) -> UKGeographyAssetSpec:
+    """Select the local-authority asset for a managed dataset identity."""
+
+    return uk_local_authority_asset_spec(
+        resolve_uk_local_authority_vintage(dataset_identity)
+    )
+
+
+LOCAL_AUTHORITY_LAD22_ASSET_SPEC = uk_local_authority_asset_spec(
+    LocalAuthorityVintage.LAD22
 )
+LOCAL_AUTHORITY_LAD23_ASSET_SPEC = uk_local_authority_asset_spec(
+    LocalAuthorityVintage.LAD23
+)
+# Backwards-compatible name for callers that do not provide a dataset identity.
+LOCAL_AUTHORITY_ASSET_SPEC = LOCAL_AUTHORITY_LAD23_ASSET_SPEC
+
+
+def verify_uk_geography_lookup_asset(
+    path: PathLike,
+    spec: UKGeographyAssetSpec,
+) -> None:
+    """Verify a bundle-managed lookup file against its pinned content hash."""
+
+    if spec.lookup_csv_sha256 is None:
+        return
+    candidate = Path(path).expanduser()
+    actual_sha256 = hashlib.sha256(candidate.read_bytes()).hexdigest()
+    if actual_sha256 != spec.lookup_csv_sha256:
+        raise ValueError(
+            f"UK {spec.geography_type} lookup CSV failed its SHA-256 check: "
+            f"{candidate}; expected {spec.lookup_csv_sha256}, got {actual_sha256}"
+        )
 
 
 def _env_path(name: str) -> Optional[Path]:
@@ -121,9 +241,6 @@ def resolve(
         raise NotImplementedError
 
 
-PathLike = Union[os.PathLike, str]
-
-
 class LocalUKGeographyAssetStrategy(UKGeographyAssetStrategy):
     """Resolve geography assets from explicit paths or local search dirs."""
 
@@ -170,6 +287,8 @@ def resolve(
         )
 
         if weight_path is not None and csv_path is not None:
+            if lookup_csv_path is None:
+                verify_uk_geography_lookup_asset(csv_path, spec)
             self.last_error = None
             return UKGeographyAssetPaths(
                 weight_matrix_path=str(weight_path),
@@ -252,6 +371,9 @@ def resolve(
             self.last_error = f"GCS download failed for {spec.geography_type}: {exc}"
             return None
 
+        if lookup_csv_path is None:
+            verify_uk_geography_lookup_asset(resolved_lookup_csv_path, spec)
+
         self.last_error = None
         return UKGeographyAssetPaths(
             weight_matrix_path=str(resolved_weight_matrix_path),

src/policyengine/outputs/__init__.py

@@ -70,13 +70,23 @@
 from policyengine.outputs.uk_geography_assets import (
     CONSTITUENCY_ASSET_SPEC,
     LOCAL_AUTHORITY_ASSET_SPEC,
+    LOCAL_AUTHORITY_LAD22_ASSET_SPEC,
+    LOCAL_AUTHORITY_LAD23_ASSET_SPEC,
     GCSUKGeographyAssetStrategy,
+    LocalAuthorityVintage,
     LocalUKGeographyAssetStrategy,
     UKGeographyAssetPaths,
     UKGeographyAssetSpec,
     UKGeographyAssetStrategy,
+    UKLocalAuthorityLookupAsset,
+    UKLocalAuthorityLookupConfiguration,
     default_uk_geography_asset_strategies,
+    get_uk_local_authority_lookup_configuration,
     resolve_uk_geography_asset_paths,
+    resolve_uk_local_authority_asset_spec,
+    resolve_uk_local_authority_vintage,
+    uk_local_authority_asset_spec,
+    verify_uk_geography_lookup_asset,
 )
 
 __all__ = [
@@ -131,11 +141,21 @@
     "compute_uk_local_authority_impacts",
     "CONSTITUENCY_ASSET_SPEC",
     "LOCAL_AUTHORITY_ASSET_SPEC",
+    "LOCAL_AUTHORITY_LAD22_ASSET_SPEC",
+    "LOCAL_AUTHORITY_LAD23_ASSET_SPEC",
+    "LocalAuthorityVintage",
     "GCSUKGeographyAssetStrategy",
     "LocalUKGeographyAssetStrategy",
     "UKGeographyAssetPaths",
     "UKGeographyAssetSpec",
     "UKGeographyAssetStrategy",
+    "UKLocalAuthorityLookupAsset",
+    "UKLocalAuthorityLookupConfiguration",
     "default_uk_geography_asset_strategies",
+    "get_uk_local_authority_lookup_configuration",
+    "resolve_uk_local_authority_asset_spec",
+    "resolve_uk_local_authority_vintage",
     "resolve_uk_geography_asset_paths",
+    "uk_local_authority_asset_spec",
+    "verify_uk_geography_lookup_asset",
 ]

src/policyengine/outputs/local_authority_impact.py

@@ -10,7 +10,9 @@
 from pydantic import ConfigDict
 
 from policyengine.core import Output
-from policyengine.data.uk_geography_assets import LOCAL_AUTHORITY_ASSET_SPEC
+from policyengine.data.uk_geography_assets import (
+    resolve_uk_local_authority_asset_spec,
+)
 from policyengine.outputs.uk_geography_assets import (
     UKGeographyAssetStrategy,
 )
@@ -63,6 +65,7 @@ def compute_uk_local_authority_impacts(
     year: str = "2025",
     asset_strategies: Optional[Sequence[UKGeographyAssetStrategy]] = None,
     download_missing_assets: bool = True,
+    dataset_identity: Optional[str] = None,
 ) -> LocalAuthorityImpact:
     """Compute per-local-authority income changes for UK.
 
@@ -71,19 +74,22 @@ def compute_uk_local_authority_impacts(
         reform_simulation: Completed reform simulation.
         weight_matrix_path: Deprecated and ignored. Local-authority outputs
             now group by ``la_code_oa`` on the household output.
-        local_authority_csv_path: Optional path to local_authorities_2021.csv.
+        local_authority_csv_path: Optional path to a local-authority lookup CSV.
             If omitted, standard local paths are checked before downloading
             from GCS. If still unavailable, results use geography codes as names.
         year: Deprecated and ignored.
         asset_strategies: Deprecated and ignored.
         download_missing_assets: Whether to download the optional lookup CSV
             from GCS when no local CSV is found.
+        dataset_identity: Managed bundle dataset identity. Explicitly mapped
+            legacy datasets use LAD22 lookup metadata; all others use LAD23.
 
     Returns:
         LocalAuthorityImpact with local_authority_results populated.
     """
+    asset_spec = resolve_uk_local_authority_asset_spec(dataset_identity)
     lookup_csv_path = resolve_uk_geography_lookup_csv_path(
-        LOCAL_AUTHORITY_ASSET_SPEC,
+        asset_spec,
         lookup_csv_path=local_authority_csv_path,
         download_missing_assets=download_missing_assets,
     )