Calibrate pension-age Housing Benefit to DWP's GB figures

.gitignore

@@ -19,3 +19,5 @@
 **/_build
 !policyengine_uk_data/storage/*.csv
 **/version.json
+# Build output: household weights by local area, derived from FRS microdata
+policyengine_uk_data/storage/local_geography_weights.csv.gz

changelog.d/hb-dwp-age-targets.changed.md

@@ -0,0 +1 @@
+Calibrate pension-age Housing Benefit to DWP's Great Britain spending and claims over Pension Credit qualifying age, replacing the OBR Housing Benefit target that the model compared with UK-wide Housing Benefit. Modelled GB Housing Benefit for 2025-26 falls from about £11.7bn to £8.0bn (DWP: £12.9bn), because pension-age Housing Benefit was about 1.6 times DWP's figure while working-age, temporary and supported-accommodation Housing Benefit remain largely unmodelled. Stop setting `would_claim_uc` for benefit units whose adults have all reached State Pension age, and build with policyengine-uk 2.102.5 so pension-age families can make new Housing Benefit claims.

policyengine_uk_data/datasets/frs.py

@@ -24,6 +24,7 @@
     add_disability_benefit_flags_from_reported_amounts,
     drop_internal_disability_reported_amounts,
 )
+from policyengine_uk_data.utils.benefit_units import claimant_or_partner_variable
 from policyengine_uk_data.utils.datasets import (
     sum_to_entity,
     categorical,
@@ -392,6 +393,38 @@ def derive_is_parent_from_frs_microdata(
     return is_adult_record & has_dependent_children
 
 
+def derive_all_claimants_over_state_pension_age(
+    person_benunit_ids,
+    is_claimant_or_partner,
+    is_over_state_pension_age,
+    benunit_ids,
+) -> np.ndarray:
+    """Identify benefit units whose claimant and any partner have all reached
+    State Pension age.
+
+    Such a unit cannot claim Universal Credit (Welfare Reform Act 2012
+    s.4(1)(b)). ``is_claimant_or_partner`` should be the variable that
+    ``claimant_or_partner_variable`` names, so the rule matches the
+    pension-age route of policyengine-uk's ``housing_benefit_eligible``.
+    """
+
+    claimant = np.asarray(is_claimant_or_partner, dtype=bool)
+    over = claimant & np.asarray(is_over_state_pension_age, dtype=bool)
+    counts = (
+        pd.DataFrame(
+            {
+                "benunit": np.asarray(person_benunit_ids),
+                "claimants": claimant.astype(int),
+                "over": over.astype(int),
+            }
+        )
+        .groupby("benunit")[["claimants", "over"]]
+        .sum()
+        .reindex(np.asarray(benunit_ids), fill_value=0)
+    )
+    return ((counts.claimants > 0) & (counts.over == counts.claimants)).to_numpy()
+
+
 def _as_non_negative_array(values) -> np.ndarray:
     values = np.asarray(values, dtype=float)
     return np.maximum(np.nan_to_num(values, nan=0.0), 0.0)
@@ -1537,10 +1570,23 @@ def _reported_benunit_mask(person_column: str) -> np.ndarray:
         pension_credit_rate,
         reported_mask=_reported_benunit_mask("pension_credit_reported"),
     )
+    # A benefit unit whose claimant and any partner have all reached State
+    # Pension age cannot claim Universal Credit, so it never gets
+    # would_claim_uc, even if it reports UC. The draw still covers every unit,
+    # so the random stream and every other unit's value are unchanged. Ages
+    # are not rolled forward, so if State Pension age rises above a claimant's
+    # survey age in a later year, that unit stays without would_claim_uc there.
     pe_benunit["would_claim_uc"] = assign_takeup_with_reported_anchors(
         generator.random(len(pe_benunit)),
         universal_credit_rate,
         reported_mask=_reported_benunit_mask("universal_credit_reported"),
+    ) & ~derive_all_claimants_over_state_pension_age(
+        person_benunit_ids=sim.calculate("person_benunit_id", year).values,
+        is_claimant_or_partner=sim.calculate(
+            claimant_or_partner_variable(sim.tax_benefit_system.variables), year
+        ).values,
+        is_over_state_pension_age=sim.calculate("is_SP_age", year).values,
+        benunit_ids=pe_benunit.benunit_id,
     )
     pe_benunit["would_claim_tfc"] = generator.random(len(pe_benunit)) < tfc_rate
 

policyengine_uk_data/targets/build_loss_matrix.py

@@ -113,7 +113,7 @@ def create_target_matrix(
             col = _compute_column(target, ctx, year)
             if col is None:
                 continue
-            df[target.name] = col
+            df[target.name] = restrict_to_countries(col, ctx.country, target.countries)
             target_names.append(target.name)
             target_values.append(val)
         except Exception as e:
@@ -122,6 +122,18 @@ def create_target_matrix(
     return df, pd.Series(target_values, index=target_names)
 
 
+def restrict_to_countries(column, household_country, countries):
+    """Zero a household column outside the countries a target covers.
+
+    ``countries`` of None means the target covers the whole UK, and the
+    column is returned unchanged.
+    """
+    if countries is None:
+        return column
+    in_scope = np.isin(np.asarray(household_country), countries)
+    return np.asarray(column, dtype=float) * in_scope
+
+
 def _resolve_value(target: Target, year: int) -> float | None:
     """Get the target value for a year, falling back to nearest year.
 

policyengine_uk_data/targets/schema.py

@@ -19,6 +19,11 @@ class Unit(str, Enum):
     RATE = "rate"
 
 
+# DWP statistics cover Great Britain: benefits for Northern Ireland residents
+# are the Northern Ireland Executive's responsibility.
+GREAT_BRITAIN = ("ENGLAND", "SCOTLAND", "WALES")
+
+
 class Target(BaseModel):
     """A single calibration target from an official statistical source.
 
@@ -41,6 +46,10 @@ class Target(BaseModel):
     is_count: bool = False
     reference_url: str | None = None
     forecast_vintage: str | None = None
+    # Countries a national source covers when that is less than the UK, as
+    # values of the model's `country` variable. The loss matrix column only
+    # counts households in these countries. None means the whole UK.
+    countries: tuple[str, ...] | None = None
 
     # For targets needing custom simulation logic (UC splits,
     # counterfactuals). Excluded from serialisation.

policyengine_uk_data/targets/sources/dwp_housing_benefit.py

@@ -0,0 +1,197 @@
+"""DWP Housing Benefit targets by age group.
+
+Housing Benefit spending and caseload over and under Pension Credit
+qualifying age, from DWP's benefit expenditure and caseload tables for the
+Spring Forecast 2026 (Housing benefits sheet, nominal £ million and
+thousands of claims, 2022-23 to 2024-25 outturn, forecast after). Financial
+year 2025-26 is stored as 2025.
+
+Coverage is Great Britain: Housing Benefit for Northern Ireland residents is
+paid under Northern Ireland legislation and sits outside DWP's figures, so
+the model columns count GB households only.
+
+These targets replace OBR EFO table 4.9 "Housing benefit (not on JSA)".
+That line is DWP-funded spending only, while DWP's tables count all
+Housing Benefit paid, including the part local authorities fund (£0.79bn
+in 2025-26). The age split sums to that full amount, so targeting both
+would ask for two different GB totals.
+
+DWP splits claims by benefit rules rather than by age alone (Notes, note
+5): from 2024-25 the two lines equal its Pension Credit plus State Pension
+benefit groups and its ESA plus other working-age groups. Under regulation 5 of
+both Housing Benefit Regulations 2006 (SI 2006/213 and 2006/214), the
+pension-age rules apply when the claimant or partner has reached the
+qualifying age for Pension Credit, unless either is on Universal Credit,
+Income Support, income-based JSA or income-related ESA. A benefit unit is
+therefore over Pension Credit qualifying age here when its claimant or
+partner has reached State Pension age and it gets none of those benefits. Mixed-age
+couples who kept pension-age Housing Benefit after May 2019 fall in the
+older group; DWP does not publish its rule for them, and this assumes they
+sit in its Pension Credit and State Pension groups.
+
+Only the older group is calibrated. policyengine-uk pays working-age
+Housing Benefit only as a continuing award to families that report it and
+do not claim Universal Credit: 267 of the 770 working-age records that
+report it, about 17,000 weighted claims in 2025-26 against DWP's 460,000.
+DWP's working-age figure also includes temporary and supported
+accommodation (together £5.2bn of Housing Benefit in 2025-26, not split by
+age), which the FRS barely samples. A test build on 2026-09-30 that also
+targeted the younger group reached DWP's £5.8bn by loading it onto about
+three effective records, and fitted the other targets no better. The
+younger group's figures stay here for diagnostics and tests.
+
+Source: https://www.gov.uk/government/publications/benefit-expenditure-and-caseload-tables-2026
+"""
+
+import numpy as np
+
+from policyengine_uk_data.targets.schema import GREAT_BRITAIN, Target, Unit
+from policyengine_uk_data.utils.benefit_units import claimant_or_partner_variable
+
+_REFERENCE_URL = (
+    "https://www.gov.uk/government/publications/"
+    "benefit-expenditure-and-caseload-tables-2026"
+)
+_VINTAGE = "spring_2026"
+
+# Benefits that keep a claim under the working-age Housing Benefit rules
+# when the claimant or partner has reached Pension Credit qualifying age.
+_WORKING_AGE_BENEFITS = (
+    "universal_credit",
+    "income_support",
+    "jsa_income",
+    "esa_income",
+)
+
+# Housing benefits sheet rows "Housing Benefit over/under Pension Credit
+# qualifying age": expenditure in £ million (nominal) and caseload in
+# thousands (annual average, rounded to the nearest thousand by DWP).
+_SPENDING_GBP_M = {
+    "over": {
+        2022: 5_890.1,
+        2023: 6_248.1,
+        2024: 6_851.0,
+        2025: 7_114.7,
+        2026: 7_268.9,
+        2027: 7_340.4,
+        2028: 7_464.4,
+        2029: 7_747.9,
+        2030: 7_941.3,
+    },
+    "under": {
+        2022: 9_689.1,
+        2023: 9_524.5,
+        2024: 8_603.5,
+        2025: 5_778.1,
+        2026: 5_205.3,
+        2027: 5_493.8,
+        2028: 5_788.5,
+        2029: 6_151.5,
+        2030: 6_405.4,
+    },
+}
+_CASELOAD_THOUSANDS = {
+    "over": {
+        2022: 1_121,
+        2023: 1_108,
+        2024: 1_104,
+        2025: 1_109,
+        2026: 1_082,
+        2027: 1_057,
+        2028: 1_040,
+        2029: 1_036,
+        2030: 1_042,
+    },
+    "under": {
+        2022: 1_388,
+        2023: 1_243,
+        2024: 968,
+        2025: 460,
+        2026: 328,
+        2027: 339,
+        2028: 349,
+        2029: 361,
+        2030: 372,
+    },
+}
+
+
+def _over_pension_credit_age(ctx) -> np.ndarray:
+    """Benefit units assessed under the pension-age Housing Benefit rules."""
+    claimant = np.asarray(
+        ctx.pe_person(
+            claimant_or_partner_variable(ctx.sim.tax_benefit_system.variables)
+        ),
+        dtype=bool,
+    )
+    over = claimant & np.asarray(ctx.pe_person("is_SP_age"), dtype=bool)
+    any_over = (
+        np.asarray(ctx.sim.map_result(over.astype(float), "person", "benunit")) > 0
+    )
+    on_working_age_benefit = np.zeros_like(any_over)
+    for benefit in _WORKING_AGE_BENEFITS:
+        on_working_age_benefit |= np.asarray(ctx.sim.calculate(benefit).values) > 0
+    return any_over & ~on_working_age_benefit
+
+
+# Age groups the calibration targets; see the module docstring.
+_CALIBRATED_AGE_GROUPS = ("over",)
+
+
+def _make_compute(age_group: str, count: bool):
+    def compute(ctx, target: Target, year: int) -> np.ndarray:
+        housing_benefit = np.asarray(
+            ctx.sim.calculate("housing_benefit").values, dtype=float
+        )
+        in_group = _over_pension_credit_age(ctx)
+        if age_group == "under":
+            in_group = ~in_group
+        value = (housing_benefit > 0) if count else housing_benefit
+        return np.asarray(ctx.household_from_family(value * in_group), dtype=float)
+
+    return compute
+
+
+def get_targets() -> list[Target]:
+    return build_targets(_CALIBRATED_AGE_GROUPS)
+
+
+def build_targets(age_groups=("over", "under")) -> list[Target]:
+    """Housing Benefit spending and claims targets for these age groups."""
+    targets = []
+    for age_group in age_groups:
+        name = f"dwp/housing_benefit/{age_group}_pension_credit_age"
+        targets.append(
+            Target(
+                name=name,
+                variable="housing_benefit",
+                source="dwp",
+                unit=Unit.GBP,
+                values={
+                    year: value * 1e6
+                    for year, value in _SPENDING_GBP_M[age_group].items()
+                },
+                reference_url=_REFERENCE_URL,
+                forecast_vintage=_VINTAGE,
+                countries=GREAT_BRITAIN,
+                custom_compute=_make_compute(age_group, count=False),
+            )
+        )
+        targets.append(
+            Target(
+                name=f"{name}_claims",
+                variable="housing_benefit",
+                source="dwp",
+                unit=Unit.COUNT,
+                values={
+                    year: value * 1e3
+                    for year, value in _CASELOAD_THOUSANDS[age_group].items()
+                },
+                is_count=True,
+                reference_url=_REFERENCE_URL,
+                forecast_vintage=_VINTAGE,
+                countries=GREAT_BRITAIN,
+                custom_compute=_make_compute(age_group, count=True),
+            )
+        )
+    return targets

policyengine_uk_data/targets/sources/obr.py

@@ -482,11 +482,10 @@ def read_49(row_num: int) -> dict[int, float]:
                 result[fy[col]] = float(val) * 1e9
         return result
 
+    # "Housing benefit (not on JSA)" is not targeted: it is DWP-funded GB
+    # spending only, and dwp_housing_benefit.py targets total GB Housing
+    # Benefit split by age instead.
     benefit_rows = {
-        "housing_benefit": (
-            "Housing benefit (not on JSA)",
-            "housing_benefit",
-        ),
         "pip": (
             "Disability living allowance and personal independence p",
             "pip",