From 5b4f904d725757f77232774621b829d6563b3960 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 30 Sep 2026 07:02:22 -0400 Subject: [PATCH 1/3] Calibrate pension-age Housing Benefit to DWP's GB figures Replace the OBR table 4.9 Housing Benefit target, which is GB DWP-funded spending but was compared with UK-wide modelled Housing Benefit, with DWP Spring 2026 targets for Housing Benefit spending and claims over Pension Credit qualifying age in Great Britain. Targets can now name the countries they cover. The figures under Pension Credit qualifying age are kept but not calibrated: the model pays working-age Housing Benefit to too few records for a target to be met without concentrating it on a handful of them. Stop drawing would_claim_uc for benefit units whose adults have all reached State Pension age, keeping every other unit's draw unchanged. Require policyengine-uk 2.102.5, which lets pension-age families make new Housing Benefit claims (PolicyEngine/policyengine-uk#1901). Co-Authored-By: Claude Opus 5.5 --- .gitignore | 2 + changelog.d/hb-dwp-age-targets.fixed.md | 1 + policyengine_uk_data/datasets/frs.py | 42 +++ .../targets/build_loss_matrix.py | 14 +- policyengine_uk_data/targets/schema.py | 9 + .../targets/sources/dwp_housing_benefit.py | 190 ++++++++++++ policyengine_uk_data/targets/sources/obr.py | 7 +- .../tests/test_dwp_housing_benefit_targets.py | 277 ++++++++++++++++++ .../tests/test_would_claim_uc_pension_age.py | 56 ++++ pyproject.toml | 3 +- uv.lock | 87 +++++- 11 files changed, 674 insertions(+), 14 deletions(-) create mode 100644 changelog.d/hb-dwp-age-targets.fixed.md create mode 100644 policyengine_uk_data/targets/sources/dwp_housing_benefit.py create mode 100644 policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py create mode 100644 policyengine_uk_data/tests/test_would_claim_uc_pension_age.py diff --git a/.gitignore b/.gitignore index 9a741bc49..04136e33e 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,5 @@ **/_build !policyengine_uk_data/storage/*.csv **/version.json +# Build output: household weights by local area, derived from FRS microdata +policyengine_uk_data/storage/local_geography_weights.csv.gz diff --git a/changelog.d/hb-dwp-age-targets.fixed.md b/changelog.d/hb-dwp-age-targets.fixed.md new file mode 100644 index 000000000..2c4c56708 --- /dev/null +++ b/changelog.d/hb-dwp-age-targets.fixed.md @@ -0,0 +1 @@ +Calibrate pension-age Housing Benefit to DWP's Great Britain spending and claims over Pension Credit qualifying age, replacing the OBR Housing Benefit target that the model compared with UK-wide Housing Benefit. Stop setting `would_claim_uc` for benefit units whose adults have all reached State Pension age, and build with policyengine-uk 2.102.5 so pension-age families can make new Housing Benefit claims. diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index e440d4869..4a95842bb 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -392,6 +392,39 @@ def derive_is_parent_from_frs_microdata( return is_adult_record & has_dependent_children +def derive_every_adult_over_state_pension_age( + person_benunit_ids, + is_adult, + is_over_state_pension_age, + benunit_ids, +) -> np.ndarray: + """Identify benefit units with an adult, all of whose adults have reached + State Pension age. + + Such a unit cannot claim Universal Credit (Welfare Reform Act 2012 + s.4(1)(b)). The inputs are policyengine-uk's ``is_adult`` and + ``is_SP_age``, as in the pension-age route of its + ``housing_benefit_eligible``, so an 18 or 19 year old dependant makes the + unit working-age here as it does there. + """ + + is_adult = np.asarray(is_adult, dtype=bool) + over = is_adult & np.asarray(is_over_state_pension_age, dtype=bool) + counts = ( + pd.DataFrame( + { + "benunit": np.asarray(person_benunit_ids), + "adults": is_adult.astype(int), + "over": over.astype(int), + } + ) + .groupby("benunit")[["adults", "over"]] + .sum() + .reindex(np.asarray(benunit_ids), fill_value=0) + ) + return ((counts.adults > 0) & (counts.over == counts.adults)).to_numpy() + + def _as_non_negative_array(values) -> np.ndarray: values = np.asarray(values, dtype=float) return np.maximum(np.nan_to_num(values, nan=0.0), 0.0) @@ -1537,10 +1570,19 @@ def _reported_benunit_mask(person_column: str) -> np.ndarray: pension_credit_rate, reported_mask=_reported_benunit_mask("pension_credit_reported"), ) + # A benefit unit whose adults have all reached State Pension age cannot + # claim Universal Credit, so it never gets would_claim_uc, even if it + # reports UC. The draw still covers every unit, so the random stream and + # every other unit's value are unchanged. pe_benunit["would_claim_uc"] = assign_takeup_with_reported_anchors( generator.random(len(pe_benunit)), universal_credit_rate, reported_mask=_reported_benunit_mask("universal_credit_reported"), + ) & ~derive_every_adult_over_state_pension_age( + person_benunit_ids=sim.calculate("person_benunit_id", year).values, + is_adult=sim.calculate("is_adult", year).values, + is_over_state_pension_age=sim.calculate("is_SP_age", year).values, + benunit_ids=pe_benunit.benunit_id, ) pe_benunit["would_claim_tfc"] = generator.random(len(pe_benunit)) < tfc_rate diff --git a/policyengine_uk_data/targets/build_loss_matrix.py b/policyengine_uk_data/targets/build_loss_matrix.py index 9552aeea2..44b34272c 100644 --- a/policyengine_uk_data/targets/build_loss_matrix.py +++ b/policyengine_uk_data/targets/build_loss_matrix.py @@ -113,7 +113,7 @@ def create_target_matrix( col = _compute_column(target, ctx, year) if col is None: continue - df[target.name] = col + df[target.name] = restrict_to_countries(col, ctx.country, target.countries) target_names.append(target.name) target_values.append(val) except Exception as e: @@ -122,6 +122,18 @@ def create_target_matrix( return df, pd.Series(target_values, index=target_names) +def restrict_to_countries(column, household_country, countries): + """Zero a household column outside the countries a target covers. + + ``countries`` of None means the target covers the whole UK, and the + column is returned unchanged. + """ + if countries is None: + return column + in_scope = np.isin(np.asarray(household_country), countries) + return np.asarray(column, dtype=float) * in_scope + + def _resolve_value(target: Target, year: int) -> float | None: """Get the target value for a year, falling back to nearest year. diff --git a/policyengine_uk_data/targets/schema.py b/policyengine_uk_data/targets/schema.py index 97b814678..0a678a228 100644 --- a/policyengine_uk_data/targets/schema.py +++ b/policyengine_uk_data/targets/schema.py @@ -19,6 +19,11 @@ class Unit(str, Enum): RATE = "rate" +# DWP statistics cover Great Britain: benefits for Northern Ireland residents +# are the Northern Ireland Executive's responsibility. +GREAT_BRITAIN = ("ENGLAND", "SCOTLAND", "WALES") + + class Target(BaseModel): """A single calibration target from an official statistical source. @@ -41,6 +46,10 @@ class Target(BaseModel): is_count: bool = False reference_url: str | None = None forecast_vintage: str | None = None + # Countries a national source covers when that is less than the UK, as + # values of the model's `country` variable. The loss matrix column only + # counts households in these countries. None means the whole UK. + countries: tuple[str, ...] | None = None # For targets needing custom simulation logic (UC splits, # counterfactuals). Excluded from serialisation. diff --git a/policyengine_uk_data/targets/sources/dwp_housing_benefit.py b/policyengine_uk_data/targets/sources/dwp_housing_benefit.py new file mode 100644 index 000000000..69d17f9b6 --- /dev/null +++ b/policyengine_uk_data/targets/sources/dwp_housing_benefit.py @@ -0,0 +1,190 @@ +"""DWP Housing Benefit targets by age group. + +Housing Benefit spending and caseload over and under Pension Credit +qualifying age, from DWP's benefit expenditure and caseload tables for the +Spring Forecast 2026 (Housing benefits sheet, nominal £ million and +thousands of claims, 2022-23 to 2024-25 outturn, forecast after). Financial +year 2025-26 is stored as 2025. + +Coverage is Great Britain: Housing Benefit for Northern Ireland residents is +paid under Northern Ireland legislation and sits outside DWP's figures, so +the model columns count GB households only. + +These targets replace OBR EFO table 4.9 "Housing benefit (not on JSA)". +That line is DWP-funded spending only, while DWP's tables count all +Housing Benefit paid, including the part local authorities fund (£0.79bn +in 2025-26). The age split sums to that full amount, so targeting both +would ask for two different GB totals. + +DWP splits claims by benefit rules rather than by age alone (Notes, note +5): the two lines equal its Pension Credit plus State Pension benefit +groups and its ESA plus other working-age groups. Under regulation 5 of +both Housing Benefit Regulations 2006 (SI 2006/213 and 2006/214), the +pension-age rules apply when the claimant or partner has reached the +qualifying age for Pension Credit, unless either is on Universal Credit, +Income Support, income-based JSA or income-related ESA. A benefit unit is +therefore over Pension Credit qualifying age here when one of its adults +has reached State Pension age and it gets none of those benefits. Mixed-age +couples who kept pension-age Housing Benefit after May 2019 fall in the +older group, as they do in DWP's benefit groups. + +Only the older group is calibrated. policyengine-uk pays working-age +Housing Benefit only as a continuing award to families that report it and +do not claim Universal Credit: 267 of the 770 working-age records that +report it, about 24,000 weighted claims in 2025-26 against DWP's 460,000. +DWP's working-age figure also includes temporary and supported +accommodation (together £5.2bn of Housing Benefit in 2025-26, not split by +age), which the FRS barely samples. A test build on 2026-09-30 that also +targeted the younger group reached DWP's £5.8bn by loading it onto about +three effective records, and fitted the other targets no better. The +younger group's figures stay here for diagnostics and tests. + +Source: https://www.gov.uk/government/publications/benefit-expenditure-and-caseload-tables-2026 +""" + +import numpy as np + +from policyengine_uk_data.targets.schema import GREAT_BRITAIN, Target, Unit + +_REFERENCE_URL = ( + "https://www.gov.uk/government/publications/" + "benefit-expenditure-and-caseload-tables-2026" +) +_VINTAGE = "spring_2026" + +# Benefits that keep a claim under the working-age Housing Benefit rules +# when the claimant or partner has reached Pension Credit qualifying age. +_WORKING_AGE_BENEFITS = ( + "universal_credit", + "income_support", + "jsa_income", + "esa_income", +) + +# Housing benefits sheet rows "Housing Benefit over/under Pension Credit +# qualifying age": expenditure in £ million (nominal) and caseload in +# thousands (annual average, rounded to the nearest thousand by DWP). +_SPENDING_GBP_M = { + "over": { + 2022: 5_890.1, + 2023: 6_248.1, + 2024: 6_851.0, + 2025: 7_114.7, + 2026: 7_268.9, + 2027: 7_340.4, + 2028: 7_464.4, + 2029: 7_747.9, + 2030: 7_941.3, + }, + "under": { + 2022: 9_689.1, + 2023: 9_524.5, + 2024: 8_603.5, + 2025: 5_778.1, + 2026: 5_205.3, + 2027: 5_493.8, + 2028: 5_788.5, + 2029: 6_151.5, + 2030: 6_405.4, + }, +} +_CASELOAD_THOUSANDS = { + "over": { + 2022: 1_121, + 2023: 1_108, + 2024: 1_104, + 2025: 1_109, + 2026: 1_082, + 2027: 1_057, + 2028: 1_040, + 2029: 1_036, + 2030: 1_042, + }, + "under": { + 2022: 1_388, + 2023: 1_243, + 2024: 968, + 2025: 460, + 2026: 328, + 2027: 339, + 2028: 349, + 2029: 361, + 2030: 372, + }, +} + + +def _over_pension_credit_age(ctx) -> np.ndarray: + """Benefit units assessed under the pension-age Housing Benefit rules.""" + adult = np.asarray(ctx.pe_person("is_adult"), dtype=bool) + over = adult & np.asarray(ctx.pe_person("is_SP_age"), dtype=bool) + any_over = ( + np.asarray(ctx.sim.map_result(over.astype(float), "person", "benunit")) > 0 + ) + on_working_age_benefit = np.zeros_like(any_over) + for benefit in _WORKING_AGE_BENEFITS: + on_working_age_benefit |= np.asarray(ctx.sim.calculate(benefit).values) > 0 + return any_over & ~on_working_age_benefit + + +# Age groups the calibration targets; see the module docstring. +_CALIBRATED_AGE_GROUPS = ("over",) + + +def _make_compute(age_group: str, count: bool): + def compute(ctx, target: Target, year: int) -> np.ndarray: + housing_benefit = np.asarray( + ctx.sim.calculate("housing_benefit").values, dtype=float + ) + in_group = _over_pension_credit_age(ctx) + if age_group == "under": + in_group = ~in_group + value = (housing_benefit > 0) if count else housing_benefit + return np.asarray(ctx.household_from_family(value * in_group), dtype=float) + + return compute + + +def get_targets() -> list[Target]: + return build_targets(_CALIBRATED_AGE_GROUPS) + + +def build_targets(age_groups=("over", "under")) -> list[Target]: + """Housing Benefit spending and claims targets for these age groups.""" + targets = [] + for age_group in age_groups: + name = f"dwp/housing_benefit/{age_group}_pension_credit_age" + targets.append( + Target( + name=name, + variable="housing_benefit", + source="dwp", + unit=Unit.GBP, + values={ + year: value * 1e6 + for year, value in _SPENDING_GBP_M[age_group].items() + }, + reference_url=_REFERENCE_URL, + forecast_vintage=_VINTAGE, + countries=GREAT_BRITAIN, + custom_compute=_make_compute(age_group, count=False), + ) + ) + targets.append( + Target( + name=f"{name}_claims", + variable="housing_benefit", + source="dwp", + unit=Unit.COUNT, + values={ + year: value * 1e3 + for year, value in _CASELOAD_THOUSANDS[age_group].items() + }, + is_count=True, + reference_url=_REFERENCE_URL, + forecast_vintage=_VINTAGE, + countries=GREAT_BRITAIN, + custom_compute=_make_compute(age_group, count=True), + ) + ) + return targets diff --git a/policyengine_uk_data/targets/sources/obr.py b/policyengine_uk_data/targets/sources/obr.py index ad4e3cf17..bb4fc0ce6 100644 --- a/policyengine_uk_data/targets/sources/obr.py +++ b/policyengine_uk_data/targets/sources/obr.py @@ -482,11 +482,10 @@ def read_49(row_num: int) -> dict[int, float]: result[fy[col]] = float(val) * 1e9 return result + # "Housing benefit (not on JSA)" is not targeted: it is DWP-funded GB + # spending only, and dwp_housing_benefit.py targets total GB Housing + # Benefit split by age instead. benefit_rows = { - "housing_benefit": ( - "Housing benefit (not on JSA)", - "housing_benefit", - ), "pip": ( "Disability living allowance and personal independence p", "pip", diff --git a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py new file mode 100644 index 000000000..f92c2976c --- /dev/null +++ b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py @@ -0,0 +1,277 @@ +"""Tests for the DWP Housing Benefit targets by age group and GB scope.""" + +from types import SimpleNamespace + +import numpy as np +import openpyxl +import pandas as pd +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk_data.targets import get_all_targets +from policyengine_uk_data.targets.build_loss_matrix import restrict_to_countries +from policyengine_uk_data.targets.schema import GREAT_BRITAIN, Unit +from policyengine_uk_data.targets.sources import dwp_housing_benefit, obr + +# Other rows of the same DWP Housing benefits sheet, transcribed separately: +# total Housing Benefit is AME within the welfare cap + AME outside it + LA +# funded (£ million), and the total caseload (thousands). +_DWP_TOTAL_GBP_M = { + 2022: 14_839.6 + 160.8 + 578.7, + 2023: 14_946.4 + 111.0 + 715.2, + 2024: 14_593.4 + 57.2 + 804.0, + 2025: 12_103.3 + 0.0 + 789.6, + 2026: 11_643.4 + 0.0 + 830.8, + 2027: 11_969.5 + 0.0 + 864.6, + 2028: 12_352.1 + 0.0 + 900.8, + 2029: 12_950.1 + 0.0 + 949.2, + 2030: 13_368.2 + 0.0 + 978.5, +} +_DWP_TOTAL_CASELOAD_K = { + 2022: 2_509, + 2023: 2_351, + 2024: 2_072, + 2025: 1_569, + 2026: 1_410, + 2027: 1_396, + 2028: 1_390, + 2029: 1_397, + 2030: 1_414, +} + +_CALIBRATED = { + "dwp/housing_benefit/over_pension_credit_age", + "dwp/housing_benefit/over_pension_credit_age_claims", +} +_ALL = _CALIBRATED | { + "dwp/housing_benefit/under_pension_credit_age", + "dwp/housing_benefit/under_pension_credit_age_claims", +} + + +def _targets(): + """Both age groups, including the one calibration leaves out.""" + return {t.name: t for t in dwp_housing_benefit.build_targets()} + + +def test_targets_cover_great_britain_in_pounds_and_claims(): + targets = _targets() + assert set(targets) == _ALL + for name, target in targets.items(): + assert target.countries == GREAT_BRITAIN + assert target.variable == "housing_benefit" + assert target.is_count == name.endswith("_claims") + assert target.unit == (Unit.COUNT if target.is_count else Unit.GBP) + over = targets["dwp/housing_benefit/over_pension_credit_age"] + under = targets["dwp/housing_benefit/under_pension_credit_age_claims"] + assert over.values[2025] == 7_114.7e6 + assert under.values[2025] == 460e3 + + +def test_age_split_reconciles_with_dwp_totals(): + """Over + under matches DWP's total, including LA-funded spending.""" + targets = _targets() + over = targets["dwp/housing_benefit/over_pension_credit_age"].values + under = targets["dwp/housing_benefit/under_pension_credit_age"].values + over_k = targets["dwp/housing_benefit/over_pension_credit_age_claims"].values + under_k = targets["dwp/housing_benefit/under_pension_credit_age_claims"].values + for year, total in _DWP_TOTAL_GBP_M.items(): + assert abs((over[year] + under[year]) / 1e6 - total) < 0.25, year + for year, total in _DWP_TOTAL_CASELOAD_K.items(): + # DWP rounds each caseload to the nearest thousand. + assert abs((over_k[year] + under_k[year]) / 1e3 - total) <= 1, year + + +def test_only_pension_age_housing_benefit_is_calibrated(): + housing_benefit_targets = { + t.name for t in get_all_targets() if t.variable == "housing_benefit" + } + assert housing_benefit_targets == _CALIBRATED + + +def test_obr_housing_benefit_row_is_not_parsed(): + wb = openpyxl.Workbook() + ws = wb.active + ws.title = "4.9" + ws["B2"] = "Housing benefit (not on JSA)1" + ws["B3"] = "Pension credit" + for row in (2, 3): + for col in "CDEFGHI": + ws[f"{col}{row}"] = 6.0 + names = {t.name for t in obr._parse_welfare(wb)} + assert "obr/pension_credit" in names + assert not any("housing_benefit" in name for name in names) + + +def _ctx( + benunit_of_person, + is_adult, + is_sp_age, + benunit_hb, + household_of_benunit, + benunit_benefits=None, +): + """Minimal loss-matrix context over explicit entity mappings. + + ``benunit_benefits`` maps benefit-unit variables such as + ``universal_credit`` to amounts; any other benefit is zero. + """ + benunit_of_person = np.asarray(benunit_of_person) + household_of_benunit = np.asarray(household_of_benunit) + n_benunits = len(benunit_hb) + n_households = household_of_benunit.max() + 1 if n_benunits else 0 + + def map_result(values, source, target): + values = np.asarray(values, dtype=float) + if (source, target) == ("person", "benunit"): + return np.bincount(benunit_of_person, values, minlength=n_benunits) + if (source, target) == ("benunit", "household"): + return np.bincount(household_of_benunit, values, minlength=n_households) + raise AssertionError((source, target)) + + person = {"is_adult": np.asarray(is_adult), "is_SP_age": np.asarray(is_sp_age)} + benunit_values = {"housing_benefit": benunit_hb, **(benunit_benefits or {})} + sim = SimpleNamespace( + map_result=map_result, + calculate=lambda variable: SimpleNamespace( + values=np.asarray( + benunit_values.get(variable, np.zeros(n_benunits)), dtype=float + ) + ), + ) + return SimpleNamespace( + sim=sim, + pe_person=lambda variable: person[variable], + household_from_family=lambda values: map_result(values, "benunit", "household"), + ) + + +def _column(ctx, name): + target = _targets()[name] + return target.custom_compute(ctx, target, 2025) + + +def test_benefit_rules_assign_mixed_age_couples(): + # Benefit units, one household each: pensioner couple; mixed-age couple + # on pension-age rules; working-age single; pensioner with an + # 18-year-old dependant; mixed-age couple whose younger partner gets + # income-related ESA; mixed-age couple on Universal Credit. + ctx = _ctx( + benunit_of_person=[0, 0, 1, 1, 2, 3, 3, 4, 4, 5, 5], + is_adult=[1] * 11, + is_sp_age=[1, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0], + benunit_hb=[5_000, 4_000, 3_000, 2_000, 1_000, 500], + household_of_benunit=[0, 1, 2, 3, 4, 5], + benunit_benefits={ + "esa_income": [0, 0, 0, 0, 4_000, 0], + "universal_credit": [0, 0, 0, 0, 0, 6_000], + }, + ) + over = _column(ctx, "dwp/housing_benefit/over_pension_credit_age") + under = _column(ctx, "dwp/housing_benefit/under_pension_credit_age") + np.testing.assert_array_equal(over, [5_000, 4_000, 0, 2_000, 0, 0]) + np.testing.assert_array_equal(under, [0, 0, 3_000, 0, 1_000, 500]) + + +@st.composite +def _population(draw): + n_benunits = draw(st.integers(1, 12)) + n_households = draw(st.integers(1, n_benunits)) + household_of_benunit = draw( + st.lists( + st.integers(0, n_households - 1), + min_size=n_benunits, + max_size=n_benunits, + ) + ) + n_people = draw(st.integers(n_benunits, 4 * n_benunits)) + # Every benefit unit has at least one member. + benunit_of_person = list(range(n_benunits)) + draw( + st.lists( + st.integers(0, n_benunits - 1), + min_size=n_people - n_benunits, + max_size=n_people - n_benunits, + ) + ) + flags = st.lists(st.booleans(), min_size=n_people, max_size=n_people) + benunit_hb = draw( + st.lists( + st.one_of(st.just(0.0), st.floats(1, 50_000)), + min_size=n_benunits, + max_size=n_benunits, + ) + ) + benefit = st.lists( + st.sampled_from([0.0, 100.0]), min_size=n_benunits, max_size=n_benunits + ) + return dict( + benunit_of_person=benunit_of_person, + is_adult=draw(flags), + is_sp_age=draw(flags), + benunit_hb=benunit_hb, + household_of_benunit=household_of_benunit, + benunit_benefits={ + name: draw(benefit) for name in dwp_housing_benefit._WORKING_AGE_BENEFITS + }, + ) + + +@settings(max_examples=200, deadline=None) +@given(_population()) +def test_age_split_partitions_housing_benefit(population): + """Over + under is all Housing Benefit, in pounds and in claims, and + the older group is exactly the units with an adult over SPA and no + working-age income benefit.""" + ctx = _ctx(**population) + household = np.asarray(population["household_of_benunit"]) + hb = np.asarray(population["benunit_hb"]) + n_households = household.max() + 1 + over = _column(ctx, "dwp/housing_benefit/over_pension_credit_age") + under = _column(ctx, "dwp/housing_benefit/under_pension_credit_age") + over_k = _column(ctx, "dwp/housing_benefit/over_pension_credit_age_claims") + under_k = _column(ctx, "dwp/housing_benefit/under_pension_credit_age_claims") + np.testing.assert_allclose( + over + under, np.bincount(household, hb, minlength=n_households) + ) + np.testing.assert_array_equal( + over_k + under_k, np.bincount(household, hb > 0, minlength=n_households) + ) + people = pd.DataFrame( + { + "benunit": population["benunit_of_person"], + "older": np.asarray(population["is_adult"]) + & np.asarray(population["is_sp_age"]), + } + ) + older_unit = ( + people.groupby("benunit").older.any().reindex(range(len(hb)), fill_value=False) + ).to_numpy() + on_working_age_benefit = np.any( + [np.asarray(v) > 0 for v in population["benunit_benefits"].values()], axis=0 + ) + expected_over = np.bincount( + household, hb * (older_unit & ~on_working_age_benefit), minlength=n_households + ) + np.testing.assert_allclose(over, expected_over) + + +_COUNTRIES = ["ENGLAND", "SCOTLAND", "WALES", "NORTHERN_IRELAND"] + + +@settings(max_examples=200, deadline=None) +@given( + st.lists( + st.tuples(st.floats(-1e9, 1e9), st.sampled_from(_COUNTRIES)), + min_size=1, + max_size=50, + ) +) +def test_country_restriction_partitions_the_uk(rows): + column = np.array([value for value, _ in rows]) + country = np.array([c for _, c in rows]) + gb = restrict_to_countries(column, country, GREAT_BRITAIN) + ni = restrict_to_countries(column, country, ("NORTHERN_IRELAND",)) + np.testing.assert_allclose(gb + ni, column) + np.testing.assert_array_equal(gb[country == "NORTHERN_IRELAND"], 0) + np.testing.assert_array_equal(restrict_to_countries(gb, country, GREAT_BRITAIN), gb) + assert restrict_to_countries(column, country, None) is column diff --git a/policyengine_uk_data/tests/test_would_claim_uc_pension_age.py b/policyengine_uk_data/tests/test_would_claim_uc_pension_age.py new file mode 100644 index 000000000..251848335 --- /dev/null +++ b/policyengine_uk_data/tests/test_would_claim_uc_pension_age.py @@ -0,0 +1,56 @@ +"""Wholly pension-age benefit units never get would_claim_uc.""" + +import numpy as np +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk_data.datasets.frs import ( + derive_every_adult_over_state_pension_age, +) + + +def test_examples(): + # Units: pensioner couple, mixed-age couple, working-age single, + # pensioner with an 18-year-old dependant, a unit with no adult, and a + # unit with no members at all. + result = derive_every_adult_over_state_pension_age( + person_benunit_ids=[10, 10, 20, 20, 30, 40, 40, 50], + is_adult=[1, 1, 1, 1, 1, 1, 1, 0], + is_over_state_pension_age=[1, 1, 1, 0, 0, 1, 0, 0], + benunit_ids=[10, 20, 30, 40, 50, 60], + ) + np.testing.assert_array_equal(result, [True, False, False, False, False, False]) + + +@settings(max_examples=300, deadline=None) +@given( + st.lists( + st.tuples(st.integers(0, 6), st.booleans(), st.booleans()), + max_size=30, + ), + st.permutations(list(range(8))), +) +def test_matches_a_direct_definition(people, benunit_order): + """Differential check against a per-unit loop, for any unit order.""" + benunit_ids = np.array(benunit_order) + result = derive_every_adult_over_state_pension_age( + person_benunit_ids=[p[0] for p in people], + is_adult=[p[1] for p in people], + is_over_state_pension_age=[p[2] for p in people], + benunit_ids=benunit_ids, + ) + for i, unit in enumerate(benunit_ids): + adults = [over for b, adult, over in people if b == unit and adult] + assert result[i] == (len(adults) > 0 and all(adults)) + + +def test_built_dataset_has_no_pension_age_uc_claimants(baseline): + year = 2025 + adult = baseline.calculate("is_adult", year).values.astype(bool) + over = adult & baseline.calculate("is_SP_age", year).values.astype(bool) + adults = baseline.map_result(adult.astype(float), "person", "benunit") + adults_over = baseline.map_result(over.astype(float), "person", "benunit") + wholly_pension_age = (adults > 0) & (adults_over == adults) + would_claim_uc = baseline.calculate("would_claim_uc", year).values.astype(bool) + assert wholly_pension_age.any() + assert not (would_claim_uc & wholly_pension_age).any() diff --git a/pyproject.toml b/pyproject.toml index beff7f1a4..e46cfa9a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "policyengine", "google-cloud-storage", "google-auth", - "policyengine-uk>=2.93.0", + "policyengine-uk>=2.102.5", "microcalibrate>=0.18.0", "microimpute>=1.0.1", "ruff>=0.9.0", @@ -38,6 +38,7 @@ dependencies = [ dev = [ "ruff>=0.9.0", "pytest", + "hypothesis", "torch", "l0-python>=0.4.0", "tables", diff --git a/uv.lock b/uv.lock index a3e9f44d9..370de706e 100644 --- a/uv.lock +++ b/uv.lock @@ -577,6 +577,75 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/f4/124858007ddf3c61e9b144107304c9152fa80b5b6c168da07d86fe583cc1/huggingface_hub-1.1.5-py3-none-any.whl", hash = "sha256:e88ecc129011f37b868586bbcfae6c56868cae80cd56a79d61575426a3aa0d7d", size = 516000, upload-time = "2025-11-20T15:49:30.926Z" }, ] +[[package]] +name = "hypothesis" +version = "6.168.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/b7/13118bbc45d6d8b9d04e2de779e2a4ff23145ea39efa692b994fb874ca72/hypothesis-6.168.3.tar.gz", hash = "sha256:a43388f9067678fef6e13bdff325b6cfa6961a590498bb37f7ff31589c83bc75", size = 511022, upload-time = "2026-09-28T05:20:58.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/cd/746a2fc1e5e5ba34f07ea6ee1ad07525a3dde5ff4836db3a7980af49a891/hypothesis-6.168.3-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d20972ca134e652e9928ecd200966a8a2857adc2812c1d74b12a872e5cd503eb", size = 791536, upload-time = "2026-09-28T05:18:57.162Z" }, + { url = "https://files.pythonhosted.org/packages/c3/0f/7a158e377b69556c8e12c25c8fd811d0103e54c24a12dab1f3b9819f202e/hypothesis-6.168.3-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:eafbec09d3e87d13d8242411d1f5868b5e879f1bd6d95e233e9ff80c527bea1c", size = 787313, upload-time = "2026-09-28T05:20:18.122Z" }, + { url = "https://files.pythonhosted.org/packages/45/c6/7df9104ee359e8fcd77791781c47e70794964ca69673665e0de2a6590f4c/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73c5627497968cc62d140e9fde1ea21e12cac7b64dc419fea8286cd34ff1ad4e", size = 1124036, upload-time = "2026-09-28T05:20:04.372Z" }, + { url = "https://files.pythonhosted.org/packages/92/0f/8a61715404a73b9a82cb1f976803428d603cdea976a9c30266c72ad6a7ce/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dc56e6dc6eeb0aeb08ac463f847279bde2336c4786faf7ee0af4b93cd031a7", size = 1147875, upload-time = "2026-09-28T05:19:15.495Z" }, + { url = "https://files.pythonhosted.org/packages/e4/3f/2b16e95cc3b9a069afa23830a012aefe377ce4a457a227e5a5026eb827ff/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:753bb501f8560d2e321ed3b62596b4496c56f15668b0a0669231ccc3e6c4e80d", size = 1149489, upload-time = "2026-09-28T05:19:18.886Z" }, + { url = "https://files.pythonhosted.org/packages/f6/e7/d7cf6dc068bb02732b2a6e38a35a0dcb7f2137608198fc79d740f69d197c/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:71ab606c472449cb872ef2a7acaec679ee4f651b7f0a477a04ebc85d8933ef8c", size = 1191992, upload-time = "2026-09-28T05:19:21.83Z" }, + { url = "https://files.pythonhosted.org/packages/5f/26/f1e5b25dec15998e8375d722825ec3dae1075cbe7cb1f74221b2312a2e33/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26928956c54748e4dfa587333741ab750246123b93484955646ff316cca27eef", size = 1169911, upload-time = "2026-09-28T05:19:27.996Z" }, + { url = "https://files.pythonhosted.org/packages/03/36/901234a49147e6bf8a5b9e54b775706a223c2b52befea1106ea2aaf15d48/hypothesis-6.168.3-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:0bdfc53c041b61c854fa3761735736b991bc2bddafee45a361cfe4d43027c1ef", size = 1129406, upload-time = "2026-09-28T05:18:42.632Z" }, + { url = "https://files.pythonhosted.org/packages/55/c1/5fc9ff91ec9fba6bb527833386c66812524b5cc75fe7d4a4f9122ee1c420/hypothesis-6.168.3-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:650528e2b1e2a45e95c2df624d4b9364b8ac5a027ba84e1eea2bc5398dd01dcd", size = 1160399, upload-time = "2026-09-28T05:20:23.911Z" }, + { url = "https://files.pythonhosted.org/packages/9b/d2/49ad1ef5c55547c8abfd0fc571b3493806dda530fd5cb3d8263dbad86a72/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:209dc54cdb1b4d6d7020ad8d09e44c49756361b7183adacea4d6f5705085b595", size = 1299906, upload-time = "2026-09-28T05:18:58.417Z" }, + { url = "https://files.pythonhosted.org/packages/0d/5f/f98a26094c4f462c4215807bed0f9a6b5508b27a3c329bcba1bc8adc7ac7/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:af8ca98cd11dc7f9427bb90483abcd9688d4df2e8e63893a30a2423b027ebb11", size = 1425538, upload-time = "2026-09-28T05:20:49.074Z" }, + { url = "https://files.pythonhosted.org/packages/d0/b8/c5b4406e3a6591e51a42f85469351d1d824e0a54b84167aec9b2353759fe/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:f8122cfdd0bc0ba843063effa22ac310bdebdb3a1319bf1e63f14baa119c72ab", size = 1377084, upload-time = "2026-09-28T05:20:06.618Z" }, + { url = "https://files.pythonhosted.org/packages/e0/01/0d84a8ea469d024c15f602799c0b8410fbe03d99fdb0370449b33c3c916f/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4bedbb379eab34f792af7ee9a05aae04e9c08bbb52e0f90f5a2110d4fe4b2fbd", size = 1281162, upload-time = "2026-09-28T05:19:42.768Z" }, + { url = "https://files.pythonhosted.org/packages/22/b5/ea6435038de1a795f005cb603531e8ccc053febc50180f3ab55583c9c60e/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:5a953115b9f5c95133ab2d04efffeec96e5658c3207923dca7285f7db3e6bbef", size = 1300433, upload-time = "2026-09-28T05:18:55.634Z" }, + { url = "https://files.pythonhosted.org/packages/6c/78/fa6c77d9f64b6d592e69e582d83cbce7d7e56897faa4762885a2123b0166/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6173558e676ad25ed1e20507fa4024a0d816dd90f715f77006e5a28f19109026", size = 1336273, upload-time = "2026-09-28T05:19:08.273Z" }, + { url = "https://files.pythonhosted.org/packages/f1/56/4acd0d778bab818c9ea336fcba768bf43bc70db7f84cc373579298df4641/hypothesis-6.168.3-cp310-abi3-win32.whl", hash = "sha256:dc66390fb12d80585aa9222bf538ce8b7aa22cf5d118250647355a1c9e8f62f4", size = 678211, upload-time = "2026-09-28T05:19:23.355Z" }, + { url = "https://files.pythonhosted.org/packages/b0/db/0a02b146ad1c30f9716362f2e1a379dfd6b0f7be03ac1e22c9da24c9e2a9/hypothesis-6.168.3-cp310-abi3-win_amd64.whl", hash = "sha256:92325b276360fe86c5bf71a568c0d53a6d140b0de36dcf029f9164a17803bb24", size = 684906, upload-time = "2026-09-28T05:18:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/a3/c0/958deaf726848f96f52250740bf39f13f476b068e22f73e2b358eaa07532/hypothesis-6.168.3-cp310-abi3-win_arm64.whl", hash = "sha256:3cf6f1eeaf41cd8d60cf1f88fde905ca1dd77c906929a507d6ac7f66f2ccba2a", size = 683292, upload-time = "2026-09-28T05:19:57.404Z" }, + { url = "https://files.pythonhosted.org/packages/0c/a2/6787da846d929e52fc3344d803299c45782dbeac287528af08380e984bf8/hypothesis-6.168.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1b230a850de63334c16654a34a2d547e0179d36b9071d4439b3e7237f6d077e7", size = 793254, upload-time = "2026-09-28T05:19:36.294Z" }, + { url = "https://files.pythonhosted.org/packages/89/ba/2893f5ca42501f4d562cba3229c8994c3a3e5fac66ef60c1e0e0b5220a5d/hypothesis-6.168.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d63b0226cd3e0d8bdd97c3384b22a21934ed4d53246c1c93575dada616672499", size = 784699, upload-time = "2026-09-28T05:18:54.194Z" }, + { url = "https://files.pythonhosted.org/packages/b7/aa/7d7349daf75b71f6f35876f8de115e974c5c04d600e86df1b2779b0883ab/hypothesis-6.168.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0369f5df055f96e117ab12e5f249668ff144731ab5280bc7a205fdf81f990b89", size = 1123016, upload-time = "2026-09-28T05:20:36.372Z" }, + { url = "https://files.pythonhosted.org/packages/00/f0/7774e1ea072708ea46cb25c4aeee5f9978b6c271764809069e8a6789858e/hypothesis-6.168.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f076bcd0f77fdcdb7797826c879099573d03a02e228ebe649ea917081b962ac0", size = 1168989, upload-time = "2026-09-28T05:20:08.44Z" }, + { url = "https://files.pythonhosted.org/packages/a7/7d/113992abed9efbd7944e3a496a382da58152dce126fddf6419ff31a0a57d/hypothesis-6.168.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b823ba1fcec8da730f29316d010b06d3f7e0c3828dcf91020e24c55e7d24652a", size = 1298626, upload-time = "2026-09-28T05:19:40.986Z" }, + { url = "https://files.pythonhosted.org/packages/d5/65/3659fa5e733027e5b37a27486e57f4053535e40853d23501c422bb275043/hypothesis-6.168.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dd2849c269d674e4618590f3b48d443bd4c06b5aef3d8d869086c2e6d213d248", size = 1335184, upload-time = "2026-09-28T05:20:20.087Z" }, + { url = "https://files.pythonhosted.org/packages/b4/6c/35aab2221b5ea65125340f236e1a765a9e9f3b28d89a389ea0033fed29f7/hypothesis-6.168.3-cp313-cp313-win_amd64.whl", hash = "sha256:3ef7d26f5789e691401d5f87eafed9bd2763f0dbe47af6d6b66012a509404766", size = 682173, upload-time = "2026-09-28T05:19:12.762Z" }, + { url = "https://files.pythonhosted.org/packages/6e/79/27b0cb56ff5d2bc92458bf6fcebdb1f0dc01f57f043a178e9f9d74498169/hypothesis-6.168.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:e2d4c68729a13df9af4998d2652cfb5d541c5609b88c306880a5dfb284938ec2", size = 793287, upload-time = "2026-09-28T05:20:38.522Z" }, + { url = "https://files.pythonhosted.org/packages/e8/bd/5342f95c3bc36586777ef8cb8eba87ac7acf0184210327fb0e6d8654e295/hypothesis-6.168.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b345f818083ec99966a43ca4f7b38feb62c6920ce28572bc2bde4948a8b7eaba", size = 784857, upload-time = "2026-09-28T05:20:42.609Z" }, + { url = "https://files.pythonhosted.org/packages/4a/5f/fc774241518a0588d362680a3084275058aab86d6bc02a2c61e1073142d0/hypothesis-6.168.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0608c610fc002978fc5de0f471770d8817e9455cb8649a47983c9f8bccfc1897", size = 1123330, upload-time = "2026-09-28T05:19:29.735Z" }, + { url = "https://files.pythonhosted.org/packages/b0/e6/131f16775a3dca5f4fe27f0d6ad9a9851600098a54a872d163f6ebf0b664/hypothesis-6.168.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9dcf6448b1ecc37f2b23f2d1b3ddfc9ff6b6910614c15a642dc82f419b96037b", size = 1169178, upload-time = "2026-09-28T05:19:53.436Z" }, + { url = "https://files.pythonhosted.org/packages/06/61/c9f5bff8b73321c9fd12f2d69666c6e4861b97b60f5f24fdd6ac293007fd/hypothesis-6.168.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f9f094041dcce5119ebd7bab71062743ab02b6e654d056b370beda78c19e2", size = 1299127, upload-time = "2026-09-28T05:20:14.486Z" }, + { url = "https://files.pythonhosted.org/packages/e4/6d/d4618f8ab12dd76c4d58ea052252759aa2bf0c4e71f4b502ba5d577e0b9f/hypothesis-6.168.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d479985fe73af97badfb72dc6d20c6a353e486a36f6d065f600026e0cf954a86", size = 1335390, upload-time = "2026-09-28T05:20:10.326Z" }, + { url = "https://files.pythonhosted.org/packages/95/b6/727161e17cc297df1aeea945de05c532033b66751625bec192bc5b5ff707/hypothesis-6.168.3-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:785e2c45f8c08e274b4bf1ccae97f1a4e09407e9a68e80790cfab1d17a4a45fa", size = 624291, upload-time = "2026-09-28T05:18:36.376Z" }, + { url = "https://files.pythonhosted.org/packages/77/23/2f9b506392a0b8712440bcb781abc5713be9ffac1303b6d10f3669050882/hypothesis-6.168.3-cp314-cp314-win_amd64.whl", hash = "sha256:320920b1e3dae8611eee8a03d063cf2187446f2a17c38cfb8a7fc1466f71eee2", size = 682064, upload-time = "2026-09-28T05:20:16.336Z" }, + { url = "https://files.pythonhosted.org/packages/50/93/efabfd95eb2b69c0c9fa1e1c83c8290aa2715d46baeb5a7764faf09c133a/hypothesis-6.168.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:35380baa981108a7f60c4eab71e46acd8d8f58440520346a1a6aba06dca7e074", size = 791875, upload-time = "2026-09-28T05:19:37.789Z" }, + { url = "https://files.pythonhosted.org/packages/1b/fc/2a0ada1623a9a33048bbf9f00a7c92513398c6b8865cba5efb854262ce30/hypothesis-6.168.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f0aaaed00438fa6673d856aaee12a4a8afbde82a9f3c5ff487cacfb064239afe", size = 783412, upload-time = "2026-09-28T05:18:48.266Z" }, + { url = "https://files.pythonhosted.org/packages/41/31/73c615d37eeb12da0209555612c32ced628d860267c23cfd2691c6207aea/hypothesis-6.168.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27e6df1576bf838e7f484d4ae5cba92114497ca30afb917056bf1ea33e95675", size = 1121662, upload-time = "2026-09-28T05:19:26.225Z" }, + { url = "https://files.pythonhosted.org/packages/92/54/1893344a3b8bbf83fa7c9bb7e96f6836580213854f46671cb749f7b4706c/hypothesis-6.168.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3bc85014577982ec6d2e266edc5cd6e7a7d3674c648791ca983c67a52f89a0c4", size = 1167761, upload-time = "2026-09-28T05:18:49.85Z" }, + { url = "https://files.pythonhosted.org/packages/26/97/a61d82968febd25a0f08ffbbfabfaeead66f68fc5fa73f3c9ece1c36fc8f/hypothesis-6.168.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ccfc29505aa1cdcc254cb9cd701fd0811fef5480addd97d4127a00df023410f7", size = 1297309, upload-time = "2026-09-28T05:19:51.711Z" }, + { url = "https://files.pythonhosted.org/packages/bc/01/fe8e4cf6d39d02f4efadaa351427fa13722086cfc54a2da3579741c5177a/hypothesis-6.168.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32d0699566aaa93f9e97a44705d7164386f1e91de78d277f53bada35e78bcc96", size = 1334260, upload-time = "2026-09-28T05:19:00.121Z" }, + { url = "https://files.pythonhosted.org/packages/39/6a/09177ebc62f4778f94379ecade6a51299d3d8c4f4b75ba636bf3a0202364/hypothesis-6.168.3-cp314-cp314t-win_amd64.whl", hash = "sha256:28d88fa174ecbd4ecbd7bb290f06d0db084a3971c3f511ae2830b65b5e25500f", size = 681992, upload-time = "2026-09-28T05:19:48.043Z" }, + { url = "https://files.pythonhosted.org/packages/f9/f6/890bf33d608cd63348d3146a3ca83362fbf09e589c5e5230a00c403070f4/hypothesis-6.168.3-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:61f5782d807b1e6aa5c9beef1054cb2037e7d1add48a64972cd6ee447778781f", size = 791231, upload-time = "2026-09-28T05:18:43.998Z" }, + { url = "https://files.pythonhosted.org/packages/03/dd/fc36b204f8aa7437c676576d9437f0422de46aa8a729e6cd5bbb0224e759/hypothesis-6.168.3-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:7aacf3cf40c7ce8f9e4924d5b57bc0b068beafdfed2347bcf160a28b4cfbba7b", size = 783171, upload-time = "2026-09-28T05:19:20.353Z" }, + { url = "https://files.pythonhosted.org/packages/5c/d2/3cd087d577db67c404991d7723e64d7cd75570cdc82a0f7329aed6f2086c/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01768a03a30dc54df7fe457c0b34016c84d598fa00d5965ededab93ba4eb3408", size = 1121152, upload-time = "2026-09-28T05:18:39.095Z" }, + { url = "https://files.pythonhosted.org/packages/42/6f/a05a68cc66e7a22701c50bf94fb9ddeb36f4f5d0de3cc130066709eda621/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1a8a4ffc6c6e6e577f2bfbcfebf7cffbb310283ba2532c729570a0a752cebaaa", size = 1144069, upload-time = "2026-09-28T05:20:44.705Z" }, + { url = "https://files.pythonhosted.org/packages/4a/f4/fdd7a093fb2fb3a0ce24256c92ad5bf67936f4669bd06d5ceae4298150ec/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b987d73eba95183a7e59cca6d1925c588aa4307922d9852cc2b4d282a2ae4128", size = 1146692, upload-time = "2026-09-28T05:20:25.83Z" }, + { url = "https://files.pythonhosted.org/packages/f1/c8/6dbd4377e935505ae4fc8ee4c7b18c69994c4bbaca015447c470218bdbee/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4569c39bd97d9573e946429ed676f3b55a7c8ed80920addd67d384a47d59b38", size = 1189480, upload-time = "2026-09-28T05:20:46.734Z" }, + { url = "https://files.pythonhosted.org/packages/ba/bf/ff288b496b690000d2686dcfa7f67855e4c5a6dddb464ed8e4880be83bb2/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6042b8707a4b25bbbfe20b110258fb7549a68951e5525608a8ce06091039e9fc", size = 1167109, upload-time = "2026-09-28T05:19:24.758Z" }, + { url = "https://files.pythonhosted.org/packages/d9/d1/3fee2bc29fc747fabbb27e174515592e390809ca465d3bbbabbcfa3235a8/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:f413629de94d38a7a2ad259697a6752526143cb49c2e2d7bdba19381c80693f6", size = 1126823, upload-time = "2026-09-28T05:18:52.636Z" }, + { url = "https://files.pythonhosted.org/packages/58/6a/585294392fa6a6d9719446a042a777ba98ecca265d9e48cbedc10c88df0f/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f071737e4e775bebba07e319eb645a880d1e0186b4d24bad23255e849d86d483", size = 1155773, upload-time = "2026-09-28T05:20:51.268Z" }, + { url = "https://files.pythonhosted.org/packages/cd/4d/fa283ff79996debf1ff08593e01f8b773eb8211b19b3d6d5d491317a65bf/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:f59a3912858d0609c26054aa1c474847c797937f5e0e1c77f0d3f08032e50f09", size = 1296671, upload-time = "2026-09-28T05:18:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/42/05/98f9c2f628da5afabb6c3e8df5d2d299d2d50bf306b14a6e26d3954c4951/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:f09c05a23a8025dd5cad07c2ed299a46778e72d49ff0dd5bf353bcc0f7dc1580", size = 1422001, upload-time = "2026-09-28T05:19:04.188Z" }, + { url = "https://files.pythonhosted.org/packages/b8/c9/f2109ade29a7ec284d55bca328d784743e8b13e321576fd34ee7e65f99af/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_i686.whl", hash = "sha256:44ace770bda3a0301739fc5a413c764de790c739df1f1a7218dd049f8594d9f5", size = 1374182, upload-time = "2026-09-28T05:20:27.764Z" }, + { url = "https://files.pythonhosted.org/packages/ce/6d/e05d5f72441564a3bebc71fa155deafd0fd3b015d6014ec8a00edf6e42bb/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:03b131043608f94a2578a2a896a1a72092acb7079815eef5b8513b08447e0e62", size = 1278402, upload-time = "2026-09-28T05:20:56.332Z" }, + { url = "https://files.pythonhosted.org/packages/96/d7/6988a7f1f69c5c530687dce03a32c157e2d878e3a2e32af9269ce016b78a/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:e9784aca26eddfe99b03a0292320db742cc8a74200ef864fdce949f526f973cd", size = 1297770, upload-time = "2026-09-28T05:19:11.387Z" }, + { url = "https://files.pythonhosted.org/packages/a2/28/5bb82b60b836bd2329e1fe01ad94efc2b14dfa806f8e50ed14b795d78770/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:2b52ac363096232bebc2add117e9178d91f4248f4cdb919fd1026b5f86a4bb16", size = 1333977, upload-time = "2026-09-28T05:20:02.483Z" }, + { url = "https://files.pythonhosted.org/packages/8e/7d/d419841b8f65481ea1a50c4ba36f2670d48b8c7a51e4e8586089564cf7f6/hypothesis-6.168.3-cp315-abi3.abi3t-win32.whl", hash = "sha256:d28e3a6b511a74ce37df5274b51f36c2b274fea7365e00b16f0c81e22acd5957", size = 675394, upload-time = "2026-09-28T05:19:55.237Z" }, + { url = "https://files.pythonhosted.org/packages/d8/d2/1de6a2ad100e44817f2e9a8e8ba3eeaf4769621f4f87f2d4966c4971fcc4/hypothesis-6.168.3-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:7b9638789548361a57d984f56619ac694a914c328181d911271618409261ff4a", size = 681699, upload-time = "2026-09-28T05:19:09.957Z" }, + { url = "https://files.pythonhosted.org/packages/c0/79/be3370fd02734d6b1d950183ae9224580d19fa8d356b95348d60f1e3eeb7/hypothesis-6.168.3-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:65d78e4357ec48ed2c67825f06740ee3599be4cfe770a6092bed07108d679ac5", size = 679849, upload-time = "2026-09-28T05:19:06.884Z" }, +] + [[package]] name = "idna" version = "3.11" @@ -1322,7 +1391,7 @@ wheels = [ [[package]] name = "policyengine-core" -version = "3.31.1" +version = "3.32.11" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dpath" }, @@ -1342,14 +1411,14 @@ dependencies = [ { name = "standard-imghdr" }, { name = "wheel" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9f/1e/cd58cb6947720a5b7850c06a988abbc0eef53e2be192ed49a17ed62d78d4/policyengine_core-3.31.1.tar.gz", hash = "sha256:3ca8f666c3e67b0c0ce992fa3dc9d1ff2c095a3860f3d719883588b2f9e44ec3", size = 500450, upload-time = "2026-08-28T05:06:43.16Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/9f/0789273f03bad9d7274e487c97f212559fbbe1977a0320b38eeff411bbe2/policyengine_core-3.32.11.tar.gz", hash = "sha256:e32f5e89d0a3abbf4d5e7c5233168c26033b45171373947f604ef18920445b5d", size = 430637, upload-time = "2026-09-30T02:10:11.155Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/74/ce/995992049664b6b4cecd1e4db76b143e07aaee8239043057436f48ccbe75/policyengine_core-3.31.1-py3-none-any.whl", hash = "sha256:308200b28aa26bb79d7f0badebac391c0a81704d45ba4ef48ac79e88f0b21e6b", size = 245719, upload-time = "2026-08-28T05:06:41.769Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8b/f6b21e7872ce4681edf3b1794549fcaed19f4d3efc1dfafdb80044952e28/policyengine_core-3.32.11-py3-none-any.whl", hash = "sha256:591acd6161b992f09070c582e4f830f636a8be863b226d2de546fffc5bbe602a", size = 247934, upload-time = "2026-09-30T02:10:09.831Z" }, ] [[package]] name = "policyengine-uk" -version = "2.93.0" +version = "2.102.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -1357,14 +1426,14 @@ dependencies = [ { name = "pydantic" }, { name = "tables" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/af/04/ab8fbb526c22d5c422eed374244d977b123241baa6f96a604c600876647a/policyengine_uk-2.93.0.tar.gz", hash = "sha256:e3b8eab824e7dad3a1fd61fa8503442a8b1b2285e05fff7ab60f93b6a9b22578", size = 1246786, upload-time = "2026-08-28T14:55:16.266Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5d/a1/4700bcb06a11541236f1e77c44d68842731a3854e15ebf7117da0480d59e/policyengine_uk-2.102.5.tar.gz", hash = "sha256:1526264cd114e40c99bebbc9e2603ec6004c5f2a91bbf5d1a89018be4d05a29b", size = 1319280, upload-time = "2026-09-30T07:53:44.344Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6f/74/5e626f9f80b9485b10bcccbef8d93a635abe309bfdfad437b5de38a80fae/policyengine_uk-2.93.0-py3-none-any.whl", hash = "sha256:4f95feffa3b27fab927f9b903f8a9f1f0d63b2fba209d6b8ac010f0f07a5ec7c", size = 2046525, upload-time = "2026-08-28T14:55:14.114Z" }, + { url = "https://files.pythonhosted.org/packages/11/a0/da29962570fc4091227cf9264b56e2ba66f4c268b391108d1f643e38c649/policyengine_uk-2.102.5-py3-none-any.whl", hash = "sha256:c72dc93cfda8d267f24bbf6675bd27f9c50604844bd6dbefaa3cab6d74f371a6", size = 2195332, upload-time = "2026-09-30T07:53:42.803Z" }, ] [[package]] name = "policyengine-uk-data" -version = "1.56.16" +version = "1.57.4" source = { editable = "." } dependencies = [ { name = "google-auth" }, @@ -1392,6 +1461,7 @@ dependencies = [ dev = [ { name = "build" }, { name = "furo" }, + { name = "hypothesis" }, { name = "itables" }, { name = "l0-python" }, { name = "pytest" }, @@ -1410,6 +1480,7 @@ requires-dist = [ { name = "google-auth" }, { name = "google-cloud-storage" }, { name = "huggingface-hub" }, + { name = "hypothesis", marker = "extra == 'dev'" }, { name = "itables", marker = "extra == 'dev'" }, { name = "l0-python", marker = "extra == 'dev'", specifier = ">=0.4.0" }, { name = "microcalibrate", specifier = ">=0.18.0" }, @@ -1419,7 +1490,7 @@ requires-dist = [ { name = "pandas" }, { name = "policyengine" }, { name = "policyengine-core", specifier = ">=3.19.4" }, - { name = "policyengine-uk", specifier = ">=2.93.0" }, + { name = "policyengine-uk", specifier = ">=2.102.5" }, { name = "pydantic", specifier = ">=2.0" }, { name = "pytest", marker = "extra == 'dev'" }, { name = "pyyaml" }, From 25971022f8f7384f8c9436e2e5734fb2fb40d0f6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 30 Sep 2026 08:05:20 -0400 Subject: [PATCH 2/3] Address review: release note, docstring precision, loss-matrix test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit State in the release note that modelled GB Housing Benefit falls to about £8.0bn. Mark the mixed-age classification as an assumption, date the benefit-group equality from 2024-25, and quote the calibrated build's working-age claims. Add a test that the calibrated columns build on the dataset, since the loss matrix skips a target whose column raises, and note that would_claim_uc uses survey-year State Pension age. Co-Authored-By: Claude Opus 5.5 --- changelog.d/hb-dwp-age-targets.changed.md | 1 + changelog.d/hb-dwp-age-targets.fixed.md | 1 - policyengine_uk_data/datasets/frs.py | 4 +++- .../targets/sources/dwp_housing_benefit.py | 9 +++++---- .../tests/test_dwp_housing_benefit_targets.py | 18 ++++++++++++++++++ 5 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 changelog.d/hb-dwp-age-targets.changed.md delete mode 100644 changelog.d/hb-dwp-age-targets.fixed.md diff --git a/changelog.d/hb-dwp-age-targets.changed.md b/changelog.d/hb-dwp-age-targets.changed.md new file mode 100644 index 000000000..759b2a2ff --- /dev/null +++ b/changelog.d/hb-dwp-age-targets.changed.md @@ -0,0 +1 @@ +Calibrate pension-age Housing Benefit to DWP's Great Britain spending and claims over Pension Credit qualifying age, replacing the OBR Housing Benefit target that the model compared with UK-wide Housing Benefit. Modelled GB Housing Benefit for 2025-26 falls from about £11.7bn to £8.0bn (DWP: £12.9bn), because pension-age Housing Benefit was about 1.6 times DWP's figure while working-age, temporary and supported-accommodation Housing Benefit remain largely unmodelled. Stop setting `would_claim_uc` for benefit units whose adults have all reached State Pension age, and build with policyengine-uk 2.102.5 so pension-age families can make new Housing Benefit claims. diff --git a/changelog.d/hb-dwp-age-targets.fixed.md b/changelog.d/hb-dwp-age-targets.fixed.md deleted file mode 100644 index 2c4c56708..000000000 --- a/changelog.d/hb-dwp-age-targets.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Calibrate pension-age Housing Benefit to DWP's Great Britain spending and claims over Pension Credit qualifying age, replacing the OBR Housing Benefit target that the model compared with UK-wide Housing Benefit. Stop setting `would_claim_uc` for benefit units whose adults have all reached State Pension age, and build with policyengine-uk 2.102.5 so pension-age families can make new Housing Benefit claims. diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index 4a95842bb..94e40c2a6 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -1573,7 +1573,9 @@ def _reported_benunit_mask(person_column: str) -> np.ndarray: # A benefit unit whose adults have all reached State Pension age cannot # claim Universal Credit, so it never gets would_claim_uc, even if it # reports UC. The draw still covers every unit, so the random stream and - # every other unit's value are unchanged. + # every other unit's value are unchanged. Ages are not rolled forward, so + # if State Pension age rises above an adult's survey age in a later year, + # that unit stays without would_claim_uc there. pe_benunit["would_claim_uc"] = assign_takeup_with_reported_anchors( generator.random(len(pe_benunit)), universal_credit_rate, diff --git a/policyengine_uk_data/targets/sources/dwp_housing_benefit.py b/policyengine_uk_data/targets/sources/dwp_housing_benefit.py index 69d17f9b6..79870d13d 100644 --- a/policyengine_uk_data/targets/sources/dwp_housing_benefit.py +++ b/policyengine_uk_data/targets/sources/dwp_housing_benefit.py @@ -17,8 +17,8 @@ would ask for two different GB totals. DWP splits claims by benefit rules rather than by age alone (Notes, note -5): the two lines equal its Pension Credit plus State Pension benefit -groups and its ESA plus other working-age groups. Under regulation 5 of +5): from 2024-25 the two lines equal its Pension Credit plus State Pension +benefit groups and its ESA plus other working-age groups. Under regulation 5 of both Housing Benefit Regulations 2006 (SI 2006/213 and 2006/214), the pension-age rules apply when the claimant or partner has reached the qualifying age for Pension Credit, unless either is on Universal Credit, @@ -26,12 +26,13 @@ therefore over Pension Credit qualifying age here when one of its adults has reached State Pension age and it gets none of those benefits. Mixed-age couples who kept pension-age Housing Benefit after May 2019 fall in the -older group, as they do in DWP's benefit groups. +older group; DWP does not publish its rule for them, and this assumes they +sit in its Pension Credit and State Pension groups. Only the older group is calibrated. policyengine-uk pays working-age Housing Benefit only as a continuing award to families that report it and do not claim Universal Credit: 267 of the 770 working-age records that -report it, about 24,000 weighted claims in 2025-26 against DWP's 460,000. +report it, about 17,000 weighted claims in 2025-26 against DWP's 460,000. DWP's working-age figure also includes temporary and supported accommodation (together £5.2bn of Housing Benefit in 2025-26, not split by age), which the FRS barely samples. A test build on 2026-09-30 that also diff --git a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py index f92c2976c..549006b42 100644 --- a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py +++ b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py @@ -275,3 +275,21 @@ def test_country_restriction_partitions_the_uk(rows): np.testing.assert_array_equal(gb[country == "NORTHERN_IRELAND"], 0) np.testing.assert_array_equal(restrict_to_countries(gb, country, GREAT_BRITAIN), gb) assert restrict_to_countries(column, country, None) is column + + +def test_calibrated_columns_build_on_the_dataset(baseline, enhanced_frs): + """The loss matrix skips a target whose column raises, so check that the + calibrated columns build on a real simulation and land near DWP.""" + from policyengine_uk_data.targets.build_loss_matrix import _SimContext + + year = 2025 + baseline.default_calculation_period = str(year) + ctx = _SimContext(baseline, str(year), enhanced_frs, None) + weight = baseline.calculate("household_weight", year).values + for target in dwp_housing_benefit.get_targets(): + column = restrict_to_countries( + target.custom_compute(ctx, target, year), ctx.country, target.countries + ) + assert np.isfinite(column).all(), target.name + ratio = (column * weight).sum() / target.values[year] + assert 0.5 < ratio < 2, (target.name, ratio) From e2e37ba72166d23e0de1a3ade1fb12990182bd55 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 1 Oct 2026 09:57:29 -0400 Subject: [PATCH 3/3] Fix create_frs smoke test; use policyengine-uk's claimant-or-partner flag The create_frs smoke test's fake simulation did not answer the variables the would_claim_uc rule now reads (person_benunit_id, is_adult, is_SP_age), which failed CI with KeyError: 'person_benunit_id'. Pick the claimant-and-partner flag the way the locked policyengine-uk does for Housing Benefit's pension-age route: is_claimant_or_partner where the release defines it, otherwise is_adult as in 2.102.5. Both the would_claim_uc rule and the age-group targets use it. Under 2.102.5 the base FRS is identical to the previous commit's in every column. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk_data/datasets/frs.py | 42 ++++++++++--------- .../targets/sources/dwp_housing_benefit.py | 14 +++++-- .../tests/test_dwp_housing_benefit_targets.py | 1 + .../tests/test_legacy_benefit_proxies.py | 9 +++- .../tests/test_would_claim_uc_pension_age.py | 28 +++++++++---- policyengine_uk_data/utils/benefit_units.py | 15 +++++++ 6 files changed, 75 insertions(+), 34 deletions(-) create mode 100644 policyengine_uk_data/utils/benefit_units.py diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index 94e40c2a6..6b5b13b27 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -24,6 +24,7 @@ add_disability_benefit_flags_from_reported_amounts, drop_internal_disability_reported_amounts, ) +from policyengine_uk_data.utils.benefit_units import claimant_or_partner_variable from policyengine_uk_data.utils.datasets import ( sum_to_entity, categorical, @@ -392,37 +393,36 @@ def derive_is_parent_from_frs_microdata( return is_adult_record & has_dependent_children -def derive_every_adult_over_state_pension_age( +def derive_all_claimants_over_state_pension_age( person_benunit_ids, - is_adult, + is_claimant_or_partner, is_over_state_pension_age, benunit_ids, ) -> np.ndarray: - """Identify benefit units with an adult, all of whose adults have reached + """Identify benefit units whose claimant and any partner have all reached State Pension age. Such a unit cannot claim Universal Credit (Welfare Reform Act 2012 - s.4(1)(b)). The inputs are policyengine-uk's ``is_adult`` and - ``is_SP_age``, as in the pension-age route of its - ``housing_benefit_eligible``, so an 18 or 19 year old dependant makes the - unit working-age here as it does there. + s.4(1)(b)). ``is_claimant_or_partner`` should be the variable that + ``claimant_or_partner_variable`` names, so the rule matches the + pension-age route of policyengine-uk's ``housing_benefit_eligible``. """ - is_adult = np.asarray(is_adult, dtype=bool) - over = is_adult & np.asarray(is_over_state_pension_age, dtype=bool) + claimant = np.asarray(is_claimant_or_partner, dtype=bool) + over = claimant & np.asarray(is_over_state_pension_age, dtype=bool) counts = ( pd.DataFrame( { "benunit": np.asarray(person_benunit_ids), - "adults": is_adult.astype(int), + "claimants": claimant.astype(int), "over": over.astype(int), } ) - .groupby("benunit")[["adults", "over"]] + .groupby("benunit")[["claimants", "over"]] .sum() .reindex(np.asarray(benunit_ids), fill_value=0) ) - return ((counts.adults > 0) & (counts.over == counts.adults)).to_numpy() + return ((counts.claimants > 0) & (counts.over == counts.claimants)).to_numpy() def _as_non_negative_array(values) -> np.ndarray: @@ -1570,19 +1570,21 @@ def _reported_benunit_mask(person_column: str) -> np.ndarray: pension_credit_rate, reported_mask=_reported_benunit_mask("pension_credit_reported"), ) - # A benefit unit whose adults have all reached State Pension age cannot - # claim Universal Credit, so it never gets would_claim_uc, even if it - # reports UC. The draw still covers every unit, so the random stream and - # every other unit's value are unchanged. Ages are not rolled forward, so - # if State Pension age rises above an adult's survey age in a later year, - # that unit stays without would_claim_uc there. + # A benefit unit whose claimant and any partner have all reached State + # Pension age cannot claim Universal Credit, so it never gets + # would_claim_uc, even if it reports UC. The draw still covers every unit, + # so the random stream and every other unit's value are unchanged. Ages + # are not rolled forward, so if State Pension age rises above a claimant's + # survey age in a later year, that unit stays without would_claim_uc there. pe_benunit["would_claim_uc"] = assign_takeup_with_reported_anchors( generator.random(len(pe_benunit)), universal_credit_rate, reported_mask=_reported_benunit_mask("universal_credit_reported"), - ) & ~derive_every_adult_over_state_pension_age( + ) & ~derive_all_claimants_over_state_pension_age( person_benunit_ids=sim.calculate("person_benunit_id", year).values, - is_adult=sim.calculate("is_adult", year).values, + is_claimant_or_partner=sim.calculate( + claimant_or_partner_variable(sim.tax_benefit_system.variables), year + ).values, is_over_state_pension_age=sim.calculate("is_SP_age", year).values, benunit_ids=pe_benunit.benunit_id, ) diff --git a/policyengine_uk_data/targets/sources/dwp_housing_benefit.py b/policyengine_uk_data/targets/sources/dwp_housing_benefit.py index 79870d13d..eff5309f8 100644 --- a/policyengine_uk_data/targets/sources/dwp_housing_benefit.py +++ b/policyengine_uk_data/targets/sources/dwp_housing_benefit.py @@ -23,8 +23,8 @@ pension-age rules apply when the claimant or partner has reached the qualifying age for Pension Credit, unless either is on Universal Credit, Income Support, income-based JSA or income-related ESA. A benefit unit is -therefore over Pension Credit qualifying age here when one of its adults -has reached State Pension age and it gets none of those benefits. Mixed-age +therefore over Pension Credit qualifying age here when its claimant or +partner has reached State Pension age and it gets none of those benefits. Mixed-age couples who kept pension-age Housing Benefit after May 2019 fall in the older group; DWP does not publish its rule for them, and this assumes they sit in its Pension Credit and State Pension groups. @@ -46,6 +46,7 @@ import numpy as np from policyengine_uk_data.targets.schema import GREAT_BRITAIN, Target, Unit +from policyengine_uk_data.utils.benefit_units import claimant_or_partner_variable _REFERENCE_URL = ( "https://www.gov.uk/government/publications/" @@ -117,8 +118,13 @@ def _over_pension_credit_age(ctx) -> np.ndarray: """Benefit units assessed under the pension-age Housing Benefit rules.""" - adult = np.asarray(ctx.pe_person("is_adult"), dtype=bool) - over = adult & np.asarray(ctx.pe_person("is_SP_age"), dtype=bool) + claimant = np.asarray( + ctx.pe_person( + claimant_or_partner_variable(ctx.sim.tax_benefit_system.variables) + ), + dtype=bool, + ) + over = claimant & np.asarray(ctx.pe_person("is_SP_age"), dtype=bool) any_over = ( np.asarray(ctx.sim.map_result(over.astype(float), "person", "benunit")) > 0 ) diff --git a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py index 549006b42..d9a805f14 100644 --- a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py +++ b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py @@ -132,6 +132,7 @@ def map_result(values, source, target): person = {"is_adult": np.asarray(is_adult), "is_SP_age": np.asarray(is_sp_age)} benunit_values = {"housing_benefit": benunit_hb, **(benunit_benefits or {})} sim = SimpleNamespace( + tax_benefit_system=SimpleNamespace(variables={"is_adult": None}), map_result=map_result, calculate=lambda variable: SimpleNamespace( values=np.asarray( diff --git a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py index 5f1acd85f..a6af94e3f 100644 --- a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py +++ b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py @@ -299,6 +299,7 @@ def __init__(self, dataset): "FakeTaxBenefitSystem", (), { + "variables": {"is_adult": None}, "parameters": lambda self, year: type( "FakeParametersRoot", (), @@ -349,7 +350,7 @@ def __init__(self, dataset): }, )() }, - )() + )(), }, )() @@ -360,6 +361,12 @@ def calculate(self, variable, year=None): return np.array([100]) if variable == "state_pension_age": return pd.Series([66]) + if variable == "person_benunit_id": + return pd.Series(self.dataset.person.person_benunit_id.values) + if variable == "is_adult": + return pd.Series(self.dataset.person.age.values >= 18) + if variable == "is_SP_age": + return pd.Series(self.dataset.person.age.values >= 66) if variable in ( "childcare_grant", "parents_learning_allowance", diff --git a/policyengine_uk_data/tests/test_would_claim_uc_pension_age.py b/policyengine_uk_data/tests/test_would_claim_uc_pension_age.py index 251848335..b6a4f634d 100644 --- a/policyengine_uk_data/tests/test_would_claim_uc_pension_age.py +++ b/policyengine_uk_data/tests/test_would_claim_uc_pension_age.py @@ -5,17 +5,18 @@ from hypothesis import strategies as st from policyengine_uk_data.datasets.frs import ( - derive_every_adult_over_state_pension_age, + derive_all_claimants_over_state_pension_age, ) +from policyengine_uk_data.utils.benefit_units import claimant_or_partner_variable def test_examples(): - # Units: pensioner couple, mixed-age couple, working-age single, - # pensioner with an 18-year-old dependant, a unit with no adult, and a - # unit with no members at all. - result = derive_every_adult_over_state_pension_age( + # Units: pensioner couple, mixed-age couple, working-age single, a + # pensioner and a younger person flagged as claimant or partner, a unit + # with no claimant, and a unit with no members at all. + result = derive_all_claimants_over_state_pension_age( person_benunit_ids=[10, 10, 20, 20, 30, 40, 40, 50], - is_adult=[1, 1, 1, 1, 1, 1, 1, 0], + is_claimant_or_partner=[1, 1, 1, 1, 1, 1, 1, 0], is_over_state_pension_age=[1, 1, 1, 0, 0, 1, 0, 0], benunit_ids=[10, 20, 30, 40, 50, 60], ) @@ -33,9 +34,9 @@ def test_examples(): def test_matches_a_direct_definition(people, benunit_order): """Differential check against a per-unit loop, for any unit order.""" benunit_ids = np.array(benunit_order) - result = derive_every_adult_over_state_pension_age( + result = derive_all_claimants_over_state_pension_age( person_benunit_ids=[p[0] for p in people], - is_adult=[p[1] for p in people], + is_claimant_or_partner=[p[1] for p in people], is_over_state_pension_age=[p[2] for p in people], benunit_ids=benunit_ids, ) @@ -44,9 +45,18 @@ def test_matches_a_direct_definition(people, benunit_order): assert result[i] == (len(adults) > 0 and all(adults)) +def test_claimant_or_partner_variable_prefers_the_legal_flag(): + assert claimant_or_partner_variable({"is_adult": 0}) == "is_adult" + assert ( + claimant_or_partner_variable({"is_adult": 0, "is_claimant_or_partner": 0}) + == "is_claimant_or_partner" + ) + + def test_built_dataset_has_no_pension_age_uc_claimants(baseline): year = 2025 - adult = baseline.calculate("is_adult", year).values.astype(bool) + claimant = claimant_or_partner_variable(baseline.tax_benefit_system.variables) + adult = baseline.calculate(claimant, year).values.astype(bool) over = adult & baseline.calculate("is_SP_age", year).values.astype(bool) adults = baseline.map_result(adult.astype(float), "person", "benunit") adults_over = baseline.map_result(over.astype(float), "person", "benunit") diff --git a/policyengine_uk_data/utils/benefit_units.py b/policyengine_uk_data/utils/benefit_units.py new file mode 100644 index 000000000..fd098770b --- /dev/null +++ b/policyengine_uk_data/utils/benefit_units.py @@ -0,0 +1,15 @@ +"""Benefit-unit membership helpers shared by the dataset build and targets.""" + + +def claimant_or_partner_variable(variables) -> str: + """Name of the policyengine-uk variable marking a benefit unit's claimant + and any partner. + + Releases that define ``is_claimant_or_partner`` use it for Housing + Benefit's pension-age route. Earlier releases, such as 2.102.5, use + ``is_adult`` (age 18 or over) there, which also counts an 18 or 19 year + old dependant. + """ + if "is_claimant_or_partner" in variables: + return "is_claimant_or_partner" + return "is_adult"