diff --git a/changelog.d/frs-shared-household-rent.added.md b/changelog.d/frs-shared-household-rent.added.md new file mode 100644 index 00000000..887843bd --- /dev/null +++ b/changelog.d/frs-shared-household-rent.added.md @@ -0,0 +1 @@ +Mark the benefit units of FRS shared households that are liable for part of the rent (liable_for_share_of_household_rent), so policyengine-uk can split the household's rent among them. A unit counts where it reports rent (SRENTAMT), housing benefit (HBOTHAMT) or a Universal Credit housing element on a linked benefit record (UCHOUSEL). diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index 7e33f440..fa8ca742 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -275,6 +275,68 @@ def frs_boarder_and_lodger_rent(person: pd.DataFrame) -> tuple[np.ndarray, np.nd return rent_paid * is_boarder, rent_paid * ~is_boarder +UNIVERSAL_CREDIT_BENEFIT_CODE = 95 + + +def frs_liable_for_share_of_household_rent( + benunit: pd.DataFrame, + person: pd.DataFrame, + household: pd.DataFrame, + benefits: pd.DataFrame | None = None, +) -> np.ndarray: + """Whether each benefit unit shares liability for its household's rent. + + In a shared household (HHSTAT 2: "shared on an equal basis", the head of + household unclear or arbitrary), each benefit unit after the first is + asked how much rent it pays (SRENTAMT, on its adults' records) and how + much housing benefit it gets (HBOTHAMT). The questionnaire takes SRENTAMT + after state help with the rent, so a unit whose share is wholly met by + Universal Credit can report zero in both. Such a unit is still liable: + it counts where a Universal Credit record linked to DWP administrative + data carries a housing element (UCHOUSEL on a BENEFIT 95 record). A later + unit with any of the three is one of the people liable for HHRENT, the + rent of the whole dwelling, and policyengine-uk splits ``rent`` among + them. Total Universal Credit alone is not evidence of a rent liability. + + ``benefits`` is the FRS benefits table. Without it, or in a release + whose benefits table has no UCHOUSEL column, only SRENTAMT and HBOTHAMT + are used. + + Scope: conventional households (HHSTAT 1) are left out. Their later + units can also report HBOTHAMT or a Universal Credit housing element, + but the data do not say whether they owe a share of HHRENT to the + landlord or rent to the householder, which policyengine-uk treats + differently (the boarder and lodger inputs). + + The FRS's own split (BURENT) is not used. In shared households of the + 2023-24 and 2024-25 releases the later units between them get all of + HHRENT and benefit unit 1 a further amount; in 2024-25 the units' BURENT + sum to about two and a half times HHRENT. The published derivation of + HHRENT from benefit unit 1's rent plus the later units' SRENTAMT and + HBOTHAMT is the May 2003 derived-variable specification; this function + does not rely on a later one. + """ + benunit_id = benunit.benunit_id.values + unit_number = benunit_id % 100 + shared = household.hhstat.reindex(benunit.household_id.values).values == 2 + srent = ( + person.srentamt.where(person.srentamt > 0, 0) + .groupby(person.benunit_id.values) + .sum() + .reindex(benunit_id, fill_value=0) + .values + ) + hb = np.maximum(0, benunit.hbothamt.fillna(0).values) + uc_housing = np.zeros(len(benunit), dtype=bool) + if benefits is not None and "uchousel" in benefits.columns: + with_housing_element = benefits.benunit_id[ + (benefits.benefit == UNIVERSAL_CREDIT_BENEFIT_CODE) + & (benefits.uchousel > 0) + ] + uc_housing = np.isin(benunit_id, with_housing_element.values) + return (unit_number > 1) & shared & ((srent > 0) | (hb > 0) | uc_housing) + + def derive_is_in_non_advanced_education( current_education, is_apprentice=None, @@ -1320,6 +1382,9 @@ def determine_education_level(fted_val, typeed2_val, age_val): * WEEKS_IN_YEAR ) pe_household["rent"] = household.hhrent.fillna(0).values * WEEKS_IN_YEAR + pe_benunit["liable_for_share_of_household_rent"] = ( + frs_liable_for_share_of_household_rent(benunit, person, household, benefits) + ) pe_household["mortgage_interest_repayment"] = ( household.mortint.fillna(0).values * WEEKS_IN_YEAR ) diff --git a/policyengine_uk_data/tests/test_frs_shared_household_rent.py b/policyengine_uk_data/tests/test_frs_shared_household_rent.py new file mode 100644 index 00000000..df97152a --- /dev/null +++ b/policyengine_uk_data/tests/test_frs_shared_household_rent.py @@ -0,0 +1,234 @@ +import numpy as np +import pandas as pd +import pytest + +from policyengine_uk_data.datasets.frs import frs_liable_for_share_of_household_rent + +CONVENTIONAL, SHARED = 1, 2 +UNIVERSAL_CREDIT, HOUSING_BENEFIT = 95, 94 + + +def frames(units, adults, hhstat): + benunit = pd.DataFrame(units, columns=["benunit_id", "household_id", "hbothamt"]) + person = pd.DataFrame(adults, columns=["person_id", "benunit_id", "srentamt"]) + household = pd.DataFrame( + {"hhstat": list(hhstat.values())}, index=list(hhstat.keys()) + ) + return benunit, person, household + + +def test_later_units_of_a_shared_household_that_pay_rent_are_liable(): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 0), (1_03, 1, 0)], + [(1_001, 1_01, 0), (1_002, 1_02, 120), (1_003, 1_03, 110)], + {1: SHARED}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + # Benefit unit 1 is the household head's, liable for the rent anyway. + assert liable.tolist() == [False, True, True] + + +def test_housing_benefit_alone_marks_a_unit_liable(): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 85)], + [(1_001, 1_01, 0), (1_002, 1_02, 0)], + {1: SHARED}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + assert liable.tolist() == [False, True] + + +@pytest.mark.parametrize("srentamt", [0, -1, np.nan]) +def test_a_later_unit_paying_nothing_is_not_liable(srentamt): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, np.nan)], + [(1_001, 1_01, 0), (1_002, 1_02, srentamt)], + {1: SHARED}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + assert liable.tolist() == [False, False] + + +def test_units_of_conventional_households_are_never_sharers(): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 50), (2_01, 2, 0), (2_02, 2, 0)], + [(1_001, 1_01, 0), (1_002, 1_02, 0), (2_001, 2_01, 0), (2_002, 2_02, 90)], + {1: CONVENTIONAL, 2: CONVENTIONAL}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + assert not liable.any() + + +def test_a_couple_counts_once_and_households_do_not_leak(): + # A couple in unit 2 of a shared household reports SRENTAMT on one + # partner's record; the next household is conventional. + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 0), (2_01, 2, 0), (2_02, 2, 0)], + [ + (1_001, 1_01, 0), + (1_002, 1_02, 200), + (1_003, 1_02, 0), + (2_001, 2_01, 0), + (2_002, 2_02, 0), + ], + {1: SHARED, 2: CONVENTIONAL}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + assert liable.tolist() == [False, True, False, False] + + +def uc_records(rows): + """FRS benefits rows: (benefit unit, benefit code, weekly housing element).""" + return pd.DataFrame(rows, columns=["benunit_id", "benefit", "uchousel"]) + + +def test_the_result_is_one_boolean_per_benefit_unit(): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 0), (1_03, 1, 40)], + [(1_001, 1_01, 0), (1_002, 1_02, 120), (1_003, 1_03, 0)], + {1: SHARED}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + assert isinstance(liable, np.ndarray) + assert liable.dtype == bool + assert liable.shape == (len(benunit),) + + +def test_the_first_unit_is_never_marked_whatever_it_reports(): + # Benefit unit 1 is liable as the household head's family; the flag is + # only for the later units. + benunit, person, household = frames( + [(1_01, 1, 60), (1_02, 1, 0)], + [(1_001, 1_01, 150), (1_002, 1_02, 0)], + {1: SHARED}, + ) + benefits = uc_records([(1_01, UNIVERSAL_CREDIT, 100)]) + liable = frs_liable_for_share_of_household_rent( + benunit, person, household, benefits + ) + assert liable.tolist() == [False, False] + + +def test_a_paying_unit_does_not_mark_a_non_paying_unit_of_its_household(): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 0), (1_03, 1, 0)], + [(1_001, 1_01, 0), (1_002, 1_02, 130), (1_003, 1_03, 0)], + {1: SHARED}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + assert liable.tolist() == [False, True, False] + + +def test_a_paying_unit_does_not_mark_the_same_numbered_unit_elsewhere(): + # Two shared households: unit 2 pays in the first, not in the second. + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 0), (2_01, 2, 0), (2_02, 2, 0)], + [(1_001, 1_01, 0), (1_002, 1_02, 130), (2_001, 2_01, 0), (2_002, 2_02, 0)], + {1: SHARED, 2: SHARED}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + assert liable.tolist() == [False, True, False, False] + + +def test_rent_on_the_second_adult_of_a_couple_counts(): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 0)], + [(1_001, 1_01, 0), (1_002, 1_02, 0), (1_003, 1_02, 140)], + {1: SHARED}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + assert liable.tolist() == [False, True] + + +def test_a_missing_code_on_one_partner_does_not_cancel_the_other_partners_rent(): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 0)], + [(1_001, 1_01, 0), (1_002, 1_02, -140), (1_003, 1_02, 140)], + {1: SHARED}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + assert liable.tolist() == [False, True] + + +@pytest.mark.parametrize("hbothamt", [-1, -9, 0, np.nan]) +def test_a_missing_or_zero_housing_benefit_amount_does_not_mark_a_unit(hbothamt): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, hbothamt)], + [(1_001, 1_01, 0), (1_002, 1_02, 0)], + {1: SHARED}, + ) + liable = frs_liable_for_share_of_household_rent(benunit, person, household) + assert liable.tolist() == [False, False] + + +def test_the_order_of_the_tables_does_not_matter(): + units = [(1_01, 1, 0), (1_02, 1, 0), (2_01, 2, 0), (2_02, 2, 70), (2_03, 2, 0)] + adults = [ + (1_001, 1_01, 0), + (1_002, 1_02, 120), + (2_001, 2_01, 0), + (2_002, 2_02, 0), + (2_003, 2_03, 0), + ] + benunit, person, household = frames(units, adults, {1: SHARED, 2: SHARED}) + expected = dict(zip(benunit.benunit_id, [False, True, False, True, False])) + shuffled_units = benunit.iloc[[3, 0, 4, 2, 1]].reset_index(drop=True) + shuffled_people = person.iloc[[4, 2, 0, 3, 1]].reset_index(drop=True) + shuffled_households = household.iloc[[1, 0]] + liable = frs_liable_for_share_of_household_rent( + shuffled_units, shuffled_people, shuffled_households + ) + assert liable.tolist() == [expected[i] for i in shuffled_units.benunit_id] + + +def test_a_universal_credit_housing_element_marks_a_unit_that_pays_nothing_itself(): + # The questionnaire takes SRENTAMT after state help with the rent, so a + # joint tenant whose share is wholly met by Universal Credit reports no + # rent and no housing benefit. + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 0), (1_03, 1, 0)], + [(1_001, 1_01, 0), (1_002, 1_02, 0), (1_003, 1_03, 0)], + {1: SHARED}, + ) + benefits = uc_records([(1_02, UNIVERSAL_CREDIT, 100)]) + liable = frs_liable_for_share_of_household_rent( + benunit, person, household, benefits + ) + assert liable.tolist() == [False, True, False] + + +@pytest.mark.parametrize( + "record", + [ + (1_02, UNIVERSAL_CREDIT, 0), + (1_02, UNIVERSAL_CREDIT, -1), + (1_02, UNIVERSAL_CREDIT, np.nan), + # A housing element amount on a record of another benefit. + (1_02, HOUSING_BENEFIT, 100), + # Another unit's Universal Credit. + (2_02, UNIVERSAL_CREDIT, 100), + ], +) +def test_other_benefit_records_do_not_mark_a_unit(record): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 0), (2_01, 2, 0), (2_02, 2, 0)], + [(1_001, 1_01, 0), (1_002, 1_02, 0), (2_001, 2_01, 0), (2_002, 2_02, 0)], + {1: SHARED, 2: CONVENTIONAL}, + ) + liable = frs_liable_for_share_of_household_rent( + benunit, person, household, uc_records([record]) + ) + assert not liable.any() + + +def test_a_benefits_table_without_the_housing_element_uses_the_other_two_signals(): + benunit, person, household = frames( + [(1_01, 1, 0), (1_02, 1, 0), (1_03, 1, 55)], + [(1_001, 1_01, 0), (1_002, 1_02, 0), (1_003, 1_03, 0)], + {1: SHARED}, + ) + benefits = uc_records([(1_02, UNIVERSAL_CREDIT, 100)]).drop(columns="uchousel") + liable = frs_liable_for_share_of_household_rent( + benunit, person, household, benefits + ) + assert liable.tolist() == [False, False, True]