From c3bc616f51b001db44bb076a874ab559067a31a5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 2 Oct 2026 13:05:40 -0400 Subject: [PATCH 1/2] Put FRS self-employment losses in trading_loss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SEINCAM2 records a self-employment loss as a negative profit ("Any losses are recorded as such", FRS methodology glossary). The build floored it at zero, so about 74,000 people's £1.28bn a year of losses (2024-25, before uprating) disappeared. policyengine-uk wants the profit and the loss as two non-negative inputs, because its programmes treat a loss differently: Income Tax and tax credits set it against other income, means-tested benefits do not, and HBAI counts it as negative income. Split SEINCAM2 into self_employment_income (the profit, unchanged) and trading_loss (the loss as a positive amount). SPI-donor rows of the enhanced FRS carry no trading loss: their self-employment profits come from the SPI, so their FRS donor's loss would sit beside another taxpayer's imputed incomes. Tests: Hypothesis properties of the split (non-negative, exclusive, conserves the reported profit, monotone) and data checks on the built FRS and enhanced FRS. Adds hypothesis to the dev extras. Co-Authored-By: Claude Opus 5.5 --- changelog.d/frs-trading-loss.added.md | 1 + policyengine_uk_data/datasets/frs.py | 22 ++++- .../datasets/imputations/income.py | 7 ++ .../tests/test_non_negative_incomes.py | 1 + .../tests/test_trading_loss.py | 84 +++++++++++++++++++ pyproject.toml | 1 + uv.lock | 73 +++++++++++++++- 7 files changed, 187 insertions(+), 2 deletions(-) create mode 100644 changelog.d/frs-trading-loss.added.md create mode 100644 policyengine_uk_data/tests/test_trading_loss.py diff --git a/changelog.d/frs-trading-loss.added.md b/changelog.d/frs-trading-loss.added.md new file mode 100644 index 000000000..bf8a5d043 --- /dev/null +++ b/changelog.d/frs-trading-loss.added.md @@ -0,0 +1 @@ +FRS self-employment losses (negative SEINCAM2) now go into policyengine-uk's `trading_loss` input as a positive amount, instead of being floored away; `self_employment_income` stays the non-negative profit. SPI-donor rows of the enhanced FRS carry no trading loss. diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index e440d4869..9c4de00b4 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -397,6 +397,23 @@ def _as_non_negative_array(values) -> np.ndarray: return np.maximum(np.nan_to_num(values, nan=0.0), 0.0) +def split_self_employment_profit(weekly_profit) -> tuple[np.ndarray, np.ndarray]: + """Split FRS weekly self-employment profit (SEINCAM2) into annual + ``self_employment_income`` and ``trading_loss``. + + SEINCAM2 keeps losses as negative values ("Any losses are recorded as + such", FRS methodology glossary). policyengine-uk wants profits and losses + as two non-negative inputs, because its programmes treat a loss + differently: Income Tax and tax credits set it against other income, + means-tested benefits do not, and HBAI counts it as negative income. So the + profit goes to ``self_employment_income`` and the loss, as a positive + amount, to ``trading_loss``. Their difference is the reported profit. + """ + annual = np.nan_to_num(np.asarray(weekly_profit, dtype=float), nan=0.0) + annual = annual * WEEKS_IN_YEAR + return np.maximum(annual, 0.0), np.maximum(-annual, 0.0) + + def allocate_reported_education_grants( reported_grants, grant_capacities: dict[str, np.ndarray] ) -> dict[str, np.ndarray]: @@ -1044,7 +1061,10 @@ def determine_education_level(fted_val, typeed2_val, age_val): pension_payment + pension_tax_paid + pension_deductions_removed ) * WEEKS_IN_YEAR - pe_person["self_employment_income"] = np.maximum(0, person.seincam2) * WEEKS_IN_YEAR + ( + pe_person["self_employment_income"], + pe_person["trading_loss"], + ) = split_self_employment_profit(person.seincam2) INVERTED_BASIC_RATE = 1.25 diff --git a/policyengine_uk_data/datasets/imputations/income.py b/policyengine_uk_data/datasets/imputations/income.py index feafcb27c..8a426cbad 100644 --- a/policyengine_uk_data/datasets/imputations/income.py +++ b/policyengine_uk_data/datasets/imputations/income.py @@ -292,6 +292,13 @@ def impute_income(dataset: UKSingleYearDataset) -> UKSingleYearDataset: target_dataset=zero_weight_copy, ) + # A trading loss belongs with the self-employment profit it came from. + # The SPI-donor rows' profits now come from the SPI, which this build + # does not draw losses from, so the FRS donor's loss would sit beside + # another taxpayer's imputed incomes. They carry none. + if "trading_loss" in zero_weight_copy.person.columns: + zero_weight_copy.person["trading_loss"] = 0.0 + dataset = impute_over_incomes( dataset, model, diff --git a/policyengine_uk_data/tests/test_non_negative_incomes.py b/policyengine_uk_data/tests/test_non_negative_incomes.py index 762d8fb0e..357440c26 100644 --- a/policyengine_uk_data/tests/test_non_negative_incomes.py +++ b/policyengine_uk_data/tests/test_non_negative_incomes.py @@ -3,6 +3,7 @@ INCOME_VARIABLES = [ "employment_income", "self_employment_income", + "trading_loss", "tax_free_savings_income", "savings_interest_income", "dividend_income", diff --git a/policyengine_uk_data/tests/test_trading_loss.py b/policyengine_uk_data/tests/test_trading_loss.py new file mode 100644 index 000000000..3a604bb94 --- /dev/null +++ b/policyengine_uk_data/tests/test_trading_loss.py @@ -0,0 +1,84 @@ +"""FRS self-employment profit is split into a profit and a trading loss. + +SEINCAM2 records a loss as a negative profit. The build puts the profit in +``self_employment_income`` and the loss, as a positive amount, in +``trading_loss``, so policyengine-uk can apply each programme's own loss rule. + +Invariants, for any weekly profit (including missing values): + +1. Both outputs are non-negative and at most one is positive. +2. Conservation: profit less loss is the annualised reported profit. +3. Profit never falls and loss never rises as the reported profit rises. + +And for the built datasets: + +4. No FRS person has both a profit and a loss, and no loss is negative. +5. SPI-donor rows of the enhanced FRS carry no trading loss; their + self-employment profits come from the SPI, not their FRS donor. +""" + +import numpy as np +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st +from hypothesis.extra.numpy import arrays + +from policyengine_uk_data.datasets.frs import ( + WEEKS_IN_YEAR, + split_self_employment_profit, +) + +weekly_profits = arrays( + np.float64, + st.integers(1, 50), + elements=st.one_of( + st.floats(-50_000, 50_000, allow_nan=False, allow_infinity=False), + st.just(0.0), + st.just(np.nan), + ), +) + + +@settings(max_examples=200, deadline=None, derandomize=True) +@given(weekly_profits) +def test_split_is_non_negative_exclusive_and_conserves_profit(weekly): + income, loss = split_self_employment_profit(weekly) + assert np.all(income >= 0) and np.all(loss >= 0) + assert np.all((income == 0) | (loss == 0)) + reported = np.nan_to_num(weekly, nan=0.0) * WEEKS_IN_YEAR + np.testing.assert_allclose(income - loss, reported, rtol=0, atol=1e-6) + + +@settings(max_examples=200, deadline=None, derandomize=True) +@given(weekly_profits, st.floats(0, 10_000, allow_nan=False)) +def test_split_is_monotone_in_reported_profit(weekly, rise): + income, loss = split_self_employment_profit(weekly) + higher_income, higher_loss = split_self_employment_profit( + np.nan_to_num(weekly, nan=0.0) + rise + ) + assert np.all(higher_income >= income) + assert np.all(higher_loss <= loss) + + +def test_frs_profit_and_loss_never_both_positive(frs): + if "trading_loss" not in frs.person.columns: + pytest.skip("Dataset built before trading_loss was added") + income = frs.person["self_employment_income"].to_numpy() + loss = frs.person["trading_loss"].to_numpy() + assert loss.min() >= 0 + assert not np.any((income > 0) & (loss > 0)) + + +def test_spi_donor_rows_carry_no_trading_loss(enhanced_frs): + person = enhanced_frs.person + if "trading_loss" not in person.columns: + pytest.skip("Dataset built before trading_loss was added") + household = enhanced_frs.household + synthetic_households = household.household_id[ + household.household_is_spi_synthetic.astype(bool) + ] + synthetic = person.person_household_id.isin(synthetic_households).to_numpy() + loss = person["trading_loss"].to_numpy() + assert synthetic.any(), "expected SPI-donor rows in the enhanced FRS" + assert np.all(loss[synthetic] == 0) + assert loss.min() >= 0 diff --git a/pyproject.toml b/pyproject.toml index beff7f1a4..58de5b1d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dependencies = [ dev = [ "ruff>=0.9.0", "pytest", + "hypothesis", "torch", "l0-python>=0.4.0", "tables", diff --git a/uv.lock b/uv.lock index a3e9f44d9..c301c6c9f 100644 --- a/uv.lock +++ b/uv.lock @@ -577,6 +577,75 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/f4/124858007ddf3c61e9b144107304c9152fa80b5b6c168da07d86fe583cc1/huggingface_hub-1.1.5-py3-none-any.whl", hash = "sha256:e88ecc129011f37b868586bbcfae6c56868cae80cd56a79d61575426a3aa0d7d", size = 516000, upload-time = "2025-11-20T15:49:30.926Z" }, ] +[[package]] +name = "hypothesis" +version = "6.168.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/b7/13118bbc45d6d8b9d04e2de779e2a4ff23145ea39efa692b994fb874ca72/hypothesis-6.168.3.tar.gz", hash = "sha256:a43388f9067678fef6e13bdff325b6cfa6961a590498bb37f7ff31589c83bc75", size = 511022, upload-time = "2026-09-28T05:20:58.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/cd/746a2fc1e5e5ba34f07ea6ee1ad07525a3dde5ff4836db3a7980af49a891/hypothesis-6.168.3-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d20972ca134e652e9928ecd200966a8a2857adc2812c1d74b12a872e5cd503eb", size = 791536, upload-time = "2026-09-28T05:18:57.162Z" }, + { url = "https://files.pythonhosted.org/packages/c3/0f/7a158e377b69556c8e12c25c8fd811d0103e54c24a12dab1f3b9819f202e/hypothesis-6.168.3-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:eafbec09d3e87d13d8242411d1f5868b5e879f1bd6d95e233e9ff80c527bea1c", size = 787313, upload-time = "2026-09-28T05:20:18.122Z" }, + { url = "https://files.pythonhosted.org/packages/45/c6/7df9104ee359e8fcd77791781c47e70794964ca69673665e0de2a6590f4c/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73c5627497968cc62d140e9fde1ea21e12cac7b64dc419fea8286cd34ff1ad4e", size = 1124036, upload-time = "2026-09-28T05:20:04.372Z" }, + { url = "https://files.pythonhosted.org/packages/92/0f/8a61715404a73b9a82cb1f976803428d603cdea976a9c30266c72ad6a7ce/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dc56e6dc6eeb0aeb08ac463f847279bde2336c4786faf7ee0af4b93cd031a7", size = 1147875, upload-time = "2026-09-28T05:19:15.495Z" }, + { url = "https://files.pythonhosted.org/packages/e4/3f/2b16e95cc3b9a069afa23830a012aefe377ce4a457a227e5a5026eb827ff/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:753bb501f8560d2e321ed3b62596b4496c56f15668b0a0669231ccc3e6c4e80d", size = 1149489, upload-time = "2026-09-28T05:19:18.886Z" }, + { url = "https://files.pythonhosted.org/packages/f6/e7/d7cf6dc068bb02732b2a6e38a35a0dcb7f2137608198fc79d740f69d197c/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:71ab606c472449cb872ef2a7acaec679ee4f651b7f0a477a04ebc85d8933ef8c", size = 1191992, upload-time = "2026-09-28T05:19:21.83Z" }, + { url = "https://files.pythonhosted.org/packages/5f/26/f1e5b25dec15998e8375d722825ec3dae1075cbe7cb1f74221b2312a2e33/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26928956c54748e4dfa587333741ab750246123b93484955646ff316cca27eef", size = 1169911, upload-time = "2026-09-28T05:19:27.996Z" }, + { url = "https://files.pythonhosted.org/packages/03/36/901234a49147e6bf8a5b9e54b775706a223c2b52befea1106ea2aaf15d48/hypothesis-6.168.3-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:0bdfc53c041b61c854fa3761735736b991bc2bddafee45a361cfe4d43027c1ef", size = 1129406, upload-time = "2026-09-28T05:18:42.632Z" }, + { url = "https://files.pythonhosted.org/packages/55/c1/5fc9ff91ec9fba6bb527833386c66812524b5cc75fe7d4a4f9122ee1c420/hypothesis-6.168.3-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:650528e2b1e2a45e95c2df624d4b9364b8ac5a027ba84e1eea2bc5398dd01dcd", size = 1160399, upload-time = "2026-09-28T05:20:23.911Z" }, + { url = "https://files.pythonhosted.org/packages/9b/d2/49ad1ef5c55547c8abfd0fc571b3493806dda530fd5cb3d8263dbad86a72/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:209dc54cdb1b4d6d7020ad8d09e44c49756361b7183adacea4d6f5705085b595", size = 1299906, upload-time = "2026-09-28T05:18:58.417Z" }, + { url = "https://files.pythonhosted.org/packages/0d/5f/f98a26094c4f462c4215807bed0f9a6b5508b27a3c329bcba1bc8adc7ac7/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:af8ca98cd11dc7f9427bb90483abcd9688d4df2e8e63893a30a2423b027ebb11", size = 1425538, upload-time = "2026-09-28T05:20:49.074Z" }, + { url = "https://files.pythonhosted.org/packages/d0/b8/c5b4406e3a6591e51a42f85469351d1d824e0a54b84167aec9b2353759fe/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:f8122cfdd0bc0ba843063effa22ac310bdebdb3a1319bf1e63f14baa119c72ab", size = 1377084, upload-time = "2026-09-28T05:20:06.618Z" }, + { url = "https://files.pythonhosted.org/packages/e0/01/0d84a8ea469d024c15f602799c0b8410fbe03d99fdb0370449b33c3c916f/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4bedbb379eab34f792af7ee9a05aae04e9c08bbb52e0f90f5a2110d4fe4b2fbd", size = 1281162, upload-time = "2026-09-28T05:19:42.768Z" }, + { url = "https://files.pythonhosted.org/packages/22/b5/ea6435038de1a795f005cb603531e8ccc053febc50180f3ab55583c9c60e/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:5a953115b9f5c95133ab2d04efffeec96e5658c3207923dca7285f7db3e6bbef", size = 1300433, upload-time = "2026-09-28T05:18:55.634Z" }, + { url = "https://files.pythonhosted.org/packages/6c/78/fa6c77d9f64b6d592e69e582d83cbce7d7e56897faa4762885a2123b0166/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6173558e676ad25ed1e20507fa4024a0d816dd90f715f77006e5a28f19109026", size = 1336273, upload-time = "2026-09-28T05:19:08.273Z" }, + { url = "https://files.pythonhosted.org/packages/f1/56/4acd0d778bab818c9ea336fcba768bf43bc70db7f84cc373579298df4641/hypothesis-6.168.3-cp310-abi3-win32.whl", hash = "sha256:dc66390fb12d80585aa9222bf538ce8b7aa22cf5d118250647355a1c9e8f62f4", size = 678211, upload-time = "2026-09-28T05:19:23.355Z" }, + { url = "https://files.pythonhosted.org/packages/b0/db/0a02b146ad1c30f9716362f2e1a379dfd6b0f7be03ac1e22c9da24c9e2a9/hypothesis-6.168.3-cp310-abi3-win_amd64.whl", hash = "sha256:92325b276360fe86c5bf71a568c0d53a6d140b0de36dcf029f9164a17803bb24", size = 684906, upload-time = "2026-09-28T05:18:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/a3/c0/958deaf726848f96f52250740bf39f13f476b068e22f73e2b358eaa07532/hypothesis-6.168.3-cp310-abi3-win_arm64.whl", hash = "sha256:3cf6f1eeaf41cd8d60cf1f88fde905ca1dd77c906929a507d6ac7f66f2ccba2a", size = 683292, upload-time = "2026-09-28T05:19:57.404Z" }, + { url = "https://files.pythonhosted.org/packages/0c/a2/6787da846d929e52fc3344d803299c45782dbeac287528af08380e984bf8/hypothesis-6.168.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1b230a850de63334c16654a34a2d547e0179d36b9071d4439b3e7237f6d077e7", size = 793254, upload-time = "2026-09-28T05:19:36.294Z" }, + { url = "https://files.pythonhosted.org/packages/89/ba/2893f5ca42501f4d562cba3229c8994c3a3e5fac66ef60c1e0e0b5220a5d/hypothesis-6.168.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d63b0226cd3e0d8bdd97c3384b22a21934ed4d53246c1c93575dada616672499", size = 784699, upload-time = "2026-09-28T05:18:54.194Z" }, + { url = "https://files.pythonhosted.org/packages/b7/aa/7d7349daf75b71f6f35876f8de115e974c5c04d600e86df1b2779b0883ab/hypothesis-6.168.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0369f5df055f96e117ab12e5f249668ff144731ab5280bc7a205fdf81f990b89", size = 1123016, upload-time = "2026-09-28T05:20:36.372Z" }, + { url = "https://files.pythonhosted.org/packages/00/f0/7774e1ea072708ea46cb25c4aeee5f9978b6c271764809069e8a6789858e/hypothesis-6.168.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f076bcd0f77fdcdb7797826c879099573d03a02e228ebe649ea917081b962ac0", size = 1168989, upload-time = "2026-09-28T05:20:08.44Z" }, + { url = "https://files.pythonhosted.org/packages/a7/7d/113992abed9efbd7944e3a496a382da58152dce126fddf6419ff31a0a57d/hypothesis-6.168.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b823ba1fcec8da730f29316d010b06d3f7e0c3828dcf91020e24c55e7d24652a", size = 1298626, upload-time = "2026-09-28T05:19:40.986Z" }, + { url = "https://files.pythonhosted.org/packages/d5/65/3659fa5e733027e5b37a27486e57f4053535e40853d23501c422bb275043/hypothesis-6.168.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dd2849c269d674e4618590f3b48d443bd4c06b5aef3d8d869086c2e6d213d248", size = 1335184, upload-time = "2026-09-28T05:20:20.087Z" }, + { url = "https://files.pythonhosted.org/packages/b4/6c/35aab2221b5ea65125340f236e1a765a9e9f3b28d89a389ea0033fed29f7/hypothesis-6.168.3-cp313-cp313-win_amd64.whl", hash = "sha256:3ef7d26f5789e691401d5f87eafed9bd2763f0dbe47af6d6b66012a509404766", size = 682173, upload-time = "2026-09-28T05:19:12.762Z" }, + { url = "https://files.pythonhosted.org/packages/6e/79/27b0cb56ff5d2bc92458bf6fcebdb1f0dc01f57f043a178e9f9d74498169/hypothesis-6.168.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:e2d4c68729a13df9af4998d2652cfb5d541c5609b88c306880a5dfb284938ec2", size = 793287, upload-time = "2026-09-28T05:20:38.522Z" }, + { url = "https://files.pythonhosted.org/packages/e8/bd/5342f95c3bc36586777ef8cb8eba87ac7acf0184210327fb0e6d8654e295/hypothesis-6.168.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b345f818083ec99966a43ca4f7b38feb62c6920ce28572bc2bde4948a8b7eaba", size = 784857, upload-time = "2026-09-28T05:20:42.609Z" }, + { url = "https://files.pythonhosted.org/packages/4a/5f/fc774241518a0588d362680a3084275058aab86d6bc02a2c61e1073142d0/hypothesis-6.168.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0608c610fc002978fc5de0f471770d8817e9455cb8649a47983c9f8bccfc1897", size = 1123330, upload-time = "2026-09-28T05:19:29.735Z" }, + { url = "https://files.pythonhosted.org/packages/b0/e6/131f16775a3dca5f4fe27f0d6ad9a9851600098a54a872d163f6ebf0b664/hypothesis-6.168.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9dcf6448b1ecc37f2b23f2d1b3ddfc9ff6b6910614c15a642dc82f419b96037b", size = 1169178, upload-time = "2026-09-28T05:19:53.436Z" }, + { url = "https://files.pythonhosted.org/packages/06/61/c9f5bff8b73321c9fd12f2d69666c6e4861b97b60f5f24fdd6ac293007fd/hypothesis-6.168.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f9f094041dcce5119ebd7bab71062743ab02b6e654d056b370beda78c19e2", size = 1299127, upload-time = "2026-09-28T05:20:14.486Z" }, + { url = "https://files.pythonhosted.org/packages/e4/6d/d4618f8ab12dd76c4d58ea052252759aa2bf0c4e71f4b502ba5d577e0b9f/hypothesis-6.168.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d479985fe73af97badfb72dc6d20c6a353e486a36f6d065f600026e0cf954a86", size = 1335390, upload-time = "2026-09-28T05:20:10.326Z" }, + { url = "https://files.pythonhosted.org/packages/95/b6/727161e17cc297df1aeea945de05c532033b66751625bec192bc5b5ff707/hypothesis-6.168.3-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:785e2c45f8c08e274b4bf1ccae97f1a4e09407e9a68e80790cfab1d17a4a45fa", size = 624291, upload-time = "2026-09-28T05:18:36.376Z" }, + { url = "https://files.pythonhosted.org/packages/77/23/2f9b506392a0b8712440bcb781abc5713be9ffac1303b6d10f3669050882/hypothesis-6.168.3-cp314-cp314-win_amd64.whl", hash = "sha256:320920b1e3dae8611eee8a03d063cf2187446f2a17c38cfb8a7fc1466f71eee2", size = 682064, upload-time = "2026-09-28T05:20:16.336Z" }, + { url = "https://files.pythonhosted.org/packages/50/93/efabfd95eb2b69c0c9fa1e1c83c8290aa2715d46baeb5a7764faf09c133a/hypothesis-6.168.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:35380baa981108a7f60c4eab71e46acd8d8f58440520346a1a6aba06dca7e074", size = 791875, upload-time = "2026-09-28T05:19:37.789Z" }, + { url = "https://files.pythonhosted.org/packages/1b/fc/2a0ada1623a9a33048bbf9f00a7c92513398c6b8865cba5efb854262ce30/hypothesis-6.168.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f0aaaed00438fa6673d856aaee12a4a8afbde82a9f3c5ff487cacfb064239afe", size = 783412, upload-time = "2026-09-28T05:18:48.266Z" }, + { url = "https://files.pythonhosted.org/packages/41/31/73c615d37eeb12da0209555612c32ced628d860267c23cfd2691c6207aea/hypothesis-6.168.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27e6df1576bf838e7f484d4ae5cba92114497ca30afb917056bf1ea33e95675", size = 1121662, upload-time = "2026-09-28T05:19:26.225Z" }, + { url = "https://files.pythonhosted.org/packages/92/54/1893344a3b8bbf83fa7c9bb7e96f6836580213854f46671cb749f7b4706c/hypothesis-6.168.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3bc85014577982ec6d2e266edc5cd6e7a7d3674c648791ca983c67a52f89a0c4", size = 1167761, upload-time = "2026-09-28T05:18:49.85Z" }, + { url = "https://files.pythonhosted.org/packages/26/97/a61d82968febd25a0f08ffbbfabfaeead66f68fc5fa73f3c9ece1c36fc8f/hypothesis-6.168.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ccfc29505aa1cdcc254cb9cd701fd0811fef5480addd97d4127a00df023410f7", size = 1297309, upload-time = "2026-09-28T05:19:51.711Z" }, + { url = "https://files.pythonhosted.org/packages/bc/01/fe8e4cf6d39d02f4efadaa351427fa13722086cfc54a2da3579741c5177a/hypothesis-6.168.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32d0699566aaa93f9e97a44705d7164386f1e91de78d277f53bada35e78bcc96", size = 1334260, upload-time = "2026-09-28T05:19:00.121Z" }, + { url = "https://files.pythonhosted.org/packages/39/6a/09177ebc62f4778f94379ecade6a51299d3d8c4f4b75ba636bf3a0202364/hypothesis-6.168.3-cp314-cp314t-win_amd64.whl", hash = "sha256:28d88fa174ecbd4ecbd7bb290f06d0db084a3971c3f511ae2830b65b5e25500f", size = 681992, upload-time = "2026-09-28T05:19:48.043Z" }, + { url = "https://files.pythonhosted.org/packages/f9/f6/890bf33d608cd63348d3146a3ca83362fbf09e589c5e5230a00c403070f4/hypothesis-6.168.3-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:61f5782d807b1e6aa5c9beef1054cb2037e7d1add48a64972cd6ee447778781f", size = 791231, upload-time = "2026-09-28T05:18:43.998Z" }, + { url = "https://files.pythonhosted.org/packages/03/dd/fc36b204f8aa7437c676576d9437f0422de46aa8a729e6cd5bbb0224e759/hypothesis-6.168.3-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:7aacf3cf40c7ce8f9e4924d5b57bc0b068beafdfed2347bcf160a28b4cfbba7b", size = 783171, upload-time = "2026-09-28T05:19:20.353Z" }, + { url = "https://files.pythonhosted.org/packages/5c/d2/3cd087d577db67c404991d7723e64d7cd75570cdc82a0f7329aed6f2086c/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01768a03a30dc54df7fe457c0b34016c84d598fa00d5965ededab93ba4eb3408", size = 1121152, upload-time = "2026-09-28T05:18:39.095Z" }, + { url = "https://files.pythonhosted.org/packages/42/6f/a05a68cc66e7a22701c50bf94fb9ddeb36f4f5d0de3cc130066709eda621/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1a8a4ffc6c6e6e577f2bfbcfebf7cffbb310283ba2532c729570a0a752cebaaa", size = 1144069, upload-time = "2026-09-28T05:20:44.705Z" }, + { url = "https://files.pythonhosted.org/packages/4a/f4/fdd7a093fb2fb3a0ce24256c92ad5bf67936f4669bd06d5ceae4298150ec/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b987d73eba95183a7e59cca6d1925c588aa4307922d9852cc2b4d282a2ae4128", size = 1146692, upload-time = "2026-09-28T05:20:25.83Z" }, + { url = "https://files.pythonhosted.org/packages/f1/c8/6dbd4377e935505ae4fc8ee4c7b18c69994c4bbaca015447c470218bdbee/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4569c39bd97d9573e946429ed676f3b55a7c8ed80920addd67d384a47d59b38", size = 1189480, upload-time = "2026-09-28T05:20:46.734Z" }, + { url = "https://files.pythonhosted.org/packages/ba/bf/ff288b496b690000d2686dcfa7f67855e4c5a6dddb464ed8e4880be83bb2/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6042b8707a4b25bbbfe20b110258fb7549a68951e5525608a8ce06091039e9fc", size = 1167109, upload-time = "2026-09-28T05:19:24.758Z" }, + { url = "https://files.pythonhosted.org/packages/d9/d1/3fee2bc29fc747fabbb27e174515592e390809ca465d3bbbabbcfa3235a8/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:f413629de94d38a7a2ad259697a6752526143cb49c2e2d7bdba19381c80693f6", size = 1126823, upload-time = "2026-09-28T05:18:52.636Z" }, + { url = "https://files.pythonhosted.org/packages/58/6a/585294392fa6a6d9719446a042a777ba98ecca265d9e48cbedc10c88df0f/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f071737e4e775bebba07e319eb645a880d1e0186b4d24bad23255e849d86d483", size = 1155773, upload-time = "2026-09-28T05:20:51.268Z" }, + { url = "https://files.pythonhosted.org/packages/cd/4d/fa283ff79996debf1ff08593e01f8b773eb8211b19b3d6d5d491317a65bf/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:f59a3912858d0609c26054aa1c474847c797937f5e0e1c77f0d3f08032e50f09", size = 1296671, upload-time = "2026-09-28T05:18:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/42/05/98f9c2f628da5afabb6c3e8df5d2d299d2d50bf306b14a6e26d3954c4951/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:f09c05a23a8025dd5cad07c2ed299a46778e72d49ff0dd5bf353bcc0f7dc1580", size = 1422001, upload-time = "2026-09-28T05:19:04.188Z" }, + { url = "https://files.pythonhosted.org/packages/b8/c9/f2109ade29a7ec284d55bca328d784743e8b13e321576fd34ee7e65f99af/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_i686.whl", hash = "sha256:44ace770bda3a0301739fc5a413c764de790c739df1f1a7218dd049f8594d9f5", size = 1374182, upload-time = "2026-09-28T05:20:27.764Z" }, + { url = "https://files.pythonhosted.org/packages/ce/6d/e05d5f72441564a3bebc71fa155deafd0fd3b015d6014ec8a00edf6e42bb/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:03b131043608f94a2578a2a896a1a72092acb7079815eef5b8513b08447e0e62", size = 1278402, upload-time = "2026-09-28T05:20:56.332Z" }, + { url = "https://files.pythonhosted.org/packages/96/d7/6988a7f1f69c5c530687dce03a32c157e2d878e3a2e32af9269ce016b78a/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:e9784aca26eddfe99b03a0292320db742cc8a74200ef864fdce949f526f973cd", size = 1297770, upload-time = "2026-09-28T05:19:11.387Z" }, + { url = "https://files.pythonhosted.org/packages/a2/28/5bb82b60b836bd2329e1fe01ad94efc2b14dfa806f8e50ed14b795d78770/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:2b52ac363096232bebc2add117e9178d91f4248f4cdb919fd1026b5f86a4bb16", size = 1333977, upload-time = "2026-09-28T05:20:02.483Z" }, + { url = "https://files.pythonhosted.org/packages/8e/7d/d419841b8f65481ea1a50c4ba36f2670d48b8c7a51e4e8586089564cf7f6/hypothesis-6.168.3-cp315-abi3.abi3t-win32.whl", hash = "sha256:d28e3a6b511a74ce37df5274b51f36c2b274fea7365e00b16f0c81e22acd5957", size = 675394, upload-time = "2026-09-28T05:19:55.237Z" }, + { url = "https://files.pythonhosted.org/packages/d8/d2/1de6a2ad100e44817f2e9a8e8ba3eeaf4769621f4f87f2d4966c4971fcc4/hypothesis-6.168.3-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:7b9638789548361a57d984f56619ac694a914c328181d911271618409261ff4a", size = 681699, upload-time = "2026-09-28T05:19:09.957Z" }, + { url = "https://files.pythonhosted.org/packages/c0/79/be3370fd02734d6b1d950183ae9224580d19fa8d356b95348d60f1e3eeb7/hypothesis-6.168.3-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:65d78e4357ec48ed2c67825f06740ee3599be4cfe770a6092bed07108d679ac5", size = 679849, upload-time = "2026-09-28T05:19:06.884Z" }, +] + [[package]] name = "idna" version = "3.11" @@ -1364,7 +1433,7 @@ wheels = [ [[package]] name = "policyengine-uk-data" -version = "1.56.16" +version = "1.57.4" source = { editable = "." } dependencies = [ { name = "google-auth" }, @@ -1392,6 +1461,7 @@ dependencies = [ dev = [ { name = "build" }, { name = "furo" }, + { name = "hypothesis" }, { name = "itables" }, { name = "l0-python" }, { name = "pytest" }, @@ -1410,6 +1480,7 @@ requires-dist = [ { name = "google-auth" }, { name = "google-cloud-storage" }, { name = "huggingface-hub" }, + { name = "hypothesis", marker = "extra == 'dev'" }, { name = "itables", marker = "extra == 'dev'" }, { name = "l0-python", marker = "extra == 'dev'", specifier = ">=0.4.0" }, { name = "microcalibrate", specifier = ">=0.18.0" }, From 94bd3d7783075fe89851dd98d8a0b9f0c3bc54dc Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 2 Oct 2026 23:00:19 -0400 Subject: [PATCH 2/2] Impute SPI-donor losses by the stage-2 QRF; uprate trading_loss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review r1 on c3bc616 (REQUEST CHANGES): - Zeroing the SPI-donor rows' losses left the calibrated enhanced FRS with £0.29bn of losses on about 23,000 weighted people, against £1.28bn on 74,000 in the FRS. SPI copies hold 28% of household weight. The SPI has no current-year loss field, so impute trading_loss on those rows by the second-stage QRF, from FRS respondents with similar demographics and imputed incomes, like the other FRS-only variables. It is appended last: microimpute imputes the outputs in order with a per-output seeded generator, so every earlier imputation is unchanged and no global random state is used. A profit and a loss are never both kept on one row, as SEINCAM2 nets a person's trades. - trading_loss gets the self-employment rows in uprating_factors.csv and uprating_growth_factors.csv (what the table generator produces once policyengine-uk#2085 gives the variable that uprating). - The data checks assert on violation counts, so a failure never prints record-level amounts. Co-Authored-By: Claude Opus 5.5 --- .../datasets/imputations/frs_only.py | 3 ++ .../datasets/imputations/income.py | 13 +++++--- .../storage/uprating_factors.csv | 1 + .../storage/uprating_growth_factors.csv | 1 + .../tests/test_trading_loss.py | 33 ++++++++++++------- 5 files changed, 35 insertions(+), 16 deletions(-) diff --git a/policyengine_uk_data/datasets/imputations/frs_only.py b/policyengine_uk_data/datasets/imputations/frs_only.py index 242fbf247..fd620e185 100644 --- a/policyengine_uk_data/datasets/imputations/frs_only.py +++ b/policyengine_uk_data/datasets/imputations/frs_only.py @@ -100,6 +100,9 @@ "jsa_income_reported", "esa_contrib_reported", "esa_income_reported", + # Self-employment losses. Last, so the QRF's sequential imputation of + # every variable above is unchanged by it. + "trading_loss", ] diff --git a/policyengine_uk_data/datasets/imputations/income.py b/policyengine_uk_data/datasets/imputations/income.py index 8a426cbad..ba1345345 100644 --- a/policyengine_uk_data/datasets/imputations/income.py +++ b/policyengine_uk_data/datasets/imputations/income.py @@ -292,12 +292,15 @@ def impute_income(dataset: UKSingleYearDataset) -> UKSingleYearDataset: target_dataset=zero_weight_copy, ) - # A trading loss belongs with the self-employment profit it came from. - # The SPI-donor rows' profits now come from the SPI, which this build - # does not draw losses from, so the FRS donor's loss would sit beside - # another taxpayer's imputed incomes. They carry none. + # The second stage imputes trading_loss from FRS respondents with similar + # demographics and imputed incomes (the SPI has no current-year loss). + # SEINCAM2 nets a person's trades, so an FRS respondent has a profit or a + # loss, never both; keep the SPI-donor rows the same. if "trading_loss" in zero_weight_copy.person.columns: - zero_weight_copy.person["trading_loss"] = 0.0 + person = zero_weight_copy.person + person["trading_loss"] = np.where( + person["self_employment_income"] > 0, 0.0, person["trading_loss"] + ) dataset = impute_over_incomes( dataset, diff --git a/policyengine_uk_data/storage/uprating_factors.csv b/policyengine_uk_data/storage/uprating_factors.csv index dacd591bd..9266f0506 100644 --- a/policyengine_uk_data/storage/uprating_factors.csv +++ b/policyengine_uk_data/storage/uprating_factors.csv @@ -73,6 +73,7 @@ statutory_paternity_pay,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38, statutory_sick_pay,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 student_loan_repayments,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 sublet_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 +trading_loss,1.0,1.0,1.063,1.089,1.141,1.194,1.231,1.27,1.315,1.365,1.365,1.365,1.365,1.365,1.365 transport_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 universal_credit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 water_and_sewerage_charges,1.0,1.0,1.0,1.092,1.14,1.21,1.283,1.349,1.4,1.46,1.46,1.46,1.46,1.46,1.46 diff --git a/policyengine_uk_data/storage/uprating_growth_factors.csv b/policyengine_uk_data/storage/uprating_growth_factors.csv index 122b4ed69..471500fc6 100644 --- a/policyengine_uk_data/storage/uprating_growth_factors.csv +++ b/policyengine_uk_data/storage/uprating_growth_factors.csv @@ -73,6 +73,7 @@ statutory_paternity_pay,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0. statutory_sick_pay,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 student_loan_repayments,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 sublet_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 +trading_loss,0,0.0,0.063,0.024,0.048,0.046,0.031,0.032,0.035,0.038,0.0,0.0,0.0,0.0,0.0 transport_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 universal_credit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 water_and_sewerage_charges,0,0.0,0.0,0.092,0.044,0.061,0.06,0.051,0.038,0.043,0.0,0.0,0.0,0.0,0.0 diff --git a/policyengine_uk_data/tests/test_trading_loss.py b/policyengine_uk_data/tests/test_trading_loss.py index 3a604bb94..9da3030ee 100644 --- a/policyengine_uk_data/tests/test_trading_loss.py +++ b/policyengine_uk_data/tests/test_trading_loss.py @@ -13,8 +13,11 @@ And for the built datasets: 4. No FRS person has both a profit and a loss, and no loss is negative. -5. SPI-donor rows of the enhanced FRS carry no trading loss; their - self-employment profits come from the SPI, not their FRS donor. +5. The same holds on the SPI-donor rows of the enhanced FRS, whose losses the + second-stage QRF imputes from FRS respondents with similar incomes. + +The data checks assert on counts only, so a failure never prints a record's +amounts (the FRS is licensed microdata). """ import numpy as np @@ -60,16 +63,24 @@ def test_split_is_monotone_in_reported_profit(weekly, rise): assert np.all(higher_loss <= loss) +def _violations(person, mask=None) -> dict: + income = person["self_employment_income"].to_numpy() + loss = person["trading_loss"].to_numpy() + keep = np.ones(len(person), dtype=bool) if mask is None else mask + return { + "negative_loss": int((loss[keep] < 0).sum()), + "profit_and_loss": int(((income > 0) & (loss > 0))[keep].sum()), + } + + def test_frs_profit_and_loss_never_both_positive(frs): if "trading_loss" not in frs.person.columns: pytest.skip("Dataset built before trading_loss was added") - income = frs.person["self_employment_income"].to_numpy() - loss = frs.person["trading_loss"].to_numpy() - assert loss.min() >= 0 - assert not np.any((income > 0) & (loss > 0)) + counts = _violations(frs.person) + assert counts == {"negative_loss": 0, "profit_and_loss": 0}, counts -def test_spi_donor_rows_carry_no_trading_loss(enhanced_frs): +def test_spi_donor_rows_never_have_both(enhanced_frs): person = enhanced_frs.person if "trading_loss" not in person.columns: pytest.skip("Dataset built before trading_loss was added") @@ -78,7 +89,7 @@ def test_spi_donor_rows_carry_no_trading_loss(enhanced_frs): household.household_is_spi_synthetic.astype(bool) ] synthetic = person.person_household_id.isin(synthetic_households).to_numpy() - loss = person["trading_loss"].to_numpy() - assert synthetic.any(), "expected SPI-donor rows in the enhanced FRS" - assert np.all(loss[synthetic] == 0) - assert loss.min() >= 0 + assert int(synthetic.sum()) > 0, "expected SPI-donor rows in the enhanced FRS" + counts = _violations(person, synthetic) + assert counts == {"negative_loss": 0, "profit_and_loss": 0}, counts + assert _violations(person)["negative_loss"] == 0