From 4bcc5f1690143746b34e48b9644ccb0680e1823d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 30 Sep 2026 21:29:36 +0400 Subject: [PATCH 1/5] Add packaged UK local-authority metadata --- .../stage12_execution.py | 14 + .../tests/test_stage12_execution.py | 14 + .../coordinates_lad22.csv | 375 +++++++++++++++++ .../coordinates_lad23.csv | 362 +++++++++++++++++ .../resources/uk_local_authorities/names.csv | 379 ++++++++++++++++++ .../uk_local_authority_metadata.py | 164 ++++++++ .../tests/test_uk_local_authority_metadata.py | 71 ++++ 7 files changed, 1379 insertions(+) create mode 100644 projects/policyengine-simulation-executor/src/policyengine_simulation_executor/resources/uk_local_authorities/coordinates_lad22.csv create mode 100644 projects/policyengine-simulation-executor/src/policyengine_simulation_executor/resources/uk_local_authorities/coordinates_lad23.csv create mode 100644 projects/policyengine-simulation-executor/src/policyengine_simulation_executor/resources/uk_local_authorities/names.csv create mode 100644 projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/uk_local_authority_metadata.py create mode 100644 projects/policyengine-simulation-executor/tests/test_uk_local_authority_metadata.py diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution.py index 75f1d2d98..93dc54f71 100644 --- a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution.py +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution.py @@ -52,6 +52,20 @@ class SimulationRole(StrEnum): STANDALONE = "standalone" +class UKLocalAuthorityRoster(StrEnum): + """Supported UK local-authority boundary configurations.""" + + LAD22 = "lad22" + LAD23 = "lad23" + + +class UKLocalAuthorityMetadata(StrictContractModel): + """Authority configuration detected from a UK simulation dataset.""" + + country: Literal["uk"] = "uk" + roster: UKLocalAuthorityRoster + + class ArtifactMediaType(StrEnum): JSON = "application/json" PARQUET = "application/vnd.apache.parquet" diff --git a/libs/policyengine-simulation-contract/tests/test_stage12_execution.py b/libs/policyengine-simulation-contract/tests/test_stage12_execution.py index 500bf35e1..0642daa9e 100644 --- a/libs/policyengine-simulation-contract/tests/test_stage12_execution.py +++ b/libs/policyengine-simulation-contract/tests/test_stage12_execution.py @@ -9,6 +9,8 @@ ReportAggregate, ReportOutputRequirements, Stage12OutputPlan, + UKLocalAuthorityMetadata, + UKLocalAuthorityRoster, stage12_output_plan_sha256, ) @@ -115,3 +117,15 @@ def test_entity_output_plan_separates_calculated_and_dataset_variables() -> None additional_variables=("constituency_code_oa",), dataset_variables=("constituency_code_oa",), ) + + +def test_uk_local_authority_metadata_is_strict_and_typed() -> None: + metadata = UKLocalAuthorityMetadata(roster=UKLocalAuthorityRoster.LAD22) + + assert metadata.country == "uk" + assert metadata.roster is UKLocalAuthorityRoster.LAD22 + + with pytest.raises(ValueError, match="extra_forbidden"): + UKLocalAuthorityMetadata.model_validate( + {"country": "uk", "roster": "lad23", "dataset": "microcosm"} + ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/resources/uk_local_authorities/coordinates_lad22.csv b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/resources/uk_local_authorities/coordinates_lad22.csv new file mode 100644 index 000000000..cc333e371 --- /dev/null +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/resources/uk_local_authorities/coordinates_lad22.csv @@ -0,0 +1,375 @@ +code,x,y +E06000001,8,19 +E06000002,9,18 +E06000003,9,19 +E06000004,8,18 +E06000005,7,18 +E06000006,1,11 +E06000007,2,11 +E06000008,4,15 +E06000009,2,15 +E06000010,10,15 +E06000011,11,16 +E06000012,11,14 +E06000013,10,14 +E06000014,9,17 +E06000015,6,11 +E06000016,8,8 +E06000017,10,9 +E06000018,8,10 +E06000019,0,8 +E06000020,2,9 +E06000021,3,10 +E06000022,1,3 +E06000023,0,3 +E06000024,0,2 +E06000025,1,4 +E06000026,-4,-2 +E06000027,-3,-2 +E06000030,2,4 +E06000031,11,9 +E06000032,10,7 +E06000033,16,6 +E06000034,15,4 +E06000035,15,1 +E06000036,4,2 +E06000037,2,2 +E06000038,2,3 +E06000039,6,4 +E06000040,4,3 +E06000041,3,3 +E06000042,6,5 +E06000043,9,-2 +E06000044,4,-1 +E06000045,2,0 +E06000046,1,-2 +E06000047,6,18 +E06000049,4,11 +E06000050,3,11 +E06000051,1,9 +E06000052,-5,-2 +E06000053,-7,-3 +E06000054,1,2 +E06000055,9,7 +E06000056,9,6 +E06000057,5,20 +E06000058,0,0 +E06000059,-1,0 +E06000060,5,5 +E06000061,9,9 +E06000062,7,6 +E07000008,12,8 +E07000009,12,9 +E07000010,13,10 +E07000011,10,8 +E07000012,11,8 +E07000026,4,18 +E07000027,2,17 +E07000028,4,19 +E07000029,3,18 +E07000030,5,18 +E07000031,4,17 +E07000032,7,11 +E07000033,10,12 +E07000034,9,12 +E07000035,7,12 +E07000036,7,9 +E07000037,7,13 +E07000038,8,12 +E07000039,6,10 +E07000040,-2,-1 +E07000041,-3,-1 +E07000042,-2,0 +E07000043,-3,1 +E07000044,-4,-3 +E07000045,-2,-2 +E07000046,-4,-1 +E07000047,-3,0 +E07000061,10,-2 +E07000062,13,-2 +E07000063,10,-1 +E07000064,12,-2 +E07000065,11,-2 +E07000066,14,5 +E07000067,14,7 +E07000068,13,5 +E07000069,15,5 +E07000070,14,6 +E07000071,15,8 +E07000072,12,5 +E07000073,13,6 +E07000074,15,7 +E07000075,15,6 +E07000076,16,8 +E07000077,13,7 +E07000078,1,5 +E07000079,2,5 +E07000080,-1,6 +E07000081,0,6 +E07000082,0,5 +E07000083,1,6 +E07000084,2,1 +E07000085,4,0 +E07000086,3,0 +E07000087,2,-1 +E07000088,3,-1 +E07000089,3,2 +E07000090,5,0 +E07000091,1,0 +E07000092,4,1 +E07000093,1,1 +E07000094,3,1 +E07000095,12,6 +E07000096,8,6 +E07000098,9,5 +E07000099,11,7 +E07000102,7,5 +E07000103,8,5 +E07000105,12,-1 +E07000106,15,0 +E07000107,13,1 +E07000108,14,-1 +E07000109,14,1 +E07000110,14,0 +E07000111,12,0 +E07000112,13,-1 +E07000113,16,0 +E07000114,15,-1 +E07000115,13,0 +E07000116,11,-1 +E07000117,6,15 +E07000118,3,14 +E07000119,4,16 +E07000120,5,15 +E07000121,3,17 +E07000122,6,16 +E07000123,5,16 +E07000124,5,17 +E07000125,6,14 +E07000126,3,15 +E07000127,2,13 +E07000128,3,16 +E07000129,7,7 +E07000130,8,9 +E07000131,8,7 +E07000132,7,8 +E07000133,11,10 +E07000134,6,9 +E07000135,9,8 +E07000136,12,12 +E07000137,12,13 +E07000138,11,12 +E07000139,11,11 +E07000140,12,11 +E07000141,12,10 +E07000142,11,13 +E07000143,14,10 +E07000144,15,12 +E07000145,15,11 +E07000146,13,11 +E07000147,14,12 +E07000148,14,11 +E07000149,15,10 +E07000163,6,17 +E07000164,10,18 +E07000165,8,17 +E07000166,7,17 +E07000167,10,16 +E07000168,10,17 +E07000169,9,16 +E07000170,8,11 +E07000171,10,13 +E07000172,7,10 +E07000173,9,10 +E07000174,9,11 +E07000175,10,11 +E07000176,10,10 +E07000177,4,5 +E07000178,4,4 +E07000179,5,4 +E07000180,3,4 +E07000181,3,5 +E07000187,0,1 +E07000188,-1,2 +E07000189,-1,1 +E07000192,3,9 +E07000193,5,11 +E07000194,4,9 +E07000195,2,10 +E07000196,2,8 +E07000197,4,10 +E07000198,5,10 +E07000199,5,9 +E07000200,14,8 +E07000202,15,9 +E07000203,14,9 +E07000207,7,2 +E07000208,8,0 +E07000209,5,1 +E07000210,6,1 +E07000211,7,0 +E07000212,5,3 +E07000213,6,3 +E07000214,5,2 +E07000215,9,-1 +E07000216,6,0 +E07000217,6,2 +E07000218,6,8 +E07000219,6,7 +E07000220,6,6 +E07000221,3,6 +E07000222,4,6 +E07000223,8,-2 +E07000224,6,-2 +E07000225,5,-1 +E07000226,8,-1 +E07000227,6,-1 +E07000228,7,-1 +E07000229,7,-2 +E07000234,2,7 +E07000235,-1,7 +E07000236,4,7 +E07000237,0,7 +E07000238,2,6 +E07000239,1,8 +E07000240,10,6 +E07000241,11,6 +E07000242,13,8 +E07000243,12,7 +E07000244,16,10 +E07000245,13,9 +E07000246,-2,1 +E08000001,4,14 +E08000002,5,14 +E08000003,5,12 +E08000004,5,13 +E08000005,7,14 +E08000006,4,13 +E08000007,6,12 +E08000008,6,13 +E08000009,4,12 +E08000010,3,13 +E08000011,2,12 +E08000012,1,13 +E08000013,3,12 +E08000014,2,14 +E08000015,1,12 +E08000016,8,14 +E08000017,9,14 +E08000018,9,13 +E08000019,8,13 +E08000021,5,19 +E08000022,6,20 +E08000023,7,20 +E08000024,7,19 +E08000025,5,8 +E08000026,5,6 +E08000027,1,7 +E08000028,3,7 +E08000029,5,7 +E08000030,4,8 +E08000031,3,8 +E08000032,7,16 +E08000033,7,15 +E08000034,8,15 +E08000035,8,16 +E08000036,9,15 +E08000037,6,19 +E09000001,11,2 +E09000002,13,3 +E09000003,10,5 +E09000004,12,1 +E09000005,10,4 +E09000006,11,0 +E09000007,11,4 +E09000008,10,0 +E09000009,9,4 +E09000010,11,5 +E09000011,11,1 +E09000012,12,3 +E09000013,8,3 +E09000014,12,4 +E09000015,8,4 +E09000016,14,3 +E09000017,7,4 +E09000018,7,3 +E09000019,11,3 +E09000020,9,3 +E09000021,7,1 +E09000022,10,2 +E09000023,10,1 +E09000024,8,1 +E09000025,13,2 +E09000026,14,4 +E09000027,8,2 +E09000028,9,1 +E09000029,9,0 +E09000030,12,2 +E09000031,13,4 +E09000032,9,2 +E09000033,10,3 +N09000001,-4,16 +N09000002,-5,16 +N09000003,-4,17 +N09000004,-5,18 +N09000005,-6,17 +N09000006,-6,16 +N09000007,-5,15 +N09000008,-4,18 +N09000009,-5,17 +N09000010,-4,15 +N09000011,-3,16 +S12000005,2,24 +S12000006,4,20 +S12000008,3,20 +S12000010,5,22 +S12000011,2,20 +S12000013,-1,27 +S12000014,2,23 +S12000017,1,26 +S12000018,0,21 +S12000019,3,21 +S12000020,2,26 +S12000021,1,20 +S12000023,4,28 +S12000026,4,21 +S12000027,5,30 +S12000028,1,19 +S12000029,2,21 +S12000030,1,24 +S12000033,4,26 +S12000034,3,26 +S12000035,0,24 +S12000036,4,22 +S12000038,1,22 +S12000039,0,23 +S12000040,3,22 +S12000041,2,25 +S12000042,3,25 +S12000045,1,23 +S12000047,3,24 +S12000048,1,25 +S12000049,1,21 +S12000050,2,22 +W06000001,-2,12 +W06000002,-2,10 +W06000003,-1,10 +W06000004,0,10 +W06000005,0,11 +W06000006,1,10 +W06000008,-2,9 +W06000009,-5,6 +W06000010,-4,6 +W06000011,-4,5 +W06000012,-3,5 +W06000013,-3,6 +W06000014,-2,4 +W06000015,-2,5 +W06000016,-3,7 +W06000018,-2,6 +W06000019,0,9 +W06000020,-2,7 +W06000021,-1,8 +W06000022,-1,5 +W06000023,-1,9 +W06000024,-2,8 diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/resources/uk_local_authorities/coordinates_lad23.csv b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/resources/uk_local_authorities/coordinates_lad23.csv new file mode 100644 index 000000000..591d36189 --- /dev/null +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/resources/uk_local_authorities/coordinates_lad23.csv @@ -0,0 +1,362 @@ +code,x,y +E06000001,8,19 +E06000002,9,18 +E06000003,9,19 +E06000004,8,18 +E06000005,7,18 +E06000006,1,11 +E06000007,2,11 +E06000008,4,15 +E06000009,2,15 +E06000010,10,15 +E06000011,11,16 +E06000012,11,14 +E06000013,10,14 +E06000014,9,17 +E06000015,6,11 +E06000016,8,8 +E06000017,10,9 +E06000018,8,10 +E06000019,0,8 +E06000020,2,9 +E06000021,3,10 +E06000022,1,3 +E06000023,0,3 +E06000024,0,2 +E06000025,1,4 +E06000026,-4,-2 +E06000027,-3,-2 +E06000030,2,4 +E06000031,11,9 +E06000032,10,7 +E06000033,16,6 +E06000034,15,4 +E06000035,15,1 +E06000036,4,2 +E06000037,2,2 +E06000038,2,3 +E06000039,6,4 +E06000040,4,3 +E06000041,3,3 +E06000042,6,5 +E06000043,9,-2 +E06000044,4,-1 +E06000045,2,0 +E06000046,1,-2 +E06000047,6,18 +E06000049,4,11 +E06000050,3,11 +E06000051,1,9 +E06000052,-5,-2 +E06000053,-7,-3 +E06000054,1,2 +E06000055,9,7 +E06000056,9,6 +E06000057,5,20 +E06000058,0,0 +E06000059,-1,0 +E06000060,5,5 +E06000061,9,9 +E06000062,7,6 +E06000063,4,18 +E06000064,4,17 +E06000065,8,17 +E06000066,0,1 +E07000008,12,8 +E07000009,12,9 +E07000010,13,10 +E07000011,10,8 +E07000012,11,8 +E07000032,7,11 +E07000033,10,12 +E07000034,9,12 +E07000035,7,12 +E07000036,7,9 +E07000037,7,13 +E07000038,8,12 +E07000039,6,10 +E07000040,-2,-1 +E07000041,-3,-1 +E07000042,-2,0 +E07000043,-3,1 +E07000044,-4,-3 +E07000045,-2,-2 +E07000046,-4,-1 +E07000047,-3,0 +E07000061,10,-2 +E07000062,13,-2 +E07000063,10,-1 +E07000064,12,-2 +E07000065,11,-2 +E07000066,14,5 +E07000067,14,7 +E07000068,13,5 +E07000069,15,5 +E07000070,14,6 +E07000071,15,8 +E07000072,12,5 +E07000073,13,6 +E07000074,15,7 +E07000075,15,6 +E07000076,16,8 +E07000077,13,7 +E07000078,1,5 +E07000079,2,5 +E07000080,-1,6 +E07000081,0,6 +E07000082,0,5 +E07000083,1,6 +E07000084,2,1 +E07000085,4,0 +E07000086,3,0 +E07000087,2,-1 +E07000088,3,-1 +E07000089,3,2 +E07000090,5,0 +E07000091,1,0 +E07000092,4,1 +E07000093,1,1 +E07000094,3,1 +E07000095,12,6 +E07000096,8,6 +E07000098,9,5 +E07000099,11,7 +E07000102,7,5 +E07000103,8,5 +E07000105,12,-1 +E07000106,15,0 +E07000107,13,1 +E07000108,14,-1 +E07000109,14,1 +E07000110,14,0 +E07000111,12,0 +E07000112,13,-1 +E07000113,16,0 +E07000114,15,-1 +E07000115,13,0 +E07000116,11,-1 +E07000117,6,15 +E07000118,3,14 +E07000119,4,16 +E07000120,5,15 +E07000121,3,17 +E07000122,6,16 +E07000123,5,16 +E07000124,5,17 +E07000125,6,14 +E07000126,3,15 +E07000127,2,13 +E07000128,3,16 +E07000129,7,7 +E07000130,8,9 +E07000131,8,7 +E07000132,7,8 +E07000133,11,10 +E07000134,6,9 +E07000135,9,8 +E07000136,12,12 +E07000137,12,13 +E07000138,11,12 +E07000139,11,11 +E07000140,12,11 +E07000141,12,10 +E07000142,11,13 +E07000143,14,10 +E07000144,15,12 +E07000145,15,11 +E07000146,13,11 +E07000147,14,12 +E07000148,14,11 +E07000149,15,10 +E07000170,8,11 +E07000171,10,13 +E07000172,7,10 +E07000173,9,10 +E07000174,9,11 +E07000175,10,11 +E07000176,10,10 +E07000177,4,5 +E07000178,4,4 +E07000179,5,4 +E07000180,3,4 +E07000181,3,5 +E07000192,3,9 +E07000193,5,11 +E07000194,4,9 +E07000195,2,10 +E07000196,2,8 +E07000197,4,10 +E07000198,5,10 +E07000199,5,9 +E07000200,14,8 +E07000202,15,9 +E07000203,14,9 +E07000207,7,2 +E07000208,8,0 +E07000209,5,1 +E07000210,6,1 +E07000211,7,0 +E07000212,5,3 +E07000213,6,3 +E07000214,5,2 +E07000215,9,-1 +E07000216,6,0 +E07000217,6,2 +E07000218,6,8 +E07000219,6,7 +E07000220,6,6 +E07000221,3,6 +E07000222,4,6 +E07000223,8,-2 +E07000224,6,-2 +E07000225,5,-1 +E07000226,8,-1 +E07000227,6,-1 +E07000228,7,-1 +E07000229,7,-2 +E07000234,2,7 +E07000235,-1,7 +E07000236,4,7 +E07000237,0,7 +E07000238,2,6 +E07000239,1,8 +E07000240,10,6 +E07000241,11,6 +E07000242,13,8 +E07000243,12,7 +E07000244,16,10 +E07000245,13,9 +E08000001,4,14 +E08000002,5,14 +E08000003,5,12 +E08000004,5,13 +E08000005,7,14 +E08000006,4,13 +E08000007,6,12 +E08000008,6,13 +E08000009,4,12 +E08000010,3,13 +E08000011,2,12 +E08000012,1,13 +E08000013,3,12 +E08000014,2,14 +E08000015,1,12 +E08000016,8,14 +E08000017,9,14 +E08000018,9,13 +E08000019,8,13 +E08000021,5,19 +E08000022,6,20 +E08000023,7,20 +E08000024,7,19 +E08000025,5,8 +E08000026,5,6 +E08000027,1,7 +E08000028,3,7 +E08000029,5,7 +E08000030,4,8 +E08000031,3,8 +E08000032,7,16 +E08000033,7,15 +E08000034,8,15 +E08000035,8,16 +E08000036,9,15 +E08000037,6,19 +E09000001,11,2 +E09000002,13,3 +E09000003,10,5 +E09000004,12,1 +E09000005,10,4 +E09000006,11,0 +E09000007,11,4 +E09000008,10,0 +E09000009,9,4 +E09000010,11,5 +E09000011,11,1 +E09000012,12,3 +E09000013,8,3 +E09000014,12,4 +E09000015,8,4 +E09000016,14,3 +E09000017,7,4 +E09000018,7,3 +E09000019,11,3 +E09000020,9,3 +E09000021,7,1 +E09000022,10,2 +E09000023,10,1 +E09000024,8,1 +E09000025,13,2 +E09000026,14,4 +E09000027,8,2 +E09000028,9,1 +E09000029,9,0 +E09000030,12,2 +E09000031,13,4 +E09000032,9,2 +E09000033,10,3 +N09000001,-4,16 +N09000002,-5,16 +N09000003,-4,17 +N09000004,-5,18 +N09000005,-6,17 +N09000006,-6,16 +N09000007,-5,15 +N09000008,-4,18 +N09000009,-5,17 +N09000010,-4,15 +N09000011,-3,16 +S12000005,2,24 +S12000006,4,20 +S12000008,3,20 +S12000010,5,22 +S12000011,2,20 +S12000013,-1,27 +S12000014,2,23 +S12000017,1,26 +S12000018,0,21 +S12000019,3,21 +S12000020,2,26 +S12000021,1,20 +S12000023,4,28 +S12000026,4,21 +S12000027,5,30 +S12000028,1,19 +S12000029,2,21 +S12000030,1,24 +S12000033,4,26 +S12000034,3,26 +S12000035,0,24 +S12000036,4,22 +S12000038,1,22 +S12000039,0,23 +S12000040,3,22 +S12000041,2,25 +S12000042,3,25 +S12000045,1,23 +S12000047,3,24 +S12000048,1,25 +S12000049,1,21 +S12000050,2,22 +W06000001,-2,12 +W06000002,-2,10 +W06000003,-1,10 +W06000004,0,10 +W06000005,0,11 +W06000006,1,10 +W06000008,-2,9 +W06000009,-5,6 +W06000010,-4,6 +W06000011,-4,5 +W06000012,-3,5 +W06000013,-3,6 +W06000014,-2,4 +W06000015,-2,5 +W06000016,-3,7 +W06000018,-2,6 +W06000019,0,9 +W06000020,-2,7 +W06000021,-1,8 +W06000022,-1,5 +W06000023,-1,9 +W06000024,-2,8 diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/resources/uk_local_authorities/names.csv b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/resources/uk_local_authorities/names.csv new file mode 100644 index 000000000..57ec2a049 --- /dev/null +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/resources/uk_local_authorities/names.csv @@ -0,0 +1,379 @@ +code,name +E06000001,Hartlepool +E06000002,Middlesbrough +E06000003,Redcar and Cleveland +E06000004,Stockton-on-Tees +E06000005,Darlington +E06000006,Halton +E06000007,Warrington +E06000008,Blackburn with Darwen +E06000009,Blackpool +E06000010,"Kingston upon Hull, City of" +E06000011,East Riding of Yorkshire +E06000012,North East Lincolnshire +E06000013,North Lincolnshire +E06000014,York +E06000015,Derby +E06000016,Leicester +E06000017,Rutland +E06000018,Nottingham +E06000019,"Herefordshire, County of" +E06000020,Telford and Wrekin +E06000021,Stoke-on-Trent +E06000022,Bath and North East Somerset +E06000023,"Bristol, City of" +E06000024,North Somerset +E06000025,South Gloucestershire +E06000026,Plymouth +E06000027,Torbay +E06000030,Swindon +E06000031,Peterborough +E06000032,Luton +E06000033,Southend-on-Sea +E06000034,Thurrock +E06000035,Medway +E06000036,Bracknell Forest +E06000037,West Berkshire +E06000038,Reading +E06000039,Slough +E06000040,Windsor and Maidenhead +E06000041,Wokingham +E06000042,Milton Keynes +E06000043,Brighton and Hove +E06000044,Portsmouth +E06000045,Southampton +E06000046,Isle of Wight +E06000047,County Durham +E06000049,Cheshire East +E06000050,Cheshire West and Chester +E06000051,Shropshire +E06000052,Cornwall +E06000053,Isles of Scilly +E06000054,Wiltshire +E06000055,Bedford +E06000056,Central Bedfordshire +E06000057,Northumberland +E06000058,"Bournemouth, Christchurch and Poole" +E06000059,Dorset +E06000060,Buckinghamshire +E06000061,North Northamptonshire +E06000062,West Northamptonshire +E06000063,Cumberland +E06000064,Westmorland and Furness +E06000065,North Yorkshire +E06000066,Somerset +E07000008,Cambridge +E07000009,East Cambridgeshire +E07000010,Fenland +E07000011,Huntingdonshire +E07000012,South Cambridgeshire +E07000026,Allerdale +E07000027,Barrow-in-Furness +E07000028,Carlisle +E07000029,Copeland +E07000030,Eden +E07000031,South Lakeland +E07000032,Amber Valley +E07000033,Bolsover +E07000034,Chesterfield +E07000035,Derbyshire Dales +E07000036,Erewash +E07000037,High Peak +E07000038,North East Derbyshire +E07000039,South Derbyshire +E07000040,East Devon +E07000041,Exeter +E07000042,Mid Devon +E07000043,North Devon +E07000044,South Hams +E07000045,Teignbridge +E07000046,Torridge +E07000047,West Devon +E07000061,Eastbourne +E07000062,Hastings +E07000063,Lewes +E07000064,Rother +E07000065,Wealden +E07000066,Basildon +E07000067,Braintree +E07000068,Brentwood +E07000069,Castle Point +E07000070,Chelmsford +E07000071,Colchester +E07000072,Epping Forest +E07000073,Harlow +E07000074,Maldon +E07000075,Rochford +E07000076,Tendring +E07000077,Uttlesford +E07000078,Cheltenham +E07000079,Cotswold +E07000080,Forest of Dean +E07000081,Gloucester +E07000082,Stroud +E07000083,Tewkesbury +E07000084,Basingstoke and Deane +E07000085,East Hampshire +E07000086,Eastleigh +E07000087,Fareham +E07000088,Gosport +E07000089,Hart +E07000090,Havant +E07000091,New Forest +E07000092,Rushmoor +E07000093,Test Valley +E07000094,Winchester +E07000095,Broxbourne +E07000096,Dacorum +E07000098,Hertsmere +E07000099,North Hertfordshire +E07000102,Three Rivers +E07000103,Watford +E07000105,Ashford +E07000106,Canterbury +E07000107,Dartford +E07000108,Dover +E07000109,Gravesham +E07000110,Maidstone +E07000111,Sevenoaks +E07000112,Folkestone and Hythe +E07000113,Swale +E07000114,Thanet +E07000115,Tonbridge and Malling +E07000116,Tunbridge Wells +E07000117,Burnley +E07000118,Chorley +E07000119,Fylde +E07000120,Hyndburn +E07000121,Lancaster +E07000122,Pendle +E07000123,Preston +E07000124,Ribble Valley +E07000125,Rossendale +E07000126,South Ribble +E07000127,West Lancashire +E07000128,Wyre +E07000129,Blaby +E07000130,Charnwood +E07000131,Harborough +E07000132,Hinckley and Bosworth +E07000133,Melton +E07000134,North West Leicestershire +E07000135,Oadby and Wigston +E07000136,Boston +E07000137,East Lindsey +E07000138,Lincoln +E07000139,North Kesteven +E07000140,South Holland +E07000141,South Kesteven +E07000142,West Lindsey +E07000143,Breckland +E07000144,Broadland +E07000145,Great Yarmouth +E07000146,King's Lynn and West Norfolk +E07000147,North Norfolk +E07000148,Norwich +E07000149,South Norfolk +E07000163,Craven +E07000164,Hambleton +E07000165,Harrogate +E07000166,Richmondshire +E07000167,Ryedale +E07000168,Scarborough +E07000169,Selby +E07000170,Ashfield +E07000171,Bassetlaw +E07000172,Broxtowe +E07000173,Gedling +E07000174,Mansfield +E07000175,Newark and Sherwood +E07000176,Rushcliffe +E07000177,Cherwell +E07000178,Oxford +E07000179,South Oxfordshire +E07000180,Vale of White Horse +E07000181,West Oxfordshire +E07000187,Mendip +E07000188,Sedgemoor +E07000189,South Somerset +E07000192,Cannock Chase +E07000193,East Staffordshire +E07000194,Lichfield +E07000195,Newcastle-under-Lyme +E07000196,South Staffordshire +E07000197,Stafford +E07000198,Staffordshire Moorlands +E07000199,Tamworth +E07000200,Babergh +E07000202,Ipswich +E07000203,Mid Suffolk +E07000207,Elmbridge +E07000208,Epsom and Ewell +E07000209,Guildford +E07000210,Mole Valley +E07000211,Reigate and Banstead +E07000212,Runnymede +E07000213,Spelthorne +E07000214,Surrey Heath +E07000215,Tandridge +E07000216,Waverley +E07000217,Woking +E07000218,North Warwickshire +E07000219,Nuneaton and Bedworth +E07000220,Rugby +E07000221,Stratford-on-Avon +E07000222,Warwick +E07000223,Adur +E07000224,Arun +E07000225,Chichester +E07000226,Crawley +E07000227,Horsham +E07000228,Mid Sussex +E07000229,Worthing +E07000234,Bromsgrove +E07000235,Malvern Hills +E07000236,Redditch +E07000237,Worcester +E07000238,Wychavon +E07000239,Wyre Forest +E07000240,St Albans +E07000241,Welwyn Hatfield +E07000242,East Hertfordshire +E07000243,Stevenage +E07000244,East Suffolk +E07000245,West Suffolk +E07000246,Somerset West and Taunton +E08000001,Bolton +E08000002,Bury +E08000003,Manchester +E08000004,Oldham +E08000005,Rochdale +E08000006,Salford +E08000007,Stockport +E08000008,Tameside +E08000009,Trafford +E08000010,Wigan +E08000011,Knowsley +E08000012,Liverpool +E08000013,St. Helens +E08000014,Sefton +E08000015,Wirral +E08000016,Barnsley +E08000017,Doncaster +E08000018,Rotherham +E08000019,Sheffield +E08000021,Newcastle upon Tyne +E08000022,North Tyneside +E08000023,South Tyneside +E08000024,Sunderland +E08000025,Birmingham +E08000026,Coventry +E08000027,Dudley +E08000028,Sandwell +E08000029,Solihull +E08000030,Walsall +E08000031,Wolverhampton +E08000032,Bradford +E08000033,Calderdale +E08000034,Kirklees +E08000035,Leeds +E08000036,Wakefield +E08000037,Gateshead +E09000001,City of London +E09000002,Barking and Dagenham +E09000003,Barnet +E09000004,Bexley +E09000005,Brent +E09000006,Bromley +E09000007,Camden +E09000008,Croydon +E09000009,Ealing +E09000010,Enfield +E09000011,Greenwich +E09000012,Hackney +E09000013,Hammersmith and Fulham +E09000014,Haringey +E09000015,Harrow +E09000016,Havering +E09000017,Hillingdon +E09000018,Hounslow +E09000019,Islington +E09000020,Kensington and Chelsea +E09000021,Kingston upon Thames +E09000022,Lambeth +E09000023,Lewisham +E09000024,Merton +E09000025,Newham +E09000026,Redbridge +E09000027,Richmond upon Thames +E09000028,Southwark +E09000029,Sutton +E09000030,Tower Hamlets +E09000031,Waltham Forest +E09000032,Wandsworth +E09000033,Westminster +N09000001,Antrim and Newtownabbey +N09000002,"Armagh City, Banbridge and Craigavon" +N09000003,Belfast +N09000004,Causeway Coast and Glens +N09000005,Derry City and Strabane +N09000006,Fermanagh and Omagh +N09000007,Lisburn and Castlereagh +N09000008,Mid and East Antrim +N09000009,Mid Ulster +N09000010,"Newry, Mourne and Down" +N09000011,Ards and North Down +S12000005,Clackmannanshire +S12000006,Dumfries and Galloway +S12000008,East Ayrshire +S12000010,East Lothian +S12000011,East Renfrewshire +S12000013,Na h-Eileanan Siar +S12000014,Falkirk +S12000017,Highland +S12000018,Inverclyde +S12000019,Midlothian +S12000020,Moray +S12000021,North Ayrshire +S12000023,Orkney Islands +S12000026,Scottish Borders +S12000027,Shetland Islands +S12000028,South Ayrshire +S12000029,South Lanarkshire +S12000030,Stirling +S12000033,Aberdeen City +S12000034,Aberdeenshire +S12000035,Argyll and Bute +S12000036,City of Edinburgh +S12000038,Renfrewshire +S12000039,West Dunbartonshire +S12000040,West Lothian +S12000041,Angus +S12000042,Dundee City +S12000045,East Dunbartonshire +S12000047,Fife +S12000048,Perth and Kinross +S12000049,Glasgow City +S12000050,North Lanarkshire +W06000001,Isle of Anglesey +W06000002,Gwynedd +W06000003,Conwy +W06000004,Denbighshire +W06000005,Flintshire +W06000006,Wrexham +W06000008,Ceredigion +W06000009,Pembrokeshire +W06000010,Carmarthenshire +W06000011,Swansea +W06000012,Neath Port Talbot +W06000013,Bridgend +W06000014,Vale of Glamorgan +W06000015,Cardiff +W06000016,Rhondda Cynon Taf +W06000018,Caerphilly +W06000019,Blaenau Gwent +W06000020,Torfaen +W06000021,Monmouthshire +W06000022,Newport +W06000023,Powys +W06000024,Merthyr Tydfil diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/uk_local_authority_metadata.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/uk_local_authority_metadata.py new file mode 100644 index 000000000..18fe1456c --- /dev/null +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/uk_local_authority_metadata.py @@ -0,0 +1,164 @@ +"""Packaged UK local-authority display metadata for temporary Stage 12 output.""" + +from __future__ import annotations + +import csv +import re +from dataclasses import dataclass +from functools import lru_cache +from importlib.resources import files +from types import MappingProxyType +from typing import Mapping + +from policyengine_simulation_contract.stage12_execution import ( + UKLocalAuthorityRoster, +) + +_CODE_PATTERN = re.compile(r"^[A-Z]\d{8}$") +_RESOURCE_PACKAGE = "policyengine_simulation_executor" +_RESOURCE_DIRECTORY = ("resources", "uk_local_authorities") + + +@dataclass(frozen=True, slots=True) +class LocalAuthorityCoordinate: + """Integer rendering coordinate for one local authority.""" + + x: int + y: int + + +@dataclass(frozen=True, slots=True) +class LocalAuthorityDisplayMetadata: + """Display-only metadata attached after authority impacts are calculated.""" + + name: str + x: int + y: int + + +@dataclass(frozen=True, slots=True) +class UKLocalAuthorityResources: + """Validated display names and coordinates for supported authority rosters.""" + + names: Mapping[str, str] + coordinates: Mapping[ + UKLocalAuthorityRoster, + Mapping[str, LocalAuthorityCoordinate], + ] + + def metadata_for( + self, + roster: UKLocalAuthorityRoster, + ) -> Mapping[str, LocalAuthorityDisplayMetadata]: + """Combine common names with one roster's rendering coordinates.""" + + return MappingProxyType( + { + code: LocalAuthorityDisplayMetadata( + name=self.names[code], + x=coordinate.x, + y=coordinate.y, + ) + for code, coordinate in self.coordinates[roster].items() + } + ) + + +def _resource_path(filename: str): + path = files(_RESOURCE_PACKAGE) + for part in _RESOURCE_DIRECTORY: + path = path.joinpath(part) + return path.joinpath(filename) + + +def _rows( + filename: str, expected_fields: tuple[str, ...] +) -> tuple[dict[str, str], ...]: + with _resource_path(filename).open("r", encoding="utf-8") as stream: + reader = csv.DictReader(stream) + if tuple(reader.fieldnames or ()) != expected_fields: + raise ValueError( + f"UK local-authority resource {filename!r} has invalid columns" + ) + rows: list[dict[str, str]] = [] + for raw in reader: + if any(raw.get(field) is None for field in expected_fields): + raise ValueError( + f"UK local-authority resource {filename!r} has an incomplete row" + ) + rows.append({field: str(raw[field]).strip() for field in expected_fields}) + return tuple(rows) + + +def _require_code(code: str, filename: str) -> None: + if not _CODE_PATTERN.fullmatch(code): + raise ValueError( + f"UK local-authority resource {filename!r} contains invalid code {code!r}" + ) + + +def _load_names() -> Mapping[str, str]: + filename = "names.csv" + names: dict[str, str] = {} + for row in _rows(filename, ("code", "name")): + code = row["code"] + name = row["name"] + _require_code(code, filename) + if not name or name == code: + raise ValueError( + f"UK local-authority resource {filename!r} has no name for {code!r}" + ) + if code in names: + raise ValueError( + f"UK local-authority resource {filename!r} repeats {code!r}" + ) + names[code] = name + return MappingProxyType(names) + + +def _load_coordinates(filename: str) -> Mapping[str, LocalAuthorityCoordinate]: + coordinates: dict[str, LocalAuthorityCoordinate] = {} + for row in _rows(filename, ("code", "x", "y")): + code = row["code"] + _require_code(code, filename) + if code in coordinates: + raise ValueError( + f"UK local-authority resource {filename!r} repeats {code!r}" + ) + try: + coordinate = LocalAuthorityCoordinate(x=int(row["x"]), y=int(row["y"])) + except ValueError as error: + raise ValueError( + f"UK local-authority resource {filename!r} has invalid coordinates " + f"for {code!r}" + ) from error + coordinates[code] = coordinate + return MappingProxyType(coordinates) + + +@lru_cache(maxsize=1) +def load_uk_local_authority_resources() -> UKLocalAuthorityResources: + """Load and validate every packaged Stage 12 local-authority resource.""" + + names = _load_names() + lad22 = _load_coordinates("coordinates_lad22.csv") + lad23 = _load_coordinates("coordinates_lad23.csv") + supported_codes = lad22.keys() | lad23.keys() + if names.keys() != supported_codes: + raise ValueError( + "UK local-authority names must exactly cover the supported rosters" + ) + for code in lad22.keys() & lad23.keys(): + if lad22[code] != lad23[code]: + raise ValueError( + f"UK local-authority coordinate differs between rosters for {code!r}" + ) + return UKLocalAuthorityResources( + names=names, + coordinates=MappingProxyType( + { + UKLocalAuthorityRoster.LAD22: lad22, + UKLocalAuthorityRoster.LAD23: lad23, + } + ), + ) diff --git a/projects/policyengine-simulation-executor/tests/test_uk_local_authority_metadata.py b/projects/policyengine-simulation-executor/tests/test_uk_local_authority_metadata.py new file mode 100644 index 000000000..8ade5459e --- /dev/null +++ b/projects/policyengine-simulation-executor/tests/test_uk_local_authority_metadata.py @@ -0,0 +1,71 @@ +"""Tests for packaged Stage 12 UK local-authority display metadata.""" + +from __future__ import annotations + +from policyengine_simulation_contract.stage12_execution import ( + UKLocalAuthorityRoster, +) + +from policyengine_simulation_executor.stage12_runtime.uk_local_authority_metadata import ( + load_uk_local_authority_resources, +) + + +LAD22_ONLY_CODES = { + "E07000026", + "E07000027", + "E07000028", + "E07000029", + "E07000030", + "E07000031", + "E07000163", + "E07000164", + "E07000165", + "E07000166", + "E07000167", + "E07000168", + "E07000169", + "E07000187", + "E07000188", + "E07000189", + "E07000246", +} +LAD23_ONLY_CODES = { + "E06000063", + "E06000064", + "E06000065", + "E06000066", +} + + +def test_packaged_resources_cover_the_exact_supported_rosters() -> None: + resources = load_uk_local_authority_resources() + lad22 = resources.coordinates[UKLocalAuthorityRoster.LAD22] + lad23 = resources.coordinates[UKLocalAuthorityRoster.LAD23] + + assert len(resources.names) == 378 + assert len(lad22) == 374 + assert len(lad23) == 361 + assert len(lad22.keys() & lad23.keys()) == 357 + assert lad22.keys() - lad23.keys() == LAD22_ONLY_CODES + assert lad23.keys() - lad22.keys() == LAD23_ONLY_CODES + assert resources.names.keys() == lad22.keys() | lad23.keys() + + +def test_shared_authorities_have_identical_coordinates() -> None: + resources = load_uk_local_authority_resources() + lad22 = resources.coordinates[UKLocalAuthorityRoster.LAD22] + lad23 = resources.coordinates[UKLocalAuthorityRoster.LAD23] + + for code in lad22.keys() & lad23.keys(): + assert lad22[code] == lad23[code] + + +def test_resource_lookup_combines_names_with_roster_coordinates() -> None: + resources = load_uk_local_authority_resources() + + cumberland = resources.metadata_for(UKLocalAuthorityRoster.LAD23)["E06000063"] + assert cumberland.name == "Cumberland" + assert isinstance(cumberland.x, int) + assert isinstance(cumberland.y, int) + assert "E06000063" not in resources.metadata_for(UKLocalAuthorityRoster.LAD22) From 960fb3d6ccca68d9374baab8a76193cdff328db0 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 30 Sep 2026 21:31:06 +0400 Subject: [PATCH 2/5] Detect Stage 12 UK local-authority rosters --- .../uk_local_authority_metadata.py | 102 ++++++++++++++++++ .../tests/test_uk_local_authority_metadata.py | 35 ++++++ 2 files changed, 137 insertions(+) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/uk_local_authority_metadata.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/uk_local_authority_metadata.py index 18fe1456c..944e29cf6 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/uk_local_authority_metadata.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/uk_local_authority_metadata.py @@ -4,6 +4,7 @@ import csv import re +from collections.abc import Iterable from dataclasses import dataclass from functools import lru_cache from importlib.resources import files @@ -11,12 +12,62 @@ from typing import Mapping from policyengine_simulation_contract.stage12_execution import ( + UKLocalAuthorityMetadata, UKLocalAuthorityRoster, ) _CODE_PATTERN = re.compile(r"^[A-Z]\d{8}$") _RESOURCE_PACKAGE = "policyengine_simulation_executor" _RESOURCE_DIRECTORY = ("resources", "uk_local_authorities") +_LAD22_ONLY_CODES = frozenset( + { + "E07000026", + "E07000027", + "E07000028", + "E07000029", + "E07000030", + "E07000031", + "E07000163", + "E07000164", + "E07000165", + "E07000166", + "E07000167", + "E07000168", + "E07000169", + "E07000187", + "E07000188", + "E07000189", + "E07000246", + } +) +_LAD23_ONLY_CODES = frozenset( + { + "E06000063", + "E06000064", + "E06000065", + "E06000066", + } +) + + +@dataclass(frozen=True, slots=True) +class LocalAuthorityRosterDefinition: + """Codes that distinguish one supported boundary configuration.""" + + roster: UKLocalAuthorityRoster + distinguishing_codes: frozenset[str] + + +_ROSTER_DEFINITIONS = ( + LocalAuthorityRosterDefinition( + roster=UKLocalAuthorityRoster.LAD22, + distinguishing_codes=_LAD22_ONLY_CODES, + ), + LocalAuthorityRosterDefinition( + roster=UKLocalAuthorityRoster.LAD23, + distinguishing_codes=_LAD23_ONLY_CODES, + ), +) @dataclass(frozen=True, slots=True) @@ -148,6 +199,10 @@ def load_uk_local_authority_resources() -> UKLocalAuthorityResources: raise ValueError( "UK local-authority names must exactly cover the supported rosters" ) + if lad22.keys() - lad23.keys() != _LAD22_ONLY_CODES: + raise ValueError("LAD22 distinguishing codes do not match its coordinate file") + if lad23.keys() - lad22.keys() != _LAD23_ONLY_CODES: + raise ValueError("LAD23 distinguishing codes do not match its coordinate file") for code in lad22.keys() & lad23.keys(): if lad22[code] != lad23[code]: raise ValueError( @@ -162,3 +217,50 @@ def load_uk_local_authority_resources() -> UKLocalAuthorityResources: } ), ) + + +def _normalise_observed_code(value: object) -> str: + if isinstance(value, bytes): + try: + code = value.decode("utf-8").strip() + except UnicodeDecodeError as error: + raise ValueError( + "UK dataset contains an invalid local-authority code" + ) from error + elif isinstance(value, str): + code = value.strip() + else: + raise TypeError("UK dataset contains a non-text local-authority code") + if not _CODE_PATTERN.fullmatch(code): + raise ValueError(f"UK dataset contains invalid local-authority code {code!r}") + return code + + +def detect_uk_local_authority_roster( + values: Iterable[object], +) -> UKLocalAuthorityMetadata: + """Identify LAD22 or LAD23 from the unscoped dataset's authority codes.""" + + observed_codes = frozenset(_normalise_observed_code(value) for value in values) + if not observed_codes: + raise ValueError("UK dataset contains no local-authority codes") + supported_codes = load_uk_local_authority_resources().names.keys() + unsupported_codes = observed_codes - supported_codes + if unsupported_codes: + first = min(unsupported_codes) + raise ValueError( + f"UK dataset contains unsupported local-authority code {first!r}" + ) + + matches = tuple( + definition + for definition in _ROSTER_DEFINITIONS + if observed_codes & definition.distinguishing_codes + ) + if len(matches) > 1: + raise ValueError("UK dataset mixes LAD22 and LAD23 local-authority codes") + if not matches: + raise ValueError( + "UK dataset local-authority configuration cannot be identified" + ) + return UKLocalAuthorityMetadata(roster=matches[0].roster) diff --git a/projects/policyengine-simulation-executor/tests/test_uk_local_authority_metadata.py b/projects/policyengine-simulation-executor/tests/test_uk_local_authority_metadata.py index 8ade5459e..8c7ab5aa4 100644 --- a/projects/policyengine-simulation-executor/tests/test_uk_local_authority_metadata.py +++ b/projects/policyengine-simulation-executor/tests/test_uk_local_authority_metadata.py @@ -2,11 +2,14 @@ from __future__ import annotations +import pytest + from policyengine_simulation_contract.stage12_execution import ( UKLocalAuthorityRoster, ) from policyengine_simulation_executor.stage12_runtime.uk_local_authority_metadata import ( + detect_uk_local_authority_roster, load_uk_local_authority_resources, ) @@ -69,3 +72,35 @@ def test_resource_lookup_combines_names_with_roster_coordinates() -> None: assert isinstance(cumberland.x, int) assert isinstance(cumberland.y, int) assert "E06000063" not in resources.metadata_for(UKLocalAuthorityRoster.LAD22) + + +def test_detector_identifies_lad22_from_a_predecessor_code() -> None: + metadata = detect_uk_local_authority_roster( + ["E06000001", " E07000026 ", "S12000033"] + ) + + assert metadata.roster is UKLocalAuthorityRoster.LAD22 + + +def test_detector_identifies_lad23_from_a_successor_code() -> None: + metadata = detect_uk_local_authority_roster( + [b"E06000001", b"E06000063", b"S12000033"] + ) + + assert metadata.roster is UKLocalAuthorityRoster.LAD23 + + +def test_detector_rejects_mixed_authority_configurations() -> None: + with pytest.raises(ValueError, match="mixes LAD22 and LAD23"): + detect_uk_local_authority_roster(["E07000026", "E06000063"]) + + +def test_detector_rejects_an_unidentifiable_configuration() -> None: + with pytest.raises(ValueError, match="cannot be identified"): + detect_uk_local_authority_roster(["E06000001", "S12000033"]) + + +@pytest.mark.parametrize("value", [None, "", "UNKNOWN", "E06000999"]) +def test_detector_rejects_missing_or_unsupported_codes(value: object) -> None: + with pytest.raises((TypeError, ValueError), match="local-authority code"): + detect_uk_local_authority_roster([value]) From d225d65eba20226ddd8a18c7b73e7227e250b1c0 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 30 Sep 2026 21:50:04 +0400 Subject: [PATCH 3/5] Carry typed UK roster metadata through Stage 12 artifacts --- .../stage12_execution.py | 3 + .../tests/test_stage12_execution.py | 8 +++ .../stage12_artifacts.py | 22 ++++++ .../stage12_qualification.py | 11 +++ .../stage12_runtime/aggregation.py | 21 ++++++ .../stage12_runtime/coordination.py | 13 +++- .../stage12_runtime/simulation.py | 33 +++++++++ .../tests/test_stage12_artifacts.py | 22 ++++++ .../tests/test_stage12_runtime.py | 68 ++++++++++++++++++- 9 files changed, 199 insertions(+), 2 deletions(-) diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution.py index 93dc54f71..a778ac606 100644 --- a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution.py +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution.py @@ -99,6 +99,9 @@ class SimulationParquetPayloadContract(StrictContractModel): calculation_provenance_metadata_key: Literal[ "policyengine.stage12.calculation_provenance" ] = "policyengine.stage12.calculation_provenance" + uk_local_authority_metadata_key: Literal[ + "policyengine.stage12.uk_local_authority_metadata" + ] = "policyengine.stage12.uk_local_authority_metadata" SIMULATION_PARQUET_PAYLOAD_CONTRACT = SimulationParquetPayloadContract() diff --git a/libs/policyengine-simulation-contract/tests/test_stage12_execution.py b/libs/policyengine-simulation-contract/tests/test_stage12_execution.py index 0642daa9e..704678b76 100644 --- a/libs/policyengine-simulation-contract/tests/test_stage12_execution.py +++ b/libs/policyengine-simulation-contract/tests/test_stage12_execution.py @@ -5,6 +5,7 @@ import pytest from policyengine_simulation_contract.stage12_execution import ( + SIMULATION_PARQUET_PAYLOAD_CONTRACT, EntityOutputPlan, ReportAggregate, ReportOutputRequirements, @@ -129,3 +130,10 @@ def test_uk_local_authority_metadata_is_strict_and_typed() -> None: UKLocalAuthorityMetadata.model_validate( {"country": "uk", "roster": "lad23", "dataset": "microcosm"} ) + + +def test_parquet_contract_names_uk_local_authority_metadata() -> None: + assert ( + SIMULATION_PARQUET_PAYLOAD_CONTRACT.uk_local_authority_metadata_key + == "policyengine.stage12.uk_local_authority_metadata" + ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_artifacts.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_artifacts.py index 7427b208d..56d3e3968 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_artifacts.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_artifacts.py @@ -22,6 +22,7 @@ ResultComparisonArtifactPayload, RowIdentity, SimulationArtifactDescriptor, + UKLocalAuthorityMetadata, stage12_output_plan_sha256, ) from pydantic import JsonValue @@ -118,6 +119,7 @@ def serialize_simulation_frames( frames: Mapping[str, pd.DataFrame], *, calculation_provenance: Mapping[str, Any] | None = None, + uk_local_authority_metadata: UKLocalAuthorityMetadata | None = None, ) -> tuple[bytes, RowIdentity]: combined, dtypes, identifier_values = _frame_payload(frames) identity_payload = canonical_json_bytes(identifier_values) @@ -140,6 +142,10 @@ def serialize_simulation_frames( metadata[PARQUET_CONTRACT.calculation_provenance_metadata_key.encode()] = ( canonical_json_bytes(calculation_provenance) ) + if uk_local_authority_metadata is not None: + metadata[PARQUET_CONTRACT.uk_local_authority_metadata_key.encode()] = ( + canonical_json_bytes(uk_local_authority_metadata.model_dump(mode="json")) + ) table = table.replace_schema_metadata(metadata) buffer = BytesIO() pq.write_table( @@ -210,6 +216,20 @@ def deserialize_calculation_provenance(payload: bytes) -> dict[str, Any] | None: return value +def deserialize_uk_local_authority_metadata( + payload: bytes, +) -> UKLocalAuthorityMetadata | None: + """Read typed UK authority metadata from a Stage 12 simulation artifact.""" + + table = pq.read_table(BytesIO(payload)) + raw = (table.schema.metadata or {}).get( + PARQUET_CONTRACT.uk_local_authority_metadata_key.encode() + ) + if raw is None: + return None + return UKLocalAuthorityMetadata.model_validate_json(raw) + + class Stage12ArtifactStore: def __init__(self, bucket_name: str, *, store: ArtifactStore | None = None): if not bucket_name: @@ -264,6 +284,7 @@ def write_simulation( simulation: PlannedSimulationExecutionInput, frames: Mapping[str, pd.DataFrame], calculation_provenance: Mapping[str, Any] | None = None, + uk_local_authority_metadata: UKLocalAuthorityMetadata | None = None, ) -> SimulationArtifactDescriptor: normalized_provenance = cast( dict[str, JsonValue] | None, @@ -276,6 +297,7 @@ def write_simulation( payload, row_identity = serialize_simulation_frames( frames, calculation_provenance=normalized_provenance, + uk_local_authority_metadata=uk_local_authority_metadata, ) artifact = self._write_immutable( simulation_path(prefix=prefix, role=simulation.role.value), diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_qualification.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_qualification.py index 635e17dc0..8ffd4561f 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_qualification.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_qualification.py @@ -43,6 +43,9 @@ SimulationCalculation, simulation_input_sha256, ) +from policyengine_simulation_executor.stage12_runtime.aggregation import ( + validate_uk_local_authority_metadata, +) from policyengine_simulation_executor.stage12_runtime.output_planning import ( plan_simulation_input, resolve_report_output_plan, @@ -187,6 +190,7 @@ def _descriptor( payload, row_identity = serialize_simulation_frames( calculation.frames, calculation_provenance=calculation.calculation_provenance, + uk_local_authority_metadata=calculation.uk_local_authority_metadata, ) return SimulationArtifactDescriptor( evaluation_id=simulation.evaluation_id, @@ -303,6 +307,11 @@ def run_existing(request: dict[str, Any]) -> Mapping[str, Any]: reform_frames=reform_calculation.frames, baseline_descriptor=baseline_descriptor, reform_descriptor=reform_descriptor, + uk_local_authority_metadata=validate_uk_local_authority_metadata( + report.baseline.geography.country, + baseline_calculation.uk_local_authority_metadata, + reform_calculation.uk_local_authority_metadata, + ), ) v2_result = v2_report.get("result") if not isinstance(v2_result, Mapping): @@ -318,10 +327,12 @@ def run_existing(request: dict[str, Any]) -> Mapping[str, Any]: v2_baseline_payload, _ = serialize_simulation_frames( baseline_calculation.frames, calculation_provenance=baseline_calculation.calculation_provenance, + uk_local_authority_metadata=baseline_calculation.uk_local_authority_metadata, ) v2_reform_payload, _ = serialize_simulation_frames( reform_calculation.frames, calculation_provenance=reform_calculation.calculation_provenance, + uk_local_authority_metadata=reform_calculation.uk_local_authority_metadata, ) return Stage12ParityReceipt( evaluation_id=str(report.evaluation_id), diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/aggregation.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/aggregation.py index 18e9b98ac..3973e125e 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/aggregation.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/aggregation.py @@ -11,11 +11,30 @@ ReportExecutionInput, SimulationArtifactDescriptor, Stage12OutputPlan, + UKLocalAuthorityMetadata, stage12_output_plan_sha256, ) from pydantic import JsonValue +def validate_uk_local_authority_metadata( + country: CountryId, + baseline: UKLocalAuthorityMetadata | None, + reform: UKLocalAuthorityMetadata | None, +) -> UKLocalAuthorityMetadata | None: + """Require matching UK authority metadata and reject it elsewhere.""" + + if country == "uk": + if baseline is None or reform is None: + raise ValueError("UK simulation artifact metadata is missing") + if baseline != reform: + raise ValueError("UK simulation artifact authority rosters do not match") + return baseline + if baseline is not None or reform is not None: + raise ValueError("non-UK simulation artifacts contain UK authority metadata") + return None + + def validate_aligned_outputs( report: ReportExecutionInput, output_plan: Stage12OutputPlan, @@ -93,7 +112,9 @@ def build_aggregate_report( reform_frames: Mapping[str, pd.DataFrame], baseline_descriptor: SimulationArtifactDescriptor, reform_descriptor: SimulationArtifactDescriptor, + uk_local_authority_metadata: UKLocalAuthorityMetadata | None = None, ) -> dict[str, Any]: + del uk_local_authority_metadata from policyengine_simulation_executor.segmented_national_reduce import ( PrecomputedSimulation, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/coordination.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/coordination.py index fe59afbaf..e6d285916 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/coordination.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/coordination.py @@ -34,9 +34,14 @@ Stage12ArtifactStore, deserialize_calculation_provenance, deserialize_simulation_frames, + deserialize_uk_local_authority_metadata, ) -from .aggregation import build_aggregate_report, validate_aligned_outputs +from .aggregation import ( + build_aggregate_report, + validate_aligned_outputs, + validate_uk_local_authority_metadata, +) from .comparison import compare_completed_report from .dependencies import ( ChildCall, @@ -411,6 +416,11 @@ def coordinate_report( raise ValueError("reform artifact digest mismatch") baseline_frames = deserialize_simulation_frames(baseline_payload) reform_frames = deserialize_simulation_frames(reform_payload) + uk_local_authority_metadata = validate_uk_local_authority_metadata( + report.baseline.geography.country, + deserialize_uk_local_authority_metadata(baseline_payload), + deserialize_uk_local_authority_metadata(reform_payload), + ) validate_output_frames(baseline_frames, output_plan) validate_output_frames(reform_frames, output_plan) aggregation_span = ( @@ -425,6 +435,7 @@ def coordinate_report( reform_frames=reform_frames, baseline_descriptor=baseline, reform_descriptor=reform, + uk_local_authority_metadata=uk_local_authority_metadata, ) aggregate_write_span = ( runtime.span(stage_plan.name(Stage.STAGE12_AGGREGATE_ARTIFACT_WRITE)) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/simulation.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/simulation.py index 14b96a23b..b5b0202c5 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/simulation.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/simulation.py @@ -19,6 +19,7 @@ SimulationArtifactDescriptor, SimulationExecutionInput, Stage12InvocationContext, + UKLocalAuthorityMetadata, stage12_output_plan_sha256, ) from policyengine_simulation_observability.stages import ( @@ -40,6 +41,30 @@ class SimulationCalculation: frames: Mapping[str, pd.DataFrame] calculation_provenance: dict[str, Any] | None = None + uk_local_authority_metadata: UKLocalAuthorityMetadata | None = None + + +def _detect_uk_local_authority_metadata( + country: CountryId, + dataset: object, +) -> UKLocalAuthorityMetadata | None: + """Inspect a complete dataset before any requested regional scoping.""" + + if country != "uk": + return None + data = getattr(dataset, "data", None) + entity_data = getattr(data, "entity_data", None) + if not isinstance(entity_data, Mapping): + raise TypeError("UK dataset contains no entity tables") + household = entity_data.get("household") + if household is None: + raise ValueError("UK dataset contains no household table") + household_frame = pd.DataFrame(household) + if "la_code_oa" not in household_frame: + raise ValueError("UK dataset household table contains no la_code_oa column") + from .uk_local_authority_metadata import detect_uk_local_authority_roster + + return detect_uk_local_authority_roster(household_frame["la_code_oa"].tolist()) def simulation_input_sha256(simulation: SimulationExecutionInput) -> str: @@ -174,6 +199,10 @@ def calculate_simulation_frames( selection=dataset_selection, country_module=country_module, ) + uk_local_authority_metadata = _detect_uk_local_authority_metadata( + country, + dataset, + ) policy_span = ( runtime.span(STAGE12_SIMULATION_STAGES.name(Stage.POLICY_NORMALIZATION)) if runtime is not None @@ -231,6 +260,7 @@ def calculate_simulation_frames( return SimulationCalculation( frames=frames, calculation_provenance=calculation_provenance, + uk_local_authority_metadata=uk_local_authority_metadata, ) @@ -336,9 +366,11 @@ def run_single_simulation( if isinstance(calculated, SimulationCalculation): frames = calculated.frames calculation_provenance = calculated.calculation_provenance + uk_local_authority_metadata = calculated.uk_local_authority_metadata else: frames = calculated calculation_provenance = None + uk_local_authority_metadata = None validate_output_frames(frames, simulation.output_plan) artifact_span = ( runtime.span( @@ -353,6 +385,7 @@ def run_single_simulation( simulation=simulation, frames=frames, calculation_provenance=calculation_provenance, + uk_local_authority_metadata=uk_local_authority_metadata, ) completed = datetime.now(UTC) persistence.replace_simulation( diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_artifacts.py b/projects/policyengine-simulation-executor/tests/test_stage12_artifacts.py index 019035377..e0bca5e5b 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_artifacts.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_artifacts.py @@ -8,12 +8,17 @@ import numpy as np import pandas as pd import pytest +from policyengine_simulation_contract.stage12_execution import ( + UKLocalAuthorityMetadata, + UKLocalAuthorityRoster, +) from policyengine_simulation_executor.stage12_artifacts import ( Stage12ArtifactStore, comparison_run_prefix, deserialize_calculation_provenance, deserialize_simulation_frames, + deserialize_uk_local_authority_metadata, serialize_simulation_frames, ) @@ -77,6 +82,23 @@ def test_parquet_retains_detached_calculation_provenance() -> None: assert deserialize_calculation_provenance(payload) == provenance +def test_parquet_retains_typed_uk_local_authority_metadata() -> None: + metadata = UKLocalAuthorityMetadata(roster=UKLocalAuthorityRoster.LAD22) + + payload, _ = serialize_simulation_frames( + _frames(), + uk_local_authority_metadata=metadata, + ) + + assert deserialize_uk_local_authority_metadata(payload) == metadata + + +def test_parquet_omits_uk_metadata_for_non_uk_simulations() -> None: + payload, _ = serialize_simulation_frames(_frames()) + + assert deserialize_uk_local_authority_metadata(payload) is None + + def test_duplicate_entity_identifiers_are_rejected() -> None: frames = _frames() frames["person"]["person_id"] = [1, 1] diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_runtime.py b/projects/policyengine-simulation-executor/tests/test_stage12_runtime.py index d5fd4166a..b1b2d1110 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_runtime.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_runtime.py @@ -38,6 +38,8 @@ SimulationRole, Stage12InvocationContext, Stage12OutputPlan, + UKLocalAuthorityMetadata, + UKLocalAuthorityRoster, stage12_output_plan_sha256, ) @@ -53,6 +55,12 @@ run_single_simulation, simulation_input_sha256, ) +from policyengine_simulation_executor.stage12_runtime.aggregation import ( + validate_uk_local_authority_metadata, +) +from policyengine_simulation_executor.stage12_runtime.simulation import ( + _detect_uk_local_authority_metadata, +) NOW = datetime(2026, 9, 15, tzinfo=UTC) EVALUATION_ID = UUID("00000000-0000-0000-0000-000000000001") @@ -330,8 +338,29 @@ def write_simulation( simulation, frames, calculation_provenance=None, + uk_local_authority_metadata=None, ): - descriptor = self.add_simulation(simulation, frames) + payload, row_identity = serialize_simulation_frames( + frames, + calculation_provenance=calculation_provenance, + uk_local_authority_metadata=uk_local_authority_metadata, + ) + uri = f"gs://private/{simulation.role.value}.parquet" + self.payloads[uri] = payload + descriptor = SimulationArtifactDescriptor( + evaluation_id=simulation.evaluation_id, + simulation_execution_id=simulation.simulation_execution_id, + role=simulation.role, + artifact=ArtifactReference( + uri=uri, + media_type=ArtifactMediaType.PARQUET, + content_sha256=sha256(payload).hexdigest(), + size_bytes=len(payload), + ), + output_plan_sha256=stage12_output_plan_sha256(_output_plan()), + row_identity=row_identity, + bundle=simulation.bundle, + ) return descriptor.model_copy( update={"calculation_provenance": calculation_provenance} ) @@ -459,6 +488,42 @@ def build_simulation(params, *, dataset, dataset_selection, **kwargs): assert set(result.frames) == {"household", "person"} +def test_uk_roster_is_detected_from_the_unscoped_dataset() -> None: + dataset = type( + "Dataset", + (), + { + "data": type( + "Data", + (), + { + "entity_data": { + "household": pd.DataFrame( + {"la_code_oa": ["E06000001", "E07000026"]} + ) + } + }, + )() + }, + )() + + metadata = _detect_uk_local_authority_metadata("uk", dataset) + + assert metadata == UKLocalAuthorityMetadata(roster=UKLocalAuthorityRoster.LAD22) + + +def test_country_metadata_alignment_rejects_missing_or_mismatched_uk_values() -> None: + lad22 = UKLocalAuthorityMetadata(roster=UKLocalAuthorityRoster.LAD22) + lad23 = UKLocalAuthorityMetadata(roster=UKLocalAuthorityRoster.LAD23) + + with pytest.raises(ValueError, match="missing"): + validate_uk_local_authority_metadata("uk", None, lad22) + with pytest.raises(ValueError, match="do not match"): + validate_uk_local_authority_metadata("uk", lad22, lad23) + assert validate_uk_local_authority_metadata("uk", lad22, lad22) == lad22 + assert validate_uk_local_authority_metadata("us", None, None) is None + + def test_single_worker_accepts_one_policy_and_persists_one_artifact() -> None: store = FakeStore() simulation = _planned_simulation(SimulationRole.BASELINE) @@ -867,6 +932,7 @@ def resolver(report): assert observed["reform_frames"]["household"]["household_net_income"].tolist() == [ 120.0 ] + assert observed["uk_local_authority_metadata"] is None assert store.parent.status is ComparisonRunLifecycleStatus.SUCCEEDED assert store.parent.aggregation_status is ComparisonRunAggregationStatus.SUCCEEDED assert store.parent.coordinator_invocation_id == "coordinator-1" From 7317d091d6676c84ca9af728fab5698ec3c27cb0 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 30 Sep 2026 22:05:18 +0400 Subject: [PATCH 4/5] Build Stage 12 local-authority output from detected roster --- .../simulation_output_builder.py | 9 +- .../simulation_output_geographic.py | 54 +++++++- .../stage12_runtime/aggregation.py | 2 +- .../test_simulation_output_geographic.py | 128 +++++++++++++++++- .../tests/test_stage12_runtime.py | 1 + 5 files changed, 187 insertions(+), 7 deletions(-) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_builder.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_builder.py index 9d0d442d9..9a2a1cfb6 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_builder.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_builder.py @@ -7,6 +7,9 @@ from typing import Any from policyengine_observability import ObservabilityRuntime +from policyengine_simulation_contract.stage12_execution import ( + UKLocalAuthorityMetadata, +) from policyengine_simulation_executor import simulation_output_budget from policyengine_simulation_executor import simulation_output_cliff @@ -49,6 +52,7 @@ class SimulationOutputBuilder: resolved_data_version: str | None = None resolved_region_code: str | None = None runtime: ObservabilityRuntime | None = None + uk_local_authority_metadata: UKLocalAuthorityMetadata | None = None _analysis: Any = field(default=None, init=False) def __post_init__(self) -> None: @@ -238,7 +242,10 @@ def _build_uk_constituency_impact(self) -> GeographicImpactOutput | None: def _build_uk_local_authority_impact(self) -> GeographicImpactOutput | None: with self._span(ANNUAL_IMPACT_STAGES.name(Stage.OUTPUT_UK_LOCAL_AUTHORITY)): return simulation_output_geographic.build_uk_local_authority_impact( - self.country, self.baseline, self.reform + self.country, + self.baseline, + self.reform, + uk_local_authority_metadata=self.uk_local_authority_metadata, ) def _model_version(self) -> str: diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py index 4e8abce55..48971c709 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py @@ -6,10 +6,14 @@ from functools import lru_cache from typing import TYPE_CHECKING, Any +import pandas as pd from policyengine.data.uk_geography_assets import ( CONSTITUENCY_ASSET_SPEC, LOCAL_AUTHORITY_ASSET_SPEC, ) +from policyengine_simulation_contract.stage12_execution import ( + UKLocalAuthorityMetadata, +) from policyengine_simulation_executor.simulation_macro_output import ( CongressionalDistrictImpactOutput, @@ -232,11 +236,59 @@ def build_uk_constituency_impact( def build_uk_local_authority_impact( - country: str, baseline, reform + country: str, + baseline, + reform, + *, + uk_local_authority_metadata: UKLocalAuthorityMetadata | None = None, ) -> GeographicImpactOutput | None: if country != "uk": return None + if uk_local_authority_metadata is not None: + from policyengine.outputs.uk_geography_impact import ( + compute_longwise_uk_geography_impacts, + ) + from policyengine_simulation_executor.stage12_runtime.uk_local_authority_metadata import ( + load_uk_local_authority_resources, + ) + + baseline_household = pd.DataFrame(baseline.output_dataset.data.household) + reform_household = pd.DataFrame(reform.output_dataset.data.household) + numeric_records = compute_longwise_uk_geography_impacts( + baseline_household=baseline_household, + reform_household=reform_household, + geography_column="la_code_oa", + result_key_prefix="local_authority", + lookup_csv_path=None, + ) + roster = uk_local_authority_metadata.roster + display_metadata = load_uk_local_authority_resources().metadata_for(roster) + records: list[dict[str, object]] = [] + for numeric_record in numeric_records: + code = numeric_record.get("local_authority_code") + if not isinstance(code, str) or not code: + raise ValueError("UK local-authority output contains an invalid code") + metadata = display_metadata.get(code) + if metadata is None: + raise ValueError( + f"UK local-authority code {code!r} is not part of the detected " + f"{roster.value.upper()} roster" + ) + records.append( + { + **numeric_record, + "local_authority_name": metadata.name, + "x": metadata.x, + "y": metadata.y, + } + ) + return _complete_uk_geography_output( + records, + code_field="local_authority_code", + name_field="local_authority_name", + ) + lookup_csv_path = _required_uk_geography_lookup_csv_path(LOCAL_AUTHORITY_ASSET_SPEC) impact = _output_module_function( "local_authority_impact", "compute_uk_local_authority_impacts" diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/aggregation.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/aggregation.py index 3973e125e..c61e608bd 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/aggregation.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/aggregation.py @@ -114,7 +114,6 @@ def build_aggregate_report( reform_descriptor: SimulationArtifactDescriptor, uk_local_authority_metadata: UKLocalAuthorityMetadata | None = None, ) -> dict[str, Any]: - del uk_local_authority_metadata from policyengine_simulation_executor.segmented_national_reduce import ( PrecomputedSimulation, ) @@ -168,6 +167,7 @@ def stand_in(dataset, policy: dict[str, JsonValue]): reform=stand_in(datasets["reform"], report.reform.policy), resolved_data_version=report.baseline.bundle.dataset.artifact_revision, resolved_region_code=report.baseline.geography.region, + uk_local_authority_metadata=uk_local_authority_metadata, ).serialize() output.update( build_spm_result( diff --git a/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py b/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py index ec9f30f75..815a84e19 100644 --- a/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py +++ b/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py @@ -3,9 +3,15 @@ from __future__ import annotations from contextlib import nullcontext +from types import SimpleNamespace +import pandas as pd import pytest from policyengine.data.uk_geography_assets import CONSTITUENCY_ASSET_SPEC +from policyengine_simulation_contract.stage12_execution import ( + UKLocalAuthorityMetadata, + UKLocalAuthorityRoster, +) from policyengine_simulation_executor import simulation_output_geographic @@ -22,6 +28,22 @@ def _complete_constituency_record() -> dict[str, object]: } +def _uk_simulation( + code: str, + income: float, +) -> SimpleNamespace: + household = pd.DataFrame( + { + "la_code_oa": [code], + "household_net_income": [income], + "household_weight": [2.0], + } + ) + return SimpleNamespace( + output_dataset=SimpleNamespace(data=SimpleNamespace(household=household)) + ) + + def test_required_uk_lookup_uses_gcp_credentials(monkeypatch) -> None: observed: list[str] = [] @@ -40,8 +62,7 @@ def resolve(spec, **kwargs): credentials, ) monkeypatch.setattr( - "policyengine.outputs.uk_geography_impact." - "resolve_uk_geography_lookup_csv_path", + "policyengine.outputs.uk_geography_impact.resolve_uk_geography_lookup_csv_path", resolve, ) @@ -59,8 +80,7 @@ def test_required_uk_lookup_rejects_missing_asset(monkeypatch) -> None: nullcontext, ) monkeypatch.setattr( - "policyengine.outputs.uk_geography_impact." - "resolve_uk_geography_lookup_csv_path", + "policyengine.outputs.uk_geography_impact.resolve_uk_geography_lookup_csv_path", lambda *args, **kwargs: None, ) @@ -115,3 +135,103 @@ def test_complete_uk_geography_output_rejects_fallback_metadata( code_field="constituency_code", name_field="constituency_name", ) + + +@pytest.mark.parametrize( + ("roster", "code", "expected_name"), + [ + (UKLocalAuthorityRoster.LAD22, "E07000026", "Allerdale"), + (UKLocalAuthorityRoster.LAD23, "E06000063", "Cumberland"), + ], +) +def test_stage12_local_authority_output_uses_detected_roster_metadata( + monkeypatch, + roster: UKLocalAuthorityRoster, + code: str, + expected_name: str, +) -> None: + monkeypatch.setattr( + simulation_output_geographic, + "_required_uk_geography_lookup_csv_path", + lambda _: pytest.fail("Stage 12 must not resolve the legacy GCS lookup"), + ) + + result = simulation_output_geographic.build_uk_local_authority_impact( + "uk", + _uk_simulation(code, 100.0), + _uk_simulation(code, 110.0), + uk_local_authority_metadata=UKLocalAuthorityMetadata(roster=roster), + ) + + assert result is not None + record = result.root[0].model_dump(mode="python") + assert record["local_authority_code"] == code + assert record["local_authority_name"] == expected_name + assert isinstance(record["x"], int) + assert isinstance(record["y"], int) + assert record["average_household_income_change"] == 10.0 + + +def test_stage12_local_authority_output_rejects_code_outside_detected_roster() -> None: + with pytest.raises(ValueError, match="not part of the detected LAD22 roster"): + simulation_output_geographic.build_uk_local_authority_impact( + "uk", + _uk_simulation("E06000063", 100.0), + _uk_simulation("E06000063", 110.0), + uk_local_authority_metadata=UKLocalAuthorityMetadata( + roster=UKLocalAuthorityRoster.LAD22 + ), + ) + + +def test_legacy_local_authority_output_still_resolves_the_gcs_lookup( + monkeypatch, +) -> None: + observed: list[object] = [] + legacy_record = { + "local_authority_code": "E06000063", + "local_authority_name": "Cumberland", + "x": 1, + "y": 2, + "average_household_income_change": 10.0, + "relative_household_income_change": 0.1, + "population": 2.0, + } + + def require_lookup(spec): + observed.append(spec) + return "/tmp/local_authorities_2021.csv" + + def output_function(module: str, function: str): + assert module == "local_authority_impact" + assert function == "compute_uk_local_authority_impacts" + + def calculate(*args, **kwargs): + observed.append(kwargs) + return SimpleNamespace(local_authority_results=[legacy_record]) + + return calculate + + monkeypatch.setattr( + simulation_output_geographic, + "_required_uk_geography_lookup_csv_path", + require_lookup, + ) + monkeypatch.setattr( + simulation_output_geographic, + "_output_module_function", + output_function, + ) + + result = simulation_output_geographic.build_uk_local_authority_impact( + "uk", + object(), + object(), + ) + + assert result is not None + assert observed[0] is simulation_output_geographic.LOCAL_AUTHORITY_ASSET_SPEC + assert observed[1] == { + "local_authority_csv_path": "/tmp/local_authorities_2021.csv", + "download_missing_assets": False, + } diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_runtime.py b/projects/policyengine-simulation-executor/tests/test_stage12_runtime.py index b1b2d1110..ff1fa425d 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_runtime.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_runtime.py @@ -745,6 +745,7 @@ def serialize(self): assert result["result"] == {"captured": True} assert observed["simulation_params"]["include_cliffs"] is True + assert observed["uk_local_authority_metadata"] is None assert labor_supply_response_is_active( observed["baseline"], observed["reform"], From efbe8cfa44fbbb9a2fc430ef4dbbc7a12b46e37a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 30 Sep 2026 22:10:42 +0400 Subject: [PATCH 5/5] Document temporary Stage 12 geography metadata --- .../stage12-validate-infrastructure.sh | 1 - .../stage-12-modal-worker-foundation.md | 44 +++++++++++++++++-- .../tests/test_stage12_infrastructure.py | 2 +- .../stage12_worker_validation.py | 15 +++++++ .../tests/test_stage12_worker_validation.py | 18 ++++++++ 5 files changed, 74 insertions(+), 6 deletions(-) diff --git a/.github/scripts/stage12-validate-infrastructure.sh b/.github/scripts/stage12-validate-infrastructure.sh index dc63c1337..d12daabad 100755 --- a/.github/scripts/stage12-validate-infrastructure.sh +++ b/.github/scripts/stage12-validate-infrastructure.sh @@ -39,7 +39,6 @@ for account in \ done uk_geography_lookup_objects=( "gs://policyengine-uk-data-private/constituencies_2024.csv" - "gs://policyengine-uk-data-private/local_authorities_2021.csv" ) for secret_name in \ "${STAGE12_MODAL_TOKEN_ID_SECRET_NAME}" \ diff --git a/docs/migration/stage-12-modal-worker-foundation.md b/docs/migration/stage-12-modal-worker-foundation.md index fff728157..86eb421e2 100644 --- a/docs/migration/stage-12-modal-worker-foundation.md +++ b/docs/migration/stage-12-modal-worker-foundation.md @@ -62,10 +62,11 @@ with the live API-owned schema. `stage12-evaluation-gcp-credentials` Modal secret. The worker application does not receive the existing general GCP credential secret. The credential's service account can read and write its environment-specific Stage 12 artifact - bucket. A conditional IAM binding separately permits it to read only - `constituencies_2024.csv` and `local_authorities_2021.csv` from - `policyengine-uk-data-private`; deployment validation downloads both lookup - files using the worker credential. + bucket. A conditional IAM binding separately permits it to read + `constituencies_2024.csv` from `policyengine-uk-data-private`; deployment + validation downloads that lookup file using the worker credential. Stage 12 + local-authority output uses packaged metadata and does not require access to + `local_authorities_2021.csv`. - `STAGE12_DATABASE_URL` is delivered from the environment-specific Secret Manager resource named by `STAGE12_DATABASE_URL_SECRET_NAME`. It authenticates as the existing shared `policyengine_v2_runtime` account; Stage 12 does not @@ -207,6 +208,41 @@ metadata so PolicyEngine can correctly recognize conditional labor-supply analysis. `include_cliffs=true` is supported by this path and is carried through both output planning and aggregation. +### Temporary UK local-authority display metadata + +Stage 12 detects the local-authority boundary configuration from the complete +source dataset before applying any requested regional scope. It examines the +dataset's `la_code_oa` values: the 17 predecessor English authorities identify +LAD22, while `E06000063` through `E06000066` identify LAD23. A mixed roster, +an unknown code, or a dataset without a distinguishing code fails the Stage 12 +run. Dataset names and release labels do not select a roster, and there is no +fallback to the newest known configuration. + +The executor temporarily packages three display-only resources: + +- one common `code,name` file containing the union of supported authority + codes; +- one `code,x,y` file for the 374-authority LAD22 configuration; and +- one `code,x,y` file for the 361-authority LAD23 configuration. + +The Enhanced FRS geography is LAD22: its 2021 output areas are crosswalked to +2022 local-authority boundaries. The output-area vintage does not make the +authority roster LAD21. Names and coordinates are attached only after numeric +local-authority impacts have been calculated; these resources never assign a +household to an authority. + +The detected roster is stored as typed metadata in both child Parquet +artifacts. The coordinator requires baseline and reform to agree before it +enriches the aggregate output. Existing non-Stage-12 calculations continue to +use the existing GCS lookup and are not changed by this path. + +This resource ownership is temporary. A future changed authority roster must +add its codes to the common names file, add a distinct coordinate file, and +register a typed detector definition. Longer term, display-coordinate rendering +should move to the front end, and Stage 14 or later work should remove these +temporary executor resources when the authoritative v2 report architecture +supersedes Stage 12. + ## Temporary direct runner endpoint > **Temporary Stage 12 interface:** The authenticated routes in this section diff --git a/projects/policyengine-simulation-entry/tests/test_stage12_infrastructure.py b/projects/policyengine-simulation-entry/tests/test_stage12_infrastructure.py index 356603594..4eb899072 100644 --- a/projects/policyengine-simulation-entry/tests/test_stage12_infrastructure.py +++ b/projects/policyengine-simulation-entry/tests/test_stage12_infrastructure.py @@ -93,7 +93,7 @@ def test_storage_validation_uses_only_the_modal_worker_credentials() -> None: assert source.count(f"runtime_gcloud {modal_account} storage cp") == 3 assert source.count(f"runtime_gcloud {modal_account} storage rm") == 2 assert "gs://policyengine-uk-data-private/constituencies_2024.csv" in source - assert "gs://policyengine-uk-data-private/local_authorities_2021.csv" in source + assert "gs://policyengine-uk-data-private/local_authorities_2021.csv" not in source assert "STAGE12_UK_GEOGRAPHY_BUCKET" not in source diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_worker_validation.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_worker_validation.py index 26baab257..5eb8f6f6a 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_worker_validation.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_worker_validation.py @@ -65,6 +65,16 @@ def _run_non_serving_calculation(country: CountryId) -> None: raise RuntimeError("Stage 12 validation calculation returned no net income") +def _check_uk_local_authority_resources() -> None: + from policyengine_simulation_executor.stage12_runtime.uk_local_authority_metadata import ( + load_uk_local_authority_resources, + ) + + resources = load_uk_local_authority_resources() + for roster in resources.coordinates: + resources.metadata_for(roster) + + def validate_country_worker( *, country: CountryId, @@ -75,6 +85,9 @@ def validate_country_worker( ), dataset_check: Callable[[str, str], None] = _check_dataset_access, calculation_check: Callable[[CountryId], None] = _run_non_serving_calculation, + local_authority_resource_check: Callable[[], None] = ( + _check_uk_local_authority_resources + ), ) -> dict[str, Any]: """Validate imports, installed bundle, secrets, data, and calculation.""" @@ -109,6 +122,8 @@ def validate_country_worker( if dataset.identity == country_bundle.default_dataset ) dataset_check(installed_dataset_path, selected_dataset.sha256) + if country == "uk": + local_authority_resource_check() calculation_check(country) return { "validated": True, diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_worker_validation.py b/projects/policyengine-simulation-executor/tests/test_stage12_worker_validation.py index b90f0bb4c..fe569f109 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_worker_validation.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_worker_validation.py @@ -49,6 +49,24 @@ def test_validation_checks_dataset_and_non_serving_calculation() -> None: assert countries == ["us"] +def test_uk_validation_loads_packaged_local_authority_resources() -> None: + resolved = load_stage12_bundle() + checks: list[str] = [] + + result = validate_country_worker( + country="uk", + expected_bundle_manifest_sha256=resolved.bundle_manifest_sha256, + environment=_environment(), + dataset_path_resolver=lambda _: "/installed/enhanced_frs_2024_25.h5", + dataset_check=lambda *_: None, + calculation_check=lambda _: None, + local_authority_resource_check=lambda: checks.append("resources-loaded"), + ) + + assert result["validated"] is True + assert checks == ["resources-loaded"] + + def test_validation_rejects_digest_mismatch_before_dataset_access() -> None: accessed: list[str] = [] with pytest.raises(RuntimeError, match="digest differs"):