From a5e53ccbfb8667e18da6eed53f6d3b709cb27672 Mon Sep 17 00:00:00 2001 From: Rohit Kumar Date: Thu, 6 Aug 2026 12:56:39 +0000 Subject: [PATCH 1/5] Fix Pandas 3.x compatibility and test discovery gap for USCensusPEP_Sex. - Replaced delim_whitespace=True with sep=r'\s+' for Pandas 3.x compatibility. - Fixed float-casting issue in county intercensal data parsing by dropping NaN rows and explicitly casting Year/Observation to int64. - Added missing __init__.py files in us_census/pep/ to resolve CI/CD test discovery gap. - Protected absl flags against duplicate definition error during test discovery. - Refactored annual_population preprocess_test.py to use TemporaryDirectory for output verification. --- scripts/us_census/pep/__init__.py | 1 + .../pep/annual_population/preprocess_test.py | 26 +++-- scripts/us_census/pep/us_pep_sex/__init__.py | 1 + scripts/us_census/pep/us_pep_sex/process.py | 24 +++-- .../population_estimate_sex.csv | 98 +++++++++---------- 5 files changed, 79 insertions(+), 71 deletions(-) create mode 100644 scripts/us_census/pep/__init__.py create mode 100644 scripts/us_census/pep/us_pep_sex/__init__.py diff --git a/scripts/us_census/pep/__init__.py b/scripts/us_census/pep/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/scripts/us_census/pep/__init__.py @@ -0,0 +1 @@ + diff --git a/scripts/us_census/pep/annual_population/preprocess_test.py b/scripts/us_census/pep/annual_population/preprocess_test.py index 11b35c1dbe..1980dc7770 100644 --- a/scripts/us_census/pep/annual_population/preprocess_test.py +++ b/scripts/us_census/pep/annual_population/preprocess_test.py @@ -42,24 +42,22 @@ def __init__(self, methodName: str = ...) -> None: files_dir = os.path.join(_MODULE_DIR, TEST_DATA_DIR, "datasets") - data_file_path = os.path.join(_MODULE_DIR, TEST_DATA_DIR, - "output_files") + with tempfile.TemporaryDirectory() as tmp_dir: + cleaned_csv_path = os.path.join(tmp_dir, + "usa_annual_population.csv") + mcf_path = os.path.join(tmp_dir, "usa_annual_population.mcf") + tmcf_path = os.path.join(tmp_dir, "usa_annual_population.tmcf") - cleaned_csv_path = os.path.join(data_file_path, - "usa_annual_population.csv") - mcf_path = os.path.join(data_file_path, "usa_annual_population.mcf") - tmcf_path = os.path.join(data_file_path, "usa_annual_population.tmcf") + process(files_dir, cleaned_csv_path, mcf_path, tmcf_path, False) - process(files_dir, cleaned_csv_path, mcf_path, tmcf_path, False) + with open(mcf_path, encoding="UTF-8") as mcf_file: + self._actual_mcf_data = mcf_file.read() - with open(mcf_path, encoding="UTF-8") as mcf_file: - self._actual_mcf_data = mcf_file.read() + with open(tmcf_path, encoding="UTF-8") as tmcf_file: + self._actual_tmcf_data = tmcf_file.read() - with open(tmcf_path, encoding="UTF-8") as tmcf_file: - self._actual_tmcf_data = tmcf_file.read() - - with open(cleaned_csv_path, encoding="utf-8") as csv_file: - self._actual_csv_data = csv_file.read() + with open(cleaned_csv_path, encoding="utf-8") as csv_file: + self._actual_csv_data = csv_file.read() def test_mcf_tmcf_files(self): """ diff --git a/scripts/us_census/pep/us_pep_sex/__init__.py b/scripts/us_census/pep/us_pep_sex/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/scripts/us_census/pep/us_pep_sex/__init__.py @@ -0,0 +1 @@ + diff --git a/scripts/us_census/pep/us_pep_sex/process.py b/scripts/us_census/pep/us_pep_sex/process.py index 8eda40aa4d..44f6a022ad 100644 --- a/scripts/us_census/pep/us_pep_sex/process.py +++ b/scripts/us_census/pep/us_pep_sex/process.py @@ -34,9 +34,11 @@ _FLAGS = flags.FLAGS -flags.DEFINE_string('mode', '', 'Options: download or process') -flags.DEFINE_string('config_path', '', - 'Path to the configuration file in the GCS bucket.') +if 'mode' not in flags.FLAGS: + flags.DEFINE_string('mode', '', 'Options: download or process') +if 'config_path' not in flags.FLAGS: + flags.DEFINE_string('config_path', '', + 'Path to the configuration file in the GCS bucket.') _MODULE_DIR = os.path.dirname(os.path.abspath(__file__)) _INPUT_FILE_PATH = os.path.join(_MODULE_DIR, 'input_files') @@ -58,7 +60,9 @@ _FLAGS = flags.FLAGS default_input_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "gcs_folder/us_pep_sex_source_files") -flags.DEFINE_string("input_path", default_input_path, "Import Data File's List") +if 'input_path' not in flags.FLAGS: + flags.DEFINE_string("input_path", default_input_path, + "Import Data File's List") _MCF_TEMPLATE = ("Node: dcid:{pv1}\n" "typeOf: dcs:StatisticalVariable\n" @@ -441,12 +445,12 @@ def _state_1980_1990(file_path: str) -> pd.DataFrame: if year == 1987: df = pd.read_table(file_path, skiprows=29, - delim_whitespace=True, + sep=r'\s+', names=column_names) else: df = pd.read_table(file_path, skiprows=28, - delim_whitespace=True, + sep=r'\s+', names=column_names) df['geo_ID'] = 'geoId/' + (df['geo_ID'].map(str)).str.zfill(2) df['Year'] = year @@ -691,6 +695,11 @@ def _county_1980_1990(file_path: str) -> pd.DataFrame: """ try: df = pd.read_csv(file_path, skiprows=5) + df = df.dropna( + subset=['Year of Estimate', 'FIPS State and County Codes']) + df['Year of Estimate'] = df['Year of Estimate'].astype('int64') + df['FIPS State and County Codes'] = df[ + 'FIPS State and County Codes'].astype('int64') # adding age groups to get total value df['Total'] = df[_COLUMNS_TO_SUM].sum(axis=1) df = df.drop(columns=_COLUMNS_TO_SUM) @@ -736,7 +745,7 @@ def _county_1990_2000(file_path: str) -> pd.DataFrame: """ try: column_names = ['Year', 'geo_ID', 'Age', 'Race-Sex', 'Ethnic', 'Value'] - df = pd.read_table(file_path, delim_whitespace=True, header=None) + df = pd.read_table(file_path, sep=r'\s+', header=None) df.columns = column_names df['Year'] = '19' + df['Year'].astype(str) df['geo_ID'] = 'geoId/' + (df['geo_ID'].map(str)).str.zfill(5) @@ -1092,6 +1101,7 @@ def process(self): value_vars=['Count_Person_Male', 'Count_Person_Female'], var_name="SV", value_name="Observation") + final_df['Observation'] = final_df['Observation'].astype('int64') subset_cols = ['Year', 'geo_ID', 'Measurement_Method', 'SV'] # 2. Drop duplicates based on those columns, keeping the first occurrence final_df.drop_duplicates(subset=subset_cols, diff --git a/scripts/us_census/pep/us_pep_sex/test_data/expected_files/population_estimate_sex.csv b/scripts/us_census/pep/us_pep_sex/test_data/expected_files/population_estimate_sex.csv index 0690373707..c29c20951a 100644 --- a/scripts/us_census/pep/us_pep_sex/test_data/expected_files/population_estimate_sex.csv +++ b/scripts/us_census/pep/us_pep_sex/test_data/expected_files/population_estimate_sex.csv @@ -37,22 +37,22 @@ Year,geo_ID,Measurement_Method,SV,Observation 1970,geoId/34,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,3479511 1970,geoId/35,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,504967 1970,geoId/36,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,7630368 -1989.0,geoId/1001.0,CensusPEPSurvey,Count_Person_Male,16565.0 -1989.0,geoId/1003.0,CensusPEPSurvey,Count_Person_Male,46734.0 -1989.0,geoId/1005.0,CensusPEPSurvey,Count_Person_Male,12022.0 -1989.0,geoId/1007.0,CensusPEPSurvey,Count_Person_Male,7999.0 -1989.0,geoId/1009.0,CensusPEPSurvey,Count_Person_Male,18948.0 -1989.0,geoId/1011.0,CensusPEPSurvey,Count_Person_Male,5370.0 -1989.0,geoId/1013.0,CensusPEPSurvey,Count_Person_Male,10293.0 -1989.0,geoId/1015.0,CensusPEPSurvey,Count_Person_Male,56331.0 -1989.0,geoId/1017.0,CensusPEPSurvey,Count_Person_Male,17546.0 -1989.0,geoId/1019.0,CensusPEPSurvey,Count_Person_Male,9566.0 -1989.0,geoId/1021.0,CensusPEPSurvey,Count_Person_Male,15709.0 -1989.0,geoId/1023.0,CensusPEPSurvey,Count_Person_Male,7682.0 -1989.0,geoId/1025.0,CensusPEPSurvey,Count_Person_Male,12996.0 -1989.0,geoId/1027.0,CensusPEPSurvey,Count_Person_Male,6336.0 -1989.0,geoId/1029.0,CensusPEPSurvey,Count_Person_Male,6282.0 -1989.0,geoId/1031.0,CensusPEPSurvey,Count_Person_Male,16307.0 +1989,geoId/01001,CensusPEPSurvey,Count_Person_Male,16565 +1989,geoId/01003,CensusPEPSurvey,Count_Person_Male,46734 +1989,geoId/01005,CensusPEPSurvey,Count_Person_Male,12022 +1989,geoId/01007,CensusPEPSurvey,Count_Person_Male,7999 +1989,geoId/01009,CensusPEPSurvey,Count_Person_Male,18948 +1989,geoId/01011,CensusPEPSurvey,Count_Person_Male,5370 +1989,geoId/01013,CensusPEPSurvey,Count_Person_Male,10293 +1989,geoId/01015,CensusPEPSurvey,Count_Person_Male,56331 +1989,geoId/01017,CensusPEPSurvey,Count_Person_Male,17546 +1989,geoId/01019,CensusPEPSurvey,Count_Person_Male,9566 +1989,geoId/01021,CensusPEPSurvey,Count_Person_Male,15709 +1989,geoId/01023,CensusPEPSurvey,Count_Person_Male,7682 +1989,geoId/01025,CensusPEPSurvey,Count_Person_Male,12996 +1989,geoId/01027,CensusPEPSurvey,Count_Person_Male,6336 +1989,geoId/01029,CensusPEPSurvey,Count_Person_Male,6282 +1989,geoId/01031,CensusPEPSurvey,Count_Person_Male,16307 2000,country/USA,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,138443407 2000,geoId/01001,CensusPEPSurvey,Count_Person_Male,21385 2000,geoId/01003,CensusPEPSurvey,Count_Person_Male,69302 @@ -271,9 +271,8 @@ Year,geo_ID,Measurement_Method,SV,Observation 2020,geoId/01009,CensusPEPSurvey,Count_Person_Male,28617 2020,geoId/01011,CensusPEPSurvey,Count_Person_Male,5461 2020,geoId/01013,CensusPEPSurvey,Count_Person_Male,9084 -2020,geoId/19,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,1575695 -2020,geoId/20,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,1451540 -2020,geoId/32,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,1572640 +2020,geoId/06,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,19752720 +2020,geoId/50,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,319764 2021,geoId/01001,CensusPEPSurvey,Count_Person_Male,28782 2021,geoId/01003,CensusPEPSurvey,Count_Person_Male,116725 2021,geoId/01005,CensusPEPSurvey,Count_Person_Male,12811 @@ -294,8 +293,8 @@ Year,geo_ID,Measurement_Method,SV,Observation 2021,geoId/01035,CensusPEPSurvey,Count_Person_Male,5456 2021,geoId/01037,CensusPEPSurvey,Count_Person_Male,5194 2021,geoId/01039,CensusPEPSurvey,Count_Person_Male,18208 -2021,geoId/06,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,19560558.0 -2021,geoId/50,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,321586.0 +2021,geoId/06,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,19560558 +2021,geoId/50,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,321586 2022,geoId/01001,CensusPEPSurvey,Count_Person_Male,29030 2022,geoId/01003,CensusPEPSurvey,Count_Person_Male,120079 2022,geoId/01005,CensusPEPSurvey,Count_Person_Male,13093 @@ -316,8 +315,8 @@ Year,geo_ID,Measurement_Method,SV,Observation 2022,geoId/01035,CensusPEPSurvey,Count_Person_Male,5397 2022,geoId/01037,CensusPEPSurvey,Count_Person_Male,5234 2022,geoId/01039,CensusPEPSurvey,Count_Person_Male,18240 -2022,geoId/06,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,19496496.0 -2022,geoId/50,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,321328.0 +2022,geoId/06,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,19496496 +2022,geoId/50,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,321328 2023,geoId/01001,CensusPEPSurvey,Count_Person_Male,29277 2023,geoId/01003,CensusPEPSurvey,Count_Person_Male,123497 2023,geoId/01005,CensusPEPSurvey,Count_Person_Male,13066 @@ -337,8 +336,8 @@ Year,geo_ID,Measurement_Method,SV,Observation 2023,geoId/01033,CensusPEPSurvey,Count_Person_Male,27995 2023,geoId/01035,CensusPEPSurvey,Count_Person_Male,5405 2023,geoId/01037,CensusPEPSurvey,Count_Person_Male,5192 -2023,geoId/06,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,38965193.0 -2023,geoId/50,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,647464.0 +2023,geoId/06,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,19445653 +2023,geoId/50,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Male,321436 1988,country/USA,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,2350311 1988,geoId/01,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,2093935 1988,geoId/02,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,256376 @@ -377,22 +376,22 @@ Year,geo_ID,Measurement_Method,SV,Observation 1970,geoId/34,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,3713699 1970,geoId/35,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,519323 1970,geoId/36,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,8271815 -1989.0,geoId/1001.0,CensusPEPSurvey,Count_Person_Female,17431.0 -1989.0,geoId/1003.0,CensusPEPSurvey,Count_Person_Female,49469.0 -1989.0,geoId/1005.0,CensusPEPSurvey,Count_Person_Female,13315.0 -1989.0,geoId/1007.0,CensusPEPSurvey,Count_Person_Female,8359.0 -1989.0,geoId/1009.0,CensusPEPSurvey,Count_Person_Female,19823.0 -1989.0,geoId/1011.0,CensusPEPSurvey,Count_Person_Female,5801.0 -1989.0,geoId/1013.0,CensusPEPSurvey,Count_Person_Female,11745.0 -1989.0,geoId/1015.0,CensusPEPSurvey,Count_Person_Female,60047.0 -1989.0,geoId/1017.0,CensusPEPSurvey,Count_Person_Female,19663.0 -1989.0,geoId/1019.0,CensusPEPSurvey,Count_Person_Female,9899.0 -1989.0,geoId/1021.0,CensusPEPSurvey,Count_Person_Female,16659.0 -1989.0,geoId/1023.0,CensusPEPSurvey,Count_Person_Female,8483.0 -1989.0,geoId/1025.0,CensusPEPSurvey,Count_Person_Female,14170.0 -1989.0,geoId/1027.0,CensusPEPSurvey,Count_Person_Female,6834.0 -1989.0,geoId/1029.0,CensusPEPSurvey,Count_Person_Female,6473.0 -1989.0,geoId/1031.0,CensusPEPSurvey,Count_Person_Female,16704.0 +1989,geoId/01001,CensusPEPSurvey,Count_Person_Female,17431 +1989,geoId/01003,CensusPEPSurvey,Count_Person_Female,49469 +1989,geoId/01005,CensusPEPSurvey,Count_Person_Female,13315 +1989,geoId/01007,CensusPEPSurvey,Count_Person_Female,8359 +1989,geoId/01009,CensusPEPSurvey,Count_Person_Female,19823 +1989,geoId/01011,CensusPEPSurvey,Count_Person_Female,5801 +1989,geoId/01013,CensusPEPSurvey,Count_Person_Female,11745 +1989,geoId/01015,CensusPEPSurvey,Count_Person_Female,60047 +1989,geoId/01017,CensusPEPSurvey,Count_Person_Female,19663 +1989,geoId/01019,CensusPEPSurvey,Count_Person_Female,9899 +1989,geoId/01021,CensusPEPSurvey,Count_Person_Female,16659 +1989,geoId/01023,CensusPEPSurvey,Count_Person_Female,8483 +1989,geoId/01025,CensusPEPSurvey,Count_Person_Female,14170 +1989,geoId/01027,CensusPEPSurvey,Count_Person_Female,6834 +1989,geoId/01029,CensusPEPSurvey,Count_Person_Female,6473 +1989,geoId/01031,CensusPEPSurvey,Count_Person_Female,16704 2000,country/USA,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,143719004 2000,geoId/01001,CensusPEPSurvey,Count_Person_Female,22636 2000,geoId/01003,CensusPEPSurvey,Count_Person_Female,72040 @@ -611,9 +610,8 @@ Year,geo_ID,Measurement_Method,SV,Observation 2020,geoId/01009,CensusPEPSurvey,Count_Person_Female,29262 2020,geoId/01011,CensusPEPSurvey,Count_Person_Female,4515 2020,geoId/01013,CensusPEPSurvey,Count_Person_Female,10420 -2020,geoId/19,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,1587866 -2020,geoId/20,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,1462265 -2020,geoId/32,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,1565619 +2020,geoId/06,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,19750480 +2020,geoId/50,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,323172 2021,geoId/01001,CensusPEPSurvey,Count_Person_Female,30421 2021,geoId/01003,CensusPEPSurvey,Count_Person_Female,122714 2021,geoId/01005,CensusPEPSurvey,Count_Person_Female,11722 @@ -634,8 +632,8 @@ Year,geo_ID,Measurement_Method,SV,Observation 2021,geoId/01035,CensusPEPSurvey,Count_Person_Female,5864 2021,geoId/01037,CensusPEPSurvey,Count_Person_Female,5120 2021,geoId/01039,CensusPEPSurvey,Count_Person_Female,19362 -2021,geoId/06,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,19584502.0 -2021,geoId/50,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,325507.0 +2021,geoId/06,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,19584502 +2021,geoId/50,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,325507 2022,geoId/01001,CensusPEPSurvey,Count_Person_Female,30696 2022,geoId/01003,CensusPEPSurvey,Count_Person_Female,126452 2022,geoId/01005,CensusPEPSurvey,Count_Person_Female,11607 @@ -656,8 +654,8 @@ Year,geo_ID,Measurement_Method,SV,Observation 2022,geoId/01035,CensusPEPSurvey,Count_Person_Female,5827 2022,geoId/01037,CensusPEPSurvey,Count_Person_Female,5073 2022,geoId/01039,CensusPEPSurvey,Count_Person_Female,19363 -2022,geoId/06,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,19544120.0 -2022,geoId/50,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,325782.0 +2022,geoId/06,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,19544120 +2022,geoId/50,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,325782 2023,geoId/01001,CensusPEPSurvey,Count_Person_Female,31065 2023,geoId/01003,CensusPEPSurvey,Count_Person_Female,130010 2023,geoId/01005,CensusPEPSurvey,Count_Person_Female,11519 @@ -677,5 +675,5 @@ Year,geo_ID,Measurement_Method,SV,Observation 2023,geoId/01033,CensusPEPSurvey,Count_Person_Female,30366 2023,geoId/01035,CensusPEPSurvey,Count_Person_Female,5769 2023,geoId/01037,CensusPEPSurvey,Count_Person_Female,5076 -2023,geoId/06,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,19445653.0 -2023,geoId/50,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,321436.0 +2023,geoId/06,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,19519540 +2023,geoId/50,dcAggregate/CensusPEPSurvey_PartialAggregate,Count_Person_Female,326028 From 689076c4717454a011de2b6c860221d07c38222e Mon Sep 17 00:00:00 2001 From: Rohit Kumar Date: Thu, 6 Aug 2026 13:28:25 +0000 Subject: [PATCH 2/5] Fix: Safely cast Observation to nullable Int64 to handle empty strings --- scripts/us_census/pep/us_pep_sex/process.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/us_census/pep/us_pep_sex/process.py b/scripts/us_census/pep/us_pep_sex/process.py index 44f6a022ad..b2376217c8 100644 --- a/scripts/us_census/pep/us_pep_sex/process.py +++ b/scripts/us_census/pep/us_pep_sex/process.py @@ -1101,7 +1101,8 @@ def process(self): value_vars=['Count_Person_Male', 'Count_Person_Female'], var_name="SV", value_name="Observation") - final_df['Observation'] = final_df['Observation'].astype('int64') + final_df['Observation'] = pd.to_numeric( + final_df['Observation'], errors='coerce').astype('Int64') subset_cols = ['Year', 'geo_ID', 'Measurement_Method', 'SV'] # 2. Drop duplicates based on those columns, keeping the first occurrence final_df.drop_duplicates(subset=subset_cols, From 883e564b07cf52b573f7939fd30fcdd5ac42c1e8 Mon Sep 17 00:00:00 2001 From: Rohit Kumar Date: Thu, 6 Aug 2026 13:30:31 +0000 Subject: [PATCH 3/5] Refactor: Use setUpClass/tearDownClass in unit tests to improve execution efficiency --- .../pep/annual_population/preprocess_test.py | 33 +++++++------ .../us_census/pep/us_pep_sex/process_test.py | 47 ++++++++++--------- 2 files changed, 43 insertions(+), 37 deletions(-) diff --git a/scripts/us_census/pep/annual_population/preprocess_test.py b/scripts/us_census/pep/annual_population/preprocess_test.py index 1980dc7770..daa852c559 100644 --- a/scripts/us_census/pep/annual_population/preprocess_test.py +++ b/scripts/us_census/pep/annual_population/preprocess_test.py @@ -37,27 +37,30 @@ class TestPreprocess(unittest.TestCase): and compare it with expected results. """ - def __init__(self, methodName: str = ...) -> None: - super().__init__(methodName) - + @classmethod + def setUpClass(cls): + cls.tmp_dir_obj = tempfile.TemporaryDirectory() + tmp_dir = cls.tmp_dir_obj.name files_dir = os.path.join(_MODULE_DIR, TEST_DATA_DIR, "datasets") - with tempfile.TemporaryDirectory() as tmp_dir: - cleaned_csv_path = os.path.join(tmp_dir, - "usa_annual_population.csv") - mcf_path = os.path.join(tmp_dir, "usa_annual_population.mcf") - tmcf_path = os.path.join(tmp_dir, "usa_annual_population.tmcf") + cleaned_csv_path = os.path.join(tmp_dir, "usa_annual_population.csv") + mcf_path = os.path.join(tmp_dir, "usa_annual_population.mcf") + tmcf_path = os.path.join(tmp_dir, "usa_annual_population.tmcf") + + process(files_dir, cleaned_csv_path, mcf_path, tmcf_path, False) - process(files_dir, cleaned_csv_path, mcf_path, tmcf_path, False) + with open(mcf_path, encoding="UTF-8") as mcf_file: + cls._actual_mcf_data = mcf_file.read() - with open(mcf_path, encoding="UTF-8") as mcf_file: - self._actual_mcf_data = mcf_file.read() + with open(tmcf_path, encoding="UTF-8") as tmcf_file: + cls._actual_tmcf_data = tmcf_file.read() - with open(tmcf_path, encoding="UTF-8") as tmcf_file: - self._actual_tmcf_data = tmcf_file.read() + with open(cleaned_csv_path, encoding="utf-8") as csv_file: + cls._actual_csv_data = csv_file.read() - with open(cleaned_csv_path, encoding="utf-8") as csv_file: - self._actual_csv_data = csv_file.read() + @classmethod + def tearDownClass(cls): + cls.tmp_dir_obj.cleanup() def test_mcf_tmcf_files(self): """ diff --git a/scripts/us_census/pep/us_pep_sex/process_test.py b/scripts/us_census/pep/us_pep_sex/process_test.py index 4edea39f19..8a315cf7f5 100644 --- a/scripts/us_census/pep/us_pep_sex/process_test.py +++ b/scripts/us_census/pep/us_pep_sex/process_test.py @@ -41,28 +41,31 @@ class TestProcess(unittest.TestCase): Comparing the data with the expected files. """ - def __init__(self, methodName: str = ...) -> None: - super().__init__(methodName) - - with tempfile.TemporaryDirectory() as tmp_dir: - cleaned_csv_file_path = os.path.join(tmp_dir, "data.csv") - mcf_file_path = os.path.join(tmp_dir, "test_census.mcf") - tmcf_file_path = os.path.join(tmp_dir, "test_census.tmcf") - - base = PopulationEstimateBySex(TEST_DATASET_DIR, - cleaned_csv_file_path, mcf_file_path, - tmcf_file_path) - base.process() - - with open(mcf_file_path, mode='r', encoding="UTF-8") as mcf_file: - self.actual_mcf_data = mcf_file.read() - - with open(tmcf_file_path, mode='r', encoding="UTF-8") as tmcf_file: - self.actual_tmcf_data = tmcf_file.read() - - with open(cleaned_csv_file_path, mode='r', - encoding="utf-8-sig") as csv_file: - self.actual_csv_data = csv_file.read() + @classmethod + def setUpClass(cls): + cls.tmp_dir_obj = tempfile.TemporaryDirectory() + tmp_dir = cls.tmp_dir_obj.name + cleaned_csv_file_path = os.path.join(tmp_dir, "data.csv") + mcf_file_path = os.path.join(tmp_dir, "test_census.mcf") + tmcf_file_path = os.path.join(tmp_dir, "test_census.tmcf") + + base = PopulationEstimateBySex(TEST_DATASET_DIR, cleaned_csv_file_path, + mcf_file_path, tmcf_file_path) + base.process() + + with open(mcf_file_path, mode='r', encoding="UTF-8") as mcf_file: + cls.actual_mcf_data = mcf_file.read() + + with open(tmcf_file_path, mode='r', encoding="UTF-8") as tmcf_file: + cls.actual_tmcf_data = tmcf_file.read() + + with open(cleaned_csv_file_path, mode='r', + encoding="utf-8-sig") as csv_file: + cls.actual_csv_data = csv_file.read() + + @classmethod + def tearDownClass(cls): + cls.tmp_dir_obj.cleanup() def test_mcf_tmcf_files(self): """ From 7233f580c264bb5becdd9477d80aa0e298f8ed9c Mon Sep 17 00:00:00 2001 From: Rohit Kumar Date: Thu, 6 Aug 2026 15:47:04 +0000 Subject: [PATCH 4/5] Fix: Guard duplicate flags and resolve namespace collision in test runner --- .../us_census/pep/annual_population/preprocess.py | 10 ++++++---- .../pep/annual_population/preprocess_test.py | 12 ++++++++++-- scripts/us_census/pep/us_pep_sex/process_test.py | 11 +++++++++-- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/scripts/us_census/pep/annual_population/preprocess.py b/scripts/us_census/pep/annual_population/preprocess.py index 53027a3de2..2013265970 100644 --- a/scripts/us_census/pep/annual_population/preprocess.py +++ b/scripts/us_census/pep/annual_population/preprocess.py @@ -54,10 +54,12 @@ _FLAGS = flags.FLAGS -flags.DEFINE_string('mode', '', 'Options: download or process') -flags.DEFINE_bool( - 'is_summary_levels', False, - 'Options: True for all summary_levels and False for only 162') +if 'mode' not in flags.FLAGS: + flags.DEFINE_string('mode', '', 'Options: download or process') +if 'is_summary_levels' not in flags.FLAGS: + flags.DEFINE_bool( + 'is_summary_levels', False, + 'Options: True for all summary_levels and False for only 162') _MODULE_DIR = os.path.dirname(os.path.abspath(__file__)) _INPUT_FILE_PATH = os.path.join(_MODULE_DIR, 'input_files') diff --git a/scripts/us_census/pep/annual_population/preprocess_test.py b/scripts/us_census/pep/annual_population/preprocess_test.py index daa852c559..8b77dd61e5 100644 --- a/scripts/us_census/pep/annual_population/preprocess_test.py +++ b/scripts/us_census/pep/annual_population/preprocess_test.py @@ -21,11 +21,19 @@ import tempfile # _MODULE_DIR is the path to where this test is running from. _MODULE_DIR = os.path.dirname(__file__) +_SCRIPTS_DIR = os.path.abspath(os.path.join(_MODULE_DIR, '../../../')) +if _SCRIPTS_DIR not in sys.path: + sys.path.insert(0, _SCRIPTS_DIR) sys.path.insert(1, _MODULE_DIR) + # pylint: disable=wrong-import-position # pylint: disable=import-error -from preprocess import process -from constants import TEST_DATA_DIR +try: + from us_census.pep.annual_population.preprocess import process + from us_census.pep.annual_population.constants import TEST_DATA_DIR +except ImportError: + from preprocess import process + from constants import TEST_DATA_DIR # pylint: enable=import-error # pylint: enable=wrong-import-position diff --git a/scripts/us_census/pep/us_pep_sex/process_test.py b/scripts/us_census/pep/us_pep_sex/process_test.py index 8a315cf7f5..c795d92478 100644 --- a/scripts/us_census/pep/us_pep_sex/process_test.py +++ b/scripts/us_census/pep/us_pep_sex/process_test.py @@ -23,9 +23,16 @@ # module_dir is the path to where this test is running from. MODULE_DIR = os.path.dirname(__file__) -sys.path.insert(0, MODULE_DIR) +_SCRIPTS_DIR = os.path.abspath(os.path.join(MODULE_DIR, '../../../')) +if _SCRIPTS_DIR not in sys.path: + sys.path.insert(0, _SCRIPTS_DIR) +sys.path.insert(1, MODULE_DIR) + # pylint: disable=wrong-import-position -from process import PopulationEstimateBySex +try: + from us_census.pep.us_pep_sex.process import PopulationEstimateBySex +except ImportError: + from process import PopulationEstimateBySex # pylint: enable=wrong-import-position TEST_DATASET_DIR = os.path.join(MODULE_DIR, "test_data", "datasets") From f38cb910517176c29e9930213ccfe68d3e3b807c Mon Sep 17 00:00:00 2001 From: rohit kumar Date: Fri, 7 Aug 2026 03:08:19 +0000 Subject: [PATCH 5/5] Load Census PEP flags only for CLI runs --- .../us_census/pep/annual_population/preprocess.py | 6 ++++-- scripts/us_census/pep/us_pep_sex/process.py | 15 ++++++++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/scripts/us_census/pep/annual_population/preprocess.py b/scripts/us_census/pep/annual_population/preprocess.py index 2013265970..892151556f 100644 --- a/scripts/us_census/pep/annual_population/preprocess.py +++ b/scripts/us_census/pep/annual_population/preprocess.py @@ -54,13 +54,14 @@ _FLAGS = flags.FLAGS -if 'mode' not in flags.FLAGS: + +def _define_flags(): flags.DEFINE_string('mode', '', 'Options: download or process') -if 'is_summary_levels' not in flags.FLAGS: flags.DEFINE_bool( 'is_summary_levels', False, 'Options: True for all summary_levels and False for only 162') + _MODULE_DIR = os.path.dirname(os.path.abspath(__file__)) _INPUT_FILE_PATH = os.path.join(_MODULE_DIR, 'input_files') os.makedirs(_INPUT_FILE_PATH, exist_ok=True) @@ -1186,4 +1187,5 @@ def main(_): if __name__ == "__main__": + _define_flags() app.run(main) diff --git a/scripts/us_census/pep/us_pep_sex/process.py b/scripts/us_census/pep/us_pep_sex/process.py index b2376217c8..ad97a1b95a 100644 --- a/scripts/us_census/pep/us_pep_sex/process.py +++ b/scripts/us_census/pep/us_pep_sex/process.py @@ -34,12 +34,6 @@ _FLAGS = flags.FLAGS -if 'mode' not in flags.FLAGS: - flags.DEFINE_string('mode', '', 'Options: download or process') -if 'config_path' not in flags.FLAGS: - flags.DEFINE_string('config_path', '', - 'Path to the configuration file in the GCS bucket.') - _MODULE_DIR = os.path.dirname(os.path.abspath(__file__)) _INPUT_FILE_PATH = os.path.join(_MODULE_DIR, 'input_files') _INPUT_URL_JSON = "input_url.json" @@ -60,10 +54,16 @@ _FLAGS = flags.FLAGS default_input_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "gcs_folder/us_pep_sex_source_files") -if 'input_path' not in flags.FLAGS: + + +def _define_flags(): + flags.DEFINE_string('mode', '', 'Options: download or process') + flags.DEFINE_string('config_path', '', + 'Path to the configuration file in the GCS bucket.') flags.DEFINE_string("input_path", default_input_path, "Import Data File's List") + _MCF_TEMPLATE = ("Node: dcid:{pv1}\n" "typeOf: dcs:StatisticalVariable\n" "populationType: dcs:Person{pv2}\n" @@ -1410,4 +1410,5 @@ def main(_): if __name__ == "__main__": + _define_flags() app.run(main)