Fix Pandas 3.x compatibility, float-casting, and test discovery gap for USCensusPEP_Sex

scripts/us_census/pep/__init__.py

@@ -0,0 +1 @@
+

scripts/us_census/pep/annual_population/preprocess.py

@@ -54,10 +54,13 @@
 
 _FLAGS = flags.FLAGS
 
-flags.DEFINE_string('mode', '', 'Options: download or process')
-flags.DEFINE_bool(
-    'is_summary_levels', False,
-    'Options: True for all summary_levels and False for only 162')
+
+def _define_flags():
+    flags.DEFINE_string('mode', '', 'Options: download or process')
+    flags.DEFINE_bool(
+        'is_summary_levels', False,
+        'Options: True for all summary_levels and False for only 162')
+
 
 _MODULE_DIR = os.path.dirname(os.path.abspath(__file__))
 _INPUT_FILE_PATH = os.path.join(_MODULE_DIR, 'input_files')
@@ -1184,4 +1187,5 @@ def main(_):
 
 
 if __name__ == "__main__":
+    _define_flags()
     app.run(main)

scripts/us_census/pep/annual_population/preprocess_test.py

@@ -21,11 +21,19 @@
 import tempfile
 # _MODULE_DIR is the path to where this test is running from.
 _MODULE_DIR = os.path.dirname(__file__)
+_SCRIPTS_DIR = os.path.abspath(os.path.join(_MODULE_DIR, '../../../'))
+if _SCRIPTS_DIR not in sys.path:
+    sys.path.insert(0, _SCRIPTS_DIR)
 sys.path.insert(1, _MODULE_DIR)
+
 # pylint: disable=wrong-import-position
 # pylint: disable=import-error
-from preprocess import process
-from constants import TEST_DATA_DIR
+try:
+    from us_census.pep.annual_population.preprocess import process
+    from us_census.pep.annual_population.constants import TEST_DATA_DIR
+except ImportError:
+    from preprocess import process
+    from constants import TEST_DATA_DIR
 # pylint: enable=import-error
 # pylint: enable=wrong-import-position
 
@@ -37,29 +45,30 @@ class TestPreprocess(unittest.TestCase):
     and compare it with expected results.
     """
 
-    def __init__(self, methodName: str = ...) -> None:
-        super().__init__(methodName)
-
+    @classmethod
+    def setUpClass(cls):
+        cls.tmp_dir_obj = tempfile.TemporaryDirectory()
+        tmp_dir = cls.tmp_dir_obj.name
         files_dir = os.path.join(_MODULE_DIR, TEST_DATA_DIR, "datasets")
 
-        data_file_path = os.path.join(_MODULE_DIR, TEST_DATA_DIR,
-                                      "output_files")
-
-        cleaned_csv_path = os.path.join(data_file_path,
-                                        "usa_annual_population.csv")
-        mcf_path = os.path.join(data_file_path, "usa_annual_population.mcf")
-        tmcf_path = os.path.join(data_file_path, "usa_annual_population.tmcf")
+        cleaned_csv_path = os.path.join(tmp_dir, "usa_annual_population.csv")
+        mcf_path = os.path.join(tmp_dir, "usa_annual_population.mcf")
+        tmcf_path = os.path.join(tmp_dir, "usa_annual_population.tmcf")
 
         process(files_dir, cleaned_csv_path, mcf_path, tmcf_path, False)
 
         with open(mcf_path, encoding="UTF-8") as mcf_file:
-            self._actual_mcf_data = mcf_file.read()
+            cls._actual_mcf_data = mcf_file.read()
 
         with open(tmcf_path, encoding="UTF-8") as tmcf_file:
-            self._actual_tmcf_data = tmcf_file.read()
+            cls._actual_tmcf_data = tmcf_file.read()
 
         with open(cleaned_csv_path, encoding="utf-8") as csv_file:
-            self._actual_csv_data = csv_file.read()
+            cls._actual_csv_data = csv_file.read()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.tmp_dir_obj.cleanup()
 
     def test_mcf_tmcf_files(self):
         """

scripts/us_census/pep/us_pep_sex/__init__.py

@@ -0,0 +1 @@
+

scripts/us_census/pep/us_pep_sex/process.py

@@ -34,10 +34,6 @@
 
 _FLAGS = flags.FLAGS
 
-flags.DEFINE_string('mode', '', 'Options: download or process')
-flags.DEFINE_string('config_path', '',
-                    'Path to the configuration file in the GCS bucket.')
-
 _MODULE_DIR = os.path.dirname(os.path.abspath(__file__))
 _INPUT_FILE_PATH = os.path.join(_MODULE_DIR, 'input_files')
 _INPUT_URL_JSON = "input_url.json"
@@ -58,7 +54,15 @@
 _FLAGS = flags.FLAGS
 default_input_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                   "gcs_folder/us_pep_sex_source_files")
-flags.DEFINE_string("input_path", default_input_path, "Import Data File's List")
+
+
+def _define_flags():
+    flags.DEFINE_string('mode', '', 'Options: download or process')
+    flags.DEFINE_string('config_path', '',
+                        'Path to the configuration file in the GCS bucket.')
+    flags.DEFINE_string("input_path", default_input_path,
+                        "Import Data File's List")
+
 
 _MCF_TEMPLATE = ("Node: dcid:{pv1}\n"
                  "typeOf: dcs:StatisticalVariable\n"
@@ -441,12 +445,12 @@ def _state_1980_1990(file_path: str) -> pd.DataFrame:
         if year == 1987:
             df = pd.read_table(file_path,
                                skiprows=29,
-                               delim_whitespace=True,
+                               sep=r'\s+',
                                names=column_names)
         else:
             df = pd.read_table(file_path,
                                skiprows=28,
-                               delim_whitespace=True,
+                               sep=r'\s+',
                                names=column_names)
         df['geo_ID'] = 'geoId/' + (df['geo_ID'].map(str)).str.zfill(2)
         df['Year'] = year
@@ -691,6 +695,11 @@ def _county_1980_1990(file_path: str) -> pd.DataFrame:
     """
     try:
         df = pd.read_csv(file_path, skiprows=5)
+        df = df.dropna(
+            subset=['Year of Estimate', 'FIPS State and County Codes'])
+        df['Year of Estimate'] = df['Year of Estimate'].astype('int64')
+        df['FIPS State and County Codes'] = df[
+            'FIPS State and County Codes'].astype('int64')
         # adding age groups to get total value
         df['Total'] = df[_COLUMNS_TO_SUM].sum(axis=1)
         df = df.drop(columns=_COLUMNS_TO_SUM)
@@ -736,7 +745,7 @@ def _county_1990_2000(file_path: str) -> pd.DataFrame:
     """
     try:
         column_names = ['Year', 'geo_ID', 'Age', 'Race-Sex', 'Ethnic', 'Value']
-        df = pd.read_table(file_path, delim_whitespace=True, header=None)
+        df = pd.read_table(file_path, sep=r'\s+', header=None)
         df.columns = column_names
         df['Year'] = '19' + df['Year'].astype(str)
         df['geo_ID'] = 'geoId/' + (df['geo_ID'].map(str)).str.zfill(5)
@@ -1092,6 +1101,8 @@ def process(self):
                 value_vars=['Count_Person_Male', 'Count_Person_Female'],
                 var_name="SV",
                 value_name="Observation")
+            final_df['Observation'] = pd.to_numeric(
+                final_df['Observation'], errors='coerce').astype('Int64')
             subset_cols = ['Year', 'geo_ID', 'Measurement_Method', 'SV']
             # 2. Drop duplicates based on those columns, keeping the first occurrence
             final_df.drop_duplicates(subset=subset_cols,
@@ -1399,4 +1410,5 @@ def main(_):
 
 
 if __name__ == "__main__":
+    _define_flags()
     app.run(main)

scripts/us_census/pep/us_pep_sex/process_test.py

@@ -23,9 +23,16 @@
 
 # module_dir is the path to where this test is running from.
 MODULE_DIR = os.path.dirname(__file__)
-sys.path.insert(0, MODULE_DIR)
+_SCRIPTS_DIR = os.path.abspath(os.path.join(MODULE_DIR, '../../../'))
+if _SCRIPTS_DIR not in sys.path:
+    sys.path.insert(0, _SCRIPTS_DIR)
+sys.path.insert(1, MODULE_DIR)
+
 # pylint: disable=wrong-import-position
-from process import PopulationEstimateBySex
+try:
+    from us_census.pep.us_pep_sex.process import PopulationEstimateBySex
+except ImportError:
+    from process import PopulationEstimateBySex
 # pylint: enable=wrong-import-position
 
 TEST_DATASET_DIR = os.path.join(MODULE_DIR, "test_data", "datasets")
@@ -41,28 +48,31 @@ class TestProcess(unittest.TestCase):
     Comparing the data with the expected files.
     """
 
-    def __init__(self, methodName: str = ...) -> None:
-        super().__init__(methodName)
+    @classmethod
+    def setUpClass(cls):
+        cls.tmp_dir_obj = tempfile.TemporaryDirectory()
+        tmp_dir = cls.tmp_dir_obj.name
+        cleaned_csv_file_path = os.path.join(tmp_dir, "data.csv")
+        mcf_file_path = os.path.join(tmp_dir, "test_census.mcf")
+        tmcf_file_path = os.path.join(tmp_dir, "test_census.tmcf")
 
-        with tempfile.TemporaryDirectory() as tmp_dir:
-            cleaned_csv_file_path = os.path.join(tmp_dir, "data.csv")
-            mcf_file_path = os.path.join(tmp_dir, "test_census.mcf")
-            tmcf_file_path = os.path.join(tmp_dir, "test_census.tmcf")
+        base = PopulationEstimateBySex(TEST_DATASET_DIR, cleaned_csv_file_path,
+                                       mcf_file_path, tmcf_file_path)
+        base.process()
 
-            base = PopulationEstimateBySex(TEST_DATASET_DIR,
-                                           cleaned_csv_file_path, mcf_file_path,
-                                           tmcf_file_path)
-            base.process()
+        with open(mcf_file_path, mode='r', encoding="UTF-8") as mcf_file:
+            cls.actual_mcf_data = mcf_file.read()
 
-            with open(mcf_file_path, mode='r', encoding="UTF-8") as mcf_file:
-                self.actual_mcf_data = mcf_file.read()
+        with open(tmcf_file_path, mode='r', encoding="UTF-8") as tmcf_file:
+            cls.actual_tmcf_data = tmcf_file.read()
 
-            with open(tmcf_file_path, mode='r', encoding="UTF-8") as tmcf_file:
-                self.actual_tmcf_data = tmcf_file.read()
+        with open(cleaned_csv_file_path, mode='r',
+                  encoding="utf-8-sig") as csv_file:
+            cls.actual_csv_data = csv_file.read()
 
-            with open(cleaned_csv_file_path, mode='r',
-                      encoding="utf-8-sig") as csv_file:
-                self.actual_csv_data = csv_file.read()
+    @classmethod
+    def tearDownClass(cls):
+        cls.tmp_dir_obj.cleanup()
 
     def test_mcf_tmcf_files(self):
         """