Added goldens for USCensusPEP_MonthlyPopulation , USCensusPEP_AgeSexRace & USCensusPEP_Sex

scripts/us_census/pep/monthly_population_estimate/golden_data/golden_observations.csv

@@ -0,0 +1,2 @@
+"Location"
+"country/USA"

scripts/us_census/pep/monthly_population_estimate/golden_data/golden_summary_report.csv

@@ -0,0 +1,7 @@
+"NumPlaces","MeasurementMethods","observationPeriods","Units","MinDate","ScalingFactors","StatVar"
+"1","[CensusPEPSurvey]","[P1M]","[]","1980-04","[]","Count_Person_USResident"
+"1","[CensusPEPSurvey]","[P1M]","[]","1990-04","[]","Count_Person_Civilian_NonInstitutionalized"
+"1","[dcAggregate/CensusPEPSurvey]","[P1M]","[]","1980-04","[]","Count_Person_InUSArmedForcesOverseas"
+"1","[CensusPEPSurvey]","[P1M]","[]","2010-04","[]","Count_Person_ResidesInHousehold"
+"1","[CensusPEPSurvey]","[P1M]","[]","1980-04","[]","Count_Person_Civilian"
+"1","[CensusPEPSurvey]","[P1M]","[]","1980-04","[]","Count_Person_USResidentOrInUSArmedForcesOverseas"

scripts/us_census/pep/monthly_population_estimate/manifest.json

@@ -19,7 +19,8 @@
                     "cleaned_csv": "output/USA_Population_Count.csv"
                 }
             ],
-            "cron_schedule": "0 7 * * 6"
+            "cron_schedule": "0 7 * * 6",
+            "validation_config_file": "validation_config.json"
         }
     ]
 }

scripts/us_census/pep/monthly_population_estimate/preprocess.py

@@ -241,8 +241,7 @@ def _transform_df(self, df: pd.DataFrame, file: str) -> pd.DataFrame:
         final_cols = [col for col in df.columns if 'year' not in col.lower()]
         # _return_year("1999") or _return_year("1999 [1]"): 1999
         # _return_year(".07 1"): pd.NA
-        df['Year'] = df['Year and Month'].apply(_return_year).fillna(
-            method='ffill', limit=12)
+        df['Year'] = df['Year and Month'].apply(_return_year).ffill(limit=12)
         # _return_year("1999") or _return_year("1999 [1]"): pd.NA
         # _return_year(".07 1"): 07
         df['Month'] = df['Year and Month'].apply(_return_month)
@@ -568,8 +567,8 @@ def _mulitply_scaling_factor(col: pd.Series) -> pd.Series:
         Series: A DataFrame column with values multiplied by the scaling factor.
     """
     res = col
-    if col not in [None, np.nan]:
-        if col.isdigit():
+    if not pd.isna(col) and col is not None:
+        if isinstance(col, str) and col.isdigit():
             res = int(col) * _SCALING_FACTOR_TXT_FILE
     return res
 
@@ -585,10 +584,10 @@ def _concat_cols(col: pd.Series) -> pd.Series:
     Returns:
         res (Series) : Concatenated DataFrame Columns
     """
-    res = col[0]
-    if col[1] is None:
+    res = col.iloc[0]
+    if pd.isna(col.iloc[1]) or col.iloc[1] is None:
         return res
-    res = col[0] + ' ' + col[1]
+    res = str(col.iloc[0]) + ' ' + str(col.iloc[1])
     return res
 
 
@@ -654,7 +653,7 @@ def download_files():
                     ]
                     df = pd.read_table(url,
                                        index_col=False,
-                                       delim_whitespace=True,
+                                       sep=r'\s+',
                                        engine='python',
                                        skiprows=17,
                                        names=cols)

scripts/us_census/pep/monthly_population_estimate/validation_config.json

@@ -0,0 +1,32 @@
+{
+    "schema_version": "1.0",
+    "rules": [
+        {
+            "rule_id": "check_deleted_records_percent",
+            "description": "Checks that the percentage of deleted points is within the threshold.",
+            "validator": "DELETED_RECORDS_PERCENT",
+            "params": {
+                "threshold": 0.1
+            }
+        },
+        {
+            "rule_id": "check_goldens_summary_report",
+            "validator": "GOLDENS_CHECK",
+            "params": {
+                "golden_files": "../../../../golden_data/golden_summary_report.csv"
+            }
+        },
+        {
+            "rule_id": "Check_goldens_output_csv",
+            "description": "Verifies the generated output CSV data matches established critical golden records",
+            "validator": "GOLDENS_CHECK",
+            "params": {
+                "golden_files": "../../../../golden_data/golden_observations.csv",
+                "input_files": "../../../../output/USA_Population_Count.csv"
+            }
+        }
+    ]
+}
+
+
+

scripts/us_census/pep/population_estimates_by_asr/county_1990_2000.py

@@ -42,7 +42,7 @@ def county1990(output_folder: str):
                 '/popest/tables/1990-2000/counties/asrh/casrh'+str(j)+'.txt'
             cols=['Year','geo_ID','Race',0,1,2,3,4,5,6,7\
                 ,8,9,10,11,12,13,14,15,16,17]
-            df = pd.read_table(url,index_col=False,delim_whitespace=True\
+            df = pd.read_table(url,index_col=False,sep=r'\s+'\
                 ,skiprows=16,skipfooter=14,engine='python',names=cols,\
                     encoding='ISO-8859-1')
             #Writing raw data to csv