From ea4cdd2362df7ed842426a56eeaa993c07b73a28 Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Tue, 7 Apr 2026 07:21:33 +0000 Subject: [PATCH 1/6] Fixed Code for 3 files --- scripts/un/energy/download.py | 4 ++-- scripts/un/energy/process.py | 3 +++ scripts/un/energy/process_test.py | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/un/energy/download.py b/scripts/un/energy/download.py index 4542b4068c..b3dab19d4c 100644 --- a/scripts/un/energy/download.py +++ b/scripts/un/energy/download.py @@ -90,7 +90,6 @@ def download_energy_dataset( str(supported_datasets)) return output_files # Download data in batches of years as the download has a limit of 100k rows. - years_list = list(range(start_year, years_per_batch + 1)) years_list = [str(y) for y in range(start_year, end_year + 1)] batch_years = [ years_list[i:i + years_per_batch] @@ -117,7 +116,8 @@ def download_energy_dataset( if download_successful: logging.info(f"Download of '{download_url}' completed.") for f in os.listdir(output): - output_files.append(os.path.join(output, f)) + if f.endswith('.csv'): + output_files.append(os.path.join(output, f)) else: logging.fatal(f"Download or processing of '{download_url}' failed") return output_files diff --git a/scripts/un/energy/process.py b/scripts/un/energy/process.py index eedbe872d8..5791f7644d 100644 --- a/scripts/un/energy/process.py +++ b/scripts/un/energy/process.py @@ -545,6 +545,9 @@ def process(in_paths: list, with open(mcf_file_path, 'w+', newline='') as f_out_mcf: # Process each CSV input file, one row at a time. for in_file in in_paths: + if not in_file.endswith('.csv'): + logging.info(f'Skipping non-CSV file: {in_file}') + continue logging.info(f'Processing data file: {in_file}') with open(in_file) as csvfile: counters['input_files'] += 1 diff --git a/scripts/un/energy/process_test.py b/scripts/un/energy/process_test.py index 776e45ccbf..bb7fb81c77 100644 --- a/scripts/un/energy/process_test.py +++ b/scripts/un/energy/process_test.py @@ -67,5 +67,4 @@ def test_un_energy_process(self): if __name__ == '__main__': - app.run() unittest.main() From 91d22a72b80b57ddae12b3a9d23be9f72ec9430d Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Wed, 8 Apr 2026 10:59:08 +0000 Subject: [PATCH 2/6] NameError Resolved --- scripts/un/energy/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/un/energy/download.py b/scripts/un/energy/download.py index b3dab19d4c..1ef6e37f13 100644 --- a/scripts/un/energy/download.py +++ b/scripts/un/energy/download.py @@ -83,6 +83,7 @@ def download_energy_dataset( Returns: A list of output files downloaded. """ + output_files = [] supported_datasets = get_all_energy_source_codes() if energy_dataset not in supported_datasets: logging.info( @@ -95,7 +96,6 @@ def download_energy_dataset( years_list[i:i + years_per_batch] for i in range(0, len(years_list), years_per_batch) ] - output_files = [] for year_batch in batch_years: start_year = year_batch[0] end_year = year_batch[-1] From 3ee47d187f528b3366071dc4bf9924d9eec3b267 Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Fri, 31 Jul 2026 08:57:46 +0000 Subject: [PATCH 3/6] Fixed Mongolia_Education --- .../common_download_script.py | 294 +++++++++++++----- ...d_colleges_by_professional_field_pvmap.csv | 4 +- ...universities_and_colleges_by_sex_pvmap.csv | 8 +- ...umber_of_kindergartens_by_region_pvmap.csv | 8 +- ...versities_and_colleges_by_region_pvmap.csv | 8 +- .../mongolia_education/run.sh | 2 +- ...ns_by_sex_and_educational_degree_pvmap.csv | 8 +- ...d_colleges_by_professional_field_pvmap.csv | 8 +- 8 files changed, 260 insertions(+), 80 deletions(-) diff --git a/statvar_imports/mongolia_imports/common_download_script.py b/statvar_imports/mongolia_imports/common_download_script.py index 17254046b4..d1801dccee 100644 --- a/statvar_imports/mongolia_imports/common_download_script.py +++ b/statvar_imports/mongolia_imports/common_download_script.py @@ -183,6 +183,80 @@ }] + + + + +def find_table_path(session, table_id): + import functools + import re + tid = table_id.upper() + if tid.endswith(".PX"): + tid = tid[:-3] + + # Base ID for fuzzy version match (e.g., DT_NSO_0300_077V1 -> DT_NSO_0300_077) + base_match = re.match(r"(DT_NSO_\d+_\d+)V\d+", tid) + base_id = base_match.group(1) if base_match else tid + + sectors_url = "https://data.1212.mn/api/v1/en/NSO" + try: + r = session.get(sectors_url) + if r.status_code != 200: + return None + sectors = r.json() + except Exception as e: + logging.error(f"Failed to fetch sectors: {e}") + return None + + candidates = [] + for sector in sectors: + sector_id = sector.get("id") + if not sector_id: + continue + subsectors_url = f"https://data.1212.mn/api/v1/en/NSO/{sector_id}" + try: + r = session.get(subsectors_url) + if r.status_code != 200: + continue + subsectors = r.json() + except Exception: + continue + + for subsector in subsectors: + subsector_id = subsector.get("id") + if not subsector_id: + continue + tables_url = f"https://data.1212.mn/api/v1/en/NSO/{sector_id}/{subsector_id}" + try: + r = session.get(tables_url) + if r.status_code != 200: + continue + tables = r.json() + except Exception: + continue + + for table in tables: + table_item_id = table.get("id", "").upper() + if table_item_id.endswith(".PX"): + table_item_id = table_item_id[:-3] + + # Exact match + if table_item_id == tid: + return sector_id, subsector_id, table.get("id") + + # Fuzzy version match candidate + if table_item_id.startswith(base_id): + candidates.append((sector_id, subsector_id, table.get("id"), table_item_id)) + + if candidates: + # Sort candidates so that we get the highest version first or exact version if possible + candidates.sort(key=lambda x: x[3], reverse=True) + logging.info(f"Fuzzy version match found for {table_id}: selected {candidates[0][2]}") + return candidates[0][0], candidates[0][1], candidates[0][2] + + return None + + def fetch_and_save_data(table_id, csv_filepath, header_mapping): """ Fetches data from the API for a given table ID, pivots it, @@ -196,95 +270,171 @@ def fetch_and_save_data(table_id, csv_filepath, header_mapping): """ logging.info(f"Processing {table_id} -> {csv_filepath}...") - url = "https://opendata.1212.mn/api/Data" - data = {"tbl_id": table_id} - headers = {"Content-Type": "application/json"} - retry_logic = Retry(total=5, backoff_factor=1, allowed_methods=["POST"]) + retry_logic = Retry(total=5, backoff_factor=1, allowed_methods=["GET", "POST"]) adapter = HTTPAdapter(max_retries=retry_logic) session = requests.Session() session.mount("https://", adapter) try: - response = session.post(url, headers=headers, data=json.dumps(data)) + # 1. Resolve table path + path_info = find_table_path(session, table_id) + if not path_info: + error_msg = f"FATAL ERROR for {table_id}: Table not found in API catalog. Aborting." + logging.fatal(error_msg) + raise RuntimeError(error_msg) + + sector_id, subsector_id, table_filename = path_info + logging.info(f"Resolved path for {table_id}: NSO/{sector_id}/{subsector_id}/{table_filename}") + + # 2. Fetch metadata (English and Mongolian) + en_meta_url = f"https://data.1212.mn/api/v1/en/NSO/{sector_id}/{subsector_id}/{table_filename}" + mn_meta_url = f"https://data.1212.mn/api/v1/mn/NSO/{sector_id}/{subsector_id}/{table_filename}" + + en_meta_res = session.get(en_meta_url) + mn_meta_res = session.get(mn_meta_url) + + if en_meta_res.status_code != 200 or mn_meta_res.status_code != 200: + error_msg = f"FATAL ERROR for {table_id}: Failed to fetch metadata (EN: {en_meta_res.status_code}, MN: {mn_meta_res.status_code})" + logging.fatal(error_msg) + raise RuntimeError(error_msg) + + en_meta = en_meta_res.json() + mn_meta = mn_meta_res.json() + + # 3. Fetch all data using POST + data_url = f"https://data.1212.mn/api/v1/en/NSO/{sector_id}/{subsector_id}/{table_filename}" + payload = { + "query": [], + "response": { + "format": "json" + } + } + headers = {"Content-Type": "application/json"} + response = session.post(data_url, headers=headers, json=payload) # Check status code first if response.status_code == 200: response_data = response.json() - if not response_data: - # Checks if the dictionary is empty[from the source we are getting data in dictionary fomate] + if not response_data or "data" not in response_data: error_msg = f"FATAL ERROR for {table_id}: No data found in the source. Aborting script." logging.fatal(error_msg) raise RuntimeError(error_msg) logging.info("Success! Response data received.") - if "DataList" in response_data and isinstance( - response_data["DataList"], list): - data_list = response_data["DataList"] - - # Check for empty DataList and log the finding before aborting - if not data_list: - logging.info( - f"Found 'DataList' structure in source, but it is empty: {table_id}." - ) - error_msg = f"FATAL ERROR for {table_id}: 'DataList' is present but contains zero records. No data found from the Source" - logging.fatal(error_msg) - raise RuntimeError(error_msg) - if len(data_list) < 40: # Updated threshold to 40 records - logging.info( - f"Found 'DataList' structure in source, but it is empty or too small: {table_id}. DataList length: {len(data_list)}" - ) - error_msg = f"FATAL ERROR for {table_id}: 'DataList' contains less than 40 records. Data is not sufficient." - logging.fatal(error_msg) - raise RuntimeError(error_msg) - - pivoted_data = {} - all_periods = set() - - for item in data_list: - period = item.get("Period", "") - row_keys = [ - item.get(key, "") for key in header_mapping['keys'] - ] - dtval_co = item.get("DTVAL_CO", "") - row_key = tuple(row_keys) - - if period: - all_periods.add(period) - if row_key not in pivoted_data: - pivoted_data[row_key] = {} - pivoted_data[row_key][period] = dtval_co - - sorted_periods = sorted(list(all_periods)) - - try: - with open(csv_filepath, 'w', newline='', - encoding='utf-8') as csvfile: - csv_writer = csv.writer(csvfile) - - csv_headers = header_mapping['cols'] + sorted_periods - csv_writer.writerow(csv_headers) - - for row_key, period_values in pivoted_data.items(): - - # It creates the row using only as many keys as there are column names. - row = list(row_key)[:len(header_mapping['cols'])] - - for period in sorted_periods: - row.append(period_values.get(period, "")) - csv_writer.writerow(row) - - logging.info( - f"Successfully created CSV file: {csv_filepath}\n") - except IOError as e: - error_msg = f"Failed to write CSV file {csv_filepath}: {e}" - logging.fatal(error_msg) - raise RuntimeError(error_msg) - else: - logging.warning( - f"Error for {table_id}: 'DataList' not found in the API response.\n" + # Map variables + variables = en_meta["variables"] + period_var_idx = -1 + for idx, var in enumerate(variables): + code_lower = var.get("code", "").lower() + text_lower = var.get("text", "").lower() + if any(p in code_lower or p in text_lower for p in ["year", "period", "time", "month", "он", "сар"]): + period_var_idx = idx + break + + class_vars = [] + for idx, (en_v, mn_v) in enumerate(zip(en_meta["variables"], mn_meta["variables"])): + if idx != period_var_idx: + class_vars.append((idx, en_v, mn_v)) + + N = len(class_vars) + + # Map POST response data to data_list format + data_list = [] + for item in response_data.get("data", []): + keys = item.get("key", []) + vals = item.get("values", []) + if not vals: + continue + + row_data = {} + row_data["DTVAL_CO"] = vals[0] + + if period_var_idx != -1: + period_key = keys[period_var_idx] + en_var = en_meta["variables"][period_var_idx] + try: + val_idx = en_var["values"].index(period_key) + row_data["Period"] = en_var["valueTexts"][val_idx] + except ValueError: + row_data["Period"] = period_key + else: + row_data["Period"] = "" + + for i, (var_idx, en_v, mn_v) in enumerate(class_vars): + suffix = str(N - 1 - i) if i < N - 1 else "" + key_val = keys[var_idx] + try: + val_idx = en_v["values"].index(key_val) + en_text = en_v["valueTexts"][val_idx] + mn_text = mn_v["valueTexts"][val_idx] + except ValueError: + en_text = key_val + mn_text = key_val + row_data[f"CODE{suffix}"] = key_val + row_data[f"SCR_ENG{suffix}"] = en_text + row_data[f"SCR_MN{suffix}"] = mn_text + data_list.append(row_data) + + # Check for empty DataList and log the finding before aborting + if not data_list: + logging.info( + f"Found empty DataList for: {table_id}." + ) + error_msg = f"FATAL ERROR for {table_id}: DataList contains zero records. No data found from the Source" + logging.fatal(error_msg) + raise RuntimeError(error_msg) + if len(data_list) < 40: # Updated threshold to 40 records + logging.info( + f"Found too small DataList: {table_id}. DataList length: {len(data_list)}" ) + error_msg = f"FATAL ERROR for {table_id}: DataList contains less than 40 records. Data is not sufficient." + logging.fatal(error_msg) + raise RuntimeError(error_msg) + + pivoted_data = {} + all_periods = set() + + for item in data_list: + period = item.get("Period", "") + row_keys = [ + item.get(key, "") for key in header_mapping['keys'] + ] + dtval_co = item.get("DTVAL_CO", "") + row_key = tuple(row_keys) + + if period: + all_periods.add(period) + if row_key not in pivoted_data: + pivoted_data[row_key] = {} + pivoted_data[row_key][period] = dtval_co + + sorted_periods = sorted(list(all_periods)) + + try: + with open(csv_filepath, 'w', newline='', + encoding='utf-8') as csvfile: + csv_writer = csv.writer(csvfile) + + csv_headers = header_mapping['cols'] + sorted_periods + csv_writer.writerow(csv_headers) + + for row_key, period_values in pivoted_data.items(): + + # It creates the row using only as many keys as there are column names. + row = list(row_key)[:len(header_mapping['cols'])] + + for period in sorted_periods: + row.append(period_values.get(period, "")) + csv_writer.writerow(row) + + logging.info( + f"Successfully created CSV file: {csv_filepath}\n") + except IOError as e: + error_msg = f"Failed to write CSV file {csv_filepath}: {e}" + logging.fatal(error_msg) + raise RuntimeError(error_msg) else: logging.warning( f"Error for {table_id}: Request failed with status code {response.status_code}\n" diff --git a/statvar_imports/mongolia_imports/mongolia_education/graduates_of_universities_and_colleges_by_professional_field_pvmap.csv b/statvar_imports/mongolia_imports/mongolia_education/graduates_of_universities_and_colleges_by_professional_field_pvmap.csv index fce3110dcf..a2a4a870a8 100644 --- a/statvar_imports/mongolia_imports/mongolia_education/graduates_of_universities_and_colleges_by_professional_field_pvmap.csv +++ b/statvar_imports/mongolia_imports/mongolia_education/graduates_of_universities_and_colleges_by_professional_field_pvmap.csv @@ -13,7 +13,7 @@ Biological and related sciences,bachelorsDegreeMajor,BiologicalAndBiomedicalScie Environment,bachelorsDegreeMajor,EnvironmentMajor,,,,,, Natural sciences,bachelorsDegreeMajor,NaturalSciencesMajor,,,,,, Mathematics and statistics,bachelorsDegreeMajor,MathAndStatisticsMajor,,,,,, -Information and Communication Technologies,bachelorsDegreeMajor,InformationAndCommunicationsTechnologiesMajor ,,,,,, +Information and Communication Technologies,bachelorsDegreeMajor,InformationAndCommunicationsTechnologiesMajor,,,,,, "Engineering, manufacturing and construction",bachelorsDegreeMajor,EngineeringMajor__EngineeringManufacturing__ConstructionEngineering,,,,,, Architecture and construction,bachelorsDegreeMajor,ArchitectureAndRelatedServicesMajor__ConstructionEngineering,,,,,, Engineering between industries,bachelorsDegreeMajor,EngineeringMajor__Industries,,,,,, @@ -22,7 +22,7 @@ Production and processing,bachelorsDegreeMajor,ProductionEngineering__Processing "Agriculture, forestry, fisheries and veterinary",bachelorsDegreeMajor,AgriculturalEngineering__ForestryEngineering__Fisheries__Veterinary,,,,,, Veterinary medicine,bachelorsDegreeMajor,VeterinaryMedicineMajor,,,,,, "Agriculture, forestry, fisheries",bachelorsDegreeMajor,AgriculturalEngineering__ForestryEngineering__Fisheries,,,,,, -Health and social protection,bachelorsDegreeMajor,HealthRelatedMajor,,,,,, +Health and welfare,bachelorsDegreeMajor,HealthRelatedMajor,,,,,, Oral and Maxillofacial Studies,bachelorsDegreeMajor,OralAndMaxillofacialStudies,,,,,, Nursing and midwifery,bachelorsDegreeMajor,NursingAndMidwifery,,,,,, Pharmacology,bachelorsDegreeMajor,Pharmacology,,,,,, diff --git a/statvar_imports/mongolia_imports/mongolia_education/number_of_full_time_teachers_in_universities_and_colleges_by_sex_pvmap.csv b/statvar_imports/mongolia_imports/mongolia_education/number_of_full_time_teachers_in_universities_and_colleges_by_sex_pvmap.csv index 34d2396aaf..5097ed1606 100644 --- a/statvar_imports/mongolia_imports/mongolia_education/number_of_full_time_teachers_in_universities_and_colleges_by_sex_pvmap.csv +++ b/statvar_imports/mongolia_imports/mongolia_education/number_of_full_time_teachers_in_universities_and_colleges_by_sex_pvmap.csv @@ -28,4 +28,10 @@ Number of teacher:Total,populationType,Teacher,measuredProperty,count,employment 2021,observationDate,2021,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count 2022,observationDate,2022,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count 2023,observationDate,2023,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count -2024,observationDate,2024,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count \ No newline at end of file +2024,observationDate,2024,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count +2025,observationDate,2025,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count +2026,observationDate,2026,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count +2027,observationDate,2027,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count +2028,observationDate,2028,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count +2029,observationDate,2029,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count +2030,observationDate,2030,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count \ No newline at end of file diff --git a/statvar_imports/mongolia_imports/mongolia_education/number_of_kindergartens_by_region_pvmap.csv b/statvar_imports/mongolia_imports/mongolia_education/number_of_kindergartens_by_region_pvmap.csv index fc29252b7f..f13fa3a559 100644 --- a/statvar_imports/mongolia_imports/mongolia_education/number_of_kindergartens_by_region_pvmap.csv +++ b/statvar_imports/mongolia_imports/mongolia_education/number_of_kindergartens_by_region_pvmap.csv @@ -24,4 +24,10 @@ Aimag,observationAbout,{Data},schoolGradeLevel,Kindergarten,populationType,Schoo 2021,observationDate,2021,value,{Number},,,, 2022,observationDate,2022,value,{Number},,,, 2023,observationDate,2023,value,{Number},,,, -2024,observationDate,2024,value,{Number},,,, \ No newline at end of file +2024,observationDate,2024,value,{Number},,,, +2025,observationDate,2025,value,{Number},,,, +2026,observationDate,2026,value,{Number},,,, +2027,observationDate,2027,value,{Number},,,, +2028,observationDate,2028,value,{Number},,,, +2029,observationDate,2029,value,{Number},,,, +2030,observationDate,2030,value,{Number},,,, \ No newline at end of file diff --git a/statvar_imports/mongolia_imports/mongolia_education/number_of_students_in_universities_and_colleges_by_region_pvmap.csv b/statvar_imports/mongolia_imports/mongolia_education/number_of_students_in_universities_and_colleges_by_region_pvmap.csv index 3e1eb07f2b..81b33f9e67 100644 --- a/statvar_imports/mongolia_imports/mongolia_education/number_of_students_in_universities_and_colleges_by_region_pvmap.csv +++ b/statvar_imports/mongolia_imports/mongolia_education/number_of_students_in_universities_and_colleges_by_region_pvmap.csv @@ -19,4 +19,10 @@ Female,gender,Female,,,, 2021,observationDate,2021,value,{Number},, 2022,observationDate,2022,value,{Number},, 2023,observationDate,2023,value,{Number},, -2024,observationDate,2024,value,{Number},, \ No newline at end of file +2024,observationDate,2024,value,{Number},, +2025,observationDate,2025,value,{Number},, +2026,observationDate,2026,value,{Number},, +2027,observationDate,2027,value,{Number},, +2028,observationDate,2028,value,{Number},, +2029,observationDate,2029,value,{Number},, +2030,observationDate,2030,value,{Number},, \ No newline at end of file diff --git a/statvar_imports/mongolia_imports/mongolia_education/run.sh b/statvar_imports/mongolia_imports/mongolia_education/run.sh index 73abefc34c..43438347f9 100644 --- a/statvar_imports/mongolia_imports/mongolia_education/run.sh +++ b/statvar_imports/mongolia_imports/mongolia_education/run.sh @@ -9,7 +9,7 @@ python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --inp python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/number_of_students_in_universities_and_colleges_by_region.csv --pv_map=$SCRIPT_PATH/number_of_students_in_universities_and_colleges_by_region_pvmap.csv --config_file=$SCRIPT_PATH/mongolia_metadata.csv --output_path=$SCRIPT_PATH/output_files/number_of_students_in_universities_and_colleges_by_region_output --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=$SCRIPT_PATH/mongolia_place_resolver.csv || { echo "Error: Processing number_of_students_in_universities_and_colleges_by_region.csv failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/number_of_kindergartens_by_region.csv --pv_map=$SCRIPT_PATH/number_of_kindergartens_by_region_pvmap.csv --config_file=$SCRIPT_PATH/mongolia_metadata.csv --output_path=$SCRIPT_PATH/output_files/number_of_kindergartens_by_region_output --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf || { echo "Error: Processing number_of_kindergartens_by_region_pvmap.csv failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/number_of_kindergartens_by_region.csv --pv_map=$SCRIPT_PATH/number_of_kindergartens_by_region_pvmap.csv --config_file=$SCRIPT_PATH/mongolia_metadata.csv --output_path=$SCRIPT_PATH/output_files/number_of_kindergartens_by_region_output --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=$SCRIPT_PATH/mongolia_place_resolver.csv || { echo "Error: Processing number_of_kindergartens_by_region_pvmap.csv failed!"; exit 1; } python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/number_of_full_time_teachers_in_universities_and_colleges_by_sex.csv --pv_map=$SCRIPT_PATH/number_of_full_time_teachers_in_universities_and_colleges_by_sex_pvmap.csv --config_file=$SCRIPT_PATH/mongolia_metadata.csv --output_path=$SCRIPT_PATH/output_files/number_of_full_time_teachers_in_universities_and_colleges_by_sex_output --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf || { echo "Error: Processing number_of_full_time_teachers_in_universities_and_colleges_by_sex.csv failed!"; exit 1; } diff --git a/statvar_imports/mongolia_imports/mongolia_education/students_in_teritary_educational_institutions_by_sex_and_educational_degree_pvmap.csv b/statvar_imports/mongolia_imports/mongolia_education/students_in_teritary_educational_institutions_by_sex_and_educational_degree_pvmap.csv index 2b14b206fb..dcef73f171 100644 --- a/statvar_imports/mongolia_imports/mongolia_education/students_in_teritary_educational_institutions_by_sex_and_educational_degree_pvmap.csv +++ b/statvar_imports/mongolia_imports/mongolia_education/students_in_teritary_educational_institutions_by_sex_and_educational_degree_pvmap.csv @@ -2,7 +2,7 @@ key,p1,v1,p2,v2,, ,,,,,, Professional fields,schoolGradeLevel,TertiaryEducation,populationType,Student,observationAbout,country/MNG Of which: Female,gender,Female,schoolGradeLevel,TertiaryEducation,, -Bachelor¹,enrollmentLevel,BachelorsDegree,schoolGradeLevel,TertiaryEducation,#Header,enrollmentLevel +Bachelor,enrollmentLevel,BachelorsDegree,schoolGradeLevel,TertiaryEducation,#Header,enrollmentLevel Master,enrollmentLevel,MastersDegree,schoolGradeLevel,TertiaryEducation,#Header,enrollmentLevel Ph.D,enrollmentLevel,DoctorateDegree,schoolGradeLevel,TertiaryEducation,#Header,enrollmentLevel Upper diploma,enrollmentLevel,UpperDiploma,schoolGradeLevel,TertiaryEducation,#Header,enrollmentLevel @@ -30,3 +30,9 @@ Upper diploma,enrollmentLevel,UpperDiploma,schoolGradeLevel,TertiaryEducation,#H 2022,observationDate,2022,value,{Number},, 2023,observationDate,2023,value,{Number},, 2024,observationDate,2024,value,{Number},, +2025,observationDate,2025,value,{Number},, +2026,observationDate,2026,value,{Number},, +2027,observationDate,2027,value,{Number},, +2028,observationDate,2028,value,{Number},, +2029,observationDate,2029,value,{Number},, +2030,observationDate,2030,value,{Number},, diff --git a/statvar_imports/mongolia_imports/mongolia_education/students_of_universities_and_colleges_by_professional_field_pvmap.csv b/statvar_imports/mongolia_imports/mongolia_education/students_of_universities_and_colleges_by_professional_field_pvmap.csv index 4341f3efca..33ac607fa6 100644 --- a/statvar_imports/mongolia_imports/mongolia_education/students_of_universities_and_colleges_by_professional_field_pvmap.csv +++ b/statvar_imports/mongolia_imports/mongolia_education/students_of_universities_and_colleges_by_professional_field_pvmap.csv @@ -29,4 +29,10 @@ General basic curriculum,bachelorsDegreeMajor,GeneralBasicCurriculumMajor,,,,,,, 2021,observationDate,2021,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool 2022,observationDate,2022,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool 2023,observationDate,2023,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool -2024,observationDate,2024,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool \ No newline at end of file +2024,observationDate,2024,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool +2025,observationDate,2025,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool +2026,observationDate,2026,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool +2027,observationDate,2027,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool +2028,observationDate,2028,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool +2029,observationDate,2029,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool +2030,observationDate,2030,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool \ No newline at end of file From 75350d1de2568f3bb9405275a466a2c7a466a0d0 Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Fri, 31 Jul 2026 09:15:05 +0000 Subject: [PATCH 4/6] gemini suggested changes done --- .../common_download_script.py | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/statvar_imports/mongolia_imports/common_download_script.py b/statvar_imports/mongolia_imports/common_download_script.py index d1801dccee..5898731f60 100644 --- a/statvar_imports/mongolia_imports/common_download_script.py +++ b/statvar_imports/mongolia_imports/common_download_script.py @@ -200,7 +200,7 @@ def find_table_path(session, table_id): sectors_url = "https://data.1212.mn/api/v1/en/NSO" try: - r = session.get(sectors_url) + r = session.get(sectors_url, timeout=60) if r.status_code != 200: return None sectors = r.json() @@ -215,7 +215,7 @@ def find_table_path(session, table_id): continue subsectors_url = f"https://data.1212.mn/api/v1/en/NSO/{sector_id}" try: - r = session.get(subsectors_url) + r = session.get(subsectors_url, timeout=60) if r.status_code != 200: continue subsectors = r.json() @@ -228,7 +228,7 @@ def find_table_path(session, table_id): continue tables_url = f"https://data.1212.mn/api/v1/en/NSO/{sector_id}/{subsector_id}" try: - r = session.get(tables_url) + r = session.get(tables_url, timeout=60) if r.status_code != 200: continue tables = r.json() @@ -290,8 +290,8 @@ def fetch_and_save_data(table_id, csv_filepath, header_mapping): en_meta_url = f"https://data.1212.mn/api/v1/en/NSO/{sector_id}/{subsector_id}/{table_filename}" mn_meta_url = f"https://data.1212.mn/api/v1/mn/NSO/{sector_id}/{subsector_id}/{table_filename}" - en_meta_res = session.get(en_meta_url) - mn_meta_res = session.get(mn_meta_url) + en_meta_res = session.get(en_meta_url, timeout=60) + mn_meta_res = session.get(mn_meta_url, timeout=60) if en_meta_res.status_code != 200 or mn_meta_res.status_code != 200: error_msg = f"FATAL ERROR for {table_id}: Failed to fetch metadata (EN: {en_meta_res.status_code}, MN: {mn_meta_res.status_code})" @@ -310,7 +310,7 @@ def fetch_and_save_data(table_id, csv_filepath, header_mapping): } } headers = {"Content-Type": "application/json"} - response = session.post(data_url, headers=headers, json=payload) + response = session.post(data_url, headers=headers, json=payload, timeout=60) # Check status code first if response.status_code == 200: @@ -324,17 +324,17 @@ def fetch_and_save_data(table_id, csv_filepath, header_mapping): logging.info("Success! Response data received.") # Map variables - variables = en_meta["variables"] + variables = en_meta.get("variables", []) period_var_idx = -1 for idx, var in enumerate(variables): - code_lower = var.get("code", "").lower() - text_lower = var.get("text", "").lower() + code_lower = var.get("code", "").lower() if var else "" + text_lower = var.get("text", "").lower() if var else "" if any(p in code_lower or p in text_lower for p in ["year", "period", "time", "month", "он", "сар"]): period_var_idx = idx break class_vars = [] - for idx, (en_v, mn_v) in enumerate(zip(en_meta["variables"], mn_meta["variables"])): + for idx, (en_v, mn_v) in enumerate(zip(en_meta.get("variables", []), mn_meta.get("variables", []))): if idx != period_var_idx: class_vars.append((idx, en_v, mn_v)) @@ -351,12 +351,14 @@ def fetch_and_save_data(table_id, csv_filepath, header_mapping): row_data = {} row_data["DTVAL_CO"] = vals[0] - if period_var_idx != -1: + if period_var_idx != -1 and period_var_idx < len(keys): period_key = keys[period_var_idx] - en_var = en_meta["variables"][period_var_idx] + en_var = en_meta.get("variables", [])[period_var_idx] if period_var_idx < len(en_meta.get("variables", [])) else None + en_values = en_var.get("values", []) if en_var else [] + en_texts = en_var.get("valueTexts", []) if en_var else [] try: - val_idx = en_var["values"].index(period_key) - row_data["Period"] = en_var["valueTexts"][val_idx] + val_idx = en_values.index(period_key) + row_data["Period"] = en_texts[val_idx] if val_idx < len(en_texts) else period_key except ValueError: row_data["Period"] = period_key else: @@ -364,11 +366,14 @@ def fetch_and_save_data(table_id, csv_filepath, header_mapping): for i, (var_idx, en_v, mn_v) in enumerate(class_vars): suffix = str(N - 1 - i) if i < N - 1 else "" - key_val = keys[var_idx] + key_val = keys[var_idx] if var_idx < len(keys) else "" + en_values = en_v.get("values", []) if en_v else [] + en_texts = en_v.get("valueTexts", []) if en_v else [] + mn_texts = mn_v.get("valueTexts", []) if mn_v else [] try: - val_idx = en_v["values"].index(key_val) - en_text = en_v["valueTexts"][val_idx] - mn_text = mn_v["valueTexts"][val_idx] + val_idx = en_values.index(key_val) + en_text = en_texts[val_idx] if val_idx < len(en_texts) else key_val + mn_text = mn_texts[val_idx] if val_idx < len(mn_texts) else key_val except ValueError: en_text = key_val mn_text = key_val From 3ec4357a3ca15dfeea9b626bb42bcc73ea1f7c3f Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Fri, 31 Jul 2026 11:46:12 +0000 Subject: [PATCH 5/6] Fix for mongolia_health --- .../mongolia_imports/common_download_script.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/statvar_imports/mongolia_imports/common_download_script.py b/statvar_imports/mongolia_imports/common_download_script.py index 5898731f60..e0632fde6f 100644 --- a/statvar_imports/mongolia_imports/common_download_script.py +++ b/statvar_imports/mongolia_imports/common_download_script.py @@ -358,9 +358,13 @@ def fetch_and_save_data(table_id, csv_filepath, header_mapping): en_texts = en_var.get("valueTexts", []) if en_var else [] try: val_idx = en_values.index(period_key) - row_data["Period"] = en_texts[val_idx] if val_idx < len(en_texts) else period_key + p_val = en_texts[val_idx] if val_idx < len(en_texts) else period_key except ValueError: - row_data["Period"] = period_key + p_val = period_key + import re + if re.match(r"^\d{4}-\d{2}$", p_val): + p_val = p_val.replace("-", "") + row_data["Period"] = p_val else: row_data["Period"] = "" From 0adf170a20b17a4c09834838ee4889ab423e7383 Mon Sep 17 00:00:00 2001 From: Kartik Samnotra Date: Sat, 1 Aug 2026 04:33:53 +0000 Subject: [PATCH 6/6] Mongolia Demographics fixed --- .../mongolia_imports/common_download_script.py | 8 ++++---- ...roup_15_and_over_and_maritalstatus_pvmap.csv | 17 +++++++++-------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/statvar_imports/mongolia_imports/common_download_script.py b/statvar_imports/mongolia_imports/common_download_script.py index e0632fde6f..adf2c96286 100644 --- a/statvar_imports/mongolia_imports/common_download_script.py +++ b/statvar_imports/mongolia_imports/common_download_script.py @@ -64,7 +64,7 @@ "filename": "resident_population_by_agegroup_15_and_over_and_maritalstatus.csv", "header_mapping": { - "keys": ["SCR_ENG", "SCR_ENG1", "SCR_ENG2", "CODE"], + "keys": ["SCR_ENG2", "SCR_ENG1", "SCR_ENG", "CODE"], "cols": ["Marital Status", "Age Group", "Gender"] } }, @@ -72,8 +72,8 @@ "id": "DT_NSO_0300_004V1", "filename": "total_population_by_region_and_urban_rural.csv", "header_mapping": { - "keys": ["SCR_ENG", "SCR_ENG1", "CODE1", "CODE"], - "cols": ["Total", "Aimag", "Код"] + "keys": ["SCR_ENG", "SCR_ENG1", "CODE"], + "cols": ["Aimag", "Total", "Код"] } }, { @@ -81,7 +81,7 @@ "filename": "number_of_households_by_region_and_urban_rural.csv", "header_mapping": { "keys": ["SCR_ENG", "SCR_ENG1", "CODE"], - "cols": ["NUMBER OF HOUSEHOLDS", "Aimag", "Код"] + "cols": ["Aimag", "NUMBER OF HOUSEHOLDS", "Код"] } }, ] diff --git a/statvar_imports/mongolia_imports/mongolia_demographics/resident_population_by_agegroup_15_and_over_and_maritalstatus_pvmap.csv b/statvar_imports/mongolia_imports/mongolia_demographics/resident_population_by_agegroup_15_and_over_and_maritalstatus_pvmap.csv index 28652958e5..9abe21918c 100644 --- a/statvar_imports/mongolia_imports/mongolia_demographics/resident_population_by_agegroup_15_and_over_and_maritalstatus_pvmap.csv +++ b/statvar_imports/mongolia_imports/mongolia_demographics/resident_population_by_agegroup_15_and_over_and_maritalstatus_pvmap.csv @@ -10,13 +10,14 @@ Single,maritalStatus,Unmarried,,,,,,,, 2000,observationDate,2000,value,{Number},observationAbout,country/MNG,populationType,Person,measuredProperty,count 2010,observationDate,2010,value,{Number},observationAbout,country/MNG,populationType,Person,measuredProperty,count 2015,observationDate,2015,value,{Number},observationAbout,country/MNG,populationType,Person,measuredProperty,count +2020,observationDate,2020,value,{Number},observationAbout,country/MNG,populationType,Person,measuredProperty,count ,,,,,,,,,, -Age Group:Population aged 15 and over-Total,age,[15 - Years],,,,,,,, -Age Group:15-19,#ignore,skip,,,,,,,, -Age Group:70+,#ignore,skip,,,,,,,, +Age Group:Total,age,[15 - Years],,,,,,,, +Age Group:20-29,age,[20 29 Years],,,,,,,, +Age Group:30-39,age,[30 39 Years],,,,,,,, +Age Group:40-49,age,[40 49 Years],,,,,,,, +Age Group:50-59,age,[50 59 Years],,,,,,,, +Age Group:60-69,age,[60 69 Years],,,,,,,, ,,,,,,,,,, -20-29,age,[20 29 Years],,,,,,,, -30-39,age,[30 39 Years],,,,,,,, -40-49,age,[40 49 Years],,,,,,,, -50-59,age,[50 59 Years],,,,,,,, -60-69,age,[60 69 Years],,,,,,,, +Male,#ignore,skip,,,,,,,, +Female,#ignore,skip,,,,,,,,