From 64bbab2f106160a29dbafef2eeeacb8eae8a5092 Mon Sep 17 00:00:00 2001 From: kcazzatoCMAP Date: Thu, 21 May 2026 12:01:54 -0500 Subject: [PATCH 1/9] update wfh columns change logic to read in the first three columns only of that file; so regardless of how many columns the input file has, it will just grab the first 3 it needs --- Database/trip_generation/trip_generation_model.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Database/trip_generation/trip_generation_model.py b/Database/trip_generation/trip_generation_model.py index 9bda35c..6a2426a 100644 --- a/Database/trip_generation/trip_generation_model.py +++ b/Database/trip_generation/trip_generation_model.py @@ -303,12 +303,10 @@ def fixed_width(outfile, df): sz.drop(szDrop, axis=1, inplace=True) # Load household work from home flag file -wfhCols = ['serial_number','wfh_flag','number_WFH_workers','tc14_not_working'] -wfhDrop = ['number_WFH_workers','tc14_not_working'] -wfh = pd.read_csv(wfhFile, names=wfhCols, engine=pdEngine) +wfhCols = ['serial_number','wfh_flag','number_WFH_workers'] +wfh = pd.read_csv(wfhFile, names=wfhCols, engine=pdEngine, usecols=[0, 1, 2]) wfh['household_record'] = wfh.index + 1 wfh['wfh_flag'] = wfh['wfh_flag'].clip(upper=1) -wfh.drop(wfhDrop, axis=1, inplace=True) # Load household vehicle type category file hhvtype = pd.read_csv(hhvtypeFile, dtype='Int64', engine=pdEngine) From 50a4338ee8d59731e0b78c64082c01f0d4f57670 Mon Sep 17 00:00:00 2001 From: kcazzatoCMAP Date: Thu, 21 May 2026 12:04:57 -0500 Subject: [PATCH 2/9] update only 2 columns The script only actually needs the first two columns so I made that update --- Database/trip_generation/trip_generation_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Database/trip_generation/trip_generation_model.py b/Database/trip_generation/trip_generation_model.py index 6a2426a..ce9a4da 100644 --- a/Database/trip_generation/trip_generation_model.py +++ b/Database/trip_generation/trip_generation_model.py @@ -303,8 +303,8 @@ def fixed_width(outfile, df): sz.drop(szDrop, axis=1, inplace=True) # Load household work from home flag file -wfhCols = ['serial_number','wfh_flag','number_WFH_workers'] -wfh = pd.read_csv(wfhFile, names=wfhCols, engine=pdEngine, usecols=[0, 1, 2]) +wfhCols = ['serial_number','wfh_flag'] +wfh = pd.read_csv(wfhFile, names=wfhCols, engine=pdEngine, usecols=[0, 1]) wfh['household_record'] = wfh.index + 1 wfh['wfh_flag'] = wfh['wfh_flag'].clip(upper=1) From 7aa65f134f352b355e7818769035f7e7807b36ee Mon Sep 17 00:00:00 2001 From: Nicholas Ferguson Date: Fri, 24 Jul 2026 17:30:40 -0500 Subject: [PATCH 3/9] feat(workflow): add branch restriction for pull requests to main Pull requests to main must come from develop. --- .github/workflows/restrict-pr-source.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/restrict-pr-source.yml diff --git a/.github/workflows/restrict-pr-source.yml b/.github/workflows/restrict-pr-source.yml new file mode 100644 index 0000000..1b30bce --- /dev/null +++ b/.github/workflows/restrict-pr-source.yml @@ -0,0 +1,23 @@ +name: Restrict PR Source Branch + +on: + pull_request: + branches: + - main + +jobs: + verify-source-branch: + runs-on: windows-latest + steps: + - name: Verify PR source branch + shell: pwsh + run: | + $allowedBranch = "develop" + $headBranch = "${{ github.head_ref }}" + + if ($headBranch -ne $allowedBranch) { + Write-Error "Pull Requests to this branch are only allowed from '$allowedBranch'. You tried to merge from '$headBranch'." + exit 1 + } + + Write-Output "Branch verification successful." From 446ec37ad8822ef80128286e7285b739cd45f891 Mon Sep 17 00:00:00 2001 From: Nicholas Ferguson Date: Fri, 24 Jul 2026 18:38:39 -0500 Subject: [PATCH 4/9] feat(workflow): add sync workflow to merge main into develop Keep develop in sync with main after merging a PR from develop to main. The merge creates a merge commit in main that puts develop one commit behind. --- .github/workflows/sync-develop.yml | 35 ++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/sync-develop.yml diff --git a/.github/workflows/sync-develop.yml b/.github/workflows/sync-develop.yml new file mode 100644 index 0000000..78dbae4 --- /dev/null +++ b/.github/workflows/sync-develop.yml @@ -0,0 +1,35 @@ +name: Sync Develop with Main + +on: + pull_request: + types: [closed] + branches: + - main + +jobs: + sync: + if: github.event.pull_request.merged == true && github.event.pull_request.head.ref == 'develop' + runs-on: windows-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: develop + fetch-depth: 0 + + - name: Configure Git + shell: pwsh + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + - name: Merge Main into Develop + shell: pwsh + run: | + git fetch origin main + try { + git merge origin/main --ff-only + } catch { + git merge origin/main -m "chore: sync main back to develop" + } + git push origin develop \ No newline at end of file From a626f9f59fd7bbce7193b2bfed51992d9290a844 Mon Sep 17 00:00:00 2001 From: Nicholas Ferguson Date: Tue, 7 Jul 2026 09:56:22 -0500 Subject: [PATCH 5/9] feat(tbmtools): Add subpackages and modules for packaging data - Restructure tbmtools as a Python project with a src layout - Integrate scripts from \Scripts\share\standard_data - Add a CLI for packaging data with an option to batch process a series of model runs --- Scripts/share/standard_data/run.cmd | 6 - .../share/standard_data/src/standard_data.py | 304 ------------------ Scripts/tbmtools/project.py | 44 --- Scripts/tbmtools/results/highway_network.py | 165 ---------- Scripts/tbmtools/results/sharing.py | 42 --- Scripts/tbmtools/results/skims.py | 70 ---- Scripts/tbmtools/results/transit_network.py | 79 ----- Scripts/tbmtools/results/trip_roster.py | 33 -- Scripts/tbmtools/results/vehicle_trips.py | 22 -- .../tbmtools/{ => src/tbmtools}/__init__.py | 0 Scripts/tbmtools/src/tbmtools/__main__.py | 4 + Scripts/tbmtools/src/tbmtools/cli.py | 13 + .../src/tbmtools}/config.yaml | 6 +- Scripts/tbmtools/src/tbmtools/data_package.py | 297 +++++++++++++++++ .../tbmtools/matrix}/__init__.py | 0 .../tbmtools/matrix/person_trip.py} | 264 +++++++++++---- Scripts/tbmtools/src/tbmtools/matrix/skim.py | 119 +++++++ .../src/tbmtools/matrix/vehicle_trip.py | 46 +++ .../tbmtools/src/tbmtools/network/__init__.py | 0 .../tbmtools/src/tbmtools/network/highway.py | 220 +++++++++++++ .../tbmtools/src/tbmtools/network/transit.py | 133 ++++++++ Scripts/tbmtools/src/tbmtools/project.py | 66 ++++ .../templates}/data_user_guide_html.txt | 0 .../templates}/data_user_guide_md.txt | 0 Scripts/tbmtools/src/tbmtools/utils.py | 198 ++++++++++++ 25 files changed, 1293 insertions(+), 838 deletions(-) delete mode 100644 Scripts/share/standard_data/run.cmd delete mode 100644 Scripts/share/standard_data/src/standard_data.py delete mode 100644 Scripts/tbmtools/project.py delete mode 100644 Scripts/tbmtools/results/highway_network.py delete mode 100644 Scripts/tbmtools/results/sharing.py delete mode 100644 Scripts/tbmtools/results/skims.py delete mode 100644 Scripts/tbmtools/results/transit_network.py delete mode 100644 Scripts/tbmtools/results/trip_roster.py delete mode 100644 Scripts/tbmtools/results/vehicle_trips.py rename Scripts/tbmtools/{ => src/tbmtools}/__init__.py (100%) create mode 100644 Scripts/tbmtools/src/tbmtools/__main__.py create mode 100644 Scripts/tbmtools/src/tbmtools/cli.py rename Scripts/{share/standard_data/hand => tbmtools/src/tbmtools}/config.yaml (83%) create mode 100644 Scripts/tbmtools/src/tbmtools/data_package.py rename Scripts/tbmtools/{results => src/tbmtools/matrix}/__init__.py (100%) rename Scripts/tbmtools/{results/person_trips.py => src/tbmtools/matrix/person_trip.py} (51%) create mode 100644 Scripts/tbmtools/src/tbmtools/matrix/skim.py create mode 100644 Scripts/tbmtools/src/tbmtools/matrix/vehicle_trip.py create mode 100644 Scripts/tbmtools/src/tbmtools/network/__init__.py create mode 100644 Scripts/tbmtools/src/tbmtools/network/highway.py create mode 100644 Scripts/tbmtools/src/tbmtools/network/transit.py create mode 100644 Scripts/tbmtools/src/tbmtools/project.py rename Scripts/{share/standard_data/hand => tbmtools/src/tbmtools/templates}/data_user_guide_html.txt (100%) rename Scripts/{share/standard_data/hand => tbmtools/src/tbmtools/templates}/data_user_guide_md.txt (100%) create mode 100644 Scripts/tbmtools/src/tbmtools/utils.py diff --git a/Scripts/share/standard_data/run.cmd b/Scripts/share/standard_data/run.cmd deleted file mode 100644 index 62998b4..0000000 --- a/Scripts/share/standard_data/run.cmd +++ /dev/null @@ -1,6 +0,0 @@ -@echo off - -call ..\..\manage\env\activate_env.cmd emme -python src\standard_data.py - -pause \ No newline at end of file diff --git a/Scripts/share/standard_data/src/standard_data.py b/Scripts/share/standard_data/src/standard_data.py deleted file mode 100644 index edaf8c9..0000000 --- a/Scripts/share/standard_data/src/standard_data.py +++ /dev/null @@ -1,304 +0,0 @@ -"""Share standard model data. - -This script prepares standard datasets from a trip-based model run for -sharing. Arguments are read from configuration files. - -This script requires the `emme-plus` Python environment and the -`tbmtools` local Python package. - -This script creates the following non-public global variables to serve as path -anchors: - - * _src_dir - source code directory containing this script - * _proj_dir - trip-based model project directory read by this script - * _out_dir - output directory for files generated by this script -""" - -from pathlib import Path -import sys -import os -import logging -import shutil -import multiprocessing - -from tqdm import tqdm -import yaml -from jinja2 import Environment, FileSystemLoader -import markdown - -sys.path.append(str(Path(__file__).resolve().parents[3])) -import tbmtools.project as tbm -from tbmtools.results import vehicle_trips -from tbmtools.results import trip_roster -from tbmtools.results import person_trips -from tbmtools.results import skims -from tbmtools.results import transit_network -from tbmtools.results import highway_network -from tbmtools.results import sharing - -__all__ = ['load_config', 'export', 'compress', 'document'] - -_src_dir = Path(__file__).resolve().parent -_proj_dir = _src_dir.parents[3] -_out_dir = _src_dir.parent.joinpath('output') - - -def load_config(): - """Load values from configuration files. - - Reads configuration settings from Database/batch_file.yaml and - Scripts/share/standard_data/hand/config.yaml. - - Returns - ------- - dict - Configuration properties as keys and configuration property - values as values. - """ - with open(_proj_dir.joinpath('Database/batch_file.yaml')) as f: - batch_file_config = yaml.safe_load(f) - with open(_src_dir.parent.joinpath('hand/config.yaml')) as f: - config = yaml.safe_load(f) - config['scenario_code'] = batch_file_config['scenario_code'] - config['model_version'] = batch_file_config['model_version'] - return config - -def export(project_file_name, trip_roster_file_name, tg_data_file_name, scenario_code): - """Export standard data files from the trip-based model. - - Outputs text log file export.log. - - Parameters - ---------- - project_file_name : str - Name of source Emme project file (.emp). - trip_roster_file_name : str - Name of output trip roster file. - tg_data_file_name : str - Name of output trip generation data file. - scenario_code : int - Project scenario code. - - Returns - ------- - dict of str: Path - Output file name configuration properties as keys and paths to - locations of exported data files as values. - """ - # Make output directory. - _out_dir.mkdir(exist_ok=True) - # Set up log file. - logging.basicConfig(filename=_out_dir.joinpath('export.log'), - filemode='w', - format='%(asctime)s - %(levelname)s - %(message)s', - level=logging.INFO) - - print(f'Writing output to {_out_dir}') - - # Start Modeller in the Emme project. - modeller = tbm.connect(_proj_dir.joinpath(project_file_name)) - logging.info(f'Connected to {modeller.desktop.project_file_name()}') - # Display a progress bar while exporting data. - export_paths = dict() - with tqdm(desc='Exporting data', total=11) as pbar: - # Export vehicle trips from emmebank. - logging.info('Exporting vehicle trips') - trip_tables_path = vehicle_trips.export_matrices(_out_dir, modeller) - export_paths['trip_tables'] = trip_tables_path - pbar.update() - # Export skims from emmebank. - logging.info('Exporting skims') - skims.flag_transit_disconnects(modeller) - skim_matrices_path = skims.export_matrices(_out_dir, modeller) - export_paths['skim_matrices'] = skim_matrices_path - pbar.update() - # Export trip roster from parquet files. - logging.info('Exporting trip roster') - trip_roster_path = trip_roster.export(_proj_dir, _out_dir, trip_roster_file_name) - export_paths['trip_roster'] = trip_roster_path - pbar.update() - # Export auto person trips from trip roster and transit person trips - # from emmebank. - logging.info('Exporting person trips') - trip_tables_path, hov_trip_tables_path = person_trips.export_auto_matrices(_proj_dir, _out_dir, trip_roster_path) - export_paths['hov_trip_tables'] = hov_trip_tables_path - pbar.update() - person_trips.export_transit_matrices(_out_dir, modeller) - pbar.update() - # Export peak (AM peak) and off-peak (midday) transit network, - # itineraries, and attributes as Emme transaction files and - # shapefiles. - logging.info('Exporting transit network') - transit_networks_path = transit_network.export(_out_dir, - scenario=int(scenario_code), - format='transaction', - modeller=modeller, - emmebank=modeller.emmebank) - export_paths['transit_networks'] = transit_networks_path - pbar.update() - peak_transit_network_path, offpeak_transit_network_path = transit_network.export(_out_dir, - scenario=int(scenario_code), - format='shape', - modeller=modeller, - emmebank=modeller.emmebank) - export_paths['peak_transit_network'] = peak_transit_network_path - export_paths['offpeak_transit_network'] = offpeak_transit_network_path - pbar.update() - # Export each time of day highway network and attributes as Emme - # transaction files. - logging.info('Exporting highway network') - highway_networks_path = highway_network.export_by_tod(_out_dir, - scenario=int(scenario_code), - format='transaction', - modeller=modeller, - emmebank=modeller.emmebank) - export_paths['highway_networks'] = highway_networks_path - pbar.update() - # Export peak highway networks and attributes as shapefiles. - am_peak_highway_network_path, pm_peak_highway_network_path = highway_network.export_by_tod(_out_dir, - scenario=int(scenario_code), - format='shape', - modeller=modeller, - emmebank=modeller.emmebank, - period=[3, 7]) - export_paths['am_peak_highway_network'] = am_peak_highway_network_path - export_paths['pm_peak_highway_network'] = pm_peak_highway_network_path - pbar.update() - # Export daily highway network and attributes as Emme transaction - # files and shapefiles. - highway_network.export_daily(_out_dir, - scenario=int(scenario_code), - format='transaction', - modeller=modeller, - emmebank=modeller.emmebank) - pbar.update() - daily_highway_network_path = highway_network.export_daily(_out_dir, - scenario=int(scenario_code), - format='shape', - modeller=modeller, - emmebank=modeller.emmebank) - export_paths['daily_highway_network'] = daily_highway_network_path - pbar.update() - # Copy TG results to output directory. - logging.info('Copying TG results') - tg_dir = _proj_dir.joinpath('Database/tg') - tg_results_file = sorted(tg_dir.joinpath('data').glob('tg_results*.csv')) - if len(tg_results_file) > 1: - raise FileNotFoundError(f'Multiple TG results files exist in {tg_dir}.') - else: - file = tg_results_file[0] - shutil.copy(file, _out_dir) - file_copy = _out_dir.joinpath(file.name) - renamed_copy = file_copy.with_name(tg_data_file_name) - if renamed_copy.exists(): - os.remove(renamed_copy) - file_copy.rename(renamed_copy) - export_paths['tg_data'] = renamed_copy - # Copy productions and attractions to output subdirectory. - logging.info('Copying productions and attractions') - files = [tg_dir.joinpath('fortran/TRIP49_PA_OUT.TXT'), - tg_dir.joinpath('fortran/TRIP49_PA_WFH_OUT.TXT')] - pa_tables_path = _out_dir.joinpath('prods_attrs') - pa_tables_path.mkdir(exist_ok=True) - for file in files: - shutil.copy(file, pa_tables_path) - export_paths['pa_tables'] = pa_tables_path - logging.info('Finished.') - return export_paths - -def compress(zip_file_names, source_paths): - """ Compress standard data files into ZIP archives. - - Outputs text log file compress.log. - - Parameters - ---------- - zip_file_names : dict - Names of compressed data files. - source_paths : dict of str: Path - Output file name configuration properties as keys and paths to - locations of data files to compress as values. - """ - # Set up log file. - logging.basicConfig(filename=_out_dir.joinpath('compress.log'), - filemode='w', - format='%(asctime)s - %(levelname)s - %(message)s', - level=logging.INFO) - - print(f'Writing compressed output to {_out_dir}') - - mp_compress_args = list() - for placeholder, file_name in zip_file_names.items(): - mp_compress_args.append((file_name, source_paths[placeholder], _out_dir)) - # Compress model data for sharing. - logging.info('Compressing outputs') - with multiprocessing.Pool(processes=min(os.cpu_count(), 61)) as pool: - # Display a progress bar while processing the tasks. - for i in tqdm(iterable=pool.imap_unordered(func=sharing.mp_compress, - iterable=mp_compress_args), - total=len(mp_compress_args), - desc='Compressing outputs'): - pass - logging.info('Finished.') - -def document(context): - """ Render a HTML data user guide. - - Reads Markdown template with placeholder variables from - ../hand/data_user_guide_md.txt and HTML template from - ../hand/data_user_guide_html.txt. - - Parameters - ---------- - context : dict of str: str - Configuration properties as keys with text to render as values. - """ - # Load the Markdown template for data user guide. - environment = Environment(loader=FileSystemLoader(_src_dir.parent.joinpath('hand'))) - md_template = environment.get_template('data_user_guide_md.txt') - # Render the Markdown template. - md_file = _out_dir.joinpath('data_user_guide.md') - with open(md_file, mode='w', encoding='utf-8') as file: - file.write(md_template.render(context)) - # Read Markdown from file. - with open(md_file, encoding='utf-8') as file: - md = file.read() - # Convert Markdown to HTML. - html = markdown.markdown(text=md, extensions=['tables']) - # Load the HTML template for data user guide. - html_template_file = _src_dir.parent.joinpath('hand/data_user_guide_html.txt') - with open(html_template_file, encoding='utf-8') as file: - html_template = file.read() - # Render the HTML template. - with open(md_file.with_suffix('.html'), mode='w', encoding='utf-8') as file: - file.write(html_template.replace('{{content}}', html)) - -def main(): - # Load configuration settings. - config = load_config() - # Tag for file names. - tag = f'_{config["model_version"]}_{config["scenario_code"]}' - # Export data files. - tagged_trip_roster_file_name = config['trip_roster'] + tag - tagged_tg_data_file_name = config['tg_data'] + tag - export_paths = export(config['project_file_name'], - tagged_trip_roster_file_name + '.csv', - tagged_tg_data_file_name + '.csv', - config['scenario_code']) - export_paths['tod_transit_networks'] = Path(config['transit_directory'], - str(config['scenario_code'])) - export_paths['database'] = _proj_dir.joinpath('Database/emmebank') - export_paths['matrices'] = _proj_dir.joinpath('Database/emmemat') - # Compress data files. - zip_file_names = {} - for placeholder, file_name in config['compressed'].items(): - tagged_file_name = file_name + tag - zip_file_names[placeholder] = tagged_file_name + '.zip' - compress(zip_file_names, source_paths=export_paths) - # Render the data user guide. - config.update(config.pop('compressed')) - document(context=config) - - -if __name__ == '__main__': - sys.exit(main()) \ No newline at end of file diff --git a/Scripts/tbmtools/project.py b/Scripts/tbmtools/project.py deleted file mode 100644 index 6f3ea3d..0000000 --- a/Scripts/tbmtools/project.py +++ /dev/null @@ -1,44 +0,0 @@ -from pathlib import Path -import argparse -import inro.emme.desktop.app as _app -import inro.modeller as _m - -def emme_project_file(path): - """ - Validate an Emme project file path. - - Parameters: path : str - Path to an Emme project file. - - Returns: str - Path to Emme project file. - """ - ext = Path(path).suffix - if ext != '.emp': - raise argparse.ArgumentTypeError('File must have an emp extension') - if not Path(path).exists(): - raise argparse.ArgumentError('File does not exist') - return path - -def connect(path): - """ - Start an Emme Desktop session and connect Emme Modeller to the Emme - project. - - Parameters: path : str or path object - Emme project file path or path to directory - containing Emme project file. - - Returns: Modeller object - """ - if isinstance(path, str): - path = Path(path) - if path.is_file(): - empfile = path - elif path.is_dir(): - empfile = sorted(path.glob('**/*.emp'))[0] - app = _app.start_dedicated(visible=False, - user_initials='CMAP', - project=empfile) - modeller = _m.Modeller(app) - return modeller \ No newline at end of file diff --git a/Scripts/tbmtools/results/highway_network.py b/Scripts/tbmtools/results/highway_network.py deleted file mode 100644 index 1372679..0000000 --- a/Scripts/tbmtools/results/highway_network.py +++ /dev/null @@ -1,165 +0,0 @@ -from pathlib import Path -import csv - -def export_by_tod(outdir, scenario, format, modeller, emmebank, period=[1, 2, 3, 4, 5, 6, 7, 8]): - """ - Write highway network and attributes to Emme transaction files or - Esri shapefiles. - - Parameters: outdir : str or path object - Path to output directory. - - scenario : int - 3-digit scenario number. - - format : str - Should be 'transaction' for Emme transaction file - format or 'shape' for Esri shapefile format. - - modeller : inro.modeller.Modeller - - emmebank : inro.emme.database.emmebank.Emmebank - - period : int or list of int, default [1, 2, 3, 4, 5, 6, 7, 8] - Time of day period(s) to export. - - Returns: None - """ - if isinstance(outdir, str): - outdir = Path(outdir).resolve() - if not isinstance(period, list): - period = [period] - # Make output subdirectories. - hwydir = outdir.joinpath('networks', 'highway') - hwydir.mkdir(parents=True, exist_ok=True) - # Construct Modeller tools. - copy_scen = modeller.tool('inro.emme.data.scenario.copy_scenario') - create_attrib = modeller.tool('inro.emme.data.extra_attribute.create_extra_attribute') - export_basenet = modeller.tool('inro.emme.data.network.base.export_base_network') - net_calc = modeller.tool('inro.emme.network_calculation.network_calculator') - net_to_shp = modeller.tool('inro.emme.data.network.export_network_as_shapefile') - change_scen = modeller.tool('inro.emme.data.scenario.change_primary_scenario') - delete_scen = modeller.tool('inro.emme.data.scenario.delete_scenario') - - hwyshpdirs = list() - for p in period: - # Copy scenario. - copy_scen(from_scenario=emmebank.scenario(p), - scenario_id=99, - scenario_title=f'Copy of p{p}', - set_as_primary=True) - # Create extra attribute. - create_attrib(extra_attribute_type='LINK', - extra_attribute_name='@vadt', - extra_attribute_description=f'adt p{p}') - # Calculate vehicle volumes. - spec = {'type': 'NETWORK_CALCULATION', - 'result': '@vadt', - 'expression': '@avauv + @avh2v + @avh3v + @avbqv + @avlqv + (@avmqv/2) + (@avhqv/3)', - 'selections': {'link': 'all'}} - net_calc(specification=spec) - - if format == 'transaction': - # Write network transaction file. - export_basenet(export_file=hwydir.joinpath(f'network_p{p}.txt')) - # Write attribute transaction file. - spec = {'type': 'NETWORK_CALCULATION', - 'expression': '@speed + @width + @parkl + @toll + @sigic + @tipid + @ftime + @emcap + @avelw + @vadt + timau', - 'selections': {'link': 'all'}} - report = net_calc(specification=spec, - full_report=True) - with open(hwydir.joinpath(f'attribs_p{p}.txt'), 'w', newline='') as f: - txtwriter = csv.writer(f, delimiter=' ') - txtwriter.writerows(report['table']) - elif format == 'shape': - # Clear user data attributes. - spec1 = {'type': 'NETWORK_CALCULATION', - 'result': 'ul1', - 'expression': '0', - 'selections': {'link': 'all'}} - spec2 = spec1.copy() - spec2['result'] = 'ul2' - spec3 = spec1.copy() - spec3['result'] = 'ul3' - net_calc(specification=[spec1, spec2, spec3]) - # Store vehicle volumes. - spec = {'type': 'NETWORK_CALCULATION', - 'result': 'ul1', - 'expression': '@vadt', - 'selections': {'link': 'all'}} - net_calc(specification=spec) - # Remove extra attributes. - for xattrib in emmebank.scenario(99).extra_attributes(): - emmebank.scenario(99).delete_extra_attribute(xattrib.id) - # Set file tag. - if p == 3: - f_tag = 'ampk' - elif p == 7: - f_tag = 'pmpk' - else: - f_tag = 'p' + str(p) - # Write shapefile. - hwyshpdir = hwydir.joinpath(f'highway_{f_tag}-{scenario}') - net_to_shp(export_path=hwyshpdir) - hwyshpdirs.append(hwyshpdir) - change_scen(p) - delete_scen(emmebank.scenario(99)) - if format == 'transaction': - return hwydir - elif format == 'shape': - return tuple(hwyshpdirs) - -def export_daily(outdir, scenario, format, modeller, emmebank): - """ - Write highway network and attributes to Emme transaction files or - Esri shapefiles. - - Parameters: outdir : str or path object - Path to output directory. - - scenario : int - 3-digit scenario number. - - format : str - Should be 'transaction' for Emme transaction file - format or 'shape' for Esri shapefile format. - - modeller : inro.modeller.Modeller - - emmebank : inro.emme.database.emmebank.Emmebank - - Returns: None - """ - if isinstance(outdir, str): - outdir = Path(outdir).resolve() - # Make output subdirectories. - hwydir = outdir.joinpath('networks', 'highway') - hwydir.mkdir(parents=True, exist_ok=True) - # Construct Emme Modeller tools. - export_basenet = modeller.tool('inro.emme.data.network.base.export_base_network') - net_calc = modeller.tool('inro.emme.network_calculation.network_calculator') - net_to_shp = modeller.tool('inro.emme.data.network.export_network_as_shapefile') - change_scen = modeller.tool('inro.emme.data.scenario.change_primary_scenario') - - change_scen(f'{scenario}29') - - if format == 'transaction': - # Write network transaction file. - export_basenet(export_file=hwydir.joinpath(f'network_daily.txt')) - # Write attribute transaction file. - spec = {'type': 'NETWORK_CALCULATION', - 'expression': '@vadt', - 'selections': {'link': 'all'}} - report = net_calc(specification=spec, - full_report=True) - with open(hwydir.joinpath(f'attribs_daily.txt'), 'w', newline='') as f: - txtwriter = csv.writer(f, delimiter=' ') - txtwriter.writerows(report['table']) - elif format == 'shape': - # Write shapefile. - hwyshpdir = hwydir.joinpath(f'highway-{scenario}') - net_to_shp(export_path=hwyshpdir) - if format == 'transaction': - return hwydir - elif format == 'shape': - return hwyshpdir \ No newline at end of file diff --git a/Scripts/tbmtools/results/sharing.py b/Scripts/tbmtools/results/sharing.py deleted file mode 100644 index 0d37493..0000000 --- a/Scripts/tbmtools/results/sharing.py +++ /dev/null @@ -1,42 +0,0 @@ -from pathlib import Path -from zipfile import ZipFile, ZIP_DEFLATED - - -def mp_compress(args): - """Wrap compress for multiprocessing. - - Unpack arguments, then call compress using the unpacked arguments. - - Parameters - ---------- - args : iterable object - Element of the iterable (iterable of iterables) passed to the - process pool using a method that does not unpack iterables. - """ - # Unpack arguments. - out_file_name, source_path, out_dir = args - # Compress. - compress(out_file_name, source_path, out_dir) - -def compress(out_file_name, source_path, out_dir): - """Compress file(s) into a ZIP file. - - Parameters - ---------- - out_file_name : str or path object - Path to destination ZIP file. - source_path : str or path object - Path to a file or a directory with files to be - compressed. - """ - # Handle arguments. - if isinstance(source_path, str): - source_path = Path(source_path).resolve() - # Compress content. - with ZipFile(out_dir.joinpath(out_file_name), mode='w', compression=ZIP_DEFLATED, compresslevel=9) as zip: - if source_path.is_file(): - zip.write(source_path, arcname=source_path.name) - elif source_path.is_dir(): - for container in source_path.iterdir(): - if container.is_file(): - zip.write(container, arcname=container.name) \ No newline at end of file diff --git a/Scripts/tbmtools/results/skims.py b/Scripts/tbmtools/results/skims.py deleted file mode 100644 index b101ac9..0000000 --- a/Scripts/tbmtools/results/skims.py +++ /dev/null @@ -1,70 +0,0 @@ -from copy import deepcopy -from pathlib import Path - -SKIM_MATRIX_IDS = {'transit': {'peak': {'in-vehicle minutes': 'mf822', - 'walk transfer minutes': 'mf823', - 'wait time': 'mf838', - 'priority mode': 'mf830', - 'average fare': 'mf828', - 'station zone': 'mf837'}, - 'off-peak': {'in-vehicle minutes': 'mf922', - 'walk transfer minutes': 'mf923', - 'wait time': 'mf938', - 'priority mode': 'mf930', - 'average fare': 'mf928', - 'station zone': 'mf937'}}, - 'highway': {'am': {'time': 'mf44', - 'distance': 'mf45'}, - 'md': {'time': 'mf46', - 'distance': 'mf47'}}} - -def flag_transit_disconnects(modeller): - """ - Flag peak and off-peak transit skim O-Ds that are not connected by - transit. Use a flag value of 9999. - """ - compute_matrices = modeller.tool('inro.emme.matrix_calculation.matrix_calculator') - transit_skims = SKIM_MATRIX_IDS['transit'] - for transitnet, transitnet_skims in transit_skims.items(): - # Flag O-Ds with negative or impossibly large values for in-vehicle minutes. - spec = {'type': 'MATRIX_CALCULATION', - 'expression': '9999', - 'result': transitnet_skims['in-vehicle minutes'], - 'constraint': {'by_value': {'od_values': transitnet_skims['in-vehicle minutes'], - 'interval_min': 0, - 'interval_max': 9999, - 'condition': 'EXCLUDE'}}} - compute_matrices(spec) - # Apply flag to other transit skim matrices. - spec['constraint']['by_value']['interval_min'] = 9999 - spec['constraint']['by_value']['condition'] = 'INCLUDE' - specs = [] - for desc, mtx_id in transitnet_skims.items(): - if desc not in ['in-vehicle minutes', 'station zone']: - spec['result'] = mtx_id - specs.append(deepcopy(spec)) - compute_matrices(specs) - -def export_matrices(outdir, modeller): - """ - Export peak and off-peak transit skims and highway time and distance - skim matrices. - """ - # Make output subdirectory. - skimdir = outdir.joinpath('skims') - skimdir.mkdir(exist_ok=True) - # Construct Modeller tool. - export_matrix_data = modeller.tool('inro.emme.data.matrix.export_matrix_to_csv') - # Export peak transit skims. - export_matrix_data(matrices=[i for i in list(SKIM_MATRIX_IDS['transit']['peak'].values())], - export_path=skimdir) - # Export off-peak transit skims. - export_matrix_data(matrices=[i for i in list(SKIM_MATRIX_IDS['transit']['off-peak'].values())], - export_path=skimdir) - # Export am highway skims. - export_matrix_data(matrices=[i for i in list(SKIM_MATRIX_IDS['highway']['am'].values())], - export_path=skimdir) - # Export md highway skims. - export_matrix_data(matrices=[i for i in list(SKIM_MATRIX_IDS['highway']['md'].values())], - export_path=skimdir) - return skimdir \ No newline at end of file diff --git a/Scripts/tbmtools/results/transit_network.py b/Scripts/tbmtools/results/transit_network.py deleted file mode 100644 index 2b32d86..0000000 --- a/Scripts/tbmtools/results/transit_network.py +++ /dev/null @@ -1,79 +0,0 @@ -from pathlib import Path -import csv - -def export(outdir, scenario, format, modeller, emmebank, period=[0, 5]): - """ - Write transit network, itineraries, and attributes to Emme - transaction files or Esri shapefiles. - - Parameters: outdir : str or path object - Path to output directory. - - scenario : int - 3-digit scenario number. - - format : str - Should be 'transaction' for Emme transaction file - format or 'shape' for Esri shapefile format. - - modeller : inro.modeller.Modeller - - emmebank : inro.emme.database.emmebank.Emmebank - - period : int or list of int, default [0, 5] - Time of day period(s) to export. - - Returns: None - """ - if isinstance(outdir, str): - outdir = Path(outdir).resolve() - if not isinstance(period, list): - period = [period] - # Make output subdirectories. - transitdir = outdir.joinpath('networks', 'transit') - transitdir.mkdir(parents=True, exist_ok=True) - # Construct Modeller tools. - export_basenet = modeller.tool('inro.emme.data.network.base.export_base_network') - export_lines = modeller.tool('inro.emme.data.network.transit.export_transit_lines') - net_calc = modeller.tool('inro.emme.network_calculation.network_calculator') - net_to_shp = modeller.tool('inro.emme.data.network.export_network_as_shapefile') - - transitshpdirs = list() - for p in period: - # Set scenario. - s = emmebank.scenario(scenario + p) - # Set file tag. - if p == 0: - f_tag = 'pk' - elif 0 in period and p == 5: - f_tag = 'op' - else: - f_tag = str(p) - if format == 'transaction': - # Write network transaction file. - export_basenet(export_file=transitdir.joinpath(f'network_{f_tag}.txt'), - scenario=s) - # Write itinerary transaction file. - export_lines(export_file=transitdir.joinpath(f'itins_{f_tag}.txt'), - scenario=s) - # Write attribute transaction file. - spec = {'type': 'NETWORK_CALCULATION', - 'expression': '@ltime + @hwytm + @zfare_link', - 'selections': {'link': 'all', - 'transit_line': 'all'}} - report = net_calc(specification=spec, - scenario=s, - full_report=True) - with open(transitdir.joinpath(f'attribs_{f_tag}.txt'), 'w', newline='') as f: - txtwriter = csv.writer(f, delimiter=' ') - txtwriter.writerows(report['table']) - elif format == 'shape': - # Write shapefiles. - transitshpdir = transitdir.joinpath(f'transit_{f_tag}-{scenario}') - net_to_shp(export_path=transitshpdir, - scenario=s) - transitshpdirs.append(transitshpdir) - if format == 'transaction': - return transitdir - elif format == 'shape': - return tuple(transitshpdirs) \ No newline at end of file diff --git a/Scripts/tbmtools/results/trip_roster.py b/Scripts/tbmtools/results/trip_roster.py deleted file mode 100644 index 8cd1439..0000000 --- a/Scripts/tbmtools/results/trip_roster.py +++ /dev/null @@ -1,33 +0,0 @@ -from pathlib import Path -import pandas as pd - -def export(projdir, outdir, out_filename): - """ - Assemble the trip roster from parquet job files and export it to a - CSV. - - Parameters: projdir : str or path object - Path to Emme project directory. - - outdir : str or path object - Path to output directory. - - Returns: path object - Path to output CSV. - """ - if isinstance(projdir, str): - projdir = Path(projdir).resolve() - if isinstance(outdir, str): - outdir = Path(outdir).resolve() - pqdir = Path(projdir).joinpath('Database/cache/choice_simulator_trips_out') - hh_types = ['typical', 'wfh', 'deadhead'] - hh_type_trip_rosters = {} - for t in hh_types: - jobfiles = sorted(pqdir.glob(f'*{t}.pq')) - dfs = [pd.read_parquet(f).reset_index().set_index('purpose') for f in jobfiles] - hh_type_trip_rosters.update({t: pd.concat(dfs)}) - complete_trip_roster = pd.concat(hh_type_trip_rosters, names=['hh_type'], sort=False) - outdir.mkdir(parents=True, exist_ok=True) - trip_roster_path = outdir.joinpath(out_filename) - complete_trip_roster.to_csv(trip_roster_path) - return trip_roster_path \ No newline at end of file diff --git a/Scripts/tbmtools/results/vehicle_trips.py b/Scripts/tbmtools/results/vehicle_trips.py deleted file mode 100644 index 9d3596f..0000000 --- a/Scripts/tbmtools/results/vehicle_trips.py +++ /dev/null @@ -1,22 +0,0 @@ -from pathlib import Path - -def export_matrices(outdir, modeller): - """ - Export vehicle trip matrices from Emme. - """ - # Make output subdirectories. - tripdir = outdir.joinpath('trips') - tripdir.mkdir(exist_ok=True) - # Construct Modeller tool. - export_matrix_data = modeller.tool('inro.emme.data.matrix.export_matrix_to_csv') - # Export vehicle trip matrices. - matrix_names = {'mf4': 'b_truck', - 'mf5': 'l_truck', - 'mf6': 'm_truck', - 'mf7': 'h_truck', - 'mf8': 'poe_auto', - 'mf9': 'poe_truck', - 'mf10': 'airport'} - export_matrix_data(matrices=list(matrix_names.keys()), - export_path=tripdir) - return tripdir \ No newline at end of file diff --git a/Scripts/tbmtools/__init__.py b/Scripts/tbmtools/src/tbmtools/__init__.py similarity index 100% rename from Scripts/tbmtools/__init__.py rename to Scripts/tbmtools/src/tbmtools/__init__.py diff --git a/Scripts/tbmtools/src/tbmtools/__main__.py b/Scripts/tbmtools/src/tbmtools/__main__.py new file mode 100644 index 0000000..bd67961 --- /dev/null +++ b/Scripts/tbmtools/src/tbmtools/__main__.py @@ -0,0 +1,4 @@ +import sys +from .cli import main + +sys.exit(main()) \ No newline at end of file diff --git a/Scripts/tbmtools/src/tbmtools/cli.py b/Scripts/tbmtools/src/tbmtools/cli.py new file mode 100644 index 0000000..aee4c5f --- /dev/null +++ b/Scripts/tbmtools/src/tbmtools/cli.py @@ -0,0 +1,13 @@ +import argparse + +from .data_package import run_pack_data + +def main(): + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + # Define `pack` subcommand. + pack_parser = subparsers.add_parser('pack') + pack_parser.add_argument('--series', help='path to a directory containing a series of model runs') + pack_parser.set_defaults(func=run_pack_data) + args = parser.parse_args() + args.func(args) \ No newline at end of file diff --git a/Scripts/share/standard_data/hand/config.yaml b/Scripts/tbmtools/src/tbmtools/config.yaml similarity index 83% rename from Scripts/share/standard_data/hand/config.yaml rename to Scripts/tbmtools/src/tbmtools/config.yaml index 1c07c41..6166b06 100644 --- a/Scripts/share/standard_data/hand/config.yaml +++ b/Scripts/tbmtools/src/tbmtools/config.yaml @@ -1,9 +1,7 @@ -# Name of EMME project file. -project_file_name: c24q2_100.emp # Path to transit transaction file directory. -transit_directory: C:\Testing\c24q2\transit +transit_directory: E:\nrf\c25q2\transit # Date of Board approval. -approval_date: June 2024 +approval_date: January 2025 # Names of output files. trip_roster: trip_roster tg_data: tg_results diff --git a/Scripts/tbmtools/src/tbmtools/data_package.py b/Scripts/tbmtools/src/tbmtools/data_package.py new file mode 100644 index 0000000..0e2eec5 --- /dev/null +++ b/Scripts/tbmtools/src/tbmtools/data_package.py @@ -0,0 +1,297 @@ +"""Package model output data products for distribution. + +This module exports model data from Emme projects, assembles package +directories, and renders documentation for the packaged results. +""" + +import logging +import os +from pathlib import Path +import shutil + +from jinja2 import Environment, FileSystemLoader +import markdown +from tqdm.contrib.concurrent import process_map + +from . import project as tbm +from .utils import compress, copy_prods_attrs, load_config +from .matrix import vehicle_trip, skim, person_trip +from .network import highway, transit + + +def compress_data(data_dirs, out_dir, config, tag): + """Compress exported data directories into ZIP archives. + + Parameters + ---------- + data_dirs : dict + Mapping of data product names to directories that should be compressed. + out_dir : pathlib.Path + Directory where the ZIP archives will be written. + config : dict + Configuration mapping containing the ``compressed`` section that + defines the output archive names. + tag : str + Tag appended to each archive filename. + + Returns + ------- + None + """ + logging.info('Compressing data files') + zip_file_names = [f'{label}_{tag}.zip' for label in config['compressed'].values()] + src_paths = [data_dirs[data_product] for data_product in config['compressed'].keys()] + process_map(compress, zip_file_names, src_paths, [out_dir] * len(zip_file_names), desc=f'Compressing {tag} data files') + + +def copy_data(src_dir, target_dir, config, tag, data_paths): + """Copy packaged data files into the output directory. + + Parameters + ---------- + src_dir : pathlib.Path + Source project directory containing the TG results data. + target_dir : pathlib.Path + Destination directory for the packaged data files. + config : dict + Configuration mapping used to name the copied TG results file. + tag : str + Tag appended to the copied TG results filename. + data_paths : dict + Mapping of exported artifact names to their paths, including the trip + roster. + + Returns + ------- + None + + Raises + ------ + FileNotFoundError + If more than one TG results file is found in the source directory. + """ + # Copy TG results. + logging.info('Copying TG results') + tg_dir = src_dir.joinpath('Database', 'tg') + tg_results_file = sorted(tg_dir.joinpath('data').glob('tg_results*.csv')) + if len(tg_results_file) > 1: + raise FileNotFoundError(f"Multiple TG results files exist in {tg_dir.joinpath('data')}.") + else: + file = tg_results_file[0] + shutil.copy(file, target_dir) + # Rename TG results. + file_copy = target_dir.joinpath(file.name) + new_path = file_copy.with_name(f'{config["tg_data"]}_{tag}.csv') + if new_path.exists(): + os.remove(new_path) + file_copy.rename(new_path) + data_paths['tg_data'] = new_path + # Copy trip roster to package directory. + shutil.copy(data_paths['trip_roster'], target_dir) + + +def export_data(proj_dir, out_dir, tag, config, modeller): + """Export model data products into a staging directory. + + Parameters + ---------- + proj_dir : pathlib.Path + Project directory containing the Emme model files and supporting data. + out_dir : pathlib.Path + Directory where exported data products will be written. + tag : str + Tag used to distinguish packaged output files. + config : dict + Configuration mapping with scenario and output naming settings. + modeller : inro.modeller.Modeller + Modeller instance used to access Emme tools for exporting data. + + Returns + ------- + dict + Mapping of exported data product names to their output paths. + """ + # Display a progress bar while exporting data. + export_paths = dict() + # Export data files. + trip_tables_path = vehicle_trip.export_auto(out_dir, + config["scenario_code"], + modeller) + export_paths['trip_tables'] = trip_tables_path + skim_matrices_path = skim.export_transit(out_dir, + config["scenario_code"], + modeller) + skim.export_highway(out_dir, + config["scenario_code"], + modeller) + export_paths['skim_matrices'] = skim_matrices_path + trip_roster_name = f"{config['trip_roster']}_{tag}.csv" + trip_roster_path = person_trip.export_trip_roster(proj_dir, + out_dir, + out_filename=trip_roster_name) + export_paths['trip_roster'] = trip_roster_path + trip_tables_path, hov_trip_tables_path = person_trip.export_auto(proj_dir, + out_dir, + trip_roster_path) + person_trip.export_transit(proj_dir, + out_dir, + config["scenario_code"], + modeller) + export_paths['hov_trip_tables'] = hov_trip_tables_path + transit_networks_path, peak_transit_network_path, offpeak_transit_network_path = transit.export_all(out_dir, + config["scenario_code"], + modeller) + export_paths['transit_networks'] = transit_networks_path + export_paths['peak_transit_network'] = peak_transit_network_path + export_paths['offpeak_transit_network'] = offpeak_transit_network_path + highway_networks_path, am_peak_highway_network_path, pm_peak_highway_network_path, daily_highway_network_path = highway.export_all(out_dir=out_dir, + scenario_code=config["scenario_code"], + modeller=modeller) + export_paths['highway_networks'] = highway_networks_path + export_paths['am_peak_highway_network'] = am_peak_highway_network_path + export_paths['pm_peak_highway_network'] = pm_peak_highway_network_path + export_paths['daily_highway_network'] = daily_highway_network_path + export_paths['tod_transit_networks'] = Path(config['transit_directory'], str(config["scenario_code"])) + export_paths['database'] = proj_dir.joinpath('Database/emmebank') + export_paths['matrices'] = proj_dir.joinpath('Database/emmemat') + export_paths['pa_tables'] = copy_prods_attrs(proj_dir, out_dir) + + return export_paths + + +def pack_data(proj_file, out_dir, config_file=Path(__file__).parent.joinpath('config.yaml'), templates=Path(__file__).parent.joinpath('templates')): + """Package model data products into a compressed archive. + + Parameters + ---------- + proj_file : pathlib.Path + Path to the Emme project file to package. + out_dir : pathlib.Path + Root output directory where the packaged data will be written. + config_file : pathlib.Path, optional + Path to the YAML configuration file used to define packaging settings. + templates : pathlib.Path, optional + Directory containing the markdown and HTML templates used to render the + data user guide. + + Returns + ------- + pathlib.Path + Path to the generated ZIP archive for the packaged data. + """ + # Load configuration settings. + proj_dir = proj_file.parent + config = load_config(config_file, proj_dir) + # Make output subdirectory. + out_subdir = out_dir.joinpath(config["model_version"], str(config["scenario_code"])) + out_subdir.mkdir(parents=True, exist_ok=True) + # Set up log file. + logging.basicConfig(filename=out_subdir.joinpath('pack.log'), + filemode='w', + format='%(asctime)s - %(levelname)s - %(message)s', + level=logging.INFO) + # Connect to EMME Modeller. + modeller = tbm.connect(proj_file) + logging.info(f'Connected to {modeller.desktop.project_file_name()}') + # Set tag for file names. + tag = f'{config["model_version"]}_{config["scenario_code"]}' + logging.info(f'Writing output to {out_subdir}') + # Export data. + export_paths = export_data(proj_dir, + out_subdir, + tag, + config, + modeller) + # Compress exported data. + pkg_dir = out_subdir.joinpath(f'{config["model_version"]}_{config["scenario_code"]}') + pkg_dir.mkdir(exist_ok=True) + compress_data(data_dirs=export_paths, + out_dir=pkg_dir, + config=config, + tag=tag) + # Copy individual data files. + copy_data(src_dir=proj_dir, + target_dir=pkg_dir, + config=config, + tag=tag, + data_paths=export_paths) + # Render the data user guide. + config.update(config.pop('compressed')) + data_user_guide_file = render_data_user_guide(templates, out_subdir, context=config) + # Copy data user guide to package directory. + shutil.copy(data_user_guide_file, pkg_dir) + # Compress package directory. + pkg = compress(f'{pkg_dir.name}.zip', pkg_dir, pkg_dir.parent) + logging.info('Finished.') + + return pkg + + +def render_data_user_guide(templates, out_dir, context): + """Render an HTML data user guide from Markdown and HTML templates. + + Parameters + ---------- + templates : pathlib.Path + Directory containing the template files used to build the guide. + out_dir : pathlib.Path + Directory where the generated Markdown and HTML files will be written. + context : dict + Mapping of configuration keys to values used to render the templates. + + Returns + ------- + pathlib.Path + Path to the rendered HTML data user guide file. + """ + # Load the Markdown template for data user guide. + environment = Environment(loader=FileSystemLoader(templates)) + md_template = environment.get_template('data_user_guide_md.txt') + # Render the Markdown template. + md_file = out_dir.joinpath('data_user_guide.md') + with open(md_file, mode='w', encoding='utf-8') as file: + file.write(md_template.render(context)) + # Read Markdown from file. + with open(md_file, encoding='utf-8') as file: + md = file.read() + # Convert Markdown to HTML. + html = markdown.markdown(text=md, extensions=['tables']) + # Load the HTML template for data user guide. + html_template_file = Path(__file__).parent.joinpath('templates', 'data_user_guide_html.txt') + with open(html_template_file, encoding='utf-8') as file: + html_template = file.read() + # Render the HTML template. + html_file = md_file.with_suffix('.html') + with open(html_file, mode='w', encoding='utf-8') as file: + file.write(html_template.replace('{{content}}', html)) + + return html_file + + +def run_pack_data(args): + """Package data for one or more Emme projects from CLI arguments. + + Parameters + ---------- + args : argparse.Namespace + Parsed command-line arguments. If ``args.series`` is provided, it should + point to a directory containing one or more Emme project files. + + Returns + ------- + None + """ + # Search for Emme project files. + if args.series: + proj_files = sorted(Path(args.series).rglob('*.emp')) + out_dir = Path(args.series).joinpath('packaged_data') + process_map(pack_data, proj_files, [out_dir] * len(proj_files), desc=f'Packing data into {out_dir}') + else: + proj_dir = Path(__file__).parents[4] + proj_files = sorted(proj_dir.glob('*.emp')) + if len(proj_files) > 1: + raise FileNotFoundError(f"Multiple EMME project files exist in {proj_dir}.") + else: + proj_file = proj_files[0] + out_dir = proj_file.parent.joinpath('packaged_data') + pack_data(proj_file, out_dir) diff --git a/Scripts/tbmtools/results/__init__.py b/Scripts/tbmtools/src/tbmtools/matrix/__init__.py similarity index 100% rename from Scripts/tbmtools/results/__init__.py rename to Scripts/tbmtools/src/tbmtools/matrix/__init__.py diff --git a/Scripts/tbmtools/results/person_trips.py b/Scripts/tbmtools/src/tbmtools/matrix/person_trip.py similarity index 51% rename from Scripts/tbmtools/results/person_trips.py rename to Scripts/tbmtools/src/tbmtools/matrix/person_trip.py index 76c634b..32081eb 100644 --- a/Scripts/tbmtools/results/person_trips.py +++ b/Scripts/tbmtools/src/tbmtools/matrix/person_trip.py @@ -1,68 +1,53 @@ -from pathlib import Path +"""Export person-trip matrices from trip rosters and Emme model outputs. + +This module provides utilities for assembling auto and transit person-trip +matrices, writing them to CSV files, and building a trip roster from +choice-simulator parquet outputs. It is used to convert trip-level data into +matrix-ready files for downstream analysis and reporting. +""" + +import logging import multiprocessing import os +from pathlib import Path + import pandas as pd -def export_matrix_from_roster(name, spec, outdir, roster, report): - """ - """ - # Create matrix indices. - max_taz = 3649 - z_range = range(1, max_taz + 1) - arrays = [[row for row in z_range for col in z_range], - [col for row in z_range for col in z_range]] - pa_index = pd.MultiIndex.from_arrays(arrays, names=['p_zone', 'a_zone']) - od_index = pd.MultiIndex.from_arrays(arrays, names=['o_zone', 'd_zone']) - # Define function to calculate production zone. - p_zone_calc = lambda x: x['o_zone'] if x['a_zone'] == x['d_zone'] else x['d_zone'] - # Select trips from roster. - select_trips = roster.loc[roster['purpose'].isin(spec['purpose']) & - roster['mode'].isin(spec['mode'])].copy() - with open(report, 'a') as f: - print(f"{name}: {select_trips['trips'].sum()}", file=f) - # Sum selected trips by index zones. - select_trips['p_zone'] = select_trips.apply(p_zone_calc, axis=1) - if spec['format'] == 'PA': - mtx_index = pa_index - elif spec['format'] == 'OD': - mtx_index = od_index - p = mtx_index.names[0] - q = mtx_index.names[1] - mtx_index_sum = select_trips[[p, q, 'trips']].groupby([p, q]).sum() - # Format as matrix and export to CSV. - mtx_header = f"{p[0]}/{q[0]}/{spec['description']}" - pd.DataFrame(index=mtx_index).merge(mtx_index_sum, how='left', on=[p, q])\ - .fillna(0)\ - .reset_index()\ - .rename(columns={p: mtx_header})\ - .pivot(index=mtx_header, columns=q, values='trips')\ - .to_csv(outdir.joinpath(f'{name}.csv')) -def export_auto_matrices(projdir, outdir, trip_roster_path): +def export_auto(proj_dir, out_dir, trip_roster_path): """ - Generate daily auto person trip matrices from trip roster and export to CSV. + Create auto person trip matrices from a trip roster and export the + matrices to CSVs. - Parameters: projdir : str or path object - Path to Emme project directory. + Parameters + ---------- + proj_dir : str or pathlib.Path + Path to the Emme project directory. If a string is provided, it is + converted to a resolved pathlib.Path. + out_dir : str or pathlib.Path + Root output directory where `trips/` and `trips/hov_trips/` are created. + trip_roster_path : str or pathlib.Path + Path to the CSV trip roster file used to build auto person matrices. - outdir : str or path object - Path to output directory. + Returns + ------- + tuple[pathlib.Path, pathlib.Path] + A tuple containing the auto trip export directory and the HOV trip + subdirectory. - Returns: path object - Path to output OMX. + Notes + ----- + The function reads the roster with pandas, generates auto person trip + matrices for multiple trip purposes and modes using `matrix_from_roster`, + writes totals to `auto_person_trip_totals.txt`, and moves HOV matrices + into the `hov_trips/` subdirectory. """ - # Handle arguments. - if isinstance(projdir, str): - projdir = Path(projdir).resolve() - if isinstance(outdir, str): - outdir = Path(outdir).resolve() - # Make output subdirectories. - tripdir = outdir.joinpath('trips') - tripdir.mkdir(exist_ok=True) - hovtripdir = tripdir.joinpath('hov_trips') - hovtripdir.mkdir(exist_ok=True) - # Read trip roster. - roster = pd.read_csv(trip_roster_path) + logging.info('Exporting auto person trips') + # Normalize arguments. + if isinstance(proj_dir, str): + proj_dir = Path(proj_dir).resolve() + if isinstance(out_dir, str): + out_dir = Path(out_dir).resolve() # Specify matrices. mtx_specs = {'hbwL_auto': {'description': 'total daily low-income hbw auto person trips', 'purpose': ['HBWL'], @@ -140,31 +125,71 @@ def export_auto_matrices(projdir, outdir, trip_roster_path): 'purpose': ['NHB'], 'mode': [3], 'format': 'OD'}} - # Export specified matrices. - report = outdir.joinpath('auto_person_trip_totals.txt') + logging.info('Exporting auto person trips') + report = out_dir.joinpath('auto_person_trip_totals.txt') if report.exists(): report.unlink() + # Read trip roster. + roster = pd.read_csv(trip_roster_path) + # Make output subdirectory. + trip_dir = out_dir.joinpath('trips') + trip_dir.mkdir(exist_ok=True) + # Export specified matrices. args = [] for name, spec in mtx_specs.items(): - args.append((name, spec, tripdir, roster, report)) + args.append((name, spec, trip_dir, roster, report)) with multiprocessing.Pool(processes=min(os.cpu_count(), 61)) as pool: - pool.starmap(export_matrix_from_roster, args) + pool.starmap(matrix_from_roster, args) # Move HOV matrices to output directory. - for p in sorted(tripdir.glob('*.csv')): + hovtrip_dir = trip_dir.joinpath('hov_trips') + hovtrip_dir.mkdir(exist_ok=True) + for p in sorted(trip_dir.glob('*.csv')): if p.stem in ['hbw_sov', 'hbw_hov2', 'hbw_hov3', 'hbs_sov', 'hbs_hov2', 'hbs_hov3', 'hbo_sov', 'hbo_hov2', 'hbo_hov3', 'nhb_sov', 'nhb_hov2', 'nhb_hov3']: - p.replace(hovtripdir.joinpath(p.name)) - return (tripdir, hovtripdir) + p.replace(hovtrip_dir.joinpath(p.name)) + + return (trip_dir, hovtrip_dir) -def export_transit_matrices(outdir, modeller): + +def export_transit(proj_dir, out_dir, scenario_code, modeller): """ - Export transit person trip matrices from Emme. + Export daily transit person trip matrices from an emmebank to CSVs. + + Parameters + ---------- + proj_dir : str or pathlib.Path + Path to the Emme project directory. If a string is provided, it is + converted to a resolved ``pathlib.Path``. + out_dir : str or pathlib.Path + Root output directory where the ``trips/`` subdirectory is created. + scenario_code : int + Scenario year code used to select the daily Emme scenario + (``{scenario_code}29``). + modeller : inro.modeller.Modeller + Modeller instance used to construct the matrix export tool. + + + Returns + ------- + pathlib.Path + Path to the directory containing exported transit trip CSV files. + + Notes + ----- + The function exports transit person trip matrices for fixed matrix IDs + from the daily scenario and writes them into the ``trips/`` directory. """ - # Make output subdirectories. - tripdir = outdir.joinpath('trips') - tripdir.mkdir(exist_ok=True) + logging.info('Exporting transit person trips') + # Normalize arguments. + if isinstance(proj_dir, str): + proj_dir = Path(proj_dir).resolve() + if isinstance(out_dir, str): + out_dir = Path(out_dir).resolve() + # Make output subdirectory. + trip_dir = out_dir.joinpath('trips') + trip_dir.mkdir(exist_ok=True) # Construct Modeller tools. export_matrix_data = modeller.tool('inro.emme.data.matrix.export_matrix_to_csv') matrix_names = {'mf38': 'visit_transit', @@ -175,5 +200,106 @@ def export_transit_matrices(outdir, modeller): 'mf43': 'nhb_transit'} # Export transit trips. export_matrix_data(matrices=[i for i in list(matrix_names.keys())], - export_path=tripdir) - return tripdir \ No newline at end of file + export_path=trip_dir, + scenario=modeller.emmebank.scenario(str(scenario_code) + '29')) + + return trip_dir + + +def export_trip_roster(proj_dir, out_dir, out_filename): + """ + Assemble a trip roster from parquet job files and export it as a CSV. + + Parameters + ---------- + proj_dir : str or pathlib.Path + Path to the Emme project directory containing + `Database/cache/choice_simulator_trips_out`. + out_dir : str or pathlib.Path + Output directory where the exported CSV file will be written. + out_filename : str + Name of the exported CSV file. + + Returns + ------- + pathlib.Path + Path to the exported trip roster CSV file. + """ + logging.info('Exporting trip roster') + if isinstance(proj_dir, str): + proj_dir = Path(proj_dir).resolve() + if isinstance(out_dir, str): + out_dir = Path(out_dir).resolve() + pq_dir = Path(proj_dir).joinpath('Database/cache/choice_simulator_trips_out') + hh_types = ['typical', 'wfh', 'deadhead'] + hh_type_trip_rosters = {} + for t in hh_types: + jobfiles = [f for f in pq_dir.glob(f'*{t}.pq') if '_util' not in f.name] + dfs = [pd.read_parquet(f).reset_index().set_index('purpose') for f in jobfiles] + hh_type_trip_rosters.update({t: pd.concat(dfs)}) + complete_trip_roster = pd.concat(hh_type_trip_rosters, names=['hh_type'], sort=False) + out_dir.mkdir(exist_ok=True) + trip_roster_path = out_dir.joinpath(out_filename) + complete_trip_roster.to_csv(trip_roster_path) + + return trip_roster_path + + +def matrix_from_roster(name, spec, out_dir, roster, report): + """ + Export a matrix from a trip roster using a specification. + + Parameters + ---------- + name : str + Base file name to use for the exported CSV (e.g. "hbw_auto"). + spec : dict + Specification dictionary containing: + - 'description' : str + - 'purpose' : list[str] + - 'mode' : iterable[int] + - 'format' : {'PA', 'OD'} + out_dir : str or pathlib.Path + Directory where the output CSV file will be written. + roster : pandas.DataFrame + Trip roster containing columns including 'purpose', 'mode', 'o_zone', + 'd_zone', and 'trips'. + report : str or pathlib.Path + Path to a text report file where trip totals are appended. + + Returns + ------- + None + """ + # Create matrix indices. + max_taz = 3649 + z_range = range(1, max_taz + 1) + arrays = [[row for row in z_range for col in z_range], + [col for row in z_range for col in z_range]] + pa_index = pd.MultiIndex.from_arrays(arrays, names=['p_zone', 'a_zone']) + od_index = pd.MultiIndex.from_arrays(arrays, names=['o_zone', 'd_zone']) + # Define function to calculate production zone. + p_zone_calc = lambda x: x['o_zone'] if x['a_zone'] == x['d_zone'] else x['d_zone'] + # Select trips from roster. + select_trips = roster.loc[roster['purpose'].isin(spec['purpose']) & + roster['mode'].isin(spec['mode'])].copy() + with open(report, 'a') as f: + print(f"{name}: {select_trips['trips'].sum()}", file=f) + # Sum selected trips by index zones. + select_trips['p_zone'] = select_trips.apply(p_zone_calc, axis=1) + if spec['format'] == 'PA': + mtx_index = pa_index + elif spec['format'] == 'OD': + mtx_index = od_index + p = mtx_index.names[0] + q = mtx_index.names[1] + mtx_index_sum = select_trips[[p, q, 'trips']].groupby([p, q]).sum() + # Format as matrix and export to CSV. + mtx_header = f"{p[0]}/{q[0]}/{spec['description']}" + pd.DataFrame(index=mtx_index).merge(mtx_index_sum, how='left', on=[p, q])\ + .fillna(0)\ + .reset_index()\ + .rename(columns={p: mtx_header})\ + .pivot(index=mtx_header, columns=q, values='trips')\ + .to_csv(out_dir.joinpath(f'{name}.csv')) + \ No newline at end of file diff --git a/Scripts/tbmtools/src/tbmtools/matrix/skim.py b/Scripts/tbmtools/src/tbmtools/matrix/skim.py new file mode 100644 index 0000000..94ca912 --- /dev/null +++ b/Scripts/tbmtools/src/tbmtools/matrix/skim.py @@ -0,0 +1,119 @@ +"""Export highway and transit skim matrices from Emme model outputs. + +This module provides helpers for exporting travel time, distance, and transit +skim matrices to CSV files for downstream analysis and reporting. +""" + +import logging + +from tbmtools.utils import flag_disconnected_transit_ods + + +def export_highway(out_dir, scenario_code, modeller): + """ + Export highway time and distance skim matrices from an emmebank to + CSVs. + + Parameters + ---------- + out_dir : str or pathlib.Path + Root output directory where the `skims/` subdirectory will be created. + scenario_code : int + Scenario year code used to select the daily Emme scenario (exported + from the scenario identified as `"{scenario_code}29"`). + modeller : inro.modeller.Modeller + Modeller instance used to construct the matrix export tool. + + Returns + ------- + pathlib.Path + Path to the created `skims/` directory containing exported highway + skim CSV files. + + Notes + ----- + This function exports AM and MD highway skim matrices: + - AM: time=`mf44`, distance=`mf45` + - MD: time=`mf46`, distance=`mf47` + + The export is performed via the modeller tool + `inro.emme.data.matrix.export_matrix_to_csv`. + """ + logging.info('Exporting highway skims') + skim_matrix_ids = {'am': {'time': 'mf44', + 'distance': 'mf45'}, + 'md': {'time': 'mf46', + 'distance': 'mf47'}} + # Make output subdirectory. + skim_dir = out_dir.joinpath('skims') + skim_dir.mkdir(exist_ok=True) + # Construct Modeller tool. + export_matrix_data = modeller.tool('inro.emme.data.matrix.export_matrix_to_csv') + # Export am highway skims. + export_matrix_data(matrices=[i for i in list(skim_matrix_ids['am'].values())], + export_path=skim_dir, + scenario=modeller.emmebank.scenario(str(scenario_code) + '29')) + # Export md highway skims. + export_matrix_data(matrices=[i for i in list(skim_matrix_ids['md'].values())], + export_path=skim_dir, + scenario=modeller.emmebank.scenario(str(scenario_code) + '29')) + + return skim_dir + + +def export_transit(out_dir, scenario_code, modeller): + """ + Export peak and off-peak transit skim matrices from an emmebank to + CSV files. + + Parameters + ---------- + out_dir : str or pathlib.Path + Root output directory where the `skims/` subdirectory is created. + scenario_code : int + Scenario year code used to select the daily Emme scenario + (`{scenario_code}29`). + modeller : inro.modeller.Modeller + Modeller instance used to construct the matrix export tool. + + Returns + ------- + pathlib.Path + Path to the created `skims/` directory containing exported transit + skim CSV files. + + Notes + ----- + The function flags disconnected transit O-Ds before exporting skim + matrices for both peak and off-peak periods using the modeller tool + `inro.emme.data.matrix.export_matrix_to_csv`. + """ + logging.info('Exporting transit skims') + skim_matrix_ids = {'peak': {'in-vehicle minutes': 'mf822', + 'walk transfer minutes': 'mf823', + 'wait time': 'mf838', + 'priority mode': 'mf830', + 'average fare': 'mf828', + 'station zone': 'mf837'}, + 'off-peak': {'in-vehicle minutes': 'mf922', + 'walk transfer minutes': 'mf923', + 'wait time': 'mf938', + 'priority mode': 'mf930', + 'average fare': 'mf928', + 'station zone': 'mf937'}} + flag_disconnected_transit_ods(skim_matrix_ids, scenario_code, modeller) + # Make output subdirectory. + skim_dir = out_dir.joinpath('skims') + skim_dir.mkdir(exist_ok=True) + # Construct Modeller tool. + export_matrix_data = modeller.tool('inro.emme.data.matrix.export_matrix_to_csv') + # Export peak transit skims. + export_matrix_data(matrices=[i for i in list(skim_matrix_ids['peak'].values())], + export_path=skim_dir, + scenario=modeller.emmebank.scenario(str(scenario_code) + '29')) + # Export off-peak transit skims. + export_matrix_data(matrices=[i for i in list(skim_matrix_ids['off-peak'].values())], + export_path=skim_dir, + scenario=modeller.emmebank.scenario(str(scenario_code) + '29')) + + return skim_dir diff --git a/Scripts/tbmtools/src/tbmtools/matrix/vehicle_trip.py b/Scripts/tbmtools/src/tbmtools/matrix/vehicle_trip.py new file mode 100644 index 0000000..bbebcf6 --- /dev/null +++ b/Scripts/tbmtools/src/tbmtools/matrix/vehicle_trip.py @@ -0,0 +1,46 @@ +"""Export vehicle-trip matrices from Emme model outputs. + +This module provides utilities for exporting vehicle trip matrices to CSV +files for downstream analysis and reporting. +""" + +import logging + + +def export_auto(out_dir, scenario_code, modeller): + """ + Export vehicle trip matrices from an emmebank to CSVs. + + Parameters + ---------- + out_dir : str or pathlib.Path + Root output directory where the `trips` subdirectory will be created. + scenario_code : int + Scenario year code used to select the daily Emme scenario. + modeller : inro.modeller.Modeller + Modeller instance used to construct the matrix export tool. + + Returns + ------- + pathlib.Path + Path to the directory containing exported vehicle trip CSV files. + """ + logging.info('Exporting vehicle trips') + # Make output subdirectories. + trip_dir = out_dir.joinpath('trips') + trip_dir.mkdir(exist_ok=True) + # Construct Modeller tool. + export_matrix_data = modeller.tool('inro.emme.data.matrix.export_matrix_to_csv') + # Export vehicle trip matrices. + matrix_names = {'mf4': 'b_truck', + 'mf5': 'l_truck', + 'mf6': 'm_truck', + 'mf7': 'h_truck', + 'mf8': 'poe_auto', + 'mf9': 'poe_truck', + 'mf10': 'airport'} + export_matrix_data(matrices=list(matrix_names.keys()), + export_path=trip_dir, + scenario=modeller.emmebank.scenario(str(scenario_code) + '29')) + + return trip_dir diff --git a/Scripts/tbmtools/src/tbmtools/network/__init__.py b/Scripts/tbmtools/src/tbmtools/network/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Scripts/tbmtools/src/tbmtools/network/highway.py b/Scripts/tbmtools/src/tbmtools/network/highway.py new file mode 100644 index 0000000..60193a1 --- /dev/null +++ b/Scripts/tbmtools/src/tbmtools/network/highway.py @@ -0,0 +1,220 @@ +"""Export highway network data and attributes from Emme scenarios. + +This module provides utilities for exporting highway networks, link +attributes, and shapefiles for multiple time-of-day periods and the daily +scenario. +""" + +import csv +import logging +from pathlib import Path + +from tbmtools.utils import calculate_vadt + + +def export_all(out_dir, scenario_code, modeller): + """ + Export highway networks and attributes for all time-of-day periods + and the daily scenario. Networks are exported to both transaction + file and shapefile formats. + + Parameters + ---------- + out_dir : str or pathlib.Path + Path to the root output directory where `networks/highway` will + be created. + scenario_code : int + Scenario year code (used as `{scenario_code}29` for the daily + highway scenario). + modeller : inro.modeller.Modeller + Modeller instance used to construct Emme export tools. + + Returns + ------- + tuple[pathlib.Path, ...] + A tuple containing the highway transaction export directory + followed by any generated shapefile directories. + """ + logging.info(f'Exporting highway networks') + # Normalize arguments. + out_dir = Path(out_dir).resolve() if isinstance(out_dir, str) else out_dir + # Create output subdirectories. + hwy_dir = out_dir.joinpath('networks', 'highway') + hwy_dir.mkdir(parents=True, exist_ok=True) + # Construct Modeller tools. + create_attrib = modeller.tool('inro.emme.data.extra_attribute.create_extra_attribute') + export_basenet = modeller.tool('inro.emme.data.network.base.export_base_network') + net_calc = modeller.tool('inro.emme.network_calculation.network_calculator') + net_to_shp = modeller.tool('inro.emme.data.network.export_network_as_shapefile') + # Export each time of day highway network and attributes as Emme + # transaction files and shapefiles. + hwyshp_dirs = [] + for p in range(1, 9): + scenario = modeller.emmebank.copy_scenario(source_id=p, destination_id=99) + calculate_vadt(create_attrib, net_calc, scenario, p) + export_transaction_files_tod(export_basenet, net_calc, hwy_dir, scenario, p) + if p in [3, 7]: + hwyshp_dirs.append(export_shapefiles_tod(net_calc, net_to_shp, out_dir, scenario, p, scenario_code)) + modeller.emmebank.delete_scenario(scenario.id) + # Export daily highway network and attributes as Emme transaction + # files and shapefiles. + daily_scenario = modeller.emmebank.scenario(f'{scenario_code}29') + export_transaction_files_day(export_basenet, net_calc, hwy_dir, daily_scenario) + hwyshp_dirs.append(export_shapefiles_day(net_to_shp, out_dir, scenario_code, daily_scenario)) + return tuple([hwy_dir] + hwyshp_dirs) + +def export_shapefiles_day(net_to_shp, out_dir, scenario_code, scenario): + """ + Export the daily highway network to shapefile format. + + Parameters + ---------- + net_to_shp : callable + Modeller tool used to export a network as a shapefile. + out_dir : str or pathlib.Path + Output directory where the `highway-{scenario_code}` shapefile + directory will be created. + scenario_code : int + Scenario year code used to name the output shapefile directory. + scenario : inro.emme.scenario.Scenario + Emme scenario object with the network to export. + + Returns + ------- + pathlib.Path + Path to the created shapefile export directory. + """ + # Write shapefiles. + hwyshp_dir = out_dir.joinpath(f'highway-{scenario_code}') + net_to_shp(export_path=hwyshp_dir, scenario=scenario) + + return hwyshp_dir + +def export_shapefiles_tod(net_calc, net_to_shp, out_dir, scenario, p, scenario_code): + """ + Export a time-of-day highway network to shapefile format. + + This function clears temporary user data attributes from the scenario + network, stores vehicle volumes in the `ul1` attribute, and exports the + network to a shapefile directory named using the period tag and scenario + code. + + Parameters + ---------- + net_calc : callable + Modeller tool used to perform network calculations. + net_to_shp : callable + Modeller tool used to export the network as a shapefile. + out_dir : str or pathlib.Path + Output directory where the `highway_{tag}-{scenario_code}` shapefile + directory will be created. + scenario : inro.emme.scenario.Scenario + Emme scenario object containing the time-of-day highway network. + p : int + Time-of-day period identifier used to determine the shapefile tag. + scenario_code : int + Scenario year code used in the exported directory name. + + Returns + ------- + pathlib.Path + Path to the created shapefile export directory. + """ + # Clear user data attributes. + spec1 = {'type': 'NETWORK_CALCULATION', + 'result': 'ul1', + 'expression': '0', + 'selections': {'link': 'all'}} + spec2 = spec1.copy() + spec2['result'] = 'ul2' + spec3 = spec1.copy() + spec3['result'] = 'ul3' + net_calc(specification=[spec1, spec2, spec3], scenario=scenario) + # Store vehicle volumes. + net_calc(specification={'type': 'NETWORK_CALCULATION', + 'result': 'ul1', + 'expression': '@vadt', + 'selections': {'link': 'all'}}, + scenario=scenario) + # Remove extra attributes. + for xattrib in scenario.extra_attributes(): + scenario.delete_extra_attribute(xattrib.id) + # Set file tag. + if p == 3: + f_tag = 'ampk' + elif p == 7: + f_tag = 'pmpk' + else: + f_tag = 'p' + str(p) + # Write shapefiles. + hwyshp_dir = out_dir.joinpath(f'highway_{f_tag}-{scenario_code}') + net_to_shp(export_path=hwyshp_dir, scenario=scenario) + return hwyshp_dir + +def export_transaction_files_day(export_basenet, net_calc, out_dir, scenario): + """ + Export the daily highway network and link attributes to transaction + files. + + Parameters + ---------- + export_basenet : callable + Modeller tool used to export the base network transaction file. + net_calc : callable + Modeller tool used to compute network attributes and return the + calculation report. + out_dir : str or pathlib.Path + Output directory where `network_daily.txt` and + `attribs_daily.txt` will be written. + scenario : inro.emme.scenario.Scenario + Emme scenario object containing the daily highway network. + + Returns + ------- + None + """ + # Write network transaction file. + export_basenet(export_file=out_dir.joinpath(f'network_daily.txt'), scenario=scenario) + # Write attribute transaction file. + report = net_calc(specification={'type': 'NETWORK_CALCULATION', + 'expression': '@vadt', + 'selections': {'link': 'all'}}, + scenario=scenario, + full_report=True) + with open(out_dir.joinpath(f'attribs_daily.txt'), 'w', newline='') as f: + csv.writer(f, delimiter=' ').writerows(report['table']) + +def export_transaction_files_tod(export_basenet, net_calc, out_dir, scenario, p): + """ + Export a time-of-day highway network and its link attributes to transaction files. + + Parameters + ---------- + export_basenet : callable + Modeller tool used to export the base network transaction file. + net_calc : callable + Modeller tool used to compute network attributes and return the + calculation report. + out_dir : str or pathlib.Path + Output directory where `network_p{p}.txt` and `attribs_p{p}.txt` + will be written. + scenario : inro.emme.scenario.Scenario + Emme scenario object containing the time-of-day highway network. + p : int + Time-of-day period identifier used to name the network and attribute + export files. + + Returns + ------- + None + """ + # Write network transaction file. + export_basenet(export_file=out_dir.joinpath(f'network_p{p}.txt'), scenario=scenario) + # Write attribute transaction file. + report = net_calc(specification={'type': 'NETWORK_CALCULATION', + 'expression': '@speed + @width + @parkl + @toll + @sigic + @tipid + @ftime + @emcap + @avelw + @vadt + timau', + 'selections': {'link': 'all'}}, + scenario=scenario, + full_report=True) + with open(out_dir.joinpath(f'attribs_p{p}.txt'), 'w', newline='') as f: + csv.writer(f, delimiter=' ').writerows(report['table']) \ No newline at end of file diff --git a/Scripts/tbmtools/src/tbmtools/network/transit.py b/Scripts/tbmtools/src/tbmtools/network/transit.py new file mode 100644 index 0000000..6790aea --- /dev/null +++ b/Scripts/tbmtools/src/tbmtools/network/transit.py @@ -0,0 +1,133 @@ +"""Export transit network data and attributes from Emme scenarios. + +This module provides utilities for exporting transit networks, itineraries, +and related attributes to transaction files and shapefiles for peak and +off-peak periods. +""" + +import csv +import logging +from pathlib import Path + + +def export_all(out_dir, scenario_code, modeller): + """ + Export transit networks, itineraries, and attributes for peak and + off-peak periods. Neworks are exported to both transaction file and + shapefile formats. + + Parameters + ---------- + out_dir : str or pathlib.Path + Path to the root output directory where `networks/transit` will + be created. + scenario_code : int + Scenario year code used to select the transit scenarios. + modeller : inro.modeller.Modeller + Modeller instance used to construct Emme export tools. + + Returns + ------- + tuple[pathlib.Path, ...] + A tuple containing the transit transaction export directory + followed by any generated shapefile directories. + """ + logging.info(f'Exporting transit network') + # Normalize arguments. + out_dir = Path(out_dir).resolve() if isinstance(out_dir, str) else out_dir + # Make output subdirectories. + transit_dir = out_dir.joinpath('networks', 'transit') + transit_dir.mkdir(parents=True, exist_ok=True) + # Construct Modeller tools. + export_basenet = modeller.tool('inro.emme.data.network.base.export_base_network') + export_lines = modeller.tool('inro.emme.data.network.transit.export_transit_lines') + net_calc = modeller.tool('inro.emme.network_calculation.network_calculator') + net_to_shp = modeller.tool('inro.emme.data.network.export_network_as_shapefile') + # Export peak and off-peak networks as Emme transaction files and + # shapefiles. + transitshp_dirs = [] + for n in [0, 5]: + # Set scenario. + s = modeller.emmebank.scenario(scenario_code + n) + # Set file tag. + if n == 0: + f_tag = 'pk' + elif n == 5: + f_tag = 'op' + export_transaction_files_tod(export_basenet, export_lines, net_calc, transit_dir, f_tag, s) + transitshp_dirs.append(export_shapefiles_tod(net_to_shp, transit_dir, f_tag, s, scenario_code)) + return tuple([transit_dir] + transitshp_dirs) + + +def export_shapefiles_tod(net_to_shp, transit_dir, f_tag, s, scenario_code): + """ + Export a transit network to shapefile format. + + Parameters + ---------- + net_to_shp : callable + Modeller tool used to export a network as shapefiles. + transit_dir : str or pathlib.Path + Output directory where the `transit_{f_tag}-{scenario_code}` + shapefile directory will be created. + f_tag : str + Transit period file tag used in the exported directory name. + s : inro.emme.scenario.Scenario + Emme scenario object containing the transit network to export. + scenario_code : int + Scenario year code used in the exported directory name. + + Returns + ------- + pathlib.Path + Path to the created shapefile export directory. + """ + # Write shapefiles. + transitshp_dir = transit_dir.joinpath(f'transit_{f_tag}-{scenario_code}') + net_to_shp(export_path=transitshp_dir, + scenario=s) + return transitshp_dir + + +def export_transaction_files_tod(export_basenet, export_lines, net_calc, transit_dir, f_tag, s): + """ + Export a transit network, its itineraries, and link attributes to + transaction files. + + Parameters + ---------- + export_basenet : callable + Modeller tool used to export the base transit network + transaction file. + export_lines : callable + Modeller tool used to export transit itinerary transaction + files. + net_calc : callable + Modeller tool used to compute transit link attributes and return + the calculation report. + transit_dir : str or pathlib.Path + Output directory where `network_{f_tag}.txt`, + `itins_{f_tag}.txt`, and `attribs_{f_tag}.txt` will be written. + f_tag : str + Transit period file tag used to name the exported files. + s : inro.emme.scenario.Scenario + Emme scenario object containing the transit network and + itinerary data. + + Returns + ------- + None + """ + # Write network transaction file. + export_basenet(export_file=transit_dir.joinpath(f'network_{f_tag}.txt'), scenario=s) + # Write itinerary transaction file. + export_lines(export_file=transit_dir.joinpath(f'itins_{f_tag}.txt'), scenario=s) + # Write attribute transaction file. + report = net_calc(specification={'type': 'NETWORK_CALCULATION', + 'expression': '@ltime + @hwytm + @zfare_link', + 'selections': {'link': 'all', 'transit_line': 'all'}}, + scenario=s, + full_report=True) + with open(transit_dir.joinpath(f'attribs_{f_tag}.txt'), 'w', newline='') as f: + csv.writer(f, delimiter=' ').writerows(report['table']) + \ No newline at end of file diff --git a/Scripts/tbmtools/src/tbmtools/project.py b/Scripts/tbmtools/src/tbmtools/project.py new file mode 100644 index 0000000..0f72292 --- /dev/null +++ b/Scripts/tbmtools/src/tbmtools/project.py @@ -0,0 +1,66 @@ +"""Helpers for working with Emme projects. + +This module provides utilities for validating Emme project paths and +connecting to Emme Desktop from Python. +""" + +from pathlib import Path +import argparse +import inro.emme.desktop.app as _app +import inro.modeller as _m + + +def emme_project_file(path): + """Validate an Emme project file path. + + Parameters + ---------- + path : str or pathlib.Path + Path to an Emme project file. + + Returns + ------- + str or pathlib.Path + The validated input path. + + Raises + ------ + argparse.ArgumentTypeError + If the path does not have an ``.emp`` extension. + argparse.ArgumentError + If the path does not exist. + """ + ext = Path(path).suffix + if ext != '.emp': + raise argparse.ArgumentTypeError('File must have an emp extension') + if not Path(path).exists(): + raise argparse.ArgumentError('File does not exist') + + return path + + +def connect(path): + """Start an Emme Desktop session and connect Modeller to a project. + + Parameters + ---------- + path : str or pathlib.Path + Path to an Emme project file or a directory containing one. + + Returns + ------- + inro.modeller.Modeller + An initialized Modeller client connected to the Emme project. + """ + if isinstance(path, str): + path = Path(path) + if path.is_file(): + empfile = path + elif path.is_dir(): + empfile = sorted(path.glob('**/*.emp'))[0] + app = _app.start_dedicated(visible=False, + user_initials='CMAP', + project=empfile) + modeller = _m.Modeller(app) + + return modeller diff --git a/Scripts/share/standard_data/hand/data_user_guide_html.txt b/Scripts/tbmtools/src/tbmtools/templates/data_user_guide_html.txt similarity index 100% rename from Scripts/share/standard_data/hand/data_user_guide_html.txt rename to Scripts/tbmtools/src/tbmtools/templates/data_user_guide_html.txt diff --git a/Scripts/share/standard_data/hand/data_user_guide_md.txt b/Scripts/tbmtools/src/tbmtools/templates/data_user_guide_md.txt similarity index 100% rename from Scripts/share/standard_data/hand/data_user_guide_md.txt rename to Scripts/tbmtools/src/tbmtools/templates/data_user_guide_md.txt diff --git a/Scripts/tbmtools/src/tbmtools/utils.py b/Scripts/tbmtools/src/tbmtools/utils.py new file mode 100644 index 0000000..25b435f --- /dev/null +++ b/Scripts/tbmtools/src/tbmtools/utils.py @@ -0,0 +1,198 @@ +""" +Utility helper functions for trip-based model data export. + +This module provides generic support routines used by the TBM export +pipeline, including network attribute calculation, ZIP compression, +transit skim flagging, configuration loading, and multiprocessing +support for compression tasks. + +Functions +--------- +calculate_vadt + Create and compute the `@vadt` extra attribute for a scenario network. +compress + Compress a single file or directory into a ZIP archive. +flag_disconnected_transit_ods + Flag disconnected transit skim O-D pairs with a numeric sentinel value. +load_config + Load project configuration values from YAML files. +mp_compress + Helper wrapper for calling `compress` from multiprocessing pools. +""" +from copy import deepcopy +import logging +from pathlib import Path +import shutil +from zipfile import ZipFile, ZIP_DEFLATED + +import yaml + + +def calculate_vadt(create_attrib, net_calc, scenario, p): + """ + Calculate vehicle average daily traffic (VADT) for a scenario and + store it in an extra attribute. + + Parameters + ---------- + create_attrib : callable + Modeller tool used to create an extra link attribute. + net_calc : callable + Modeller tool used to perform network calculations. + scenario : inro.emme.scenario.Scenario + Emme scenario containing the network on which VADT is calculated. + p : int + Time-of-day period identifier used in the attribute description. + + Returns + ------- + None + This function updates the scenario network by adding the `@vadt` + extra attribute and computing its values. + """ + # Create extra attribute. + create_attrib(extra_attribute_type='LINK', + extra_attribute_name='@vadt', + extra_attribute_description=f'adt p{p}', + scenario=scenario) + # Calculate vehicle volumes. + spec = {'type': 'NETWORK_CALCULATION', + 'result': '@vadt', + 'expression': '@avauv + @avh2v + @avh3v + @avbqv + @avlqv + (@avmqv/2) + (@avhqv/3)', + 'selections': {'link': 'all'}} + net_calc(specification=spec, scenario=scenario) + + +def compress(out_file_name, source_path, out_dir): + """ + Compress a file or directory into a ZIP archive. + + Parameters + ---------- + out_file_name : str or path-like + Name of the destination ZIP file. + source_path : str or path-like + Path to the file or directory to compress. + out_dir : pathlib.Path + Directory where the ZIP file will be written. + + Returns + ------- + pathlib.Path + Path to the created ZIP archive. + """ + # Handle arguments. + if isinstance(source_path, str): + source_path = Path(source_path).resolve() + # Compress content. + out_file = out_dir.joinpath(out_file_name) + with ZipFile(out_file, mode='w', compression=ZIP_DEFLATED, compresslevel=9) as zip: + if source_path.is_file(): + zip.write(source_path, arcname=source_path.name) + elif source_path.is_dir(): + for container in source_path.iterdir(): + if container.is_file(): + zip.write(container, arcname=container.name) + + return out_file + + +def copy_prods_attrs(proj_dir, out_dir): + """Copy production and attraction tables into an output directory. + + Parameters + ---------- + proj_dir : pathlib.Path + Project directory containing the production/attraction table files. + out_dir : pathlib.Path + Directory where the copied tables will be written. + + Returns + ------- + pathlib.Path + Path to the directory containing the copied production and attraction + tables. + """ + # Copy productions and attractions to output subdirectory. + logging.info('Copying productions and attractions') + files = [proj_dir.joinpath('Database', 'tg', 'fortran', 'TRIP49_PA_OUT.TXT'), + proj_dir.joinpath('Database', 'tg', 'fortran', 'TRIP49_PA_WFH_OUT.TXT')] + pa_tables_path = out_dir.joinpath('prods_attrs') + pa_tables_path.mkdir(exist_ok=True) + for file in files: + shutil.copy(file, pa_tables_path) + return pa_tables_path + + +def flag_disconnected_transit_ods(skim_matrix_ids, scenario_code, modeller): + """ + Flag transit skim O-D pairs that are not connected by transit. + + Parameters + ---------- + skim_matrix_ids : dict + Dictionary of skim matrix IDs grouped by transit period, + e.g. {'peak': {...}, 'off-peak': {...}}. Each group must include + 'in-vehicle minutes' and other skim matrix identifiers. + scenario_code : int + Scenario year code used to select the daily scenario + (`"{scenario_code}29"`). + modeller : inro.modeller.Modeller + Modeller instance used to construct the matrix calculation tool. + + Returns + ------- + None + The function updates skim matrices in the specified scenario by + assigning a flag value of `9999` for disconnected transit O-D pairs. + """ + # Flag peak and off-peak transit skim O-Ds that are not connected by + # transit. Use a flag value of 9999. + compute_matrices = modeller.tool('inro.emme.matrix_calculation.matrix_calculator') + for transitnet, transitnet_skims in skim_matrix_ids.items(): + # Flag O-Ds with negative or impossibly large values for in-vehicle minutes. + spec = {'type': 'MATRIX_CALCULATION', + 'expression': '9999', + 'result': transitnet_skims['in-vehicle minutes'], + 'constraint': {'by_value': {'od_values': transitnet_skims['in-vehicle minutes'], + 'interval_min': 0, + 'interval_max': 9999, + 'condition': 'EXCLUDE'}}} + compute_matrices(spec, scenario=modeller.emmebank.scenario(str(scenario_code) + '29')) + # Apply flag to other transit skim matrices. + spec['constraint']['by_value']['interval_min'] = 9999 + spec['constraint']['by_value']['condition'] = 'INCLUDE' + specs = [] + for desc, mtx_id in transitnet_skims.items(): + if desc not in ['in-vehicle minutes', 'station zone']: + spec['result'] = mtx_id + specs.append(deepcopy(spec)) + compute_matrices(specs, scenario=modeller.emmebank.scenario(str(scenario_code) + '29')) + + +def load_config(file, proj_dir): + """ + Load model configuration values from YAML files. + + Parameters + ---------- + file : str or pathlib.Path + Path to the YAML configuration file to load. + proj_dir : pathlib.Path + Path to the Emme project directory containing + `Database/batch_file.yaml`. + + Returns + ------- + dict + Configuration dictionary containing values from + ``Database/batch_file.yaml`` and the supplied YAML file, with + ``scenario_code`` and ``model_version`` added from the batch file. + """ + with open(proj_dir.joinpath('Database/batch_file.yaml')) as f: + batch_file_config = yaml.safe_load(f) + with open(file) as f: + config = yaml.safe_load(f) + config['scenario_code'] = batch_file_config['scenario_code'] + config['model_version'] = batch_file_config['model_version'] + return config From 905fddd7803c9c8efa4c99922605bba9ddf60256 Mon Sep 17 00:00:00 2001 From: Nicholas Ferguson Date: Tue, 7 Jul 2026 10:00:03 -0500 Subject: [PATCH 6/9] test(tbmtools): Add unit tests for packaging data --- Scripts/tbmtools/tests/__init__.py | 0 Scripts/tbmtools/tests/data/.gitignore | 6 + Scripts/tbmtools/tests/data/config.yaml | 23 +++ Scripts/tbmtools/tests/test_data_package.py | 211 ++++++++++++++++++++ 4 files changed, 240 insertions(+) create mode 100644 Scripts/tbmtools/tests/__init__.py create mode 100644 Scripts/tbmtools/tests/data/.gitignore create mode 100644 Scripts/tbmtools/tests/data/config.yaml create mode 100644 Scripts/tbmtools/tests/test_data_package.py diff --git a/Scripts/tbmtools/tests/__init__.py b/Scripts/tbmtools/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Scripts/tbmtools/tests/data/.gitignore b/Scripts/tbmtools/tests/data/.gitignore new file mode 100644 index 0000000..a0917ab --- /dev/null +++ b/Scripts/tbmtools/tests/data/.gitignore @@ -0,0 +1,6 @@ +# Ignore exerything +* +# Don't ignore +!.gitignore +!/templates +!config.yaml \ No newline at end of file diff --git a/Scripts/tbmtools/tests/data/config.yaml b/Scripts/tbmtools/tests/data/config.yaml new file mode 100644 index 0000000..7bfbd72 --- /dev/null +++ b/Scripts/tbmtools/tests/data/config.yaml @@ -0,0 +1,23 @@ +# Path to transit transaction file directory. +transit_directory: E:\nrf\tbm\cmap_trip-based_model\Scripts\tbmtools\tests\data\transit +# Date of Board approval. +approval_date: June 2025 +# Names of output files. +trip_roster: trip_roster +tg_data: tg_results +compressed: + pa_tables: prods_attrs + trip_tables: trips + hov_trip_tables: hovtrips + highway_networks: emmenet_highway + am_peak_highway_network: highwayshp_ampk + pm_peak_highway_network: highwayshp_pmpk + daily_highway_network: highwayshp + transit_networks: emmenet_transit + tod_transit_networks: emmenet_transit_tod + peak_transit_network: transitshp_pk + offpeak_transit_network: transitshp_op + skim_matrices: skims + database: emmebank + matrices: emmemat + \ No newline at end of file diff --git a/Scripts/tbmtools/tests/test_data_package.py b/Scripts/tbmtools/tests/test_data_package.py new file mode 100644 index 0000000..b0f03df --- /dev/null +++ b/Scripts/tbmtools/tests/test_data_package.py @@ -0,0 +1,211 @@ +import filecmp +from itertools import islice, zip_longest +from pathlib import Path +import shutil +import sys +import tempfile +import unittest +import zipfile + +sys.path.append(str(Path(__file__).parents[1].joinpath('src'))) + +from tbmtools import project as tbm +from tbmtools.matrix import person_trip, skim, vehicle_trip +from tbmtools.network import highway, transit + + +class TestDataPackage(unittest.TestCase): + + def test_vehicle_trip_matrices(self): + with tempfile.TemporaryDirectory() as out_dir: + expected_trips_dir = Path(out_dir).joinpath('expected_trips') + with zipfile.ZipFile(expected_dir.joinpath('trips_c25q2_700.zip'), 'r') as z: + z.extractall(expected_trips_dir) + test_trips_dir = vehicle_trip.export_auto(Path(out_dir), 700, modeller) + # Compare the directories. + comp = filecmp.dircmp(expected_trips_dir, + test_trips_dir, + ignore=[f.name for f in expected_trips_dir.iterdir() if f.name not in [f'mf{n}.csv' for n in range(4, 11)]]) + self.assertEqual(comp.left_only, [], + f'Files only in expected: {comp.left_only}') + self.assertEqual(comp.right_only, [], + f'Files only in test: {comp.right_only}') + self.assertEqual(comp.diff_files, [], + f'Files with different data: {comp.diff_files}') + + def test_transit_skim_matrices(self): + with tempfile.TemporaryDirectory() as out_dir: + expected_skims_dir = Path(out_dir).joinpath('expected_skims') + with zipfile.ZipFile(expected_dir.joinpath('skims_c25q2_700.zip'), 'r') as z: + z.extractall(expected_skims_dir) + test_skims_dir = skim.export_transit(Path(out_dir), 700, modeller) + # Compare the directories. + comp = filecmp.dircmp(expected_skims_dir, + test_skims_dir, + ignore=[f.name for f in expected_skims_dir.iterdir() if f.name not in [f'mf{n}.csv' for n in [822, 823, 838, 830, 828, 837, 922, 923, 938, 930, 928, 937]]]) + self.assertEqual(comp.left_only, [], + f'Files only in expected: {comp.left_only}') + self.assertEqual(comp.right_only, [], + f'Files only in test: {comp.right_only}') + self.assertEqual(comp.diff_files, [], + f'Files with different data: {comp.diff_files}') + + def test_highway_skim_matrices(self): + with tempfile.TemporaryDirectory() as out_dir: + expected_skims_dir = Path(out_dir).joinpath('expected_skims') + with zipfile.ZipFile(expected_dir.joinpath('skims_c25q2_700.zip'), 'r') as z: + z.extractall(expected_skims_dir) + test_skims_dir = skim.export_highway(Path(out_dir), 700, modeller) + # Compare the directories. + comp = filecmp.dircmp(expected_skims_dir, + test_skims_dir, + ignore=[f.name for f in expected_skims_dir.iterdir() if f.name not in [f'mf{n}.csv' for n in range(44, 48)]]) + self.assertEqual(comp.left_only, [], + f'Files only in expected: {comp.left_only}') + self.assertEqual(comp.right_only, [], + f'Files only in test: {comp.right_only}') + self.assertEqual(comp.diff_files, [], + f'Files with different data: {comp.diff_files}') + + def test_trip_roster(self): + with tempfile.TemporaryDirectory() as out_dir: + expected_trip_roster_file = Path(expected_dir).joinpath('trip_roster_c25q2_700.csv') + test_trip_roster_file = person_trip.export_trip_roster(Path(modeller.desktop.project_file_name()).parent, Path(out_dir), 'trip_roster') + # Compare the directories. + comp = filecmp.cmp(expected_trip_roster_file, + test_trip_roster_file, + shallow=False) + self.assertTrue(comp) + + def test_auto_person_trip_matrices(self): + with tempfile.TemporaryDirectory() as out_dir: + expected_trips_dir = Path(out_dir).joinpath('expected_trips') + expected_hovtrips_dir = Path(out_dir).joinpath('expected_hov_trips') + with zipfile.ZipFile(expected_dir.joinpath('trips_c25q2_700.zip'), 'r') as z1: + z1.extractall(expected_trips_dir) + with zipfile.ZipFile(expected_dir.joinpath('hovtrips_c25q2_700.zip'), 'r') as z2: + z2.extractall(expected_hovtrips_dir) + test_trip_roster_file = person_trip.export_trip_roster(Path(modeller.desktop.project_file_name()).parent, Path(out_dir), 'trip_roster') + test_trips_dir, test_hovtrips_dir = person_trip.export_auto(Path(modeller.desktop.project_file_name()).parent, Path(out_dir), test_trip_roster_file) + # Compare the directories. + comp1 = filecmp.dircmp(expected_trips_dir, + test_trips_dir, + ignore=[f'mf{n}.csv' for n in [4, 5, 6, 7, 8, 9, 10, 38, 39, 40, 41, 42, 43]] + ['hov_trips']) + self.assertEqual(comp1.left_only, [], + f'Files only in expected: {comp1.left_only}') + self.assertEqual(comp1.right_only, [], + f'Files only in test: {comp1.right_only}') + self.assertEqual(comp1.diff_files, [], + f'Files with different data: {comp1.diff_files}') + comp2 = filecmp.dircmp(expected_hovtrips_dir, + test_hovtrips_dir) + self.assertEqual(comp2.left_only, [], + f'Files only in expected: {comp2.left_only}') + self.assertEqual(comp2.right_only, [], + f'Files only in test: {comp2.right_only}') + self.assertEqual(comp2.diff_files, [], + f'Files with different data: {comp2.diff_files}') + + def test_transit_person_trip_matrices(self): + with tempfile.TemporaryDirectory() as out_dir: + expected_trips_dir = Path(out_dir).joinpath('expected_trips') + with zipfile.ZipFile(expected_dir.joinpath('trips_c25q2_700.zip'), 'r') as z: + z.extractall(expected_trips_dir) + test_trips_dir = person_trip.export_transit(Path(modeller.desktop.project_file_name()).parent, Path(out_dir), 700, modeller) + # Compare the directories. + comp = filecmp.dircmp(expected_trips_dir, + test_trips_dir, + ignore=[f.name for f in expected_trips_dir.iterdir() if f.name not in [f'mf{n}.csv' for n in range(38, 44)]]) + self.assertEqual(comp.left_only, [], + f'Files only in expected: {comp.left_only}') + self.assertEqual(comp.right_only, [], + f'Files only in test: {comp.right_only}') + self.assertEqual(comp.diff_files, [], + f'Files with different data: {comp.diff_files}') + + def test_transit_network_transaction_files(self): + with tempfile.TemporaryDirectory() as out_dir: + expected_transit_network_dir = Path(out_dir).joinpath('expected_transit_network') + with zipfile.ZipFile(expected_dir.joinpath('emmenet_transit_c25q2_700.zip'), 'r') as z: + z.extractall(expected_transit_network_dir) + test_transit_network_dir = transit.export_all(Path(out_dir), 700, modeller)[0] + # Compare the directories. + comp = filecmp.dircmp(expected_transit_network_dir, + test_transit_network_dir, + ignore=['transit_op-700', 'transit_pk-700']) + self.assertEqual(comp.left_only, [], + f'Files only in expected: {comp.left_only}') + self.assertEqual(comp.right_only, [], + f'Files only in test: {comp.right_only}') + if comp.diff_files: + line_diff_files = [] + for file in comp.diff_files: + if diff_skip_lines(expected_transit_network_dir.joinpath(file), test_transit_network_dir.joinpath(file), lines_to_skip=4): + line_diff_files.append(file) + comp.diff_files = line_diff_files + self.assertEqual(comp.diff_files, [], + f'Files with different data: {comp.diff_files}') + + def test_higway_network_transaction_files(self): + with tempfile.TemporaryDirectory() as out_dir: + expected_highway_network_dir = Path(out_dir).joinpath('expected_highway_network') + with zipfile.ZipFile(expected_dir.joinpath('emmenet_highway_c25q2_700.zip'), 'r') as z: + z.extractall(expected_highway_network_dir) + test_highway_network_dir = highway.export_all(Path(out_dir), 700, modeller)[0] + # Compare the directories. + comp = filecmp.dircmp(expected_highway_network_dir, + test_highway_network_dir) + self.assertEqual(comp.left_only, [], + f'Files only in expected: {comp.left_only}') + self.assertEqual(comp.right_only, [], + f'Files only in test: {comp.right_only}') + if comp.diff_files: + line_diff_files = [] + for file in comp.diff_files: + if diff_skip_lines(expected_highway_network_dir.joinpath(file), test_highway_network_dir.joinpath(file), lines_to_skip=4): + line_diff_files.append(file) + comp.diff_files = line_diff_files + self.assertEqual(comp.diff_files, [], + f'Files with different data: {comp.diff_files}') + + +def diff_skip_lines(file1, file2, lines_to_skip=1): + line_diff = False # Files are identical + with open(file1, 'r') as f1, open(file2, 'r') as f2: + # Skip the specified number of lines + skipped_f1 = islice(f1, lines_to_skip, None) + skipped_f2 = islice(f2, lines_to_skip, None) + with open(Path(__file__).parent.joinpath(f'diff_{file1.name}'), 'w') as f3: + f3.write('expected, test\n') + # Compare remaining lines side-by-side + for line1, line2 in zip_longest(skipped_f1, skipped_f2): + if line1 != line2: + f3.write(f'{repr(line1)}, {repr(line2)}\n') + line_diff = True # Files differ + if not line_diff: + Path(__file__).parent.joinpath(f'diff_{file1.name}').unlink() + + return line_diff + + +def setUpModule(): + # Connect to testing project. + data_dir = Path(__file__).parent.joinpath('data') + global modeller + modeller = tbm.connect(data_dir) + # Extract expected test data. + expected_file = sorted(data_dir.glob('*.zip'))[0] + global expected_dir + expected_dir = data_dir.joinpath('expected') + expected_dir.mkdir() + with zipfile.ZipFile(expected_file, 'r') as z: + z.extractall(expected_dir) + + +def tearDownModule(): + # Delete expected test data directory. + shutil.rmtree(expected_dir) + + +if __name__ == '__main__': + unittest.main() From 8c7060851edd94a0f8017a53bb874715c021e145 Mon Sep 17 00:00:00 2001 From: kcazzatoCMAP Date: Mon, 3 Aug 2026 08:40:30 -0500 Subject: [PATCH 7/9] remove DW DW PR removes tollway.data.mac from setup; script output is included in the punklink data so this is no longer needed --- Database/post_macros/tollway.data.mac | 92 --------------------------- 1 file changed, 92 deletions(-) delete mode 100644 Database/post_macros/tollway.data.mac diff --git a/Database/post_macros/tollway.data.mac b/Database/post_macros/tollway.data.mac deleted file mode 100644 index 8eb60b2..0000000 --- a/Database/post_macros/tollway.data.mac +++ /dev/null @@ -1,92 +0,0 @@ -~# tollway.data.mac -~# Craig Heither, rev. 10-23-2021 -~# -~# **************************************************************************************** -~# Macro punches network results for analysis of tollway volumes - 6 vehicle class version. -~# - timau recalculated for vdf=7 using speed from incoming link. -~# - toll collection facilities on ramps are flagged for later analysis. -~# - @atype punched to identify urban vs. rural. -~# -~# Must submit with 3-digit scenario number: -~# (e.g. "~error -~# -~o|39 -~# -~## -- Set Up Register Values -- -~t2=report\moves.rpt -~t3=data\moves.longhaul.data -~# ------------------------- -~!if exist %t2% (del %t2% /Q) -~!if exist %t3% (del %t3% /Q) -reports=%t2% -~# -~# ===================================================================== -~/ ## OBTAIN LINK DATA ## -~# -~## -- Set Register To Count Time Periods -- -~x=1 -~# -~:period -~# -~## -- Write separate file for each time period -- -~t1=data\tollway_pd%x%.data -~!if exist %t1% (del %t1% /Q) -batchout=%t1% -~# -~## -- Copy Time Period Network To Temporary Scenario -- -1.22 -~+;2;99998;~?e -~+; ;q;~$>next -~+;y;q -~:next -~##+;1.22;3;%1%2%x%;99998; ;y;q -~+;1.22;3;%x%;99998; ;y;q -~# -~## -- Set Userfields And Punch -- -2.41 -~## -- Store Time Period In tmpl1 -- -~+;1;y;tmpl1;0; ;all;4 -~+;1;y;tmpl1;%x%; ;all;4 -~# -~## -- Flag Toll Links on Ramps -- -~## -- (based on incoming and outgoing links - store flag in tmpl2) -- -~+;1;y;ui1;0; ;all;4 -~+;1;y;ui2;0; ;all;4 -~+;1;y;ui1;(vdf.eq.3 .or. vdf.eq.5 .or. vdf.eq.8); ;2;all;4 -~+;1;y;uj2;(vdf.eq.3 .or. vdf.eq.5 .or. vdf.eq.8); ;2;all;4 -~+;1;y;tmpl2;(ui2+uj1).ge.2; ;vdf=7; ;4 -~# -~/ -- Punch Link Data: Period %x% -- -~+;1;n;tmpl1+len+lan+vdf+@zone+@emcap+timau+ -@ftime+@avs1v+@avs2v+@avs3v+@avh2v+@avh3v+@avbqv+@avlqv+ -@avmqv+@avhqv+@atype+tmpl2+@busveq+@imarea+@speed -~+; ;all;3;q -~/ -~## -- Delete Temporary Scenario -- -~+;1.22;2;99998;y;q -~# -~## -- Iterate Through Time Periods -- -~x+1 -~+;~?x<9;~$period -~# -~# -~$>end -~# -~:error -~/ +++++++++++++++++++++++++++++++++++++++ -~/ SUBMIT WITH 3-DIGIT SCENARIO!!!!!!! -~/ +++++++++++++++++++++++++++++++++++++++ -~/ -~:end -~o=6 -batchout= -reports= -~/ -- end of macro -- From f0c2b061834eea1317b1e80cfd2105e57102856c Mon Sep 17 00:00:00 2001 From: kcazzatoCMAP Date: Mon, 3 Aug 2026 08:50:40 -0500 Subject: [PATCH 8/9] TO useful macros initial add initial updated scripts from TKO translated useful macros; removed old macros --- .../summarize_transit_boardings.mac | 97 ---------- .../summarize_transit_boardings.py | 177 ++++++++++++++++++ .../delete.initial.batchin.scenarios | 47 ----- Database/useful_macros/delete.scenarios | 21 --- .../delete_initial_batchin_scenarios.py | 54 ++++++ Database/useful_macros/delete_scenarios.py | 107 +++++++++++ 6 files changed, 338 insertions(+), 165 deletions(-) delete mode 100644 Database/transit_asmt_macros/summarize_transit_boardings.mac create mode 100644 Database/transit_asmt_macros/summarize_transit_boardings.py delete mode 100644 Database/useful_macros/delete.initial.batchin.scenarios delete mode 100644 Database/useful_macros/delete.scenarios create mode 100644 Database/useful_macros/delete_initial_batchin_scenarios.py create mode 100644 Database/useful_macros/delete_scenarios.py diff --git a/Database/transit_asmt_macros/summarize_transit_boardings.mac b/Database/transit_asmt_macros/summarize_transit_boardings.mac deleted file mode 100644 index 900821c..0000000 --- a/Database/transit_asmt_macros/summarize_transit_boardings.mac +++ /dev/null @@ -1,97 +0,0 @@ - -~# SUMMARIZE_TRANSIT_BOARDINGS.MAC -~# Craig Heither, 10-25-2022 -~# -~# Summarize transit boardings for congested transit assignment. -~# submit with: ~ -~# e.g.: " ~>%t3% -~"Line,Scenario,TotalBoard,PassengerMiles -~> -~# #### -~# -~:full_loop -~# -~:scenario_loop -s=%x% -~?y=1 -~+;~t1=CTA_Rail_Blue_Line;~t2=cbl___ -~?y=2 -~+;~t1=CTA_Rail_Brown_Line;~t2=cbr___ -~?y=3 -~+;~t1=CTA_Rail_Green_Line;~t2=cg____ -~?y=4 -~+;~t1=CTA_Rail_Orange_Line;~t2=cor___ -~?y=5 -~+;~t1=CTA_Rail_Pink_Line;~t2=cpk___ -~?y=6 -~+;~t1=CTA_Rail_Purple_Line;~t2=cpr___ -~?y=7 -~+;~t1=CTA_Rail_Red_Line;~t2=crd___ -~?y=8 -~+;~t1=CTA_Rail_Yellow_Line;~t2=cye___ -~?y=9 -~+;~t1=Metra_BNSF;~t2=mbn___ -~?y=10 -~+;~t1=Metra_Heritage_Corridor;~t2=mhc___ -~?y=11 -~+;~t1=Metra_Electric;~t2=mme___ -~?y=12 -~+;~t1=Metra_Milwaukee_North;~t2=mmn___ -~?y=13 -~+;~t1=Metra_Milwaukee_West;~t2=mmw___ -~?y=14 -~+;~t1=Metra_North_Central;~t2=mnc___ -~?y=15 -~+;~t1=Metra_Rock_Island;~t2=mri___ -~?y=16 -~+;~t1=Metra_SouthWest_Service;~t2=msw___ -~?y=17 -~+;~t1=Metra_UP_North;~t2=mun___ -~?y=18 -~+;~t1=Metra_UP_Northwest;~t2=mnw___ -~?y=19 -~+;~t1=Metra_UP_West;~t2=muw___ -~?y=20 -~+;~t1=NICTD_South_Shore;~t2=mss___ -~?y=21 -~+;~t1=CTA_Bus_Regular;~t2=mod=B -~?y=22 -~+;~t1=CTA_Bus_Express;~t2=mod=E -~?y=23 -~+;~t1=Pace_Bus_Regular;~t2=mod=P -~?y=24 -~+;~t1=Pace_Bus_Express;~t2=mod=Q -~?y=25 -~+;~t1=Pace_Bus_Local;~t2=mod=L -~# -~# ##-- Boardings -- ## -2.41 -~+;1;n;board; ;%t2%; ;*;5;4;ms900;allbrd;All line boards;1 -~+;1;n;voltr*len; ;%t2%; ;*;5;4;ms901;pssmle;Passenger miles;1;q -~# -~>>%t3% -~"%t1%,%s%,%ms900.0%,%ms901.0% -~> -~x+2 -~+;~?x<%z%;~$scenario_loop -~# ##-- next line --## -~+;~x-8;~y+1 -~+;~?y<26;~$full_loop -~# -~# ##-- Delete matrices --## -3.12 -~+;2;ms900;y -~+;2;ms901;y;q -~/ summary done! -q diff --git a/Database/transit_asmt_macros/summarize_transit_boardings.py b/Database/transit_asmt_macros/summarize_transit_boardings.py new file mode 100644 index 0000000..ececcc1 --- /dev/null +++ b/Database/transit_asmt_macros/summarize_transit_boardings.py @@ -0,0 +1,177 @@ +''' +SUMMARIZE_TRANSIT_BOARDINGS.PY + Craig Heither, 10-25-2022 + Translated from Emme macro to Python by Tim O'Leary, 9-26-2025 + + Summarize transit boardings for congested transit assignment. + No inputs necessary-- reads batch_file.yaml to determine scen numbers. + Will not run unless transit assignment has been completed. +''' + +#libraries +import os +import sys +from pathlib import Path +import yaml +import pandas as pd + +#startup emme +proj_dir = Path(__file__).resolve().parents[2] +db = proj_dir.joinpath('Database') +sys.path.append(str(proj_dir.joinpath('Scripts'))) +from tbmtools import project as tbm + +#connect to modeller +modeller = tbm.connect(proj_dir) +emmebank = modeller.emmebank + +#output location +out_boarding_csv = os.path.join( + db, 'transit_asmt_macros/report/Boarding_summary.csv' +) +out_transit_punch = os.path.join( + db, 'transit_asmt_macros/report/transit_punch_segment.csv' +) + +if os.path.exists(out_boarding_csv): + os.remove(out_boarding_csv) + +#get scenario info from batch_file.yaml for transit asmt scenarios +db = proj_dir.joinpath('Database') +with open(os.path.join(db, 'batch_file.yaml')) as f: + lines_without_backslashes = ''.join([line.replace('\\','/') for line in f]) + config = yaml.safe_load(lines_without_backslashes) +yr = str(config['scenario_code'])[0] # e.g., '2' from '200' + +trnt_scens = { + f'Night (6pm-6am)': f'{yr}21', + f'AM (6am-9am)': f'{yr}23', + f'Midday (9am-4pm)': f'{yr}25', + f'PM (4pm-6pm)': f'{yr}27', +} + +tr_dfs = [] + +for period_name, scen_num in trnt_scens.items(): + + scen = emmebank.scenario(scen_num) + if scen is None: + raise ValueError(f'Scenario {scen_num} not found in Emmebank') + + network = scen.get_network() + links = network.links() + + trpunch_list = [] + for link in network.links(): + segments = link.segments() #empty if link does not have transit segments + inode = network.node(link.i_node) + jnode = network.node(link.j_node) + #for each segment (if it has any), add this info to table: + for segment in segments: + line = network.transit_line(str(segment.id).split('-')[0]) + seg = [ + link.id, + segment.id, + link.length, + line.headway, + segment['transit_boardings'], + segment['transit_volume'], + segment['data1'], + inode['@zone'], + jnode['@zone'] + ] + + trpunch_list.append(seg) + + columns = ['id', 'segment_id', 'length', 'headway', 'transit_boardings','transit_volume','ltime','inode_zone','jnode_zone'] + trpunch = pd.DataFrame(data=trpunch_list, columns=columns) + + trpunch.eval('pmt = transit_volume * length', inplace=True) + trpunch.eval('pht = transit_volume * ltime / 60', inplace=True) + + #use transit line info to determine mode + def line_mode(x): + ''' + function for .map() method used directly below this function + to determine mode name from segment_id + + segment_id format: [alpha-code][5-digit number], e.g., 'b12535', 'cbl10011' + + b -> CTA local bus + e -> CTA express + p -> Pace regular bus + l -> Pace feeder bus + q -> Pace express bus + c** -> CTA rail lines (3-letter code) + m** -> Metra lines (3-letter code) + + returns: the english name representation of the mode code + + ''' + + line = x.split('-')[0] + name = ''.join([char for char in line if char.isalpha()]) + name_key = { + 'cbl': 'CTA Blue Line', + 'cbr': 'CTA Brown Line', + 'cg': 'CTA Green Line', + 'cor': 'CTA Orange Line', + 'cpk': 'CTA Pink Line', + 'cpr': 'CTA Purple Line', + 'crd': 'CTA Red Line', + 'cye': 'CTA Yellow Line', + 'mbn': 'Metra BNSF', + 'mhc': 'Metra Heritage Corridor', + 'mme': 'Metra Electric', + 'mmn': 'Metra Milwaukee North', + 'mmw': 'Metra Milwaukee West', + 'mnc': 'Metra North Central', + 'mri': 'Metra Rock Island', + 'msw': 'Metra SouthWest Service', + 'mun': 'Metra UP North', + 'mnw': 'Metra UP Northwest', + 'muw': 'Metra UP West', + 'mss': 'NICTD South Shore', + 'b': 'CTA Regular Bus', + 'e': 'CTA Express Bus', + 'p': 'Pace Regular Bus', + 'q': 'Pace Express Bus', + 'l': 'Pace Feeder Bus', + } + nam = [code for code in name_key.keys() if code in name] + nam = max(nam, key=len) if len(nam) > 0 else None + return name_key[nam] + + trpunch['mode'] = trpunch['segment_id'].map(line_mode) + + if len(trpunch.loc[trpunch['mode'].isnull()]) > 0: + print(f'SOME TRANSIT SEGMENTS WERE NOT MAPPED TO A MODE!') + print(trpunch.loc[trpunch['mode'].isnull()]) + raise ValueError('Some transit segments were not mapped to a mode (must be c, m, p, q, l, b, or e). Look at printed output of errors above.') + + trpunch['tod'] = scen_num + trpunch['tod_name'] = period_name + + tr_dfs.append(trpunch) + +tr_all = pd.concat(tr_dfs, ignore_index=True) + +#output all segment info to csv (similar to punch link) +tr_all.to_csv(out_transit_punch, index=False) + +#summary by time-of-day +summary_tod = tr_all.groupby(['tod_name', 'tod', 'mode']).agg({ + 'transit_boardings':'sum', + 'pmt':'sum' +}).reset_index() +#total daily +summary_daily = summary_tod.groupby('mode').agg({ + 'transit_boardings':'sum', + 'pmt':'sum' +}).reset_index() +summary_daily['tod'] = 'All' +summary_daily['tod_name'] = 'Daily' + +summary = pd.concat([summary_tod, summary_daily], ignore_index=True) +summary.to_csv(out_boarding_csv, index=False) +print(f'Summary transit boarding information written to: {out_boarding_csv}') \ No newline at end of file diff --git a/Database/useful_macros/delete.initial.batchin.scenarios b/Database/useful_macros/delete.initial.batchin.scenarios deleted file mode 100644 index 07f53cd..0000000 --- a/Database/useful_macros/delete.initial.batchin.scenarios +++ /dev/null @@ -1,47 +0,0 @@ -~/ delete.initial.batchin.scenarios -~/ Craig Heither, 3/31/09 -~/ Deletes scenarios -~/ submit with 3-digit scenario number -~/ -~# Heither, modified 07-26-2016: delete build_turn.rpt -~# ------------------------------------------------------------- -~x=%0% -~+;~?!x=1;~$>error -~/ -~! if exist report\build_turn.rpt (del report\build_turn.rpt) -~/ -~/ DELETE SCENARIOS xxx0,xxx00-xxx08 IF THEY EXIST -~/ -~r1=%1% -~r1*10 -1.22 -2 -%r1% -~?e -~+;~/Scenario %r1% does not exist; ;q;~$>next1 -y -q -~:next1 -~/ -~r1*10 -~/ -~:loop -1.22 -2 -%r1% -~?e -~+;~/Scenario %r1% does not exist; ;q;~$>next2 -y -q -~:next2 -~r1+1 -~+;~?r1<%1%09;~$loop -~/ END LOOP -~/ -~/ -~$>end -~:error -~/ SUBMIT WITH PROJECT DIRECTORY NAME!!!!!!! -~/ -~:end -~/end of macro diff --git a/Database/useful_macros/delete.scenarios b/Database/useful_macros/delete.scenarios deleted file mode 100644 index 7a4396a..0000000 --- a/Database/useful_macros/delete.scenarios +++ /dev/null @@ -1,21 +0,0 @@ -~/ macro for deleting a range of network scenarios -~/ -~/ r11 = counter -~/ Created by DBE 21May2004, modified kww 6/6 -~/ modified by cmh 3/09: continues running if scenario does not exist -~x=%1% -~:loop -~x+1 -~r11+1 -1.22 -2 -%x% -~?e -~+; ;q;~$>next -yes -q -~:next -~?!x=%2% -~$loop -~:done -s= \ No newline at end of file diff --git a/Database/useful_macros/delete_initial_batchin_scenarios.py b/Database/useful_macros/delete_initial_batchin_scenarios.py new file mode 100644 index 0000000..6e5a7e1 --- /dev/null +++ b/Database/useful_macros/delete_initial_batchin_scenarios.py @@ -0,0 +1,54 @@ +''' +delete_initial_batchin_scenarios.py + +author: Craig Heither, 3/31/09 + - deletes scenarios + - submit with 3-digit scenario number + - modified 7/26/2016: also delete build.turn.rpt +translated: Tim O'Leary, 9/26/2025 to Python 3 + - reads batch_file.yaml to retrieve 3-digit scenario number +''' + +import os +import sys +from pathlib import Path +import yaml + +proj_dir = Path(__file__).resolve().parents[2] +sys.path.append(str(proj_dir.joinpath('Scripts'))) +from tbmtools import project as tbm + +print('delete initial batchin scenarios (*0000-*0008)') +print('(and build_turn.rpt if it exists)') +print(' executing...') + +#connect to modeller +modeller = tbm.connect(proj_dir) +emmebank = modeller.emmebank + +#define tools +delete_scenario = modeller.tool('inro.emme.data.scenario.delete_scenario') + +#get scenario info from batch_file.yaml for scenario numbers to delete +db = proj_dir.joinpath('Database') +with open(os.path.join(db, 'batch_file.yaml')) as f: + lines_without_backslashes = ''.join([line.replace('\\','/') for line in f]) + config = yaml.safe_load(lines_without_backslashes) +scen_yr = config['scenario_code'] # e.g., '200' +scenarios_to_delete = [f'{scen_yr}0{i}' for i in range(0,9)] + [f'{scen_yr}0'] + +#delete scenarios +for scen in scenarios_to_delete: + if emmebank.scenario(scen): #returns None if DNE + delete_scenario(emmebank.scenario(scen)) + else: + print(f'scenario {scen} does not exist') + +#delete build_turn.rpt if it exists +rptfile = db.joinpath('report/build_turn.rpt') +if os.path.exists(rptfile): + os.remove(rptfile) +else: + print('build_turn.rpt does not exist') + +print('done') \ No newline at end of file diff --git a/Database/useful_macros/delete_scenarios.py b/Database/useful_macros/delete_scenarios.py new file mode 100644 index 0000000..d30b5cf --- /dev/null +++ b/Database/useful_macros/delete_scenarios.py @@ -0,0 +1,107 @@ +''' +`delete_scenarios.py` +Author: Tim O'Leary +Date: 9/26/2025 + +Description: This script deletes specified scenarios from the emme databank. It can +be called as a subscript or by itself. Contains delete_scenarios() function that can take +a string or a list of strings as input. + +Translated to Python 3 from 'delete.scenarios' Emme macro +Written by DBE 5/2004 and modified to handle non-existent scenarios by CH 3/2009 +Added functionality with Python translation: +- Can call a single scenario, a range of scenarios, or list +- Runs interactively with user input if called by itself, or the function 'delete_scenarios()' can be imported and called by another script +''' + +print('delete_scenarios.py\nstarting up...') +import os +import sys +from pathlib import Path +import textwrap + +proj_dir = Path(__file__).resolve().parents[2] +sys.path.append(str(proj_dir.joinpath('Scripts'))) +from tbmtools import project as tbm + +#connect to modeller +modeller = tbm.connect(proj_dir) +emmebank = modeller.emmebank + +#define tools +delete_scenario = modeller.tool('inro.emme.data.scenario.delete_scenario') + +def delete_scenarios(scens_to_delete): + ''' + Deletes specified scenarios from the databank. + Input may be one of the following: + - a single scenario (e.g., '20003') + - a range of scenarios separated by a hyphen (e.g., '20021-20028') + - a list of scenarios (e.g., '221, 22003, 22004')) + ''' + + bad_input_msg = textwrap.dedent(f'''\ + Input not recognized. Please input scenario IDs as integers in one of the following formats: + \t- a single scenario ID (e.g., '20003'); + \t- a range of IDs with hyphen between (e.g., '20021-20028'); or + \t- a list of IDs (e.g., '221, 22003, 22004').''') + + #if passed function with no input + if scens_to_delete is None or scens_to_delete == '': + raise ValueError(bad_input_msg) + + #check for incorrectly formatted input + if '-' in str(scens_to_delete): + input_type = 'range' + scens_check = str(scens_to_delete).split('-') + scens_check = [m.strip() for m in scens_check] + else: + input_type = 'list' + scens_check = str(scens_to_delete).replace('[','').replace(']','').split(',') + scens_check = [m.strip() for m in scens_check] + + #make sure prefixes and suffixes aren't invalid, and check if mixed matrix types + not_a_number = [m for m in scens_check if not m.isnumeric()] + if not_a_number: + raise ValueError(bad_input_msg) + + #handle range input + if input_type == 'range': + min = int(scens_check[0]) + max = int(scens_check[1]) + if min >= max: + raise ValueError('Your min should be less than your max.') + scens = [m for m in range(min, max+1)] + else: + scens = scens_check + + #delete matrices + for scen in scens: + if emmebank.scenario(scen): + delete_scenario(emmebank.scenario(scen)) + else: + print(f' - scenario {scen} not in dabatank, skipping...') + +print(''' + DELETE_SCENARIOS.PY + This script will delete specified scenarios from the emme databank! + Input may be one of the following: + - a single scenario ID (e.g., '20027') + - a range of scenarios, min and max separated by a hyphen (e.g., '221-227') + - a list of scenario IDs, separated by commas (e.g., '5, 8, 20005, 20008')) + ''') + +while True: + try: + user_input = input('Enter scenarios to delete (or "exit" to quit): ') + if user_input.lower() == 'exit': + break + delete_scenarios(user_input) + print('\n - done! want to go again?\n') + except ValueError as e: + print(f'\n!! Error: {e}\n') + except Exception as e: + print(f'\n!! Unexpected error occurred: {e}\n') +print('bye-bye, then!') +input('press any key to exit') +sys.exit() From e5be1a40c286693f942cce557ddd51c48bb70d98 Mon Sep 17 00:00:00 2001 From: kcazzatoCMAP Date: Mon, 3 Aug 2026 10:09:07 -0500 Subject: [PATCH 9/9] uv update run_transit run_transit_assignment.bat was never updated to uv; doing that and adding summarize_transit_boardings.py to full submit --- Database/Submit_Full_Regional_Model_SOLA.bat | 7 ++- .../run_transit_assignment.bat | 50 +++++++++++-------- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/Database/Submit_Full_Regional_Model_SOLA.bat b/Database/Submit_Full_Regional_Model_SOLA.bat index 370a534..ea9827d 100644 --- a/Database/Submit_Full_Regional_Model_SOLA.bat +++ b/Database/Submit_Full_Regional_Model_SOLA.bat @@ -382,14 +382,19 @@ if "%transitAsmt%" EQU "T" ( uv run transit_asmt_macros/setup_transit_asmt_2_initialize_matrices.py %file1% %RSPrun% if %ERRORLEVEL% NEQ 0 (goto issue) REM -- Fill matrices with demand (point to conda environment) - uv run transit_asmt_macros\setup_transit_asmt_3_TOD_transit_demand.py %RSPrun% + uv run transit_asmt_macros/setup_transit_asmt_3_TOD_transit_demand.py %RSPrun% if %ERRORLEVEL% NEQ 0 (goto issue) @ECHO End Transit Assignment setup >> model_run_timestamp.txt @ECHO Submit Transit Assignment >> model_run_timestamp.txt cd transit_asmt_macros uv run cmap_transit_assignment_runner.py %file1% 1 %val% if %ERRORLEVEL% GTR 0 (goto issue) + REM -- Summarize transit boardings cd .. + set /a val21=%val%+21 + uv run transit_asmt_macros\summarize_transit_boardings.py %val21% + if %ERRORLEVEL% GTR 0 (goto issue) + @echo. REM -- Delete transit assignment matrices uv run transit_asmt_macros\delete_transit_skims.py %file1% if %ERRORLEVEL% GTR 0 (goto issue) diff --git a/Database/transit_asmt_macros/run_transit_assignment.bat b/Database/transit_asmt_macros/run_transit_assignment.bat index f143a5a..8829931 100644 --- a/Database/transit_asmt_macros/run_transit_assignment.bat +++ b/Database/transit_asmt_macros/run_transit_assignment.bat @@ -15,18 +15,18 @@ REM Heither, rev. 10-21-2024 (updated for c24q4) @echo - RSP: set to True [it doesn't matter if it is an actual RSP, this merely sets a flag] @echo ------------------------------------------------------------------------------------------------- -cd %~dp0 cd .. +@echo %cd% echo. rem -- Read model run settings from batch_file.yaml -- for /f "eol=# skip=2 tokens=2 delims=:" %%a in (batch_file.yaml) do (set val=%%a & goto break1) :break1 -for /f "eol=# skip=10 tokens=2 delims=:" %%f in (batch_file.yaml) do (set transitAsmt=%%f & goto break2) -:break2 -for /f "eol=# skip=12 tokens=2 delims=:" %%h in (batch_file.yaml) do (set selLineFile=%%h & goto break4) +for /f "eol=# skip=10 tokens=2 delims=:" %%f in (batch_file.yaml) do (set transitAsmt=%%f & goto break4) :break4 -for /f "eol=# skip=18 tokens=2 delims=:" %%k in (batch_file.yaml) do (set RSPrun=%%k & goto break5) +for /f "eol=# skip=12 tokens=2 delims=:" %%i in (batch_file.yaml) do (set selLineFile=%%i & goto break5) :break5 +for /f "eol=# skip=18 tokens=2 delims=:" %%l in (batch_file.yaml) do (set RSPrun=%%l & goto break8) +:break8 set val=%val:~1,3% set transitAsmt=%transitAsmt:~1,1% @@ -59,34 +59,42 @@ echo file1 = %file1% call :CheckEmpty %infile% :filepass if exist %infile% (del %infile% /Q) -cd Database/transit_asmt_macros +cd Database if exist usemacro_* (del usemacro_* /Q) -rem Activate Emme env -call %~dp0..\..\Scripts\manage\env\activate_env.cmd emme - -REM -- Submit with name of .emp file -python cmap_transit_assignment_runner.py %file1% 1 %val% +@ECHO Begin Transit Assignment setup: %date% %time% >> model_run_timestamp.txt +REM -- Create matrices to hold TOD transit demand +if "%RSPrun%" EQU "T" (@ECHO -- Creating HBW transit demand matrices >> model_run_timestamp.txt) +uv run transit_asmt_macros/setup_transit_asmt_2_initialize_matrices.py %file1% %RSPrun% +if %ERRORLEVEL% NEQ 0 (goto issue) +REM -- Fill matrices with demand (point to conda environment) +uv run transit_asmt_macros/setup_transit_asmt_3_TOD_transit_demand.py %RSPrun% +if %ERRORLEVEL% NEQ 0 (goto issue) +@ECHO End Transit Assignment setup >> model_run_timestamp.txt +@ECHO Submit Transit Assignment >> model_run_timestamp.txt +cd transit_asmt_macros +uv run cmap_transit_assignment_runner.py %file1% 1 %val% +if %ERRORLEVEL% GTR 0 (goto issue) REM -- Summarize transit boardings cd .. set /a val21=%val%+21 -call emme -ng 000 -m transit_asmt_macros/summarize_transit_boardings.mac %val21% -echo. -REM -- Delete transit assignment matrices -python transit_asmt_macros/delete_transit_skims.py %file1% +uv run transit_asmt_macros\summarize_transit_boardings.py %val21% +if %ERRORLEVEL% GTR 0 (goto issue) @echo. -@echo MATRICES DELETED. - +REM -- Delete transit assignment matrices +uv run transit_asmt_macros\delete_transit_skims.py %file1% +if %ERRORLEVEL% GTR 0 (goto issue) if "%check2%" NEQ "None" ( REM -- Run select line analysis - call python transit_asmt_macros\transit_select_line.py %file1% %val% %selLineFile% + uv run transit_asmt_macros\transit_select_line.py %file1% %val% %selLineFile% if %ERRORLEVEL% GTR 0 (goto issue) - @ECHO -- Completed Select Line Analysis + @ECHO -- Completed Select Line Analysis >> model_run_timestamp.txt REM -- Summarize select line boardings - call python transit_asmt_macros\select_line_boardings.py %file1% %val% %RSPrun% %selLineFile% + uv run transit_asmt_macros\select_line_boardings.py %file1% %val% %RSPrun% %selLineFile% if %ERRORLEVEL% GTR 0 (goto issue) - @ECHO -- Completed Select Line Boarding Analysis + @ECHO -- Completed Select Line Boarding Analysis >> model_run_timestamp.txt ) +@ECHO End Transit Assignment: %date% %time% >> model_run_timestamp.txt goto last REM ======================================================================