Skip failed sampling choices

activitysim/abm/models/joint_tour_destination.py

@@ -7,11 +7,10 @@
 import pandas as pd
 
 from activitysim.abm.models.util import tour_destination
-from activitysim.core import estimation, los, tracing, workflow, expressions
+from activitysim.core import estimation, expressions, los, tracing, workflow
 from activitysim.core.configuration.logit import TourLocationComponentSettings
 from activitysim.core.util import assign_in_place
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -94,7 +93,6 @@ def joint_tour_destination(
         state.settings.downcast_int,
         state.settings.downcast_float,
     )
-    state.add_table("tours", tours)
 
     if want_logsums:
         joint_tours[logsum_column_name] = choices_df["logsum"]
@@ -105,6 +103,10 @@ def joint_tour_destination(
             state.settings.downcast_float,
         )
 
+    # add_table drops the tours of households skipped under skip_failed_choices,
+    # so assign every column first
+    state.add_table("tours", tours)
+
     tracing.print_summary("destination", joint_tours.destination, describe=True)
 
     if want_sample_table:

activitysim/abm/models/location_choice.py

@@ -463,6 +463,9 @@ def location_presample(
         n_total_alts=n_total_alts,
     )
 
+    if state.settings.skip_failed_choices and taz_sample.empty:
+        return taz_sample.rename(columns={DEST_TAZ: alt_dest_col_name})
+
     # print(f"taz_sample\n{taz_sample}")
     #            dest_TAZ      prob  pick_count
     # person_id
@@ -869,6 +872,12 @@ def run_location_choice(
             ),
         )
 
+        if state.settings.skip_failed_choices:
+            # Failed sampling choices have no alternatives for logsums or simulation.
+            choosers = choosers[choosers.index.isin(location_sample_df.index)]
+            if choosers.empty:
+                continue
+
         # - location_logsums
         # skip logsum calculations if LOGSUM_SETTINGS is None
         if model_settings.LOGSUM_SETTINGS:

activitysim/abm/models/non_mandatory_destination.py

@@ -7,7 +7,7 @@
 import pandas as pd
 
 from activitysim.abm.models.util import tour_destination
-from activitysim.core import estimation, los, tracing, workflow, expressions
+from activitysim.core import estimation, expressions, logit, los, tracing, workflow
 from activitysim.core.configuration.logit import TourLocationComponentSettings
 from activitysim.core.util import assign_in_place
 
@@ -123,9 +123,12 @@ def non_mandatory_tour_destination(
             state.settings.downcast_float,
         )
 
-    assert all(
-        ~tours["destination"].isna()
-    ), f"Tours are missing destination: {tours[tours['destination'].isna()]}"
+    # tours of households skipped under skip_failed_choices have no destination;
+    # add_table drops them
+    missing = tours["destination"].isna() & ~tours["household_id"].isin(
+        logit.skipped_household_ids(state)
+    )
+    assert not missing.any(), f"Tours are missing destination: {tours[missing]}"
 
     state.add_table("tours", tours)
 

activitysim/abm/models/util/tour_destination.py

@@ -15,6 +15,7 @@
     config,
     estimation,
     expressions,
+    logit,
     los,
     simulate,
     tracing,
@@ -175,7 +176,9 @@ def _destination_sample(
 
     # remember person_id in chosen alts so we can merge with persons in subsequent steps
     # (broadcasts person_id onto all alternatives sharing the same tour_id index value)
-    choices[chooser_id_column] = choosers[chooser_id_column]
+    # reindex: assigning a Series to an empty sample would create rows for
+    # choosers whose sampling failed
+    choices[chooser_id_column] = choosers[chooser_id_column].reindex(choices.index)
 
     return choices
 
@@ -664,6 +667,10 @@ def destination_presample(
         n_total_alts=len(full_taz_index) if full_taz_index is not None else None,
     )
 
+    if state.settings.skip_failed_choices and taz_sample.empty:
+        # every chooser failed sampling, so there is no TAZ to choose a MAZ for
+        return taz_sample.rename(columns={DEST_TAZ: alt_dest_col_name})
+
     # choose a MAZ for each DEST_TAZ choice, choice probability based on MAZ size_term fraction of TAZ total
     maz_choices = choose_MAZ_for_TAZ(
         state,
@@ -749,7 +756,7 @@ def run_destination_sample(
 
     # remember person_id in chosen alts so we can merge with persons in subsequent steps
     # (broadcasts person_id onto all alternatives sharing the same tour_id index value)
-    choices[chooser_id_column] = tours[chooser_id_column]
+    choices[chooser_id_column] = tours[chooser_id_column].reindex(choices.index)
 
     return choices
 
@@ -1027,6 +1034,19 @@ def run_tour_destination(
             trace_label=tracing.extend_trace_label(segment_trace_label, "sample"),
         )
 
+        if state.settings.skip_failed_choices:
+            # Failed sampling choices have no alternatives for logsums or
+            # simulation. Record their households (the sample choosers carry no
+            # household_id) so add_table drops them with the rest of the household.
+            failed = ~choosers.index.isin(location_sample_df.index)
+            if failed.any():
+                logit.record_skipped_households(
+                    state, choosers.loc[failed, "household_id"], segment_trace_label
+                )
+                choosers = choosers[~failed]
+            if choosers.empty:
+                continue
+
         # - destination_logsums
         # if LOGSUM_SETTINGS is set to 'None', we skip this step
         if model_settings.LOGSUM_SETTINGS:

activitysim/abm/models/util/tour_od.py

@@ -683,6 +683,16 @@ def od_presample(
         trace_label,
     )
 
+    alt_dest_col_name = model_settings.ALT_DEST_COL_NAME
+    chooser_orig_col_name = model_settings.CHOOSER_ORIG_COL_NAME
+
+    if state.settings.skip_failed_choices and orig_MAZ_dest_TAZ_sample.empty:
+        # every chooser failed sampling, so there is no TAZ to choose a MAZ for
+        return pd.DataFrame(
+            columns=[alt_dest_col_name, chooser_orig_col_name, "prob", "pick_count"],
+            index=orig_MAZ_dest_TAZ_sample.index,
+        )
+
     orig_MAZ_dest_TAZ_sample[ORIG_MAZ] = (
         orig_MAZ_dest_TAZ_sample[alt_od_col_name].str.split("_").str[0].astype(int)
     )
@@ -705,8 +715,6 @@ def od_presample(
     # outputs
     assert DEST_MAZ in maz_choices
 
-    alt_dest_col_name = model_settings.ALT_DEST_COL_NAME
-    chooser_orig_col_name = model_settings.CHOOSER_ORIG_COL_NAME
     maz_choices = maz_choices.rename(
         columns={DEST_MAZ: alt_dest_col_name, ORIG_MAZ: chooser_orig_col_name}
     )
@@ -1161,6 +1169,21 @@ def run_tour_od(
             ),
         )
 
+        if state.settings.skip_failed_choices:
+            # Failed sampling choices have no alternatives for logsums or
+            # simulation; record their households so add_table drops them.
+            failed = ~choosers.index.isin(od_sample_df.index)
+            if failed.any():
+                if "household_id" in choosers:
+                    logit.record_skipped_households(
+                        state,
+                        choosers.loc[failed, "household_id"],
+                        tracing.extend_trace_label(trace_label, segment_name),
+                    )
+                choosers = choosers[~failed]
+            if choosers.empty:
+                continue
+
         if model_settings.ORIG_FILTER == "original_MAZ > 0":
             pass
         elif model_settings.ORIG_FILTER == "external_TAZ > 0":
@@ -1232,6 +1255,13 @@ def run_tour_od(
 
     if len(choices_list) > 0:
         choices_df = pd.concat(choices_list)
+    else:
+        # every segment was empty, e.g. all of its choosers failed sampling
+        logger.warning("%s no choices", trace_label)
+        choices_df = pd.DataFrame(
+            columns=["choice", origin_col_name, model_settings.DEST_COL_NAME]
+            + (["logsum"] if want_logsums else [])
+        )
 
     if len(sample_list) > 0:
         save_sample_df = pd.concat(sample_list)

activitysim/abm/test/test_location_choice.py

@@ -5,6 +5,7 @@
 
 import pandas as pd
 import pandas.testing as pdt
+import pytest
 
 from activitysim.abm.models import location_choice
 
@@ -58,6 +59,7 @@ def test_estimation_override_preserves_destination_choice_logsum(monkeypatch):
     state = SimpleNamespace(
         settings=SimpleNamespace(
             sample_method="inverse_cdf",
+            skip_failed_choices=False,
             trace_hh_id=None,
             use_explicit_error_terms=False,
         )
@@ -88,3 +90,69 @@ def test_estimation_override_preserves_destination_choice_logsum(monkeypatch):
     )
     pdt.assert_frame_equal(choices, expected)
     assert sample is None
+
+
+@pytest.mark.parametrize("sampled_person_ids", [[], [2, 2]])
+def test_location_choice_skips_failed_samples(monkeypatch, sampled_person_ids):
+    """Only persons with sampled alternatives reach logsums and final choice."""
+    persons = pd.DataFrame(
+        {"segment": [1, 1]}, index=pd.Index([1, 2], name="person_id")
+    )
+    location_sample = pd.DataFrame(
+        {"alt_dest": [101, 202][: len(sampled_person_ids)]},
+        index=pd.Index(sampled_person_ids, name="person_id"),
+    )
+    expected_persons = persons.loc[persons.index.isin(sampled_person_ids)]
+    sample_model = Mock(return_value=location_sample)
+    logsums_model = Mock(
+        return_value=location_sample.assign(**{location_choice.ALT_LOGSUM: 0.0})
+    )
+    simulate_model = Mock(
+        return_value=pd.DataFrame({"choice": 101}, index=expected_persons.index)
+    )
+    monkeypatch.setattr(location_choice, "run_location_sample", sample_model)
+    monkeypatch.setattr(location_choice, "run_location_logsums", logsums_model)
+    monkeypatch.setattr(location_choice, "run_location_simulate", simulate_model)
+    shadow_price_calculator = Mock()
+    shadow_price_calculator.dest_size_terms.return_value = pd.Series(
+        [1.0, 1.0], index=[101, 202]
+    )
+    model_settings = SimpleNamespace(
+        ALT_DEST_COL_NAME="alt_dest",
+        CHOOSER_SEGMENT_COLUMN_NAME="segment",
+        LOGSUM_SETTINGS="tour_mode_choice.yaml",
+        SEGMENT_IDS={"workers": 1},
+    )
+    state = SimpleNamespace(
+        settings=SimpleNamespace(
+            skip_failed_choices=True, use_explicit_error_terms=True
+        )
+    )
+
+    choices, sample = location_choice.run_location_choice(
+        state=state,
+        persons_merged_df=persons,
+        network_los=Mock(),
+        shadow_price_calculator=shadow_price_calculator,
+        want_logsums=False,
+        want_sample_table=True,
+        estimator=None,
+        model_settings=model_settings,
+        chunk_size=0,
+        chunk_tag="workplace_location",
+        trace_label="workplace_location",
+    )
+
+    if expected_persons.empty:
+        assert choices.empty
+        assert sample is None
+        logsums_model.assert_not_called()
+        simulate_model.assert_not_called()
+    else:
+        pdt.assert_index_equal(choices.index, expected_persons.index)
+        pdt.assert_frame_equal(logsums_model.call_args.args[2], expected_persons)
+        pdt.assert_frame_equal(simulate_model.call_args.args[2], expected_persons)
+        pdt.assert_index_equal(
+            sample.index.get_level_values("person_id").unique(),
+            expected_persons.index,
+        )

activitysim/abm/test/test_misc/test_location_choice_sampling.py

@@ -1,5 +1,8 @@
 from __future__ import annotations
 
+from types import SimpleNamespace
+from unittest.mock import Mock
+
 import pandas as pd
 
 from activitysim.abm.models import location_choice
@@ -23,6 +26,58 @@ def get_skim_dict(self, layer):
         return _DummySkimDict()
 
 
+def test_location_presample_skips_empty_taz_sample(monkeypatch):
+    state = workflow.State().default_settings()
+    state.settings.skip_failed_choices = True
+    state.settings.use_explicit_error_terms = True
+    taz_sample = pd.DataFrame(
+        columns=[location_choice.DEST_TAZ, "prob", "pick_count"],
+        index=pd.Index([], name="person_id"),
+    )
+    monkeypatch.setattr(
+        location_choice, "_location_sample", Mock(return_value=taz_sample)
+    )
+    monkeypatch.setattr(
+        location_choice.shadow_pricing,
+        "load_shadow_price_calculator",
+        Mock(return_value=SimpleNamespace(use_shadow_pricing=False)),
+    )
+    choose_maz = Mock()
+    monkeypatch.setattr(
+        location_choice.tour_destination, "choose_MAZ_for_TAZ", choose_maz
+    )
+    persons = pd.DataFrame(
+        {location_choice.HOME_MAZ: [9001], location_choice.HOME_TAZ: [90]},
+        index=pd.Index([1001], name="person_id"),
+    )
+    size_terms = pd.DataFrame(
+        {
+            "size_term": [1.0, 2.0],
+            "shadow_price_size_term_adjustment": [1.0, 1.0],
+            "shadow_price_utility_adjustment": [0.0, 0.0],
+        },
+        index=pd.Index([101, 102], name="zone_id"),
+    )
+
+    out = location_choice.location_presample(
+        state,
+        "workers",
+        persons,
+        _DummyNetworkLos({101: 1, 102: 2}),
+        size_terms,
+        estimator=None,
+        model_settings=SimpleNamespace(ALT_DEST_COL_NAME="zone_id"),
+        chunk_size=0,
+        chunk_tag="workplace_location",
+        trace_label="workplace_location",
+    )
+
+    pd.testing.assert_frame_equal(
+        out, taz_sample.rename(columns={location_choice.DEST_TAZ: "zone_id"})
+    )
+    choose_maz.assert_not_called()
+
+
 def test_location_presample_uses_taz_stable_mapping(monkeypatch):
     captured = {}