From 440fd322a58fd9b754a1cb76ff45223f98d7f0f7 Mon Sep 17 00:00:00 2001 From: Sijia Wang Date: Tue, 16 Jun 2026 15:36:00 -0400 Subject: [PATCH 1/5] take segmentation from model setting --- activitysim/abm/tables/shadow_pricing.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/activitysim/abm/tables/shadow_pricing.py b/activitysim/abm/tables/shadow_pricing.py index 04c5eafc21..ee7c13ed62 100644 --- a/activitysim/abm/tables/shadow_pricing.py +++ b/activitysim/abm/tables/shadow_pricing.py @@ -176,6 +176,7 @@ def __init__( ) self.model_selector = model_settings.MODEL_SELECTOR + self.chooser_segment_column = model_settings.CHOOSER_SEGMENT_COLUMN_NAME if (self.num_processes > 1) and not state.settings.fail_fast: # if we are multiprocessing, then fail_fast should be true or we will wait forever for failed processes @@ -826,9 +827,24 @@ def update_shadow_prices(self, state): sampled_persons = pd.DataFrame() persons_merged = state.get_dataframe("persons_merged") - # need to join the segment to the choices to sample correct persons - segment_to_name_dict = self.shadow_settings.SEGMENT_TO_NAME - segment_name = segment_to_name_dict[self.model_selector] + # Use the model chooser segmentation to keep shadow-pricing resampling + # consistent with segment_ids in location choice settings. + segment_name = self.chooser_segment_column + if segment_name not in persons_merged.columns: + raise SystemConfigurationError( + f"Missing chooser segment column '{segment_name}' in persons_merged " + f"for {self.model_selector} simulation shadow pricing" + ) + + # Fail fast on obvious misconfiguration instead of silently sampling no one. + segment_values = set(self.segment_ids.values()) + chooser_values = set(persons_merged[segment_name].dropna().unique()) + if not segment_values.intersection(chooser_values): + raise SystemConfigurationError( + f"No overlap between SEGMENT_IDS values ({sorted(segment_values)}) and " + f"persons_merged['{segment_name}'] values for {self.model_selector} " + "simulation shadow pricing" + ) if type(self.choices_synced) != pd.DataFrame: self.choices_synced = self.choices_synced.to_frame() From f82f6e152b33583ec114e207e607fa0bbccf4d51 Mon Sep 17 00:00:00 2001 From: Sijia Wang Date: Tue, 16 Jun 2026 16:38:32 -0400 Subject: [PATCH 2/5] deprecate `SEGMENT_TO_NAME` --- activitysim/abm/tables/shadow_pricing.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/activitysim/abm/tables/shadow_pricing.py b/activitysim/abm/tables/shadow_pricing.py index ee7c13ed62..0cfe1f081d 100644 --- a/activitysim/abm/tables/shadow_pricing.py +++ b/activitysim/abm/tables/shadow_pricing.py @@ -62,12 +62,6 @@ TALLY_CHECKOUT = (1, -1) TALLY_PENDING_PERSONS = (2, -1) -default_segment_to_name_dict = { - # model_selector : persons_segment_name - "school": "school_segment", - "workplace": "income_segment", -} - def size_table_name(model_selector): """ @@ -134,12 +128,6 @@ class ShadowPriceSettings(PydanticReadable, extra="forbid"): WRITE_ITERATION_CHOICES: bool = False - SEGMENT_TO_NAME: dict[str, str] = { - "school": "school_segment", - "workplace": "income_segment", - } # pydantic uses deep copy, so mutable default value is ok here - """Mapping from model_selector to persons_segment_name.""" - class ShadowPriceCalculator: def __init__( From 7b8d25697d62e74525c2e30391feb5a54c3a0fcb Mon Sep 17 00:00:00 2001 From: Sijia Wang Date: Mon, 21 Sep 2026 12:17:35 -0400 Subject: [PATCH 3/5] add unit test for customized segments, and non-overlapping segment values --- .../test_misc/test_shadow_pricing_simulate.py | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/activitysim/abm/test/test_misc/test_shadow_pricing_simulate.py b/activitysim/abm/test/test_misc/test_shadow_pricing_simulate.py index 22de62a6d8..3307df285a 100644 --- a/activitysim/abm/test/test_misc/test_shadow_pricing_simulate.py +++ b/activitysim/abm/test/test_misc/test_shadow_pricing_simulate.py @@ -10,6 +10,7 @@ from activitysim.abm.tables import shadow_pricing from activitysim.core import workflow, los from activitysim.core.configuration.logit import TourLocationComponentSettings +from activitysim.core.exceptions import SystemConfigurationError from activitysim.abm.models.location_choice import run_location_choice @@ -341,6 +342,9 @@ def persons() -> pd.DataFrame: } ) + persons["school_segment_string"] = persons["school_segment"].astype(str) + persons["school_segment_string_99"] = "99" + return persons @@ -578,3 +582,111 @@ def test_shadow_pricing_simulate(state, model_settings, network_los): choices_df.index ), ) + + +def test_shadow_pricing_simulate_custom_segment(state, model_settings, network_los): + """Run simulation shadow pricing with string-valued chooser segments.""" + segment_ids = { + "university": "3", + "highschool": "2", + "gradeschool": "1", + } + custom_model_settings = model_settings.model_copy( + update={ + "CHOOSER_SEGMENT_COLUMN_NAME": "school_segment_string", + "SEGMENT_IDS": segment_ids, + } + ) + custom_model_settings.LOGSUM_SETTINGS = None + + spc = shadow_pricing.load_shadow_price_calculator(state, custom_model_settings) + + max_iterations = 5 + chooser_segment_column = "school_segment_string" + save_sample_df = choices_df = None + persons_merged = state.get_dataframe("persons_merged") + + for iteration in range(1, max_iterations + 1): + old_shadow_prices = spc.shadow_prices["highschool"].values + persons_merged_df_ = persons_merged.copy() + + if spc.use_shadow_pricing and iteration > 1: + spc.update_shadow_prices(state) + + if spc.shadow_settings.SHADOW_PRICE_METHOD == "simulation": + persons_merged_df_ = persons_merged_df_[ + persons_merged_df_.index.isin(spc.sampled_persons.index) + ].sort_index() + + choices_df_, save_sample_df = run_location_choice( + state, + persons_merged_df_, + network_los, + shadow_price_calculator=spc, + want_logsums=False, + want_sample_table=False, + estimator=None, + model_settings=custom_model_settings, + chunk_size=0, + chunk_tag="school_location_string_segment", + trace_label=f"school_location_string_segment_{iteration}", + ) + + if spc.use_shadow_pricing: + if ( + spc.shadow_settings.SHADOW_PRICE_METHOD == "simulation" + and iteration > 1 + ): + if len(choices_df_) != 0: + choices_df = pd.concat([choices_df, choices_df_], axis=0) + choices_df_index = choices_df_.index.name + choices_df = choices_df.reset_index() + choices_df = choices_df.drop_duplicates( + subset=[choices_df_index], keep="last" + ) + choices_df = choices_df.set_index(choices_df_index).sort_index() + else: + choices_df = choices_df_.copy() + + new_shadow_prices = spc.shadow_prices["highschool"].values + assert not any((old_shadow_prices == -999) & (new_shadow_prices != -999)) + check_shadow_prices(spc, iteration) + + spc.set_choices( + choices=choices_df["choice"], + segment_ids=persons_merged[chooser_segment_column].reindex( + choices_df.index + ), + ) + + +def test_shadow_pricing_simulate_segment_values_do_not_overlap(state, model_settings): + custom_segment_column = "school_segment_string_99" + custom_model_settings = model_settings.model_copy( + update={ + "CHOOSER_SEGMENT_COLUMN_NAME": custom_segment_column, + "SEGMENT_IDS": { + "university": "3", + "highschool": "2", + "gradeschool": "1", + }, + } + ) + persons_merged = state.get_dataframe("persons_merged") + state.settings.use_shadow_pricing = True + + try: + spc = shadow_pricing.load_shadow_price_calculator(state, custom_model_settings) + choices = pd.Series(22660, index=persons_merged.index, name="choice", dtype=int) + spc.set_choices(choices, persons_merged[custom_segment_column]) + + with pytest.raises(SystemConfigurationError) as error: + spc.update_shadow_prices(state) + + assert str(error.value) == ( + "No overlap between SEGMENT_IDS values (['1', '2', '3']) and " + "persons_merged['school_segment_string_99'] values for school " + "simulation shadow pricing" + ) + finally: + persons_merged.pop(custom_segment_column) From 9021cfbb71f4f22939c904cb923d9ba246c143b2 Mon Sep 17 00:00:00 2001 From: Sijia Wang Date: Wed, 21 Oct 2026 17:24:59 -0400 Subject: [PATCH 4/5] address copilot review comments --- activitysim/abm/tables/shadow_pricing.py | 33 ++++++++++-- .../test_misc/test_shadow_pricing_simulate.py | 53 ++++++++++++++++++- 2 files changed, 79 insertions(+), 7 deletions(-) diff --git a/activitysim/abm/tables/shadow_pricing.py b/activitysim/abm/tables/shadow_pricing.py index 9692ab8e0a..9bbecd4c21 100644 --- a/activitysim/abm/tables/shadow_pricing.py +++ b/activitysim/abm/tables/shadow_pricing.py @@ -6,11 +6,13 @@ import logging import multiprocessing import time +import warnings from collections import OrderedDict from typing import Any, Literal import numpy as np import pandas as pd +from pydantic import model_validator from activitysim.abm.tables.size_terms import size_terms as get_size_terms from activitysim.abm.tables.size_terms import tour_destination_size_terms @@ -128,6 +130,21 @@ class ShadowPriceSettings(PydanticReadable, extra="forbid"): WRITE_ITERATION_CHOICES: bool = False + SEGMENT_TO_NAME: dict[str, str] | None = None + """Deprecated compatibility setting; chooser segmentation comes from model settings.""" + + @model_validator(mode="before") + @classmethod + def warn_deprecated_segment_to_name(cls, data: Any) -> Any: + if isinstance(data, dict) and "SEGMENT_TO_NAME" in data: + warnings.warn( + "SEGMENT_TO_NAME is deprecated and ignored; use " + "CHOOSER_SEGMENT_COLUMN_NAME in location choice settings instead", + DeprecationWarning, + stacklevel=2, + ) + return data + class ShadowPriceCalculator: def __init__( @@ -867,19 +884,25 @@ def update_shadow_prices(self, state): f"for {self.model_selector} simulation shadow pricing" ) + if type(self.choices_synced) != pd.DataFrame: + self.choices_synced = self.choices_synced.to_frame() + # Fail fast on obvious misconfiguration instead of silently sampling no one. segment_values = set(self.segment_ids.values()) - chooser_values = set(persons_merged[segment_name].dropna().unique()) + chooser_values = set( + persons_merged[segment_name] + .reindex(self.choices_synced.index) + .dropna() + .unique() + ) if not segment_values.intersection(chooser_values): raise SystemConfigurationError( f"No overlap between SEGMENT_IDS values ({sorted(segment_values)}) and " - f"persons_merged['{segment_name}'] values for {self.model_selector} " + f"modeled chooser values in persons_merged['{segment_name}'] " + f"for {self.model_selector} " "simulation shadow pricing" ) - if type(self.choices_synced) != pd.DataFrame: - self.choices_synced = self.choices_synced.to_frame() - choices_synced = self.choices_synced.merge( persons_merged[segment_name], how="left", diff --git a/activitysim/abm/test/test_misc/test_shadow_pricing_simulate.py b/activitysim/abm/test/test_misc/test_shadow_pricing_simulate.py index 371d4f4991..288edc7000 100644 --- a/activitysim/abm/test/test_misc/test_shadow_pricing_simulate.py +++ b/activitysim/abm/test/test_misc/test_shadow_pricing_simulate.py @@ -652,7 +652,9 @@ def test_shadow_pricing_simulate_custom_segment(state, model_settings, network_l ) -def test_shadow_pricing_simulate_segment_values_do_not_overlap(state, model_settings): +def test_shadow_pricing_simulate_segment_values_do_not_overlap( + state, model_settings, network_los +): custom_segment_column = "school_segment_string_99" custom_model_settings = model_settings.model_copy( update={ @@ -677,13 +679,60 @@ def test_shadow_pricing_simulate_segment_values_do_not_overlap(state, model_sett assert str(error.value) == ( "No overlap between SEGMENT_IDS values (['1', '2', '3']) and " - "persons_merged['school_segment_string_99'] values for school " + "modeled chooser values in persons_merged['school_segment_string_99'] for school " "simulation shadow pricing" ) finally: persons_merged.pop(custom_segment_column) +def test_shadow_pricing_simulate_segment_overlap_ignores_unmodeled_choosers( + state, model_settings, network_los +): + custom_segment_column = "school_segment_string_99" + custom_model_settings = model_settings.model_copy( + update={ + "CHOOSER_SEGMENT_COLUMN_NAME": custom_segment_column, + "SEGMENT_IDS": { + "university": "3", + "highschool": "2", + "gradeschool": "1", + }, + } + ) + persons_merged = state.get_dataframe("persons_merged") + persons_merged[custom_segment_column] = "99" + unmodeled_chooser = persons_merged.index[0] + persons_merged.loc[unmodeled_chooser, custom_segment_column] = "1" + modeled_choosers = persons_merged.drop(index=unmodeled_chooser) + state.settings.use_shadow_pricing = True + + try: + spc = shadow_pricing.load_shadow_price_calculator(state, custom_model_settings) + choices = pd.Series( + 22660, index=modeled_choosers.index, name="choice", dtype=int + ) + spc.set_choices(choices, modeled_choosers[custom_segment_column]) + + with pytest.raises( + SystemConfigurationError, match="No overlap between SEGMENT_IDS" + ): + spc.update_shadow_prices(state) + finally: + persons_merged.pop(custom_segment_column) + + +def test_shadow_price_settings_accept_deprecated_segment_to_name(): + legacy_setting = {"school": "school_segment"} + + with pytest.warns(DeprecationWarning, match="SEGMENT_TO_NAME is deprecated"): + settings = shadow_pricing.ShadowPriceSettings.model_validate( + {"SEGMENT_TO_NAME": legacy_setting} + ) + + assert settings.SEGMENT_TO_NAME == legacy_setting + + def test_shadow_pricing_dedicated_rng_channel_eet_only( state, model_settings, network_los ): From a68705dde6599458c5273fc16294a6dec47cf142 Mon Sep 17 00:00:00 2001 From: Sijia Wang Date: Tue, 27 Oct 2026 11:11:16 -0400 Subject: [PATCH 5/5] update dev guide and change log --- docs/dev-guide/changes.md | 12 +++++++++++ docs/dev-guide/components/shadow_pricing.md | 22 +++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/docs/dev-guide/changes.md b/docs/dev-guide/changes.md index 2afe63ff89..16361539b5 100644 --- a/docs/dev-guide/changes.md +++ b/docs/dev-guide/changes.md @@ -14,6 +14,18 @@ branch (i.e., the main branch on GitHub), but not yet released in a stable versi of ActivitySim. No unreleased changes are currently documented. +### Shadow Pricing Re-Simulation Segmentation + +In version 1.6.1, the segmentation used for shadow pricing re-simulation now +inherits the segmentation used in the work and school location choice simulation, +e.g., defined as `CHOOSER_SEGMENT_COLUMN_NAME` in the `workplace_location.yaml`. +The legacy setting `SEGMENT_TO_NAME` in the `shadow_pricing.yaml` is now deprecated +and ignored. This ensures consistency between the segmentation used for shadow +pricing re-simulation and the segmentation used for work and school location +choice simulation. See +[Issue #1082](https://github.com/ActivitySim/activitysim/issues/1082) and +[PR #1083](https://github.com/ActivitySim/activitysim/pull/1083). + ## v1.6 diff --git a/docs/dev-guide/components/shadow_pricing.md b/docs/dev-guide/components/shadow_pricing.md index c618858fe8..9a9c3e760c 100644 --- a/docs/dev-guide/components/shadow_pricing.md +++ b/docs/dev-guide/components/shadow_pricing.md @@ -50,10 +50,32 @@ zone on or off for selection in the subsequent iterations. For this reason, warm for this approach is not applicable.) This process repeats until the overall convergence criteria is met or the maximum number of allowed iterations is reached. +Note that the simulation approach compares the modeled employment/enrollment share against the target +employment/enrollment share for each zone, rather than the absolute number. Therefore, it does not require +the absolute total workers/students to match the regional employment/enrollment totals. + Because the simulation approach only re-simulates workers / students who were over-assigned in the previous iteration, run time is significantly less (~90%) than the CTRAMP or DaySim approaches which re-simulate all workers and students at each iteration. +When choosing workers or students for re-simulation, the simulation approach selects individuals from +overassigned zones by shadow pricing segment. In version 1.6.0 and prior, the segmentation was defaulted +to use income groups for workers and school segments for students, and changable via the `SEGMENT_TO_NAME` +setting in the `shadow_pricing.yaml`. However, this setting was easily overlooked or misconfigured, +which could cause the shadow pricing to be accidentally skipped or cause a mismatch between the segmentation +used for shadow pricing re-simulation and the segmentation used for work and school location choice simulation, +if the latter uses a different segmentation scheme from the default income and school segments. +In version 1.6.1, the segmentation used for shadow pricing re-simulation now inherits the segmentation +used in the work and school location choice simulation, e.g., defined as `CHOOSER_SEGMENT_COLUMN_NAME` in +the `workplace_location.yaml`. The legacy setting `SEGMENT_TO_NAME` in the `shadow_pricing.yaml` is now +deprecated and ignored. + +The segmentation in shadow pricing re-simulation, e.g., occupation or income groups, accommodates but +does not require occupation- or income-specific land use employment controls. For example, +the example SANDAG model segments workplace location choice by occupation, but all occupation segments +share the same `total_emp_revised` target. Under the SANDAG configuration, ActivitySim combines the modeled +workers across segments sharing that target when evaluating whether a zone is over- or under-assigned. + ## Configuration ```{eval-rst}