From eb809d9874a3516e504a055e06c2181976c669c9 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Mon, 8 Jun 2026 22:03:54 -0700 Subject: [PATCH 01/96] initial calibration commit --- activitysim/cli/run.py | 26 +- activitysim/core/calibration.py | 1067 +++++++++++++++++++++++++++++++ 2 files changed, 1091 insertions(+), 2 deletions(-) create mode 100644 activitysim/core/calibration.py diff --git a/activitysim/cli/run.py b/activitysim/cli/run.py index ad91c4f167..289a27914f 100644 --- a/activitysim/cli/run.py +++ b/activitysim/cli/run.py @@ -14,7 +14,7 @@ import numpy as np -from activitysim.core import chunk, config, mem, timing, tracing, workflow +from activitysim.core import calibration, chunk, config, mem, timing, tracing, workflow from activitysim.core.configuration import FileSystem, Settings from activitysim.core.run_id import RunId @@ -418,7 +418,29 @@ def run(args): check_model_settings(state, extension_settings=extension_checker_settings) try: - if state.settings.multiprocess: + if calibration.calibration_enabled(state): + logger.info("run calibration workflow") + + calibration_result = calibration.run_calibration_loop( + state=state, + models=state.settings.models, + memory_sidecar_process=memory_sidecar_process, + ) + + logger.info( + "calibration workflow complete converged=%s completed_global_iterations=%s", + calibration_result.converged, + calibration_result.completed_global_iterations, + ) + + if state.settings.cleanup_pipeline_after_run: + state.checkpoint.cleanup() + else: + state.checkpoint.close_store() + + mem.log_global_hwm() + + elif state.settings.multiprocess: logger.info("run multiprocess simulation") from activitysim.core import mp_tasks diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py new file mode 100644 index 0000000000..902bdbba61 --- /dev/null +++ b/activitysim/core/calibration.py @@ -0,0 +1,1067 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +import importlib +import importlib.util +import inspect +import json +import logging +import math +import os +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +from pydantic import model_validator + +from activitysim.core import workflow +from activitysim.core.configuration import PydanticReadable +from activitysim.core.configuration.base import PydanticBase + +logger = logging.getLogger("calibration") + +CALIBRATION_SETTINGS_FILE_NAME = "calibration.yaml" +CALIBRATION_OUTPUT_DIR = "calibration" +CALIBRATION_PROGRESS_FILE = "calibration/calibration_progress.json" +CALIBRATION_ITERATION_FILE = "calibration/calibration_iteration_records.csv" +CALIBRATION_SUMMARY_FILE = "calibration/calibration_iteration_summary.csv" +CALIBRATION_FINAL_COEFFICIENTS_FILE = "calibration/final_calibrated_coefficients.csv" + +CALIBRATION_REQUIRED_COLUMNS = [ + "description", + "coefficient", + "model_value", + "target_value", + "hold_fast", + "min", + "max", + "damping", + "method", + "tolerance", +] + +MP_INJECTABLES = [ + "data_dir", + "configs_dir", + "data_model_dir", + "output_dir", + "cache_dir", + "settings_file_name", + "imported_extensions", + "run_timestamp", + "run_id", +] + + +class CalibrationRunSettings(PydanticBase): + """Run-control settings for calibration.""" + + resume_after: str + calibrate_models: list[str] + restart_after: list[str] = [] + + +class CalibrationReportsSettings(PydanticBase): + """Reporting settings for a calibrated component.""" + + generic: bool = True + bespoke: str | None = None + + +class CalibrationComponentSettings(PydanticBase): + """Settings for one calibratable model component.""" + + calibration_spec: str + helper_module: str | None = None + submodel_max_iterations: int = 1 + reports: CalibrationReportsSettings = CalibrationReportsSettings() + + +class CalibrationConfig(PydanticReadable): + """Top-level calibration configuration.""" + + enable: bool = False + max_iterations: int = 1 + run: CalibrationRunSettings + model_settings: dict[str, CalibrationComponentSettings] = {} + + @model_validator(mode="after") + def validate_model_settings(self): + """Validate that configured components are aligned with run settings.""" + for component in self.run.calibrate_models: + if component not in self.model_settings: + raise ValueError( + f"calibration model '{component}' is not in model_settings" + ) + + for component in self.run.restart_after: + if component not in self.run.calibrate_models: + raise ValueError( + f"restart_after component '{component}' is not in calibrate_models" + ) + + if self.max_iterations < 1: + raise ValueError("max_iterations must be >= 1") + + return self + + +@dataclass +class CalibrationComponentResult: + """Result details from calibrating one component.""" + + component: str + converged: bool + component_iterations: int + + +@dataclass +class CalibrationRunResult: + """Result details from a complete global calibration loop.""" + + converged: bool + completed_global_iterations: int + + +def read_calibration_settings(state: workflow.State) -> CalibrationConfig | None: + """Read and validate calibration settings if calibration.yaml exists.""" + return CalibrationConfig.read_settings_file( + state.filesystem, + CALIBRATION_SETTINGS_FILE_NAME, + mandatory=False, + ) + + +def calibration_enabled(state: workflow.State) -> bool: + """Return True when calibration.yaml exists and is enabled.""" + settings = read_calibration_settings(state) + return bool(settings and settings.enable) + + +def run_calibration_loop( + state: workflow.State, + models: list[str], + memory_sidecar_process=None, +) -> CalibrationRunResult: + """ + Run the global calibration workflow. + + This function intentionally minimizes changes to the existing run mechanics: + it always reuses ActivitySim's normal model execution paths and only adds + calibration orchestration around them. + """ + calibration_settings = read_calibration_settings(state) + if not calibration_settings or not calibration_settings.enable: + raise RuntimeError("calibration loop called while calibration is disabled") + + _ensure_calibration_output_dir(state) + + # If there is recoverable calibration progress from a prior interrupted run, + # continue from that iteration. Coefficient updates are persisted in config + # coefficient files, so restarting from a later global iteration is compatible + # with current checkpoint semantics. + progress = _read_progress(state) + start_global_iter = int(progress.get("next_global_iteration", 1)) if progress else 1 + + original_pipeline_name = state.filesystem.pipeline_file_name + + try: + for global_iter in range( + start_global_iter, calibration_settings.max_iterations + 1 + ): + logger.info( + "calibration global iteration %s/%s", + global_iter, + calibration_settings.max_iterations, + ) + + # Run ActivitySim normally from resume_after through production model steps. + _run_model_system_for_iteration( + state, + models=models, + resume_after=calibration_settings.run.resume_after, + global_iter=global_iter, + memory_sidecar_process=memory_sidecar_process, + ) + + all_converged = True + restart_triggered = False + + for component in calibration_settings.run.calibrate_models: + component_settings = calibration_settings.model_settings[component] + prior_step = _prior_step_name(models, component) + if prior_step is None: + prior_step = calibration_settings.run.resume_after + + component_result = _calibrate_component( + state=state, + component_name=component, + component_settings=component_settings, + prior_step=prior_step, + global_iter=global_iter, + ) + + all_converged = all_converged and component_result.converged + + if component in calibration_settings.run.restart_after: + # Restart global loop from resume_after after this component. + restart_triggered = True + break + + _write_progress( + state, + { + "next_global_iteration": global_iter + 1, + "last_completed_global_iteration": global_iter, + }, + ) + + if all_converged and not restart_triggered: + _write_final_coefficients_snapshot(state, calibration_settings) + _clear_progress(state) + return CalibrationRunResult( + converged=True, + completed_global_iterations=global_iter, + ) + + _write_final_coefficients_snapshot(state, calibration_settings) + _clear_progress(state) + return CalibrationRunResult( + converged=False, + completed_global_iterations=calibration_settings.max_iterations, + ) + finally: + state.filesystem.pipeline_file_name = original_pipeline_name + + +def _run_model_system_for_iteration( + state: workflow.State, + models: list[str], + resume_after: str, + global_iter: int, + memory_sidecar_process=None, +) -> None: + """Run the normal ActivitySim model flow for one global calibration iteration.""" + if global_iter > 1: + # Seed a fresh pipeline from the configured resume checkpoint to avoid + # duplicate checkpoint-name collisions across global calibration loops. + prior_pipeline = state.checkpoint.store.filename + state.checkpoint.close_store() + state.filesystem.pipeline_file_name = f"pipeline_calibration_iter_{global_iter}" + state.checkpoint.restore_from(prior_pipeline, checkpoint_name=resume_after) + + _run_in_configured_mode( + state, + models=models, + resume_after=resume_after, + memory_sidecar_process=memory_sidecar_process, + ) + + +def _calibrate_component( + state: workflow.State, + component_name: str, + component_settings: CalibrationComponentSettings, + prior_step: str, + global_iter: int, +) -> CalibrationComponentResult: + """Run iterative coefficient calibration for one component.""" + model_settings_file = _infer_model_settings_file(component_name) + model_settings = state.filesystem.read_model_settings( + model_settings_file, mandatory=True + ) + + coefficients_df = state.filesystem.read_model_coefficients( + model_settings=model_settings + ) + helper_symbols, bespoke_callable = _load_helper_symbols( + state, + component_settings, + ) + + calibration_spec_df = _read_calibration_spec( + state, component_settings.calibration_spec + ) + + utility_coeff_names = _extract_utility_coefficient_names(state, model_settings) + _validate_calibration_coefficients_against_utility_spec( + component_name, + calibration_spec_df, + utility_coeff_names, + ) + + coefficients_df = _ensure_coefficients_exist( + component_name, + calibration_spec_df, + coefficients_df, + ) + + _warn_if_initial_values_outside_bounds( + component_name, calibration_spec_df, coefficients_df + ) + + component_converged = False + component_iterations = 0 + + for component_iter in range(1, component_settings.submodel_max_iterations + 1): + component_iterations = component_iter + + # Re-run only this component from its prior checkpoint so model values + # reflect the current candidate coefficients for this component. + if state.settings.multiprocess: + # In multiprocess mode, preserve the standard multiprocess orchestration + # so table coalescing semantics match the initial global run path. + _run_in_configured_mode( + state, + models=state.settings.models, + resume_after=prior_step, + ) + else: + run_model_name = ( + f"{component_name}.calibration_component_iter={component_iter};" + f"calibration_global_iter={global_iter}" + ) + state.run(models=[run_model_name], resume_after=prior_step) + + eval_context = _build_expression_context(state, helper_symbols) + + ( + row_records, + summary_record, + new_coefficients_df, + component_converged, + ) = _evaluate_and_update( + component_name=component_name, + calibration_spec_df=calibration_spec_df, + coefficients_df=coefficients_df, + eval_context=eval_context, + global_iter=global_iter, + component_iter=component_iter, + ) + + coefficients_df = new_coefficients_df + + _persist_coefficients_to_config(state, model_settings, coefficients_df) + _append_iteration_records(state, row_records) + _append_summary_records(state, [summary_record]) + + if component_settings.reports.generic: + _write_generic_report(state, component_name, row_records) + + if bespoke_callable is not None: + # Preserve compatibility with helper modules that expect a global + # `state` symbol and/or no explicit arguments. + _run_bespoke_report(bespoke_callable, state) + + if component_converged: + break + + return CalibrationComponentResult( + component=component_name, + converged=component_converged, + component_iterations=component_iterations, + ) + + +def _read_calibration_spec(state: workflow.State, file_name: str) -> pd.DataFrame: + """Read calibration spec CSV and validate required columns.""" + path = state.filesystem.get_config_file_path(file_name) + df = pd.read_csv(path, comment="#") + + missing = [c for c in CALIBRATION_REQUIRED_COLUMNS if c not in df.columns] + if missing: + raise ValueError( + f"calibration_spec {file_name} is missing required columns: {missing}" + ) + + df = df[CALIBRATION_REQUIRED_COLUMNS].copy() + df["description"] = df["description"].astype(str) + df["coefficient"] = df["coefficient"].astype(str) + + # Normalize booleans and defaults to keep row math deterministic. + df["hold_fast"] = ( + df["hold_fast"] + .fillna(False) + .astype(str) + .str.strip() + .str.lower() + .isin(["1", "true", "t", "yes", "y"]) + ) + + for c in ["min", "max", "damping", "tolerance"]: + df[c] = pd.to_numeric(df[c], errors="coerce") + + df["method"] = df["method"].astype(str).str.strip().str.lower() + + bad_methods = df.loc[~df["method"].isin(["log_ratio", "odds_ratio"]), "method"] + if not bad_methods.empty: + raise ValueError( + f"unsupported calibration method(s): {sorted(set(bad_methods))}" + ) + + if df["damping"].isna().any(): + raise ValueError("calibration_spec damping must be numeric") + if df["tolerance"].isna().any(): + raise ValueError("calibration_spec tolerance must be numeric") + + return df + + +def _extract_utility_coefficient_names( + state: workflow.State, + model_settings: dict[str, Any] | Any, +) -> set[str]: + """ + Extract coefficient tokens from configured utility spec files. + + The extraction scans all settings keys ending with "SPEC" and parses + tokens from utility columns (all non-description/expression columns). + """ + names: set[str] = set() + + model_settings_dict = _settings_to_dict(model_settings) + spec_keys = [ + k for k in model_settings_dict.keys() if str(k).upper().endswith("SPEC") + ] + for key in spec_keys: + spec_file = model_settings_dict.get(key) + if not spec_file: + continue + + spec_path = state.filesystem.get_config_file_path(spec_file) + try: + raw = pd.read_csv(spec_path, comment="#") + except Exception: + # Do not fail hard on optional or model-specific supplemental specs. + continue + + utility_columns = [ + c + for c in raw.columns + if c + not in [ + "Description", + "Expression", + "Label", + "description", + "expression", + "label", + ] + ] + + for col in utility_columns: + for value in raw[col].dropna().tolist(): + if isinstance(value, (int, float, np.number)): + continue + text = str(value) + for token in re.findall(r"[A-Za-z_][A-Za-z0-9_]*", text): + names.add(token) + + return names + + +def _validate_calibration_coefficients_against_utility_spec( + component_name: str, + calibration_spec_df: pd.DataFrame, + utility_coeff_names: set[str], +) -> None: + """Ensure calibration coefficients are referenced in utility specifications.""" + missing = sorted( + set(calibration_spec_df["coefficient"].tolist()) - set(utility_coeff_names) + ) + if missing: + raise ValueError( + f"calibration coefficients not present in model utility specification for {component_name}: {missing}" + ) + + +def _ensure_coefficients_exist( + component_name: str, + calibration_spec_df: pd.DataFrame, + coefficients_df: pd.DataFrame, +) -> pd.DataFrame: + """Add calibration coefficients missing from coefficient file with value 0.0.""" + coeffs = coefficients_df.copy() + + for coefficient_name in calibration_spec_df["coefficient"].tolist(): + if coefficient_name not in coeffs.index: + logger.warning( + "component %s coefficient %s missing from coefficient file, adding with value 0.0", + component_name, + coefficient_name, + ) + coeffs.loc[coefficient_name, "value"] = 0.0 + + coeffs["value"] = pd.to_numeric(coeffs["value"], errors="coerce") + if coeffs["value"].isna().any(): + bad = coeffs[coeffs["value"].isna()].index.tolist() + raise ValueError(f"non-numeric coefficient values found: {bad}") + + return coeffs + + +def _warn_if_initial_values_outside_bounds( + component_name: str, + calibration_spec_df: pd.DataFrame, + coefficients_df: pd.DataFrame, +) -> None: + """Warn when initial coefficient values violate provided bounds.""" + for _, row in calibration_spec_df.iterrows(): + coefficient_name = row["coefficient"] + current_value = float(coefficients_df.loc[coefficient_name, "value"]) + lower = row["min"] + upper = row["max"] + + if not pd.isna(lower) and current_value < float(lower): + logger.warning( + "component %s coefficient %s starts below min bound (%s < %s)", + component_name, + coefficient_name, + current_value, + lower, + ) + if not pd.isna(upper) and current_value > float(upper): + logger.warning( + "component %s coefficient %s starts above max bound (%s > %s)", + component_name, + coefficient_name, + current_value, + upper, + ) + + +def _build_expression_context( + state: workflow.State, + helper_symbols: dict[str, Any], +) -> dict[str, Any]: + """Create the evaluation context for model_value and target_value expressions.""" + context: dict[str, Any] = { + "state": state, + "np": np, + "pd": pd, + } + + # Load active tables into context for direct expression access. + for table_name in list(state.existing_table_names): + try: + context[table_name] = state.get_dataframe(table_name, as_copy=False) + except Exception: + # Some entries may not be available as dataframes in all contexts. + continue + + context.update(helper_symbols) + return context + + +def _evaluate_and_update( + component_name: str, + calibration_spec_df: pd.DataFrame, + coefficients_df: pd.DataFrame, + eval_context: dict[str, Any], + global_iter: int, + component_iter: int, +) -> tuple[list[dict[str, Any]], dict[str, Any], pd.DataFrame, bool]: + """Evaluate spec rows, update coefficients, and return detailed records.""" + updated = coefficients_df.copy() + records: list[dict[str, Any]] = [] + + max_difference = -math.inf + max_difference_component = "" + max_difference_coefficient = "" + max_change = -math.inf + max_change_component = "" + max_change_coefficient = "" + + num_converged = 0 + + for _, row in calibration_spec_df.iterrows(): + coefficient_name = row["coefficient"] + description = row["description"] + method = row["method"] + hold_fast = bool(row["hold_fast"]) + + prev_value = float(updated.loc[coefficient_name, "value"]) + + model_value = _eval_numeric_value( + row["model_value"], + eval_context, + component_name, + description, + "model_value", + ) + target_value = _eval_numeric_value( + row["target_value"], + eval_context, + component_name, + description, + "target_value", + ) + + difference = target_value - model_value + prc_difference = _safe_percent_difference(difference, target_value) + + tolerance = float(row["tolerance"]) + converged = abs(difference) <= tolerance + + damping = float(row["damping"]) + raw_delta = _compute_delta( + method=method, + model_value=model_value, + target_value=target_value, + damping=damping, + component_name=component_name, + description=description, + ) + + candidate_value = prev_value if hold_fast else prev_value + raw_delta + + under_min = False + over_max = False + + lower = row["min"] + upper = row["max"] + + if not pd.isna(lower) and candidate_value < float(lower): + candidate_value = float(lower) + under_min = True + if not pd.isna(upper) and candidate_value > float(upper): + candidate_value = float(upper) + over_max = True + + if not np.isfinite(candidate_value): + raise RuntimeError( + f"non-finite next coefficient for {component_name} / {description} / {coefficient_name}" + ) + + updated.loc[coefficient_name, "value"] = candidate_value + + abs_diff = abs(difference) + abs_change = abs(candidate_value - prev_value) + + if abs_diff > max_difference: + max_difference = abs_diff + max_difference_component = component_name + max_difference_coefficient = coefficient_name + + if abs_change > max_change: + max_change = abs_change + max_change_component = component_name + max_change_coefficient = coefficient_name + + if converged: + num_converged += 1 + + records.append( + { + "global_iter": global_iter, + "component_iter": component_iter, + "description": description, + "component": component_name, + "coefficient": coefficient_name, + "target_value": target_value, + "model_value": model_value, + "difference": difference, + "prc_difference": prc_difference, + "hold_fast": hold_fast, + "prev_coefficient": prev_value, + "next_coefficient": candidate_value, + "converged": converged, + "under_min": under_min, + "over_max": over_max, + } + ) + + total_rows = len(calibration_spec_df) + num_unconverged = total_rows - num_converged + component_converged = num_unconverged == 0 + + summary_record = { + "global_iter": global_iter, + "component_iter": component_iter, + "component": component_name, + "max_difference": max_difference if max_difference != -math.inf else 0.0, + "max_difference_component": max_difference_component, + "max_difference_coefficient": max_difference_coefficient, + "max_change": max_change if max_change != -math.inf else 0.0, + "max_change_component": max_change_component, + "max_change_coefficient": max_change_coefficient, + "num_converged_iter": num_converged, + "tot_converged": num_converged, + "num_unconverged": num_unconverged, + } + + return records, summary_record, updated, component_converged + + +def _eval_numeric_value( + raw_value: Any, + context: dict[str, Any], + component_name: str, + description: str, + field_name: str, +) -> float: + """Evaluate numeric or expression value and enforce finite numeric result.""" + if isinstance(raw_value, (int, float, np.number)) and not pd.isna(raw_value): + value = float(raw_value) + else: + try: + value = eval(str(raw_value), {}, context) + except Exception as err: + raise RuntimeError( + f"error evaluating {field_name} for {component_name} / {description}: {raw_value}" + ) from err + + try: + value = float(value) + except Exception as err: + raise RuntimeError( + f"{field_name} did not evaluate to a numeric value for {component_name} / {description}: {value}" + ) from err + + if not np.isfinite(value): + raise RuntimeError( + f"{field_name} evaluated to non-finite value for {component_name} / {description}: {value}" + ) + + return value + + +def _compute_delta( + method: str, + model_value: float, + target_value: float, + damping: float, + component_name: str, + description: str, +) -> float: + """Compute damped coefficient delta using selected method.""" + if damping < 0: + raise RuntimeError( + f"negative damping not allowed for {component_name} / {description}: {damping}" + ) + + if method == "log_ratio": + if model_value <= 0 or target_value <= 0: + raise RuntimeError( + f"log_ratio requires positive model and target values for {component_name} / {description}" + ) + delta = math.log(target_value / model_value) * damping + + elif method == "odds_ratio": + # Formula requested by the calibration outline. + numerator = (target_value * model_value) - target_value + denominator = (target_value * model_value) - model_value + + if numerator <= 0 or denominator <= 0: + raise RuntimeError( + f"odds_ratio produced invalid numerator/denominator for {component_name} / {description}" + ) + + ratio = numerator / denominator + if ratio <= 0 or not np.isfinite(ratio): + raise RuntimeError( + f"odds_ratio produced invalid ratio for {component_name} / {description}" + ) + + delta = math.log(ratio) * damping + + else: + raise RuntimeError(f"unsupported calibration method: {method}") + + if not np.isfinite(delta): + raise RuntimeError( + f"coefficient delta is non-finite for {component_name} / {description}" + ) + + return delta + + +def _safe_percent_difference(difference: float, target_value: float) -> float: + """Return a stable percentage difference with zero-target handling.""" + if target_value == 0: + return math.inf if difference != 0 else 0.0 + return (difference / target_value) * 100.0 + + +def _persist_coefficients_to_config( + state: workflow.State, + model_settings: dict[str, Any] | Any, + coefficients_df: pd.DataFrame, +) -> None: + """Write updated coefficients back to the component coefficient file in configs.""" + coeff_file = _setting_value(model_settings, "COEFFICIENTS") + if not coeff_file: + raise RuntimeError("component model settings missing COEFFICIENTS") + + output = coefficients_df.copy() + output.index.name = "coefficient_name" + + coeff_path = state.filesystem.get_config_file_path(coeff_file) + output.to_csv(coeff_path) + + +def _append_iteration_records( + state: workflow.State, records: list[dict[str, Any]] +) -> None: + """Append per-coefficient calibration iteration records.""" + if not records: + return + path = state.get_output_file_path(CALIBRATION_ITERATION_FILE) + df = pd.DataFrame(records) + _append_csv(df, path) + + +def _append_summary_records( + state: workflow.State, records: list[dict[str, Any]] +) -> None: + """Append per-iteration summary records.""" + if not records: + return + path = state.get_output_file_path(CALIBRATION_SUMMARY_FILE) + df = pd.DataFrame(records) + _append_csv(df, path) + + +def _append_csv(df: pd.DataFrame, path: Path) -> None: + """Append a dataframe to a CSV file with header-once behavior.""" + os.makedirs(path.parent, exist_ok=True) + write_header = not path.exists() + df.to_csv(path, mode="a", index=False, header=write_header) + + +def _write_generic_report( + state: workflow.State, + component_name: str, + row_records: list[dict[str, Any]], +) -> None: + """Write a simple dashboard-friendly generic report for the current component iteration.""" + if not row_records: + return + + df = pd.DataFrame(row_records) + report = ( + df[ + [ + "global_iter", + "component_iter", + "component", + "description", + "difference", + "prc_difference", + "converged", + ] + ] + .copy() + .sort_values(["global_iter", "component_iter", "description"]) + ) + + path = state.get_output_file_path( + f"calibration/{component_name}_generic_report.csv" + ) + _append_csv(report, path) + + +def _run_bespoke_report(bespoke_callable, state: workflow.State) -> None: + """Run optional bespoke report callback from helper module.""" + try: + # Support either no-argument callback or callback(state). + sig = inspect.signature(bespoke_callable) + if len(sig.parameters) == 0: + bespoke_callable() + else: + bespoke_callable(state) + except TypeError: + bespoke_callable() + + +def _load_helper_symbols( + state: workflow.State, + component_settings: CalibrationComponentSettings, +) -> tuple[dict[str, Any], Any | None]: + """Load helper module and return evaluation symbols and bespoke function.""" + if not component_settings.helper_module: + return {}, None + + module = _load_helper_module(state, component_settings.helper_module) + symbols = { + name: obj for name, obj in vars(module).items() if not name.startswith("__") + } + + bespoke = None + if component_settings.reports and component_settings.reports.bespoke: + fn_name = component_settings.reports.bespoke + if not hasattr(module, fn_name): + raise RuntimeError( + f"helper module does not define bespoke function {fn_name}" + ) + bespoke = getattr(module, fn_name) + + return symbols, bespoke + + +def _load_helper_module(state: workflow.State, helper_module: str): + """Load helper module by file path or import path.""" + if helper_module.endswith(".py"): + helper_path = state.filesystem.get_config_file_path(helper_module) + module_name = Path(helper_module).stem + + spec = importlib.util.spec_from_file_location(module_name, helper_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"unable to load helper module from {helper_path}") + + module = importlib.util.module_from_spec(spec) + # Expose state as a global for compatibility with helper examples. + module.state = state + spec.loader.exec_module(module) + return module + + module = importlib.import_module(helper_module) + setattr(module, "state", state) + return module + + +def _infer_model_settings_file(component_name: str) -> str: + """Infer model settings yaml filename from component step name.""" + # This follows the dominant naming convention in the existing codebase. + if component_name.endswith("_simulate"): + base = component_name[: -len("_simulate")] + else: + base = component_name + return f"{base}.yaml" + + +def _settings_to_dict(model_settings: dict[str, Any] | Any) -> dict[str, Any]: + """Convert pydantic or dict model settings to a plain dictionary.""" + if isinstance(model_settings, dict): + return model_settings + if hasattr(model_settings, "model_dump"): + return model_settings.model_dump() + return dict(model_settings) + + +def _setting_value(model_settings: dict[str, Any] | Any, key: str, default=None): + """Read a setting value from dict-like or attribute-based settings.""" + if isinstance(model_settings, dict): + return model_settings.get(key, default) + return getattr(model_settings, key, default) + + +def _prior_step_name(models: list[str], component_name: str) -> str | None: + """Return the step name immediately preceding component_name in models.""" + if component_name not in models: + return None + idx = models.index(component_name) + if idx == 0: + return None + return models[idx - 1] + + +def _write_final_coefficients_snapshot( + state: workflow.State, + calibration_settings: CalibrationConfig, +) -> None: + """Write a combined final coefficients file snapshot for calibrated components.""" + frames = [] + for component_name in calibration_settings.run.calibrate_models: + model_settings_file = _infer_model_settings_file(component_name) + model_settings = state.filesystem.read_model_settings( + model_settings_file, mandatory=True + ) + coeff_df = state.filesystem.read_model_coefficients( + model_settings=model_settings + ).copy() + coeff_df = coeff_df.reset_index().rename(columns={"index": "coefficient_name"}) + coeff_df.insert(0, "component", component_name) + frames.append(coeff_df) + + if not frames: + return + + final_df = pd.concat(frames, ignore_index=True) + path = state.get_output_file_path(CALIBRATION_FINAL_COEFFICIENTS_FILE) + os.makedirs(path.parent, exist_ok=True) + final_df.to_csv(path, index=False) + + +def _ensure_calibration_output_dir(state: workflow.State) -> None: + """Ensure output/calibration exists.""" + path = state.get_output_file_path(CALIBRATION_OUTPUT_DIR) + os.makedirs(path, exist_ok=True) + + +def _read_progress(state: workflow.State) -> dict[str, Any] | None: + """Read persisted calibration progress metadata if it exists.""" + path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) + if not path.exists(): + return None + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def _write_progress(state: workflow.State, payload: dict[str, Any]) -> None: + """Write calibration progress metadata.""" + path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) + os.makedirs(path.parent, exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2) + + +def _clear_progress(state: workflow.State) -> None: + """Delete calibration progress metadata after successful completion.""" + path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) + if path.exists(): + path.unlink() + + +def _run_in_configured_mode( + state: workflow.State, + models: list[str], + resume_after: str | None, + memory_sidecar_process=None, +) -> None: + """Run models using the same single/multiprocess mode as the parent run.""" + if state.settings.multiprocess: + _run_multiprocess_with_overrides( + state, + models=models, + resume_after=resume_after, + ) + return + + state.run( + models=models, + resume_after=resume_after, + memory_sidecar_process=memory_sidecar_process, + ) + + +def _run_multiprocess_with_overrides( + state: workflow.State, + models: list[str], + resume_after: str | None, +) -> None: + """Run multiprocess with temporary settings overrides for calibration passes.""" + from activitysim.core import mp_tasks + + original_models = state.settings.models + original_resume_after = state.settings.resume_after + + state.settings.models = models + state.settings.resume_after = resume_after + + try: + injectables = {} + for key in MP_INJECTABLES: + try: + injectables[key] = state.get_injectable(key) + except KeyError: + pass + injectables["settings"] = state.settings + mp_tasks.run_multiprocess(state, injectables) + finally: + state.settings.models = original_models + state.settings.resume_after = original_resume_after From 087659786cebe36b0cdc24453052305b8a128ed3 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Tue, 9 Jun 2026 10:26:32 -0700 Subject: [PATCH 02/96] do not validate settings if file not found and not mandatory --- activitysim/core/configuration/filesystem.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/activitysim/core/configuration/filesystem.py b/activitysim/core/configuration/filesystem.py index 71074ac62d..0de99ef1f1 100644 --- a/activitysim/core/configuration/filesystem.py +++ b/activitysim/core/configuration/filesystem.py @@ -724,6 +724,7 @@ def backfill_settings(settings, backfill): file_name = "%s.yaml" % (file_name,) inheriting = False + settings_file_found = False settings = {} if isinstance(include_stack, list): source_file_paths = include_stack.copy() @@ -732,6 +733,7 @@ def backfill_settings(settings, backfill): for dir in configs_dir_list: file_path = os.path.join(dir, file_name) if os.path.exists(file_path): + settings_file_found = True if inheriting: # we must be inheriting logger.debug( @@ -835,7 +837,7 @@ def backfill_settings(settings, backfill): settings.pop("inherit_settings", None) settings.pop("include_settings", None) - if validator_class is not None: + if validator_class is not None and (mandatory or settings_file_found): settings = validator_class.model_validate(settings) if include_stack: From 6571db170406f1b9e6fbd1ab52d1880a9b9aed14 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 2 Jul 2026 14:07:47 -0400 Subject: [PATCH 03/96] calibration settings, bespoke reports changes --- activitysim/core/calibration.py | 11 +++++++---- activitysim/core/estimation.py | 5 +++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 902bdbba61..a570392fbe 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -79,6 +79,7 @@ class CalibrationComponentSettings(PydanticBase): helper_module: str | None = None submodel_max_iterations: int = 1 reports: CalibrationReportsSettings = CalibrationReportsSettings() + survey_file: str class CalibrationConfig(PydanticReadable): @@ -355,7 +356,7 @@ def _calibrate_component( if bespoke_callable is not None: # Preserve compatibility with helper modules that expect a global # `state` symbol and/or no explicit arguments. - _run_bespoke_report(bespoke_callable, state) + _run_bespoke_report(bespoke_callable, state, component_settings) if component_converged: break @@ -865,15 +866,17 @@ def _write_generic_report( _append_csv(report, path) -def _run_bespoke_report(bespoke_callable, state: workflow.State) -> None: +def _run_bespoke_report(bespoke_callable, state: workflow.State, component_settings: CalibrationComponentSettings) -> None: """Run optional bespoke report callback from helper module.""" try: - # Support either no-argument callback or callback(state). + # Support no-argument callback, callback(state), or callback(state, component_settings). sig = inspect.signature(bespoke_callable) if len(sig.parameters) == 0: bespoke_callable() - else: + elif len(sig.parameters) == 1: bespoke_callable(state) + else: + bespoke_callable(state, component_settings) except TypeError: bespoke_callable() diff --git a/activitysim/core/estimation.py b/activitysim/core/estimation.py index bbbe376ee4..9b6fcbab4c 100644 --- a/activitysim/core/estimation.py +++ b/activitysim/core/estimation.py @@ -798,8 +798,9 @@ def initialize_settings(self, state): if not self.settings: # if the model self.settings file is not found, we are not in estimation mode. self.enabled = False - else: - self.enabled = self.settings.enable + return + + self.enabled = self.settings.enable self.bundles = self.settings.bundles self.estimation_table_types = self.settings.estimation_table_types From 6ccf84d3750317aa6b90f8220a5c04783c0e6a58 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Fri, 3 Jul 2026 15:56:00 -0400 Subject: [PATCH 04/96] Fix global iterations --- activitysim/core/calibration.py | 129 +++++++++++++++++++++++--------- 1 file changed, 93 insertions(+), 36 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index a570392fbe..99eabc0187 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -12,7 +12,7 @@ import re from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, Optional import numpy as np import pandas as pd @@ -60,9 +60,11 @@ class CalibrationRunSettings(PydanticBase): """Run-control settings for calibration.""" - resume_after: str + resume_after: Optional[str] = None calibrate_models: list[str] restart_after: list[str] = [] + global_iterations: int = 1 + complete_steps: bool = True class CalibrationReportsSettings(PydanticBase): @@ -86,7 +88,6 @@ class CalibrationConfig(PydanticReadable): """Top-level calibration configuration.""" enable: bool = False - max_iterations: int = 1 run: CalibrationRunSettings model_settings: dict[str, CalibrationComponentSettings] = {} @@ -105,7 +106,7 @@ def validate_model_settings(self): f"restart_after component '{component}' is not in calibrate_models" ) - if self.max_iterations < 1: + if self.run.global_iterations < 1: raise ValueError("max_iterations must be >= 1") return self @@ -159,6 +160,16 @@ def run_calibration_loop( if not calibration_settings or not calibration_settings.enable: raise RuntimeError("calibration loop called while calibration is disabled") + assert all( + [c in models for c in calibration_settings.run.calibrate_models] + ), f"settings.yaml steps list does not include calibration model{'s' if len([c for c in calibration_settings.run.calibrate_models if c not in models]) != 1 else ''} {[c for c in calibration_settings.run.calibrate_models if c not in models]}" + + # sort calibration models into main model order + calibration_settings.run.calibrate_models = sorted( + calibration_settings.run.calibrate_models, key=lambda x: models.index(x) + ) + first_calib_model_idx = models.index(calibration_settings.run.calibrate_models[0]) + _ensure_calibration_output_dir(state) # If there is recoverable calibration progress from a prior interrupted run, @@ -172,31 +183,48 @@ def run_calibration_loop( try: for global_iter in range( - start_global_iter, calibration_settings.max_iterations + 1 + start_global_iter, + start_global_iter + calibration_settings.run.global_iterations, ): logger.info( "calibration global iteration %s/%s", - global_iter, - calibration_settings.max_iterations, + global_iter - start_global_iter, + calibration_settings.run.global_iterations, ) # Run ActivitySim normally from resume_after through production model steps. - _run_model_system_for_iteration( + _run_precursor_components( state, - models=models, - resume_after=calibration_settings.run.resume_after, + models=models[:first_calib_model_idx], + resume_after=calibration_settings.run.resume_after + if global_iter == start_global_iter + else _prior_step_name( + models, calibration_settings.run.calibrate_models[0] + ), global_iter=global_iter, memory_sidecar_process=memory_sidecar_process, ) all_converged = True - restart_triggered = False + last_calibrated_component = None for component in calibration_settings.run.calibrate_models: component_settings = calibration_settings.model_settings[component] + prior_step = _prior_step_name(models, component) - if prior_step is None: - prior_step = calibration_settings.run.resume_after + + if last_calibrated_component is not None: + + # run all models b/w the last calibrated model and the current one + _run_intermediate_components( + state, + models=models[ + models.index(last_calibrated_component) + + 1 : models.index(component) + ], + resume_after=last_calibrated_component, + memory_sidecar_process=memory_sidecar_process, + ) component_result = _calibrate_component( state=state, @@ -208,10 +236,15 @@ def run_calibration_loop( all_converged = all_converged and component_result.converged - if component in calibration_settings.run.restart_after: - # Restart global loop from resume_after after this component. - restart_triggered = True - break + last_calibrated_component = component + + if calibration_settings.run.complete_steps: + _run_subsequent_components( + state, + models=models[models.index(last_calibrated_component) + 1 :], + resume_after=last_calibrated_component, + memory_sidecar_process=memory_sidecar_process, + ) _write_progress( state, @@ -221,25 +254,16 @@ def run_calibration_loop( }, ) - if all_converged and not restart_triggered: - _write_final_coefficients_snapshot(state, calibration_settings) - _clear_progress(state) - return CalibrationRunResult( - converged=True, - completed_global_iterations=global_iter, - ) - _write_final_coefficients_snapshot(state, calibration_settings) - _clear_progress(state) return CalibrationRunResult( converged=False, - completed_global_iterations=calibration_settings.max_iterations, + completed_global_iterations=calibration_settings.run.global_iterations, ) finally: state.filesystem.pipeline_file_name = original_pipeline_name -def _run_model_system_for_iteration( +def _run_precursor_components( state: workflow.State, models: list[str], resume_after: str, @@ -247,6 +271,10 @@ def _run_model_system_for_iteration( memory_sidecar_process=None, ) -> None: """Run the normal ActivitySim model flow for one global calibration iteration.""" + + assert (resume_after is None) or ( + resume_after in models + ), f"resume_after step {resume_after} not in models preceding calibration models" if global_iter > 1: # Seed a fresh pipeline from the configured resume checkpoint to avoid # duplicate checkpoint-name collisions across global calibration loops. @@ -254,7 +282,38 @@ def _run_model_system_for_iteration( state.checkpoint.close_store() state.filesystem.pipeline_file_name = f"pipeline_calibration_iter_{global_iter}" state.checkpoint.restore_from(prior_pipeline, checkpoint_name=resume_after) + else: + + _run_in_configured_mode( + state, + models=models, + resume_after=resume_after, + memory_sidecar_process=memory_sidecar_process, + ) + + +def _run_intermediate_components( + state: workflow.State, + models: list[str], + resume_after: str, + memory_sidecar_process=None, +) -> None: + # don't modify the pipeline, just run the models needed + _run_in_configured_mode( + state, + models=models, + resume_after=resume_after, + memory_sidecar_process=memory_sidecar_process, + ) + +def _run_subsequent_components( + state: workflow.State, + models: list[str], + resume_after: str, + memory_sidecar_process=None, +) -> None: + # don't modify the pipeline, just run the models needed _run_in_configured_mode( state, models=models, @@ -360,6 +419,7 @@ def _calibrate_component( if component_converged: break + state.checkpoint.add(component_name) return CalibrationComponentResult( component=component_name, @@ -866,7 +926,11 @@ def _write_generic_report( _append_csv(report, path) -def _run_bespoke_report(bespoke_callable, state: workflow.State, component_settings: CalibrationComponentSettings) -> None: +def _run_bespoke_report( + bespoke_callable, + state: workflow.State, + component_settings: CalibrationComponentSettings, +) -> None: """Run optional bespoke report callback from helper module.""" try: # Support no-argument callback, callback(state), or callback(state, component_settings). @@ -1013,13 +1077,6 @@ def _write_progress(state: workflow.State, payload: dict[str, Any]) -> None: json.dump(payload, f, indent=2) -def _clear_progress(state: workflow.State) -> None: - """Delete calibration progress metadata after successful completion.""" - path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) - if path.exists(): - path.unlink() - - def _run_in_configured_mode( state: workflow.State, models: list[str], From f2ea9478712265c3ad4c7ad230ab02824c027bbf Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Fri, 3 Jul 2026 18:09:51 -0400 Subject: [PATCH 05/96] Loading from 2+ global iters --- activitysim/core/calibration.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 99eabc0187..a575f50ce0 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -197,7 +197,7 @@ def run_calibration_loop( state, models=models[:first_calib_model_idx], resume_after=calibration_settings.run.resume_after - if global_iter == start_global_iter + if global_iter == 1 else _prior_step_name( models, calibration_settings.run.calibrate_models[0] ), @@ -255,6 +255,18 @@ def run_calibration_loop( ) _write_final_coefficients_snapshot(state, calibration_settings) + + iteration_records = pd.read_csv(state.get_output_file_path(CALIBRATION_ITERATION_FILE)) + + for component in iteration_records.component.unique(): + + ax = iteration_records.loc[iteration_records.component == component].set_index(['global_iter','component_iter','coefficient']).next_coefficient.unstack('coefficient').plot() + ax.xaxis.set_label("Component iteration") + ax.yaxis.set_label("Coefficient value") + + ax.legend(title="Coefficient label") + ax.figure.savefig(os.path.join(state.filesystem.output_dir,f"{component}_coefficient_progress.png")) + return CalibrationRunResult( converged=False, completed_global_iterations=calibration_settings.run.global_iterations, From 72758e6a170649501a72c22f3ab2f31bcb2c89e8 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Mon, 6 Jul 2026 09:37:16 -0400 Subject: [PATCH 06/96] Bugfix in plotting coefficient momentum --- activitysim/core/calibration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index a575f50ce0..29e0c8f657 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -261,8 +261,8 @@ def run_calibration_loop( for component in iteration_records.component.unique(): ax = iteration_records.loc[iteration_records.component == component].set_index(['global_iter','component_iter','coefficient']).next_coefficient.unstack('coefficient').plot() - ax.xaxis.set_label("Component iteration") - ax.yaxis.set_label("Coefficient value") + ax.xaxis.set_label_text("Component iteration") + ax.yaxis.set_label_text("Coefficient value") ax.legend(title="Coefficient label") ax.figure.savefig(os.path.join(state.filesystem.output_dir,f"{component}_coefficient_progress.png")) From b065459890398d72323a6bab026e54d9711e08c3 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Mon, 6 Jul 2026 09:44:44 -0400 Subject: [PATCH 07/96] Add coef_delta to calibration records --- activitysim/core/calibration.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 29e0c8f657..bfa90f798f 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -675,7 +675,7 @@ def _evaluate_and_update( ) difference = target_value - model_value - prc_difference = _safe_percent_difference(difference, target_value) + pct_difference = _safe_percent_difference(difference, target_value) tolerance = float(row["tolerance"]) converged = abs(difference) <= tolerance @@ -738,9 +738,10 @@ def _evaluate_and_update( "target_value": target_value, "model_value": model_value, "difference": difference, - "prc_difference": prc_difference, + "pct_difference": pct_difference, "hold_fast": hold_fast, "prev_coefficient": prev_value, + "coef_delta": abs_change, "next_coefficient": candidate_value, "converged": converged, "under_min": under_min, @@ -924,7 +925,7 @@ def _write_generic_report( "component", "description", "difference", - "prc_difference", + "pct_difference", "converged", ] ] From 3c135c7a389bc23dca2421653423ca4f7a4fe096 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Mon, 6 Jul 2026 16:45:51 -0400 Subject: [PATCH 08/96] Update graphs, handle zero modeled vals --- activitysim/core/calibration.py | 118 +++++++++++++++++++++++--------- 1 file changed, 87 insertions(+), 31 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index bfa90f798f..99b4116662 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -4,7 +4,6 @@ import importlib import importlib.util -import inspect import json import logging import math @@ -16,6 +15,7 @@ import numpy as np import pandas as pd +import matplotlib.pyplot as plt from pydantic import model_validator from activitysim.core import workflow @@ -31,6 +31,8 @@ CALIBRATION_SUMMARY_FILE = "calibration/calibration_iteration_summary.csv" CALIBRATION_FINAL_COEFFICIENTS_FILE = "calibration/final_calibrated_coefficients.csv" +DEFAULT_INCREMENT = 2.0 + CALIBRATION_REQUIRED_COLUMNS = [ "description", "coefficient", @@ -64,7 +66,7 @@ class CalibrationRunSettings(PydanticBase): calibrate_models: list[str] restart_after: list[str] = [] global_iterations: int = 1 - complete_steps: bool = True + complete_steps: bool = False class CalibrationReportsSettings(PydanticBase): @@ -238,7 +240,11 @@ def run_calibration_loop( last_calibrated_component = component - if calibration_settings.run.complete_steps: + if calibration_settings.run.complete_steps or ( + start_global_iter + calibration_settings.run.global_iterations + == global_iter + 1 + ): + # finish the full model chain _run_subsequent_components( state, models=models[models.index(last_calibrated_component) + 1 :], @@ -256,16 +262,64 @@ def run_calibration_loop( _write_final_coefficients_snapshot(state, calibration_settings) - iteration_records = pd.read_csv(state.get_output_file_path(CALIBRATION_ITERATION_FILE)) + iteration_records = ( + pd.read_csv(state.get_output_file_path(CALIBRATION_ITERATION_FILE)) + .set_index(["global_iter", "component_iter", "coefficient"]) + .sort_index() + ) for component in iteration_records.component.unique(): - ax = iteration_records.loc[iteration_records.component == component].set_index(['global_iter','component_iter','coefficient']).next_coefficient.unstack('coefficient').plot() + ax = ( + iteration_records.loc[iteration_records.component == component] + .next_coefficient.unstack("coefficient") + .plot() + ) ax.xaxis.set_label_text("Component iteration") ax.yaxis.set_label_text("Coefficient value") - + ax.legend(title="Coefficient label") - ax.figure.savefig(os.path.join(state.filesystem.output_dir,f"{component}_coefficient_progress.png")) + ax.figure.savefig( + os.path.join( + state.filesystem.output_dir, + "calibration", + f"{component}_coefficient_progress.png", + ) + ) + + last_global = iteration_records.index.get_level_values("global_iter")[-1] + last_comp = iteration_records.loc[last_global].index.get_level_values( + "component_iter" + )[-1] + + last_records = iteration_records.xs( + (last_global, last_comp), level=("global_iter", "component_iter") + )[["target_value", "model_value"]] + ax = last_records.plot.bar() + ax.xaxis.set_tick_params(rotation=45) + ax.xaxis.set_label_text("Component value") + plt.tight_layout() + ax.figure.savefig( + os.path.join( + state.filesystem.output_dir, + "calibration", + f"{component}_final_components.png", + ) + ) + + _ = plt.subplots() + + pct_diff = last_records.diff(axis=1).model_value / last_records.target_value + ax = pct_diff.plot.bar() + ax.xaxis.set_tick_params(rotation=45) + plt.tight_layout() + ax.figure.savefig( + os.path.join( + state.filesystem.output_dir, + "calibration", + f"{component}_final_pct_change.png", + ) + ) return CalibrationRunResult( converged=False, @@ -427,7 +481,8 @@ def _calibrate_component( if bespoke_callable is not None: # Preserve compatibility with helper modules that expect a global # `state` symbol and/or no explicit arguments. - _run_bespoke_report(bespoke_callable, state, component_settings) + kwargs = {"state": state, "component_settings": component_settings} + bespoke_callable(**kwargs) if component_converged: break @@ -657,6 +712,12 @@ def _evaluate_and_update( method = row["method"] hold_fast = bool(row["hold_fast"]) + default_increment = ( + row["default_increment"] + if "default_increment" in row.index + else DEFAULT_INCREMENT + ) + prev_value = float(updated.loc[coefficient_name, "value"]) model_value = _eval_numeric_value( @@ -688,6 +749,7 @@ def _evaluate_and_update( damping=damping, component_name=component_name, description=description, + default_increment=default_increment, ) candidate_value = prev_value if hold_fast else prev_value + raw_delta @@ -811,6 +873,7 @@ def _compute_delta( damping: float, component_name: str, description: str, + default_increment: float, ) -> float: """Compute damped coefficient delta using selected method.""" if damping < 0: @@ -820,9 +883,15 @@ def _compute_delta( if method == "log_ratio": if model_value <= 0 or target_value <= 0: - raise RuntimeError( - f"log_ratio requires positive model and target values for {component_name} / {description}" + logger.warning( + f"log_ratio requires positive model and target values for {component_name} / {description}. Falling back to default increment {default_increment}" ) + if model_value <= 0 and target_value > 0: + return default_increment + elif model_value > 0 and target_value <= 0: + return -default_increment + else: + return 0 delta = math.log(target_value / model_value) * damping elif method == "odds_ratio": @@ -831,9 +900,15 @@ def _compute_delta( denominator = (target_value * model_value) - model_value if numerator <= 0 or denominator <= 0: - raise RuntimeError( - f"odds_ratio produced invalid numerator/denominator for {component_name} / {description}" + logger.warning( + f"odds_ratio produced invalid numerator/denominator for {component_name} / {description}. Falling back to default increment {default_increment}" ) + if model_value <= 0 and target_value > 0: + return default_increment + elif model_value > 0 and target_value <= 0: + return -default_increment + else: + return 0 ratio = numerator / denominator if ratio <= 0 or not np.isfinite(ratio): @@ -939,25 +1014,6 @@ def _write_generic_report( _append_csv(report, path) -def _run_bespoke_report( - bespoke_callable, - state: workflow.State, - component_settings: CalibrationComponentSettings, -) -> None: - """Run optional bespoke report callback from helper module.""" - try: - # Support no-argument callback, callback(state), or callback(state, component_settings). - sig = inspect.signature(bespoke_callable) - if len(sig.parameters) == 0: - bespoke_callable() - elif len(sig.parameters) == 1: - bespoke_callable(state) - else: - bespoke_callable(state, component_settings) - except TypeError: - bespoke_callable() - - def _load_helper_symbols( state: workflow.State, component_settings: CalibrationComponentSettings, From b1db935cd9679d6c5501d3dc8caccc0650952b01 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Mon, 6 Jul 2026 17:29:35 -0400 Subject: [PATCH 09/96] Batch output graphs --- activitysim/core/calibration.py | 109 +++++++++++++++++++------------- 1 file changed, 64 insertions(+), 45 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 99b4116662..8dca21f413 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -32,6 +32,7 @@ CALIBRATION_FINAL_COEFFICIENTS_FILE = "calibration/final_calibrated_coefficients.csv" DEFAULT_INCREMENT = 2.0 +MAX_COEFFS_IN_GRAPH = 10 CALIBRATION_REQUIRED_COLUMNS = [ "description", @@ -270,56 +271,74 @@ def run_calibration_loop( for component in iteration_records.component.unique(): - ax = ( - iteration_records.loc[iteration_records.component == component] - .next_coefficient.unstack("coefficient") - .plot() - ) - ax.xaxis.set_label_text("Component iteration") - ax.yaxis.set_label_text("Coefficient value") - - ax.legend(title="Coefficient label") - ax.figure.savefig( - os.path.join( - state.filesystem.output_dir, - "calibration", - f"{component}_coefficient_progress.png", + recs = iteration_records.loc[iteration_records.component == component] + coefs = sorted(recs.index.get_level_values("coefficient").unique()) + n_sets = math.ceil(len(coefs) / MAX_COEFFS_IN_GRAPH) + for coef_set in range(n_sets): + set_coefs = coefs[ + coef_set + * MAX_COEFFS_IN_GRAPH : min( + len(coefs), (coef_set + 1) * MAX_COEFFS_IN_GRAPH + ) + ] + ax = ( + recs[recs.index.get_level_values("coefficient").isin(set_coefs)] + .next_coefficient.unstack("coefficient") + .plot() + ) + ax.xaxis.set_label_text("Component iteration") + ax.yaxis.set_label_text("Coefficient value") + + ax.legend(title="Coefficient label") + ax.figure.savefig( + os.path.join( + state.filesystem.output_dir, + "calibration", + f"{component}_coefficient_progress_set_{coef_set}.png", + ) ) - ) - last_global = iteration_records.index.get_level_values("global_iter")[-1] - last_comp = iteration_records.loc[last_global].index.get_level_values( - "component_iter" - )[-1] - - last_records = iteration_records.xs( - (last_global, last_comp), level=("global_iter", "component_iter") - )[["target_value", "model_value"]] - ax = last_records.plot.bar() - ax.xaxis.set_tick_params(rotation=45) - ax.xaxis.set_label_text("Component value") - plt.tight_layout() - ax.figure.savefig( - os.path.join( - state.filesystem.output_dir, - "calibration", - f"{component}_final_components.png", + last_global = recs[ + recs.index.get_level_values("coefficient").isin(set_coefs) + ].index.get_level_values("global_iter")[-1] + last_comp = ( + recs[recs.index.get_level_values("coefficient").isin(set_coefs)] + .loc[last_global] + .index.get_level_values("component_iter")[-1] ) - ) - _ = plt.subplots() - - pct_diff = last_records.diff(axis=1).model_value / last_records.target_value - ax = pct_diff.plot.bar() - ax.xaxis.set_tick_params(rotation=45) - plt.tight_layout() - ax.figure.savefig( - os.path.join( - state.filesystem.output_dir, - "calibration", - f"{component}_final_pct_change.png", + last_records = recs[ + recs.index.get_level_values("coefficient").isin(set_coefs) + ].xs((last_global, last_comp), level=("global_iter", "component_iter"))[ + ["target_value", "model_value"] + ] + ax = last_records.plot.bar() + ax.xaxis.set_tick_params(rotation=45) + ax.xaxis.set_label_text("Component value") + plt.tight_layout() + ax.figure.savefig( + os.path.join( + state.filesystem.output_dir, + "calibration", + f"{component}_final_components_set_{coef_set}.png", + ) + ) + + _ = plt.subplots() + + pct_diff = ( + last_records.diff(axis=1).model_value / last_records.target_value + ) + ax = pct_diff.plot.bar() + ax.xaxis.set_tick_params(rotation=45) + plt.tight_layout() + ax.figure.savefig( + os.path.join( + state.filesystem.output_dir, + "calibration", + f"{component}_final_pct_change_set_{coef_set}.png", + ) ) - ) return CalibrationRunResult( converged=False, From 0b1f72346d8b0d00247d46b75a8c09213f4d3213 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Mon, 6 Jul 2026 17:43:01 -0400 Subject: [PATCH 10/96] Minor plotting updates --- activitysim/core/calibration.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 8dca21f413..b501c342f5 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -24,6 +24,8 @@ logger = logging.getLogger("calibration") +plt.style.use("seaborn-v0_8-darkgrid") + CALIBRATION_SETTINGS_FILE_NAME = "calibration.yaml" CALIBRATION_OUTPUT_DIR = "calibration" CALIBRATION_PROGRESS_FILE = "calibration/calibration_progress.json" @@ -312,7 +314,7 @@ def run_calibration_loop( ].xs((last_global, last_comp), level=("global_iter", "component_iter"))[ ["target_value", "model_value"] ] - ax = last_records.plot.bar() + ax = last_records.plot.barh() ax.xaxis.set_tick_params(rotation=45) ax.xaxis.set_label_text("Component value") plt.tight_layout() @@ -329,8 +331,10 @@ def run_calibration_loop( pct_diff = ( last_records.diff(axis=1).model_value / last_records.target_value ) - ax = pct_diff.plot.bar() + ax = pct_diff.plot.barh() ax.xaxis.set_tick_params(rotation=45) + ax.xaxis.set_label_text("Coefficient") + ax.yaxis.set_label_text("% Change") plt.tight_layout() ax.figure.savefig( os.path.join( From 96527ff8d22b72b12633ab77e4d92635ac12b782 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Mon, 6 Jul 2026 19:44:26 -0700 Subject: [PATCH 11/96] expose context to bespoke functions --- activitysim/core/calibration.py | 45 ++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index b501c342f5..068ff50ac1 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -427,7 +427,7 @@ def _calibrate_component( coefficients_df = state.filesystem.read_model_coefficients( model_settings=model_settings ) - helper_symbols, bespoke_callable = _load_helper_symbols( + helper_symbols, bespoke_callable, helper_module = _load_helper_symbols( state, component_settings, ) @@ -477,6 +477,7 @@ def _calibrate_component( state.run(models=[run_model_name], resume_after=prior_step) eval_context = _build_expression_context(state, helper_symbols) + _bind_context_to_helper_module_globals(helper_module, eval_context) ( row_records, @@ -703,11 +704,47 @@ def _build_expression_context( except Exception: # Some entries may not be available as dataframes in all contexts. continue + try: + network_los = state.get_injectable("network_los") + context["network_los"] = network_los + context["skim_dict"] = network_los.get_default_skim_dict() + except Exception: + # Network LOS may not be available in all contexts. + pass context.update(helper_symbols) return context +def _bind_context_to_helper_module_globals( + helper_module: Any | None, + eval_context: dict[str, Any], +) -> None: + """Bind eval context names into helper module globals. + + Helper functions referenced by calibration expressions resolve free names + from their module globals, not from eval locals. This syncs the current + evaluation context into the helper module globals each component iteration. + """ + if helper_module is None: + return + + excluded = { + "__builtins__", + "__name__", + "__package__", + "__loader__", + "__spec__", + "__file__", + "__cached__", + } + + filtered_context = { + key: value for key, value in eval_context.items() if key not in excluded + } + helper_module.__dict__.update(filtered_context) + + def _evaluate_and_update( component_name: str, calibration_spec_df: pd.DataFrame, @@ -1040,10 +1077,10 @@ def _write_generic_report( def _load_helper_symbols( state: workflow.State, component_settings: CalibrationComponentSettings, -) -> tuple[dict[str, Any], Any | None]: +) -> tuple[dict[str, Any], Any | None, Any | None]: """Load helper module and return evaluation symbols and bespoke function.""" if not component_settings.helper_module: - return {}, None + return {}, None, None module = _load_helper_module(state, component_settings.helper_module) symbols = { @@ -1059,7 +1096,7 @@ def _load_helper_symbols( ) bespoke = getattr(module, fn_name) - return symbols, bespoke + return symbols, bespoke, module def _load_helper_module(state: workflow.State, helper_module: str): From d347f9c7f25796c10d6fb5db30737842e287dd80 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Mon, 6 Jul 2026 21:44:41 -0700 Subject: [PATCH 12/96] output updates --- activitysim/core/calibration.py | 252 +++++++++++++++++++------------- 1 file changed, 147 insertions(+), 105 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 068ff50ac1..2fd2888f99 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -15,6 +15,7 @@ import numpy as np import pandas as pd +import matplotlib import matplotlib.pyplot as plt from pydantic import model_validator @@ -25,6 +26,7 @@ logger = logging.getLogger("calibration") plt.style.use("seaborn-v0_8-darkgrid") +matplotlib.use('Agg') # Forces non-interactive background rendering CALIBRATION_SETTINGS_FILE_NAME = "calibration.yaml" CALIBRATION_OUTPUT_DIR = "calibration" @@ -238,6 +240,7 @@ def run_calibration_loop( prior_step=prior_step, global_iter=global_iter, ) + _write_component_plots(state, component) all_converged = all_converged and component_result.converged @@ -265,85 +268,6 @@ def run_calibration_loop( _write_final_coefficients_snapshot(state, calibration_settings) - iteration_records = ( - pd.read_csv(state.get_output_file_path(CALIBRATION_ITERATION_FILE)) - .set_index(["global_iter", "component_iter", "coefficient"]) - .sort_index() - ) - - for component in iteration_records.component.unique(): - - recs = iteration_records.loc[iteration_records.component == component] - coefs = sorted(recs.index.get_level_values("coefficient").unique()) - n_sets = math.ceil(len(coefs) / MAX_COEFFS_IN_GRAPH) - for coef_set in range(n_sets): - set_coefs = coefs[ - coef_set - * MAX_COEFFS_IN_GRAPH : min( - len(coefs), (coef_set + 1) * MAX_COEFFS_IN_GRAPH - ) - ] - ax = ( - recs[recs.index.get_level_values("coefficient").isin(set_coefs)] - .next_coefficient.unstack("coefficient") - .plot() - ) - ax.xaxis.set_label_text("Component iteration") - ax.yaxis.set_label_text("Coefficient value") - - ax.legend(title="Coefficient label") - ax.figure.savefig( - os.path.join( - state.filesystem.output_dir, - "calibration", - f"{component}_coefficient_progress_set_{coef_set}.png", - ) - ) - - last_global = recs[ - recs.index.get_level_values("coefficient").isin(set_coefs) - ].index.get_level_values("global_iter")[-1] - last_comp = ( - recs[recs.index.get_level_values("coefficient").isin(set_coefs)] - .loc[last_global] - .index.get_level_values("component_iter")[-1] - ) - - last_records = recs[ - recs.index.get_level_values("coefficient").isin(set_coefs) - ].xs((last_global, last_comp), level=("global_iter", "component_iter"))[ - ["target_value", "model_value"] - ] - ax = last_records.plot.barh() - ax.xaxis.set_tick_params(rotation=45) - ax.xaxis.set_label_text("Component value") - plt.tight_layout() - ax.figure.savefig( - os.path.join( - state.filesystem.output_dir, - "calibration", - f"{component}_final_components_set_{coef_set}.png", - ) - ) - - _ = plt.subplots() - - pct_diff = ( - last_records.diff(axis=1).model_value / last_records.target_value - ) - ax = pct_diff.plot.barh() - ax.xaxis.set_tick_params(rotation=45) - ax.xaxis.set_label_text("Coefficient") - ax.yaxis.set_label_text("% Change") - plt.tight_layout() - ax.figure.savefig( - os.path.join( - state.filesystem.output_dir, - "calibration", - f"{component}_final_pct_change_set_{coef_set}.png", - ) - ) - return CalibrationRunResult( converged=False, completed_global_iterations=calibration_settings.run.global_iterations, @@ -476,7 +400,7 @@ def _calibrate_component( ) state.run(models=[run_model_name], resume_after=prior_step) - eval_context = _build_expression_context(state, helper_symbols) + eval_context = _build_expression_context(state, helper_symbols, component_name) _bind_context_to_helper_module_globals(helper_module, eval_context) ( @@ -496,20 +420,20 @@ def _calibrate_component( coefficients_df = new_coefficients_df _persist_coefficients_to_config(state, model_settings, coefficients_df) - _append_iteration_records(state, row_records) + _append_iteration_records(state, component_name, row_records) _append_summary_records(state, [summary_record]) if component_settings.reports.generic: _write_generic_report(state, component_name, row_records) if bespoke_callable is not None: - # Preserve compatibility with helper modules that expect a global - # `state` symbol and/or no explicit arguments. - kwargs = {"state": state, "component_settings": component_settings} - bespoke_callable(**kwargs) + # Helper module globals were already updated from eval_context, + # so bespoke functions can access tables/injectables directly. + bespoke_callable() if component_converged: break + state.checkpoint.add(component_name) return CalibrationComponentResult( @@ -689,12 +613,14 @@ def _warn_if_initial_values_outside_bounds( def _build_expression_context( state: workflow.State, helper_symbols: dict[str, Any], + component_name: str, ) -> dict[str, Any]: """Create the evaluation context for model_value and target_value expressions.""" context: dict[str, Any] = { "state": state, "np": np, "pd": pd, + "component_output_dir": _component_output_dir(state, component_name), } # Load active tables into context for direct expression access. @@ -758,10 +684,8 @@ def _evaluate_and_update( records: list[dict[str, Any]] = [] max_difference = -math.inf - max_difference_component = "" max_difference_coefficient = "" max_change = -math.inf - max_change_component = "" max_change_coefficient = "" num_converged = 0 @@ -814,18 +738,18 @@ def _evaluate_and_update( candidate_value = prev_value if hold_fast else prev_value + raw_delta - under_min = False - over_max = False + at_min = False + at_max = False lower = row["min"] upper = row["max"] - if not pd.isna(lower) and candidate_value < float(lower): + if not pd.isna(lower) and candidate_value <= float(lower): candidate_value = float(lower) - under_min = True - if not pd.isna(upper) and candidate_value > float(upper): + at_min = True + if not pd.isna(upper) and candidate_value >= float(upper): candidate_value = float(upper) - over_max = True + at_max = True if not np.isfinite(candidate_value): raise RuntimeError( @@ -839,12 +763,10 @@ def _evaluate_and_update( if abs_diff > max_difference: max_difference = abs_diff - max_difference_component = component_name max_difference_coefficient = coefficient_name if abs_change > max_change: max_change = abs_change - max_change_component = component_name max_change_coefficient = coefficient_name if converged: @@ -866,8 +788,8 @@ def _evaluate_and_update( "coef_delta": abs_change, "next_coefficient": candidate_value, "converged": converged, - "under_min": under_min, - "over_max": over_max, + "at_min": at_min, + "at_max": at_max, } ) @@ -880,10 +802,8 @@ def _evaluate_and_update( "component_iter": component_iter, "component": component_name, "max_difference": max_difference if max_difference != -math.inf else 0.0, - "max_difference_component": max_difference_component, "max_difference_coefficient": max_difference_coefficient, "max_change": max_change if max_change != -math.inf else 0.0, - "max_change_component": max_change_component, "max_change_coefficient": max_change_coefficient, "num_converged_iter": num_converged, "tot_converged": num_converged, @@ -1014,14 +934,22 @@ def _persist_coefficients_to_config( def _append_iteration_records( - state: workflow.State, records: list[dict[str, Any]] + state: workflow.State, component_name: str, records: list[dict[str, Any]] ) -> None: """Append per-coefficient calibration iteration records.""" if not records: return - path = state.get_output_file_path(CALIBRATION_ITERATION_FILE) df = pd.DataFrame(records) - _append_csv(df, path) + + # Save a global iteration history file + global_path = state.get_output_file_path(CALIBRATION_ITERATION_FILE) + _append_csv(df, global_path) + + # Also write component-local iteration history + component_path = _component_output_dir(state, component_name) / Path( + CALIBRATION_ITERATION_FILE + ).name + _append_csv(df, component_path) def _append_summary_records( @@ -1042,6 +970,122 @@ def _append_csv(df: pd.DataFrame, path: Path) -> None: df.to_csv(path, mode="a", index=False, header=write_header) +def _component_output_dir(state: workflow.State, component_name: str) -> Path: + """Return output/calibration/ and ensure it exists.""" + component_dir = state.get_output_file_path( + f"calibration/{component_name}" + ) + os.makedirs(component_dir, exist_ok=True) + return component_dir + + +def _write_component_plots(state: workflow.State, component_name: str) -> None: + """Write/update all standard plots for one calibrated component.""" + recs = _read_component_iteration_records(state, component_name) + if recs is None or recs.empty: + return + + # Segment coefficients into manageable sets for plotting + coefs = sorted(recs.index.get_level_values("coefficient").unique()) + n_sets = math.ceil(len(coefs) / MAX_COEFFS_IN_GRAPH) + for coef_set in range(n_sets): + set_coefs = coefs[ + coef_set * MAX_COEFFS_IN_GRAPH : min( + len(coefs), (coef_set + 1) * MAX_COEFFS_IN_GRAPH + ) + ] + _plot_coefficient_progress(state, component_name, recs, set_coefs, coef_set) + last_records = _component_last_records(recs, set_coefs) + _plot_component_values(state, component_name, last_records, coef_set) + _plot_component_pct_change(state, component_name, last_records, coef_set) + + +def _read_component_iteration_records( + state: workflow.State, component_name: str +) -> pd.DataFrame | None: + """Read all iteration records for a single component.""" + path = state.get_output_file_path(CALIBRATION_ITERATION_FILE) + if not path.exists(): + return None + + iteration_records = ( + pd.read_csv(path) + .set_index(["global_iter", "component_iter", "coefficient"]) + .sort_index() + ) + return iteration_records.loc[iteration_records.component == component_name] + + +def _plot_coefficient_progress( + state: workflow.State, + component_name: str, + recs: pd.DataFrame, + set_coefs: list[str], + coef_set: int, +) -> None: + """Plot coefficient value progression for one coefficient subset.""" + component_dir = _component_output_dir(state, component_name) + ax = ( + recs[recs.index.get_level_values("coefficient").isin(set_coefs)] + .next_coefficient.unstack("coefficient") + .plot(figsize=(10,5)) + ) + ax.xaxis.set_label_text("Component iteration") + ax.yaxis.set_label_text("Coefficient value") + ax.legend(title="Coefficient label", loc="center left", bbox_to_anchor=(1.02, 0.5)) + plt.tight_layout() + ax.figure.savefig( + component_dir / f"coefficient_progress_set_{coef_set}.png", + bbox_inches="tight", + ) + plt.close(ax.figure) + + +def _component_last_records(recs: pd.DataFrame, set_coefs: list[str]) -> pd.DataFrame: + """Select target/model values for the latest iteration and coefficient subset.""" + filtered = recs[recs.index.get_level_values("coefficient").isin(set_coefs)] + last_global = filtered.index.get_level_values("global_iter")[-1] + last_comp = filtered.loc[last_global].index.get_level_values("component_iter")[-1] + return filtered.xs((last_global, last_comp), level=("global_iter", "component_iter"))[ + ["target_value", "model_value"] + ] + + +def _plot_component_values( + state: workflow.State, + component_name: str, + last_records: pd.DataFrame, + coef_set: int, +) -> None: + """Plot final target/model component values for one coefficient subset.""" + component_dir = _component_output_dir(state, component_name) + ax = last_records.plot.bar(figsize=(10,5)) + ax.xaxis.set_tick_params(rotation=45) + ax.xaxis.set_label_text("Component value") + plt.tight_layout() + ax.figure.savefig(component_dir / f"final_components_set_{coef_set}.png") + plt.close(ax.figure) + + +def _plot_component_pct_change( + state: workflow.State, + component_name: str, + last_records: pd.DataFrame, + coef_set: int, +) -> None: + """Plot final percent difference for one coefficient subset.""" + component_dir = _component_output_dir(state, component_name) + fig, ax = plt.subplots(figsize=(10,5)) + pct_diff = last_records.diff(axis=1).model_value / last_records.target_value + ax = pct_diff.plot.bar(ax=ax) + ax.xaxis.set_tick_params(rotation=45) + ax.xaxis.set_label_text("Coefficient") + ax.yaxis.set_label_text("% Difference between Model and Target") + plt.tight_layout() + ax.figure.savefig(component_dir / f"final_pct_change_set_{coef_set}.png") + plt.close(ax.figure) + + def _write_generic_report( state: workflow.State, component_name: str, @@ -1068,9 +1112,7 @@ def _write_generic_report( .sort_values(["global_iter", "component_iter", "description"]) ) - path = state.get_output_file_path( - f"calibration/{component_name}_generic_report.csv" - ) + path = _component_output_dir(state, component_name) / "generic_report.csv" _append_csv(report, path) From 9b026082731c4f22023272e3fa826930806ef7d8 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 7 Jul 2026 11:54:48 -0400 Subject: [PATCH 13/96] add component settings --- activitysim/core/calibration.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 2fd2888f99..e9dd40cc31 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -400,7 +400,7 @@ def _calibrate_component( ) state.run(models=[run_model_name], resume_after=prior_step) - eval_context = _build_expression_context(state, helper_symbols, component_name) + eval_context = _build_expression_context(state, helper_symbols, component_name, component_settings) _bind_context_to_helper_module_globals(helper_module, eval_context) ( @@ -614,6 +614,7 @@ def _build_expression_context( state: workflow.State, helper_symbols: dict[str, Any], component_name: str, + component_settings: CalibrationComponentSettings, ) -> dict[str, Any]: """Create the evaluation context for model_value and target_value expressions.""" context: dict[str, Any] = { @@ -621,6 +622,7 @@ def _build_expression_context( "np": np, "pd": pd, "component_output_dir": _component_output_dir(state, component_name), + "component_settings": component_settings, } # Load active tables into context for direct expression access. From 4475471e1a4057e5e66af6712f7d445c0bbebc90 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 7 Jul 2026 12:17:59 -0400 Subject: [PATCH 14/96] Add calibration configs --- .../auto_ownership_calib_helper.py | 36 ++++++++ .../auto_ownership_calibration.csv | 5 ++ .../configs_calibration/calibration.yaml | 36 ++++++++ .../tour_mode_choice_calib_helper.py | 36 ++++++++ .../tour_mode_choice_calibration.csv | 61 +++++++++++++ .../workplace_location_calib_helper.py | 89 +++++++++++++++++++ .../workplace_location_calibration.csv | 7 ++ 7 files changed, 270 insertions(+) create mode 100644 activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py create mode 100644 activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calibration.csv create mode 100644 activitysim/examples/prototype_mtc/configs_calibration/calibration.yaml create mode 100644 activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py create mode 100644 activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calibration.csv create mode 100644 activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py create mode 100644 activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calibration.csv diff --git a/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py new file mode 100644 index 0000000000..516f5cd85f --- /dev/null +++ b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py @@ -0,0 +1,36 @@ +import pandas as pd +import os + +SURVEY_DATA_FOLDER = r"U:\projects\clients\ASIM\autocalibration\calibration_test_mtc\data" + +def report_auto_ownership(): + model_hhs = households + survey_hhs = None + try: + survey_hhs = pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_households.csv")) + except FileNotFoundError: + raise FileNotFoundError(f"No survey file override_households.csv found in {SURVEY_DATA_FOLDER}!") + + model_summary = model_hhs.auto_ownership.value_counts(normalize=True).sort_index().fillna(0) + # survey_summary = survey_hhs.groupby("auto_ownership").household_weight.sum() + # survey_summary = survey_summary / survey_hhs.household_weight.sum() + survey_summary = survey_hhs.auto_ownership.value_counts(normalize=True).sort_index().fillna(0) + + summary_df = ( + pd.DataFrame({"model": model_summary, "survey": survey_summary}) + .reset_index() + .rename(columns={"index": "num_autos"}) + ) + + print(summary_df) + + # plot comparing model and survey distributions + import matplotlib.pyplot as plt + + summary_df.plot(x="auto_ownership", y=["model", "survey"], kind="bar") + plt.title("Auto Ownership Distribution: Model vs Survey") + plt.xlabel("Number of Autos") + plt.ylabel("Proportion of Households") + plt.legend(title="Data Source") + plt.savefig(os.path.join(component_output_dir, "auto_ownership_comparison.png")) + plt.close() \ No newline at end of file diff --git a/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calibration.csv b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calibration.csv new file mode 100644 index 0000000000..b320b379b2 --- /dev/null +++ b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calibration.csv @@ -0,0 +1,5 @@ +description,coefficient,model_value,target_value,hold_fast,min,max,damping,method,tolerance +0 auto ownership share,coef_calib_auto_0,len(households[households.auto_ownership ==0]) / len(households),0.06812,FALSE,-5,5,1,log_ratio,0.02 +2 auto ownership share,coef_calib_auto_2,len(households[households.auto_ownership ==2]) / len(households),0.348413,FALSE,-5,5,1,log_ratio,0.01 +3 auto ownership share,coef_calib_auto_3,len(households[households.auto_ownership ==3]) / len(households),0.13718,FALSE,-5,5,1,log_ratio,0.01 +4 auto ownership share,coef_calib_auto_4,len(households[households.auto_ownership ==4]) / len(households),0.057501,FALSE,-5,5,1,log_ratio,0.01 diff --git a/activitysim/examples/prototype_mtc/configs_calibration/calibration.yaml b/activitysim/examples/prototype_mtc/configs_calibration/calibration.yaml new file mode 100644 index 0000000000..c4949c0718 --- /dev/null +++ b/activitysim/examples/prototype_mtc/configs_calibration/calibration.yaml @@ -0,0 +1,36 @@ +enable: True +run: + calibrate_models: + - workplace_location + - auto_ownership_simulate + - tour_mode_choice_simulate + global_iterations: 3 + +model_settings: + workplace_location: + calibration_spec: workplace_location_calibration.csv + helper_module: workplace_location_calib_helper.py + submodel_max_iterations: 3 + reports: + generic: true + bespoke: report_workplace_location + survey_file: survey_persons.csv + + auto_ownership_simulate: + calibration_spec: auto_ownership_calibration.csv + helper_module: auto_ownership_calib_helper.py + submodel_max_iterations: 3 + reports: + generic: true + bespoke: report_auto_ownership + survey_file: survey_households.csv + + tour_mode_choice_simulate: + calibration_spec: tour_mode_choice_calibration.csv + helper_module: tour_mode_choice_calib_helper.py + submodel_max_iterations: 3 + reports: + generic: true + bespoke: report_tour_mode_choice + survey_file: survey_tours.csv + diff --git a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py new file mode 100644 index 0000000000..a87708a795 --- /dev/null +++ b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py @@ -0,0 +1,36 @@ +import pandas as pd +import os + +def report_tour_mode_choice(): + model_tours = state.get_table("tours") + survey_tours = None + for data_dir in state.filesystem.data_dir: + try: + survey_tours = pd.read_csv(os.path.join(data_dir, component_settings.survey_file)) + break + except FileNotFoundError: + pass + assert survey_tours is not None, f"No survey file {component_settings.survey_file} found in data dirs!" + + model_summary = model_tours.tour_mode.value_counts(normalize=True).sort_index().fillna(0) + survey_summary = survey_tours.groupby("tour_mode").tour_weight.sum() + survey_summary = survey_summary / survey_tours.tour_weight.sum() + + summary_df = ( + pd.DataFrame({"model": model_summary, "survey": survey_summary}) + .reset_index() + .rename(columns={"index": "tour_mode"}) + ) + + print(summary_df) + + # plot comparing model and survey distributions + import matplotlib.pyplot as plt + + summary_df.plot(x="tour_mode", y=["model", "survey"], kind="bar") + plt.title("Tour Mode Choice Distribution: Model vs Survey") + plt.xlabel("Tour Mode") + plt.ylabel("Proportion of Tours") + plt.legend(title="Data Source") + plt.savefig("tour_mode_choice_comparison.png") + plt.close() \ No newline at end of file diff --git a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calibration.csv b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calibration.csv new file mode 100644 index 0000000000..a8c1c50af0 --- /dev/null +++ b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calibration.csv @@ -0,0 +1,61 @@ +description,coefficient,model_value,target_value,hold_fast,min,max,damping,method,tolerance,default_increment +DRIVEALONEPAY share for zero auto HHs,coef_calib_DRIVEALONEPAY_zero_auto,"len(tours[(tours.tour_mode == ""DRIVEALONEPAY"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.036379517,FALSE,-5,5,1,log_ratio,0.02,2 +SHARED2FREE share for zero auto HHs,coef_calib_SHARED2FREE_zero_auto,"len(tours[(tours.tour_mode == ""SHARED2FREE"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.098670942,FALSE,-5,5,1,log_ratio,0.01,2 +SHARED2PAY share for zero auto HHs,coef_calib_SHARED2PAY_zero_auto,"len(tours[(tours.tour_mode == ""SHARED2PAY"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.043195465,FALSE,-5,5,1,log_ratio,0.01,2 +SHARED3FREE share for zero auto HHs,coef_calib_SHARED3FREE_zero_auto,"len(tours[(tours.tour_mode == ""SHARED3FREE"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.054018294,FALSE,-5,5,1,log_ratio,0.01,2 +SHARED3PAY share for zero auto HHs,coef_calib_SHARED3PAY_zero_auto,"len(tours[(tours.tour_mode == ""SHARED3PAY"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.027328167,FALSE,-5,5,1,log_ratio,0.01,2 +WALK share for zero auto HHs,coef_calib_WALK_zero_auto,"len(tours[(tours.tour_mode == ""WALK"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.059569409,FALSE,-5,5,1,log_ratio,0.01,2 +BIKE share for zero auto HHs,coef_calib_BIKE_zero_auto,"len(tours[(tours.tour_mode == ""BIKE"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.064270507,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_LOC share for zero auto HHs,coef_calib_WALK_LOC_zero_auto,"len(tours[(tours.tour_mode == ""WALK_LOC"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.031069537,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_LRF share for zero auto HHs,coef_calib_WALK_LRF_zero_auto,"len(tours[(tours.tour_mode == ""WALK_LRF"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.055759405,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_EXP share for zero auto HHs,coef_calib_WALK_EXP_zero_auto,"len(tours[(tours.tour_mode == ""WALK_EXP"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.012576768,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_HVY share for zero auto HHs,coef_calib_WALK_HVY_zero_auto,"len(tours[(tours.tour_mode == ""WALK_HVY"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.004320335,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_COM share for zero auto HHs,coef_calib_WALK_COM_zero_auto,"len(tours[(tours.tour_mode == ""WALK_COM"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.055175324,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_LOC share for zero auto HHs,coef_calib_DRIVE_LOC_zero_auto,"len(tours[(tours.tour_mode == ""DRIVE_LOC"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.086416787,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_LRF share for zero auto HHs,coef_calib_DRIVE_LRF_zero_auto,"len(tours[(tours.tour_mode == ""DRIVE_LRF"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.067921667,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_EXP share for zero auto HHs,coef_calib_DRIVE_EXP_zero_auto,"len(tours[(tours.tour_mode == ""DRIVE_EXP"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.044918204,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_HVY share for zero auto HHs,coef_calib_DRIVE_HVY_zero_auto,"len(tours[(tours.tour_mode == ""DRIVE_HVY"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.033260131,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_COM share for zero auto HHs,coef_calib_DRIVE_COM_zero_auto,"len(tours[(tours.tour_mode == ""DRIVE_COM"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.021919061,FALSE,-5,5,1,log_ratio,0.01,2 +TAXI share for zero auto HHs,coef_calib_TAXI_zero_auto,"len(tours[(tours.tour_mode == ""TAXI"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.076918756,FALSE,-5,5,1,log_ratio,0.01,2 +TNC_SINGLE share for zero auto HHs,coef_calib_TNC_SINGLE_zero_auto,"len(tours[(tours.tour_mode == ""TNC_SINGLE"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.07478863,FALSE,-5,5,1,log_ratio,0.01,2 +TNC_SHARED share for zero auto HHs,coef_calib_TNC_SHARED_zero_auto,"len(tours[(tours.tour_mode == ""TNC_SHARED"") & tours.household_id.isin(households[households.auto_ownership == 0].index)]) / len(tours[tours.household_id.isin(households[households.auto_ownership == 0].index)])",0.051523091,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVEALONEPAY share for auto insufficient HHs,coef_calib_DRIVEALONEPAY_auto_insuff,"len(tours[(tours.tour_mode == ""DRIVEALONEPAY"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.036379517,FALSE,-5,5,1,log_ratio,0.02,2 +SHARED2FREE share for auto insufficient HHs,coef_calib_SHARED2FREE_auto_insuff,"len(tours[(tours.tour_mode == ""SHARED2FREE"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.098670942,FALSE,-5,5,1,log_ratio,0.01,2 +SHARED2PAY share for auto insufficient HHs,coef_calib_SHARED2PAY_auto_insuff,"len(tours[(tours.tour_mode == ""SHARED2PAY"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.043195465,FALSE,-5,5,1,log_ratio,0.01,2 +SHARED3FREE share for auto insufficient HHs,coef_calib_SHARED3FREE_auto_insuff,"len(tours[(tours.tour_mode == ""SHARED3FREE"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.054018294,FALSE,-5,5,1,log_ratio,0.01,2 +SHARED3PAY share for auto insufficient HHs,coef_calib_SHARED3PAY_auto_insuff,"len(tours[(tours.tour_mode == ""SHARED3PAY"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.027328167,FALSE,-5,5,1,log_ratio,0.01,2 +WALK share for auto insufficient HHs,coef_calib_WALK_auto_insuff,"len(tours[(tours.tour_mode == ""WALK"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.059569409,FALSE,-5,5,1,log_ratio,0.01,2 +BIKE share for auto insufficient HHs,coef_calib_BIKE_auto_insuff,"len(tours[(tours.tour_mode == ""BIKE"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.064270507,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_LOC share for auto insufficient HHs,coef_calib_WALK_LOC_auto_insuff,"len(tours[(tours.tour_mode == ""WALK_LOC"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.031069537,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_LRF share for auto insufficient HHs,coef_calib_WALK_LRF_auto_insuff,"len(tours[(tours.tour_mode == ""WALK_LRF"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.055759405,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_EXP share for auto insufficient HHs,coef_calib_WALK_EXP_auto_insuff,"len(tours[(tours.tour_mode == ""WALK_EXP"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.012576768,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_HVY share for auto insufficient HHs,coef_calib_WALK_HVY_auto_insuff,"len(tours[(tours.tour_mode == ""WALK_HVY"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.004320335,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_COM share for auto insufficient HHs,coef_calib_WALK_COM_auto_insuff,"len(tours[(tours.tour_mode == ""WALK_COM"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.055175324,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_LOC share for auto insufficient HHs,coef_calib_DRIVE_LOC_auto_insuff,"len(tours[(tours.tour_mode == ""DRIVE_LOC"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.086416787,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_LRF share for auto insufficient HHs,coef_calib_DRIVE_LRF_auto_insuff,"len(tours[(tours.tour_mode == ""DRIVE_LRF"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.067921667,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_EXP share for auto insufficient HHs,coef_calib_DRIVE_EXP_auto_insuff,"len(tours[(tours.tour_mode == ""DRIVE_EXP"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.044918204,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_HVY share for auto insufficient HHs,coef_calib_DRIVE_HVY_auto_insuff,"len(tours[(tours.tour_mode == ""DRIVE_HVY"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.033260131,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_COM share for auto insufficient HHs,coef_calib_DRIVE_COM_auto_insuff,"len(tours[(tours.tour_mode == ""DRIVE_COM"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.021919061,FALSE,-5,5,1,log_ratio,0.01,2 +TAXI share for auto insufficient HHs,coef_calib_TAXI_auto_insuff,"len(tours[(tours.tour_mode == ""TAXI"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.076918756,FALSE,-5,5,1,log_ratio,0.01,2 +TNC_SINGLE share for auto insufficient HHs,coef_calib_TNC_SINGLE_auto_insuff,"len(tours[(tours.tour_mode == ""TNC_SINGLE"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.07478863,FALSE,-5,5,1,log_ratio,0.01,2 +TNC_SHARED share for auto insufficient HHs,coef_calib_TNC_SHARED_auto_insuff,"len(tours[(tours.tour_mode == ""TNC_SHARED"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership < households.num_workers)].index)])",0.051523091,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVEALONEPAY share for auto sufficient HHs,coef_calib_DRIVEALONEPAY_auto_suff,"len(tours[(tours.tour_mode == ""DRIVEALONEPAY"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.036379517,FALSE,-5,5,1,log_ratio,0.02,2 +SHARED2FREE share for auto sufficient HHs,coef_calib_SHARED2FREE_auto_suff,"len(tours[(tours.tour_mode == ""SHARED2FREE"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.098670942,FALSE,-5,5,1,log_ratio,0.01,2 +SHARED2PAY share for auto sufficient HHs,coef_calib_SHARED2PAY_auto_suff,"len(tours[(tours.tour_mode == ""SHARED2PAY"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.043195465,FALSE,-5,5,1,log_ratio,0.01,2 +SHARED3FREE share for auto sufficient HHs,coef_calib_SHARED3FREE_auto_suff,"len(tours[(tours.tour_mode == ""SHARED3FREE"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.054018294,FALSE,-5,5,1,log_ratio,0.01,2 +SHARED3PAY share for auto sufficient HHs,coef_calib_SHARED3PAY_auto_suff,"len(tours[(tours.tour_mode == ""SHARED3PAY"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.027328167,FALSE,-5,5,1,log_ratio,0.01,2 +WALK share for auto sufficient HHs,coef_calib_WALK_auto_suff,"len(tours[(tours.tour_mode == ""WALK"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.059569409,FALSE,-5,5,1,log_ratio,0.01,2 +BIKE share for auto sufficient HHs,coef_calib_BIKE_auto_suff,"len(tours[(tours.tour_mode == ""BIKE"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.064270507,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_LOC share for auto sufficient HHs,coef_calib_WALK_LOC_auto_suff,"len(tours[(tours.tour_mode == ""WALK_LOC"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.031069537,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_LRF share for auto sufficient HHs,coef_calib_WALK_LRF_auto_suff,"len(tours[(tours.tour_mode == ""WALK_LRF"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.055759405,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_EXP share for auto sufficient HHs,coef_calib_WALK_EXP_auto_suff,"len(tours[(tours.tour_mode == ""WALK_EXP"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.012576768,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_HVY share for auto sufficient HHs,coef_calib_WALK_HVY_auto_suff,"len(tours[(tours.tour_mode == ""WALK_HVY"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.004320335,FALSE,-5,5,1,log_ratio,0.01,2 +WALK_COM share for auto sufficient HHs,coef_calib_WALK_COM_auto_suff,"len(tours[(tours.tour_mode == ""WALK_COM"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.055175324,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_LOC share for auto sufficient HHs,coef_calib_DRIVE_LOC_auto_suff,"len(tours[(tours.tour_mode == ""DRIVE_LOC"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.086416787,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_LRF share for auto sufficient HHs,coef_calib_DRIVE_LRF_auto_suff,"len(tours[(tours.tour_mode == ""DRIVE_LRF"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.067921667,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_EXP share for auto sufficient HHs,coef_calib_DRIVE_EXP_auto_suff,"len(tours[(tours.tour_mode == ""DRIVE_EXP"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.044918204,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_HVY share for auto sufficient HHs,coef_calib_DRIVE_HVY_auto_suff,"len(tours[(tours.tour_mode == ""DRIVE_HVY"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.033260131,FALSE,-5,5,1,log_ratio,0.01,2 +DRIVE_COM share for auto sufficient HHs,coef_calib_DRIVE_COM_auto_suff,"len(tours[(tours.tour_mode == ""DRIVE_COM"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.021919061,FALSE,-5,5,1,log_ratio,0.01,2 +TAXI share for auto sufficient HHs,coef_calib_TAXI_auto_suff,"len(tours[(tours.tour_mode == ""TAXI"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.076918756,FALSE,-5,5,1,log_ratio,0.01,2 +TNC_SINGLE share for auto sufficient HHs,coef_calib_TNC_SINGLE_auto_suff,"len(tours[(tours.tour_mode == ""TNC_SINGLE"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.07478863,FALSE,-5,5,1,log_ratio,0.01,2 +TNC_SHARED share for auto sufficient HHs,coef_calib_TNC_SHARED_auto_suff,"len(tours[(tours.tour_mode == ""TNC_SHARED"") & tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)]) / len(tours[tours.household_id.isin(households[(households.auto_ownership > 0) & (households.auto_ownership >= households.num_workers)].index)])",0.051523091,FALSE,-5,5,1,log_ratio,0.01,2 diff --git a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py new file mode 100644 index 0000000000..ed7555c427 --- /dev/null +++ b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py @@ -0,0 +1,89 @@ +import matplotlib.pyplot as plt +import pandas as pd +import os +from functools import lru_cache + +SURVEY_DATA_FOLDER = r"U:\projects\clients\ASIM\autocalibration\calibration_test_mtc\data" + +def compute_distances(origins, destinations): + # Compute distances between origins and destinations using the network level of service + # using non-time-dependent DIST skim + distances = skim_dict.lookup(origins, destinations, 'DIST') + # time dependent example + # distances = skim_dict.lookup_3d(origins, destinations, 'AM', 'SOV_DIST') + return distances + + +@lru_cache(maxsize=1) +def _survey_persons() -> pd.DataFrame: + """Load survey persons once per Python process.""" + return pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_persons.csv")) + + +@lru_cache(maxsize=1) +def _survey_households() -> pd.DataFrame: + """Load survey households once per Python process.""" + return pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_households.csv")) + + +@lru_cache(maxsize=1) +def _survey_worker_distances(): + """Compute survey worker distances once and reuse across calibration rows.""" + survey_persons = _survey_persons() + survey_workers = survey_persons[survey_persons["workplace_zone_id"] > 0] + survey_home_zone_ids = _survey_households().set_index("household_id")["home_zone_id"] + survey_home_zone_ids = survey_workers["household_id"].map(survey_home_zone_ids) + survey_workplace_zone_ids = survey_workers["workplace_zone_id"] + return compute_distances(survey_home_zone_ids, survey_workplace_zone_ids) + + +def summarize_model(min_dist=1, max_dist=2): + """Summarize the model results for workplaces within the specified distance range.""" + workers = persons[persons['workplace_zone_id'] > 0] + home_zone_ids = workers['home_zone_id'] + workplace_zone_ids = workers['workplace_zone_id'] + + distances = compute_distances(home_zone_ids, workplace_zone_ids) + + # Filter distances within the specified range + mask = (distances >= min_dist) & (distances < max_dist) + filtered_distances = distances[mask] + + share = len(filtered_distances) / len(distances) if len(distances) > 0 else 0 + return share + +def summarize_survey(min_dist=1, max_dist=2): + """Summarize the survey results for workplaces within the specified distance range.""" + + distances = _survey_worker_distances() + + # Filter distances within the specified range + mask = (distances >= min_dist) & (distances < max_dist) + filtered_distances = distances[mask] + + share = len(filtered_distances) / len(distances) if len(distances) > 0 else 0 + return share + +def report_workplace_location(): + """Workplace location distance frequency plot comparing model results with observed data.""" + print("summarizing workplace location model") + model_persons = persons + model_workers = model_persons[model_persons['workplace_zone_id'] > 0] + model_home_zone_ids = model_workers['home_zone_id'] + model_workplace_zone_ids = model_workers['workplace_zone_id'] + + model_distances = compute_distances(model_home_zone_ids, model_workplace_zone_ids) + + # survey_distances = _survey_worker_distances() + survey_distances = model_distances + + # Here you can add code to compare model_distances and survey_distances, + # for example by plotting histograms or computing summary statistics. + plt.hist(model_distances, bins=20, density=True, alpha=0.5, label='Model') + plt.hist(survey_distances, bins=20, density=True, alpha=0.5, label='Survey') + plt.xlabel('Distance') + plt.ylabel('Frequency') + plt.legend() + # component_output_dir set in the evaluation context + plt.savefig(os.path.join(component_output_dir, 'workplace_location_comparison.png')) + plt.close() \ No newline at end of file diff --git a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calibration.csv b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calibration.csv new file mode 100644 index 0000000000..4a7193acd2 --- /dev/null +++ b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calibration.csv @@ -0,0 +1,7 @@ +description,coefficient,model_value,target_value,hold_fast,min,max,damping,method,tolerance +#Distance 1 to 2 mi share,coef_calib_dist_1_2,"summarize_model(min_dist=.1, max_dist=.2)","summarize_survey(min_dist=.1, max_dist=.2)",FALSE,-5,5,1,log_ratio,0.02 +#Distance 5 to 15 mi share,coef_calib_dist_5_15,"summarize_model(min_dist=.15, max_dist=.5)","summarize_survey(min_dist=.15, max_dist=.5)",FALSE,-5,5,1,log_ratio,0.01 +#Distance 15+ mi share,coef_calib_dist_15_up,"summarize_model(min_dist=.5, max_dist=999)","summarize_survey(min_dist=.5, max_dist=999)",FALSE,-5,5,1,log_ratio,0.01 +Distance 1 to 2 mi share,coef_calib_dist_1_2,"summarize_model(min_dist=.1, max_dist=.2)",0.1,FALSE,-5,5,1,log_ratio,0.02 +Distance 5 to 15 mi share,coef_calib_dist_5_15,"summarize_model(min_dist=.15, max_dist=.5)",0.65,FALSE,-5,5,1,log_ratio,0.01 +Distance 15+ mi share,coef_calib_dist_15_up,"summarize_model(min_dist=.5, max_dist=999)",0.15,FALSE,-5,5,1,log_ratio,0.01 From 1bf010c54926b780739ee3be84060aa909542ba3 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:03:56 -0700 Subject: [PATCH 15/96] passing context dict --- activitysim/core/calibration.py | 36 ++------------- .../prototype_mtc/configs/auto_ownership.csv | 4 ++ .../auto_ownership_calib_helper.py | 22 +++------- .../tour_mode_choice_calib_helper.py | 21 +++------ .../workplace_location_calib_helper.py | 44 +++++++++++-------- 5 files changed, 46 insertions(+), 81 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index e9dd40cc31..e60e05dc0e 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -401,7 +401,6 @@ def _calibrate_component( state.run(models=[run_model_name], resume_after=prior_step) eval_context = _build_expression_context(state, helper_symbols, component_name, component_settings) - _bind_context_to_helper_module_globals(helper_module, eval_context) ( row_records, @@ -427,9 +426,7 @@ def _calibrate_component( _write_generic_report(state, component_name, row_records) if bespoke_callable is not None: - # Helper module globals were already updated from eval_context, - # so bespoke functions can access tables/injectables directly. - bespoke_callable() + bespoke_callable(eval_context) if component_converged: break @@ -641,38 +638,11 @@ def _build_expression_context( pass context.update(helper_symbols) + # Explicit function-call context used by calibration expressions. + context["context"] = context return context -def _bind_context_to_helper_module_globals( - helper_module: Any | None, - eval_context: dict[str, Any], -) -> None: - """Bind eval context names into helper module globals. - - Helper functions referenced by calibration expressions resolve free names - from their module globals, not from eval locals. This syncs the current - evaluation context into the helper module globals each component iteration. - """ - if helper_module is None: - return - - excluded = { - "__builtins__", - "__name__", - "__package__", - "__loader__", - "__spec__", - "__file__", - "__cached__", - } - - filtered_context = { - key: value for key, value in eval_context.items() if key not in excluded - } - helper_module.__dict__.update(filtered_context) - - def _evaluate_and_update( component_name: str, calibration_spec_df: pd.DataFrame, diff --git a/activitysim/examples/prototype_mtc/configs/auto_ownership.csv b/activitysim/examples/prototype_mtc/configs/auto_ownership.csv index aa30bdb4db..9f00def376 100644 --- a/activitysim/examples/prototype_mtc/configs/auto_ownership.csv +++ b/activitysim/examples/prototype_mtc/configs/auto_ownership.csv @@ -28,3 +28,7 @@ util_retail_transit_workers,"Retail accessibility (0.66*PK + 0.34*OP) by transit util_retail_non_motor_no_workers,"Retail accessibility by non-motorized, if 0 workers",(num_workers==0)*nmRetail,,coef_retail_non_motor,coef_retail_non_motor,coef_retail_non_motor,coef_retail_non_motor util_retail_non_motor_workers,"Retail accessibility by non-motorized, if 1+ workers",(num_workers>0)*nmRetail,,coef_retail_non_motor,coef_retail_non_motor,coef_retail_non_motor,coef_retail_non_motor util_auto_time_saving_per_worker,Auto time savings per worker to work,"@np.where(df.num_workers > 0, df.hh_work_auto_savings_ratio / df.num_workers, 0)",,coef_cars1_auto_time_saving_per_worker,coef_cars2_auto_time_saving_per_worker,coef_cars3_auto_time_saving_per_worker,coef_cars4_auto_time_saving_per_worker +util_calib_auto_0,Calibration utility for 0 autos,"@df.auto_ownership == 0",,coef_calib_auto_0 +util_calib_auto_2,Calibration utility for 2 autos,"@df.auto_ownership == 2",,coef_calib_auto_2 +util_calib_auto_3,Calibration utility for 3 autos,"@df.auto_ownership == 3",,coef_calib_auto_3 +util_calib_auto_4,Calibration utility for 4 autos,"@df.auto_ownership == 4",,coef_calib_auto_4 \ No newline at end of file diff --git a/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py index 516f5cd85f..ee6f319ce7 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py @@ -1,36 +1,26 @@ +import matplotlib.pyplot as plt import pandas as pd import os -SURVEY_DATA_FOLDER = r"U:\projects\clients\ASIM\autocalibration\calibration_test_mtc\data" +SURVEY_DATA_FOLDER = r"C:\Users\david.hensle\OneDrive - Resource Systems Group, Inc\Documents\projects\activitysim\rsg_activitysim\activitysim\examples\example_estimation\data_test\survey_data" -def report_auto_ownership(): - model_hhs = households - survey_hhs = None - try: - survey_hhs = pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_households.csv")) - except FileNotFoundError: - raise FileNotFoundError(f"No survey file override_households.csv found in {SURVEY_DATA_FOLDER}!") +def report_auto_ownership(context): + model_hhs = context["households"] + survey_hhs = pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_households.csv")) model_summary = model_hhs.auto_ownership.value_counts(normalize=True).sort_index().fillna(0) - # survey_summary = survey_hhs.groupby("auto_ownership").household_weight.sum() - # survey_summary = survey_summary / survey_hhs.household_weight.sum() survey_summary = survey_hhs.auto_ownership.value_counts(normalize=True).sort_index().fillna(0) - summary_df = ( pd.DataFrame({"model": model_summary, "survey": survey_summary}) .reset_index() .rename(columns={"index": "num_autos"}) ) - print(summary_df) - # plot comparing model and survey distributions - import matplotlib.pyplot as plt - summary_df.plot(x="auto_ownership", y=["model", "survey"], kind="bar") plt.title("Auto Ownership Distribution: Model vs Survey") plt.xlabel("Number of Autos") plt.ylabel("Proportion of Households") plt.legend(title="Data Source") - plt.savefig(os.path.join(component_output_dir, "auto_ownership_comparison.png")) + plt.savefig(os.path.join(context["component_output_dir"], "auto_ownership_comparison.png")) plt.close() \ No newline at end of file diff --git a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py index a87708a795..bbec248ca7 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py @@ -1,16 +1,13 @@ import pandas as pd +import matplotlib.pyplot as plt import os -def report_tour_mode_choice(): - model_tours = state.get_table("tours") +SURVEY_DATA_FOLDER = r"C:\Users\david.hensle\OneDrive - Resource Systems Group, Inc\Documents\projects\activitysim\rsg_activitysim\activitysim\examples\example_estimation\data_test\survey_data" + +def report_tour_mode_choice(context): + model_tours = context["tours"] survey_tours = None - for data_dir in state.filesystem.data_dir: - try: - survey_tours = pd.read_csv(os.path.join(data_dir, component_settings.survey_file)) - break - except FileNotFoundError: - pass - assert survey_tours is not None, f"No survey file {component_settings.survey_file} found in data dirs!" + survey_tours = pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_tours.csv")) model_summary = model_tours.tour_mode.value_counts(normalize=True).sort_index().fillna(0) survey_summary = survey_tours.groupby("tour_mode").tour_weight.sum() @@ -22,15 +19,11 @@ def report_tour_mode_choice(): .rename(columns={"index": "tour_mode"}) ) - print(summary_df) - # plot comparing model and survey distributions - import matplotlib.pyplot as plt - summary_df.plot(x="tour_mode", y=["model", "survey"], kind="bar") plt.title("Tour Mode Choice Distribution: Model vs Survey") plt.xlabel("Tour Mode") plt.ylabel("Proportion of Tours") plt.legend(title="Data Source") - plt.savefig("tour_mode_choice_comparison.png") + plt.savefig(os.path.join(context["component_output_dir"], "tour_mode_choice_comparison.png")) plt.close() \ No newline at end of file diff --git a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py index ed7555c427..3c2496667f 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py @@ -1,49 +1,58 @@ +""" +Workplace location calibration helper functions. + +Notes: + The lru_cache decorator to ensure that survey data is loaded and computed only once per Python process. + Context is developed in _build_expression_context() in activitysim.core.calibration. +""" + import matplotlib.pyplot as plt import pandas as pd import os from functools import lru_cache -SURVEY_DATA_FOLDER = r"U:\projects\clients\ASIM\autocalibration\calibration_test_mtc\data" +SURVEY_DATA_FOLDER = r"C:\Users\david.hensle\OneDrive - Resource Systems Group, Inc\Documents\projects\activitysim\rsg_activitysim\activitysim\examples\example_estimation\data_test\survey_data" -def compute_distances(origins, destinations): +def compute_distances(context, origins, destinations): # Compute distances between origins and destinations using the network level of service # using non-time-dependent DIST skim - distances = skim_dict.lookup(origins, destinations, 'DIST') + distances = context["skim_dict"].lookup(origins.clip(upper=24), destinations.clip(upper=24), 'DIST') # time dependent example # distances = skim_dict.lookup_3d(origins, destinations, 'AM', 'SOV_DIST') return distances -@lru_cache(maxsize=1) +# @lru_cache(maxsize=1) def _survey_persons() -> pd.DataFrame: """Load survey persons once per Python process.""" return pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_persons.csv")) -@lru_cache(maxsize=1) +# @lru_cache(maxsize=1) def _survey_households() -> pd.DataFrame: """Load survey households once per Python process.""" return pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_households.csv")) -@lru_cache(maxsize=1) -def _survey_worker_distances(): +# @lru_cache(maxsize=1) +def _survey_worker_distances(context): """Compute survey worker distances once and reuse across calibration rows.""" survey_persons = _survey_persons() survey_workers = survey_persons[survey_persons["workplace_zone_id"] > 0] survey_home_zone_ids = _survey_households().set_index("household_id")["home_zone_id"] survey_home_zone_ids = survey_workers["household_id"].map(survey_home_zone_ids) survey_workplace_zone_ids = survey_workers["workplace_zone_id"] - return compute_distances(survey_home_zone_ids, survey_workplace_zone_ids) + return compute_distances(context, survey_home_zone_ids, survey_workplace_zone_ids) -def summarize_model(min_dist=1, max_dist=2): +def summarize_model(context, min_dist=1, max_dist=2): """Summarize the model results for workplaces within the specified distance range.""" + persons = context['persons'] workers = persons[persons['workplace_zone_id'] > 0] home_zone_ids = workers['home_zone_id'] workplace_zone_ids = workers['workplace_zone_id'] - distances = compute_distances(home_zone_ids, workplace_zone_ids) + distances = compute_distances(context, home_zone_ids, workplace_zone_ids) # Filter distances within the specified range mask = (distances >= min_dist) & (distances < max_dist) @@ -52,10 +61,10 @@ def summarize_model(min_dist=1, max_dist=2): share = len(filtered_distances) / len(distances) if len(distances) > 0 else 0 return share -def summarize_survey(min_dist=1, max_dist=2): +def summarize_survey(context, min_dist=1, max_dist=2): """Summarize the survey results for workplaces within the specified distance range.""" - distances = _survey_worker_distances() + distances = _survey_worker_distances(context) # Filter distances within the specified range mask = (distances >= min_dist) & (distances < max_dist) @@ -64,18 +73,17 @@ def summarize_survey(min_dist=1, max_dist=2): share = len(filtered_distances) / len(distances) if len(distances) > 0 else 0 return share -def report_workplace_location(): +def report_workplace_location(context): """Workplace location distance frequency plot comparing model results with observed data.""" print("summarizing workplace location model") - model_persons = persons + model_persons = context["persons"] model_workers = model_persons[model_persons['workplace_zone_id'] > 0] model_home_zone_ids = model_workers['home_zone_id'] model_workplace_zone_ids = model_workers['workplace_zone_id'] - model_distances = compute_distances(model_home_zone_ids, model_workplace_zone_ids) + model_distances = compute_distances(context, model_home_zone_ids, model_workplace_zone_ids) - # survey_distances = _survey_worker_distances() - survey_distances = model_distances + survey_distances = _survey_worker_distances(context) # Here you can add code to compare model_distances and survey_distances, # for example by plotting histograms or computing summary statistics. @@ -85,5 +93,5 @@ def report_workplace_location(): plt.ylabel('Frequency') plt.legend() # component_output_dir set in the evaluation context - plt.savefig(os.path.join(component_output_dir, 'workplace_location_comparison.png')) + plt.savefig(os.path.join(context["component_output_dir"], 'workplace_location_comparison.png')) plt.close() \ No newline at end of file From dc731d0df8bf2d1e4cfd2b9bfd7b4acf7c4837fc Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:04:11 -0700 Subject: [PATCH 16/96] updated specs for calibration --- .../configs/auto_ownership_coefficients.csv | 20 +++++++++++-------- .../configs/workplace_location.csv | 6 +++++- .../workplace_location_coefficients.csv | 4 ++++ .../workplace_location_calibration.csv | 12 +++++------ 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv b/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv index b9d7fd07b0..4ade078423 100644 --- a/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv +++ b/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv @@ -1,12 +1,12 @@ coefficient_name,value,constrain -coef_cars1_drivers_2,0,T -coef_cars1_drivers_3,0,T -coef_cars1_persons_16_17,0,T -coef_cars234_asc_marin,0,T -coef_cars1_persons_25_34,0,T -coef_cars1_num_workers_clip_3,0,T -coef_cars1_hh_income_30_up,0,T -coef_cars1_density_0_10_no_workers,0,T +coef_cars1_drivers_2,0.0,T +coef_cars1_drivers_3,0.0,T +coef_cars1_persons_16_17,0.0,T +coef_cars234_asc_marin,0.0,T +coef_cars1_persons_25_34,0.0,T +coef_cars1_num_workers_clip_3,0.0,T +coef_cars1_hh_income_30_up,0.0,T +coef_cars1_density_0_10_no_workers,0.0,T coef_cars1_density_10_up_workers,-0.0152,F coef_retail_non_motor,-0.03,T coef_cars4_asc,-5.313,F @@ -66,3 +66,7 @@ coef_cars3_drivers_3,5.5131,F coef_cars2_drivers_4_up,6.3662,F coef_cars3_drivers_4_up,8.5148,F coef_cars4_drivers_4_up,9.5807,F +coef_calib_auto_0,-5.0,T +coef_calib_auto_2,5.0,T +coef_calib_auto_3,5.0,T +coef_calib_auto_4,5.0,T diff --git a/activitysim/examples/prototype_mtc/configs/workplace_location.csv b/activitysim/examples/prototype_mtc/configs/workplace_location.csv index 3ac8b59e96..67ca4a22b8 100644 --- a/activitysim/examples/prototype_mtc/configs/workplace_location.csv +++ b/activitysim/examples/prototype_mtc/configs/workplace_location.csv @@ -11,4 +11,8 @@ util_size_variable,Size variable,@(df['size_term'] * df['shadow_price_size_term_ util_utility_adjustment,utility adjustment,@df['shadow_price_utility_adjustment'],1 util_no_attractions,No attractions,@df['size_term']==0,-999 util_mode_logsum,Mode choice logsum,mode_choice_logsum,coef_mode_logsum -util_sample_of_corrections_factor,Sample of alternatives correction factor,"@np.minimum(np.log(df.pick_count/df.prob), 60)",1 \ No newline at end of file +util_sample_of_corrections_factor,Sample of alternatives correction factor,"@np.minimum(np.log(df.pick_count/df.prob), 60)",1 +util_calib_dist_0_2,Calibration utility 0 to 2 mi,"@(_DIST >= 0) & (_DIST < 2)",coef_calib_dist_0_2 +util_calib_dist_2_5,Calibration utility 2 to 5 mi,"@(_DIST >= 2) & (_DIST < 5)",coef_calib_dist_2_5 +util_calib_dist_5_15,Calibration utility 5 to 15 mi,"@(_DIST >= 5) & (_DIST < 15)",coef_calib_dist_5_15 +util_calib_dist_15_up,Calibration utility 15+ mi,"@(_DIST >= 15)",coef_calib_dist_15_up diff --git a/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv b/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv index b3ac834103..7b1b52e4ac 100644 --- a/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv +++ b/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv @@ -7,3 +7,7 @@ coef_dist_15_up,-0.0917,F coef_dist_0_5_high,0.15,F coef_dist_5_up_high,0.02,F coef_mode_logsum,0.3,F +coef_calib_dist_0_2,-5.0,T +coef_calib_dist_2_5,0.0,T +coef_calib_dist_5_15,-2.771779906602003,T +coef_calib_dist_15_up,0.749738309309575,T diff --git a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calibration.csv b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calibration.csv index 4a7193acd2..c12ada8045 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calibration.csv +++ b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calibration.csv @@ -1,7 +1,7 @@ description,coefficient,model_value,target_value,hold_fast,min,max,damping,method,tolerance -#Distance 1 to 2 mi share,coef_calib_dist_1_2,"summarize_model(min_dist=.1, max_dist=.2)","summarize_survey(min_dist=.1, max_dist=.2)",FALSE,-5,5,1,log_ratio,0.02 -#Distance 5 to 15 mi share,coef_calib_dist_5_15,"summarize_model(min_dist=.15, max_dist=.5)","summarize_survey(min_dist=.15, max_dist=.5)",FALSE,-5,5,1,log_ratio,0.01 -#Distance 15+ mi share,coef_calib_dist_15_up,"summarize_model(min_dist=.5, max_dist=999)","summarize_survey(min_dist=.5, max_dist=999)",FALSE,-5,5,1,log_ratio,0.01 -Distance 1 to 2 mi share,coef_calib_dist_1_2,"summarize_model(min_dist=.1, max_dist=.2)",0.1,FALSE,-5,5,1,log_ratio,0.02 -Distance 5 to 15 mi share,coef_calib_dist_5_15,"summarize_model(min_dist=.15, max_dist=.5)",0.65,FALSE,-5,5,1,log_ratio,0.01 -Distance 15+ mi share,coef_calib_dist_15_up,"summarize_model(min_dist=.5, max_dist=999)",0.15,FALSE,-5,5,1,log_ratio,0.01 +#Distance 1 to 2 mi share,coef_calib_dist_1_2,"summarize_model(context, min_dist=.1, max_dist=.2)","summarize_survey(context, min_dist=.1, max_dist=.2)",FALSE,-5,5,1,log_ratio,0.02 +#Distance 5 to 15 mi share,coef_calib_dist_5_15,"summarize_model(context, min_dist=.15, max_dist=.5)","summarize_survey(context, min_dist=.15, max_dist=.5)",FALSE,-5,5,1,log_ratio,0.01 +#Distance 15+ mi share,coef_calib_dist_15_up,"summarize_model(context, min_dist=.5, max_dist=999)","summarize_survey(context, min_dist=.5, max_dist=999)",FALSE,-5,5,1,log_ratio,0.01 +Distance 0 to 2 mi share,coef_calib_dist_0_2,"summarize_model(context, min_dist=.0, max_dist=.5)",0.05,FALSE,-5,5,1,log_ratio,0.02 +Distance 5 to 15 mi share,coef_calib_dist_5_15,"summarize_model(context, min_dist=.5, max_dist=1)",0.25,FALSE,-5,5,1,log_ratio,0.01 +Distance 15+ mi share,coef_calib_dist_15_up,"summarize_model(context, min_dist=2, max_dist=999)",0.1,FALSE,-5,5,1,log_ratio,0.01 From 034545cae892b603572f28b869187ce5ca88f5be Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 7 Jul 2026 13:09:38 -0400 Subject: [PATCH 17/96] Add tour mode choice calibration coefficients --- .../configs/tour_mode_choice.csv | 3 + .../configs/tour_mode_choice_coefficients.csv | 104 ++++++++++++++---- ...tour_mode_choice_coefficients_template.csv | 60 ++++++++++ 3 files changed, 145 insertions(+), 22 deletions(-) diff --git a/activitysim/examples/prototype_mtc/configs/tour_mode_choice.csv b/activitysim/examples/prototype_mtc/configs/tour_mode_choice.csv index 5bd4898e03..80a900d7b0 100644 --- a/activitysim/examples/prototype_mtc/configs/tour_mode_choice.csv +++ b/activitysim/examples/prototype_mtc/configs/tour_mode_choice.csv @@ -342,3 +342,6 @@ util_Walk_not_available_for_long_distances,Walk not available for long distances util_Bike_not_available_for_long_distances,Bike not available for long distances,@od_skims.max('DISTBIKE') > 8,,,,,,,,-999,,,,,,,,,,,,, util_Drive_alone_not_available_for_escort_tours,Drive alone not available for escort tours,is_escort,-999,-999,,,,,,,,,,,,,,,,,,, #, max(c_densityIndexOrigin*originDensityIndex,originDensityIndexMax),,,,,,,,,1,1,1,1,1,1,1,,,,,, +util_calib_zero_autos,Calibration utility for zero-auto households,1,,coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3PAY_zero_auto,coef_calib_WALK_zero_auto,coef_calib_BIKE_zero_auto,coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_COM_zero_auto,coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_COM_zero_auto,coef_calib_TAXI_zero_auto,coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SHARED_zero_auto +util_calib_auto_insufficient,Calibration utility for auto-insufficient households,1,,coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3PAY_auto_insuff,coef_calib_WALK_auto_insuff,coef_calib_BIKE_auto_insuff,coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_COM_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_COM_auto_insuff,coef_calib_TAXI_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SHARED_auto_insuff +util_calib_auto_sufficient,Calibration utility for auto-sufficient hou8seholds,1,,coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3PAY_auto_suff,coef_calib_WALK_auto_suff,coef_calib_BIKE_auto_suff,coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_COM_auto_suff,coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_COM_auto_suff,coef_calib_TAXI_auto_suff,coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SHARED_auto_suff diff --git a/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv b/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv index 9693953808..a6d5c07487 100644 --- a/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv +++ b/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv @@ -1,6 +1,6 @@ coefficient_name,value,constrain -coef_one,1,T -coef_nest_root,1.00,T +coef_one,1.0,T +coef_nest_root,1.0,T coef_nest_AUTO,0.72,T coef_nest_AUTO_DRIVEALONE,0.35,T coef_nest_AUTO_SHAREDRIDE2,0.35,T @@ -14,23 +14,23 @@ coef_ivt_eatout_escort_othdiscr_othmaint_shopping_social,-0.0175,F coef_ivt_school_univ,-0.0224,F coef_ivt_work,-0.0134,F coef_ivt_atwork,-0.0188,F -coef_topology_walk_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,15,F +coef_topology_walk_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,15.0,F coef_topology_walk_multiplier_atwork,7.5,F -coef_topology_bike_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,20,F -coef_topology_bike_multiplier_atwork,10,F +coef_topology_bike_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,20.0,F +coef_topology_bike_multiplier_atwork,10.0,F coef_topology_trn_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,2.2,F -coef_topology_trn_multiplier_atwork,2,F -coef_age1619_da_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work,0,F +coef_topology_trn_multiplier_atwork,2.0,F +coef_age1619_da_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work,0.0,F coef_age1619_da_multiplier_school_univ,-1.3813,F coef_age1619_da_multiplier_atwork,0.0032336,F -coef_age010_trn_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work,0,F +coef_age010_trn_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work,0.0,F coef_age010_trn_multiplier_school_univ,-1.5548,F coef_age010_trn_multiplier_atwork,0.000722,F coef_age16p_sr_multiplier_eatout_escort_othdiscr_othmaint_shopping_social,-1.366,F -coef_age16p_sr_multiplier_school_univ_work_atwork,0,F -coef_hhsize1_sr_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_atwork,0,F +coef_age16p_sr_multiplier_school_univ_work_atwork,0.0,F +coef_hhsize1_sr_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_atwork,0.0,F coef_hhsize1_sr_multiplier_work,-0.734588,F -coef_hhsize2_sr_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work_atwork,0,F +coef_hhsize2_sr_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work_atwork,0.0,F coef_hhsize2_sr_multiplier_school_univ,-0.6359,F walk_ASC_no_auto_eatout,5.1251173,F walk_ASC_no_auto_escort,2.8012068,F @@ -92,9 +92,9 @@ bike_ASC_auto_sufficient_social,-1.368071,F bike_ASC_auto_sufficient_univ,-1.9397832,F bike_ASC_auto_sufficient_work,-1.5800232,F bike_ASC_auto_sufficient_atwork,15.72017,F -sr2_ASC_no_auto_all,0,F +sr2_ASC_no_auto_all,0.0,F sr2_ASC_auto_deficient_eatout,0.5882345,F -sr2_ASC_auto_deficient_escort,0,F +sr2_ASC_auto_deficient_escort,0.0,F sr2_ASC_auto_deficient_othdiscr,0.6601513,F sr2_ASC_auto_deficient_othmaint,0.2621527,F sr2_ASC_auto_deficient_school,0.12474365,F @@ -104,7 +104,7 @@ sr2_ASC_auto_deficient_univ,-1.6922346,F sr2_ASC_auto_deficient_work,-0.33803123,F sr2_ASC_auto_deficient_atwork,-2.1102421,F sr2_ASC_auto_sufficient_eatout,0.86280555,F -sr2_ASC_auto_sufficient_escort,0,F +sr2_ASC_auto_sufficient_escort,0.0,F sr2_ASC_auto_sufficient_othdiscr,0.49684617,F sr2_ASC_auto_sufficient_othmaint,0.25817883,F sr2_ASC_auto_sufficient_school,-1.6062657,F @@ -173,7 +173,7 @@ walk_transit_ASC_auto_sufficient_social,-0.3453759,F walk_transit_ASC_auto_sufficient_univ,0.4731163,F walk_transit_ASC_auto_sufficient_work,-0.8916507,F walk_transit_ASC_auto_sufficient_atwork,-3.401027,F -drive_transit_ASC_no_auto_all,0,F +drive_transit_ASC_no_auto_all,0.0,F drive_transit_ASC_auto_deficient_eatout,0.5998061,F drive_transit_ASC_auto_deficient_escort,-1.1537067,F drive_transit_ASC_auto_deficient_othdiscr,0.3199308,F @@ -196,7 +196,7 @@ drive_transit_ASC_auto_sufficient_work,-1.0045459,F drive_transit_ASC_auto_sufficient_atwork,-999.21466,F taxi_ASC_no_auto_eatout_othdiscr_social,0.9923,F taxi_ASC_no_auto_escort_othmaint_shopping,1.8939,F -taxi_ASC_no_auto_school_univ,-7,T +taxi_ASC_no_auto_school_univ,-7.0,T taxi_ASC_no_auto_work,4.7291,F taxi_ASC_no_auto_atwork,4.1021,F taxi_ASC_auto_deficient_eatout_othdiscr_social,-3.1317,F @@ -213,7 +213,7 @@ taxi_ASC_auto_sufficient_work,-4.8509,F taxi_ASC_auto_sufficient_atwork,-2.8804,F tnc_single_ASC_no_auto_eatout_othdiscr_social,1.6852,F tnc_single_ASC_no_auto_escort_othmaint_shopping,1.8605,F -tnc_single_ASC_no_auto_school,-7,T +tnc_single_ASC_no_auto_school,-7.0,T tnc_single_ASC_no_auto_univ,-2.519,F tnc_single_ASC_no_auto_work,5.7855,F tnc_single_ASC_no_auto_atwork,4.4982,F @@ -231,7 +231,7 @@ tnc_single_ASC_auto_sufficient_work,-4.1946,F tnc_single_ASC_auto_sufficient_atwork,-2.7988,F tnc_shared_ASC_no_auto_eatout_othdiscr_social,0.6464,F tnc_shared_ASC_no_auto_escort_othmaint_shopping,0.9361,F -tnc_shared_ASC_no_auto_school,-7,T +tnc_shared_ASC_no_auto_school,-7.0,T tnc_shared_ASC_no_auto_univ,-5.8116,F tnc_shared_ASC_no_auto_work,3.2429,F tnc_shared_ASC_no_auto_atwork,3.3672,F @@ -253,16 +253,16 @@ joint_walk_ASC_auto_sufficient_all,-3.2352157,F joint_bike_ASC_no_auto_all,-2.8671598,F joint_bike_ASC_auto_deficient_all,-6.076415,F joint_bike_ASC_auto_sufficient_all,-6.3760657,F -joint_sr2_ASC_no_auto_all,0,T -joint_sr2_ASC_auto_deficient_all,0,T -joint_sr2_ASC_auto_sufficient_all,0,T +joint_sr2_ASC_no_auto_all,0.0,T +joint_sr2_ASC_auto_deficient_all,0.0,T +joint_sr2_ASC_auto_sufficient_all,0.0,T joint_sr3p_ASC_no_auto_all,0.5630671,F joint_sr3p_ASC_auto_deficient_all,-1.8841692,F joint_sr3p_ASC_auto_sufficient_all,-2.234826,F joint_walk_transit_ASC_no_auto_all,0.62292415,F joint_walk_transit_ASC_auto_deficient_all,-5.1634483,F joint_walk_transit_ASC_auto_sufficient_all,-18.264534,F -joint_drive_transit_ASC_no_auto_all,0,T +joint_drive_transit_ASC_no_auto_all,0.0,T joint_drive_transit_ASC_auto_deficient_all,-5.9632215,F joint_drive_transit_ASC_auto_sufficient_all,-8.045285,F joint_taxi_ASC_no_auto_all,-4.5792,F @@ -306,3 +306,63 @@ drive_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,0.525,F drive_transit_CBD_ASC_school_univ,0.672,F drive_transit_CBD_ASC_work,1.1,F drive_transit_CBD_ASC_atwork,0.564,F +coef_calib_DRIVEALONEPAY_zero_auto,5.0,T +coef_calib_SHARED2FREE_zero_auto,5.0,T +coef_calib_SHARED2PAY_zero_auto,5.0,T +coef_calib_SHARED3FREE_zero_auto,5.0,T +coef_calib_SHARED3PAY_zero_auto,5.0,T +coef_calib_WALK_zero_auto,1.19369549873279,T +coef_calib_BIKE_zero_auto,5.0,T +coef_calib_WALK_LOC_zero_auto,0.29206606912731364,T +coef_calib_WALK_LRF_zero_auto,1.2520906218382721,T +coef_calib_WALK_EXP_zero_auto,5.0,T +coef_calib_WALK_HVY_zero_auto,2.427455130545431,T +coef_calib_WALK_COM_zero_auto,5.0,T +coef_calib_DRIVE_LOC_zero_auto,5.0,T +coef_calib_DRIVE_LRF_zero_auto,5.0,T +coef_calib_DRIVE_EXP_zero_auto,5.0,T +coef_calib_DRIVE_HVY_zero_auto,5.0,T +coef_calib_DRIVE_COM_zero_auto,5.0,T +coef_calib_TAXI_zero_auto,5.0,T +coef_calib_TNC_SINGLE_zero_auto,3.2347512774613625,T +coef_calib_TNC_SHARED_zero_auto,5.0,T +coef_calib_DRIVEALONEPAY_auto_insuff,5.0,T +coef_calib_SHARED2FREE_auto_insuff,4.655634851986755,T +coef_calib_SHARED2PAY_auto_insuff,5.0,T +coef_calib_SHARED3FREE_auto_insuff,5.0,T +coef_calib_SHARED3PAY_auto_insuff,5.0,T +coef_calib_WALK_auto_insuff,3.1931900192458924,T +coef_calib_BIKE_auto_insuff,4.992944512095712,T +coef_calib_WALK_LOC_auto_insuff,5.0,T +coef_calib_WALK_LRF_auto_insuff,3.9302088455457014,T +coef_calib_WALK_EXP_auto_insuff,5.0,T +coef_calib_WALK_HVY_auto_insuff,1.3844675036448821,T +coef_calib_WALK_COM_auto_insuff,5.0,T +coef_calib_DRIVE_LOC_auto_insuff,5.0,T +coef_calib_DRIVE_LRF_auto_insuff,5.0,T +coef_calib_DRIVE_EXP_auto_insuff,5.0,T +coef_calib_DRIVE_HVY_auto_insuff,5.0,T +coef_calib_DRIVE_COM_auto_insuff,5.0,T +coef_calib_TAXI_auto_insuff,5.0,T +coef_calib_TNC_SINGLE_auto_insuff,3.9140984082873835,T +coef_calib_TNC_SHARED_auto_insuff,5.0,T +coef_calib_DRIVEALONEPAY_auto_suff,5.0,T +coef_calib_SHARED2FREE_auto_suff,2.396741245612774,T +coef_calib_SHARED2PAY_auto_suff,5.0,T +coef_calib_SHARED3FREE_auto_suff,4.928377149112955,T +coef_calib_SHARED3PAY_auto_suff,5.0,T +coef_calib_WALK_auto_suff,3.077985217800374,T +coef_calib_BIKE_auto_suff,5.0,T +coef_calib_WALK_LOC_auto_suff,5.0,T +coef_calib_WALK_LRF_auto_suff,4.399523089022793,T +coef_calib_WALK_EXP_auto_suff,5.0,T +coef_calib_WALK_HVY_auto_suff,2.8217459359394326,T +coef_calib_WALK_COM_auto_suff,5.0,T +coef_calib_DRIVE_LOC_auto_suff,5.0,T +coef_calib_DRIVE_LRF_auto_suff,5.0,T +coef_calib_DRIVE_EXP_auto_suff,5.0,T +coef_calib_DRIVE_HVY_auto_suff,5.0,T +coef_calib_DRIVE_COM_auto_suff,5.0,T +coef_calib_TAXI_auto_suff,5.0,T +coef_calib_TNC_SINGLE_auto_suff,4.29485267200215,T +coef_calib_TNC_SHARED_auto_suff,5.0,T diff --git a/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients_template.csv b/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients_template.csv index b1b009a3f0..212e02d0ea 100644 --- a/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients_template.csv +++ b/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients_template.csv @@ -85,3 +85,63 @@ heavy_rail_ASC,heavy_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_at commuter_rail_ASC,commuter_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,commuter_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,commuter_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,commuter_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,commuter_rail_ASC_school_univ,commuter_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,commuter_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,commuter_rail_ASC_school_univ,commuter_rail_ASC_work,commuter_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork walk_transit_CBD_ASC,walk_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,walk_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,walk_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,walk_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,walk_transit_CBD_ASC_school_univ,walk_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,walk_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,walk_transit_CBD_ASC_school_univ,walk_transit_CBD_ASC_work,walk_transit_CBD_ASC_atwork drive_transit_CBD_ASC,drive_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,drive_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,drive_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,drive_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,drive_transit_CBD_ASC_school_univ,drive_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,drive_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,drive_transit_CBD_ASC_school_univ,drive_transit_CBD_ASC_work,drive_transit_CBD_ASC_atwork +coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_DRIVEALONEPAY_zero_auto,coef_calib_DRIVEALONEPAY_zero_auto +coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2FREE_zero_auto,coef_calib_SHARED2FREE_zero_auto +coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED2PAY_zero_auto,coef_calib_SHARED2PAY_zero_auto +coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3FREE_zero_auto,coef_calib_SHARED3FREE_zero_auto +coef_calib_SHARED3PAY_zero_auto,coef_calib_SHARED3PAY_zero_auto,coef_calib_SHARED3PAY_zero_auto,coef_calib_SHARED3PAY_zero_auto,coef_calib_SHARED3PAY_zero_auto,coef_calib_SHARED3PAY_zero_auto,coef_calib_SHARED3PAY_zero_auto,coef_calib_SHARED3PAY_zero_auto,coef_calib_SHARED3PAY_zero_auto,coef_calib_SHARED3PAY_zero_auto,coef_calib_SHARED3PAY_zero_auto +coef_calib_WALK_zero_auto,coef_calib_WALK_zero_auto,coef_calib_WALK_zero_auto,coef_calib_WALK_zero_auto,coef_calib_WALK_zero_auto,coef_calib_WALK_zero_auto,coef_calib_WALK_zero_auto,coef_calib_WALK_zero_auto,coef_calib_WALK_zero_auto,coef_calib_WALK_zero_auto,coef_calib_WALK_zero_auto +coef_calib_BIKE_zero_auto,coef_calib_BIKE_zero_auto,coef_calib_BIKE_zero_auto,coef_calib_BIKE_zero_auto,coef_calib_BIKE_zero_auto,coef_calib_BIKE_zero_auto,coef_calib_BIKE_zero_auto,coef_calib_BIKE_zero_auto,coef_calib_BIKE_zero_auto,coef_calib_BIKE_zero_auto,coef_calib_BIKE_zero_auto +coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LOC_zero_auto,coef_calib_WALK_LOC_zero_auto +coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_LRF_zero_auto,coef_calib_WALK_LRF_zero_auto +coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_EXP_zero_auto,coef_calib_WALK_EXP_zero_auto +coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_HVY_zero_auto,coef_calib_WALK_HVY_zero_auto +coef_calib_WALK_COM_zero_auto,coef_calib_WALK_COM_zero_auto,coef_calib_WALK_COM_zero_auto,coef_calib_WALK_COM_zero_auto,coef_calib_WALK_COM_zero_auto,coef_calib_WALK_COM_zero_auto,coef_calib_WALK_COM_zero_auto,coef_calib_WALK_COM_zero_auto,coef_calib_WALK_COM_zero_auto,coef_calib_WALK_COM_zero_auto,coef_calib_WALK_COM_zero_auto +coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LOC_zero_auto,coef_calib_DRIVE_LOC_zero_auto +coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_LRF_zero_auto,coef_calib_DRIVE_LRF_zero_auto +coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_EXP_zero_auto,coef_calib_DRIVE_EXP_zero_auto +coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_HVY_zero_auto,coef_calib_DRIVE_HVY_zero_auto +coef_calib_DRIVE_COM_zero_auto,coef_calib_DRIVE_COM_zero_auto,coef_calib_DRIVE_COM_zero_auto,coef_calib_DRIVE_COM_zero_auto,coef_calib_DRIVE_COM_zero_auto,coef_calib_DRIVE_COM_zero_auto,coef_calib_DRIVE_COM_zero_auto,coef_calib_DRIVE_COM_zero_auto,coef_calib_DRIVE_COM_zero_auto,coef_calib_DRIVE_COM_zero_auto,coef_calib_DRIVE_COM_zero_auto +coef_calib_TAXI_zero_auto,coef_calib_TAXI_zero_auto,coef_calib_TAXI_zero_auto,coef_calib_TAXI_zero_auto,coef_calib_TAXI_zero_auto,coef_calib_TAXI_zero_auto,coef_calib_TAXI_zero_auto,coef_calib_TAXI_zero_auto,coef_calib_TAXI_zero_auto,coef_calib_TAXI_zero_auto,coef_calib_TAXI_zero_auto +coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SINGLE_zero_auto,coef_calib_TNC_SINGLE_zero_auto +coef_calib_TNC_SHARED_zero_auto,coef_calib_TNC_SHARED_zero_auto,coef_calib_TNC_SHARED_zero_auto,coef_calib_TNC_SHARED_zero_auto,coef_calib_TNC_SHARED_zero_auto,coef_calib_TNC_SHARED_zero_auto,coef_calib_TNC_SHARED_zero_auto,coef_calib_TNC_SHARED_zero_auto,coef_calib_TNC_SHARED_zero_auto,coef_calib_TNC_SHARED_zero_auto,coef_calib_TNC_SHARED_zero_auto +coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_DRIVEALONEPAY_auto_insuff,coef_calib_DRIVEALONEPAY_auto_insuff +coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2FREE_auto_insuff,coef_calib_SHARED2FREE_auto_insuff +coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED2PAY_auto_insuff,coef_calib_SHARED2PAY_auto_insuff +coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3FREE_auto_insuff,coef_calib_SHARED3FREE_auto_insuff +coef_calib_SHARED3PAY_auto_insuff,coef_calib_SHARED3PAY_auto_insuff,coef_calib_SHARED3PAY_auto_insuff,coef_calib_SHARED3PAY_auto_insuff,coef_calib_SHARED3PAY_auto_insuff,coef_calib_SHARED3PAY_auto_insuff,coef_calib_SHARED3PAY_auto_insuff,coef_calib_SHARED3PAY_auto_insuff,coef_calib_SHARED3PAY_auto_insuff,coef_calib_SHARED3PAY_auto_insuff,coef_calib_SHARED3PAY_auto_insuff +coef_calib_WALK_auto_insuff,coef_calib_WALK_auto_insuff,coef_calib_WALK_auto_insuff,coef_calib_WALK_auto_insuff,coef_calib_WALK_auto_insuff,coef_calib_WALK_auto_insuff,coef_calib_WALK_auto_insuff,coef_calib_WALK_auto_insuff,coef_calib_WALK_auto_insuff,coef_calib_WALK_auto_insuff,coef_calib_WALK_auto_insuff +coef_calib_BIKE_auto_insuff,coef_calib_BIKE_auto_insuff,coef_calib_BIKE_auto_insuff,coef_calib_BIKE_auto_insuff,coef_calib_BIKE_auto_insuff,coef_calib_BIKE_auto_insuff,coef_calib_BIKE_auto_insuff,coef_calib_BIKE_auto_insuff,coef_calib_BIKE_auto_insuff,coef_calib_BIKE_auto_insuff,coef_calib_BIKE_auto_insuff +coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LOC_auto_insuff,coef_calib_WALK_LOC_auto_insuff +coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_LRF_auto_insuff,coef_calib_WALK_LRF_auto_insuff +coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_EXP_auto_insuff,coef_calib_WALK_EXP_auto_insuff +coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_HVY_auto_insuff,coef_calib_WALK_HVY_auto_insuff +coef_calib_WALK_COM_auto_insuff,coef_calib_WALK_COM_auto_insuff,coef_calib_WALK_COM_auto_insuff,coef_calib_WALK_COM_auto_insuff,coef_calib_WALK_COM_auto_insuff,coef_calib_WALK_COM_auto_insuff,coef_calib_WALK_COM_auto_insuff,coef_calib_WALK_COM_auto_insuff,coef_calib_WALK_COM_auto_insuff,coef_calib_WALK_COM_auto_insuff,coef_calib_WALK_COM_auto_insuff +coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff,coef_calib_DRIVE_LOC_auto_insuff +coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff,coef_calib_DRIVE_LRF_auto_insuff +coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff,coef_calib_DRIVE_EXP_auto_insuff +coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff,coef_calib_DRIVE_HVY_auto_insuff +coef_calib_DRIVE_COM_auto_insuff,coef_calib_DRIVE_COM_auto_insuff,coef_calib_DRIVE_COM_auto_insuff,coef_calib_DRIVE_COM_auto_insuff,coef_calib_DRIVE_COM_auto_insuff,coef_calib_DRIVE_COM_auto_insuff,coef_calib_DRIVE_COM_auto_insuff,coef_calib_DRIVE_COM_auto_insuff,coef_calib_DRIVE_COM_auto_insuff,coef_calib_DRIVE_COM_auto_insuff,coef_calib_DRIVE_COM_auto_insuff +coef_calib_TAXI_auto_insuff,coef_calib_TAXI_auto_insuff,coef_calib_TAXI_auto_insuff,coef_calib_TAXI_auto_insuff,coef_calib_TAXI_auto_insuff,coef_calib_TAXI_auto_insuff,coef_calib_TAXI_auto_insuff,coef_calib_TAXI_auto_insuff,coef_calib_TAXI_auto_insuff,coef_calib_TAXI_auto_insuff,coef_calib_TAXI_auto_insuff +coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff,coef_calib_TNC_SINGLE_auto_insuff +coef_calib_TNC_SHARED_auto_insuff,coef_calib_TNC_SHARED_auto_insuff,coef_calib_TNC_SHARED_auto_insuff,coef_calib_TNC_SHARED_auto_insuff,coef_calib_TNC_SHARED_auto_insuff,coef_calib_TNC_SHARED_auto_insuff,coef_calib_TNC_SHARED_auto_insuff,coef_calib_TNC_SHARED_auto_insuff,coef_calib_TNC_SHARED_auto_insuff,coef_calib_TNC_SHARED_auto_insuff,coef_calib_TNC_SHARED_auto_insuff +coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_DRIVEALONEPAY_auto_suff,coef_calib_DRIVEALONEPAY_auto_suff +coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2FREE_auto_suff,coef_calib_SHARED2FREE_auto_suff +coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED2PAY_auto_suff,coef_calib_SHARED2PAY_auto_suff +coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3FREE_auto_suff,coef_calib_SHARED3FREE_auto_suff +coef_calib_SHARED3PAY_auto_suff,coef_calib_SHARED3PAY_auto_suff,coef_calib_SHARED3PAY_auto_suff,coef_calib_SHARED3PAY_auto_suff,coef_calib_SHARED3PAY_auto_suff,coef_calib_SHARED3PAY_auto_suff,coef_calib_SHARED3PAY_auto_suff,coef_calib_SHARED3PAY_auto_suff,coef_calib_SHARED3PAY_auto_suff,coef_calib_SHARED3PAY_auto_suff,coef_calib_SHARED3PAY_auto_suff +coef_calib_WALK_auto_suff,coef_calib_WALK_auto_suff,coef_calib_WALK_auto_suff,coef_calib_WALK_auto_suff,coef_calib_WALK_auto_suff,coef_calib_WALK_auto_suff,coef_calib_WALK_auto_suff,coef_calib_WALK_auto_suff,coef_calib_WALK_auto_suff,coef_calib_WALK_auto_suff,coef_calib_WALK_auto_suff +coef_calib_BIKE_auto_suff,coef_calib_BIKE_auto_suff,coef_calib_BIKE_auto_suff,coef_calib_BIKE_auto_suff,coef_calib_BIKE_auto_suff,coef_calib_BIKE_auto_suff,coef_calib_BIKE_auto_suff,coef_calib_BIKE_auto_suff,coef_calib_BIKE_auto_suff,coef_calib_BIKE_auto_suff,coef_calib_BIKE_auto_suff +coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LOC_auto_suff,coef_calib_WALK_LOC_auto_suff +coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_LRF_auto_suff,coef_calib_WALK_LRF_auto_suff +coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_EXP_auto_suff,coef_calib_WALK_EXP_auto_suff +coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_HVY_auto_suff,coef_calib_WALK_HVY_auto_suff +coef_calib_WALK_COM_auto_suff,coef_calib_WALK_COM_auto_suff,coef_calib_WALK_COM_auto_suff,coef_calib_WALK_COM_auto_suff,coef_calib_WALK_COM_auto_suff,coef_calib_WALK_COM_auto_suff,coef_calib_WALK_COM_auto_suff,coef_calib_WALK_COM_auto_suff,coef_calib_WALK_COM_auto_suff,coef_calib_WALK_COM_auto_suff,coef_calib_WALK_COM_auto_suff +coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LOC_auto_suff,coef_calib_DRIVE_LOC_auto_suff +coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_LRF_auto_suff,coef_calib_DRIVE_LRF_auto_suff +coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_EXP_auto_suff,coef_calib_DRIVE_EXP_auto_suff +coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_HVY_auto_suff,coef_calib_DRIVE_HVY_auto_suff +coef_calib_DRIVE_COM_auto_suff,coef_calib_DRIVE_COM_auto_suff,coef_calib_DRIVE_COM_auto_suff,coef_calib_DRIVE_COM_auto_suff,coef_calib_DRIVE_COM_auto_suff,coef_calib_DRIVE_COM_auto_suff,coef_calib_DRIVE_COM_auto_suff,coef_calib_DRIVE_COM_auto_suff,coef_calib_DRIVE_COM_auto_suff,coef_calib_DRIVE_COM_auto_suff,coef_calib_DRIVE_COM_auto_suff +coef_calib_TAXI_auto_suff,coef_calib_TAXI_auto_suff,coef_calib_TAXI_auto_suff,coef_calib_TAXI_auto_suff,coef_calib_TAXI_auto_suff,coef_calib_TAXI_auto_suff,coef_calib_TAXI_auto_suff,coef_calib_TAXI_auto_suff,coef_calib_TAXI_auto_suff,coef_calib_TAXI_auto_suff,coef_calib_TAXI_auto_suff +coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SINGLE_auto_suff,coef_calib_TNC_SINGLE_auto_suff +coef_calib_TNC_SHARED_auto_suff,coef_calib_TNC_SHARED_auto_suff,coef_calib_TNC_SHARED_auto_suff,coef_calib_TNC_SHARED_auto_suff,coef_calib_TNC_SHARED_auto_suff,coef_calib_TNC_SHARED_auto_suff,coef_calib_TNC_SHARED_auto_suff,coef_calib_TNC_SHARED_auto_suff,coef_calib_TNC_SHARED_auto_suff,coef_calib_TNC_SHARED_auto_suff,coef_calib_TNC_SHARED_auto_suff From fbf24d8289c2861c54ecc8b9227ff3e9aeedf423 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 7 Jul 2026 13:24:05 -0400 Subject: [PATCH 18/96] Update workplace location survey file directory --- .../configs/workplace_location_coefficients.csv | 8 ++++---- .../workplace_location_calib_helper.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv b/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv index 7b1b52e4ac..28be9b2e5a 100644 --- a/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv +++ b/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv @@ -7,7 +7,7 @@ coef_dist_15_up,-0.0917,F coef_dist_0_5_high,0.15,F coef_dist_5_up_high,0.02,F coef_mode_logsum,0.3,F -coef_calib_dist_0_2,-5.0,T -coef_calib_dist_2_5,0.0,T -coef_calib_dist_5_15,-2.771779906602003,T -coef_calib_dist_15_up,0.749738309309575,T +coef_calib_dist_0_2,1.0,T +coef_calib_dist_2_5,1.0,T +coef_calib_dist_5_15,1.0,T +coef_calib_dist_15_up,1.0,T diff --git a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py index 3c2496667f..207af38bc5 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py @@ -11,7 +11,7 @@ import os from functools import lru_cache -SURVEY_DATA_FOLDER = r"C:\Users\david.hensle\OneDrive - Resource Systems Group, Inc\Documents\projects\activitysim\rsg_activitysim\activitysim\examples\example_estimation\data_test\survey_data" +SURVEY_DATA_FOLDER = "activitysim/examples/example_estimation/data_sf/survey_data" def compute_distances(context, origins, destinations): # Compute distances between origins and destinations using the network level of service From 33ee99393a194ae1a204702ecf94f880e52b5e6a Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 7 Jul 2026 13:57:53 -0400 Subject: [PATCH 19/96] auto ownership & tour mc relative paths --- .../configs/auto_ownership_coefficients.csv | 24 +-- .../configs/tour_mode_choice_coefficients.csv | 164 +++++++++--------- .../auto_ownership_calib_helper.py | 2 +- .../tour_mode_choice_calib_helper.py | 2 +- 4 files changed, 96 insertions(+), 96 deletions(-) diff --git a/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv b/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv index 4ade078423..d8c072de3b 100644 --- a/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv +++ b/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv @@ -1,12 +1,12 @@ coefficient_name,value,constrain -coef_cars1_drivers_2,0.0,T -coef_cars1_drivers_3,0.0,T -coef_cars1_persons_16_17,0.0,T -coef_cars234_asc_marin,0.0,T -coef_cars1_persons_25_34,0.0,T -coef_cars1_num_workers_clip_3,0.0,T -coef_cars1_hh_income_30_up,0.0,T -coef_cars1_density_0_10_no_workers,0.0,T +coef_cars1_drivers_2,0,T +coef_cars1_drivers_3,0,T +coef_cars1_persons_16_17,0,T +coef_cars234_asc_marin,0,T +coef_cars1_persons_25_34,0,T +coef_cars1_num_workers_clip_3,0,T +coef_cars1_hh_income_30_up,0,T +coef_cars1_density_0_10_no_workers,0,T coef_cars1_density_10_up_workers,-0.0152,F coef_retail_non_motor,-0.03,T coef_cars4_asc,-5.313,F @@ -66,7 +66,7 @@ coef_cars3_drivers_3,5.5131,F coef_cars2_drivers_4_up,6.3662,F coef_cars3_drivers_4_up,8.5148,F coef_cars4_drivers_4_up,9.5807,F -coef_calib_auto_0,-5.0,T -coef_calib_auto_2,5.0,T -coef_calib_auto_3,5.0,T -coef_calib_auto_4,5.0,T +coef_calib_auto_0,1,T +coef_calib_auto_2,1,T +coef_calib_auto_3,1,T +coef_calib_auto_4,1,T diff --git a/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv b/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv index a6d5c07487..afe19ff500 100644 --- a/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv +++ b/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv @@ -1,6 +1,6 @@ coefficient_name,value,constrain -coef_one,1.0,T -coef_nest_root,1.0,T +coef_one,1,T +coef_nest_root,1,T coef_nest_AUTO,0.72,T coef_nest_AUTO_DRIVEALONE,0.35,T coef_nest_AUTO_SHAREDRIDE2,0.35,T @@ -14,23 +14,23 @@ coef_ivt_eatout_escort_othdiscr_othmaint_shopping_social,-0.0175,F coef_ivt_school_univ,-0.0224,F coef_ivt_work,-0.0134,F coef_ivt_atwork,-0.0188,F -coef_topology_walk_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,15.0,F +coef_topology_walk_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,15,F coef_topology_walk_multiplier_atwork,7.5,F -coef_topology_bike_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,20.0,F -coef_topology_bike_multiplier_atwork,10.0,F +coef_topology_bike_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,20,F +coef_topology_bike_multiplier_atwork,10,F coef_topology_trn_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,2.2,F -coef_topology_trn_multiplier_atwork,2.0,F -coef_age1619_da_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work,0.0,F +coef_topology_trn_multiplier_atwork,2,F +coef_age1619_da_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work,0,F coef_age1619_da_multiplier_school_univ,-1.3813,F coef_age1619_da_multiplier_atwork,0.0032336,F -coef_age010_trn_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work,0.0,F +coef_age010_trn_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work,0,F coef_age010_trn_multiplier_school_univ,-1.5548,F coef_age010_trn_multiplier_atwork,0.000722,F coef_age16p_sr_multiplier_eatout_escort_othdiscr_othmaint_shopping_social,-1.366,F -coef_age16p_sr_multiplier_school_univ_work_atwork,0.0,F -coef_hhsize1_sr_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_atwork,0.0,F +coef_age16p_sr_multiplier_school_univ_work_atwork,0,F +coef_hhsize1_sr_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_atwork,0,F coef_hhsize1_sr_multiplier_work,-0.734588,F -coef_hhsize2_sr_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work_atwork,0.0,F +coef_hhsize2_sr_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work_atwork,0,F coef_hhsize2_sr_multiplier_school_univ,-0.6359,F walk_ASC_no_auto_eatout,5.1251173,F walk_ASC_no_auto_escort,2.8012068,F @@ -92,9 +92,9 @@ bike_ASC_auto_sufficient_social,-1.368071,F bike_ASC_auto_sufficient_univ,-1.9397832,F bike_ASC_auto_sufficient_work,-1.5800232,F bike_ASC_auto_sufficient_atwork,15.72017,F -sr2_ASC_no_auto_all,0.0,F +sr2_ASC_no_auto_all,0,F sr2_ASC_auto_deficient_eatout,0.5882345,F -sr2_ASC_auto_deficient_escort,0.0,F +sr2_ASC_auto_deficient_escort,0,F sr2_ASC_auto_deficient_othdiscr,0.6601513,F sr2_ASC_auto_deficient_othmaint,0.2621527,F sr2_ASC_auto_deficient_school,0.12474365,F @@ -104,7 +104,7 @@ sr2_ASC_auto_deficient_univ,-1.6922346,F sr2_ASC_auto_deficient_work,-0.33803123,F sr2_ASC_auto_deficient_atwork,-2.1102421,F sr2_ASC_auto_sufficient_eatout,0.86280555,F -sr2_ASC_auto_sufficient_escort,0.0,F +sr2_ASC_auto_sufficient_escort,0,F sr2_ASC_auto_sufficient_othdiscr,0.49684617,F sr2_ASC_auto_sufficient_othmaint,0.25817883,F sr2_ASC_auto_sufficient_school,-1.6062657,F @@ -173,7 +173,7 @@ walk_transit_ASC_auto_sufficient_social,-0.3453759,F walk_transit_ASC_auto_sufficient_univ,0.4731163,F walk_transit_ASC_auto_sufficient_work,-0.8916507,F walk_transit_ASC_auto_sufficient_atwork,-3.401027,F -drive_transit_ASC_no_auto_all,0.0,F +drive_transit_ASC_no_auto_all,0,F drive_transit_ASC_auto_deficient_eatout,0.5998061,F drive_transit_ASC_auto_deficient_escort,-1.1537067,F drive_transit_ASC_auto_deficient_othdiscr,0.3199308,F @@ -196,7 +196,7 @@ drive_transit_ASC_auto_sufficient_work,-1.0045459,F drive_transit_ASC_auto_sufficient_atwork,-999.21466,F taxi_ASC_no_auto_eatout_othdiscr_social,0.9923,F taxi_ASC_no_auto_escort_othmaint_shopping,1.8939,F -taxi_ASC_no_auto_school_univ,-7.0,T +taxi_ASC_no_auto_school_univ,-7,T taxi_ASC_no_auto_work,4.7291,F taxi_ASC_no_auto_atwork,4.1021,F taxi_ASC_auto_deficient_eatout_othdiscr_social,-3.1317,F @@ -213,7 +213,7 @@ taxi_ASC_auto_sufficient_work,-4.8509,F taxi_ASC_auto_sufficient_atwork,-2.8804,F tnc_single_ASC_no_auto_eatout_othdiscr_social,1.6852,F tnc_single_ASC_no_auto_escort_othmaint_shopping,1.8605,F -tnc_single_ASC_no_auto_school,-7.0,T +tnc_single_ASC_no_auto_school,-7,T tnc_single_ASC_no_auto_univ,-2.519,F tnc_single_ASC_no_auto_work,5.7855,F tnc_single_ASC_no_auto_atwork,4.4982,F @@ -231,7 +231,7 @@ tnc_single_ASC_auto_sufficient_work,-4.1946,F tnc_single_ASC_auto_sufficient_atwork,-2.7988,F tnc_shared_ASC_no_auto_eatout_othdiscr_social,0.6464,F tnc_shared_ASC_no_auto_escort_othmaint_shopping,0.9361,F -tnc_shared_ASC_no_auto_school,-7.0,T +tnc_shared_ASC_no_auto_school,-7,T tnc_shared_ASC_no_auto_univ,-5.8116,F tnc_shared_ASC_no_auto_work,3.2429,F tnc_shared_ASC_no_auto_atwork,3.3672,F @@ -253,16 +253,16 @@ joint_walk_ASC_auto_sufficient_all,-3.2352157,F joint_bike_ASC_no_auto_all,-2.8671598,F joint_bike_ASC_auto_deficient_all,-6.076415,F joint_bike_ASC_auto_sufficient_all,-6.3760657,F -joint_sr2_ASC_no_auto_all,0.0,T -joint_sr2_ASC_auto_deficient_all,0.0,T -joint_sr2_ASC_auto_sufficient_all,0.0,T +joint_sr2_ASC_no_auto_all,0,T +joint_sr2_ASC_auto_deficient_all,0,T +joint_sr2_ASC_auto_sufficient_all,0,T joint_sr3p_ASC_no_auto_all,0.5630671,F joint_sr3p_ASC_auto_deficient_all,-1.8841692,F joint_sr3p_ASC_auto_sufficient_all,-2.234826,F joint_walk_transit_ASC_no_auto_all,0.62292415,F joint_walk_transit_ASC_auto_deficient_all,-5.1634483,F joint_walk_transit_ASC_auto_sufficient_all,-18.264534,F -joint_drive_transit_ASC_no_auto_all,0.0,T +joint_drive_transit_ASC_no_auto_all,0,T joint_drive_transit_ASC_auto_deficient_all,-5.9632215,F joint_drive_transit_ASC_auto_sufficient_all,-8.045285,F joint_taxi_ASC_no_auto_all,-4.5792,F @@ -306,63 +306,63 @@ drive_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,0.525,F drive_transit_CBD_ASC_school_univ,0.672,F drive_transit_CBD_ASC_work,1.1,F drive_transit_CBD_ASC_atwork,0.564,F -coef_calib_DRIVEALONEPAY_zero_auto,5.0,T -coef_calib_SHARED2FREE_zero_auto,5.0,T -coef_calib_SHARED2PAY_zero_auto,5.0,T -coef_calib_SHARED3FREE_zero_auto,5.0,T -coef_calib_SHARED3PAY_zero_auto,5.0,T -coef_calib_WALK_zero_auto,1.19369549873279,T -coef_calib_BIKE_zero_auto,5.0,T -coef_calib_WALK_LOC_zero_auto,0.29206606912731364,T -coef_calib_WALK_LRF_zero_auto,1.2520906218382721,T -coef_calib_WALK_EXP_zero_auto,5.0,T -coef_calib_WALK_HVY_zero_auto,2.427455130545431,T -coef_calib_WALK_COM_zero_auto,5.0,T -coef_calib_DRIVE_LOC_zero_auto,5.0,T -coef_calib_DRIVE_LRF_zero_auto,5.0,T -coef_calib_DRIVE_EXP_zero_auto,5.0,T -coef_calib_DRIVE_HVY_zero_auto,5.0,T -coef_calib_DRIVE_COM_zero_auto,5.0,T -coef_calib_TAXI_zero_auto,5.0,T -coef_calib_TNC_SINGLE_zero_auto,3.2347512774613625,T -coef_calib_TNC_SHARED_zero_auto,5.0,T -coef_calib_DRIVEALONEPAY_auto_insuff,5.0,T -coef_calib_SHARED2FREE_auto_insuff,4.655634851986755,T -coef_calib_SHARED2PAY_auto_insuff,5.0,T -coef_calib_SHARED3FREE_auto_insuff,5.0,T -coef_calib_SHARED3PAY_auto_insuff,5.0,T -coef_calib_WALK_auto_insuff,3.1931900192458924,T -coef_calib_BIKE_auto_insuff,4.992944512095712,T -coef_calib_WALK_LOC_auto_insuff,5.0,T -coef_calib_WALK_LRF_auto_insuff,3.9302088455457014,T -coef_calib_WALK_EXP_auto_insuff,5.0,T -coef_calib_WALK_HVY_auto_insuff,1.3844675036448821,T -coef_calib_WALK_COM_auto_insuff,5.0,T -coef_calib_DRIVE_LOC_auto_insuff,5.0,T -coef_calib_DRIVE_LRF_auto_insuff,5.0,T -coef_calib_DRIVE_EXP_auto_insuff,5.0,T -coef_calib_DRIVE_HVY_auto_insuff,5.0,T -coef_calib_DRIVE_COM_auto_insuff,5.0,T -coef_calib_TAXI_auto_insuff,5.0,T -coef_calib_TNC_SINGLE_auto_insuff,3.9140984082873835,T -coef_calib_TNC_SHARED_auto_insuff,5.0,T -coef_calib_DRIVEALONEPAY_auto_suff,5.0,T -coef_calib_SHARED2FREE_auto_suff,2.396741245612774,T -coef_calib_SHARED2PAY_auto_suff,5.0,T -coef_calib_SHARED3FREE_auto_suff,4.928377149112955,T -coef_calib_SHARED3PAY_auto_suff,5.0,T -coef_calib_WALK_auto_suff,3.077985217800374,T -coef_calib_BIKE_auto_suff,5.0,T -coef_calib_WALK_LOC_auto_suff,5.0,T -coef_calib_WALK_LRF_auto_suff,4.399523089022793,T -coef_calib_WALK_EXP_auto_suff,5.0,T -coef_calib_WALK_HVY_auto_suff,2.8217459359394326,T -coef_calib_WALK_COM_auto_suff,5.0,T -coef_calib_DRIVE_LOC_auto_suff,5.0,T -coef_calib_DRIVE_LRF_auto_suff,5.0,T -coef_calib_DRIVE_EXP_auto_suff,5.0,T -coef_calib_DRIVE_HVY_auto_suff,5.0,T -coef_calib_DRIVE_COM_auto_suff,5.0,T -coef_calib_TAXI_auto_suff,5.0,T -coef_calib_TNC_SINGLE_auto_suff,4.29485267200215,T -coef_calib_TNC_SHARED_auto_suff,5.0,T +coef_calib_DRIVEALONEPAY_zero_auto,1,T +coef_calib_SHARED2FREE_zero_auto,1,T +coef_calib_SHARED2PAY_zero_auto,1,T +coef_calib_SHARED3FREE_zero_auto,1,T +coef_calib_SHARED3PAY_zero_auto,1,T +coef_calib_WALK_zero_auto,1,T +coef_calib_BIKE_zero_auto,1,T +coef_calib_WALK_LOC_zero_auto,1,T +coef_calib_WALK_LRF_zero_auto,1,T +coef_calib_WALK_EXP_zero_auto,1,T +coef_calib_WALK_HVY_zero_auto,1,T +coef_calib_WALK_COM_zero_auto,1,T +coef_calib_DRIVE_LOC_zero_auto,1,T +coef_calib_DRIVE_LRF_zero_auto,1,T +coef_calib_DRIVE_EXP_zero_auto,1,T +coef_calib_DRIVE_HVY_zero_auto,1,T +coef_calib_DRIVE_COM_zero_auto,1,T +coef_calib_TAXI_zero_auto,1,T +coef_calib_TNC_SINGLE_zero_auto,1,T +coef_calib_TNC_SHARED_zero_auto,1,T +coef_calib_DRIVEALONEPAY_auto_insuff,1,T +coef_calib_SHARED2FREE_auto_insuff,1,T +coef_calib_SHARED2PAY_auto_insuff,1,T +coef_calib_SHARED3FREE_auto_insuff,1,T +coef_calib_SHARED3PAY_auto_insuff,1,T +coef_calib_WALK_auto_insuff,1,T +coef_calib_BIKE_auto_insuff,1,T +coef_calib_WALK_LOC_auto_insuff,1,T +coef_calib_WALK_LRF_auto_insuff,1,T +coef_calib_WALK_EXP_auto_insuff,1,T +coef_calib_WALK_HVY_auto_insuff,1,T +coef_calib_WALK_COM_auto_insuff,1,T +coef_calib_DRIVE_LOC_auto_insuff,1,T +coef_calib_DRIVE_LRF_auto_insuff,1,T +coef_calib_DRIVE_EXP_auto_insuff,1,T +coef_calib_DRIVE_HVY_auto_insuff,1,T +coef_calib_DRIVE_COM_auto_insuff,1,T +coef_calib_TAXI_auto_insuff,1,T +coef_calib_TNC_SINGLE_auto_insuff,1,T +coef_calib_TNC_SHARED_auto_insuff,1,T +coef_calib_DRIVEALONEPAY_auto_suff,1,T +coef_calib_SHARED2FREE_auto_suff,1,T +coef_calib_SHARED2PAY_auto_suff,1,T +coef_calib_SHARED3FREE_auto_suff,1,T +coef_calib_SHARED3PAY_auto_suff,1,T +coef_calib_WALK_auto_suff,1,T +coef_calib_BIKE_auto_suff,1,T +coef_calib_WALK_LOC_auto_suff,1,T +coef_calib_WALK_LRF_auto_suff,1,T +coef_calib_WALK_EXP_auto_suff,1,T +coef_calib_WALK_HVY_auto_suff,1,T +coef_calib_WALK_COM_auto_suff,1,T +coef_calib_DRIVE_LOC_auto_suff,1,T +coef_calib_DRIVE_LRF_auto_suff,1,T +coef_calib_DRIVE_EXP_auto_suff,1,T +coef_calib_DRIVE_HVY_auto_suff,1,T +coef_calib_DRIVE_COM_auto_suff,1,T +coef_calib_TAXI_auto_suff,1,T +coef_calib_TNC_SINGLE_auto_suff,1,T +coef_calib_TNC_SHARED_auto_suff,1,T diff --git a/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py index ee6f319ce7..e5640fb6f3 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py @@ -2,7 +2,7 @@ import pandas as pd import os -SURVEY_DATA_FOLDER = r"C:\Users\david.hensle\OneDrive - Resource Systems Group, Inc\Documents\projects\activitysim\rsg_activitysim\activitysim\examples\example_estimation\data_test\survey_data" +SURVEY_DATA_FOLDER = "activitysim/examples/example_estimation/data_sf/survey_data" def report_auto_ownership(context): model_hhs = context["households"] diff --git a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py index bbec248ca7..9b29b2deb5 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py @@ -2,7 +2,7 @@ import matplotlib.pyplot as plt import os -SURVEY_DATA_FOLDER = r"C:\Users\david.hensle\OneDrive - Resource Systems Group, Inc\Documents\projects\activitysim\rsg_activitysim\activitysim\examples\example_estimation\data_test\survey_data" +SURVEY_DATA_FOLDER = "activitysim/examples/example_estimation/data_sf/survey_data" def report_tour_mode_choice(context): model_tours = context["tours"] From ff71c98c29c639d1dc4630d3b6289ef14bc1a85a Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 7 Jul 2026 17:50:56 -0400 Subject: [PATCH 20/96] Allow no weights in tour mc summary --- .../configs_calibration/tour_mode_choice_calib_helper.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py index 9b29b2deb5..4fd8b31ca7 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py @@ -10,8 +10,11 @@ def report_tour_mode_choice(context): survey_tours = pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_tours.csv")) model_summary = model_tours.tour_mode.value_counts(normalize=True).sort_index().fillna(0) - survey_summary = survey_tours.groupby("tour_mode").tour_weight.sum() - survey_summary = survey_summary / survey_tours.tour_weight.sum() + if 'tour_weight' in survey_tours.columns: + survey_summary = survey_tours.groupby("tour_mode").tour_weight.sum() + survey_summary = survey_summary / survey_tours.tour_weight.sum() + else: + survey_summary = survey_tours.groupby("tour_mode").size() summary_df = ( pd.DataFrame({"model": model_summary, "survey": survey_summary}) From 217b3e7c8dbaaea6fb9e1c2d27783b20065d56b5 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Mon, 13 Jul 2026 11:17:05 -0400 Subject: [PATCH 21/96] Remove duplicated checkpoint directories --- activitysim/core/calibration.py | 14 ++++++++------ activitysim/core/mp_tasks.py | 8 ++++---- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index e60e05dc0e..c1090b9ca1 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -382,7 +382,10 @@ def _calibrate_component( for component_iter in range(1, component_settings.submodel_max_iterations + 1): component_iterations = component_iter - + run_model_name = ( + f"{component_name}.component_i{component_iter};" + f"global_i{global_iter}" + ) # Re-run only this component from its prior checkpoint so model values # reflect the current candidate coefficients for this component. if state.settings.multiprocess: @@ -390,14 +393,10 @@ def _calibrate_component( # so table coalescing semantics match the initial global run path. _run_in_configured_mode( state, - models=state.settings.models, + models=state.settings.models[:state.settings.models.index(component_name) + 1], resume_after=prior_step, ) else: - run_model_name = ( - f"{component_name}.calibration_component_iter={component_iter};" - f"calibration_global_iter={global_iter}" - ) state.run(models=[run_model_name], resume_after=prior_step) eval_context = _build_expression_context(state, helper_symbols, component_name, component_settings) @@ -1251,9 +1250,11 @@ def _run_multiprocess_with_overrides( from activitysim.core import mp_tasks original_models = state.settings.models + original_mp_steps = state.settings.multiprocess_steps original_resume_after = state.settings.resume_after state.settings.models = models + state.settings.multiprocess_steps = [step for step in original_mp_steps if step.begin in models] state.settings.resume_after = resume_after try: @@ -1268,3 +1269,4 @@ def _run_multiprocess_with_overrides( finally: state.settings.models = original_models state.settings.resume_after = original_resume_after + state.settings.multiprocess_steps = original_mp_steps diff --git a/activitysim/core/mp_tasks.py b/activitysim/core/mp_tasks.py index c8507138f4..70cd70ebb9 100644 --- a/activitysim/core/mp_tasks.py +++ b/activitysim/core/mp_tasks.py @@ -738,7 +738,7 @@ def apportion_pipeline(state: workflow.State, sub_proc_names, step_info): ) # - write table to pipeline - pipeline_path.joinpath(table_name).mkdir(parents=True, exist_ok=True) + # pipeline_path.joinpath(table_name).mkdir(parents=True, exist_ok=True) ParquetStore(pipeline_path).put( table_name=table_name, @@ -755,9 +755,9 @@ def apportion_pipeline(state: workflow.State, sub_proc_names, step_info): f"writing checkpoints ({checkpoints_df.shape}) " f"to {CHECKPOINT_TABLE_NAME} in {pipeline_path}", ) - pipeline_path.joinpath(CHECKPOINT_TABLE_NAME).mkdir( - parents=True, exist_ok=True - ) + # pipeline_path.joinpath(CHECKPOINT_TABLE_NAME).mkdir( + # parents=True, exist_ok=True + # ) ParquetStore(pipeline_path).put( table_name=CHECKPOINT_TABLE_NAME, df=checkpoints_df, From aadd5293778fefe0312a923d8eebb45bcb5b7c30 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 14 Jul 2026 14:39:28 -0500 Subject: [PATCH 22/96] Minor bugfixes --- activitysim/core/calibration.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index c1090b9ca1..05be6d762e 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -61,6 +61,7 @@ "imported_extensions", "run_timestamp", "run_id", + "pipeline_file_name", ] @@ -311,6 +312,8 @@ def _run_intermediate_components( resume_after: str, memory_sidecar_process=None, ) -> None: + if len(models) == 0: + return # don't modify the pipeline, just run the models needed _run_in_configured_mode( state, From 67fd29fcd986bf71ed503bc1b8e0d366f375302b Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 14 Jul 2026 15:33:10 -0500 Subject: [PATCH 23/96] Separate Claude attempt at MP calibration --- activitysim/core/calibration.py | 360 ++++++++++++++++++++++++++++++-- activitysim/core/mp_tasks.py | 131 ++++++------ 2 files changed, 410 insertions(+), 81 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 05be6d762e..de21667436 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -7,6 +7,7 @@ import json import logging import math +import multiprocessing import os import re from dataclasses import dataclass @@ -22,6 +23,7 @@ from activitysim.core import workflow from activitysim.core.configuration import PydanticReadable from activitysim.core.configuration.base import PydanticBase +from activitysim.core.configuration.top import MultiprocessStep logger = logging.getLogger("calibration") @@ -189,6 +191,12 @@ def run_calibration_loop( original_pipeline_name = state.filesystem.pipeline_file_name + # Initialize shared resources for multiprocess mode (skims, shadow pricing). + # These are allocated once and reused across all calibration iterations. + shared_data_buffers = None + if state.settings.multiprocess: + shared_data_buffers = _initialize_mp_shared_resources(state) + try: for global_iter in range( start_global_iter, @@ -211,6 +219,7 @@ def run_calibration_loop( ), global_iter=global_iter, memory_sidecar_process=memory_sidecar_process, + shared_data_buffers=shared_data_buffers, ) all_converged = True @@ -232,6 +241,7 @@ def run_calibration_loop( ], resume_after=last_calibrated_component, memory_sidecar_process=memory_sidecar_process, + shared_data_buffers=shared_data_buffers, ) component_result = _calibrate_component( @@ -240,6 +250,7 @@ def run_calibration_loop( component_settings=component_settings, prior_step=prior_step, global_iter=global_iter, + shared_data_buffers=shared_data_buffers, ) _write_component_plots(state, component) @@ -257,6 +268,7 @@ def run_calibration_loop( models=models[models.index(last_calibrated_component) + 1 :], resume_after=last_calibrated_component, memory_sidecar_process=memory_sidecar_process, + shared_data_buffers=shared_data_buffers, ) _write_progress( @@ -283,12 +295,10 @@ def _run_precursor_components( resume_after: str, global_iter: int, memory_sidecar_process=None, + shared_data_buffers: dict | None = None, ) -> None: """Run the normal ActivitySim model flow for one global calibration iteration.""" - assert (resume_after is None) or ( - resume_after in models - ), f"resume_after step {resume_after} not in models preceding calibration models" if global_iter > 1: # Seed a fresh pipeline from the configured resume checkpoint to avoid # duplicate checkpoint-name collisions across global calibration loops. @@ -297,12 +307,12 @@ def _run_precursor_components( state.filesystem.pipeline_file_name = f"pipeline_calibration_iter_{global_iter}" state.checkpoint.restore_from(prior_pipeline, checkpoint_name=resume_after) else: - _run_in_configured_mode( state, models=models, resume_after=resume_after, memory_sidecar_process=memory_sidecar_process, + shared_data_buffers=shared_data_buffers, ) @@ -311,15 +321,16 @@ def _run_intermediate_components( models: list[str], resume_after: str, memory_sidecar_process=None, + shared_data_buffers: dict | None = None, ) -> None: if len(models) == 0: return - # don't modify the pipeline, just run the models needed _run_in_configured_mode( state, models=models, resume_after=resume_after, memory_sidecar_process=memory_sidecar_process, + shared_data_buffers=shared_data_buffers, ) @@ -328,13 +339,14 @@ def _run_subsequent_components( models: list[str], resume_after: str, memory_sidecar_process=None, + shared_data_buffers: dict | None = None, ) -> None: - # don't modify the pipeline, just run the models needed _run_in_configured_mode( state, models=models, resume_after=resume_after, memory_sidecar_process=memory_sidecar_process, + shared_data_buffers=shared_data_buffers, ) @@ -344,6 +356,7 @@ def _calibrate_component( component_settings: CalibrationComponentSettings, prior_step: str, global_iter: int, + shared_data_buffers: dict | None = None, ) -> CalibrationComponentResult: """Run iterative coefficient calibration for one component.""" model_settings_file = _infer_model_settings_file(component_name) @@ -383,6 +396,22 @@ def _calibrate_component( component_converged = False component_iterations = 0 + # Determine the checkpoint name to restore from for component re-runs. + # In MP mode, the checkpoint that represents prior_step's completed state + # is the last checkpoint in the pipeline before we run the component. + # We capture it once and reuse across component iterations. + mp_restore_checkpoint = None + if state.settings.multiprocess and shared_data_buffers is not None: + # The pipeline should already be open from _restore_parent_state_from_pipeline + # called after precursor/intermediate models ran. The last checkpoint + # in the pipeline represents the state at prior_step. + if state.checkpoint.checkpoints: + mp_restore_checkpoint = state.checkpoint.last_checkpoint.get( + "checkpoint_name", "_" + ) + else: + mp_restore_checkpoint = "_" + for component_iter in range(1, component_settings.submodel_max_iterations + 1): component_iterations = component_iter run_model_name = ( @@ -391,13 +420,15 @@ def _calibrate_component( ) # Re-run only this component from its prior checkpoint so model values # reflect the current candidate coefficients for this component. - if state.settings.multiprocess: - # In multiprocess mode, preserve the standard multiprocess orchestration - # so table coalescing semantics match the initial global run path. - _run_in_configured_mode( + if state.settings.multiprocess and shared_data_buffers is not None: + # Use direct MP orchestration with explicit checkpoint control. + # This ensures we always apportion from prior_step's state, + # even after multiple component iterations. + _run_mp_single_component( state, - models=state.settings.models[:state.settings.models.index(component_name) + 1], - resume_after=prior_step, + component_name=component_name, + restore_checkpoint=mp_restore_checkpoint, + shared_data_buffers=shared_data_buffers, ) else: state.run(models=[run_model_name], resume_after=prior_step) @@ -1222,19 +1253,158 @@ def _write_progress(state: workflow.State, payload: dict[str, Any]) -> None: json.dump(payload, f, indent=2) +def _run_mp_single_component( + state: workflow.State, + component_name: str, + restore_checkpoint: str, + shared_data_buffers: dict, +) -> None: + """Run a single component in multiprocess mode with explicit checkpoint control. + + This directly orchestrates the apportion → simulate → coalesce flow + without going through run_multiprocess/get_run_list, giving us precise + control over which checkpoint to apportion from. This is essential for + calibration component re-runs where we must always restart from prior_step's + state regardless of what other checkpoints exist in the pipeline. + + Parameters + ---------- + state : workflow.State + component_name : str + The model component to run. + restore_checkpoint : str + The checkpoint name to restore from before apportioning. + This should be the checkpoint representing prior_step's state. + shared_data_buffers : dict + Pre-allocated shared memory buffers for skims/shadow pricing. + """ + from activitysim.core import mp_tasks + + # Determine slice info from original settings + original_steps = state.settings.multiprocess_steps + all_models = state.settings.models + + slice_info = None + num_processes = state.settings.num_processes or 2 + chunk_size = state.settings.chunk_size or 0 + + # Find which original step this component belongs to + step_boundaries = [] + for i, step in enumerate(original_steps): + step_boundaries.append(all_models.index(step.begin)) + step_boundaries.append(len(all_models)) + + component_idx = all_models.index(component_name) + for i, step in enumerate(original_steps): + if step_boundaries[i] <= component_idx < step_boundaries[i + 1]: + if step.slice: + slice_info = step.slice.model_dump() + if step.num_processes: + num_processes = step.num_processes + if step.chunk_size: + chunk_size = step.chunk_size + break + + step_name = f"calibration_{component_name}" + + # Build step_info dict matching what mp_tasks functions expect + step_info = { + "name": step_name, + "models": [component_name], + "num_processes": num_processes, + "chunk_size": chunk_size, + "step_num": 0, + "slice": slice_info, + "last_checkpoint_in_previous_multiprocess_step": restore_checkpoint, + } + + injectables = _build_calibration_injectables(state) + + if num_processes == 1: + sub_proc_names = [step_name] + else: + sub_proc_names = [f"{step_name}_{i}" for i in range(num_processes)] + + fail_fast = state.settings.fail_fast + + # Apportion pipeline (split tables across sub-processes) + if num_processes > 1 and slice_info is not None: + mp_tasks.run_sub_task( + state, + multiprocessing.Process( + target=mp_tasks.mp_apportion_pipeline, + name=f"{step_name}_apportion", + args=(injectables, sub_proc_names, step_info), + ), + ) + + # For multi-process runs, subprocesses must restore from the apportioned + # pipeline (which has one checkpoint). Use LAST_CHECKPOINT so they don't + # overwrite the apportioned data with a fresh pipeline. + # For single-process runs (no apportion), use restore_checkpoint to resume + # from the correct point in the main pipeline. + if num_processes > 1: + sim_resume_after = "_" # LAST_CHECKPOINT in apportioned sub-pipeline + else: + sim_resume_after = restore_checkpoint + + # Run simulations in sub-processes + completed = mp_tasks.run_sub_simulations( + state, + injectables, + shared_data_buffers, + step_info, + sub_proc_names, + sim_resume_after, + [], # previously_completed + fail_fast, + ) + + if len(completed) != num_processes: + from activitysim.core.exceptions import SubprocessError + + raise SubprocessError( + f"{num_processes - len(completed)} processes failed in " + f"calibration step {step_name}" + ) + + # Coalesce sub-process pipelines back into main pipeline + if num_processes > 1 and slice_info is not None: + mp_tasks.run_sub_task( + state, + multiprocessing.Process( + target=mp_tasks.mp_coalesce_pipelines, + name=f"{step_name}_coalesce", + args=(injectables, sub_proc_names, slice_info), + ), + ) + + # Restore coalesced results into parent state + _restore_parent_state_from_pipeline(state) + + def _run_in_configured_mode( state: workflow.State, models: list[str], resume_after: str | None, memory_sidecar_process=None, + shared_data_buffers: dict | None = None, ) -> None: """Run models using the same single/multiprocess mode as the parent run.""" + if not models: + return + if state.settings.multiprocess: _run_multiprocess_with_overrides( state, models=models, resume_after=resume_after, + shared_data_buffers=shared_data_buffers, ) + # After multiprocess completes, the coalesced pipeline exists on disk. + # Restore it into the parent process state so tables are accessible + # for calibration expression evaluation. + _restore_parent_state_from_pipeline(state) return state.run( @@ -1248,6 +1418,7 @@ def _run_multiprocess_with_overrides( state: workflow.State, models: list[str], resume_after: str | None, + shared_data_buffers: dict | None = None, ) -> None: """Run multiprocess with temporary settings overrides for calibration passes.""" from activitysim.core import mp_tasks @@ -1256,20 +1427,167 @@ def _run_multiprocess_with_overrides( original_mp_steps = state.settings.multiprocess_steps original_resume_after = state.settings.resume_after + # Build valid multiprocess_steps for the requested model subset. + calibration_mp_steps = _build_calibration_mp_steps( + models=models, + original_steps=original_mp_steps, + all_models=original_models, + ) + state.settings.models = models - state.settings.multiprocess_steps = [step for step in original_mp_steps if step.begin in models] + state.settings.multiprocess_steps = calibration_mp_steps state.settings.resume_after = resume_after try: - injectables = {} - for key in MP_INJECTABLES: - try: - injectables[key] = state.get_injectable(key) - except KeyError: - pass - injectables["settings"] = state.settings - mp_tasks.run_multiprocess(state, injectables) + injectables = _build_calibration_injectables(state) + mp_tasks.run_multiprocess( + state, + injectables, + shared_data_buffers=shared_data_buffers, + skip_final_checkpoint=True, + ) finally: state.settings.models = original_models state.settings.resume_after = original_resume_after state.settings.multiprocess_steps = original_mp_steps + + +def _restore_parent_state_from_pipeline(state: workflow.State) -> None: + """Restore coalesced pipeline tables into the parent process state. + + After a multiprocess run, the parent's in-memory state is stale. + This loads the latest checkpoint from the pipeline store so that + calibration expressions can evaluate against model outputs. + """ + if state.checkpoint.store_is_open(): + state.checkpoint.close_store() + state.checkpoint.restore(resume_after="_") + + +def _initialize_mp_shared_resources(state: workflow.State) -> dict: + """Allocate shared data buffers (skims, shadow pricing) once for reuse. + + This mirrors the allocation logic in mp_tasks.run_multiprocess but + is called once at calibration start rather than on every sub-run. + """ + from activitysim.core import mp_tasks, tracing + + shared_data_buffers = {} + sharrow_enabled = state.settings.sharrow + + t0 = tracing.print_elapsed_time() + if not sharrow_enabled: + shared_data_buffers.update(mp_tasks.allocate_shared_skim_buffers(state)) + t0 = tracing.print_elapsed_time("calibration: allocate shared skim buffer", t0) + + shared_data_buffers.update(mp_tasks.allocate_shared_shadow_pricing_buffers(state)) + t0 = tracing.print_elapsed_time( + "calibration: allocate shared shadow_pricing buffer", t0 + ) + + shared_data_buffers.update( + mp_tasks.allocate_shared_shadow_pricing_buffers_choice(state) + ) + t0 = tracing.print_elapsed_time( + "calibration: allocate shared shadow_pricing choice buffer", t0 + ) + + # Load skim data into the shared buffers. + if sharrow_enabled: + shared_data_buffers["skim_dataset"] = "sh.Dataset:skim_dataset" + from activitysim.core import flow, skim_dataset # noqa: F401 + + state.get_injectable("skim_dataset") + else: + if len(shared_data_buffers) > 0: + injectables = _build_calibration_injectables(state) + mp_tasks.run_sub_task( + state, + multiprocessing.Process( + target=mp_tasks.mp_setup_skims, + name="mp_setup_skims_calibration", + args=(injectables,), + kwargs=shared_data_buffers, + ), + ) + + # Make skims available in the parent process for expression evaluation. + state.add_injectable("data_buffers", shared_data_buffers) + try: + state.get_injectable("network_los") + except Exception: + logger.warning( + "calibration: could not resolve network_los in parent process; " + "skim-dependent expressions may fail" + ) + + return shared_data_buffers + + +def _build_calibration_mp_steps( + models: list[str], + original_steps: list[MultiprocessStep], + all_models: list[str], +) -> list[MultiprocessStep]: + """Build valid MultiprocessStep objects for a calibration model subset. + + The key challenge is that get_run_list() in mp_tasks requires: + - The first step's begin == models[0] + - Steps are ordered and non-overlapping + - Each step's begin is in the models list + + We intersect the original multiprocess_steps with the requested model + subset and construct new steps that satisfy these constraints. + """ + if not models: + return [] + + # Determine which original step each model in the full list belongs to. + # Build a mapping: model_name -> original step index + model_to_step: dict[str, int] = {} + step_boundaries = [] + for i, step in enumerate(original_steps): + begin_idx = all_models.index(step.begin) + step_boundaries.append(begin_idx) + step_boundaries.append(len(all_models)) + + for i, step in enumerate(original_steps): + for model_idx in range(step_boundaries[i], step_boundaries[i + 1]): + model_to_step[all_models[model_idx]] = i + + # Group the requested models by their original step + from collections import OrderedDict + + step_model_groups: OrderedDict[int, list[str]] = OrderedDict() + for model in models: + step_idx = model_to_step.get(model) + if step_idx is None: + continue + step_model_groups.setdefault(step_idx, []).append(model) + + # Build new MultiprocessStep for each group + new_steps = [] + for step_idx, step_models in step_model_groups.items(): + orig_step = original_steps[step_idx] + new_step = MultiprocessStep( + name=f"calibration_{orig_step.name}", + begin=step_models[0], + num_processes=orig_step.num_processes, + slice=orig_step.slice, + chunk_size=orig_step.chunk_size, + ) + new_steps.append(new_step) + + return new_steps + + +def _build_calibration_injectables(state: workflow.State) -> dict: + """Build the injectables dict for multiprocess sub-processes.""" + injectables = {} + for key in MP_INJECTABLES: + try: + injectables[key] = state.get_injectable(key) + except KeyError: + pass + injectables["settings"] = state.settings + return injectables diff --git a/activitysim/core/mp_tasks.py b/activitysim/core/mp_tasks.py index 70cd70ebb9..6adf85f853 100644 --- a/activitysim/core/mp_tasks.py +++ b/activitysim/core/mp_tasks.py @@ -558,12 +558,10 @@ def apportion_pipeline(state: workflow.State, sub_proc_names, step_info): # ensure that if we are resuming, we don't apportion any tables from future model steps last_checkpoint_in_previous_multiprocess_step = step_info.get( - "last_checkpoint_in_previous_multiprocess_step", None + "last_checkpoint_in_previous_multiprocess_step", LAST_CHECKPOINT ) if last_checkpoint_in_previous_multiprocess_step is None: - raise CheckpointNameNotFoundError( - "missing last_checkpoint_in_previous_multiprocess_step" - ) + last_checkpoint_in_previous_multiprocess_step = LAST_CHECKPOINT state.checkpoint.restore(resume_after=last_checkpoint_in_previous_multiprocess_step) # ensure all tables are in the pipeline @@ -1588,7 +1586,12 @@ def drop_breadcrumb(state: workflow.State, step_name, crumb, value=True): write_breadcrumbs(state, breadcrumbs) -def run_multiprocess(state: workflow.State, injectables): +def run_multiprocess( + state: workflow.State, + injectables, + shared_data_buffers=None, + skip_final_checkpoint=False, +): """ run the steps in run_list, possibly resuming after checkpoint specified by resume_after @@ -1616,6 +1619,13 @@ def run_multiprocess(state: workflow.State, injectables): annotated run_list (including prior run breadcrumbs if resuming) injectables : dict dict of values to inject in sub-processes + shared_data_buffers : dict, optional + Pre-allocated shared data buffers (skims, shadow pricing). If provided, + allocation and skim loading are skipped (useful for calibration loops + that call run_multiprocess repeatedly). + skip_final_checkpoint : bool, default False + If True, skip writing the final checkpoint at the end of the run. + Useful when the caller manages checkpoints externally. """ state.trace_memory_info("run_multiprocess.start") @@ -1645,60 +1655,61 @@ def find_breadcrumb(crumb, default=None): sharrow_enabled = state.settings.sharrow - # - allocate shared data - shared_data_buffers = {} - - state.trace_memory_info("allocate_shared_skim_buffer.before") - - t0 = tracing.print_elapsed_time() - if not sharrow_enabled: - shared_data_buffers.update(allocate_shared_skim_buffers(state)) - t0 = tracing.print_elapsed_time("allocate shared skim buffer", t0) - state.trace_memory_info("allocate_shared_skim_buffer.completed") - - # combine shared_skim_buffer and shared_shadow_pricing_buffer in shared_data_buffer - t0 = tracing.print_elapsed_time() - shared_data_buffers.update(allocate_shared_shadow_pricing_buffers(state)) - t0 = tracing.print_elapsed_time("allocate shared shadow_pricing buffer", t0) - state.trace_memory_info("allocate_shared_shadow_pricing_buffers.completed") - - # combine shared_shadow_pricing_buffers to pool choices across all processes - t0 = tracing.print_elapsed_time() - shared_data_buffers.update(allocate_shared_shadow_pricing_buffers_choice(state)) - t0 = tracing.print_elapsed_time("allocate shared shadow_pricing choice buffer", t0) - state.trace_memory_info("allocate_shared_shadow_pricing_buffers_choice.completed") - - start_time = time.time() - if sharrow_enabled: - shared_data_buffers["skim_dataset"] = "sh.Dataset:skim_dataset" - - # Loading skim_dataset must be done in the main process, not a subprocess, - # so that this min process can hold on to the shared memory and then cleanly - # release it on exit. - from . import flow, skim_dataset # make injectables known # noqa: F401 - - state.get_injectable("skim_dataset") - - tracing.print_elapsed_time("setup skim_dataset", t0) - state.trace_memory_info("skim_dataset.completed") - - # - mp_setup_skims - else: # not sharrow_enabled - if len(shared_data_buffers) > 0: - start_time = time.time() - run_sub_task( - state, - multiprocessing.Process( - target=mp_setup_skims, - name="mp_setup_skims", - args=(injectables,), - kwargs=shared_data_buffers, - ), - ) + # - allocate shared data (skip if pre-allocated buffers were provided) + if shared_data_buffers is None: + shared_data_buffers = {} + + state.trace_memory_info("allocate_shared_skim_buffer.before") + + t0 = tracing.print_elapsed_time() + if not sharrow_enabled: + shared_data_buffers.update(allocate_shared_skim_buffers(state)) + t0 = tracing.print_elapsed_time("allocate shared skim buffer", t0) + state.trace_memory_info("allocate_shared_skim_buffer.completed") + + # combine shared_skim_buffer and shared_shadow_pricing_buffer in shared_data_buffer + t0 = tracing.print_elapsed_time() + shared_data_buffers.update(allocate_shared_shadow_pricing_buffers(state)) + t0 = tracing.print_elapsed_time("allocate shared shadow_pricing buffer", t0) + state.trace_memory_info("allocate_shared_shadow_pricing_buffers.completed") + + # combine shared_shadow_pricing_buffers to pool choices across all processes + t0 = tracing.print_elapsed_time() + shared_data_buffers.update(allocate_shared_shadow_pricing_buffers_choice(state)) + t0 = tracing.print_elapsed_time("allocate shared shadow_pricing choice buffer", t0) + state.trace_memory_info("allocate_shared_shadow_pricing_buffers_choice.completed") + + start_time = time.time() + if sharrow_enabled: + shared_data_buffers["skim_dataset"] = "sh.Dataset:skim_dataset" + + # Loading skim_dataset must be done in the main process, not a subprocess, + # so that this min process can hold on to the shared memory and then cleanly + # release it on exit. + from . import flow, skim_dataset # make injectables known # noqa: F401 + + state.get_injectable("skim_dataset") + + tracing.print_elapsed_time("setup skim_dataset", t0) + state.trace_memory_info("skim_dataset.completed") + + # - mp_setup_skims + else: # not sharrow_enabled + if len(shared_data_buffers) > 0: + start_time = time.time() + run_sub_task( + state, + multiprocessing.Process( + target=mp_setup_skims, + name="mp_setup_skims", + args=(injectables,), + kwargs=shared_data_buffers, + ), + ) - tracing.print_elapsed_time("setup shared_data_buffers", t0) - state.trace_memory_info("mp_setup_skims.completed") - state.run.log_runtime("mp_setup_skims", start_time=start_time, force=True) + tracing.print_elapsed_time("setup shared_data_buffers", t0) + state.trace_memory_info("mp_setup_skims.completed") + state.run.log_runtime("mp_setup_skims", start_time=start_time, force=True) # - for each step in run list for step_info in run_list["multiprocess_steps"]: @@ -1769,7 +1780,7 @@ def find_breadcrumb(crumb, default=None): drop_breadcrumb(state, step_name, "coalesce") # add checkpoint with final tables even if not intermediate checkpointing - if not state.should_save_checkpoint(): + if not skip_final_checkpoint and not state.should_save_checkpoint(): state.checkpoint.restore(resume_after="_") state.checkpoint.add(FINAL_CHECKPOINT_NAME) state.checkpoint.close_store() @@ -2069,7 +2080,7 @@ def get_run_list(state: workflow.State): # remember there should always be a final checkpoint with same name as multiprocess_step name multiprocess_steps[istep][ "last_checkpoint_in_previous_multiprocess_step" - ] = (multiprocess_steps[istep - 1].get("name") if istep > 0 else None) + ] = (multiprocess_steps[istep - 1].get("name") if istep > 0 else LAST_CHECKPOINT) # - build individual step model lists based on starts starts.append(len(models)) # so last step gets remaining models in list From 6de9e4850a6ea8c0fe30a6ee63670a4398fc79f1 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 14 Jul 2026 15:54:58 -0500 Subject: [PATCH 24/96] MultiprocessStep creation fix --- activitysim/core/calibration.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index de21667436..1826f1dc82 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1565,18 +1565,24 @@ def _build_calibration_mp_steps( continue step_model_groups.setdefault(step_idx, []).append(model) - # Build new MultiprocessStep for each group + # Build new MultiprocessStep for each group. + # Some original steps (e.g. mp_initialize) omit num_processes, slice, + # and chunk_size — these default to None on MultiprocessStep and + # get_run_list() applies global defaults when they are absent. new_steps = [] for step_idx, step_models in step_model_groups.items(): orig_step = original_steps[step_idx] - new_step = MultiprocessStep( - name=f"calibration_{orig_step.name}", - begin=step_models[0], - num_processes=orig_step.num_processes, - slice=orig_step.slice, - chunk_size=orig_step.chunk_size, - ) - new_steps.append(new_step) + kwargs: dict[str, Any] = { + "name": f"calibration_{orig_step.name}", + "begin": step_models[0], + } + if orig_step.num_processes is not None: + kwargs["num_processes"] = orig_step.num_processes + if orig_step.slice is not None: + kwargs["slice"] = orig_step.slice + if orig_step.chunk_size is not None: + kwargs["chunk_size"] = orig_step.chunk_size + new_steps.append(MultiprocessStep(**kwargs)) return new_steps From de894c24021ea251ddbcaae016d7a3a74dad6fd7 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 14 Jul 2026 16:13:07 -0500 Subject: [PATCH 25/96] Alleged checkpoint fix --- activitysim/core/mp_tasks.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/activitysim/core/mp_tasks.py b/activitysim/core/mp_tasks.py index 6adf85f853..258b95450f 100644 --- a/activitysim/core/mp_tasks.py +++ b/activitysim/core/mp_tasks.py @@ -575,6 +575,18 @@ def apportion_pipeline(state: workflow.State, sub_proc_names, step_info): # for the subprocess pipelines, keep only the last row of checkpoints and patch the last checkpoint name checkpoints_df = checkpoints_df.tail(1).copy() + # Drop table columns not present at the restored checkpoint. This is + # necessary because get_inventory() reads from the on-disk store which + # may contain later checkpoints that added tables not yet created at the + # restore point (e.g. during calibration iteration re-runs). + extra_cols = [ + c + for c in checkpoints_df.columns + if c not in NON_TABLE_COLUMNS and c not in checkpointed_tables + ] + if extra_cols: + checkpoints_df = checkpoints_df.drop(columns=extra_cols) + # load all tables from pipeline checkpoint_name = multiprocess_step_name tables = {} From 3c7870094f0b14570045d6e83342d968ff588e2c Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 14 Jul 2026 16:24:00 -0500 Subject: [PATCH 26/96] Deduplicate checkpoints --- activitysim/core/workflow/checkpoint.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/activitysim/core/workflow/checkpoint.py b/activitysim/core/workflow/checkpoint.py index 7391e1c9b9..ec721a4790 100644 --- a/activitysim/core/workflow/checkpoint.py +++ b/activitysim/core/workflow/checkpoint.py @@ -181,7 +181,14 @@ def _get_store_checkpoint_from_named_checkpoint( if checkpoint_name == LAST_CHECKPOINT: checkpoint_name = cp_df.index[-1] try: - return cp_df.loc[checkpoint_name, table_name] + result = cp_df.loc[checkpoint_name, table_name] + # If checkpoint_name appears multiple times in the index (e.g. when + # run_simulation adds a final checkpoint with the same name as the + # apportion checkpoint), loc returns a Series. Take the last value + # which represents the most recent state. + if isinstance(result, pd.Series): + result = result.iloc[-1] + return result except KeyError: if checkpoint_name not in cp_df.index: raise CheckpointNameNotFoundError(checkpoint_name) From 9856ee65926ca56ab271b465a4c7ed5b9547f5e5 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 14 Jul 2026 16:45:24 -0500 Subject: [PATCH 27/96] Prevent resume_after setting --- activitysim/core/calibration.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 1826f1dc82..91a18a6175 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1436,7 +1436,11 @@ def _run_multiprocess_with_overrides( state.settings.models = models state.settings.multiprocess_steps = calibration_mp_steps - state.settings.resume_after = resume_after + # Always None: calibration manages pipeline state externally via + # _restore_parent_state_from_pipeline and checkpoint.add, so the MP + # system's breadcrumb-based resume logic must not be triggered. + # Apportion uses LAST_CHECKPOINT to read from the current pipeline state. + state.settings.resume_after = None try: injectables = _build_calibration_injectables(state) From fd7950ab69eaeda0f037d3adc3d6567f28e95a31 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 15 Jul 2026 10:10:35 -0500 Subject: [PATCH 28/96] More resume_after fixes --- activitysim/core/calibration.py | 1 + activitysim/core/mp_tasks.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 91a18a6175..5ebdfa12c0 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1449,6 +1449,7 @@ def _run_multiprocess_with_overrides( injectables, shared_data_buffers=shared_data_buffers, skip_final_checkpoint=True, + force_resume=True, ) finally: state.settings.models = original_models diff --git a/activitysim/core/mp_tasks.py b/activitysim/core/mp_tasks.py index 258b95450f..a96a317b9d 100644 --- a/activitysim/core/mp_tasks.py +++ b/activitysim/core/mp_tasks.py @@ -1603,6 +1603,7 @@ def run_multiprocess( injectables, shared_data_buffers=None, skip_final_checkpoint=False, + force_resume=False, ): """ run the steps in run_list, possibly resuming after checkpoint specified by resume_after @@ -1638,6 +1639,10 @@ def run_multiprocess( skip_final_checkpoint : bool, default False If True, skip writing the final checkpoint at the end of the run. Useful when the caller manages checkpoints externally. + force_resume : bool, default False + If True, all subprocess steps resume from LAST_CHECKPOINT regardless + of step_num. Use when the pipeline already has data that must be + preserved (e.g. calibration sub-runs). """ state.trace_memory_info("run_multiprocess.start") @@ -1755,6 +1760,12 @@ def find_breadcrumb(crumb, default=None): if not skip_phase("simulate"): resume_after = step_info.get("resume_after", None) + # When force_resume is set (e.g. calibration sub-runs), always + # resume from the last checkpoint so subprocesses don't discard + # existing pipeline data by starting fresh. + if resume_after is None and force_resume: + resume_after = LAST_CHECKPOINT + previously_completed = find_breadcrumb("completed", default=[]) completed = run_sub_simulations( From 97a29d0dd3848418f1068d4338ffa59885e0e9aa Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 16 Jul 2026 09:59:13 -0500 Subject: [PATCH 29/96] update existing table status --- activitysim/core/calibration.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 5ebdfa12c0..f980bbca2a 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1463,11 +1463,22 @@ def _restore_parent_state_from_pipeline(state: workflow.State) -> None: After a multiprocess run, the parent's in-memory state is stale. This loads the latest checkpoint from the pipeline store so that calibration expressions can evaluate against model outputs. + + All tables are explicitly re-checkpointed so that subsequent apportion + subprocesses can load them from a direct file path without relying on + checkpoint backtracking through potentially ambiguous checkpoint history. """ if state.checkpoint.store_is_open(): state.checkpoint.close_store() state.checkpoint.restore(resume_after="_") + # After restore, all tables are clean (status=False). Mark them dirty so + # the next checkpoint.add() writes them to disk at a known checkpoint name. + # This ensures apportion subprocesses find table files at a single, + # unambiguous checkpoint rather than needing to backtrack through history. + for table_name in list(state.existing_table_names): + state.existing_table_status[table_name] = True + def _initialize_mp_shared_resources(state: workflow.State) -> dict: """Allocate shared data buffers (skims, shadow pricing) once for reuse. From dbdff183c8daf9c798633980bbedabaa6a3c58ef Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 16 Jul 2026 10:26:17 -0500 Subject: [PATCH 30/96] Checkpoint rename --- activitysim/core/calibration.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index f980bbca2a..b79e5ab8e4 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1585,11 +1585,13 @@ def _build_calibration_mp_steps( # Some original steps (e.g. mp_initialize) omit num_processes, slice, # and chunk_size — these default to None on MultiprocessStep and # get_run_list() applies global defaults when they are absent. + # Step names include the first model to ensure uniqueness across multiple + # intermediate runs that draw from the same original step. new_steps = [] for step_idx, step_models in step_model_groups.items(): orig_step = original_steps[step_idx] kwargs: dict[str, Any] = { - "name": f"calibration_{orig_step.name}", + "name": f"calibration_{orig_step.name}_{step_models[0]}", "begin": step_models[0], } if orig_step.num_processes is not None: From d2dd7ae49d8adc4e49314acd815c3b18654206b9 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 16 Jul 2026 10:47:40 -0500 Subject: [PATCH 31/96] Store last checkpoint --- activitysim/core/calibration.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index b79e5ab8e4..747a0b7edc 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1405,6 +1405,11 @@ def _run_in_configured_mode( # Restore it into the parent process state so tables are accessible # for calibration expression evaluation. _restore_parent_state_from_pipeline(state) + # Add a checkpoint named after the last model so that model-name + # references (e.g. _prior_step_name, resume_after on global_iter > 1) + # resolve correctly. Without this, only the step-level coalesce name + # exists in the pipeline. + state.checkpoint.add(models[-1]) return state.run( From 510b5d49aff8e6a203607f9c4660808621d5d2a7 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 16 Jul 2026 12:15:51 -0500 Subject: [PATCH 32/96] Blacken! It works! --- activitysim/core/calibration.py | 35 ++++++------- activitysim/core/mp_tasks.py | 14 +++-- .../auto_ownership_calib_helper.py | 19 +++++-- .../tour_mode_choice_calib_helper.py | 13 +++-- .../workplace_location_calib_helper.py | 51 ++++++++++++------- 5 files changed, 84 insertions(+), 48 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 747a0b7edc..a0c1651eb0 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -28,7 +28,7 @@ logger = logging.getLogger("calibration") plt.style.use("seaborn-v0_8-darkgrid") -matplotlib.use('Agg') # Forces non-interactive background rendering +matplotlib.use("Agg") # Forces non-interactive background rendering CALIBRATION_SETTINGS_FILE_NAME = "calibration.yaml" CALIBRATION_OUTPUT_DIR = "calibration" @@ -415,8 +415,7 @@ def _calibrate_component( for component_iter in range(1, component_settings.submodel_max_iterations + 1): component_iterations = component_iter run_model_name = ( - f"{component_name}.component_i{component_iter};" - f"global_i{global_iter}" + f"{component_name}.component_i{component_iter};" f"global_i{global_iter}" ) # Re-run only this component from its prior checkpoint so model values # reflect the current candidate coefficients for this component. @@ -433,7 +432,9 @@ def _calibrate_component( else: state.run(models=[run_model_name], resume_after=prior_step) - eval_context = _build_expression_context(state, helper_symbols, component_name, component_settings) + eval_context = _build_expression_context( + state, helper_symbols, component_name, component_settings + ) ( row_records, @@ -951,9 +952,10 @@ def _append_iteration_records( _append_csv(df, global_path) # Also write component-local iteration history - component_path = _component_output_dir(state, component_name) / Path( - CALIBRATION_ITERATION_FILE - ).name + component_path = ( + _component_output_dir(state, component_name) + / Path(CALIBRATION_ITERATION_FILE).name + ) _append_csv(df, component_path) @@ -977,9 +979,7 @@ def _append_csv(df: pd.DataFrame, path: Path) -> None: def _component_output_dir(state: workflow.State, component_name: str) -> Path: """Return output/calibration/ and ensure it exists.""" - component_dir = state.get_output_file_path( - f"calibration/{component_name}" - ) + component_dir = state.get_output_file_path(f"calibration/{component_name}") os.makedirs(component_dir, exist_ok=True) return component_dir @@ -995,7 +995,8 @@ def _write_component_plots(state: workflow.State, component_name: str) -> None: n_sets = math.ceil(len(coefs) / MAX_COEFFS_IN_GRAPH) for coef_set in range(n_sets): set_coefs = coefs[ - coef_set * MAX_COEFFS_IN_GRAPH : min( + coef_set + * MAX_COEFFS_IN_GRAPH : min( len(coefs), (coef_set + 1) * MAX_COEFFS_IN_GRAPH ) ] @@ -1033,7 +1034,7 @@ def _plot_coefficient_progress( ax = ( recs[recs.index.get_level_values("coefficient").isin(set_coefs)] .next_coefficient.unstack("coefficient") - .plot(figsize=(10,5)) + .plot(figsize=(10, 5)) ) ax.xaxis.set_label_text("Component iteration") ax.yaxis.set_label_text("Coefficient value") @@ -1051,9 +1052,9 @@ def _component_last_records(recs: pd.DataFrame, set_coefs: list[str]) -> pd.Data filtered = recs[recs.index.get_level_values("coefficient").isin(set_coefs)] last_global = filtered.index.get_level_values("global_iter")[-1] last_comp = filtered.loc[last_global].index.get_level_values("component_iter")[-1] - return filtered.xs((last_global, last_comp), level=("global_iter", "component_iter"))[ - ["target_value", "model_value"] - ] + return filtered.xs( + (last_global, last_comp), level=("global_iter", "component_iter") + )[["target_value", "model_value"]] def _plot_component_values( @@ -1064,7 +1065,7 @@ def _plot_component_values( ) -> None: """Plot final target/model component values for one coefficient subset.""" component_dir = _component_output_dir(state, component_name) - ax = last_records.plot.bar(figsize=(10,5)) + ax = last_records.plot.bar(figsize=(10, 5)) ax.xaxis.set_tick_params(rotation=45) ax.xaxis.set_label_text("Component value") plt.tight_layout() @@ -1080,7 +1081,7 @@ def _plot_component_pct_change( ) -> None: """Plot final percent difference for one coefficient subset.""" component_dir = _component_output_dir(state, component_name) - fig, ax = plt.subplots(figsize=(10,5)) + fig, ax = plt.subplots(figsize=(10, 5)) pct_diff = last_records.diff(axis=1).model_value / last_records.target_value ax = pct_diff.plot.bar(ax=ax) ax.xaxis.set_tick_params(rotation=45) diff --git a/activitysim/core/mp_tasks.py b/activitysim/core/mp_tasks.py index a96a317b9d..4e881211f2 100644 --- a/activitysim/core/mp_tasks.py +++ b/activitysim/core/mp_tasks.py @@ -1693,8 +1693,12 @@ def find_breadcrumb(crumb, default=None): # combine shared_shadow_pricing_buffers to pool choices across all processes t0 = tracing.print_elapsed_time() shared_data_buffers.update(allocate_shared_shadow_pricing_buffers_choice(state)) - t0 = tracing.print_elapsed_time("allocate shared shadow_pricing choice buffer", t0) - state.trace_memory_info("allocate_shared_shadow_pricing_buffers_choice.completed") + t0 = tracing.print_elapsed_time( + "allocate shared shadow_pricing choice buffer", t0 + ) + state.trace_memory_info( + "allocate_shared_shadow_pricing_buffers_choice.completed" + ) start_time = time.time() if sharrow_enabled: @@ -2103,7 +2107,11 @@ def get_run_list(state: workflow.State): # remember there should always be a final checkpoint with same name as multiprocess_step name multiprocess_steps[istep][ "last_checkpoint_in_previous_multiprocess_step" - ] = (multiprocess_steps[istep - 1].get("name") if istep > 0 else LAST_CHECKPOINT) + ] = ( + multiprocess_steps[istep - 1].get("name") + if istep > 0 + else LAST_CHECKPOINT + ) # - build individual step model lists based on starts starts.append(len(models)) # so last step gets remaining models in list diff --git a/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py index e5640fb6f3..3e6a882fb8 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_calib_helper.py @@ -4,12 +4,19 @@ SURVEY_DATA_FOLDER = "activitysim/examples/example_estimation/data_sf/survey_data" + def report_auto_ownership(context): model_hhs = context["households"] - survey_hhs = pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_households.csv")) + survey_hhs = pd.read_csv( + os.path.join(SURVEY_DATA_FOLDER, "override_households.csv") + ) - model_summary = model_hhs.auto_ownership.value_counts(normalize=True).sort_index().fillna(0) - survey_summary = survey_hhs.auto_ownership.value_counts(normalize=True).sort_index().fillna(0) + model_summary = ( + model_hhs.auto_ownership.value_counts(normalize=True).sort_index().fillna(0) + ) + survey_summary = ( + survey_hhs.auto_ownership.value_counts(normalize=True).sort_index().fillna(0) + ) summary_df = ( pd.DataFrame({"model": model_summary, "survey": survey_summary}) .reset_index() @@ -22,5 +29,7 @@ def report_auto_ownership(context): plt.xlabel("Number of Autos") plt.ylabel("Proportion of Households") plt.legend(title="Data Source") - plt.savefig(os.path.join(context["component_output_dir"], "auto_ownership_comparison.png")) - plt.close() \ No newline at end of file + plt.savefig( + os.path.join(context["component_output_dir"], "auto_ownership_comparison.png") + ) + plt.close() diff --git a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py index 4fd8b31ca7..b647ef85bf 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_calib_helper.py @@ -4,13 +4,16 @@ SURVEY_DATA_FOLDER = "activitysim/examples/example_estimation/data_sf/survey_data" + def report_tour_mode_choice(context): model_tours = context["tours"] survey_tours = None survey_tours = pd.read_csv(os.path.join(SURVEY_DATA_FOLDER, "override_tours.csv")) - model_summary = model_tours.tour_mode.value_counts(normalize=True).sort_index().fillna(0) - if 'tour_weight' in survey_tours.columns: + model_summary = ( + model_tours.tour_mode.value_counts(normalize=True).sort_index().fillna(0) + ) + if "tour_weight" in survey_tours.columns: survey_summary = survey_tours.groupby("tour_mode").tour_weight.sum() survey_summary = survey_summary / survey_tours.tour_weight.sum() else: @@ -28,5 +31,7 @@ def report_tour_mode_choice(context): plt.xlabel("Tour Mode") plt.ylabel("Proportion of Tours") plt.legend(title="Data Source") - plt.savefig(os.path.join(context["component_output_dir"], "tour_mode_choice_comparison.png")) - plt.close() \ No newline at end of file + plt.savefig( + os.path.join(context["component_output_dir"], "tour_mode_choice_comparison.png") + ) + plt.close() diff --git a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py index 207af38bc5..b9926eb0a1 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py @@ -13,10 +13,13 @@ SURVEY_DATA_FOLDER = "activitysim/examples/example_estimation/data_sf/survey_data" + def compute_distances(context, origins, destinations): # Compute distances between origins and destinations using the network level of service # using non-time-dependent DIST skim - distances = context["skim_dict"].lookup(origins.clip(upper=24), destinations.clip(upper=24), 'DIST') + distances = context["skim_dict"].lookup( + origins.clip(upper=24), destinations.clip(upper=24), "DIST" + ) # time dependent example # distances = skim_dict.lookup_3d(origins, destinations, 'AM', 'SOV_DIST') return distances @@ -39,7 +42,9 @@ def _survey_worker_distances(context): """Compute survey worker distances once and reuse across calibration rows.""" survey_persons = _survey_persons() survey_workers = survey_persons[survey_persons["workplace_zone_id"] > 0] - survey_home_zone_ids = _survey_households().set_index("household_id")["home_zone_id"] + survey_home_zone_ids = _survey_households().set_index("household_id")[ + "home_zone_id" + ] survey_home_zone_ids = survey_workers["household_id"].map(survey_home_zone_ids) survey_workplace_zone_ids = survey_workers["workplace_zone_id"] return compute_distances(context, survey_home_zone_ids, survey_workplace_zone_ids) @@ -47,11 +52,11 @@ def _survey_worker_distances(context): def summarize_model(context, min_dist=1, max_dist=2): """Summarize the model results for workplaces within the specified distance range.""" - persons = context['persons'] - workers = persons[persons['workplace_zone_id'] > 0] - home_zone_ids = workers['home_zone_id'] - workplace_zone_ids = workers['workplace_zone_id'] - + persons = context["persons"] + workers = persons[persons["workplace_zone_id"] > 0] + home_zone_ids = workers["home_zone_id"] + workplace_zone_ids = workers["workplace_zone_id"] + distances = compute_distances(context, home_zone_ids, workplace_zone_ids) # Filter distances within the specified range @@ -61,6 +66,7 @@ def summarize_model(context, min_dist=1, max_dist=2): share = len(filtered_distances) / len(distances) if len(distances) > 0 else 0 return share + def summarize_survey(context, min_dist=1, max_dist=2): """Summarize the survey results for workplaces within the specified distance range.""" @@ -73,25 +79,32 @@ def summarize_survey(context, min_dist=1, max_dist=2): share = len(filtered_distances) / len(distances) if len(distances) > 0 else 0 return share + def report_workplace_location(context): """Workplace location distance frequency plot comparing model results with observed data.""" print("summarizing workplace location model") model_persons = context["persons"] - model_workers = model_persons[model_persons['workplace_zone_id'] > 0] - model_home_zone_ids = model_workers['home_zone_id'] - model_workplace_zone_ids = model_workers['workplace_zone_id'] + model_workers = model_persons[model_persons["workplace_zone_id"] > 0] + model_home_zone_ids = model_workers["home_zone_id"] + model_workplace_zone_ids = model_workers["workplace_zone_id"] - model_distances = compute_distances(context, model_home_zone_ids, model_workplace_zone_ids) + model_distances = compute_distances( + context, model_home_zone_ids, model_workplace_zone_ids + ) survey_distances = _survey_worker_distances(context) - - # Here you can add code to compare model_distances and survey_distances, + + # Here you can add code to compare model_distances and survey_distances, # for example by plotting histograms or computing summary statistics. - plt.hist(model_distances, bins=20, density=True, alpha=0.5, label='Model') - plt.hist(survey_distances, bins=20, density=True, alpha=0.5, label='Survey') - plt.xlabel('Distance') - plt.ylabel('Frequency') + plt.hist(model_distances, bins=20, density=True, alpha=0.5, label="Model") + plt.hist(survey_distances, bins=20, density=True, alpha=0.5, label="Survey") + plt.xlabel("Distance") + plt.ylabel("Frequency") plt.legend() # component_output_dir set in the evaluation context - plt.savefig(os.path.join(context["component_output_dir"], 'workplace_location_comparison.png')) - plt.close() \ No newline at end of file + plt.savefig( + os.path.join( + context["component_output_dir"], "workplace_location_comparison.png" + ) + ) + plt.close() From 0ee33d374201c4e16aadd8d510b42d274e0059bc Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 16 Jul 2026 14:53:35 -0500 Subject: [PATCH 33/96] Set calibration coefficients to zero --- .../configs/auto_ownership_coefficients.csv | 8 +- .../configs/tour_mode_choice_coefficients.csv | 120 +++++++++--------- .../workplace_location_coefficients.csv | 8 +- 3 files changed, 68 insertions(+), 68 deletions(-) diff --git a/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv b/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv index d8c072de3b..b285538d7f 100644 --- a/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv +++ b/activitysim/examples/prototype_mtc/configs/auto_ownership_coefficients.csv @@ -66,7 +66,7 @@ coef_cars3_drivers_3,5.5131,F coef_cars2_drivers_4_up,6.3662,F coef_cars3_drivers_4_up,8.5148,F coef_cars4_drivers_4_up,9.5807,F -coef_calib_auto_0,1,T -coef_calib_auto_2,1,T -coef_calib_auto_3,1,T -coef_calib_auto_4,1,T +coef_calib_auto_0,0,T +coef_calib_auto_2,0,T +coef_calib_auto_3,0,T +coef_calib_auto_4,0,T diff --git a/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv b/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv index afe19ff500..394fd090d8 100644 --- a/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv +++ b/activitysim/examples/prototype_mtc/configs/tour_mode_choice_coefficients.csv @@ -306,63 +306,63 @@ drive_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,0.525,F drive_transit_CBD_ASC_school_univ,0.672,F drive_transit_CBD_ASC_work,1.1,F drive_transit_CBD_ASC_atwork,0.564,F -coef_calib_DRIVEALONEPAY_zero_auto,1,T -coef_calib_SHARED2FREE_zero_auto,1,T -coef_calib_SHARED2PAY_zero_auto,1,T -coef_calib_SHARED3FREE_zero_auto,1,T -coef_calib_SHARED3PAY_zero_auto,1,T -coef_calib_WALK_zero_auto,1,T -coef_calib_BIKE_zero_auto,1,T -coef_calib_WALK_LOC_zero_auto,1,T -coef_calib_WALK_LRF_zero_auto,1,T -coef_calib_WALK_EXP_zero_auto,1,T -coef_calib_WALK_HVY_zero_auto,1,T -coef_calib_WALK_COM_zero_auto,1,T -coef_calib_DRIVE_LOC_zero_auto,1,T -coef_calib_DRIVE_LRF_zero_auto,1,T -coef_calib_DRIVE_EXP_zero_auto,1,T -coef_calib_DRIVE_HVY_zero_auto,1,T -coef_calib_DRIVE_COM_zero_auto,1,T -coef_calib_TAXI_zero_auto,1,T -coef_calib_TNC_SINGLE_zero_auto,1,T -coef_calib_TNC_SHARED_zero_auto,1,T -coef_calib_DRIVEALONEPAY_auto_insuff,1,T -coef_calib_SHARED2FREE_auto_insuff,1,T -coef_calib_SHARED2PAY_auto_insuff,1,T -coef_calib_SHARED3FREE_auto_insuff,1,T -coef_calib_SHARED3PAY_auto_insuff,1,T -coef_calib_WALK_auto_insuff,1,T -coef_calib_BIKE_auto_insuff,1,T -coef_calib_WALK_LOC_auto_insuff,1,T -coef_calib_WALK_LRF_auto_insuff,1,T -coef_calib_WALK_EXP_auto_insuff,1,T -coef_calib_WALK_HVY_auto_insuff,1,T -coef_calib_WALK_COM_auto_insuff,1,T -coef_calib_DRIVE_LOC_auto_insuff,1,T -coef_calib_DRIVE_LRF_auto_insuff,1,T -coef_calib_DRIVE_EXP_auto_insuff,1,T -coef_calib_DRIVE_HVY_auto_insuff,1,T -coef_calib_DRIVE_COM_auto_insuff,1,T -coef_calib_TAXI_auto_insuff,1,T -coef_calib_TNC_SINGLE_auto_insuff,1,T -coef_calib_TNC_SHARED_auto_insuff,1,T -coef_calib_DRIVEALONEPAY_auto_suff,1,T -coef_calib_SHARED2FREE_auto_suff,1,T -coef_calib_SHARED2PAY_auto_suff,1,T -coef_calib_SHARED3FREE_auto_suff,1,T -coef_calib_SHARED3PAY_auto_suff,1,T -coef_calib_WALK_auto_suff,1,T -coef_calib_BIKE_auto_suff,1,T -coef_calib_WALK_LOC_auto_suff,1,T -coef_calib_WALK_LRF_auto_suff,1,T -coef_calib_WALK_EXP_auto_suff,1,T -coef_calib_WALK_HVY_auto_suff,1,T -coef_calib_WALK_COM_auto_suff,1,T -coef_calib_DRIVE_LOC_auto_suff,1,T -coef_calib_DRIVE_LRF_auto_suff,1,T -coef_calib_DRIVE_EXP_auto_suff,1,T -coef_calib_DRIVE_HVY_auto_suff,1,T -coef_calib_DRIVE_COM_auto_suff,1,T -coef_calib_TAXI_auto_suff,1,T -coef_calib_TNC_SINGLE_auto_suff,1,T -coef_calib_TNC_SHARED_auto_suff,1,T +coef_calib_DRIVEALONEPAY_zero_auto,0,T +coef_calib_SHARED2FREE_zero_auto,0,T +coef_calib_SHARED2PAY_zero_auto,0,T +coef_calib_SHARED3FREE_zero_auto,0,T +coef_calib_SHARED3PAY_zero_auto,0,T +coef_calib_WALK_zero_auto,0,T +coef_calib_BIKE_zero_auto,0,T +coef_calib_WALK_LOC_zero_auto,0,T +coef_calib_WALK_LRF_zero_auto,0,T +coef_calib_WALK_EXP_zero_auto,0,T +coef_calib_WALK_HVY_zero_auto,0,T +coef_calib_WALK_COM_zero_auto,0,T +coef_calib_DRIVE_LOC_zero_auto,0,T +coef_calib_DRIVE_LRF_zero_auto,0,T +coef_calib_DRIVE_EXP_zero_auto,0,T +coef_calib_DRIVE_HVY_zero_auto,0,T +coef_calib_DRIVE_COM_zero_auto,0,T +coef_calib_TAXI_zero_auto,0,T +coef_calib_TNC_SINGLE_zero_auto,0,T +coef_calib_TNC_SHARED_zero_auto,0,T +coef_calib_DRIVEALONEPAY_auto_insuff,0,T +coef_calib_SHARED2FREE_auto_insuff,0,T +coef_calib_SHARED2PAY_auto_insuff,0,T +coef_calib_SHARED3FREE_auto_insuff,0,T +coef_calib_SHARED3PAY_auto_insuff,0,T +coef_calib_WALK_auto_insuff,0,T +coef_calib_BIKE_auto_insuff,0,T +coef_calib_WALK_LOC_auto_insuff,0,T +coef_calib_WALK_LRF_auto_insuff,0,T +coef_calib_WALK_EXP_auto_insuff,0,T +coef_calib_WALK_HVY_auto_insuff,0,T +coef_calib_WALK_COM_auto_insuff,0,T +coef_calib_DRIVE_LOC_auto_insuff,0,T +coef_calib_DRIVE_LRF_auto_insuff,0,T +coef_calib_DRIVE_EXP_auto_insuff,0,T +coef_calib_DRIVE_HVY_auto_insuff,0,T +coef_calib_DRIVE_COM_auto_insuff,0,T +coef_calib_TAXI_auto_insuff,0,T +coef_calib_TNC_SINGLE_auto_insuff,0,T +coef_calib_TNC_SHARED_auto_insuff,0,T +coef_calib_DRIVEALONEPAY_auto_suff,0,T +coef_calib_SHARED2FREE_auto_suff,0,T +coef_calib_SHARED2PAY_auto_suff,0,T +coef_calib_SHARED3FREE_auto_suff,0,T +coef_calib_SHARED3PAY_auto_suff,0,T +coef_calib_WALK_auto_suff,0,T +coef_calib_BIKE_auto_suff,0,T +coef_calib_WALK_LOC_auto_suff,0,T +coef_calib_WALK_LRF_auto_suff,0,T +coef_calib_WALK_EXP_auto_suff,0,T +coef_calib_WALK_HVY_auto_suff,0,T +coef_calib_WALK_COM_auto_suff,0,T +coef_calib_DRIVE_LOC_auto_suff,0,T +coef_calib_DRIVE_LRF_auto_suff,0,T +coef_calib_DRIVE_EXP_auto_suff,0,T +coef_calib_DRIVE_HVY_auto_suff,0,T +coef_calib_DRIVE_COM_auto_suff,0,T +coef_calib_TAXI_auto_suff,0,T +coef_calib_TNC_SINGLE_auto_suff,0,T +coef_calib_TNC_SHARED_auto_suff,0,T diff --git a/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv b/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv index 28be9b2e5a..2c3f64c5ce 100644 --- a/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv +++ b/activitysim/examples/prototype_mtc/configs/workplace_location_coefficients.csv @@ -7,7 +7,7 @@ coef_dist_15_up,-0.0917,F coef_dist_0_5_high,0.15,F coef_dist_5_up_high,0.02,F coef_mode_logsum,0.3,F -coef_calib_dist_0_2,1.0,T -coef_calib_dist_2_5,1.0,T -coef_calib_dist_5_15,1.0,T -coef_calib_dist_15_up,1.0,T +coef_calib_dist_0_2,0,T +coef_calib_dist_2_5,0,T +coef_calib_dist_5_15,0,T +coef_calib_dist_15_up,0,T From fcfcb3c9a68507182f8fe27a73cc969a6a3be31d Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 16 Jul 2026 15:01:27 -0500 Subject: [PATCH 34/96] Add override CI test coefficients for calibration --- .../auto_ownership_coefficients.csv | 72 ++++ .../tour_mode_choice_coefficients.csv | 368 ++++++++++++++++++ .../workplace_location_coefficients.csv | 13 + 3 files changed, 453 insertions(+) create mode 100644 activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_coefficients.csv create mode 100644 activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_coefficients.csv create mode 100644 activitysim/examples/prototype_mtc/configs_calibration/workplace_location_coefficients.csv diff --git a/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_coefficients.csv b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_coefficients.csv new file mode 100644 index 0000000000..b285538d7f --- /dev/null +++ b/activitysim/examples/prototype_mtc/configs_calibration/auto_ownership_coefficients.csv @@ -0,0 +1,72 @@ +coefficient_name,value,constrain +coef_cars1_drivers_2,0,T +coef_cars1_drivers_3,0,T +coef_cars1_persons_16_17,0,T +coef_cars234_asc_marin,0,T +coef_cars1_persons_25_34,0,T +coef_cars1_num_workers_clip_3,0,T +coef_cars1_hh_income_30_up,0,T +coef_cars1_density_0_10_no_workers,0,T +coef_cars1_density_10_up_workers,-0.0152,F +coef_retail_non_motor,-0.03,T +coef_cars4_asc,-5.313,F +coef_cars3_asc,-3.2502,F +coef_cars34_persons_16_17,-1.7313,F +coef_cars2_asc,-1.0846,F +coef_cars34_persons_18_24,-1.0107,F +coef_cars2_persons_18_24,-1.0095,F +coef_cars2_persons_16_17,-0.881,F +coef_cars34_persons_25_34,-0.8596,F +coef_cars1_asc_county,-0.566,F +coef_retail_transit_workers,-0.5117,F +coef_cars2_persons_25_34,-0.4849,F +coef_cars2_asc_county,-0.4429,F +coef_cars1_persons_18_24,-0.4087,F +coef_cars34_density_0_10_no_workers,-0.3654,F +coef_retail_transit_no_workers,-0.3053,F +coef_cars1_asc_marin,-0.2434,F +coef_cars34_asc_county,-0.2372,F +coef_cars2_density_0_10_no_workers,-0.2028,F +coef_cars34_density_10_up_no_workers,-0.1766,F +coef_cars2_density_10_up_no_workers,-0.1106,F +coef_cars2_density_10_up_workers,-0.1106,F +coef_cars1_density_10_up_no_workers,-0.0152,F +coef_cars2_hh_income_30_up,0.0083,F +coef_cars3_hh_income_30_up,0.011,F +coef_cars4_hh_income_30_up,0.0147,F +coef_cars1_presence_children_5_17,0.0158,F +coef_cars1_hh_income_0_30k,0.0383,F +coef_cars2_hh_income_0_30k,0.054,F +coef_cars3_hh_income_0_30k,0.0559,F +coef_cars4_hh_income_0_30k,0.0619,F +coef_retail_auto_no_workers,0.0626,F +coef_cars34_asc_san_francisco,0.1458,F +coef_retail_auto_workers,0.1646,F +coef_cars2_presence_children_5_17,0.2936,F +coef_cars2_num_workers_clip_3,0.2936,F +coef_cars1_presence_children_0_4,0.3669,F +coef_cars1_asc_san_francisco,0.4259,F +coef_cars2_asc_san_francisco,0.4683,F +coef_cars1_auto_time_saving_per_worker,0.4707,F +coef_cars34_presence_children_5_17,0.4769,F +coef_cars3_auto_time_saving_per_worker,0.5705,F +coef_cars2_auto_time_saving_per_worker,0.6142,F +coef_cars3_num_workers_clip_3,0.6389,F +coef_cars234_presence_children_0_4,0.7627,F +coef_cars4_auto_time_saving_per_worker,0.7693,F +coef_cars4_num_workers_clip_3,0.8797,F +coef_cars1_asc,1.1865,F +coef_cars1_drivers_4_up,2.0107,F +coef_cars4_drivers_2,2.6616,F +coef_cars2_drivers_2,3.0773,F +coef_cars3_drivers_2,3.1962,F +coef_cars2_drivers_3,3.5401,F +coef_cars4_drivers_3,5.208,F +coef_cars3_drivers_3,5.5131,F +coef_cars2_drivers_4_up,6.3662,F +coef_cars3_drivers_4_up,8.5148,F +coef_cars4_drivers_4_up,9.5807,F +coef_calib_auto_0,0,T +coef_calib_auto_2,0,T +coef_calib_auto_3,0,T +coef_calib_auto_4,0,T diff --git a/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_coefficients.csv b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_coefficients.csv new file mode 100644 index 0000000000..394fd090d8 --- /dev/null +++ b/activitysim/examples/prototype_mtc/configs_calibration/tour_mode_choice_coefficients.csv @@ -0,0 +1,368 @@ +coefficient_name,value,constrain +coef_one,1,T +coef_nest_root,1,T +coef_nest_AUTO,0.72,T +coef_nest_AUTO_DRIVEALONE,0.35,T +coef_nest_AUTO_SHAREDRIDE2,0.35,T +coef_nest_AUTO_SHAREDRIDE3,0.35,T +coef_nest_NONMOTORIZED,0.72,T +coef_nest_TRANSIT,0.72,T +coef_nest_TRANSIT_WALKACCESS,0.5,T +coef_nest_TRANSIT_DRIVEACCESS,0.5,T +coef_nest_RIDEHAIL,0.36,T +coef_ivt_eatout_escort_othdiscr_othmaint_shopping_social,-0.0175,F +coef_ivt_school_univ,-0.0224,F +coef_ivt_work,-0.0134,F +coef_ivt_atwork,-0.0188,F +coef_topology_walk_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,15,F +coef_topology_walk_multiplier_atwork,7.5,F +coef_topology_bike_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,20,F +coef_topology_bike_multiplier_atwork,10,F +coef_topology_trn_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_work,2.2,F +coef_topology_trn_multiplier_atwork,2,F +coef_age1619_da_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work,0,F +coef_age1619_da_multiplier_school_univ,-1.3813,F +coef_age1619_da_multiplier_atwork,0.0032336,F +coef_age010_trn_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work,0,F +coef_age010_trn_multiplier_school_univ,-1.5548,F +coef_age010_trn_multiplier_atwork,0.000722,F +coef_age16p_sr_multiplier_eatout_escort_othdiscr_othmaint_shopping_social,-1.366,F +coef_age16p_sr_multiplier_school_univ_work_atwork,0,F +coef_hhsize1_sr_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_atwork,0,F +coef_hhsize1_sr_multiplier_work,-0.734588,F +coef_hhsize2_sr_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work_atwork,0,F +coef_hhsize2_sr_multiplier_school_univ,-0.6359,F +walk_ASC_no_auto_eatout,5.1251173,F +walk_ASC_no_auto_escort,2.8012068,F +walk_ASC_no_auto_othdiscr,3.2665946,F +walk_ASC_no_auto_othmaint,1.287299,F +walk_ASC_no_auto_school,18.414557,F +walk_ASC_no_auto_shopping,2.3768773,F +walk_ASC_no_auto_social,1.8680915,F +walk_ASC_no_auto_univ,6.408967,F +walk_ASC_no_auto_work,5.7672157,F +walk_ASC_no_auto_atwork,6.669213,F +walk_ASC_auto_deficient_eatout,3.274605,F +walk_ASC_auto_deficient_escort,-0.90204656,F +walk_ASC_auto_deficient_othdiscr,2.2494075,F +walk_ASC_auto_deficient_othmaint,1.3690404,F +walk_ASC_auto_deficient_school,3.2573624,F +walk_ASC_auto_deficient_shopping,2.2701733,F +walk_ASC_auto_deficient_social,2.870184,F +walk_ASC_auto_deficient_univ,4.50591,F +walk_ASC_auto_deficient_work,2.4010417,F +walk_ASC_auto_deficient_atwork,0.92546093,F +walk_ASC_auto_sufficient_eatout,1.5516903,F +walk_ASC_auto_sufficient_escort,-0.8116066,F +walk_ASC_auto_sufficient_othdiscr,1.2633476,F +walk_ASC_auto_sufficient_othmaint,0.7999634,F +walk_ASC_auto_sufficient_school,0.6476856,F +walk_ASC_auto_sufficient_shopping,0.7312663,F +walk_ASC_auto_sufficient_social,1.7072186,F +walk_ASC_auto_sufficient_univ,1.0607665,F +walk_ASC_auto_sufficient_work,0.053265337,F +walk_ASC_auto_sufficient_atwork,0.677216,F +bike_ASC_no_auto_eatout,0.86807096,F +bike_ASC_no_auto_escort,-0.716212,F +bike_ASC_no_auto_othdiscr,-0.3764232,F +bike_ASC_no_auto_othmaint,1.5394334,F +bike_ASC_no_auto_school,12.098735,F +bike_ASC_no_auto_shopping,0.8341555,F +bike_ASC_no_auto_social,0.02058321,F +bike_ASC_no_auto_univ,4.2945156,F +bike_ASC_no_auto_work,3.1940088,F +bike_ASC_no_auto_atwork,-0.90725845,F +bike_ASC_auto_deficient_eatout,-1.5691106,F +bike_ASC_auto_deficient_escort,-4.527928,F +bike_ASC_auto_deficient_othdiscr,-0.09246834,F +bike_ASC_auto_deficient_othmaint,-1.5184649,F +bike_ASC_auto_deficient_school,-0.5280678,F +bike_ASC_auto_deficient_shopping,-0.87584466,F +bike_ASC_auto_deficient_social,0.6345214,F +bike_ASC_auto_deficient_univ,-0.669235,F +bike_ASC_auto_deficient_work,0.25318968,F +bike_ASC_auto_deficient_atwork,-0.8074083,F +bike_ASC_auto_sufficient_eatout,-1.2003471,F +bike_ASC_auto_sufficient_escort,-5.0631084,F +bike_ASC_auto_sufficient_othdiscr,-1.0714597,F +bike_ASC_auto_sufficient_othmaint,-2.8083024,F +bike_ASC_auto_sufficient_school,-2.1134686,F +bike_ASC_auto_sufficient_shopping,-2.5662103,F +bike_ASC_auto_sufficient_social,-1.368071,F +bike_ASC_auto_sufficient_univ,-1.9397832,F +bike_ASC_auto_sufficient_work,-1.5800232,F +bike_ASC_auto_sufficient_atwork,15.72017,F +sr2_ASC_no_auto_all,0,F +sr2_ASC_auto_deficient_eatout,0.5882345,F +sr2_ASC_auto_deficient_escort,0,F +sr2_ASC_auto_deficient_othdiscr,0.6601513,F +sr2_ASC_auto_deficient_othmaint,0.2621527,F +sr2_ASC_auto_deficient_school,0.12474365,F +sr2_ASC_auto_deficient_shopping,0.24409756,F +sr2_ASC_auto_deficient_social,1.8558528,F +sr2_ASC_auto_deficient_univ,-1.6922346,F +sr2_ASC_auto_deficient_work,-0.33803123,F +sr2_ASC_auto_deficient_atwork,-2.1102421,F +sr2_ASC_auto_sufficient_eatout,0.86280555,F +sr2_ASC_auto_sufficient_escort,0,F +sr2_ASC_auto_sufficient_othdiscr,0.49684617,F +sr2_ASC_auto_sufficient_othmaint,0.25817883,F +sr2_ASC_auto_sufficient_school,-1.6062657,F +sr2_ASC_auto_sufficient_shopping,0.19770707,F +sr2_ASC_auto_sufficient_social,0.5236025,F +sr2_ASC_auto_sufficient_univ,-1.859427,F +sr2_ASC_auto_sufficient_work,-1.0857458,F +sr2_ASC_auto_sufficient_atwork,-1.4450618,F +sr3p_ASC_no_auto_eatout,0.3219998,F +sr3p_ASC_no_auto_escort,-1.8129267,F +sr3p_ASC_no_auto_othdiscr,0.27216902,F +sr3p_ASC_no_auto_othmaint,-0.8031854,F +sr3p_ASC_no_auto_school,-6.0240827,F +sr3p_ASC_no_auto_shopping,-0.27978948,F +sr3p_ASC_no_auto_social,-1.4036902,F +sr3p_ASC_no_auto_univ,-6.056001,F +sr3p_ASC_no_auto_work,-0.5831269,F +sr3p_ASC_no_auto_atwork,0.5826626,F +sr3p_ASC_auto_deficient_eatout,0.04605236,F +sr3p_ASC_auto_deficient_escort,-0.40818766,F +sr3p_ASC_auto_deficient_othdiscr,1.0470966,F +sr3p_ASC_auto_deficient_othmaint,-1.3493925,F +sr3p_ASC_auto_deficient_school,0.7149571,F +sr3p_ASC_auto_deficient_shopping,-0.073370166,F +sr3p_ASC_auto_deficient_social,1.5007243,F +sr3p_ASC_auto_deficient_univ,-1.7277422,F +sr3p_ASC_auto_deficient_work,-0.8527042,F +sr3p_ASC_auto_deficient_atwork,-2.514658,F +sr3p_ASC_auto_sufficient_eatout,0.8468596,F +sr3p_ASC_auto_sufficient_escort,-0.05741253,F +sr3p_ASC_auto_sufficient_othdiscr,0.58850205,F +sr3p_ASC_auto_sufficient_othmaint,-0.07549867,F +sr3p_ASC_auto_sufficient_school,-1.0201935,F +sr3p_ASC_auto_sufficient_shopping,-0.077571295,F +sr3p_ASC_auto_sufficient_social,0.50617886,F +sr3p_ASC_auto_sufficient_univ,-1.9047098,F +sr3p_ASC_auto_sufficient_work,-1.4699702,F +sr3p_ASC_auto_sufficient_atwork,-1.652174,F +walk_transit_ASC_no_auto_eatout,2.5936368,F +walk_transit_ASC_no_auto_escort,-2.2172081,F +walk_transit_ASC_no_auto_othdiscr,2.2437785,F +walk_transit_ASC_no_auto_othmaint,2.5643456,F +walk_transit_ASC_no_auto_school,21.383749,F +walk_transit_ASC_no_auto_shopping,2.1067476,F +walk_transit_ASC_no_auto_social,1.3814651,F +walk_transit_ASC_no_auto_univ,8.786037,F +walk_transit_ASC_no_auto_work,5.0354166,F +walk_transit_ASC_no_auto_atwork,2.7041876,F +walk_transit_ASC_auto_deficient_eatout,-0.03896324,F +walk_transit_ASC_auto_deficient_escort,-4.960704,F +walk_transit_ASC_auto_deficient_othdiscr,0.9530884,F +walk_transit_ASC_auto_deficient_othmaint,-3.0597258,F +walk_transit_ASC_auto_deficient_school,4.120708,F +walk_transit_ASC_auto_deficient_shopping,-0.8476569,F +walk_transit_ASC_auto_deficient_social,0.97444487,F +walk_transit_ASC_auto_deficient_univ,3.1362555,F +walk_transit_ASC_auto_deficient_work,0.65302855,F +walk_transit_ASC_auto_deficient_atwork,-2.9988291,F +walk_transit_ASC_auto_sufficient_eatout,-1.1126906,F +walk_transit_ASC_auto_sufficient_escort,-4.934847,F +walk_transit_ASC_auto_sufficient_othdiscr,-0.80636793,F +walk_transit_ASC_auto_sufficient_othmaint,-1.5471172,F +walk_transit_ASC_auto_sufficient_school,0.74590874,F +walk_transit_ASC_auto_sufficient_shopping,-2.2036798,F +walk_transit_ASC_auto_sufficient_social,-0.3453759,F +walk_transit_ASC_auto_sufficient_univ,0.4731163,F +walk_transit_ASC_auto_sufficient_work,-0.8916507,F +walk_transit_ASC_auto_sufficient_atwork,-3.401027,F +drive_transit_ASC_no_auto_all,0,F +drive_transit_ASC_auto_deficient_eatout,0.5998061,F +drive_transit_ASC_auto_deficient_escort,-1.1537067,F +drive_transit_ASC_auto_deficient_othdiscr,0.3199308,F +drive_transit_ASC_auto_deficient_othmaint,-0.29943228,F +drive_transit_ASC_auto_deficient_school,5.3252654,F +drive_transit_ASC_auto_deficient_shopping,-0.41849178,F +drive_transit_ASC_auto_deficient_social,1.5627195,F +drive_transit_ASC_auto_deficient_univ,1.8501176,F +drive_transit_ASC_auto_deficient_work,0.10081567,F +drive_transit_ASC_auto_deficient_atwork,-998.8196,F +drive_transit_ASC_auto_sufficient_eatout,-0.96951586,F +drive_transit_ASC_auto_sufficient_escort,-4.6014247,F +drive_transit_ASC_auto_sufficient_othdiscr,-0.3785917,F +drive_transit_ASC_auto_sufficient_othmaint,-2.6249478,F +drive_transit_ASC_auto_sufficient_school,1.40135,F +drive_transit_ASC_auto_sufficient_shopping,-2.1718938,F +drive_transit_ASC_auto_sufficient_social,-0.61585575,F +drive_transit_ASC_auto_sufficient_univ,1.3587753,F +drive_transit_ASC_auto_sufficient_work,-1.0045459,F +drive_transit_ASC_auto_sufficient_atwork,-999.21466,F +taxi_ASC_no_auto_eatout_othdiscr_social,0.9923,F +taxi_ASC_no_auto_escort_othmaint_shopping,1.8939,F +taxi_ASC_no_auto_school_univ,-7,T +taxi_ASC_no_auto_work,4.7291,F +taxi_ASC_no_auto_atwork,4.1021,F +taxi_ASC_auto_deficient_eatout_othdiscr_social,-3.1317,F +taxi_ASC_auto_deficient_escort_othmaint_shopping,0.1766,F +taxi_ASC_auto_deficient_school,-0.3338,F +taxi_ASC_auto_deficient_univ,4.2492,F +taxi_ASC_auto_deficient_work,-1.4766,F +taxi_ASC_auto_deficient_atwork,-4.4046,F +taxi_ASC_auto_sufficient_eatout_othdiscr_social,-3.0374,F +taxi_ASC_auto_sufficient_escort_othmaint_shopping,-1.8055,F +taxi_ASC_auto_sufficient_school,-2.4294,F +taxi_ASC_auto_sufficient_univ,-0.3131,F +taxi_ASC_auto_sufficient_work,-4.8509,F +taxi_ASC_auto_sufficient_atwork,-2.8804,F +tnc_single_ASC_no_auto_eatout_othdiscr_social,1.6852,F +tnc_single_ASC_no_auto_escort_othmaint_shopping,1.8605,F +tnc_single_ASC_no_auto_school,-7,T +tnc_single_ASC_no_auto_univ,-2.519,F +tnc_single_ASC_no_auto_work,5.7855,F +tnc_single_ASC_no_auto_atwork,4.4982,F +tnc_single_ASC_auto_deficient_eatout_othdiscr_social,-2.9623,F +tnc_single_ASC_auto_deficient_escort_othmaint_shopping,0.6748,F +tnc_single_ASC_auto_deficient_school,-0.5524,F +tnc_single_ASC_auto_deficient_univ,1.0221,F +tnc_single_ASC_auto_deficient_work,-0.8013,F +tnc_single_ASC_auto_deficient_atwork,-3.7626,F +tnc_single_ASC_auto_sufficient_eatout_othdiscr_social,-2.3239,F +tnc_single_ASC_auto_sufficient_escort_othmaint_shopping,-1.45,F +tnc_single_ASC_auto_sufficient_school,-2.8375,F +tnc_single_ASC_auto_sufficient_univ,0.2088,F +tnc_single_ASC_auto_sufficient_work,-4.1946,F +tnc_single_ASC_auto_sufficient_atwork,-2.7988,F +tnc_shared_ASC_no_auto_eatout_othdiscr_social,0.6464,F +tnc_shared_ASC_no_auto_escort_othmaint_shopping,0.9361,F +tnc_shared_ASC_no_auto_school,-7,T +tnc_shared_ASC_no_auto_univ,-5.8116,F +tnc_shared_ASC_no_auto_work,3.2429,F +tnc_shared_ASC_no_auto_atwork,3.3672,F +tnc_shared_ASC_auto_deficient_eatout_othdiscr_social,-4.3576,F +tnc_shared_ASC_auto_deficient_escort_othmaint_shopping,-0.3863,F +tnc_shared_ASC_auto_deficient_school,-1.4746,F +tnc_shared_ASC_auto_deficient_univ,3.25,F +tnc_shared_ASC_auto_deficient_work,-2.1435,F +tnc_shared_ASC_auto_deficient_atwork,-4.5089,F +tnc_shared_ASC_auto_sufficient_eatout_othdiscr_social,-3.6638,F +tnc_shared_ASC_auto_sufficient_escort_othmaint_shopping,-2.4365,F +tnc_shared_ASC_auto_sufficient_school,-3.7219,F +tnc_shared_ASC_auto_sufficient_univ,-0.9068,F +tnc_shared_ASC_auto_sufficient_work,-5.3575,F +tnc_shared_ASC_auto_sufficient_atwork,-3.5397,F +joint_walk_ASC_no_auto_all,-0.21274701,F +joint_walk_ASC_auto_deficient_all,-1.9607706,F +joint_walk_ASC_auto_sufficient_all,-3.2352157,F +joint_bike_ASC_no_auto_all,-2.8671598,F +joint_bike_ASC_auto_deficient_all,-6.076415,F +joint_bike_ASC_auto_sufficient_all,-6.3760657,F +joint_sr2_ASC_no_auto_all,0,T +joint_sr2_ASC_auto_deficient_all,0,T +joint_sr2_ASC_auto_sufficient_all,0,T +joint_sr3p_ASC_no_auto_all,0.5630671,F +joint_sr3p_ASC_auto_deficient_all,-1.8841692,F +joint_sr3p_ASC_auto_sufficient_all,-2.234826,F +joint_walk_transit_ASC_no_auto_all,0.62292415,F +joint_walk_transit_ASC_auto_deficient_all,-5.1634483,F +joint_walk_transit_ASC_auto_sufficient_all,-18.264534,F +joint_drive_transit_ASC_no_auto_all,0,T +joint_drive_transit_ASC_auto_deficient_all,-5.9632215,F +joint_drive_transit_ASC_auto_sufficient_all,-8.045285,F +joint_taxi_ASC_no_auto_all,-4.5792,F +joint_taxi_ASC_auto_deficient_all,-9.8157,F +joint_taxi_ASC_auto_sufficient_all,-11.7099,T +joint_tnc_single_ASC_no_auto_all,-4.4917,F +joint_tnc_single_ASC_auto_deficient_all,-9.8961,F +joint_tnc_single_ASC_auto_sufficient_all,-14.0159,T +joint_tnc_shared_ASC_no_auto_all,-4.3002,F +joint_tnc_shared_ASC_auto_deficient_all,-11.1572,F +joint_tnc_shared_ASC_auto_sufficient_all,-13.205,T +local_bus_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,-0.090703264,F +local_bus_ASC_school_univ,-0.06508621,F +local_bus_ASC_work,0.06689507,F +walk_light_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,0.76895475,F +walk_light_rail_ASC_school_univ,1.6814003,F +walk_light_rail_ASC_work,0.8255567,F +drive_light_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,0.76895475,F +drive_light_rail_ASC_school_univ,1.6814003,F +drive_light_rail_ASC_work,0.8255567,F +walk_ferry_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,0.9401238,F +walk_ferry_ASC_school_univ,2.0202317,F +walk_ferry_ASC_work,0.93322605,F +drive_ferry_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,0.9401238,F +drive_ferry_ASC_school_univ,2.0202317,F +drive_ferry_ASC_work,0.93322605,F +express_bus_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,0.9692316,F +express_bus_ASC_school_univ,0.32496938,F +express_bus_ASC_work,-0.5165474,F +heavy_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,0.7706121,F +heavy_rail_ASC_school_univ,0.96200377,F +heavy_rail_ASC_work,0.64772975,F +commuter_rail_ASC_eatout_escort_othdiscr_othmaint_shopping_social_atwork,0.7270185,F +commuter_rail_ASC_school_univ,1.0336206,F +commuter_rail_ASC_work,0.725503,F +walk_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,0.525,F +walk_transit_CBD_ASC_school_univ,0.672,F +walk_transit_CBD_ASC_work,0.804,F +walk_transit_CBD_ASC_atwork,0.564,F +drive_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,0.525,F +drive_transit_CBD_ASC_school_univ,0.672,F +drive_transit_CBD_ASC_work,1.1,F +drive_transit_CBD_ASC_atwork,0.564,F +coef_calib_DRIVEALONEPAY_zero_auto,0,T +coef_calib_SHARED2FREE_zero_auto,0,T +coef_calib_SHARED2PAY_zero_auto,0,T +coef_calib_SHARED3FREE_zero_auto,0,T +coef_calib_SHARED3PAY_zero_auto,0,T +coef_calib_WALK_zero_auto,0,T +coef_calib_BIKE_zero_auto,0,T +coef_calib_WALK_LOC_zero_auto,0,T +coef_calib_WALK_LRF_zero_auto,0,T +coef_calib_WALK_EXP_zero_auto,0,T +coef_calib_WALK_HVY_zero_auto,0,T +coef_calib_WALK_COM_zero_auto,0,T +coef_calib_DRIVE_LOC_zero_auto,0,T +coef_calib_DRIVE_LRF_zero_auto,0,T +coef_calib_DRIVE_EXP_zero_auto,0,T +coef_calib_DRIVE_HVY_zero_auto,0,T +coef_calib_DRIVE_COM_zero_auto,0,T +coef_calib_TAXI_zero_auto,0,T +coef_calib_TNC_SINGLE_zero_auto,0,T +coef_calib_TNC_SHARED_zero_auto,0,T +coef_calib_DRIVEALONEPAY_auto_insuff,0,T +coef_calib_SHARED2FREE_auto_insuff,0,T +coef_calib_SHARED2PAY_auto_insuff,0,T +coef_calib_SHARED3FREE_auto_insuff,0,T +coef_calib_SHARED3PAY_auto_insuff,0,T +coef_calib_WALK_auto_insuff,0,T +coef_calib_BIKE_auto_insuff,0,T +coef_calib_WALK_LOC_auto_insuff,0,T +coef_calib_WALK_LRF_auto_insuff,0,T +coef_calib_WALK_EXP_auto_insuff,0,T +coef_calib_WALK_HVY_auto_insuff,0,T +coef_calib_WALK_COM_auto_insuff,0,T +coef_calib_DRIVE_LOC_auto_insuff,0,T +coef_calib_DRIVE_LRF_auto_insuff,0,T +coef_calib_DRIVE_EXP_auto_insuff,0,T +coef_calib_DRIVE_HVY_auto_insuff,0,T +coef_calib_DRIVE_COM_auto_insuff,0,T +coef_calib_TAXI_auto_insuff,0,T +coef_calib_TNC_SINGLE_auto_insuff,0,T +coef_calib_TNC_SHARED_auto_insuff,0,T +coef_calib_DRIVEALONEPAY_auto_suff,0,T +coef_calib_SHARED2FREE_auto_suff,0,T +coef_calib_SHARED2PAY_auto_suff,0,T +coef_calib_SHARED3FREE_auto_suff,0,T +coef_calib_SHARED3PAY_auto_suff,0,T +coef_calib_WALK_auto_suff,0,T +coef_calib_BIKE_auto_suff,0,T +coef_calib_WALK_LOC_auto_suff,0,T +coef_calib_WALK_LRF_auto_suff,0,T +coef_calib_WALK_EXP_auto_suff,0,T +coef_calib_WALK_HVY_auto_suff,0,T +coef_calib_WALK_COM_auto_suff,0,T +coef_calib_DRIVE_LOC_auto_suff,0,T +coef_calib_DRIVE_LRF_auto_suff,0,T +coef_calib_DRIVE_EXP_auto_suff,0,T +coef_calib_DRIVE_HVY_auto_suff,0,T +coef_calib_DRIVE_COM_auto_suff,0,T +coef_calib_TAXI_auto_suff,0,T +coef_calib_TNC_SINGLE_auto_suff,0,T +coef_calib_TNC_SHARED_auto_suff,0,T diff --git a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_coefficients.csv b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_coefficients.csv new file mode 100644 index 0000000000..2c3f64c5ce --- /dev/null +++ b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_coefficients.csv @@ -0,0 +1,13 @@ +coefficient_name,value,constrain +coef_dist_0_1,-0.8428,F +coef_dist_1_2,-0.3104,F +coef_dist_2_5,-0.3783,F +coef_dist_5_15,-0.1285,F +coef_dist_15_up,-0.0917,F +coef_dist_0_5_high,0.15,F +coef_dist_5_up_high,0.02,F +coef_mode_logsum,0.3,F +coef_calib_dist_0_2,0,T +coef_calib_dist_2_5,0,T +coef_calib_dist_5_15,0,T +coef_calib_dist_15_up,0,T From 9d9f00615ef06825073298b81f84f45973e64ef4 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 16 Jul 2026 16:23:05 -0500 Subject: [PATCH 35/96] Add estimation mode regression test coeffs --- .../test_auto_ownership.csv | 4 ++ .../test_tour_mode_choice.csv | 60 +++++++++++++++++++ .../test_workplace_location.csv | 4 ++ 3 files changed, 68 insertions(+) diff --git a/activitysim/estimation/test/test_larch_estimation/test_auto_ownership.csv b/activitysim/estimation/test/test_larch_estimation/test_auto_ownership.csv index 2a39f593d9..dcd75d90d9 100644 --- a/activitysim/estimation/test/test_larch_estimation/test_auto_ownership.csv +++ b/activitysim/estimation/test/test_larch_estimation/test_auto_ownership.csv @@ -65,3 +65,7 @@ coef_retail_auto_workers,-0.53111245140175878,0,0,,,0,-0.53111245140175878 coef_retail_non_motor,-0.029999999999999999,0,0,,,1,-0.029999999999999999 coef_retail_transit_no_workers,-0.33344704884648813,0,0,,,0,-0.33344704884648813 coef_retail_transit_workers,-0.46438215846637165,0,0,,,0,-0.46438215846637165 +coef_calib_auto_0,0,0,0,,,1,0 +coef_calib_auto_2,0,0,0,,,1,0 +coef_calib_auto_3,0,0,0,,,1,0 +coef_calib_auto_4,0,0,0,,,1,0 diff --git a/activitysim/estimation/test/test_larch_estimation/test_tour_mode_choice.csv b/activitysim/estimation/test/test_larch_estimation/test_tour_mode_choice.csv index 2b0c82486a..dbc03dc56a 100644 --- a/activitysim/estimation/test/test_larch_estimation/test_tour_mode_choice.csv +++ b/activitysim/estimation/test/test_larch_estimation/test_tour_mode_choice.csv @@ -300,3 +300,63 @@ walk_transit_ASC_auto_deficient_atwork,-2.9988291,0,0,,,0,-2.9988291 walk_transit_ASC_auto_sufficient_atwork,-3.401027,0,0,,,0,-3.401027 walk_transit_ASC_no_auto_atwork,2.7041876,0,0,,,0,2.7041876 walk_transit_CBD_ASC_atwork,0.56399999999999995,0,0,,,0,0.56399999999999995 +coef_calib_DRIVEALONEPAY_zero_auto,0,0,0,,,1,0 +coef_calib_SHARED2FREE_zero_auto,0,0,0,,,1,0 +coef_calib_SHARED2PAY_zero_auto,0,0,0,,,1,0 +coef_calib_SHARED3FREE_zero_auto,0,0,0,,,1,0 +coef_calib_SHARED3PAY_zero_auto,0,0,0,,,1,0 +coef_calib_WALK_zero_auto,0,0,0,,,1,0 +coef_calib_BIKE_zero_auto,0,0,0,,,1,0 +coef_calib_WALK_LOC_zero_auto,0,0,0,,,1,0 +coef_calib_WALK_LRF_zero_auto,0,0,0,,,1,0 +coef_calib_WALK_EXP_zero_auto,0,0,0,,,1,0 +coef_calib_WALK_HVY_zero_auto,0,0,0,,,1,0 +coef_calib_WALK_COM_zero_auto,0,0,0,,,1,0 +coef_calib_DRIVE_LOC_zero_auto,0,0,0,,,1,0 +coef_calib_DRIVE_LRF_zero_auto,0,0,0,,,1,0 +coef_calib_DRIVE_EXP_zero_auto,0,0,0,,,1,0 +coef_calib_DRIVE_HVY_zero_auto,0,0,0,,,1,0 +coef_calib_DRIVE_COM_zero_auto,0,0,0,,,1,0 +coef_calib_TAXI_zero_auto,0,0,0,,,1,0 +coef_calib_TNC_SINGLE_zero_auto,0,0,0,,,1,0 +coef_calib_TNC_SHARED_zero_auto,0,0,0,,,1,0 +coef_calib_DRIVEALONEPAY_auto_insuff,0,0,0,,,1,0 +coef_calib_SHARED2FREE_auto_insuff,0,0,0,,,1,0 +coef_calib_SHARED2PAY_auto_insuff,0,0,0,,,1,0 +coef_calib_SHARED3FREE_auto_insuff,0,0,0,,,1,0 +coef_calib_SHARED3PAY_auto_insuff,0,0,0,,,1,0 +coef_calib_WALK_auto_insuff,0,0,0,,,1,0 +coef_calib_BIKE_auto_insuff,0,0,0,,,1,0 +coef_calib_WALK_LOC_auto_insuff,0,0,0,,,1,0 +coef_calib_WALK_LRF_auto_insuff,0,0,0,,,1,0 +coef_calib_WALK_EXP_auto_insuff,0,0,0,,,1,0 +coef_calib_WALK_HVY_auto_insuff,0,0,0,,,1,0 +coef_calib_WALK_COM_auto_insuff,0,0,0,,,1,0 +coef_calib_DRIVE_LOC_auto_insuff,0,0,0,,,1,0 +coef_calib_DRIVE_LRF_auto_insuff,0,0,0,,,1,0 +coef_calib_DRIVE_EXP_auto_insuff,0,0,0,,,1,0 +coef_calib_DRIVE_HVY_auto_insuff,0,0,0,,,1,0 +coef_calib_DRIVE_COM_auto_insuff,0,0,0,,,1,0 +coef_calib_TAXI_auto_insuff,0,0,0,,,1,0 +coef_calib_TNC_SINGLE_auto_insuff,0,0,0,,,1,0 +coef_calib_TNC_SHARED_auto_insuff,0,0,0,,,1,0 +coef_calib_DRIVEALONEPAY_auto_suff,0,0,0,,,1,0 +coef_calib_SHARED2FREE_auto_suff,0,0,0,,,1,0 +coef_calib_SHARED2PAY_auto_suff,0,0,0,,,1,0 +coef_calib_SHARED3FREE_auto_suff,0,0,0,,,1,0 +coef_calib_SHARED3PAY_auto_suff,0,0,0,,,1,0 +coef_calib_WALK_auto_suff,0,0,0,,,1,0 +coef_calib_BIKE_auto_suff,0,0,0,,,1,0 +coef_calib_WALK_LOC_auto_suff,0,0,0,,,1,0 +coef_calib_WALK_LRF_auto_suff,0,0,0,,,1,0 +coef_calib_WALK_EXP_auto_suff,0,0,0,,,1,0 +coef_calib_WALK_HVY_auto_suff,0,0,0,,,1,0 +coef_calib_WALK_COM_auto_suff,0,0,0,,,1,0 +coef_calib_DRIVE_LOC_auto_suff,0,0,0,,,1,0 +coef_calib_DRIVE_LRF_auto_suff,0,0,0,,,1,0 +coef_calib_DRIVE_EXP_auto_suff,0,0,0,,,1,0 +coef_calib_DRIVE_HVY_auto_suff,0,0,0,,,1,0 +coef_calib_DRIVE_COM_auto_suff,0,0,0,,,1,0 +coef_calib_TAXI_auto_suff,0,0,0,,,1,0 +coef_calib_TNC_SINGLE_auto_suff,0,0,0,,,1,0 +coef_calib_TNC_SHARED_auto_suff,0,0,0,,,1,0 diff --git a/activitysim/estimation/test/test_larch_estimation/test_workplace_location.csv b/activitysim/estimation/test/test_larch_estimation/test_workplace_location.csv index 528d77a9ba..30c16478da 100644 --- a/activitysim/estimation/test/test_larch_estimation/test_workplace_location.csv +++ b/activitysim/estimation/test/test_larch_estimation/test_workplace_location.csv @@ -33,3 +33,7 @@ work_veryhigh_HEREMPN,-5.6530572756846489,-5.6530572756846489,-1.422958374023437 work_veryhigh_MWTEMPN,-6,-6,-1.4024237394332886,0 work_veryhigh_OTHEMPN,-5.8240005358464035,-5.8240005358464035,-1.9241486787796021,0 work_veryhigh_RETEMPN,-2.3751556873321533,-2.3751556873321533,-2.3751556873321533,0 +coef_calib_dist_0_2,0,0,0,0 +coef_calib_dist_2_5,0,0,0,0 +coef_calib_dist_5_15,0,0,0,0 +coef_calib_dist_15_up,0,0,0,0 From afebfe3e0bbb2077c17349c22fa664e6bcef5463 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 16 Jul 2026 17:04:46 -0500 Subject: [PATCH 36/96] More coefficient adds --- ...n_model_workplace_location_SLSQP_None_.csv | 4 ++ ...t_simple_simulate_auto_ownership_BHHH_.csv | 4 ++ .../test_tour_and_subtour_mode_choice.csv | 60 +++++++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/activitysim/estimation/test/test_larch_estimation/test_location_model_workplace_location_SLSQP_None_.csv b/activitysim/estimation/test/test_larch_estimation/test_location_model_workplace_location_SLSQP_None_.csv index d4b8d728d6..72d28ac897 100644 --- a/activitysim/estimation/test/test_larch_estimation/test_location_model_workplace_location_SLSQP_None_.csv +++ b/activitysim/estimation/test/test_larch_estimation/test_location_model_workplace_location_SLSQP_None_.csv @@ -33,3 +33,7 @@ work_veryhigh_HEREMPN,-5.6489927637825792,-5.6489927637825792,-1.422958374023437 work_veryhigh_MWTEMPN,-6,-6,-1.4024237394332886,0 work_veryhigh_OTHEMPN,-5.8313629173672439,-5.8313629173672439,-1.9241486787796021,0 work_veryhigh_RETEMPN,-2.3751556873321533,-2.3751556873321533,-2.3751556873321533,0 +coef_calib_dist_0_2,0,0,0,0 +coef_calib_dist_2_5,0,0,0,0 +coef_calib_dist_5_15,0,0,0,0 +coef_calib_dist_15_up,0,0,0,0 diff --git a/activitysim/estimation/test/test_larch_estimation/test_simple_simulate_auto_ownership_BHHH_.csv b/activitysim/estimation/test/test_larch_estimation/test_simple_simulate_auto_ownership_BHHH_.csv index a45f944823..7047821b45 100644 --- a/activitysim/estimation/test/test_larch_estimation/test_simple_simulate_auto_ownership_BHHH_.csv +++ b/activitysim/estimation/test/test_larch_estimation/test_simple_simulate_auto_ownership_BHHH_.csv @@ -65,3 +65,7 @@ coef_retail_auto_workers,-0.53111240158828621,0.1646,0,,,-0.53111240158828621 coef_retail_non_motor,-0.029999999999999999,-0.029999999999999999,0,-0.029999999999999999,-0.029999999999999999,-0.029999999999999999 coef_retail_transit_no_workers,-0.33344704526636021,-0.30530000000000002,0,,,-0.33344704526636021 coef_retail_transit_workers,-0.46438215677534161,-0.51170000000000004,0,,,-0.46438215677534161 +coef_calib_auto_0,0,0,0,0,0,0 +coef_calib_auto_2,0,0,0,0,0,0 +coef_calib_auto_3,0,0,0,0,0,0 +coef_calib_auto_4,0,0,0,0,0,0 diff --git a/activitysim/estimation/test/test_larch_estimation/test_tour_and_subtour_mode_choice.csv b/activitysim/estimation/test/test_larch_estimation/test_tour_and_subtour_mode_choice.csv index 8d8064cbb4..dee933e942 100644 --- a/activitysim/estimation/test/test_larch_estimation/test_tour_and_subtour_mode_choice.csv +++ b/activitysim/estimation/test/test_larch_estimation/test_tour_and_subtour_mode_choice.csv @@ -300,3 +300,63 @@ walk_transit_CBD_ASC_atwork,0.34851829905878767,0.34851829905878767,0.5640000104 walk_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,0.69144903771302224,0.69144903771302224,0.52499997615814209,0 walk_transit_CBD_ASC_school_univ,0.78893465707209709,0.78893465707209709,0.67199999094009399,0 walk_transit_CBD_ASC_work,0.97702352207787502,0.97702352207787502,0.80400002002716064,0 +coef_calib_DRIVEALONEPAY_zero_auto,0,0,0,0 +coef_calib_SHARED2FREE_zero_auto,0,0,0,0 +coef_calib_SHARED2PAY_zero_auto,0,0,0,0 +coef_calib_SHARED3FREE_zero_auto,0,0,0,0 +coef_calib_SHARED3PAY_zero_auto,0,0,0,0 +coef_calib_WALK_zero_auto,0,0,0,0 +coef_calib_BIKE_zero_auto,0,0,0,0 +coef_calib_WALK_LOC_zero_auto,0,0,0,0 +coef_calib_WALK_LRF_zero_auto,0,0,0,0 +coef_calib_WALK_EXP_zero_auto,0,0,0,0 +coef_calib_WALK_HVY_zero_auto,0,0,0,0 +coef_calib_WALK_COM_zero_auto,0,0,0,0 +coef_calib_DRIVE_LOC_zero_auto,0,0,0,0 +coef_calib_DRIVE_LRF_zero_auto,0,0,0,0 +coef_calib_DRIVE_EXP_zero_auto,0,0,0,0 +coef_calib_DRIVE_HVY_zero_auto,0,0,0,0 +coef_calib_DRIVE_COM_zero_auto,0,0,0,0 +coef_calib_TAXI_zero_auto,0,0,0,0 +coef_calib_TNC_SINGLE_zero_auto,0,0,0,0 +coef_calib_TNC_SHARED_zero_auto,0,0,0,0 +coef_calib_DRIVEALONEPAY_auto_insuff,0,0,0,0 +coef_calib_SHARED2FREE_auto_insuff,0,0,0,0 +coef_calib_SHARED2PAY_auto_insuff,0,0,0,0 +coef_calib_SHARED3FREE_auto_insuff,0,0,0,0 +coef_calib_SHARED3PAY_auto_insuff,0,0,0,0 +coef_calib_WALK_auto_insuff,0,0,0,0 +coef_calib_BIKE_auto_insuff,0,0,0,0 +coef_calib_WALK_LOC_auto_insuff,0,0,0,0 +coef_calib_WALK_LRF_auto_insuff,0,0,0,0 +coef_calib_WALK_EXP_auto_insuff,0,0,0,0 +coef_calib_WALK_HVY_auto_insuff,0,0,0,0 +coef_calib_WALK_COM_auto_insuff,0,0,0,0 +coef_calib_DRIVE_LOC_auto_insuff,0,0,0,0 +coef_calib_DRIVE_LRF_auto_insuff,0,0,0,0 +coef_calib_DRIVE_EXP_auto_insuff,0,0,0,0 +coef_calib_DRIVE_HVY_auto_insuff,0,0,0,0 +coef_calib_DRIVE_COM_auto_insuff,0,0,0,0 +coef_calib_TAXI_auto_insuff,0,0,0,0 +coef_calib_TNC_SINGLE_auto_insuff,0,0,0,0 +coef_calib_TNC_SHARED_auto_insuff,0,0,0,0 +coef_calib_DRIVEALONEPAY_auto_suff,0,0,0,0 +coef_calib_SHARED2FREE_auto_suff,0,0,0,0 +coef_calib_SHARED2PAY_auto_suff,0,0,0,0 +coef_calib_SHARED3FREE_auto_suff,0,0,0,0 +coef_calib_SHARED3PAY_auto_suff,0,0,0,0 +coef_calib_WALK_auto_suff,0,0,0,0 +coef_calib_BIKE_auto_suff,0,0,0,0 +coef_calib_WALK_LOC_auto_suff,0,0,0,0 +coef_calib_WALK_LRF_auto_suff,0,0,0,0 +coef_calib_WALK_EXP_auto_suff,0,0,0,0 +coef_calib_WALK_HVY_auto_suff,0,0,0,0 +coef_calib_WALK_COM_auto_suff,0,0,0,0 +coef_calib_DRIVE_LOC_auto_suff,0,0,0,0 +coef_calib_DRIVE_LRF_auto_suff,0,0,0,0 +coef_calib_DRIVE_EXP_auto_suff,0,0,0,0 +coef_calib_DRIVE_HVY_auto_suff,0,0,0,0 +coef_calib_DRIVE_COM_auto_suff,0,0,0,0 +coef_calib_TAXI_auto_suff,0,0,0,0 +coef_calib_TNC_SINGLE_auto_suff,0,0,0,0 +coef_calib_TNC_SHARED_auto_suff,0,0,0,0 From 4cda8e6b89706b60398a6f172f2cac05ed6de62d Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Fri, 17 Jul 2026 16:24:58 -0500 Subject: [PATCH 37/96] Add calibration documentation RST --- docs/calibration.rst | 713 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 713 insertions(+) create mode 100644 docs/calibration.rst diff --git a/docs/calibration.rst b/docs/calibration.rst new file mode 100644 index 0000000000..e64f1afb0f --- /dev/null +++ b/docs/calibration.rst @@ -0,0 +1,713 @@ +.. _calibration: + +=========================== +ActivitySim Auto-Calibration +=========================== + +ActivitySim includes an automated calibration framework that iteratively adjusts +model coefficients to match observed survey targets. Calibration wraps around +ActivitySim's normal model execution — it reuses the standard model runner and +only adds orchestration logic around it. + +Overview +======== + +The calibration loop works as follows: + +1. Run all model steps preceding the first calibrated component (the "precursor" models). +2. For each calibrated component, iteratively: + + - Run the component from its prior checkpoint. + - Evaluate model output shares and compare against survey targets. + - Compute coefficient adjustments using a damped log-ratio or odds-ratio method. + - Write updated coefficients back to the config file on disk. + - Check convergence; stop early if all coefficients are within tolerance. + +3. Run any intermediate (non-calibrated) model steps between calibrated components. +4. Optionally run the remaining model steps after the last calibrated component. +5. Repeat the entire process for a configurable number of **global iterations**. +6. Write a final snapshot of all calibrated coefficients. + +Both single-process and multiprocess execution modes are fully supported. Shared +resources (skim buffers, shadow pricing) are allocated once and reused across all +calibration iterations. + +Quick Start +=========== + +1. Prepare Your Configs Directory +---------------------------------- + +Add the following files to your ActivitySim configs directory (or a +calibration-specific overlay directory): + +- ``calibration.yaml`` — top-level calibration configuration +- One **calibration spec CSV** per calibrated component +- One **coefficients CSV** per calibrated component (may already exist from estimation) +- Optionally, one **helper module** (``.py``) per component for custom expressions or reports + +2. Create ``calibration.yaml`` +------------------------------- + +.. code-block:: yaml + + enable: True + + run: + calibrate_models: + - workplace_location + - auto_ownership_simulate + - tour_mode_choice_simulate + resume_after: null # checkpoint to resume from on global iteration 1 + restart_after: [] # components after which to restart (advanced) + global_iterations: 3 # number of full calibration passes + complete_steps: false # run model steps after the last calibrated component + + model_settings: + workplace_location: + calibration_spec: workplace_location_calibration.csv + helper_module: workplace_location_calib_helper.py + submodel_max_iterations: 3 + survey_file: survey_persons.csv + reports: + generic: true + bespoke: report_workplace_location + + auto_ownership_simulate: + calibration_spec: auto_ownership_calibration.csv + helper_module: auto_ownership_calib_helper.py + submodel_max_iterations: 3 + survey_file: survey_households.csv + reports: + generic: true + bespoke: report_auto_ownership + + tour_mode_choice_simulate: + calibration_spec: tour_mode_choice_calibration.csv + helper_module: tour_mode_choice_calib_helper.py + submodel_max_iterations: 3 + survey_file: survey_tours.csv + reports: + generic: true + bespoke: report_tour_mode_choice + +3. Run Calibration +------------------- + +Calibration is triggered through the standard ``activitysim run`` command. No +special subcommand is needed — when ``calibration.yaml`` exists in the config +directory with ``enable: True``, the calibration loop runs automatically instead +of the normal model flow. + +.. code-block:: bash + + activitysim run -c configs_calibration -c configs -d data -o output + +Use multiple ``-c`` flags to layer a calibration-specific config directory on top +of your base configs. The calibration configs override base configs via +ActivitySim's standard config resolution order (earlier directories take priority). + + +Configuration Reference +======================= + +``calibration.yaml`` — Top-Level Settings +------------------------------------------- + +.. list-table:: + :header-rows: 1 + :widths: 20 15 15 50 + + * - Field + - Type + - Default + - Description + * - ``enable`` + - ``bool`` + - ``False`` + - Master switch. Set to ``True`` to activate calibration. + * - ``run`` + - object + - *required* + - Run-control settings (see below). + * - ``model_settings`` + - dict + - ``{}`` + - Per-component calibration config, keyed by component name. + +**Validation rules:** + +- Every component listed in ``run.calibrate_models`` must have a corresponding + entry in ``model_settings``. +- Every component in ``run.restart_after`` must also appear in ``run.calibrate_models``. +- ``run.global_iterations`` must be ≥ 1. + +``run`` — Run Control Settings +------------------------------- + +.. list-table:: + :header-rows: 1 + :widths: 20 15 10 55 + + * - Field + - Type + - Default + - Description + * - ``calibrate_models`` + - ``list[str]`` + - *required* + - Model component names to calibrate. Must match names in ``settings.yaml`` + ``models`` list. + * - ``resume_after`` + - ``str`` or ``null`` + - ``null`` + - Checkpoint to resume from on the first global iteration. Equivalent to + ``resume_after`` in ``settings.yaml``. Use this to skip expensive + initialization steps that do not change across calibration iterations. + * - ``restart_after`` + - ``list[str]`` + - ``[]`` + - Components after which to restart. + * - ``global_iterations`` + - ``int`` + - ``1`` + - Number of full outer-loop calibration passes over all components. Each + global iteration re-runs all precursor models and re-calibrates all + components from scratch using the latest coefficients. + * - ``complete_steps`` + - ``bool`` + - ``False`` + - Whether to run all model steps after the last calibrated component. + When ``False``, only runs them on the final global iteration. Set to + ``True`` if downstream model outputs are needed every iteration (e.g., + for dashboarding). + +``model_settings.`` — Per-Component Settings +------------------------------------------------------------- + +.. list-table:: + :header-rows: 1 + :widths: 25 15 10 50 + + * - Field + - Type + - Default + - Description + * - ``calibration_spec`` + - ``str`` + - *required* + - Filename of the calibration spec CSV (must be in a configs directory). + * - ``helper_module`` + - ``str`` or ``null`` + - ``null`` + - Python file path (e.g., ``my_helper.py``) or importable module path. + Functions defined in this module are available as evaluation context for + ``model_value`` and ``target_value`` expressions. + * - ``submodel_max_iterations`` + - ``int`` + - ``1`` + - Maximum number of inner-loop iterations per component per global + iteration. The component re-runs from its prior checkpoint each iteration. + * - ``survey_file`` + - ``str`` + - *required* + - Survey data CSV filename. Made available via + ``component_settings.survey_file`` in the expression context. + * - ``reports.generic`` + - ``bool`` + - ``True`` + - Write a generic CSV report each iteration. + * - ``reports.bespoke`` + - ``str`` or ``null`` + - ``null`` + - Name of a function in the helper module to call for custom reporting. The + function receives the full evaluation context dict. + + +Calibration Spec CSV +==================== + +The calibration spec is a CSV file that defines — for each coefficient — how to +compute model and target values, the adjustment method, convergence tolerance, +and bounds. Each row represents one coefficient to calibrate. + +Required Columns +----------------- + +.. list-table:: + :header-rows: 1 + :widths: 20 15 65 + + * - Column + - Type + - Description + * - ``description`` + - ``str`` + - Human-readable label for the coefficient (used in reports and plots). + * - ``coefficient`` + - ``str`` + - Name of the coefficient in the component's coefficients CSV file. Must + also appear in the component's utility specification. + * - ``model_value`` + - numeric or expression + - The model's current output for the target metric. Can be a literal number + or a Python expression evaluated against the expression context. + * - ``target_value`` + - numeric or expression + - The observed/survey target. Can be a literal number or a Python expression. + * - ``hold_fast`` + - ``bool`` + - If ``True``, the coefficient is evaluated but never updated. Useful for + monitoring convergence of a held coefficient. + * - ``min`` + - numeric + - Lower bound for the coefficient value. Leave blank for no bound. + * - ``max`` + - numeric + - Upper bound for the coefficient value. Leave blank for no bound. + * - ``damping`` + - numeric + - Damping factor (≥ 0) applied to the computed delta. ``1.0`` means no + damping; values < 1 slow convergence for stability. + * - ``method`` + - ``str`` + - Adjustment method: ``log_ratio`` or ``odds_ratio``. + * - ``tolerance`` + - numeric + - Absolute difference threshold. A coefficient is "converged" when + ``|target_value - model_value| <= tolerance``. + +Optional Columns +----------------- + +.. list-table:: + :header-rows: 1 + :widths: 20 15 10 55 + + * - Column + - Type + - Default + - Description + * - ``default_increment`` + - numeric + - ``2.0`` + - Fallback delta when the chosen method encounters invalid inputs (e.g., + zero or negative values for ``log_ratio``). + +Comment Rows +------------- + +Lines beginning with ``#`` are treated as comments and ignored. This is useful +for temporarily disabling individual coefficients without removing them. + +Example: Auto Ownership +------------------------- + +.. code-block:: text + + description,coefficient,model_value,target_value,hold_fast,min,max,damping,method,tolerance + 0 auto ownership share,coef_calib_auto_0,len(households[households.auto_ownership==0]) / len(households),0.06812,FALSE,-5,5,1,log_ratio,0.02 + 2 auto ownership share,coef_calib_auto_2,len(households[households.auto_ownership==2]) / len(households),0.348413,FALSE,-5,5,1,log_ratio,0.01 + 3 auto ownership share,coef_calib_auto_3,len(households[households.auto_ownership==3]) / len(households),0.13718,FALSE,-5,5,1,log_ratio,0.01 + 4 auto ownership share,coef_calib_auto_4,len(households[households.auto_ownership==4]) / len(households),0.057501,FALSE,-5,5,1,log_ratio,0.01 + +In this example: + +- ``model_value`` is a Python expression that computes the share of households + with a given auto ownership level from the pipeline ``households`` table. +- ``target_value`` is a fixed numeric target derived from observed survey data. +- Each coefficient is bounded to ``[-5, 5]`` with no damping (``1.0``) and uses + the ``log_ratio`` method. + +Example: Workplace Location (Using Helper Functions) +----------------------------------------------------- + +.. code-block:: text + + description,coefficient,model_value,target_value,hold_fast,min,max,damping,method,tolerance + Distance 0 to 2 mi share,coef_calib_dist_0_2,"summarize_model(context, min_dist=.0, max_dist=.5)",0.05,FALSE,-5,5,1,log_ratio,0.02 + Distance 5 to 15 mi share,coef_calib_dist_5_15,"summarize_model(context, min_dist=.5, max_dist=1)",0.25,FALSE,-5,5,1,log_ratio,0.01 + Distance 15+ mi share,coef_calib_dist_15_up,"summarize_model(context, min_dist=2, max_dist=999)",0.1,FALSE,-5,5,1,log_ratio,0.01 + +Here ``summarize_model()`` is a function defined in the helper module. Both +``model_value`` and ``target_value`` can call helper functions via ``context``. + + +Adjustment Methods +================== + +``log_ratio`` +-------------- + +Computes the coefficient delta as: + +.. math:: + + \Delta = \ln\!\left(\frac{\text{target\_value}}{\text{model\_value}}\right) \times \text{damping} + +Requires both ``model_value`` and ``target_value`` to be positive. If either is +zero or negative, falls back to ``default_increment`` (positive or negative +depending on direction). + +Best suited for **share-based targets** where both model and target represent +proportions. + +``odds_ratio`` +--------------- + +Computes the coefficient delta as: + +.. math:: + + \Delta = \ln\!\left(\frac{T \cdot M - T}{T \cdot M - M}\right) \times \text{damping} + +where :math:`T` = ``target_value`` and :math:`M` = ``model_value``. + +Falls back to ``default_increment`` when the numerator or denominator is +non-positive. Appropriate for **logit-based models** where coefficients operate +in utility space. + +Damping +-------- + +The ``damping`` factor multiplies the computed delta. Use values less than 1.0 +(e.g., ``0.5``) to slow convergence and improve stability when coefficients +oscillate. A value of ``1.0`` applies the full computed adjustment. + +Bounds +------- + +When ``min`` and/or ``max`` are specified, the candidate coefficient value is +clamped after the delta is applied. The iteration record tracks whether clamping +occurred via ``at_min`` and ``at_max`` flags. + + +Expression Context +================== + +The ``model_value`` and ``target_value`` fields in the calibration spec can +contain Python expressions. These expressions are evaluated with access to the +following context variables: + +.. list-table:: + :header-rows: 1 + :widths: 25 20 55 + + * - Variable + - Type + - Description + * - ``state`` + - ``workflow.State`` + - The ActivitySim state object. + * - ``np`` + - module + - NumPy. + * - ``pd`` + - module + - pandas. + * - ``households`` + - ``DataFrame`` + - The ``households`` pipeline table (if it exists). + * - ``persons`` + - ``DataFrame`` + - The ``persons`` pipeline table (if it exists). + * - ``tours`` + - ``DataFrame`` + - The ``tours`` pipeline table (if it exists). + * - ``trips`` + - ``DataFrame`` + - The ``trips`` pipeline table (if it exists). + * - *(other tables)* + - ``DataFrame`` + - Any other registered pipeline table. + * - ``network_los`` + - object + - Network level of service (if available). + * - ``skim_dict`` + - ``SkimDict`` + - Default skim dictionary (if available). + * - ``component_output_dir`` + - ``Path`` + - Output directory for the current component (``output/calibration//``). + * - ``component_settings`` + - object + - The ``CalibrationComponentSettings`` for the current component. + * - ``context`` + - ``dict`` + - Self-reference to the full context dict, for passing to helper functions. + * - *(helper symbols)* + - varies + - All public names from the helper module (functions, variables, classes). + +Writing Expressions +-------------------- + +**Inline expressions** work well for simple share computations: + +.. code-block:: python + + len(households[households.auto_ownership == 0]) / len(households) + +**Helper function calls** are preferred for complex logic: + +.. code-block:: python + + summarize_model(context, min_dist=0.5, max_dist=1.0) + +The ``context`` variable gives helper functions access to everything: pipeline +tables, skims, pandas, numpy, and other helpers. + + +Helper Modules +============== + +A helper module is a Python file placed in the configs directory (or an +importable Python module) that provides functions for use in calibration spec +expressions and custom reports. + +Loading +-------- + +Specify the helper module in ``calibration.yaml``: + +.. code-block:: yaml + + model_settings: + workplace_location: + helper_module: workplace_location_calib_helper.py + +File paths ending in ``.py`` are loaded from the config directory. Other values +are treated as Python import paths (e.g., ``mypackage.calibration_helpers``). + +All public names (functions, variables, classes) from the module are injected +into the expression context. + +Example Helper Module +---------------------- + +.. code-block:: python + + """Workplace location calibration helpers.""" + + import pandas as pd + + + def compute_distances(context, origins, destinations): + """Compute distances between zones using the skim dictionary.""" + return context["skim_dict"].lookup(origins, destinations, "DIST") + + + def summarize_model(context, min_dist=1, max_dist=2): + """Compute the share of workers with workplace distance in [min_dist, max_dist).""" + persons = context["persons"] + workers = persons[persons["workplace_zone_id"] > 0] + distances = compute_distances( + context, workers["home_zone_id"], workers["workplace_zone_id"] + ) + mask = (distances >= min_dist) & (distances < max_dist) + return len(distances[mask]) / len(distances) if len(distances) > 0 else 0 + + + def report_workplace_location(context): + """Custom bespoke report called after each iteration.""" + import matplotlib.pyplot as plt + import os + + # Custom plotting logic here... + plt.savefig(os.path.join(context["component_output_dir"], "custom_report.png")) + plt.close() + +Bespoke Reports +---------------- + +If ``reports.bespoke`` is set to a function name (e.g., +``report_workplace_location``), that function is called after each component +iteration with the full expression context dict. Use this for custom plots, +tables, or dashboards beyond what the generic report provides. + + +Output Files +============ + +All calibration output is written under ``output/calibration/``. + +Global Files +------------- + +.. list-table:: + :header-rows: 1 + :widths: 40 60 + + * - File + - Description + * - ``calibration_progress.json`` + - Tracks ``next_global_iteration`` for crash recovery. If a run is + interrupted, restarting will resume from the last completed global + iteration. + * - ``calibration_iteration_records.csv`` + - Appended per-coefficient detail for every iteration across all + components. Columns include ``global_iter``, ``component_iter``, + ``coefficient``, ``target_value``, ``model_value``, ``difference``, + ``pct_difference``, ``prev_coefficient``, ``next_coefficient``, + ``converged``, ``at_min``, ``at_max``. + * - ``calibration_iteration_summary.csv`` + - One row per component iteration with summary statistics: + ``max_difference``, ``max_change``, ``num_converged``, + ``num_unconverged``. + * - ``final_calibrated_coefficients.csv`` + - Combined snapshot of all calibrated coefficients at the end of the run, + with ``component`` and ``coefficient_name`` columns. + +Per-Component Files (``calibration//``) +-------------------------------------------------------- + +.. list-table:: + :header-rows: 1 + :widths: 40 60 + + * - File + - Description + * - ``calibration_iteration_records.csv`` + - Component-specific iteration records (same schema as the global file). + * - ``generic_report.csv`` + - Simplified iteration report with ``description``, ``difference``, + ``pct_difference``, and ``converged``. + * - ``coefficient_progress_set_N.png`` + - Line plot of coefficient values across iterations (up to 10 coefficients + per plot). + * - ``final_components_set_N.png`` + - Bar chart comparing final ``target_value`` vs ``model_value``. + * - ``final_pct_change_set_N.png`` + - Bar chart of final percent difference between model and target. + +Updated Config Files +--------------------- + +Coefficient CSV files in the configs directory are **updated in-place** after +each iteration. This means: + +- The calibrated coefficients persist across runs. +- You can inspect intermediate coefficient values at any time. +- To reset, restore the original coefficient files from version control. + + +Crash Recovery +============== + +Calibration progress is persisted to ``calibration_progress.json`` after each +completed global iteration. If a run is interrupted: + +1. The coefficient files on disk reflect the state at the last completed iteration. +2. Restarting ``activitysim run`` with the same configuration will resume from + the ``next_global_iteration`` recorded in the progress file. + +To force a fresh start, delete ``output/calibration/calibration_progress.json`` +and restore original coefficient files. + + +Multiprocess Mode +================= + +Calibration works with ActivitySim's multiprocess execution mode. When +``multiprocess: True`` is set in ``settings.yaml``: + +- Shared resources (skim buffers, shadow pricing buffers) are allocated **once** + at calibration start and reused across all iterations, avoiding repeated + expensive allocations. +- Precursor, intermediate, and subsequent model steps use ActivitySim's normal + multiprocess orchestration. +- Calibrated component re-runs use direct apportion → simulate → coalesce + orchestration with explicit checkpoint control, ensuring correct state + restoration on each iteration. +- After each multiprocess run, the coalesced pipeline is loaded back into the + parent process so calibration expressions can evaluate model outputs. + +No special configuration is needed — the calibration framework automatically +respects ``num_processes``, ``multiprocess_steps``, ``slice``, and +``chunk_size`` from the existing ``settings.yaml``. + + +Convergence +=========== + +A coefficient is considered **converged** when: + +.. math:: + + |\text{target\_value} - \text{model\_value}| \leq \text{tolerance} + +A component is converged when **all** of its coefficients are converged. The +component inner loop stops early upon convergence. + +The overall calibration run completes after all ``global_iterations`` have +executed. Global convergence is tracked but does not currently trigger early +termination of the outer loop — use ``global_iterations`` to control the total +number of passes. + + +Coefficient Requirements +======================== + +Calibration coefficients must satisfy these requirements: + +1. **Present in utility specification**: Every ``coefficient`` in the calibration + spec must appear as a token in the component's utility expression CSV (the + files referenced by settings keys ending in ``SPEC``). A validation error is + raised at startup if any are missing. + +2. **Present in coefficients file**: If a calibration coefficient is not found in + the component's coefficients CSV, it is automatically added with an initial + value of ``0.0`` and a warning is logged. + +3. **Numeric values**: All coefficient values must be numeric. Non-numeric values + raise an error. + + +Working Example +=============== + +A complete working example is included in the repository at:: + + activitysim/examples/prototype_mtc/configs_calibration/ + +This example calibrates three components — ``workplace_location``, +``auto_ownership_simulate``, and ``tour_mode_choice_simulate`` — and includes: + +- ``calibration.yaml`` — top-level configuration +- ``*_calibration.csv`` — calibration specs with inline expressions and helper + function calls +- ``*_calib_helper.py`` — helper modules with custom summary functions and + bespoke reports +- ``*_coefficients.csv`` — initial coefficient values + +To run the example: + +.. code-block:: bash + + activitysim run \ + -c activitysim/examples/prototype_mtc/configs_calibration \ + -c activitysim/examples/prototype_mtc/configs \ + -d activitysim/examples/prototype_mtc/data \ + -o output + + +Tips +==== + +- **Start with** ``submodel_max_iterations: 1`` and a small number of + ``global_iterations`` to verify your setup before committing to a long + calibration run. +- **Use** ``damping < 1.0`` if coefficients oscillate between iterations. +- **Set** ``hold_fast: True`` on reference-category coefficients that should be + fixed (e.g., the base alternative in a logit model). +- **Comment out rows** in the calibration spec with ``#`` to temporarily exclude + coefficients without modifying the file structure. +- **Check** ``calibration_iteration_records.csv`` to diagnose convergence issues + — look for coefficients hitting bounds (``at_min``/``at_max``) or oscillating. +- **Use** ``complete_steps: true`` if you need full model outputs each iteration + (e.g., for trip generation downstream of mode choice). +- **Version control your coefficient files** so you can diff changes and reset to + initial values. +- **Use the** ``resume_after`` **setting** to skip expensive upstream steps (like + skims loading or accessibility computation) that don't change across + calibration iterations. From 15df92cc348836e459b2f81843efff7fdfbc0987 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Mon, 20 Jul 2026 21:47:31 -0700 Subject: [PATCH 38/96] move calib documentation into users guide --- docs/{ => users-guide}/calibration.rst | 4 ++-- docs/users-guide/index.rst | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) rename docs/{ => users-guide}/calibration.rst (99%) diff --git a/docs/calibration.rst b/docs/users-guide/calibration.rst similarity index 99% rename from docs/calibration.rst rename to docs/users-guide/calibration.rst index e64f1afb0f..db0cf719d0 100644 --- a/docs/calibration.rst +++ b/docs/users-guide/calibration.rst @@ -343,7 +343,7 @@ Computes the coefficient delta as: .. math:: - \Delta = \ln\!\left(\frac{\text{target\_value}}{\text{model\_value}}\right) \times \text{damping} + \Delta = \ln\!\left(\frac{\text{target_value}}{\text{model_value}}\right) \times \text{damping} Requires both ``model_value`` and ``target_value`` to be positive. If either is zero or negative, falls back to ``default_increment`` (positive or negative @@ -634,7 +634,7 @@ A coefficient is considered **converged** when: .. math:: - |\text{target\_value} - \text{model\_value}| \leq \text{tolerance} + \text{target_value} - \text{model_value} \leq \text{tolerance} A component is converged when **all** of its coefficients are converged. The component inner loop stops early upon convergence. diff --git a/docs/users-guide/index.rst b/docs/users-guide/index.rst index d464a6cd0d..3dc88a373e 100644 --- a/docs/users-guide/index.rst +++ b/docs/users-guide/index.rst @@ -45,6 +45,7 @@ Contents example_models example_performance estimation-mode/index + calibration .. toctree:: :maxdepth: 1 other_examples From 4e38277ba94ffd142318ffb51eb18d984fac8b24 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Mon, 20 Jul 2026 21:48:16 -0700 Subject: [PATCH 39/96] all converged fix and dead restart_after --- activitysim/core/calibration.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index a0c1651eb0..547f6499c3 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -72,7 +72,6 @@ class CalibrationRunSettings(PydanticBase): resume_after: Optional[str] = None calibrate_models: list[str] - restart_after: list[str] = [] global_iterations: int = 1 complete_steps: bool = False @@ -110,12 +109,6 @@ def validate_model_settings(self): f"calibration model '{component}' is not in model_settings" ) - for component in self.run.restart_after: - if component not in self.run.calibrate_models: - raise ValueError( - f"restart_after component '{component}' is not in calibrate_models" - ) - if self.run.global_iterations < 1: raise ValueError("max_iterations must be >= 1") @@ -279,10 +272,18 @@ def run_calibration_loop( }, ) + if all_converged: + logger.info( + "calibration converged after global iteration %s/%s", + global_iter - start_global_iter, + calibration_settings.run.global_iterations, + ) + break + _write_final_coefficients_snapshot(state, calibration_settings) return CalibrationRunResult( - converged=False, + converged=all_converged, completed_global_iterations=calibration_settings.run.global_iterations, ) finally: @@ -415,7 +416,7 @@ def _calibrate_component( for component_iter in range(1, component_settings.submodel_max_iterations + 1): component_iterations = component_iter run_model_name = ( - f"{component_name}.component_i{component_iter};" f"global_i{global_iter}" + f"{component_name}.c_i{component_iter};" f"g_i{global_iter}" ) # Re-run only this component from its prior checkpoint so model values # reflect the current candidate coefficients for this component. From 391d9a1371449c7fe00daa03001f1c57482bba42 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Mon, 20 Jul 2026 21:51:27 -0700 Subject: [PATCH 40/96] blacken --- activitysim/core/calibration.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 547f6499c3..fa554043d9 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -415,9 +415,7 @@ def _calibrate_component( for component_iter in range(1, component_settings.submodel_max_iterations + 1): component_iterations = component_iter - run_model_name = ( - f"{component_name}.c_i{component_iter};" f"g_i{global_iter}" - ) + run_model_name = f"{component_name}.c_i{component_iter};" f"g_i{global_iter}" # Re-run only this component from its prior checkpoint so model values # reflect the current candidate coefficients for this component. if state.settings.multiprocess and shared_data_buffers is not None: From eb4e6bad79fc116cd4a77e54020edfee26ab2347 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 21 Jul 2026 12:29:57 -0400 Subject: [PATCH 41/96] Fix resume_after bug --- activitysim/core/calibration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index fa554043d9..faf09dd8fc 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -205,7 +205,7 @@ def run_calibration_loop( _run_precursor_components( state, models=models[:first_calib_model_idx], - resume_after=calibration_settings.run.resume_after + resume_after=state.settings.resume_after if global_iter == 1 else _prior_step_name( models, calibration_settings.run.calibrate_models[0] From de119ed12c35190fde86d83f16a05b746203aecf Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 22 Jul 2026 16:57:16 -0400 Subject: [PATCH 42/96] partial MP resume_after fix --- activitysim/core/calibration.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index faf09dd8fc..6e01bcb159 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -206,7 +206,7 @@ def run_calibration_loop( state, models=models[:first_calib_model_idx], resume_after=state.settings.resume_after - if global_iter == 1 + if global_iter == start_global_iter else _prior_step_name( models, calibration_settings.run.calibrate_models[0] ), @@ -300,7 +300,7 @@ def _run_precursor_components( ) -> None: """Run the normal ActivitySim model flow for one global calibration iteration.""" - if global_iter > 1: + if global_iter > 1 and resume_after is not None: # Seed a fresh pipeline from the configured resume checkpoint to avoid # duplicate checkpoint-name collisions across global calibration loops. prior_pipeline = state.checkpoint.store.filename @@ -1454,7 +1454,7 @@ def _run_multiprocess_with_overrides( injectables, shared_data_buffers=shared_data_buffers, skip_final_checkpoint=True, - force_resume=True, + force_resume=resume_after is not None, ) finally: state.settings.models = original_models From f0e9ccb7411ed1168bde40bfdf189d69830c2dfc Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 29 Jul 2026 19:42:14 -0400 Subject: [PATCH 43/96] More checkpointing work --- activitysim/core/calibration.py | 181 ++++++++++++++++++------ activitysim/core/workflow/checkpoint.py | 10 +- 2 files changed, 141 insertions(+), 50 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 6e01bcb159..11df42f26a 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -163,6 +163,12 @@ def run_calibration_loop( if not calibration_settings or not calibration_settings.enable: raise RuntimeError("calibration loop called while calibration is disabled") + if state.settings.duplicate_step_execution != "allow": + state.settings.duplicate_step_execution = "allow" + logger.warning( + "Overriding duplicate_step_execution setting: must be enabled for calibration" + ) + assert all( [c in models for c in calibration_settings.run.calibrate_models] ), f"settings.yaml steps list does not include calibration model{'s' if len([c for c in calibration_settings.run.calibrate_models if c not in models]) != 1 else ''} {[c for c in calibration_settings.run.calibrate_models if c not in models]}" @@ -172,6 +178,12 @@ def run_calibration_loop( calibration_settings.run.calibrate_models, key=lambda x: models.index(x) ) first_calib_model_idx = models.index(calibration_settings.run.calibrate_models[0]) + last_calib_model_idx = models.index(calibration_settings.run.calibrate_models[-1]) + first_model_idx = ( + models.index(state.settings.resume_after) + 1 + if state.settings.resume_after + else None + ) _ensure_calibration_output_dir(state) @@ -191,6 +203,23 @@ def run_calibration_loop( shared_data_buffers = _initialize_mp_shared_resources(state) try: + # skip precursors if, on first iter, resume_after exists and is >= first_calib_model_idx + if ( + state.settings.resume_after is None + or first_model_idx < first_calib_model_idx + ): + # Run ActivitySim normally from resume_after through production model steps. + _run_precursor_components( + state, + models=models[:first_calib_model_idx] + if first_model_idx is None + else models[first_model_idx:first_calib_model_idx], + resume_after=state.settings.resume_after, + global_iter=start_global_iter, + memory_sidecar_process=memory_sidecar_process, + shared_data_buffers=shared_data_buffers, + ) + for global_iter in range( start_global_iter, start_global_iter + calibration_settings.run.global_iterations, @@ -201,24 +230,20 @@ def run_calibration_loop( calibration_settings.run.global_iterations, ) - # Run ActivitySim normally from resume_after through production model steps. - _run_precursor_components( - state, - models=models[:first_calib_model_idx], - resume_after=state.settings.resume_after - if global_iter == start_global_iter - else _prior_step_name( - models, calibration_settings.run.calibrate_models[0] - ), - global_iter=global_iter, - memory_sidecar_process=memory_sidecar_process, - shared_data_buffers=shared_data_buffers, - ) - - all_converged = True + # suppress early termination on first iteration if resume_after is after all calibrated models + all_converged = ( + first_model_idx is not None and first_model_idx <= last_calib_model_idx + ) or global_iter > start_global_iter last_calibrated_component = None for component in calibration_settings.run.calibrate_models: + # on the first global iter, skip model if it's before or == resume_after + if ( + global_iter == start_global_iter + and first_model_idx is not None + and first_model_idx > models.index(component) + ): + continue component_settings = calibration_settings.model_settings[component] prior_step = _prior_step_name(models, component) @@ -251,15 +276,32 @@ def run_calibration_loop( last_calibrated_component = component - if calibration_settings.run.complete_steps or ( - start_global_iter + calibration_settings.run.global_iterations - == global_iter + 1 + if ( + calibration_settings.run.complete_steps + or ( + start_global_iter + calibration_settings.run.global_iterations + == global_iter + 1 + ) + or ( + global_iter == start_global_iter + and state.settings.resume_after is not None + and first_model_idx > last_calib_model_idx + ) ): + subsequent_components = ( + models[first_model_idx:] + if global_iter == start_global_iter + and first_model_idx > last_calib_model_idx + else models[models.index(last_calibrated_component) + 1 :] + ) # finish the full model chain _run_subsequent_components( state, - models=models[models.index(last_calibrated_component) + 1 :], - resume_after=last_calibrated_component, + models=subsequent_components, + resume_after=state.settings.resume_after + if global_iter == start_global_iter + and first_model_idx > last_calib_model_idx + else last_calibrated_component, memory_sidecar_process=memory_sidecar_process, shared_data_buffers=shared_data_buffers, ) @@ -300,21 +342,22 @@ def _run_precursor_components( ) -> None: """Run the normal ActivitySim model flow for one global calibration iteration.""" - if global_iter > 1 and resume_after is not None: - # Seed a fresh pipeline from the configured resume checkpoint to avoid - # duplicate checkpoint-name collisions across global calibration loops. - prior_pipeline = state.checkpoint.store.filename - state.checkpoint.close_store() - state.filesystem.pipeline_file_name = f"pipeline_calibration_iter_{global_iter}" - state.checkpoint.restore_from(prior_pipeline, checkpoint_name=resume_after) - else: - _run_in_configured_mode( - state, - models=models, - resume_after=resume_after, - memory_sidecar_process=memory_sidecar_process, - shared_data_buffers=shared_data_buffers, - ) + # if global_iter > 1 and resume_after is not None: + # # Seed a fresh pipeline from the configured resume checkpoint to avoid + # # duplicate checkpoint-name collisions across global calibration loops. + # prior_pipeline = state.checkpoint.store.filename + # state.checkpoint.close_store() + # state.filesystem.pipeline_file_name = f"pipeline_calibration_iter_{global_iter}" + # state.checkpoint.restore_from(prior_pipeline, checkpoint_name=resume_after) + # else: + + _run_in_configured_mode( + state, + models=models, + resume_after=resume_after, + memory_sidecar_process=memory_sidecar_process, + shared_data_buffers=shared_data_buffers, + ) def _run_intermediate_components( @@ -425,7 +468,9 @@ def _calibrate_component( _run_mp_single_component( state, component_name=component_name, - restore_checkpoint=mp_restore_checkpoint, + restore_checkpoint=mp_restore_checkpoint + if component_iter == 1 + else "_", shared_data_buffers=shared_data_buffers, ) else: @@ -1305,11 +1350,9 @@ def _run_mp_single_component( chunk_size = step.chunk_size break - step_name = f"calibration_{component_name}" - # Build step_info dict matching what mp_tasks functions expect step_info = { - "name": step_name, + "name": component_name, "models": [component_name], "num_processes": num_processes, "chunk_size": chunk_size, @@ -1321,9 +1364,9 @@ def _run_mp_single_component( injectables = _build_calibration_injectables(state) if num_processes == 1: - sub_proc_names = [step_name] + sub_proc_names = [component_name] else: - sub_proc_names = [f"{step_name}_{i}" for i in range(num_processes)] + sub_proc_names = [f"{component_name}_{i}" for i in range(num_processes)] fail_fast = state.settings.fail_fast @@ -1333,7 +1376,7 @@ def _run_mp_single_component( state, multiprocessing.Process( target=mp_tasks.mp_apportion_pipeline, - name=f"{step_name}_apportion", + name=f"{component_name}_apportion", args=(injectables, sub_proc_names, step_info), ), ) @@ -1365,7 +1408,7 @@ def _run_mp_single_component( raise SubprocessError( f"{num_processes - len(completed)} processes failed in " - f"calibration step {step_name}" + f"calibration step {component_name}" ) # Coalesce sub-process pipelines back into main pipeline @@ -1374,7 +1417,7 @@ def _run_mp_single_component( state, multiprocessing.Process( target=mp_tasks.mp_coalesce_pipelines, - name=f"{step_name}_coalesce", + name=f"{component_name}_coalesce", args=(injectables, sub_proc_names, slice_info), ), ) @@ -1394,6 +1437,8 @@ def _run_in_configured_mode( if not models: return + _prep_model_data(state, models) + if state.settings.multiprocess: _run_multiprocess_with_overrides( state, @@ -1419,6 +1464,52 @@ def _run_in_configured_mode( ) +def _prep_model_data(state, models): + checkpoint = False + _restore_parent_state_from_pipeline(state) + if "compute_accessibility" in models and state.is_table("accessibility"): + state.add_table( + "accessibility", pd.DataFrame(index=state.get_table("accessibility").index) + ) + checkpoint = True + if "mandatory_tour_frequency" in models and state.is_table("tours"): + state.add_table( + "tours", + state.get_table("tours")[ + state.get_table("tours").tour_category != "mandatory" + ], + ) + checkpoint = True + + if "non_mandatory_tour_frequency" in models and state.is_table("tours"): + state.add_table( + "tours", + state.get_table("tours")[ + state.get_table("tours").tour_category != "non_mandatory" + ], + ) + checkpoint = True + + if "joint_tour_frequency" in models and state.is_table("tours"): + state.add_table( + "tours", + state.get_table("tours")[state.get_table("tours").tour_category != "joint"], + ) + checkpoint = True + + if "atwork_subtour_frequency" in models and state.is_table("tours"): + state.add_table( + "tours", + state.get_table("tours")[ + state.get_table("tours").tour_category != "atwork" + ], + ) + checkpoint = True + + if checkpoint: + state.checkpoint.add(state.checkpoint.checkpoints[-1]["checkpoint_name"]) + + def _run_multiprocess_with_overrides( state: workflow.State, models: list[str], @@ -1596,7 +1687,7 @@ def _build_calibration_mp_steps( for step_idx, step_models in step_model_groups.items(): orig_step = original_steps[step_idx] kwargs: dict[str, Any] = { - "name": f"calibration_{orig_step.name}_{step_models[0]}", + "name": orig_step.name, "begin": step_models[0], } if orig_step.num_processes is not None: diff --git a/activitysim/core/workflow/checkpoint.py b/activitysim/core/workflow/checkpoint.py index ec721a4790..63b5583254 100644 --- a/activitysim/core/workflow/checkpoint.py +++ b/activitysim/core/workflow/checkpoint.py @@ -767,7 +767,7 @@ def load(self, checkpoint_name: str, store=None): try: # truncate rows after target checkpoint - i = checkpoints[checkpoints[CHECKPOINT_NAME] == checkpoint_name].index[0] + i = checkpoints[checkpoints[CHECKPOINT_NAME] == checkpoint_name].index[-1] checkpoints = checkpoints.loc[:i] # if the store is not open in read-only mode, @@ -1231,10 +1231,10 @@ def load_dataframe(self, table_name, checkpoint_name=None): return self._obj.get_dataframe(table_name) # find the requested checkpoint - checkpoint = next( - (x for x in self.checkpoints if x["checkpoint_name"] == checkpoint_name), - None, - ) + matching_checkpoints = [ + x for x in self.checkpoints if x["checkpoint_name"] == checkpoint_name + ] + checkpoint = matching_checkpoints[-1] if matching_checkpoints else None if checkpoint is None: raise CheckpointNameNotFoundError( "checkpoint '%s' not in checkpoints." % checkpoint_name From c9afece562f8cdbb890cf94cfe564e97ea8f9450 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 29 Jul 2026 19:45:17 -0400 Subject: [PATCH 44/96] Add tour mode choice handler --- activitysim/core/calibration.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 11df42f26a..7aae3cbb3e 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1497,7 +1497,9 @@ def _prep_model_data(state, models): ) checkpoint = True - if "atwork_subtour_frequency" in models and state.is_table("tours"): + if ( + "atwork_subtour_frequency" in models or "tour_mode_choice_simulate" in models + ) and state.is_table("tours"): state.add_table( "tours", state.get_table("tours")[ From ba8a14287f1e8993c273ba99ae11cb78c40e17f4 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 30 Jul 2026 11:32:17 -0700 Subject: [PATCH 45/96] Clean checkpointing --- activitysim/core/calibration.py | 131 ++++++++++++++++++++------------ 1 file changed, 82 insertions(+), 49 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 7aae3cbb3e..2c01f277b2 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1437,7 +1437,7 @@ def _run_in_configured_mode( if not models: return - _prep_model_data(state, models) + _prep_model_data(state, models, resume_after=resume_after) if state.settings.multiprocess: _run_multiprocess_with_overrides( @@ -1464,52 +1464,73 @@ def _run_in_configured_mode( ) -def _prep_model_data(state, models): - checkpoint = False - _restore_parent_state_from_pipeline(state) - if "compute_accessibility" in models and state.is_table("accessibility"): - state.add_table( - "accessibility", pd.DataFrame(index=state.get_table("accessibility").index) - ) - checkpoint = True - if "mandatory_tour_frequency" in models and state.is_table("tours"): - state.add_table( - "tours", - state.get_table("tours")[ - state.get_table("tours").tour_category != "mandatory" - ], - ) - checkpoint = True - - if "non_mandatory_tour_frequency" in models and state.is_table("tours"): - state.add_table( - "tours", - state.get_table("tours")[ - state.get_table("tours").tour_category != "non_mandatory" - ], - ) - checkpoint = True +def _prep_model_data(state, models, resume_after=None): + """Restore the pipeline to the correct state before running models. - if "joint_tour_frequency" in models and state.is_table("tours"): - state.add_table( - "tours", - state.get_table("tours")[state.get_table("tours").tour_category != "joint"], - ) - checkpoint = True - - if ( - "atwork_subtour_frequency" in models or "tour_mode_choice_simulate" in models - ) and state.is_table("tours"): - state.add_table( - "tours", - state.get_table("tours")[ - state.get_table("tours").tour_category != "atwork" - ], - ) - checkpoint = True + When a specific ``resume_after`` checkpoint exists in the pipeline, we + restore from it directly. This gives the exact state as it was when that + model completed — without downstream data that later models may have added + (e.g. tours from mandatory_tour_frequency polluting the state for a re-run + of that same model). + + If ``resume_after`` is not available as a named checkpoint (e.g. first + calibration run from a multiprocess pipeline with only step-level + checkpoints), fall back to LAST_CHECKPOINT which is the most recently + written state. + """ + if resume_after: + # Try to restore from the exact resume_after checkpoint + try: + if state.checkpoint.store_is_open(): + checkpoint_names = [ + cp.get("checkpoint_name", "") for cp in state.checkpoint.checkpoints + ] + else: + from activitysim.core.workflow.checkpoint import HdfStore, ParquetStore + + pipeline_path = Path(state.checkpoint.default_pipeline_file_path()) + if state.settings.checkpoint_format == "hdf": + store = HdfStore(pipeline_path, mode="r") + else: + store = ParquetStore(pipeline_path, mode="r") + try: + checkpoint_names = store.list_checkpoint_names() + finally: + store.close() + + if resume_after in checkpoint_names: + _restore_parent_state_from_pipeline(state, checkpoint_name=resume_after) + return + + # Resolve to step-level checkpoint if model-level not found. + # Find the multiprocess step that contains resume_after and use + # its step name as the checkpoint. + resolved = None + all_models = state.settings.models + mp_steps = state.settings.multiprocess_steps + if mp_steps and resume_after in all_models: + resume_idx = all_models.index(resume_after) + step_boundaries = [all_models.index(s.begin) for s in mp_steps] + step_boundaries.append(len(all_models)) + for i, step in enumerate(mp_steps): + if step_boundaries[i] <= resume_idx < step_boundaries[i + 1]: + if step.name in checkpoint_names: + resolved = step.name + break + + if resolved: + _restore_parent_state_from_pipeline(state, checkpoint_name=resolved) + return + except Exception: + logger.warning( + "calibration: could not restore from checkpoint %r, " + "falling back to LAST_CHECKPOINT", + resume_after, + ) - if checkpoint: - state.checkpoint.add(state.checkpoint.checkpoints[-1]["checkpoint_name"]) + # Fallback: load LAST_CHECKPOINT (appropriate after a coalesce that + # only ran the desired models) + _restore_parent_state_from_pipeline(state) def _run_multiprocess_with_overrides( @@ -1555,20 +1576,32 @@ def _run_multiprocess_with_overrides( state.settings.multiprocess_steps = original_mp_steps -def _restore_parent_state_from_pipeline(state: workflow.State) -> None: - """Restore coalesced pipeline tables into the parent process state. +def _restore_parent_state_from_pipeline( + state: workflow.State, checkpoint_name: str = "_" +) -> None: + """Restore pipeline tables into the parent process state. After a multiprocess run, the parent's in-memory state is stale. - This loads the latest checkpoint from the pipeline store so that + This loads a specific checkpoint from the pipeline store so that calibration expressions can evaluate against model outputs. + Parameters + ---------- + checkpoint_name : str, default "_" + The checkpoint to restore from. Use a model-level checkpoint name + (e.g. the prior step name) to get the exact state at that point, + avoiding pollution from downstream models that may have added rows + to shared tables like ``tours``. The default ``"_"`` loads the + last checkpoint, which is appropriate immediately after a coalesce + that only ran the desired models. + All tables are explicitly re-checkpointed so that subsequent apportion subprocesses can load them from a direct file path without relying on checkpoint backtracking through potentially ambiguous checkpoint history. """ if state.checkpoint.store_is_open(): state.checkpoint.close_store() - state.checkpoint.restore(resume_after="_") + state.checkpoint.restore(resume_after=checkpoint_name) # After restore, all tables are clean (status=False). Mark them dirty so # the next checkpoint.add() writes them to disk at a known checkpoint name. From c7f3c7756c8587b9524e55c9952765e1675c9553 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 30 Jul 2026 12:24:34 -0700 Subject: [PATCH 46/96] More checkpointing fixes --- activitysim/core/calibration.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 2c01f277b2..b9fb437e4c 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -219,6 +219,14 @@ def run_calibration_loop( memory_sidecar_process=memory_sidecar_process, shared_data_buffers=shared_data_buffers, ) + else: + # Precursors skipped — but the pipeline must still be initialized + # at the resume_after point so that _calibrate_component (and its + # apportion subprocess) starts from the correct state without + # downstream model data. + _prep_model_data(state, [], resume_after=state.settings.resume_after) + state.checkpoint.add(state.settings.resume_after) + state.checkpoint.close_store() for global_iter in range( start_global_iter, @@ -1440,6 +1448,14 @@ def _run_in_configured_mode( _prep_model_data(state, models, resume_after=resume_after) if state.settings.multiprocess: + # Write the restored state as a checkpoint so LAST_CHECKPOINT on disk + # reflects the correct (clean) state for the apportion subprocess. + # Without this, Parquet stores retain a stale LAST_CHECKPOINT from a + # prior run that may include downstream data (e.g. non-mandatory tours + # polluting a re-run of non_mandatory_tour_frequency). + state.checkpoint.add(resume_after or models[0]) + state.checkpoint.close_store() + _run_multiprocess_with_overrides( state, models=models, From c9aca4575828640de5c8359afa1c44bf03970e77 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 30 Jul 2026 12:58:42 -0700 Subject: [PATCH 47/96] Restore from subprocess checkpoints --- activitysim/core/calibration.py | 256 ++++++++++++++++++++++++++++---- 1 file changed, 230 insertions(+), 26 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index b9fb437e4c..ebe5fe4668 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -224,9 +224,23 @@ def run_calibration_loop( # at the resume_after point so that _calibrate_component (and its # apportion subprocess) starts from the correct state without # downstream model data. - _prep_model_data(state, [], resume_after=state.settings.resume_after) - state.checkpoint.add(state.settings.resume_after) - state.checkpoint.close_store() + extra_models = _prep_model_data( + state, [], resume_after=state.settings.resume_after + ) + if extra_models: + # No model-level checkpoint exists for resume_after; we must + # run models from the prior step through resume_after to + # recreate the correct intermediate state. + _run_in_configured_mode( + state, + models=extra_models, + resume_after=None, + memory_sidecar_process=memory_sidecar_process, + shared_data_buffers=shared_data_buffers, + ) + else: + state.checkpoint.add(state.settings.resume_after) + state.checkpoint.close_store() for global_iter in range( start_global_iter, @@ -1445,7 +1459,12 @@ def _run_in_configured_mode( if not models: return - _prep_model_data(state, models, resume_after=resume_after) + extra_models = _prep_model_data(state, models, resume_after=resume_after) + if extra_models: + # Models from the step beginning through resume_after must run first + # to recreate the correct intermediate state (since no model-level + # checkpoint existed for resume_after). + models = extra_models + models if state.settings.multiprocess: # Write the restored state as a checkpoint so LAST_CHECKPOINT on disk @@ -1483,19 +1502,22 @@ def _run_in_configured_mode( def _prep_model_data(state, models, resume_after=None): """Restore the pipeline to the correct state before running models. - When a specific ``resume_after`` checkpoint exists in the pipeline, we - restore from it directly. This gives the exact state as it was when that - model completed — without downstream data that later models may have added - (e.g. tours from mandatory_tour_frequency polluting the state for a re-run - of that same model). - - If ``resume_after`` is not available as a named checkpoint (e.g. first - calibration run from a multiprocess pipeline with only step-level - checkpoints), fall back to LAST_CHECKPOINT which is the most recently - written state. + Resolution priority: + 1. Direct model-level checkpoint in the main pipeline (fastest, exact). + 2. Subprocess pipelines from a prior multiprocess run — performs a + "coalesce at specific checkpoint" to recover the exact intermediate + state without re-running anything. + 3. Previous step checkpoint + re-run models from step begin through + resume_after (slowest but always works). + + Returns + ------- + list[str] + Models that must be prepended to the caller's models list to reach + the correct state at ``resume_after``. Empty when the exact + checkpoint was found and restored directly (paths 1 or 2). """ if resume_after: - # Try to restore from the exact resume_after checkpoint try: if state.checkpoint.store_is_open(): checkpoint_names = [ @@ -1514,14 +1536,16 @@ def _prep_model_data(state, models, resume_after=None): finally: store.close() + # Path 1: direct model-level checkpoint in main pipeline if resume_after in checkpoint_names: _restore_parent_state_from_pipeline(state, checkpoint_name=resume_after) - return + return [] + + # Path 2: subprocess pipelines (model-level checkpoints preserved) + if _restore_from_subprocess_pipelines(state, resume_after): + return [] - # Resolve to step-level checkpoint if model-level not found. - # Find the multiprocess step that contains resume_after and use - # its step name as the checkpoint. - resolved = None + # Path 3: restore from previous step and re-run all_models = state.settings.models mp_steps = state.settings.multiprocess_steps if mp_steps and resume_after in all_models: @@ -1530,13 +1554,20 @@ def _prep_model_data(state, models, resume_after=None): step_boundaries.append(len(all_models)) for i, step in enumerate(mp_steps): if step_boundaries[i] <= resume_idx < step_boundaries[i + 1]: - if step.name in checkpoint_names: - resolved = step.name + if i > 0 and mp_steps[i - 1].name in checkpoint_names: + _restore_parent_state_from_pipeline( + state, checkpoint_name=mp_steps[i - 1].name + ) + step_begin_idx = step_boundaries[i] + extra_models = all_models[step_begin_idx : resume_idx + 1] + return extra_models + elif i == 0: + _restore_parent_state_from_pipeline( + state, checkpoint_name="_" + ) + extra_models = all_models[: resume_idx + 1] + return extra_models break - - if resolved: - _restore_parent_state_from_pipeline(state, checkpoint_name=resolved) - return except Exception: logger.warning( "calibration: could not restore from checkpoint %r, " @@ -1547,6 +1578,179 @@ def _prep_model_data(state, models, resume_after=None): # Fallback: load LAST_CHECKPOINT (appropriate after a coalesce that # only ran the desired models) _restore_parent_state_from_pipeline(state) + return [] + + +def _restore_from_subprocess_pipelines( + state: workflow.State, resume_after: str +) -> bool: + """Restore state from subprocess pipelines at a specific model checkpoint. + + Subprocess pipelines retain model-level checkpoints that don't exist in + the main pipeline. This performs a "coalesce at checkpoint" — reading + mirrored tables from one subprocess and concatenating sliced tables from + all subprocesses at the specified checkpoint name. + + Parameters + ---------- + state : workflow.State + resume_after : str + Model-level checkpoint name to restore from. + + Returns + ------- + bool + True if the restore succeeded; False if subprocess pipelines don't + exist or don't contain the requested checkpoint. + """ + from activitysim.core.workflow.checkpoint import ( + CHECKPOINT_NAME, + CHECKPOINT_TABLE_NAME, + HdfStore, + NON_TABLE_COLUMNS, + ParquetStore, + ) + + all_models = state.settings.models + mp_steps = state.settings.multiprocess_steps + if not mp_steps or resume_after not in all_models: + return False + + # Find the multiprocess step containing resume_after + resume_idx = all_models.index(resume_after) + step_boundaries = [all_models.index(s.begin) for s in mp_steps] + step_boundaries.append(len(all_models)) + + enclosing_step = None + num_processes = state.settings.num_processes or 2 + slice_info = None + for i, step in enumerate(mp_steps): + if step_boundaries[i] <= resume_idx < step_boundaries[i + 1]: + enclosing_step = step + if step.num_processes: + num_processes = step.num_processes + if step.slice: + slice_info = ( + step.slice.model_dump() + if hasattr(step.slice, "model_dump") + else step.slice + ) + break + + if enclosing_step is None or num_processes <= 1: + return False + + # Build subprocess pipeline file paths + step_name = enclosing_step.name + pipeline_file_name = state.filesystem.pipeline_file_name + sub_proc_names = [f"{step_name}_{i}" for i in range(num_processes)] + + def _subprocess_path(proc_name): + base = state.get_output_file_path(pipeline_file_name, prefix=proc_name) + if state.settings.checkpoint_format == "hdf": + return base + pq = Path(str(base)).with_suffix(ParquetStore.extension) + return pq if pq.exists() else base + + first_path = _subprocess_path(sub_proc_names[0]) + if not first_path.exists(): + return False + + # Open first subprocess pipeline and verify checkpoint exists + if state.settings.checkpoint_format == "hdf": + first_store = HdfStore(first_path, mode="r") + else: + first_store = ParquetStore(first_path, mode="r") + + try: + cp_names = first_store.list_checkpoint_names() + if resume_after not in cp_names: + return False + + # Read checkpoint row to get table→checkpoint mapping + cp_df = first_store.get_dataframe(CHECKPOINT_TABLE_NAME) + cp_row = cp_df[cp_df[CHECKPOINT_NAME] == resume_after].iloc[-1] + + table_map = {} + for col in cp_row.index: + if col not in NON_TABLE_COLUMNS and cp_row[col]: + table_map[col] = cp_row[col] + + # Read all tables from first subprocess at this checkpoint + tables = {} + for table_name, cp_for_table in table_map.items(): + try: + tables[table_name] = first_store.get_dataframe(table_name, cp_for_table) + except (FileNotFoundError, KeyError): + logger.warning( + f"calibration: subprocess pipeline missing table " + f"{table_name} at {cp_for_table}" + ) + finally: + first_store.close() + + if not tables: + return False + + # Determine sliced tables that need concatenation across processes + sliced_table_names = set(slice_info.get("tables", [])) if slice_info else set() + + # Read sliced tables from remaining subprocesses and concatenate + if num_processes > 1 and sliced_table_names: + omnibus = {t: [tables[t]] for t in sliced_table_names if t in tables} + + for proc_name in sub_proc_names[1:]: + proc_path = _subprocess_path(proc_name) + if not proc_path.exists(): + logger.warning( + f"calibration: subprocess pipeline not found: {proc_path}" + ) + return False + + if state.settings.checkpoint_format == "hdf": + proc_store = HdfStore(proc_path, mode="r") + else: + proc_store = ParquetStore(proc_path, mode="r") + + try: + proc_cp_df = proc_store.get_dataframe(CHECKPOINT_TABLE_NAME) + proc_row = proc_cp_df[proc_cp_df[CHECKPOINT_NAME] == resume_after].iloc[ + -1 + ] + + for table_name in list(omnibus.keys()): + cp_for_table = proc_row.get(table_name, "") + if cp_for_table: + omnibus[table_name].append( + proc_store.get_dataframe(table_name, cp_for_table) + ) + finally: + proc_store.close() + + # Replace sliced tables with concatenated versions + for table_name, dfs in omnibus.items(): + tables[table_name] = pd.concat(dfs, sort=False) + + # Load into parent state + state.init_state() + if state.checkpoint.store_is_open(): + state.checkpoint.close_store() + state.checkpoint.open_store(overwrite=False) + + for table_name, df in tables.items(): + state.add_table(table_name, df) + + # Mark all tables dirty for subsequent checkpoint.add + for table_name in list(state.existing_table_names): + state.existing_table_status[table_name] = True + + logger.info( + "calibration: restored %d tables from subprocess pipelines at " + "checkpoint '%s'", + len(tables), + resume_after, + ) + return True def _run_multiprocess_with_overrides( From a19638bab8e6def343f5945da339dad3cb3c9953 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Fri, 31 Jul 2026 10:15:52 -0700 Subject: [PATCH 48/96] resume_after bugfix --- activitysim/core/calibration.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index ebe5fe4668..469e8b929b 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1469,10 +1469,14 @@ def _run_in_configured_mode( if state.settings.multiprocess: # Write the restored state as a checkpoint so LAST_CHECKPOINT on disk # reflects the correct (clean) state for the apportion subprocess. - # Without this, Parquet stores retain a stale LAST_CHECKPOINT from a - # prior run that may include downstream data (e.g. non-mandatory tours - # polluting a re-run of non_mandatory_tour_frequency). - state.checkpoint.add(resume_after or models[0]) + # When extra_models were prepended, the state is from a PRIOR step + # (not the actual resume_after point) — use a non-conflicting name + # so it won't be mistakenly loaded as the resume_after state on a + # subsequent restart. + if extra_models: + state.checkpoint.add(f"_calibration_staging") + else: + state.checkpoint.add(resume_after or models[0]) state.checkpoint.close_store() _run_multiprocess_with_overrides( From 2fd3db779c153ba0f7513100def557c372e5d46f Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 4 Aug 2026 17:42:51 -0500 Subject: [PATCH 49/96] add subproc reuse --- activitysim/core/calibration.py | 71 ++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 10 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 469e8b929b..411e063ac9 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -225,7 +225,7 @@ def run_calibration_loop( # apportion subprocess) starts from the correct state without # downstream model data. extra_models = _prep_model_data( - state, [], resume_after=state.settings.resume_after + state, resume_after=state.settings.resume_after ) if extra_models: # No model-level checkpoint exists for resume_after; we must @@ -1459,7 +1459,7 @@ def _run_in_configured_mode( if not models: return - extra_models = _prep_model_data(state, models, resume_after=resume_after) + extra_models = _prep_model_data(state, resume_after=resume_after) if extra_models: # Models from the step beginning through resume_after must run first # to recreate the correct intermediate state (since no model-level @@ -1479,11 +1479,19 @@ def _run_in_configured_mode( state.checkpoint.add(resume_after or models[0]) state.checkpoint.close_store() + # When subprocess pipelines from a prior run already have the + # resume_after checkpoint (Path 2 in _prep_model_data), subprocesses + # can skip models before resume_after by reusing those pipelines + # instead of freshly apportioning. Signal this by passing + # can_reuse_subprocs=True. + can_reuse = not extra_models and resume_after is not None + _run_multiprocess_with_overrides( state, models=models, resume_after=resume_after, shared_data_buffers=shared_data_buffers, + can_reuse_subprocs=can_reuse, ) # After multiprocess completes, the coalesced pipeline exists on disk. # Restore it into the parent process state so tables are accessible @@ -1503,7 +1511,7 @@ def _run_in_configured_mode( ) -def _prep_model_data(state, models, resume_after=None): +def _prep_model_data(state, resume_after=None): """Restore the pipeline to the correct state before running models. Resolution priority: @@ -1762,8 +1770,22 @@ def _run_multiprocess_with_overrides( models: list[str], resume_after: str | None, shared_data_buffers: dict | None = None, + can_reuse_subprocs: bool = False, ) -> None: - """Run multiprocess with temporary settings overrides for calibration passes.""" + """Run multiprocess with temporary settings overrides for calibration passes. + + Parameters + ---------- + can_reuse_subprocs : bool, default False + When True, subprocess pipelines from a prior run are assumed to exist + and contain the ``resume_after`` checkpoint. Breadcrumbs are written + so that ``get_run_list`` populates ``step_info["resume_after"]``, + apportion is skipped (reusing existing subprocess pipelines), and + subprocesses resume from their model-level checkpoint — skipping + already-completed models. + """ + from collections import OrderedDict + from activitysim.core import mp_tasks original_models = state.settings.models @@ -1779,11 +1801,40 @@ def _run_multiprocess_with_overrides( state.settings.models = models state.settings.multiprocess_steps = calibration_mp_steps - # Always None: calibration manages pipeline state externally via - # _restore_parent_state_from_pipeline and checkpoint.add, so the MP - # system's breadcrumb-based resume logic must not be triggered. - # Apportion uses LAST_CHECKPOINT to read from the current pipeline state. - state.settings.resume_after = None + + if can_reuse_subprocs and resume_after: + # Enable the normal resume mechanism: set resume_after in settings + # and write breadcrumbs so get_run_list can properly populate + # step_info["resume_after"]. Apportion will be skipped (prior + # subprocess pipelines reused), and subprocesses will restore from + # their existing model-level checkpoint. + state.settings.resume_after = resume_after + + # Write minimal breadcrumbs indicating the step containing + # resume_after has completed apportion (so it's skipped) but + # simulate/coalesce need re-running. + breadcrumbs = OrderedDict() + for step in calibration_mp_steps: + step_dict = {"name": step.name, "apportion": True} + breadcrumbs[step.name] = step_dict + # Find the step containing resume_after + all_models = state.settings.models + if resume_after in all_models: + step_begin = all_models.index(step.begin) + step_models_in_step = [ + m for m in all_models[step_begin:] if m in models + ] + if resume_after in step_models_in_step: + # This step contains resume_after — stop here. + # get_breadcrumbs will mark simulate/coalesce for re-run. + break + + mp_tasks.write_breadcrumbs(state, breadcrumbs) + else: + # No reuse: calibration manages pipeline state externally via + # _restore_parent_state_from_pipeline and checkpoint.add, so the MP + # system's breadcrumb-based resume logic must not be triggered. + state.settings.resume_after = None try: injectables = _build_calibration_injectables(state) @@ -1792,7 +1843,7 @@ def _run_multiprocess_with_overrides( injectables, shared_data_buffers=shared_data_buffers, skip_final_checkpoint=True, - force_resume=resume_after is not None, + force_resume=resume_after is not None and not can_reuse_subprocs, ) finally: state.settings.models = original_models From cc344c7c78f0fec28f3b1f085fd1dc867f94227a Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 5 Aug 2026 13:41:01 -0500 Subject: [PATCH 50/96] Modify breadcrumb logic --- activitysim/core/calibration.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 411e063ac9..b75171febe 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1803,11 +1803,19 @@ def _run_multiprocess_with_overrides( state.settings.multiprocess_steps = calibration_mp_steps if can_reuse_subprocs and resume_after: - # Enable the normal resume mechanism: set resume_after in settings - # and write breadcrumbs so get_run_list can properly populate - # step_info["resume_after"]. Apportion will be skipped (prior - # subprocess pipelines reused), and subprocesses will restore from - # their existing model-level checkpoint. + # Include resume_after in the models list so get_breadcrumbs can + # locate the step containing it. Subprocesses will skip this model + # (it's already checkpointed in their pipeline) and run the rest. + models = [resume_after] + models + + # Rebuild steps with resume_after included. + calibration_mp_steps = _build_calibration_mp_steps( + models=models, + original_steps=original_mp_steps, + all_models=original_models, + ) + state.settings.models = models + state.settings.multiprocess_steps = calibration_mp_steps state.settings.resume_after = resume_after # Write minimal breadcrumbs indicating the step containing From fcc65672d3387e97e5fa245c70ded4ce89249038 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 08:59:27 -0500 Subject: [PATCH 51/96] Add RNG channel reloading --- activitysim/core/calibration.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index b75171febe..201c0abd28 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1744,6 +1744,8 @@ def _subprocess_path(proc_name): tables[table_name] = pd.concat(dfs, sort=False) # Load into parent state + prior_rng_channels = list(state.get_injectable("rng_channels", [])) + state.init_state() if state.checkpoint.store_is_open(): state.checkpoint.close_store() @@ -1752,6 +1754,8 @@ def _subprocess_path(proc_name): for table_name, df in tables.items(): state.add_table(table_name, df) + _reregister_rng_channels(state, prior_rng_channels) + # Mark all tables dirty for subsequent checkpoint.add for table_name in list(state.existing_table_names): state.existing_table_status[table_name] = True @@ -1859,6 +1863,19 @@ def _run_multiprocess_with_overrides( state.settings.multiprocess_steps = original_mp_steps +def _reregister_rng_channels(state: workflow.State, prior_channels: list[str]) -> None: + """Re-register RNG channels that were lost during init_state().""" + current_channels = set(state.get_injectable("rng_channels", [])) + for channel_name in prior_channels: + if channel_name not in current_channels and state.is_table(channel_name): + try: + state.rng().add_channel(channel_name, state.get_dataframe(channel_name)) + current_channels.add(channel_name) + except Exception: + pass + state.add_injectable("rng_channels", list(current_channels)) + + def _restore_parent_state_from_pipeline( state: workflow.State, checkpoint_name: str = "_" ) -> None: @@ -1882,10 +1899,17 @@ def _restore_parent_state_from_pipeline( subprocesses can load them from a direct file path without relying on checkpoint backtracking through potentially ambiguous checkpoint history. """ + # Capture RNG channels before restore — models may have dynamically + # added channels (e.g. "vehicles") that aren't in the default + # rng_channels injectable and would be lost by init_state(). + prior_rng_channels = list(state.get_injectable("rng_channels", [])) + if state.checkpoint.store_is_open(): state.checkpoint.close_store() state.checkpoint.restore(resume_after=checkpoint_name) + _reregister_rng_channels(state, prior_rng_channels) + # After restore, all tables are clean (status=False). Mark them dirty so # the next checkpoint.add() writes them to disk at a known checkpoint name. # This ensures apportion subprocesses find table files at a single, From a854b67560445c09880e617eb3dcba25e4e847ad Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 10:01:17 -0400 Subject: [PATCH 52/96] Make survey file optional --- activitysim/core/calibration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 201c0abd28..9048fb7b40 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -90,7 +90,7 @@ class CalibrationComponentSettings(PydanticBase): helper_module: str | None = None submodel_max_iterations: int = 1 reports: CalibrationReportsSettings = CalibrationReportsSettings() - survey_file: str + survey_file: Optional[str] = None class CalibrationConfig(PydanticReadable): From 1b4eb8bcdffed4438c89193aa19c155437f72434 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 10:15:17 -0500 Subject: [PATCH 53/96] Add single-process channel restoration --- activitysim/core/calibration.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 9048fb7b40..8c67ace215 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1504,12 +1504,18 @@ def _run_in_configured_mode( state.checkpoint.add(models[-1]) return + # Single-process: checkpoint.restore inside state.run calls init_state() + # which loses dynamically-added RNG channels (e.g. vehicles). + prior_rng_channels = list(state.get_injectable("rng_channels", [])) + state.run( models=models, resume_after=resume_after, memory_sidecar_process=memory_sidecar_process, ) + _reregister_rng_channels(state, prior_rng_channels) + def _prep_model_data(state, resume_after=None): """Restore the pipeline to the correct state before running models. From 7b66d4f98eabd5589f5911ed890f802f145a47e8 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 10:51:23 -0500 Subject: [PATCH 54/96] Update rng_channels --- activitysim/core/calibration.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 8c67ace215..f75993e2c4 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1504,9 +1504,11 @@ def _run_in_configured_mode( state.checkpoint.add(models[-1]) return - # Single-process: checkpoint.restore inside state.run calls init_state() - # which loses dynamically-added RNG channels (e.g. vehicles). - prior_rng_channels = list(state.get_injectable("rng_channels", [])) + # Ensure rng_channels injectable includes all currently-registered + # channels (not just the defaults). checkpoint.load reads this injectable + # after init_state() to re-register channels; without this, dynamically- + # added channels like "vehicles" are lost mid-run. + _sync_rng_channels_injectable(state) state.run( models=models, @@ -1514,8 +1516,6 @@ def _run_in_configured_mode( memory_sidecar_process=memory_sidecar_process, ) - _reregister_rng_channels(state, prior_rng_channels) - def _prep_model_data(state, resume_after=None): """Restore the pipeline to the correct state before running models. @@ -1869,6 +1869,16 @@ def _run_multiprocess_with_overrides( state.settings.multiprocess_steps = original_mp_steps +def _sync_rng_channels_injectable(state: workflow.State) -> None: + """Update rng_channels injectable to include all registered channels.""" + rng = state.rng() + if hasattr(rng, "channels"): + all_channels = list( + set(state.get_injectable("rng_channels", [])) | set(rng.channels.keys()) + ) + state.add_injectable("rng_channels", all_channels) + + def _reregister_rng_channels(state: workflow.State, prior_channels: list[str]) -> None: """Re-register RNG channels that were lost during init_state().""" current_channels = set(state.get_injectable("rng_channels", [])) From c0c937daced2b68f85f2b8a72de635012b825095 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 11:24:37 -0500 Subject: [PATCH 55/96] Modify single-process run approach --- activitysim/core/calibration.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index f75993e2c4..0c80e7cd7d 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -496,7 +496,17 @@ def _calibrate_component( shared_data_buffers=shared_data_buffers, ) else: - state.run(models=[run_model_name], resume_after=prior_step) + # Restore to prior_step ourselves then run the model directly. + # state.run(resume_after=prior_step) would trigger + # checkpoint.restore → init_state which creates a fresh RNG. + # If prior_step is before the calibrated model created its table + # (e.g. vehicles), the table won't be in that checkpoint and the + # RNG channel won't be registered — causing a crash when the + # model tries to use it. By restoring here and calling by_name, + # we keep the RNG channels from _prep_model_data intact. + _prep_model_data(state, resume_after=prior_step) + state.checkpoint.add(prior_step) + state.run.by_name(run_model_name) eval_context = _build_expression_context( state, helper_symbols, component_name, component_settings From cf2c743fd662fc673098e9f587ca978dca37b9b9 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 12:24:15 -0500 Subject: [PATCH 56/96] handle index_to_channel --- activitysim/core/calibration.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 0c80e7cd7d..3ca1aa972e 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1761,6 +1761,7 @@ def _subprocess_path(proc_name): # Load into parent state prior_rng_channels = list(state.get_injectable("rng_channels", [])) + prior_index_to_channel = dict(state.rng().index_to_channel) if hasattr(state.rng(), "index_to_channel") else {} state.init_state() if state.checkpoint.store_is_open(): @@ -1770,7 +1771,7 @@ def _subprocess_path(proc_name): for table_name, df in tables.items(): state.add_table(table_name, df) - _reregister_rng_channels(state, prior_rng_channels) + _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) # Mark all tables dirty for subsequent checkpoint.add for table_name in list(state.existing_table_names): @@ -1880,16 +1881,18 @@ def _run_multiprocess_with_overrides( def _sync_rng_channels_injectable(state: workflow.State) -> None: - """Update rng_channels injectable to include all registered channels.""" + """Update rng_channels injectable and preserve index_to_channel mapping.""" rng = state.rng() if hasattr(rng, "channels"): all_channels = list( set(state.get_injectable("rng_channels", [])) | set(rng.channels.keys()) ) state.add_injectable("rng_channels", all_channels) + if hasattr(rng, "index_to_channel"): + state.add_injectable("_prior_index_to_channel", dict(rng.index_to_channel)) -def _reregister_rng_channels(state: workflow.State, prior_channels: list[str]) -> None: +def _reregister_rng_channels(state: workflow.State, prior_channels: list[str], prior_index_to_channel: dict[str, str] = None) -> None: """Re-register RNG channels that were lost during init_state().""" current_channels = set(state.get_injectable("rng_channels", [])) for channel_name in prior_channels: @@ -1899,6 +1902,20 @@ def _reregister_rng_channels(state: workflow.State, prior_channels: list[str]) - current_channels.add(channel_name) except Exception: pass + # Pre-register empty channels for index_to_channel mappings that were + # lost but whose table doesn't exist yet (e.g. vehicles before vehicle + # type choice runs). The model will extend the domain via add_channel. + if prior_index_to_channel: + for index_name, channel_name in prior_index_to_channel.items(): + if index_name not in state.rng().index_to_channel: + if channel_name not in state.rng().channels: + empty_df = pd.DataFrame( + index=pd.Index([], dtype="int64", name=index_name) + ) + state.rng().add_channel(channel_name, empty_df) + else: + state.rng().index_to_channel[index_name] = channel_name + current_channels.add(channel_name) state.add_injectable("rng_channels", list(current_channels)) @@ -1925,16 +1942,17 @@ def _restore_parent_state_from_pipeline( subprocesses can load them from a direct file path without relying on checkpoint backtracking through potentially ambiguous checkpoint history. """ - # Capture RNG channels before restore — models may have dynamically + # Capture RNG state before restore — models may have dynamically # added channels (e.g. "vehicles") that aren't in the default # rng_channels injectable and would be lost by init_state(). prior_rng_channels = list(state.get_injectable("rng_channels", [])) + prior_index_to_channel = dict(state.rng().index_to_channel) if hasattr(state.rng(), "index_to_channel") else {} if state.checkpoint.store_is_open(): state.checkpoint.close_store() state.checkpoint.restore(resume_after=checkpoint_name) - _reregister_rng_channels(state, prior_rng_channels) + _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) # After restore, all tables are clean (status=False). Mark them dirty so # the next checkpoint.add() writes them to disk at a known checkpoint name. From ec6e2695554b38ad4df4caaf24970de51b7be905 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 13:55:45 -0500 Subject: [PATCH 57/96] Modify configured mode to use same path --- activitysim/core/calibration.py | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 3ca1aa972e..64a4e0fa4b 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1514,17 +1514,13 @@ def _run_in_configured_mode( state.checkpoint.add(models[-1]) return - # Ensure rng_channels injectable includes all currently-registered - # channels (not just the defaults). checkpoint.load reads this injectable - # after init_state() to re-register channels; without this, dynamically- - # added channels like "vehicles" are lost mid-run. - _sync_rng_channels_injectable(state) - - state.run( - models=models, - resume_after=resume_after, - memory_sidecar_process=memory_sidecar_process, - ) + # Run models individually via by_name, avoiding state.run()'s internal + # checkpoint.restore which would create a fresh RNG and lose channels + # for tables not yet created at the resume_after checkpoint (e.g. vehicles). + _prep_model_data(state, resume_after=resume_after) + state.checkpoint.add(resume_after or models[0]) + for model in models: + state.run.by_name(model) def _prep_model_data(state, resume_after=None): @@ -1880,18 +1876,6 @@ def _run_multiprocess_with_overrides( state.settings.multiprocess_steps = original_mp_steps -def _sync_rng_channels_injectable(state: workflow.State) -> None: - """Update rng_channels injectable and preserve index_to_channel mapping.""" - rng = state.rng() - if hasattr(rng, "channels"): - all_channels = list( - set(state.get_injectable("rng_channels", [])) | set(rng.channels.keys()) - ) - state.add_injectable("rng_channels", all_channels) - if hasattr(rng, "index_to_channel"): - state.add_injectable("_prior_index_to_channel", dict(rng.index_to_channel)) - - def _reregister_rng_channels(state: workflow.State, prior_channels: list[str], prior_index_to_channel: dict[str, str] = None) -> None: """Re-register RNG channels that were lost during init_state().""" current_channels = set(state.get_injectable("rng_channels", [])) From c287efb904fafe8fbe703eae1f9c712a6db2c44b Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 14:04:40 -0500 Subject: [PATCH 58/96] Remove memory sidecar process from calib --- activitysim/cli/run.py | 1 - activitysim/core/calibration.py | 12 ------------ 2 files changed, 13 deletions(-) diff --git a/activitysim/cli/run.py b/activitysim/cli/run.py index 289a27914f..4d7a1929f4 100644 --- a/activitysim/cli/run.py +++ b/activitysim/cli/run.py @@ -424,7 +424,6 @@ def run(args): calibration_result = calibration.run_calibration_loop( state=state, models=state.settings.models, - memory_sidecar_process=memory_sidecar_process, ) logger.info( diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 64a4e0fa4b..2fcb5998b8 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -150,7 +150,6 @@ def calibration_enabled(state: workflow.State) -> bool: def run_calibration_loop( state: workflow.State, models: list[str], - memory_sidecar_process=None, ) -> CalibrationRunResult: """ Run the global calibration workflow. @@ -216,7 +215,6 @@ def run_calibration_loop( else models[first_model_idx:first_calib_model_idx], resume_after=state.settings.resume_after, global_iter=start_global_iter, - memory_sidecar_process=memory_sidecar_process, shared_data_buffers=shared_data_buffers, ) else: @@ -235,7 +233,6 @@ def run_calibration_loop( state, models=extra_models, resume_after=None, - memory_sidecar_process=memory_sidecar_process, shared_data_buffers=shared_data_buffers, ) else: @@ -280,7 +277,6 @@ def run_calibration_loop( + 1 : models.index(component) ], resume_after=last_calibrated_component, - memory_sidecar_process=memory_sidecar_process, shared_data_buffers=shared_data_buffers, ) @@ -324,7 +320,6 @@ def run_calibration_loop( if global_iter == start_global_iter and first_model_idx > last_calib_model_idx else last_calibrated_component, - memory_sidecar_process=memory_sidecar_process, shared_data_buffers=shared_data_buffers, ) @@ -359,7 +354,6 @@ def _run_precursor_components( models: list[str], resume_after: str, global_iter: int, - memory_sidecar_process=None, shared_data_buffers: dict | None = None, ) -> None: """Run the normal ActivitySim model flow for one global calibration iteration.""" @@ -377,7 +371,6 @@ def _run_precursor_components( state, models=models, resume_after=resume_after, - memory_sidecar_process=memory_sidecar_process, shared_data_buffers=shared_data_buffers, ) @@ -386,7 +379,6 @@ def _run_intermediate_components( state: workflow.State, models: list[str], resume_after: str, - memory_sidecar_process=None, shared_data_buffers: dict | None = None, ) -> None: if len(models) == 0: @@ -395,7 +387,6 @@ def _run_intermediate_components( state, models=models, resume_after=resume_after, - memory_sidecar_process=memory_sidecar_process, shared_data_buffers=shared_data_buffers, ) @@ -404,14 +395,12 @@ def _run_subsequent_components( state: workflow.State, models: list[str], resume_after: str, - memory_sidecar_process=None, shared_data_buffers: dict | None = None, ) -> None: _run_in_configured_mode( state, models=models, resume_after=resume_after, - memory_sidecar_process=memory_sidecar_process, shared_data_buffers=shared_data_buffers, ) @@ -1462,7 +1451,6 @@ def _run_in_configured_mode( state: workflow.State, models: list[str], resume_after: str | None, - memory_sidecar_process=None, shared_data_buffers: dict | None = None, ) -> None: """Run models using the same single/multiprocess mode as the parent run.""" From 52ee5cd3b9ccc1de75efe12e9e3b8e3a4211bad7 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 15:01:00 -0500 Subject: [PATCH 59/96] Add reloading of tables in reregister_rng --- activitysim/core/calibration.py | 62 ++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 2fcb5998b8..65008ab9e0 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1864,7 +1864,12 @@ def _run_multiprocess_with_overrides( state.settings.multiprocess_steps = original_mp_steps -def _reregister_rng_channels(state: workflow.State, prior_channels: list[str], prior_index_to_channel: dict[str, str] = None) -> None: +def _reregister_rng_channels( + state: workflow.State, + prior_channels: list[str], + prior_index_to_channel: dict[str, str] = None, + table_checkpoint_map: dict[str, str] = None, +) -> None: """Re-register RNG channels that were lost during init_state().""" current_channels = set(state.get_injectable("rng_channels", [])) for channel_name in prior_channels: @@ -1874,19 +1879,31 @@ def _reregister_rng_channels(state: workflow.State, prior_channels: list[str], p current_channels.add(channel_name) except Exception: pass - # Pre-register empty channels for index_to_channel mappings that were - # lost but whose table doesn't exist yet (e.g. vehicles before vehicle - # type choice runs). The model will extend the domain via add_channel. + # Re-register channels whose tables don't exist at the restored checkpoint + # but DO exist elsewhere in the pipeline store (from a prior iteration). if prior_index_to_channel: for index_name, channel_name in prior_index_to_channel.items(): if index_name not in state.rng().index_to_channel: - if channel_name not in state.rng().channels: - empty_df = pd.DataFrame( - index=pd.Index([], dtype="int64", name=index_name) - ) - state.rng().add_channel(channel_name, empty_df) - else: - state.rng().index_to_channel[index_name] = channel_name + # Try loading the actual table from the store at its last-written checkpoint + loaded = False + if table_checkpoint_map and channel_name in table_checkpoint_map: + cp_name = table_checkpoint_map[channel_name] + if cp_name and state.checkpoint.store_is_open(): + try: + df = state.checkpoint._read_df(channel_name, checkpoint_name=cp_name) + state.rng().add_channel(channel_name, df) + loaded = True + except Exception: + pass + if not loaded: + # Fallback: register with empty domain; model must call add_channel + if channel_name not in state.rng().channels: + empty_df = pd.DataFrame( + index=pd.Index([], dtype="int64", name=index_name) + ) + state.rng().add_channel(channel_name, empty_df) + else: + state.rng().index_to_channel[index_name] = channel_name current_channels.add(channel_name) state.add_injectable("rng_channels", list(current_channels)) @@ -1920,11 +1937,32 @@ def _restore_parent_state_from_pipeline( prior_rng_channels = list(state.get_injectable("rng_channels", [])) prior_index_to_channel = dict(state.rng().index_to_channel) if hasattr(state.rng(), "index_to_channel") else {} + # Capture where each table was last written BEFORE the truncating restore, + # so we can load actual table data for RNG channels that won't exist at + # the restored checkpoint (e.g. vehicles before vehicle_type_choice runs). + table_checkpoint_map = {} + try: + if not state.checkpoint.store_is_open(): + state.checkpoint.open_store(overwrite=False, mode="r") + _opened = True + else: + _opened = False + from activitysim.core.workflow.checkpoint import CHECKPOINT_TABLE_NAME, CHECKPOINT_NAME, NON_TABLE_COLUMNS + full_cp_df = state.checkpoint.store.get_dataframe(CHECKPOINT_TABLE_NAME) + last_row = full_cp_df.iloc[-1] + for col in last_row.index: + if col not in NON_TABLE_COLUMNS and last_row[col]: + table_checkpoint_map[col] = last_row[col] + if _opened: + state.checkpoint.close_store() + except Exception: + pass + if state.checkpoint.store_is_open(): state.checkpoint.close_store() state.checkpoint.restore(resume_after=checkpoint_name) - _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) + _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel, table_checkpoint_map) # After restore, all tables are clean (status=False). Mark them dirty so # the next checkpoint.add() writes them to disk at a known checkpoint name. From af641ed1c4def316ca944ffed7fb1f5b266a187e Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 16:07:57 -0500 Subject: [PATCH 60/96] Maybe a different approach to checkpoint restoration --- activitysim/core/calibration.py | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 65008ab9e0..32b1699c05 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1937,26 +1937,15 @@ def _restore_parent_state_from_pipeline( prior_rng_channels = list(state.get_injectable("rng_channels", [])) prior_index_to_channel = dict(state.rng().index_to_channel) if hasattr(state.rng(), "index_to_channel") else {} - # Capture where each table was last written BEFORE the truncating restore, - # so we can load actual table data for RNG channels that won't exist at - # the restored checkpoint (e.g. vehicles before vehicle_type_choice runs). + # Build map of table_name → checkpoint_name from the in-memory checkpoint + # history (which still has entries from the prior iteration, before truncation). table_checkpoint_map = {} - try: - if not state.checkpoint.store_is_open(): - state.checkpoint.open_store(overwrite=False, mode="r") - _opened = True - else: - _opened = False - from activitysim.core.workflow.checkpoint import CHECKPOINT_TABLE_NAME, CHECKPOINT_NAME, NON_TABLE_COLUMNS - full_cp_df = state.checkpoint.store.get_dataframe(CHECKPOINT_TABLE_NAME) - last_row = full_cp_df.iloc[-1] - for col in last_row.index: - if col not in NON_TABLE_COLUMNS and last_row[col]: - table_checkpoint_map[col] = last_row[col] - if _opened: - state.checkpoint.close_store() - except Exception: - pass + from activitysim.core.workflow.checkpoint import NON_TABLE_COLUMNS + if state.checkpoint.checkpoints: + last_entry = state.checkpoint.checkpoints[-1] + for key, val in last_entry.items(): + if key not in NON_TABLE_COLUMNS and val: + table_checkpoint_map[key] = val if state.checkpoint.store_is_open(): state.checkpoint.close_store() From b38b329d81d8ad321cc6064d4f642a17b8e35ebe Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Tue, 11 Aug 2026 17:38:12 -0500 Subject: [PATCH 61/96] Restore correct checkpoint --- activitysim/core/calibration.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 32b1699c05..eeb43955e5 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1937,13 +1937,12 @@ def _restore_parent_state_from_pipeline( prior_rng_channels = list(state.get_injectable("rng_channels", [])) prior_index_to_channel = dict(state.rng().index_to_channel) if hasattr(state.rng(), "index_to_channel") else {} - # Build map of table_name → checkpoint_name from the in-memory checkpoint - # history (which still has entries from the prior iteration, before truncation). + # Build map of table_name → checkpoint_name by scanning the full in-memory + # checkpoint history for the last non-empty value for each table. table_checkpoint_map = {} from activitysim.core.workflow.checkpoint import NON_TABLE_COLUMNS - if state.checkpoint.checkpoints: - last_entry = state.checkpoint.checkpoints[-1] - for key, val in last_entry.items(): + for entry in state.checkpoint.checkpoints: + for key, val in entry.items(): if key not in NON_TABLE_COLUMNS and val: table_checkpoint_map[key] = val From b4520562d39c2540d2baa341e8c8ce69229a57cb Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 10:44:29 -0500 Subject: [PATCH 62/96] Fix add_table bug --- activitysim/core/calibration.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index eeb43955e5..194e23a97a 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1502,10 +1502,10 @@ def _run_in_configured_mode( state.checkpoint.add(models[-1]) return - # Run models individually via by_name, avoiding state.run()'s internal - # checkpoint.restore which would create a fresh RNG and lose channels - # for tables not yet created at the resume_after checkpoint (e.g. vehicles). - _prep_model_data(state, resume_after=resume_after) + # State is already at the correct point from _prep_model_data above. + # Do NOT call _prep_model_data again — the second call would build its + # table_checkpoint_map from the now-truncated in-memory checkpoint history, + # losing references to tables created after resume_after (e.g. vehicles). state.checkpoint.add(resume_after or models[0]) for model in models: state.run.by_name(model) @@ -1892,6 +1892,11 @@ def _reregister_rng_channels( try: df = state.checkpoint._read_df(channel_name, checkpoint_name=cp_name) state.rng().add_channel(channel_name, df) + # Also add the table to state so that @workflow.table + # factories are not re-triggered. Without this, the + # factory would call add_channel again with the same + # indices, hitting the disjoint-index assertion. + state.add_table(channel_name, df) loaded = True except Exception: pass From d169bd488703196ffe98dc1df589a6cfcfbe469c Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 11:11:54 -0500 Subject: [PATCH 63/96] Different approach to checkpoint/channel restoration --- activitysim/core/calibration.py | 52 +++++++++------------------------ 1 file changed, 14 insertions(+), 38 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 194e23a97a..1b4021f072 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1868,7 +1868,6 @@ def _reregister_rng_channels( state: workflow.State, prior_channels: list[str], prior_index_to_channel: dict[str, str] = None, - table_checkpoint_map: dict[str, str] = None, ) -> None: """Re-register RNG channels that were lost during init_state().""" current_channels = set(state.get_injectable("rng_channels", [])) @@ -1879,36 +1878,22 @@ def _reregister_rng_channels( current_channels.add(channel_name) except Exception: pass - # Re-register channels whose tables don't exist at the restored checkpoint - # but DO exist elsewhere in the pipeline store (from a prior iteration). + # For channels whose tables don't exist at the restored checkpoint, + # register an empty channel. Do NOT pre-load from a later checkpoint + # in the store — that data may include modifications from downstream + # models and would pollute the pre-model state. The empty channel + # allows the model's normal table factory to create the table fresh + # and extend the channel without hitting the disjoint-index assertion. if prior_index_to_channel: for index_name, channel_name in prior_index_to_channel.items(): if index_name not in state.rng().index_to_channel: - # Try loading the actual table from the store at its last-written checkpoint - loaded = False - if table_checkpoint_map and channel_name in table_checkpoint_map: - cp_name = table_checkpoint_map[channel_name] - if cp_name and state.checkpoint.store_is_open(): - try: - df = state.checkpoint._read_df(channel_name, checkpoint_name=cp_name) - state.rng().add_channel(channel_name, df) - # Also add the table to state so that @workflow.table - # factories are not re-triggered. Without this, the - # factory would call add_channel again with the same - # indices, hitting the disjoint-index assertion. - state.add_table(channel_name, df) - loaded = True - except Exception: - pass - if not loaded: - # Fallback: register with empty domain; model must call add_channel - if channel_name not in state.rng().channels: - empty_df = pd.DataFrame( - index=pd.Index([], dtype="int64", name=index_name) - ) - state.rng().add_channel(channel_name, empty_df) - else: - state.rng().index_to_channel[index_name] = channel_name + if channel_name not in state.rng().channels: + empty_df = pd.DataFrame( + index=pd.Index([], dtype="int64", name=index_name) + ) + state.rng().add_channel(channel_name, empty_df) + else: + state.rng().index_to_channel[index_name] = channel_name current_channels.add(channel_name) state.add_injectable("rng_channels", list(current_channels)) @@ -1942,20 +1927,11 @@ def _restore_parent_state_from_pipeline( prior_rng_channels = list(state.get_injectable("rng_channels", [])) prior_index_to_channel = dict(state.rng().index_to_channel) if hasattr(state.rng(), "index_to_channel") else {} - # Build map of table_name → checkpoint_name by scanning the full in-memory - # checkpoint history for the last non-empty value for each table. - table_checkpoint_map = {} - from activitysim.core.workflow.checkpoint import NON_TABLE_COLUMNS - for entry in state.checkpoint.checkpoints: - for key, val in entry.items(): - if key not in NON_TABLE_COLUMNS and val: - table_checkpoint_map[key] = val - if state.checkpoint.store_is_open(): state.checkpoint.close_store() state.checkpoint.restore(resume_after=checkpoint_name) - _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel, table_checkpoint_map) + _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) # After restore, all tables are clean (status=False). Mark them dirty so # the next checkpoint.add() writes them to disk at a known checkpoint name. From 2eb889dfed8fbad4ebf1d342212f32cfa54a7e66 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 11:59:46 -0500 Subject: [PATCH 64/96] Add table invalidation --- activitysim/core/calibration.py | 60 +++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 1b4021f072..94fd2fb6f7 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -74,6 +74,11 @@ class CalibrationRunSettings(PydanticBase): calibrate_models: list[str] global_iterations: int = 1 complete_steps: bool = False + invalidate_tables: list[str] | None = None + """Tables to drop after each restore so their @workflow.table factories + regenerate from current data. If None (default), automatically detects + factory tables that depend on other tables and have RNG channels (e.g. + vehicles). Set to an explicit list to override.""" class CalibrationReportsSettings(PydanticBase): @@ -1898,6 +1903,56 @@ def _reregister_rng_channels( state.add_injectable("rng_channels", list(current_channels)) +def _invalidate_derived_tables(state: workflow.State) -> None: + """Drop factory-produced tables that may be stale after a calibration restore. + + When a calibrated model (e.g. auto_ownership) changes a table that a + @workflow.table factory depends on (e.g. vehicles depends on households), + the checkpoint may contain a stale version of that factory table. Dropping + it forces the factory to regenerate from current data on next access. + + Auto-detection rule: invalidate any table that is (a) registered as a + @workflow.table factory, (b) has DataFrame parameters (= table dependencies), + and (c) is in RANDOM_CHANNELS. This currently matches only 'vehicles' but + will automatically cover future factory tables with the same pattern. + """ + settings = read_calibration_settings(state) + if not settings: + return + + tables_to_invalidate = settings.run.invalidate_tables + if not tables_to_invalidate: + tables_to_invalidate = _detect_derived_rng_tables(state) + + for table_name in tables_to_invalidate: + if state.is_table(table_name): + state.drop_table(table_name) + state.rng().drop_channel(table_name) + logger.debug("calibration: invalidated derived table '%s'", table_name) + + +def _detect_derived_rng_tables(state: workflow.State) -> list[str]: + """Identify factory tables with table dependencies and RNG channels.""" + import inspect + + from activitysim.abm.models.util.canonical_ids import RANDOM_CHANNELS + + result = [] + for table_name, factory_func in state._LOADABLE_TABLES.items(): + if table_name not in RANDOM_CHANNELS: + continue + sig = inspect.signature(factory_func) + has_table_dep = any( + p.annotation is pd.DataFrame + or (p.annotation is inspect.Parameter.empty and p.name != "state") + for p in sig.parameters.values() + if p.name != "state" + ) + if has_table_dep: + result.append(table_name) + return result + + def _restore_parent_state_from_pipeline( state: workflow.State, checkpoint_name: str = "_" ) -> None: @@ -1933,6 +1988,11 @@ def _restore_parent_state_from_pipeline( _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) + # Drop derived tables so their factories regenerate from current data. + # Without this, a stale vehicles table (based on old auto_ownership values) + # would be loaded from the checkpoint and never refreshed. + _invalidate_derived_tables(state) + # After restore, all tables are clean (status=False). Mark them dirty so # the next checkpoint.add() writes them to disk at a known checkpoint name. # This ensures apportion subprocesses find table files at a single, From 67b87a421abcce285e8da0763af6e5c4751b05de Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 13:08:21 -0500 Subject: [PATCH 65/96] Additional precursor guard --- activitysim/core/calibration.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 94fd2fb6f7..5ab0badf2b 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -240,6 +240,25 @@ def run_calibration_loop( resume_after=None, shared_data_buffers=shared_data_buffers, ) + elif not any( + cp.get("checkpoint_name") == state.settings.resume_after + for cp in state.checkpoint.checkpoints + ): + # _prep_model_data took its fallback path — the pipeline either + # doesn't exist or doesn't contain resume_after's checkpoint. + # The restored state is incomplete (precursor models never ran). + logger.warning( + "calibration: resume_after=%r not found in restored pipeline; " + "running precursor models", + state.settings.resume_after, + ) + _run_precursor_components( + state, + models=models[:first_calib_model_idx], + resume_after=None, + global_iter=start_global_iter, + shared_data_buffers=shared_data_buffers, + ) else: state.checkpoint.add(state.settings.resume_after) state.checkpoint.close_store() From 8a19f6c817e4826888800430cfb3754f415c8c9d Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 13:10:16 -0500 Subject: [PATCH 66/96] Blacken --- activitysim/core/calibration.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 5ab0badf2b..82d19f8caf 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1769,7 +1769,11 @@ def _subprocess_path(proc_name): # Load into parent state prior_rng_channels = list(state.get_injectable("rng_channels", [])) - prior_index_to_channel = dict(state.rng().index_to_channel) if hasattr(state.rng(), "index_to_channel") else {} + prior_index_to_channel = ( + dict(state.rng().index_to_channel) + if hasattr(state.rng(), "index_to_channel") + else {} + ) state.init_state() if state.checkpoint.store_is_open(): @@ -1999,7 +2003,11 @@ def _restore_parent_state_from_pipeline( # added channels (e.g. "vehicles") that aren't in the default # rng_channels injectable and would be lost by init_state(). prior_rng_channels = list(state.get_injectable("rng_channels", [])) - prior_index_to_channel = dict(state.rng().index_to_channel) if hasattr(state.rng(), "index_to_channel") else {} + prior_index_to_channel = ( + dict(state.rng().index_to_channel) + if hasattr(state.rng(), "index_to_channel") + else {} + ) if state.checkpoint.store_is_open(): state.checkpoint.close_store() From 21a0d7854b867ecc103d166cc67d713fe8d65211 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 13:42:27 -0500 Subject: [PATCH 67/96] Fix calibrate_component precursor calls --- activitysim/core/calibration.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 82d19f8caf..cfd69dac9d 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -517,7 +517,12 @@ def _calibrate_component( # RNG channel won't be registered — causing a crash when the # model tries to use it. By restoring here and calling by_name, # we keep the RNG channels from _prep_model_data intact. - _prep_model_data(state, resume_after=prior_step) + extra_models = _prep_model_data(state, resume_after=prior_step) + if extra_models: + # prior_step checkpoint not found directly; run intermediate + # models (e.g. annotators) to recreate the correct state. + for m in extra_models: + state.run.by_name(m) state.checkpoint.add(prior_step) state.run.by_name(run_model_name) From 0437b7081c74ebf5898f611d8efd6afd3b691a1f Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 14:33:00 -0500 Subject: [PATCH 68/96] Add temporary diagnostic tracing --- activitysim/core/calibration.py | 67 +++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index cfd69dac9d..8d768a35f6 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -524,6 +524,34 @@ def _calibrate_component( for m in extra_models: state.run.by_name(m) state.checkpoint.add(prior_step) + + # Diagnostic: verify households state before running calibrated model + if state.is_table("households"): + _hh = state.get_dataframe("households") + logger.debug( + "calibration: households before %s has %d rows, columns: %s", + run_model_name, + len(_hh), + list(_hh.columns), + ) + if "num_drivers" not in _hh.columns: + logger.error( + "calibration: households missing 'num_drivers' before %s. " + "prior_step=%r, extra_models=%r, checkpoint_names=%s", + run_model_name, + prior_step, + extra_models, + [ + cp.get("checkpoint_name", "") + for cp in state.checkpoint.checkpoints + ], + ) + else: + logger.error( + "calibration: households table not in state before %s", + run_model_name, + ) + state.run.by_name(run_model_name) eval_context = _build_expression_context( @@ -2018,13 +2046,52 @@ def _restore_parent_state_from_pipeline( state.checkpoint.close_store() state.checkpoint.restore(resume_after=checkpoint_name) + # DEBUG: trace where num_drivers disappears + _trace_col = "num_drivers" + if state.is_table("households"): + _hh = state.get_dataframe("households") + logger.debug( + "calibration TRACE [after restore]: households has %d cols, " + "%s present=%s, checkpoint=%r", + len(_hh.columns), + _trace_col, + _trace_col in _hh.columns, + checkpoint_name, + ) + else: + logger.debug( + "calibration TRACE [after restore]: households NOT in state, checkpoint=%r", + checkpoint_name, + ) + _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) + if state.is_table("households"): + _hh = state.get_dataframe("households") + logger.debug( + "calibration TRACE [after _reregister_rng_channels]: %s present=%s", + _trace_col, + _trace_col in _hh.columns, + ) + # Drop derived tables so their factories regenerate from current data. # Without this, a stale vehicles table (based on old auto_ownership values) # would be loaded from the checkpoint and never refreshed. _invalidate_derived_tables(state) + if state.is_table("households"): + _hh = state.get_dataframe("households") + logger.debug( + "calibration TRACE [after _invalidate_derived_tables]: %s present=%s", + _trace_col, + _trace_col in _hh.columns, + ) + else: + logger.debug( + "calibration TRACE [after _invalidate_derived_tables]: " + "households NOT in state (was it invalidated?)" + ) + # After restore, all tables are clean (status=False). Mark them dirty so # the next checkpoint.add() writes them to disk at a known checkpoint name. # This ensures apportion subprocesses find table files at a single, From 2f588283e934cf2156b6e3dbea0b3d71882394f2 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 14:58:49 -0500 Subject: [PATCH 69/96] More diagnostic logging --- activitysim/core/calibration.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 8d768a35f6..8ef1abad96 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1980,12 +1980,24 @@ def _invalidate_derived_tables(state: workflow.State) -> None: if not tables_to_invalidate: tables_to_invalidate = _detect_derived_rng_tables(state) + logger.debug( + "calibration: tables detected for invalidation: %s", tables_to_invalidate + ) + tables_before = set(state.existing_table_names) + for table_name in tables_to_invalidate: if state.is_table(table_name): state.drop_table(table_name) state.rng().drop_channel(table_name) logger.debug("calibration: invalidated derived table '%s'", table_name) + tables_after = set(state.existing_table_names) + lost = tables_before - tables_after - set(tables_to_invalidate) + if lost: + logger.error( + "calibration: tables unexpectedly removed during invalidation: %s", lost + ) + def _detect_derived_rng_tables(state: workflow.State) -> list[str]: """Identify factory tables with table dependencies and RNG channels.""" @@ -1998,9 +2010,15 @@ def _detect_derived_rng_tables(state: workflow.State) -> list[str]: if table_name not in RANDOM_CHANNELS: continue sig = inspect.signature(factory_func) + # Only match parameters that are actual table dependencies: + # annotated as pd.DataFrame, or positional without a default value. has_table_dep = any( p.annotation is pd.DataFrame - or (p.annotation is inspect.Parameter.empty and p.name != "state") + or ( + p.annotation is inspect.Parameter.empty + and p.default is inspect.Parameter.empty + and p.name != "state" + ) for p in sig.parameters.values() if p.name != "state" ) From 656352522fba18b98b2e65566cb946c5c305c3cb Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 15:19:10 -0500 Subject: [PATCH 70/96] Change to default tables_to_invalidate --- activitysim/core/calibration.py | 34 ++++----------------------------- 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 8ef1abad96..3bbaa45b38 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1977,8 +1977,10 @@ def _invalidate_derived_tables(state: workflow.State) -> None: return tables_to_invalidate = settings.run.invalidate_tables - if not tables_to_invalidate: - tables_to_invalidate = _detect_derived_rng_tables(state) + if tables_to_invalidate is None: + # Default: vehicles is the only RNG-channel table whose row identity + # depends on another table's values (households.auto_ownership). + tables_to_invalidate = ["vehicles"] logger.debug( "calibration: tables detected for invalidation: %s", tables_to_invalidate @@ -1999,34 +2001,6 @@ def _invalidate_derived_tables(state: workflow.State) -> None: ) -def _detect_derived_rng_tables(state: workflow.State) -> list[str]: - """Identify factory tables with table dependencies and RNG channels.""" - import inspect - - from activitysim.abm.models.util.canonical_ids import RANDOM_CHANNELS - - result = [] - for table_name, factory_func in state._LOADABLE_TABLES.items(): - if table_name not in RANDOM_CHANNELS: - continue - sig = inspect.signature(factory_func) - # Only match parameters that are actual table dependencies: - # annotated as pd.DataFrame, or positional without a default value. - has_table_dep = any( - p.annotation is pd.DataFrame - or ( - p.annotation is inspect.Parameter.empty - and p.default is inspect.Parameter.empty - and p.name != "state" - ) - for p in sig.parameters.values() - if p.name != "state" - ) - if has_table_dep: - result.append(table_name) - return result - - def _restore_parent_state_from_pipeline( state: workflow.State, checkpoint_name: str = "_" ) -> None: From 210454e553d62e5cd261ac8922462a26209da8ce Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 15:28:02 -0500 Subject: [PATCH 71/96] Change when table invalidation occurs --- activitysim/core/calibration.py | 106 +++++++++----------------------- 1 file changed, 30 insertions(+), 76 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 3bbaa45b38..86d10142f1 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -75,10 +75,33 @@ class CalibrationRunSettings(PydanticBase): global_iterations: int = 1 complete_steps: bool = False invalidate_tables: list[str] | None = None - """Tables to drop after each restore so their @workflow.table factories - regenerate from current data. If None (default), automatically detects - factory tables that depend on other tables and have RNG channels (e.g. - vehicles). Set to an explicit list to override.""" + """Tables to drop from state after each calibration restore so their + ``@workflow.table`` factories regenerate from current data. + + Default (None): invalidates ``["vehicles"]``. Set to ``[]`` to disable. + + A table should be listed here when ALL of the following are true: + + 1. It is created by a ``@workflow.table`` factory from another table's + values (not just from input data files). + 2. That source table is modified by a calibrated model or by a model + whose outputs change when calibrated coefficients change. + 3. The factory uses source-table values to determine **row identity** + (index values) or **row count**, not just column values. + + The canonical example is ``vehicles``: its factory repeats household + rows by ``households["auto_ownership"]`` and derives ``vehicle_id`` + from ``household_id``. When ``auto_ownership_simulate`` is calibrated, + different coefficients produce different ownership counts, so the + stale vehicles table loaded from a prior checkpoint would have the + wrong number of rows and wrong vehicle IDs. Dropping it forces the + factory to regenerate vehicles consistent with the current households. + + Tables that only read *column values* from upstream tables (without + affecting row identity) generally do NOT need invalidation — their + content will be correct as long as the upstream table is correct at + the restored checkpoint. + """ class CalibrationReportsSettings(PydanticBase): @@ -413,6 +436,7 @@ def _run_intermediate_components( resume_after=resume_after, shared_data_buffers=shared_data_buffers, ) + _invalidate_derived_tables(state) def _run_subsequent_components( @@ -427,6 +451,7 @@ def _run_subsequent_components( resume_after=resume_after, shared_data_buffers=shared_data_buffers, ) + _invalidate_derived_tables(state) def _calibrate_component( @@ -523,35 +548,8 @@ def _calibrate_component( # models (e.g. annotators) to recreate the correct state. for m in extra_models: state.run.by_name(m) + _invalidate_derived_tables(state) state.checkpoint.add(prior_step) - - # Diagnostic: verify households state before running calibrated model - if state.is_table("households"): - _hh = state.get_dataframe("households") - logger.debug( - "calibration: households before %s has %d rows, columns: %s", - run_model_name, - len(_hh), - list(_hh.columns), - ) - if "num_drivers" not in _hh.columns: - logger.error( - "calibration: households missing 'num_drivers' before %s. " - "prior_step=%r, extra_models=%r, checkpoint_names=%s", - run_model_name, - prior_step, - extra_models, - [ - cp.get("checkpoint_name", "") - for cp in state.checkpoint.checkpoints - ], - ) - else: - logger.error( - "calibration: households table not in state before %s", - run_model_name, - ) - state.run.by_name(run_model_name) eval_context = _build_expression_context( @@ -2038,52 +2036,8 @@ def _restore_parent_state_from_pipeline( state.checkpoint.close_store() state.checkpoint.restore(resume_after=checkpoint_name) - # DEBUG: trace where num_drivers disappears - _trace_col = "num_drivers" - if state.is_table("households"): - _hh = state.get_dataframe("households") - logger.debug( - "calibration TRACE [after restore]: households has %d cols, " - "%s present=%s, checkpoint=%r", - len(_hh.columns), - _trace_col, - _trace_col in _hh.columns, - checkpoint_name, - ) - else: - logger.debug( - "calibration TRACE [after restore]: households NOT in state, checkpoint=%r", - checkpoint_name, - ) - _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) - if state.is_table("households"): - _hh = state.get_dataframe("households") - logger.debug( - "calibration TRACE [after _reregister_rng_channels]: %s present=%s", - _trace_col, - _trace_col in _hh.columns, - ) - - # Drop derived tables so their factories regenerate from current data. - # Without this, a stale vehicles table (based on old auto_ownership values) - # would be loaded from the checkpoint and never refreshed. - _invalidate_derived_tables(state) - - if state.is_table("households"): - _hh = state.get_dataframe("households") - logger.debug( - "calibration TRACE [after _invalidate_derived_tables]: %s present=%s", - _trace_col, - _trace_col in _hh.columns, - ) - else: - logger.debug( - "calibration TRACE [after _invalidate_derived_tables]: " - "households NOT in state (was it invalidated?)" - ) - # After restore, all tables are clean (status=False). Mark them dirty so # the next checkpoint.add() writes them to disk at a known checkpoint name. # This ensures apportion subprocesses find table files at a single, From e8f6a33a6094133a62d1a219d45f271c74e6f4d5 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Wed, 12 Aug 2026 16:10:27 -0500 Subject: [PATCH 72/96] Fix SP checkpointing --- activitysim/core/calibration.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 86d10142f1..e05d502c87 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -1564,6 +1564,10 @@ def _run_in_configured_mode( state.checkpoint.add(resume_after or models[0]) for model in models: state.run.by_name(model) + # Ensure final model's state is persisted even if should_save_checkpoint + # returned False for it — _calibrate_component needs to restore to it. + if models: + state.checkpoint.add(models[-1]) def _prep_model_data(state, resume_after=None): From fb923bb4e52a063d7cb9639343568210cc59e362 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:01:25 -0700 Subject: [PATCH 73/96] fixing resume after and rng calib invariance --- activitysim/core/calibration.py | 90 +++++++++++++++++++++++++---- activitysim/core/workflow/runner.py | 14 ++++- 2 files changed, 91 insertions(+), 13 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index e05d502c87..a0d0ec1ce1 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -75,6 +75,15 @@ class CalibrationRunSettings(PydanticBase): global_iterations: int = 1 complete_steps: bool = False invalidate_tables: list[str] | None = None + + @model_validator(mode="after") + def validate_run_settings(self): + if not self.calibrate_models: + raise ValueError( + "calibration.run.calibrate_models must contain at least one model name" + ) + return self + """Tables to drop from state after each calibration restore so their ``@workflow.table`` factories regenerate from current data. @@ -196,9 +205,29 @@ def run_calibration_loop( "Overriding duplicate_step_execution setting: must be enabled for calibration" ) - assert all( - [c in models for c in calibration_settings.run.calibrate_models] - ), f"settings.yaml steps list does not include calibration model{'s' if len([c for c in calibration_settings.run.calibrate_models if c not in models]) != 1 else ''} {[c for c in calibration_settings.run.calibrate_models if c not in models]}" + if not calibration_settings.run.calibrate_models: + raise ValueError( + "calibration.run.calibrate_models must contain at least one model name" + ) + + missing_calibration_models = [ + component + for component in calibration_settings.run.calibrate_models + if component not in models + ] + if missing_calibration_models: + raise ValueError( + "settings.yaml models list does not include configured calibration " + f"model(s): {missing_calibration_models}" + ) + + resume_after = state.settings.resume_after + if resume_after is not None and resume_after not in models: + raise ValueError( + f"settings.yaml resume_after={resume_after!r} is not present in the " + "settings.yaml models list. Calibration requires resume_after to be " + "a model-level checkpoint name." + ) # sort calibration models into main model order calibration_settings.run.calibrate_models = sorted( @@ -206,12 +235,27 @@ def run_calibration_loop( ) first_calib_model_idx = models.index(calibration_settings.run.calibrate_models[0]) last_calib_model_idx = models.index(calibration_settings.run.calibrate_models[-1]) - first_model_idx = ( - models.index(state.settings.resume_after) + 1 - if state.settings.resume_after - else None + first_model_idx = models.index(resume_after) + 1 if resume_after else None + first_calibration_restart_step = _prior_step_name( + models, calibration_settings.run.calibrate_models[0] ) + if resume_after is not None: + skipped_calibration_models = [ + component + for component in calibration_settings.run.calibrate_models + if models.index(component) <= models.index(resume_after) + ] + if skipped_calibration_models: + logger.warning( + "Calibration is honoring settings.yaml resume_after=%r using strict " + "ActivitySim semantics. The following calibrated model(s) occur at " + "or before resume_after and will be skipped during the first global " + "iteration: %s", + resume_after, + skipped_calibration_models, + ) + _ensure_calibration_output_dir(state) # If there is recoverable calibration progress from a prior interrupted run, @@ -290,6 +334,31 @@ def run_calibration_loop( start_global_iter, start_global_iter + calibration_settings.run.global_iterations, ): + # Every global iteration after the first begins from the immutable + # checkpoint directly before the first calibrated model. This makes + # global reruns independent of the state left by the final calibrated + # model in the preceding iteration. + if global_iter > start_global_iter: + logger.info( + "Restarting global calibration iteration %s from checkpoint %r", + global_iter, + first_calibration_restart_step, + ) + extra_models = _prep_model_data( + state, resume_after=first_calibration_restart_step + ) + if extra_models: + _run_in_configured_mode( + state, + models=extra_models, + resume_after=None, + shared_data_buffers=shared_data_buffers, + ) + _invalidate_derived_tables(state) + if first_calibration_restart_step is not None: + state.checkpoint.add(first_calibration_restart_step) + state.checkpoint.close_store() + logger.info( "calibration global iteration %s/%s", global_iter - start_global_iter, @@ -528,9 +597,10 @@ def _calibrate_component( _run_mp_single_component( state, component_name=component_name, - restore_checkpoint=mp_restore_checkpoint - if component_iter == 1 - else "_", + # Always restore from the same immutable pre-component + # checkpoint. LAST_CHECKPOINT may point to the prior iteration's + # coalesced component output and is therefore not a safe baseline. + restore_checkpoint=mp_restore_checkpoint, shared_data_buffers=shared_data_buffers, ) else: diff --git a/activitysim/core/workflow/runner.py b/activitysim/core/workflow/runner.py index 79ecd0ed4f..9596e6c47c 100644 --- a/activitysim/core/workflow/runner.py +++ b/activitysim/core/workflow/runner.py @@ -270,7 +270,10 @@ def _pre_run_step(self, model_name: str) -> bool | None: f"Cannot run model '{model_name}' more than once" ) - self._obj.rng().begin_step(model_name) + # Parse the canonical workflow step before initializing the RNG. + # Arguments appended to a model invocation (for example calibration + # iteration labels) may affect logging and checkpoint names, but must + # never affect the deterministic random stream for the model itself. # check for args if "." in model_name: @@ -285,6 +288,11 @@ def _pre_run_step(self, model_name: str) -> bool | None: step_name = model_name args = {} + self.rng_step_name = ( + step_name[1:] if step_name.startswith(NO_CHECKPOINT_PREFIX) else step_name + ) + self._obj.rng().begin_step(self.rng_step_name) + # check for no_checkpoint prefix if step_name[0] == NO_CHECKPOINT_PREFIX: step_name = step_name[1:] @@ -351,7 +359,7 @@ def by_name(self, model_name, **kwargs): except Exception: self.t0 = self._log_elapsed_time(f"run.{model_name} UNTIL ERROR", self.t0) self._obj.add_injectable("step_args", None) - self._obj.rng().end_step(model_name) + self._obj.rng().end_step(self.rng_step_name) raise else: @@ -361,7 +369,7 @@ def by_name(self, model_name, **kwargs): self._obj.add_injectable("step_args", None) - self._obj.rng().end_step(model_name) + self._obj.rng().end_step(self.rng_step_name) if self.checkpoint: self._obj.checkpoint.add(model_name) else: From 92bbc84c763961d2fe8e72b53aff425f8c03b099 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:43:41 -0700 Subject: [PATCH 74/96] fixing pipeline and rng restarting issues --- activitysim/abm/models/initialize.py | 11 +++++++++-- activitysim/core/calibration.py | 8 ++++++++ activitysim/core/random.py | 5 +++++ activitysim/core/test/test_random.py | 14 +++++++++++++- activitysim/core/workflow/runner.py | 4 +++- 5 files changed, 38 insertions(+), 4 deletions(-) diff --git a/activitysim/abm/models/initialize.py b/activitysim/abm/models/initialize.py index 84fe001e90..d986e111d7 100644 --- a/activitysim/abm/models/initialize.py +++ b/activitysim/abm/models/initialize.py @@ -154,12 +154,19 @@ def initialize_households( households = state.get_dataframe("households") assert not households._is_view chunk_sizer.log_df(trace_label, "households", households) - del households - chunk_sizer.log_df(trace_label, "households", None) persons = state.get_dataframe("persons") assert not persons._is_view chunk_sizer.log_df(trace_label, "persons", persons) + + rng = state.get_rn_generator() + if "households" not in rng.channels: + rng.add_channel("households", households) + if "persons" not in rng.channels: + rng.add_channel("persons", persons) + + del households + chunk_sizer.log_df(trace_label, "households", None) del persons chunk_sizer.log_df(trace_label, "persons", None) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index a0d0ec1ce1..825cb16c51 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -258,6 +258,14 @@ def run_calibration_loop( _ensure_calibration_output_dir(state) + if state.settings.resume_after is None: + # compute_accessibility requires its accessibility table to be empty; + # unlike most model steps, it will not overwrite a prior result. + # Remove a cached result before restore clears table-status metadata, + # so the table factory recreates its empty placeholder for the replay. + state.drop_table("accessibility") + state.checkpoint.restore() + # If there is recoverable calibration progress from a prior interrupted run, # continue from that iteration. Coefficient updates are persisted in config # coefficient files, so restarting from a later global iteration is compatible diff --git a/activitysim/core/random.py b/activitysim/core/random.py index 37b1976403..1c95683b26 100644 --- a/activitysim/core/random.py +++ b/activitysim/core/random.py @@ -503,6 +503,11 @@ def drop_channel(self, channel_name): if channel_name in self.channels: logger.debug("Dropping channel '%s'" % (channel_name,)) del self.channels[channel_name] + self.index_to_channel = { + index_name: mapped_channel_name + for index_name, mapped_channel_name in self.index_to_channel.items() + if mapped_channel_name != channel_name + } else: logger.error( "drop_channel called with unknown channel '%s'" % (channel_name,) diff --git a/activitysim/core/test/test_random.py b/activitysim/core/test/test_random.py index bcbc602685..5e1a66bda4 100644 --- a/activitysim/core/test/test_random.py +++ b/activitysim/core/test/test_random.py @@ -8,7 +8,7 @@ import pytest from activitysim.core import random -from activitysim.core.exceptions import DuplicateLoadableObjectError +from activitysim.core.exceptions import DuplicateLoadableObjectError, TableIndexError def test_basic(): @@ -126,3 +126,15 @@ def test_channel(): npt.assert_almost_equal(np.asanyarray(rands).flatten(), test1_expected_rands2) rng.end_step("test_step") + + +def test_drop_channel_removes_index_mapping(): + rng = random.Random() + persons = pd.DataFrame(index=pd.Index([1], name="person_id")) + + rng.add_channel("persons", persons) + rng.drop_channel("persons") + + assert "person_id" not in rng.index_to_channel + with pytest.raises(TableIndexError, match="No channel with index name 'person_id'"): + rng.get_channel_for_df(persons) diff --git a/activitysim/core/workflow/runner.py b/activitysim/core/workflow/runner.py index 9596e6c47c..14ef972049 100644 --- a/activitysim/core/workflow/runner.py +++ b/activitysim/core/workflow/runner.py @@ -322,6 +322,7 @@ def by_name(self, model_name, **kwargs): model_name is assumed to be the name of a registered workflow step """ self.t0 = time.time() + self.rng_step_name = None try: should_skip = self._pre_run_step(model_name) if should_skip: @@ -359,7 +360,8 @@ def by_name(self, model_name, **kwargs): except Exception: self.t0 = self._log_elapsed_time(f"run.{model_name} UNTIL ERROR", self.t0) self._obj.add_injectable("step_args", None) - self._obj.rng().end_step(self.rng_step_name) + if self.rng_step_name is not None: + self._obj.rng().end_step(self.rng_step_name) raise else: From af1e21bec767f23a4bc0ca43a67a84491e789959 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Mon, 17 Aug 2026 10:46:59 -0700 Subject: [PATCH 75/96] fixing more pipeline restart issues --- activitysim/core/calibration.py | 69 ++++++++++++++++++++------ activitysim/core/test/test_pipeline.py | 16 ++++++ activitysim/core/workflow/state.py | 3 ++ 3 files changed, 72 insertions(+), 16 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 825cb16c51..61785a9e09 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -20,7 +20,7 @@ import matplotlib.pyplot as plt from pydantic import model_validator -from activitysim.core import workflow +from activitysim.core import simulate, workflow from activitysim.core.configuration import PydanticReadable from activitysim.core.configuration.base import PydanticBase from activitysim.core.configuration.top import MultiprocessStep @@ -38,7 +38,7 @@ CALIBRATION_FINAL_COEFFICIENTS_FILE = "calibration/final_calibrated_coefficients.csv" DEFAULT_INCREMENT = 2.0 -MAX_COEFFS_IN_GRAPH = 10 +MAX_COEFFS_IN_GRAPH = 15 CALIBRATION_REQUIRED_COLUMNS = [ "description", @@ -513,7 +513,6 @@ def _run_intermediate_components( resume_after=resume_after, shared_data_buffers=shared_data_buffers, ) - _invalidate_derived_tables(state) def _run_subsequent_components( @@ -528,7 +527,6 @@ def _run_subsequent_components( resume_after=resume_after, shared_data_buffers=shared_data_buffers, ) - _invalidate_derived_tables(state) def _calibrate_component( @@ -721,11 +719,18 @@ def _extract_utility_coefficient_names( model_settings: dict[str, Any] | Any, ) -> set[str]: """ - Extract coefficient tokens from configured utility spec files. + Extract coefficient names used by the configured utility specifications. - The extraction scans all settings keys ending with "SPEC" and parses - tokens from utility columns (all non-description/expression columns). + Templated logit models map utility-spec row labels to actual coefficient + names by segment, so their template cell values are the source of truth. + Other models use coefficient tokens in utility-spec columns. """ + if _setting_value(model_settings, "COEFFICIENT_TEMPLATE"): + template = simulate.read_model_coefficient_template( + state.filesystem, model_settings + ) + return {str(name) for name in template.to_numpy().ravel()} + names: set[str] = set() model_settings_dict = _settings_to_dict(model_settings) @@ -1147,14 +1152,22 @@ def _append_iteration_records( # Save a global iteration history file global_path = state.get_output_file_path(CALIBRATION_ITERATION_FILE) - _append_csv(df, global_path) + _append_csv( + df, + global_path, + unique_on=["global_iter", "component_iter", "component", "coefficient"], + ) # Also write component-local iteration history component_path = ( _component_output_dir(state, component_name) / Path(CALIBRATION_ITERATION_FILE).name ) - _append_csv(df, component_path) + _append_csv( + df, + component_path, + unique_on=["global_iter", "component_iter", "component", "coefficient"], + ) def _append_summary_records( @@ -1165,12 +1178,26 @@ def _append_summary_records( return path = state.get_output_file_path(CALIBRATION_SUMMARY_FILE) df = pd.DataFrame(records) - _append_csv(df, path) + _append_csv( + df, + path, + unique_on=["global_iter", "component_iter", "component"], + ) -def _append_csv(df: pd.DataFrame, path: Path) -> None: - """Append a dataframe to a CSV file with header-once behavior.""" +def _append_csv( + df: pd.DataFrame, path: Path, unique_on: list[str] | None = None +) -> None: + """Append a dataframe to a CSV file, replacing rows with matching keys.""" os.makedirs(path.parent, exist_ok=True) + if unique_on and path.exists(): + existing = pd.read_csv(path) + df = pd.concat([existing, df], ignore_index=True).drop_duplicates( + subset=unique_on, keep="last" + ) + df.to_csv(path, index=False) + return + write_header = not path.exists() df.to_csv(path, mode="a", index=False, header=write_header) @@ -1317,7 +1344,11 @@ def _write_generic_report( ) path = _component_output_dir(state, component_name) / "generic_report.csv" - _append_csv(report, path) + _append_csv( + report, + path, + unique_on=["global_iter", "component_iter", "component", "description"], + ) def _load_helper_symbols( @@ -2058,9 +2089,14 @@ def _invalidate_derived_tables(state: workflow.State) -> None: tables_to_invalidate = settings.run.invalidate_tables if tables_to_invalidate is None: - # Default: vehicles is the only RNG-channel table whose row identity - # depends on another table's values (households.auto_ownership). - tables_to_invalidate = ["vehicles"] + # Vehicles needs regeneration only when calibration changes + # households.auto_ownership. Downstream calibration components must + # retain vehicle_type_choice's vehicle attributes. + tables_to_invalidate = ( + ["vehicles"] + if "auto_ownership_simulate" in settings.run.calibrate_models + else [] + ) logger.debug( "calibration: tables detected for invalidation: %s", tables_to_invalidate @@ -2071,6 +2107,7 @@ def _invalidate_derived_tables(state: workflow.State) -> None: if state.is_table(table_name): state.drop_table(table_name) state.rng().drop_channel(table_name) + state.get_dataframe(table_name, as_copy=False) logger.debug("calibration: invalidated derived table '%s'", table_name) tables_after = set(state.existing_table_names) diff --git a/activitysim/core/test/test_pipeline.py b/activitysim/core/test/test_pipeline.py index 12f31dbc66..95dca6e436 100644 --- a/activitysim/core/test/test_pipeline.py +++ b/activitysim/core/test/test_pipeline.py @@ -5,6 +5,7 @@ import logging import os +import pandas as pd import pytest import tables @@ -128,6 +129,21 @@ def test_pipeline_checkpoint_drop(state): close_handlers() +def test_get_table_returns_current_table_after_recreation(state): + original = pd.DataFrame({"value": [1]}, index=pd.Index([1], name="id")) + recreated = pd.DataFrame({"value": [2]}, index=pd.Index([1], name="id")) + + state.add_table("recreated_table", original) + state.checkpoint.add("before_drop") + state.drop_table("recreated_table") + state.add_table("recreated_table", recreated) + + pd.testing.assert_frame_equal(state.get_table("recreated_table"), recreated) + + state.checkpoint.close_store() + close_handlers() + + # if __name__ == "__main__": # # print "\n\ntest_pipeline_run" diff --git a/activitysim/core/workflow/state.py b/activitysim/core/workflow/state.py index de22b0687d..fbf85addfc 100644 --- a/activitysim/core/workflow/state.py +++ b/activitysim/core/workflow/state.py @@ -1185,6 +1185,9 @@ def get_table(self, table_name, checkpoint_name=None): df : pandas.DataFrame """ + if checkpoint_name is None and table_name in self._context: + return self._context[table_name] + if table_name not in self.checkpoint.last_checkpoint and self.is_table( table_name ): From 022b4fb83111454b5c5c1900fcea1bd89d7ed8a1 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:34:08 -0700 Subject: [PATCH 76/96] appropriately handling crashing in middle of iteration --- activitysim/core/calibration.py | 210 +++++++++++++++++++++++++++---- docs/users-guide/calibration.rst | 30 +++-- 2 files changed, 207 insertions(+), 33 deletions(-) diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py index 61785a9e09..cdeaa32e17 100644 --- a/activitysim/core/calibration.py +++ b/activitysim/core/calibration.py @@ -10,6 +10,7 @@ import multiprocessing import os import re +import shutil from dataclasses import dataclass from pathlib import Path from typing import Any, Optional @@ -36,6 +37,7 @@ CALIBRATION_ITERATION_FILE = "calibration/calibration_iteration_records.csv" CALIBRATION_SUMMARY_FILE = "calibration/calibration_iteration_summary.csv" CALIBRATION_FINAL_COEFFICIENTS_FILE = "calibration/final_calibrated_coefficients.csv" +CALIBRATION_RECOVERY_DIR = "calibration/recovery" DEFAULT_INCREMENT = 2.0 MAX_COEFFS_IN_GRAPH = 15 @@ -258,6 +260,62 @@ def run_calibration_loop( _ensure_calibration_output_dir(state) + progress = _read_progress(state) + if progress and progress.get("complete"): + logger.info( + "calibration progress is already complete; remove %s to start a " + "fresh calibration run", + CALIBRATION_PROGRESS_FILE, + ) + return CalibrationRunResult( + converged=bool(progress.get("converged", False)), + completed_global_iterations=int( + progress.get( + "last_completed_global_iteration", + calibration_settings.run.global_iterations, + ) + ), + ) + + interrupted_iteration = ( + progress.get("in_progress_iteration") if progress else None + ) + if interrupted_iteration is not None: + interrupted_iteration = int(interrupted_iteration) + logger.warning( + "recovering interrupted calibration global iteration %s", + interrupted_iteration, + ) + _restore_coefficient_backups(state, calibration_settings) + progress = { + "in_progress_iteration": None, + "next_global_iteration": interrupted_iteration, + "last_completed_global_iteration": interrupted_iteration - 1, + } + _write_progress(state, progress) + + # Progress files from earlier versions contain next_global_iteration, so + # they remain compatible with the corrected total-count semantics. + start_global_iter = int(progress.get("next_global_iteration", 1)) if progress else 1 + completed_global_iterations = start_global_iter - 1 + + if start_global_iter > calibration_settings.run.global_iterations: + logger.info( + "calibration progress already reached configured global_iterations=%s", + calibration_settings.run.global_iterations, + ) + converged = bool(progress.get("converged", False)) if progress else False + _write_final_coefficients_snapshot(state, calibration_settings) + _write_completed_progress( + state, + completed_global_iterations, + converged, + ) + return CalibrationRunResult( + converged=converged, + completed_global_iterations=completed_global_iterations, + ) + if state.settings.resume_after is None: # compute_accessibility requires its accessibility table to be empty; # unlike most model steps, it will not overwrite a prior result. @@ -266,13 +324,6 @@ def run_calibration_loop( state.drop_table("accessibility") state.checkpoint.restore() - # If there is recoverable calibration progress from a prior interrupted run, - # continue from that iteration. Coefficient updates are persisted in config - # coefficient files, so restarting from a later global iteration is compatible - # with current checkpoint semantics. - progress = _read_progress(state) - start_global_iter = int(progress.get("next_global_iteration", 1)) if progress else 1 - original_pipeline_name = state.filesystem.pipeline_file_name # Initialize shared resources for multiprocess mode (skims, shadow pricing). @@ -340,8 +391,14 @@ def run_calibration_loop( for global_iter in range( start_global_iter, - start_global_iter + calibration_settings.run.global_iterations, + calibration_settings.run.global_iterations + 1, ): + _begin_global_iteration_transaction( + state, + calibration_settings, + global_iter, + ) + # Every global iteration after the first begins from the immutable # checkpoint directly before the first calibrated model. This makes # global reruns independent of the state left by the final calibrated @@ -369,7 +426,7 @@ def run_calibration_loop( logger.info( "calibration global iteration %s/%s", - global_iter - start_global_iter, + global_iter, calibration_settings.run.global_iterations, ) @@ -420,10 +477,7 @@ def run_calibration_loop( if ( calibration_settings.run.complete_steps - or ( - start_global_iter + calibration_settings.run.global_iterations - == global_iter + 1 - ) + or global_iter == calibration_settings.run.global_iterations or ( global_iter == start_global_iter and state.settings.resume_after is not None @@ -447,27 +501,40 @@ def run_calibration_loop( shared_data_buffers=shared_data_buffers, ) - _write_progress( - state, - { - "next_global_iteration": global_iter + 1, - "last_completed_global_iteration": global_iter, - }, + completed_global_iterations = global_iter + iteration_is_complete = ( + all_converged + or global_iter == calibration_settings.run.global_iterations ) + if not iteration_is_complete: + _write_progress( + state, + { + "in_progress_iteration": None, + "next_global_iteration": global_iter + 1, + "last_completed_global_iteration": global_iter, + "converged": all_converged, + }, + ) if all_converged: logger.info( "calibration converged after global iteration %s/%s", - global_iter - start_global_iter, + global_iter, calibration_settings.run.global_iterations, ) break _write_final_coefficients_snapshot(state, calibration_settings) + _write_completed_progress( + state, + completed_global_iterations, + all_converged, + ) return CalibrationRunResult( converged=all_converged, - completed_global_iterations=calibration_settings.run.global_iterations, + completed_global_iterations=completed_global_iterations, ) finally: state.filesystem.pipeline_file_name = original_pipeline_name @@ -1466,6 +1533,83 @@ def _ensure_calibration_output_dir(state: workflow.State) -> None: os.makedirs(path, exist_ok=True) +def _calibration_coefficient_paths( + state: workflow.State, + calibration_settings: CalibrationConfig, +) -> list[Path]: + """Return the unique coefficient files modified by this calibration run.""" + paths: list[Path] = [] + seen: set[str] = set() + + for component_name in calibration_settings.run.calibrate_models: + model_settings_file = _infer_model_settings_file(component_name) + model_settings = state.filesystem.read_model_settings( + model_settings_file, mandatory=True + ) + coefficient_file = _setting_value(model_settings, "COEFFICIENTS") + if not coefficient_file: + raise RuntimeError( + f"component {component_name} model settings missing COEFFICIENTS" + ) + + path = Path(state.filesystem.get_config_file_path(coefficient_file)).resolve() + key = os.path.normcase(str(path)) + if key not in seen: + paths.append(path) + seen.add(key) + + return paths + + +def _begin_global_iteration_transaction( + state: workflow.State, + calibration_settings: CalibrationConfig, + global_iteration: int, +) -> None: + """Snapshot coefficients and durably mark a global iteration in progress.""" + recovery_dir = state.get_output_file_path(CALIBRATION_RECOVERY_DIR) + os.makedirs(recovery_dir, exist_ok=True) + + for file_number, coefficient_path in enumerate( + _calibration_coefficient_paths(state, calibration_settings) + ): + if not coefficient_path.exists(): + raise FileNotFoundError( + f"calibration coefficient file not found: {coefficient_path}" + ) + + backup_name = f"{file_number:03d}_{coefficient_path.name}" + shutil.copyfile(coefficient_path, recovery_dir / backup_name) + + # Write the marker only after all backups exist. If backup creation is + # interrupted, the previous between-iteration progress remains valid. + _write_progress( + state, + { + "in_progress_iteration": global_iteration, + "next_global_iteration": global_iteration, + "last_completed_global_iteration": global_iteration - 1, + }, + ) + + +def _restore_coefficient_backups( + state: workflow.State, + calibration_settings: CalibrationConfig, +) -> None: + """Restore the coefficient backups for an interrupted global iteration.""" + recovery_dir = state.get_output_file_path(CALIBRATION_RECOVERY_DIR) + for file_number, coefficient_path in enumerate( + _calibration_coefficient_paths(state, calibration_settings) + ): + backup_path = recovery_dir / f"{file_number:03d}_{coefficient_path.name}" + if not backup_path.exists(): + raise RuntimeError( + f"cannot recover interrupted calibration iteration: missing {backup_path}" + ) + shutil.copyfile(backup_path, coefficient_path) + + def _read_progress(state: workflow.State) -> dict[str, Any] | None: """Read persisted calibration progress metadata if it exists.""" path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) @@ -1476,11 +1620,31 @@ def _read_progress(state: workflow.State) -> dict[str, Any] | None: def _write_progress(state: workflow.State, payload: dict[str, Any]) -> None: - """Write calibration progress metadata.""" + """Atomically write calibration progress metadata.""" path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) os.makedirs(path.parent, exist_ok=True) - with open(path, "w", encoding="utf-8") as f: + temporary_path = path.with_name(f".{path.name}.tmp") + with open(temporary_path, "w", encoding="utf-8") as f: json.dump(payload, f, indent=2) + os.replace(temporary_path, path) + + +def _write_completed_progress( + state: workflow.State, + completed_global_iterations: int, + converged: bool, +) -> None: + """Mark calibration complete after all final output has been written.""" + _write_progress( + state, + { + "complete": True, + "in_progress_iteration": None, + "next_global_iteration": completed_global_iterations + 1, + "last_completed_global_iteration": completed_global_iterations, + "converged": converged, + }, + ) def _run_mp_single_component( diff --git a/docs/users-guide/calibration.rst b/docs/users-guide/calibration.rst index db0cf719d0..2d817f32fe 100644 --- a/docs/users-guide/calibration.rst +++ b/docs/users-guide/calibration.rst @@ -541,9 +541,11 @@ Global Files * - File - Description * - ``calibration_progress.json`` - - Tracks ``next_global_iteration`` for crash recovery. If a run is - interrupted, restarting will resume from the last completed global - iteration. + - Tracks the active and next global iterations for crash recovery, and + whether calibration is complete. + * - ``recovery/`` + - Reusable start-of-iteration coefficient backups used to roll back and + replay an interrupted global iteration. * - ``calibration_iteration_records.csv`` - Appended per-coefficient detail for every iteration across all components. Columns include ``global_iter``, ``component_iter``, @@ -584,7 +586,7 @@ Updated Config Files --------------------- Coefficient CSV files in the configs directory are **updated in-place** after -each iteration. This means: +each component iteration. This means: - The calibrated coefficients persist across runs. - You can inspect intermediate coefficient values at any time. @@ -594,15 +596,23 @@ each iteration. This means: Crash Recovery ============== -Calibration progress is persisted to ``calibration_progress.json`` after each -completed global iteration. If a run is interrupted: +Before each global iteration, calibration replaces the files in its recovery +directory with a copy of every coefficient file that it may modify, then records +the active iteration in ``calibration_progress.json``. If a run is interrupted: -1. The coefficient files on disk reflect the state at the last completed iteration. -2. Restarting ``activitysim run`` with the same configuration will resume from - the ``next_global_iteration`` recorded in the progress file. +1. Restarting ``activitysim run`` restores all coefficient files from the + start-of-iteration recovery snapshot. +2. The interrupted global iteration is replayed from that consistent boundary. +3. Remaining iterations run only until the configured total + ``global_iterations`` is reached. + +The progress file is written using atomic replacement. Once progress is marked +complete, rerunning with the same output directory does not apply additional +calibration iterations. To force a fresh start, delete ``output/calibration/calibration_progress.json`` -and restore original coefficient files. +and restore original coefficient files. Recovery snapshots can also be removed +after a completed run if they are no longer needed. Multiprocess Mode From d79517e12be980fb6edfc0bfa250ce2937206e7b Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:18:03 -0700 Subject: [PATCH 77/96] refactor into separate files --- activitysim/core/calibration.py | 2466 ----------------- activitysim/core/calibration/README.md | 27 + activitysim/core/calibration/__init__.py | 70 + activitysim/core/calibration/coefficients.py | 85 + activitysim/core/calibration/component.py | 514 ++++ activitysim/core/calibration/execution.py | 262 ++ activitysim/core/calibration/expressions.py | 199 ++ activitysim/core/calibration/multiprocess.py | 604 ++++ activitysim/core/calibration/orchestrator.py | 455 +++ activitysim/core/calibration/recovery.py | 103 + activitysim/core/calibration/reporting.py | 268 ++ activitysim/core/calibration/settings.py | 132 + .../workplace_location_calib_helper.py | 3 +- 13 files changed, 2721 insertions(+), 2467 deletions(-) delete mode 100644 activitysim/core/calibration.py create mode 100644 activitysim/core/calibration/README.md create mode 100644 activitysim/core/calibration/__init__.py create mode 100644 activitysim/core/calibration/coefficients.py create mode 100644 activitysim/core/calibration/component.py create mode 100644 activitysim/core/calibration/execution.py create mode 100644 activitysim/core/calibration/expressions.py create mode 100644 activitysim/core/calibration/multiprocess.py create mode 100644 activitysim/core/calibration/orchestrator.py create mode 100644 activitysim/core/calibration/recovery.py create mode 100644 activitysim/core/calibration/reporting.py create mode 100644 activitysim/core/calibration/settings.py diff --git a/activitysim/core/calibration.py b/activitysim/core/calibration.py deleted file mode 100644 index cdeaa32e17..0000000000 --- a/activitysim/core/calibration.py +++ /dev/null @@ -1,2466 +0,0 @@ -# ActivitySim -# See full license in LICENSE.txt. -from __future__ import annotations - -import importlib -import importlib.util -import json -import logging -import math -import multiprocessing -import os -import re -import shutil -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Optional - -import numpy as np -import pandas as pd -import matplotlib -import matplotlib.pyplot as plt -from pydantic import model_validator - -from activitysim.core import simulate, workflow -from activitysim.core.configuration import PydanticReadable -from activitysim.core.configuration.base import PydanticBase -from activitysim.core.configuration.top import MultiprocessStep - -logger = logging.getLogger("calibration") - -plt.style.use("seaborn-v0_8-darkgrid") -matplotlib.use("Agg") # Forces non-interactive background rendering - -CALIBRATION_SETTINGS_FILE_NAME = "calibration.yaml" -CALIBRATION_OUTPUT_DIR = "calibration" -CALIBRATION_PROGRESS_FILE = "calibration/calibration_progress.json" -CALIBRATION_ITERATION_FILE = "calibration/calibration_iteration_records.csv" -CALIBRATION_SUMMARY_FILE = "calibration/calibration_iteration_summary.csv" -CALIBRATION_FINAL_COEFFICIENTS_FILE = "calibration/final_calibrated_coefficients.csv" -CALIBRATION_RECOVERY_DIR = "calibration/recovery" - -DEFAULT_INCREMENT = 2.0 -MAX_COEFFS_IN_GRAPH = 15 - -CALIBRATION_REQUIRED_COLUMNS = [ - "description", - "coefficient", - "model_value", - "target_value", - "hold_fast", - "min", - "max", - "damping", - "method", - "tolerance", -] - -MP_INJECTABLES = [ - "data_dir", - "configs_dir", - "data_model_dir", - "output_dir", - "cache_dir", - "settings_file_name", - "imported_extensions", - "run_timestamp", - "run_id", - "pipeline_file_name", -] - - -class CalibrationRunSettings(PydanticBase): - """Run-control settings for calibration.""" - - resume_after: Optional[str] = None - calibrate_models: list[str] - global_iterations: int = 1 - complete_steps: bool = False - invalidate_tables: list[str] | None = None - - @model_validator(mode="after") - def validate_run_settings(self): - if not self.calibrate_models: - raise ValueError( - "calibration.run.calibrate_models must contain at least one model name" - ) - return self - - """Tables to drop from state after each calibration restore so their - ``@workflow.table`` factories regenerate from current data. - - Default (None): invalidates ``["vehicles"]``. Set to ``[]`` to disable. - - A table should be listed here when ALL of the following are true: - - 1. It is created by a ``@workflow.table`` factory from another table's - values (not just from input data files). - 2. That source table is modified by a calibrated model or by a model - whose outputs change when calibrated coefficients change. - 3. The factory uses source-table values to determine **row identity** - (index values) or **row count**, not just column values. - - The canonical example is ``vehicles``: its factory repeats household - rows by ``households["auto_ownership"]`` and derives ``vehicle_id`` - from ``household_id``. When ``auto_ownership_simulate`` is calibrated, - different coefficients produce different ownership counts, so the - stale vehicles table loaded from a prior checkpoint would have the - wrong number of rows and wrong vehicle IDs. Dropping it forces the - factory to regenerate vehicles consistent with the current households. - - Tables that only read *column values* from upstream tables (without - affecting row identity) generally do NOT need invalidation — their - content will be correct as long as the upstream table is correct at - the restored checkpoint. - """ - - -class CalibrationReportsSettings(PydanticBase): - """Reporting settings for a calibrated component.""" - - generic: bool = True - bespoke: str | None = None - - -class CalibrationComponentSettings(PydanticBase): - """Settings for one calibratable model component.""" - - calibration_spec: str - helper_module: str | None = None - submodel_max_iterations: int = 1 - reports: CalibrationReportsSettings = CalibrationReportsSettings() - survey_file: Optional[str] = None - - -class CalibrationConfig(PydanticReadable): - """Top-level calibration configuration.""" - - enable: bool = False - run: CalibrationRunSettings - model_settings: dict[str, CalibrationComponentSettings] = {} - - @model_validator(mode="after") - def validate_model_settings(self): - """Validate that configured components are aligned with run settings.""" - for component in self.run.calibrate_models: - if component not in self.model_settings: - raise ValueError( - f"calibration model '{component}' is not in model_settings" - ) - - if self.run.global_iterations < 1: - raise ValueError("max_iterations must be >= 1") - - return self - - -@dataclass -class CalibrationComponentResult: - """Result details from calibrating one component.""" - - component: str - converged: bool - component_iterations: int - - -@dataclass -class CalibrationRunResult: - """Result details from a complete global calibration loop.""" - - converged: bool - completed_global_iterations: int - - -def read_calibration_settings(state: workflow.State) -> CalibrationConfig | None: - """Read and validate calibration settings if calibration.yaml exists.""" - return CalibrationConfig.read_settings_file( - state.filesystem, - CALIBRATION_SETTINGS_FILE_NAME, - mandatory=False, - ) - - -def calibration_enabled(state: workflow.State) -> bool: - """Return True when calibration.yaml exists and is enabled.""" - settings = read_calibration_settings(state) - return bool(settings and settings.enable) - - -def run_calibration_loop( - state: workflow.State, - models: list[str], -) -> CalibrationRunResult: - """ - Run the global calibration workflow. - - This function intentionally minimizes changes to the existing run mechanics: - it always reuses ActivitySim's normal model execution paths and only adds - calibration orchestration around them. - """ - calibration_settings = read_calibration_settings(state) - if not calibration_settings or not calibration_settings.enable: - raise RuntimeError("calibration loop called while calibration is disabled") - - if state.settings.duplicate_step_execution != "allow": - state.settings.duplicate_step_execution = "allow" - logger.warning( - "Overriding duplicate_step_execution setting: must be enabled for calibration" - ) - - if not calibration_settings.run.calibrate_models: - raise ValueError( - "calibration.run.calibrate_models must contain at least one model name" - ) - - missing_calibration_models = [ - component - for component in calibration_settings.run.calibrate_models - if component not in models - ] - if missing_calibration_models: - raise ValueError( - "settings.yaml models list does not include configured calibration " - f"model(s): {missing_calibration_models}" - ) - - resume_after = state.settings.resume_after - if resume_after is not None and resume_after not in models: - raise ValueError( - f"settings.yaml resume_after={resume_after!r} is not present in the " - "settings.yaml models list. Calibration requires resume_after to be " - "a model-level checkpoint name." - ) - - # sort calibration models into main model order - calibration_settings.run.calibrate_models = sorted( - calibration_settings.run.calibrate_models, key=lambda x: models.index(x) - ) - first_calib_model_idx = models.index(calibration_settings.run.calibrate_models[0]) - last_calib_model_idx = models.index(calibration_settings.run.calibrate_models[-1]) - first_model_idx = models.index(resume_after) + 1 if resume_after else None - first_calibration_restart_step = _prior_step_name( - models, calibration_settings.run.calibrate_models[0] - ) - - if resume_after is not None: - skipped_calibration_models = [ - component - for component in calibration_settings.run.calibrate_models - if models.index(component) <= models.index(resume_after) - ] - if skipped_calibration_models: - logger.warning( - "Calibration is honoring settings.yaml resume_after=%r using strict " - "ActivitySim semantics. The following calibrated model(s) occur at " - "or before resume_after and will be skipped during the first global " - "iteration: %s", - resume_after, - skipped_calibration_models, - ) - - _ensure_calibration_output_dir(state) - - progress = _read_progress(state) - if progress and progress.get("complete"): - logger.info( - "calibration progress is already complete; remove %s to start a " - "fresh calibration run", - CALIBRATION_PROGRESS_FILE, - ) - return CalibrationRunResult( - converged=bool(progress.get("converged", False)), - completed_global_iterations=int( - progress.get( - "last_completed_global_iteration", - calibration_settings.run.global_iterations, - ) - ), - ) - - interrupted_iteration = ( - progress.get("in_progress_iteration") if progress else None - ) - if interrupted_iteration is not None: - interrupted_iteration = int(interrupted_iteration) - logger.warning( - "recovering interrupted calibration global iteration %s", - interrupted_iteration, - ) - _restore_coefficient_backups(state, calibration_settings) - progress = { - "in_progress_iteration": None, - "next_global_iteration": interrupted_iteration, - "last_completed_global_iteration": interrupted_iteration - 1, - } - _write_progress(state, progress) - - # Progress files from earlier versions contain next_global_iteration, so - # they remain compatible with the corrected total-count semantics. - start_global_iter = int(progress.get("next_global_iteration", 1)) if progress else 1 - completed_global_iterations = start_global_iter - 1 - - if start_global_iter > calibration_settings.run.global_iterations: - logger.info( - "calibration progress already reached configured global_iterations=%s", - calibration_settings.run.global_iterations, - ) - converged = bool(progress.get("converged", False)) if progress else False - _write_final_coefficients_snapshot(state, calibration_settings) - _write_completed_progress( - state, - completed_global_iterations, - converged, - ) - return CalibrationRunResult( - converged=converged, - completed_global_iterations=completed_global_iterations, - ) - - if state.settings.resume_after is None: - # compute_accessibility requires its accessibility table to be empty; - # unlike most model steps, it will not overwrite a prior result. - # Remove a cached result before restore clears table-status metadata, - # so the table factory recreates its empty placeholder for the replay. - state.drop_table("accessibility") - state.checkpoint.restore() - - original_pipeline_name = state.filesystem.pipeline_file_name - - # Initialize shared resources for multiprocess mode (skims, shadow pricing). - # These are allocated once and reused across all calibration iterations. - shared_data_buffers = None - if state.settings.multiprocess: - shared_data_buffers = _initialize_mp_shared_resources(state) - - try: - # skip precursors if, on first iter, resume_after exists and is >= first_calib_model_idx - if ( - state.settings.resume_after is None - or first_model_idx < first_calib_model_idx - ): - # Run ActivitySim normally from resume_after through production model steps. - _run_precursor_components( - state, - models=models[:first_calib_model_idx] - if first_model_idx is None - else models[first_model_idx:first_calib_model_idx], - resume_after=state.settings.resume_after, - global_iter=start_global_iter, - shared_data_buffers=shared_data_buffers, - ) - else: - # Precursors skipped — but the pipeline must still be initialized - # at the resume_after point so that _calibrate_component (and its - # apportion subprocess) starts from the correct state without - # downstream model data. - extra_models = _prep_model_data( - state, resume_after=state.settings.resume_after - ) - if extra_models: - # No model-level checkpoint exists for resume_after; we must - # run models from the prior step through resume_after to - # recreate the correct intermediate state. - _run_in_configured_mode( - state, - models=extra_models, - resume_after=None, - shared_data_buffers=shared_data_buffers, - ) - elif not any( - cp.get("checkpoint_name") == state.settings.resume_after - for cp in state.checkpoint.checkpoints - ): - # _prep_model_data took its fallback path — the pipeline either - # doesn't exist or doesn't contain resume_after's checkpoint. - # The restored state is incomplete (precursor models never ran). - logger.warning( - "calibration: resume_after=%r not found in restored pipeline; " - "running precursor models", - state.settings.resume_after, - ) - _run_precursor_components( - state, - models=models[:first_calib_model_idx], - resume_after=None, - global_iter=start_global_iter, - shared_data_buffers=shared_data_buffers, - ) - else: - state.checkpoint.add(state.settings.resume_after) - state.checkpoint.close_store() - - for global_iter in range( - start_global_iter, - calibration_settings.run.global_iterations + 1, - ): - _begin_global_iteration_transaction( - state, - calibration_settings, - global_iter, - ) - - # Every global iteration after the first begins from the immutable - # checkpoint directly before the first calibrated model. This makes - # global reruns independent of the state left by the final calibrated - # model in the preceding iteration. - if global_iter > start_global_iter: - logger.info( - "Restarting global calibration iteration %s from checkpoint %r", - global_iter, - first_calibration_restart_step, - ) - extra_models = _prep_model_data( - state, resume_after=first_calibration_restart_step - ) - if extra_models: - _run_in_configured_mode( - state, - models=extra_models, - resume_after=None, - shared_data_buffers=shared_data_buffers, - ) - _invalidate_derived_tables(state) - if first_calibration_restart_step is not None: - state.checkpoint.add(first_calibration_restart_step) - state.checkpoint.close_store() - - logger.info( - "calibration global iteration %s/%s", - global_iter, - calibration_settings.run.global_iterations, - ) - - # suppress early termination on first iteration if resume_after is after all calibrated models - all_converged = ( - first_model_idx is not None and first_model_idx <= last_calib_model_idx - ) or global_iter > start_global_iter - - last_calibrated_component = None - for component in calibration_settings.run.calibrate_models: - # on the first global iter, skip model if it's before or == resume_after - if ( - global_iter == start_global_iter - and first_model_idx is not None - and first_model_idx > models.index(component) - ): - continue - component_settings = calibration_settings.model_settings[component] - - prior_step = _prior_step_name(models, component) - - if last_calibrated_component is not None: - - # run all models b/w the last calibrated model and the current one - _run_intermediate_components( - state, - models=models[ - models.index(last_calibrated_component) - + 1 : models.index(component) - ], - resume_after=last_calibrated_component, - shared_data_buffers=shared_data_buffers, - ) - - component_result = _calibrate_component( - state=state, - component_name=component, - component_settings=component_settings, - prior_step=prior_step, - global_iter=global_iter, - shared_data_buffers=shared_data_buffers, - ) - _write_component_plots(state, component) - - all_converged = all_converged and component_result.converged - - last_calibrated_component = component - - if ( - calibration_settings.run.complete_steps - or global_iter == calibration_settings.run.global_iterations - or ( - global_iter == start_global_iter - and state.settings.resume_after is not None - and first_model_idx > last_calib_model_idx - ) - ): - subsequent_components = ( - models[first_model_idx:] - if global_iter == start_global_iter - and first_model_idx > last_calib_model_idx - else models[models.index(last_calibrated_component) + 1 :] - ) - # finish the full model chain - _run_subsequent_components( - state, - models=subsequent_components, - resume_after=state.settings.resume_after - if global_iter == start_global_iter - and first_model_idx > last_calib_model_idx - else last_calibrated_component, - shared_data_buffers=shared_data_buffers, - ) - - completed_global_iterations = global_iter - iteration_is_complete = ( - all_converged - or global_iter == calibration_settings.run.global_iterations - ) - if not iteration_is_complete: - _write_progress( - state, - { - "in_progress_iteration": None, - "next_global_iteration": global_iter + 1, - "last_completed_global_iteration": global_iter, - "converged": all_converged, - }, - ) - - if all_converged: - logger.info( - "calibration converged after global iteration %s/%s", - global_iter, - calibration_settings.run.global_iterations, - ) - break - - _write_final_coefficients_snapshot(state, calibration_settings) - _write_completed_progress( - state, - completed_global_iterations, - all_converged, - ) - - return CalibrationRunResult( - converged=all_converged, - completed_global_iterations=completed_global_iterations, - ) - finally: - state.filesystem.pipeline_file_name = original_pipeline_name - - -def _run_precursor_components( - state: workflow.State, - models: list[str], - resume_after: str, - global_iter: int, - shared_data_buffers: dict | None = None, -) -> None: - """Run the normal ActivitySim model flow for one global calibration iteration.""" - - # if global_iter > 1 and resume_after is not None: - # # Seed a fresh pipeline from the configured resume checkpoint to avoid - # # duplicate checkpoint-name collisions across global calibration loops. - # prior_pipeline = state.checkpoint.store.filename - # state.checkpoint.close_store() - # state.filesystem.pipeline_file_name = f"pipeline_calibration_iter_{global_iter}" - # state.checkpoint.restore_from(prior_pipeline, checkpoint_name=resume_after) - # else: - - _run_in_configured_mode( - state, - models=models, - resume_after=resume_after, - shared_data_buffers=shared_data_buffers, - ) - - -def _run_intermediate_components( - state: workflow.State, - models: list[str], - resume_after: str, - shared_data_buffers: dict | None = None, -) -> None: - if len(models) == 0: - return - _run_in_configured_mode( - state, - models=models, - resume_after=resume_after, - shared_data_buffers=shared_data_buffers, - ) - - -def _run_subsequent_components( - state: workflow.State, - models: list[str], - resume_after: str, - shared_data_buffers: dict | None = None, -) -> None: - _run_in_configured_mode( - state, - models=models, - resume_after=resume_after, - shared_data_buffers=shared_data_buffers, - ) - - -def _calibrate_component( - state: workflow.State, - component_name: str, - component_settings: CalibrationComponentSettings, - prior_step: str, - global_iter: int, - shared_data_buffers: dict | None = None, -) -> CalibrationComponentResult: - """Run iterative coefficient calibration for one component.""" - model_settings_file = _infer_model_settings_file(component_name) - model_settings = state.filesystem.read_model_settings( - model_settings_file, mandatory=True - ) - - coefficients_df = state.filesystem.read_model_coefficients( - model_settings=model_settings - ) - helper_symbols, bespoke_callable, helper_module = _load_helper_symbols( - state, - component_settings, - ) - - calibration_spec_df = _read_calibration_spec( - state, component_settings.calibration_spec - ) - - utility_coeff_names = _extract_utility_coefficient_names(state, model_settings) - _validate_calibration_coefficients_against_utility_spec( - component_name, - calibration_spec_df, - utility_coeff_names, - ) - - coefficients_df = _ensure_coefficients_exist( - component_name, - calibration_spec_df, - coefficients_df, - ) - - _warn_if_initial_values_outside_bounds( - component_name, calibration_spec_df, coefficients_df - ) - - component_converged = False - component_iterations = 0 - - # Determine the checkpoint name to restore from for component re-runs. - # In MP mode, the checkpoint that represents prior_step's completed state - # is the last checkpoint in the pipeline before we run the component. - # We capture it once and reuse across component iterations. - mp_restore_checkpoint = None - if state.settings.multiprocess and shared_data_buffers is not None: - # The pipeline should already be open from _restore_parent_state_from_pipeline - # called after precursor/intermediate models ran. The last checkpoint - # in the pipeline represents the state at prior_step. - if state.checkpoint.checkpoints: - mp_restore_checkpoint = state.checkpoint.last_checkpoint.get( - "checkpoint_name", "_" - ) - else: - mp_restore_checkpoint = "_" - - for component_iter in range(1, component_settings.submodel_max_iterations + 1): - component_iterations = component_iter - run_model_name = f"{component_name}.c_i{component_iter};" f"g_i{global_iter}" - # Re-run only this component from its prior checkpoint so model values - # reflect the current candidate coefficients for this component. - if state.settings.multiprocess and shared_data_buffers is not None: - # Use direct MP orchestration with explicit checkpoint control. - # This ensures we always apportion from prior_step's state, - # even after multiple component iterations. - _run_mp_single_component( - state, - component_name=component_name, - # Always restore from the same immutable pre-component - # checkpoint. LAST_CHECKPOINT may point to the prior iteration's - # coalesced component output and is therefore not a safe baseline. - restore_checkpoint=mp_restore_checkpoint, - shared_data_buffers=shared_data_buffers, - ) - else: - # Restore to prior_step ourselves then run the model directly. - # state.run(resume_after=prior_step) would trigger - # checkpoint.restore → init_state which creates a fresh RNG. - # If prior_step is before the calibrated model created its table - # (e.g. vehicles), the table won't be in that checkpoint and the - # RNG channel won't be registered — causing a crash when the - # model tries to use it. By restoring here and calling by_name, - # we keep the RNG channels from _prep_model_data intact. - extra_models = _prep_model_data(state, resume_after=prior_step) - if extra_models: - # prior_step checkpoint not found directly; run intermediate - # models (e.g. annotators) to recreate the correct state. - for m in extra_models: - state.run.by_name(m) - _invalidate_derived_tables(state) - state.checkpoint.add(prior_step) - state.run.by_name(run_model_name) - - eval_context = _build_expression_context( - state, helper_symbols, component_name, component_settings - ) - - ( - row_records, - summary_record, - new_coefficients_df, - component_converged, - ) = _evaluate_and_update( - component_name=component_name, - calibration_spec_df=calibration_spec_df, - coefficients_df=coefficients_df, - eval_context=eval_context, - global_iter=global_iter, - component_iter=component_iter, - ) - - coefficients_df = new_coefficients_df - - _persist_coefficients_to_config(state, model_settings, coefficients_df) - _append_iteration_records(state, component_name, row_records) - _append_summary_records(state, [summary_record]) - - if component_settings.reports.generic: - _write_generic_report(state, component_name, row_records) - - if bespoke_callable is not None: - bespoke_callable(eval_context) - - if component_converged: - break - - state.checkpoint.add(component_name) - - return CalibrationComponentResult( - component=component_name, - converged=component_converged, - component_iterations=component_iterations, - ) - - -def _read_calibration_spec(state: workflow.State, file_name: str) -> pd.DataFrame: - """Read calibration spec CSV and validate required columns.""" - path = state.filesystem.get_config_file_path(file_name) - df = pd.read_csv(path, comment="#") - - missing = [c for c in CALIBRATION_REQUIRED_COLUMNS if c not in df.columns] - if missing: - raise ValueError( - f"calibration_spec {file_name} is missing required columns: {missing}" - ) - - df = df[CALIBRATION_REQUIRED_COLUMNS].copy() - df["description"] = df["description"].astype(str) - df["coefficient"] = df["coefficient"].astype(str) - - # Normalize booleans and defaults to keep row math deterministic. - df["hold_fast"] = ( - df["hold_fast"] - .fillna(False) - .astype(str) - .str.strip() - .str.lower() - .isin(["1", "true", "t", "yes", "y"]) - ) - - for c in ["min", "max", "damping", "tolerance"]: - df[c] = pd.to_numeric(df[c], errors="coerce") - - df["method"] = df["method"].astype(str).str.strip().str.lower() - - bad_methods = df.loc[~df["method"].isin(["log_ratio", "odds_ratio"]), "method"] - if not bad_methods.empty: - raise ValueError( - f"unsupported calibration method(s): {sorted(set(bad_methods))}" - ) - - if df["damping"].isna().any(): - raise ValueError("calibration_spec damping must be numeric") - if df["tolerance"].isna().any(): - raise ValueError("calibration_spec tolerance must be numeric") - - return df - - -def _extract_utility_coefficient_names( - state: workflow.State, - model_settings: dict[str, Any] | Any, -) -> set[str]: - """ - Extract coefficient names used by the configured utility specifications. - - Templated logit models map utility-spec row labels to actual coefficient - names by segment, so their template cell values are the source of truth. - Other models use coefficient tokens in utility-spec columns. - """ - if _setting_value(model_settings, "COEFFICIENT_TEMPLATE"): - template = simulate.read_model_coefficient_template( - state.filesystem, model_settings - ) - return {str(name) for name in template.to_numpy().ravel()} - - names: set[str] = set() - - model_settings_dict = _settings_to_dict(model_settings) - spec_keys = [ - k for k in model_settings_dict.keys() if str(k).upper().endswith("SPEC") - ] - for key in spec_keys: - spec_file = model_settings_dict.get(key) - if not spec_file: - continue - - spec_path = state.filesystem.get_config_file_path(spec_file) - try: - raw = pd.read_csv(spec_path, comment="#") - except Exception: - # Do not fail hard on optional or model-specific supplemental specs. - continue - - utility_columns = [ - c - for c in raw.columns - if c - not in [ - "Description", - "Expression", - "Label", - "description", - "expression", - "label", - ] - ] - - for col in utility_columns: - for value in raw[col].dropna().tolist(): - if isinstance(value, (int, float, np.number)): - continue - text = str(value) - for token in re.findall(r"[A-Za-z_][A-Za-z0-9_]*", text): - names.add(token) - - return names - - -def _validate_calibration_coefficients_against_utility_spec( - component_name: str, - calibration_spec_df: pd.DataFrame, - utility_coeff_names: set[str], -) -> None: - """Ensure calibration coefficients are referenced in utility specifications.""" - missing = sorted( - set(calibration_spec_df["coefficient"].tolist()) - set(utility_coeff_names) - ) - if missing: - raise ValueError( - f"calibration coefficients not present in model utility specification for {component_name}: {missing}" - ) - - -def _ensure_coefficients_exist( - component_name: str, - calibration_spec_df: pd.DataFrame, - coefficients_df: pd.DataFrame, -) -> pd.DataFrame: - """Add calibration coefficients missing from coefficient file with value 0.0.""" - coeffs = coefficients_df.copy() - - for coefficient_name in calibration_spec_df["coefficient"].tolist(): - if coefficient_name not in coeffs.index: - logger.warning( - "component %s coefficient %s missing from coefficient file, adding with value 0.0", - component_name, - coefficient_name, - ) - coeffs.loc[coefficient_name, "value"] = 0.0 - - coeffs["value"] = pd.to_numeric(coeffs["value"], errors="coerce") - if coeffs["value"].isna().any(): - bad = coeffs[coeffs["value"].isna()].index.tolist() - raise ValueError(f"non-numeric coefficient values found: {bad}") - - return coeffs - - -def _warn_if_initial_values_outside_bounds( - component_name: str, - calibration_spec_df: pd.DataFrame, - coefficients_df: pd.DataFrame, -) -> None: - """Warn when initial coefficient values violate provided bounds.""" - for _, row in calibration_spec_df.iterrows(): - coefficient_name = row["coefficient"] - current_value = float(coefficients_df.loc[coefficient_name, "value"]) - lower = row["min"] - upper = row["max"] - - if not pd.isna(lower) and current_value < float(lower): - logger.warning( - "component %s coefficient %s starts below min bound (%s < %s)", - component_name, - coefficient_name, - current_value, - lower, - ) - if not pd.isna(upper) and current_value > float(upper): - logger.warning( - "component %s coefficient %s starts above max bound (%s > %s)", - component_name, - coefficient_name, - current_value, - upper, - ) - - -def _build_expression_context( - state: workflow.State, - helper_symbols: dict[str, Any], - component_name: str, - component_settings: CalibrationComponentSettings, -) -> dict[str, Any]: - """Create the evaluation context for model_value and target_value expressions.""" - context: dict[str, Any] = { - "state": state, - "np": np, - "pd": pd, - "component_output_dir": _component_output_dir(state, component_name), - "component_settings": component_settings, - } - - # Load active tables into context for direct expression access. - for table_name in list(state.existing_table_names): - try: - context[table_name] = state.get_dataframe(table_name, as_copy=False) - except Exception: - # Some entries may not be available as dataframes in all contexts. - continue - try: - network_los = state.get_injectable("network_los") - context["network_los"] = network_los - context["skim_dict"] = network_los.get_default_skim_dict() - except Exception: - # Network LOS may not be available in all contexts. - pass - - context.update(helper_symbols) - # Explicit function-call context used by calibration expressions. - context["context"] = context - return context - - -def _evaluate_and_update( - component_name: str, - calibration_spec_df: pd.DataFrame, - coefficients_df: pd.DataFrame, - eval_context: dict[str, Any], - global_iter: int, - component_iter: int, -) -> tuple[list[dict[str, Any]], dict[str, Any], pd.DataFrame, bool]: - """Evaluate spec rows, update coefficients, and return detailed records.""" - updated = coefficients_df.copy() - records: list[dict[str, Any]] = [] - - max_difference = -math.inf - max_difference_coefficient = "" - max_change = -math.inf - max_change_coefficient = "" - - num_converged = 0 - - for _, row in calibration_spec_df.iterrows(): - coefficient_name = row["coefficient"] - description = row["description"] - method = row["method"] - hold_fast = bool(row["hold_fast"]) - - default_increment = ( - row["default_increment"] - if "default_increment" in row.index - else DEFAULT_INCREMENT - ) - - prev_value = float(updated.loc[coefficient_name, "value"]) - - model_value = _eval_numeric_value( - row["model_value"], - eval_context, - component_name, - description, - "model_value", - ) - target_value = _eval_numeric_value( - row["target_value"], - eval_context, - component_name, - description, - "target_value", - ) - - difference = target_value - model_value - pct_difference = _safe_percent_difference(difference, target_value) - - tolerance = float(row["tolerance"]) - converged = abs(difference) <= tolerance - - damping = float(row["damping"]) - raw_delta = _compute_delta( - method=method, - model_value=model_value, - target_value=target_value, - damping=damping, - component_name=component_name, - description=description, - default_increment=default_increment, - ) - - candidate_value = prev_value if hold_fast else prev_value + raw_delta - - at_min = False - at_max = False - - lower = row["min"] - upper = row["max"] - - if not pd.isna(lower) and candidate_value <= float(lower): - candidate_value = float(lower) - at_min = True - if not pd.isna(upper) and candidate_value >= float(upper): - candidate_value = float(upper) - at_max = True - - if not np.isfinite(candidate_value): - raise RuntimeError( - f"non-finite next coefficient for {component_name} / {description} / {coefficient_name}" - ) - - updated.loc[coefficient_name, "value"] = candidate_value - - abs_diff = abs(difference) - abs_change = abs(candidate_value - prev_value) - - if abs_diff > max_difference: - max_difference = abs_diff - max_difference_coefficient = coefficient_name - - if abs_change > max_change: - max_change = abs_change - max_change_coefficient = coefficient_name - - if converged: - num_converged += 1 - - records.append( - { - "global_iter": global_iter, - "component_iter": component_iter, - "description": description, - "component": component_name, - "coefficient": coefficient_name, - "target_value": target_value, - "model_value": model_value, - "difference": difference, - "pct_difference": pct_difference, - "hold_fast": hold_fast, - "prev_coefficient": prev_value, - "coef_delta": abs_change, - "next_coefficient": candidate_value, - "converged": converged, - "at_min": at_min, - "at_max": at_max, - } - ) - - total_rows = len(calibration_spec_df) - num_unconverged = total_rows - num_converged - component_converged = num_unconverged == 0 - - summary_record = { - "global_iter": global_iter, - "component_iter": component_iter, - "component": component_name, - "max_difference": max_difference if max_difference != -math.inf else 0.0, - "max_difference_coefficient": max_difference_coefficient, - "max_change": max_change if max_change != -math.inf else 0.0, - "max_change_coefficient": max_change_coefficient, - "num_converged_iter": num_converged, - "tot_converged": num_converged, - "num_unconverged": num_unconverged, - } - - return records, summary_record, updated, component_converged - - -def _eval_numeric_value( - raw_value: Any, - context: dict[str, Any], - component_name: str, - description: str, - field_name: str, -) -> float: - """Evaluate numeric or expression value and enforce finite numeric result.""" - if isinstance(raw_value, (int, float, np.number)) and not pd.isna(raw_value): - value = float(raw_value) - else: - try: - value = eval(str(raw_value), {}, context) - except Exception as err: - raise RuntimeError( - f"error evaluating {field_name} for {component_name} / {description}: {raw_value}" - ) from err - - try: - value = float(value) - except Exception as err: - raise RuntimeError( - f"{field_name} did not evaluate to a numeric value for {component_name} / {description}: {value}" - ) from err - - if not np.isfinite(value): - raise RuntimeError( - f"{field_name} evaluated to non-finite value for {component_name} / {description}: {value}" - ) - - return value - - -def _compute_delta( - method: str, - model_value: float, - target_value: float, - damping: float, - component_name: str, - description: str, - default_increment: float, -) -> float: - """Compute damped coefficient delta using selected method.""" - if damping < 0: - raise RuntimeError( - f"negative damping not allowed for {component_name} / {description}: {damping}" - ) - - if method == "log_ratio": - if model_value <= 0 or target_value <= 0: - logger.warning( - f"log_ratio requires positive model and target values for {component_name} / {description}. Falling back to default increment {default_increment}" - ) - if model_value <= 0 and target_value > 0: - return default_increment - elif model_value > 0 and target_value <= 0: - return -default_increment - else: - return 0 - delta = math.log(target_value / model_value) * damping - - elif method == "odds_ratio": - # Formula requested by the calibration outline. - numerator = (target_value * model_value) - target_value - denominator = (target_value * model_value) - model_value - - if numerator <= 0 or denominator <= 0: - logger.warning( - f"odds_ratio produced invalid numerator/denominator for {component_name} / {description}. Falling back to default increment {default_increment}" - ) - if model_value <= 0 and target_value > 0: - return default_increment - elif model_value > 0 and target_value <= 0: - return -default_increment - else: - return 0 - - ratio = numerator / denominator - if ratio <= 0 or not np.isfinite(ratio): - raise RuntimeError( - f"odds_ratio produced invalid ratio for {component_name} / {description}" - ) - - delta = math.log(ratio) * damping - - else: - raise RuntimeError(f"unsupported calibration method: {method}") - - if not np.isfinite(delta): - raise RuntimeError( - f"coefficient delta is non-finite for {component_name} / {description}" - ) - - return delta - - -def _safe_percent_difference(difference: float, target_value: float) -> float: - """Return a stable percentage difference with zero-target handling.""" - if target_value == 0: - return math.inf if difference != 0 else 0.0 - return (difference / target_value) * 100.0 - - -def _persist_coefficients_to_config( - state: workflow.State, - model_settings: dict[str, Any] | Any, - coefficients_df: pd.DataFrame, -) -> None: - """Write updated coefficients back to the component coefficient file in configs.""" - coeff_file = _setting_value(model_settings, "COEFFICIENTS") - if not coeff_file: - raise RuntimeError("component model settings missing COEFFICIENTS") - - output = coefficients_df.copy() - output.index.name = "coefficient_name" - - coeff_path = state.filesystem.get_config_file_path(coeff_file) - output.to_csv(coeff_path) - - -def _append_iteration_records( - state: workflow.State, component_name: str, records: list[dict[str, Any]] -) -> None: - """Append per-coefficient calibration iteration records.""" - if not records: - return - df = pd.DataFrame(records) - - # Save a global iteration history file - global_path = state.get_output_file_path(CALIBRATION_ITERATION_FILE) - _append_csv( - df, - global_path, - unique_on=["global_iter", "component_iter", "component", "coefficient"], - ) - - # Also write component-local iteration history - component_path = ( - _component_output_dir(state, component_name) - / Path(CALIBRATION_ITERATION_FILE).name - ) - _append_csv( - df, - component_path, - unique_on=["global_iter", "component_iter", "component", "coefficient"], - ) - - -def _append_summary_records( - state: workflow.State, records: list[dict[str, Any]] -) -> None: - """Append per-iteration summary records.""" - if not records: - return - path = state.get_output_file_path(CALIBRATION_SUMMARY_FILE) - df = pd.DataFrame(records) - _append_csv( - df, - path, - unique_on=["global_iter", "component_iter", "component"], - ) - - -def _append_csv( - df: pd.DataFrame, path: Path, unique_on: list[str] | None = None -) -> None: - """Append a dataframe to a CSV file, replacing rows with matching keys.""" - os.makedirs(path.parent, exist_ok=True) - if unique_on and path.exists(): - existing = pd.read_csv(path) - df = pd.concat([existing, df], ignore_index=True).drop_duplicates( - subset=unique_on, keep="last" - ) - df.to_csv(path, index=False) - return - - write_header = not path.exists() - df.to_csv(path, mode="a", index=False, header=write_header) - - -def _component_output_dir(state: workflow.State, component_name: str) -> Path: - """Return output/calibration/ and ensure it exists.""" - component_dir = state.get_output_file_path(f"calibration/{component_name}") - os.makedirs(component_dir, exist_ok=True) - return component_dir - - -def _write_component_plots(state: workflow.State, component_name: str) -> None: - """Write/update all standard plots for one calibrated component.""" - recs = _read_component_iteration_records(state, component_name) - if recs is None or recs.empty: - return - - # Segment coefficients into manageable sets for plotting - coefs = sorted(recs.index.get_level_values("coefficient").unique()) - n_sets = math.ceil(len(coefs) / MAX_COEFFS_IN_GRAPH) - for coef_set in range(n_sets): - set_coefs = coefs[ - coef_set - * MAX_COEFFS_IN_GRAPH : min( - len(coefs), (coef_set + 1) * MAX_COEFFS_IN_GRAPH - ) - ] - _plot_coefficient_progress(state, component_name, recs, set_coefs, coef_set) - last_records = _component_last_records(recs, set_coefs) - _plot_component_values(state, component_name, last_records, coef_set) - _plot_component_pct_change(state, component_name, last_records, coef_set) - - -def _read_component_iteration_records( - state: workflow.State, component_name: str -) -> pd.DataFrame | None: - """Read all iteration records for a single component.""" - path = state.get_output_file_path(CALIBRATION_ITERATION_FILE) - if not path.exists(): - return None - - iteration_records = ( - pd.read_csv(path) - .set_index(["global_iter", "component_iter", "coefficient"]) - .sort_index() - ) - return iteration_records.loc[iteration_records.component == component_name] - - -def _plot_coefficient_progress( - state: workflow.State, - component_name: str, - recs: pd.DataFrame, - set_coefs: list[str], - coef_set: int, -) -> None: - """Plot coefficient value progression for one coefficient subset.""" - component_dir = _component_output_dir(state, component_name) - ax = ( - recs[recs.index.get_level_values("coefficient").isin(set_coefs)] - .next_coefficient.unstack("coefficient") - .plot(figsize=(10, 5)) - ) - ax.xaxis.set_label_text("Component iteration") - ax.yaxis.set_label_text("Coefficient value") - ax.legend(title="Coefficient label", loc="center left", bbox_to_anchor=(1.02, 0.5)) - plt.tight_layout() - ax.figure.savefig( - component_dir / f"coefficient_progress_set_{coef_set}.png", - bbox_inches="tight", - ) - plt.close(ax.figure) - - -def _component_last_records(recs: pd.DataFrame, set_coefs: list[str]) -> pd.DataFrame: - """Select target/model values for the latest iteration and coefficient subset.""" - filtered = recs[recs.index.get_level_values("coefficient").isin(set_coefs)] - last_global = filtered.index.get_level_values("global_iter")[-1] - last_comp = filtered.loc[last_global].index.get_level_values("component_iter")[-1] - return filtered.xs( - (last_global, last_comp), level=("global_iter", "component_iter") - )[["target_value", "model_value"]] - - -def _plot_component_values( - state: workflow.State, - component_name: str, - last_records: pd.DataFrame, - coef_set: int, -) -> None: - """Plot final target/model component values for one coefficient subset.""" - component_dir = _component_output_dir(state, component_name) - ax = last_records.plot.bar(figsize=(10, 5)) - ax.xaxis.set_tick_params(rotation=45) - ax.xaxis.set_label_text("Component value") - plt.tight_layout() - ax.figure.savefig(component_dir / f"final_components_set_{coef_set}.png") - plt.close(ax.figure) - - -def _plot_component_pct_change( - state: workflow.State, - component_name: str, - last_records: pd.DataFrame, - coef_set: int, -) -> None: - """Plot final percent difference for one coefficient subset.""" - component_dir = _component_output_dir(state, component_name) - fig, ax = plt.subplots(figsize=(10, 5)) - pct_diff = last_records.diff(axis=1).model_value / last_records.target_value - ax = pct_diff.plot.bar(ax=ax) - ax.xaxis.set_tick_params(rotation=45) - ax.xaxis.set_label_text("Coefficient") - ax.yaxis.set_label_text("% Difference between Model and Target") - plt.tight_layout() - ax.figure.savefig(component_dir / f"final_pct_change_set_{coef_set}.png") - plt.close(ax.figure) - - -def _write_generic_report( - state: workflow.State, - component_name: str, - row_records: list[dict[str, Any]], -) -> None: - """Write a simple dashboard-friendly generic report for the current component iteration.""" - if not row_records: - return - - df = pd.DataFrame(row_records) - report = ( - df[ - [ - "global_iter", - "component_iter", - "component", - "description", - "difference", - "pct_difference", - "converged", - ] - ] - .copy() - .sort_values(["global_iter", "component_iter", "description"]) - ) - - path = _component_output_dir(state, component_name) / "generic_report.csv" - _append_csv( - report, - path, - unique_on=["global_iter", "component_iter", "component", "description"], - ) - - -def _load_helper_symbols( - state: workflow.State, - component_settings: CalibrationComponentSettings, -) -> tuple[dict[str, Any], Any | None, Any | None]: - """Load helper module and return evaluation symbols and bespoke function.""" - if not component_settings.helper_module: - return {}, None, None - - module = _load_helper_module(state, component_settings.helper_module) - symbols = { - name: obj for name, obj in vars(module).items() if not name.startswith("__") - } - - bespoke = None - if component_settings.reports and component_settings.reports.bespoke: - fn_name = component_settings.reports.bespoke - if not hasattr(module, fn_name): - raise RuntimeError( - f"helper module does not define bespoke function {fn_name}" - ) - bespoke = getattr(module, fn_name) - - return symbols, bespoke, module - - -def _load_helper_module(state: workflow.State, helper_module: str): - """Load helper module by file path or import path.""" - if helper_module.endswith(".py"): - helper_path = state.filesystem.get_config_file_path(helper_module) - module_name = Path(helper_module).stem - - spec = importlib.util.spec_from_file_location(module_name, helper_path) - if spec is None or spec.loader is None: - raise RuntimeError(f"unable to load helper module from {helper_path}") - - module = importlib.util.module_from_spec(spec) - # Expose state as a global for compatibility with helper examples. - module.state = state - spec.loader.exec_module(module) - return module - - module = importlib.import_module(helper_module) - setattr(module, "state", state) - return module - - -def _infer_model_settings_file(component_name: str) -> str: - """Infer model settings yaml filename from component step name.""" - # This follows the dominant naming convention in the existing codebase. - if component_name.endswith("_simulate"): - base = component_name[: -len("_simulate")] - else: - base = component_name - return f"{base}.yaml" - - -def _settings_to_dict(model_settings: dict[str, Any] | Any) -> dict[str, Any]: - """Convert pydantic or dict model settings to a plain dictionary.""" - if isinstance(model_settings, dict): - return model_settings - if hasattr(model_settings, "model_dump"): - return model_settings.model_dump() - return dict(model_settings) - - -def _setting_value(model_settings: dict[str, Any] | Any, key: str, default=None): - """Read a setting value from dict-like or attribute-based settings.""" - if isinstance(model_settings, dict): - return model_settings.get(key, default) - return getattr(model_settings, key, default) - - -def _prior_step_name(models: list[str], component_name: str) -> str | None: - """Return the step name immediately preceding component_name in models.""" - if component_name not in models: - return None - idx = models.index(component_name) - if idx == 0: - return None - return models[idx - 1] - - -def _write_final_coefficients_snapshot( - state: workflow.State, - calibration_settings: CalibrationConfig, -) -> None: - """Write a combined final coefficients file snapshot for calibrated components.""" - frames = [] - for component_name in calibration_settings.run.calibrate_models: - model_settings_file = _infer_model_settings_file(component_name) - model_settings = state.filesystem.read_model_settings( - model_settings_file, mandatory=True - ) - coeff_df = state.filesystem.read_model_coefficients( - model_settings=model_settings - ).copy() - coeff_df = coeff_df.reset_index().rename(columns={"index": "coefficient_name"}) - coeff_df.insert(0, "component", component_name) - frames.append(coeff_df) - - if not frames: - return - - final_df = pd.concat(frames, ignore_index=True) - path = state.get_output_file_path(CALIBRATION_FINAL_COEFFICIENTS_FILE) - os.makedirs(path.parent, exist_ok=True) - final_df.to_csv(path, index=False) - - -def _ensure_calibration_output_dir(state: workflow.State) -> None: - """Ensure output/calibration exists.""" - path = state.get_output_file_path(CALIBRATION_OUTPUT_DIR) - os.makedirs(path, exist_ok=True) - - -def _calibration_coefficient_paths( - state: workflow.State, - calibration_settings: CalibrationConfig, -) -> list[Path]: - """Return the unique coefficient files modified by this calibration run.""" - paths: list[Path] = [] - seen: set[str] = set() - - for component_name in calibration_settings.run.calibrate_models: - model_settings_file = _infer_model_settings_file(component_name) - model_settings = state.filesystem.read_model_settings( - model_settings_file, mandatory=True - ) - coefficient_file = _setting_value(model_settings, "COEFFICIENTS") - if not coefficient_file: - raise RuntimeError( - f"component {component_name} model settings missing COEFFICIENTS" - ) - - path = Path(state.filesystem.get_config_file_path(coefficient_file)).resolve() - key = os.path.normcase(str(path)) - if key not in seen: - paths.append(path) - seen.add(key) - - return paths - - -def _begin_global_iteration_transaction( - state: workflow.State, - calibration_settings: CalibrationConfig, - global_iteration: int, -) -> None: - """Snapshot coefficients and durably mark a global iteration in progress.""" - recovery_dir = state.get_output_file_path(CALIBRATION_RECOVERY_DIR) - os.makedirs(recovery_dir, exist_ok=True) - - for file_number, coefficient_path in enumerate( - _calibration_coefficient_paths(state, calibration_settings) - ): - if not coefficient_path.exists(): - raise FileNotFoundError( - f"calibration coefficient file not found: {coefficient_path}" - ) - - backup_name = f"{file_number:03d}_{coefficient_path.name}" - shutil.copyfile(coefficient_path, recovery_dir / backup_name) - - # Write the marker only after all backups exist. If backup creation is - # interrupted, the previous between-iteration progress remains valid. - _write_progress( - state, - { - "in_progress_iteration": global_iteration, - "next_global_iteration": global_iteration, - "last_completed_global_iteration": global_iteration - 1, - }, - ) - - -def _restore_coefficient_backups( - state: workflow.State, - calibration_settings: CalibrationConfig, -) -> None: - """Restore the coefficient backups for an interrupted global iteration.""" - recovery_dir = state.get_output_file_path(CALIBRATION_RECOVERY_DIR) - for file_number, coefficient_path in enumerate( - _calibration_coefficient_paths(state, calibration_settings) - ): - backup_path = recovery_dir / f"{file_number:03d}_{coefficient_path.name}" - if not backup_path.exists(): - raise RuntimeError( - f"cannot recover interrupted calibration iteration: missing {backup_path}" - ) - shutil.copyfile(backup_path, coefficient_path) - - -def _read_progress(state: workflow.State) -> dict[str, Any] | None: - """Read persisted calibration progress metadata if it exists.""" - path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) - if not path.exists(): - return None - with open(path, "r", encoding="utf-8") as f: - return json.load(f) - - -def _write_progress(state: workflow.State, payload: dict[str, Any]) -> None: - """Atomically write calibration progress metadata.""" - path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) - os.makedirs(path.parent, exist_ok=True) - temporary_path = path.with_name(f".{path.name}.tmp") - with open(temporary_path, "w", encoding="utf-8") as f: - json.dump(payload, f, indent=2) - os.replace(temporary_path, path) - - -def _write_completed_progress( - state: workflow.State, - completed_global_iterations: int, - converged: bool, -) -> None: - """Mark calibration complete after all final output has been written.""" - _write_progress( - state, - { - "complete": True, - "in_progress_iteration": None, - "next_global_iteration": completed_global_iterations + 1, - "last_completed_global_iteration": completed_global_iterations, - "converged": converged, - }, - ) - - -def _run_mp_single_component( - state: workflow.State, - component_name: str, - restore_checkpoint: str, - shared_data_buffers: dict, -) -> None: - """Run a single component in multiprocess mode with explicit checkpoint control. - - This directly orchestrates the apportion → simulate → coalesce flow - without going through run_multiprocess/get_run_list, giving us precise - control over which checkpoint to apportion from. This is essential for - calibration component re-runs where we must always restart from prior_step's - state regardless of what other checkpoints exist in the pipeline. - - Parameters - ---------- - state : workflow.State - component_name : str - The model component to run. - restore_checkpoint : str - The checkpoint name to restore from before apportioning. - This should be the checkpoint representing prior_step's state. - shared_data_buffers : dict - Pre-allocated shared memory buffers for skims/shadow pricing. - """ - from activitysim.core import mp_tasks - - # Determine slice info from original settings - original_steps = state.settings.multiprocess_steps - all_models = state.settings.models - - slice_info = None - num_processes = state.settings.num_processes or 2 - chunk_size = state.settings.chunk_size or 0 - - # Find which original step this component belongs to - step_boundaries = [] - for i, step in enumerate(original_steps): - step_boundaries.append(all_models.index(step.begin)) - step_boundaries.append(len(all_models)) - - component_idx = all_models.index(component_name) - for i, step in enumerate(original_steps): - if step_boundaries[i] <= component_idx < step_boundaries[i + 1]: - if step.slice: - slice_info = step.slice.model_dump() - if step.num_processes: - num_processes = step.num_processes - if step.chunk_size: - chunk_size = step.chunk_size - break - - # Build step_info dict matching what mp_tasks functions expect - step_info = { - "name": component_name, - "models": [component_name], - "num_processes": num_processes, - "chunk_size": chunk_size, - "step_num": 0, - "slice": slice_info, - "last_checkpoint_in_previous_multiprocess_step": restore_checkpoint, - } - - injectables = _build_calibration_injectables(state) - - if num_processes == 1: - sub_proc_names = [component_name] - else: - sub_proc_names = [f"{component_name}_{i}" for i in range(num_processes)] - - fail_fast = state.settings.fail_fast - - # Apportion pipeline (split tables across sub-processes) - if num_processes > 1 and slice_info is not None: - mp_tasks.run_sub_task( - state, - multiprocessing.Process( - target=mp_tasks.mp_apportion_pipeline, - name=f"{component_name}_apportion", - args=(injectables, sub_proc_names, step_info), - ), - ) - - # For multi-process runs, subprocesses must restore from the apportioned - # pipeline (which has one checkpoint). Use LAST_CHECKPOINT so they don't - # overwrite the apportioned data with a fresh pipeline. - # For single-process runs (no apportion), use restore_checkpoint to resume - # from the correct point in the main pipeline. - if num_processes > 1: - sim_resume_after = "_" # LAST_CHECKPOINT in apportioned sub-pipeline - else: - sim_resume_after = restore_checkpoint - - # Run simulations in sub-processes - completed = mp_tasks.run_sub_simulations( - state, - injectables, - shared_data_buffers, - step_info, - sub_proc_names, - sim_resume_after, - [], # previously_completed - fail_fast, - ) - - if len(completed) != num_processes: - from activitysim.core.exceptions import SubprocessError - - raise SubprocessError( - f"{num_processes - len(completed)} processes failed in " - f"calibration step {component_name}" - ) - - # Coalesce sub-process pipelines back into main pipeline - if num_processes > 1 and slice_info is not None: - mp_tasks.run_sub_task( - state, - multiprocessing.Process( - target=mp_tasks.mp_coalesce_pipelines, - name=f"{component_name}_coalesce", - args=(injectables, sub_proc_names, slice_info), - ), - ) - - # Restore coalesced results into parent state - _restore_parent_state_from_pipeline(state) - - -def _run_in_configured_mode( - state: workflow.State, - models: list[str], - resume_after: str | None, - shared_data_buffers: dict | None = None, -) -> None: - """Run models using the same single/multiprocess mode as the parent run.""" - if not models: - return - - extra_models = _prep_model_data(state, resume_after=resume_after) - if extra_models: - # Models from the step beginning through resume_after must run first - # to recreate the correct intermediate state (since no model-level - # checkpoint existed for resume_after). - models = extra_models + models - - if state.settings.multiprocess: - # Write the restored state as a checkpoint so LAST_CHECKPOINT on disk - # reflects the correct (clean) state for the apportion subprocess. - # When extra_models were prepended, the state is from a PRIOR step - # (not the actual resume_after point) — use a non-conflicting name - # so it won't be mistakenly loaded as the resume_after state on a - # subsequent restart. - if extra_models: - state.checkpoint.add(f"_calibration_staging") - else: - state.checkpoint.add(resume_after or models[0]) - state.checkpoint.close_store() - - # When subprocess pipelines from a prior run already have the - # resume_after checkpoint (Path 2 in _prep_model_data), subprocesses - # can skip models before resume_after by reusing those pipelines - # instead of freshly apportioning. Signal this by passing - # can_reuse_subprocs=True. - can_reuse = not extra_models and resume_after is not None - - _run_multiprocess_with_overrides( - state, - models=models, - resume_after=resume_after, - shared_data_buffers=shared_data_buffers, - can_reuse_subprocs=can_reuse, - ) - # After multiprocess completes, the coalesced pipeline exists on disk. - # Restore it into the parent process state so tables are accessible - # for calibration expression evaluation. - _restore_parent_state_from_pipeline(state) - # Add a checkpoint named after the last model so that model-name - # references (e.g. _prior_step_name, resume_after on global_iter > 1) - # resolve correctly. Without this, only the step-level coalesce name - # exists in the pipeline. - state.checkpoint.add(models[-1]) - return - - # State is already at the correct point from _prep_model_data above. - # Do NOT call _prep_model_data again — the second call would build its - # table_checkpoint_map from the now-truncated in-memory checkpoint history, - # losing references to tables created after resume_after (e.g. vehicles). - state.checkpoint.add(resume_after or models[0]) - for model in models: - state.run.by_name(model) - # Ensure final model's state is persisted even if should_save_checkpoint - # returned False for it — _calibrate_component needs to restore to it. - if models: - state.checkpoint.add(models[-1]) - - -def _prep_model_data(state, resume_after=None): - """Restore the pipeline to the correct state before running models. - - Resolution priority: - 1. Direct model-level checkpoint in the main pipeline (fastest, exact). - 2. Subprocess pipelines from a prior multiprocess run — performs a - "coalesce at specific checkpoint" to recover the exact intermediate - state without re-running anything. - 3. Previous step checkpoint + re-run models from step begin through - resume_after (slowest but always works). - - Returns - ------- - list[str] - Models that must be prepended to the caller's models list to reach - the correct state at ``resume_after``. Empty when the exact - checkpoint was found and restored directly (paths 1 or 2). - """ - if resume_after: - try: - if state.checkpoint.store_is_open(): - checkpoint_names = [ - cp.get("checkpoint_name", "") for cp in state.checkpoint.checkpoints - ] - else: - from activitysim.core.workflow.checkpoint import HdfStore, ParquetStore - - pipeline_path = Path(state.checkpoint.default_pipeline_file_path()) - if state.settings.checkpoint_format == "hdf": - store = HdfStore(pipeline_path, mode="r") - else: - store = ParquetStore(pipeline_path, mode="r") - try: - checkpoint_names = store.list_checkpoint_names() - finally: - store.close() - - # Path 1: direct model-level checkpoint in main pipeline - if resume_after in checkpoint_names: - _restore_parent_state_from_pipeline(state, checkpoint_name=resume_after) - return [] - - # Path 2: subprocess pipelines (model-level checkpoints preserved) - if _restore_from_subprocess_pipelines(state, resume_after): - return [] - - # Path 3: restore from previous step and re-run - all_models = state.settings.models - mp_steps = state.settings.multiprocess_steps - if mp_steps and resume_after in all_models: - resume_idx = all_models.index(resume_after) - step_boundaries = [all_models.index(s.begin) for s in mp_steps] - step_boundaries.append(len(all_models)) - for i, step in enumerate(mp_steps): - if step_boundaries[i] <= resume_idx < step_boundaries[i + 1]: - if i > 0 and mp_steps[i - 1].name in checkpoint_names: - _restore_parent_state_from_pipeline( - state, checkpoint_name=mp_steps[i - 1].name - ) - step_begin_idx = step_boundaries[i] - extra_models = all_models[step_begin_idx : resume_idx + 1] - return extra_models - elif i == 0: - _restore_parent_state_from_pipeline( - state, checkpoint_name="_" - ) - extra_models = all_models[: resume_idx + 1] - return extra_models - break - except Exception: - logger.warning( - "calibration: could not restore from checkpoint %r, " - "falling back to LAST_CHECKPOINT", - resume_after, - ) - - # Fallback: load LAST_CHECKPOINT (appropriate after a coalesce that - # only ran the desired models) - _restore_parent_state_from_pipeline(state) - return [] - - -def _restore_from_subprocess_pipelines( - state: workflow.State, resume_after: str -) -> bool: - """Restore state from subprocess pipelines at a specific model checkpoint. - - Subprocess pipelines retain model-level checkpoints that don't exist in - the main pipeline. This performs a "coalesce at checkpoint" — reading - mirrored tables from one subprocess and concatenating sliced tables from - all subprocesses at the specified checkpoint name. - - Parameters - ---------- - state : workflow.State - resume_after : str - Model-level checkpoint name to restore from. - - Returns - ------- - bool - True if the restore succeeded; False if subprocess pipelines don't - exist or don't contain the requested checkpoint. - """ - from activitysim.core.workflow.checkpoint import ( - CHECKPOINT_NAME, - CHECKPOINT_TABLE_NAME, - HdfStore, - NON_TABLE_COLUMNS, - ParquetStore, - ) - - all_models = state.settings.models - mp_steps = state.settings.multiprocess_steps - if not mp_steps or resume_after not in all_models: - return False - - # Find the multiprocess step containing resume_after - resume_idx = all_models.index(resume_after) - step_boundaries = [all_models.index(s.begin) for s in mp_steps] - step_boundaries.append(len(all_models)) - - enclosing_step = None - num_processes = state.settings.num_processes or 2 - slice_info = None - for i, step in enumerate(mp_steps): - if step_boundaries[i] <= resume_idx < step_boundaries[i + 1]: - enclosing_step = step - if step.num_processes: - num_processes = step.num_processes - if step.slice: - slice_info = ( - step.slice.model_dump() - if hasattr(step.slice, "model_dump") - else step.slice - ) - break - - if enclosing_step is None or num_processes <= 1: - return False - - # Build subprocess pipeline file paths - step_name = enclosing_step.name - pipeline_file_name = state.filesystem.pipeline_file_name - sub_proc_names = [f"{step_name}_{i}" for i in range(num_processes)] - - def _subprocess_path(proc_name): - base = state.get_output_file_path(pipeline_file_name, prefix=proc_name) - if state.settings.checkpoint_format == "hdf": - return base - pq = Path(str(base)).with_suffix(ParquetStore.extension) - return pq if pq.exists() else base - - first_path = _subprocess_path(sub_proc_names[0]) - if not first_path.exists(): - return False - - # Open first subprocess pipeline and verify checkpoint exists - if state.settings.checkpoint_format == "hdf": - first_store = HdfStore(first_path, mode="r") - else: - first_store = ParquetStore(first_path, mode="r") - - try: - cp_names = first_store.list_checkpoint_names() - if resume_after not in cp_names: - return False - - # Read checkpoint row to get table→checkpoint mapping - cp_df = first_store.get_dataframe(CHECKPOINT_TABLE_NAME) - cp_row = cp_df[cp_df[CHECKPOINT_NAME] == resume_after].iloc[-1] - - table_map = {} - for col in cp_row.index: - if col not in NON_TABLE_COLUMNS and cp_row[col]: - table_map[col] = cp_row[col] - - # Read all tables from first subprocess at this checkpoint - tables = {} - for table_name, cp_for_table in table_map.items(): - try: - tables[table_name] = first_store.get_dataframe(table_name, cp_for_table) - except (FileNotFoundError, KeyError): - logger.warning( - f"calibration: subprocess pipeline missing table " - f"{table_name} at {cp_for_table}" - ) - finally: - first_store.close() - - if not tables: - return False - - # Determine sliced tables that need concatenation across processes - sliced_table_names = set(slice_info.get("tables", [])) if slice_info else set() - - # Read sliced tables from remaining subprocesses and concatenate - if num_processes > 1 and sliced_table_names: - omnibus = {t: [tables[t]] for t in sliced_table_names if t in tables} - - for proc_name in sub_proc_names[1:]: - proc_path = _subprocess_path(proc_name) - if not proc_path.exists(): - logger.warning( - f"calibration: subprocess pipeline not found: {proc_path}" - ) - return False - - if state.settings.checkpoint_format == "hdf": - proc_store = HdfStore(proc_path, mode="r") - else: - proc_store = ParquetStore(proc_path, mode="r") - - try: - proc_cp_df = proc_store.get_dataframe(CHECKPOINT_TABLE_NAME) - proc_row = proc_cp_df[proc_cp_df[CHECKPOINT_NAME] == resume_after].iloc[ - -1 - ] - - for table_name in list(omnibus.keys()): - cp_for_table = proc_row.get(table_name, "") - if cp_for_table: - omnibus[table_name].append( - proc_store.get_dataframe(table_name, cp_for_table) - ) - finally: - proc_store.close() - - # Replace sliced tables with concatenated versions - for table_name, dfs in omnibus.items(): - tables[table_name] = pd.concat(dfs, sort=False) - - # Load into parent state - prior_rng_channels = list(state.get_injectable("rng_channels", [])) - prior_index_to_channel = ( - dict(state.rng().index_to_channel) - if hasattr(state.rng(), "index_to_channel") - else {} - ) - - state.init_state() - if state.checkpoint.store_is_open(): - state.checkpoint.close_store() - state.checkpoint.open_store(overwrite=False) - - for table_name, df in tables.items(): - state.add_table(table_name, df) - - _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) - - # Mark all tables dirty for subsequent checkpoint.add - for table_name in list(state.existing_table_names): - state.existing_table_status[table_name] = True - - logger.info( - "calibration: restored %d tables from subprocess pipelines at " - "checkpoint '%s'", - len(tables), - resume_after, - ) - return True - - -def _run_multiprocess_with_overrides( - state: workflow.State, - models: list[str], - resume_after: str | None, - shared_data_buffers: dict | None = None, - can_reuse_subprocs: bool = False, -) -> None: - """Run multiprocess with temporary settings overrides for calibration passes. - - Parameters - ---------- - can_reuse_subprocs : bool, default False - When True, subprocess pipelines from a prior run are assumed to exist - and contain the ``resume_after`` checkpoint. Breadcrumbs are written - so that ``get_run_list`` populates ``step_info["resume_after"]``, - apportion is skipped (reusing existing subprocess pipelines), and - subprocesses resume from their model-level checkpoint — skipping - already-completed models. - """ - from collections import OrderedDict - - from activitysim.core import mp_tasks - - original_models = state.settings.models - original_mp_steps = state.settings.multiprocess_steps - original_resume_after = state.settings.resume_after - - # Build valid multiprocess_steps for the requested model subset. - calibration_mp_steps = _build_calibration_mp_steps( - models=models, - original_steps=original_mp_steps, - all_models=original_models, - ) - - state.settings.models = models - state.settings.multiprocess_steps = calibration_mp_steps - - if can_reuse_subprocs and resume_after: - # Include resume_after in the models list so get_breadcrumbs can - # locate the step containing it. Subprocesses will skip this model - # (it's already checkpointed in their pipeline) and run the rest. - models = [resume_after] + models - - # Rebuild steps with resume_after included. - calibration_mp_steps = _build_calibration_mp_steps( - models=models, - original_steps=original_mp_steps, - all_models=original_models, - ) - state.settings.models = models - state.settings.multiprocess_steps = calibration_mp_steps - state.settings.resume_after = resume_after - - # Write minimal breadcrumbs indicating the step containing - # resume_after has completed apportion (so it's skipped) but - # simulate/coalesce need re-running. - breadcrumbs = OrderedDict() - for step in calibration_mp_steps: - step_dict = {"name": step.name, "apportion": True} - breadcrumbs[step.name] = step_dict - # Find the step containing resume_after - all_models = state.settings.models - if resume_after in all_models: - step_begin = all_models.index(step.begin) - step_models_in_step = [ - m for m in all_models[step_begin:] if m in models - ] - if resume_after in step_models_in_step: - # This step contains resume_after — stop here. - # get_breadcrumbs will mark simulate/coalesce for re-run. - break - - mp_tasks.write_breadcrumbs(state, breadcrumbs) - else: - # No reuse: calibration manages pipeline state externally via - # _restore_parent_state_from_pipeline and checkpoint.add, so the MP - # system's breadcrumb-based resume logic must not be triggered. - state.settings.resume_after = None - - try: - injectables = _build_calibration_injectables(state) - mp_tasks.run_multiprocess( - state, - injectables, - shared_data_buffers=shared_data_buffers, - skip_final_checkpoint=True, - force_resume=resume_after is not None and not can_reuse_subprocs, - ) - finally: - state.settings.models = original_models - state.settings.resume_after = original_resume_after - state.settings.multiprocess_steps = original_mp_steps - - -def _reregister_rng_channels( - state: workflow.State, - prior_channels: list[str], - prior_index_to_channel: dict[str, str] = None, -) -> None: - """Re-register RNG channels that were lost during init_state().""" - current_channels = set(state.get_injectable("rng_channels", [])) - for channel_name in prior_channels: - if channel_name not in current_channels and state.is_table(channel_name): - try: - state.rng().add_channel(channel_name, state.get_dataframe(channel_name)) - current_channels.add(channel_name) - except Exception: - pass - # For channels whose tables don't exist at the restored checkpoint, - # register an empty channel. Do NOT pre-load from a later checkpoint - # in the store — that data may include modifications from downstream - # models and would pollute the pre-model state. The empty channel - # allows the model's normal table factory to create the table fresh - # and extend the channel without hitting the disjoint-index assertion. - if prior_index_to_channel: - for index_name, channel_name in prior_index_to_channel.items(): - if index_name not in state.rng().index_to_channel: - if channel_name not in state.rng().channels: - empty_df = pd.DataFrame( - index=pd.Index([], dtype="int64", name=index_name) - ) - state.rng().add_channel(channel_name, empty_df) - else: - state.rng().index_to_channel[index_name] = channel_name - current_channels.add(channel_name) - state.add_injectable("rng_channels", list(current_channels)) - - -def _invalidate_derived_tables(state: workflow.State) -> None: - """Drop factory-produced tables that may be stale after a calibration restore. - - When a calibrated model (e.g. auto_ownership) changes a table that a - @workflow.table factory depends on (e.g. vehicles depends on households), - the checkpoint may contain a stale version of that factory table. Dropping - it forces the factory to regenerate from current data on next access. - - Auto-detection rule: invalidate any table that is (a) registered as a - @workflow.table factory, (b) has DataFrame parameters (= table dependencies), - and (c) is in RANDOM_CHANNELS. This currently matches only 'vehicles' but - will automatically cover future factory tables with the same pattern. - """ - settings = read_calibration_settings(state) - if not settings: - return - - tables_to_invalidate = settings.run.invalidate_tables - if tables_to_invalidate is None: - # Vehicles needs regeneration only when calibration changes - # households.auto_ownership. Downstream calibration components must - # retain vehicle_type_choice's vehicle attributes. - tables_to_invalidate = ( - ["vehicles"] - if "auto_ownership_simulate" in settings.run.calibrate_models - else [] - ) - - logger.debug( - "calibration: tables detected for invalidation: %s", tables_to_invalidate - ) - tables_before = set(state.existing_table_names) - - for table_name in tables_to_invalidate: - if state.is_table(table_name): - state.drop_table(table_name) - state.rng().drop_channel(table_name) - state.get_dataframe(table_name, as_copy=False) - logger.debug("calibration: invalidated derived table '%s'", table_name) - - tables_after = set(state.existing_table_names) - lost = tables_before - tables_after - set(tables_to_invalidate) - if lost: - logger.error( - "calibration: tables unexpectedly removed during invalidation: %s", lost - ) - - -def _restore_parent_state_from_pipeline( - state: workflow.State, checkpoint_name: str = "_" -) -> None: - """Restore pipeline tables into the parent process state. - - After a multiprocess run, the parent's in-memory state is stale. - This loads a specific checkpoint from the pipeline store so that - calibration expressions can evaluate against model outputs. - - Parameters - ---------- - checkpoint_name : str, default "_" - The checkpoint to restore from. Use a model-level checkpoint name - (e.g. the prior step name) to get the exact state at that point, - avoiding pollution from downstream models that may have added rows - to shared tables like ``tours``. The default ``"_"`` loads the - last checkpoint, which is appropriate immediately after a coalesce - that only ran the desired models. - - All tables are explicitly re-checkpointed so that subsequent apportion - subprocesses can load them from a direct file path without relying on - checkpoint backtracking through potentially ambiguous checkpoint history. - """ - # Capture RNG state before restore — models may have dynamically - # added channels (e.g. "vehicles") that aren't in the default - # rng_channels injectable and would be lost by init_state(). - prior_rng_channels = list(state.get_injectable("rng_channels", [])) - prior_index_to_channel = ( - dict(state.rng().index_to_channel) - if hasattr(state.rng(), "index_to_channel") - else {} - ) - - if state.checkpoint.store_is_open(): - state.checkpoint.close_store() - state.checkpoint.restore(resume_after=checkpoint_name) - - _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) - - # After restore, all tables are clean (status=False). Mark them dirty so - # the next checkpoint.add() writes them to disk at a known checkpoint name. - # This ensures apportion subprocesses find table files at a single, - # unambiguous checkpoint rather than needing to backtrack through history. - for table_name in list(state.existing_table_names): - state.existing_table_status[table_name] = True - - -def _initialize_mp_shared_resources(state: workflow.State) -> dict: - """Allocate shared data buffers (skims, shadow pricing) once for reuse. - - This mirrors the allocation logic in mp_tasks.run_multiprocess but - is called once at calibration start rather than on every sub-run. - """ - from activitysim.core import mp_tasks, tracing - - shared_data_buffers = {} - sharrow_enabled = state.settings.sharrow - - t0 = tracing.print_elapsed_time() - if not sharrow_enabled: - shared_data_buffers.update(mp_tasks.allocate_shared_skim_buffers(state)) - t0 = tracing.print_elapsed_time("calibration: allocate shared skim buffer", t0) - - shared_data_buffers.update(mp_tasks.allocate_shared_shadow_pricing_buffers(state)) - t0 = tracing.print_elapsed_time( - "calibration: allocate shared shadow_pricing buffer", t0 - ) - - shared_data_buffers.update( - mp_tasks.allocate_shared_shadow_pricing_buffers_choice(state) - ) - t0 = tracing.print_elapsed_time( - "calibration: allocate shared shadow_pricing choice buffer", t0 - ) - - # Load skim data into the shared buffers. - if sharrow_enabled: - shared_data_buffers["skim_dataset"] = "sh.Dataset:skim_dataset" - from activitysim.core import flow, skim_dataset # noqa: F401 - - state.get_injectable("skim_dataset") - else: - if len(shared_data_buffers) > 0: - injectables = _build_calibration_injectables(state) - mp_tasks.run_sub_task( - state, - multiprocessing.Process( - target=mp_tasks.mp_setup_skims, - name="mp_setup_skims_calibration", - args=(injectables,), - kwargs=shared_data_buffers, - ), - ) - - # Make skims available in the parent process for expression evaluation. - state.add_injectable("data_buffers", shared_data_buffers) - try: - state.get_injectable("network_los") - except Exception: - logger.warning( - "calibration: could not resolve network_los in parent process; " - "skim-dependent expressions may fail" - ) - - return shared_data_buffers - - -def _build_calibration_mp_steps( - models: list[str], - original_steps: list[MultiprocessStep], - all_models: list[str], -) -> list[MultiprocessStep]: - """Build valid MultiprocessStep objects for a calibration model subset. - - The key challenge is that get_run_list() in mp_tasks requires: - - The first step's begin == models[0] - - Steps are ordered and non-overlapping - - Each step's begin is in the models list - - We intersect the original multiprocess_steps with the requested model - subset and construct new steps that satisfy these constraints. - """ - if not models: - return [] - - # Determine which original step each model in the full list belongs to. - # Build a mapping: model_name -> original step index - model_to_step: dict[str, int] = {} - step_boundaries = [] - for i, step in enumerate(original_steps): - begin_idx = all_models.index(step.begin) - step_boundaries.append(begin_idx) - step_boundaries.append(len(all_models)) - - for i, step in enumerate(original_steps): - for model_idx in range(step_boundaries[i], step_boundaries[i + 1]): - model_to_step[all_models[model_idx]] = i - - # Group the requested models by their original step - from collections import OrderedDict - - step_model_groups: OrderedDict[int, list[str]] = OrderedDict() - for model in models: - step_idx = model_to_step.get(model) - if step_idx is None: - continue - step_model_groups.setdefault(step_idx, []).append(model) - - # Build new MultiprocessStep for each group. - # Some original steps (e.g. mp_initialize) omit num_processes, slice, - # and chunk_size — these default to None on MultiprocessStep and - # get_run_list() applies global defaults when they are absent. - # Step names include the first model to ensure uniqueness across multiple - # intermediate runs that draw from the same original step. - new_steps = [] - for step_idx, step_models in step_model_groups.items(): - orig_step = original_steps[step_idx] - kwargs: dict[str, Any] = { - "name": orig_step.name, - "begin": step_models[0], - } - if orig_step.num_processes is not None: - kwargs["num_processes"] = orig_step.num_processes - if orig_step.slice is not None: - kwargs["slice"] = orig_step.slice - if orig_step.chunk_size is not None: - kwargs["chunk_size"] = orig_step.chunk_size - new_steps.append(MultiprocessStep(**kwargs)) - - return new_steps - - -def _build_calibration_injectables(state: workflow.State) -> dict: - """Build the injectables dict for multiprocess sub-processes.""" - injectables = {} - for key in MP_INJECTABLES: - try: - injectables[key] = state.get_injectable(key) - except KeyError: - pass - injectables["settings"] = state.settings - return injectables diff --git a/activitysim/core/calibration/README.md b/activitysim/core/calibration/README.md new file mode 100644 index 0000000000..66c92b3802 --- /dev/null +++ b/activitysim/core/calibration/README.md @@ -0,0 +1,27 @@ +# ActivitySim calibration package + +This package implements ActivitySim's automated coefficient calibration workflow. +The modules are divided by responsibility so orchestration, model execution, and +calibration math can evolve independently. + +- `__init__.py` defines the public API and compatibility access to former private + attributes from `activitysim.core.calibration`. +- `settings.py` defines calibration configuration models, result types, and + configuration loading. +- `orchestrator.py` coordinates global iterations and model-component sequencing. +- `component.py` runs component iterations, validates calibration specifications, + and calculates coefficient updates. +- `expressions.py` builds expression contexts, evaluates target/model expressions, + and loads optional helper modules. +- `coefficients.py` locates, reads, and writes model coefficient files. +- `reporting.py` writes iteration histories, summaries, plots, and final snapshots. +- `recovery.py` maintains calibration progress and start-of-iteration coefficient + backups. +- `execution.py` restores pipeline state and dispatches model execution. +- `multiprocess.py` contains subprocess orchestration, shared-resource setup, and + multiprocess pipeline restoration. + +Dependencies should flow from `orchestrator.py` into the focused modules. Lower +level modules should not import the orchestrator. Multiprocess code imports back +into `execution.py` only inside the single-component runner to avoid an import-time +cycle. diff --git a/activitysim/core/calibration/__init__.py b/activitysim/core/calibration/__init__.py new file mode 100644 index 0000000000..442046abd7 --- /dev/null +++ b/activitysim/core/calibration/__init__.py @@ -0,0 +1,70 @@ +# ActivitySim +# See full license in LICENSE.txt. +"""Automated calibration support for ActivitySim model runs. + +The public API is intentionally small. Private attributes from the former +single-file module remain available through ``__getattr__`` for compatibility +while callers migrate to the package modules. +""" + +from . import ( + coefficients, + component, + execution, + expressions, + multiprocess, + recovery, + reporting, + settings, +) +from .orchestrator import run_calibration_loop +from .settings import ( + CalibrationComponentResult, + CalibrationComponentSettings, + CalibrationConfig, + CalibrationReportsSettings, + CalibrationRunResult, + CalibrationRunSettings, + calibration_enabled, + read_calibration_settings, +) + +__all__ = [ + "CalibrationComponentResult", + "CalibrationComponentSettings", + "CalibrationConfig", + "CalibrationReportsSettings", + "CalibrationRunResult", + "CalibrationRunSettings", + "calibration_enabled", + "read_calibration_settings", + "run_calibration_loop", +] + +_COMPATIBILITY_MODULES = ( + settings, + component, + expressions, + coefficients, + reporting, + recovery, + execution, + multiprocess, +) + + +def __getattr__(name): + for module in _COMPATIBILITY_MODULES: + if hasattr(module, name): + return getattr(module, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__(): + compatibility_names = { + name + for module in _COMPATIBILITY_MODULES + for name in vars(module) + if not name.startswith("__") + } + return sorted(set(globals()) | compatibility_names) diff --git a/activitysim/core/calibration/coefficients.py b/activitysim/core/calibration/coefficients.py new file mode 100644 index 0000000000..9aa15bcd4c --- /dev/null +++ b/activitysim/core/calibration/coefficients.py @@ -0,0 +1,85 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +import pandas as pd + +from activitysim.core import workflow + +from .settings import CalibrationConfig + + +def _persist_coefficients_to_config( + state: workflow.State, + model_settings: dict[str, Any] | Any, + coefficients_df: pd.DataFrame, +) -> None: + """Write updated coefficients back to the component coefficient file in configs.""" + coeff_file = _setting_value(model_settings, "COEFFICIENTS") + if not coeff_file: + raise RuntimeError("component model settings missing COEFFICIENTS") + + output = coefficients_df.copy() + output.index.name = "coefficient_name" + + coeff_path = state.filesystem.get_config_file_path(coeff_file) + output.to_csv(coeff_path) + + +def _infer_model_settings_file(component_name: str) -> str: + """Infer model settings yaml filename from component step name.""" + # This follows the dominant naming convention in the existing codebase. + if component_name.endswith("_simulate"): + base = component_name[: -len("_simulate")] + else: + base = component_name + return f"{base}.yaml" + + +def _settings_to_dict(model_settings: dict[str, Any] | Any) -> dict[str, Any]: + """Convert pydantic or dict model settings to a plain dictionary.""" + if isinstance(model_settings, dict): + return model_settings + if hasattr(model_settings, "model_dump"): + return model_settings.model_dump() + return dict(model_settings) + + +def _setting_value(model_settings: dict[str, Any] | Any, key: str, default=None): + """Read a setting value from dict-like or attribute-based settings.""" + if isinstance(model_settings, dict): + return model_settings.get(key, default) + return getattr(model_settings, key, default) + + +def _calibration_coefficient_paths( + state: workflow.State, + calibration_settings: CalibrationConfig, +) -> list[Path]: + """Return the unique coefficient files modified by this calibration run.""" + paths: list[Path] = [] + seen: set[str] = set() + + for component_name in calibration_settings.run.calibrate_models: + model_settings_file = _infer_model_settings_file(component_name) + model_settings = state.filesystem.read_model_settings( + model_settings_file, mandatory=True + ) + coefficient_file = _setting_value(model_settings, "COEFFICIENTS") + if not coefficient_file: + raise RuntimeError( + f"component {component_name} model settings missing COEFFICIENTS" + ) + + path = Path(state.filesystem.get_config_file_path(coefficient_file)).resolve() + key = os.path.normcase(str(path)) + if key not in seen: + paths.append(path) + seen.add(key) + + return paths + diff --git a/activitysim/core/calibration/component.py b/activitysim/core/calibration/component.py new file mode 100644 index 0000000000..7a3285ca3a --- /dev/null +++ b/activitysim/core/calibration/component.py @@ -0,0 +1,514 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +import logging +import math +import re +from typing import Any + +import numpy as np +import pandas as pd + +from activitysim.core import simulate, workflow + +from .coefficients import ( + _infer_model_settings_file, + _persist_coefficients_to_config, + _setting_value, + _settings_to_dict, +) +from .execution import _invalidate_derived_tables, _prep_model_data +from .expressions import ( + _build_expression_context, + _compute_delta, + _eval_numeric_value, + _load_helper_symbols, +) +from .multiprocess import _run_mp_single_component +from .reporting import ( + _append_iteration_records, + _append_summary_records, + _write_generic_report, +) +from .settings import CalibrationComponentResult, CalibrationComponentSettings + +logger = logging.getLogger("calibration") + +DEFAULT_INCREMENT = 2.0 +CALIBRATION_REQUIRED_COLUMNS = [ + "description", + "coefficient", + "model_value", + "target_value", + "hold_fast", + "min", + "max", + "damping", + "method", + "tolerance", +] + + +def _calibrate_component( + state: workflow.State, + component_name: str, + component_settings: CalibrationComponentSettings, + prior_step: str, + global_iter: int, + shared_data_buffers: dict | None = None, +) -> CalibrationComponentResult: + """Run iterative coefficient calibration for one component.""" + model_settings_file = _infer_model_settings_file(component_name) + model_settings = state.filesystem.read_model_settings( + model_settings_file, mandatory=True + ) + + coefficients_df = state.filesystem.read_model_coefficients( + model_settings=model_settings + ) + helper_symbols, bespoke_callable, helper_module = _load_helper_symbols( + state, + component_settings, + ) + + calibration_spec_df = _read_calibration_spec( + state, component_settings.calibration_spec + ) + + utility_coeff_names = _extract_utility_coefficient_names(state, model_settings) + _validate_calibration_coefficients_against_utility_spec( + component_name, + calibration_spec_df, + utility_coeff_names, + ) + + coefficients_df = _ensure_coefficients_exist( + component_name, + calibration_spec_df, + coefficients_df, + ) + + _warn_if_initial_values_outside_bounds( + component_name, calibration_spec_df, coefficients_df + ) + + component_converged = False + component_iterations = 0 + + # Determine the checkpoint name to restore from for component re-runs. + # In MP mode, the checkpoint that represents prior_step's completed state + # is the last checkpoint in the pipeline before we run the component. + # We capture it once and reuse across component iterations. + mp_restore_checkpoint = None + if state.settings.multiprocess and shared_data_buffers is not None: + # The pipeline should already be open from _restore_parent_state_from_pipeline + # called after precursor/intermediate models ran. The last checkpoint + # in the pipeline represents the state at prior_step. + if state.checkpoint.checkpoints: + mp_restore_checkpoint = state.checkpoint.last_checkpoint.get( + "checkpoint_name", "_" + ) + else: + mp_restore_checkpoint = "_" + + for component_iter in range(1, component_settings.submodel_max_iterations + 1): + component_iterations = component_iter + run_model_name = f"{component_name}.c_i{component_iter};" f"g_i{global_iter}" + # Re-run only this component from its prior checkpoint so model values + # reflect the current candidate coefficients for this component. + if state.settings.multiprocess and shared_data_buffers is not None: + # Use direct MP orchestration with explicit checkpoint control. + # This ensures we always apportion from prior_step's state, + # even after multiple component iterations. + _run_mp_single_component( + state, + component_name=component_name, + # Always restore from the same immutable pre-component + # checkpoint. LAST_CHECKPOINT may point to the prior iteration's + # coalesced component output and is therefore not a safe baseline. + restore_checkpoint=mp_restore_checkpoint, + shared_data_buffers=shared_data_buffers, + ) + else: + # Restore to prior_step ourselves then run the model directly. + # state.run(resume_after=prior_step) would trigger + # checkpoint.restore → init_state which creates a fresh RNG. + # If prior_step is before the calibrated model created its table + # (e.g. vehicles), the table won't be in that checkpoint and the + # RNG channel won't be registered — causing a crash when the + # model tries to use it. By restoring here and calling by_name, + # we keep the RNG channels from _prep_model_data intact. + extra_models = _prep_model_data(state, resume_after=prior_step) + if extra_models: + # prior_step checkpoint not found directly; run intermediate + # models (e.g. annotators) to recreate the correct state. + for m in extra_models: + state.run.by_name(m) + _invalidate_derived_tables(state) + state.checkpoint.add(prior_step) + state.run.by_name(run_model_name) + + eval_context = _build_expression_context( + state, helper_symbols, component_name, component_settings + ) + + ( + row_records, + summary_record, + new_coefficients_df, + component_converged, + ) = _evaluate_and_update( + component_name=component_name, + calibration_spec_df=calibration_spec_df, + coefficients_df=coefficients_df, + eval_context=eval_context, + global_iter=global_iter, + component_iter=component_iter, + ) + + coefficients_df = new_coefficients_df + + _persist_coefficients_to_config(state, model_settings, coefficients_df) + _append_iteration_records(state, component_name, row_records) + _append_summary_records(state, [summary_record]) + + if component_settings.reports.generic: + _write_generic_report(state, component_name, row_records) + + if bespoke_callable is not None: + bespoke_callable(eval_context) + + if component_converged: + break + + state.checkpoint.add(component_name) + + return CalibrationComponentResult( + component=component_name, + converged=component_converged, + component_iterations=component_iterations, + ) + + +def _read_calibration_spec(state: workflow.State, file_name: str) -> pd.DataFrame: + """Read calibration spec CSV and validate required columns.""" + path = state.filesystem.get_config_file_path(file_name) + df = pd.read_csv(path, comment="#") + + missing = [c for c in CALIBRATION_REQUIRED_COLUMNS if c not in df.columns] + if missing: + raise ValueError( + f"calibration_spec {file_name} is missing required columns: {missing}" + ) + + df = df[CALIBRATION_REQUIRED_COLUMNS].copy() + df["description"] = df["description"].astype(str) + df["coefficient"] = df["coefficient"].astype(str) + + # Normalize booleans and defaults to keep row math deterministic. + df["hold_fast"] = ( + df["hold_fast"] + .fillna(False) + .astype(str) + .str.strip() + .str.lower() + .isin(["1", "true", "t", "yes", "y"]) + ) + + for c in ["min", "max", "damping", "tolerance"]: + df[c] = pd.to_numeric(df[c], errors="coerce") + + df["method"] = df["method"].astype(str).str.strip().str.lower() + + bad_methods = df.loc[~df["method"].isin(["log_ratio", "odds_ratio"]), "method"] + if not bad_methods.empty: + raise ValueError( + f"unsupported calibration method(s): {sorted(set(bad_methods))}" + ) + + if df["damping"].isna().any(): + raise ValueError("calibration_spec damping must be numeric") + if df["tolerance"].isna().any(): + raise ValueError("calibration_spec tolerance must be numeric") + + return df + + +def _extract_utility_coefficient_names( + state: workflow.State, + model_settings: dict[str, Any] | Any, +) -> set[str]: + """ + Extract coefficient names used by the configured utility specifications. + + Templated logit models map utility-spec row labels to actual coefficient + names by segment, so their template cell values are the source of truth. + Other models use coefficient tokens in utility-spec columns. + """ + if _setting_value(model_settings, "COEFFICIENT_TEMPLATE"): + template = simulate.read_model_coefficient_template( + state.filesystem, model_settings + ) + return {str(name) for name in template.to_numpy().ravel()} + + names: set[str] = set() + + model_settings_dict = _settings_to_dict(model_settings) + spec_keys = [ + k for k in model_settings_dict.keys() if str(k).upper().endswith("SPEC") + ] + for key in spec_keys: + spec_file = model_settings_dict.get(key) + if not spec_file: + continue + + spec_path = state.filesystem.get_config_file_path(spec_file) + try: + raw = pd.read_csv(spec_path, comment="#") + except Exception: + # Do not fail hard on optional or model-specific supplemental specs. + continue + + utility_columns = [ + c + for c in raw.columns + if c + not in [ + "Description", + "Expression", + "Label", + "description", + "expression", + "label", + ] + ] + + for col in utility_columns: + for value in raw[col].dropna().tolist(): + if isinstance(value, (int, float, np.number)): + continue + text = str(value) + for token in re.findall(r"[A-Za-z_][A-Za-z0-9_]*", text): + names.add(token) + + return names + + +def _validate_calibration_coefficients_against_utility_spec( + component_name: str, + calibration_spec_df: pd.DataFrame, + utility_coeff_names: set[str], +) -> None: + """Ensure calibration coefficients are referenced in utility specifications.""" + missing = sorted( + set(calibration_spec_df["coefficient"].tolist()) - set(utility_coeff_names) + ) + if missing: + raise ValueError( + f"calibration coefficients not present in model utility specification for {component_name}: {missing}" + ) + + +def _ensure_coefficients_exist( + component_name: str, + calibration_spec_df: pd.DataFrame, + coefficients_df: pd.DataFrame, +) -> pd.DataFrame: + """Add calibration coefficients missing from coefficient file with value 0.0.""" + coeffs = coefficients_df.copy() + + for coefficient_name in calibration_spec_df["coefficient"].tolist(): + if coefficient_name not in coeffs.index: + logger.warning( + "component %s coefficient %s missing from coefficient file, adding with value 0.0", + component_name, + coefficient_name, + ) + coeffs.loc[coefficient_name, "value"] = 0.0 + + coeffs["value"] = pd.to_numeric(coeffs["value"], errors="coerce") + if coeffs["value"].isna().any(): + bad = coeffs[coeffs["value"].isna()].index.tolist() + raise ValueError(f"non-numeric coefficient values found: {bad}") + + return coeffs + + +def _warn_if_initial_values_outside_bounds( + component_name: str, + calibration_spec_df: pd.DataFrame, + coefficients_df: pd.DataFrame, +) -> None: + """Warn when initial coefficient values violate provided bounds.""" + for _, row in calibration_spec_df.iterrows(): + coefficient_name = row["coefficient"] + current_value = float(coefficients_df.loc[coefficient_name, "value"]) + lower = row["min"] + upper = row["max"] + + if not pd.isna(lower) and current_value < float(lower): + logger.warning( + "component %s coefficient %s starts below min bound (%s < %s)", + component_name, + coefficient_name, + current_value, + lower, + ) + if not pd.isna(upper) and current_value > float(upper): + logger.warning( + "component %s coefficient %s starts above max bound (%s > %s)", + component_name, + coefficient_name, + current_value, + upper, + ) + + +def _evaluate_and_update( + component_name: str, + calibration_spec_df: pd.DataFrame, + coefficients_df: pd.DataFrame, + eval_context: dict[str, Any], + global_iter: int, + component_iter: int, +) -> tuple[list[dict[str, Any]], dict[str, Any], pd.DataFrame, bool]: + """Evaluate spec rows, update coefficients, and return detailed records.""" + updated = coefficients_df.copy() + records: list[dict[str, Any]] = [] + + max_difference = -math.inf + max_difference_coefficient = "" + max_change = -math.inf + max_change_coefficient = "" + + num_converged = 0 + + for _, row in calibration_spec_df.iterrows(): + coefficient_name = row["coefficient"] + description = row["description"] + method = row["method"] + hold_fast = bool(row["hold_fast"]) + + default_increment = ( + row["default_increment"] + if "default_increment" in row.index + else DEFAULT_INCREMENT + ) + + prev_value = float(updated.loc[coefficient_name, "value"]) + + model_value = _eval_numeric_value( + row["model_value"], + eval_context, + component_name, + description, + "model_value", + ) + target_value = _eval_numeric_value( + row["target_value"], + eval_context, + component_name, + description, + "target_value", + ) + + difference = target_value - model_value + pct_difference = _safe_percent_difference(difference, target_value) + + tolerance = float(row["tolerance"]) + converged = abs(difference) <= tolerance + + damping = float(row["damping"]) + raw_delta = _compute_delta( + method=method, + model_value=model_value, + target_value=target_value, + damping=damping, + component_name=component_name, + description=description, + default_increment=default_increment, + ) + + candidate_value = prev_value if hold_fast else prev_value + raw_delta + + at_min = False + at_max = False + + lower = row["min"] + upper = row["max"] + + if not pd.isna(lower) and candidate_value <= float(lower): + candidate_value = float(lower) + at_min = True + if not pd.isna(upper) and candidate_value >= float(upper): + candidate_value = float(upper) + at_max = True + + if not np.isfinite(candidate_value): + raise RuntimeError( + f"non-finite next coefficient for {component_name} / {description} / {coefficient_name}" + ) + + updated.loc[coefficient_name, "value"] = candidate_value + + abs_diff = abs(difference) + abs_change = abs(candidate_value - prev_value) + + if abs_diff > max_difference: + max_difference = abs_diff + max_difference_coefficient = coefficient_name + + if abs_change > max_change: + max_change = abs_change + max_change_coefficient = coefficient_name + + if converged: + num_converged += 1 + + records.append( + { + "global_iter": global_iter, + "component_iter": component_iter, + "description": description, + "component": component_name, + "coefficient": coefficient_name, + "target_value": target_value, + "model_value": model_value, + "difference": difference, + "pct_difference": pct_difference, + "hold_fast": hold_fast, + "prev_coefficient": prev_value, + "coef_delta": abs_change, + "next_coefficient": candidate_value, + "converged": converged, + "at_min": at_min, + "at_max": at_max, + } + ) + + total_rows = len(calibration_spec_df) + num_unconverged = total_rows - num_converged + component_converged = num_unconverged == 0 + + summary_record = { + "global_iter": global_iter, + "component_iter": component_iter, + "component": component_name, + "max_difference": max_difference if max_difference != -math.inf else 0.0, + "max_difference_coefficient": max_difference_coefficient, + "max_change": max_change if max_change != -math.inf else 0.0, + "max_change_coefficient": max_change_coefficient, + "num_converged_iter": num_converged, + "tot_converged": num_converged, + "num_unconverged": num_unconverged, + } + + return records, summary_record, updated, component_converged + + +def _safe_percent_difference(difference: float, target_value: float) -> float: + """Return a stable percentage difference with zero-target handling.""" + if target_value == 0: + return math.inf if difference != 0 else 0.0 + return (difference / target_value) * 100.0 diff --git a/activitysim/core/calibration/execution.py b/activitysim/core/calibration/execution.py new file mode 100644 index 0000000000..0c7f8e98ab --- /dev/null +++ b/activitysim/core/calibration/execution.py @@ -0,0 +1,262 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +import logging +from pathlib import Path + +from activitysim.core import workflow + +from .multiprocess import ( + _reregister_rng_channels, + _restore_from_subprocess_pipelines, + _run_multiprocess_with_overrides, +) +from .settings import read_calibration_settings + +logger = logging.getLogger("calibration") + + +def _run_in_configured_mode( + state: workflow.State, + models: list[str], + resume_after: str | None, + shared_data_buffers: dict | None = None, +) -> None: + """Run models using the same single/multiprocess mode as the parent run.""" + if not models: + return + + extra_models = _prep_model_data(state, resume_after=resume_after) + if extra_models: + # Models from the step beginning through resume_after must run first + # to recreate the correct intermediate state (since no model-level + # checkpoint existed for resume_after). + models = extra_models + models + + if state.settings.multiprocess: + # Write the restored state as a checkpoint so LAST_CHECKPOINT on disk + # reflects the correct (clean) state for the apportion subprocess. + # When extra_models were prepended, the state is from a PRIOR step + # (not the actual resume_after point) — use a non-conflicting name + # so it won't be mistakenly loaded as the resume_after state on a + # subsequent restart. + if extra_models: + state.checkpoint.add("_calibration_staging") + else: + state.checkpoint.add(resume_after or models[0]) + state.checkpoint.close_store() + + # When subprocess pipelines from a prior run already have the + # resume_after checkpoint (Path 2 in _prep_model_data), subprocesses + # can skip models before resume_after by reusing those pipelines + # instead of freshly apportioning. Signal this by passing + # can_reuse_subprocs=True. + can_reuse = not extra_models and resume_after is not None + + _run_multiprocess_with_overrides( + state, + models=models, + resume_after=resume_after, + shared_data_buffers=shared_data_buffers, + can_reuse_subprocs=can_reuse, + ) + # After multiprocess completes, the coalesced pipeline exists on disk. + # Restore it into the parent process state so tables are accessible + # for calibration expression evaluation. + _restore_parent_state_from_pipeline(state) + # Add a checkpoint named after the last model so that model-name + # references (e.g. _prior_step_name, resume_after on global_iter > 1) + # resolve correctly. Without this, only the step-level coalesce name + # exists in the pipeline. + state.checkpoint.add(models[-1]) + return + + # State is already at the correct point from _prep_model_data above. + # Do NOT call _prep_model_data again — the second call would build its + # table_checkpoint_map from the now-truncated in-memory checkpoint history, + # losing references to tables created after resume_after (e.g. vehicles). + state.checkpoint.add(resume_after or models[0]) + for model in models: + state.run.by_name(model) + # Ensure final model's state is persisted even if should_save_checkpoint + # returned False for it — _calibrate_component needs to restore to it. + if models: + state.checkpoint.add(models[-1]) + + +def _prep_model_data(state, resume_after=None): + """Restore the pipeline to the correct state before running models. + + Resolution priority: + 1. Direct model-level checkpoint in the main pipeline (fastest, exact). + 2. Subprocess pipelines from a prior multiprocess run — performs a + "coalesce at specific checkpoint" to recover the exact intermediate + state without re-running anything. + 3. Previous step checkpoint + re-run models from step begin through + resume_after (slowest but always works). + + Returns + ------- + list[str] + Models that must be prepended to the caller's models list to reach + the correct state at ``resume_after``. Empty when the exact + checkpoint was found and restored directly (paths 1 or 2). + """ + if resume_after: + try: + if state.checkpoint.store_is_open(): + checkpoint_names = [ + cp.get("checkpoint_name", "") for cp in state.checkpoint.checkpoints + ] + else: + from activitysim.core.workflow.checkpoint import HdfStore, ParquetStore + + pipeline_path = Path(state.checkpoint.default_pipeline_file_path()) + if state.settings.checkpoint_format == "hdf": + store = HdfStore(pipeline_path, mode="r") + else: + store = ParquetStore(pipeline_path, mode="r") + try: + checkpoint_names = store.list_checkpoint_names() + finally: + store.close() + + # Path 1: direct model-level checkpoint in main pipeline + if resume_after in checkpoint_names: + _restore_parent_state_from_pipeline(state, checkpoint_name=resume_after) + return [] + + # Path 2: subprocess pipelines (model-level checkpoints preserved) + if _restore_from_subprocess_pipelines(state, resume_after): + return [] + + # Path 3: restore from previous step and re-run + all_models = state.settings.models + mp_steps = state.settings.multiprocess_steps + if mp_steps and resume_after in all_models: + resume_idx = all_models.index(resume_after) + step_boundaries = [all_models.index(s.begin) for s in mp_steps] + step_boundaries.append(len(all_models)) + for i, step in enumerate(mp_steps): + if step_boundaries[i] <= resume_idx < step_boundaries[i + 1]: + if i > 0 and mp_steps[i - 1].name in checkpoint_names: + _restore_parent_state_from_pipeline( + state, checkpoint_name=mp_steps[i - 1].name + ) + step_begin_idx = step_boundaries[i] + extra_models = all_models[step_begin_idx : resume_idx + 1] + return extra_models + elif i == 0: + _restore_parent_state_from_pipeline( + state, checkpoint_name="_" + ) + extra_models = all_models[: resume_idx + 1] + return extra_models + break + except Exception: + logger.warning( + "calibration: could not restore from checkpoint %r, " + "falling back to LAST_CHECKPOINT", + resume_after, + ) + + # Fallback: load LAST_CHECKPOINT (appropriate after a coalesce that + # only ran the desired models) + _restore_parent_state_from_pipeline(state) + return [] + + +def _invalidate_derived_tables(state: workflow.State) -> None: + """Drop factory-produced tables that may be stale after a calibration restore. + + When a calibrated model (e.g. auto_ownership) changes a table that a + @workflow.table factory depends on (e.g. vehicles depends on households), + the checkpoint may contain a stale version of that factory table. Dropping + it forces the factory to regenerate from current data on next access. + + Auto-detection rule: invalidate any table that is (a) registered as a + @workflow.table factory, (b) has DataFrame parameters (= table dependencies), + and (c) is in RANDOM_CHANNELS. This currently matches only 'vehicles' but + will automatically cover future factory tables with the same pattern. + """ + settings = read_calibration_settings(state) + if not settings: + return + + tables_to_invalidate = settings.run.invalidate_tables + if tables_to_invalidate is None: + # Vehicles needs regeneration only when calibration changes + # households.auto_ownership. Downstream calibration components must + # retain vehicle_type_choice's vehicle attributes. + tables_to_invalidate = ( + ["vehicles"] + if "auto_ownership_simulate" in settings.run.calibrate_models + else [] + ) + + logger.debug( + "calibration: tables detected for invalidation: %s", tables_to_invalidate + ) + tables_before = set(state.existing_table_names) + + for table_name in tables_to_invalidate: + if state.is_table(table_name): + state.drop_table(table_name) + state.rng().drop_channel(table_name) + state.get_dataframe(table_name, as_copy=False) + logger.debug("calibration: invalidated derived table '%s'", table_name) + + tables_after = set(state.existing_table_names) + lost = tables_before - tables_after - set(tables_to_invalidate) + if lost: + logger.error( + "calibration: tables unexpectedly removed during invalidation: %s", lost + ) + + +def _restore_parent_state_from_pipeline( + state: workflow.State, checkpoint_name: str = "_" +) -> None: + """Restore pipeline tables into the parent process state. + + After a multiprocess run, the parent's in-memory state is stale. + This loads a specific checkpoint from the pipeline store so that + calibration expressions can evaluate against model outputs. + + Parameters + ---------- + checkpoint_name : str, default "_" + The checkpoint to restore from. Use a model-level checkpoint name + (e.g. the prior step name) to get the exact state at that point, + avoiding pollution from downstream models that may have added rows + to shared tables like ``tours``. The default ``"_"`` loads the + last checkpoint, which is appropriate immediately after a coalesce + that only ran the desired models. + + All tables are explicitly re-checkpointed so that subsequent apportion + subprocesses can load them from a direct file path without relying on + checkpoint backtracking through potentially ambiguous checkpoint history. + """ + # Capture RNG state before restore — models may have dynamically + # added channels (e.g. "vehicles") that aren't in the default + # rng_channels injectable and would be lost by init_state(). + prior_rng_channels = list(state.get_injectable("rng_channels", [])) + prior_index_to_channel = ( + dict(state.rng().index_to_channel) + if hasattr(state.rng(), "index_to_channel") + else {} + ) + + if state.checkpoint.store_is_open(): + state.checkpoint.close_store() + state.checkpoint.restore(resume_after=checkpoint_name) + + _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) + + # After restore, all tables are clean (status=False). Mark them dirty so + # the next checkpoint.add() writes them to disk at a known checkpoint name. + # This ensures apportion subprocesses find table files at a single, + # unambiguous checkpoint rather than needing to backtrack through history. + for table_name in list(state.existing_table_names): + state.existing_table_status[table_name] = True diff --git a/activitysim/core/calibration/expressions.py b/activitysim/core/calibration/expressions.py new file mode 100644 index 0000000000..9338b7b39c --- /dev/null +++ b/activitysim/core/calibration/expressions.py @@ -0,0 +1,199 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +import importlib +import importlib.util +import logging +import math +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +from activitysim.core import workflow + +from .reporting import _component_output_dir +from .settings import CalibrationComponentSettings + +logger = logging.getLogger("calibration") + + +def _build_expression_context( + state: workflow.State, + helper_symbols: dict[str, Any], + component_name: str, + component_settings: CalibrationComponentSettings, +) -> dict[str, Any]: + """Create the evaluation context for model_value and target_value expressions.""" + context: dict[str, Any] = { + "state": state, + "np": np, + "pd": pd, + "component_output_dir": _component_output_dir(state, component_name), + "component_settings": component_settings, + } + + # Load active tables into context for direct expression access. + for table_name in list(state.existing_table_names): + try: + context[table_name] = state.get_dataframe(table_name, as_copy=False) + except Exception: + # Some entries may not be available as dataframes in all contexts. + continue + try: + network_los = state.get_injectable("network_los") + context["network_los"] = network_los + context["skim_dict"] = network_los.get_default_skim_dict() + except Exception: + # Network LOS may not be available in all contexts. + pass + + context.update(helper_symbols) + # Explicit function-call context used by calibration expressions. + context["context"] = context + return context + + +def _eval_numeric_value( + raw_value: Any, + context: dict[str, Any], + component_name: str, + description: str, + field_name: str, +) -> float: + """Evaluate numeric or expression value and enforce finite numeric result.""" + if isinstance(raw_value, (int, float, np.number)) and not pd.isna(raw_value): + value = float(raw_value) + else: + try: + value = eval(str(raw_value), {}, context) + except Exception as err: + raise RuntimeError( + f"error evaluating {field_name} for {component_name} / {description}: {raw_value}" + ) from err + + try: + value = float(value) + except Exception as err: + raise RuntimeError( + f"{field_name} did not evaluate to a numeric value for {component_name} / {description}: {value}" + ) from err + + if not np.isfinite(value): + raise RuntimeError( + f"{field_name} evaluated to non-finite value for {component_name} / {description}: {value}" + ) + + return value + + +def _compute_delta( + method: str, + model_value: float, + target_value: float, + damping: float, + component_name: str, + description: str, + default_increment: float, +) -> float: + """Compute damped coefficient delta using selected method.""" + if damping < 0: + raise RuntimeError( + f"negative damping not allowed for {component_name} / {description}: {damping}" + ) + + if method == "log_ratio": + if model_value <= 0 or target_value <= 0: + logger.warning( + f"log_ratio requires positive model and target values for {component_name} / {description}. Falling back to default increment {default_increment}" + ) + if model_value <= 0 and target_value > 0: + return default_increment + elif model_value > 0 and target_value <= 0: + return -default_increment + else: + return 0 + delta = math.log(target_value / model_value) * damping + + elif method == "odds_ratio": + # Formula requested by the calibration outline. + numerator = (target_value * model_value) - target_value + denominator = (target_value * model_value) - model_value + + if numerator <= 0 or denominator <= 0: + logger.warning( + f"odds_ratio produced invalid numerator/denominator for {component_name} / {description}. Falling back to default increment {default_increment}" + ) + if model_value <= 0 and target_value > 0: + return default_increment + elif model_value > 0 and target_value <= 0: + return -default_increment + else: + return 0 + + ratio = numerator / denominator + if ratio <= 0 or not np.isfinite(ratio): + raise RuntimeError( + f"odds_ratio produced invalid ratio for {component_name} / {description}" + ) + + delta = math.log(ratio) * damping + + else: + raise RuntimeError(f"unsupported calibration method: {method}") + + if not np.isfinite(delta): + raise RuntimeError( + f"coefficient delta is non-finite for {component_name} / {description}" + ) + + return delta + + +def _load_helper_symbols( + state: workflow.State, + component_settings: CalibrationComponentSettings, +) -> tuple[dict[str, Any], Any | None, Any | None]: + """Load helper module and return evaluation symbols and bespoke function.""" + if not component_settings.helper_module: + return {}, None, None + + module = _load_helper_module(state, component_settings.helper_module) + symbols = { + name: obj for name, obj in vars(module).items() if not name.startswith("__") + } + + bespoke = None + if component_settings.reports and component_settings.reports.bespoke: + fn_name = component_settings.reports.bespoke + if not hasattr(module, fn_name): + raise RuntimeError( + f"helper module does not define bespoke function {fn_name}" + ) + bespoke = getattr(module, fn_name) + + return symbols, bespoke, module + + +def _load_helper_module(state: workflow.State, helper_module: str): + """Load helper module by file path or import path.""" + if helper_module.endswith(".py"): + helper_path = state.filesystem.get_config_file_path(helper_module) + module_name = Path(helper_module).stem + + spec = importlib.util.spec_from_file_location(module_name, helper_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"unable to load helper module from {helper_path}") + + module = importlib.util.module_from_spec(spec) + # Expose state as a global for compatibility with helper examples. + module.state = state + spec.loader.exec_module(module) + return module + + module = importlib.import_module(helper_module) + setattr(module, "state", state) + return module + diff --git a/activitysim/core/calibration/multiprocess.py b/activitysim/core/calibration/multiprocess.py new file mode 100644 index 0000000000..b828005007 --- /dev/null +++ b/activitysim/core/calibration/multiprocess.py @@ -0,0 +1,604 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +import logging +import multiprocessing +from pathlib import Path +from typing import Any + +import pandas as pd + +from activitysim.core import workflow +from activitysim.core.configuration.top import MultiprocessStep + +logger = logging.getLogger("calibration") + +MP_INJECTABLES = [ + "data_dir", + "configs_dir", + "data_model_dir", + "output_dir", + "cache_dir", + "settings_file_name", + "imported_extensions", + "run_timestamp", + "run_id", + "pipeline_file_name", +] + + +def _run_mp_single_component( + state: workflow.State, + component_name: str, + restore_checkpoint: str, + shared_data_buffers: dict, +) -> None: + """Run a single component in multiprocess mode with explicit checkpoint control. + + This directly orchestrates the apportion → simulate → coalesce flow + without going through run_multiprocess/get_run_list, giving us precise + control over which checkpoint to apportion from. This is essential for + calibration component re-runs where we must always restart from prior_step's + state regardless of what other checkpoints exist in the pipeline. + + Parameters + ---------- + state : workflow.State + component_name : str + The model component to run. + restore_checkpoint : str + The checkpoint name to restore from before apportioning. + This should be the checkpoint representing prior_step's state. + shared_data_buffers : dict + Pre-allocated shared memory buffers for skims/shadow pricing. + """ + from activitysim.core import mp_tasks + + from .execution import _restore_parent_state_from_pipeline + + # Determine slice info from original settings + original_steps = state.settings.multiprocess_steps + all_models = state.settings.models + + slice_info = None + num_processes = state.settings.num_processes or 2 + chunk_size = state.settings.chunk_size or 0 + + # Find which original step this component belongs to + step_boundaries = [] + for i, step in enumerate(original_steps): + step_boundaries.append(all_models.index(step.begin)) + step_boundaries.append(len(all_models)) + + component_idx = all_models.index(component_name) + for i, step in enumerate(original_steps): + if step_boundaries[i] <= component_idx < step_boundaries[i + 1]: + if step.slice: + slice_info = step.slice.model_dump() + if step.num_processes: + num_processes = step.num_processes + if step.chunk_size: + chunk_size = step.chunk_size + break + + # Build step_info dict matching what mp_tasks functions expect + step_info = { + "name": component_name, + "models": [component_name], + "num_processes": num_processes, + "chunk_size": chunk_size, + "step_num": 0, + "slice": slice_info, + "last_checkpoint_in_previous_multiprocess_step": restore_checkpoint, + } + + injectables = _build_calibration_injectables(state) + + if num_processes == 1: + sub_proc_names = [component_name] + else: + sub_proc_names = [f"{component_name}_{i}" for i in range(num_processes)] + + fail_fast = state.settings.fail_fast + + # Apportion pipeline (split tables across sub-processes) + if num_processes > 1 and slice_info is not None: + mp_tasks.run_sub_task( + state, + multiprocessing.Process( + target=mp_tasks.mp_apportion_pipeline, + name=f"{component_name}_apportion", + args=(injectables, sub_proc_names, step_info), + ), + ) + + # For multi-process runs, subprocesses must restore from the apportioned + # pipeline (which has one checkpoint). Use LAST_CHECKPOINT so they don't + # overwrite the apportioned data with a fresh pipeline. + # For single-process runs (no apportion), use restore_checkpoint to resume + # from the correct point in the main pipeline. + if num_processes > 1: + sim_resume_after = "_" # LAST_CHECKPOINT in apportioned sub-pipeline + else: + sim_resume_after = restore_checkpoint + + # Run simulations in sub-processes + completed = mp_tasks.run_sub_simulations( + state, + injectables, + shared_data_buffers, + step_info, + sub_proc_names, + sim_resume_after, + [], # previously_completed + fail_fast, + ) + + if len(completed) != num_processes: + from activitysim.core.exceptions import SubprocessError + + raise SubprocessError( + f"{num_processes - len(completed)} processes failed in " + f"calibration step {component_name}" + ) + + # Coalesce sub-process pipelines back into main pipeline + if num_processes > 1 and slice_info is not None: + mp_tasks.run_sub_task( + state, + multiprocessing.Process( + target=mp_tasks.mp_coalesce_pipelines, + name=f"{component_name}_coalesce", + args=(injectables, sub_proc_names, slice_info), + ), + ) + + # Restore coalesced results into parent state + _restore_parent_state_from_pipeline(state) + + +def _restore_from_subprocess_pipelines( + state: workflow.State, resume_after: str +) -> bool: + """Restore state from subprocess pipelines at a specific model checkpoint. + + Subprocess pipelines retain model-level checkpoints that don't exist in + the main pipeline. This performs a "coalesce at checkpoint" — reading + mirrored tables from one subprocess and concatenating sliced tables from + all subprocesses at the specified checkpoint name. + + Parameters + ---------- + state : workflow.State + resume_after : str + Model-level checkpoint name to restore from. + + Returns + ------- + bool + True if the restore succeeded; False if subprocess pipelines don't + exist or don't contain the requested checkpoint. + """ + from activitysim.core.workflow.checkpoint import ( + CHECKPOINT_NAME, + CHECKPOINT_TABLE_NAME, + HdfStore, + NON_TABLE_COLUMNS, + ParquetStore, + ) + + all_models = state.settings.models + mp_steps = state.settings.multiprocess_steps + if not mp_steps or resume_after not in all_models: + return False + + # Find the multiprocess step containing resume_after + resume_idx = all_models.index(resume_after) + step_boundaries = [all_models.index(s.begin) for s in mp_steps] + step_boundaries.append(len(all_models)) + + enclosing_step = None + num_processes = state.settings.num_processes or 2 + slice_info = None + for i, step in enumerate(mp_steps): + if step_boundaries[i] <= resume_idx < step_boundaries[i + 1]: + enclosing_step = step + if step.num_processes: + num_processes = step.num_processes + if step.slice: + slice_info = ( + step.slice.model_dump() + if hasattr(step.slice, "model_dump") + else step.slice + ) + break + + if enclosing_step is None or num_processes <= 1: + return False + + # Build subprocess pipeline file paths + step_name = enclosing_step.name + pipeline_file_name = state.filesystem.pipeline_file_name + sub_proc_names = [f"{step_name}_{i}" for i in range(num_processes)] + + def _subprocess_path(proc_name): + base = state.get_output_file_path(pipeline_file_name, prefix=proc_name) + if state.settings.checkpoint_format == "hdf": + return base + pq = Path(str(base)).with_suffix(ParquetStore.extension) + return pq if pq.exists() else base + + first_path = _subprocess_path(sub_proc_names[0]) + if not first_path.exists(): + return False + + # Open first subprocess pipeline and verify checkpoint exists + if state.settings.checkpoint_format == "hdf": + first_store = HdfStore(first_path, mode="r") + else: + first_store = ParquetStore(first_path, mode="r") + + try: + cp_names = first_store.list_checkpoint_names() + if resume_after not in cp_names: + return False + + # Read checkpoint row to get table→checkpoint mapping + cp_df = first_store.get_dataframe(CHECKPOINT_TABLE_NAME) + cp_row = cp_df[cp_df[CHECKPOINT_NAME] == resume_after].iloc[-1] + + table_map = {} + for col in cp_row.index: + if col not in NON_TABLE_COLUMNS and cp_row[col]: + table_map[col] = cp_row[col] + + # Read all tables from first subprocess at this checkpoint + tables = {} + for table_name, cp_for_table in table_map.items(): + try: + tables[table_name] = first_store.get_dataframe(table_name, cp_for_table) + except (FileNotFoundError, KeyError): + logger.warning( + f"calibration: subprocess pipeline missing table " + f"{table_name} at {cp_for_table}" + ) + finally: + first_store.close() + + if not tables: + return False + + # Determine sliced tables that need concatenation across processes + sliced_table_names = set(slice_info.get("tables", [])) if slice_info else set() + + # Read sliced tables from remaining subprocesses and concatenate + if num_processes > 1 and sliced_table_names: + omnibus = {t: [tables[t]] for t in sliced_table_names if t in tables} + + for proc_name in sub_proc_names[1:]: + proc_path = _subprocess_path(proc_name) + if not proc_path.exists(): + logger.warning( + f"calibration: subprocess pipeline not found: {proc_path}" + ) + return False + + if state.settings.checkpoint_format == "hdf": + proc_store = HdfStore(proc_path, mode="r") + else: + proc_store = ParquetStore(proc_path, mode="r") + + try: + proc_cp_df = proc_store.get_dataframe(CHECKPOINT_TABLE_NAME) + proc_row = proc_cp_df[proc_cp_df[CHECKPOINT_NAME] == resume_after].iloc[ + -1 + ] + + for table_name in list(omnibus.keys()): + cp_for_table = proc_row.get(table_name, "") + if cp_for_table: + omnibus[table_name].append( + proc_store.get_dataframe(table_name, cp_for_table) + ) + finally: + proc_store.close() + + # Replace sliced tables with concatenated versions + for table_name, dfs in omnibus.items(): + tables[table_name] = pd.concat(dfs, sort=False) + + # Load into parent state + prior_rng_channels = list(state.get_injectable("rng_channels", [])) + prior_index_to_channel = ( + dict(state.rng().index_to_channel) + if hasattr(state.rng(), "index_to_channel") + else {} + ) + + state.init_state() + if state.checkpoint.store_is_open(): + state.checkpoint.close_store() + state.checkpoint.open_store(overwrite=False) + + for table_name, df in tables.items(): + state.add_table(table_name, df) + + _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) + + # Mark all tables dirty for subsequent checkpoint.add + for table_name in list(state.existing_table_names): + state.existing_table_status[table_name] = True + + logger.info( + "calibration: restored %d tables from subprocess pipelines at " + "checkpoint '%s'", + len(tables), + resume_after, + ) + return True + + +def _run_multiprocess_with_overrides( + state: workflow.State, + models: list[str], + resume_after: str | None, + shared_data_buffers: dict | None = None, + can_reuse_subprocs: bool = False, +) -> None: + """Run multiprocess with temporary settings overrides for calibration passes. + + Parameters + ---------- + can_reuse_subprocs : bool, default False + When True, subprocess pipelines from a prior run are assumed to exist + and contain the ``resume_after`` checkpoint. Breadcrumbs are written + so that ``get_run_list`` populates ``step_info["resume_after"]``, + apportion is skipped (reusing existing subprocess pipelines), and + subprocesses resume from their model-level checkpoint — skipping + already-completed models. + """ + from collections import OrderedDict + + from activitysim.core import mp_tasks + + original_models = state.settings.models + original_mp_steps = state.settings.multiprocess_steps + original_resume_after = state.settings.resume_after + + # Build valid multiprocess_steps for the requested model subset. + calibration_mp_steps = _build_calibration_mp_steps( + models=models, + original_steps=original_mp_steps, + all_models=original_models, + ) + + state.settings.models = models + state.settings.multiprocess_steps = calibration_mp_steps + + if can_reuse_subprocs and resume_after: + # Include resume_after in the models list so get_breadcrumbs can + # locate the step containing it. Subprocesses will skip this model + # (it's already checkpointed in their pipeline) and run the rest. + models = [resume_after] + models + + # Rebuild steps with resume_after included. + calibration_mp_steps = _build_calibration_mp_steps( + models=models, + original_steps=original_mp_steps, + all_models=original_models, + ) + state.settings.models = models + state.settings.multiprocess_steps = calibration_mp_steps + state.settings.resume_after = resume_after + + # Write minimal breadcrumbs indicating the step containing + # resume_after has completed apportion (so it's skipped) but + # simulate/coalesce need re-running. + breadcrumbs = OrderedDict() + for step in calibration_mp_steps: + step_dict = {"name": step.name, "apportion": True} + breadcrumbs[step.name] = step_dict + # Find the step containing resume_after + all_models = state.settings.models + if resume_after in all_models: + step_begin = all_models.index(step.begin) + step_models_in_step = [ + m for m in all_models[step_begin:] if m in models + ] + if resume_after in step_models_in_step: + # This step contains resume_after — stop here. + # get_breadcrumbs will mark simulate/coalesce for re-run. + break + + mp_tasks.write_breadcrumbs(state, breadcrumbs) + else: + # No reuse: calibration manages pipeline state externally via + # _restore_parent_state_from_pipeline and checkpoint.add, so the MP + # system's breadcrumb-based resume logic must not be triggered. + state.settings.resume_after = None + + try: + injectables = _build_calibration_injectables(state) + mp_tasks.run_multiprocess( + state, + injectables, + shared_data_buffers=shared_data_buffers, + skip_final_checkpoint=True, + force_resume=resume_after is not None and not can_reuse_subprocs, + ) + finally: + state.settings.models = original_models + state.settings.resume_after = original_resume_after + state.settings.multiprocess_steps = original_mp_steps + + +def _reregister_rng_channels( + state: workflow.State, + prior_channels: list[str], + prior_index_to_channel: dict[str, str] = None, +) -> None: + """Re-register RNG channels that were lost during init_state().""" + current_channels = set(state.get_injectable("rng_channels", [])) + for channel_name in prior_channels: + if channel_name not in current_channels and state.is_table(channel_name): + try: + state.rng().add_channel(channel_name, state.get_dataframe(channel_name)) + current_channels.add(channel_name) + except Exception: + pass + # For channels whose tables don't exist at the restored checkpoint, + # register an empty channel. Do NOT pre-load from a later checkpoint + # in the store — that data may include modifications from downstream + # models and would pollute the pre-model state. The empty channel + # allows the model's normal table factory to create the table fresh + # and extend the channel without hitting the disjoint-index assertion. + if prior_index_to_channel: + for index_name, channel_name in prior_index_to_channel.items(): + if index_name not in state.rng().index_to_channel: + if channel_name not in state.rng().channels: + empty_df = pd.DataFrame( + index=pd.Index([], dtype="int64", name=index_name) + ) + state.rng().add_channel(channel_name, empty_df) + else: + state.rng().index_to_channel[index_name] = channel_name + current_channels.add(channel_name) + state.add_injectable("rng_channels", list(current_channels)) + + +def _initialize_mp_shared_resources(state: workflow.State) -> dict: + """Allocate shared data buffers (skims, shadow pricing) once for reuse. + + This mirrors the allocation logic in mp_tasks.run_multiprocess but + is called once at calibration start rather than on every sub-run. + """ + from activitysim.core import mp_tasks, tracing + + shared_data_buffers = {} + sharrow_enabled = state.settings.sharrow + + t0 = tracing.print_elapsed_time() + if not sharrow_enabled: + shared_data_buffers.update(mp_tasks.allocate_shared_skim_buffers(state)) + t0 = tracing.print_elapsed_time("calibration: allocate shared skim buffer", t0) + + shared_data_buffers.update(mp_tasks.allocate_shared_shadow_pricing_buffers(state)) + t0 = tracing.print_elapsed_time( + "calibration: allocate shared shadow_pricing buffer", t0 + ) + + shared_data_buffers.update( + mp_tasks.allocate_shared_shadow_pricing_buffers_choice(state) + ) + t0 = tracing.print_elapsed_time( + "calibration: allocate shared shadow_pricing choice buffer", t0 + ) + + # Load skim data into the shared buffers. + if sharrow_enabled: + shared_data_buffers["skim_dataset"] = "sh.Dataset:skim_dataset" + from activitysim.core import flow, skim_dataset # noqa: F401 + + state.get_injectable("skim_dataset") + else: + if len(shared_data_buffers) > 0: + injectables = _build_calibration_injectables(state) + mp_tasks.run_sub_task( + state, + multiprocessing.Process( + target=mp_tasks.mp_setup_skims, + name="mp_setup_skims_calibration", + args=(injectables,), + kwargs=shared_data_buffers, + ), + ) + + # Make skims available in the parent process for expression evaluation. + state.add_injectable("data_buffers", shared_data_buffers) + try: + state.get_injectable("network_los") + except Exception: + logger.warning( + "calibration: could not resolve network_los in parent process; " + "skim-dependent expressions may fail" + ) + + return shared_data_buffers + + +def _build_calibration_mp_steps( + models: list[str], + original_steps: list[MultiprocessStep], + all_models: list[str], +) -> list[MultiprocessStep]: + """Build valid MultiprocessStep objects for a calibration model subset. + + The key challenge is that get_run_list() in mp_tasks requires: + - The first step's begin == models[0] + - Steps are ordered and non-overlapping + - Each step's begin is in the models list + + We intersect the original multiprocess_steps with the requested model + subset and construct new steps that satisfy these constraints. + """ + if not models: + return [] + + # Determine which original step each model in the full list belongs to. + # Build a mapping: model_name -> original step index + model_to_step: dict[str, int] = {} + step_boundaries = [] + for i, step in enumerate(original_steps): + begin_idx = all_models.index(step.begin) + step_boundaries.append(begin_idx) + step_boundaries.append(len(all_models)) + + for i, step in enumerate(original_steps): + for model_idx in range(step_boundaries[i], step_boundaries[i + 1]): + model_to_step[all_models[model_idx]] = i + + # Group the requested models by their original step + from collections import OrderedDict + + step_model_groups: OrderedDict[int, list[str]] = OrderedDict() + for model in models: + step_idx = model_to_step.get(model) + if step_idx is None: + continue + step_model_groups.setdefault(step_idx, []).append(model) + + # Build new MultiprocessStep for each group. + # Some original steps (e.g. mp_initialize) omit num_processes, slice, + # and chunk_size — these default to None on MultiprocessStep and + # get_run_list() applies global defaults when they are absent. + # Step names include the first model to ensure uniqueness across multiple + # intermediate runs that draw from the same original step. + new_steps = [] + for step_idx, step_models in step_model_groups.items(): + orig_step = original_steps[step_idx] + kwargs: dict[str, Any] = { + "name": orig_step.name, + "begin": step_models[0], + } + if orig_step.num_processes is not None: + kwargs["num_processes"] = orig_step.num_processes + if orig_step.slice is not None: + kwargs["slice"] = orig_step.slice + if orig_step.chunk_size is not None: + kwargs["chunk_size"] = orig_step.chunk_size + new_steps.append(MultiprocessStep(**kwargs)) + + return new_steps + + +def _build_calibration_injectables(state: workflow.State) -> dict: + """Build the injectables dict for multiprocess sub-processes.""" + injectables = {} + for key in MP_INJECTABLES: + try: + injectables[key] = state.get_injectable(key) + except KeyError: + pass + injectables["settings"] = state.settings + return injectables diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py new file mode 100644 index 0000000000..a4973f9625 --- /dev/null +++ b/activitysim/core/calibration/orchestrator.py @@ -0,0 +1,455 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +import logging + +from activitysim.core import workflow + +from .component import _calibrate_component +from .execution import ( + _invalidate_derived_tables, + _prep_model_data, + _run_in_configured_mode, +) +from .multiprocess import _initialize_mp_shared_resources +from .recovery import ( + CALIBRATION_PROGRESS_FILE, + _begin_global_iteration_transaction, + _read_progress, + _restore_coefficient_backups, + _write_completed_progress, + _write_progress, +) +from .reporting import ( + _ensure_calibration_output_dir, + _write_component_plots, + _write_final_coefficients_snapshot, +) +from .settings import ( + CalibrationRunResult, + read_calibration_settings, +) + +logger = logging.getLogger("calibration") + + +def run_calibration_loop( + state: workflow.State, + models: list[str], +) -> CalibrationRunResult: + """ + Run the global calibration workflow. + + This function intentionally minimizes changes to the existing run mechanics: + it always reuses ActivitySim's normal model execution paths and only adds + calibration orchestration around them. + """ + calibration_settings = read_calibration_settings(state) + if not calibration_settings or not calibration_settings.enable: + raise RuntimeError("calibration loop called while calibration is disabled") + + if state.settings.duplicate_step_execution != "allow": + state.settings.duplicate_step_execution = "allow" + logger.warning( + "Overriding duplicate_step_execution setting: must be enabled for calibration" + ) + + if not calibration_settings.run.calibrate_models: + raise ValueError( + "calibration.run.calibrate_models must contain at least one model name" + ) + + missing_calibration_models = [ + component + for component in calibration_settings.run.calibrate_models + if component not in models + ] + if missing_calibration_models: + raise ValueError( + "settings.yaml models list does not include configured calibration " + f"model(s): {missing_calibration_models}" + ) + + resume_after = state.settings.resume_after + if resume_after is not None and resume_after not in models: + raise ValueError( + f"settings.yaml resume_after={resume_after!r} is not present in the " + "settings.yaml models list. Calibration requires resume_after to be " + "a model-level checkpoint name." + ) + + # sort calibration models into main model order + calibration_settings.run.calibrate_models = sorted( + calibration_settings.run.calibrate_models, key=lambda x: models.index(x) + ) + first_calib_model_idx = models.index(calibration_settings.run.calibrate_models[0]) + last_calib_model_idx = models.index(calibration_settings.run.calibrate_models[-1]) + first_model_idx = models.index(resume_after) + 1 if resume_after else None + first_calibration_restart_step = _prior_step_name( + models, calibration_settings.run.calibrate_models[0] + ) + + if resume_after is not None: + skipped_calibration_models = [ + component + for component in calibration_settings.run.calibrate_models + if models.index(component) <= models.index(resume_after) + ] + if skipped_calibration_models: + logger.warning( + "Calibration is honoring settings.yaml resume_after=%r using strict " + "ActivitySim semantics. The following calibrated model(s) occur at " + "or before resume_after and will be skipped during the first global " + "iteration: %s", + resume_after, + skipped_calibration_models, + ) + + _ensure_calibration_output_dir(state) + + progress = _read_progress(state) + if progress and progress.get("complete"): + logger.info( + "calibration progress is already complete; remove %s to start a " + "fresh calibration run", + CALIBRATION_PROGRESS_FILE, + ) + return CalibrationRunResult( + converged=bool(progress.get("converged", False)), + completed_global_iterations=int( + progress.get( + "last_completed_global_iteration", + calibration_settings.run.global_iterations, + ) + ), + ) + + interrupted_iteration = ( + progress.get("in_progress_iteration") if progress else None + ) + if interrupted_iteration is not None: + interrupted_iteration = int(interrupted_iteration) + logger.warning( + "recovering interrupted calibration global iteration %s", + interrupted_iteration, + ) + _restore_coefficient_backups(state, calibration_settings) + progress = { + "in_progress_iteration": None, + "next_global_iteration": interrupted_iteration, + "last_completed_global_iteration": interrupted_iteration - 1, + } + _write_progress(state, progress) + + # Progress files from earlier versions contain next_global_iteration, so + # they remain compatible with the corrected total-count semantics. + start_global_iter = int(progress.get("next_global_iteration", 1)) if progress else 1 + completed_global_iterations = start_global_iter - 1 + + if start_global_iter > calibration_settings.run.global_iterations: + logger.info( + "calibration progress already reached configured global_iterations=%s", + calibration_settings.run.global_iterations, + ) + converged = bool(progress.get("converged", False)) if progress else False + _write_final_coefficients_snapshot(state, calibration_settings) + _write_completed_progress( + state, + completed_global_iterations, + converged, + ) + return CalibrationRunResult( + converged=converged, + completed_global_iterations=completed_global_iterations, + ) + + if state.settings.resume_after is None: + # compute_accessibility requires its accessibility table to be empty; + # unlike most model steps, it will not overwrite a prior result. + # Remove a cached result before restore clears table-status metadata, + # so the table factory recreates its empty placeholder for the replay. + state.drop_table("accessibility") + state.checkpoint.restore() + + original_pipeline_name = state.filesystem.pipeline_file_name + + # Initialize shared resources for multiprocess mode (skims, shadow pricing). + # These are allocated once and reused across all calibration iterations. + shared_data_buffers = None + if state.settings.multiprocess: + shared_data_buffers = _initialize_mp_shared_resources(state) + + try: + # skip precursors if, on first iter, resume_after exists and is >= first_calib_model_idx + if ( + state.settings.resume_after is None + or first_model_idx < first_calib_model_idx + ): + # Run ActivitySim normally from resume_after through production model steps. + _run_precursor_components( + state, + models=models[:first_calib_model_idx] + if first_model_idx is None + else models[first_model_idx:first_calib_model_idx], + resume_after=state.settings.resume_after, + global_iter=start_global_iter, + shared_data_buffers=shared_data_buffers, + ) + else: + # Precursors skipped — but the pipeline must still be initialized + # at the resume_after point so that _calibrate_component (and its + # apportion subprocess) starts from the correct state without + # downstream model data. + extra_models = _prep_model_data( + state, resume_after=state.settings.resume_after + ) + if extra_models: + # No model-level checkpoint exists for resume_after; we must + # run models from the prior step through resume_after to + # recreate the correct intermediate state. + _run_in_configured_mode( + state, + models=extra_models, + resume_after=None, + shared_data_buffers=shared_data_buffers, + ) + elif not any( + cp.get("checkpoint_name") == state.settings.resume_after + for cp in state.checkpoint.checkpoints + ): + # _prep_model_data took its fallback path — the pipeline either + # doesn't exist or doesn't contain resume_after's checkpoint. + # The restored state is incomplete (precursor models never ran). + logger.warning( + "calibration: resume_after=%r not found in restored pipeline; " + "running precursor models", + state.settings.resume_after, + ) + _run_precursor_components( + state, + models=models[:first_calib_model_idx], + resume_after=None, + global_iter=start_global_iter, + shared_data_buffers=shared_data_buffers, + ) + else: + state.checkpoint.add(state.settings.resume_after) + state.checkpoint.close_store() + + for global_iter in range( + start_global_iter, + calibration_settings.run.global_iterations + 1, + ): + _begin_global_iteration_transaction( + state, + calibration_settings, + global_iter, + ) + + # Every global iteration after the first begins from the immutable + # checkpoint directly before the first calibrated model. This makes + # global reruns independent of the state left by the final calibrated + # model in the preceding iteration. + if global_iter > start_global_iter: + logger.info( + "Restarting global calibration iteration %s from checkpoint %r", + global_iter, + first_calibration_restart_step, + ) + extra_models = _prep_model_data( + state, resume_after=first_calibration_restart_step + ) + if extra_models: + _run_in_configured_mode( + state, + models=extra_models, + resume_after=None, + shared_data_buffers=shared_data_buffers, + ) + _invalidate_derived_tables(state) + if first_calibration_restart_step is not None: + state.checkpoint.add(first_calibration_restart_step) + state.checkpoint.close_store() + + logger.info( + "calibration global iteration %s/%s", + global_iter, + calibration_settings.run.global_iterations, + ) + + # suppress early termination on first iteration if resume_after is after all calibrated models + all_converged = ( + first_model_idx is not None and first_model_idx <= last_calib_model_idx + ) or global_iter > start_global_iter + + last_calibrated_component = None + for component in calibration_settings.run.calibrate_models: + # on the first global iter, skip model if it's before or == resume_after + if ( + global_iter == start_global_iter + and first_model_idx is not None + and first_model_idx > models.index(component) + ): + continue + component_settings = calibration_settings.model_settings[component] + + prior_step = _prior_step_name(models, component) + + if last_calibrated_component is not None: + + # run all models b/w the last calibrated model and the current one + _run_intermediate_components( + state, + models=models[ + models.index(last_calibrated_component) + + 1 : models.index(component) + ], + resume_after=last_calibrated_component, + shared_data_buffers=shared_data_buffers, + ) + + component_result = _calibrate_component( + state=state, + component_name=component, + component_settings=component_settings, + prior_step=prior_step, + global_iter=global_iter, + shared_data_buffers=shared_data_buffers, + ) + _write_component_plots(state, component) + + all_converged = all_converged and component_result.converged + + last_calibrated_component = component + + if ( + calibration_settings.run.complete_steps + or global_iter == calibration_settings.run.global_iterations + or ( + global_iter == start_global_iter + and state.settings.resume_after is not None + and first_model_idx > last_calib_model_idx + ) + ): + subsequent_components = ( + models[first_model_idx:] + if global_iter == start_global_iter + and first_model_idx > last_calib_model_idx + else models[models.index(last_calibrated_component) + 1 :] + ) + # finish the full model chain + _run_subsequent_components( + state, + models=subsequent_components, + resume_after=state.settings.resume_after + if global_iter == start_global_iter + and first_model_idx > last_calib_model_idx + else last_calibrated_component, + shared_data_buffers=shared_data_buffers, + ) + + completed_global_iterations = global_iter + iteration_is_complete = ( + all_converged + or global_iter == calibration_settings.run.global_iterations + ) + if not iteration_is_complete: + _write_progress( + state, + { + "in_progress_iteration": None, + "next_global_iteration": global_iter + 1, + "last_completed_global_iteration": global_iter, + "converged": all_converged, + }, + ) + + if all_converged: + logger.info( + "calibration converged after global iteration %s/%s", + global_iter, + calibration_settings.run.global_iterations, + ) + break + + _write_final_coefficients_snapshot(state, calibration_settings) + _write_completed_progress( + state, + completed_global_iterations, + all_converged, + ) + + return CalibrationRunResult( + converged=all_converged, + completed_global_iterations=completed_global_iterations, + ) + finally: + state.filesystem.pipeline_file_name = original_pipeline_name + + +def _run_precursor_components( + state: workflow.State, + models: list[str], + resume_after: str, + global_iter: int, + shared_data_buffers: dict | None = None, +) -> None: + """Run the normal ActivitySim model flow for one global calibration iteration.""" + + # if global_iter > 1 and resume_after is not None: + # # Seed a fresh pipeline from the configured resume checkpoint to avoid + # # duplicate checkpoint-name collisions across global calibration loops. + # prior_pipeline = state.checkpoint.store.filename + # state.checkpoint.close_store() + # state.filesystem.pipeline_file_name = f"pipeline_calibration_iter_{global_iter}" + # state.checkpoint.restore_from(prior_pipeline, checkpoint_name=resume_after) + # else: + + _run_in_configured_mode( + state, + models=models, + resume_after=resume_after, + shared_data_buffers=shared_data_buffers, + ) + + +def _run_intermediate_components( + state: workflow.State, + models: list[str], + resume_after: str, + shared_data_buffers: dict | None = None, +) -> None: + if len(models) == 0: + return + _run_in_configured_mode( + state, + models=models, + resume_after=resume_after, + shared_data_buffers=shared_data_buffers, + ) + + +def _run_subsequent_components( + state: workflow.State, + models: list[str], + resume_after: str, + shared_data_buffers: dict | None = None, +) -> None: + _run_in_configured_mode( + state, + models=models, + resume_after=resume_after, + shared_data_buffers=shared_data_buffers, + ) + + +def _prior_step_name(models: list[str], component_name: str) -> str | None: + """Return the step name immediately preceding component_name in models.""" + if component_name not in models: + return None + idx = models.index(component_name) + if idx == 0: + return None + return models[idx - 1] + diff --git a/activitysim/core/calibration/recovery.py b/activitysim/core/calibration/recovery.py new file mode 100644 index 0000000000..b46c779ba8 --- /dev/null +++ b/activitysim/core/calibration/recovery.py @@ -0,0 +1,103 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +import json +import os +import shutil +from typing import Any + +from activitysim.core import workflow + +from .coefficients import _calibration_coefficient_paths +from .settings import CalibrationConfig + +CALIBRATION_PROGRESS_FILE = "calibration/calibration_progress.json" +CALIBRATION_RECOVERY_DIR = "calibration/recovery" + + +def _begin_global_iteration_transaction( + state: workflow.State, + calibration_settings: CalibrationConfig, + global_iteration: int, +) -> None: + """Snapshot coefficients and durably mark a global iteration in progress.""" + recovery_dir = state.get_output_file_path(CALIBRATION_RECOVERY_DIR) + os.makedirs(recovery_dir, exist_ok=True) + + for file_number, coefficient_path in enumerate( + _calibration_coefficient_paths(state, calibration_settings) + ): + if not coefficient_path.exists(): + raise FileNotFoundError( + f"calibration coefficient file not found: {coefficient_path}" + ) + + backup_name = f"{file_number:03d}_{coefficient_path.name}" + shutil.copyfile(coefficient_path, recovery_dir / backup_name) + + # Write the marker only after all backups exist. If backup creation is + # interrupted, the previous between-iteration progress remains valid. + _write_progress( + state, + { + "in_progress_iteration": global_iteration, + "next_global_iteration": global_iteration, + "last_completed_global_iteration": global_iteration - 1, + }, + ) + + +def _restore_coefficient_backups( + state: workflow.State, + calibration_settings: CalibrationConfig, +) -> None: + """Restore the coefficient backups for an interrupted global iteration.""" + recovery_dir = state.get_output_file_path(CALIBRATION_RECOVERY_DIR) + for file_number, coefficient_path in enumerate( + _calibration_coefficient_paths(state, calibration_settings) + ): + backup_path = recovery_dir / f"{file_number:03d}_{coefficient_path.name}" + if not backup_path.exists(): + raise RuntimeError( + f"cannot recover interrupted calibration iteration: missing {backup_path}" + ) + shutil.copyfile(backup_path, coefficient_path) + + +def _read_progress(state: workflow.State) -> dict[str, Any] | None: + """Read persisted calibration progress metadata if it exists.""" + path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) + if not path.exists(): + return None + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def _write_progress(state: workflow.State, payload: dict[str, Any]) -> None: + """Atomically write calibration progress metadata.""" + path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) + os.makedirs(path.parent, exist_ok=True) + temporary_path = path.with_name(f".{path.name}.tmp") + with open(temporary_path, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2) + os.replace(temporary_path, path) + + +def _write_completed_progress( + state: workflow.State, + completed_global_iterations: int, + converged: bool, +) -> None: + """Mark calibration complete after all final output has been written.""" + _write_progress( + state, + { + "complete": True, + "in_progress_iteration": None, + "next_global_iteration": completed_global_iterations + 1, + "last_completed_global_iteration": completed_global_iterations, + "converged": converged, + }, + ) + diff --git a/activitysim/core/calibration/reporting.py b/activitysim/core/calibration/reporting.py new file mode 100644 index 0000000000..ff09e8ab1f --- /dev/null +++ b/activitysim/core/calibration/reporting.py @@ -0,0 +1,268 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +import math +import os +from pathlib import Path +from typing import Any + +import matplotlib +import matplotlib.pyplot as plt +import pandas as pd + +from activitysim.core import workflow + +from .coefficients import _infer_model_settings_file +from .settings import CalibrationConfig + +plt.style.use("seaborn-v0_8-darkgrid") +matplotlib.use("Agg") + +CALIBRATION_OUTPUT_DIR = "calibration" +CALIBRATION_ITERATION_FILE = "calibration/calibration_iteration_records.csv" +CALIBRATION_SUMMARY_FILE = "calibration/calibration_iteration_summary.csv" +CALIBRATION_FINAL_COEFFICIENTS_FILE = "calibration/final_calibrated_coefficients.csv" +MAX_COEFFS_IN_GRAPH = 15 + + +def _append_iteration_records( + state: workflow.State, component_name: str, records: list[dict[str, Any]] +) -> None: + """Append per-coefficient calibration iteration records.""" + if not records: + return + df = pd.DataFrame(records) + + # Save a global iteration history file + global_path = state.get_output_file_path(CALIBRATION_ITERATION_FILE) + _append_csv( + df, + global_path, + unique_on=["global_iter", "component_iter", "component", "coefficient"], + ) + + # Also write component-local iteration history + component_path = ( + _component_output_dir(state, component_name) + / Path(CALIBRATION_ITERATION_FILE).name + ) + _append_csv( + df, + component_path, + unique_on=["global_iter", "component_iter", "component", "coefficient"], + ) + + +def _append_summary_records( + state: workflow.State, records: list[dict[str, Any]] +) -> None: + """Append per-iteration summary records.""" + if not records: + return + path = state.get_output_file_path(CALIBRATION_SUMMARY_FILE) + df = pd.DataFrame(records) + _append_csv( + df, + path, + unique_on=["global_iter", "component_iter", "component"], + ) + + +def _append_csv( + df: pd.DataFrame, path: Path, unique_on: list[str] | None = None +) -> None: + """Append a dataframe to a CSV file, replacing rows with matching keys.""" + os.makedirs(path.parent, exist_ok=True) + if unique_on and path.exists(): + existing = pd.read_csv(path) + df = pd.concat([existing, df], ignore_index=True).drop_duplicates( + subset=unique_on, keep="last" + ) + df.to_csv(path, index=False) + return + + write_header = not path.exists() + df.to_csv(path, mode="a", index=False, header=write_header) + + +def _component_output_dir(state: workflow.State, component_name: str) -> Path: + """Return output/calibration/ and ensure it exists.""" + component_dir = state.get_output_file_path(f"calibration/{component_name}") + os.makedirs(component_dir, exist_ok=True) + return component_dir + + +def _write_component_plots(state: workflow.State, component_name: str) -> None: + """Write/update all standard plots for one calibrated component.""" + recs = _read_component_iteration_records(state, component_name) + if recs is None or recs.empty: + return + + # Segment coefficients into manageable sets for plotting + coefs = sorted(recs.index.get_level_values("coefficient").unique()) + n_sets = math.ceil(len(coefs) / MAX_COEFFS_IN_GRAPH) + for coef_set in range(n_sets): + set_coefs = coefs[ + coef_set + * MAX_COEFFS_IN_GRAPH : min( + len(coefs), (coef_set + 1) * MAX_COEFFS_IN_GRAPH + ) + ] + _plot_coefficient_progress(state, component_name, recs, set_coefs, coef_set) + last_records = _component_last_records(recs, set_coefs) + _plot_component_values(state, component_name, last_records, coef_set) + _plot_component_pct_change(state, component_name, last_records, coef_set) + + +def _read_component_iteration_records( + state: workflow.State, component_name: str +) -> pd.DataFrame | None: + """Read all iteration records for a single component.""" + path = state.get_output_file_path(CALIBRATION_ITERATION_FILE) + if not path.exists(): + return None + + iteration_records = ( + pd.read_csv(path) + .set_index(["global_iter", "component_iter", "coefficient"]) + .sort_index() + ) + return iteration_records.loc[iteration_records.component == component_name] + + +def _plot_coefficient_progress( + state: workflow.State, + component_name: str, + recs: pd.DataFrame, + set_coefs: list[str], + coef_set: int, +) -> None: + """Plot coefficient value progression for one coefficient subset.""" + component_dir = _component_output_dir(state, component_name) + ax = ( + recs[recs.index.get_level_values("coefficient").isin(set_coefs)] + .next_coefficient.unstack("coefficient") + .plot(figsize=(10, 5)) + ) + ax.xaxis.set_label_text("Component iteration") + ax.yaxis.set_label_text("Coefficient value") + ax.legend(title="Coefficient label", loc="center left", bbox_to_anchor=(1.02, 0.5)) + plt.tight_layout() + ax.figure.savefig( + component_dir / f"coefficient_progress_set_{coef_set}.png", + bbox_inches="tight", + ) + plt.close(ax.figure) + + +def _component_last_records(recs: pd.DataFrame, set_coefs: list[str]) -> pd.DataFrame: + """Select target/model values for the latest iteration and coefficient subset.""" + filtered = recs[recs.index.get_level_values("coefficient").isin(set_coefs)] + last_global = filtered.index.get_level_values("global_iter")[-1] + last_comp = filtered.loc[last_global].index.get_level_values("component_iter")[-1] + return filtered.xs( + (last_global, last_comp), level=("global_iter", "component_iter") + )[["target_value", "model_value"]] + + +def _plot_component_values( + state: workflow.State, + component_name: str, + last_records: pd.DataFrame, + coef_set: int, +) -> None: + """Plot final target/model component values for one coefficient subset.""" + component_dir = _component_output_dir(state, component_name) + ax = last_records.plot.bar(figsize=(10, 5)) + ax.xaxis.set_tick_params(rotation=45) + ax.xaxis.set_label_text("Component value") + plt.tight_layout() + ax.figure.savefig(component_dir / f"final_components_set_{coef_set}.png") + plt.close(ax.figure) + + +def _plot_component_pct_change( + state: workflow.State, + component_name: str, + last_records: pd.DataFrame, + coef_set: int, +) -> None: + """Plot final percent difference for one coefficient subset.""" + component_dir = _component_output_dir(state, component_name) + fig, ax = plt.subplots(figsize=(10, 5)) + pct_diff = last_records.diff(axis=1).model_value / last_records.target_value + ax = pct_diff.plot.bar(ax=ax) + ax.xaxis.set_tick_params(rotation=45) + ax.xaxis.set_label_text("Coefficient") + ax.yaxis.set_label_text("% Difference between Model and Target") + plt.tight_layout() + ax.figure.savefig(component_dir / f"final_pct_change_set_{coef_set}.png") + plt.close(ax.figure) + + +def _write_generic_report( + state: workflow.State, + component_name: str, + row_records: list[dict[str, Any]], +) -> None: + """Write a simple dashboard-friendly generic report for the current component iteration.""" + if not row_records: + return + + df = pd.DataFrame(row_records) + report = ( + df[ + [ + "global_iter", + "component_iter", + "component", + "description", + "difference", + "pct_difference", + "converged", + ] + ] + .copy() + .sort_values(["global_iter", "component_iter", "description"]) + ) + + path = _component_output_dir(state, component_name) / "generic_report.csv" + _append_csv( + report, + path, + unique_on=["global_iter", "component_iter", "component", "description"], + ) + + +def _write_final_coefficients_snapshot( + state: workflow.State, + calibration_settings: CalibrationConfig, +) -> None: + """Write a combined final coefficients file snapshot for calibrated components.""" + frames = [] + for component_name in calibration_settings.run.calibrate_models: + model_settings_file = _infer_model_settings_file(component_name) + model_settings = state.filesystem.read_model_settings( + model_settings_file, mandatory=True + ) + coeff_df = state.filesystem.read_model_coefficients( + model_settings=model_settings + ).copy() + coeff_df = coeff_df.reset_index().rename(columns={"index": "coefficient_name"}) + coeff_df.insert(0, "component", component_name) + frames.append(coeff_df) + + if not frames: + return + + final_df = pd.concat(frames, ignore_index=True) + path = state.get_output_file_path(CALIBRATION_FINAL_COEFFICIENTS_FILE) + os.makedirs(path.parent, exist_ok=True) + final_df.to_csv(path, index=False) + + +def _ensure_calibration_output_dir(state: workflow.State) -> None: + """Ensure output/calibration exists.""" + path = state.get_output_file_path(CALIBRATION_OUTPUT_DIR) + os.makedirs(path, exist_ok=True) diff --git a/activitysim/core/calibration/settings.py b/activitysim/core/calibration/settings.py new file mode 100644 index 0000000000..b88db13da3 --- /dev/null +++ b/activitysim/core/calibration/settings.py @@ -0,0 +1,132 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +from pydantic import model_validator + +from activitysim.core import workflow +from activitysim.core.configuration import PydanticReadable +from activitysim.core.configuration.base import PydanticBase + +CALIBRATION_SETTINGS_FILE_NAME = "calibration.yaml" + + +class CalibrationRunSettings(PydanticBase): + """Run-control settings for calibration.""" + + resume_after: Optional[str] = None + calibrate_models: list[str] + global_iterations: int = 1 + complete_steps: bool = False + invalidate_tables: list[str] | None = None + + @model_validator(mode="after") + def validate_run_settings(self): + if not self.calibrate_models: + raise ValueError( + "calibration.run.calibrate_models must contain at least one model name" + ) + return self + + """Tables to drop from state after each calibration restore so their + ``@workflow.table`` factories regenerate from current data. + + Default (None): invalidates ``["vehicles"]``. Set to ``[]`` to disable. + + A table should be listed here when ALL of the following are true: + + 1. It is created by a ``@workflow.table`` factory from another table's + values (not just from input data files). + 2. That source table is modified by a calibrated model or by a model + whose outputs change when calibrated coefficients change. + 3. The factory uses source-table values to determine **row identity** + (index values) or **row count**, not just column values. + + The canonical example is ``vehicles``: its factory repeats household + rows by ``households["auto_ownership"]`` and derives ``vehicle_id`` + from ``household_id``. When ``auto_ownership_simulate`` is calibrated, + different coefficients produce different ownership counts, so the + stale vehicles table loaded from a prior checkpoint would have the + wrong number of rows and wrong vehicle IDs. Dropping it forces the + factory to regenerate vehicles consistent with the current households. + + Tables that only read *column values* from upstream tables (without + affecting row identity) generally do NOT need invalidation — their + content will be correct as long as the upstream table is correct at + the restored checkpoint. + """ + + +class CalibrationReportsSettings(PydanticBase): + """Reporting settings for a calibrated component.""" + + generic: bool = True + bespoke: str | None = None + + +class CalibrationComponentSettings(PydanticBase): + """Settings for one calibratable model component.""" + + calibration_spec: str + helper_module: str | None = None + submodel_max_iterations: int = 1 + reports: CalibrationReportsSettings = CalibrationReportsSettings() + survey_file: Optional[str] = None + + +class CalibrationConfig(PydanticReadable): + """Top-level calibration configuration.""" + + enable: bool = False + run: CalibrationRunSettings + model_settings: dict[str, CalibrationComponentSettings] = {} + + @model_validator(mode="after") + def validate_model_settings(self): + """Validate that configured components are aligned with run settings.""" + for component in self.run.calibrate_models: + if component not in self.model_settings: + raise ValueError( + f"calibration model '{component}' is not in model_settings" + ) + + if self.run.global_iterations < 1: + raise ValueError("max_iterations must be >= 1") + + return self + + +@dataclass +class CalibrationComponentResult: + """Result details from calibrating one component.""" + + component: str + converged: bool + component_iterations: int + + +@dataclass +class CalibrationRunResult: + """Result details from a complete global calibration loop.""" + + converged: bool + completed_global_iterations: int + + +def read_calibration_settings(state: workflow.State) -> CalibrationConfig | None: + """Read and validate calibration settings if calibration.yaml exists.""" + return CalibrationConfig.read_settings_file( + state.filesystem, + CALIBRATION_SETTINGS_FILE_NAME, + mandatory=False, + ) + + +def calibration_enabled(state: workflow.State) -> bool: + """Return True when calibration.yaml exists and is enabled.""" + settings = read_calibration_settings(state) + return bool(settings and settings.enable) + diff --git a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py index b9926eb0a1..217cb46580 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py +++ b/activitysim/examples/prototype_mtc/configs_calibration/workplace_location_calib_helper.py @@ -3,7 +3,8 @@ Notes: The lru_cache decorator to ensure that survey data is loaded and computed only once per Python process. - Context is developed in _build_expression_context() in activitysim.core.calibration. +Context is developed in _build_expression_context() in +activitysim.core.calibration.expressions. """ import matplotlib.pyplot as plt From ce7844f9550a9a3db84911ed4d8e5ca74513d679 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:25:54 -0700 Subject: [PATCH 78/96] if everything is converged, will continue to run downstream models --- activitysim/core/calibration/orchestrator.py | 29 ++++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py index a4973f9625..97953bb1a7 100644 --- a/activitysim/core/calibration/orchestrator.py +++ b/activitysim/core/calibration/orchestrator.py @@ -323,19 +323,24 @@ def run_calibration_loop( last_calibrated_component = component + iteration_is_complete = ( + all_converged + or global_iter == calibration_settings.run.global_iterations + ) + resumed_after_all_calibrated_models = ( + global_iter == start_global_iter + and state.settings.resume_after is not None + and first_model_idx > last_calib_model_idx + ) + if ( calibration_settings.run.complete_steps - or global_iter == calibration_settings.run.global_iterations - or ( - global_iter == start_global_iter - and state.settings.resume_after is not None - and first_model_idx > last_calib_model_idx - ) + or iteration_is_complete + or resumed_after_all_calibrated_models ): subsequent_components = ( models[first_model_idx:] - if global_iter == start_global_iter - and first_model_idx > last_calib_model_idx + if resumed_after_all_calibrated_models else models[models.index(last_calibrated_component) + 1 :] ) # finish the full model chain @@ -343,17 +348,12 @@ def run_calibration_loop( state, models=subsequent_components, resume_after=state.settings.resume_after - if global_iter == start_global_iter - and first_model_idx > last_calib_model_idx + if resumed_after_all_calibrated_models else last_calibrated_component, shared_data_buffers=shared_data_buffers, ) completed_global_iterations = global_iter - iteration_is_complete = ( - all_converged - or global_iter == calibration_settings.run.global_iterations - ) if not iteration_is_complete: _write_progress( state, @@ -452,4 +452,3 @@ def _prior_step_name(models: list[str], component_name: str) -> str | None: if idx == 0: return None return models[idx - 1] - From e9dcffbebb12e3fa02580ea26142e58bb9fc637b Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:26:38 -0700 Subject: [PATCH 79/96] blacken --- activitysim/core/calibration/coefficients.py | 1 - activitysim/core/calibration/expressions.py | 1 - activitysim/core/calibration/orchestrator.py | 4 +--- activitysim/core/calibration/recovery.py | 1 - activitysim/core/calibration/settings.py | 1 - 5 files changed, 1 insertion(+), 7 deletions(-) diff --git a/activitysim/core/calibration/coefficients.py b/activitysim/core/calibration/coefficients.py index 9aa15bcd4c..a41d36f7ad 100644 --- a/activitysim/core/calibration/coefficients.py +++ b/activitysim/core/calibration/coefficients.py @@ -82,4 +82,3 @@ def _calibration_coefficient_paths( seen.add(key) return paths - diff --git a/activitysim/core/calibration/expressions.py b/activitysim/core/calibration/expressions.py index 9338b7b39c..72263c9c96 100644 --- a/activitysim/core/calibration/expressions.py +++ b/activitysim/core/calibration/expressions.py @@ -196,4 +196,3 @@ def _load_helper_module(state: workflow.State, helper_module: str): module = importlib.import_module(helper_module) setattr(module, "state", state) return module - diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py index 97953bb1a7..2da1292304 100644 --- a/activitysim/core/calibration/orchestrator.py +++ b/activitysim/core/calibration/orchestrator.py @@ -125,9 +125,7 @@ def run_calibration_loop( ), ) - interrupted_iteration = ( - progress.get("in_progress_iteration") if progress else None - ) + interrupted_iteration = progress.get("in_progress_iteration") if progress else None if interrupted_iteration is not None: interrupted_iteration = int(interrupted_iteration) logger.warning( diff --git a/activitysim/core/calibration/recovery.py b/activitysim/core/calibration/recovery.py index b46c779ba8..2128480d94 100644 --- a/activitysim/core/calibration/recovery.py +++ b/activitysim/core/calibration/recovery.py @@ -100,4 +100,3 @@ def _write_completed_progress( "converged": converged, }, ) - diff --git a/activitysim/core/calibration/settings.py b/activitysim/core/calibration/settings.py index b88db13da3..b99964796c 100644 --- a/activitysim/core/calibration/settings.py +++ b/activitysim/core/calibration/settings.py @@ -129,4 +129,3 @@ def calibration_enabled(state: workflow.State) -> bool: """Return True when calibration.yaml exists and is enabled.""" settings = read_calibration_settings(state) return bool(settings and settings.enable) - From adf17f94d29dac4990f5d8e8f56f2875649a633b Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:06:38 -0700 Subject: [PATCH 80/96] remove coefficient rollback functionality --- activitysim/core/calibration/README.md | 14 +++++- activitysim/core/calibration/coefficients.py | 46 +++++++------------- activitysim/core/calibration/component.py | 6 ++- activitysim/core/calibration/orchestrator.py | 38 ++++++++-------- activitysim/core/calibration/recovery.py | 45 ++----------------- activitysim/core/calibration/reporting.py | 7 ++- activitysim/core/calibration/settings.py | 1 + 7 files changed, 60 insertions(+), 97 deletions(-) diff --git a/activitysim/core/calibration/README.md b/activitysim/core/calibration/README.md index 66c92b3802..6e7028f8c2 100644 --- a/activitysim/core/calibration/README.md +++ b/activitysim/core/calibration/README.md @@ -15,12 +15,22 @@ calibration math can evolve independently. and loads optional helper modules. - `coefficients.py` locates, reads, and writes model coefficient files. - `reporting.py` writes iteration histories, summaries, plots, and final snapshots. -- `recovery.py` maintains calibration progress and start-of-iteration coefficient - backups. +- `recovery.py` durably tracks calibration progress. Coefficient files are the + authoritative current state and are never rolled back when a run resumes. - `execution.py` restores pipeline state and dispatches model execution. - `multiprocess.py` contains subprocess orchestration, shared-resource setup, and multiprocess pipeline restoration. +Component settings may specify `model_settings_file` when a workflow step does +not follow the conventional `.yaml` naming pattern. Components +that share model settings may point to the same coefficient file. + +When a run is interrupted, calibration continues the active global iteration. +Any coefficient updates written before the interruption, including subsequent +manual edits, are preserved. On the first resumed iteration, top-level +`settings.yaml` `resume_after` uses standard ActivitySim semantics: the named +model is treated as complete and execution begins with the following model. + Dependencies should flow from `orchestrator.py` into the focused modules. Lower level modules should not import the orchestrator. Multiprocess code imports back into `execution.py` only inside the single-component runner to avoid an import-time diff --git a/activitysim/core/calibration/coefficients.py b/activitysim/core/calibration/coefficients.py index a41d36f7ad..134298e040 100644 --- a/activitysim/core/calibration/coefficients.py +++ b/activitysim/core/calibration/coefficients.py @@ -10,7 +10,7 @@ from activitysim.core import workflow -from .settings import CalibrationConfig +from .settings import CalibrationComponentSettings def _persist_coefficients_to_config( @@ -26,8 +26,10 @@ def _persist_coefficients_to_config( output = coefficients_df.copy() output.index.name = "coefficient_name" - coeff_path = state.filesystem.get_config_file_path(coeff_file) - output.to_csv(coeff_path) + coeff_path = Path(state.filesystem.get_config_file_path(coeff_file)) + temporary_path = coeff_path.with_name(f".{coeff_path.name}.tmp") + output.to_csv(temporary_path) + os.replace(temporary_path, coeff_path) def _infer_model_settings_file(component_name: str) -> str: @@ -40,6 +42,16 @@ def _infer_model_settings_file(component_name: str) -> str: return f"{base}.yaml" +def _resolve_model_settings_file( + component_name: str, + component_settings: CalibrationComponentSettings, +) -> str: + """Return an explicit component settings file or infer the conventional name.""" + return component_settings.model_settings_file or _infer_model_settings_file( + component_name + ) + + def _settings_to_dict(model_settings: dict[str, Any] | Any) -> dict[str, Any]: """Convert pydantic or dict model settings to a plain dictionary.""" if isinstance(model_settings, dict): @@ -54,31 +66,3 @@ def _setting_value(model_settings: dict[str, Any] | Any, key: str, default=None) if isinstance(model_settings, dict): return model_settings.get(key, default) return getattr(model_settings, key, default) - - -def _calibration_coefficient_paths( - state: workflow.State, - calibration_settings: CalibrationConfig, -) -> list[Path]: - """Return the unique coefficient files modified by this calibration run.""" - paths: list[Path] = [] - seen: set[str] = set() - - for component_name in calibration_settings.run.calibrate_models: - model_settings_file = _infer_model_settings_file(component_name) - model_settings = state.filesystem.read_model_settings( - model_settings_file, mandatory=True - ) - coefficient_file = _setting_value(model_settings, "COEFFICIENTS") - if not coefficient_file: - raise RuntimeError( - f"component {component_name} model settings missing COEFFICIENTS" - ) - - path = Path(state.filesystem.get_config_file_path(coefficient_file)).resolve() - key = os.path.normcase(str(path)) - if key not in seen: - paths.append(path) - seen.add(key) - - return paths diff --git a/activitysim/core/calibration/component.py b/activitysim/core/calibration/component.py index 7a3285ca3a..82539ebf1f 100644 --- a/activitysim/core/calibration/component.py +++ b/activitysim/core/calibration/component.py @@ -13,8 +13,8 @@ from activitysim.core import simulate, workflow from .coefficients import ( - _infer_model_settings_file, _persist_coefficients_to_config, + _resolve_model_settings_file, _setting_value, _settings_to_dict, ) @@ -59,7 +59,9 @@ def _calibrate_component( shared_data_buffers: dict | None = None, ) -> CalibrationComponentResult: """Run iterative coefficient calibration for one component.""" - model_settings_file = _infer_model_settings_file(component_name) + model_settings_file = _resolve_model_settings_file( + component_name, component_settings + ) model_settings = state.filesystem.read_model_settings( model_settings_file, mandatory=True ) diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py index 2da1292304..a93d5205d0 100644 --- a/activitysim/core/calibration/orchestrator.py +++ b/activitysim/core/calibration/orchestrator.py @@ -15,9 +15,8 @@ from .multiprocess import _initialize_mp_shared_resources from .recovery import ( CALIBRATION_PROGRESS_FILE, - _begin_global_iteration_transaction, + _mark_global_iteration_in_progress, _read_progress, - _restore_coefficient_backups, _write_completed_progress, _write_progress, ) @@ -110,6 +109,14 @@ def run_calibration_loop( progress = _read_progress(state) if progress and progress.get("complete"): + if resume_after is not None: + raise RuntimeError( + f"settings.yaml resume_after={resume_after!r} cannot be honored " + "because calibration progress is already complete. Remove " + f"{CALIBRATION_PROGRESS_FILE} or use a new output directory to " + "start a new calibration run; current coefficient values will " + "be preserved." + ) logger.info( "calibration progress is already complete; remove %s to start a " "fresh calibration run", @@ -127,22 +134,18 @@ def run_calibration_loop( interrupted_iteration = progress.get("in_progress_iteration") if progress else None if interrupted_iteration is not None: - interrupted_iteration = int(interrupted_iteration) + start_global_iter = int(interrupted_iteration) logger.warning( - "recovering interrupted calibration global iteration %s", - interrupted_iteration, + "continuing interrupted calibration global iteration %s using the " + "current coefficient files", + start_global_iter, + ) + else: + # Progress files from earlier versions contain next_global_iteration, so + # they remain compatible with the corrected total-count semantics. + start_global_iter = ( + int(progress.get("next_global_iteration", 1)) if progress else 1 ) - _restore_coefficient_backups(state, calibration_settings) - progress = { - "in_progress_iteration": None, - "next_global_iteration": interrupted_iteration, - "last_completed_global_iteration": interrupted_iteration - 1, - } - _write_progress(state, progress) - - # Progress files from earlier versions contain next_global_iteration, so - # they remain compatible with the corrected total-count semantics. - start_global_iter = int(progress.get("next_global_iteration", 1)) if progress else 1 completed_global_iterations = start_global_iter - 1 if start_global_iter > calibration_settings.run.global_iterations: @@ -239,9 +242,8 @@ def run_calibration_loop( start_global_iter, calibration_settings.run.global_iterations + 1, ): - _begin_global_iteration_transaction( + _mark_global_iteration_in_progress( state, - calibration_settings, global_iter, ) diff --git a/activitysim/core/calibration/recovery.py b/activitysim/core/calibration/recovery.py index 2128480d94..c7a4dc28dd 100644 --- a/activitysim/core/calibration/recovery.py +++ b/activitysim/core/calibration/recovery.py @@ -4,40 +4,18 @@ import json import os -import shutil from typing import Any from activitysim.core import workflow -from .coefficients import _calibration_coefficient_paths -from .settings import CalibrationConfig - CALIBRATION_PROGRESS_FILE = "calibration/calibration_progress.json" -CALIBRATION_RECOVERY_DIR = "calibration/recovery" -def _begin_global_iteration_transaction( +def _mark_global_iteration_in_progress( state: workflow.State, - calibration_settings: CalibrationConfig, global_iteration: int, ) -> None: - """Snapshot coefficients and durably mark a global iteration in progress.""" - recovery_dir = state.get_output_file_path(CALIBRATION_RECOVERY_DIR) - os.makedirs(recovery_dir, exist_ok=True) - - for file_number, coefficient_path in enumerate( - _calibration_coefficient_paths(state, calibration_settings) - ): - if not coefficient_path.exists(): - raise FileNotFoundError( - f"calibration coefficient file not found: {coefficient_path}" - ) - - backup_name = f"{file_number:03d}_{coefficient_path.name}" - shutil.copyfile(coefficient_path, recovery_dir / backup_name) - - # Write the marker only after all backups exist. If backup creation is - # interrupted, the previous between-iteration progress remains valid. + """Durably mark a global iteration in progress.""" _write_progress( state, { @@ -48,29 +26,12 @@ def _begin_global_iteration_transaction( ) -def _restore_coefficient_backups( - state: workflow.State, - calibration_settings: CalibrationConfig, -) -> None: - """Restore the coefficient backups for an interrupted global iteration.""" - recovery_dir = state.get_output_file_path(CALIBRATION_RECOVERY_DIR) - for file_number, coefficient_path in enumerate( - _calibration_coefficient_paths(state, calibration_settings) - ): - backup_path = recovery_dir / f"{file_number:03d}_{coefficient_path.name}" - if not backup_path.exists(): - raise RuntimeError( - f"cannot recover interrupted calibration iteration: missing {backup_path}" - ) - shutil.copyfile(backup_path, coefficient_path) - - def _read_progress(state: workflow.State) -> dict[str, Any] | None: """Read persisted calibration progress metadata if it exists.""" path = state.get_output_file_path(CALIBRATION_PROGRESS_FILE) if not path.exists(): return None - with open(path, "r", encoding="utf-8") as f: + with open(path, encoding="utf-8") as f: return json.load(f) diff --git a/activitysim/core/calibration/reporting.py b/activitysim/core/calibration/reporting.py index ff09e8ab1f..a4fbd60010 100644 --- a/activitysim/core/calibration/reporting.py +++ b/activitysim/core/calibration/reporting.py @@ -13,7 +13,7 @@ from activitysim.core import workflow -from .coefficients import _infer_model_settings_file +from .coefficients import _resolve_model_settings_file from .settings import CalibrationConfig plt.style.use("seaborn-v0_8-darkgrid") @@ -242,7 +242,10 @@ def _write_final_coefficients_snapshot( """Write a combined final coefficients file snapshot for calibrated components.""" frames = [] for component_name in calibration_settings.run.calibrate_models: - model_settings_file = _infer_model_settings_file(component_name) + component_settings = calibration_settings.model_settings[component_name] + model_settings_file = _resolve_model_settings_file( + component_name, component_settings + ) model_settings = state.filesystem.read_model_settings( model_settings_file, mandatory=True ) diff --git a/activitysim/core/calibration/settings.py b/activitysim/core/calibration/settings.py index b99964796c..7bbf5d6250 100644 --- a/activitysim/core/calibration/settings.py +++ b/activitysim/core/calibration/settings.py @@ -71,6 +71,7 @@ class CalibrationComponentSettings(PydanticBase): """Settings for one calibratable model component.""" calibration_spec: str + model_settings_file: str | None = None helper_module: str | None = None submodel_max_iterations: int = 1 reports: CalibrationReportsSettings = CalibrationReportsSettings() From 66e1037f5ef97de9c15c99d9889a9db7f9bd2724 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 19 Aug 2026 17:44:46 -0700 Subject: [PATCH 81/96] better handling of internal memory handling between iterations --- activitysim/core/calibration/component.py | 11 +- activitysim/core/calibration/execution.py | 133 ++++++++++--------- activitysim/core/calibration/orchestrator.py | 2 - activitysim/core/calibration/settings.py | 31 +---- 4 files changed, 77 insertions(+), 100 deletions(-) diff --git a/activitysim/core/calibration/component.py b/activitysim/core/calibration/component.py index 82539ebf1f..caa6b1d8d1 100644 --- a/activitysim/core/calibration/component.py +++ b/activitysim/core/calibration/component.py @@ -18,7 +18,7 @@ _setting_value, _settings_to_dict, ) -from .execution import _invalidate_derived_tables, _prep_model_data +from .execution import _prep_model_data from .expressions import ( _build_expression_context, _compute_delta, @@ -136,18 +136,15 @@ def _calibrate_component( # Restore to prior_step ourselves then run the model directly. # state.run(resume_after=prior_step) would trigger # checkpoint.restore → init_state which creates a fresh RNG. - # If prior_step is before the calibrated model created its table - # (e.g. vehicles), the table won't be in that checkpoint and the - # RNG channel won't be registered — causing a crash when the - # model tries to use it. By restoring here and calling by_name, - # we keep the RNG channels from _prep_model_data intact. + # _prep_model_data also performs an exact calibration rewind, + # removing tables and RNG channels that do not exist at prior_step + # so normal model execution can recreate them at the right point. extra_models = _prep_model_data(state, resume_after=prior_step) if extra_models: # prior_step checkpoint not found directly; run intermediate # models (e.g. annotators) to recreate the correct state. for m in extra_models: state.run.by_name(m) - _invalidate_derived_tables(state) state.checkpoint.add(prior_step) state.run.by_name(run_model_name) diff --git a/activitysim/core/calibration/execution.py b/activitysim/core/calibration/execution.py index 0c7f8e98ab..8180ae22a9 100644 --- a/activitysim/core/calibration/execution.py +++ b/activitysim/core/calibration/execution.py @@ -5,15 +5,21 @@ import logging from pathlib import Path +import pandas as pd + from activitysim.core import workflow +from activitysim.core.workflow.checkpoint import ( + CHECKPOINT_NAME, + CHECKPOINT_TABLE_NAME, + LAST_CHECKPOINT, + NON_TABLE_COLUMNS, +) from .multiprocess import ( _reregister_rng_channels, _restore_from_subprocess_pipelines, _run_multiprocess_with_overrides, ) -from .settings import read_calibration_settings - logger = logging.getLogger("calibration") @@ -167,62 +173,15 @@ def _prep_model_data(state, resume_after=None): return [] -def _invalidate_derived_tables(state: workflow.State) -> None: - """Drop factory-produced tables that may be stale after a calibration restore. - - When a calibrated model (e.g. auto_ownership) changes a table that a - @workflow.table factory depends on (e.g. vehicles depends on households), - the checkpoint may contain a stale version of that factory table. Dropping - it forces the factory to regenerate from current data on next access. - - Auto-detection rule: invalidate any table that is (a) registered as a - @workflow.table factory, (b) has DataFrame parameters (= table dependencies), - and (c) is in RANDOM_CHANNELS. This currently matches only 'vehicles' but - will automatically cover future factory tables with the same pattern. - """ - settings = read_calibration_settings(state) - if not settings: - return - - tables_to_invalidate = settings.run.invalidate_tables - if tables_to_invalidate is None: - # Vehicles needs regeneration only when calibration changes - # households.auto_ownership. Downstream calibration components must - # retain vehicle_type_choice's vehicle attributes. - tables_to_invalidate = ( - ["vehicles"] - if "auto_ownership_simulate" in settings.run.calibrate_models - else [] - ) - - logger.debug( - "calibration: tables detected for invalidation: %s", tables_to_invalidate - ) - tables_before = set(state.existing_table_names) - - for table_name in tables_to_invalidate: - if state.is_table(table_name): - state.drop_table(table_name) - state.rng().drop_channel(table_name) - state.get_dataframe(table_name, as_copy=False) - logger.debug("calibration: invalidated derived table '%s'", table_name) - - tables_after = set(state.existing_table_names) - lost = tables_before - tables_after - set(tables_to_invalidate) - if lost: - logger.error( - "calibration: tables unexpectedly removed during invalidation: %s", lost - ) - - def _restore_parent_state_from_pipeline( state: workflow.State, checkpoint_name: str = "_" ) -> None: """Restore pipeline tables into the parent process state. - After a multiprocess run, the parent's in-memory state is stale. - This loads a specific checkpoint from the pipeline store so that - calibration expressions can evaluate against model outputs. + After a multiprocess run or calibration rewind, the parent's in-memory + state may contain tables created after the requested checkpoint. Remove + those tables before loading the checkpoint so the restored state exactly + represents that point in the model sequence. Parameters ---------- @@ -238,15 +197,65 @@ def _restore_parent_state_from_pipeline( subprocesses can load them from a direct file path without relying on checkpoint backtracking through potentially ambiguous checkpoint history. """ - # Capture RNG state before restore — models may have dynamically - # added channels (e.g. "vehicles") that aren't in the default - # rng_channels injectable and would be lost by init_state(). - prior_rng_channels = list(state.get_injectable("rng_channels", [])) - prior_index_to_channel = ( - dict(state.rng().index_to_channel) - if hasattr(state.rng(), "index_to_channel") - else {} - ) + # Read the target checkpoint manifest before restore truncates checkpoint + # history. Tables represented by columns in this manifest are pipeline- + # managed; a false/empty value means the table did not yet exist. + checkpoints = state.checkpoint.store.get_dataframe(CHECKPOINT_TABLE_NAME) + if checkpoint_name == LAST_CHECKPOINT: + target_checkpoint = checkpoints.iloc[-1] + else: + matching_checkpoints = checkpoints[ + checkpoints[CHECKPOINT_NAME] == checkpoint_name + ] + if matching_checkpoints.empty: + # Let checkpoint.restore raise its normal, more specific exception. + target_checkpoint = None + else: + target_checkpoint = matching_checkpoints.iloc[-1] + + stale_tables: set[str] = set() + if target_checkpoint is not None: + pipeline_tables = set(checkpoints.columns) - set(NON_TABLE_COLUMNS) + target_tables = { + table_name + for table_name in pipeline_tables + if pd.notna(target_checkpoint[table_name]) + and bool(target_checkpoint[table_name]) + } + stale_tables = pipeline_tables - target_tables + + # Capture RNG state before restore — models may have dynamically added + # channels that aren't in the default rng_channels injectable. Do not carry + # channels for stale tables across the rewind; their table factories will + # register fresh channels when normal downstream execution recreates them. + prior_rng_channels = [ + channel_name + for channel_name in state.get_injectable("rng_channels", []) + if channel_name not in stale_tables + ] + prior_index_to_channel = { + index_name: channel_name + for index_name, channel_name in getattr( + state.rng(), "index_to_channel", {} + ).items() + if channel_name not in stale_tables + } + + for table_name in stale_tables: + # Use State.drop rather than drop_table: a prior restore may have reset + # salient-table metadata while leaving the cached DataFrame in context. + if table_name in state: + state.drop(table_name) + logger.debug( + "calibration: exact restore removed post-checkpoint table '%s'", + table_name, + ) + if table_name in state.rng().channels: + state.rng().drop_channel(table_name) + + # checkpoint.load uses this injectable to decide which restored tables get + # RNG channels. Remove stale channel names before it performs that work. + state.add_injectable("rng_channels", prior_rng_channels) if state.checkpoint.store_is_open(): state.checkpoint.close_store() diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py index a93d5205d0..facfa88eea 100644 --- a/activitysim/core/calibration/orchestrator.py +++ b/activitysim/core/calibration/orchestrator.py @@ -8,7 +8,6 @@ from .component import _calibrate_component from .execution import ( - _invalidate_derived_tables, _prep_model_data, _run_in_configured_mode, ) @@ -267,7 +266,6 @@ def run_calibration_loop( resume_after=None, shared_data_buffers=shared_data_buffers, ) - _invalidate_derived_tables(state) if first_calibration_restart_step is not None: state.checkpoint.add(first_calibration_restart_step) state.checkpoint.close_store() diff --git a/activitysim/core/calibration/settings.py b/activitysim/core/calibration/settings.py index 7bbf5d6250..23b1588b47 100644 --- a/activitysim/core/calibration/settings.py +++ b/activitysim/core/calibration/settings.py @@ -21,6 +21,8 @@ class CalibrationRunSettings(PydanticBase): calibrate_models: list[str] global_iterations: int = 1 complete_steps: bool = False + # Deprecated compatibility setting retained so existing configurations + # continue to parse. Exact calibration restores supersede this setting. invalidate_tables: list[str] | None = None @model_validator(mode="after") @@ -31,35 +33,6 @@ def validate_run_settings(self): ) return self - """Tables to drop from state after each calibration restore so their - ``@workflow.table`` factories regenerate from current data. - - Default (None): invalidates ``["vehicles"]``. Set to ``[]`` to disable. - - A table should be listed here when ALL of the following are true: - - 1. It is created by a ``@workflow.table`` factory from another table's - values (not just from input data files). - 2. That source table is modified by a calibrated model or by a model - whose outputs change when calibrated coefficients change. - 3. The factory uses source-table values to determine **row identity** - (index values) or **row count**, not just column values. - - The canonical example is ``vehicles``: its factory repeats household - rows by ``households["auto_ownership"]`` and derives ``vehicle_id`` - from ``household_id``. When ``auto_ownership_simulate`` is calibrated, - different coefficients produce different ownership counts, so the - stale vehicles table loaded from a prior checkpoint would have the - wrong number of rows and wrong vehicle IDs. Dropping it forces the - factory to regenerate vehicles consistent with the current households. - - Tables that only read *column values* from upstream tables (without - affecting row identity) generally do NOT need invalidation — their - content will be correct as long as the upstream table is correct at - the restored checkpoint. - """ - - class CalibrationReportsSettings(PydanticBase): """Reporting settings for a calibrated component.""" From 1f5586322bf0afaf86bbc7b18153308e1c77d599 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:29:41 -0700 Subject: [PATCH 82/96] run modes and resume after tests --- activitysim/core/calibration/execution.py | 16 +- activitysim/core/calibration/multiprocess.py | 24 ++- activitysim/core/calibration/settings.py | 1 + .../prototype_mtc/test/calibration/README.md | 26 +++ .../configs/auto_ownership_calibration.csv | 2 + .../test/calibration/configs/calibration.yaml | 29 +++ .../test/calibration/configs/settings.yaml | 42 ++++ .../configs/tour_mode_choice_calibration.csv | 2 + .../tour_mode_choice_calibration_failing.csv | 2 + .../workplace_location_calib_helper.py | 13 ++ .../workplace_location_calibration.csv | 2 + .../test/calibration/configs_mp/settings.yaml | 62 ++++++ .../test/calibration/test_run_modes.py | 202 ++++++++++++++++++ 13 files changed, 408 insertions(+), 15 deletions(-) create mode 100644 activitysim/examples/prototype_mtc/test/calibration/README.md create mode 100644 activitysim/examples/prototype_mtc/test/calibration/configs/auto_ownership_calibration.csv create mode 100644 activitysim/examples/prototype_mtc/test/calibration/configs/calibration.yaml create mode 100644 activitysim/examples/prototype_mtc/test/calibration/configs/settings.yaml create mode 100644 activitysim/examples/prototype_mtc/test/calibration/configs/tour_mode_choice_calibration.csv create mode 100644 activitysim/examples/prototype_mtc/test/calibration/configs/tour_mode_choice_calibration_failing.csv create mode 100644 activitysim/examples/prototype_mtc/test/calibration/configs/workplace_location_calib_helper.py create mode 100644 activitysim/examples/prototype_mtc/test/calibration/configs/workplace_location_calibration.csv create mode 100644 activitysim/examples/prototype_mtc/test/calibration/configs_mp/settings.yaml create mode 100644 activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py diff --git a/activitysim/core/calibration/execution.py b/activitysim/core/calibration/execution.py index 8180ae22a9..7afe6bab81 100644 --- a/activitysim/core/calibration/execution.py +++ b/activitysim/core/calibration/execution.py @@ -20,6 +20,7 @@ _restore_from_subprocess_pipelines, _run_multiprocess_with_overrides, ) + logger = logging.getLogger("calibration") @@ -53,19 +54,16 @@ def _run_in_configured_mode( state.checkpoint.add(resume_after or models[0]) state.checkpoint.close_store() - # When subprocess pipelines from a prior run already have the - # resume_after checkpoint (Path 2 in _prep_model_data), subprocesses - # can skip models before resume_after by reusing those pipelines - # instead of freshly apportioning. Signal this by passing - # can_reuse_subprocs=True. - can_reuse = not extra_models and resume_after is not None - _run_multiprocess_with_overrides( state, models=models, resume_after=resume_after, shared_data_buffers=shared_data_buffers, - can_reuse_subprocs=can_reuse, + # _prep_model_data may have restored either the main pipeline or + # coalesced subprocess pipelines. Freshly apportion the exact + # restored parent state in both cases; an empty extra_models list + # alone is not evidence that subprocess pipelines are reusable. + can_reuse_subprocs=False, ) # After multiprocess completes, the coalesced pipeline exists on disk. # Restore it into the parent process state so tables are accessible @@ -144,7 +142,7 @@ def _prep_model_data(state, resume_after=None): resume_idx = all_models.index(resume_after) step_boundaries = [all_models.index(s.begin) for s in mp_steps] step_boundaries.append(len(all_models)) - for i, step in enumerate(mp_steps): + for i, _step in enumerate(mp_steps): if step_boundaries[i] <= resume_idx < step_boundaries[i + 1]: if i > 0 and mp_steps[i - 1].name in checkpoint_names: _restore_parent_state_from_pipeline( diff --git a/activitysim/core/calibration/multiprocess.py b/activitysim/core/calibration/multiprocess.py index b828005007..9408787ffb 100644 --- a/activitysim/core/calibration/multiprocess.py +++ b/activitysim/core/calibration/multiprocess.py @@ -441,12 +441,13 @@ def _reregister_rng_channels( """Re-register RNG channels that were lost during init_state().""" current_channels = set(state.get_injectable("rng_channels", [])) for channel_name in prior_channels: - if channel_name not in current_channels and state.is_table(channel_name): + if channel_name not in state.rng().channels and state.is_table(channel_name): try: state.rng().add_channel(channel_name, state.get_dataframe(channel_name)) - current_channels.add(channel_name) except Exception: pass + if channel_name in state.rng().channels: + current_channels.add(channel_name) # For channels whose tables don't exist at the restored checkpoint, # register an empty channel. Do NOT pre-load from a later checkpoint # in the store — that data may include modifications from downstream @@ -457,12 +458,23 @@ def _reregister_rng_channels( for index_name, channel_name in prior_index_to_channel.items(): if index_name not in state.rng().index_to_channel: if channel_name not in state.rng().channels: - empty_df = pd.DataFrame( - index=pd.Index([], dtype="int64", name=index_name) - ) - state.rng().add_channel(channel_name, empty_df) + if state.is_table(channel_name): + # The channel may have been registered dynamically and + # therefore be absent from the rng_channels injectable. + # Populate it from the exact table restored from the + # target checkpoint rather than creating an empty + # channel for an existing domain. + state.rng().add_channel( + channel_name, state.get_dataframe(channel_name) + ) + else: + empty_df = pd.DataFrame( + index=pd.Index([], dtype="int64", name=index_name) + ) + state.rng().add_channel(channel_name, empty_df) else: state.rng().index_to_channel[index_name] = channel_name + if channel_name in state.rng().channels: current_channels.add(channel_name) state.add_injectable("rng_channels", list(current_channels)) diff --git a/activitysim/core/calibration/settings.py b/activitysim/core/calibration/settings.py index 23b1588b47..dbd8accaab 100644 --- a/activitysim/core/calibration/settings.py +++ b/activitysim/core/calibration/settings.py @@ -33,6 +33,7 @@ def validate_run_settings(self): ) return self + class CalibrationReportsSettings(PydanticBase): """Reporting settings for a calibrated component.""" diff --git a/activitysim/examples/prototype_mtc/test/calibration/README.md b/activitysim/examples/prototype_mtc/test/calibration/README.md new file mode 100644 index 0000000000..f8288ef562 --- /dev/null +++ b/activitysim/examples/prototype_mtc/test/calibration/README.md @@ -0,0 +1,26 @@ +# Prototype MTC calibration run-mode test + +This fixture exercises calibration with three real prototype MTC models: + +- `workplace_location` evaluates its target with the `DIST` network skim; +- `auto_ownership_simulate` provides a second upstream coefficient update; and +- `tour_mode_choice_simulate` is the downstream restart target. + +The test runs uninterrupted single- and multiprocess references. It then runs +each mode with a deliberately invalid tour-mode calibration expression, verifies +that global iteration 1 remains in progress, replaces the expression, and +restarts with `resume_after: non_mandatory_tour_scheduling`. The resumed runs +must preserve the workplace and auto-ownership coefficient files and match the +uninterrupted final tables and calibrated coefficients across both modes. + +Each run copies mutable coefficient files into its temporary config directory; +the shared prototype MTC configs and data remain unchanged. The 25-household +sample minimizes simulation work, although process startup and pipeline +apportion/coalesce make the full matrix an integration test rather than a fast +unit test. + +Run from the repository root with: + +```powershell +.venv\Scripts\python.exe -m pytest activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py -q +``` diff --git a/activitysim/examples/prototype_mtc/test/calibration/configs/auto_ownership_calibration.csv b/activitysim/examples/prototype_mtc/test/calibration/configs/auto_ownership_calibration.csv new file mode 100644 index 0000000000..ecddc7f5dd --- /dev/null +++ b/activitysim/examples/prototype_mtc/test/calibration/configs/auto_ownership_calibration.csv @@ -0,0 +1,2 @@ +description,coefficient,model_value,target_value,hold_fast,min,max,damping,method,tolerance +Zero-auto household share,coef_calib_auto_0,len(households[households.auto_ownership == 0]) / len(households),0.271828,FALSE,-10,10,1,log_ratio,0 diff --git a/activitysim/examples/prototype_mtc/test/calibration/configs/calibration.yaml b/activitysim/examples/prototype_mtc/test/calibration/configs/calibration.yaml new file mode 100644 index 0000000000..2b4aa8d159 --- /dev/null +++ b/activitysim/examples/prototype_mtc/test/calibration/configs/calibration.yaml @@ -0,0 +1,29 @@ +enable: true + +run: + calibrate_models: + - workplace_location + - auto_ownership_simulate + - tour_mode_choice_simulate + global_iterations: 1 + complete_steps: true + +model_settings: + workplace_location: + calibration_spec: workplace_location_calibration.csv + helper_module: workplace_location_calib_helper.py + submodel_max_iterations: 1 + reports: + generic: false + + auto_ownership_simulate: + calibration_spec: auto_ownership_calibration.csv + submodel_max_iterations: 1 + reports: + generic: false + + tour_mode_choice_simulate: + calibration_spec: tour_mode_choice_calibration.csv + submodel_max_iterations: 1 + reports: + generic: false diff --git a/activitysim/examples/prototype_mtc/test/calibration/configs/settings.yaml b/activitysim/examples/prototype_mtc/test/calibration/configs/settings.yaml new file mode 100644 index 0000000000..897735df4a --- /dev/null +++ b/activitysim/examples/prototype_mtc/test/calibration/configs/settings.yaml @@ -0,0 +1,42 @@ +inherit_settings: true + +strict: false +multiprocess: false +rng_base_seed: 0 +households_sample_size: 25 +chunk_size: 0 +use_shadow_pricing: false +want_dest_choice_sample_tables: false +cleanup_pipeline_after_run: false + +models: + - initialize_landuse + - initialize_households + - compute_accessibility + - school_location + - workplace_location + - auto_ownership_simulate + - free_parking + - cdap_simulate + - mandatory_tour_frequency + - mandatory_tour_scheduling + - joint_tour_frequency + - joint_tour_composition + - joint_tour_participation + - joint_tour_destination + - joint_tour_scheduling + - non_mandatory_tour_frequency + - non_mandatory_tour_destination + - non_mandatory_tour_scheduling + - tour_mode_choice_simulate + - write_tables + +output_tables: + h5_store: false + action: include + prefix: final_ + sort: true + tables: + - households + - persons + - tours diff --git a/activitysim/examples/prototype_mtc/test/calibration/configs/tour_mode_choice_calibration.csv b/activitysim/examples/prototype_mtc/test/calibration/configs/tour_mode_choice_calibration.csv new file mode 100644 index 0000000000..048b703c76 --- /dev/null +++ b/activitysim/examples/prototype_mtc/test/calibration/configs/tour_mode_choice_calibration.csv @@ -0,0 +1,2 @@ +description,coefficient,model_value,target_value,hold_fast,min,max,damping,method,tolerance +Walk tour share for auto-sufficient households,coef_calib_WALK_auto_suff,len(tours[tours.tour_mode == 'WALK']) / len(tours),0.271828,FALSE,-10,10,1,log_ratio,0 diff --git a/activitysim/examples/prototype_mtc/test/calibration/configs/tour_mode_choice_calibration_failing.csv b/activitysim/examples/prototype_mtc/test/calibration/configs/tour_mode_choice_calibration_failing.csv new file mode 100644 index 0000000000..3a5922f642 --- /dev/null +++ b/activitysim/examples/prototype_mtc/test/calibration/configs/tour_mode_choice_calibration_failing.csv @@ -0,0 +1,2 @@ +description,coefficient,model_value,target_value,hold_fast,min,max,damping,method,tolerance +Walk tour share for auto-sufficient households,coef_calib_WALK_auto_suff,missing_model_value,0.271828,FALSE,-10,10,1,log_ratio,0 diff --git a/activitysim/examples/prototype_mtc/test/calibration/configs/workplace_location_calib_helper.py b/activitysim/examples/prototype_mtc/test/calibration/configs/workplace_location_calib_helper.py new file mode 100644 index 0000000000..fd081a8ef2 --- /dev/null +++ b/activitysim/examples/prototype_mtc/test/calibration/configs/workplace_location_calib_helper.py @@ -0,0 +1,13 @@ +from __future__ import annotations + + +def workplace_distance_share(context, minimum, maximum): + """Return the modeled worker share in a DIST skim interval.""" + persons = context["persons"] + workers = persons[persons["workplace_zone_id"] > 0] + distances = context["skim_dict"].lookup( + workers["home_zone_id"], + workers["workplace_zone_id"], + "DIST", + ) + return ((distances >= minimum) & (distances < maximum)).mean() diff --git a/activitysim/examples/prototype_mtc/test/calibration/configs/workplace_location_calibration.csv b/activitysim/examples/prototype_mtc/test/calibration/configs/workplace_location_calibration.csv new file mode 100644 index 0000000000..2eb39163d6 --- /dev/null +++ b/activitysim/examples/prototype_mtc/test/calibration/configs/workplace_location_calibration.csv @@ -0,0 +1,2 @@ +description,coefficient,model_value,target_value,hold_fast,min,max,damping,method,tolerance +Workplace distance under 2 miles,coef_calib_dist_0_2,"workplace_distance_share(context, 0, 2)",0.271828,FALSE,-10,10,1,log_ratio,0 diff --git a/activitysim/examples/prototype_mtc/test/calibration/configs_mp/settings.yaml b/activitysim/examples/prototype_mtc/test/calibration/configs_mp/settings.yaml new file mode 100644 index 0000000000..33d33c537b --- /dev/null +++ b/activitysim/examples/prototype_mtc/test/calibration/configs_mp/settings.yaml @@ -0,0 +1,62 @@ +inherit_settings: true + +strict: false +multiprocess: true +fail_fast: true +num_processes: 2 +rng_base_seed: 0 +households_sample_size: 25 +chunk_size: 0 +use_shadow_pricing: false +want_dest_choice_sample_tables: false +cleanup_pipeline_after_run: false + +models: + - initialize_landuse + - initialize_households + - compute_accessibility + - school_location + - workplace_location + - auto_ownership_simulate + - free_parking + - cdap_simulate + - mandatory_tour_frequency + - mandatory_tour_scheduling + - joint_tour_frequency + - joint_tour_composition + - joint_tour_participation + - joint_tour_destination + - joint_tour_scheduling + - non_mandatory_tour_frequency + - non_mandatory_tour_destination + - non_mandatory_tour_scheduling + - tour_mode_choice_simulate + - write_tables + +multiprocess_steps: + - name: mp_initialize + begin: initialize_landuse + - name: mp_accessibility + begin: compute_accessibility + slice: + tables: + - accessibility + exclude: true + - name: mp_households + begin: school_location + slice: + tables: + - households + - persons + - name: mp_summarize + begin: write_tables + +output_tables: + h5_store: false + action: include + prefix: final_ + sort: true + tables: + - households + - persons + - tours diff --git a/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py b/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py new file mode 100644 index 0000000000..607efdcd24 --- /dev/null +++ b/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path + +import pandas as pd +import pandas.testing as pdt +import pytest +import yaml + + +HERE = Path(__file__).resolve().parent +EXAMPLE_ROOT = HERE.parents[1] +TEST_ROOT = EXAMPLE_ROOT / "test" +BASE_CONFIGS = EXAMPLE_ROOT / "configs" +BASE_MP_CONFIGS = EXAMPLE_ROOT / "configs_mp" +DATA = EXAMPLE_ROOT / "data" +SIMULATION = TEST_ROOT / "simulation.py" + +COEFFICIENT_FILES = ( + "workplace_location_coefficients.csv", + "auto_ownership_coefficients.csv", + "tour_mode_choice_coefficients.csv", +) + +RESULT_TABLES = ( + "final_households.csv", + "final_persons.csv", + "final_tours.csv", +) + + +def _prepare_run(tmp_path: Path, name: str, multiprocess: bool, failing: bool): + run_dir = tmp_path / name + configs_dir = run_dir / "configs" + output_dir = run_dir / "output" + shutil.copytree(HERE / "configs", configs_dir) + output_dir.mkdir(parents=True) + + settings_source = ( + HERE / "configs_mp" / "settings.yaml" + if multiprocess + else HERE / "configs" / "settings.yaml" + ) + shutil.copyfile(settings_source, configs_dir / "settings.yaml") + + for file_name in COEFFICIENT_FILES: + shutil.copyfile(BASE_CONFIGS / file_name, configs_dir / file_name) + + if failing: + shutil.copyfile( + configs_dir / "tour_mode_choice_calibration_failing.csv", + configs_dir / "tour_mode_choice_calibration.csv", + ) + + return configs_dir, output_dir + + +def _run(configs_dir: Path, output_dir: Path, multiprocess: bool): + args = [ + sys.executable, + str(SIMULATION), + "-c", + str(configs_dir), + ] + if multiprocess: + args.extend(["-c", str(BASE_MP_CONFIGS)]) + args.extend( + [ + "-c", + str(BASE_CONFIGS), + "-d", + str(DATA), + "-o", + str(output_dir), + ] + ) + return subprocess.run(args, check=False, capture_output=True, text=True) + + +def _resume(configs_dir: Path): + shutil.copyfile( + HERE / "configs" / "tour_mode_choice_calibration.csv", + configs_dir / "tour_mode_choice_calibration.csv", + ) + settings_path = configs_dir / "settings.yaml" + with open(settings_path, encoding="utf-8") as stream: + settings = yaml.safe_load(stream) + settings["resume_after"] = "non_mandatory_tour_scheduling" + with open(settings_path, "w", encoding="utf-8") as stream: + yaml.safe_dump(settings, stream, sort_keys=False) + + +def _assert_success(result: subprocess.CompletedProcess): + assert result.returncode == 0, result.stdout + result.stderr + + +def _assert_equivalent(left_output: Path, right_output: Path): + for file_name in RESULT_TABLES: + left = pd.read_csv(left_output / file_name).sort_index(axis=1) + right = pd.read_csv(right_output / file_name).sort_index(axis=1) + pdt.assert_frame_equal(left, right, check_dtype=False) + + left_coefficients = pd.read_csv( + left_output / "calibration" / "final_calibrated_coefficients.csv" + ).sort_values(["component", "coefficient_name"]) + right_coefficients = pd.read_csv( + right_output / "calibration" / "final_calibrated_coefficients.csv" + ).sort_values(["component", "coefficient_name"]) + pdt.assert_frame_equal( + left_coefficients.reset_index(drop=True), + right_coefficients.reset_index(drop=True), + check_dtype=False, + check_exact=False, + rtol=1e-12, + atol=1e-12, + ) + + +def _run_reference(root: Path, name: str, multiprocess: bool) -> Path: + configs, output = _prepare_run(root, name, multiprocess, failing=False) + _assert_success(_run(configs, output, multiprocess)) + return output + + +def _run_resumed(root: Path, name: str, multiprocess: bool) -> Path: + configs, output = _prepare_run(root, name, multiprocess, failing=True) + failed = _run(configs, output, multiprocess) + assert failed.returncode != 0 + + progress_path = output / "calibration" / "calibration_progress.json" + with open(progress_path, encoding="utf-8") as stream: + progress = json.load(stream) + assert progress["in_progress_iteration"] == 1 + + upstream_coefficients = { + file_name: (configs / file_name).read_bytes() + for file_name in COEFFICIENT_FILES[:2] + } + + _resume(configs) + _assert_success(_run(configs, output, multiprocess)) + + for file_name, contents in upstream_coefficients.items(): + assert (configs / file_name).read_bytes() == contents + return output + + +@pytest.fixture(scope="module") +def run_root(tmp_path_factory) -> Path: + return tmp_path_factory.mktemp("calibration_run_modes") + + +@pytest.fixture(scope="module") +def single_output(run_root: Path) -> Path: + return _run_reference(run_root, "single", multiprocess=False) + + +@pytest.fixture(scope="module") +def multiprocess_output(run_root: Path) -> Path: + return _run_reference(run_root, "multiprocess", multiprocess=True) + + +@pytest.fixture(scope="module") +def single_resumed_output(run_root: Path) -> Path: + return _run_resumed(run_root, "single_resumed", multiprocess=False) + + +@pytest.fixture(scope="module") +def multiprocess_resumed_output(run_root: Path) -> Path: + return _run_resumed(run_root, "multiprocess_resumed", multiprocess=True) + + +def test_single_and_multiprocess_are_equivalent( + single_output: Path, + multiprocess_output: Path, +): + _assert_equivalent(single_output, multiprocess_output) + + +def test_single_resume_matches_uninterrupted( + single_output: Path, + single_resumed_output: Path, +): + _assert_equivalent(single_output, single_resumed_output) + + +def test_multiprocess_resume_matches_uninterrupted( + multiprocess_output: Path, + multiprocess_resumed_output: Path, +): + _assert_equivalent(multiprocess_output, multiprocess_resumed_output) + + +def test_resumed_single_and_multiprocess_are_equivalent( + single_resumed_output: Path, + multiprocess_resumed_output: Path, +): + _assert_equivalent(single_resumed_output, multiprocess_resumed_output) From 97fbf166d684c7f30f205ba9bb449922fddcd13a Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:58:21 -0700 Subject: [PATCH 83/96] nailing down resume_after and iteration counting plus some bug fixes including in RNG and odds ratio --- activitysim/core/calibration/README.md | 69 ++++++- activitysim/core/calibration/component.py | 97 +++++++--- activitysim/core/calibration/expressions.py | 20 +- activitysim/core/calibration/multiprocess.py | 15 +- activitysim/core/calibration/orchestrator.py | 182 +++++++++++++++--- activitysim/core/calibration/recovery.py | 10 + activitysim/core/calibration/reporting.py | 91 +++++++-- activitysim/core/test/test_calibration.py | 155 +++++++++++++++ .../core/test/test_calibration_reporting.py | 59 ++++++ .../prototype_mtc/test/calibration/README.md | 4 + .../test/calibration/test_run_modes.py | 150 ++++++++++++++- 11 files changed, 752 insertions(+), 100 deletions(-) create mode 100644 activitysim/core/test/test_calibration.py create mode 100644 activitysim/core/test/test_calibration_reporting.py diff --git a/activitysim/core/calibration/README.md b/activitysim/core/calibration/README.md index 6e7028f8c2..ae87db4ae0 100644 --- a/activitysim/core/calibration/README.md +++ b/activitysim/core/calibration/README.md @@ -25,11 +25,70 @@ Component settings may specify `model_settings_file` when a workflow step does not follow the conventional `.yaml` naming pattern. Components that share model settings may point to the same coefficient file. -When a run is interrupted, calibration continues the active global iteration. -Any coefficient updates written before the interruption, including subsequent -manual edits, are preserved. On the first resumed iteration, top-level -`settings.yaml` `resume_after` uses standard ActivitySim semantics: the named -model is treated as complete and execution begins with the following model. +## Global iterations, recovery attempts, and `resume_after` + +`calibration.yaml` `run.global_iterations` is the maximum total number of +logical global calibration iterations, not the number to execute on each +ActivitySim invocation. Calibration records durable progress in +`output/calibration/calibration_progress.json`: + +- a new output directory starts at global iteration 1; +- a crash re-enters the interrupted global iteration; +- a cleanly completed iteration advances to the next global iteration; +- convergence can finish the run before the configured maximum; and +- after a completed run, increasing `global_iterations` continues at the next + unfinished global iteration. An unchanged or lower value remains a no-op. + +Recovery attempts are distinct from global iterations. The first execution of a +global iteration is attempt 1. Restarting an interrupted global iteration creates +attempt 2, then attempt 3 if another restart is needed. A recovery attempt does +not consume an additional global iteration. + +Calibration uses the top-level `settings.yaml` `resume_after` setting. It has +strict ActivitySim semantics: the named model is treated as complete, its +checkpoint is restored, and execution begins with the following model. The name +must occur in the top-level `models` list and must be a model-level checkpoint; +the `_` shorthand for the last checkpoint is not accepted in calibration mode. +The similarly named `calibration.yaml` `run.resume_after` compatibility field is +not used by the orchestrator. + +On the first global iteration entered by an invocation, calibrated models at or +before `resume_after` are skipped. Later global iterations in the same invocation +restart immediately before the first calibrated model and run the normal complete +calibration sequence. For example, if global iteration 3 was interrupted after +calibrated `model_a` completed: + +- `resume_after: model_a` preserves model A's attempt-1 result and resumes with + the following model under attempt 2; +- `resume_after: initialize` rewinds pipeline state and reruns model A under + attempt 2; and +- no `resume_after` replays the interrupted iteration from the beginning under + attempt 2. + +Coefficient files are the authoritative current state and are never rolled back +during recovery. Updates written before the interruption, along with subsequent +manual edits, become the starting coefficients for the new attempt. Rewinding +across a completed calibrated model therefore applies another calibration update +to its current coefficients. Calibration logs a warning when `resume_after` +causes this kind of rewind. + +Iteration histories are append-only across attempts. Standard record, summary, +and generic-report rows include `global_iter`, `attempt`, and `component_iter`, +so a rerun does not replace the coefficient transition written by an earlier +attempt. The coefficient trajectory plots show the complete sequence, including +the initial value and every subsequent update. X-axis labels use the compact +form `G-A-C`, for example `G3-A2-C1`. + +After each calibrated component completes, the progress file records its attempt, +component-iteration count, and convergence result. A restart that skips that +component retains this state. This allows a run that crashed in downstream, +non-calibrated models to finish the same terminal global iteration without losing +the calibrated components' convergence decision. + +Calibration is marked complete only after the terminal global iteration has run +all remaining models in the top-level `models` list and the final coefficient +snapshot has been written. A crash after the last calibrated model but before the +last production model therefore leaves the global iteration in progress. Dependencies should flow from `orchestrator.py` into the focused modules. Lower level modules should not import the orchestrator. Multiprocess code imports back diff --git a/activitysim/core/calibration/component.py b/activitysim/core/calibration/component.py index caa6b1d8d1..6683b670a6 100644 --- a/activitysim/core/calibration/component.py +++ b/activitysim/core/calibration/component.py @@ -50,12 +50,40 @@ ] +def _run_component_model( + state: workflow.State, + component_name: str, + run_model_name: str, + prior_step: str | None, + mp_restore_checkpoint: str | None, + shared_data_buffers: dict | None, +) -> None: + """Run one component simulation from its fixed pre-component state.""" + if state.settings.multiprocess and shared_data_buffers is not None: + _run_mp_single_component( + state, + component_name=component_name, + run_label=run_model_name.replace(";", "_").replace(".", "_"), + restore_checkpoint=mp_restore_checkpoint, + shared_data_buffers=shared_data_buffers, + ) + return + + extra_models = _prep_model_data(state, resume_after=prior_step) + if extra_models: + for model_name in extra_models: + state.run.by_name(model_name) + state.checkpoint.add(prior_step) + state.run.by_name(run_model_name) + + def _calibrate_component( state: workflow.State, component_name: str, component_settings: CalibrationComponentSettings, prior_step: str, global_iter: int, + attempt: int, shared_data_buffers: dict | None = None, ) -> CalibrationComponentResult: """Run iterative coefficient calibration for one component.""" @@ -116,41 +144,25 @@ def _calibrate_component( for component_iter in range(1, component_settings.submodel_max_iterations + 1): component_iterations = component_iter - run_model_name = f"{component_name}.c_i{component_iter};" f"g_i{global_iter}" - # Re-run only this component from its prior checkpoint so model values - # reflect the current candidate coefficients for this component. - if state.settings.multiprocess and shared_data_buffers is not None: - # Use direct MP orchestration with explicit checkpoint control. - # This ensures we always apportion from prior_step's state, - # even after multiple component iterations. - _run_mp_single_component( - state, - component_name=component_name, - # Always restore from the same immutable pre-component - # checkpoint. LAST_CHECKPOINT may point to the prior iteration's - # coalesced component output and is therefore not a safe baseline. - restore_checkpoint=mp_restore_checkpoint, - shared_data_buffers=shared_data_buffers, - ) - else: - # Restore to prior_step ourselves then run the model directly. - # state.run(resume_after=prior_step) would trigger - # checkpoint.restore → init_state which creates a fresh RNG. - # _prep_model_data also performs an exact calibration rewind, - # removing tables and RNG channels that do not exist at prior_step - # so normal model execution can recreate them at the right point. - extra_models = _prep_model_data(state, resume_after=prior_step) - if extra_models: - # prior_step checkpoint not found directly; run intermediate - # models (e.g. annotators) to recreate the correct state. - for m in extra_models: - state.run.by_name(m) - state.checkpoint.add(prior_step) - state.run.by_name(run_model_name) + run_model_name = ( + f"{component_name}.c_i{component_iter};" + f"g_i{global_iter};a_i{attempt}" + ) + _run_component_model( + state=state, + component_name=component_name, + run_model_name=run_model_name, + prior_step=prior_step, + mp_restore_checkpoint=mp_restore_checkpoint, + shared_data_buffers=shared_data_buffers, + ) eval_context = _build_expression_context( state, helper_symbols, component_name, component_settings ) + eval_context["calibration_global_iteration"] = global_iter + eval_context["calibration_attempt"] = attempt + eval_context["calibration_component_iteration"] = component_iter ( row_records, @@ -164,6 +176,7 @@ def _calibrate_component( eval_context=eval_context, global_iter=global_iter, component_iter=component_iter, + attempt=attempt, ) coefficients_df = new_coefficients_df @@ -181,6 +194,21 @@ def _calibrate_component( if component_converged: break + if component_iter == component_settings.submodel_max_iterations: + # The update just persisted above has not yet been simulated. Run + # the component once more so final pipeline tables and downstream + # models use the coefficient values left in the config file. + _run_component_model( + state=state, + component_name=component_name, + run_model_name=( + f"{component_name}.c_final;g_i{global_iter};a_i{attempt}" + ), + prior_step=prior_step, + mp_restore_checkpoint=mp_restore_checkpoint, + shared_data_buffers=shared_data_buffers, + ) + state.checkpoint.add(component_name) return CalibrationComponentResult( @@ -371,6 +399,7 @@ def _evaluate_and_update( eval_context: dict[str, Any], global_iter: int, component_iter: int, + attempt: int = 1, ) -> tuple[list[dict[str, Any]], dict[str, Any], pd.DataFrame, bool]: """Evaluate spec rows, update coefficients, and return detailed records.""" updated = coefficients_df.copy() @@ -429,7 +458,9 @@ def _evaluate_and_update( default_increment=default_increment, ) - candidate_value = prev_value if hold_fast else prev_value + raw_delta + candidate_value = ( + prev_value if hold_fast or converged else prev_value + raw_delta + ) at_min = False at_max = False @@ -468,6 +499,7 @@ def _evaluate_and_update( records.append( { "global_iter": global_iter, + "attempt": attempt, "component_iter": component_iter, "description": description, "component": component_name, @@ -492,6 +524,7 @@ def _evaluate_and_update( summary_record = { "global_iter": global_iter, + "attempt": attempt, "component_iter": component_iter, "component": component_name, "max_difference": max_difference if max_difference != -math.inf else 0.0, diff --git a/activitysim/core/calibration/expressions.py b/activitysim/core/calibration/expressions.py index 72263c9c96..e65b0c194e 100644 --- a/activitysim/core/calibration/expressions.py +++ b/activitysim/core/calibration/expressions.py @@ -107,7 +107,9 @@ def _compute_delta( if method == "log_ratio": if model_value <= 0 or target_value <= 0: logger.warning( - f"log_ratio requires positive model and target values for {component_name} / {description}. Falling back to default increment {default_increment}" + f"log_ratio requires positive model and target values for " + f"{component_name} / {description}. Falling back to default " + f"increment {default_increment}" ) if model_value <= 0 and target_value > 0: return default_increment @@ -118,22 +120,20 @@ def _compute_delta( delta = math.log(target_value / model_value) * damping elif method == "odds_ratio": - # Formula requested by the calibration outline. - numerator = (target_value * model_value) - target_value - denominator = (target_value * model_value) - model_value - - if numerator <= 0 or denominator <= 0: + if not (0 < model_value < 1 and 0 < target_value < 1): logger.warning( - f"odds_ratio produced invalid numerator/denominator for {component_name} / {description}. Falling back to default increment {default_increment}" + f"odds_ratio requires model and target values strictly between " + f"zero and one for {component_name} / {description}. Falling " + f"back to default increment {default_increment}" ) - if model_value <= 0 and target_value > 0: + if target_value > model_value: return default_increment - elif model_value > 0 and target_value <= 0: + elif target_value < model_value: return -default_increment else: return 0 - ratio = numerator / denominator + ratio = (target_value * (1 - model_value)) / (model_value * (1 - target_value)) if ratio <= 0 or not np.isfinite(ratio): raise RuntimeError( f"odds_ratio produced invalid ratio for {component_name} / {description}" diff --git a/activitysim/core/calibration/multiprocess.py b/activitysim/core/calibration/multiprocess.py index 9408787ffb..6815ae604b 100644 --- a/activitysim/core/calibration/multiprocess.py +++ b/activitysim/core/calibration/multiprocess.py @@ -31,6 +31,7 @@ def _run_mp_single_component( state: workflow.State, component_name: str, + run_label: str, restore_checkpoint: str, shared_data_buffers: dict, ) -> None: @@ -47,6 +48,10 @@ def _run_mp_single_component( state : workflow.State component_name : str The model component to run. + run_label : str + Unique name for this calibration execution's subprocess pipelines and + checkpoint. It must differ from ``component_name`` so restoring the + apportioned checkpoint does not cause ActivitySim to skip the model. restore_checkpoint : str The checkpoint name to restore from before apportioning. This should be the checkpoint representing prior_step's state. @@ -84,7 +89,7 @@ def _run_mp_single_component( # Build step_info dict matching what mp_tasks functions expect step_info = { - "name": component_name, + "name": run_label, "models": [component_name], "num_processes": num_processes, "chunk_size": chunk_size, @@ -96,9 +101,9 @@ def _run_mp_single_component( injectables = _build_calibration_injectables(state) if num_processes == 1: - sub_proc_names = [component_name] + sub_proc_names = [run_label] else: - sub_proc_names = [f"{component_name}_{i}" for i in range(num_processes)] + sub_proc_names = [f"{run_label}_{i}" for i in range(num_processes)] fail_fast = state.settings.fail_fast @@ -108,7 +113,7 @@ def _run_mp_single_component( state, multiprocessing.Process( target=mp_tasks.mp_apportion_pipeline, - name=f"{component_name}_apportion", + name=f"{run_label}_apportion", args=(injectables, sub_proc_names, step_info), ), ) @@ -149,7 +154,7 @@ def _run_mp_single_component( state, multiprocessing.Process( target=mp_tasks.mp_coalesce_pipelines, - name=f"{component_name}_coalesce", + name=f"{run_label}_coalesce", args=(injectables, sub_proc_names, slice_info), ), ) diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py index facfa88eea..378b27702b 100644 --- a/activitysim/core/calibration/orchestrator.py +++ b/activitysim/core/calibration/orchestrator.py @@ -108,36 +108,62 @@ def run_calibration_loop( progress = _read_progress(state) if progress and progress.get("complete"): - if resume_after is not None: - raise RuntimeError( - f"settings.yaml resume_after={resume_after!r} cannot be honored " - "because calibration progress is already complete. Remove " - f"{CALIBRATION_PROGRESS_FILE} or use a new output directory to " - "start a new calibration run; current coefficient values will " - "be preserved." + completed_global_iterations = int( + progress.get("last_completed_global_iteration", 0) + ) + completed_for_global_iterations = int( + progress.get( + "configured_global_iterations", completed_global_iterations ) - logger.info( - "calibration progress is already complete; remove %s to start a " - "fresh calibration run", - CALIBRATION_PROGRESS_FILE, ) - return CalibrationRunResult( - converged=bool(progress.get("converged", False)), - completed_global_iterations=int( - progress.get( - "last_completed_global_iteration", - calibration_settings.run.global_iterations, + if calibration_settings.run.global_iterations > completed_for_global_iterations: + logger.info( + "calibration global_iterations increased from %s to %s; " + "continuing with global iteration %s", + completed_for_global_iterations, + calibration_settings.run.global_iterations, + completed_global_iterations + 1, + ) + progress = { + "complete": False, + "in_progress_iteration": None, + "next_global_iteration": completed_global_iterations + 1, + "last_completed_global_iteration": completed_global_iterations, + "converged": bool(progress.get("converged", False)), + "configured_global_iterations": calibration_settings.run.global_iterations, + "attempt": 0, + "completed_components": {}, + } + _write_progress(state, progress) + else: + if resume_after is not None: + raise RuntimeError( + f"settings.yaml resume_after={resume_after!r} cannot be honored " + "because calibration progress is already complete. Remove " + f"{CALIBRATION_PROGRESS_FILE} or use a new output directory to " + "start a new calibration run; current coefficient values will " + "be preserved." ) - ), - ) + logger.info( + "calibration progress is already complete; remove %s to start a " + "fresh calibration run", + CALIBRATION_PROGRESS_FILE, + ) + return CalibrationRunResult( + converged=bool(progress.get("converged", False)), + completed_global_iterations=completed_global_iterations, + ) interrupted_iteration = progress.get("in_progress_iteration") if progress else None if interrupted_iteration is not None: start_global_iter = int(interrupted_iteration) + start_attempt = int(progress.get("attempt", 1)) + 1 + start_completed_components = dict(progress.get("completed_components", {})) logger.warning( - "continuing interrupted calibration global iteration %s using the " - "current coefficient files", + "continuing interrupted calibration global iteration %s as attempt %s " + "using the current coefficient files", start_global_iter, + start_attempt, ) else: # Progress files from earlier versions contain next_global_iteration, so @@ -145,8 +171,28 @@ def run_calibration_loop( start_global_iter = ( int(progress.get("next_global_iteration", 1)) if progress else 1 ) + start_attempt = 1 + start_completed_components = {} completed_global_iterations = start_global_iter - 1 + if interrupted_iteration is not None and resume_after is not None: + rerun_completed_components = [ + component + for component in start_completed_components + if component in calibration_settings.run.calibrate_models + and models.index(component) > models.index(resume_after) + ] + if rerun_completed_components: + logger.warning( + "resume_after=%r rewinds across completed calibrated component(s) " + "%s. Their coefficient values will not be rolled back; rerun " + "results will be appended as attempt %s of global iteration %s.", + resume_after, + rerun_completed_components, + start_attempt, + start_global_iter, + ) + if start_global_iter > calibration_settings.run.global_iterations: logger.info( "calibration progress already reached configured global_iterations=%s", @@ -158,6 +204,11 @@ def run_calibration_loop( state, completed_global_iterations, converged, + calibration_settings.run.global_iterations, + attempt=int(progress.get("attempt", 1)) if progress else 1, + completed_components=( + dict(progress.get("completed_components", {})) if progress else {} + ), ) return CalibrationRunResult( converged=converged, @@ -241,9 +292,17 @@ def run_calibration_loop( start_global_iter, calibration_settings.run.global_iterations + 1, ): + attempt = start_attempt if global_iter == start_global_iter else 1 + completed_components = ( + dict(start_completed_components) + if global_iter == start_global_iter + else {} + ) _mark_global_iteration_in_progress( state, global_iter, + attempt, + completed_components, ) # Every global iteration after the first begins from the immutable @@ -271,24 +330,36 @@ def run_calibration_loop( state.checkpoint.close_store() logger.info( - "calibration global iteration %s/%s", + "calibration global iteration %s/%s attempt %s", global_iter, calibration_settings.run.global_iterations, + attempt, ) - # suppress early termination on first iteration if resume_after is after all calibrated models - all_converged = ( - first_model_idx is not None and first_model_idx <= last_calib_model_idx - ) or global_iter > start_global_iter + skipped_components = _skipped_calibration_components( + calibration_models=calibration_settings.run.calibrate_models, + models=models, + first_model_idx=first_model_idx, + global_iter=global_iter, + start_global_iter=start_global_iter, + ) + if skipped_components: + all_converged = all( + bool(completed_components.get(component, {}).get("converged")) + for component in skipped_components + ) + else: + all_converged = _components_ran_for_convergence( + first_model_idx=first_model_idx, + last_calib_model_idx=last_calib_model_idx, + global_iter=global_iter, + start_global_iter=start_global_iter, + ) last_calibrated_component = None for component in calibration_settings.run.calibrate_models: # on the first global iter, skip model if it's before or == resume_after - if ( - global_iter == start_global_iter - and first_model_idx is not None - and first_model_idx > models.index(component) - ): + if component in skipped_components: continue component_settings = calibration_settings.model_settings[component] @@ -313,12 +384,25 @@ def run_calibration_loop( component_settings=component_settings, prior_step=prior_step, global_iter=global_iter, + attempt=attempt, shared_data_buffers=shared_data_buffers, ) _write_component_plots(state, component) all_converged = all_converged and component_result.converged + completed_components[component] = { + "attempt": attempt, + "converged": component_result.converged, + "component_iterations": component_result.component_iterations, + } + _mark_global_iteration_in_progress( + state, + global_iter, + attempt, + completed_components, + ) + last_calibrated_component = component iteration_is_complete = ( @@ -360,6 +444,8 @@ def run_calibration_loop( "next_global_iteration": global_iter + 1, "last_completed_global_iteration": global_iter, "converged": all_converged, + "attempt": 0, + "completed_components": {}, }, ) @@ -376,6 +462,9 @@ def run_calibration_loop( state, completed_global_iterations, all_converged, + calibration_settings.run.global_iterations, + attempt=attempt, + completed_components=completed_components, ) return CalibrationRunResult( @@ -450,3 +539,34 @@ def _prior_step_name(models: list[str], component_name: str) -> str | None: if idx == 0: return None return models[idx - 1] + + +def _components_ran_for_convergence( + first_model_idx: int | None, + last_calib_model_idx: int, + global_iter: int, + start_global_iter: int, +) -> bool: + """Return whether component results can establish convergence this iteration.""" + return ( + first_model_idx is None + or first_model_idx <= last_calib_model_idx + or global_iter > start_global_iter + ) + + +def _skipped_calibration_components( + calibration_models: list[str], + models: list[str], + first_model_idx: int | None, + global_iter: int, + start_global_iter: int, +) -> list[str]: + """Return calibrated components skipped by resume_after this iteration.""" + if global_iter != start_global_iter or first_model_idx is None: + return [] + return [ + component + for component in calibration_models + if first_model_idx > models.index(component) + ] diff --git a/activitysim/core/calibration/recovery.py b/activitysim/core/calibration/recovery.py index c7a4dc28dd..7abd5f5421 100644 --- a/activitysim/core/calibration/recovery.py +++ b/activitysim/core/calibration/recovery.py @@ -14,6 +14,8 @@ def _mark_global_iteration_in_progress( state: workflow.State, global_iteration: int, + attempt: int, + completed_components: dict[str, dict[str, Any]] | None = None, ) -> None: """Durably mark a global iteration in progress.""" _write_progress( @@ -22,6 +24,8 @@ def _mark_global_iteration_in_progress( "in_progress_iteration": global_iteration, "next_global_iteration": global_iteration, "last_completed_global_iteration": global_iteration - 1, + "attempt": attempt, + "completed_components": completed_components or {}, }, ) @@ -49,6 +53,9 @@ def _write_completed_progress( state: workflow.State, completed_global_iterations: int, converged: bool, + configured_global_iterations: int, + attempt: int = 1, + completed_components: dict[str, dict[str, Any]] | None = None, ) -> None: """Mark calibration complete after all final output has been written.""" _write_progress( @@ -59,5 +66,8 @@ def _write_completed_progress( "next_global_iteration": completed_global_iterations + 1, "last_completed_global_iteration": completed_global_iterations, "converged": converged, + "configured_global_iterations": configured_global_iterations, + "attempt": attempt, + "completed_components": completed_components or {}, }, ) diff --git a/activitysim/core/calibration/reporting.py b/activitysim/core/calibration/reporting.py index a4fbd60010..6db1490b1d 100644 --- a/activitysim/core/calibration/reporting.py +++ b/activitysim/core/calibration/reporting.py @@ -39,7 +39,13 @@ def _append_iteration_records( _append_csv( df, global_path, - unique_on=["global_iter", "component_iter", "component", "coefficient"], + unique_on=[ + "global_iter", + "attempt", + "component_iter", + "component", + "coefficient", + ], ) # Also write component-local iteration history @@ -50,7 +56,13 @@ def _append_iteration_records( _append_csv( df, component_path, - unique_on=["global_iter", "component_iter", "component", "coefficient"], + unique_on=[ + "global_iter", + "attempt", + "component_iter", + "component", + "coefficient", + ], ) @@ -65,7 +77,7 @@ def _append_summary_records( _append_csv( df, path, - unique_on=["global_iter", "component_iter", "component"], + unique_on=["global_iter", "attempt", "component_iter", "component"], ) @@ -76,6 +88,10 @@ def _append_csv( os.makedirs(path.parent, exist_ok=True) if unique_on and path.exists(): existing = pd.read_csv(path) + if "attempt" in unique_on and "attempt" not in existing.columns: + # Histories created before recovery attempts were introduced belong + # to the first attempt of their logical global iteration. + existing["attempt"] = 1 df = pd.concat([existing, df], ignore_index=True).drop_duplicates( subset=unique_on, keep="last" ) @@ -125,9 +141,12 @@ def _read_component_iteration_records( iteration_records = ( pd.read_csv(path) - .set_index(["global_iter", "component_iter", "coefficient"]) - .sort_index() ) + if "attempt" not in iteration_records.columns: + iteration_records["attempt"] = 1 + iteration_records = iteration_records.set_index( + ["global_iter", "attempt", "component_iter", "coefficient"] + ).sort_index() return iteration_records.loc[iteration_records.component == component_name] @@ -140,12 +159,11 @@ def _plot_coefficient_progress( ) -> None: """Plot coefficient value progression for one coefficient subset.""" component_dir = _component_output_dir(state, component_name) - ax = ( - recs[recs.index.get_level_values("coefficient").isin(set_coefs)] - .next_coefficient.unstack("coefficient") - .plot(figsize=(10, 5)) - ) - ax.xaxis.set_label_text("Component iteration") + trajectory, step_labels = _coefficient_trajectory(recs, set_coefs) + ax = trajectory.plot(figsize=(10, 5)) + ax.set_xticks(range(len(step_labels))) + ax.set_xticklabels(step_labels, rotation=45, ha="right") + ax.xaxis.set_label_text("Calibration update (global-attempt-component)") ax.yaxis.set_label_text("Coefficient value") ax.legend(title="Coefficient label", loc="center left", bbox_to_anchor=(1.02, 0.5)) plt.tight_layout() @@ -156,13 +174,47 @@ def _plot_coefficient_progress( plt.close(ax.figure) +def _coefficient_trajectory( + recs: pd.DataFrame, + set_coefs: list[str], +) -> tuple[pd.DataFrame, list[str]]: + """Build the complete ordered coefficient path and compact step labels.""" + filtered = recs[ + recs.index.get_level_values("coefficient").isin(set_coefs) + ].reset_index() + history = filtered.pivot( + index=["global_iter", "attempt", "component_iter"], + columns="coefficient", + values="next_coefficient", + ).sort_index() + initial_values = ( + filtered.sort_values(["global_iter", "attempt", "component_iter"]) + .groupby("coefficient", sort=False) + .first()["prev_coefficient"] + .reindex(history.columns) + ) + trajectory = pd.concat( + [pd.DataFrame([initial_values], index=["Start"]), history.reset_index(drop=True)] + ) + step_labels = ["Start"] + [ + f"G{global_iter}-A{attempt}-C{component_iter}" + for global_iter, attempt, component_iter in history.index + ] + return trajectory, step_labels + + def _component_last_records(recs: pd.DataFrame, set_coefs: list[str]) -> pd.DataFrame: """Select target/model values for the latest iteration and coefficient subset.""" filtered = recs[recs.index.get_level_values("coefficient").isin(set_coefs)] last_global = filtered.index.get_level_values("global_iter")[-1] - last_comp = filtered.loc[last_global].index.get_level_values("component_iter")[-1] + last_attempt = filtered.loc[last_global].index.get_level_values("attempt")[-1] + last_comp = ( + filtered.loc[(last_global, last_attempt)] + .index.get_level_values("component_iter")[-1] + ) return filtered.xs( - (last_global, last_comp), level=("global_iter", "component_iter") + (last_global, last_attempt, last_comp), + level=("global_iter", "attempt", "component_iter"), )[["target_value", "model_value"]] @@ -215,6 +267,7 @@ def _write_generic_report( df[ [ "global_iter", + "attempt", "component_iter", "component", "description", @@ -224,14 +277,22 @@ def _write_generic_report( ] ] .copy() - .sort_values(["global_iter", "component_iter", "description"]) + .sort_values( + ["global_iter", "attempt", "component_iter", "description"] + ) ) path = _component_output_dir(state, component_name) / "generic_report.csv" _append_csv( report, path, - unique_on=["global_iter", "component_iter", "component", "description"], + unique_on=[ + "global_iter", + "attempt", + "component_iter", + "component", + "description", + ], ) diff --git a/activitysim/core/test/test_calibration.py b/activitysim/core/test/test_calibration.py new file mode 100644 index 0000000000..e8b50bc37a --- /dev/null +++ b/activitysim/core/test/test_calibration.py @@ -0,0 +1,155 @@ +from __future__ import annotations + +import math + +import pandas as pd +import pytest + +from activitysim.core.calibration.component import _evaluate_and_update +from activitysim.core.calibration.expressions import _compute_delta +from activitysim.core.calibration.orchestrator import ( + _components_ran_for_convergence, +) + + +@pytest.mark.parametrize( + ("method", "model_value", "target_value", "damping", "expected"), + [ + ("log_ratio", 0.25, 0.5, 0.5, math.log(2) * 0.5), + ( + "odds_ratio", + 0.4, + 0.6, + 0.5, + math.log((0.6 * (1 - 0.4)) / (0.4 * (1 - 0.6))) * 0.5, + ), + ], +) +def test_compute_delta(method, model_value, target_value, damping, expected): + delta = _compute_delta( + method=method, + model_value=model_value, + target_value=target_value, + damping=damping, + component_name="test_component", + description="test target", + default_increment=2.0, + ) + + assert delta == pytest.approx(expected) + + +@pytest.mark.parametrize( + ("model_value", "target_value", "expected"), + [(0.0, 0.5, 2.0), (1.0, 0.5, -2.0), (0.0, 0.0, 0.0)], +) +def test_odds_ratio_boundary_fallback_has_correct_direction( + model_value, target_value, expected +): + delta = _compute_delta( + method="odds_ratio", + model_value=model_value, + target_value=target_value, + damping=1.0, + component_name="test_component", + description="test target", + default_increment=2.0, + ) + + assert delta == expected + + +def test_evaluate_and_update_only_changes_eligible_coefficients(): + calibration_spec = pd.DataFrame( + [ + { + "description": "unconverged", + "coefficient": "coef_update", + "model_value": 0.25, + "target_value": 0.5, + "hold_fast": False, + "min": -10.0, + "max": 10.0, + "damping": 0.5, + "method": "log_ratio", + "tolerance": 0.01, + }, + { + "description": "within tolerance", + "coefficient": "coef_converged", + "model_value": 0.49, + "target_value": 0.5, + "hold_fast": False, + "min": -10.0, + "max": 10.0, + "damping": 1.0, + "method": "log_ratio", + "tolerance": 0.02, + }, + { + "description": "held fixed", + "coefficient": "coef_held", + "model_value": 0.25, + "target_value": 0.5, + "hold_fast": True, + "min": -10.0, + "max": 10.0, + "damping": 1.0, + "method": "log_ratio", + "tolerance": 0.01, + }, + ] + ) + coefficients = pd.DataFrame( + {"value": [1.0, 2.0, 3.0]}, + index=["coef_update", "coef_converged", "coef_held"], + ) + + records, _, updated, component_converged = _evaluate_and_update( + component_name="test_component", + calibration_spec_df=calibration_spec, + coefficients_df=coefficients, + eval_context={}, + global_iter=1, + component_iter=1, + ) + + assert updated.loc["coef_update", "value"] == pytest.approx(1.0 + math.log(2) * 0.5) + assert updated.loc["coef_converged", "value"] == 2.0 + assert updated.loc["coef_held", "value"] == 3.0 + assert records[1]["coef_delta"] == 0.0 + assert records[2]["coef_delta"] == 0.0 + assert component_converged is False + + +@pytest.mark.parametrize( + ( + "first_model_idx", + "last_calib_model_idx", + "global_iter", + "start_global_iter", + "expected", + ), + [ + (None, 10, 1, 1, True), + (5, 10, 1, 1, True), + (11, 10, 1, 1, False), + (11, 10, 2, 1, True), + ], +) +def test_components_ran_for_convergence( + first_model_idx, + last_calib_model_idx, + global_iter, + start_global_iter, + expected, +): + assert ( + _components_ran_for_convergence( + first_model_idx=first_model_idx, + last_calib_model_idx=last_calib_model_idx, + global_iter=global_iter, + start_global_iter=start_global_iter, + ) + is expected + ) diff --git a/activitysim/core/test/test_calibration_reporting.py b/activitysim/core/test/test_calibration_reporting.py new file mode 100644 index 0000000000..c1e4c684c7 --- /dev/null +++ b/activitysim/core/test/test_calibration_reporting.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from pathlib import Path + +import pandas as pd + +from activitysim.core.calibration.reporting import ( + _append_iteration_records, + _coefficient_trajectory, + _read_component_iteration_records, +) + + +class _State: + def __init__(self, output_dir: Path): + self.output_dir = output_dir + + def get_output_file_path(self, file_name: str) -> Path: + return self.output_dir / file_name + + +def _record(attempt: int, previous: float, next_value: float) -> dict: + return { + "global_iter": 1, + "attempt": attempt, + "component_iter": 1, + "description": "test target", + "component": "model_a", + "coefficient": "coef_a", + "target_value": 0.5, + "model_value": 0.25, + "difference": 0.25, + "pct_difference": 50.0, + "hold_fast": False, + "prev_coefficient": previous, + "coef_delta": next_value - previous, + "next_coefficient": next_value, + "converged": False, + "at_min": False, + "at_max": False, + } + + +def test_recovery_attempts_preserve_complete_coefficient_trajectory(tmp_path): + state = _State(tmp_path) + _append_iteration_records(state, "model_a", [_record(1, 1.0, 1.5)]) + _append_iteration_records(state, "model_a", [_record(2, 1.5, 1.75)]) + + stored = pd.read_csv( + tmp_path / "calibration" / "calibration_iteration_records.csv" + ) + assert list(stored["attempt"]) == [1, 2] + assert stored.loc[1, "prev_coefficient"] == stored.loc[0, "next_coefficient"] + + records = _read_component_iteration_records(state, "model_a") + trajectory, labels = _coefficient_trajectory(records, ["coef_a"]) + + assert labels == ["Start", "G1-A1-C1", "G1-A2-C1"] + assert list(trajectory["coef_a"]) == [1.0, 1.5, 1.75] diff --git a/activitysim/examples/prototype_mtc/test/calibration/README.md b/activitysim/examples/prototype_mtc/test/calibration/README.md index f8288ef562..9054f5937e 100644 --- a/activitysim/examples/prototype_mtc/test/calibration/README.md +++ b/activitysim/examples/prototype_mtc/test/calibration/README.md @@ -12,6 +12,10 @@ that global iteration 1 remains in progress, replaces the expression, and restarts with `resume_after: non_mandatory_tour_scheduling`. The resumed runs must preserve the workplace and auto-ownership coefficient files and match the uninterrupted final tables and calibrated coefficients across both modes. +The fixture also rewinds a failed run to `initialize_landuse`, verifies that the +completed calibrated models rerun as attempt 2 of the same global iteration, +and confirms that attempt-1 and attempt-2 coefficient transitions are both +retained with a continuous `next_coefficient` to `prev_coefficient` chain. Each run copies mutable coefficient files into its temporary config directory; the shared prototype MTC configs and data remain unchanged. The 25-household diff --git a/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py b/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py index 607efdcd24..f48fb9ca2a 100644 --- a/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py +++ b/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py @@ -81,7 +81,10 @@ def _run(configs_dir: Path, output_dir: Path, multiprocess: bool): return subprocess.run(args, check=False, capture_output=True, text=True) -def _resume(configs_dir: Path): +def _resume( + configs_dir: Path, + resume_after: str = "non_mandatory_tour_scheduling", +): shutil.copyfile( HERE / "configs" / "tour_mode_choice_calibration.csv", configs_dir / "tour_mode_choice_calibration.csv", @@ -89,11 +92,20 @@ def _resume(configs_dir: Path): settings_path = configs_dir / "settings.yaml" with open(settings_path, encoding="utf-8") as stream: settings = yaml.safe_load(stream) - settings["resume_after"] = "non_mandatory_tour_scheduling" + settings["resume_after"] = resume_after with open(settings_path, "w", encoding="utf-8") as stream: yaml.safe_dump(settings, stream, sort_keys=False) +def _set_global_iterations(configs_dir: Path, global_iterations: int): + calibration_path = configs_dir / "calibration.yaml" + with open(calibration_path, encoding="utf-8") as stream: + calibration = yaml.safe_load(stream) + calibration["run"]["global_iterations"] = global_iterations + with open(calibration_path, "w", encoding="utf-8") as stream: + yaml.safe_dump(calibration, stream, sort_keys=False) + + def _assert_success(result: subprocess.CompletedProcess): assert result.returncode == 0, result.stdout + result.stderr @@ -149,6 +161,89 @@ def _run_resumed(root: Path, name: str, multiprocess: bool) -> Path: return output +def _run_with_increased_global_iterations( + root: Path, name: str, completed_output: Path, multiprocess: bool +) -> Path: + run_dir = root / name + configs = run_dir / "configs" + output = run_dir / "output" + shutil.copytree(completed_output.parent / "configs", configs) + shutil.copytree(completed_output, output) + + _set_global_iterations(configs, 2) + _assert_success(_run(configs, output, multiprocess)) + + with open( + output / "calibration" / "calibration_progress.json", encoding="utf-8" + ) as stream: + progress = json.load(stream) + assert progress["complete"] is True + assert progress["last_completed_global_iteration"] == 2 + assert progress["configured_global_iterations"] == 2 + + records = pd.read_csv( + output / "calibration" / "calibration_iteration_records.csv" + ) + assert set(records["global_iter"]) == {1, 2} + return output + + +def _run_rewound_attempt(root: Path, name: str, multiprocess: bool) -> Path: + configs, output = _prepare_run(root, name, multiprocess, failing=True) + failed = _run(configs, output, multiprocess) + assert failed.returncode != 0 + + progress_path = output / "calibration" / "calibration_progress.json" + with open(progress_path, encoding="utf-8") as stream: + progress = json.load(stream) + assert progress["in_progress_iteration"] == 1 + assert progress["attempt"] == 1 + assert set(progress["completed_components"]) == { + "workplace_location", + "auto_ownership_simulate", + } + + records_path = output / "calibration" / "calibration_iteration_records.csv" + first_attempt = pd.read_csv(records_path) + workplace_first = first_attempt[ + first_attempt["component"] == "workplace_location" + ] + assert set(workplace_first["attempt"]) == {1} + + _resume(configs, resume_after="initialize_landuse") + _assert_success(_run(configs, output, multiprocess)) + + records = pd.read_csv(records_path) + workplace = records[records["component"] == "workplace_location"].sort_values( + ["global_iter", "attempt", "component_iter"] + ) + assert set(workplace["global_iter"]) == {1} + assert set(workplace["attempt"]) == {1, 2} + + attempt_1 = workplace[workplace["attempt"] == 1].iloc[-1] + attempt_2 = workplace[workplace["attempt"] == 2].iloc[0] + assert attempt_2["prev_coefficient"] == pytest.approx( + attempt_1["next_coefficient"] + ) + + assert ( + output + / "calibration" + / "workplace_location" + / "coefficient_progress_set_0.png" + ).exists() + + with open(progress_path, encoding="utf-8") as stream: + progress = json.load(stream) + assert progress["complete"] is True + assert progress["attempt"] == 2 + assert all( + component["attempt"] == 2 + for component in progress["completed_components"].values() + ) + return output + + @pytest.fixture(scope="module") def run_root(tmp_path_factory) -> Path: return tmp_path_factory.mktemp("calibration_run_modes") @@ -174,6 +269,40 @@ def multiprocess_resumed_output(run_root: Path) -> Path: return _run_resumed(run_root, "multiprocess_resumed", multiprocess=True) +@pytest.fixture(scope="module") +def single_increased_iterations_output( + run_root: Path, single_output: Path +) -> Path: + return _run_with_increased_global_iterations( + run_root, + "single_increased_iterations", + completed_output=single_output, + multiprocess=False, + ) + + +@pytest.fixture(scope="module") +def multiprocess_increased_iterations_output( + run_root: Path, multiprocess_output: Path +) -> Path: + return _run_with_increased_global_iterations( + run_root, + "multiprocess_increased_iterations", + completed_output=multiprocess_output, + multiprocess=True, + ) + + +@pytest.fixture(scope="module") +def single_rewound_output(run_root: Path) -> Path: + return _run_rewound_attempt(run_root, "single_rewound", multiprocess=False) + + +@pytest.fixture(scope="module") +def multiprocess_rewound_output(run_root: Path) -> Path: + return _run_rewound_attempt(run_root, "multiprocess_rewound", multiprocess=True) + + def test_single_and_multiprocess_are_equivalent( single_output: Path, multiprocess_output: Path, @@ -200,3 +329,20 @@ def test_resumed_single_and_multiprocess_are_equivalent( multiprocess_resumed_output: Path, ): _assert_equivalent(single_resumed_output, multiprocess_resumed_output) + + +def test_increased_global_iterations_continue_completed_run_in_all_modes( + single_increased_iterations_output: Path, + multiprocess_increased_iterations_output: Path, +): + _assert_equivalent( + single_increased_iterations_output, + multiprocess_increased_iterations_output, + ) + + +def test_rewinding_completed_calibrated_model_appends_attempt_in_all_modes( + single_rewound_output: Path, + multiprocess_rewound_output: Path, +): + _assert_equivalent(single_rewound_output, multiprocess_rewound_output) From 910a0e004e14529c252ad53d26bc013ef9d8b1de Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:59:03 -0700 Subject: [PATCH 84/96] blacken --- activitysim/core/calibration/component.py | 3 +-- activitysim/core/calibration/orchestrator.py | 4 +--- activitysim/core/calibration/reporting.py | 20 ++++++++---------- .../core/test/test_calibration_reporting.py | 4 +--- .../test/calibration/test_run_modes.py | 21 +++++-------------- 5 files changed, 17 insertions(+), 35 deletions(-) diff --git a/activitysim/core/calibration/component.py b/activitysim/core/calibration/component.py index 6683b670a6..1f470c5fcd 100644 --- a/activitysim/core/calibration/component.py +++ b/activitysim/core/calibration/component.py @@ -145,8 +145,7 @@ def _calibrate_component( for component_iter in range(1, component_settings.submodel_max_iterations + 1): component_iterations = component_iter run_model_name = ( - f"{component_name}.c_i{component_iter};" - f"g_i{global_iter};a_i{attempt}" + f"{component_name}.c_i{component_iter};" f"g_i{global_iter};a_i{attempt}" ) _run_component_model( state=state, diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py index 378b27702b..c8281d27c2 100644 --- a/activitysim/core/calibration/orchestrator.py +++ b/activitysim/core/calibration/orchestrator.py @@ -112,9 +112,7 @@ def run_calibration_loop( progress.get("last_completed_global_iteration", 0) ) completed_for_global_iterations = int( - progress.get( - "configured_global_iterations", completed_global_iterations - ) + progress.get("configured_global_iterations", completed_global_iterations) ) if calibration_settings.run.global_iterations > completed_for_global_iterations: logger.info( diff --git a/activitysim/core/calibration/reporting.py b/activitysim/core/calibration/reporting.py index 6db1490b1d..c4dee4ca0f 100644 --- a/activitysim/core/calibration/reporting.py +++ b/activitysim/core/calibration/reporting.py @@ -139,9 +139,7 @@ def _read_component_iteration_records( if not path.exists(): return None - iteration_records = ( - pd.read_csv(path) - ) + iteration_records = pd.read_csv(path) if "attempt" not in iteration_records.columns: iteration_records["attempt"] = 1 iteration_records = iteration_records.set_index( @@ -194,7 +192,10 @@ def _coefficient_trajectory( .reindex(history.columns) ) trajectory = pd.concat( - [pd.DataFrame([initial_values], index=["Start"]), history.reset_index(drop=True)] + [ + pd.DataFrame([initial_values], index=["Start"]), + history.reset_index(drop=True), + ] ) step_labels = ["Start"] + [ f"G{global_iter}-A{attempt}-C{component_iter}" @@ -208,10 +209,9 @@ def _component_last_records(recs: pd.DataFrame, set_coefs: list[str]) -> pd.Data filtered = recs[recs.index.get_level_values("coefficient").isin(set_coefs)] last_global = filtered.index.get_level_values("global_iter")[-1] last_attempt = filtered.loc[last_global].index.get_level_values("attempt")[-1] - last_comp = ( - filtered.loc[(last_global, last_attempt)] - .index.get_level_values("component_iter")[-1] - ) + last_comp = filtered.loc[(last_global, last_attempt)].index.get_level_values( + "component_iter" + )[-1] return filtered.xs( (last_global, last_attempt, last_comp), level=("global_iter", "attempt", "component_iter"), @@ -277,9 +277,7 @@ def _write_generic_report( ] ] .copy() - .sort_values( - ["global_iter", "attempt", "component_iter", "description"] - ) + .sort_values(["global_iter", "attempt", "component_iter", "description"]) ) path = _component_output_dir(state, component_name) / "generic_report.csv" diff --git a/activitysim/core/test/test_calibration_reporting.py b/activitysim/core/test/test_calibration_reporting.py index c1e4c684c7..d0d459cec2 100644 --- a/activitysim/core/test/test_calibration_reporting.py +++ b/activitysim/core/test/test_calibration_reporting.py @@ -46,9 +46,7 @@ def test_recovery_attempts_preserve_complete_coefficient_trajectory(tmp_path): _append_iteration_records(state, "model_a", [_record(1, 1.0, 1.5)]) _append_iteration_records(state, "model_a", [_record(2, 1.5, 1.75)]) - stored = pd.read_csv( - tmp_path / "calibration" / "calibration_iteration_records.csv" - ) + stored = pd.read_csv(tmp_path / "calibration" / "calibration_iteration_records.csv") assert list(stored["attempt"]) == [1, 2] assert stored.loc[1, "prev_coefficient"] == stored.loc[0, "next_coefficient"] diff --git a/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py b/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py index f48fb9ca2a..6daeb44ac7 100644 --- a/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py +++ b/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py @@ -181,9 +181,7 @@ def _run_with_increased_global_iterations( assert progress["last_completed_global_iteration"] == 2 assert progress["configured_global_iterations"] == 2 - records = pd.read_csv( - output / "calibration" / "calibration_iteration_records.csv" - ) + records = pd.read_csv(output / "calibration" / "calibration_iteration_records.csv") assert set(records["global_iter"]) == {1, 2} return output @@ -205,9 +203,7 @@ def _run_rewound_attempt(root: Path, name: str, multiprocess: bool) -> Path: records_path = output / "calibration" / "calibration_iteration_records.csv" first_attempt = pd.read_csv(records_path) - workplace_first = first_attempt[ - first_attempt["component"] == "workplace_location" - ] + workplace_first = first_attempt[first_attempt["component"] == "workplace_location"] assert set(workplace_first["attempt"]) == {1} _resume(configs, resume_after="initialize_landuse") @@ -222,15 +218,10 @@ def _run_rewound_attempt(root: Path, name: str, multiprocess: bool) -> Path: attempt_1 = workplace[workplace["attempt"] == 1].iloc[-1] attempt_2 = workplace[workplace["attempt"] == 2].iloc[0] - assert attempt_2["prev_coefficient"] == pytest.approx( - attempt_1["next_coefficient"] - ) + assert attempt_2["prev_coefficient"] == pytest.approx(attempt_1["next_coefficient"]) assert ( - output - / "calibration" - / "workplace_location" - / "coefficient_progress_set_0.png" + output / "calibration" / "workplace_location" / "coefficient_progress_set_0.png" ).exists() with open(progress_path, encoding="utf-8") as stream: @@ -270,9 +261,7 @@ def multiprocess_resumed_output(run_root: Path) -> Path: @pytest.fixture(scope="module") -def single_increased_iterations_output( - run_root: Path, single_output: Path -) -> Path: +def single_increased_iterations_output(run_root: Path, single_output: Path) -> Path: return _run_with_increased_global_iterations( run_root, "single_increased_iterations", From 2ef98285e3221d7db88875e59b5e8b53aa76a11b Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Mon, 24 Aug 2026 22:07:34 -0700 Subject: [PATCH 85/96] removing unused calibration.yaml settings --- activitysim/core/calibration/README.md | 2 -- activitysim/core/calibration/settings.py | 3 --- .../configs_calibration/calibration.yaml | 3 --- docs/users-guide/calibration.rst | 21 +++---------------- 4 files changed, 3 insertions(+), 26 deletions(-) diff --git a/activitysim/core/calibration/README.md b/activitysim/core/calibration/README.md index ae87db4ae0..f6c78cda00 100644 --- a/activitysim/core/calibration/README.md +++ b/activitysim/core/calibration/README.md @@ -49,8 +49,6 @@ strict ActivitySim semantics: the named model is treated as complete, its checkpoint is restored, and execution begins with the following model. The name must occur in the top-level `models` list and must be a model-level checkpoint; the `_` shorthand for the last checkpoint is not accepted in calibration mode. -The similarly named `calibration.yaml` `run.resume_after` compatibility field is -not used by the orchestrator. On the first global iteration entered by an invocation, calibrated models at or before `resume_after` are skipped. Later global iterations in the same invocation diff --git a/activitysim/core/calibration/settings.py b/activitysim/core/calibration/settings.py index dbd8accaab..83f8bdd200 100644 --- a/activitysim/core/calibration/settings.py +++ b/activitysim/core/calibration/settings.py @@ -3,7 +3,6 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Optional from pydantic import model_validator @@ -17,7 +16,6 @@ class CalibrationRunSettings(PydanticBase): """Run-control settings for calibration.""" - resume_after: Optional[str] = None calibrate_models: list[str] global_iterations: int = 1 complete_steps: bool = False @@ -49,7 +47,6 @@ class CalibrationComponentSettings(PydanticBase): helper_module: str | None = None submodel_max_iterations: int = 1 reports: CalibrationReportsSettings = CalibrationReportsSettings() - survey_file: Optional[str] = None class CalibrationConfig(PydanticReadable): diff --git a/activitysim/examples/prototype_mtc/configs_calibration/calibration.yaml b/activitysim/examples/prototype_mtc/configs_calibration/calibration.yaml index c4949c0718..61ac34a57d 100644 --- a/activitysim/examples/prototype_mtc/configs_calibration/calibration.yaml +++ b/activitysim/examples/prototype_mtc/configs_calibration/calibration.yaml @@ -14,7 +14,6 @@ model_settings: reports: generic: true bespoke: report_workplace_location - survey_file: survey_persons.csv auto_ownership_simulate: calibration_spec: auto_ownership_calibration.csv @@ -23,7 +22,6 @@ model_settings: reports: generic: true bespoke: report_auto_ownership - survey_file: survey_households.csv tour_mode_choice_simulate: calibration_spec: tour_mode_choice_calibration.csv @@ -32,5 +30,4 @@ model_settings: reports: generic: true bespoke: report_tour_mode_choice - survey_file: survey_tours.csv diff --git a/docs/users-guide/calibration.rst b/docs/users-guide/calibration.rst index 2d817f32fe..ac79a1af2f 100644 --- a/docs/users-guide/calibration.rst +++ b/docs/users-guide/calibration.rst @@ -58,7 +58,6 @@ calibration-specific overlay directory): - workplace_location - auto_ownership_simulate - tour_mode_choice_simulate - resume_after: null # checkpoint to resume from on global iteration 1 restart_after: [] # components after which to restart (advanced) global_iterations: 3 # number of full calibration passes complete_steps: false # run model steps after the last calibrated component @@ -68,7 +67,6 @@ calibration-specific overlay directory): calibration_spec: workplace_location_calibration.csv helper_module: workplace_location_calib_helper.py submodel_max_iterations: 3 - survey_file: survey_persons.csv reports: generic: true bespoke: report_workplace_location @@ -77,7 +75,6 @@ calibration-specific overlay directory): calibration_spec: auto_ownership_calibration.csv helper_module: auto_ownership_calib_helper.py submodel_max_iterations: 3 - survey_file: survey_households.csv reports: generic: true bespoke: report_auto_ownership @@ -86,7 +83,6 @@ calibration-specific overlay directory): calibration_spec: tour_mode_choice_calibration.csv helper_module: tour_mode_choice_calib_helper.py submodel_max_iterations: 3 - survey_file: survey_tours.csv reports: generic: true bespoke: report_tour_mode_choice @@ -158,12 +154,6 @@ Configuration Reference - *required* - Model component names to calibrate. Must match names in ``settings.yaml`` ``models`` list. - * - ``resume_after`` - - ``str`` or ``null`` - - ``null`` - - Checkpoint to resume from on the first global iteration. Equivalent to - ``resume_after`` in ``settings.yaml``. Use this to skip expensive - initialization steps that do not change across calibration iterations. * - ``restart_after`` - ``list[str]`` - ``[]`` @@ -208,11 +198,6 @@ Configuration Reference - ``1`` - Maximum number of inner-loop iterations per component per global iteration. The component re-runs from its prior checkpoint each iteration. - * - ``survey_file`` - - ``str`` - - *required* - - Survey data CSV filename. Made available via - ``component_settings.survey_file`` in the expression context. * - ``reports.generic`` - ``bool`` - ``True`` @@ -718,6 +703,6 @@ Tips (e.g., for trip generation downstream of mode choice). - **Version control your coefficient files** so you can diff changes and reset to initial values. -- **Use the** ``resume_after`` **setting** to skip expensive upstream steps (like - skims loading or accessibility computation) that don't change across - calibration iterations. +- **Use the top-level** ``settings.yaml`` ``resume_after`` **setting** to skip + expensive upstream steps (like skims loading or accessibility computation) + that don't change across calibration iterations. From cd8f8b58b14cb09e34c46442bc38f35cefc53614 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Tue, 25 Aug 2026 16:27:48 -0700 Subject: [PATCH 86/96] clarifying resume_after readme --- activitysim/core/calibration/README.md | 44 ++++++++++++++++---------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/activitysim/core/calibration/README.md b/activitysim/core/calibration/README.md index f6c78cda00..5863987b67 100644 --- a/activitysim/core/calibration/README.md +++ b/activitysim/core/calibration/README.md @@ -44,24 +44,36 @@ global iteration is attempt 1. Restarting an interrupted global iteration create attempt 2, then attempt 3 if another restart is needed. A recovery attempt does not consume an additional global iteration. -Calibration uses the top-level `settings.yaml` `resume_after` setting. It has -strict ActivitySim semantics: the named model is treated as complete, its -checkpoint is restored, and execution begins with the following model. The name -must occur in the top-level `models` list and must be a model-level checkpoint; -the `_` shorthand for the last checkpoint is not accepted in calibration mode. - -On the first global iteration entered by an invocation, calibrated models at or -before `resume_after` are skipped. Later global iterations in the same invocation -restart immediately before the first calibrated model and run the normal complete -calibration sequence. For example, if global iteration 3 was interrupted after +Calibration uses only the top-level `settings.yaml` `resume_after` setting. On +the first global iteration executed by an ActivitySim invocation, including a +new attempt of an interrupted iteration, it behaves like `resume_after` in a +non-calibration run: + +- when set to a model name, that model is treated as complete, its checkpoint is + restored, and execution begins with the following model; and +- when unset or `null`, execution starts at the beginning of the top-level + `models` list. Calibration does not automatically continue after the last + model completed by the preceding attempt. + +A named value must occur in the top-level `models` list and must identify a +model-level checkpoint. Calibration does not define a special `initialize` +value. A value such as `initialize_landuse` has ordinary model-name semantics: +that model is skipped and execution begins with the next model. The `_` shorthand +for the last checkpoint is not accepted in calibration mode. + +`resume_after` affects only the first global iteration entered by the current +invocation. Calibrated models at or before a named resume point are skipped in +that iteration. Any later global iterations in the same invocation ignore +`resume_after`, restart immediately before the first calibrated model, and run +the normal complete calibration sequence. + +For example, suppose global iteration 3 attempt 1 was interrupted after calibrated `model_a` completed: -- `resume_after: model_a` preserves model A's attempt-1 result and resumes with - the following model under attempt 2; -- `resume_after: initialize` rewinds pipeline state and reruns model A under - attempt 2; and -- no `resume_after` replays the interrupted iteration from the beginning under - attempt 2. +- `resume_after: model_a` starts attempt 2 after model A, preserving its + attempt-1 result; and +- `resume_after: null` starts attempt 2 at the beginning of the complete model + list, so model A runs again. Coefficient files are the authoritative current state and are never rolled back during recovery. Updates written before the interruption, along with subsequent From 896fc066373b20130ba7796996df99ef5ce4d805 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:33:19 -0700 Subject: [PATCH 87/96] aligning tests and language around restart functionality also fixing some bugs captured by AI review --- activitysim/abm/models/settings_checker.py | 28 +- activitysim/cli/run.py | 7 +- activitysim/core/calibration/README.md | 38 ++- activitysim/core/calibration/__init__.py | 3 +- activitysim/core/calibration/component.py | 22 +- activitysim/core/calibration/orchestrator.py | 308 +++++++++++++----- activitysim/core/calibration/settings.py | 8 +- .../core/calibration/test/test_calibration.py | 301 +++++++++++++++++ .../test/test_calibration_restart.py | 202 ++++++++++++ activitysim/core/test/test_calibration.py | 155 --------- .../core/test/test_calibration_reporting.py | 57 ---- ...modes.py => test_calibration_run_modes.py} | 0 docs/users-guide/calibration.rst | 66 ++-- 13 files changed, 863 insertions(+), 332 deletions(-) create mode 100644 activitysim/core/calibration/test/test_calibration.py create mode 100644 activitysim/core/calibration/test/test_calibration_restart.py delete mode 100644 activitysim/core/test/test_calibration.py delete mode 100644 activitysim/core/test/test_calibration_reporting.py rename activitysim/examples/prototype_mtc/test/calibration/{test_run_modes.py => test_calibration_run_modes.py} (100%) diff --git a/activitysim/abm/models/settings_checker.py b/activitysim/abm/models/settings_checker.py index c65d0f5772..12b6666392 100644 --- a/activitysim/abm/models/settings_checker.py +++ b/activitysim/abm/models/settings_checker.py @@ -4,24 +4,26 @@ from pydantic import BaseModel as PydanticBase from typing import Type, Optional +from activitysim.core import config +from activitysim.core.calibration.settings import ( + CALIBRATION_SETTINGS_FILE_NAME, + CalibrationConfig, +) from activitysim.core.configuration.base import PydanticReadable - -# import core settings from activitysim.core.configuration.logit import ( LogitNestSpec, TourLocationComponentSettings, TourModeComponentSettings, TemplatedLogitComponentSettings, ) -from activitysim.core import config from activitysim.core.configuration.network import NetworkSettings +from activitysim.core.exceptions import ModelConfigurationError from activitysim.core.workflow import State from activitysim.core.simulate import ( eval_coefficients, eval_nest_coefficients, read_model_coefficient_template, ) -from activitysim.core.exceptions import ModelConfigurationError # import model settings from activitysim.abm.models.accessibility import AccessibilitySettings @@ -637,6 +639,24 @@ def check_model_settings( # Collect all errors all_errors = [] + # calibration.yaml is optional and is not itself a model step, so validate + # it explicitly using the same Pydantic/error-aggregation path as model + # settings files. + try: + CalibrationConfig.read_settings_file( + state.filesystem, + CALIBRATION_SETTINGS_FILE_NAME, + mandatory=False, + ) + except Exception as error: + all_errors.append( + SettingsCheckerError( + "calibration", + error, + CALIBRATION_SETTINGS_FILE_NAME, + ) + ) + # additional logging set up formatter = logging.Formatter( "%(asctime)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S" diff --git a/activitysim/cli/run.py b/activitysim/cli/run.py index 4d7a1929f4..49a048a53e 100644 --- a/activitysim/cli/run.py +++ b/activitysim/cli/run.py @@ -336,8 +336,13 @@ def run(args): resume_after = state.settings.resume_after # cleanup if not resuming - if not resume_after: + preserve_calibration_outputs = calibration.calibration_run_should_preserve_outputs( + state + ) + if not resume_after and not preserve_calibration_outputs: cleanup_output_files(state) + elif preserve_calibration_outputs: + logger.info("preserving output files based on calibration restart preflight") elif state.settings.cleanup_trace_files_on_resume: tracing.delete_trace_files(state) diff --git a/activitysim/core/calibration/README.md b/activitysim/core/calibration/README.md index 5863987b67..2d4e939e54 100644 --- a/activitysim/core/calibration/README.md +++ b/activitysim/core/calibration/README.md @@ -27,17 +27,33 @@ that share model settings may point to the same coefficient file. ## Global iterations, recovery attempts, and `resume_after` -`calibration.yaml` `run.global_iterations` is the maximum total number of -logical global calibration iterations, not the number to execute on each -ActivitySim invocation. Calibration records durable progress in -`output/calibration/calibration_progress.json`: - -- a new output directory starts at global iteration 1; -- a crash re-enters the interrupted global iteration; -- a cleanly completed iteration advances to the next global iteration; -- convergence can finish the run before the configured maximum; and -- after a completed run, increasing `global_iterations` continues at the next - unfinished global iteration. An unchanged or lower value remains a no-op. +`calibration.yaml` `run.global_iterations` is the desired total number of +completed logical calibration iterations, not the number to execute on each +ActivitySim invocation. The run-control contract is: + +1. An unchanged setting on a completed run is a no-op, detected before normal + output cleanup so the pipeline and final outputs are preserved. +2. If a completed run's setting is changed to a value greater than the number + actually completed, calibration continues until the new total. This includes + lowering a previous maximum after early convergence, such as changing 5 to 3 + after convergence completed iteration 2. +3. A changed setting less than or equal to the number already completed is a + no-op; completed coefficient updates are never undone implicitly. +4. Top-level `settings.yaml` `resume_after` has normal ActivitySim semantics for + the first global iteration entered by the current invocation. Later global + iterations ignore it and execute every calibrated component. +5. A global iteration counts only if it has at least one durable calibrated + component result, either from the current attempt or an earlier attempt of + that same logical iteration. +6. `global_iterations` cannot be lowered below an interrupted iteration because + its coefficient files may already contain updates from that iteration. The + run stops with instructions to resume the iteration or deliberately reset + progress and coefficients. +7. Startup logs report the detected completed count, requested target, selected + action, starting iteration and attempt, and `resume_after` value. + +Calibration records the state needed to apply this contract in +`output/calibration/calibration_progress.json`. Recovery attempts are distinct from global iterations. The first execution of a global iteration is attempt 1. Restarting an interrupted global iteration creates diff --git a/activitysim/core/calibration/__init__.py b/activitysim/core/calibration/__init__.py index 442046abd7..d01414fdcf 100644 --- a/activitysim/core/calibration/__init__.py +++ b/activitysim/core/calibration/__init__.py @@ -17,7 +17,7 @@ reporting, settings, ) -from .orchestrator import run_calibration_loop +from .orchestrator import calibration_run_should_preserve_outputs, run_calibration_loop from .settings import ( CalibrationComponentResult, CalibrationComponentSettings, @@ -36,6 +36,7 @@ "CalibrationReportsSettings", "CalibrationRunResult", "CalibrationRunSettings", + "calibration_run_should_preserve_outputs", "calibration_enabled", "read_calibration_settings", "run_calibration_loop", diff --git a/activitysim/core/calibration/component.py b/activitysim/core/calibration/component.py index 1f470c5fcd..2009cc4787 100644 --- a/activitysim/core/calibration/component.py +++ b/activitysim/core/calibration/component.py @@ -185,10 +185,28 @@ def _calibrate_component( _append_summary_records(state, [summary_record]) if component_settings.reports.generic: - _write_generic_report(state, component_name, row_records) + try: + _write_generic_report(state, component_name, row_records) + except Exception as e: + logger.exception( + "calibration component %s iteration %s completed, but its " + "optional generic report could not be written: %s", + component_name, + component_iter, + e, + ) if bespoke_callable is not None: - bespoke_callable(eval_context) + try: + bespoke_callable(eval_context) + except Exception as e: + logger.exception( + "calibration component %s iteration %s completed, but its " + "optional bespoke report could not be written: %s", + component_name, + component_iter, + e, + ) if component_converged: break diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py index c8281d27c2..481e37f2bb 100644 --- a/activitysim/core/calibration/orchestrator.py +++ b/activitysim/core/calibration/orchestrator.py @@ -3,6 +3,8 @@ from __future__ import annotations import logging +from dataclasses import dataclass +from typing import Literal from activitysim.core import workflow @@ -25,6 +27,7 @@ _write_final_coefficients_snapshot, ) from .settings import ( + CALIBRATION_SETTINGS_FILE_NAME, CalibrationRunResult, read_calibration_settings, ) @@ -32,6 +35,126 @@ logger = logging.getLogger("calibration") +@dataclass(frozen=True) +class CalibrationRestartPlan: + """Side-effect-free decision about how persisted calibration should continue.""" + + action: Literal["run", "finalize", "noop", "error"] + start_global_iteration: int | None + attempt: int | None + completed_global_iterations: int + completed_components: dict + message: str | None = None + + +def _plan_calibration_restart( + progress: dict | None, + global_iterations: int, +) -> CalibrationRestartPlan: + """Apply the user-facing global-iteration restart contract. + + The contract is intentionally based on completed logical iterations: + + 1. An unchanged target on a completed run is a no-op. + 2. A changed target above the completed count extends to that new total, + even when it is below the previous maximum after early convergence. + 3. A changed target already satisfied by the completed count is a no-op. + 4. ``resume_after`` is applied separately by the orchestrator, only to the + first global iteration entered by the current invocation. + 5. The orchestrator counts an iteration only when it has a durable + calibrated-component result from some attempt of that iteration. + 6. An active iteration cannot be discarded by lowering the target. + 7. A lower target at a clean boundary is finalized without another + coefficient update so downstream outputs reflect current coefficients. + """ + if not progress: + return CalibrationRestartPlan("run", 1, 1, 0, {}) + + completed = int(progress.get("last_completed_global_iteration", 0)) + completed_components = dict(progress.get("completed_components", {})) + + if progress.get("complete"): + previous_target = int(progress.get("configured_global_iterations", completed)) + if global_iterations == previous_target or global_iterations <= completed: + return CalibrationRestartPlan( + "noop", None, None, completed, completed_components + ) + return CalibrationRestartPlan("run", completed + 1, 1, completed, {}) + + interrupted = progress.get("in_progress_iteration") + if interrupted is not None: + interrupted = int(interrupted) + if global_iterations < interrupted: + return CalibrationRestartPlan( + "error", + None, + None, + completed, + completed_components, + "Cannot set calibration run.global_iterations to " + f"{global_iterations} because global iteration {interrupted} is " + "currently in progress and coefficient files may already contain " + f"iteration {interrupted} updates. Set global_iterations to at " + f"least {interrupted} to resume, or deliberately reset calibration " + "progress and restore the desired coefficient files before starting " + "a new run.", + ) + return CalibrationRestartPlan( + "run", + interrupted, + int(progress.get("attempt", 1)) + 1, + completed, + completed_components, + ) + + next_iteration = int(progress.get("next_global_iteration", completed + 1)) + if next_iteration > global_iterations: + return CalibrationRestartPlan("finalize", next_iteration, 1, completed, {}) + return CalibrationRestartPlan("run", next_iteration, 1, completed, {}) + + +def calibration_run_should_preserve_outputs(state: workflow.State) -> bool: + """Return whether preflight must preserve outputs before orchestration.""" + # Read the small set of preflight fields without schema validation. The + # normal settings checker runs after cleanup selection and must remain the + # place that aggregates calibration.yaml validation errors. + calibration_settings = state.filesystem.read_settings_file( + CALIBRATION_SETTINGS_FILE_NAME, + mandatory=False, + ) + if not calibration_settings or not calibration_settings.get("enable", False): + return False + + progress = _read_progress(state) + if not progress or not progress.get("complete"): + if not progress: + return False + configured_global_iterations = int( + calibration_settings.get("run", {}).get("global_iterations", 1) + ) + plan = _plan_calibration_restart(progress, configured_global_iterations) + return plan.action in {"noop", "error"} + + +def _validate_counted_iteration_has_calibration( + calibration_models: list[str], + skipped_components: list[str], + completed_components: dict, +) -> None: + """Do not count a new logical iteration that has no calibration result.""" + if set(skipped_components) != set(calibration_models): + return + if any(component in completed_components for component in calibration_models): + return + raise RuntimeError( + "settings.yaml resume_after skips every calibrated model in the first " + "pending global iteration, and that iteration has no durable calibrated " + "component result from an earlier attempt. Choose a resume_after before " + "the last calibrated model, or remove resume_after, so the requested " + "global calibration iteration performs a coefficient update." + ) + + def run_calibration_loop( state: workflow.State, models: list[str], @@ -88,6 +211,7 @@ def run_calibration_loop( models, calibration_settings.run.calibrate_models[0] ) + skipped_calibration_models = [] if resume_after is not None: skipped_calibration_models = [ component @@ -107,71 +231,82 @@ def run_calibration_loop( _ensure_calibration_output_dir(state) progress = _read_progress(state) - if progress and progress.get("complete"): - completed_global_iterations = int( - progress.get("last_completed_global_iteration", 0) + restart_plan = _plan_calibration_restart( + progress, + calibration_settings.run.global_iterations, + ) + logger.info( + "calibration restart plan: action=%s completed=%s requested=%s " + "start_iteration=%s attempt=%s resume_after=%r", + restart_plan.action, + restart_plan.completed_global_iterations, + calibration_settings.run.global_iterations, + restart_plan.start_global_iteration, + restart_plan.attempt, + resume_after, + ) + if restart_plan.action == "error": + raise RuntimeError(restart_plan.message) + if restart_plan.action == "noop": + logger.info( + "calibration progress is already complete for the requested target; " + "remove %s to start a fresh calibration run", + CALIBRATION_PROGRESS_FILE, ) - completed_for_global_iterations = int( - progress.get("configured_global_iterations", completed_global_iterations) + return CalibrationRunResult( + converged=bool(progress.get("converged", False)), + completed_global_iterations=restart_plan.completed_global_iterations, ) - if calibration_settings.run.global_iterations > completed_for_global_iterations: - logger.info( - "calibration global_iterations increased from %s to %s; " - "continuing with global iteration %s", - completed_for_global_iterations, - calibration_settings.run.global_iterations, - completed_global_iterations + 1, - ) - progress = { - "complete": False, - "in_progress_iteration": None, - "next_global_iteration": completed_global_iterations + 1, - "last_completed_global_iteration": completed_global_iterations, - "converged": bool(progress.get("converged", False)), - "configured_global_iterations": calibration_settings.run.global_iterations, - "attempt": 0, - "completed_components": {}, - } - _write_progress(state, progress) - else: - if resume_after is not None: - raise RuntimeError( - f"settings.yaml resume_after={resume_after!r} cannot be honored " - "because calibration progress is already complete. Remove " - f"{CALIBRATION_PROGRESS_FILE} or use a new output directory to " - "start a new calibration run; current coefficient values will " - "be preserved." - ) - logger.info( - "calibration progress is already complete; remove %s to start a " - "fresh calibration run", - CALIBRATION_PROGRESS_FILE, - ) - return CalibrationRunResult( - converged=bool(progress.get("converged", False)), - completed_global_iterations=completed_global_iterations, + + progress_was_complete = bool(progress and progress.get("complete")) + if progress_was_complete: + previous_target = int( + progress.get( + "configured_global_iterations", + restart_plan.completed_global_iterations, ) + ) + logger.info( + "calibration global_iterations changed from %s to %s; continuing " + "with global iteration %s", + previous_target, + calibration_settings.run.global_iterations, + restart_plan.start_global_iteration, + ) + progress = { + "complete": False, + "in_progress_iteration": None, + "next_global_iteration": restart_plan.start_global_iteration, + "last_completed_global_iteration": ( + restart_plan.completed_global_iterations + ), + "converged": bool(progress.get("converged", False)), + "configured_global_iterations": calibration_settings.run.global_iterations, + "attempt": 0, + "completed_components": {}, + } + _write_progress(state, progress) interrupted_iteration = progress.get("in_progress_iteration") if progress else None + start_global_iter = restart_plan.start_global_iteration + start_attempt = restart_plan.attempt + start_completed_components = restart_plan.completed_components + completed_global_iterations = restart_plan.completed_global_iterations + if interrupted_iteration is not None: - start_global_iter = int(interrupted_iteration) - start_attempt = int(progress.get("attempt", 1)) + 1 - start_completed_components = dict(progress.get("completed_components", {})) logger.warning( "continuing interrupted calibration global iteration %s as attempt %s " "using the current coefficient files", start_global_iter, start_attempt, ) - else: - # Progress files from earlier versions contain next_global_iteration, so - # they remain compatible with the corrected total-count semantics. - start_global_iter = ( - int(progress.get("next_global_iteration", 1)) if progress else 1 + + if restart_plan.action == "run": + _validate_counted_iteration_has_calibration( + calibration_settings.run.calibrate_models, + skipped_calibration_models, + start_completed_components, ) - start_attempt = 1 - start_completed_components = {} - completed_global_iterations = start_global_iter - 1 if interrupted_iteration is not None and resume_after is not None: rerun_completed_components = [ @@ -191,28 +326,6 @@ def run_calibration_loop( start_global_iter, ) - if start_global_iter > calibration_settings.run.global_iterations: - logger.info( - "calibration progress already reached configured global_iterations=%s", - calibration_settings.run.global_iterations, - ) - converged = bool(progress.get("converged", False)) if progress else False - _write_final_coefficients_snapshot(state, calibration_settings) - _write_completed_progress( - state, - completed_global_iterations, - converged, - calibration_settings.run.global_iterations, - attempt=int(progress.get("attempt", 1)) if progress else 1, - completed_components=( - dict(progress.get("completed_components", {})) if progress else {} - ), - ) - return CalibrationRunResult( - converged=converged, - completed_global_iterations=completed_global_iterations, - ) - if state.settings.resume_after is None: # compute_accessibility requires its accessibility table to be empty; # unlike most model steps, it will not overwrite a prior result. @@ -230,6 +343,42 @@ def run_calibration_loop( shared_data_buffers = _initialize_mp_shared_resources(state) try: + if restart_plan.action == "finalize": + # The target was lowered at a clean boundary between iterations. + # No active coefficient update is being discarded, but the normal + # model sequence still runs so final outputs use current coefficients. + logger.info( + "calibration global_iterations=%s is below next global iteration " + "%s; running the final model sequence without another " + "calibration update", + calibration_settings.run.global_iterations, + start_global_iter, + ) + final_models = ( + models if first_model_idx is None else models[first_model_idx:] + ) + _run_in_configured_mode( + state, + models=final_models, + resume_after=state.settings.resume_after, + shared_data_buffers=shared_data_buffers, + ) + completed_global_iterations = restart_plan.completed_global_iterations + converged = bool(progress.get("converged", False)) if progress else False + _write_final_coefficients_snapshot(state, calibration_settings) + _write_completed_progress( + state, + completed_global_iterations, + converged, + calibration_settings.run.global_iterations, + attempt=int(progress.get("attempt", 1)) if progress else 1, + completed_components={}, + ) + return CalibrationRunResult( + converged=converged, + completed_global_iterations=completed_global_iterations, + ) + # skip precursors if, on first iter, resume_after exists and is >= first_calib_model_idx if ( state.settings.resume_after is None @@ -385,8 +534,6 @@ def run_calibration_loop( attempt=attempt, shared_data_buffers=shared_data_buffers, ) - _write_component_plots(state, component) - all_converged = all_converged and component_result.converged completed_components[component] = { @@ -401,6 +548,15 @@ def run_calibration_loop( completed_components, ) + try: + _write_component_plots(state, component) + except Exception: + logger.exception( + "calibration component %s completed, but its optional " + "standard plots could not be written", + component, + ) + last_calibrated_component = component iteration_is_complete = ( diff --git a/activitysim/core/calibration/settings.py b/activitysim/core/calibration/settings.py index 83f8bdd200..464d553a7a 100644 --- a/activitysim/core/calibration/settings.py +++ b/activitysim/core/calibration/settings.py @@ -13,7 +13,7 @@ CALIBRATION_SETTINGS_FILE_NAME = "calibration.yaml" -class CalibrationRunSettings(PydanticBase): +class CalibrationRunSettings(PydanticBase, extra="forbid"): """Run-control settings for calibration.""" calibrate_models: list[str] @@ -32,14 +32,14 @@ def validate_run_settings(self): return self -class CalibrationReportsSettings(PydanticBase): +class CalibrationReportsSettings(PydanticBase, extra="forbid"): """Reporting settings for a calibrated component.""" generic: bool = True bespoke: str | None = None -class CalibrationComponentSettings(PydanticBase): +class CalibrationComponentSettings(PydanticBase, extra="forbid"): """Settings for one calibratable model component.""" calibration_spec: str @@ -49,7 +49,7 @@ class CalibrationComponentSettings(PydanticBase): reports: CalibrationReportsSettings = CalibrationReportsSettings() -class CalibrationConfig(PydanticReadable): +class CalibrationConfig(PydanticReadable, extra="forbid"): """Top-level calibration configuration.""" enable: bool = False diff --git a/activitysim/core/calibration/test/test_calibration.py b/activitysim/core/calibration/test/test_calibration.py new file mode 100644 index 0000000000..4352d007d1 --- /dev/null +++ b/activitysim/core/calibration/test/test_calibration.py @@ -0,0 +1,301 @@ +from __future__ import annotations + +import math +import copy + +import pandas as pd +import pytest +from pathlib import Path +from pydantic import ValidationError + +from activitysim.core.calibration.component import _evaluate_and_update +from activitysim.core.calibration.expressions import _compute_delta +from activitysim.core.calibration.orchestrator import ( + _components_ran_for_convergence, +) +from activitysim.core.calibration.reporting import ( + _append_iteration_records, + _coefficient_trajectory, + _read_component_iteration_records, +) +from activitysim.core.calibration.settings import CalibrationConfig + + +@pytest.mark.parametrize( + ("method", "model_value", "target_value", "damping", "expected"), + [ + # log_ratio: delta = log(target / model) * damping = log(2) * 0.5 + ("log_ratio", 0.25, 0.5, 0.5, math.log(2) * 0.5), + ( + # odds_ratio: delta = log((t*(1-m)) / (m*(1-t))) * damping + "odds_ratio", + 0.4, + 0.6, + 0.5, + math.log((0.6 * (1 - 0.4)) / (0.4 * (1 - 0.6))) * 0.5, + ), + ], +) +def test_compute_delta(method, model_value, target_value, damping, expected): + delta = _compute_delta( + method=method, + model_value=model_value, + target_value=target_value, + damping=damping, + component_name="test_component", + description="test target", + default_increment=2.0, + ) + + assert delta == pytest.approx(expected) + + +@pytest.mark.parametrize( + ("model_value", "target_value", "expected"), + [ + ( + 0.0, + 0.5, + 2.0, + ), # model is zero but target is not: fallback must nudge coefficient upward + ( + 1.0, + 0.5, + -2.0, + ), # model is one but target is not: fallback must nudge coefficient downward + (0.0, 0.0, 0.0), # both at boundary zero: no change needed + ], +) +def test_odds_ratio_boundary_fallback_has_correct_direction( + model_value, target_value, expected +): + delta = _compute_delta( + method="odds_ratio", + model_value=model_value, + target_value=target_value, + damping=1.0, + component_name="test_component", + description="test target", + default_increment=2.0, + ) + + assert delta == expected + + +def test_evaluate_and_update_only_changes_eligible_coefficients(): + calibration_spec = pd.DataFrame( + [ + { + "description": "unconverged", + "coefficient": "coef_update", + "model_value": 0.25, + "target_value": 0.5, + "hold_fast": False, + "min": -10.0, + "max": 10.0, + "damping": 0.5, + "method": "log_ratio", + "tolerance": 0.01, + }, + { + "description": "within tolerance", + "coefficient": "coef_converged", + "model_value": 0.49, + "target_value": 0.5, + "hold_fast": False, + "min": -10.0, + "max": 10.0, + "damping": 1.0, + "method": "log_ratio", + "tolerance": 0.02, + }, + { + "description": "held fixed", + "coefficient": "coef_held", + "model_value": 0.25, + "target_value": 0.5, + "hold_fast": True, + "min": -10.0, + "max": 10.0, + "damping": 1.0, + "method": "log_ratio", + "tolerance": 0.01, + }, + ] + ) + coefficients = pd.DataFrame( + {"value": [1.0, 2.0, 3.0]}, + index=["coef_update", "coef_converged", "coef_held"], + ) + + records, _, updated, component_converged = _evaluate_and_update( + component_name="test_component", + calibration_spec_df=calibration_spec, + coefficients_df=coefficients, + eval_context={}, + global_iter=1, + component_iter=1, + ) + + assert updated.loc["coef_update", "value"] == pytest.approx( + 1.0 + math.log(2) * 0.5 + ) # unconverged: full log-ratio delta applied + assert ( + updated.loc["coef_converged", "value"] == 2.0 + ) # within tolerance: value must not change + assert ( + updated.loc["coef_held", "value"] == 3.0 + ) # hold_fast=True: value must not change + assert records[1]["coef_delta"] == 0.0 # converged row records zero change + assert records[2]["coef_delta"] == 0.0 # held row records zero change + assert ( + component_converged is False + ) # one unconverged row means the whole component is not done + + +@pytest.mark.parametrize( + ( + "first_model_idx", + "last_calib_model_idx", + "global_iter", + "start_global_iter", + "expected", + ), + [ + ( + None, + 10, + 1, + 1, + True, + ), # no pre-calibration model ran: always counts toward convergence + ( + 5, + 10, + 1, + 1, + True, + ), # model ran before last calibration model in pipeline: counts + ( + 11, + 10, + 1, + 1, + False, + ), # model ran after last calibration model: skipped on first global iter + ( + 11, + 10, + 2, + 1, + True, + ), # same position but later global iter: skip only applies to first pass + ], +) +def test_components_ran_for_convergence( + first_model_idx, + last_calib_model_idx, + global_iter, + start_global_iter, + expected, +): + assert ( + _components_ran_for_convergence( + first_model_idx=first_model_idx, + last_calib_model_idx=last_calib_model_idx, + global_iter=global_iter, + start_global_iter=start_global_iter, + ) + is expected + ) + + +@pytest.mark.parametrize( + ("location", "unknown_setting"), + [ + ((), "cleanup_pipeline_after_run"), # unknown at top-level CalibrationConfig + (("run",), "restart_after"), # unknown inside CalibrationRunSettings + ( + ("model_settings", "test_component"), + "survey_file", + ), # unknown inside CalibrationComponentSettings + ( + ("model_settings", "test_component", "reports"), + "unexpected_report", + ), # unknown inside CalibrationReportSettings + ], +) +def test_calibration_settings_reject_unknown_fields(location, unknown_setting): + settings = { + "enable": True, + "run": { + "calibrate_models": ["test_component"], + "global_iterations": 1, + }, + "model_settings": { + "test_component": { + "calibration_spec": "test_calibration.csv", + "reports": {"generic": True}, + } + }, + } + invalid_settings = copy.deepcopy(settings) + container = invalid_settings + for key in location: + container = container[key] + container[unknown_setting] = True + + with pytest.raises(ValidationError) as error: + CalibrationConfig.model_validate(invalid_settings) + + assert error.value.errors()[0]["loc"] == (*location, unknown_setting) + + +class _State: + def __init__(self, output_dir: Path): + self.output_dir = output_dir + + def get_output_file_path(self, file_name: str) -> Path: + return self.output_dir / file_name + + +def _record(attempt: int, previous: float, next_value: float) -> dict: + return { + "global_iter": 1, + "attempt": attempt, + "component_iter": 1, + "description": "test target", + "component": "model_a", + "coefficient": "coef_a", + "target_value": 0.5, + "model_value": 0.25, + "difference": 0.25, + "pct_difference": 50.0, + "hold_fast": False, + "prev_coefficient": previous, + "coef_delta": next_value - previous, + "next_coefficient": next_value, + "converged": False, + "at_min": False, + "at_max": False, + } + + +def test_recovery_attempts_preserve_complete_coefficient_trajectory(tmp_path): + state = _State(tmp_path) + # attempt 1: coefficient moves 1.0 → 1.5 + _append_iteration_records(state, "model_a", [_record(1, 1.0, 1.5)]) + # attempt 2: picks up exactly where attempt 1 ended, 1.5 → 1.75 + _append_iteration_records(state, "model_a", [_record(2, 1.5, 1.75)]) + + stored = pd.read_csv(tmp_path / "calibration" / "calibration_iteration_records.csv") + assert list(stored["attempt"]) == [1, 2] + # the chain must be unbroken: attempt 2 prev_coefficient must equal attempt 1 next_coefficient + assert stored.loc[1, "prev_coefficient"] == stored.loc[0, "next_coefficient"] + + records = _read_component_iteration_records(state, "model_a") + trajectory, labels = _coefficient_trajectory(records, ["coef_a"]) + + # labels: one "Start" entry (initial value) plus one label per recorded iteration + assert labels == ["Start", "G1-A1-C1", "G1-A2-C1"] + assert list(trajectory["coef_a"]) == [1.0, 1.5, 1.75] diff --git a/activitysim/core/calibration/test/test_calibration_restart.py b/activitysim/core/calibration/test/test_calibration_restart.py new file mode 100644 index 0000000000..1e907f67a7 --- /dev/null +++ b/activitysim/core/calibration/test/test_calibration_restart.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import pytest + +from activitysim.core.calibration.orchestrator import ( + _plan_calibration_restart, + _skipped_calibration_components, + _validate_counted_iteration_has_calibration, +) + + +@pytest.mark.parametrize( + ( + "progress", + "global_iterations", + "expected_action", + "expected_start", + "expected_attempt", + ), + [ + # no prior progress file: start fresh from iteration 1, attempt 1 + (None, 2, "run", 1, 1), + ( + # interrupted mid-run at iteration 3; resume with an incremented attempt number + { + "in_progress_iteration": 3, + "last_completed_global_iteration": 2, + "attempt": 1, + "completed_components": {}, + }, + 3, + "run", + 3, + 2, + ), + ( + # calibration iterations done but downstream finalization models still pending + { + "last_completed_global_iteration": 2, + "next_global_iteration": 3, + }, + 2, + "finalize", + 3, + 1, + ), + ( + # fully complete and global_iterations unchanged: nothing left to do + { + "complete": True, + "last_completed_global_iteration": 2, + "configured_global_iterations": 2, + }, + 2, + "noop", + None, + None, + ), + ( + # fully complete but global_iterations increased: extend the calibration run + { + "complete": True, + "last_completed_global_iteration": 2, + "configured_global_iterations": 2, + }, + 4, + "run", + 3, + 1, + ), + # Convergence completed only two iterations against a target of five. + # Leaving the target unchanged is a no-op, while changing it to a value + # above the completed count explicitly requests more iterations. + ( + # converged early at iter 2; configured target of 5 not changed → no-op + { + "complete": True, + "last_completed_global_iteration": 2, + "configured_global_iterations": 5, + }, + 5, + "noop", + None, + None, + ), + ( + # converged early at iter 2; target lowered to 3 → run one more iteration + { + "complete": True, + "last_completed_global_iteration": 2, + "configured_global_iterations": 5, + }, + 3, + "run", + 3, + 1, + ), + ( + # converged early at iter 2; target lowered to 4 → run two more iterations starting at 3 + { + "complete": True, + "last_completed_global_iteration": 2, + "configured_global_iterations": 5, + }, + 4, + "run", + 3, + 1, + ), + ( + # converged early at iter 2; target lowered to 2 (≤ completed count) → no-op + { + "complete": True, + "last_completed_global_iteration": 2, + "configured_global_iterations": 5, + }, + 2, + "noop", + None, + None, + ), + ], +) +def test_plan_calibration_restart( + progress, + global_iterations, + expected_action, + expected_start, + expected_attempt, +): + plan = _plan_calibration_restart(progress, global_iterations) + + assert plan.action == expected_action + assert plan.start_global_iteration == expected_start + assert plan.attempt == expected_attempt + + +def test_cannot_lower_limit_below_interrupted_iteration(): + # iteration 3 is in progress; lowering global_iterations to 2 would abandon it mid-run + plan = _plan_calibration_restart( + { + "in_progress_iteration": 3, + "last_completed_global_iteration": 2, + "attempt": 1, + "completed_components": {"model_a": {"attempt": 1, "converged": True}}, + }, + global_iterations=2, + ) + + assert plan.action == "error" + # error message must identify the blocked iteration and the minimum required setting + assert "global iteration 3 is currently in progress" in plan.message + assert "global_iterations to at least 3" in plan.message + + +def test_resume_after_only_skips_components_in_first_entered_iteration(): + # model_a ran and completed in a prior attempt during iteration 3; model_b had not yet started + # first_model_idx is the pipeline position immediately after model_a + models = ["initialize", "model_a", "intermediate", "model_b", "output"] + calibration_models = ["model_a", "model_b"] + first_model_idx = models.index("model_a") + 1 + + # re-entering the interrupted iteration: model_a already ran, so it must be skipped + assert _skipped_calibration_components( + calibration_models, + models, + first_model_idx, + global_iter=3, + start_global_iter=3, + ) == ["model_a"] + # on a later full iteration nothing was partially skipped before, run everything normally + assert ( + _skipped_calibration_components( + calibration_models, + models, + first_model_idx, + global_iter=4, + start_global_iter=3, + ) + == [] + ) + + +def test_new_iteration_cannot_skip_every_calibrated_component(): + # with no prior-attempt results in completed_components, skipping all calibration + # models would produce an iteration with zero calibration updates — that is a bug + with pytest.raises(RuntimeError, match="skips every calibrated model"): + _validate_counted_iteration_has_calibration( + calibration_models=["model_a", "model_b"], + skipped_components=["model_a", "model_b"], + completed_components={}, + ) + + +def test_prior_attempt_result_allows_downstream_only_recovery(): + # model_a has a completed result from a prior attempt, so skipping both calibration + # models is valid: downstream models will still pick up model_a's updated coefficients + _validate_counted_iteration_has_calibration( + calibration_models=["model_a", "model_b"], + skipped_components=["model_a", "model_b"], + completed_components={"model_a": {"attempt": 1, "converged": True}}, + ) diff --git a/activitysim/core/test/test_calibration.py b/activitysim/core/test/test_calibration.py deleted file mode 100644 index e8b50bc37a..0000000000 --- a/activitysim/core/test/test_calibration.py +++ /dev/null @@ -1,155 +0,0 @@ -from __future__ import annotations - -import math - -import pandas as pd -import pytest - -from activitysim.core.calibration.component import _evaluate_and_update -from activitysim.core.calibration.expressions import _compute_delta -from activitysim.core.calibration.orchestrator import ( - _components_ran_for_convergence, -) - - -@pytest.mark.parametrize( - ("method", "model_value", "target_value", "damping", "expected"), - [ - ("log_ratio", 0.25, 0.5, 0.5, math.log(2) * 0.5), - ( - "odds_ratio", - 0.4, - 0.6, - 0.5, - math.log((0.6 * (1 - 0.4)) / (0.4 * (1 - 0.6))) * 0.5, - ), - ], -) -def test_compute_delta(method, model_value, target_value, damping, expected): - delta = _compute_delta( - method=method, - model_value=model_value, - target_value=target_value, - damping=damping, - component_name="test_component", - description="test target", - default_increment=2.0, - ) - - assert delta == pytest.approx(expected) - - -@pytest.mark.parametrize( - ("model_value", "target_value", "expected"), - [(0.0, 0.5, 2.0), (1.0, 0.5, -2.0), (0.0, 0.0, 0.0)], -) -def test_odds_ratio_boundary_fallback_has_correct_direction( - model_value, target_value, expected -): - delta = _compute_delta( - method="odds_ratio", - model_value=model_value, - target_value=target_value, - damping=1.0, - component_name="test_component", - description="test target", - default_increment=2.0, - ) - - assert delta == expected - - -def test_evaluate_and_update_only_changes_eligible_coefficients(): - calibration_spec = pd.DataFrame( - [ - { - "description": "unconverged", - "coefficient": "coef_update", - "model_value": 0.25, - "target_value": 0.5, - "hold_fast": False, - "min": -10.0, - "max": 10.0, - "damping": 0.5, - "method": "log_ratio", - "tolerance": 0.01, - }, - { - "description": "within tolerance", - "coefficient": "coef_converged", - "model_value": 0.49, - "target_value": 0.5, - "hold_fast": False, - "min": -10.0, - "max": 10.0, - "damping": 1.0, - "method": "log_ratio", - "tolerance": 0.02, - }, - { - "description": "held fixed", - "coefficient": "coef_held", - "model_value": 0.25, - "target_value": 0.5, - "hold_fast": True, - "min": -10.0, - "max": 10.0, - "damping": 1.0, - "method": "log_ratio", - "tolerance": 0.01, - }, - ] - ) - coefficients = pd.DataFrame( - {"value": [1.0, 2.0, 3.0]}, - index=["coef_update", "coef_converged", "coef_held"], - ) - - records, _, updated, component_converged = _evaluate_and_update( - component_name="test_component", - calibration_spec_df=calibration_spec, - coefficients_df=coefficients, - eval_context={}, - global_iter=1, - component_iter=1, - ) - - assert updated.loc["coef_update", "value"] == pytest.approx(1.0 + math.log(2) * 0.5) - assert updated.loc["coef_converged", "value"] == 2.0 - assert updated.loc["coef_held", "value"] == 3.0 - assert records[1]["coef_delta"] == 0.0 - assert records[2]["coef_delta"] == 0.0 - assert component_converged is False - - -@pytest.mark.parametrize( - ( - "first_model_idx", - "last_calib_model_idx", - "global_iter", - "start_global_iter", - "expected", - ), - [ - (None, 10, 1, 1, True), - (5, 10, 1, 1, True), - (11, 10, 1, 1, False), - (11, 10, 2, 1, True), - ], -) -def test_components_ran_for_convergence( - first_model_idx, - last_calib_model_idx, - global_iter, - start_global_iter, - expected, -): - assert ( - _components_ran_for_convergence( - first_model_idx=first_model_idx, - last_calib_model_idx=last_calib_model_idx, - global_iter=global_iter, - start_global_iter=start_global_iter, - ) - is expected - ) diff --git a/activitysim/core/test/test_calibration_reporting.py b/activitysim/core/test/test_calibration_reporting.py deleted file mode 100644 index d0d459cec2..0000000000 --- a/activitysim/core/test/test_calibration_reporting.py +++ /dev/null @@ -1,57 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pandas as pd - -from activitysim.core.calibration.reporting import ( - _append_iteration_records, - _coefficient_trajectory, - _read_component_iteration_records, -) - - -class _State: - def __init__(self, output_dir: Path): - self.output_dir = output_dir - - def get_output_file_path(self, file_name: str) -> Path: - return self.output_dir / file_name - - -def _record(attempt: int, previous: float, next_value: float) -> dict: - return { - "global_iter": 1, - "attempt": attempt, - "component_iter": 1, - "description": "test target", - "component": "model_a", - "coefficient": "coef_a", - "target_value": 0.5, - "model_value": 0.25, - "difference": 0.25, - "pct_difference": 50.0, - "hold_fast": False, - "prev_coefficient": previous, - "coef_delta": next_value - previous, - "next_coefficient": next_value, - "converged": False, - "at_min": False, - "at_max": False, - } - - -def test_recovery_attempts_preserve_complete_coefficient_trajectory(tmp_path): - state = _State(tmp_path) - _append_iteration_records(state, "model_a", [_record(1, 1.0, 1.5)]) - _append_iteration_records(state, "model_a", [_record(2, 1.5, 1.75)]) - - stored = pd.read_csv(tmp_path / "calibration" / "calibration_iteration_records.csv") - assert list(stored["attempt"]) == [1, 2] - assert stored.loc[1, "prev_coefficient"] == stored.loc[0, "next_coefficient"] - - records = _read_component_iteration_records(state, "model_a") - trajectory, labels = _coefficient_trajectory(records, ["coef_a"]) - - assert labels == ["Start", "G1-A1-C1", "G1-A2-C1"] - assert list(trajectory["coef_a"]) == [1.0, 1.5, 1.75] diff --git a/activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py b/activitysim/examples/prototype_mtc/test/calibration/test_calibration_run_modes.py similarity index 100% rename from activitysim/examples/prototype_mtc/test/calibration/test_run_modes.py rename to activitysim/examples/prototype_mtc/test/calibration/test_calibration_run_modes.py diff --git a/docs/users-guide/calibration.rst b/docs/users-guide/calibration.rst index ac79a1af2f..ede52d8cea 100644 --- a/docs/users-guide/calibration.rst +++ b/docs/users-guide/calibration.rst @@ -58,7 +58,6 @@ calibration-specific overlay directory): - workplace_location - auto_ownership_simulate - tour_mode_choice_simulate - restart_after: [] # components after which to restart (advanced) global_iterations: 3 # number of full calibration passes complete_steps: false # run model steps after the last calibrated component @@ -135,8 +134,13 @@ Configuration Reference - Every component listed in ``run.calibrate_models`` must have a corresponding entry in ``model_settings``. -- Every component in ``run.restart_after`` must also appear in ``run.calibrate_models``. - ``run.global_iterations`` must be ≥ 1. +- Unknown fields are rejected at every level of ``calibration.yaml``, following + the same strict settings-model convention used by checked ActivitySim model + settings. When the settings checker is enabled, calibration validation errors + are included in its normal aggregated error report. General run settings such + as ``cleanup_pipeline_after_run`` belong in the top-level ``settings.yaml`` + file. ``run`` — Run Control Settings ------------------------------- @@ -154,10 +158,6 @@ Configuration Reference - *required* - Model component names to calibrate. Must match names in ``settings.yaml`` ``models`` list. - * - ``restart_after`` - - ``list[str]`` - - ``[]`` - - Components after which to restart. * - ``global_iterations`` - ``int`` - ``1`` @@ -581,23 +581,47 @@ each component iteration. This means: Crash Recovery ============== -Before each global iteration, calibration replaces the files in its recovery -directory with a copy of every coefficient file that it may modify, then records -the active iteration in ``calibration_progress.json``. If a run is interrupted: - -1. Restarting ``activitysim run`` restores all coefficient files from the - start-of-iteration recovery snapshot. -2. The interrupted global iteration is replayed from that consistent boundary. -3. Remaining iterations run only until the configured total - ``global_iterations`` is reached. - -The progress file is written using atomic replacement. Once progress is marked -complete, rerunning with the same output directory does not apply additional -calibration iterations. +Calibration follows this run-control contract: + +1. ``global_iterations`` is the desired total number of calibration iterations. +2. If a completed run's setting is changed to a value greater than the number + actually completed, calibration continues until the new total. This applies + even when the new value is below the previous maximum after early convergence. + For example, changing 5 to 3 after convergence at iteration 2 requests one + additional iteration. +3. A changed setting less than or equal to the number already completed will not + run and an error message is thrown; completed coefficient updates are not + implicitly undone. +4. Top-level ``settings.yaml`` ``resume_after`` follows normal ActivitySim + semantics for the first global iteration entered by the current invocation. + Later global iterations ignore it and run every calibrated component. +5. A logical global iteration counts only if it has at least one durable + calibrated-component result from the current attempt or an earlier attempt + of that iteration. A new iteration cannot be consumed by a ``resume_after`` + value that skips every calibrated model. +6. ``global_iterations`` cannot be lowered below an interrupted iteration. + ActivitySim raises an error explaining that coefficient files may already + contain updates from the active iteration and instructs the user either to + resume it or deliberately reset progress and coefficient files. +7. At startup, ActivitySim logs the detected completed count, requested target, + selected action, starting iteration and attempt, and ``resume_after`` value. +8. Coefficient updates are stored in the configs directory and are **not** + rolled back after an interruption. This allows a new run to resume from the + last known coefficients and allows the user to update coefficients manually. +9. Coefficient updates are uniquely numbered by global iteration, calibration + iteration, and attempt. Attempt number is typically only 1 unless the model + crashes after coefficient updates are made but before component fully + completes (e.g. plotting functions cause crash). + +Calibration records the active global iteration and recovery attempt in +``calibration_progress.json``. Coefficient files are the authoritative current +state and are not rolled back after an interruption. Restarting re-enters the +interrupted global iteration using those current coefficients; the top-level +``settings.yaml`` ``resume_after`` value determines where model execution +resumes. To force a fresh start, delete ``output/calibration/calibration_progress.json`` -and restore original coefficient files. Recovery snapshots can also be removed -after a completed run if they are no longer needed. +and restore the desired starting coefficient files. Multiprocess Mode From f414f230632e176052fd6dda13c0df0d1e89c343 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:34:16 -0700 Subject: [PATCH 88/96] setting checking cleanup --- activitysim/core/calibration/component.py | 8 +-- activitysim/core/calibration/orchestrator.py | 59 +++++++++++++------ activitysim/core/calibration/settings.py | 21 +++++-- .../core/calibration/test/test_calibration.py | 46 +++++++++++++++ .../test/test_calibration_restart.py | 42 +++++++++++++ docs/users-guide/calibration.rst | 12 ++-- 6 files changed, 155 insertions(+), 33 deletions(-) diff --git a/activitysim/core/calibration/component.py b/activitysim/core/calibration/component.py index 2009cc4787..c82577c351 100644 --- a/activitysim/core/calibration/component.py +++ b/activitysim/core/calibration/component.py @@ -190,11 +190,11 @@ def _calibrate_component( except Exception as e: logger.exception( "calibration component %s iteration %s completed, but its " - "optional generic report could not be written: %s", + "generic report could not be written.", component_name, component_iter, - e, ) + raise RuntimeError(e) if bespoke_callable is not None: try: @@ -202,11 +202,11 @@ def _calibrate_component( except Exception as e: logger.exception( "calibration component %s iteration %s completed, but its " - "optional bespoke report could not be written: %s", + "bespoke report could not be written.", component_name, component_iter, - e, ) + raise RuntimeError(e) if component_converged: break diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py index 481e37f2bb..a7a8196a34 100644 --- a/activitysim/core/calibration/orchestrator.py +++ b/activitysim/core/calibration/orchestrator.py @@ -115,23 +115,43 @@ def _plan_calibration_restart( def calibration_run_should_preserve_outputs(state: workflow.State) -> bool: """Return whether preflight must preserve outputs before orchestration.""" - # Read the small set of preflight fields without schema validation. The - # normal settings checker runs after cleanup selection and must remain the - # place that aggregates calibration.yaml validation errors. - calibration_settings = state.filesystem.read_settings_file( - CALIBRATION_SETTINGS_FILE_NAME, - mandatory=False, - ) - if not calibration_settings or not calibration_settings.get("enable", False): + # Keep missing and disabled calibration files on the ordinary ActivitySim + # cleanup path. Only enabled calibration runs get calibration-specific + # protection during preflight. + try: + raw_settings = state.filesystem.read_settings_file( + CALIBRATION_SETTINGS_FILE_NAME, + mandatory=False, + ) + except Exception: + logger.debug( + "preserving outputs because calibration settings could not be read", + exc_info=True, + ) + return True + if not raw_settings or not raw_settings.get("enable", False): + return False + + # Calibration validation normally runs after output cleanup. Preserve the + # outputs if enabled settings are invalid, then let the normal settings + # checker aggregate and report the validation error later. + try: + calibration_settings = read_calibration_settings(state) + except Exception: + logger.debug( + "preserving outputs because calibration settings validation failed", + exc_info=True, + ) + return True + + if not calibration_settings or not calibration_settings.enable: return False progress = _read_progress(state) if not progress or not progress.get("complete"): if not progress: return False - configured_global_iterations = int( - calibration_settings.get("run", {}).get("global_iterations", 1) - ) + configured_global_iterations = calibration_settings.run.global_iterations plan = _plan_calibration_restart(progress, configured_global_iterations) return plan.action in {"noop", "error"} @@ -258,6 +278,16 @@ def run_calibration_loop( completed_global_iterations=restart_plan.completed_global_iterations, ) + # Validate the requested restart before changing a formerly complete + # progress record. A rejected resume_after must leave durable progress in + # exactly the state in which it was found. + if restart_plan.action == "run": + _validate_counted_iteration_has_calibration( + calibration_settings.run.calibrate_models, + skipped_calibration_models, + restart_plan.completed_components, + ) + progress_was_complete = bool(progress and progress.get("complete")) if progress_was_complete: previous_target = int( @@ -301,13 +331,6 @@ def run_calibration_loop( start_attempt, ) - if restart_plan.action == "run": - _validate_counted_iteration_has_calibration( - calibration_settings.run.calibrate_models, - skipped_calibration_models, - start_completed_components, - ) - if interrupted_iteration is not None and resume_after is not None: rerun_completed_components = [ component diff --git a/activitysim/core/calibration/settings.py b/activitysim/core/calibration/settings.py index 464d553a7a..d35c4cd50e 100644 --- a/activitysim/core/calibration/settings.py +++ b/activitysim/core/calibration/settings.py @@ -4,7 +4,7 @@ from dataclasses import dataclass -from pydantic import model_validator +from pydantic import Field, model_validator from activitysim.core import workflow from activitysim.core.configuration import PydanticReadable @@ -17,7 +17,7 @@ class CalibrationRunSettings(PydanticBase, extra="forbid"): """Run-control settings for calibration.""" calibrate_models: list[str] - global_iterations: int = 1 + global_iterations: int = Field(default=1, ge=1) complete_steps: bool = False # Deprecated compatibility setting retained so existing configurations # continue to parse. Exact calibration restores supersede this setting. @@ -29,6 +29,18 @@ def validate_run_settings(self): raise ValueError( "calibration.run.calibrate_models must contain at least one model name" ) + duplicate_models = sorted( + { + model + for model in self.calibrate_models + if self.calibrate_models.count(model) > 1 + } + ) + if duplicate_models: + raise ValueError( + "calibration.run.calibrate_models contains duplicate model " + f"name(s): {duplicate_models}" + ) return self @@ -45,7 +57,7 @@ class CalibrationComponentSettings(PydanticBase, extra="forbid"): calibration_spec: str model_settings_file: str | None = None helper_module: str | None = None - submodel_max_iterations: int = 1 + submodel_max_iterations: int = Field(default=1, ge=1) reports: CalibrationReportsSettings = CalibrationReportsSettings() @@ -65,9 +77,6 @@ def validate_model_settings(self): f"calibration model '{component}' is not in model_settings" ) - if self.run.global_iterations < 1: - raise ValueError("max_iterations must be >= 1") - return self diff --git a/activitysim/core/calibration/test/test_calibration.py b/activitysim/core/calibration/test/test_calibration.py index 4352d007d1..6cd47b9885 100644 --- a/activitysim/core/calibration/test/test_calibration.py +++ b/activitysim/core/calibration/test/test_calibration.py @@ -251,6 +251,52 @@ def test_calibration_settings_reject_unknown_fields(location, unknown_setting): assert error.value.errors()[0]["loc"] == (*location, unknown_setting) +@pytest.mark.parametrize( + ("field", "value"), + [ + ("global_iterations", 0), + ("submodel_max_iterations", 0), + ], +) +def test_calibration_iteration_limits_must_be_positive(field, value): + settings = { + "enable": True, + "run": { + "calibrate_models": ["test_component"], + "global_iterations": value if field == "global_iterations" else 1, + }, + "model_settings": { + "test_component": { + "calibration_spec": "test_calibration.csv", + "submodel_max_iterations": ( + value if field == "submodel_max_iterations" else 1 + ), + } + }, + } + + with pytest.raises(ValidationError) as error: + CalibrationConfig.model_validate(settings) + + assert error.value.errors()[0]["type"] == "greater_than_equal" + + +def test_calibration_models_must_be_unique(): + settings = { + "enable": True, + "run": { + "calibrate_models": ["test_component", "test_component"], + "global_iterations": 1, + }, + "model_settings": { + "test_component": {"calibration_spec": "test_calibration.csv"} + }, + } + + with pytest.raises(ValidationError, match="duplicate model name"): + CalibrationConfig.model_validate(settings) + + class _State: def __init__(self, output_dir: Path): self.output_dir = output_dir diff --git a/activitysim/core/calibration/test/test_calibration_restart.py b/activitysim/core/calibration/test/test_calibration_restart.py index 1e907f67a7..b1809115e5 100644 --- a/activitysim/core/calibration/test/test_calibration_restart.py +++ b/activitysim/core/calibration/test/test_calibration_restart.py @@ -3,12 +3,54 @@ import pytest from activitysim.core.calibration.orchestrator import ( + calibration_run_should_preserve_outputs, _plan_calibration_restart, _skipped_calibration_components, _validate_counted_iteration_has_calibration, ) +class _PreflightFilesystem: + def __init__(self, calibration_settings): + self.calibration_settings = calibration_settings + + def read_settings_file(self, _file_name, mandatory=False): + assert mandatory is False + return self.calibration_settings + + +class _PreflightState: + def __init__(self, calibration_settings): + self.filesystem = _PreflightFilesystem(calibration_settings) + + +def test_preflight_preserves_outputs_when_calibration_settings_are_invalid(monkeypatch): + def invalid_settings(_state): + raise ValueError("invalid calibration settings") + + monkeypatch.setattr( + "activitysim.core.calibration.orchestrator.read_calibration_settings", + invalid_settings, + ) + + state = _PreflightState({"enable": True}) + + assert calibration_run_should_preserve_outputs(state) is True + + +@pytest.mark.parametrize("calibration_settings", [None, {"enable": False}]) +def test_preflight_leaves_non_calibration_cleanup_unchanged( + monkeypatch, calibration_settings +): + monkeypatch.setattr( + "activitysim.core.calibration.orchestrator.read_calibration_settings", + lambda _state: pytest.fail("non-calibration settings must not be validated"), + ) + state = _PreflightState(calibration_settings) + + assert calibration_run_should_preserve_outputs(state) is False + + @pytest.mark.parametrize( ( "progress", diff --git a/docs/users-guide/calibration.rst b/docs/users-guide/calibration.rst index ede52d8cea..d8ec7bce43 100644 --- a/docs/users-guide/calibration.rst +++ b/docs/users-guide/calibration.rst @@ -134,7 +134,9 @@ Configuration Reference - Every component listed in ``run.calibrate_models`` must have a corresponding entry in ``model_settings``. +- Component names in ``run.calibrate_models`` must be unique. - ``run.global_iterations`` must be ≥ 1. +- Each component's ``submodel_max_iterations`` must be ≥ 1. - Unknown fields are rejected at every level of ``calibration.yaml``, following the same strict settings-model convention used by checked ActivitySim model settings. When the settings checker is enabled, calibration validation errors @@ -653,15 +655,15 @@ A coefficient is considered **converged** when: .. math:: - \text{target_value} - \text{model_value} \leq \text{tolerance} + \left|\text{target_value} - \text{model_value}\right| \leq \text{tolerance} A component is converged when **all** of its coefficients are converged. The component inner loop stops early upon convergence. -The overall calibration run completes after all ``global_iterations`` have -executed. Global convergence is tracked but does not currently trigger early -termination of the outer loop — use ``global_iterations`` to control the total -number of passes. +The outer loop also stops early when every calibrated component converges in +the same global iteration. Otherwise, it runs until ``global_iterations`` have +completed. In either case, the remaining downstream models run once with the +final coefficient values before the calibration run is marked complete. Coefficient Requirements From 6c59ee2b2bab16ef58b6be1ef4bf1640266ff788 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 26 Aug 2026 14:52:37 -0700 Subject: [PATCH 89/96] addressing RNG and resume review comments --- activitysim/core/calibration/component.py | 5 +- activitysim/core/calibration/multiprocess.py | 67 ++++++++--- activitysim/core/calibration/orchestrator.py | 10 +- .../core/calibration/test/test_calibration.py | 109 +++++++++++++++++- activitysim/core/random.py | 2 +- activitysim/core/test/extensions/steps.py | 8 ++ activitysim/core/test/test_pipeline.py | 26 +++++ activitysim/core/workflow/runner.py | 33 ++++-- 8 files changed, 226 insertions(+), 34 deletions(-) diff --git a/activitysim/core/calibration/component.py b/activitysim/core/calibration/component.py index c82577c351..9d32d61b1c 100644 --- a/activitysim/core/calibration/component.py +++ b/activitysim/core/calibration/component.py @@ -74,7 +74,10 @@ def _run_component_model( for model_name in extra_models: state.run.by_name(model_name) state.checkpoint.add(prior_step) - state.run.by_name(run_model_name) + # The labeled invocation keeps calibration checkpoints unique, while the + # canonical component RNG name supplies common random numbers across + # component iterations, global iterations, and recovery attempts. + state.run.by_name_with_rng(run_model_name, rng_step_name=component_name) def _calibrate_component( diff --git a/activitysim/core/calibration/multiprocess.py b/activitysim/core/calibration/multiprocess.py index 6815ae604b..c188d1bb13 100644 --- a/activitysim/core/calibration/multiprocess.py +++ b/activitysim/core/calibration/multiprocess.py @@ -188,8 +188,8 @@ def _restore_from_subprocess_pipelines( from activitysim.core.workflow.checkpoint import ( CHECKPOINT_NAME, CHECKPOINT_TABLE_NAME, - HdfStore, NON_TABLE_COLUMNS, + HdfStore, ParquetStore, ) @@ -313,35 +313,64 @@ def _subprocess_path(proc_name): for table_name, dfs in omnibus.items(): tables[table_name] = pd.concat(dfs, sort=False) - # Load into parent state - prior_rng_channels = list(state.get_injectable("rng_channels", [])) - prior_index_to_channel = ( - dict(state.rng().index_to_channel) - if hasattr(state.rng(), "index_to_channel") - else {} + _install_restored_subprocess_state(state, tables, resume_after) + + logger.info( + "calibration: restored %d tables from subprocess pipelines at " + "checkpoint '%s'", + len(tables), + resume_after, ) + return True - state.init_state() + +def _install_restored_subprocess_state( + state: workflow.State, + tables: dict[str, pd.DataFrame], + checkpoint_name: str, +) -> None: + """Install an exact subprocess snapshot and make it a parent checkpoint.""" + restored_table_names = set(tables) + + # init_state() resets salient-table and RNG bookkeeping but deliberately + # retains cached context values. Remove later tables before that reset so + # they cannot be returned from the context after this rewind. + stale_table_names = set(state.registered_tables()) - restored_table_names + prior_rng_channels = [ + channel_name + for channel_name in state.get_injectable("rng_channels", []) + if channel_name in restored_table_names + ] + prior_index_to_channel = { + index_name: channel_name + for index_name, channel_name in getattr( + state.rng(), "index_to_channel", {} + ).items() + if channel_name in restored_table_names + } + + for table_name in stale_table_names: + if table_name in state: + state.drop(table_name) + if table_name in state.rng().channels: + state.rng().drop_channel(table_name) + + # Close the current handle before init_state() forgets it. if state.checkpoint.store_is_open(): state.checkpoint.close_store() + state.init_state() state.checkpoint.open_store(overwrite=False) for table_name, df in tables.items(): state.add_table(table_name, df) + state.add_injectable("rng_channels", prior_rng_channels) _reregister_rng_channels(state, prior_rng_channels, prior_index_to_channel) - # Mark all tables dirty for subsequent checkpoint.add - for table_name in list(state.existing_table_names): - state.existing_table_status[table_name] = True - - logger.info( - "calibration: restored %d tables from subprocess pipelines at " - "checkpoint '%s'", - len(tables), - resume_after, - ) - return True + # Persist the reconstructed model-level state in the parent pipeline. This + # both makes resume_after observable to the caller and avoids repeating the + # subprocess coalesce on later restores. + state.checkpoint.add(checkpoint_name) def _run_multiprocess_with_overrides( diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py index a7a8196a34..5c381912d1 100644 --- a/activitysim/core/calibration/orchestrator.py +++ b/activitysim/core/calibration/orchestrator.py @@ -455,7 +455,15 @@ def run_calibration_loop( shared_data_buffers=shared_data_buffers, ) else: - state.checkpoint.add(state.settings.resume_after) + # Direct restores already end at this checkpoint, while a + # subprocess restore now persists it in the parent pipeline. + # Only add it here for an alternate restore implementation + # that made the checkpoint visible but did not select it. + if ( + state.checkpoint.last_checkpoint.get("checkpoint_name") + != state.settings.resume_after + ): + state.checkpoint.add(state.settings.resume_after) state.checkpoint.close_store() for global_iter in range( diff --git a/activitysim/core/calibration/test/test_calibration.py b/activitysim/core/calibration/test/test_calibration.py index 6cd47b9885..a3abb7d096 100644 --- a/activitysim/core/calibration/test/test_calibration.py +++ b/activitysim/core/calibration/test/test_calibration.py @@ -1,15 +1,18 @@ from __future__ import annotations -import math import copy +import math +from pathlib import Path import pandas as pd import pytest -from pathlib import Path from pydantic import ValidationError from activitysim.core.calibration.component import _evaluate_and_update from activitysim.core.calibration.expressions import _compute_delta +from activitysim.core.calibration.multiprocess import ( + _install_restored_subprocess_state, +) from activitysim.core.calibration.orchestrator import ( _components_ran_for_convergence, ) @@ -19,6 +22,7 @@ _read_component_iteration_records, ) from activitysim.core.calibration.settings import CalibrationConfig +from activitysim.core.random import Random @pytest.mark.parametrize( @@ -297,6 +301,107 @@ def test_calibration_models_must_be_unique(): CalibrationConfig.model_validate(settings) +class _RestoreCheckpoint: + def __init__(self, state): + self.state = state + self.is_open = True + self.added = [] + self.last_checkpoint = {} + + def initialize(self): + self.is_open = False + self.last_checkpoint = {} + + def store_is_open(self): + return self.is_open + + def close_store(self): + self.is_open = False + + def open_store(self, overwrite=False): + assert overwrite is False + self.is_open = True + + def add(self, checkpoint_name): + self.added.append(checkpoint_name) + self.last_checkpoint = {"checkpoint_name": checkpoint_name} + for table_name in self.state.existing_table_status: + self.state.existing_table_status[table_name] = False + + +class _RestoreState: + def __init__(self): + households = pd.DataFrame(index=pd.Index([1], name="household_id")) + tours = pd.DataFrame(index=pd.Index([10], name="tour_id")) + self.context = { + "households": households, + "tours": tours, + "rng_channels": ["households", "tours"], + } + self.existing_table_status = {"households": False, "tours": False} + self._rng = Random() + self._rng.add_channel("households", households) + self._rng.add_channel("tours", tours) + self.checkpoint = _RestoreCheckpoint(self) + + def __contains__(self, key): + return key in self.context + + def registered_tables(self): + return [name for name in self.existing_table_status if name in self.context] + + def get_injectable(self, name, default=None): + return self.context.get(name, default) + + def add_injectable(self, name, value): + self.context[name] = value + + def rng(self): + return self._rng + + def drop(self, name): + del self.context[name] + + def init_state(self): + # Match workflow.State: reset bookkeeping and RNG, retain context. + self.checkpoint.initialize() + self._rng = Random() + self.existing_table_status = {} + + def add_table(self, name, table): + self.context[name] = table + self.existing_table_status[name] = True + + def is_table(self, name): + return name in self.existing_table_status + + def get_dataframe(self, name): + return self.context[name].copy() + + +def test_subprocess_restore_is_exact_and_durable(): + state = _RestoreState() + restored_households = pd.DataFrame( + {"value": [2]}, index=pd.Index([1], name="household_id") + ) + + _install_restored_subprocess_state( + state, + tables={"households": restored_households}, + checkpoint_name="model_a", + ) + + assert "tours" not in state.context + assert "tours" not in state.rng().channels + assert "tour_id" not in state.rng().index_to_channel + assert state.get_injectable("rng_channels") == ["households"] + pd.testing.assert_frame_equal( + state.get_dataframe("households"), restored_households + ) + assert state.checkpoint.last_checkpoint == {"checkpoint_name": "model_a"} + assert state.checkpoint.added == ["model_a"] + + class _State: def __init__(self, output_dir: Path): self.output_dir = output_dir diff --git a/activitysim/core/random.py b/activitysim/core/random.py index 9c87b7abd3..3a71acd177 100644 --- a/activitysim/core/random.py +++ b/activitysim/core/random.py @@ -816,7 +816,7 @@ def drop_channel(self, channel_name): for index_name, mapped_channel_name in self.index_to_channel.items() if mapped_channel_name != channel_name } - + # Also clear any index_to_channel entries that pointed at the # dropped channel; a stale mapping would otherwise survive and # could mis-route a subsequent channel registered against the diff --git a/activitysim/core/test/extensions/steps.py b/activitysim/core/test/extensions/steps.py index 0ac63f0952..cf59fb5d19 100644 --- a/activitysim/core/test/extensions/steps.py +++ b/activitysim/core/test/extensions/steps.py @@ -58,3 +58,11 @@ def create_households(state: workflow.State) -> None: state.get_rn_generator().add_channel("households", df) state.tracing.register_traceable_table("households", df) + + +@workflow.step +def record_random_draw(state: workflow.State) -> None: + """Record one global RNG draw for runner stream-name tests.""" + draws = list(state.get_injectable("recorded_random_draws", [])) + draws.append(state.get_rn_generator().get_global_rng().rand()) + state.add_injectable("recorded_random_draws", draws) diff --git a/activitysim/core/test/test_pipeline.py b/activitysim/core/test/test_pipeline.py index 95dca6e436..2c9471bfa6 100644 --- a/activitysim/core/test/test_pipeline.py +++ b/activitysim/core/test/test_pipeline.py @@ -144,6 +144,32 @@ def test_get_table_returns_current_table_after_recreation(state): close_handlers() +def test_runner_rng_name_override_is_explicit(state): + state.run.by_name("_record_random_draw.label=one") + state.run.by_name("_record_random_draw.label=two") + default_one, default_two = state.get_injectable("recorded_random_draws") + + # Normal parameterized invocations retain distinct ActivitySim streams. + assert default_one != default_two + + state.run.by_name_with_rng( + "_record_random_draw.calibration=one", + rng_step_name="record_random_draw", + ) + state.run.by_name_with_rng( + "_record_random_draw.calibration=two", + rng_step_name="record_random_draw", + ) + override_one, override_two = state.get_injectable("recorded_random_draws")[-2:] + + # Calibration can explicitly request common random numbers while keeping + # distinct invocation names for logging and checkpoint management. + assert override_one == override_two + + state.checkpoint.close_store() + close_handlers() + + # if __name__ == "__main__": # # print "\n\ntest_pipeline_run" diff --git a/activitysim/core/workflow/runner.py b/activitysim/core/workflow/runner.py index 14ef972049..e144d9e4b7 100644 --- a/activitysim/core/workflow/runner.py +++ b/activitysim/core/workflow/runner.py @@ -5,6 +5,7 @@ import time from collections.abc import Callable, Iterable from datetime import timedelta +from typing import Any from activitysim.core import tracing from activitysim.core.exceptions import DuplicateWorkflowNameError @@ -246,7 +247,9 @@ def log_runtime(self, model_name, start_time=None, timing=None, force=False): self.timing_notes.clear() - def _pre_run_step(self, model_name: str) -> bool | None: + def _pre_run_step( + self, model_name: str, rng_step_name: str | None = None + ) -> bool | None: """ Parameters @@ -270,11 +273,6 @@ def _pre_run_step(self, model_name: str) -> bool | None: f"Cannot run model '{model_name}' more than once" ) - # Parse the canonical workflow step before initializing the RNG. - # Arguments appended to a model invocation (for example calibration - # iteration labels) may affect logging and checkpoint names, but must - # never affect the deterministic random stream for the model itself. - # check for args if "." in model_name: step_name, arg_string = model_name.split(".", 1) @@ -288,9 +286,11 @@ def _pre_run_step(self, model_name: str) -> bool | None: step_name = model_name args = {} - self.rng_step_name = ( - step_name[1:] if step_name.startswith(NO_CHECKPOINT_PREFIX) else step_name - ) + # Preserve ActivitySim's normal behavior: the complete invocation name, + # including arguments and a no-checkpoint prefix, identifies its random + # stream. Specialized callers such as calibration may explicitly reuse + # a canonical stream while retaining a unique checkpoint/logging name. + self.rng_step_name = model_name if rng_step_name is None else rng_step_name self._obj.rng().begin_step(self.rng_step_name) # check for no_checkpoint prefix @@ -321,10 +321,23 @@ def by_name(self, model_name, **kwargs): model_name : str model_name is assumed to be the name of a registered workflow step """ + return self._by_name(model_name, rng_step_name=None, **kwargs) + + def by_name_with_rng(self, model_name: str, rng_step_name: str, **kwargs) -> None: + """Run a model using an explicit deterministic random-stream name. + + This specialized entry point lets calibration retain a unique labeled + invocation for checkpointing and logging while reusing the canonical + component's random stream. Ordinary ``by_name`` callers retain the + complete invocation name as their stream identifier. + """ + return self._by_name(model_name, rng_step_name=rng_step_name, **kwargs) + + def _by_name(self, model_name: str, rng_step_name: str | None, **kwargs) -> None: self.t0 = time.time() self.rng_step_name = None try: - should_skip = self._pre_run_step(model_name) + should_skip = self._pre_run_step(model_name, rng_step_name=rng_step_name) if should_skip: return From 0b953b2bc35c6fab394f87fcfcfb7992f3ebc295 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:10:13 -0700 Subject: [PATCH 90/96] clearer logging at max iterations --- activitysim/cli/run.py | 40 +++++++++++++++++--- activitysim/core/calibration/orchestrator.py | 6 +++ activitysim/core/calibration/settings.py | 2 + 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/activitysim/cli/run.py b/activitysim/cli/run.py index 49a048a53e..c6795a567c 100644 --- a/activitysim/cli/run.py +++ b/activitysim/cli/run.py @@ -424,18 +424,46 @@ def run(args): try: if calibration.calibration_enabled(state): - logger.info("run calibration workflow") + logger.info("evaluate calibration workflow") calibration_result = calibration.run_calibration_loop( state=state, models=state.settings.models, ) - logger.info( - "calibration workflow complete converged=%s completed_global_iterations=%s", - calibration_result.converged, - calibration_result.completed_global_iterations, - ) + if calibration_result.model_system_ran: + logger.info( + "calibration workflow complete converged=%s " + "completed_global_iterations=%s configured_global_iterations=%s", + calibration_result.converged, + calibration_result.completed_global_iterations, + calibration_result.configured_global_iterations, + ) + else: + if ( + calibration_result.completed_global_iterations + >= calibration_result.configured_global_iterations + ): + logger.info( + "calibration workflow skipped: no model system steps were run " + "because completed_global_iterations=%s has reached the " + "configured run.global_iterations=%s limit. Increase " + "run.global_iterations to request additional iterations, or " + "remove output/calibration/calibration_progress.json to start a fresh run.", + calibration_result.completed_global_iterations, + calibration_result.configured_global_iterations, + ) + else: + logger.info( + "calibration workflow skipped: no model system steps were run " + "because the prior calibration run is already complete " + "(completed_global_iterations=%s, run.global_iterations=%s). " + "Change run.global_iterations to a new value above the completed " + "count to request additional iterations, or remove " + "output/calibration/calibration_progress.json to start a fresh run.", + calibration_result.completed_global_iterations, + calibration_result.configured_global_iterations, + ) if state.settings.cleanup_pipeline_after_run: state.checkpoint.cleanup() diff --git a/activitysim/core/calibration/orchestrator.py b/activitysim/core/calibration/orchestrator.py index 5c381912d1..6b88ceb19b 100644 --- a/activitysim/core/calibration/orchestrator.py +++ b/activitysim/core/calibration/orchestrator.py @@ -276,6 +276,8 @@ def run_calibration_loop( return CalibrationRunResult( converged=bool(progress.get("converged", False)), completed_global_iterations=restart_plan.completed_global_iterations, + configured_global_iterations=(calibration_settings.run.global_iterations), + model_system_ran=False, ) # Validate the requested restart before changing a formerly complete @@ -400,6 +402,9 @@ def run_calibration_loop( return CalibrationRunResult( converged=converged, completed_global_iterations=completed_global_iterations, + configured_global_iterations=( + calibration_settings.run.global_iterations + ), ) # skip precursors if, on first iter, resume_after exists and is >= first_calib_model_idx @@ -655,6 +660,7 @@ def run_calibration_loop( return CalibrationRunResult( converged=all_converged, completed_global_iterations=completed_global_iterations, + configured_global_iterations=calibration_settings.run.global_iterations, ) finally: state.filesystem.pipeline_file_name = original_pipeline_name diff --git a/activitysim/core/calibration/settings.py b/activitysim/core/calibration/settings.py index d35c4cd50e..15e40c78ad 100644 --- a/activitysim/core/calibration/settings.py +++ b/activitysim/core/calibration/settings.py @@ -95,6 +95,8 @@ class CalibrationRunResult: converged: bool completed_global_iterations: int + configured_global_iterations: int + model_system_ran: bool = True def read_calibration_settings(state: workflow.State) -> CalibrationConfig | None: From cb2790dcd7908c329602f421fccf4e8847b391bb Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Mon, 21 Sep 2026 12:37:54 -0500 Subject: [PATCH 91/96] Update calibration readme --- docs/users-guide/calibration.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/users-guide/calibration.rst b/docs/users-guide/calibration.rst index d8ec7bce43..e17c2d1259 100644 --- a/docs/users-guide/calibration.rst +++ b/docs/users-guide/calibration.rst @@ -163,9 +163,11 @@ Configuration Reference * - ``global_iterations`` - ``int`` - ``1`` - - Number of full outer-loop calibration passes over all components. Each - global iteration re-runs all precursor models and re-calibrates all - components from scratch using the latest coefficients. + - Number of full outer-loop calibration passes over all components. + Initially, all precursor models are run to generate starting values. + Each global iteration then re-runs and re-calibrates all components + specified in ``calibrate_models``, plus any intermediate components, + from scratch using the latest coefficients. * - ``complete_steps`` - ``bool`` - ``False`` From b05fda8504bfbfc08967ce4b2ea666d81b72306e Mon Sep 17 00:00:00 2001 From: Jeff Newman Date: Mon, 21 Sep 2026 16:54:36 -0500 Subject: [PATCH 92/96] Fix calibration diagnostics after the final component simulation At the component iteration limit, calibration already ran an extra simulation with the newly persisted coefficients. However, that call bypassed expression evaluation and reporting, leaving convergence status, histories, and reports describing the preceding simulation. Final model outputs could match a target while calibration still reported failure. Move the existing final simulation into an evaluation-only loop pass. Rebuild the expression context, evaluate targets and convergence, and refresh histories and enabled generic and bespoke reports. Skip coefficient adjustments, bound clipping, and coefficient persistence on this pass so reported results describe the coefficients actually simulated. Give the final evaluation its own history index at submodel_max_iterations + 1, preserving the preceding coefficient update and recording zero final coefficient change. Retain the existing count of calibration update iterations and the early-convergence behavior. Add regression coverage for final-pass convergence and nonconvergence, early convergence, refreshed reports and CSV histories, and evaluation without coefficient adjustment or bound clipping. Document the final-pass behavior and history numbering. Validation: 39 calibration unit tests and 6 integration tests passed, including single/multiprocess equivalence, restart, continuation, and rewind scenarios. Black and git diff --check passed. --- activitysim/core/calibration/component.py | 85 +++++----- .../core/calibration/test/test_calibration.py | 146 +++++++++++++++++- docs/users-guide/calibration.rst | 8 + 3 files changed, 192 insertions(+), 47 deletions(-) diff --git a/activitysim/core/calibration/component.py b/activitysim/core/calibration/component.py index 9d32d61b1c..4b947d5a76 100644 --- a/activitysim/core/calibration/component.py +++ b/activitysim/core/calibration/component.py @@ -145,11 +145,17 @@ def _calibrate_component( else: mp_restore_checkpoint = "_" - for component_iter in range(1, component_settings.submodel_max_iterations + 1): - component_iterations = component_iter - run_model_name = ( - f"{component_name}.c_i{component_iter};" f"g_i{global_iter};a_i{attempt}" - ) + # The last pass evaluates the final updated coefficients without changing + # them again. Give it its own history index to retain the last update. + for component_iter in range(1, component_settings.submodel_max_iterations + 2): + evaluation_only = component_iter > component_settings.submodel_max_iterations + if evaluation_only: + run_model_name = f"{component_name}.c_final;g_i{global_iter};a_i{attempt}" + else: + component_iterations = component_iter + run_model_name = ( + f"{component_name}.c_i{component_iter};g_i{global_iter};a_i{attempt}" + ) _run_component_model( state=state, component_name=component_name, @@ -179,11 +185,13 @@ def _calibrate_component( global_iter=global_iter, component_iter=component_iter, attempt=attempt, + update_coefficients=not evaluation_only, ) coefficients_df = new_coefficients_df - _persist_coefficients_to_config(state, model_settings, coefficients_df) + if not evaluation_only: + _persist_coefficients_to_config(state, model_settings, coefficients_df) _append_iteration_records(state, component_name, row_records) _append_summary_records(state, [summary_record]) @@ -214,21 +222,6 @@ def _calibrate_component( if component_converged: break - if component_iter == component_settings.submodel_max_iterations: - # The update just persisted above has not yet been simulated. Run - # the component once more so final pipeline tables and downstream - # models use the coefficient values left in the config file. - _run_component_model( - state=state, - component_name=component_name, - run_model_name=( - f"{component_name}.c_final;g_i{global_iter};a_i{attempt}" - ), - prior_step=prior_step, - mp_restore_checkpoint=mp_restore_checkpoint, - shared_data_buffers=shared_data_buffers, - ) - state.checkpoint.add(component_name) return CalibrationComponentResult( @@ -420,8 +413,9 @@ def _evaluate_and_update( global_iter: int, component_iter: int, attempt: int = 1, + update_coefficients: bool = True, ) -> tuple[list[dict[str, Any]], dict[str, Any], pd.DataFrame, bool]: - """Evaluate spec rows, update coefficients, and return detailed records.""" + """Evaluate spec rows and optionally update coefficients, returning records.""" updated = coefficients_df.copy() records: list[dict[str, Any]] = [] @@ -467,33 +461,32 @@ def _evaluate_and_update( tolerance = float(row["tolerance"]) converged = abs(difference) <= tolerance - damping = float(row["damping"]) - raw_delta = _compute_delta( - method=method, - model_value=model_value, - target_value=target_value, - damping=damping, - component_name=component_name, - description=description, - default_increment=default_increment, - ) - - candidate_value = ( - prev_value if hold_fast or converged else prev_value + raw_delta - ) - - at_min = False - at_max = False - + candidate_value = prev_value lower = row["min"] upper = row["max"] + if update_coefficients: + damping = float(row["damping"]) + raw_delta = _compute_delta( + method=method, + model_value=model_value, + target_value=target_value, + damping=damping, + component_name=component_name, + description=description, + default_increment=default_increment, + ) + + candidate_value = ( + prev_value if hold_fast or converged else prev_value + raw_delta + ) + + if not pd.isna(lower) and candidate_value <= float(lower): + candidate_value = float(lower) + if not pd.isna(upper) and candidate_value >= float(upper): + candidate_value = float(upper) - if not pd.isna(lower) and candidate_value <= float(lower): - candidate_value = float(lower) - at_min = True - if not pd.isna(upper) and candidate_value >= float(upper): - candidate_value = float(upper) - at_max = True + at_min = not pd.isna(lower) and candidate_value <= float(lower) + at_max = not pd.isna(upper) and candidate_value >= float(upper) if not np.isfinite(candidate_value): raise RuntimeError( diff --git a/activitysim/core/calibration/test/test_calibration.py b/activitysim/core/calibration/test/test_calibration.py index a3abb7d096..e7b1366975 100644 --- a/activitysim/core/calibration/test/test_calibration.py +++ b/activitysim/core/calibration/test/test_calibration.py @@ -3,11 +3,14 @@ import copy import math from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock import pandas as pd import pytest from pydantic import ValidationError +from activitysim.core.calibration import component from activitysim.core.calibration.component import _evaluate_and_update from activitysim.core.calibration.expressions import _compute_delta from activitysim.core.calibration.multiprocess import ( @@ -21,7 +24,10 @@ _coefficient_trajectory, _read_component_iteration_records, ) -from activitysim.core.calibration.settings import CalibrationConfig +from activitysim.core.calibration.settings import ( + CalibrationComponentSettings, + CalibrationConfig, +) from activitysim.core.random import Random @@ -450,3 +456,141 @@ def test_recovery_attempts_preserve_complete_coefficient_trajectory(tmp_path): # labels: one "Start" entry (initial value) plus one label per recorded iteration assert labels == ["Start", "G1-A1-C1", "G1-A2-C1"] assert list(trajectory["coef_a"]) == [1.0, 1.5, 1.75] + + +@pytest.mark.parametrize( + "model_values, expected_converged", + [([0.25, 0.5], True), ([0.25, 0.75], False), ([0.5], True)], +) +def test_component_reports_final_simulation( + monkeypatch, tmp_path, model_values, expected_converged +): + """Report final results without losing the last update or updating again.""" + coefficients = pd.DataFrame({"value": [0.0]}, index=["coef"]) + spec = pd.DataFrame( + [ + dict( + description="target", + coefficient="coef", + model_value="modeled", + target_value=0.5, + hold_fast=False, + min=-10.0, + max=10.0, + damping=1.0, + method="log_ratio", + tolerance=0.01, + ) + ] + ) + state = SimpleNamespace( + settings=SimpleNamespace(multiprocess=False), + filesystem=SimpleNamespace( + read_model_settings=Mock(return_value={}), + read_model_coefficients=Mock(return_value=coefficients), + ), + checkpoint=SimpleNamespace(add=Mock()), + get_output_file_path=lambda name: tmp_path / name, + ) + settings = CalibrationComponentSettings( + calibration_spec="calibration.csv", submodel_max_iterations=1 + ) + bespoke = Mock() + generic = Mock() + persist = Mock() + runs = [] + values = iter(model_values) + + def run_model(**kwargs): + runs.append(kwargs["run_model_name"]) + state.modeled = next(values) + + monkeypatch.setattr(component, "_run_component_model", run_model) + monkeypatch.setattr(component, "_read_calibration_spec", lambda *args: spec) + monkeypatch.setattr( + component, "_extract_utility_coefficient_names", lambda *args: {"coef"} + ) + monkeypatch.setattr( + component, "_load_helper_symbols", lambda *args: ({}, bespoke, None) + ) + monkeypatch.setattr( + component, + "_build_expression_context", + lambda *args: {"modeled": state.modeled}, + ) + monkeypatch.setattr(component, "_persist_coefficients_to_config", persist) + monkeypatch.setattr(component, "_write_generic_report", generic) + + result = component._calibrate_component( + state, "test_component", settings, "prior_step", global_iter=2, attempt=3 + ) + + assert result.converged is expected_converged + assert result.component_iterations == 1 + assert len(runs) == len(model_values) + if len(model_values) == 2: + assert runs[-1] == "test_component.c_final;g_i2;a_i3" + persist.assert_called_once() + persisted = persist.call_args.args[2] + expected_coefficient = math.log(2) if len(model_values) == 2 else 0.0 + assert persisted.loc["coef", "value"] == pytest.approx(expected_coefficient) + + history = pd.read_csv(tmp_path / "calibration/calibration_iteration_records.csv") + local_history = pd.read_csv( + tmp_path / "calibration/test_component/calibration_iteration_records.csv" + ) + pd.testing.assert_frame_equal(history, local_history) + assert history["model_value"].tolist() == model_values + assert history["component_iter"].tolist() == list(range(1, len(model_values) + 1)) + final = history.iloc[-1] + assert bool(final["converged"]) is expected_converged + assert final["prev_coefficient"] == pytest.approx(expected_coefficient) + assert final["next_coefficient"] == pytest.approx(expected_coefficient) + assert final["coef_delta"] == 0.0 + if len(model_values) == 2: + assert history.iloc[0]["coef_delta"] == pytest.approx(math.log(2)) + + summary = pd.read_csv(tmp_path / "calibration/calibration_iteration_summary.csv") + assert len(summary) == len(model_values) + assert summary.iloc[-1]["num_unconverged"] == int(not expected_converged) + assert summary.iloc[-1]["max_difference"] == abs(0.5 - model_values[-1]) + assert summary.iloc[-1]["max_change"] == 0.0 + assert generic.call_count == bespoke.call_count == len(model_values) + assert generic.call_args.args[2][0]["model_value"] == model_values[-1] + assert bespoke.call_args.args[0] == dict( + modeled=model_values[-1], + calibration_global_iteration=2, + calibration_attempt=3, + calibration_component_iteration=len(model_values), + ) + state.checkpoint.add.assert_called_once_with("test_component") + + +def test_evaluation_only_does_not_clip_or_compute_adjustments(monkeypatch): + coefficients = pd.DataFrame({"value": [5.0]}, index=["coef"]) + spec = pd.DataFrame( + [ + dict( + description="target", + coefficient="coef", + model_value=0.0, + target_value=0.5, + hold_fast=False, + min=-1.0, + max=1.0, + damping=1.0, + method="log_ratio", + tolerance=0.01, + ) + ] + ) + compute_delta = Mock(side_effect=AssertionError("must not compute an update")) + monkeypatch.setattr(component, "_compute_delta", compute_delta) + records, summary, updated, converged = _evaluate_and_update( + "test_component", spec, coefficients, {}, 1, 2, update_coefficients=False + ) + pd.testing.assert_frame_equal(updated, coefficients) + assert not converged + assert records[0]["coef_delta"] == summary["max_change"] == 0.0 + assert records[0]["at_max"] + compute_delta.assert_not_called() diff --git a/docs/users-guide/calibration.rst b/docs/users-guide/calibration.rst index e17c2d1259..a34efcae1f 100644 --- a/docs/users-guide/calibration.rst +++ b/docs/users-guide/calibration.rst @@ -662,6 +662,14 @@ A coefficient is considered **converged** when: A component is converged when **all** of its coefficients are converged. The component inner loop stops early upon convergence. +If the component reaches ``submodel_max_iterations`` without converging, it +runs once more with the last updated coefficients. This final pass evaluates +convergence and refreshes the histories, generic reports, and bespoke reports +without adjusting or rewriting coefficients. Its history entry uses +``component_iter = submodel_max_iterations + 1`` and records zero coefficient +change; the component iteration count still counts only calibration update +passes. + The outer loop also stops early when every calibrated component converges in the same global iteration. Otherwise, it runs until ``global_iterations`` have completed. In either case, the remaining downstream models run once with the From d29dfa6c3c6ccbe47d7fa8e5f2788cd08e159ace Mon Sep 17 00:00:00 2001 From: Jeff Newman Date: Mon, 21 Sep 2026 17:09:53 -0500 Subject: [PATCH 93/96] Preserve configured calibration fallback increments when reading CSVs The calibration CSV reader selected only required columns, silently removing the optional default_increment column. Although the coefficient updater supported this setting, it always received rows without it and fell back to 2.0. A configured increment of 0.25 therefore produced a 2.0 adjustment when a log-ratio or odds-ratio calculation needed the fallback. Retain default_increment during CSV loading and validate it as a finite, nonnegative numeric magnitude. Use 2.0 for omitted columns and blank cells, and preserve explicit zero values so users can disable fallback adjustments for individual rows. Leave the separate fallback damping behavior unchanged. Document these semantics and add CSV-to-update regression coverage for both adjustment methods and directions, custom and zero increments, missing and blank values, and invalid inputs. Validation: all 51 calibration unit tests passed. Black and staged whitespace checks passed. --- activitysim/core/calibration/component.py | 14 ++- .../core/calibration/test/test_calibration.py | 90 +++++++++++++++++++ docs/users-guide/calibration.rst | 7 +- 3 files changed, 108 insertions(+), 3 deletions(-) diff --git a/activitysim/core/calibration/component.py b/activitysim/core/calibration/component.py index 4b947d5a76..423499ed2e 100644 --- a/activitysim/core/calibration/component.py +++ b/activitysim/core/calibration/component.py @@ -242,7 +242,19 @@ def _read_calibration_spec(state: workflow.State, file_name: str) -> pd.DataFram f"calibration_spec {file_name} is missing required columns: {missing}" ) - df = df[CALIBRATION_REQUIRED_COLUMNS].copy() + # Keep the optional fallback magnitude instead of discarding it with + # unrelated columns. Missing columns and blank cells use the default. + if "default_increment" not in df.columns: + df["default_increment"] = DEFAULT_INCREMENT + df = df[CALIBRATION_REQUIRED_COLUMNS + ["default_increment"]].copy() + increments = df["default_increment"].replace(r"^\s*$", np.nan, regex=True) + increments = pd.to_numeric(increments.fillna(DEFAULT_INCREMENT), errors="coerce") + if (~np.isfinite(increments) | (increments < 0)).any(): + raise ValueError( + f"calibration_spec {file_name} default_increment must be finite, " + "numeric, and nonnegative" + ) + df["default_increment"] = increments df["description"] = df["description"].astype(str) df["coefficient"] = df["coefficient"].astype(str) diff --git a/activitysim/core/calibration/test/test_calibration.py b/activitysim/core/calibration/test/test_calibration.py index e7b1366975..989a9e5292 100644 --- a/activitysim/core/calibration/test/test_calibration.py +++ b/activitysim/core/calibration/test/test_calibration.py @@ -594,3 +594,93 @@ def test_evaluation_only_does_not_clip_or_compute_adjustments(monkeypatch): assert records[0]["coef_delta"] == summary["max_change"] == 0.0 assert records[0]["at_max"] compute_delta.assert_not_called() + + +@pytest.fixture +def calibration_spec_csv(tmp_path): + """Write real CSV inputs so tests exercise optional-column parsing.""" + + def read(rows): + path = tmp_path / "calibration.csv" + pd.DataFrame(rows).to_csv(path, index=False) + state = SimpleNamespace( + filesystem=SimpleNamespace(get_config_file_path=lambda name: path) + ) + return component._read_calibration_spec(state, path.name) + + return read + + +@pytest.mark.parametrize( + "method, model_value, target_value, direction", + [ + ("log_ratio", 0.0, 0.5, 1), + ("log_ratio", 0.5, 0.0, -1), + ("odds_ratio", 0.0, 0.5, 1), + ("odds_ratio", 1.0, 0.5, -1), + ], +) +@pytest.mark.parametrize("include_increment", [False, True]) +def test_csv_default_increment_reaches_coefficient_update( + calibration_spec_csv, + method, + model_value, + target_value, + direction, + include_increment, +): + rows = [] + for i, increment in enumerate([0.25, None, 0.0, " "]): + row = dict( + description=f"target {i}", + coefficient=f"coef_{i}", + model_value=model_value, + target_value=target_value, + hold_fast=False, + min=-10.0, + max=10.0, + damping=1.0, + method=method, + tolerance=0.01, + ) + if include_increment: + row["default_increment"] = increment + rows.append(row) + spec = calibration_spec_csv(rows) + magnitudes = [0.25, 2.0, 0.0, 2.0] if include_increment else [2.0] * 4 + assert spec["default_increment"].tolist() == magnitudes + coefficients = pd.DataFrame( + {"value": [1.0] * 4}, index=[f"coef_{i}" for i in range(4)] + ) + + records, _, updated, converged = _evaluate_and_update( + "test_component", spec, coefficients, {}, 1, 1 + ) + + assert not converged + assert updated["value"].tolist() == pytest.approx( + [1.0 + direction * magnitude for magnitude in magnitudes] + ) + assert [row["coef_delta"] for row in records] == pytest.approx(magnitudes) + + +@pytest.mark.parametrize("increment", ["invalid", "inf", "-inf", -0.25]) +def test_csv_rejects_invalid_default_increment(calibration_spec_csv, increment): + row = dict( + description="target", + coefficient="coef", + model_value=0.0, + target_value=0.5, + hold_fast=False, + min=-10.0, + max=10.0, + damping=1.0, + method="log_ratio", + tolerance=0.01, + default_increment=increment, + ) + with pytest.raises( + ValueError, + match="calibration.csv default_increment must be finite, numeric, and nonnegative", + ): + calibration_spec_csv([row]) diff --git a/docs/users-guide/calibration.rst b/docs/users-guide/calibration.rst index a34efcae1f..40e083eb10 100644 --- a/docs/users-guide/calibration.rst +++ b/docs/users-guide/calibration.rst @@ -280,8 +280,11 @@ Optional Columns * - ``default_increment`` - numeric - ``2.0`` - - Fallback delta when the chosen method encounters invalid inputs (e.g., - zero or negative values for ``log_ratio``). + - Finite, nonnegative fallback magnitude when the chosen method encounters + invalid inputs (e.g., zero or negative values for ``log_ratio``). The + adjustment direction is determined by the model and target values. + An omitted column or blank cell uses ``2.0``; ``0`` disables the fallback + adjustment for that row. Comment Rows ------------- From 73451496f7c1b927ae21600e13d4a33632ec4feb Mon Sep 17 00:00:00 2001 From: Jeff Newman Date: Mon, 21 Sep 2026 17:30:23 -0500 Subject: [PATCH 94/96] Preserve held and converged calibration coefficients outside bounds The updater preserved held or converged values before applying bounds, which could still change them. For example, a held coefficient of 5 with bounds [-1, 1] was silently clipped to 1. Clipping a converged coefficient also changed the value used by the simulation that established convergence, leaving the reported result inconsistent with the persisted coefficient. Calculate adjustments and apply bounds only to coefficients eligible for an update: neither held nor converged, and outside an evaluation-only pass. Continue evaluating target differences and reporting bound flags for fixed values; hold_fast does not imply convergence. Retain the existing warning policy for initial values outside bounds and document that held or converged values remain unchanged. Users requiring a hard constraint must correct starting values or bounds before calibration. Bounds continue to clip eligible coefficient updates normally. Add regression coverage for held and converged coefficients above and below bounds, unchanged coefficient records, target convergence status, initial warnings, and clipping of eligible updates in both directions. Validation: all 59 calibration unit tests passed. Black and staged whitespace checks passed. --- activitysim/core/calibration/component.py | 8 +- .../core/calibration/test/test_calibration.py | 88 +++++++++++++++++++ docs/users-guide/calibration.rst | 16 ++++ 3 files changed, 108 insertions(+), 4 deletions(-) diff --git a/activitysim/core/calibration/component.py b/activitysim/core/calibration/component.py index 423499ed2e..ea8ae8cf6c 100644 --- a/activitysim/core/calibration/component.py +++ b/activitysim/core/calibration/component.py @@ -476,7 +476,9 @@ def _evaluate_and_update( candidate_value = prev_value lower = row["min"] upper = row["max"] - if update_coefficients: + # Bounds constrain proposed updates, not held or converged values. + # Clipping a converged value would invalidate its evaluated result. + if update_coefficients and not hold_fast and not converged: damping = float(row["damping"]) raw_delta = _compute_delta( method=method, @@ -488,9 +490,7 @@ def _evaluate_and_update( default_increment=default_increment, ) - candidate_value = ( - prev_value if hold_fast or converged else prev_value + raw_delta - ) + candidate_value = prev_value + raw_delta if not pd.isna(lower) and candidate_value <= float(lower): candidate_value = float(lower) diff --git a/activitysim/core/calibration/test/test_calibration.py b/activitysim/core/calibration/test/test_calibration.py index 989a9e5292..59e07ea689 100644 --- a/activitysim/core/calibration/test/test_calibration.py +++ b/activitysim/core/calibration/test/test_calibration.py @@ -684,3 +684,91 @@ def test_csv_rejects_invalid_default_increment(calibration_spec_csv, increment): match="calibration.csv default_increment must be finite, numeric, and nonnegative", ): calibration_spec_csv([row]) + + +@pytest.mark.parametrize("initial_value", [-5.0, 5.0]) +@pytest.mark.parametrize( + "hold_fast, model_value, expected_converged", + [(True, 0.25, False), (True, 0.5, True), (False, 0.5, True)], +) +def test_bounds_preserve_held_and_converged_coefficients( + monkeypatch, caplog, initial_value, hold_fast, model_value, expected_converged +): + spec = pd.DataFrame( + [ + dict( + description="target", + coefficient="coef", + model_value=model_value, + target_value=0.5, + hold_fast=hold_fast, + min=-1.0, + max=1.0, + damping=1.0, + method="log_ratio", + tolerance=0.01, + ) + ] + ) + coefficients = pd.DataFrame({"value": [initial_value]}, index=["coef"]) + compute_delta = Mock(side_effect=AssertionError("fixed values need no update")) + monkeypatch.setattr(component, "_compute_delta", compute_delta) + + component._warn_if_initial_values_outside_bounds( + "test_component", spec, coefficients + ) + assert "test_component coefficient coef starts" in caplog.text + assert ( + "below min bound" if initial_value < 0 else "above max bound" + ) in caplog.text + records, summary, updated, converged = _evaluate_and_update( + "test_component", spec, coefficients, {}, 1, 1 + ) + + pd.testing.assert_frame_equal(updated, coefficients) + compute_delta.assert_not_called() + assert converged is expected_converged + assert records[0]["converged"] is expected_converged + assert records[0]["prev_coefficient"] == initial_value + assert records[0]["next_coefficient"] == initial_value + assert records[0]["coef_delta"] == summary["max_change"] == 0.0 + assert records[0]["at_min"] == (initial_value < 0) + assert records[0]["at_max"] == (initial_value > 0) + assert summary["num_unconverged"] == int(not expected_converged) + + +@pytest.mark.parametrize( + "initial_value, model_value, target_value, expected_value", + [(0.9, 0.25, 0.5, 1.0), (-0.9, 0.5, 0.25, -1.0)], +) +def test_bounds_still_clip_eligible_updates( + initial_value, model_value, target_value, expected_value +): + spec = pd.DataFrame( + [ + dict( + description="target", + coefficient="coef", + model_value=model_value, + target_value=target_value, + hold_fast=False, + min=-1.0, + max=1.0, + damping=1.0, + method="log_ratio", + tolerance=0.01, + ) + ] + ) + coefficients = pd.DataFrame({"value": [initial_value]}, index=["coef"]) + + records, summary, updated, converged = _evaluate_and_update( + "test_component", spec, coefficients, {}, 1, 1 + ) + + assert updated.loc["coef", "value"] == expected_value + assert not converged + assert records[0]["coef_delta"] == pytest.approx(0.1) + assert summary["max_change"] == pytest.approx(0.1) + assert records[0]["at_min"] == (expected_value == -1.0) + assert records[0]["at_max"] == (expected_value == 1.0) diff --git a/docs/users-guide/calibration.rst b/docs/users-guide/calibration.rst index 40e083eb10..814a76ec3b 100644 --- a/docs/users-guide/calibration.rst +++ b/docs/users-guide/calibration.rst @@ -266,6 +266,22 @@ Required Columns - Absolute difference threshold. A coefficient is "converged" when ``|target_value - model_value| <= tolerance``. +Bounds and Fixed Coefficients +----------------------------- + +Bounds apply only when updating a coefficient that is neither held nor +converged. A coefficient with ``hold_fast: True`` or a model value within +``tolerance`` of its target retains its current value, even if that coefficient +is outside the configured bounds. This preserves the coefficient used by the +simulation that established convergence. + +Starting values outside the bounds generate warnings, not errors. If a held +or converged value violates an intended hard constraint, correct the starting +coefficient or bounds before calibration; the framework will not silently +clip it. Held coefficients still have their target differences evaluated and +are not automatically marked converged. The ``at_min`` and ``at_max`` flags +continue to identify values at or beyond their respective bounds. + Optional Columns ----------------- From 3913e44932cf3ebafc5b8c01fd53b7f06e007a88 Mon Sep 17 00:00:00 2001 From: Jeff Newman Date: Mon, 21 Sep 2026 20:05:09 -0500 Subject: [PATCH 95/96] Validate calibration coefficients through referenced utility template rows Templated models previously accepted every coefficient listed anywhere in COEFFICIENT_TEMPLATE, including entries whose generic names never appeared in the utility specification. This allowed calibration to adjust unused coefficients without affecting the model's utilities. Extract tokens from utility columns first, then map only referenced template rows to their segment-specific coefficient names. Preserve direct-token validation for models without templates and the template reader's handling of blank cells. Labels, descriptions, expressions, and comments do not count as coefficient references in utility columns. Add regression coverage for referenced and unused template rows, multiple segments, blank-cell inheritance, supplemental specifications, and direct coefficient references. Document that this is a static reference check, not a guarantee that a segment occurs in the simulation sample. Validation: 61 calibration unit tests passed. The stricter check detects exactly the six disconnected zero-auto SHARED2 coefficients in the original SANDAG calibration tables; corrected SANDAG configurations pass five tests. Both prototype MTC calibration configurations also validate. Formatting and staged whitespace checks passed. No full regional calibration was run. --- activitysim/core/calibration/component.py | 19 ++++---- .../core/calibration/test/test_calibration.py | 47 +++++++++++++++++++ docs/users-guide/calibration.rst | 8 +++- 3 files changed, 63 insertions(+), 11 deletions(-) diff --git a/activitysim/core/calibration/component.py b/activitysim/core/calibration/component.py index ea8ae8cf6c..d613c85d2c 100644 --- a/activitysim/core/calibration/component.py +++ b/activitysim/core/calibration/component.py @@ -294,16 +294,10 @@ def _extract_utility_coefficient_names( """ Extract coefficient names used by the configured utility specifications. - Templated logit models map utility-spec row labels to actual coefficient - names by segment, so their template cell values are the source of truth. - Other models use coefficient tokens in utility-spec columns. + Templated logit models map generic tokens in utility-spec columns to + coefficient names by segment. Only referenced template rows contribute + names; unused template entries do not affect the utilities. """ - if _setting_value(model_settings, "COEFFICIENT_TEMPLATE"): - template = simulate.read_model_coefficient_template( - state.filesystem, model_settings - ) - return {str(name) for name in template.to_numpy().ravel()} - names: set[str] = set() model_settings_dict = _settings_to_dict(model_settings) @@ -344,6 +338,13 @@ def _extract_utility_coefficient_names( for token in re.findall(r"[A-Za-z_][A-Za-z0-9_]*", text): names.add(token) + if _setting_value(model_settings, "COEFFICIENT_TEMPLATE"): + template = simulate.read_model_coefficient_template( + state.filesystem, model_settings + ) + referenced_rows = template.loc[template.index.isin(names)] + return {str(name) for name in referenced_rows.to_numpy().ravel()} + return names diff --git a/activitysim/core/calibration/test/test_calibration.py b/activitysim/core/calibration/test/test_calibration.py index 59e07ea689..2f696bc7f3 100644 --- a/activitysim/core/calibration/test/test_calibration.py +++ b/activitysim/core/calibration/test/test_calibration.py @@ -772,3 +772,50 @@ def test_bounds_still_clip_eligible_updates( assert summary["max_change"] == pytest.approx(0.1) assert records[0]["at_min"] == (expected_value == -1.0) assert records[0]["at_max"] == (expected_value == 1.0) + + +@pytest.mark.parametrize("templated", [False, True]) +def test_utility_coefficient_names_follow_referenced_template_rows(tmp_path, templated): + # Only alternative columns count. Labels, descriptions, expressions, and + # commented rows must not make an otherwise unused template row reachable. + (tmp_path / "spec.csv").write_text( + "Label,Description,Expression,A,B\n" + "coef_unused,coef_unused,coef_unused,coef_generic + 2 * coef_shared,0\n" + "#comment,ignored,1,coef_unused,coef_unused\n" + ) + (tmp_path / "sample.csv").write_text( + "description,expression,utility\nignored,1,-coef_extra\n" + ) + (tmp_path / "template.csv").write_text( + "coefficient_name,work,school\n" + "coef_generic,coef_work,coef_school\n" + "coef_shared,,\n" + "coef_extra,coef_extra_work,coef_extra_school\n" + "coef_unused,coef_unused_work,coef_unused_school\n" + ) + state = SimpleNamespace( + filesystem=SimpleNamespace(get_config_file_path=lambda name: tmp_path / name) + ) + settings = {"SPEC": "spec.csv", "SAMPLE_SPEC": "sample.csv"} + if templated: + settings["COEFFICIENT_TEMPLATE"] = "template.csv" + names = component._extract_utility_coefficient_names(state, settings) + expected = ( + { + "coef_work", + "coef_school", + "coef_shared", + "coef_extra_work", + "coef_extra_school", + } + if templated + else {"coef_generic", "coef_shared", "coef_extra"} + ) + assert names == expected + component._validate_calibration_coefficients_against_utility_spec( + "test_component", pd.DataFrame({"coefficient": sorted(expected)}), names + ) + with pytest.raises(ValueError, match="coef_unused_work"): + component._validate_calibration_coefficients_against_utility_spec( + "test_component", pd.DataFrame({"coefficient": ["coef_unused_work"]}), names + ) diff --git a/docs/users-guide/calibration.rst b/docs/users-guide/calibration.rst index 814a76ec3b..f3909dd74d 100644 --- a/docs/users-guide/calibration.rst +++ b/docs/users-guide/calibration.rst @@ -702,8 +702,12 @@ Calibration coefficients must satisfy these requirements: 1. **Present in utility specification**: Every ``coefficient`` in the calibration spec must appear as a token in the component's utility expression CSV (the - files referenced by settings keys ending in ``SPEC``). A validation error is - raised at startup if any are missing. + files referenced by settings keys ending in ``SPEC``). For templated models, + a utility-column token must reference a row in ``COEFFICIENT_TEMPLATE`` that + maps to the calibration coefficient. Merely listing a coefficient in an + unused template row is insufficient. A validation error is raised before + the component runs if any are missing. This checks static references; it + does not guarantee that the corresponding segment occurs in the sample. 2. **Present in coefficients file**: If a calibration coefficient is not found in the component's coefficients CSV, it is automatically added with an initial From 16fd863ae386fac23ebde4c8f3009b7cd8b70bef Mon Sep 17 00:00:00 2001 From: Jeff Newman Date: Mon, 21 Sep 2026 21:52:14 -0500 Subject: [PATCH 96/96] Compare estimation parameters in canonical name order Calibration constants were appended to four expected parameter tables, while Larch includes them in alphabetical order. Positional CSV comparisons therefore paired different coefficients and failed despite passing likelihood checks. Reorder those expected rows without changing any values or tolerances, and sort unique-name parameter tables before regression comparison. Preserve the existing grouped order for concatenated segment tables with repeated names. Add focused checks that row reordering passes while missing, extra, renamed, duplicate, and materially changed parameters fail. Check that the four affected baselines retain unique, sorted names. Add a focused Windows estimation workflow using the same locked dependencies as Core Testing. Rebuild estimation data from a fresh cache and upload complete CSV comparisons and JUnit results so numerical discrepancies can be inspected without relying on truncated console output. Local validation: seven parameter-comparison tests passed; all expected CSV row contents are unchanged. Windows estimation validation follows in Actions. --- .github/workflows/estimation-tests.yml | 52 ++++++++ .../estimation/test/test_larch_estimation.py | 4 + ...n_model_workplace_location_SLSQP_None_.csv | 8 +- ...t_simple_simulate_auto_ownership_BHHH_.csv | 8 +- .../test_tour_and_subtour_mode_choice.csv | 120 +++++++++--------- .../test_workplace_location.csv | 8 +- .../test/test_parameter_regression.py | 62 +++++++++ 7 files changed, 190 insertions(+), 72 deletions(-) create mode 100644 .github/workflows/estimation-tests.yml create mode 100644 activitysim/estimation/test/test_parameter_regression.py diff --git a/.github/workflows/estimation-tests.yml b/.github/workflows/estimation-tests.yml new file mode 100644 index 0000000000..74b581aad7 --- /dev/null +++ b/.github/workflows/estimation-tests.yml @@ -0,0 +1,52 @@ +name: Estimation Mode Testing + +on: + workflow_dispatch: + push: + paths: + - 'activitysim/estimation/**' + - '.github/workflows/estimation-tests.yml' + +permissions: + contents: read + +jobs: + estimation_mode: + name: Windows estimation mode tests + runs-on: windows-latest + timeout-minutes: 60 + defaults: + run: + shell: pwsh + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + version: '0.7.12' + enable-cache: true + cache-dependency-glob: uv.lock + - uses: actions/setup-python@v5 + with: + python-version-file: .python-version + - name: Install locked dependencies + run: uv sync --locked + - name: Clear generated estimation data + run: >- + uv run python -c "import pathlib, platformdirs, shutil; + shutil.rmtree(pathlib.Path(platformdirs.user_cache_dir(appname='ActivitySim')) + / 'estimation-test-base', ignore_errors=True)" + - name: Test estimation mode and parameter comparisons + run: >- + uv run pytest activitysim/estimation/test/test_larch_estimation.py + activitysim/estimation/test/test_parameter_regression.py + --durations=0 --basetemp="${{ runner.temp }}/estimation-tests" + --junitxml="${{ runner.temp }}/estimation-results.xml" + - name: Save complete estimation comparisons + if: always() + uses: actions/upload-artifact@v4 + with: + name: estimation-regression-results + path: | + ${{ runner.temp }}/estimation-results.xml + ${{ runner.temp }}/estimation-tests/**/*.csv + retention-days: 14 diff --git a/activitysim/estimation/test/test_larch_estimation.py b/activitysim/estimation/test/test_larch_estimation.py index 09db283665..d158f04108 100644 --- a/activitysim/estimation/test/test_larch_estimation.py +++ b/activitysim/estimation/test/test_larch_estimation.py @@ -35,6 +35,10 @@ def est_data(): def _regression_check(dataframe_regression, df, basename=None, rtol=None): + # Single-model parameters have unique names. The nonmandatory-frequency + # check concatenates segment tables with repeated names; retain its grouping. + if df.index.is_unique: + df = df.sort_index() if rtol is None: rtol = 0.1 dataframe_regression.check( diff --git a/activitysim/estimation/test/test_larch_estimation/test_location_model_workplace_location_SLSQP_None_.csv b/activitysim/estimation/test/test_larch_estimation/test_location_model_workplace_location_SLSQP_None_.csv index 72d28ac897..368f49af43 100644 --- a/activitysim/estimation/test/test_larch_estimation/test_location_model_workplace_location_SLSQP_None_.csv +++ b/activitysim/estimation/test/test_larch_estimation/test_location_model_workplace_location_SLSQP_None_.csv @@ -1,6 +1,10 @@ param_name,value,best,initvalue,nullvalue -999,-999,-999,-999,0 1,1,1,1,0 +coef_calib_dist_0_2,0,0,0,0 +coef_calib_dist_15_up,0,0,0,0 +coef_calib_dist_2_5,0,0,0,0 +coef_calib_dist_5_15,0,0,0,0 coef_dist_0_1,-0.29614981943016305,-0.29614981943016305,-0.84280002117156982,0 coef_dist_0_5_high,-0.037934685148704593,-0.037934685148704593,0.15000000596046448,0 coef_dist_15_up,-0.091700002551078796,-0.091700002551078796,-0.091700002551078796,0 @@ -33,7 +37,3 @@ work_veryhigh_HEREMPN,-5.6489927637825792,-5.6489927637825792,-1.422958374023437 work_veryhigh_MWTEMPN,-6,-6,-1.4024237394332886,0 work_veryhigh_OTHEMPN,-5.8313629173672439,-5.8313629173672439,-1.9241486787796021,0 work_veryhigh_RETEMPN,-2.3751556873321533,-2.3751556873321533,-2.3751556873321533,0 -coef_calib_dist_0_2,0,0,0,0 -coef_calib_dist_2_5,0,0,0,0 -coef_calib_dist_5_15,0,0,0,0 -coef_calib_dist_15_up,0,0,0,0 diff --git a/activitysim/estimation/test/test_larch_estimation/test_simple_simulate_auto_ownership_BHHH_.csv b/activitysim/estimation/test/test_larch_estimation/test_simple_simulate_auto_ownership_BHHH_.csv index 7047821b45..e1e746c0d3 100644 --- a/activitysim/estimation/test/test_larch_estimation/test_simple_simulate_auto_ownership_BHHH_.csv +++ b/activitysim/estimation/test/test_larch_estimation/test_simple_simulate_auto_ownership_BHHH_.csv @@ -1,4 +1,8 @@ param_name,value,initvalue,nullvalue,minimum,maximum,best +coef_calib_auto_0,0,0,0,0,0,0 +coef_calib_auto_2,0,0,0,0,0,0 +coef_calib_auto_3,0,0,0,0,0,0 +coef_calib_auto_4,0,0,0,0,0,0 coef_cars1_asc,4.7447107380921025,1.1865000000000001,0,,,4.7447107380921025 coef_cars1_asc_county,-0.56599882256075029,-0.56599999999999995,0,,,-0.56599882256075029 coef_cars1_asc_marin,-0.24339921273778259,-0.24340000000000001,0,,,-0.24339921273778259 @@ -65,7 +69,3 @@ coef_retail_auto_workers,-0.53111240158828621,0.1646,0,,,-0.53111240158828621 coef_retail_non_motor,-0.029999999999999999,-0.029999999999999999,0,-0.029999999999999999,-0.029999999999999999,-0.029999999999999999 coef_retail_transit_no_workers,-0.33344704526636021,-0.30530000000000002,0,,,-0.33344704526636021 coef_retail_transit_workers,-0.46438215677534161,-0.51170000000000004,0,,,-0.46438215677534161 -coef_calib_auto_0,0,0,0,0,0,0 -coef_calib_auto_2,0,0,0,0,0,0 -coef_calib_auto_3,0,0,0,0,0,0 -coef_calib_auto_4,0,0,0,0,0,0 diff --git a/activitysim/estimation/test/test_larch_estimation/test_tour_and_subtour_mode_choice.csv b/activitysim/estimation/test/test_larch_estimation/test_tour_and_subtour_mode_choice.csv index dee933e942..349af82eea 100644 --- a/activitysim/estimation/test/test_larch_estimation/test_tour_and_subtour_mode_choice.csv +++ b/activitysim/estimation/test/test_larch_estimation/test_tour_and_subtour_mode_choice.csv @@ -39,6 +39,66 @@ coef_age1619_da_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work, coef_age1619_da_multiplier_school_univ,-1.5953290804661184,-1.5953290804661184,-1.3812999725341797,0 coef_age16p_sr_multiplier_eatout_escort_othdiscr_othmaint_shopping_social,-1.1377269298937951,-1.1377269298937951,-1.3660000562667847,0 coef_age16p_sr_multiplier_school_univ_work_atwork,-0.70763139612535009,-0.70763139612535009,0,0 +coef_calib_BIKE_auto_insuff,0,0,0,0 +coef_calib_BIKE_auto_suff,0,0,0,0 +coef_calib_BIKE_zero_auto,0,0,0,0 +coef_calib_DRIVEALONEPAY_auto_insuff,0,0,0,0 +coef_calib_DRIVEALONEPAY_auto_suff,0,0,0,0 +coef_calib_DRIVEALONEPAY_zero_auto,0,0,0,0 +coef_calib_DRIVE_COM_auto_insuff,0,0,0,0 +coef_calib_DRIVE_COM_auto_suff,0,0,0,0 +coef_calib_DRIVE_COM_zero_auto,0,0,0,0 +coef_calib_DRIVE_EXP_auto_insuff,0,0,0,0 +coef_calib_DRIVE_EXP_auto_suff,0,0,0,0 +coef_calib_DRIVE_EXP_zero_auto,0,0,0,0 +coef_calib_DRIVE_HVY_auto_insuff,0,0,0,0 +coef_calib_DRIVE_HVY_auto_suff,0,0,0,0 +coef_calib_DRIVE_HVY_zero_auto,0,0,0,0 +coef_calib_DRIVE_LOC_auto_insuff,0,0,0,0 +coef_calib_DRIVE_LOC_auto_suff,0,0,0,0 +coef_calib_DRIVE_LOC_zero_auto,0,0,0,0 +coef_calib_DRIVE_LRF_auto_insuff,0,0,0,0 +coef_calib_DRIVE_LRF_auto_suff,0,0,0,0 +coef_calib_DRIVE_LRF_zero_auto,0,0,0,0 +coef_calib_SHARED2FREE_auto_insuff,0,0,0,0 +coef_calib_SHARED2FREE_auto_suff,0,0,0,0 +coef_calib_SHARED2FREE_zero_auto,0,0,0,0 +coef_calib_SHARED2PAY_auto_insuff,0,0,0,0 +coef_calib_SHARED2PAY_auto_suff,0,0,0,0 +coef_calib_SHARED2PAY_zero_auto,0,0,0,0 +coef_calib_SHARED3FREE_auto_insuff,0,0,0,0 +coef_calib_SHARED3FREE_auto_suff,0,0,0,0 +coef_calib_SHARED3FREE_zero_auto,0,0,0,0 +coef_calib_SHARED3PAY_auto_insuff,0,0,0,0 +coef_calib_SHARED3PAY_auto_suff,0,0,0,0 +coef_calib_SHARED3PAY_zero_auto,0,0,0,0 +coef_calib_TAXI_auto_insuff,0,0,0,0 +coef_calib_TAXI_auto_suff,0,0,0,0 +coef_calib_TAXI_zero_auto,0,0,0,0 +coef_calib_TNC_SHARED_auto_insuff,0,0,0,0 +coef_calib_TNC_SHARED_auto_suff,0,0,0,0 +coef_calib_TNC_SHARED_zero_auto,0,0,0,0 +coef_calib_TNC_SINGLE_auto_insuff,0,0,0,0 +coef_calib_TNC_SINGLE_auto_suff,0,0,0,0 +coef_calib_TNC_SINGLE_zero_auto,0,0,0,0 +coef_calib_WALK_COM_auto_insuff,0,0,0,0 +coef_calib_WALK_COM_auto_suff,0,0,0,0 +coef_calib_WALK_COM_zero_auto,0,0,0,0 +coef_calib_WALK_EXP_auto_insuff,0,0,0,0 +coef_calib_WALK_EXP_auto_suff,0,0,0,0 +coef_calib_WALK_EXP_zero_auto,0,0,0,0 +coef_calib_WALK_HVY_auto_insuff,0,0,0,0 +coef_calib_WALK_HVY_auto_suff,0,0,0,0 +coef_calib_WALK_HVY_zero_auto,0,0,0,0 +coef_calib_WALK_LOC_auto_insuff,0,0,0,0 +coef_calib_WALK_LOC_auto_suff,0,0,0,0 +coef_calib_WALK_LOC_zero_auto,0,0,0,0 +coef_calib_WALK_LRF_auto_insuff,0,0,0,0 +coef_calib_WALK_LRF_auto_suff,0,0,0,0 +coef_calib_WALK_LRF_zero_auto,0,0,0,0 +coef_calib_WALK_auto_insuff,0,0,0,0 +coef_calib_WALK_auto_suff,0,0,0,0 +coef_calib_WALK_zero_auto,0,0,0,0 coef_hhsize1_sr_multiplier_eatout_escort_othdiscr_othmaint_school_shopping_social_univ_atwork,0.020926329864022951,0.020926329864022951,0,0 coef_hhsize1_sr_multiplier_work,-0.678704061342841,-0.678704061342841,-0.73458802700042725,0 coef_hhsize2_sr_multiplier_eatout_escort_othdiscr_othmaint_shopping_social_work_atwork,-0.0037484351975650902,-0.0037484351975650902,0,0 @@ -300,63 +360,3 @@ walk_transit_CBD_ASC_atwork,0.34851829905878767,0.34851829905878767,0.5640000104 walk_transit_CBD_ASC_eatout_escort_othdiscr_othmaint_shopping_social,0.69144903771302224,0.69144903771302224,0.52499997615814209,0 walk_transit_CBD_ASC_school_univ,0.78893465707209709,0.78893465707209709,0.67199999094009399,0 walk_transit_CBD_ASC_work,0.97702352207787502,0.97702352207787502,0.80400002002716064,0 -coef_calib_DRIVEALONEPAY_zero_auto,0,0,0,0 -coef_calib_SHARED2FREE_zero_auto,0,0,0,0 -coef_calib_SHARED2PAY_zero_auto,0,0,0,0 -coef_calib_SHARED3FREE_zero_auto,0,0,0,0 -coef_calib_SHARED3PAY_zero_auto,0,0,0,0 -coef_calib_WALK_zero_auto,0,0,0,0 -coef_calib_BIKE_zero_auto,0,0,0,0 -coef_calib_WALK_LOC_zero_auto,0,0,0,0 -coef_calib_WALK_LRF_zero_auto,0,0,0,0 -coef_calib_WALK_EXP_zero_auto,0,0,0,0 -coef_calib_WALK_HVY_zero_auto,0,0,0,0 -coef_calib_WALK_COM_zero_auto,0,0,0,0 -coef_calib_DRIVE_LOC_zero_auto,0,0,0,0 -coef_calib_DRIVE_LRF_zero_auto,0,0,0,0 -coef_calib_DRIVE_EXP_zero_auto,0,0,0,0 -coef_calib_DRIVE_HVY_zero_auto,0,0,0,0 -coef_calib_DRIVE_COM_zero_auto,0,0,0,0 -coef_calib_TAXI_zero_auto,0,0,0,0 -coef_calib_TNC_SINGLE_zero_auto,0,0,0,0 -coef_calib_TNC_SHARED_zero_auto,0,0,0,0 -coef_calib_DRIVEALONEPAY_auto_insuff,0,0,0,0 -coef_calib_SHARED2FREE_auto_insuff,0,0,0,0 -coef_calib_SHARED2PAY_auto_insuff,0,0,0,0 -coef_calib_SHARED3FREE_auto_insuff,0,0,0,0 -coef_calib_SHARED3PAY_auto_insuff,0,0,0,0 -coef_calib_WALK_auto_insuff,0,0,0,0 -coef_calib_BIKE_auto_insuff,0,0,0,0 -coef_calib_WALK_LOC_auto_insuff,0,0,0,0 -coef_calib_WALK_LRF_auto_insuff,0,0,0,0 -coef_calib_WALK_EXP_auto_insuff,0,0,0,0 -coef_calib_WALK_HVY_auto_insuff,0,0,0,0 -coef_calib_WALK_COM_auto_insuff,0,0,0,0 -coef_calib_DRIVE_LOC_auto_insuff,0,0,0,0 -coef_calib_DRIVE_LRF_auto_insuff,0,0,0,0 -coef_calib_DRIVE_EXP_auto_insuff,0,0,0,0 -coef_calib_DRIVE_HVY_auto_insuff,0,0,0,0 -coef_calib_DRIVE_COM_auto_insuff,0,0,0,0 -coef_calib_TAXI_auto_insuff,0,0,0,0 -coef_calib_TNC_SINGLE_auto_insuff,0,0,0,0 -coef_calib_TNC_SHARED_auto_insuff,0,0,0,0 -coef_calib_DRIVEALONEPAY_auto_suff,0,0,0,0 -coef_calib_SHARED2FREE_auto_suff,0,0,0,0 -coef_calib_SHARED2PAY_auto_suff,0,0,0,0 -coef_calib_SHARED3FREE_auto_suff,0,0,0,0 -coef_calib_SHARED3PAY_auto_suff,0,0,0,0 -coef_calib_WALK_auto_suff,0,0,0,0 -coef_calib_BIKE_auto_suff,0,0,0,0 -coef_calib_WALK_LOC_auto_suff,0,0,0,0 -coef_calib_WALK_LRF_auto_suff,0,0,0,0 -coef_calib_WALK_EXP_auto_suff,0,0,0,0 -coef_calib_WALK_HVY_auto_suff,0,0,0,0 -coef_calib_WALK_COM_auto_suff,0,0,0,0 -coef_calib_DRIVE_LOC_auto_suff,0,0,0,0 -coef_calib_DRIVE_LRF_auto_suff,0,0,0,0 -coef_calib_DRIVE_EXP_auto_suff,0,0,0,0 -coef_calib_DRIVE_HVY_auto_suff,0,0,0,0 -coef_calib_DRIVE_COM_auto_suff,0,0,0,0 -coef_calib_TAXI_auto_suff,0,0,0,0 -coef_calib_TNC_SINGLE_auto_suff,0,0,0,0 -coef_calib_TNC_SHARED_auto_suff,0,0,0,0 diff --git a/activitysim/estimation/test/test_larch_estimation/test_workplace_location.csv b/activitysim/estimation/test/test_larch_estimation/test_workplace_location.csv index 30c16478da..bde1bf21f0 100644 --- a/activitysim/estimation/test/test_larch_estimation/test_workplace_location.csv +++ b/activitysim/estimation/test/test_larch_estimation/test_workplace_location.csv @@ -1,6 +1,10 @@ param_name,value,best,initvalue,nullvalue -999,-999,-999,-999,0 1,1,1,1,0 +coef_calib_dist_0_2,0,0,0,0 +coef_calib_dist_15_up,0,0,0,0 +coef_calib_dist_2_5,0,0,0,0 +coef_calib_dist_5_15,0,0,0,0 coef_dist_0_1,-0.30581956658755333,-0.30581956658755333,-0.84280002117156982,0 coef_dist_0_5_high,-0.037591580411684179,-0.037591580411684179,0.15000000596046448,0 coef_dist_15_up,-0.091700002551078796,-0.091700002551078796,-0.091700002551078796,0 @@ -33,7 +37,3 @@ work_veryhigh_HEREMPN,-5.6530572756846489,-5.6530572756846489,-1.422958374023437 work_veryhigh_MWTEMPN,-6,-6,-1.4024237394332886,0 work_veryhigh_OTHEMPN,-5.8240005358464035,-5.8240005358464035,-1.9241486787796021,0 work_veryhigh_RETEMPN,-2.3751556873321533,-2.3751556873321533,-2.3751556873321533,0 -coef_calib_dist_0_2,0,0,0,0 -coef_calib_dist_2_5,0,0,0,0 -coef_calib_dist_5_15,0,0,0,0 -coef_calib_dist_15_up,0,0,0,0 diff --git a/activitysim/estimation/test/test_parameter_regression.py b/activitysim/estimation/test/test_parameter_regression.py new file mode 100644 index 0000000000..79c98dceb6 --- /dev/null +++ b/activitysim/estimation/test/test_parameter_regression.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from pathlib import Path + +import pandas as pd +import pytest +from pytest_regressions.dataframe_regression import DataFrameRegressionFixture + +from activitysim.estimation.test.test_larch_estimation import _regression_check + + +@pytest.fixture +def parameter_regression(tmp_path, request): + expected = pd.DataFrame( + {"value": [1.0, 2.0], "initvalue": [0.0, 0.0]}, + index=pd.Index(["coef_a", "coef_b"], name="param_name"), + ) + expected.to_csv(tmp_path / "parameters.csv") + regression = DataFrameRegressionFixture(tmp_path, tmp_path, request) + return regression, expected + + +def test_parameter_regression_accepts_reordered_rows(parameter_regression): + regression, expected = parameter_regression + _regression_check(regression, expected.iloc[::-1], basename="parameters") + + +@pytest.mark.parametrize( + "change", ["missing", "extra", "renamed", "duplicate", "value"] +) +def test_parameter_regression_rejects_changed_parameters(parameter_regression, change): + regression, expected = parameter_regression + actual = expected.copy() + if change == "missing": + actual = actual.iloc[:1] + elif change == "extra": + actual.loc["coef_c"] = [3.0, 0.0] + elif change == "renamed": + actual = actual.rename(index={"coef_b": "coef_c"}) + elif change == "duplicate": + actual.index = ["coef_a", "coef_a"] + else: + actual.loc["coef_b", "value"] = 20.0 + with pytest.raises(AssertionError): + _regression_check(regression, actual, basename="parameters") + + +def test_parameter_baselines_have_unique_sorted_names(): + baseline_dir = Path(__file__).with_name("test_larch_estimation") + for filename in [ + "test_simple_simulate_auto_ownership_BHHH_.csv", + "test_location_model_workplace_location_SLSQP_None_.csv", + "test_workplace_location.csv", + "test_tour_and_subtour_mode_choice.csv", + ]: + path = baseline_dir / filename + baseline = pd.read_csv(path) + if "param_name" not in baseline: + continue + names = baseline["param_name"] + assert names.is_unique, path.name + assert names.tolist() == sorted(names), path.name