From 28a4ab88b1c7ab4198a416554d33a6662c2fc6ff Mon Sep 17 00:00:00 2001 From: JoeJimFlood Date: Mon, 2 Feb 2026 17:14:41 -0800 Subject: [PATCH 1/5] Changes to allow for the use of geographies in between the meta and seed geographies --- populationsim/steps/final_seed_balancing.py | 2 ++ populationsim/steps/integerize_final_seed_weights.py | 3 +++ populationsim/steps/meta_control_factoring.py | 4 ++++ 3 files changed, 9 insertions(+) diff --git a/populationsim/steps/final_seed_balancing.py b/populationsim/steps/final_seed_balancing.py index 6ec058d4..2cfbf601 100644 --- a/populationsim/steps/final_seed_balancing.py +++ b/populationsim/steps/final_seed_balancing.py @@ -60,6 +60,8 @@ def final_seed_balancing(settings, crosswalk, control_spec, incidence_table): # we use all control_spec rows, so no need to filter on geography as for initial_seed_balancing seed_controls_df = get_control_table(seed_geography) + intermediate_geographies = geographies[1:geographies.index(seed_geography)] + control_spec = control_spec[control_spec.geography.apply(lambda x: x not in intermediate_geographies)] assert (seed_controls_df.columns == control_spec.target).all() # determine master_control_index if specified in settings diff --git a/populationsim/steps/integerize_final_seed_weights.py b/populationsim/steps/integerize_final_seed_weights.py index 73f56a58..8b52f1d1 100644 --- a/populationsim/steps/integerize_final_seed_weights.py +++ b/populationsim/steps/integerize_final_seed_weights.py @@ -50,6 +50,9 @@ def integerize_final_seed_weights(settings, crosswalk, control_spec, incidence_t seed_weights_df = get_weight_table(seed_geography) # FIXME - I assume we want to integerize using meta controls too? + geographies = settings["geographies"] + intermediate_geographies = geographies[1:geographies.index(seed_geography)] + control_spec = control_spec[control_spec.geography.apply(lambda x: x not in intermediate_geographies)] control_cols = control_spec.target assert (seed_controls_df.columns == control_cols).all() diff --git a/populationsim/steps/meta_control_factoring.py b/populationsim/steps/meta_control_factoring.py index 394aa18f..96f3254c 100644 --- a/populationsim/steps/meta_control_factoring.py +++ b/populationsim/steps/meta_control_factoring.py @@ -122,6 +122,10 @@ def meta_control_factoring(settings, control_spec, incidence_table): seed_controls_df = get_control_table(seed_geography) + # Remove controls for geographies between the seed and the meta geographies + intermediate_geographies = geographies[1:geographies.index(seed_geography)] + control_spec = control_spec[control_spec.geography.apply(lambda x: x not in intermediate_geographies)] + assert len(seed_controls_df.index) == len(seed_level_meta_controls.index) seed_controls_df = pd.concat([seed_controls_df, seed_level_meta_controls], axis=1) From a8cfc42c1e89abd712fd558e6a65635b368e070d Mon Sep 17 00:00:00 2001 From: JoeJimFlood Date: Wed, 18 Mar 2026 10:31:43 -0700 Subject: [PATCH 2/5] Added test of allowing intermediate geographies --- .../configs_intermediate/controls.csv | 11 + .../configs_intermediate/logging.yaml | 54 ++++ .../configs_intermediate/repop_controls.csv | 3 + .../configs_intermediate/settings.yaml | 133 +++++++++ .../data_intermediate/district_controls.csv | 4 + .../data_intermediate/geo_cross_walk.csv | 39 +++ .../data_intermediate/region_controls.csv | 2 + .../data_intermediate/repop_taz_controls.csv | 5 + .../data_intermediate/seed_households.csv | 63 +++++ .../data_intermediate/seed_persons.csv | 254 ++++++++++++++++++ .../data_intermediate/taz_controls.csv | 37 +++ .../data_intermediate/tract_controls.csv | 13 + tests/test_intermediate_geography.py | 47 ++++ 13 files changed, 665 insertions(+) create mode 100644 examples/example_test/configs_intermediate/controls.csv create mode 100644 examples/example_test/configs_intermediate/logging.yaml create mode 100644 examples/example_test/configs_intermediate/repop_controls.csv create mode 100644 examples/example_test/configs_intermediate/settings.yaml create mode 100644 examples/example_test/data_intermediate/district_controls.csv create mode 100644 examples/example_test/data_intermediate/geo_cross_walk.csv create mode 100644 examples/example_test/data_intermediate/region_controls.csv create mode 100644 examples/example_test/data_intermediate/repop_taz_controls.csv create mode 100644 examples/example_test/data_intermediate/seed_households.csv create mode 100644 examples/example_test/data_intermediate/seed_persons.csv create mode 100644 examples/example_test/data_intermediate/taz_controls.csv create mode 100644 examples/example_test/data_intermediate/tract_controls.csv create mode 100644 tests/test_intermediate_geography.py diff --git a/examples/example_test/configs_intermediate/controls.csv b/examples/example_test/configs_intermediate/controls.csv new file mode 100644 index 00000000..71e3e97c --- /dev/null +++ b/examples/example_test/configs_intermediate/controls.csv @@ -0,0 +1,11 @@ +target,geography,seed_table,importance,control_field,expression +num_hh,TAZ,households,1000000000,HHBASE,(households.WGTP > 0) & (households.WGTP < np.inf) +hh_size_1,TAZ,households,1000,HHSIZE1,households.NP == 1 +hh_size_2,TAZ,households,1000,HHSIZE2,households.NP == 2 +hh_size_3,TAZ,households,1000,HHSIZE3,households.NP == 3 +hh_size_4_plus,TAZ,households,1000,HHSIZE4,households.NP >= 4 +students_by_housing_type,TAZ,persons,1000,OSUFAM,persons.OSUTAG == 1 +hh_by_type,TRACT,households,100,SF,households.HTYPE == 1 +persons_occ_1,DISTRICT,persons,1000,OCCP1,persons.OCCP == 1 +persons_occ_2,REGION,persons,100,OCCP2,persons.OCCP == 2 +persons_occ_3,REGION,persons,100,OCCP3,persons.OCCP == 3 diff --git a/examples/example_test/configs_intermediate/logging.yaml b/examples/example_test/configs_intermediate/logging.yaml new file mode 100644 index 00000000..06baa76a --- /dev/null +++ b/examples/example_test/configs_intermediate/logging.yaml @@ -0,0 +1,54 @@ +# Config for logging +# ------------------ +# See http://docs.python.org/2.7/library/logging.config.html#configuration-dictionary-schema + +logging: + version: 1 + disable_existing_loggers: true + + + # Configuring the default (root) logger is highly recommended + root: + level: NOTSET + handlers: [console] + + loggers: + + populationsim: + level: DEBUG + handlers: [console, logfile] + propagate: false + + orca: + level: WARN + handlers: [console, logfile] + propagate: false + + handlers: + + logfile: + class: logging.FileHandler + filename: !!python/object/apply:populationsim.core.config.log_file_path ['populationsim.log'] + mode: w + formatter: fileFormatter + level: NOTSET + + console: + class: logging.StreamHandler + stream: ext://sys.stdout + formatter: simpleFormatter + level: NOTSET + #level: WARN + + formatters: + + simpleFormatter: + class: logging.Formatter + # format: '%(levelname)s - %(name)s - %(message)s' + format: '%(levelname)s - %(message)s' + datefmt: '%d/%m/%Y %H:%M:%S' + + fileFormatter: + class: logging.Formatter + format: '%(asctime)s - %(levelname)s - %(name)s - %(message)s' + datefmt: '%d/%m/%Y %H:%M:%S' diff --git a/examples/example_test/configs_intermediate/repop_controls.csv b/examples/example_test/configs_intermediate/repop_controls.csv new file mode 100644 index 00000000..f9c17fda --- /dev/null +++ b/examples/example_test/configs_intermediate/repop_controls.csv @@ -0,0 +1,3 @@ +target,geography,seed_table,importance,control_field,expression +num_hh,TAZ,households,1000000000,HHBASE,(households.WGTP > 0) & (households.WGTP < np.inf) +muggle,TAZ,households,1000,MUGGLE,households.wizard == 0 diff --git a/examples/example_test/configs_intermediate/settings.yaml b/examples/example_test/configs_intermediate/settings.yaml new file mode 100644 index 00000000..f9a7e92b --- /dev/null +++ b/examples/example_test/configs_intermediate/settings.yaml @@ -0,0 +1,133 @@ +# This flag turns off integerization entirely (default False) +NO_INTEGERIZATION_EVER: True + +# These settings are ignored if not integerizing +INTEGERIZE_WITH_BACKSTOPPED_CONTROLS: True +SUB_BALANCE_WITH_FLOAT_SEED_WEIGHTS: False +GROUP_BY_INCIDENCE_SIGNATURE: True +INTEGERIZER_TIMEOUT: 300 # seconds +USE_SIMUL_INTEGERIZER: True + +geographies: [REGION, DISTRICT, PUMA, TRACT, TAZ] +seed_geography: PUMA + +# Tracing +# ------------------------------------------------------------------ +trace_geography: + TAZ: 100 + TRACT: 1 + + +household_weight_col: WGTP +household_id_col: hh_id +total_hh_control: num_hh +max_expansion_factor: 5 + +min_expansion_factor: 0.2 + +#control_file_name: controls.csv +#data_dir: data + +# input_pre_processor input_table_list +input_table_list: + - tablename: households + filename : seed_households.csv + index_col: hh_id + rename_columns: + SERIALNO: hh_id + - tablename: persons + # expression_filename: seed_persons_expressions.csv + filename : seed_persons.csv + rename_columns: + SERIALNO: hh_id + SPORDER: per_num + - tablename: geo_cross_walk + filename : geo_cross_walk.csv + rename_columns: + TRACTCE: TRACT + - tablename: TAZ_control_data + filename : taz_controls.csv + - tablename: TRACT_control_data + filename : tract_controls.csv + - tablename: DISTRICT_control_data + filename : district_controls.csv + - tablename: REGION_control_data + filename: region_controls.csv + + +run_list: + steps: + - input_pre_processor + - setup_data_structures + - initial_seed_balancing + - meta_control_factoring + - final_seed_balancing + - integerize_final_seed_weights + - sub_balancing.geography=TRACT + - sub_balancing.geography=TAZ + - expand_households + - summarize + - write_tables + - write_synthetic_population + + #resume_after: expand_households + +# "-m repop" command line option allows specification of repop run_list` +# expand_households options are append or replace +repop: + steps: + - input_pre_processor.table_list=repop_input_table_list + - repop_setup_data_structures + - initial_seed_balancing.final=true + - integerize_final_seed_weights.repop + - repop_balancing + - expand_households.repop;replace + - write_tables.repop + - write_synthetic_population.repop + + resume_after: summarize + +repop_control_file_name: repop_controls.csv + +repop_input_table_list: + - filename : repop_taz_controls.csv + tablename: TAZ_control_data + + + +# output_tables can specify either a list of output tables to include or to skip +# if neither is specified, then no checkpointed tables will be written + +output_tables: + action: include + tables: + - PUMA_weights + - TAZ_weights + - TRACT_weights + - trace_TAZ_weights + - trace_TRACT_weights + - summary_hh_weights + - summary_TAZ + +# - expanded_household_ids + +# +#output_tables: +# action: include +# tables: +# - expanded_household_ids + +output_synthetic_population: + household_id: household_id + households: + filename: synthetic_households.csv + columns: + - NP + - WGTP + - HTYPE + persons: + filename: synthetic_persons.csv + columns: + - per_num + - OSUTAG + - OCCP diff --git a/examples/example_test/data_intermediate/district_controls.csv b/examples/example_test/data_intermediate/district_controls.csv new file mode 100644 index 00000000..10a24314 --- /dev/null +++ b/examples/example_test/data_intermediate/district_controls.csv @@ -0,0 +1,4 @@ +DISTRICT,OCCP1,HHBASE,HHSIZE1,HHSIZE2,HHSIZE3,HHSIZE4,OSUFAM +1,720,450,90,144,144,72,318 +2,720,450,90,144,144,72,318 +3,960,600,120,192,192,96,424 diff --git a/examples/example_test/data_intermediate/geo_cross_walk.csv b/examples/example_test/data_intermediate/geo_cross_walk.csv new file mode 100644 index 00000000..6ff1f1b6 --- /dev/null +++ b/examples/example_test/data_intermediate/geo_cross_walk.csv @@ -0,0 +1,39 @@ +TAZ,TRACTCE,PUMA,DISTRICT,REGION +100,1,600,1,1 +101,1,600,1,1 +102,1,600,1,1 +103,2,600,1,1 +104,2,600,1,1 +105,2,600,1,1 +200,21,601,1,1 +201,21,601,1,1 +202,21,601,1,1 +203,22,601,1,1 +204,22,601,1,1 +205,22,601,1,1 +300,31,602,2,1 +301,31,602,2,1 +302,31,602,2,1 +303,32,602,2,1 +304,32,602,2,1 +305,32,602,2,1 +400,41,603,2,1 +401,41,603,2,1 +402,41,603,2,1 +403,42,603,2,1 +404,42,603,2,1 +405,42,603,2,1 +500,51,604,3,1 +501,51,604,3,1 +502,51,604,3,1 +503,52,604,3,1 +504,52,604,3,1 +505,52,604,3,1 +600,61,605,3,1 +601,61,605,3,1 +602,61,605,3,1 +603,62,605,3,1 +604,62,605,3,1 +605,62,605,3,1 +#,,,, +700,72,606,1,1 diff --git a/examples/example_test/data_intermediate/region_controls.csv b/examples/example_test/data_intermediate/region_controls.csv new file mode 100644 index 00000000..7771eae9 --- /dev/null +++ b/examples/example_test/data_intermediate/region_controls.csv @@ -0,0 +1,2 @@ +REGION,OCCP2,OCCP3 +1,300,1560 diff --git a/examples/example_test/data_intermediate/repop_taz_controls.csv b/examples/example_test/data_intermediate/repop_taz_controls.csv new file mode 100644 index 00000000..e5bd529a --- /dev/null +++ b/examples/example_test/data_intermediate/repop_taz_controls.csv @@ -0,0 +1,5 @@ +TAZ,TRACT,PUMA,DISTRICT,HHBASE,HHSIZE1,HHSIZE2,HHSIZE3,HHSIZE4,OSUFAM,SF,MUGGLE +100,1,600,1,26,6,8,8,4,53,26,20 +101,1,600,1,26,6,8,8,4,53,26,21 +205,22,601,1,51,11,16,16,8,0,51,40 +301,31,602,2,26,6,8,8,4,53,26,19 diff --git a/examples/example_test/data_intermediate/seed_households.csv b/examples/example_test/data_intermediate/seed_households.csv new file mode 100644 index 00000000..764d998e --- /dev/null +++ b/examples/example_test/data_intermediate/seed_households.csv @@ -0,0 +1,63 @@ +SERIALNO,NP,WGTP,HTYPE,PUMA,wizard +1,1,30,1,600,1 +2,2,48,1,600,1 +3,3,48,1,600,0 +4,4,14,1,600,0 +5,5,10,1,600,0 +6,10,5,0,600,0 +101,1,30,1,601,1 +102,2,48,1,601,0 +103,3,48,1,601,0 +104,4,14,1,601,0 +105,5,10,1,601,0 +106,10,5,0,601,0 +111,1,30,1,601,0 +112,2,48,1,601,0 +113,3,48,1,601,0 +114,4,14,1,601,0 +115,5,10,1,601,0 +116,10,5,0,601,0 +201,1,30,1,602,1 +202,2,48,1,602,0 +203,3,48,1,602,0 +204,4,14,1,602,0 +205,5,10,1,602,0 +206,10,5,0,602,0 +301,1,30,1,603,1 +302,2,48,1,603,0 +303,3,48,1,603,0 +304,4,14,1,603,0 +305,5,10,1,603,0 +306,10,5,0,603,0 +311,1,30,1,603,0 +312,2,48,1,603,0 +313,3,48,1,603,0 +314,4,14,1,603,0 +315,5,10,1,603,0 +316,10,5,0,603,0 +401,1,30,1,604,1 +402,2,48,1,604,0 +403,3,48,1,604,0 +404,4,14,1,604,0 +405,5,10,1,604,0 +406,10,5,0,604,0 +411,1,30,1,604,0 +412,2,48,1,604,0 +413,3,48,1,604,0 +414,4,14,1,604,0 +415,5,10,1,604,0 +416,10,5,0,604,0 +501,1,30,1,605,1 +502,2,48,1,605,0 +503,3,48,1,605,0 +504,4,14,1,605,0 +505,5,10,1,605,0 +506,10,5,0,605,0 +511,1,30,1,605,0 +512,2,48,1,605,0 +513,3,48,1,605,0 +514,4,14,1,605,0 +515,5,10,1,605,0 +516,10,5,0,605,0 +# this hh outside of seed zones should get dropped +600,10,5,0,606,0 diff --git a/examples/example_test/data_intermediate/seed_persons.csv b/examples/example_test/data_intermediate/seed_persons.csv new file mode 100644 index 00000000..dc362688 --- /dev/null +++ b/examples/example_test/data_intermediate/seed_persons.csv @@ -0,0 +1,254 @@ +SERIALNO,SPORDER,PUMA,OSUTAG,OCCP,WGTP +1,1,600,0,2,30 +2,1,600,0,1,48 +2,2,600,0,1,48 +3,1,600,0,1,48 +3,2,600,0,1,48 +3,3,600,1,3,48 +4,1,600,0,1,14 +4,2,600,0,1,14 +4,3,600,1,3,14 +4,4,600,1,3,14 +5,1,600,0,1,10 +5,2,600,0,1,10 +5,3,600,1,3,10 +5,4,600,1,3,10 +5,5,600,1,3,10 +6,1,600,2,3,5 +6,2,600,2,3,5 +6,3,600,2,3,5 +6,4,600,2,3,5 +6,5,600,2,3,5 +6,6,600,2,3,5 +6,7,600,2,3,5 +6,8,600,2,3,5 +6,9,600,2,3,5 +6,10,600,2,3,5 +101,1,601,0,2,30 +102,1,601,0,1,48 +102,2,601,0,1,48 +103,1,601,0,1,48 +103,2,601,0,1,48 +103,3,601,1,3,48 +104,1,601,0,1,14 +104,2,601,0,1,14 +104,3,601,1,3,14 +104,4,601,1,3,14 +105,1,601,0,1,10 +105,2,601,0,1,10 +105,3,601,1,3,10 +105,4,601,1,3,10 +105,5,601,1,3,10 +106,1,601,2,3,5 +106,2,601,2,3,5 +106,3,601,2,3,5 +106,4,601,2,3,5 +106,5,601,2,3,5 +106,6,601,2,3,5 +106,7,601,2,3,5 +106,8,601,2,3,5 +106,9,601,2,3,5 +106,10,601,2,3,5 +111,1,601,0,2,30 +112,1,601,0,1,48 +112,2,601,0,1,48 +113,1,601,0,1,48 +113,2,601,0,1,48 +113,3,601,1,3,48 +114,1,601,0,1,14 +114,2,601,0,1,14 +114,3,601,1,3,14 +114,4,601,1,3,14 +115,1,601,0,1,10 +115,2,601,0,1,10 +115,3,601,1,3,10 +115,4,601,1,3,10 +115,5,601,1,3,10 +116,1,601,2,3,5 +116,2,601,2,3,5 +116,3,601,2,3,5 +116,4,601,2,3,5 +116,5,601,2,3,5 +116,6,601,2,3,5 +116,7,601,2,3,5 +116,8,601,2,3,5 +116,9,601,2,3,5 +116,10,601,2,3,5 +201,1,602,0,2,30 +202,1,602,0,1,48 +202,2,602,0,1,48 +203,1,602,0,1,48 +203,2,602,0,1,48 +203,3,602,1,3,48 +204,1,602,0,1,14 +204,2,602,0,1,14 +204,3,602,1,3,14 +204,4,602,1,3,14 +205,1,602,0,1,10 +205,2,602,0,1,10 +205,3,602,1,3,10 +205,4,602,1,3,10 +205,5,602,1,3,10 +206,1,602,2,3,5 +206,2,602,2,3,5 +206,3,602,2,3,5 +206,4,602,2,3,5 +206,5,602,2,3,5 +206,6,602,2,3,5 +206,7,602,2,3,5 +206,8,602,2,3,5 +206,9,602,2,3,5 +206,10,602,2,3,5 +301,1,603,0,2,30 +302,1,603,0,1,48 +302,2,603,0,1,48 +303,1,603,0,1,48 +303,2,603,0,1,48 +303,3,603,1,3,48 +304,1,603,0,1,14 +304,2,603,0,1,14 +304,3,603,1,3,14 +304,4,603,1,3,14 +305,1,603,0,1,10 +305,2,603,0,1,10 +305,3,603,1,3,10 +305,4,603,1,3,10 +305,5,603,1,3,10 +306,1,603,2,3,5 +306,2,603,2,3,5 +306,3,603,2,3,5 +306,4,603,2,3,5 +306,5,603,2,3,5 +306,6,603,2,3,5 +306,7,603,2,3,5 +306,8,603,2,3,5 +306,9,603,2,3,5 +306,10,603,2,3,5 +311,1,603,0,2,30 +312,1,603,0,1,48 +312,2,603,0,1,48 +313,1,603,0,1,48 +313,2,603,0,1,48 +313,3,603,1,3,48 +314,1,603,0,1,14 +314,2,603,0,1,14 +314,3,603,1,3,14 +314,4,603,1,3,14 +315,1,603,0,1,10 +315,2,603,0,1,10 +315,3,603,1,3,10 +315,4,603,1,3,10 +315,5,603,1,3,10 +316,1,603,2,3,5 +316,2,603,2,3,5 +316,3,603,2,3,5 +316,4,603,2,3,5 +316,5,603,2,3,5 +316,6,603,2,3,5 +316,7,603,2,3,5 +316,8,603,2,3,5 +316,9,603,2,3,5 +316,10,603,2,3,5 +401,1,604,0,2,30 +402,1,604,0,1,48 +402,2,604,0,1,48 +403,1,604,0,1,48 +403,2,604,0,1,48 +403,3,604,1,3,48 +404,1,604,0,1,14 +404,2,604,0,1,14 +404,3,604,1,3,14 +404,4,604,1,3,14 +405,1,604,0,1,10 +405,2,604,0,1,10 +405,3,604,1,3,10 +405,4,604,1,3,10 +405,5,604,1,3,10 +406,1,604,2,3,5 +406,2,604,2,3,5 +406,3,604,2,3,5 +406,4,604,2,3,5 +406,5,604,2,3,5 +406,6,604,2,3,5 +406,7,604,2,3,5 +406,8,604,2,3,5 +406,9,604,2,3,5 +406,10,604,2,3,5 +411,1,604,0,2,30 +412,1,604,0,1,48 +412,2,604,0,1,48 +413,1,604,0,1,48 +413,2,604,0,1,48 +413,3,604,1,3,48 +414,1,604,0,1,14 +414,2,604,0,1,14 +414,3,604,1,3,14 +414,4,604,1,3,14 +415,1,604,0,1,10 +415,2,604,0,1,10 +415,3,604,1,3,10 +415,4,604,1,3,10 +415,5,604,1,3,10 +416,1,604,2,3,5 +416,2,604,2,3,5 +416,3,604,2,3,5 +416,4,604,2,3,5 +416,5,604,2,3,5 +416,6,604,2,3,5 +416,7,604,2,3,5 +416,8,604,2,3,5 +416,9,604,2,3,5 +416,10,604,2,3,5 +501,1,605,0,2,30 +502,1,605,0,1,48 +502,2,605,0,1,48 +503,1,605,0,1,48 +503,2,605,0,1,48 +503,3,605,1,3,48 +504,1,605,0,1,14 +504,2,605,0,1,14 +504,3,605,1,3,14 +504,4,605,1,3,14 +505,1,605,0,1,10 +505,2,605,0,1,10 +505,3,605,1,3,10 +505,4,605,1,3,10 +505,5,605,1,3,10 +506,1,605,2,3,5 +506,2,605,2,3,5 +506,3,605,2,3,5 +506,4,605,2,3,5 +506,5,605,2,3,5 +506,6,605,2,3,5 +506,7,605,2,3,5 +506,8,605,2,3,5 +506,9,605,2,3,5 +506,10,605,2,3,5 +511,1,605,0,2,30 +512,1,605,0,1,48 +512,2,605,0,1,48 +513,1,605,0,1,48 +513,2,605,0,1,48 +513,3,605,1,3,48 +514,1,605,0,1,14 +514,2,605,0,1,14 +514,3,605,1,3,14 +514,4,605,1,3,14 +515,1,605,0,1,10 +515,2,605,0,1,10 +515,3,605,1,3,10 +515,4,605,1,3,10 +515,5,605,1,3,10 +516,1,605,2,3,5 +516,2,605,2,3,5 +516,3,605,2,3,5 +516,4,605,2,3,5 +516,5,605,2,3,5 +516,6,605,2,3,5 +516,7,605,2,3,5 +516,8,605,2,3,5 +516,9,605,2,3,5 +516,10,605,2,3,5 +# this 2 person hh outside of seed zones should get dropped +600,1,606,2,3,5 +600,2,606,1,3,10 diff --git a/examples/example_test/data_intermediate/taz_controls.csv b/examples/example_test/data_intermediate/taz_controls.csv new file mode 100644 index 00000000..002742cb --- /dev/null +++ b/examples/example_test/data_intermediate/taz_controls.csv @@ -0,0 +1,37 @@ +TAZ,TRACT,PUMA,DISTRICT,HHBASE,HHSIZE1,HHSIZE2,HHSIZE3,HHSIZE4,OSUFAM,SF +100,1,600,1,25,5,8,8,4,53,25 +101,1,600,1,25,5,8,8,4,53,25 +102,1,600,1,25,5,8,8,4,0,25 +103,2,600,1,25,5,8,8,4,0,25 +104,2,600,1,25,5,8,8,4,0,25 +105,2,600,1,25,5,8,8,4,0,25 +200,21,601,1,50,10,16,16,8,106,50 +201,21,601,1,50,10,16,16,8,106,50 +202,21,601,1,50,10,16,16,8,0,50 +203,22,601,1,50,10,16,16,8,0,50 +204,22,601,1,50,10,16,16,8,0,50 +205,22,601,1,50,10,16,16,8,0,50 +300,31,602,2,25,5,8,8,4,53,25 +301,31,602,2,25,5,8,8,4,53,25 +302,31,602,2,25,5,8,8,4,0,25 +303,32,602,2,25,5,8,8,4,0,25 +304,32,602,2,25,5,8,8,4,0,25 +305,32,602,2,25,5,8,8,4,0,25 +400,41,603,2,50,10,16,16,8,106,50 +401,41,603,2,50,10,16,16,8,106,50 +402,41,603,2,50,10,16,16,8,0,50 +403,42,603,2,50,10,16,16,8,0,50 +404,42,603,2,50,10,16,16,8,0,50 +405,42,603,2,50,10,16,16,8,0,50 +500,51,604,3,50,10,16,16,8,106,50 +501,51,604,3,50,10,16,16,8,106,50 +502,51,604,3,50,10,16,16,8,0,50 +503,52,604,3,50,10,16,16,8,0,50 +504,52,604,3,50,10,16,16,8,0,50 +505,52,604,3,50,10,16,16,8,0,50 +600,61,605,3,50,10,16,16,8,106,50 +601,61,605,3,50,10,16,16,8,106,50 +602,61,605,3,50,10,16,16,8,0,50 +603,62,605,3,50,10,16,16,8,0,50 +604,62,605,3,50,10,16,16,8,0,50 +605,62,605,3,50,10,16,16,8,0,50 diff --git a/examples/example_test/data_intermediate/tract_controls.csv b/examples/example_test/data_intermediate/tract_controls.csv new file mode 100644 index 00000000..4308af17 --- /dev/null +++ b/examples/example_test/data_intermediate/tract_controls.csv @@ -0,0 +1,13 @@ +TRACT,PUMA,HHBASE,HHSIZE1,HHSIZE2,HHSIZE3,HHSIZE4,OSUFAM,SF +1,600,75,15,24,24,12,106,75 +2,600,75,15,24,24,12,0,75 +21,601,150,30,48,48,24,212,150 +22,601,150,30,48,48,24,0,150 +31,602,75,15,24,24,12,106,75 +32,602,75,15,24,24,12,0,75 +41,603,150,30,48,48,24,212,150 +42,603,150,30,48,48,24,0,150 +51,604,150,30,48,48,24,212,150 +52,604,150,30,48,48,24,0,150 +61,605,150,30,48,48,24,212,150 +62,605,150,30,48,48,24,0,150 diff --git a/tests/test_intermediate_geography.py b/tests/test_intermediate_geography.py new file mode 100644 index 00000000..757ae2fe --- /dev/null +++ b/tests/test_intermediate_geography.py @@ -0,0 +1,47 @@ +import numpy as np +import pandas as pd +from pathlib import Path + +from populationsim.core import tracing, inject, pipeline + + +def teardown_function(func): + inject.clear_cache() + inject.reinject_decorated_tables() + + +def test_intermediate_geography(): + + example_dir = Path(__file__).parent.parent / "examples" / "example_test" + configs_dir = example_dir / "configs_intermediate" + data_dir = example_dir / "data_intermediate" + output_dir = Path(__file__).parent / "output" + + inject.add_injectable("data_dir", data_dir) + inject.add_injectable("configs_dir", configs_dir) + inject.add_injectable("output_dir", output_dir) + + inject.clear_cache() + + tracing.config_logger() + + _MODELS = [ + "input_pre_processor", + "setup_data_structures", + "initial_seed_balancing", + "meta_control_factoring", + "final_seed_balancing", + "integerize_final_seed_weights", + "sub_balancing.geography=TRACT", + "sub_balancing.geography=TAZ", + "expand_households", + "summarize", + "write_tables", + "write_synthetic_population", + ] + + pipeline.run(models=_MODELS, resume_after=None) + + pipeline.close_pipeline() + + inject.clear_cache() \ No newline at end of file From 3e578ed9dff1d86654d857e4169292a23e232547 Mon Sep 17 00:00:00 2001 From: nick-fournier <45876721+nick-fournier@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:07:22 -0700 Subject: [PATCH 3/5] Assert the intermediate-geography behaviour in its test The test ran the pipeline and asserted nothing, so it only caught a crash. Give it the isolation the rest of the suite has and check what the feature actually does. Reverting the control_spec filtering now fails this test with KeyError: "['persons_occ_1'] not in index" from meta_control_factoring, so completing the run is itself meaningful -- but the run alone does not pin which controls survive where. It now also asserts that the intermediate geography's control is kept out of the seed-level control table, that the meta geography's controls are still factored into it, and that the intermediate control still constrains its own geography rather than being discarded. Adds a setup_function restoring the decorated `settings` injectable. Without it this module inherits whatever settings an earlier test pinned, and fails on a `geographies` list that is not its own. Also asserts sub-geography balancing conserves the seed household total, and that expand_households produces nothing, since this example sets NO_INTEGERIZATION_EVER. Co-Authored-By: Claude Opus 5 (1M context) --- tests/test_intermediate_geography.py | 104 ++++++++++++++++++++------- 1 file changed, 79 insertions(+), 25 deletions(-) diff --git a/tests/test_intermediate_geography.py b/tests/test_intermediate_geography.py index 757ae2fe..28c4cfd9 100644 --- a/tests/test_intermediate_geography.py +++ b/tests/test_intermediate_geography.py @@ -1,11 +1,46 @@ -import numpy as np -import pandas as pd from pathlib import Path -from populationsim.core import tracing, inject, pipeline +import pytest + +from populationsim.core import inject, pipeline, tracing + +# examples/example_test/configs_intermediate declares +# geographies: [REGION, DISTRICT, PUMA, TRACT, TAZ] +# seed_geography: PUMA +# so DISTRICT sits between the meta geography (REGION) and the seed geography +# (PUMA). Its controls.csv puts persons_occ_1 at DISTRICT and persons_occ_2 and +# persons_occ_3 at REGION, which is what lets this example tell the two apart. +SEED_GEOGRAPHY = "PUMA" +INTERMEDIATE_GEOGRAPHY = "DISTRICT" +INTERMEDIATE_CONTROL = "persons_occ_1" +META_CONTROLS = ("persons_occ_2", "persons_occ_3") + +_MODELS = [ + "input_pre_processor", + "setup_data_structures", + "initial_seed_balancing", + "meta_control_factoring", + "final_seed_balancing", + "integerize_final_seed_weights", + "sub_balancing.geography=TRACT", + "sub_balancing.geography=TAZ", + "expand_households", + "summarize", + "write_tables", + "write_synthetic_population", +] + + +def setup_function(): + # Other test modules pin `settings` as a plain injectable via + # config.override_setting; restore the decorated one so this example's own + # settings -- in particular its five-level `geographies` list -- are read. + inject.reinject_decorated_tables() def teardown_function(func): + if pipeline.is_open(): + pipeline.close_pipeline() inject.clear_cache() inject.reinject_decorated_tables() @@ -13,35 +48,54 @@ def teardown_function(func): def test_intermediate_geography(): example_dir = Path(__file__).parent.parent / "examples" / "example_test" - configs_dir = example_dir / "configs_intermediate" - data_dir = example_dir / "data_intermediate" - output_dir = Path(__file__).parent / "output" - inject.add_injectable("data_dir", data_dir) - inject.add_injectable("configs_dir", configs_dir) - inject.add_injectable("output_dir", output_dir) + inject.add_injectable("data_dir", example_dir / "data_intermediate") + inject.add_injectable("configs_dir", example_dir / "configs_intermediate") + inject.add_injectable("output_dir", Path(__file__).parent / "output") inject.clear_cache() tracing.config_logger() - _MODELS = [ - "input_pre_processor", - "setup_data_structures", - "initial_seed_balancing", - "meta_control_factoring", - "final_seed_balancing", - "integerize_final_seed_weights", - "sub_balancing.geography=TRACT", - "sub_balancing.geography=TAZ", - "expand_households", - "summarize", - "write_tables", - "write_synthetic_population", + # Completing at all is the primary regression. Without the control_spec + # filtering in meta_control_factoring / final_seed_balancing / + # integerize_final_seed_weights, the seed-level controls are indexed with + # the intermediate geography's target and this raises + # KeyError: "['persons_occ_1'] not in index". + pipeline.run(models=_MODELS, resume_after=None) + + crosswalk = pipeline.get_table("crosswalk") + assert list(crosswalk.columns) == [ + "REGION", + INTERMEDIATE_GEOGRAPHY, + SEED_GEOGRAPHY, + "TRACT", + "TAZ", ] - pipeline.run(models=_MODELS, resume_after=None) + # The behaviour under test: controls belonging to a geography between the + # meta and seed geographies are kept out of the seed-level control table, + # while the meta geography's controls are still factored into it. + seed_controls = pipeline.get_table(f"{SEED_GEOGRAPHY}_controls") + assert INTERMEDIATE_CONTROL not in seed_controls.columns + for target in META_CONTROLS: + assert target in seed_controls.columns + + # The intermediate control is excluded from the seed level, not discarded: + # it still constrains its own geography. + intermediate_controls = pipeline.get_table(f"{INTERMEDIATE_GEOGRAPHY}_controls") + assert INTERMEDIATE_CONTROL in intermediate_controls.columns - pipeline.close_pipeline() + # Sub-geography balancing ran and preserved the seed-level household total. + seed_total = pipeline.get_table(f"{SEED_GEOGRAPHY}_weights")[ + "balanced_weight" + ].sum() + assert seed_total > 0 + for geography in ("TRACT", "TAZ"): + weights = pipeline.get_table(f"{geography}_weights")["balanced_weight"] + assert (weights >= 0).all() + assert weights.sum() == pytest.approx(seed_total, rel=1e-6) - inject.clear_cache() \ No newline at end of file + # This example sets NO_INTEGERIZATION_EVER, so expand_households produces + # nothing -- assert that explicitly rather than leaving it implied. + assert pipeline.get_table("expanded_household_ids").empty From 1e29447c607b3ca61bed58aa320fbab4e810baaa Mon Sep 17 00:00:00 2001 From: nick-fournier <45876721+nick-fournier@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:03:11 -0700 Subject: [PATCH 4/5] Drop duplicated fixtures from the intermediate example Seven of the twelve fixture files were byte-identical copies of the base example_test fixtures: seed_households.csv, seed_persons.csv, taz_controls.csv, tract_controls.csv, repop_taz_controls.csv, logging.yaml and repop_controls.csv -- 429 of the example's 728 lines. The two repop files were never exercised at all, since the test runs no repop steps. configs_dir and data_dir both resolve through config.cascading_input_file_path, which accepts a list of directories and takes the first one holding a given file, so the test can layer the intermediate directories over the base example. That is already how tests/test_steps.py supplies its configs. The *_intermediate directories now carry only what this example really changes: its settings and controls, the crosswalk with the extra geography levels, and the DISTRICT and REGION control data. Beyond the size, the copies were a correctness risk: editing a seed table in examples/example_test/data would have left this example testing stale inputs with no indication anything had diverged. Verified: full suite passes on python 3.12 / pandas 3.0.3 and on python 3.10 / pandas 2.3.3. Co-Authored-By: Claude Opus 5 (1M context) --- .../configs_intermediate/logging.yaml | 54 ---- .../configs_intermediate/repop_controls.csv | 3 - .../data_intermediate/repop_taz_controls.csv | 5 - .../data_intermediate/seed_households.csv | 63 ----- .../data_intermediate/seed_persons.csv | 254 ------------------ .../data_intermediate/taz_controls.csv | 37 --- .../data_intermediate/tract_controls.csv | 13 - tests/test_intermediate_geography.py | 15 +- 8 files changed, 13 insertions(+), 431 deletions(-) delete mode 100644 examples/example_test/configs_intermediate/logging.yaml delete mode 100644 examples/example_test/configs_intermediate/repop_controls.csv delete mode 100644 examples/example_test/data_intermediate/repop_taz_controls.csv delete mode 100644 examples/example_test/data_intermediate/seed_households.csv delete mode 100644 examples/example_test/data_intermediate/seed_persons.csv delete mode 100644 examples/example_test/data_intermediate/taz_controls.csv delete mode 100644 examples/example_test/data_intermediate/tract_controls.csv diff --git a/examples/example_test/configs_intermediate/logging.yaml b/examples/example_test/configs_intermediate/logging.yaml deleted file mode 100644 index 06baa76a..00000000 --- a/examples/example_test/configs_intermediate/logging.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# Config for logging -# ------------------ -# See http://docs.python.org/2.7/library/logging.config.html#configuration-dictionary-schema - -logging: - version: 1 - disable_existing_loggers: true - - - # Configuring the default (root) logger is highly recommended - root: - level: NOTSET - handlers: [console] - - loggers: - - populationsim: - level: DEBUG - handlers: [console, logfile] - propagate: false - - orca: - level: WARN - handlers: [console, logfile] - propagate: false - - handlers: - - logfile: - class: logging.FileHandler - filename: !!python/object/apply:populationsim.core.config.log_file_path ['populationsim.log'] - mode: w - formatter: fileFormatter - level: NOTSET - - console: - class: logging.StreamHandler - stream: ext://sys.stdout - formatter: simpleFormatter - level: NOTSET - #level: WARN - - formatters: - - simpleFormatter: - class: logging.Formatter - # format: '%(levelname)s - %(name)s - %(message)s' - format: '%(levelname)s - %(message)s' - datefmt: '%d/%m/%Y %H:%M:%S' - - fileFormatter: - class: logging.Formatter - format: '%(asctime)s - %(levelname)s - %(name)s - %(message)s' - datefmt: '%d/%m/%Y %H:%M:%S' diff --git a/examples/example_test/configs_intermediate/repop_controls.csv b/examples/example_test/configs_intermediate/repop_controls.csv deleted file mode 100644 index f9c17fda..00000000 --- a/examples/example_test/configs_intermediate/repop_controls.csv +++ /dev/null @@ -1,3 +0,0 @@ -target,geography,seed_table,importance,control_field,expression -num_hh,TAZ,households,1000000000,HHBASE,(households.WGTP > 0) & (households.WGTP < np.inf) -muggle,TAZ,households,1000,MUGGLE,households.wizard == 0 diff --git a/examples/example_test/data_intermediate/repop_taz_controls.csv b/examples/example_test/data_intermediate/repop_taz_controls.csv deleted file mode 100644 index e5bd529a..00000000 --- a/examples/example_test/data_intermediate/repop_taz_controls.csv +++ /dev/null @@ -1,5 +0,0 @@ -TAZ,TRACT,PUMA,DISTRICT,HHBASE,HHSIZE1,HHSIZE2,HHSIZE3,HHSIZE4,OSUFAM,SF,MUGGLE -100,1,600,1,26,6,8,8,4,53,26,20 -101,1,600,1,26,6,8,8,4,53,26,21 -205,22,601,1,51,11,16,16,8,0,51,40 -301,31,602,2,26,6,8,8,4,53,26,19 diff --git a/examples/example_test/data_intermediate/seed_households.csv b/examples/example_test/data_intermediate/seed_households.csv deleted file mode 100644 index 764d998e..00000000 --- a/examples/example_test/data_intermediate/seed_households.csv +++ /dev/null @@ -1,63 +0,0 @@ -SERIALNO,NP,WGTP,HTYPE,PUMA,wizard -1,1,30,1,600,1 -2,2,48,1,600,1 -3,3,48,1,600,0 -4,4,14,1,600,0 -5,5,10,1,600,0 -6,10,5,0,600,0 -101,1,30,1,601,1 -102,2,48,1,601,0 -103,3,48,1,601,0 -104,4,14,1,601,0 -105,5,10,1,601,0 -106,10,5,0,601,0 -111,1,30,1,601,0 -112,2,48,1,601,0 -113,3,48,1,601,0 -114,4,14,1,601,0 -115,5,10,1,601,0 -116,10,5,0,601,0 -201,1,30,1,602,1 -202,2,48,1,602,0 -203,3,48,1,602,0 -204,4,14,1,602,0 -205,5,10,1,602,0 -206,10,5,0,602,0 -301,1,30,1,603,1 -302,2,48,1,603,0 -303,3,48,1,603,0 -304,4,14,1,603,0 -305,5,10,1,603,0 -306,10,5,0,603,0 -311,1,30,1,603,0 -312,2,48,1,603,0 -313,3,48,1,603,0 -314,4,14,1,603,0 -315,5,10,1,603,0 -316,10,5,0,603,0 -401,1,30,1,604,1 -402,2,48,1,604,0 -403,3,48,1,604,0 -404,4,14,1,604,0 -405,5,10,1,604,0 -406,10,5,0,604,0 -411,1,30,1,604,0 -412,2,48,1,604,0 -413,3,48,1,604,0 -414,4,14,1,604,0 -415,5,10,1,604,0 -416,10,5,0,604,0 -501,1,30,1,605,1 -502,2,48,1,605,0 -503,3,48,1,605,0 -504,4,14,1,605,0 -505,5,10,1,605,0 -506,10,5,0,605,0 -511,1,30,1,605,0 -512,2,48,1,605,0 -513,3,48,1,605,0 -514,4,14,1,605,0 -515,5,10,1,605,0 -516,10,5,0,605,0 -# this hh outside of seed zones should get dropped -600,10,5,0,606,0 diff --git a/examples/example_test/data_intermediate/seed_persons.csv b/examples/example_test/data_intermediate/seed_persons.csv deleted file mode 100644 index dc362688..00000000 --- a/examples/example_test/data_intermediate/seed_persons.csv +++ /dev/null @@ -1,254 +0,0 @@ -SERIALNO,SPORDER,PUMA,OSUTAG,OCCP,WGTP -1,1,600,0,2,30 -2,1,600,0,1,48 -2,2,600,0,1,48 -3,1,600,0,1,48 -3,2,600,0,1,48 -3,3,600,1,3,48 -4,1,600,0,1,14 -4,2,600,0,1,14 -4,3,600,1,3,14 -4,4,600,1,3,14 -5,1,600,0,1,10 -5,2,600,0,1,10 -5,3,600,1,3,10 -5,4,600,1,3,10 -5,5,600,1,3,10 -6,1,600,2,3,5 -6,2,600,2,3,5 -6,3,600,2,3,5 -6,4,600,2,3,5 -6,5,600,2,3,5 -6,6,600,2,3,5 -6,7,600,2,3,5 -6,8,600,2,3,5 -6,9,600,2,3,5 -6,10,600,2,3,5 -101,1,601,0,2,30 -102,1,601,0,1,48 -102,2,601,0,1,48 -103,1,601,0,1,48 -103,2,601,0,1,48 -103,3,601,1,3,48 -104,1,601,0,1,14 -104,2,601,0,1,14 -104,3,601,1,3,14 -104,4,601,1,3,14 -105,1,601,0,1,10 -105,2,601,0,1,10 -105,3,601,1,3,10 -105,4,601,1,3,10 -105,5,601,1,3,10 -106,1,601,2,3,5 -106,2,601,2,3,5 -106,3,601,2,3,5 -106,4,601,2,3,5 -106,5,601,2,3,5 -106,6,601,2,3,5 -106,7,601,2,3,5 -106,8,601,2,3,5 -106,9,601,2,3,5 -106,10,601,2,3,5 -111,1,601,0,2,30 -112,1,601,0,1,48 -112,2,601,0,1,48 -113,1,601,0,1,48 -113,2,601,0,1,48 -113,3,601,1,3,48 -114,1,601,0,1,14 -114,2,601,0,1,14 -114,3,601,1,3,14 -114,4,601,1,3,14 -115,1,601,0,1,10 -115,2,601,0,1,10 -115,3,601,1,3,10 -115,4,601,1,3,10 -115,5,601,1,3,10 -116,1,601,2,3,5 -116,2,601,2,3,5 -116,3,601,2,3,5 -116,4,601,2,3,5 -116,5,601,2,3,5 -116,6,601,2,3,5 -116,7,601,2,3,5 -116,8,601,2,3,5 -116,9,601,2,3,5 -116,10,601,2,3,5 -201,1,602,0,2,30 -202,1,602,0,1,48 -202,2,602,0,1,48 -203,1,602,0,1,48 -203,2,602,0,1,48 -203,3,602,1,3,48 -204,1,602,0,1,14 -204,2,602,0,1,14 -204,3,602,1,3,14 -204,4,602,1,3,14 -205,1,602,0,1,10 -205,2,602,0,1,10 -205,3,602,1,3,10 -205,4,602,1,3,10 -205,5,602,1,3,10 -206,1,602,2,3,5 -206,2,602,2,3,5 -206,3,602,2,3,5 -206,4,602,2,3,5 -206,5,602,2,3,5 -206,6,602,2,3,5 -206,7,602,2,3,5 -206,8,602,2,3,5 -206,9,602,2,3,5 -206,10,602,2,3,5 -301,1,603,0,2,30 -302,1,603,0,1,48 -302,2,603,0,1,48 -303,1,603,0,1,48 -303,2,603,0,1,48 -303,3,603,1,3,48 -304,1,603,0,1,14 -304,2,603,0,1,14 -304,3,603,1,3,14 -304,4,603,1,3,14 -305,1,603,0,1,10 -305,2,603,0,1,10 -305,3,603,1,3,10 -305,4,603,1,3,10 -305,5,603,1,3,10 -306,1,603,2,3,5 -306,2,603,2,3,5 -306,3,603,2,3,5 -306,4,603,2,3,5 -306,5,603,2,3,5 -306,6,603,2,3,5 -306,7,603,2,3,5 -306,8,603,2,3,5 -306,9,603,2,3,5 -306,10,603,2,3,5 -311,1,603,0,2,30 -312,1,603,0,1,48 -312,2,603,0,1,48 -313,1,603,0,1,48 -313,2,603,0,1,48 -313,3,603,1,3,48 -314,1,603,0,1,14 -314,2,603,0,1,14 -314,3,603,1,3,14 -314,4,603,1,3,14 -315,1,603,0,1,10 -315,2,603,0,1,10 -315,3,603,1,3,10 -315,4,603,1,3,10 -315,5,603,1,3,10 -316,1,603,2,3,5 -316,2,603,2,3,5 -316,3,603,2,3,5 -316,4,603,2,3,5 -316,5,603,2,3,5 -316,6,603,2,3,5 -316,7,603,2,3,5 -316,8,603,2,3,5 -316,9,603,2,3,5 -316,10,603,2,3,5 -401,1,604,0,2,30 -402,1,604,0,1,48 -402,2,604,0,1,48 -403,1,604,0,1,48 -403,2,604,0,1,48 -403,3,604,1,3,48 -404,1,604,0,1,14 -404,2,604,0,1,14 -404,3,604,1,3,14 -404,4,604,1,3,14 -405,1,604,0,1,10 -405,2,604,0,1,10 -405,3,604,1,3,10 -405,4,604,1,3,10 -405,5,604,1,3,10 -406,1,604,2,3,5 -406,2,604,2,3,5 -406,3,604,2,3,5 -406,4,604,2,3,5 -406,5,604,2,3,5 -406,6,604,2,3,5 -406,7,604,2,3,5 -406,8,604,2,3,5 -406,9,604,2,3,5 -406,10,604,2,3,5 -411,1,604,0,2,30 -412,1,604,0,1,48 -412,2,604,0,1,48 -413,1,604,0,1,48 -413,2,604,0,1,48 -413,3,604,1,3,48 -414,1,604,0,1,14 -414,2,604,0,1,14 -414,3,604,1,3,14 -414,4,604,1,3,14 -415,1,604,0,1,10 -415,2,604,0,1,10 -415,3,604,1,3,10 -415,4,604,1,3,10 -415,5,604,1,3,10 -416,1,604,2,3,5 -416,2,604,2,3,5 -416,3,604,2,3,5 -416,4,604,2,3,5 -416,5,604,2,3,5 -416,6,604,2,3,5 -416,7,604,2,3,5 -416,8,604,2,3,5 -416,9,604,2,3,5 -416,10,604,2,3,5 -501,1,605,0,2,30 -502,1,605,0,1,48 -502,2,605,0,1,48 -503,1,605,0,1,48 -503,2,605,0,1,48 -503,3,605,1,3,48 -504,1,605,0,1,14 -504,2,605,0,1,14 -504,3,605,1,3,14 -504,4,605,1,3,14 -505,1,605,0,1,10 -505,2,605,0,1,10 -505,3,605,1,3,10 -505,4,605,1,3,10 -505,5,605,1,3,10 -506,1,605,2,3,5 -506,2,605,2,3,5 -506,3,605,2,3,5 -506,4,605,2,3,5 -506,5,605,2,3,5 -506,6,605,2,3,5 -506,7,605,2,3,5 -506,8,605,2,3,5 -506,9,605,2,3,5 -506,10,605,2,3,5 -511,1,605,0,2,30 -512,1,605,0,1,48 -512,2,605,0,1,48 -513,1,605,0,1,48 -513,2,605,0,1,48 -513,3,605,1,3,48 -514,1,605,0,1,14 -514,2,605,0,1,14 -514,3,605,1,3,14 -514,4,605,1,3,14 -515,1,605,0,1,10 -515,2,605,0,1,10 -515,3,605,1,3,10 -515,4,605,1,3,10 -515,5,605,1,3,10 -516,1,605,2,3,5 -516,2,605,2,3,5 -516,3,605,2,3,5 -516,4,605,2,3,5 -516,5,605,2,3,5 -516,6,605,2,3,5 -516,7,605,2,3,5 -516,8,605,2,3,5 -516,9,605,2,3,5 -516,10,605,2,3,5 -# this 2 person hh outside of seed zones should get dropped -600,1,606,2,3,5 -600,2,606,1,3,10 diff --git a/examples/example_test/data_intermediate/taz_controls.csv b/examples/example_test/data_intermediate/taz_controls.csv deleted file mode 100644 index 002742cb..00000000 --- a/examples/example_test/data_intermediate/taz_controls.csv +++ /dev/null @@ -1,37 +0,0 @@ -TAZ,TRACT,PUMA,DISTRICT,HHBASE,HHSIZE1,HHSIZE2,HHSIZE3,HHSIZE4,OSUFAM,SF -100,1,600,1,25,5,8,8,4,53,25 -101,1,600,1,25,5,8,8,4,53,25 -102,1,600,1,25,5,8,8,4,0,25 -103,2,600,1,25,5,8,8,4,0,25 -104,2,600,1,25,5,8,8,4,0,25 -105,2,600,1,25,5,8,8,4,0,25 -200,21,601,1,50,10,16,16,8,106,50 -201,21,601,1,50,10,16,16,8,106,50 -202,21,601,1,50,10,16,16,8,0,50 -203,22,601,1,50,10,16,16,8,0,50 -204,22,601,1,50,10,16,16,8,0,50 -205,22,601,1,50,10,16,16,8,0,50 -300,31,602,2,25,5,8,8,4,53,25 -301,31,602,2,25,5,8,8,4,53,25 -302,31,602,2,25,5,8,8,4,0,25 -303,32,602,2,25,5,8,8,4,0,25 -304,32,602,2,25,5,8,8,4,0,25 -305,32,602,2,25,5,8,8,4,0,25 -400,41,603,2,50,10,16,16,8,106,50 -401,41,603,2,50,10,16,16,8,106,50 -402,41,603,2,50,10,16,16,8,0,50 -403,42,603,2,50,10,16,16,8,0,50 -404,42,603,2,50,10,16,16,8,0,50 -405,42,603,2,50,10,16,16,8,0,50 -500,51,604,3,50,10,16,16,8,106,50 -501,51,604,3,50,10,16,16,8,106,50 -502,51,604,3,50,10,16,16,8,0,50 -503,52,604,3,50,10,16,16,8,0,50 -504,52,604,3,50,10,16,16,8,0,50 -505,52,604,3,50,10,16,16,8,0,50 -600,61,605,3,50,10,16,16,8,106,50 -601,61,605,3,50,10,16,16,8,106,50 -602,61,605,3,50,10,16,16,8,0,50 -603,62,605,3,50,10,16,16,8,0,50 -604,62,605,3,50,10,16,16,8,0,50 -605,62,605,3,50,10,16,16,8,0,50 diff --git a/examples/example_test/data_intermediate/tract_controls.csv b/examples/example_test/data_intermediate/tract_controls.csv deleted file mode 100644 index 4308af17..00000000 --- a/examples/example_test/data_intermediate/tract_controls.csv +++ /dev/null @@ -1,13 +0,0 @@ -TRACT,PUMA,HHBASE,HHSIZE1,HHSIZE2,HHSIZE3,HHSIZE4,OSUFAM,SF -1,600,75,15,24,24,12,106,75 -2,600,75,15,24,24,12,0,75 -21,601,150,30,48,48,24,212,150 -22,601,150,30,48,48,24,0,150 -31,602,75,15,24,24,12,106,75 -32,602,75,15,24,24,12,0,75 -41,603,150,30,48,48,24,212,150 -42,603,150,30,48,48,24,0,150 -51,604,150,30,48,48,24,212,150 -52,604,150,30,48,48,24,0,150 -61,605,150,30,48,48,24,212,150 -62,605,150,30,48,48,24,0,150 diff --git a/tests/test_intermediate_geography.py b/tests/test_intermediate_geography.py index 28c4cfd9..040c317b 100644 --- a/tests/test_intermediate_geography.py +++ b/tests/test_intermediate_geography.py @@ -49,8 +49,19 @@ def test_intermediate_geography(): example_dir = Path(__file__).parent.parent / "examples" / "example_test" - inject.add_injectable("data_dir", example_dir / "data_intermediate") - inject.add_injectable("configs_dir", example_dir / "configs_intermediate") + # configs_dir and data_dir both cascade: the first directory holding a + # given file wins. The *_intermediate directories therefore carry only what + # this example actually changes -- its settings, controls, and the crosswalk + # with the extra geography levels -- and everything shared (seed households + # and persons, TAZ and TRACT controls, logging) resolves from the base + # example. Copying those would leave this example silently testing stale + # inputs whenever the originals changed. + inject.add_injectable( + "data_dir", [example_dir / "data_intermediate", example_dir / "data"] + ) + inject.add_injectable( + "configs_dir", [example_dir / "configs_intermediate", example_dir / "configs"] + ) inject.add_injectable("output_dir", Path(__file__).parent / "output") inject.clear_cache() From d4196c4ebf8f7a33e896a5714d94922abe9666f4 Mon Sep 17 00:00:00 2001 From: nick-fournier <45876721+nick-fournier@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:16:44 -0700 Subject: [PATCH 5/5] Cover integerized intermediate hierarchies and document control handling --- docs/application_configuration.rst | 9 ++ .../configs_intermediate/settings.yaml | 122 ++---------------- populationsim/steps/final_seed_balancing.py | 7 +- .../steps/integerize_final_seed_weights.py | 6 +- populationsim/steps/meta_control_factoring.py | 4 +- tests/test_intermediate_geography.py | 111 +++++++++++++--- 6 files changed, 118 insertions(+), 141 deletions(-) diff --git a/docs/application_configuration.rst b/docs/application_configuration.rst index bdd22a0e..ef4c364e 100644 --- a/docs/application_configuration.rst +++ b/docs/application_configuration.rst @@ -345,6 +345,15 @@ These settings control the functionality of the PopulationSim algorithm. The set | seed_geography | PUMA | Seed geography name from the list of geographies | +----------------+---------------------+---------------------------------------------------------------------------------+ +The first geography is the meta geography. One or more intermediate geographies +may appear between it and the seed geography, for example +``geographies: [REGION, DISTRICT, PUMA, TRACT, TAZ]`` with +``seed_geography: PUMA``. Include each level in the geographic crosswalk. +Controls assigned to intermediate levels remain available in their control tables +and aggregate summaries, but are not distributed to seed zones or enforced during +seed balancing or integerization. Meta-level controls are still factored into the +seed controls as usual. + **Tracing**: diff --git a/examples/example_test/configs_intermediate/settings.yaml b/examples/example_test/configs_intermediate/settings.yaml index f9a7e92b..715b474a 100644 --- a/examples/example_test/configs_intermediate/settings.yaml +++ b/examples/example_test/configs_intermediate/settings.yaml @@ -1,133 +1,29 @@ -# This flag turns off integerization entirely (default False) -NO_INTEGERIZATION_EVER: True - -# These settings are ignored if not integerizing -INTEGERIZE_WITH_BACKSTOPPED_CONTROLS: True -SUB_BALANCE_WITH_FLOAT_SEED_WEIGHTS: False -GROUP_BY_INCIDENCE_SIGNATURE: True -INTEGERIZER_TIMEOUT: 300 # seconds -USE_SIMUL_INTEGERIZER: True +# Reuse the base example; only the geography hierarchy and input list differ. +inherit_settings: True geographies: [REGION, DISTRICT, PUMA, TRACT, TAZ] seed_geography: PUMA -# Tracing -# ------------------------------------------------------------------ -trace_geography: - TAZ: 100 - TRACT: 1 - - -household_weight_col: WGTP -household_id_col: hh_id -total_hh_control: num_hh -max_expansion_factor: 5 - -min_expansion_factor: 0.2 - -#control_file_name: controls.csv -#data_dir: data - -# input_pre_processor input_table_list input_table_list: - tablename: households - filename : seed_households.csv + filename: seed_households.csv index_col: hh_id rename_columns: SERIALNO: hh_id - tablename: persons - # expression_filename: seed_persons_expressions.csv - filename : seed_persons.csv + filename: seed_persons.csv rename_columns: SERIALNO: hh_id SPORDER: per_num - tablename: geo_cross_walk - filename : geo_cross_walk.csv + filename: geo_cross_walk.csv rename_columns: TRACTCE: TRACT - tablename: TAZ_control_data - filename : taz_controls.csv + filename: taz_controls.csv - tablename: TRACT_control_data - filename : tract_controls.csv + filename: tract_controls.csv - tablename: DISTRICT_control_data - filename : district_controls.csv + filename: district_controls.csv - tablename: REGION_control_data - filename: region_controls.csv - - -run_list: - steps: - - input_pre_processor - - setup_data_structures - - initial_seed_balancing - - meta_control_factoring - - final_seed_balancing - - integerize_final_seed_weights - - sub_balancing.geography=TRACT - - sub_balancing.geography=TAZ - - expand_households - - summarize - - write_tables - - write_synthetic_population - - #resume_after: expand_households - -# "-m repop" command line option allows specification of repop run_list` -# expand_households options are append or replace -repop: - steps: - - input_pre_processor.table_list=repop_input_table_list - - repop_setup_data_structures - - initial_seed_balancing.final=true - - integerize_final_seed_weights.repop - - repop_balancing - - expand_households.repop;replace - - write_tables.repop - - write_synthetic_population.repop - - resume_after: summarize - -repop_control_file_name: repop_controls.csv - -repop_input_table_list: - - filename : repop_taz_controls.csv - tablename: TAZ_control_data - - - -# output_tables can specify either a list of output tables to include or to skip -# if neither is specified, then no checkpointed tables will be written - -output_tables: - action: include - tables: - - PUMA_weights - - TAZ_weights - - TRACT_weights - - trace_TAZ_weights - - trace_TRACT_weights - - summary_hh_weights - - summary_TAZ - -# - expanded_household_ids - -# -#output_tables: -# action: include -# tables: -# - expanded_household_ids - -output_synthetic_population: - household_id: household_id - households: - filename: synthetic_households.csv - columns: - - NP - - WGTP - - HTYPE - persons: - filename: synthetic_persons.csv - columns: - - per_num - - OSUTAG - - OCCP + filename: region_controls.csv diff --git a/populationsim/steps/final_seed_balancing.py b/populationsim/steps/final_seed_balancing.py index 2cfbf601..755dd3f8 100644 --- a/populationsim/steps/final_seed_balancing.py +++ b/populationsim/steps/final_seed_balancing.py @@ -12,7 +12,6 @@ ) from populationsim.balancing import do_balancing - logger = logging.getLogger(__name__) @@ -58,10 +57,10 @@ def final_seed_balancing(settings, crosswalk, control_spec, incidence_table): ) return - # we use all control_spec rows, so no need to filter on geography as for initial_seed_balancing + # Seed controls include factored meta targets, but no intermediate targets. seed_controls_df = get_control_table(seed_geography) - intermediate_geographies = geographies[1:geographies.index(seed_geography)] - control_spec = control_spec[control_spec.geography.apply(lambda x: x not in intermediate_geographies)] + intermediate_geographies = geographies[1 : geographies.index(seed_geography)] + control_spec = control_spec[~control_spec.geography.isin(intermediate_geographies)] assert (seed_controls_df.columns == control_spec.target).all() # determine master_control_index if specified in settings diff --git a/populationsim/steps/integerize_final_seed_weights.py b/populationsim/steps/integerize_final_seed_weights.py index 8b52f1d1..ca9cd6b9 100644 --- a/populationsim/steps/integerize_final_seed_weights.py +++ b/populationsim/steps/integerize_final_seed_weights.py @@ -49,10 +49,10 @@ def integerize_final_seed_weights(settings, crosswalk, control_spec, incidence_t seed_weights_df = get_weight_table(seed_geography) - # FIXME - I assume we want to integerize using meta controls too? + # Match the seed controls, including meta targets but excluding intermediate ones. geographies = settings["geographies"] - intermediate_geographies = geographies[1:geographies.index(seed_geography)] - control_spec = control_spec[control_spec.geography.apply(lambda x: x not in intermediate_geographies)] + intermediate_geographies = geographies[1 : geographies.index(seed_geography)] + control_spec = control_spec[~control_spec.geography.isin(intermediate_geographies)] control_cols = control_spec.target assert (seed_controls_df.columns == control_cols).all() diff --git a/populationsim/steps/meta_control_factoring.py b/populationsim/steps/meta_control_factoring.py index 96f3254c..a2c0b703 100644 --- a/populationsim/steps/meta_control_factoring.py +++ b/populationsim/steps/meta_control_factoring.py @@ -123,8 +123,8 @@ def meta_control_factoring(settings, control_spec, incidence_table): seed_controls_df = get_control_table(seed_geography) # Remove controls for geographies between the seed and the meta geographies - intermediate_geographies = geographies[1:geographies.index(seed_geography)] - control_spec = control_spec[control_spec.geography.apply(lambda x: x not in intermediate_geographies)] + intermediate_geographies = geographies[1 : geographies.index(seed_geography)] + control_spec = control_spec[~control_spec.geography.isin(intermediate_geographies)] assert len(seed_controls_df.index) == len(seed_level_meta_controls.index) seed_controls_df = pd.concat([seed_controls_df, seed_level_meta_controls], axis=1) diff --git a/tests/test_intermediate_geography.py b/tests/test_intermediate_geography.py index 040c317b..803e9f03 100644 --- a/tests/test_intermediate_geography.py +++ b/tests/test_intermediate_geography.py @@ -1,8 +1,10 @@ from pathlib import Path +import numpy as np +import pandas as pd import pytest -from populationsim.core import inject, pipeline, tracing +from populationsim.core import config, inject, pipeline, tracing # examples/example_test/configs_intermediate declares # geographies: [REGION, DISTRICT, PUMA, TRACT, TAZ] @@ -45,7 +47,9 @@ def teardown_function(func): inject.reinject_decorated_tables() -def test_intermediate_geography(): +@pytest.mark.parametrize("integerize", [False, True], ids=["fractional", "integerized"]) +@pytest.mark.parametrize("intermediate_levels", [1, 2], ids=["one-level", "two-levels"]) +def test_intermediate_geography(tmp_path, integerize, intermediate_levels): example_dir = Path(__file__).parent.parent / "examples" / "example_test" @@ -62,10 +66,46 @@ def test_intermediate_geography(): inject.add_injectable( "configs_dir", [example_dir / "configs_intermediate", example_dir / "configs"] ) - inject.add_injectable("output_dir", Path(__file__).parent / "output") + inject.add_injectable("output_dir", tmp_path) inject.clear_cache() + config.override_setting("NO_INTEGERIZATION_EVER", not integerize) + intermediate_targets = {INTERMEDIATE_GEOGRAPHY: INTERMEDIATE_CONTROL} + meta_controls = META_CONTROLS + geographies = ["REGION", INTERMEDIATE_GEOGRAPHY, SEED_GEOGRAPHY, "TRACT", "TAZ"] + + if intermediate_levels == 2: + # Insert AREA above DISTRICT, with its own target. Both intermediate + # targets must be excluded; filtering only the level next to seed fails. + inputs_dir = tmp_path / "inputs" + inputs_dir.mkdir() + crosswalk = pd.read_csv( + example_dir / "data_intermediate" / "geo_cross_walk.csv", comment="#" + ) + crosswalk["AREA"] = crosswalk["DISTRICT"].map({1: 1, 2: 1, 3: 2}) + crosswalk.to_csv(inputs_dir / "geo_cross_walk.csv", index=False) + pd.DataFrame({"AREA": [1, 2], "OCCP2": [180, 120]}).to_csv( + inputs_dir / "area_controls.csv", index=False + ) + controls = pd.read_csv(example_dir / "configs_intermediate" / "controls.csv") + controls.loc[controls.target == "persons_occ_2", "geography"] = "AREA" + controls.to_csv(inputs_dir / "controls.csv", index=False) + geographies.insert(1, "AREA") + config.override_setting("geographies", geographies) + settings = inject.get_injectable("settings") + config.override_setting( + "input_table_list", + settings["input_table_list"] + + [{"tablename": "AREA_control_data", "filename": "area_controls.csv"}], + ) + for directory in ("data_dir", "configs_dir"): + inject.add_injectable( + directory, [inputs_dir] + inject.get_injectable(directory) + ) + intermediate_targets["AREA"] = "persons_occ_2" + meta_controls = ("persons_occ_3",) + tracing.config_logger() # Completing at all is the primary regression. Without the control_spec @@ -76,26 +116,40 @@ def test_intermediate_geography(): pipeline.run(models=_MODELS, resume_after=None) crosswalk = pipeline.get_table("crosswalk") - assert list(crosswalk.columns) == [ - "REGION", - INTERMEDIATE_GEOGRAPHY, - SEED_GEOGRAPHY, - "TRACT", - "TAZ", - ] + assert list(crosswalk.columns) == geographies # The behaviour under test: controls belonging to a geography between the # meta and seed geographies are kept out of the seed-level control table, # while the meta geography's controls are still factored into it. seed_controls = pipeline.get_table(f"{SEED_GEOGRAPHY}_controls") - assert INTERMEDIATE_CONTROL not in seed_controls.columns - for target in META_CONTROLS: + for target in intermediate_targets.values(): + assert target not in seed_controls.columns + for target in meta_controls: assert target in seed_controls.columns - # The intermediate control is excluded from the seed level, not discarded: - # it still constrains its own geography. - intermediate_controls = pipeline.get_table(f"{INTERMEDIATE_GEOGRAPHY}_controls") - assert INTERMEDIATE_CONTROL in intermediate_controls.columns + # Intermediate targets remain available for reporting. This feature does + # not distribute them to seed zones or enforce them during balancing. + for geography, target in intermediate_targets.items(): + intermediate_controls = pipeline.get_table(f"{geography}_controls") + assert target in intermediate_controls.columns + for sub_geography in ("TRACT", "TAZ"): + summary = pipeline.get_table(f"summary_{sub_geography}_{geography}") + assert set(summary["id"]) == set(crosswalk[geography]) + np.testing.assert_allclose( + summary[f"{target}_control"], + intermediate_controls.loc[summary["id"], target], + ) + weights = pipeline.get_table(f"{sub_geography}_weights") + incidence = pipeline.get_table("incidence_table") + weight_col = "integer_weight" if integerize else "balanced_weight" + expected_results = ( + (weights[weight_col] * weights["hh_id"].map(incidence[target])) + .groupby(weights[geography]) + .sum() + ) + np.testing.assert_allclose( + summary[f"{target}_result"], expected_results.loc[summary["id"]] + ) # Sub-geography balancing ran and preserved the seed-level household total. seed_total = pipeline.get_table(f"{SEED_GEOGRAPHY}_weights")[ @@ -107,6 +161,25 @@ def test_intermediate_geography(): assert (weights >= 0).all() assert weights.sum() == pytest.approx(seed_total, rel=1e-6) - # This example sets NO_INTEGERIZATION_EVER, so expand_households produces - # nothing -- assert that explicitly rather than leaving it implied. - assert pipeline.get_table("expanded_household_ids").empty + expanded = pipeline.get_table("expanded_household_ids") + if integerize: + seed_weights = pipeline.get_table(f"{SEED_GEOGRAPHY}_weights") + assert (seed_weights.integer_weight >= 0).all() + assert (seed_weights.integer_weight % 1 == 0).all() + seed_totals = seed_weights.groupby(SEED_GEOGRAPHY).integer_weight.sum() + pd.testing.assert_series_equal( + seed_totals, seed_controls.num_hh, check_dtype=False, check_names=False + ) + taz_weights = pipeline.get_table("TAZ_weights") + expected_counts = taz_weights.groupby("TAZ").integer_weight.sum() + pd.testing.assert_series_equal( + expanded.groupby("TAZ").size(), + expected_counts, + check_dtype=False, + check_names=False, + ) + assert len(expanded) == expected_counts.sum() + assert (tmp_path / "synthetic_households.csv").exists() + assert (tmp_path / "synthetic_persons.csv").exists() + else: + assert expanded.empty