Update development branch

1_code/_setup/00_mirror_setup_inputs.R

@@ -29,9 +29,9 @@
 #   - Not mirrored, deliberately:
 #     - the v2 coefficient outputs on the drives (the bird
 #       coefficient CSVs, COEFS.RData, the mammal coefficient
-#       tables). Parity is scored against ABMIexploreR (08), so
+#       tables). Parity is scored against ABMIexploreR (06), so
 #       nothing in _setup/ reads them.
-#     - _setup/08's species-coefs.RData, which 08 downloads from
+#     - _setup/06's species-coefs.RData, which 06 downloads from
 #       the public ABMIexploreR repository at a pinned commit.
 #     - 0_data/v2_scripts/, which is tracked in git.
 #   - Run once, from the repository root, by someone who can
@@ -259,7 +259,7 @@ writeLines(c(
   "- `inputs_manifest.csv` gives each file's original location,",
   "  size, md5 and the date it was copied.",
   "- The originals were left in place; the manifest names them.",
-  "- 04 and 05's inputs are not here: their output,",
+  "- The retired 04 and 05's inputs are not here: their output,",
   "  `v2_results/`, is published frozen under `../v2_results/`."
 ), setup_input("README.md"))
 

1_code/_setup/02_validate_test_dataset.R

@@ -1258,8 +1258,8 @@ if (file.exists(bird_data_file)) {
 }
 
 ## 3.6.4 Harmonized lookups ----
-# _setup/09's outputs against the files checked above, so a
-# lookup left stale by a rebuild of 01 or 06 fails
+# _setup/07's outputs against the files checked above, so a
+# lookup left stale by a rebuild of 01 or 04 fails
 lookup_dir <- file.path(data_dir, "lookup")
 harmonized <- c(
   "species_queue.csv", "factor_levels.csv", "dataset_manifest.csv",
@@ -1272,7 +1272,7 @@ note(
   "lookups", "harmonized lookups written", all(present),
   if (all(present)) "" else paste0(
     "missing ", paste(harmonized[!present], collapse = ", "),
-    "; run 09_harmonize_lookups.R"
+    "; run 07_harmonize_lookups.R"
   )
 )
 

1_code/_setup/08_harmonize_abmiexplorer_results.R → 1_code/_setup/06_harmonize_abmiexplorer_results.R

@@ -30,23 +30,24 @@
 #     models the harness reproduces). Only the data file is
 #     downloaded, not the package and its dependencies.
 #
-#   - Checked against 05's reference on 2026-10-02, at the pinned
-#     commit:
+#   - Checked against the retired 05's reference on 2026-10-02,
+#     at the pinned commit:
 #     - Plants and birds are the same models. Every shared
 #       species and term matches to ~1e-15, except species where
-#       some v2 draws failed. 05 drops the failed draws (v2_n
+#       some v2 draws failed. Old 05 drops the failed draws (v2_n
 #       below 100); the package holds 100 finite draws for every
 #       cell, so its median moves slightly for those species.
 #     - The package is the published subset: about 10% fewer
 #       plant species than COEFS.RData, and 124 of 133 birds.
 #       Bryum.All, AMRO and YEWA, in the parity_check set, are
 #       not in it.
-#     - Mammals are the same models as 05's 2024 coefficient
-#       tables, presence, abundance and total, north and south,
-#       to ~1e-16 once back-transformed. The package stores them
-#       on the link scale (logit for presence, log for the other
-#       two) where the tables are on the response scale, which
-#       the harness writes, so they are back-transformed here.
+#     - Mammals are the same models as old 05's 2024
+#       coefficient tables, presence, abundance and total, north
+#       and south, to ~1e-16 once back-transformed. The package
+#       stores them on the link scale (logit for presence, log for
+#       the other two) where the tables are on the response scale,
+#       which the harness writes, so they are back-transformed
+#       here.
 #       Its 100 draws are one fit repeated, as v2 fits mammal
 #       habitat once, so they are written as v2_n 1 and the
 #       comparison reads iteration 1, as it does against 05.
@@ -115,7 +116,7 @@ for (one in c(source_dir, results_dir)) {
 }
 
 ## 1.4 Name the output columns ----
-# Identical to 05, so the comparison reads either file.
+# Identical to old 05, so the comparison reads either file.
 result_columns <- c(
   "taxon", "region", "stage", "part", "season",
   "species", "species_v2", "term",
@@ -191,7 +192,7 @@ mammal_parts <- c(
 
 # The package stores mammal habitat on the link scale; v2's
 # tables, and the harness, are on the response scale. Verified
-# exact against 05 on 2026-10-02. Mammal climate is on the link
+# exact against old 05 on 2026-10-02. Mammal climate is on the link
 # scale in both, so it is left alone.
 mammal_inverse_links <- list(
   presence = stats::plogis, abundance = exp, total = exp
@@ -213,7 +214,8 @@ part_of <- function(slug, stage, hurdle = NA) {
 
 #' Package Terms Under the Names the Harness Writes
 #'
-#' Each pair was checked on 2026-10-02 against 05's reference:
+#' Each pair was checked on 2026-10-02 against old 05's
+#' reference:
 #' the package term's medians equal the v2 term's for every
 #' shared species in the north. The package collapsed the fen
 #' age classes and renamed the human-footprint classes; birds

1_code/_setup/09_harmonize_lookups.R → 1_code/_setup/07_harmonize_lookups.R

@@ -3,7 +3,7 @@
 # author: Brendan Casey
 # created: 2026-10-03
 # inputs:
-#   in 0_data/test_dataset/lookup/ (all written by 01 and 06):
+#   in 0_data/test_dataset/lookup/ (all written by 01 and 04):
 #     - modelled_species.csv, mammal_modelled_species.csv,
 #       bird_modelled_species.csv
 #     - bird_factor_levels.csv
@@ -57,7 +57,7 @@ read_lookup <- function(name) {
   if (!file.exists(path)) {
     stop(
       "Missing ", path, ". Run 01_harmonize_model_ready_v2.R ",
-      "(and 06 for the bird matrix) first.",
+      "(and 04 for the bird matrix) first.",
       call. = FALSE
     )
   }

1_code/_setup/10_publish_datasets.R → 1_code/_setup/08_publish_datasets.R

@@ -4,7 +4,7 @@
 # created: 2026-10-05
 # inputs:
 #   - 0_data/test_dataset/, from _setup/01-09, checked by 02
-#   - 0_data/v2_results/abmiexplorer/, from _setup/08
+#   - 0_data/v2_results/abmiexplorer/, from _setup/06
 # outputs:
 #   in //ABMI-DATA2/science/sdmMethodsDev/0_data/ (SDM_SHARE_ROOT
 #   overrides it), for each dataset:
@@ -180,7 +180,7 @@ publish_dataset <- function(name, version) {
     paste0("- R: ", R.version.string),
     "",
     "Built by 1_code/_setup/ in the sdmMethodsDev repository and",
-    "published by 1_code/_setup/10_publish_datasets.R. Never",
+    "published by 1_code/_setup/08_publish_datasets.R. Never",
     "edited after publishing; checksums.csv lists every file."
   ), file.path(partial, "publish_record.md"))
 

1_code/_setup/README.md

@@ -20,27 +20,28 @@ source("1_code/_setup/run.R")
 
 `run.R` runs the steps in dependency order, with on/off switches in
 section 1.2, and logs timings to `2_pipeline/_setup/run_log.csv`. `00`,
-`03` and `10` are off by default. Run order: `01`, `06`, `09`, `02`,
-`03`, `07`, `08`, then `10`. There is no `04` or `05`; they were
-retired once `08` replaced them.
+`03` and `08` are off by default. Run order: `01`, `04`, `07`, `02`,
+`03`, `05`, `06`, then `08`. The original `04` and `05` were retired
+once `06` replaced them; the later scripts were renumbered on
+2026-10-08 to close the gap.
 
 ## Scripts
 
 | Script | Reads | Writes |
 | --- | --- | --- |
 | `00_mirror_setup_inputs.R` | Every external input, at its original location | `setup_inputs/` on ABMI-DATA2, with `inputs_manifest.csv`. Run once; never overwrites |
 | `01_harmonize_model_ready_v2.R` | The `model_ready_v2` snapshot, mammal camera climate, WildTrax species lookup, v2 prediction matrices, bird `Stratified.Rdata` | `0_data/test_dataset/`: responses, `sites.csv`, `covariates.csv`, `bird_offsets.csv`, and most of `lookup/` |
-| `02_validate_test_dataset.R` | The test dataset and the same sources | Nothing; prints a pass / fail tally. Run after `01`, `06` and `09` |
+| `02_validate_test_dataset.R` | The test dataset and the same sources | Nothing; prints a pass / fail tally. Run after `01`, `04` and `07` |
 | `03_rerun_bryophyte_v2_reference.R` | v2 plant project bryophyte data, frozen v2 functions in `0_data/v2_scripts/plants/` | `2_pipeline/v2_reference/`: `bryophyte-species-models.Rdata`, `bryophyte-bootstrap-ids.Rdata`. Takes hours |
-| `06_harmonize_bird_translation_lookup.R` | v2's bird `Xn-veg-v2024.Rdata` | `lookup/bird_veg_age_matrix.csv`. Re-run whenever `01` rebuilds `lookup/` |
-| `07_harmonize_v2_plant_bootstrap_ids.R` | v2's stored plant draws; `03`'s bryophyte draws | `lookup/v2_bootstrap_ids/<taxon>/<species>.rds`, for `plant_bootstrap = "v2_ids"` |
-| `08_harmonize_abmiexplorer_results.R` | ABMIexploreR's `species-coefs.RData` at a pinned commit; bird `birdlist.csv` | `0_data/external/abmiexplorer/<sha>/`; `0_data/v2_results/abmiexplorer/v2_results.csv` and `v2_results_coverage.csv` |
-| `09_harmonize_lookups.R` | `0_data/test_dataset/lookup/` only; offline, seconds | `species_queue.csv`, `factor_levels.csv`, `dataset_manifest.csv`, bird prediction grids |
-| `10_publish_datasets.R` | `0_data/test_dataset/`, `0_data/v2_results/` | A new `<version>/` of each on ABMI-DATA2, with `checksums.csv` and `publish_record.md` |
+| `04_harmonize_bird_translation_lookup.R` | v2's bird `Xn-veg-v2024.Rdata` | `lookup/bird_veg_age_matrix.csv`. Re-run whenever `01` rebuilds `lookup/` |
+| `05_harmonize_v2_plant_bootstrap_ids.R` | v2's stored plant draws; `03`'s bryophyte draws | `lookup/v2_bootstrap_ids/<taxon>/<species>.rds`, for `plant_bootstrap = "v2_ids"` |
+| `06_harmonize_abmiexplorer_results.R` | ABMIexploreR's `species-coefs.RData` at a pinned commit; bird `birdlist.csv` | `0_data/external/abmiexplorer/<sha>/`; `0_data/v2_results/abmiexplorer/v2_results.csv` and `v2_results_coverage.csv` |
+| `07_harmonize_lookups.R` | `0_data/test_dataset/lookup/` only; offline, seconds | `species_queue.csv`, `factor_levels.csv`, `dataset_manifest.csv`, bird prediction grids |
+| `08_publish_datasets.R` | `0_data/test_dataset/`, `0_data/v2_results/` | A new `<version>/` of each on ABMI-DATA2, with `checksums.csv` and `publish_record.md` |
 | `run.R` | The scripts above | Runs them in order |
 | `utils/input_paths.R` | | `setup_input()`: paths inside the input mirror |
 
-After `10`, move the pin in `1_code/harness/data_source.R`. A
+After `08`, move the pin in `1_code/harness/data_source.R`. A
 published version is never overwritten.
 
 ## The test dataset
@@ -74,16 +75,16 @@ Each overrides a default path.
 
 | Variable | Read by | Points at |
 | --- | --- | --- |
-| `SDM_SETUP_INPUT_ROOT` | `00`–`08` | The input mirror |
+| `SDM_SETUP_INPUT_ROOT` | `00`–`06` | The input mirror |
 | `SDM_SNAPSHOT_V2` | `01`, `02` | The `model_ready_v2` snapshot folder |
 | `SDM_MAMMAL_CAMERA_CLIMATE` | `01` | `abmi-camera-climate_2023.Rdata` |
 | `SDM_MAMMAL_CLIMATE_PRED` | `01`, `02` | `All Species Climate Predictions.csv` |
 | `SDM_V2_LOOKUP` | `01` | The v2 folder holding the plant prediction matrices |
 | `SDM_WT_SPECIES` | `01` | `WildTrax Species Strings.RData` |
 | `SDM_BIRD_DATA` | `01`, `02` | The bird `Stratified.Rdata` v2 was fitted on |
-| `SDM_V2_PROJECT` | `03`, `07` | The v2 plant project (`VegetationModels`) |
-| `SDM_V2_BIRD_ROOT` | `06` | The v2 bird project (`BirdModels`) |
-| `SDM_V2_BOOT_TAXA`, `SDM_V2_BOOT_SPECIES` | `07` | Comma-separated subsets to copy |
-| `SDM_ABMIEXPLORER_SHA` | `08` | The ABMIexploreR commit to download |
-| `SDM_V2_BIRD_LOOKUP` | `08` | The v2 bird `birdlist.csv` |
-| `SDM_SHARE_ROOT`, `SDM_DATASET_VERSION`, `SDM_PUBLISH` | `10` | Publish root, version folder (default today, `YYYY-MM-DD`), datasets |
+| `SDM_V2_PROJECT` | `03`, `05` | The v2 plant project (`VegetationModels`) |
+| `SDM_V2_BIRD_ROOT` | `04` | The v2 bird project (`BirdModels`) |
+| `SDM_V2_BOOT_TAXA`, `SDM_V2_BOOT_SPECIES` | `05` | Comma-separated subsets to copy |
+| `SDM_ABMIEXPLORER_SHA` | `06` | The ABMIexploreR commit to download |
+| `SDM_V2_BIRD_LOOKUP` | `06` | The v2 bird `birdlist.csv` |
+| `SDM_SHARE_ROOT`, `SDM_DATASET_VERSION`, `SDM_PUBLISH` | `08` | Publish root, version folder (default today, `YYYY-MM-DD`), datasets |

1_code/_setup/run.R

@@ -7,11 +7,11 @@
 #     (see their headers); most read the input mirror on
 #     ABMI-DATA2 through utils/input_paths.R
 # outputs:
-#   - 0_data/test_dataset/, from 01, 06 and 09
-#   - 0_data/test_dataset/lookup/v2_bootstrap_ids/, from 07
-#   - 0_data/v2_results/abmiexplorer/, from 08
+#   - 0_data/test_dataset/, from 01, 04 and 07
+#   - 0_data/test_dataset/lookup/v2_bootstrap_ids/, from 05
+#   - 0_data/v2_results/abmiexplorer/, from 06
 #   - 2_pipeline/v2_reference/, from 03
-#   - a new published version on ABMI-DATA2, from 10, only if
+#   - a new published version on ABMI-DATA2, from 08, only if
 #     switched on
 #   - 2_pipeline/_setup/run_log.csv: per step, start, end,
 #     minutes and whether it finished
@@ -24,7 +24,7 @@
 #       someone who can reach the original drives.
 #     - 03 re-runs the v2 bryophyte models: hours, and its
 #       output only changes if its sources do.
-#     - 10 publishes a new version. Run it deliberately, after
+#     - 08 publishes a new version. Run it deliberately, after
 #       checking 02's tally, then move the pin in
 #       1_code/harness/data_source.R.
 #   - Each script runs in its own environment; an error stops
@@ -41,18 +41,18 @@
 # Base R only; each script loads its own.
 
 ## 1.2 Choose the steps ----
-# In run order: 09 reads 01's and 06's lookups, 02 checks the
-# finished dataset, 07 reads 03's bryophyte draws.
+# In run order: 07 reads 01's and 04's lookups, 02 checks the
+# finished dataset, 05 reads 03's bryophyte draws.
 steps <- c(
   "00_mirror_setup_inputs.R" = FALSE,
   "01_harmonize_model_ready_v2.R" = TRUE,
-  "06_harmonize_bird_translation_lookup.R" = TRUE,
-  "09_harmonize_lookups.R" = TRUE,
+  "04_harmonize_bird_translation_lookup.R" = TRUE,
+  "07_harmonize_lookups.R" = TRUE,
   "02_validate_test_dataset.R" = TRUE,
   "03_rerun_bryophyte_v2_reference.R" = FALSE,
-  "07_harmonize_v2_plant_bootstrap_ids.R" = TRUE,
-  "08_harmonize_abmiexplorer_results.R" = TRUE,
-  "10_publish_datasets.R" = FALSE
+  "05_harmonize_v2_plant_bootstrap_ids.R" = TRUE,
+  "06_harmonize_abmiexplorer_results.R" = TRUE,
+  "08_publish_datasets.R" = FALSE
 )
 
 ## 1.3 Check the paths ----
@@ -96,7 +96,7 @@ log_path <- file.path(log_dir, "run_log.csv")
 #' @return TRUE, invisibly; stops if the script errors.
 #'
 #' @example # Example usage of the function
-#' # run_step("09_harmonize_lookups.R", setup_dir, log_path)
+#' # run_step("07_harmonize_lookups.R", setup_dir, log_path)
 run_step <- function(script, setup_dir, log_path) {
   # Step 1: Announce and time the step
   message("\n== ", script, " (", format(Sys.time(), "%H:%M"), ")")

1_code/experiments/exp_000_parity_v2/01_compare_to_v2.R

@@ -7,7 +7,7 @@
 #     n_bootstraps, v2_bootstraps, focal_species
 #   - `results$coefficient_summary`, from collect_results()
 #   - in config$v2_reference_dir:
-#     - v2_results.csv, built by _setup/08 from the pinned
+#     - v2_results.csv, built by _setup/06 from the pinned
 #       ABMIexploreR coefficients
 # outputs:
 #   - in out_dir/tables/: parity_terms.csv, parity_summary.csv

1_code/experiments/exp_000_parity_v2/README.md

@@ -35,7 +35,7 @@ the gate, set `species = NULL` and `n_bootstraps = 100`.
 
 `utils/parity_targets.R` holds the targets the compare step reads.
 `v2_self_agreement.R` is not part of `run.R`: run it once, after
-`_setup/03` and `_setup/08`, to write
+`_setup/03` and `_setup/06`, to write
 `tables/v2_self_agreement.csv` and `v2_self_agreement_summary.csv`.
 
 ## Inputs and outputs
@@ -69,7 +69,7 @@ If this table and the specs disagree, the specs are right.
 
 ## The v2 reference
 
-`v2_results.csv` is built by `_setup/08` from ABMIexploreR's published
+`v2_results.csv` is built by `_setup/06` from ABMIexploreR's published
 coefficients (`data/species-coefs.RData`), pinned to commit `848eeed`.
 On 2026-10-02 it matched the retired by-hand build from the network
 drives to about 1e-15 on every shared species and term. Gaps:

1_code/experiments/exp_000_parity_v2/run.R

@@ -54,13 +54,13 @@ config <- experiment_config(
 
   # 100 is v2's and the only count the gate reads; below ~20 the
   # 10-90% bands are unstable. 5 checks the plumbing.
-  n_bootstraps = 5,
+  n_bootstraps = 100,
 
   # Base seed; NULL is unseeded, as v2. Birds replay v2's draws.
   seed = 20260909,
 
   # "spatial_block" draws v2's plant bootstrap afresh;
-  # "v2_ids" replays v2's stored draws (needs _setup/07).
+  # "v2_ids" replays v2's stored draws (needs _setup/05).
   plant_bootstrap = "spatial_block",
 
   # NULL fits each spec's v2 candidate sets
@@ -75,13 +75,14 @@ config <- experiment_config(
 
 ## 1.3 The v2 reference ----
 # ABMIexploreR's published coefficients, pinned to a commit and
-# harmonized by _setup/08
+# harmonized by _setup/06
 config$v2_reference <- "abmiexplorer"
 config$v2_reference_dir <- file.path(
-  v2_results_dir(), "abmiexplorer"
+  v2_results_dir(),
+  "abmiexplorer"
 )
 config$v2_reference_builder <-
-  "1_code/_setup/08_harmonize_abmiexplorer_results.R"
+  "1_code/_setup/06_harmonize_abmiexplorer_results.R"
 
 # 2. Run ----
 # `fit = FALSE` re-summarizes existing stores without fitting

1_code/experiments/exp_000_parity_v2/v2_self_agreement.R

@@ -7,7 +7,7 @@
 #     from 1_code/_setup/03_rerun_bryophyte_v2_reference.R
 #   - abmiexplorer/v2_results.csv from the published v2 results
 #     (see 1_code/harness/data_source.R), the bryophyte climate
-#     rows, from _setup/08. These are v2's original run: they
+#     rows, from _setup/06. These are v2's original run: they
 #     match v2's COEFS.RData to ~1e-15
 # outputs:
 #   in 3_output/exp_000_parity_v2/tables/:
@@ -40,7 +40,7 @@
 #   - Climate only: the regenerated reference has no habitat
 #     stage. The rate is assumed to carry to the plant habitat
 #     stage; that is an assumption, stated in the README.
-#   - Run once, from the repository root, after _setup/03 and 08.
+#   - Run once, from the repository root, after _setup/03 and 06.
 #     Not part of run.R's sequence.
 # ---
 

1_code/experiments/exp_001_xgboost/run.R

@@ -29,17 +29,19 @@ load_framework()
 
 ## 1.2 The change: fit the habitat stage with xgboost ----
 xgb_settings <- list(
-  nrounds = 1000,          # the most rounds; early stopping picks
-  eta = 0.05,              # learning rate
-  max_depth = 3,           # tree complexity
-  subsample = 0.5,         # share of rows each tree sees
-  min_child_weight = 1     # smallest leaf, in weighted rows
+  nrounds = 1000, # the most rounds; early stopping picks
+  eta = 0.05, # learning rate
+  max_depth = 3, # tree complexity
+  subsample = 0.5, # share of rows each tree sees
+  min_child_weight = 1 # smallest leaf, in weighted rows
 )
 
 # One call per run: each spec names its own habitat stage
 specs <- lapply(
-  standard_specs(), replace_stage_method,
-  engine = "xgboost", control = xgb_settings
+  standard_specs(),
+  replace_stage_method,
+  engine = "xgboost",
+  control = xgb_settings
 )
 
 ## 1.3 Configure the run ----
@@ -49,7 +51,7 @@ config <- experiment_config(
   taxa = names(specs),
   species = "parity_check",
   # 5 checks the pipeline runs; 100 for a result to read
-  n_bootstraps = 5,
+  n_bootstraps = 100,
   seed = 20260909,
   specs = specs,
   workers = 12

1_code/harness/data_load.R

@@ -12,7 +12,7 @@
 # outputs: none; returns objects in memory
 # notes:
 #   - Taxon-agnostic: file locations and covariate keys come from
-#     lookup/dataset_manifest.csv (_setup/09). The key is
+#     lookup/dataset_manifest.csv (_setup/07). The key is
 #     `<taxon>_<region>` for mammals, whose north and south files
 #     hold different values for the same deployment.
 #   - Reads are column-selective (covariates.csv is ~218 MB, 457
@@ -50,7 +50,7 @@ dataset_manifest <- function(data_dir) {
       "Dataset manifest not found:
   ", path,
       "
-Run 1_code/_setup/09_harmonize_lookups.R first.",
+Run 1_code/_setup/07_harmonize_lookups.R first.",
       call. = FALSE
     )
   }