diff --git a/1_code/_setup/00_mirror_setup_inputs.R b/1_code/_setup/00_mirror_setup_inputs.R index 3d52e72..9d037b0 100644 --- a/1_code/_setup/00_mirror_setup_inputs.R +++ b/1_code/_setup/00_mirror_setup_inputs.R @@ -111,6 +111,11 @@ sources <- list( "Results/Habitat Modeling/2024/Climate/Predictions/", "All Species Climate Predictions.csv" ), + # The table v2's mammal climate stage fits on + paste0( + "Results/Density/R Dataset SpTable for ABMI North and ", + "South mammal coefficients 2022.RData" + ), "Data/Lookup Tables/WildTrax Species Strings.RData" ) ) diff --git a/1_code/_setup/01_harmonize_model_ready_v2.R b/1_code/_setup/01_harmonize_model_ready_v2.R index 523fcd8..468fc66 100644 --- a/1_code/_setup/01_harmonize_model_ready_v2.R +++ b/1_code/_setup/01_harmonize_model_ready_v2.R @@ -16,6 +16,9 @@ # coefficients 2024.RData # from the ABMI Mammals shared drive: # - Lookup Tables/WildTrax Species Strings.RData +# - Results/Density/R Dataset SpTable for ABMI North and +# South mammal coefficients 2022.RData, the table v2's +# mammal climate stage fits on (section 1.7b) # from the work_abmi shared drive, read-only: # - 1_projects/active/sdmMethodsDev/remote/birds_data_v2/ # Stratified.Rdata, the copy v2's birds were fitted on @@ -62,6 +65,10 @@ # - Mammal covariates and prediction matrices come from the # SpTable files (matrices checked identical to the ABMI # Mammals drive CSVs); the climate block is replaced (1.6). +# - The mammal climate stage has its own rows, covariate key +# `mammal_province`, from v2's combined 2022 table (1.7b). +# They are written after every other taxon's rows, so the +# existing rows keep their order. # - Birds come from Stratified.Rdata (1.10), read-only # (SDM_BIRD_DATA overrides). Prediction matrices come from # the vegetation models project (1.8; SDM_V2_LOOKUP @@ -288,6 +295,24 @@ mammal_climate_pred_file <- Sys.getenv( ) ) +## 1.7b Name the mammal climate table ---- +# v2's climate stage (MammalModels climate/01_prepare-data.R) +# fits on this combined table, not the 2024 regional ones: it +# holds 120 deployments the regional habitat filters drop. It +# also carries the archived prep script's bug that blanks summer +# densities when WinterDays < 10; kept, as v2 fitted on it. +mammal_climate_data_file <- Sys.getenv( + "SDM_MAMMAL_CLIMATE_DATA", + unset = setup_input( + "abmi_mammals", + "Results/Density", + paste0( + "R Dataset SpTable for ABMI North and South mammal ", + "coefficients 2022.RData" + ) + ) +) + ## 1.8 Name the v2 prediction-matrix lookups ---- # The v2 scripts read these CSVs, not the snapshot's veg.pm and # soil.pm. The veg CSV adds Peatland, Mineral, Upland and @@ -1157,6 +1182,152 @@ read_mammal_region <- function(path, region, native_sp) { )) } +## 2.9b read_mammal_province() ---- + +#' Read v2's Combined Mammal Table for the Climate Stage +#' +#' Same unit, site and climate rules as read_mammal_region(), on +#' the 2022 combined file's own layout (`FirstSpCol.s` to +#' `LastSpCol.w`). The only habitat columns kept are the two the +#' region filters read. There is no species queue or prediction +#' matrix: the climate stage fits whatever the regions fit. +#' +#' @param path Character. Path to the combined 2022 .RData file. +#' @param native_sp Character. WildTrax common-name strings. +#' @return A list with `species`, `sites`, `climate` and `veg`, or +#' NULL when the file is absent. +#' +#' @example # Example usage of the function +#' # out <- read_mammal_province(mammal_climate_data_file, +#' # native_sp) +read_mammal_province <- function(path, native_sp) { + if (!file.exists(path)) { + return(NULL) + } + + env <- load_rdata(path) + d <- as.data.frame(env$d, stringsAsFactors = FALSE) + + # Step 1: The species block, from the file's own indices + span <- env$FirstSpCol.s:env$LastSpCol.w + + if (anyNA(span) || max(span) > ncol(d)) { + stop( + "Species column indices in ", basename(path), + " fall outside the data.", + call. = FALSE + ) + } + + species_cols <- names(d)[span] + harmonized <- normalize_mammal_column( + species_cols, native_sp, "mammal province" + ) + + if (anyDuplicated(harmonized) > 0) { + stop( + "Normalizing mammal province names merged distinct ", + "columns: ", + paste( + unique(harmonized[duplicated(harmonized)]), + collapse = ", " + ), + call. = FALSE + ) + } + + # Step 2: Units. Four deployments appear twice; they get + # numbered ids, and the climate stage pools them as v2 does. + unit_id <- make_unique_ids( + paste0("mammal_province|", d$location_project), + "mammal province" + ) + + values <- d[, species_cols, drop = FALSE] + names(values) <- harmonized + + species <- data.frame( + survey_unit_id = unit_id, + values, + check.names = FALSE, + stringsAsFactors = FALSE + ) + + sites <- data.frame( + survey_unit_id = unit_id, + survey_design = "mammal_camera", + region = "province", + site = NA_character_, + site_year = NA_character_, + year = NA_integer_, + quadrant = NA_character_, + on_off_grid = NA_character_, + location = as.character(d$location), + project = as.character(d$project), + nearest_site = as.character(d$NearestSite), + lured = d$Lured, + summer_days = as.numeric(d$SummerDays), + winter_days = as.numeric(d$WinterDays), + lat = as.numeric(d$Lat), + long = as.numeric(d$Long), + elevation = NA_real_, + easting = NA_real_, + northing = NA_real_, + nr = as.character(d$NR), + nsr = as.character(d$NSR), + luf = as.character(d$LUF), + stringsAsFactors = FALSE + ) + + # Step 3: Climate, joined on location as v2's climate stage + # does (01_prepare-data.R:81-106); the SpTable's own climate + # columns are not used + camera_climate <- read_camera_climate( + mammal_camera_climate_file, mammal_camera_climate_cols + ) + + if (is.null(camera_climate)) { + stop( + "Camera climate not found at ", mammal_camera_climate_file, + "; the mammal climate rows cannot be built without it.", + call. = FALSE + ) + } + + matched <- match(as.character(d$location), camera_climate$location) + + climate <- data.frame( + survey_unit_id = unit_id, + location_project = as.character(d$location_project), + camera_climate[matched, mammal_camera_climate_cols, drop = FALSE], + check.names = FALSE, + stringsAsFactors = FALSE + ) + rownames(climate) <- NULL + + # Step 4: The habitat columns the region filters name + veg <- data.frame( + survey_unit_id = unit_id, + Water = as.numeric(d$Water), + WetlandMargin = as.numeric(d$WetlandMargin), + stringsAsFactors = FALSE + ) + + message( + " mammal province: ", nrow(species), " units x ", + length(species_cols), " species-season columns; camera ", + "climate matched ", sum(!is.na(matched)), " (", + round(100 * mean(!is.na(matched)), 1), "%)" + ) + + return(list( + species = species, + sites = sites, + climate = climate, + veg = veg + )) +} + ## 2.10 read_bird_data() ---- #' Read the Bird Data Package @@ -1425,6 +1596,21 @@ mammal_parts <- lapply( ) names(mammal_parts) <- names(mammal_files) +# Kept apart from mammal_parts: no species queue, prediction +# matrix or habitat block, so the per-region steps skip it +mammal_province <- read_mammal_province( + mammal_climate_data_file, native_sp +) + +if (is.null(mammal_province)) { + warning( + "Mammal climate table not found at ", mammal_climate_data_file, + "; no mammal_province rows are written, so the fitted mammal ", + "climate stage cannot run. Set SDM_MAMMAL_CLIMATE_DATA.", + call. = FALSE + ) +} + ## 3.4 Read the bird data package ---- message("Reading bird data ...") @@ -1481,6 +1667,29 @@ mammal_species <- rbind( align_columns(mammal_parts$south$species, mammal_species_cols) ) +## 4.3 Append the climate stage's rows ---- +# Last, so the regional rows keep their order. Species the +# regions never record are dropped: nothing could model them. +if (!is.null(mammal_province)) { + province_only <- setdiff( + names(mammal_province$species), + c("survey_unit_id", mammal_species_cols) + ) + + if (length(province_only) > 0) { + message( + " mammal province: ", length(province_only), + " species-season column(s) not in the regional files, ", + "dropped: ", paste(province_only, collapse = ", ") + ) + } + + mammal_species <- rbind( + mammal_species, + align_columns(mammal_province$species, mammal_species_cols) + ) +} + message( " mammal combined: ", nrow(mammal_species), @@ -1498,7 +1707,9 @@ all_sites <- do.call( c( lapply(plant_parts, function(x) x$sites), lapply(mammal_parts, function(x) x$sites), - list(bird = bird_parts$sites) + list(bird = bird_parts$sites), + # Last, so existing rows keep their order (section 4.3) + list(mammal_province = mammal_province$sites) ) ) rownames(all_sites) <- NULL @@ -1645,7 +1856,16 @@ covariate_blocks <- c( soil = bird_parts$soil, design = bird_parts$design ) - ) + ), + # Last, so existing rows keep their order (section 4.3) + if (!is.null(mammal_province)) { + list( + mammal_province = list( + climate = mammal_province$climate, + veg = mammal_province$veg + ) + ) + } ) merged_covariates <- lapply( @@ -1736,7 +1956,7 @@ covariate_taxa <- list( bryophyte = "bryophyte", lichen = "lichen", mite = "mite", - mammal = c("mammal_north", "mammal_south"), + mammal = c("mammal_north", "mammal_south", "mammal_province"), bird = "bird" ) diff --git a/1_code/_setup/02_validate_test_dataset.R b/1_code/_setup/02_validate_test_dataset.R index a932d23..98c76d4 100644 --- a/1_code/_setup/02_validate_test_dataset.R +++ b/1_code/_setup/02_validate_test_dataset.R @@ -1088,6 +1088,148 @@ if (file.exists(climate_pred_path) && ) } +## 3.6.1b Mammal climate-stage rows ---- +# The mammal_province rows: v2's combined 2022 table, which the +# v2 climate stage fits on. Traced back to it and to the camera +# climate, then checked to reproduce v2's training set: the +# deployments with a pooled presence must be the ones v2's +# prediction file covers. +mammal_climate_data_file <- Sys.getenv( + "SDM_MAMMAL_CLIMATE_DATA", + unset = setup_input( + "abmi_mammals", + "Results/Density", + paste0( + "R Dataset SpTable for ABMI North and South mammal ", + "coefficients 2022.RData" + ) + ) +) +mammal_camera_climate_file <- Sys.getenv( + "SDM_MAMMAL_CAMERA_CLIMATE", + unset = setup_input( + "ab_data_v2023", "abmi-camera-climate_2023.Rdata" + ) +) + +province_rows <- which( + startsWith(mam_csv$survey_unit_id, "mammal_province|") +) + +if (length(province_rows) > 0 && + file.exists(mammal_climate_data_file) && + file.exists(mammal_camera_climate_file)) { + label <- "mammal province" + env <- new.env() + load(mammal_climate_data_file, envir = env) + d <- as.data.frame(env$d, check.names = FALSE) + + # Response: keys and every species column the CSV kept + keys <- sub("__[0-9]+$", "", mam_csv$survey_unit_id[province_rows]) + note( + label, "keys match source, in order", + identical(keys, paste0("mammal_province|", d$location_project)), + paste(length(province_rows), "rows") + ) + + src_cols <- names(d)[env$FirstSpCol.s:env$LastSpCol.w] + out_cols <- harmonized_name(src_cols) + kept <- out_cols %in% names(mam_csv) + cmp <- compare_blocks( + mam_csv[province_rows, out_cols[kept], with = FALSE], + d[, src_cols[kept]] + ) + note( + label, "species values match source", cmp$n == 0, + sprintf("%d of %d columns kept; %d cells differ", + sum(kept), length(kept), cmp$n) + ) + + s <- sites[match(mam_csv$survey_unit_id[province_rows], + survey_unit_id)] + note( + label, "site fields match source", + isTRUE(all.equal(s$lat, as.numeric(d$Lat))) && + isTRUE(all.equal(s$long, as.numeric(d$Long))) && + identical( + as.numeric(s$summer_days), as.numeric(d$SummerDays) + ) && + identical( + as.numeric(s$winter_days), as.numeric(d$WinterDays) + ) && + identical(s$lured, as.character(d$Lured)) + ) + + # Climate: rebuilt here with v2's join (01_prepare-data.R) + cenv <- new.env() + load(mammal_camera_climate_file, envir = cenv) + cc <- cenv$camera.climate + cc_location <- sub( + "^CMU-|^NWSAR-", "", sub("_[0-9]{4}$", "", rownames(cc)) + ) + first <- !duplicated(cc_location) + matched <- match(d$location, cc_location[first]) + climate_cols <- c("FFP", "MAP", "TD", "CMD", "EMT") + + cov_rows <- covariates[taxon == "mammal_province"][ + match(mam_csv$survey_unit_id[province_rows], survey_unit_id) + ] + cmp <- compare_blocks( + cov_rows[, ..climate_cols], + cc[first, climate_cols][matched, ] + ) + note( + label, "climate matches the camera file (v2's join)", + cmp$n == 0, + sprintf("%d of %d deployments matched", sum(!is.na(matched)), + nrow(d)) + ) + + # v2's training set: per deployment, base weighted.mean over + # the two seasons (zero-day seasons drop out), and FFP present + source_pred <- as.data.frame(fread(mammal_climate_pred_file)) + pred_species <- setdiff(names(source_pred), "location_project") + pred_cols <- harmonized_name(paste0(pred_species, "Summer")) + pred_base <- sub("_Summer$", "", pred_cols) + checkable <- paste0(pred_base, "_Winter") %in% names(mam_csv) & + pred_cols %in% names(mam_csv) + + agree <- vapply(which(checkable), function(i) { + summer <- mam_csv[[pred_cols[i]]][province_rows] + winter_col <- paste0(pred_base[i], "_Winter") + winter <- mam_csv[[winter_col]][province_rows] + pooled <- mapply( + function(a, b, x, y) weighted.mean(c(a, b), c(x, y)), + summer, winter, s$summer_days, s$winter_days + ) + usable <- !is.na(pooled) & !is.na(cov_rows$FFP) + setequal( + keys[usable], + paste0("mammal_province|", source_pred$location_project[ + !is.na(source_pred[[pred_species[i]]]) + ]) + ) + }, logical(1)) + + note( + label, "pooled presence reproduces v2's training set", + all(agree), + sprintf( + "%d of %d species agree; not checkable: %s", + sum(agree), length(agree), + paste(pred_species[!checkable], collapse = ", ") + ) + ) +} else { + note( + "mammal province", "climate-stage rows available", FALSE, + paste( + "no mammal_province rows, or source not found;", + "set SDM_MAMMAL_CLIMATE_DATA" + ) + ) +} + ## 3.6.2 Bird response, offsets and covariates ---- # Birds come from one file rather than the snapshot. Its three # frames are stored row-aligned on surveyid, so the key is @@ -1317,8 +1459,10 @@ if (all(present)) { named <- c( file.path(data_dir, manifest$response_file), file.path(data_dir, stats::na.omit(manifest$offset_file)), - file.path(lookup_dir, - paste0(manifest$grid, "_prediction_matrix.csv")) + # The mammal province row has no grid + file.path(lookup_dir, paste0( + stats::na.omit(manifest$grid), "_prediction_matrix.csv" + )) ) note( diff --git a/1_code/_setup/07_harmonize_lookups.R b/1_code/_setup/07_harmonize_lookups.R index 793540a..17feefc 100644 --- a/1_code/_setup/07_harmonize_lookups.R +++ b/1_code/_setup/07_harmonize_lookups.R @@ -343,13 +343,33 @@ manifest <- rbind( ) ) +# The mammal climate stage's province-wide rows (_setup/01 +# section 4.3), when 01 wrote them. Last, so existing rows keep +# their order. No grid: the climate stage predicts no habitat. +column_map <- fread(file.path(lookup_dir, "covariate_columns.csv")) + +if ("mammal_province" %in% column_map$taxon) { + manifest <- rbind(manifest, data.frame( + taxon = "mammal", + region = "province", + response_file = "mammal.csv", + offset_file = NA_character_, + covariate_key = "mammal_province", + grid = NA_character_, + bootstrap_ids = NA_character_, + bootstrap_layout = NA_character_, + stringsAsFactors = FALSE + )) +} + ## 5.1 Check every named file exists ---- dataset_dir <- dirname(lookup_dir) named <- c( file.path(dataset_dir, manifest$response_file), file.path(dataset_dir, stats::na.omit(manifest$offset_file)), - file.path(lookup_dir, - paste0(manifest$grid, "_prediction_matrix.csv")) + file.path(lookup_dir, paste0( + stats::na.omit(manifest$grid), "_prediction_matrix.csv" + )) ) absent <- unique(named[!file.exists(named)]) diff --git a/1_code/experiments/README.md b/1_code/experiments/README.md index 03dae98..9e74136 100644 --- a/1_code/experiments/README.md +++ b/1_code/experiments/README.md @@ -1,15 +1,17 @@ # experiments One folder per methods question, keyed `exp_NNN_short_description`. -Each folder has a `run.R` entry point that calls `experiment_config()` -and `run_experiment()` (see the [harness README](../harness/README.md)), -and a `README.md` stating the question. The same id names the -experiment's folders in `2_pipeline/` and `3_output/`. +The same id names the experiment's folders in `2_pipeline/` and +`3_output/`. + +**An experiment edits files in its own folder only.** +`1_code/harness/`, `methods/` and `modules/` are the shared back-end; +everything an experiment changes is set in its folder (see +[What an experiment can change](#what-an-experiment-can-change)). | Folder | Holds | Status | | --- | --- | --- | -| `_shared/` | `focal_species.R`: `focal_species_sets()`, the named species sets (`full`, `parity_check`, `plants_only`, `one_each`) | | -| `_template/` | The `run.R` and `README.md` that `new_experiment()` fills in | | +| `_template/` | The `run.R`, `README.md`, `species.csv` and `hooks/` that `new_experiment()` copies | | | [`exp_000_parity_v2/`](exp_000_parity_v2/README.md) | The v2 parity check and the baseline for every other experiment | Not yet a gate | | [`exp_001_xgboost/`](exp_001_xgboost/README.md) | Habitat stage fitted with xgboost, every taxon | Test; 5-draw check only | | [`exp_002_soilgrids/`](exp_002_soilgrids/README.md) | SoilGrids 0–5 cm terms in the climate stage, plants and birds | Test; 5-draw check only | @@ -17,6 +19,16 @@ experiment's folders in `2_pipeline/` and `3_output/`. exp_000 is the required gate before iterative R&D: every other experiment is compared against its tables. +## An experiment's folder + +| File | Holds | +| --- | --- | +| `run.R` | The entry point: loads the framework and the hooks, makes any spec changes, calls `experiment_config()` and `run_experiment()` (see the [harness README](../harness/README.md)) | +| `README.md` | The question, design, result and caveats | +| `species.csv` | The species fitted: `taxon,species,note`, one row each. The template's lists every species a run can fit (1,117); delete the rows not fitted. The rows left also set the runs, so `run.R` needs no `taxa` | +| `hooks/` | Optional. The experiment's own methods (an engine, selection rule, resampler or metric) or functions a spec field names, loaded by `load_framework(hooks = )` | +| `NN_*.R`, `utils/` | Optional. Step scripts, scripts that write inputs to `2_pipeline//inputs/`, and helpers | + ## Adding an experiment 1. Scaffold it from the repository root: @@ -30,17 +42,43 @@ experiment is compared against its tables. `2_pipeline/` and `3_output/`, copies `_template/` with the id, title, author and date filled in, and creates `2_pipeline//logs/` and `3_output//` with `figures/`, - `tables/` and a stub `report.md`. Leave `id` in `run.R` as it is. + `tables/` and a stub `report.md`. Leave `id` and `exp_dir` in + `run.R` as they are. 2. State the question and design in its `README.md`. -3. Change one thing relative to exp_000: `stage_models`, - `covariate_files` or `specs`. Keep exp_000's species, draws and seed - unless they are the question. -4. Never add columns to `0_data/` or change `_setup/` for an - experiment. Covariates the dataset lacks are written by a script in - the experiment's folder to `2_pipeline//inputs/` and named in - `covariate_files`; `exp_002_soilgrids` is the example. -5. Run it at 5 draws to check it works, then at 100. Compare it with - an exp_000 run at the same number of draws. +3. Cut `species.csv` down to the species to fit. For a + like-for-like comparison with exp_000, keep its species + (`exp_000_parity_v2/species.csv`), draws and seed unless they + are the question. +4. Change one thing relative to exp_000, in `run.R` or `hooks/` + (below). +5. Run it at 5 draws to check it works, then at 100. Compare it + with an exp_000 run at the same number of draws. + +## What an experiment can change + +All from its own folder: + +| To change | Use | +| --- | --- | +| Species, and so the runs | Rows of `species.csv` | +| Regions fitted | `experiment_config(regions = )` | +| Region definitions, grid constants | `modify_region()` on a spec | +| Candidate formulas, added covariates | `stage_models` with `extend_models()`, `models_from_covariates()` or formulas | +| Removed covariates | `stage_models` with `drop_candidates()` or `strip_terms()` | +| Covariates the dataset does not hold | A script that writes them to `2_pipeline//inputs/`, named in `covariate_files` | +| Engine, selection rule, engine settings | `replace_stage_method()`, or `modify_stage()` | +| Any other stage setting (`ic`, `min_detections`, `scope`, weighting, carrying, `grid_constants`) | `modify_stage()` | +| Resampling scheme and its settings, including spatial block cut points (`long_breaks`, `lat_breaks`) | `modify_resample()` | +| Metrics scored, and those compared with exp_000 | `experiment_config(metrics = , compare_metrics = )` | +| A new engine, rule, resampler or metric | A file in `hooks/` that registers it | +| Per-unit predictions | `experiment_config(unit_predictions = "oob")` | +| Analysis beyond the standard tables | A step script, `steps = list(x = script_step(...))` | + +`modify_stage()`, `modify_region()` and `modify_resample()` validate +the spec and record the change, so `run_record.md` says what differs +from v2. Never add columns to `0_data/` or change `_setup/` for an +experiment; `exp_002_soilgrids` is the example of bringing new +covariates. Each lever is walked through in [`docs/getting_started.md`](../../docs/getting_started.md#test-a-change). diff --git a/1_code/experiments/_shared/focal_species.R b/1_code/experiments/_shared/focal_species.R deleted file mode 100644 index 9f999d5..0000000 --- a/1_code/experiments/_shared/focal_species.R +++ /dev/null @@ -1,154 +0,0 @@ -# --- -# title: Focal Species Sets, Shared by Every Experiment -# author: Brendan Casey -# created: 2026-09-11 -# inputs: none -# outputs: none; returns objects in memory -# notes: -# - Named species sets shared by experiments, so a later -# experiment can run exactly exp_000's species. Sets follow -# taxon_values() (names are data slugs, e.g. "mite"); NULL -# means every species. -# - A taxon a set does not name is unrestricted, not excluded; -# narrow `taxa` alongside a partial set. -# - Available species: list_species(data_dir, "mite", "north"). -# --- - -# 1. Setup ---- - -## 1.1 Load packages ---- -# Base R only; the harness supplies everything else. - -# 2. focal_species_sets() ---- - -#' Every Named Focal-Species Set -#' -#' @return A named list of species vectors, each in the harness's -#' named-vector convention. -#' -#' @example # Example usage of the function -#' # names(focal_species_sets()) -#' # focal_species_sets()$parity_check -focal_species_sets <- function() { - list( - - ## 2.1 full ---- - # The full parity run - full = NULL, - - ## 2.2 parity_check ---- - # Two per taxon, chosen so the parity test discriminates: rare - # species have bands wide enough to pass anything. Criteria: - # well above 20 detections in both regions; prevalence ~5-45%; - # all 100 v2 draws usable; and one of each pair balanced - # between north and south. - # - # Detections north / south, and prevalence north / south: - # Ceratodon.purpureus 2140 / 554 44.1 / 24.0 - # Bryum.All 1941 / 856 40.0 / 37.1 - # Physcia.adscendens 2232 / 506 38.4 / 21.6 - # Cladonia.chlorophaea 2236 / 218 38.5 / 9.3 - # Ceratozetes.gracilis 1190 / 133 20.8 / 5.6 - # Trhypochthonius.tectorum 546 / 197 9.6 / 8.3 - # Galium.boreale 2875 / 588 41.8 / 22.2 - # Vicia.americana 2596 / 700 37.8 / 26.4 - # Moose_Summer 1585 / 225 31.2 / 17.8 - # Coyote_Summer 1362 / 893 26.8 / 70.8 - # AMRO 41853 /23613 24.4 / 25.9 - # YEWA 22337 /15887 13.0 / 17.5 - parity_check = c( - # Bryum.All is a genus-level aggregate rather than a - # single species, and is here because it is the only - # bryophyte with a strong south signal as well as a - # strong north one. Swap in Pylaisia.polyantha for two - # true species, at the cost of the southern test. - bryophyte = "Ceratodon.purpureus", - bryophyte = "Bryum.All", - - # The two best-detected lichens, so the tightest bands of - # any plant-group taxon. - lichen = "Physcia.adscendens", - lichen = "Cladonia.chlorophaea", - - # Ceratozetes is the strongest mite in the north; - # Trhypochthonius is the most even across the regions. - mite = "Ceratozetes.gracilis", - mite = "Trhypochthonius.tectorum", - - # Both common in both regions, which makes them the most - # discriminating pair in the set. - vascular_plant = "Galium.boreale", - vascular_plant = "Vicia.americana", - - # Both have a published climate reference. Coyote carries - # the south, where moose is thinner. Both seasons, because - # v2's mammal references average summer and winter; each - # season's spec takes its own and skips the other. - mammal = "Moose_Summer", - mammal = "Moose_Winter", - mammal = "Coyote_Summer", - mammal = "Coyote_Winter", - - # Gated on climate, and on landcover once translated to - # v2's standardized terms. - bird = "AMRO", - bird = "YEWA" - ), - - ## 2.3 plants_only ---- - # Pair with a narrowed `taxa`, or mammals and birds run every - # species (experiment_config() warns) - plants_only = c( - bryophyte = "Ceratodon.purpureus", - bryophyte = "Bryum.All", - lichen = "Physcia.adscendens", - lichen = "Cladonia.chlorophaea", - mite = "Ceratozetes.gracilis", - mite = "Trhypochthonius.tectorum", - vascular_plant = "Galium.boreale", - vascular_plant = "Vicia.americana" - ), - - ## 2.4 one_each ---- - # Shortest run touching every module, for plumbing checks - one_each = c( - bryophyte = "Ceratodon.purpureus", - lichen = "Physcia.adscendens", - mite = "Ceratozetes.gracilis", - vascular_plant = "Galium.boreale", - mammal = "Moose_Summer", - mammal = "Moose_Winter", - bird = "AMRO" - ) - ) -} - -# 3. get_focal_species() ---- - -#' Look Up One Focal-Species Set by Name -#' -#' An unrecognized value (e.g. an inline species vector) is -#' returned unchanged. -#' -#' @param name Character. A name from focal_species_sets(), or -#' a species vector to pass through, or NULL for all species. -#' @param sets Named list. The registry to look in. -#' @return A named character vector, or NULL. -#' -#' @example # Example usage of the function -#' # get_focal_species("one_each") -#' # get_focal_species(c(mite = "Oppiella.nova")) -get_focal_species <- function(name, - sets = focal_species_sets()) { - if (is.null(name) || length(name) == 0) { - return(NULL) - } - - if (length(name) > 1 || !name %in% names(sets)) { - return(name) - } - - sets[[name]] -} - -# End of script ---- diff --git a/1_code/experiments/_template/README.md b/1_code/experiments/_template/README.md index 5ab9715..4a36d1a 100644 --- a/1_code/experiments/_template/README.md +++ b/1_code/experiments/_template/README.md @@ -11,10 +11,25 @@ One sentence: what does this experiment ask? - **What changes relative to exp_000:** the one thing this experiment varies (a covariate, a model set, an engine, a - selection rule). + selection rule, a resampling scheme). - **What stays the same:** taxa, species, draws and seed, unless they are the question. -- **How it is set:** `stage_models` or `specs` in `run.R`. +- **How it is set:** `stage_models` (with `extend_models()`, + `drop_candidates()` or `strip_terms()`), `covariate_files`, + `regions`, `metrics`, or a spec change (`replace_stage_method()`, + `modify_stage()`, `modify_region()`, `modify_resample()`) in + `run.R`; a method of this experiment's own in `hooks/`. + +## Files + +Everything this experiment changes is in this folder; nothing +outside it is edited. + +| File | Holds | +| --- | --- | +| `run.R` | The configuration and the run | +| `species.csv` | `taxon,species,note`. Starts as every species a run can fit; delete the rows not fitted. The rows left also set the runs. For a like-for-like comparison keep exp_000's species (`exp_000_parity_v2/species.csv`) | +| `hooks/` | Optional. Methods or functions of this experiment's own, loaded by `load_framework(hooks = )` | ## How to read the result diff --git a/1_code/experiments/_template/hooks/.gitkeep b/1_code/experiments/_template/hooks/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/1_code/experiments/_template/run.R b/1_code/experiments/_template/run.R index 93b2129..ab0f7df 100644 --- a/1_code/experiments/_template/run.R +++ b/1_code/experiments/_template/run.R @@ -5,9 +5,12 @@ # inputs: # - the published test dataset (harness/data_source.R) # - 3_output/exp_000_parity_v2/tables/, the v2 baseline +# - species.csv in this folder (columns taxon, species, note) +# - hooks/*.R in this folder, if any # outputs: # in 2_pipeline/exp_NNN_short_description/: -# - // result stores, and run_log.csv +# - // result stores, run_log.csv and +# session_info.txt # in 3_output/exp_NNN_short_description/: # - tables/ (summaries, and comparison_metrics.csv against the # baseline), run_record.md @@ -15,55 +18,93 @@ # - Start an experiment with new_experiment("description") # (harness/new_experiment.R), which numbers it, fills this # header and creates its 2_pipeline/ and 3_output/ folders. -# Copying by hand instead, set `id` below to the folder's name. -# - State the question in the README. -# - Change one thing relative to exp_000, so a difference in the -# comparison is a difference in that thing. Use the same -# species, draws and seed as exp_000 unless they are the -# question. -# - The usual levers, from least to most invasive: +# Copying by hand instead, set `id` and `exp_dir` below to the +# folder's name. +# - Edit files in this folder only. harness/, methods/ and +# modules/ are the shared back-end; everything an experiment +# changes is set here. +# - State the question in the README. Change one thing relative +# to exp_000, so a difference in the comparison is a +# difference in that thing. species.csv lists every species +# in the dataset: before running, delete the rows this +# experiment does not fit, and say why in `note` if useful. +# The rows left set both the species and the runs; nothing +# else needs editing. For a like-for-like comparison keep +# exp_000's species (exp_000_parity_v2/species.csv), draws +# and seed, unless they are the question. +# - The levers, from least to most invasive: # - `stage_models`: different candidate formulas for a stage -# (new covariates, a different model set). +# (new covariates, a different model set). extend_models() +# adds terms to a set; drop_candidates() and strip_terms() +# remove them (all in harness/model_sets.R). # - `covariate_files`: covariates the dataset does not hold, -# from a CSV this experiment's own script writes to +# from a CSV a script in this folder writes to # 2_pipeline//inputs/. Never add them to 0_data/ or # _setup/, which stay the v2 data. -# - `specs`: a spec with a stage changed - a different engine -# or selection rule. For a new habitat method in every -# taxon, mammals included, one call: +# - A spec change (section 1.2), made here, never in modules/: +# replace_stage_method(spec, engine = "xgboost") +# modify_stage(spec, "habitat", min_detections = 10L) +# modify_region(spec, "north", filter = ~ nr == "Boreal") +# modify_resample(spec, scheme = "spatial_cv", +# replace = TRUE) +# Over every run at once: # specs <- lapply(standard_specs(), replace_stage_method, # engine = "xgboost") -# See exp_001_xgboost and docs/getting_started.md. -# - A new method in 1_code/methods/, then named in a spec; -# see 1_code/methods/README.md. +# - `regions`, `metrics`, `compare_metrics`: fit one region, +# score or compare on chosen metrics. +# - A method of your own (engine, selection rule, resampler, +# metric) in hooks/, registered with register_engine(), +# register_selection(), register_resampler() or +# register_metric(); see 1_code/methods/_templates/ and +# 1_code/methods/README.md for each contract. Name it, then +# use it as a built-in one. A hook may not reuse a built-in +# name. +# - experiment_config() checks everything (species, regions, +# metrics, specs, covariates) and lists every problem before +# anything is fitted. # --- # 1. Setup ---- -## 1.1 Load the framework ---- +## 1.1 Load the framework and this experiment's hooks ---- +exp_dir <- "1_code/experiments/exp_NNN_short_description" + source("1_code/harness/harness.R") -load_framework() +load_framework(hooks = file.path(exp_dir, "hooks")) + +## 1.2 Spec changes ---- +# The v2 specs, changed where this experiment tests something. +# For example, every climate candidate with AHM added is a +# `stage_models` change (section 1.3); a habitat stage fitted with +# a different engine is a spec change here. +specs <- standard_specs() -## 1.2 Configure the run ---- +## 1.3 Configure the run ---- config <- experiment_config( id = "exp_NNN_short_description", - # The runs to fit, from names(standard_specs()) - taxa = c( - "bryophyte", "lichen", "mite", "vascular_plant", - "mammal_summer", "mammal_winter", "bird" - ), + specs = specs, - # The same species exp_000 ran, so the comparison is like for - # like - species = "parity_check", + # This folder's species.csv, cut down to the species to fit. It + # also sets the runs: only those that fit a listed species + # (a Moose_Summer row runs mammal_summer), so there is no + # `taxa` to keep in step with it. + species = file.path(exp_dir, "species.csv"), n_bootstraps = 100, seed = 20260909, # The change this experiment tests. For example, every climate - # candidate with elevation added: + # candidate with AHM added (extend_models()), or without + # the spatial terms (drop_candidates()): + # stage_models = list( + # climate = extend_models("climate_plant_v2_full", "AHM") + # ), # stage_models = list( - # climate = extend_models("climate_plant_v2_full", "elevation") + # climate = drop_candidates( + # "climate_plant_v2_full", + # c("Easting", "Northing", "Easting2", "Northing2", + # "EastingNorthing") + # ) # ), stage_models = NULL, diff --git a/1_code/experiments/_template/species.csv b/1_code/experiments/_template/species.csv new file mode 100644 index 0000000..41a8b0d --- /dev/null +++ b/1_code/experiments/_template/species.csv @@ -0,0 +1,1118 @@ +taxon,species,note +vascular_plant,Abies.balsamea, +vascular_plant,Abies.bifolia, +vascular_plant,Achillea.alpina, +vascular_plant,Achillea.borealis, +vascular_plant,Actaea.rubra, +vascular_plant,Adoxa.moschatellina, +vascular_plant,Agoseris.glauca, +vascular_plant,Agrimonia.striata, +vascular_plant,Agropyron.fragile.cristatum, +vascular_plant,Agrostis.scabra, +vascular_plant,Allium.textile, +vascular_plant,Alnus.alnobetula, +vascular_plant,Alnus.incana, +vascular_plant,Alopecurus.aequalis, +vascular_plant,Amaranthus.retroflexus, +vascular_plant,Amelanchier.alnifolia, +vascular_plant,Andromeda.polifolia, +vascular_plant,Androsace.occidentalis, +vascular_plant,Androsace.septentrionalis, +vascular_plant,Anemonastrum.canadense, +vascular_plant,Anemone.cylindrica, +vascular_plant,Anemone.multifida, +vascular_plant,Anemone.parviflora, +vascular_plant,Antennaria, +vascular_plant,Antennaria.microphylla.parvifolia, +vascular_plant,Antennaria.neglecta, +vascular_plant,Anticlea.elegans, +vascular_plant,Aphyllon.fasciculatum, +vascular_plant,Apocynum.androsaemifolium, +vascular_plant,Aquilegia.brevistyla, +vascular_plant,Aralia.nudicaulis, +vascular_plant,Arctostaphylos.uva.ursi, +vascular_plant,Arctous.rubra, +vascular_plant,Arnica.chamissonis, +vascular_plant,Arnica.cordifolia, +vascular_plant,Arnica.fulgens, +vascular_plant,Artemisia.absinthium, +vascular_plant,Artemisia.biennis, +vascular_plant,Artemisia.campestris, +vascular_plant,Artemisia.cana, +vascular_plant,Artemisia.frigida, +vascular_plant,Artemisia.ludoviciana, +vascular_plant,Astragalus, +vascular_plant,Astragalus.agrestis, +vascular_plant,Astragalus.alpinus, +vascular_plant,Astragalus.americanus.canadensis, +vascular_plant,Astragalus.crassicarpus, +vascular_plant,Astragalus.flexuosus, +vascular_plant,Astragalus.laxmannii, +vascular_plant,Astragalus.pectinatus, +vascular_plant,Athyrium.filix.femina, +vascular_plant,Atriplex.gardneri.canescens, +vascular_plant,Avena.fatua, +vascular_plant,Axyris.amaranthoides, +vascular_plant,Bassia.scoparia, +vascular_plant,Beckmannia.syzigachne, +vascular_plant,Betula.neoalaskana.papyrifera, +vascular_plant,Betula.occidentalis, +vascular_plant,Betula.x.sargentii, +vascular_plant,Bistorta.vivipara, +vascular_plant,Blitum.nuttallianum, +vascular_plant,Boechera.grahamii.collinsii.retrofracta.pedunculocarpa, +vascular_plant,Botrypus.virginianus, +vascular_plant,Bouteloua.gracilis, +vascular_plant,Brassica.napus, +vascular_plant,Brassica.rapa, +vascular_plant,Bromus.ciliatus, +vascular_plant,Bromus.inermis, +vascular_plant,Bromus.pumpellianus, +vascular_plant,Bromus.riparius, +vascular_plant,Bromus.tectorum, +vascular_plant,Calamagrostis, +vascular_plant,Calamagrostis.canadensis, +vascular_plant,Calamagrostis.montanensis, +vascular_plant,Calamagrostis.stricta, +vascular_plant,Calla.palustris, +vascular_plant,Caltha.palustris, +vascular_plant,Campanula, +vascular_plant,Capnoides.sempervirens, +vascular_plant,Capsella.bursa.pastoris, +vascular_plant,Cardamine.pensylvanica, +vascular_plant,Carex, +vascular_plant,Carex.aquatilis, +vascular_plant,Carex.atherodes, +vascular_plant,Carex.atratiformis, +vascular_plant,Carex.aurea, +vascular_plant,Carex.bebbii, +vascular_plant,Carex.brunnescens, +vascular_plant,Carex.canescens, +vascular_plant,Carex.capillaris, +vascular_plant,Carex.chordorrhiza, +vascular_plant,Carex.crawfordii, +vascular_plant,Carex.deflexa, +vascular_plant,Carex.deweyana, +vascular_plant,Carex.diandra, +vascular_plant,Carex.disperma, +vascular_plant,Carex.duriuscula, +vascular_plant,Carex.filifolia, +vascular_plant,Carex.foenea, +vascular_plant,Carex.gynocrates, +vascular_plant,Carex.inops, +vascular_plant,Carex.interior, +vascular_plant,Carex.lasiocarpa, +vascular_plant,Carex.leptalea, +vascular_plant,Carex.limosa, +vascular_plant,Carex.magellanica, +vascular_plant,Carex.media, +vascular_plant,Carex.microptera, +vascular_plant,Carex.obtusata, +vascular_plant,Carex.pellita, +vascular_plant,Carex.praegracilis, +vascular_plant,Carex.praticola, +vascular_plant,Carex.rostrata, +vascular_plant,Carex.siccata, +vascular_plant,Carex.tenuiflora, +vascular_plant,Carex.trisperma, +vascular_plant,Carex.utriculata, +vascular_plant,Carex.vaginata, +vascular_plant,Castilleja.miniata, +vascular_plant,Cerastium.arvense, +vascular_plant,Cerastium.fontanum, +vascular_plant,Chamaedaphne.calyculata, +vascular_plant,Chamaenerion.angustifolium, +vascular_plant,Chenopodium, +vascular_plant,Chenopodium.album, +vascular_plant,Chenopodium.other., +vascular_plant,Chrysosplenium.tetrandrum.iowense, +vascular_plant,Cicuta.bulbifera, +vascular_plant,Cicuta.maculata, +vascular_plant,Cinna.latifolia, +vascular_plant,Circaea.alpina, +vascular_plant,Cirsium.arvense, +vascular_plant,Cirsium.flodmanii, +vascular_plant,Cirsium.undulatum, +vascular_plant,Clematis.occidentalis, +vascular_plant,Coeloglossum.viride, +vascular_plant,Collomia.linearis, +vascular_plant,Comandra.umbellata, +vascular_plant,Comarum.palustre, +vascular_plant,Coptidium.lapponicum, +vascular_plant,Coptis.trifolia, +vascular_plant,Corallorhiza.maculata, +vascular_plant,Corallorhiza.trifida, +vascular_plant,Cornus.canadensis, +vascular_plant,Cornus.sericea, +vascular_plant,Corylus.cornuta, +vascular_plant,Crepis.tectorum, +vascular_plant,Dactylis.glomerata, +vascular_plant,Dalea.purpurea, +vascular_plant,Dasiphora.fruticosa, +vascular_plant,Delphinium.glaucum, +vascular_plant,Dendrolycopodium.dendroideum, +vascular_plant,Deschampsia.cespitosa, +vascular_plant,Descurainia.sophia, +vascular_plant,Diphasiastrum.complanatum, +vascular_plant,Distichlis.spicata, +vascular_plant,Dracocephalum.parviflorum, +vascular_plant,Drosera.rotundifolia, +vascular_plant,Drymocallis.arguta, +vascular_plant,Dryopteris.carthusiana.expansa, +vascular_plant,Echinochloa.crus.galli, +vascular_plant,Elaeagnus.commutata, +vascular_plant,Eleocharis.acicularis, +vascular_plant,Eleocharis.palustris.s.l., +vascular_plant,Elymus.albicans.lanceolatus, +vascular_plant,Elymus.repens, +vascular_plant,Elymus.trachycaulus, +vascular_plant,Empetrum.nigrum, +vascular_plant,Endotropis.alnifolia, +vascular_plant,Epilobium.ciliatum, +vascular_plant,Epilobium.leptophyllum, +vascular_plant,Epilobium.palustre, +vascular_plant,Equisetum.arvense, +vascular_plant,Equisetum.fluviatile, +vascular_plant,Equisetum.hyemale, +vascular_plant,Equisetum.pratense, +vascular_plant,Equisetum.scirpoides, +vascular_plant,Equisetum.sylvaticum, +vascular_plant,Erigeron.acris, +vascular_plant,Erigeron.caespitosus, +vascular_plant,Erigeron.canadensis, +vascular_plant,Erigeron.glabellus, +vascular_plant,Erigeron.philadelphicus, +vascular_plant,Erigeron.pumilus, +vascular_plant,Eriophorum.angustifolium, +vascular_plant,Eriophorum.brachyantherum, +vascular_plant,Eriophorum.russeolum, +vascular_plant,Eriophorum.vaginatum, +vascular_plant,Erodium.cicutarium, +vascular_plant,Erysimum.asperum, +vascular_plant,Erysimum.cheiranthoides, +vascular_plant,Erysimum.inconspicuum, +vascular_plant,Escobaria.vivipara, +vascular_plant,Eurybia.conspicua, +vascular_plant,Fagopyrum.tataricum, +vascular_plant,Fallopia.convolvulus, +vascular_plant,Festuca.campestris.hallii, +vascular_plant,Festuca.rubra, +vascular_plant,Festuca.saximontana, +vascular_plant,Fragaria.vesca.virginiana, +vascular_plant,Gaillardia.aristata, +vascular_plant,Galearis.rotundifolia, +vascular_plant,Galeopsis.bifida.tetrahit, +vascular_plant,Galium.aparine.spurium, +vascular_plant,Galium.boreale, +vascular_plant,Galium.labradoricum, +vascular_plant,Galium.trifidum, +vascular_plant,Galium.triflorum, +vascular_plant,Gaultheria.hispidula, +vascular_plant,Gentianella.amarella, +vascular_plant,Geocaulon.lividum, +vascular_plant,Geranium.bicknellii, +vascular_plant,Geranium.richardsonii, +vascular_plant,Geranium.viscosissimum, +vascular_plant,Geum.aleppicum.macrophyllum.rivale, +vascular_plant,Geum.triflorum, +vascular_plant,Glyceria.grandis, +vascular_plant,Glyceria.pulchella, +vascular_plant,Glyceria.striata, +vascular_plant,Glycyrrhiza.lepidota, +vascular_plant,Gnaphalium.palustre, +vascular_plant,Goodyera.repens, +vascular_plant,Grindelia.squarrosa, +vascular_plant,Gutierrezia.sarothrae, +vascular_plant,Gymnocarpium.dryopteris, +vascular_plant,Halenia.deflexa, +vascular_plant,Halerpestes.cymbalaria, +vascular_plant,Hedysarum.americanum, +vascular_plant,Hedysarum.sulphurescens, +vascular_plant,Helictochloa.hookeri, +vascular_plant,Heracleum.maximum, +vascular_plant,Hesperostipa.comata.curtiseta, +vascular_plant,Heterotheca.villosa, +vascular_plant,Heuchera.richardsonii, +vascular_plant,Hieracium.umbellatum, +vascular_plant,Hippuris.vulgaris, +vascular_plant,Hordeum.jubatum, +vascular_plant,Hordeum.vulgare, +vascular_plant,Hymenoxys.richardsonii, +vascular_plant,Impatiens.capensis.noli.tangere, +vascular_plant,Juncus.balticus, +vascular_plant,Juncus.bufonius, +vascular_plant,Juncus.tenuis, +vascular_plant,Juniperus.communis, +vascular_plant,Juniperus.horizontalis, +vascular_plant,Kalmia.polifolia, +vascular_plant,Koeleria.macrantha, +vascular_plant,Krascheninnikovia.lanata, +vascular_plant,Lactuca.serriola, +vascular_plant,Lappula.squarrosa, +vascular_plant,Larix.laricina, +vascular_plant,Lathyrus.ochroleucus, +vascular_plant,Lemna.minor.turionifera, +vascular_plant,Lepidium.densiflorum.ramosissimum, +vascular_plant,Leymus.innovatus, +vascular_plant,Liatris.punctata, +vascular_plant,Lilium.philadelphicum, +vascular_plant,Linaria.vulgaris, +vascular_plant,Linnaea.borealis, +vascular_plant,Linum.compactum, +vascular_plant,Linum.lewisii, +vascular_plant,Lithospermum.incisum, +vascular_plant,Lithospermum.ruderale, +vascular_plant,Lonicera.dioica.involucrata, +vascular_plant,Lonicera.villosa, +vascular_plant,Luzula.parviflora, +vascular_plant,Lycopodium.lagopus, +vascular_plant,Lygodesmia.juncea, +vascular_plant,Lysimachia.ciliata, +vascular_plant,Lysimachia.latifolia, +vascular_plant,Lysimachia.thyrsiflora, +vascular_plant,Maianthemum.amplexicaule, +vascular_plant,Maianthemum.canadense, +vascular_plant,Maianthemum.stellatum, +vascular_plant,Maianthemum.trifolium, +vascular_plant,Malva.rotundifolia.sensu.lato, +vascular_plant,Matricaria.discoidea, +vascular_plant,Medicago.lupulina, +vascular_plant,Medicago.sativa, +vascular_plant,Melampyrum.lineare, +vascular_plant,Melilotus.albus, +vascular_plant,Melilotus.officinalis, +vascular_plant,Mentha.canadensis, +vascular_plant,Menyanthes.trifoliata, +vascular_plant,Mertensia.paniculata, +vascular_plant,Mitella.nuda, +vascular_plant,Moehringia.lateriflora, +vascular_plant,Monarda.fistulosa, +vascular_plant,Moneses.uniflora, +vascular_plant,Muhlenbergia.richardsonis, +vascular_plant,Mulgedium.pulchellum, +vascular_plant,Myrica.gale, +vascular_plant,Nassella.viridula, +vascular_plant,Neottia.cordata, +vascular_plant,Oenothera.suffrutescens, +vascular_plant,Opuntia.fragilis.polyacantha, +vascular_plant,Orthilia.secunda, +vascular_plant,Orthocarpus.luteus, +vascular_plant,Oryzopsis.asperifolia, +vascular_plant,Osmorhiza.depauperata, +vascular_plant,Oxybasis.salina, +vascular_plant,Oxytropis.campestris, +vascular_plant,Oxytropis.deflexa, +vascular_plant,Oxytropis.sericea, +vascular_plant,Packera.cana, +vascular_plant,Packera.paupercula, +vascular_plant,Parnassia.palustris, +vascular_plant,Pascopyrum.smithii, +vascular_plant,Pedicularis.bracteosa, +vascular_plant,Pedicularis.groenlandica, +vascular_plant,Pedicularis.labradorica, +vascular_plant,Penstemon.gracilis, +vascular_plant,Penstemon.procerus, +vascular_plant,Persicaria.amphibia, +vascular_plant,Persicaria.maculosa.lapathifolium, +vascular_plant,Petasites.frigidus, +vascular_plant,Phalaris.arundinacea, +vascular_plant,Phleum.pratense, +vascular_plant,Phlox.hoodii, +vascular_plant,Picea.engelmannii, +vascular_plant,Picea.glauca, +vascular_plant,Picea.mariana, +vascular_plant,Pinus.banksiana, +vascular_plant,Pinus.contorta, +vascular_plant,Piptatheropsis.pungens, +vascular_plant,Pisum.sativum, +vascular_plant,Plantago.major, +vascular_plant,Plantago.patagonica, +vascular_plant,Platanthera, +vascular_plant,Platanthera.aquilonis.huronensis, +vascular_plant,Platanthera.obtusata, +vascular_plant,Platanthera.orbiculata, +vascular_plant,Poa.arida, +vascular_plant,Poa.palustris.interior.nemoralis, +vascular_plant,Poa.pratensis, +vascular_plant,Poa.secunda, +vascular_plant,Polemonium.acutiflorum, +vascular_plant,Polygonum.aviculare, +vascular_plant,Polygonum.erectum, +vascular_plant,Populus.balsamifera, +vascular_plant,Populus.tremuloides, +vascular_plant,Potentilla, +vascular_plant,Potentilla.anserina, +vascular_plant,Potentilla.argentea, +vascular_plant,Potentilla.bipinnatifida, +vascular_plant,Potentilla.concinna, +vascular_plant,Potentilla.gracilis, +vascular_plant,Potentilla.hippiana, +vascular_plant,Potentilla.norvegica, +vascular_plant,Potentilla.pensylvanica, +vascular_plant,Prosartes.trachycarpa, +vascular_plant,Prunus.pensylvanica.virginiana, +vascular_plant,Puccinellia.distans, +vascular_plant,Puccinellia.nuttalliana, +vascular_plant,Pulsatilla.nuttalliana, +vascular_plant,Pyrola, +vascular_plant,Pyrola.asarifolia, +vascular_plant,Pyrola.chlorantha, +vascular_plant,Pyrola.minor, +vascular_plant,Ranunculus.abortivus, +vascular_plant,Ranunculus.acris, +vascular_plant,Ranunculus.gmelinii, +vascular_plant,Ranunculus.macounii, +vascular_plant,Ranunculus.sceleratus, +vascular_plant,Ratibida.columnifera, +vascular_plant,Rhinanthus.minor, +vascular_plant,Rhododendron.groenlandicum, +vascular_plant,Rhododendron.tomentosum, +vascular_plant,Ribes, +vascular_plant,Ribes.americanum, +vascular_plant,Ribes.glandulosum, +vascular_plant,Ribes.hirtellum.inerme.oxyacanthoides, +vascular_plant,Ribes.hudsonianum, +vascular_plant,Ribes.lacustre, +vascular_plant,Ribes.triste, +vascular_plant,Rorippa.palustris, +vascular_plant,Rosa.acicularis.woodsii, +vascular_plant,Rosa.arkansana, +vascular_plant,Rubus.arcticus, +vascular_plant,Rubus.chamaemorus, +vascular_plant,Rubus.idaeus, +vascular_plant,Rubus.pedatus, +vascular_plant,Rubus.pubescens, +vascular_plant,Rumex, +vascular_plant,Rumex.crispus.pseudonatronatus, +vascular_plant,Rumex.occidentalis, +vascular_plant,Rumex.triangulivalvis, +vascular_plant,Salix, +vascular_plant,Salix.arbusculoides, +vascular_plant,Salix.athabascensis, +vascular_plant,Salix.bebbiana, +vascular_plant,Salix.candida, +vascular_plant,Salix.discolor, +vascular_plant,Salix.drummondiana, +vascular_plant,Salix.glauca, +vascular_plant,Salix.lasiandra, +vascular_plant,Salix.maccalliana, +vascular_plant,Salix.myrtillifolia, +vascular_plant,Salix.pedicellaris, +vascular_plant,Salix.petiolaris, +vascular_plant,Salix.planifolia, +vascular_plant,Salix.pseudomonticola, +vascular_plant,Salix.pseudomyrsinites, +vascular_plant,Salix.pyrifolia, +vascular_plant,Salix.scouleriana, +vascular_plant,Salix.serissima, +vascular_plant,Salsola.tragus.collina, +vascular_plant,Sanicula.marilandica, +vascular_plant,Schizachne.purpurascens, +vascular_plant,Schoenoplectus.acutus.tabernaemontani, +vascular_plant,Scirpus.microcarpus, +vascular_plant,Scutellaria.galericulata, +vascular_plant,Selaginella.densa, +vascular_plant,Senecio.eremophilus, +vascular_plant,Senecio.vulgaris, +vascular_plant,Setaria.viridis, +vascular_plant,Shepherdia.canadensis, +vascular_plant,Sibbaldia.tridentata, +vascular_plant,Silene.drummondii, +vascular_plant,Silene.latifolia, +vascular_plant,Sisymbrium.altissimum.loeselii, +vascular_plant,Sisyrinchium.montanum, +vascular_plant,Sium.suave, +vascular_plant,Solanum.triflorum, +vascular_plant,Solidago, +vascular_plant,Solidago.altissima.lepida, +vascular_plant,Solidago.glutinosa, +vascular_plant,Solidago.missouriensis, +vascular_plant,Solidago.multiradiata, +vascular_plant,Solidago.rigida, +vascular_plant,Sonchus.arvensis, +vascular_plant,Sonchus.asper.oleraceus, +vascular_plant,Sorbus.scopulina, +vascular_plant,Sphaeralcea.coccinea, +vascular_plant,Spinulum.annotinum, +vascular_plant,Spiraea.lucida, +vascular_plant,Spiranthes.romanzoffiana, +vascular_plant,Stachys.pilosa, +vascular_plant,Stellaria.calycantha, +vascular_plant,Stellaria.longifolia, +vascular_plant,Stellaria.longipes, +vascular_plant,Stellaria.media, +vascular_plant,Streptopus.amplexifolius, +vascular_plant,Symphoricarpos.albus, +vascular_plant,Symphoricarpos.occidentalis, +vascular_plant,Symphyotrichum, +vascular_plant,Symphyotrichum.boreale, +vascular_plant,Symphyotrichum.ciliolatum, +vascular_plant,Symphyotrichum.ericoides.falcatum, +vascular_plant,Symphyotrichum.laeve, +vascular_plant,Symphyotrichum.lanceolatum, +vascular_plant,Symphyotrichum.puniceum, +vascular_plant,Taraxacum.officinale.erythrospermum, +vascular_plant,Thalictrum.sparsiflorum, +vascular_plant,Thalictrum.venulosum, +vascular_plant,Thermopsis.rhombifolia, +vascular_plant,Thlaspi.arvense, +vascular_plant,Toxicoscordion.venenosum, +vascular_plant,Tragopogon.dubius, +vascular_plant,Trifolium.hybridum, +vascular_plant,Trifolium.pratense, +vascular_plant,Trifolium.repens, +vascular_plant,Triglochin.maritima, +vascular_plant,Tripleurospermum.inodorum, +vascular_plant,Trisetum.spicatum, +vascular_plant,Triticum, +vascular_plant,Typha.latifolia, +vascular_plant,Urtica.gracilis, +vascular_plant,Utricularia.vulgaris, +vascular_plant,Vaccinium.cespitosum, +vascular_plant,Vaccinium.membranaceum, +vascular_plant,Vaccinium.myrtilloides, +vascular_plant,Vaccinium.oxycoccos, +vascular_plant,Vaccinium.uliginosum, +vascular_plant,Vaccinium.vitis.idaea, +vascular_plant,Veronica.americana, +vascular_plant,Veronica.peregrina, +vascular_plant,Viburnum.edule, +vascular_plant,Viburnum.opulus, +vascular_plant,Vicia.americana, +vascular_plant,Viola, +vascular_plant,Viola.adunca, +vascular_plant,Viola.canadensis, +vascular_plant,Viola.renifolia, +vascular_plant,Xanthisma.spinulosum, +vascular_plant,Zizia.aptera, +bryophyte,Abietinella.abietina, +bryophyte,Amblystegium.serpens, +bryophyte,Anastrophyllum.hellerianum, +bryophyte,Aneura.pinguis, +bryophyte,Aulacomnium.palustre, +bryophyte,Barbilophozia.hatcheri, +bryophyte,Barbilophozia.kunzeana, +bryophyte,Barbula.convoluta, +bryophyte,Barbula.unguiculata, +bryophyte,Blepharostoma.trichophyllum, +bryophyte,Brachythecium.All, +bryophyte,Bryoerythrophyllum.recurvirostrum, +bryophyte,Bryum.All, +bryophyte,Calliergon.cordifolium, +bryophyte,Calliergon.giganteum, +bryophyte,Calliergon.richardsonii, +bryophyte,Calypogeia.muelleriana, +bryophyte,Calypogeia.neesiana, +bryophyte,Calypogeia.sphagnicola, +bryophyte,Campyliadelphus.chrysophyllus, +bryophyte,Campylium.stellatum, +bryophyte,Campylophyllum.hispidulum, +bryophyte,Cephalozia.connivens, +bryophyte,Cephalozia.lunulifolia, +bryophyte,Cephalozia.pleniceps, +bryophyte,Cephaloziella.All, +bryophyte,Ceratodon.purpureus, +bryophyte,Chiloscyphus.pallescens.polyanthos, +bryophyte,Climacium.dendroides, +bryophyte,Dicranella.schreberiana, +bryophyte,Dicranum.acutifolium, +bryophyte,Dicranum.elongatum, +bryophyte,Dicranum.flagellare, +bryophyte,Dicranum.fragilifolium, +bryophyte,Dicranum.fuscescens, +bryophyte,Dicranum.groenlandicum, +bryophyte,Dicranum.montanum, +bryophyte,Dicranum.polysetum, +bryophyte,Dicranum.scoparium, +bryophyte,Dicranum.undulatum, +bryophyte,Distichium.capillaceum, +bryophyte,Ditrichum.flexicaule, +bryophyte,Drepanocladus.aduncus, +bryophyte,Drepanocladus.polygamus, +bryophyte,Eurhynchiastrum.pulchellum, +bryophyte,Funaria.hygrometrica, +bryophyte,Geocalyx.graveolens, +bryophyte,Hamatocaulis.vernicosus, +bryophyte,Haplocladium.microphyllum, +bryophyte,Helodium.blandowii, +bryophyte,Herzogiella.turfacea, +bryophyte,Hygroamblystegium.varium, +bryophyte,Hylocomium.splendens, +bryophyte,Hypnum.lindbergii, +bryophyte,Hypnum.pallescens, +bryophyte,Hypnum.pratense, +bryophyte,Hypnum.revolutum, +bryophyte,Isopterygiopsis.pulchella, +bryophyte,Jamesoniella.autumnalis, +bryophyte,Lepidozia.reptans, +bryophyte,Leptobryum.pyriforme, +bryophyte,Leptodictyum.riparium, +bryophyte,Lophocolea.heterophylla, +bryophyte,Lophocolea.minor, +bryophyte,Lophozia.ascendens, +bryophyte,Lophozia.excisa, +bryophyte,Lophozia.heterocolpos, +bryophyte,Lophozia.longidens, +bryophyte,Lophozia.ventricosa, +bryophyte,Marchantia.polymorpha, +bryophyte,Meesia.uliginosa, +bryophyte,Mnium.spinulosum, +bryophyte,Mylia.anomala, +bryophyte,Myurella.julacea, +bryophyte,Oncophorus.wahlenbergii, +bryophyte,Orthotrichum.elegans.speciosum, +bryophyte,Orthotrichum.obtusifolium, +bryophyte,Plagiochila.asplenioides.porelloides, +bryophyte,Plagiomnium.ciliare, +bryophyte,Plagiomnium.cuspidatum, +bryophyte,Plagiomnium.drummondii, +bryophyte,Plagiomnium.ellipticum, +bryophyte,Plagiomnium.medium, +bryophyte,Plagiothecium.denticulatum, +bryophyte,Plagiothecium.laetum, +bryophyte,Platydictya.jungermannioides, +bryophyte,Platygyrium.repens, +bryophyte,Pleurozium.schreberi, +bryophyte,Pohlia.cruda, +bryophyte,Pohlia.nutans, +bryophyte,Pohlia.wahlenbergii, +bryophyte,Polytrichum.commune, +bryophyte,Polytrichum.juniperinum, +bryophyte,Polytrichum.piliferum, +bryophyte,Polytrichum.strictum, +bryophyte,Pseudocampylium.radicale, +bryophyte,Ptilidium.ciliare, +bryophyte,Ptilidium.pulcherrimum, +bryophyte,Ptilium.crista.castrensis, +bryophyte,Ptychostomum.pseudotriquetrum, +bryophyte,Pylaisia.polyantha, +bryophyte,Rhizomnium.gracile, +bryophyte,Rhizomnium.pseudopunctatum, +bryophyte,Rhytidiadelphus.triquetrus, +bryophyte,Riccardia.latifrons, +bryophyte,Riccardia.palmata, +bryophyte,Sanionia.uncinata, +bryophyte,Sarmentypnum.exannulatum, +bryophyte,Scapania.glaucocephala, +bryophyte,Scapania.irrigua, +bryophyte,Scorpidium.revolvens, +bryophyte,Sphagnum.angustifolium, +bryophyte,Sphagnum.capillifolium, +bryophyte,Sphagnum.centrale, +bryophyte,Sphagnum.divinum.medium, +bryophyte,Sphagnum.fallax, +bryophyte,Sphagnum.fuscum, +bryophyte,Sphagnum.girgensohnii, +bryophyte,Sphagnum.riparium, +bryophyte,Sphagnum.russowii, +bryophyte,Sphagnum.squarrosum, +bryophyte,Sphagnum.teres, +bryophyte,Sphagnum.warnstorfii, +bryophyte,Straminergon.stramineum, +bryophyte,Syntrichia.norvegica.ruralis, +bryophyte,Tetraphis.pellucida, +bryophyte,Tetraplodon.angustatus, +bryophyte,Thuidium.recognitum, +bryophyte,Timmia.austriaca, +bryophyte,Tomentypnum.falcifolium, +bryophyte,Tomentypnum.nitens, +bryophyte,Tortula.acaulon, +bryophyte,Tritomaria.exsectiformis, +bryophyte,Warnstorfia.fluitans, +lichen,Alectoria.sarmentosa, +lichen,Allocalicium.adaequatum, +lichen,Blennothallia.crispa, +lichen,Bryoria.fremontii, +lichen,Bryoria.furcellata, +lichen,Bryoria.fuscescens.glabra.lanestris.vrangiana, +lichen,Bryoria.kockiana, +lichen,Bryoria.nadvornikiana, +lichen,Bryoria.pikei, +lichen,Bryoria.pseudofuscescens, +lichen,Bryoria.simplicior, +lichen,Calicium.salicinum, +lichen,Calicium.tigillare, +lichen,Calicium.trabinellum, +lichen,Candelaria.pacifica, +lichen,Cetraria.aculeata.muricata, +lichen,Cetraria.arenaria, +lichen,Cetraria.ericetorum.ssp.reticulata, +lichen,Chaenotheca.chrysocephala, +lichen,Cladonia.acuminata, +lichen,Cladonia.amaurocraea, +lichen,Cladonia.arbuscula.ssp.beringiana, +lichen,Cladonia.arbuscula.ssp.mitis, +lichen,Cladonia.bacilliformis, +lichen,Cladonia.borealis, +lichen,Cladonia.botrytes, +lichen,Cladonia.cariosa, +lichen,Cladonia.carneola, +lichen,Cladonia.cenotea, +lichen,Cladonia.cf.ochrochlora, +lichen,Cladonia.cf.umbricola, +lichen,Cladonia.chlorophaea, +lichen,Cladonia.coccifera, +lichen,Cladonia.coniocraea, +lichen,Cladonia.coniocraea.f.pycnotheliza, +lichen,Cladonia.cornuta.ssp.cornuta, +lichen,Cladonia.crispata.var.crispata, +lichen,Cladonia.cristatella, +lichen,Cladonia.cryptochlorophaea, +lichen,Cladonia.cyanipes, +lichen,Cladonia.deformis, +lichen,Cladonia.ecmocyna, +lichen,Cladonia.fimbriata, +lichen,Cladonia.gracilis.ssp.gracilis.elongata.vulnerata, +lichen,Cladonia.gracilis.ssp.turbinata, +lichen,Cladonia.grayi, +lichen,Cladonia.imbricarica, +lichen,Cladonia.macilenta.var.bacillaris, +lichen,Cladonia.merochlorophaea, +lichen,Cladonia.multiformis, +lichen,Cladonia.norvegica, +lichen,Cladonia.phyllophora, +lichen,Cladonia.pleurota, +lichen,Cladonia.pocillum.magyarica, +lichen,Cladonia.pyxidata.s.l., +lichen,Cladonia.rangiferina, +lichen,Cladonia.rei, +lichen,Cladonia.robbinsii, +lichen,Cladonia.scabriuscula, +lichen,Cladonia.stellaris, +lichen,Cladonia.stygia, +lichen,Cladonia.subulata, +lichen,Cladonia.sulphurina, +lichen,Cladonia.symphycarpa, +lichen,Cladonia.uncialis, +lichen,Cladonia.verticillata, +lichen,Collema.furfuraceum, +lichen,Enchylium.tenax.grp., +lichen,Evernia.mesomorpha, +lichen,Flavocetraria.cucullata, +lichen,Flavocetraria.nivalis, +lichen,Flavopunctelia.flaventior, +lichen,Fulgidea.oligospora, +lichen,Heterodermia.galactophylla, +lichen,Heterodermia.japonica, +lichen,Hypocenomyce.scalaris, +lichen,Hypogymnia.austerodes, +lichen,Hypogymnia.bitteri, +lichen,Hypogymnia.dichroma, +lichen,Hypogymnia.occidentalis, +lichen,Hypogymnia.physodes, +lichen,Hypogymnia.protea, +lichen,Hypogymnia.tubulosa, +lichen,Hypogymnia.wilfiana, +lichen,Icmadophila.ericetorum, +lichen,Imshaugia.aleurites, +lichen,Imshaugia.placorodia, +lichen,Kaernefeltia.merrillii, +lichen,Leptogium.saturninum.grp., +lichen,Letharia.vulpina, +lichen,Lichenomphalia.cf.umbellifera, +lichen,Lobaria.pulmonaria, +lichen,Melanelixia.albertana, +lichen,Melanelixia.subaurifera, +lichen,Melanohalea.cf.exasperata, +lichen,Melanohalea.cf.subelegantula, +lichen,Melanohalea.cf.subolivacea, +lichen,Melanohalea.elegantula, +lichen,Melanohalea.exasperatula, +lichen,Melanohalea.septentrionalis.olivacea, +lichen,Melanohalea.trabeculata, +lichen,Mycocalicium.subtile, +lichen,Nephroma.bellum, +lichen,Nephroma.parile, +lichen,Nephroma.resupinatum, +lichen,Nodobryoria.oregana, +lichen,Parmelia.hygrophila, +lichen,Parmelia.sulcata, +lichen,Parmeliopsis.ambigua, +lichen,Parmeliopsis.hyperopta, +lichen,Peltigera.aphthosa.grp., +lichen,Peltigera.appalachiensis, +lichen,Peltigera.britannica, +lichen,Peltigera.canina.grp., +lichen,Peltigera.castanea, +lichen,Peltigera.collina, +lichen,Peltigera.didactyla.grp., +lichen,Peltigera.elisabethae, +lichen,Peltigera.evansiana, +lichen,Peltigera.extenuata, +lichen,Peltigera.latiloba, +lichen,Peltigera.lepidophora, +lichen,Peltigera.leucophlebia.grp., +lichen,Peltigera.malacea.grp., +lichen,Peltigera.neckeri, +lichen,Peltigera.neopolydactyla.occidentalis, +lichen,Peltigera.ponojensis.monticola.grp, +lichen,Peltigera.praetextata, +lichen,Peltigera.retifoveata, +lichen,Peltigera.rufescens.wulingensis.papuana, +lichen,Peltigera.scabrosa, +lichen,Phaeocalicium.compressulum, +lichen,Phaeocalicium.populneum, +lichen,Phaeophyscia.ciliata, +lichen,Phaeophyscia.constipata, +lichen,Phaeophyscia.kairamoi, +lichen,Phaeophyscia.nigricans, +lichen,Phaeophyscia.orbicularis, +lichen,Physcia.adscendens, +lichen,Physcia.adscendens.m.tenella, +lichen,Physcia.aipolia.alnophila, +lichen,Physcia.caesia, +lichen,Physcia.dubia, +lichen,Physcia.stellaris.grp, +lichen,Physconia.detersa, +lichen,Physconia.enteroxantha, +lichen,Physconia.grumosa, +lichen,Physconia.labrata, +lichen,Physconia.muscigena, +lichen,Physconia.perisidiosa, +lichen,Platismatia.glauca, +lichen,Polycauliona.candelaria, +lichen,Polycauliona.polycarpa, +lichen,Psoroma.hypnorum, +lichen,Punctelia.caseana, +lichen,Ramalina.dilacerata, +lichen,Ramalina.filigrana.ined., +lichen,Ramalina.obtusata, +lichen,Ramalina.pollinaria.grp., +lichen,Ramalina.sinensis, +lichen,Ramalina.thrausta, +lichen,Rhizoplaca.chrysoleuca, +lichen,Rhizoplaca.melanophthalma, +lichen,Scytinium.intermedium, +lichen,Scytinium.subtile.grp., +lichen,Scytinium.tenuissimum, +lichen,Scytinium.teretiusculum, +lichen,Stenocybe.pullatula, +lichen,Stereocaulon.spp., +lichen,Tuckermannopsis.americana, +lichen,Tuckermannopsis.chlorophylla, +lichen,Tuckermannopsis.orbata, +lichen,Tuckermannopsis.platyphylla, +lichen,Tuckermannopsis.sepincola, +lichen,Usnea.cavernosa, +lichen,Usnea.dasopoga, +lichen,Usnea.glabrata, +lichen,Usnea.hirta, +lichen,Usnea.lapponica, +lichen,Usnea.scabrata, +lichen,Usnea.scabrata.m.dasopoga, +lichen,Usnea.substerilis.subfloridana, +lichen,Verrucariaceae, +lichen,Vulpicida.pinastri, +lichen,Xanthomendoza.fallax, +lichen,Xanthomendoza.fulva, +lichen,Xanthomendoza.hasseana.montana, +lichen,Xanthoparmelia.camtschadalis, +lichen,Xanthoparmelia.chlorochroa, +lichen,Xanthoparmelia.wyomingica, +lichen,Xanthoria.elegans, +mite,Achipteria.coleoptrata, +mite,Achipteria.sp.1.DEW, +mite,Aeroppia.sp.1.DEW, +mite,Allosuctobelba.gigantea, +mite,Allosuctobelba.sp.2.DEW, +mite,Anachipteria.howardi, +mite,Anachipteria.sp.1.DEW, +mite,Atropacarus.striculus, +mite,Banksinoma.lanceolata.canadensis, +mite,Banksinoma.spinifera, +mite,Camisia.biurus, +mite,Camisia.biverrucata, +mite,Camisia.sp.1.DEW, +mite,Carabodes.granulatus, +mite,Carabodes.labyrinthicus, +mite,Carabodes.polyporetes, +mite,Cepheus.sp.1.DEW, +mite,Cepheus.sp.2.DEW, +mite,Ceratoppia.bipilis, +mite,Ceratoppia.quadridentata.arctica, +mite,Ceratozetes.cuspidatus, +mite,Ceratozetes.gracilis, +mite,Ceratozetes.mediocris, +mite,Ceratozetes.thienemanni, +mite,Chamobates.cuspidatus, +mite,Dentizetes.ledensis, +mite,Dentizetes.rudentiger, +mite,Dentizetes.sp.1.LML, +mite,Diapterobates.humeralis, +mite,Dorycranosus.acutidens, +mite,Dyobelba.sp.1.DEW, +mite,Eniochthonius.crosbyi, +mite,Eniochthonius.minutissimus, +mite,Eobrachychthonius.latior, +mite,Epidamaeus.arcticola, +mite,Epidamaeus.coxalis, +mite,Epidamaeus.floccosus, +mite,Epidamaeus.koyukon, +mite,Epidamaeus.sp.1.DEW, +mite,Epidamaeus.sp.2.DEW, +mite,Epidamaeus.sp.3.DEW, +mite,Epidamaeus.sp.5.DEW, +mite,Eremaeus.translamellatus, +mite,Eueremaeus.foveolatus, +mite,Eueremaeus.marshalli.quadrilamellatus, +mite,Eueremaeus.masinasin, +mite,Eueremaeus.tetrosus, +mite,Eupelops.septentrionalis, +mite,Euphthiracarus.flavus, +mite,Eupterotegaeus.rostratus, +mite,Fuscozetes.fuscipes, +mite,Galumna.sp.1.DEW, +mite,Gustavia.sp.1.DEW, +mite,Gymnodamaeus.ornatus, +mite,Hafenferrefia.sp.1.DEW, +mite,Hemileius.haydeni, +mite,Heminothrus.longisetosus, +mite,Hermanniella.robusta, +mite,Hoplophthiracarus.illinoisensis, +mite,Hydrozetes.sp.2.DEW, +mite,Hydrozetes.sp.E.RAN, +mite,Hypochthonius.rufulus, +mite,Lanibelba.sp.1.DEW, +mite,Limnozetes.canadensis.s.l., +mite,Lucoppia.burrowsii, +mite,Mainothrus.badius, +mite,Malaconothrus.mollisetosus, +mite,Multioppia.sp.1.DEW, +mite,Mycobates.incurvatus, +mite,Mycobates.perates, +mite,Nanhermannia.sp.1.DEW, +mite,Neogymnobates.luteus, +mite,Neogymnobates.sp.1.DEW, +mite,Neonothrus.humicola, +mite,Neoribates.sp.1.DEW, +mite,Neoribates.sp.2.DEW, +mite,Nothrus.anauniensis, +mite,Nothrus.borussicus, +mite,Nothrus.pratensis, +mite,Nothrus.sp.B.DEW, +mite,Oppiella.washburni, +mite,Oribatella.abmi, +mite,Oribatella.jacoti, +mite,Oribatella.pawnee, +mite,Oribatella.yukonensis, +mite,Oribatodes.mirabilis, +mite,Oribatula.sp.1.DEW, +mite,Peloptulus.sp.1.DEW, +mite,Peloribates.canadensis, +mite,Peloribates.pilosus, +mite,Peloribates.sp.3.DEW, +mite,Peloribates.sp.4.DEW, +mite,Pergalumna.sp.1.DEW, +mite,Phthiracarus.borealis, +mite,Phthiracarus.boresetosus, +mite,Pilogalumna.sp.1.DEW, +mite,Pilogalumna.sp.2.DEW, +mite,Platynothrus.peltifer, +mite,Platynothrus.yamasakii, +mite,Podoribates.longipes, +mite,Propelops.alaskensis, +mite,Propelops.canadensis, +mite,Protoribates.haughlandae, +mite,Protoribates.robustior, +mite,Protoribotritia.sp.1.DEW, +mite,Punctoribates.palustris, +mite,Quatrobelba.montana, +mite,Rhysotritia.ardua, +mite,Roynortonella.gildersleeveae, +mite,Roynortonella.sp.1.DEW, +mite,Scheloribates.pallidulus, +mite,Scutozetes.lanceolatus, +mite,Sphaerozetes.arcticus, +mite,Suctobelbella.punctata, +mite,Tectocepheus.sarekensis, +mite,Tectocepheus.velatus, +mite,Tegoribates.americanus, +mite,Trhypochthonius.cladonicola, +mite,Trhypochthonius.tectorum, +mite,Trichoribates.striatus, +mite,Trimalaconothrus.maior, +mite,Unduloribates.dianae, +mite,Zetomimus.francisi, +mite,Zygoribatula.sp.1.DEW, +mite,Zygoribatula.sp.2.DEW, +mammal,Badger_Summer, +mammal,Badger_Winter, +mammal,BlackBear_Summer, +mammal,BlackBear_Winter, +mammal,CanadaLynx_Summer, +mammal,CanadaLynx_Winter, +mammal,Cougar_Summer, +mammal,Cougar_Winter, +mammal,Coyote_Summer, +mammal,Coyote_Winter, +mammal,Deer_Summer, +mammal,Deer_Winter, +mammal,ElkWapiti_Summer, +mammal,ElkWapiti_Winter, +mammal,Fisher_Summer, +mammal,Fisher_Winter, +mammal,GrayWolf_Summer, +mammal,GrayWolf_Winter, +mammal,GrizzlyBear_Summer, +mammal,Marten_Summer, +mammal,Marten_Winter, +mammal,Moose_Summer, +mammal,Moose_Winter, +mammal,MuleDeer_Summer, +mammal,MuleDeer_Winter, +mammal,Porcupine_Summer, +mammal,Porcupine_Winter, +mammal,Pronghorn_Summer, +mammal,Pronghorn_Winter, +mammal,RedFox_Summer, +mammal,RedFox_Winter, +mammal,RedSquirrel_Summer, +mammal,RedSquirrel_Winter, +mammal,RichardsonsGroundSquirrel_Summer, +mammal,RichardsonsGroundSquirrel_Winter, +mammal,SnowshoeHare_Summer, +mammal,SnowshoeHare_Winter, +mammal,StripedSkunk_Summer, +mammal,StripedSkunk_Winter, +mammal,WeaselsAndErmine_Summer, +mammal,WeaselsAndErmine_Winter, +mammal,WhiteTailedDeer_Summer, +mammal,WhiteTailedDeer_Winter, +mammal,WhiteTailedJackRabbit_Summer, +mammal,WhiteTailedJackRabbit_Winter, +mammal,Wolverine_Summer, +mammal,Wolverine_Winter, +mammal,WoodlandCaribou_Summer, +mammal,WoodlandCaribou_Winter, +bird,ALFL, +bird,AMCR, +bird,AMGO, +bird,AMRE, +bird,AMRO, +bird,ATTW, +bird,BAIS, +bird,BANS, +bird,BAOR, +bird,BARS, +bird,BAWW, +bird,BBMA, +bird,BBWA, +bird,BBWO, +bird,BCCH, +bird,BHCO, +bird,BHVI, +bird,BLBW, +bird,BLJA, +bird,BLPW, +bird,BOBO, +bird,BOCH, +bird,BRBL, +bird,BRCR, +bird,BRSP, +bird,BRTH, +bird,BTNW, +bird,CAJA, +bird,CAWA, +bird,CCLO, +bird,CCSP, +bird,CEDW, +bird,CHSP, +bird,CLNU, +bird,CMWA, +bird,COGR, +bird,COLO, +bird,CONI, +bird,CONW, +bird,CORA, +bird,COYE, +bird,CSWA, +bird,DEJU, +bird,DOWO, +bird,EAKI, +bird,EAPH, +bird,EUST, +bird,EVGR, +bird,FOSP, +bird,GCKI, +bird,GRAP, +bird,GRCA, +bird,GRSP, +bird,GRYE, +bird,HAWO, +bird,HETH, +bird,HOGR, +bird,HOLA, +bird,HOSP, +bird,HOWR, +bird,KILL, +bird,LARB, +bird,LASP, +bird,LBCU, +bird,LEFL, +bird,LEYE, +bird,LISP, +bird,MAGO, +bird,MAWA, +bird,MAWR, +bird,MOCH, +bird,MODO, +bird,MOWA, +bird,NESP, +bird,NOFL, +bird,NOPI, +bird,NOWA, +bird,OCWA, +bird,OSFL, +bird,OVEN, +bird,PAWA, +bird,PHVI, +bird,PISI, +bird,PIWO, +bird,PUFI, +bird,RBGR, +bird,RBNU, +bird,RCKI, +bird,REVI, +bird,RNEP, +bird,ROPI, +bird,RTHA, +bird,RUBL, +bird,RUGR, +bird,RWBL, +bird,SAVS, +bird,SEWR, +bird,SORA, +bird,SOSA, +bird,SOSP, +bird,SPPI, +bird,SPSA, +bird,STGR, +bird,SWSP, +bird,SWTH, +bird,TBLO, +bird,TEWA, +bird,TRES, +bird,UPSA, +bird,VATH, +bird,VEER, +bird,VESP, +bird,WAVI, +bird,WBNU, +bird,WCSP, +bird,WEKI, +bird,WEME, +bird,WETA, +bird,WEWP, +bird,WILL, +bird,WIPH, +bird,WISN, +bird,WIWA, +bird,WIWR, +bird,WTSP, +bird,WWCR, +bird,YBFL, +bird,YBSA, +bird,YERA, +bird,YEWA, +bird,YHBL, +bird,YRWA, diff --git a/1_code/experiments/exp_000_parity_v2/README.md b/1_code/experiments/exp_000_parity_v2/README.md index 8945636..a048d51 100644 --- a/1_code/experiments/exp_000_parity_v2/README.md +++ b/1_code/experiments/exp_000_parity_v2/README.md @@ -23,8 +23,12 @@ source("1_code/experiments/exp_000_parity_v2/run.R") Every decision is in `run.R`'s `experiment_config()` call (section 1.2); section 1.3 points the run at the v2 reference. As committed, -`run.R` runs the `parity_check` species at 5 draws on 12 workers. For -the gate, set `species = NULL` and `n_bootstraps = 100`. +`run.R` runs the species in this folder's `species.csv` (two per +taxon, chosen so the test discriminates; its `note` column gives +each one's detections and prevalence) at 100 draws on 12 workers. +For the gate, set `species = NULL`, which runs every species. Later +experiments copy this `species.csv` for a like-for-like +comparison. `run_experiment()` fits, summarizes, then runs this experiment's steps: | Step | Script | Writes, in `3_output/exp_000_parity_v2/` | diff --git a/1_code/experiments/exp_000_parity_v2/run.R b/1_code/experiments/exp_000_parity_v2/run.R index ef43a41..5e197a4 100644 --- a/1_code/experiments/exp_000_parity_v2/run.R +++ b/1_code/experiments/exp_000_parity_v2/run.R @@ -7,6 +7,7 @@ # - the published v2 results, the ABMIexploreR reference # (section 1.3; see harness/data_source.R) # - the framework: 1_code/harness/, methods/, modules/ +# - species.csv in this folder # outputs: # in 2_pipeline/exp_000_parity_v2/: # - // result stores, and run_log.csv @@ -47,10 +48,14 @@ config <- experiment_config( "bird" ), - # A focal_species_sets() name, a vector named by taxon, or NULL - # for every species. Shorten trials by species, not draws, so - # the bands stay honest. - species = "parity_check", + # This folder's species.csv, or NULL for every species. Two per + # taxon, chosen so the test discriminates (rare species have + # bands wide enough to pass anything): well above 20 detections + # in both regions, prevalence ~5-45%, all 100 v2 draws usable, + # and one of each pair balanced between north and south. Its + # `note` column gives each species' numbers. Shorten trials by + # species, not draws, so the bands stay honest. + species = file.path(exp_dir, "species.csv"), # 100 is v2's and the only count the gate reads; below ~20 the # 10-90% bands are unstable. 5 checks the plumbing. diff --git a/1_code/experiments/exp_000_parity_v2/species.csv b/1_code/experiments/exp_000_parity_v2/species.csv new file mode 100644 index 0000000..bbe6932 --- /dev/null +++ b/1_code/experiments/exp_000_parity_v2/species.csv @@ -0,0 +1,15 @@ +taxon,species,note +bryophyte,Ceratodon.purpureus,"Detections N/S 2140/554; prevalence 44.1/24.0%" +bryophyte,Bryum.All,"Genus aggregate; the only bryophyte strong in the south as well as the north (1941/856; 40.0/37.1%). Pylaisia.polyantha is the true-species swap, at the cost of the southern test" +lichen,Physcia.adscendens,"Best-detected lichen (2232/506; 38.4/21.6%)" +lichen,Cladonia.chlorophaea,"Second best-detected lichen (2236/218; 38.5/9.3%)" +mite,Ceratozetes.gracilis,"Strongest mite in the north (1190/133; 20.8/5.6%)" +mite,Trhypochthonius.tectorum,"Most even mite across regions (546/197; 9.6/8.3%)" +vascular_plant,Galium.boreale,"Common in both regions (2875/588; 41.8/22.2%)" +vascular_plant,Vicia.americana,"Common in both regions (2596/700; 37.8/26.4%)" +mammal,Moose_Summer,"Published climate reference; both seasons because v2's references average them (1585/225; 31.2/17.8%)" +mammal,Moose_Winter,"Winter half of Moose" +mammal,Coyote_Summer,"Carries the south, where moose is thinner (1362/893; 26.8/70.8%)" +mammal,Coyote_Winter,"Winter half of Coyote" +bird,AMRO,"Gated on climate and translated landcover (41853/23613; 24.4/25.9%)" +bird,YEWA,"Gated on climate and translated landcover (22337/15887; 13.0/17.5%)" diff --git a/1_code/experiments/exp_001_xgboost/README.md b/1_code/experiments/exp_001_xgboost/README.md index 4f5afe9..4cdd398 100644 --- a/1_code/experiments/exp_001_xgboost/README.md +++ b/1_code/experiments/exp_001_xgboost/README.md @@ -20,7 +20,7 @@ draws with tuned settings. | Climate stage | v2: GLM candidates, AICc model averaging (mammals: v2's precomputed prediction) | The same | | Habitat stage (each spec's `habitat_stage`; birds: landcover) | v2: GLM candidates; IVW averaging (plants), staged BIC (birds), or the hurdle with best-by-AICc halves (mammals); v2's post-processing | `xgboost` with the `single` rule, on every covariate v2's habitat candidates use, `Climate` included | | Climate → habitat | Carried as `Climate` | The same | -| Species, draws, seed | `parity_check`, 20260909 | The same | +| Species, draws, seed | exp_000's (`species.csv`, two per taxon), 20260909 | The same | - **What changes:** the habitat stage's engine, selection rule and candidate set. `run.R` makes the change with one call for every run: diff --git a/1_code/experiments/exp_001_xgboost/run.R b/1_code/experiments/exp_001_xgboost/run.R index 19d82d6..a1b02ed 100644 --- a/1_code/experiments/exp_001_xgboost/run.R +++ b/1_code/experiments/exp_001_xgboost/run.R @@ -5,6 +5,7 @@ # inputs: # - the published test dataset (harness/data_source.R) # - 3_output/exp_000_parity_v2/tables/, the v2 baseline +# - species.csv in this folder: exp_000's species # outputs: # in 2_pipeline/exp_001_xgboost/: # - // result stores, and run_log.csv @@ -24,6 +25,8 @@ # 1. Setup ---- ## 1.1 Load the framework ---- +exp_dir <- "1_code/experiments/exp_001_xgboost" + source("1_code/harness/harness.R") load_framework() @@ -49,7 +52,7 @@ specs <- lapply( config <- experiment_config( id = "exp_001_xgboost", taxa = names(specs), - species = "parity_check", + species = file.path(exp_dir, "species.csv"), # 5 checks the pipeline runs; 100 for a result to read n_bootstraps = 100, seed = 20260909, diff --git a/1_code/experiments/exp_001_xgboost/species.csv b/1_code/experiments/exp_001_xgboost/species.csv new file mode 100644 index 0000000..629cc62 --- /dev/null +++ b/1_code/experiments/exp_001_xgboost/species.csv @@ -0,0 +1,15 @@ +taxon,species +bryophyte,Ceratodon.purpureus +bryophyte,Bryum.All +lichen,Physcia.adscendens +lichen,Cladonia.chlorophaea +mite,Ceratozetes.gracilis +mite,Trhypochthonius.tectorum +vascular_plant,Galium.boreale +vascular_plant,Vicia.americana +mammal,Moose_Summer +mammal,Moose_Winter +mammal,Coyote_Summer +mammal,Coyote_Winter +bird,AMRO +bird,YEWA diff --git a/1_code/experiments/exp_002_soilgrids/README.md b/1_code/experiments/exp_002_soilgrids/README.md index 8c4d393..32ac486 100644 --- a/1_code/experiments/exp_002_soilgrids/README.md +++ b/1_code/experiments/exp_002_soilgrids/README.md @@ -50,7 +50,7 @@ included. | Climate stage | v2's candidates (plants 58, birds 25), model-averaged | The same candidates, each with the nine soil terms added | | Climate → habitat | Carried as `Climate` | The same; the carried prediction now includes soil | | Habitat stage | v2 | v2, unchanged | -| Species, draws, seed | `parity_check`, 20260909 | The same | +| Species, draws, seed | exp_000's (`species.csv`, two per taxon), 20260909 | The same | - **Why the climate stage.** It is fitted once, province-wide, so one set of formulas serves every region, and it is v2's broad-scale environmental diff --git a/1_code/experiments/exp_002_soilgrids/run.R b/1_code/experiments/exp_002_soilgrids/run.R index 367d145..da714fb 100644 --- a/1_code/experiments/exp_002_soilgrids/run.R +++ b/1_code/experiments/exp_002_soilgrids/run.R @@ -8,6 +8,7 @@ # soil covariates, written by 01_extract_soilgrids_covariates.R # in this folder (run here when the file is missing) # - 3_output/exp_000_parity_v2/tables/, the v2 baseline +# - species.csv in this folder: exp_000's species # outputs: # in 2_pipeline/exp_002_soilgrids/: # - // result stores, and run_log.csv @@ -25,6 +26,8 @@ # 1. Setup ---- ## 1.1 Load the framework ---- +exp_dir <- "1_code/experiments/exp_002_soilgrids" + source("1_code/harness/harness.R") load_framework() @@ -72,7 +75,7 @@ stage_models <- c( config <- experiment_config( id = "exp_002_soilgrids", taxa = c(plant_taxa, "bird"), - species = "parity_check", + species = file.path(exp_dir, "species.csv"), # 5 checks the pipeline runs; 100 for a result to read n_bootstraps = 5, seed = 20260909, diff --git a/1_code/experiments/exp_002_soilgrids/species.csv b/1_code/experiments/exp_002_soilgrids/species.csv new file mode 100644 index 0000000..629cc62 --- /dev/null +++ b/1_code/experiments/exp_002_soilgrids/species.csv @@ -0,0 +1,15 @@ +taxon,species +bryophyte,Ceratodon.purpureus +bryophyte,Bryum.All +lichen,Physcia.adscendens +lichen,Cladonia.chlorophaea +mite,Ceratozetes.gracilis +mite,Trhypochthonius.tectorum +vascular_plant,Galium.boreale +vascular_plant,Vicia.americana +mammal,Moose_Summer +mammal,Moose_Winter +mammal,Coyote_Summer +mammal,Coyote_Winter +bird,AMRO +bird,YEWA diff --git a/1_code/harness/README.md b/1_code/harness/README.md index a75af9d..704922f 100644 --- a/1_code/harness/README.md +++ b/1_code/harness/README.md @@ -9,9 +9,17 @@ Load everything with: ```r source("1_code/harness/harness.R") -load_framework() # harness/, methods/, modules/, experiments/_shared/ +load_framework() # harness/, methods/, modules/ + +# An experiment also loads its own methods from hooks/ +load_framework(hooks = "1_code/experiments//hooks") ``` +Experiments never edit this folder. Everything an experiment +changes is an `experiment_config()` argument, a spec change made in +its `run.R`, or a function in its `hooks/` (see the +[experiments README](../experiments/README.md)). + `new_experiment.R` is the exception: it is base R and runs without `load_framework()`. @@ -19,39 +27,41 @@ load_framework() # harness/, methods/, modules/, experiments/_shared/ | File | Holds | | --- | --- | -| `harness.R` | `load_framework()`: sources the harness, methods, modules and `experiments/_shared/`, in that order | -| `experiment.R` | `experiment_config()`, `run_experiment()`, `script_step()` | +| `harness.R` | `load_framework()`: sources the harness, methods and modules, then an experiment's hooks (`load_hooks()`); workers load the same hooks | +| `experiment.R` | `experiment_config()` (checks everything, lists every problem at once), `run_experiment()`, `script_step()`, `runs_for_species()` | | `new_experiment.R` | `new_experiment()`: scaffold an experiment's three folders | | `data_source.R` | The pinned published dataset versions; `test_dataset_dir()`, `v2_results_dir()`, `verify_published()` | -| `data_load.R` | `lookup/dataset_manifest.csv`, loaders, `covariate_files` joins, model frames | +| `data_load.R` | `lookup/dataset_manifest.csv`, loaders, `covariate_files` joins, model frames; `covariate_columns_needed()` | | `cache.R` | Session cache for dataset reads | | `covariate_sets.R` | Named covariate sets, derived columns, term renaming | -| `model_sets.R` | v2 candidate formulas, spliced verbatim; `extend_models()`, `models_from_covariates()` | -| `spec.R` | `validate_spec()`, `apply_stage_models()`, `replace_stage_method()` | -| `registry.R` | `register_engine()` and the other `register_*()`; `list_methods()` | +| `model_sets.R` | v2 candidate formulas, spliced verbatim; `extend_models()` and `models_from_covariates()` to add or build, `drop_candidates()` and `strip_terms()` to reduce | +| `spec.R` | `validate_spec()`, `apply_stage_models()`, `replace_stage_method()`; `modify_stage()`, `modify_region()`, `modify_resample()` for an experiment's spec changes; `check_spec_covariates()` | +| `registry.R` | `register_engine()` and the other `register_*()` (metrics take a `direction`); `list_methods()`. A hook may not reuse a registered name | | `engines.R` | Engine interface and `check_engine()` | | `selection.R` | Plumbing for selection rules: `fit_candidates()`, `combined_predictor()`, `selection_result()` | -| `resample.R` | Per-species seeds, `with_seed()`, spatial blocks | -| `species.R` | Species lists and focal-species resolution | +| `resample.R` | Per-species seeds, `with_seed()`, spatial blocks; `v2_block_breaks()` | +| `species.R` | Species lists; `read_species_file()` and `check_species_names()` for an experiment's `species.csv` | | `predict_grid.R` | Projects a final model onto a habitat grid | | `eval_metrics.R` | Scores a prediction with every registered metric | -| `run_model.R` | `run_specs()`: prepare each spec, fit species in parallel, merge shards | +| `run_model.R` | `run_specs()`: prepare each spec, fit species in parallel, merge shards; `region_request()` and `province_request()`, shared with the configuration check | | `result.R` | The result store: writes and reads its files | | `summarise.R` | `collect_results()`: stores to summary tables | -| `compare.R` | `compare_experiments()`: one experiment against another | -| `run_record.R` | `run_record()`: writes `run_record.md` | +| `compare.R` | `compare_experiments()`: one experiment against another; `default_compare_metrics()` | +| `run_record.R` | `run_record()`: writes `run_record.md`, with provenance | ## What `run_experiment()` does 1. **Configure.** `experiment_config()` checks every decision before - anything is fitted. + anything is fitted, and stops once with every problem listed: + species names (with close matches), runs, draws, regions, + metrics, each spec, and every covariate the formulas name. 2. **Fit.** Every spec is validated and each region's data and grid are loaded once. Each species is one job; with `workers` above 1, jobs run in parallel R sessions. Each species draws under its own seed, so results do not depend on the worker count. A failed species or draw is logged and the run continues. 3. **Store.** Shards are merged into one store per run and region in - `2_pipeline//`, plus `run_log.csv`. + `2_pipeline//`, plus `run_log.csv` and `session_info.txt`. 4. **Summarize.** `collect_results()` writes `3_output//tables/`. 5. **Compare.** Every experiment except the baseline writes `tables/comparison_metrics.csv` against exp_000. @@ -66,8 +76,8 @@ Nothing outside `2_pipeline//` and `3_output//` is written. | Argument | Default | Sets | | --- | --- | --- | | `id` | | Folder name under `2_pipeline/` and `3_output/` | -| `taxa` | all | Runs, from `names(standard_specs())` | -| `species` | every species | A `focal_species_sets()` name, or a vector named by taxon | +| `taxa` | see note | Runs, from `names(standard_specs())`. Left out with a species list: only the runs that fit a listed species (a `Moose_Summer` row runs `mammal_summer`). Left out with no species list: every run | +| `species` | every species | The experiment's `species.csv` (`taxon,species,note`), or a vector named by taxon. Names are checked against the dataset's species queue | | `n_bootstraps` | `100` | Draws per species; v2 used 100. 5 checks a run works | | `seed` | `20260909` | Base seed; `NULL` leaves draws unseeded, as v2 did | | `stage_models` | v2's | Replacement candidate sets, keyed `stage` or `taxon.stage` | @@ -78,6 +88,9 @@ Nothing outside `2_pipeline//` and `3_output//` is written. | `unit_predictions` | `"none"` | `"oob"` or `"all"` writes per-unit predictions | | `baseline` | `"exp_000_parity_v2"` | Experiment to compare with; `NULL` skips | | `v2_bootstraps` | `100` | Draws a run needs to be gated | +| `regions` | all | Regions to fit, e.g. `"north"` | +| `metrics` | all registered | Metrics scored per draw | +| `compare_metrics` | `default_compare_metrics()` | Prefixed metrics (e.g. `"oob_auc"`) compared with the baseline | | `project_root`, `data_dir`, `pipeline_dir`, `out_dir` | derived | Path overrides | ## Outputs @@ -94,10 +107,18 @@ read), plus `mean`, `sd`, `n` and `boot1` (the draw-1 value). | `metrics.csv` | Metrics per species and draw | | `grid_predictions.csv` | The final model's prediction for each habitat type | | `unit_predictions.csv` | Only with `unit_predictions = "oob"` or `"all"`; several GB for birds | -| `meta.json` | Stages, engines, rules, covariates, resampling, seed, overrides | +| `meta.json` | Stages, engines, rules, covariates, resampling, seed, overrides, hook files | `2_pipeline//run_log.csv` records every species, region and draw: -`ok`, or why not. +`ok`, or why not. `2_pipeline//session_info.txt` holds the full +`sessionInfo()` of the run. + +`3_output//run_record.md` is the committed record. Its +Provenance section gives the full git commit, how many files under +`1_code/` differ from it, R version, platform, modelling package +versions, and each hook file with its md5. Its Configuration +section gives every setting, the species file, the hooks, and each +change from v2. ### Summary tables: `3_output//tables/` diff --git a/1_code/harness/compare.R b/1_code/harness/compare.R index 625ef43..235eb25 100644 --- a/1_code/harness/compare.R +++ b/1_code/harness/compare.R @@ -28,6 +28,9 @@ library(data.table) # keyed joins (version: 1.16.4) #' Which Way Is Better for a Metric #' +#' A direction given at registration (register_metric()) wins; +#' otherwise the built-in table below. +#' #' @param metric Character vector of metric names, with or #' without an `insample_` or `oob_` prefix. #' @return Character vector: "higher", "lower", "toward_one" or @@ -38,13 +41,41 @@ library(data.table) # keyed joins (version: 1.16.4) better_direction <- function(metric) { base <- sub("^(insample_|oob_)", "", metric) - ifelse( + built_in <- ifelse( base %in% c("rmse"), "lower", ifelse( base %in% c("calibration_slope"), "toward_one", ifelse(base %in% c("prevalence", "n"), "none", "higher") ) ) + + registered <- vapply(base, function(one) { + if (!one %in% registered_names("metric")) { + return(NA_character_) + } + get_method("metric", one, entry = TRUE)$direction %||% + NA_character_ + }, character(1), USE.NAMES = FALSE) + + ifelse(is.na(registered), built_in, registered) +} + +## 2.1 default_compare_metrics() ---- + +#' The Metrics an Experiment Is Compared on by Default +#' +#' Out-of-bag fit metrics: in-sample metrics flatter flexible +#' methods. +#' +#' @return A character vector of prefixed metric names. +#' +#' @example # Example usage of the function +#' # default_compare_metrics() +default_compare_metrics <- function() { + c( + "oob_auc", "oob_deviance_explained", "oob_rmse", + "oob_spearman", "oob_calibration_slope" + ) } # 3. compare_experiments() ---- @@ -94,10 +125,7 @@ compare_experiments <- function( values <- c("n", "median", "p10", "p90") # Step 1: Model fit, per species and metric - metrics <- metrics %||% c( - "oob_auc", "oob_deviance_explained", "oob_rmse", - "oob_spearman", "oob_calibration_slope" - ) + metrics <- metrics %||% default_compare_metrics() candidate <- read_summary(candidate_dir, "metric_summary") baseline <- read_summary(baseline_dir, "metric_summary") @@ -135,11 +163,15 @@ compare_experiments <- function( default = NA )] - # Step 2: Per taxon, region and metric + # Step 2: Per taxon, region and metric. Draw counts as doubles: + # median() keeps an odd-sized group's integer, and data.table + # needs one type across groups. summary <- joined[, list( species = .N, - draws_baseline = stats::median(n_baseline, na.rm = TRUE), - draws_candidate = stats::median(n_candidate, na.rm = TRUE), + draws_baseline = as.numeric(stats::median(n_baseline, + na.rm = TRUE)), + draws_candidate = as.numeric(stats::median(n_candidate, + na.rm = TRUE)), median_baseline = stats::median(median_baseline, na.rm = TRUE), median_candidate = stats::median(median_candidate, na.rm = TRUE), median_difference = stats::median(difference, na.rm = TRUE), diff --git a/1_code/harness/data_load.R b/1_code/harness/data_load.R index 7abaa94..0839518 100644 --- a/1_code/harness/data_load.R +++ b/1_code/harness/data_load.R @@ -14,7 +14,9 @@ # - Taxon-agnostic: file locations and covariate keys come from # lookup/dataset_manifest.csv (_setup/07). The key is # `_` for mammals, whose north and south files -# hold different values for the same deployment. +# hold different values for the same deployment. A manifest +# row with region "province" names a separate province-wide +# table (province_region()). # - Reads are column-selective (covariates.csv is ~218 MB, 457 # columns). build_model_data() loads once per taxon and # region; model_frame() cuts per-species frames from it. @@ -120,6 +122,32 @@ covariate_key <- function(taxon, region = NULL, data_dir) { manifest_entry(data_dir, taxon, region)$covariate_key } +## 2.4 province_region() ---- + +#' Name the Region a Taxon's Province-Wide Data Are Stored Under +#' +#' Most taxa store one set of rows that every region filters, so +#' the province frame is read with no region. A taxon whose +#' province-wide data are a separate table (the mammal climate +#' stage) has a manifest row with region "province". +#' +#' @param data_dir Character. The test dataset folder. +#' @param taxon Character. Taxon slug. +#' @return "province", or NULL when the taxon has no such row. +#' +#' @example # Example usage of the function +#' # province_region(data_dir, "mammal") # "province" +#' # province_region(data_dir, "lichen") # NULL +province_region <- function(data_dir, taxon) { + manifest <- dataset_manifest(data_dir) + + if (any(manifest$taxon == taxon & manifest$region == "province")) { + "province" + } else { + NULL + } +} + # 3. load_sites() ---- #' Load the Survey Unit Table @@ -259,6 +287,44 @@ load_offsets <- function(taxon, data_dir, species = NULL) { ) } +## 5.1 covariate_columns_needed() ---- + +#' Split Requested Covariates into Stored, Derivable and Unknown +#' +#' Reads the taxon's covariate catalogue only, not the data. +#' Shared by load_covariates() and the configuration check. +#' +#' @param taxon Character. Taxon slug. +#' @param data_dir Character. The test dataset folder. +#' @param columns Character vector of master column names. +#' @param region Character or NULL. +#' @return A list of `derivable` (computed from other columns), +#' `to_read` (stored columns to read, derivation inputs +#' included) and `unknown` (neither stored nor derivable). +#' +#' @example # Example usage of the function +#' # covariate_columns_needed("lichen", data_dir, "MAP")$unknown +covariate_columns_needed <- function(taxon, data_dir, columns, + region = NULL) { + stored <- available_covariates( + taxon, data_dir = data_dir, region = region, + include_derived = FALSE + ) + + derivable <- derivable_covariates(columns, stored) + to_read <- setdiff(columns, derivable) + + for (one in derivable) { + to_read <- unique(c(to_read, derived_covariates()[[one]]$inputs)) + } + + list( + derivable = derivable, + to_read = to_read, + unknown = setdiff(to_read, stored) + ) +} + # 6. load_covariates() ---- #' Load Covariates for One Taxon and Region @@ -290,19 +356,10 @@ load_covariates <- function( # Step 1: Check the taxon's catalogue, not the file header: the # wide table holds other taxa's columns, which read back as all # NA here. - stored <- available_covariates( - taxon, data_dir = data_dir, region = region, - include_derived = FALSE - ) - - derivable <- derivable_covariates(columns, stored) - to_read <- setdiff(columns, derivable) - - for (one in derivable) { - to_read <- unique(c(to_read, derived_covariates()[[one]]$inputs)) - } - - unknown <- setdiff(to_read, stored) + needed <- covariate_columns_needed(taxon, data_dir, columns, region) + derivable <- needed$derivable + to_read <- needed$to_read + unknown <- needed$unknown if (length(unknown) > 0) { stop( @@ -764,8 +821,11 @@ build_model_data <- function( #' model weights, or NULL. #' @param response_name,offset_name,weight_name Character. Column #' names to write into the frame. +#' @param companions Named character vector of other response +#' columns to carry, as `_`, or NULL. The +#' mammal climate stage pools a species' two seasons. #' @return A data frame with the covariates plus the response, -#' and the offset and weight when they apply. +#' and the offset, weight and companions when they apply. #' #' @example # Example usage of the function #' # d <- model_frame(md, "Aulacomnium.palustre", @@ -777,7 +837,8 @@ model_frame <- function( weight_column = NULL, response_name = "response", offset_name = "offset", - weight_name = "weight" + weight_name = "weight", + companions = NULL ) { if (!species %in% names(model_data$response)) { stop( @@ -826,6 +887,23 @@ model_frame <- function( frame[[weight_name]] <- frame[[weight_column]] } + # Step 4: Attach companion responses, for a stage's own + # response transform to read (fit_one_stage()) + for (name in names(companions)) { + column <- companions[[name]] + + if (!column %in% names(model_data$response)) { + stop( + "Companion `", column, "` of `", species, "` is not a ", + "column of the ", model_data$taxon, " response table.", + call. = FALSE + ) + } + + frame[[paste0(response_name, "_", name)]] <- + model_data$response[[column]] + } + frame } diff --git a/1_code/harness/experiment.R b/1_code/harness/experiment.R index fc28df1..9346460 100644 --- a/1_code/harness/experiment.R +++ b/1_code/harness/experiment.R @@ -5,15 +5,19 @@ # inputs: # the published test dataset (see data_source.R) # outputs: -# in 2_pipeline//: the result stores and run_log.csv +# in 2_pipeline//: the result stores, run_log.csv and +# session_info.txt # in 3_output//: tables/, run_record.md, and whatever the # experiment's own steps write # notes: # - The part every experiment shares, so each run.R is a short -# list of decisions. experiment_config() checks them before -# anything is fitted; run_experiment() fits, summarizes, -# compares with the baseline (exp_000, v2), runs the -# experiment's own steps and writes the run record. +# list of decisions. experiment_config() checks them all +# before anything is fitted and reports every problem at +# once; run_experiment() fits, summarizes, compares with the +# baseline (exp_000, v2), runs the experiment's own steps and +# writes the run record. +# - The argument defaults here, with standard_specs(), are the +# v2 defaults: an experiment passes only what it changes. # - Steps receive everything through `config` and `results`. # --- @@ -30,10 +34,14 @@ library(data.table) # run log writing (version: 1.16.4) #' "exp_001_covariate_scale". Names its 2_pipeline/ and #' 3_output/ folders. #' @param taxa Character vector of runs, from -#' names(standard_specs()), or NULL for all of them. -#' @param species A focal-species set name (see -#' focal_species_sets()), a species vector named by taxon, or -#' NULL for every species. +#' names(standard_specs()), or NULL. NULL with a species list +#' runs only the runs that fit a listed species (e.g. a +#' `Moose_Summer` row runs `mammal_summer`); NULL with no species +#' list runs every run. +#' @param species The path to the experiment's species.csv (see +#' read_species_file()), a species vector named by taxon, or +#' NULL for every species. Names are checked against the +#' dataset's species queue. #' @param n_bootstraps Integer. Draws per species. 100 is v2's. #' @param seed Integer or NULL. The base seed; NULL leaves the #' draws unseeded, as v2 did. @@ -55,15 +63,25 @@ library(data.table) # run log writing (version: 1.16.4) #' itself. #' @param v2_bootstraps Integer. v2's draw count, which a run must #' reach to be gated. +#' @param regions Character vector of regions to fit, e.g. +#' "north", or NULL for every region each spec defines. +#' @param metrics Character vector of registered metrics to score +#' each draw with, or NULL for all of them (default_metrics()). +#' @param compare_metrics Character vector of prefixed metrics +#' (e.g. "oob_auc") to compare with the baseline, or NULL for +#' default_compare_metrics(). #' @param project_root Character. The repository root. #' @param data_dir,pipeline_dir,out_dir Character, or NULL for #' test_dataset_dir(), 2_pipeline/ and 3_output/. -#' @return A list of class `experiment_config`. +#' @return A list of class `experiment_config`. Every problem +#' found is reported in one error before anything is fitted. #' #' @example # Example usage of the function -#' # config <- experiment_config("exp_001_test", taxa = "lichen", -#' # species = "one_each", -#' # n_bootstraps = 5) +#' # config <- experiment_config( +#' # "exp_001_test", +#' # species = "1_code/experiments/exp_001_test/species.csv", +#' # n_bootstraps = 5 +#' # ) experiment_config <- function( id, taxa = NULL, @@ -78,13 +96,67 @@ experiment_config <- function( unit_predictions = "none", baseline = "exp_000_parity_v2", v2_bootstraps = 100L, + regions = NULL, + metrics = NULL, + compare_metrics = NULL, project_root = getOption("sdm.project_root", "."), data_dir = NULL, pipeline_dir = NULL, out_dir = NULL ) { - # Step 1: The runs + # Problems are collected and reported together, so one run of + # run.R shows everything to fix + problems <- character(0) + problem <- function(...) { + problems <<- c(problems, paste0(...)) + } + specs <- specs %||% standard_specs(plant_bootstrap) + data_dir <- test_dataset_dir(data_dir) + + # Step 1: The species: the experiment's species.csv, or a + # vector named by taxon + species_file <- is.character(species) && length(species) == 1L && + is.null(names(species)) + focal <- species + + if (species_file) { + focal <- tryCatch( + read_species_file(species), + error = function(e) { + problem(conditionMessage(e)) + NULL + } + ) + } + + for (one in check_species_names(focal, data_dir)) { + problem(one) + } + + # Step 2: The runs. Without `taxa`, a species list names them: + # only runs that fit a listed species, so editing species.csv + # alone sets what runs. + taxa_given <- !is.null(taxa) + + if (!taxa_given && !is.null(focal) && length(problems) == 0) { + covered <- runs_for_species(specs, focal, data_dir) + taxa <- names(specs)[vapply(covered, length, integer(1)) > 0] + uncovered <- setdiff( + paste(names(focal), focal), + unlist(lapply(names(covered), function(key) { + paste(specs[[key]]$taxon, covered[[key]]) + })) + ) + + if (length(uncovered) > 0) { + problem( + "No run fits ", paste(uncovered, collapse = ", "), + " (e.g. a species not \"modelled\" in any region)." + ) + } + } + taxa <- taxa %||% names(specs) unknown <- setdiff(taxa, names(specs)) @@ -98,44 +170,57 @@ experiment_config <- function( specs <- specs[taxa] - # Step 2: The draw count, checked now rather than hours in + # A run given in `taxa` whose taxon the list does not name runs + # every species, so say so: it makes a partial list the longest + # run + if (taxa_given && !is.null(focal)) { + slugs <- vapply(specs, `[[`, character(1), "taxon") + unnamed <- names(slugs)[!slugs %in% names(focal)] + + if (length(unnamed) > 0) { + message( + "Note: ", paste(unnamed, collapse = ", "), " are not named ", + "in the species list, so every species will be run for ", + "them. Leave `taxa` out to run only the listed species." + ) + } + } + + # Step 3: The draw count, checked now rather than hours in if (!is.numeric(n_bootstraps) || length(n_bootstraps) != 1L || is.na(n_bootstraps) || n_bootstraps < 1 || n_bootstraps != round(n_bootstraps)) { - stop( - "n_bootstraps must be a single whole number of 1 or more.", - call. = FALSE + problem( + "n_bootstraps must be a single whole number of 1 or more." ) - } - - n_bootstraps <- as.integer(n_bootstraps) - data_dir <- test_dataset_dir(data_dir) - - # Stored draws have a hard ceiling; drawn ones do not - for (key in names(specs)) { - spec <- specs[[key]] - - if (!identical(spec$resample$scheme, "precomputed") || - isTRUE(spec$resample$per_species)) { - next - } - - location <- manifest_entry(data_dir, spec$taxon)$bootstrap_ids - stored <- length(names(fread( - file.path(data_dir, location), nrows = 0 - ))) - - if (n_bootstraps > stored) { - stop( - "n_bootstraps is ", n_bootstraps, " but only ", stored, - " ", spec$taxon, " draws are stored. Lower it, or drop `", - key, "`.", - call. = FALSE - ) + } else { + n_bootstraps <- as.integer(n_bootstraps) + + # Stored draws have a hard ceiling; drawn ones do not + for (key in names(specs)) { + spec <- specs[[key]] + + if (!identical(spec$resample$scheme, "precomputed") || + isTRUE(spec$resample$per_species)) { + next + } + + location <- manifest_entry(data_dir, spec$taxon)$bootstrap_ids + stored <- length(names(fread( + file.path(data_dir, location), nrows = 0 + ))) + + if (n_bootstraps > stored) { + problem( + "n_bootstraps is ", n_bootstraps, " but only ", stored, + " ", spec$taxon, " draws are stored. Lower it, or drop `", + key, "`." + ) + } } } - if (n_bootstraps < v2_bootstraps) { + if (is.integer(n_bootstraps) && n_bootstraps < v2_bootstraps) { message( "Note: running ", n_bootstraps, " of ", v2_bootstraps, " v2 draws. This is a trial; its parity numbers are not a ", @@ -143,47 +228,109 @@ experiment_config <- function( ) } - # Step 3: Covariates the experiment adds from its own files. + # Step 4: Covariates the experiment adds from its own files. # Absolute paths, so parallel workers find them too. if (!is.null(covariate_files)) { covariate_files <- normalizePath( covariate_files, winslash = "/", mustWork = FALSE ) - check_covariate_files(covariate_files, data_dir) - - specs <- lapply(specs, function(spec) { - spec$covariate_files <- covariate_files - spec + files_ok <- tryCatch({ + check_covariate_files(covariate_files, data_dir) + TRUE + }, error = function(e) { + problem(conditionMessage(e)) + FALSE }) + + if (files_ok) { + specs <- lapply(specs, function(spec) { + spec$covariate_files <- covariate_files + spec + }) + } } - # Step 4: The species. A taxon the set does not name runs every - # species, so say so: it makes a partial set the longest run. - focal <- get_focal_species(species) + # Step 5: Regions and metrics + for (key in names(specs)) { + absent <- setdiff(regions, names(specs[[key]]$regions)) - if (!is.null(focal)) { - slugs <- vapply(specs, `[[`, character(1), "taxon") - unnamed <- names(slugs)[!slugs %in% names(focal)] + if (length(absent) > 0) { + problem( + "Run `", key, "` has no region(s) ", + paste(absent, collapse = ", "), ". Its regions: ", + paste(names(specs[[key]]$regions), collapse = ", "), "." + ) + } + } - if (length(unnamed) > 0) { - message( - "Note: ", paste(unnamed, collapse = ", "), " are not named ", - "in the species set, so every species will be run for ", - "them. Narrow `taxa` to avoid it." + unknown_metrics <- setdiff(metrics, registered_names("metric")) + + if (length(unknown_metrics) > 0) { + problem( + "Unknown metric(s): ", paste(unknown_metrics, collapse = ", "), + ". Registered: ", + paste(registered_names("metric"), collapse = ", "), "." + ) + } + + # Prefixed as the summaries are; v2val_ rows come from the plant + # specs' own validation, not the registry + compare_base <- sub("^(insample_|oob_)", "", compare_metrics) + unknown_compare <- compare_metrics[ + !compare_base %in% registered_names("metric") & + !startsWith(compare_base, "v2val_") + ] + + if (length(unknown_compare) > 0) { + problem( + "Unknown compare_metrics: ", + paste(unknown_compare, collapse = ", "), ". Use a registered ", + "metric with an `oob_` or `insample_` prefix, e.g. oob_auc." + ) + } + + # Step 6: Every spec as it will run, and the covariates it + # names, checked before any data is read + for (key in names(specs)) { + spec <- apply_stage_models(specs[[key]], stage_models) + spec_problems <- validate_spec(spec, stop_on_error = FALSE) + + if (is.character(spec_problems)) { + problem( + "Run `", key, "`:\n - ", + paste(spec_problems, collapse = "\n - ") ) + next + } + + for (one in check_spec_covariates( + spec, data_dir, + regions = intersect(regions %||% names(spec$regions), + names(spec$regions)), + label = key + )) { + problem(one) } } + if (length(problems) > 0) { + stop( + "experiment_config() found ", length(problems), + " problem(s):\n - ", paste(problems, collapse = "\n - "), + call. = FALSE + ) + } + structure( list( id = id, taxa = taxa, specs = specs, - # The set's name, for the record; "inline" for a vector - species_set = if (is.character(species) && - length(species) == 1L && - is.null(names(species))) { - species + # For the record: the file's name, "inline" or "all" + species_set = if (species_file) { + basename(species) + } else if (is.null(focal)) { + "all" } else { "inline" }, @@ -194,6 +341,11 @@ experiment_config <- function( stage_models = stage_models, covariate_files = covariate_files, plant_bootstrap = plant_bootstrap, + regions = regions, + metrics = metrics, + compare_metrics = compare_metrics, + # Loaded by load_framework(hooks = ); workers load them too + hooks = getOption("sdm.hook_files"), workers = as.integer(workers), unit_predictions = unit_predictions, # The baseline does not compare with itself @@ -208,7 +360,37 @@ experiment_config <- function( ) } -## 2.1 print.experiment_config() ---- +## 2.1 runs_for_species() ---- + +#' Which Listed Species Each Run Fits +#' +#' A run fits a listed species when the species is in the run's +#' queue: its taxon, tier and, for mammals, season. +#' +#' @param specs Named list of specs. +#' @param species A species vector named by taxon. +#' @param data_dir Character. The test dataset folder. +#' @return A named list, one character vector of species per run +#' (empty when the run fits none). +#' +#' @example # Example usage of the function +#' # runs_for_species(standard_specs(), c(mammal = "Moose_Summer"), +#' # data_dir) +runs_for_species <- function(specs, species, data_dir) { + lapply(specs, function(spec) { + listed <- unname(species[names(species) == spec$taxon]) + + if (length(listed) == 0) { + return(character(0)) + } + + intersect(listed, list_species( + data_dir, spec$taxon, tier = spec$tier, season = spec$season + )) + }) +} + +## 2.2 print.experiment_config() ---- #' Print an Experiment's Configuration #' @@ -235,6 +417,14 @@ print.experiment_config <- function(x, ...) { } else { paste(changes, collapse = "\n ") }, "\n", sep = "") + if (!is.null(x$regions)) { + cat(" regions: ", paste(x$regions, collapse = ", "), "\n", + sep = "") + } + if (!is.null(x$hooks)) { + cat(" hooks: ", paste(basename(x$hooks), collapse = ", "), + "\n", sep = "") + } cat(" workers: ", x$workers, "\n", sep = "") cat(" baseline: ", x$baseline %||% "none (this is the baseline)", "\n", sep = "") @@ -283,8 +473,10 @@ run_experiment <- function( data_dir = config$data_dir, pipeline_dir = config$pipeline_dir, species = config$focal_species, + regions = config$regions, iterations = seq_len(config$n_bootstraps), stage_models = config$stage_models, + metrics = config$metrics, boot_seed = config$seed, workers = config$workers, unit_predictions = config$unit_predictions @@ -292,6 +484,11 @@ run_experiment <- function( fwrite(run_log, file.path(config$pipeline_dir, "run_log.csv"), na = "") + # The full session, beside the stores it produced + writeLines( + utils::capture.output(utils::sessionInfo()), + file.path(config$pipeline_dir, "session_info.txt") + ) cat("\nRun log:\n") print(table(run_log$taxon, run_log$status)) } @@ -323,7 +520,8 @@ run_experiment <- function( out_file = file.path(config$out_dir, "run_record.md"), exp_id = config$id, config = record_config(config, run_log), - collected = results + collected = results, + hook_files = config$hooks ) cat("Run record: ", record_file, "\n", sep = "") } @@ -374,6 +572,16 @@ record_config <- function(config, run_log = NULL) { } else { length(config$focal_species) }, + species_file = config$species_set, + regions = config$regions %||% "all", + metrics = config$metrics %||% "all registered", + compare_metrics = config$compare_metrics %||% + default_compare_metrics(), + hooks = if (is.null(config$hooks)) { + "none" + } else { + basename(config$hooks) + }, n_bootstraps = config$n_bootstraps, v2_bootstraps = config$v2_bootstraps, boot_seed = config$seed %||% "unseeded", @@ -478,16 +686,25 @@ compare_to_baseline <- function( return(results) } + compared <- config$compare_metrics %||% default_compare_metrics() comparison <- compare_experiments( candidate_dir = config$out_dir, - baseline_dir = baseline_dir + baseline_dir = baseline_dir, + metrics = compared ) - # The headline, for the console; per-species rows are written + # The headline, for the console; per-species rows are written. + # An experiment's own metrics are shown when it names no default. + headline <- intersect( + c("oob_auc", "oob_deviance_explained", "oob_calibration_slope"), + compared + ) shown <- comparison$summary[ - comparison$summary$metric %in% c( - "oob_auc", "oob_deviance_explained", "oob_calibration_slope" - ), + comparison$summary$metric %in% (if (length(headline) > 0) { + headline + } else { + compared + }), c("taxon", "region", "metric", "median_baseline", "median_candidate", "median_difference", "share_better") ] diff --git a/1_code/harness/harness.R b/1_code/harness/harness.R index 1531714..efae4d1 100644 --- a/1_code/harness/harness.R +++ b/1_code/harness/harness.R @@ -11,9 +11,10 @@ # - Single entry point: experiments source this file and call # load_framework(). # - Load order: harness/ (including the method registry), -# methods/ (each file registers itself), modules/, then -# experiments/_shared/. Files or folders starting with "_" -# are shared helpers and load first. +# methods/ (each file registers itself), modules/, any +# experiments/_shared/ files, then the experiment's own +# hooks/. Files or folders starting with "_" are shared +# helpers and load first. # --- # 1. Setup ---- @@ -124,22 +125,26 @@ load_modules <- function( # 5. load_framework() ---- -#' Load the Harness, Methods, Taxon Modules and Shared Settings +#' Load the Harness, Methods, Taxon Modules and an Experiment's +#' Hooks #' #' What an experiment calls first. Everything is sourced into one #' environment, the caller's by default. #' #' @param project_root Character. The repository root. #' @param envir Environment to source into. +#' @param hooks Character or NULL. The experiment's `hooks/` +#' folder, or a vector of `.R` files; see load_hooks(). #' @return The project root, invisibly. #' #' @example # Example usage of the function #' # source("1_code/harness/harness.R") -#' # load_framework() +#' # load_framework(hooks = "1_code/experiments/exp_003_x/hooks") #' # list_methods() load_framework <- function( project_root = ".", - envir = parent.frame() + envir = parent.frame(), + hooks = NULL ) { project_root <- normalizePath(project_root, winslash = "/") code_dir <- file.path(project_root, "1_code") @@ -148,6 +153,7 @@ load_framework <- function( load_methods(file.path(code_dir, "methods"), envir) load_modules(file.path(code_dir, "modules"), envir) + # Optional; an absent folder sources nothing shared <- sort(list.files( file.path(code_dir, "experiments", "_shared"), pattern = "\\.R$", full.names = TRUE @@ -157,10 +163,81 @@ load_framework <- function( # Parallel workers load the framework from here themselves options(sdm.project_root = project_root) + load_hooks(hooks, envir) + invisible(project_root) } -# 6. source_all() ---- +# 6. load_hooks() ---- + +#' Source an Experiment's Own Methods and Functions +#' +#' Hooks register methods with the register_*() functions, or +#' define functions a spec field names. A hook may not replace a +#' method already registered: that would change the name's meaning +#' for everything else in the session. Re-sourcing the same file +#' is allowed. The files are kept in `sdm.hook_files`, so parallel +#' workers source them too and the run record lists them. +#' +#' @param hooks Character or NULL. A folder (its `.R` files, in +#' file-name order) or a vector of `.R` files. +#' @param envir Environment to source into. +#' @return The hook files, invisibly. +#' +#' @example # Example usage of the function +#' # load_hooks("1_code/experiments/exp_003_x/hooks") +load_hooks <- function(hooks, envir = parent.frame()) { + if (is.null(hooks) || length(hooks) == 0) { + options(sdm.hook_files = NULL) + return(invisible(character(0))) + } + + files <- unlist(lapply(hooks, function(one) { + if (dir.exists(one)) { + sort(list.files(one, pattern = "\\.R$", full.names = TRUE)) + } else { + one + } + })) + + absent <- files[!file.exists(files)] + + if (length(absent) > 0) { + stop( + "Hook file(s) not found:\n ", + paste(absent, collapse = "\n "), + call. = FALSE + ) + } + + files <- normalizePath(files, winslash = "/") + + # register_method() reads this to refuse a clash; see registry.R + for (path in files) { + options(sdm.hook_loading = path) + tryCatch( + source(path, local = envir), + error = function(e) { + message <- conditionMessage(e) + + # A clash already names the hook + if (!startsWith(message, "Hook ")) { + message <- paste0("Hook ", basename(path), " failed: ", + message) + } + + stop(message, call. = FALSE) + }, + finally = options(sdm.hook_loading = NULL) + ) + } + + options(sdm.hook_files = files) + + invisible(files) +} + +# 7. source_all() ---- #' Source Files in Order, Failing Clearly on a Missing One #' diff --git a/1_code/harness/model_sets.R b/1_code/harness/model_sets.R index db11d9a..0df2dbe 100644 --- a/1_code/harness/model_sets.R +++ b/1_code/harness/model_sets.R @@ -432,6 +432,91 @@ models_union <- function(models, sets = model_sets()) { paste0(". ~ . + ", paste(terms, collapse = " + ")) } +## 3.4 drop_candidates() ---- + +#' Drop Every Candidate That Uses Any of the Given Terms +#' +#' The opposite of extend_models(), one of two ways: the remaining +#' candidates stay as written, so the set is smaller. A term +#' counts as used wherever it appears in a formula, including +#' inside an interaction or I(). The intercept-only candidate +#' (". ~ .") is kept when the set has it. A staged set (a list of +#' groups, e.g. bird landcover) is reduced group by group; check +#' the result with lengths(), as an empty group leaves a staged +#' rule nothing to step through. +#' +#' @param models A model_sets() name, formulas as text, or a list +#' of groups of them (a staged set). +#' @param terms Character vector of covariate names. +#' @param sets Named list of model sets. +#' @return The candidates that use none of `terms`, in the same +#' shape as `models`. +#' +#' @example # Example usage of the function +#' # drop_candidates("climate_plant_v2_full", +#' # c("Easting", "Northing")) +drop_candidates <- function(models, terms, sets = model_sets()) { + models <- get_model_set(models, sets) + + # Staged sets: each group on its own + if (is.list(models)) { + return(lapply(models, drop_candidates, terms = terms, + sets = sets)) + } + + uses <- vapply(models, function(formula) { + any(all.vars(stats::as.formula(formula)) %in% terms) + }, logical(1)) + + unname(models[!uses]) +} + +## 3.5 strip_terms() ---- + +#' Remove the Given Terms from Every Candidate +#' +#' The other way to reduce a set: every candidate stays, less the +#' terms. Interactions and I() terms that use a removed covariate +#' go too. Candidates left identical are merged, and a candidate +#' left with no terms becomes the intercept-only ". ~ .". Staged +#' sets are reduced group by group. +#' +#' @param models A model_sets() name, formulas as text, or a list +#' of groups of them (a staged set). +#' @param terms Character vector of covariate names. +#' @param sets Named list of model sets. +#' @return The reduced candidates, in the same shape as `models`. +#' +#' @example # Example usage of the function +#' # strip_terms("climate_plant_v2_full", "bio15") +strip_terms <- function(models, terms, sets = model_sets()) { + models <- get_model_set(models, sets) + + # Staged sets: each group on its own + if (is.list(models)) { + return(lapply(models, strip_terms, terms = terms, sets = sets)) + } + + reduced <- vapply(models, function(formula) { + labels <- setdiff(attr( + stats::terms(stats::as.formula(formula), allowDotAsName = TRUE), + "term.labels" + ), ".") + + keep <- labels[!vapply(labels, function(label) { + any(all.vars(stats::as.formula(paste("~", label))) %in% terms) + }, logical(1))] + + if (length(keep) == 0) { + ". ~ ." + } else { + paste0(". ~ . + ", paste(keep, collapse = " + ")) + } + }, character(1)) + + unique(unname(reduced)) +} + # 4. get_model_set() ---- #' Look Up One Model Set diff --git a/1_code/harness/new_experiment.R b/1_code/harness/new_experiment.R index 4e5752a..16ec9c4 100644 --- a/1_code/harness/new_experiment.R +++ b/1_code/harness/new_experiment.R @@ -275,7 +275,10 @@ fill_template <- function(template_dir, code_dir, id, number, slug, "[YYYY-MM-DD]" = format(Sys.Date(), "%Y-%m-%d") ) - files <- list.files(template_dir, recursive = TRUE) + # all.files: hooks/.gitkeep keeps the empty hooks/ folder + files <- list.files( + template_dir, recursive = TRUE, all.files = TRUE + ) for (file in files) { text <- readLines(file.path(template_dir, file), warn = FALSE) diff --git a/1_code/harness/registry.R b/1_code/harness/registry.R index f8e6c73..ae3dd33 100644 --- a/1_code/harness/registry.R +++ b/1_code/harness/registry.R @@ -9,6 +9,9 @@ # here by name, so adding a method is one new file in # 1_code/methods/ with no harness change. Re-sourcing a method # file replaces its entry. +# - An experiment's hooks/ register here too (see load_hooks() +# in harness.R), but may not replace a method another file +# registered. # - Selection rules declare the engine capabilities they need; # validate_spec() (spec.R) checks them before any data is read. # --- @@ -83,6 +86,8 @@ engine_capabilities <- function() { #' @param description Character. One line on what it does. #' @param requires Character vector of engine capabilities the #' method needs. Used by selection rules. +#' @param direction Character or NULL. For metrics: which way is +#' better; see register_metric(). #' @return The name, invisibly. #' #' @example # Example usage of the function @@ -92,7 +97,8 @@ register_method <- function( name, value, description, - requires = character(0) + requires = character(0), + direction = NULL ) { if (!kind %in% names(method_kinds())) { stop( @@ -125,11 +131,34 @@ register_method <- function( } entries <- get(kind, envir = .sdm_registry) + + # A hook may re-register only its own names (load_hooks()) + hook <- getOption("sdm.hook_loading") + + if (!is.null(hook) && name %in% names(entries) && + !identical(entries[[name]]$source, hook)) { + stop( + "Hook ", basename(hook), " registers ", method_kinds()[[kind]], + " `", name, "`, which ", + if (is.null(entries[[name]]$source)) { + "is built in" + } else { + paste0("hook ", basename(entries[[name]]$source), + " already registered") + }, + ". Give it another name.", + call. = FALSE + ) + } + entries[[name]] <- list( name = name, value = value, description = description, - requires = requires + requires = requires, + direction = direction, + # NULL for a built-in method; the file for a hook + source = hook ) assign(kind, entries, envir = .sdm_registry) @@ -260,11 +289,18 @@ register_resampler <- function(name, fn, description) { #' @param name Character. The name metrics are reported under. #' @param fn The metric function. #' @param description Character. One line. +#' @param direction Character or NULL. Which way is better: +#' "higher", "lower", "toward_one" or "none". NULL keeps the +#' built-in table in better_direction() (compare.R), which +#' treats an unknown metric as "higher"; a hook metric should +#' state it. #' @return The name, invisibly. #' #' @example # Example usage of the function -#' # register_metric("rmse", metric_rmse, "Root mean squared error") -register_metric <- function(name, fn, description) { +#' # register_metric("mae", metric_mae, "Mean absolute error", +#' # direction = "lower") +register_metric <- function(name, fn, description, + direction = NULL) { if (!all(c("observed", "predicted") %in% names(formals(fn)))) { stop( "Metric `", name, "` must take `observed` and `predicted`.", @@ -272,7 +308,28 @@ register_metric <- function(name, fn, description) { ) } - register_method("metric", name, fn, description) + allowed <- c("higher", "lower", "toward_one", "none") + + if (!is.null(direction) && + (length(direction) != 1L || !direction %in% allowed)) { + stop( + "Metric `", name, "`: `direction` must be one of ", + paste(allowed, collapse = ", "), ".", + call. = FALSE + ) + } + + if (is.null(direction) && !is.null(getOption("sdm.hook_loading"))) { + warning( + "Hook metric `", name, "` has no `direction`, so comparisons ", + "treat higher as better. Pass direction = \"lower\" if not.", + call. = FALSE + ) + } + + register_method( + "metric", name, fn, description, direction = direction + ) } ## 2.8 unregister_method() ---- diff --git a/1_code/harness/resample.R b/1_code/harness/resample.R index 4503358..a8cecee 100644 --- a/1_code/harness/resample.R +++ b/1_code/harness/resample.R @@ -126,7 +126,9 @@ with_seed <- function(seed, code) { #' Cut Survey Units into Coarse Spatial Blocks #' -#' Default cut points are the v2 plant grid. +#' Default cut points are the v2 plant grid (v2_block_breaks()). +#' An experiment sets others as `long_breaks` and `lat_breaks` in +#' a spec's `resample` list. #' #' @param long,lat Numeric vectors of coordinates. #' @param long_breaks,lat_breaks Numeric vectors of cut points. @@ -140,8 +142,8 @@ with_seed <- function(seed, code) { spatial_blocks <- function( long, lat, - long_breaks = c(-121, -116, -112, -109), - lat_breaks = c(48, 51, 54, 57, 61) + long_breaks = v2_block_breaks()$long, + lat_breaks = v2_block_breaks()$lat ) { cells <- interaction( droplevels(cut(long, long_breaks)), @@ -161,6 +163,23 @@ spatial_blocks <- function( labels[match(cells, present)] } +## 3.1 v2_block_breaks() ---- + +#' The v2 Plant Grid's Block Cut Points +#' +#' The one copy of the defaults every blocked resampler uses. +#' +#' @return A list of `long` and `lat` numeric vectors. +#' +#' @example # Example usage of the function +#' # v2_block_breaks()$lat +v2_block_breaks <- function() { + list( + long = c(-121, -116, -112, -109), + lat = c(48, 51, 54, 57, 61) + ) +} + # 4. resample_for_species() ---- diff --git a/1_code/harness/run_model.R b/1_code/harness/run_model.R index 9c1c8de..1f20c85 100644 --- a/1_code/harness/run_model.R +++ b/1_code/harness/run_model.R @@ -260,6 +260,75 @@ prepare_spec <- function( #' # data_dir) prepare_region <- function(spec, region, species, data_dir) { region_spec <- spec$regions[[region]] + + # Steps 1-2: The resolved stages and the columns to load + request <- region_request(spec, region, data_dir) + spec <- request$spec + rename <- request$rename + weight_column <- request$weight_column + covariates <- request$covariates + + # Step 3: Load once for the region; species are cut from it + model_data <- build_model_data( + taxon = spec$taxon, + data_dir = data_dir, + species = species, + covariates = covariates, + region = region, + region_filter = region_spec$filter, + weight_column = weight_column, + aliases = spec$aliases %||% list(), + covariate_files = spec$covariate_files + ) + + # Step 4: The grid (the spec's, else the manifest's). Columns + # are renamed to the data's names; row names stay as v2 wrote + # them. + grid_name <- region_spec$grid %||% + manifest_entry(data_dir, spec$taxon, region)$grid + grid <- load_prediction_grid(data_dir, grid_name) + + if (!is.null(grid)) { + names(grid) <- apply_term_map(names(grid), rename) + + # e.g. v2 drops the mammal north's WetlandMargin and Climate + grid <- grid[ + !rownames(grid) %in% region_spec$grid_drop_rows, + !names(grid) %in% region_spec$grid_drop_cols, + drop = FALSE + ] + } + + list( + name = region, + spec = spec, + species = species, + model_data = model_data, + grid = grid, + grid_name = grid_name, + weight_column = weight_column, + covariates = covariates + ) +} + +## 3.2.1 region_request() ---- + +#' Resolve One Region's Stages and the Columns They Need +#' +#' Shared by prepare_region() and the configuration check +#' (check_spec_covariates()), so both resolve names the same way. +#' Reads only lookups, not data. +#' +#' @param spec A taxon spec. +#' @param region Character. +#' @param data_dir Character. +#' @return A list of the spec (with resolved stages), the term +#' map, the weight column and the covariates to load. +#' +#' @example # Example usage of the function +#' # region_request(lichen_spec(), "north", data_dir)$covariates +region_request <- function(spec, region, data_dir) { + region_spec <- spec$regions[[region]] rename <- term_map( data_dir, spec$taxon, region, block = region_spec$term_block ) @@ -301,44 +370,9 @@ prepare_region <- function(spec, region, species, data_dir) { spec, spec_covariates(spec), region_spec$filter ) - # Step 3: Load once for the region; species are cut from it - model_data <- build_model_data( - taxon = spec$taxon, - data_dir = data_dir, - species = species, - covariates = covariates, - region = region, - region_filter = region_spec$filter, - weight_column = weight_column, - aliases = spec$aliases %||% list(), - covariate_files = spec$covariate_files - ) - - # Step 4: The grid (the spec's, else the manifest's). Columns - # are renamed to the data's names; row names stay as v2 wrote - # them. - grid_name <- region_spec$grid %||% - manifest_entry(data_dir, spec$taxon, region)$grid - grid <- load_prediction_grid(data_dir, grid_name) - - if (!is.null(grid)) { - names(grid) <- apply_term_map(names(grid), rename) - - # e.g. v2 drops the mammal north's WetlandMargin and Climate - grid <- grid[ - !rownames(grid) %in% region_spec$grid_drop_rows, - !names(grid) %in% region_spec$grid_drop_cols, - drop = FALSE - ] - } - list( - name = region, spec = spec, - species = species, - model_data = model_data, - grid = grid, - grid_name = grid_name, + rename = rename, weight_column = weight_column, covariates = covariates ) @@ -348,10 +382,12 @@ prepare_region <- function(spec, region, species, data_dir) { #' Assemble the Province-Wide Data a Spec's Province Scope Needs #' -#' v2 fits plant and bird climate stages, and draws the plant -#' bootstrap, province-wide; only habitat is per region. Specs -#' declare this with `scope = "province"`. Loads the unfiltered -#' data once per spec. +#' v2 fits plant, bird and mammal climate stages, and draws the +#' plant bootstrap, province-wide; only habitat is per region. +#' Specs declare this with `scope = "province"`. Loads the +#' unfiltered data once per spec: the taxon's own rows, or its +#' separate province table where the manifest names one (the +#' mammal climate stage; province_region()). #' #' @param spec A taxon spec, with stage models already resolved. #' @param data_dir Character. @@ -362,13 +398,57 @@ prepare_region <- function(spec, region, species, data_dir) { #' @example # Example usage of the function #' # province_context(spec, data_dir, c("Physcia.adscendens")) province_context <- function(spec, data_dir, species) { + request <- province_request(spec) + + if (is.null(request) || length(species) == 0) { + return(NULL) + } + + # Companion responses (e.g. the other season) load alongside + companions <- if (is.function(spec$response_companions)) { + unlist(lapply(species, spec$response_companions), + use.names = FALSE) + } else { + character(0) + } + + list( + model_data = build_model_data( + taxon = spec$taxon, + data_dir = data_dir, + species = unique(c(species, companions)), + covariates = request$covariates, + region = province_region(data_dir, spec$taxon), + region_filter = NULL, + weight_column = NULL, + aliases = spec$aliases %||% list(), + covariate_files = spec$covariate_files + ), + stages = request$stages, + draws = request$draws + ) +} + +## 3.3.1 province_request() ---- + +#' The Province-Scope Stages and Draws, and the Columns They Need +#' +#' Shared by province_context() and the configuration check +#' (check_spec_covariates()). +#' +#' @param spec A taxon spec. +#' @return A list of `stages`, `draws` and `covariates`, or NULL +#' when the spec has no province scope. +#' +#' @example # Example usage of the function +#' # province_request(lichen_spec())$covariates +province_request <- function(spec) { province_stages <- Filter( function(s) identical(s$scope, "province"), spec$stages ) province_draws <- identical(spec$resample$scope, "province") - if ((length(province_stages) == 0 && !province_draws) || - length(species) == 0) { + if (length(province_stages) == 0 && !province_draws) { return(NULL) } @@ -383,9 +463,15 @@ province_context <- function(spec, data_dir, species) { } } + # Not the spec's weight column: province frames are unweighted + # (province_context()), and a separate province table need not + # carry it stage_names <- vapply(province_stages, `[[`, character(1), "name") covariates <- if (length(stage_names) > 0) { - spec_covariates(spec, stage_names = stage_names) + setdiff( + spec_covariates(spec, stage_names = stage_names), + spec$weight_column + ) } else { character(0) } @@ -395,19 +481,9 @@ province_context <- function(spec, data_dir, species) { covariates <- covariate_request(spec, covariates, filters) list( - model_data = build_model_data( - taxon = spec$taxon, - data_dir = data_dir, - species = species, - covariates = covariates, - region = NULL, - region_filter = NULL, - weight_column = NULL, - aliases = spec$aliases %||% list(), - covariate_files = spec$covariate_files - ), stages = province_stages, - draws = province_draws + draws = province_draws, + covariates = covariates ) } @@ -415,8 +491,9 @@ province_context <- function(spec, data_dir, species) { #' The Columns to Load, Given the Covariates the Formulas Name #' -#' Adds the stored-draw key and region-filter columns; drops -#' aliases and site fields, which do not come from +#' Adds the stored-draw key, the province key (which maps +#' province draws to region units) and region-filter columns; +#' drops aliases and site fields, which do not come from #' covariates.csv. #' #' @param spec A taxon spec. @@ -442,6 +519,7 @@ covariate_request <- function(spec, covariates, filters = NULL) { unique(c( covariates, spec$resample$id_column, + spec$province_key, setdiff(filter_terms, c(site_columns(), "survey_unit_id")) )) } @@ -479,11 +557,14 @@ run_jobs <- function(jobs, prepared, workers = 1L, verbose = TRUE) { "called, so workers can load it too.", call. = FALSE) } - # Step 1: Start the workers and load the framework + # Step 1: Start the workers and load the framework, with the + # experiment's hooks, so its own methods are known there too cluster <- parallel::makePSOCKcluster(min(workers, length(jobs))) on.exit(parallel::stopCluster(cluster), add = TRUE) - parallel::clusterCall(cluster, load_worker, project_root) + parallel::clusterCall( + cluster, load_worker, project_root, getOption("sdm.hook_files") + ) # Step 2: Each job carries only its own spec's prepared data tasks <- lapply(jobs, function(job) { @@ -498,11 +579,12 @@ run_jobs <- function(jobs, prepared, workers = 1L, verbose = TRUE) { #' Load the Framework in a Background Session #' #' @param project_root Character. The repository root. +#' @param hooks Character or NULL. The experiment's hook files. #' @return NULL, invisibly. #' #' @example # Example usage of the function -#' # parallel::clusterCall(cluster, load_worker, root) -load_worker <- function(project_root) { +#' # parallel::clusterCall(cluster, load_worker, root, NULL) +load_worker <- function(project_root, hooks = NULL) { session <- globalenv() source( @@ -510,7 +592,7 @@ load_worker <- function(project_root) { local = session ) load <- get("load_framework", envir = session) - load(project_root, envir = session) + load(project_root, envir = session, hooks = hooks) invisible(NULL) } @@ -564,7 +646,10 @@ fit_species <- function(species, p) { if (!is.null(p$province)) { province <- list(frame = model_frame( p$province$model_data, species = species, - transform = spec$response_transform %||% "identity" + transform = spec$response_transform %||% "identity", + companions = if (is.function(spec$response_companions)) { + spec$response_companions(species) + } )) if (p$province$draws) { @@ -600,8 +685,15 @@ fit_species <- function(species, p) { } # Step 2: The draws. Province-wide draws are filtered to the - # region's units when fitted, as v2 subsets them. - draws <- if (!is.null(province$draws)) { + # region's units when fitted, as v2 subsets them; where the + # province table has its own unit ids, they are first mapped + # to the region's units through `province_key`. + draws <- if (is.list(province$draws) && + !is.null(spec$province_key)) { + map_province_draws( + province$draws, province$frame, frame, spec$province_key + ) + } else if (!is.null(province$draws)) { province$draws } else { tryCatch( @@ -628,9 +720,17 @@ fit_species <- function(species, p) { if (!is.null(province) && length(p$province$stages) > 0) { key <- as.character(boot) + # On the province draw, not the region's mapped copy (the + # same draw unless `province_key` maps it) + province_draw <- if (is.list(province$draws)) { + province$draws[[i]] + } else { + draws[[i]] + } + if (is.null(province_fits[[key]])) { province_fits[[key]] <<- fit_province_stages( - p$province$stages, province$frame, draws[[i]], spec, + p$province$stages, province$frame, province_draw, spec, species ) } @@ -683,6 +783,52 @@ fit_province_stages <- function(stages, frame, draw, spec, species) { stats::setNames(fits, vapply(stages, `[[`, character(1), "name")) } +## 4.6 map_province_draws() ---- + +#' Map Province Draws onto a Region's Own Units +#' +#' For taxa whose province table has its own unit ids (the mammal +#' climate rows), a shared key column links the two. Each region +#' unit stands for the first province unit with its key, and is +#' drawn as often as that unit is. Region units whose key the +#' province table lacks are never drawn, as v2's mammal habitat +#' models drop deployments with no climate prediction. +#' +#' @param draws List of character vectors of province unit ids. +#' @param province_frame The species' province-wide frame. +#' @param region_frame The species' region frame. +#' @param key Character. The column both frames carry. +#' @return A list of character vectors of region unit ids, in +#' region frame order, repeats kept. Draw 1 (every province +#' unit) gives each mapped region unit once. +#' +#' @example # Example usage of the function +#' # map_province_draws(draws, province$frame, frame, +#' # "location_project") +map_province_draws <- function(draws, province_frame, region_frame, + key) { + for (one in list(province_frame, region_frame)) { + if (!key %in% names(one)) { + stop("Province key `", key, "` is not in the frame.", + call. = FALSE) + } + } + + province_ids <- as.character(province_frame$survey_unit_id) + stands_for <- province_ids[match( + as.character(region_frame[[key]]), + as.character(province_frame[[key]]) + )] + region_ids <- as.character(region_frame$survey_unit_id) + + lapply(draws, function(draw) { + counts <- table(factor(draw, levels = unique(draw))) + n <- as.integer(counts[match(stands_for, names(counts))]) + n[is.na(n)] <- 0L + rep(region_ids, n) + }) +} + # 5. Fitting one draw ---- ## 5.1 run_stage() ---- @@ -1186,6 +1332,12 @@ run_meta <- function(p, region) { grid = region$grid_name %||% NA_character_, unit_predictions = p$unit_predictions, models_overridden = spec$models_overridden %||% NA_character_, + # The experiment's own methods, when it has hooks + hooks = if (is.null(getOption("sdm.hook_files"))) { + NA_character_ + } else { + basename(getOption("sdm.hook_files")) + }, v2_coverage = if (is.null(spec$v2_coverage)) { NA_character_ } else { diff --git a/1_code/harness/run_record.R b/1_code/harness/run_record.R index 33d53f7..899f30a 100644 --- a/1_code/harness/run_record.R +++ b/1_code/harness/run_record.R @@ -161,6 +161,8 @@ kv_lines <- function(values, digits = 4L) { #' @param digits Integer. Decimal places for every number. #' @param collected collect_results() output for this pipeline #' directory, or NULL to read the stores here. +#' @param hook_files Character or NULL. The experiment's hook +#' files; recorded with their md5 under Provenance. #' @return The path written, invisibly. #' #' @example # Example usage of the function @@ -175,7 +177,8 @@ run_record <- function( config = list(), provenance = TRUE, digits = 4L, - collected = NULL + collected = NULL, + hook_files = NULL ) { if (is.null(collected)) { collected <- collect_results(pipeline_dir) @@ -193,11 +196,22 @@ run_record <- function( ## 3.1 Provenance ---- if (provenance) { - commit <- tryCatch( - system2("git", c("rev-parse", "--short", "HEAD"), - stdout = TRUE, stderr = FALSE), - error = function(e) NA_character_, - warning = function(e) NA_character_ + git <- function(args) { + tryCatch( + system2("git", args, stdout = TRUE, stderr = FALSE), + error = function(e) NA_character_, + warning = function(e) NA_character_ + ) + } + commit <- git(c("rev-parse", "HEAD")) + # Code that differs from the commit, which then does not + # fully describe the run + changed <- git(c("status", "--porcelain", "--", "1_code")) + + # Versions of the modelling packages this session loaded + packages <- intersect( + c("data.table", "arm", "xgboost", "mgcv", "MuMIn"), + loadedNamespaces() ) lines <- c( @@ -210,10 +224,37 @@ run_record <- function( written_at = format( Sys.time(), "%Y-%m-%d %H:%M:%S", tz = "UTC" ), - git_commit = if (length(commit) == 1) commit else "-", + git_commit = if (length(commit) == 1 && !is.na(commit)) { + commit + } else { + "-" + }, + git_uncommitted_code = if (anyNA(changed)) { + "-" + } else if (length(changed) > 0) { + paste(length(changed), "file(s) under 1_code/ differ") + } else { + "none" + }, r_version = paste( R.version$major, R.version$minor, sep = "." ), + platform = R.version$platform, + packages = if (length(packages) == 0) { + "-" + } else { + versions <- vapply(packages, function(p) { + as.character(utils::packageVersion(p)) + }, character(1)) + paste(packages, versions) + }, + hooks = if (length(hook_files) == 0) { + "none" + } else { + paste0(basename(hook_files), " (md5 ", + unname(tools::md5sum(hook_files)), ")") + }, + session_info = file.path(pipeline_dir, "session_info.txt"), pipeline_dir = pipeline_dir ), digits), "" diff --git a/1_code/harness/spec.R b/1_code/harness/spec.R index 9f4ba9c..233b616 100644 --- a/1_code/harness/spec.R +++ b/1_code/harness/spec.R @@ -12,6 +12,9 @@ # data is read. Unknown fields are errors, because a misspelt # field is otherwise silently ignored; arguments of the named # rule or resampler count as known fields. +# - Experiments change a v2 spec with replace_stage_method(), +# modify_stage(), modify_region() and modify_resample(), never +# by editing a module; each records what it changed. # --- # 1. Setup ---- @@ -43,6 +46,10 @@ spec_fields <- function() { "notes", "models_overridden", "covariate_files", # The stage a methods experiment replaces by default "habitat_stage", + # Where the province table has its own unit ids: the column + # that maps its draws to region units, and other response + # columns its frame carries (the mammal climate stage) + "province_key", "response_companions", # Recorded facts a module keeps for its own reports "use_protocol", "protocol_is_v2", "climate_source" ), @@ -124,6 +131,29 @@ validate_spec <- function(spec, stop_on_error = TRUE) { } } + # Step 2b: The province table's key and companions + if (!is.null(spec$province_key)) { + if (!is.character(spec$province_key) || + length(spec$province_key) != 1L) { + problem(label, ": `province_key` must be one column name.") + } + + if (!identical(spec$resample$scope, "province")) { + problem( + label, ": `province_key` maps province-wide draws, so ", + "`resample$scope` must be \"province\"." + ) + } + } + + if (!is.null(spec$response_companions) && + !is.function(spec$response_companions)) { + problem( + label, ": `response_companions` must be a function of the ", + "species, returning named response columns." + ) + } + # Step 3: Stages, in order stage_names <- character(0) carried <- character(0) @@ -362,6 +392,117 @@ spec_covariates <- function(spec, stage_names = NULL) { unique(c(covariates, extra, spec$weight_column)) } +## 4.1 check_spec_covariates() ---- + +#' Find Covariates a Spec Names That the Data Does Not Hold +#' +#' Resolves names as a run does (region_request() and +#' province_request() in run_model.R), then checks the taxon's +#' covariate catalogue and the experiment's covariate_files. Reads +#' lookups only, so it is quick enough to run when an experiment +#' is configured. +#' +#' @param spec A taxon spec, with any stage_models applied. +#' @param data_dir Character. The test dataset folder. +#' @param regions Character vector of regions to check, or NULL +#' for every region the spec defines. +#' @param label Character. The run's name, for messages. +#' @return A character vector of problems, empty when every +#' covariate is found. +#' +#' @example # Example usage of the function +#' # check_spec_covariates(lichen_spec(), data_dir) +check_spec_covariates <- function(spec, data_dir, regions = NULL, + label = spec$taxon) { + problems <- character(0) + file_columns <- covariate_file_columns(spec$covariate_files) + + check <- function(columns, region) { + where <- paste0( + "Run `", label, "`", + if (is.null(region)) " (province-wide)" else + paste0(" (", region, ")") + ) + columns <- setdiff(unique(columns), file_columns) + + tryCatch({ + unknown <- covariate_columns_needed( + spec$taxon, data_dir, columns, region + )$unknown + available <- available_covariates( + data_dir, spec$taxon, region = region + ) + + for (one in unknown) { + close <- available[agrep( + one, available, max.distance = 0.2, ignore.case = TRUE + )] + problems <<- c(problems, paste0( + where, ": covariate `", one, "` is not in the dataset ", + "or covariate_files", + if (length(close) > 0) { + paste0("; did you mean ", + paste(utils::head(close, 3), collapse = ", "), "?") + } else { + "." + } + )) + } + }, error = function(e) { + problems <<- c( + problems, paste0(where, ": ", conditionMessage(e)) + ) + }) + } + + for (region in regions %||% names(spec$regions)) { + request <- tryCatch( + region_request(spec, region, data_dir), + error = function(e) { + problems <<- c(problems, paste0( + "Run `", label, "` (", region, "): ", conditionMessage(e) + )) + NULL + } + ) + + if (!is.null(request)) { + check(c(request$covariates, request$weight_column), region) + } + } + + province <- tryCatch( + province_request(spec), + error = function(e) { + problems <<- c(problems, paste0( + "Run `", label, "`: ", conditionMessage(e) + )) + NULL + } + ) + + if (!is.null(province)) { + # NULL for most taxa; "province" where the manifest names a + # separate province table + region <- tryCatch( + province_region(data_dir, spec$taxon), error = function(e) NULL + ) + + if (!is.null(spec$province_key) && is.null(region)) { + problems <- c(problems, paste0( + "Run `", label, "`: the spec reads a separate province ", + "table (`province_key`), but this test dataset has no ", + "`province` row for ", spec$taxon, " in its manifest. ", + "Use a dataset version that has one." + )) + } else { + check(province$covariates, region) + } + } + + unique(problems) +} + # 5. apply_stage_models() ---- #' Swap a Spec's Candidate Sets for an Experiment's Own @@ -549,6 +690,210 @@ replace_stage_method <- function( spec } +## 6.1 modify_stage() ---- + +#' Change Fields of One Stage of a Spec, in an Experiment +#' +#' How an experiment changes a v2 setting without editing the +#' taxon module: the module stays the record of v2, and the change +#' is recorded in `models_overridden` (so experiment_changes(), +#' meta.json and the run record show it) and in `v2_coverage`. +#' The spec is validated, so a misspelt field fails here. +#' +#' @param spec A taxon spec. +#' @param stage Character. The stage's name. +#' @param ... Named fields to set; NULL removes a field. Fields a +#' stage's rule takes as arguments are accepted too. +#' @return The spec. +#' +#' @example # Example usage of the function +#' # spec <- modify_stage(lichen_spec(), "habitat", +#' # min_detections = 10L) +modify_stage <- function(spec, stage, ...) { + fields <- spec_change_fields(list(...)) + names_in <- vapply(spec$stages, `[[`, character(1), "name") + index <- match(stage, names_in) + + if (length(stage) != 1L || is.na(index)) { + stop( + "Spec `", spec$taxon, "` has no stage `", stage, "`. Stages: ", + paste(names_in, collapse = ", "), + call. = FALSE + ) + } + + for (field in names(fields)) { + spec$stages[[index]][field] <- list(fields[[field]]) + } + + changes <- describe_spec_change(stage, fields) + spec <- record_spec_change(spec, changes, coverage_key = stage) + + validate_spec(spec) +} + +## 6.2 modify_region() ---- + +#' Change Fields of One Region of a Spec, in an Experiment +#' +#' As modify_stage(), for a region: its `filter`, `grid`, +#' `grid_constants`, `habitat_models` and the rest of +#' spec_fields()$region. +#' +#' @param spec A taxon spec. +#' @param region Character. The region's name. +#' @param ... Named fields to set; NULL removes a field. +#' @return The spec. +#' +#' @example # Example usage of the function +#' # spec <- modify_region(lichen_spec(), "north", +#' # filter = ~ nr == "Boreal") +modify_region <- function(spec, region, ...) { + fields <- spec_change_fields(list(...)) + + if (length(region) != 1L || !region %in% names(spec$regions)) { + stop( + "Spec `", spec$taxon, "` has no region `", region, + "`. Regions: ", paste(names(spec$regions), collapse = ", "), + call. = FALSE + ) + } + + for (field in names(fields)) { + spec$regions[[region]][field] <- list(fields[[field]]) + } + + changes <- describe_spec_change(paste("region", region), fields) + spec <- record_spec_change(spec, changes) + + validate_spec(spec) +} + +## 6.3 modify_resample() ---- + +#' Change a Spec's Resampling, in an Experiment +#' +#' As modify_stage(), for `spec$resample`: the scheme and its +#' settings (e.g. `min_detections`, `long_breaks`, `lat_breaks`). +#' +#' @param spec A taxon spec. +#' @param ... Named settings; NULL removes one. +#' @param replace Logical. Drop the current settings first, keeping +#' only `scope` unless it is given. Use it to switch scheme, as +#' the old scheme's settings are not the new one's. +#' @return The spec. +#' +#' @example # Example usage of the function +#' # spec <- modify_resample(lichen_spec(), scheme = "spatial_cv", +#' # replace = TRUE) +modify_resample <- function(spec, ..., replace = FALSE) { + fields <- spec_change_fields(list(...)) + + if (replace) { + spec$resample <- spec$resample["scope"] + spec$resample <- spec$resample[ + !vapply(spec$resample, is.null, logical(1)) + ] + } + + for (field in names(fields)) { + spec$resample[field] <- list(fields[[field]]) + } + + changes <- describe_spec_change( + if (replace) "resample (replaced)" else "resample", fields + ) + spec <- record_spec_change( + spec, changes, coverage_key = "resampling" + ) + + validate_spec(spec) +} + +## 6.4 spec_change_fields() ---- + +#' Check the Fields Passed to a modify_*() Function +#' +#' @param fields A list from `list(...)`. +#' @return The list, when every entry is named. +#' +#' @example # Example usage of the function +#' # spec_change_fields(list(ic = "BIC")) +spec_change_fields <- function(fields) { + if (length(fields) == 0) { + stop("Name at least one field to change.", call. = FALSE) + } + + if (is.null(names(fields)) || any(!nzchar(names(fields)))) { + stop( + "Every field to change must be named, e.g. ic = \"BIC\".", + call. = FALSE + ) + } + + fields +} + +## 6.5 describe_spec_change() ---- + +#' Describe Changed Fields for the Record +#' +#' @param where Character. What was changed, e.g. a stage name. +#' @param fields Named list of new values. +#' @return A character vector, one "where: field = value" each. +#' +#' @example # Example usage of the function +#' # describe_spec_change("habitat", list(ic = "BIC")) +describe_spec_change <- function(where, fields) { + values <- vapply(fields, function(value) { + text <- if (is.null(value)) { + "removed" + } else if (is.function(value)) { + "a custom function" + } else { + paste(deparse(value, width.cutoff = 500L), collapse = " ") + } + + if (nchar(text) > 80) paste0(substr(text, 1, 77), "...") else text + }, character(1)) + + paste0(where, ": ", names(fields), " = ", values) +} + +## 6.6 record_spec_change() ---- + +#' Record a Spec Change Where the Run Record Reads It +#' +#' @param spec A taxon spec. +#' @param changes Character vector from describe_spec_change(). +#' @param coverage_key Character or NULL. The `v2_coverage` entry +#' the change makes no longer v2's. +#' @return The spec. +#' +#' @example # Example usage of the function +#' # record_spec_change(spec, "habitat: ic = \"BIC\"", "habitat") +record_spec_change <- function(spec, changes, coverage_key = NULL) { + spec$models_overridden <- c(spec$models_overridden, changes) + + if (!is.null(coverage_key)) { + # Earlier changes to the same entry are kept + prior <- spec$v2_coverage[[coverage_key]] + earlier <- if (!is.null(prior) && + startsWith(prior$note, "Changed: ")) { + sub("\\.$", "; ", prior$note) + } else { + "Changed: " + } + + spec$v2_coverage[[coverage_key]] <- v2_status( + "not reproduced", + paste0(earlier, paste(changes, collapse = "; "), ".") + ) + } + + spec +} + # 7. v2 coverage ---- ## 7.1 v2_status() ---- diff --git a/1_code/harness/species.R b/1_code/harness/species.R index 56ced7b..726219a 100644 --- a/1_code/harness/species.R +++ b/1_code/harness/species.R @@ -237,4 +237,131 @@ taxon_values <- function(x, taxon) { out } +# 7. Experiment species files ---- + +## 7.1 read_species_file() ---- + +#' Read an Experiment's species.csv +#' +#' One row per species, with columns `taxon` (the data slug, e.g. +#' "mite") and `species`. Other columns, such as a `note` on why a +#' species was chosen, are ignored. The template lists every +#' species in the dataset; an experiment deletes the rows it does +#' not fit. A taxon the file does not name runs every species. +#' +#' @param path Character. The CSV. +#' @return A character vector of species named by taxon, as +#' taxon_values() reads. +#' +#' @example # Example usage of the function +#' # read_species_file( +#' # "1_code/experiments/exp_000_parity_v2/species.csv" +#' # ) +read_species_file <- function(path) { + if (!is.character(path) || length(path) != 1L || + !file.exists(path)) { + stop("Species file not found: ", path, call. = FALSE) + } + + rows <- fread(path, colClasses = "character", na.strings = "") + absent <- setdiff(c("taxon", "species"), names(rows)) + + if (length(absent) > 0) { + stop( + "Species file ", basename(path), " has no ", + paste0("`", absent, "`", collapse = " or "), " column. ", + "It needs `taxon` and `species`.", + call. = FALSE + ) + } + + blank <- which(is.na(rows$taxon) | is.na(rows$species) | + !nzchar(trimws(rows$taxon)) | + !nzchar(trimws(rows$species))) + + if (length(blank) > 0) { + stop( + "Species file ", basename(path), " has blank `taxon` or ", + "`species` in data row(s) ", paste(blank, collapse = ", "), ".", + call. = FALSE + ) + } + + keys <- paste(trimws(rows$taxon), trimws(rows$species)) + + if (anyDuplicated(keys)) { + stop( + "Species file ", basename(path), " repeats: ", + paste(unique(keys[duplicated(keys)]), collapse = ", "), ".", + call. = FALSE + ) + } + + stats::setNames(trimws(rows$species), trimws(rows$taxon)) +} + +## 7.2 check_species_names() ---- + +#' Find Species and Taxon Names the Dataset Does Not Hold +#' +#' The reference is the dataset's species queue, so every +#' experiment spells species the same way. Close matches are +#' suggested for a misspelt name. +#' +#' @param species A species vector named by taxon, or NULL. +#' @param data_dir Character. The test dataset folder. +#' @return A character vector of problems, empty when every name +#' is known. +#' +#' @example # Example usage of the function +#' # check_species_names(c(lichen = "Physcia.adscendns"), data_dir) +check_species_names <- function(species, data_dir) { + if (is.null(species) || length(species) == 0) { + return(character(0)) + } + + catalogue <- species_catalogue(data_dir) + taxa <- names(species) + + if (is.null(taxa) || any(is.na(taxa) | !nzchar(taxa))) { + return(paste0( + "Every species must be named by its taxon, e.g. ", + "c(lichen = \"Physcia.adscendens\"), or come from a ", + "species.csv." + )) + } + + problems <- character(0) + unknown_taxa <- setdiff(unique(taxa), catalogue$taxon) + + if (length(unknown_taxa) > 0) { + problems <- c(problems, paste0( + "Unknown taxon slug(s) in the species list: ", + paste(unknown_taxa, collapse = ", "), ". Slugs: ", + paste(unique(catalogue$taxon), collapse = ", "), "." + )) + } + + for (taxon in setdiff(unique(taxa), unknown_taxa)) { + available <- unique(catalogue$species[catalogue$taxon == taxon]) + + for (one in setdiff(species[taxa == taxon], available)) { + close <- available[agrep( + one, available, max.distance = 0.2, ignore.case = TRUE + )] + problems <- c(problems, paste0( + "Species `", one, "` is not in the ", taxon, " queue", + if (length(close) > 0) { + paste0("; did you mean ", + paste(utils::head(close, 3), collapse = ", "), "?") + } else { + ". See list_species()." + } + )) + } + } + + problems +} + # End of script ---- diff --git a/1_code/methods/README.md b/1_code/methods/README.md index 6c5788c..a2e2d10 100644 --- a/1_code/methods/README.md +++ b/1_code/methods/README.md @@ -1,10 +1,14 @@ # Methods -Every method a spec can name lives here, one file each. A file -defines its method and registers it under a name; a spec names it; -the harness looks it up. Adding a method is adding a file. No +Every built-in method a spec can name lives here, one file each. A +file defines its method and registers it under a name; a spec names +it; the harness looks it up. Adding a method is adding a file. No harness file changes. +A method added here is shared: every experiment sees it. A method +one experiment is testing goes in that experiment's `hooks/` folder +instead (see [Methods in an experiment's hooks](#methods-in-an-experiments-hooks)). + ```r source("1_code/harness/harness.R") load_framework() @@ -38,9 +42,15 @@ skeleton for an engine and a selection rule; it is not loaded. | selection | `hurdle` | Presence, then abundance given presence, each through an inner rule (`part_selection`); reports both and their product on the grid (v2 mammals) | those of `part_selection` | | resampler | `precomputed` | Replay draws stored with the dataset (v2 birds; plant `v2_ids`) | | | resampler | `spatial_block` | Bootstrap within coarse spatial blocks (v2 plants, mammals) | | -| resampler | `spatial_cv` | Spatial block cross-validation; each draw holds out one fold | | +| resampler | `spatial_cv` | Spatial block cross-validation; each draw holds out one fold, so `n_bootstraps` is the number of folds | | | metric | `auc`, `deviance_explained`, `rmse`, `spearman`, `calibration_slope`, `prevalence`, `n` | Scored in-sample and out-of-bag for every draw | | +Both blocked resamplers cut blocks at the v2 plant grid +(`v2_block_breaks()`) unless the spec's `resample` list sets +`long_breaks` and `lat_breaks`, e.g. +`modify_resample(spec, scheme = "spatial_cv", replace = TRUE, +lat_breaks = c(48, 50, 52, 54, 56, 58, 61))`. + `validate_spec()` checks a spec against this table before any data is read. A rule paired with an engine that lacks what it needs, an unregistered name, or a misspelt field stops the run with a @@ -137,7 +147,42 @@ species' draws depend only on its seed. **Metric.** A function of `observed` and `predicted`, both on the response scale, returning one number. Return `NA` rather than -stopping when the metric does not apply to a draw. +stopping when the metric does not apply to a draw. Register it with +a `direction` (`"higher"`, `"lower"`, `"toward_one"` or `"none"`) +so comparisons with the baseline know which way is better; without +one, higher is assumed. + +## Methods in an experiment's hooks + +An experiment tests a method of its own without touching this +folder. Put the file in the experiment's `hooks/`, register the +method there with the same `register_*()` call, and load it: + +```r +exp_dir <- "1_code/experiments/exp_003_my_question" +source("1_code/harness/harness.R") +load_framework(hooks = file.path(exp_dir, "hooks")) +``` + +```r +# hooks/mae.R +register_metric( + "mae", + function(observed, predicted) mean(abs(observed - predicted)), + "Mean absolute error", direction = "lower" +) +``` + +The contracts are the ones above. Parallel workers load the same +hook files. A hook may not reuse a name already registered, built +in or from another hook file: that would change the name's meaning +for everything else in the session. The run record lists each hook +file with its md5. A hook can also define a plain function that a +spec field names, such as a `response_transform` or a +`post_process` step, set with `modify_stage()`. + +When a hook method proves itself, move it here so every experiment +can use it. ## Checklist for a new method @@ -147,5 +192,5 @@ stopping when the metric does not apply to a draw. it `requires`. - [ ] `check_engine()` passes, for an engine. - [ ] `Rscript 1_code/tests/run_tests.R` passes. -- [ ] Tried on one species and a few draws (`species = "one_each"`, +- [ ] Tried on a few species and draws (a short `species.csv`, `n_bootstraps = 5`) before a full run. diff --git a/1_code/methods/resampling/spatial_block.R b/1_code/methods/resampling/spatial_block.R index 19717e0..d37d1d6 100644 --- a/1_code/methods/resampling/spatial_block.R +++ b/1_code/methods/resampling/spatial_block.R @@ -33,6 +33,8 @@ #' @param response Numeric vector, one per row of `frame`, used #' only for the detection check. #' @param max_attempts Integer. Redraws before failing. +#' @param long_breaks,lat_breaks Numeric vectors of block cut +#' points; the v2 plant grid by default (v2_block_breaks()). #' @param seed Integer or NULL. NULL leaves the stream unseeded, #' which is what v2 did. #' @return A list of character vectors, one per iteration. @@ -47,6 +49,8 @@ resample_spatial_block <- function( min_detections = 0L, response = NULL, max_attempts = 100L, + long_breaks = v2_block_breaks()$long, + lat_breaks = v2_block_breaks()$lat, seed = harness_seed() ) { for (one in c(long_column, lat_column)) { @@ -60,7 +64,8 @@ resample_spatial_block <- function( units <- as.character(frame$survey_unit_id) blocks <- spatial_blocks( - frame[[long_column]], frame[[lat_column]] + frame[[long_column]], frame[[lat_column]], + long_breaks = long_breaks, lat_breaks = lat_breaks ) by_block <- split(seq_along(units), blocks) @@ -116,6 +121,8 @@ resample_spatial_block <- function( #' @param context List with `spec` and `response_name`. #' @param min_detections,long_column,lat_column,max_attempts #' Settings from the spec's `resample` list. +#' @param long_breaks,lat_breaks Block cut points, also from +#' `resample`; the v2 plant grid by default. #' @return A list of character vectors of survey unit ids. #' #' @example # Example usage of the function @@ -124,7 +131,8 @@ resample_spatial_block <- function( resampler_spatial_block <- function( frame, iterations, seed, context, min_detections = 0L, long_column = "long", lat_column = "lat", - max_attempts = 100L + max_attempts = 100L, long_breaks = v2_block_breaks()$long, + lat_breaks = v2_block_breaks()$lat ) { response <- frame[[context$response_name]] @@ -156,7 +164,8 @@ resampler_spatial_block <- function( frame = frame, iterations = iterations, long_column = long_column, lat_column = lat_column, min_detections = min_detections, response = response, - max_attempts = max_attempts, seed = seed + max_attempts = max_attempts, long_breaks = long_breaks, + lat_breaks = lat_breaks, seed = seed ) } diff --git a/1_code/methods/resampling/spatial_cv.R b/1_code/methods/resampling/spatial_cv.R index 2d63b3f..7e9b243 100644 --- a/1_code/methods/resampling/spatial_cv.R +++ b/1_code/methods/resampling/spatial_cv.R @@ -25,6 +25,8 @@ #' coordinate columns. #' @param folds Integer. Number of folds. #' @param long_column,lat_column Character. Coordinate columns. +#' @param long_breaks,lat_breaks Numeric vectors of block cut +#' points; the v2 plant grid by default (v2_block_breaks()). #' @param seed Integer or NULL. Defaults to harness_seed(). #' @return A list of lists, each with `train` and `test` #' character vectors of survey unit ids. @@ -37,11 +39,14 @@ resample_spatial_cv <- function( folds = 5L, long_column = "long", lat_column = "lat", + long_breaks = v2_block_breaks()$long, + lat_breaks = v2_block_breaks()$lat, seed = harness_seed() ) { units <- as.character(frame$survey_unit_id) blocks <- spatial_blocks( - frame[[long_column]], frame[[lat_column]] + frame[[long_column]], frame[[lat_column]], + long_breaks = long_breaks, lat_breaks = lat_breaks ) present <- unique(blocks[!is.na(blocks)]) @@ -77,8 +82,9 @@ resample_spatial_cv <- function( #' folds. #' @param seed Integer or NULL. #' @param context Unused. -#' @param long_column,lat_column Settings from the spec's -#' `resample` list. +#' @param long_column,lat_column,long_breaks,lat_breaks Settings +#' from the spec's `resample` list; the cut points default to +#' the v2 plant grid. #' @return A list of character vectors: each fold's training #' units. #' @@ -86,11 +92,14 @@ resample_spatial_cv <- function( #' # resampler_spatial_cv(frame, 1:5, 42L, list()) resampler_spatial_cv <- function( frame, iterations, seed, context, - long_column = "long", lat_column = "lat" + long_column = "long", lat_column = "lat", + long_breaks = v2_block_breaks()$long, + lat_breaks = v2_block_breaks()$lat ) { splits <- resample_spatial_cv( frame, folds = length(iterations), long_column = long_column, - lat_column = lat_column, seed = seed + lat_column = lat_column, long_breaks = long_breaks, + lat_breaks = lat_breaks, seed = seed ) lapply(splits, `[[`, "train") diff --git a/1_code/modules/README.md b/1_code/modules/README.md index 7714b7e..271b4b1 100644 --- a/1_code/modules/README.md +++ b/1_code/modules/README.md @@ -5,6 +5,13 @@ code: it states what v2 does for a taxon as a spec list, and the [harness](../harness/) runs it. A change here affects every experiment; review it as such. +Experiments never edit a module to vary a taxon. They change the +spec the module returns, in their own `run.R`: module arguments +(`use_protocol`, `climate_source`, `tier`, `bootstrap`), +`replace_stage_method()`, or `modify_stage()`, `modify_region()` +and `modify_resample()`, which validate the change and record it. +A module stays the record of what v2 does. + `load_framework()` sources `_shared/` first, then every other folder, each in file-name order. diff --git a/1_code/modules/mammals/climate.R b/1_code/modules/mammals/climate.R new file mode 100644 index 0000000..33a4453 --- /dev/null +++ b/1_code/modules/mammals/climate.R @@ -0,0 +1,200 @@ +# --- +# title: The v2 Mammal Climate Stage +# author: Brendan Casey; model logic by Marcus Becker (v2) +# created: 2026-10-14 +# inputs: +# in the test dataset (see harness/data_source.R), the +# mammal_province rows of mammal.csv, sites.csv and +# covariates.csv: v2's combined 2022 table, which v2's climate +# stage fits on (_setup/01 section 1.7b) +# outputs: none; returns objects in memory +# notes: +# - Ports ABbiodiversity/MammalModels (2874dec), +# 1_code/2_habitat-modeling/climate/: the response built by +# 01_prepare-data.R, and the regional 03_basic-models.R join +# that keeps deployments with a climate prediction. The nine +# GLMs of 02_run-models.R are the spec's climate stage. +# - Seasons are pooled: one presence per deployment from the +# days-weighted mean density, with base weighted.mean(), which +# drops zero-day seasons. data.table's grouped weighted.mean +# does not, and would lose 97 of v2's Moose sites. +# - v2 keeps deployments by `FFP`: rows the camera climate does +# not reach are dropped from every candidate, the null +# included. +# - The data carry the archived v2 prep script's bug (summer +# densities blanked when WinterDays < 10). Kept for parity. +# --- + +# 1. Setup ---- + +## 1.1 Load packages ---- +# Base R only; the harness supplies everything else. + +# 2. mammal_season_pair() ---- + +#' Name Both Seasons' Response Columns for a Species +#' +#' The spec's `response_companions`: the climate stage pools the +#' two seasons whichever season the run models. +#' +#' @param species Character. A harness species name, e.g. +#' "Moose_Summer". +#' @return A named character vector, `summer` and `winter`. +#' +#' @example # Example usage of the function +#' # mammal_season_pair("Moose_Winter") +#' # # summer winter +#' # # "Moose_Summer" "Moose_Winter" +mammal_season_pair <- function(species) { + base <- sub("_(Summer|Winter)$", "", species) + + c( + summer = paste0(base, "_Summer"), + winter = paste0(base, "_Winter") + ) +} + +# 3. mammal_pooled_density() ---- + +#' Pool Both Seasons' Density per Deployment, as v2 +#' +#' v2 groups by `location_project`, so a deployment listed twice +#' is pooled over both rows' seasons. Bootstrap repeats of one +#' unit are not separate rows of the source, so units are pooled +#' once each. +#' +#' @param frame A province frame carrying `response_summer`, +#' `response_winter`, `summer_days`, `winter_days`, +#' `location_project` and `survey_unit_id`. +#' @return A list of `density`, the pooled density for each row +#' (NA where base weighted.mean gives NA or NaN), and `lead`, +#' TRUE on rows of the first unit of each deployment. +#' +#' @example # Example usage of the function +#' # pooled <- mammal_pooled_density(frame) +mammal_pooled_density <- function(frame) { + needed <- c( + "response_summer", "response_winter", "summer_days", + "winter_days", "location_project", "survey_unit_id" + ) + absent <- setdiff(needed, names(frame)) + + if (length(absent) > 0) { + stop( + "The mammal climate response needs ", + paste(absent, collapse = ", "), " in the frame. Is the ", + "spec's `response_companions` set?", + call. = FALSE + ) + } + + units <- as.character(frame$survey_unit_id) + key <- as.character(frame$location_project) + + # Step 1: One pooled value per deployment, over its distinct + # units (01_prepare-data.R:156-166) + distinct <- which(!duplicated(units)) + groups <- split(distinct, key[distinct]) + + pooled <- vapply(groups, function(rows) { + stats::weighted.mean( + c(frame$response_summer[rows], frame$response_winter[rows]), + c(frame$summer_days[rows], frame$winter_days[rows]) + ) + }, numeric(1)) + + lead_units <- vapply(groups, function(rows) units[rows[1]], + character(1)) + + list( + density = unname(pooled[key]), + lead = units %in% lead_units + ) +} + +# 4. mammal_climate_response() ---- + +#' Presence for the Climate Stage, as v2 Builds It +#' +#' The climate stage's `response_transform`. Present where the +#' pooled density is above zero (01_prepare-data.R:167-175). NA, +#' so `glm` drops the row, on other units of a deployment already +#' counted, where the pooled density is NA, and where the camera +#' climate is missing (`FFP` NA). +#' +#' @param values Numeric. The run's own season, unused: the +#' pooled response reads both seasons from the frame. +#' @param frame The province frame (see mammal_pooled_density()), +#' with `FFP`. +#' @return An integer vector of 0, 1 and NA, one per row. +#' +#' @example # Example usage of the function +#' # mammal_climate_response(d$response_raw, d) +mammal_climate_response <- function(values, frame) { + pooled <- mammal_pooled_density(frame) + + present <- as.integer(pooled$density > 0) + present[!pooled$lead | is.na(pooled$density) | + is.na(frame$FFP)] <- NA_integer_ + + present +} + +# 5. mammal_climate_available() ---- + +#' Keep Regional Rows Whose Deployment Has a Climate Response +#' +#' The spec's `species_frame` when climate is fitted. v2 joins its +#' climate prediction to each region with `inner_join` and drops +#' rows without one (north 03_basic-models.R:359-362, south +#' 329-332), so the habitat models see only deployments the +#' climate stage could fit. As mammal_precomputed_climate(), the +#' response is then recomputed, since lure correction and scaling +#' depend on the rows kept. +#' +#' @param frame A one-species region frame, with +#' `location_project`. +#' @param species Character. The harness species name. +#' @param data_dir Character. +#' @param spec The mammal spec. +#' @return The frame, without rows lacking a climate response. +#' +#' @example # Example usage of the function +#' # frame <- mammal_climate_available(frame, sp, data_dir, spec) +mammal_climate_available <- function(frame, species, data_dir, + spec) { + pair <- mammal_season_pair(species) + + # Step 1: The province rows, read as the harness reads them + province <- build_model_data( + taxon = spec$taxon, + data_dir = data_dir, + species = unname(pair), + covariates = c("FFP", "location_project"), + region = province_region(data_dir, spec$taxon) + ) + province_frame <- model_frame( + province, species = species, companions = pair + ) + + # Step 2: Deployments with a climate response + response <- mammal_climate_response(NULL, province_frame) + available <- unique( + province_frame$location_project[!is.na(response)] + ) + + # Step 3: Keep those regional rows, with a count, and recompute + frame <- frame[ + frame$location_project %in% available & + !is.na(frame$response_raw), , + drop = FALSE + ] + + frame$response <- apply_response_transform( + frame$response_raw, spec$response_transform, frame + ) + + frame +} + +# End of script ---- diff --git a/1_code/modules/mammals/spec.R b/1_code/modules/mammals/spec.R index fd62ed4..7c2995d 100644 --- a/1_code/modules/mammals/spec.R +++ b/1_code/modules/mammals/spec.R @@ -5,12 +5,13 @@ # inputs: none # outputs: none; returns objects in memory # notes: -# - v2 reads mammal climate from mammal_climate_predictions.csv -# (ABbiodiversity/MammalModels, 1_code/2_habitat-modeling/ -# climate/: nine binomial GLMs over FFP, MAP, CMD, TD, AICc -# averaged). `climate_source = "fitted"` instead fits that set -# as a stage, on the climate from -# abmi-camera-climate_2023.Rdata that v2 used. +# - Climate is fitted as a province-wide stage, carried into the +# regional hurdle as `Climate`, as for birds and plants. It +# ports ABbiodiversity/MammalModels 1_code/2_habitat-modeling/ +# climate/ (nine binomial GLMs over FFP, MAP, CMD, TD, AICc +# averaged) on v2's own training table, the mammal_province +# rows; see climate.R. `climate_source = "precomputed"` reads +# v2's prediction (mammal_climate_predictions.csv) instead. # - North and south are separate covariate keys: overlapping # deployments with disagreeing covariates. # - Weights are season days, so the weight column follows the @@ -123,11 +124,11 @@ mammal_agp_response <- function(values, frame) { #' (best by AICc), turned into v2's tables by #' mammal_v2_habitat_tables() (hurdle.R). #' -#' @param climate_source Character. "precomputed", the v2 -#' default, reads each species' climate prediction from -#' mammal_climate_predictions.csv and drops the deployments -#' without one, as v2 does; "fitted" fits the adapted climate -#' stage instead. +#' @param climate_source Character. "fitted", the default, fits +#' v2's climate stage province-wide on v2's training table; +#' "precomputed" reads each species' climate prediction from +#' mammal_climate_predictions.csv. Both drop the deployments +#' without a climate prediction, as v2 does. #' @param season Character. "summer" or "winter"; selects the #' weight column and the species queue. #' @param tier Character. "modelled" is the full habitat model, @@ -137,29 +138,36 @@ mammal_agp_response <- function(values, frame) { #' #' @example # Example usage of the function #' # spec <- mammal_spec() -#' # spec <- mammal_spec(climate_source = "fitted") +#' # spec <- mammal_spec(climate_source = "precomputed") mammal_spec <- function( - climate_source = c("precomputed", "fitted"), + climate_source = c("fitted", "precomputed"), season = c("summer", "winter"), tier = "modelled" ) { climate_source <- match.arg(climate_source) season <- match.arg(season) + fitted <- climate_source == "fitted" stages <- list() - if (climate_source == "fitted") { + if (fitted) { stages <- list( list( name = "climate", - # The full v2 set (CMD and TD from the camera climate file) + # climate/00_models.R and 02_run-models.R models = "climate_bird_mammal_v2", engine = "glm", selection = "aic_average", ic = "AICc", - carry_as = "Climate", - # Fitted on presence; the abundance half reuses it - response_transform = mammal_pa_response + # Province-wide and unweighted, seasons pooled, fitted on + # v2's own table (the mammal_province rows) + scope = "province", + weighted = FALSE, + response_transform = mammal_climate_response, + # v2 keeps deployments by FFP, whatever the candidates + extra_covariates = "FFP", + # plogis(link), as v2's site predictions + carry_as = "Climate" ) ) } @@ -173,17 +181,20 @@ mammal_spec <- function( # Presence; the abundance half reads the raw count response_transform = mammal_pa_response, - # v2 reads climate from a separate pipeline's prediction - species_frame = if (climate_source == "precomputed") { - mammal_precomputed_climate - } else { - NULL - }, - supplied_columns = if (climate_source == "precomputed") { - "Climate" + # Either way, only deployments with a climate prediction + # reach the habitat stage, as v2's join leaves them + species_frame = if (fitted) { + mammal_climate_available } else { - NULL + mammal_precomputed_climate }, + supplied_columns = if (fitted) NULL else "Climate", + + # The climate rows have their own unit ids: province draws + # reach each region through the deployment, and the climate + # response reads both seasons + province_key = if (fitted) "location_project" else NULL, + response_companions = if (fitted) mammal_season_pair else NULL, family = "binomial", weight_column = paste0("wt_", season), @@ -267,30 +278,35 @@ mammal_spec <- function( ) )), - resample = list( - scheme = "spatial_block", - min_detections = 20L - # No seed here: the experiment's seed is used + resample = c( + list( + scheme = "spatial_block", + min_detections = 20L + ), + # One draw for the province, mapped to each region, as for + # birds and plants. No seed here: the experiment's is used. + if (fitted) list(scope = "province") ), climate_source = climate_source, v2_coverage = list( - climate = if (climate_source == "precomputed") { + climate = if (fitted) { v2_status( "reproduced", paste( - "v2's precomputed climate prediction, read per species;", - "deployments without one are dropped, as in v2." + "v2's nine climate GLMs, AICc averaged, unweighted,", + "on v2's own table with seasons pooled, province-wide;", + "carried as plogis(link). v2 fits once, so draw 1 is", + "like for like." ) ) } else { v2_status( - "partial", + "reproduced", paste( - "The full v2 9-model set, fitted as a stage. v2 fits", - "it once, in a separate pipeline, and reads back a", - "prediction; use climate_source = \"precomputed\"." + "v2's precomputed climate prediction, read per species;", + "deployments without one are dropped, as in v2." ) ) }, @@ -306,9 +322,14 @@ mammal_spec <- function( resampling = v2_status( "partial", paste( - "Spatial-block bootstrap. v2 fits the habitat stage", - "once, so only iteration 1, the full data, is like for", - "like." + "Spatial-block bootstrap", + if (fitted) { + "drawn province-wide and mapped to regions." + } else { + "per region." + }, + "v2 fits once, so only iteration 1, the full data, is", + "like for like." ) ), season = v2_status( @@ -321,8 +342,11 @@ mammal_spec <- function( ), notes = paste0( - if (climate_source == "fitted") { - "Climate fitted with the full 9-model v2 set." + if (fitted) { + paste( + "Climate fitted province-wide with v2's 9-model set on", + "v2's own table, seasons pooled." + ) } else { "Climate read from v2's precomputed predictions." }, diff --git a/1_code/tests/README.md b/1_code/tests/README.md index 74a4a3e..2f2b6de 100644 --- a/1_code/tests/README.md +++ b/1_code/tests/README.md @@ -8,8 +8,9 @@ from the repository root. | `run_tests.R` | Loads the framework and runs `testthat/`. Takes seconds. Dataset checks are skipped when the published dataset cannot be reached | | `compare_stores.R` | Compares two sets of result stores, matched on keys, to a tolerance (default 1e-10) | | `testthat/test-dataset.R` | Manifest, species queue, covariate keys, bird grid, covariate cache, `covariate_files` joins | +| `testthat/test-experiment_config.R` | Hooks (loading, the clash rule, metric direction), `species.csv` and the runs it sets, `modify_stage()` / `modify_region()` / `modify_resample()`, `drop_candidates()` / `strip_terms()`, block cut points, and each problem `experiment_config()` reports | | `testthat/test-methods.R` | Every engine passes `check_engine()`; registry lookups; metrics | -| `testthat/test-new_experiment.R` | Slugs, numbering and scaffolding of `new_experiment()` | +| `testthat/test-new_experiment.R` | Slugs, numbering and scaffolding of `new_experiment()`, including `species.csv` and `hooks/` | | `testthat/test-results.R` | Shard merging, seeds, spatial CV folds, baseline comparison, draw summaries | | `testthat/test-specs.R` | Every standard spec validates; `replace_stage_method()`; hurdle; error reporting; `stage_models` | @@ -19,8 +20,8 @@ Rscript 1_code/tests/compare_stores.R [max_boot] [file,file] ``` The tests do not replace the parity check. For a change meant to leave -results alone, run the `parity_check` species at 5 draws to a scratch -folder and compare its stores with exp_000's, as in +results alone, run exp_000's species (its `species.csv`) at 5 draws to +a scratch folder and compare its stores with exp_000's, as in [`docs/getting_started.md`](../../docs/getting_started.md#check-that-nothing-broke). Draws are seeded per species and made in order, so draws 1–5 match those of a 100-draw run. diff --git a/1_code/tests/testthat/test-dataset.R b/1_code/tests/testthat/test-dataset.R index 0cd3aef..b2b6a4a 100644 --- a/1_code/tests/testthat/test-dataset.R +++ b/1_code/tests/testthat/test-dataset.R @@ -19,9 +19,10 @@ test_that("the manifest names files that exist", { expect_true(all(file.exists( file.path(data_dir, manifest$response_file) ))) + # The mammal province row has no grid expect_true(all(file.exists(file.path( data_dir, "lookup", - paste0(manifest$grid, "_prediction_matrix.csv") + paste0(stats::na.omit(manifest$grid), "_prediction_matrix.csv") )))) }) diff --git a/1_code/tests/testthat/test-experiment_config.R b/1_code/tests/testthat/test-experiment_config.R new file mode 100644 index 0000000..093ced4 --- /dev/null +++ b/1_code/tests/testthat/test-experiment_config.R @@ -0,0 +1,385 @@ +# --- +# title: Tests - Experiment-Local Configuration +# author: Brendan Casey +# created: 2026-10-12 +# inputs: the published test dataset (`data_dir`), for the +# experiment_config() checks; those are skipped without it +# outputs: none; testthat results +# notes: +# - What an experiment changes from its own folder: hooks, +# species.csv, spec changes (modify_*()), block cut points, +# metric direction, and the problems experiment_config() +# reports before anything is fitted. +# - Hook files are written under tempfile() and every method +# they register is removed again. +# - Run through 1_code/tests/run_tests.R, which loads the +# framework and sets `data_dir`. +# --- + +# A hook file holding the given lines +hook_file <- function(lines, name = "hook.R") { + dir <- tempfile("hooks_") + dir.create(dir) + path <- file.path(dir, name) + writeLines(lines, path) + path +} + +test_that("a hook registers a method and records its file", { + path <- hook_file(c( + "register_metric('test_hook_mae',", + " function(observed, predicted) {", + " mean(abs(observed - predicted))", + " },", + " 'Test only', direction = 'lower')" + )) + on.exit({ + unregister_method("metric", "test_hook_mae") + options(sdm.hook_files = NULL) + }) + + load_hooks(dirname(path), environment()) + + expect_true("test_hook_mae" %in% registered_names("metric")) + expect_equal(getOption("sdm.hook_files"), + normalizePath(path, winslash = "/")) + # The registered direction wins over the built-in table + expect_equal(better_direction("oob_test_hook_mae"), "lower") + expect_equal(better_direction(c("oob_auc", "oob_rmse")), + c("higher", "lower")) + + # Re-sourcing the same file re-registers its own name + expect_silent(load_hooks(path, environment())) +}) + +test_that("a hook may not replace a built-in or another hook's", { + clash <- hook_file( + "register_engine(get_engine('glm'))" + ) + on.exit(options(sdm.hook_files = NULL)) + + expect_error(load_hooks(clash, environment()), "is built in") + + first <- hook_file( + "register_metric('test_hook_twice', + function(observed, predicted) 1, 'Test only', + direction = 'none')", "first.R" + ) + second <- hook_file( + "register_metric('test_hook_twice', + function(observed, predicted) 2, 'Test only', + direction = 'none')", "second.R" + ) + on.exit(unregister_method("metric", "test_hook_twice"), add = TRUE) + + load_hooks(first, environment()) + expect_error(load_hooks(second, environment()), + "already registered") +}) + +test_that("a hook metric without a direction warns", { + path <- hook_file( + "register_metric('test_hook_nodir', + function(observed, predicted) 1, 'Test only')" + ) + on.exit({ + unregister_method("metric", "test_hook_nodir") + options(sdm.hook_files = NULL) + }) + + expect_warning(load_hooks(path, environment()), "no `direction`") +}) + +test_that("a missing hook file is an error", { + expect_error(load_hooks("no/such/hook.R"), "not found") +}) + +test_that("species.csv is read, and its mistakes are caught", { + path <- tempfile(fileext = ".csv") + writeLines(c("taxon,species,note", "lichen,Physcia.adscendens,a", + "mite,Oppiella.nova,"), path) + + expect_equal( + read_species_file(path), + c(lichen = "Physcia.adscendens", mite = "Oppiella.nova") + ) + + expect_error(read_species_file("no/such/species.csv"), "not found") + + writeLines(c("taxon,name", "lichen,Physcia.adscendens"), path) + expect_error(read_species_file(path), "no `species` column") + + writeLines(c("taxon,species", "lichen,a", "lichen,a"), path) + expect_error(read_species_file(path), "repeats") + + writeLines(c("taxon,species", "lichen,"), path) + expect_error(read_species_file(path), "blank") +}) + +test_that("the template lists every species, with empty notes", { + template <- file.path( + getOption("sdm.project_root", "."), "1_code", "experiments", + "_template", "species.csv" + ) + rows <- data.table::fread(template, colClasses = "character", + na.strings = "") + + expect_equal(names(rows), c("taxon", "species", "note")) + expect_true(all(is.na(rows$note))) + expect_false(anyDuplicated(paste(rows$taxon, rows$species)) > 0) + expect_length(read_species_file(template), nrow(rows)) +}) + +test_that("modify_stage() changes, records and validates", { + spec <- modify_stage(lichen_spec(), "habitat", min_detections = 10L) + habitat <- spec$stages[[2]] + + expect_equal(habitat$min_detections, 10L) + expect_match( + spec$models_overridden, "habitat: min_detections = 10L", + fixed = TRUE, all = FALSE + ) + expect_equal(spec$v2_coverage$habitat$status, "not reproduced") + + # A second change to the same stage keeps the first in the note + spec <- modify_stage(spec, "habitat", ic = "AIC") + expect_match(spec$v2_coverage$habitat$note, "min_detections") + expect_match(spec$v2_coverage$habitat$note, "ic") + + # NULL removes a field + spec <- modify_stage(spec, "habitat", min_detections = NULL) + expect_null(spec$stages[[2]]$min_detections) + + expect_error(modify_stage(lichen_spec(), "habitt", ic = "AIC"), + "no stage `habitt`") + expect_error(modify_stage(lichen_spec(), "habitat", min_detect = 1), + "unknown field") + expect_error( + modify_stage(lichen_spec(), "habitat", engine = "nope"), + "Unknown fitting engine" + ) + expect_error(modify_stage(lichen_spec(), "habitat", 10L), "named") +}) + +test_that("modify_region() and modify_resample() change and record", { + spec <- modify_region(lichen_spec(), "north", + filter = ~ nr == "Boreal") + expect_equal(spec$regions$north$filter, ~ nr == "Boreal") + expect_match(spec$models_overridden, "region north: filter", + all = FALSE) + expect_error(modify_region(lichen_spec(), "east", grid = "veg"), + "no region `east`") + + # Switching scheme drops the old settings but keeps the scope + spec <- modify_resample( + lichen_spec(), scheme = "spatial_cv", replace = TRUE, + lat_breaks = c(48, 52, 56, 61) + ) + expect_equal( + spec$resample, + list(scope = "province", scheme = "spatial_cv", + lat_breaks = c(48, 52, 56, 61)) + ) + expect_equal(spec$v2_coverage$resampling$status, "not reproduced") + + # Without replace, the old scheme's settings stay and are refused + expect_error( + modify_resample(bird_spec(), scheme = "spatial_cv"), + "unknown field" + ) +}) + +test_that("drop_candidates() and strip_terms() reduce a set", { + models <- c(". ~ .", ". ~ . + a", ". ~ . + a + b", + ". ~ . + b + I(b^2)", ". ~ . + a:b + c") + + # Dropping removes every candidate that names the term + expect_equal(drop_candidates(models, "b"), c(". ~ .", ". ~ . + a")) + + # Stripping keeps each candidate, less the term, and merges + # the duplicates it makes + expect_equal( + strip_terms(models, "b"), + c(". ~ .", ". ~ . + a", ". ~ . + c") + ) + + # Staged sets keep their groups + staged <- list(c(". ~ . + a", ". ~ . + b"), ". ~ . + c") + expect_equal(drop_candidates(staged, "b"), + list(". ~ . + a", ". ~ . + c")) + + # A v2 set by name: no spatial term survives, the null stays + spatial <- c("Easting", "Northing", "Easting2", "Northing2", + "EastingNorthing") + reduced <- drop_candidates("climate_plant_v2_full", spatial) + expect_false(any(grepl("Easting|Northing", reduced))) + expect_true(". ~ ." %in% reduced) + expect_lt(length(reduced), length(get_model_set( + "climate_plant_v2_full" + ))) +}) + +test_that("block cut points can be set and default to v2's", { + long <- c(-120, -118, -114, -110) + lat <- c(49, 52, 55, 58) + + expect_identical( + spatial_blocks(long, lat), + spatial_blocks(long, lat, v2_block_breaks()$long, + v2_block_breaks()$lat) + ) + # One longitude band: the four units fall in one column of + # latitude blocks + expect_length( + unique(spatial_blocks(long, lat, long_breaks = c(-121, -109))), + 4L + ) + expect_length( + unique(spatial_blocks(long, lat, long_breaks = c(-121, -109), + lat_breaks = c(48, 61))), + 1L + ) +}) + +# The checks below read the dataset's lookups +skip_if_not(dir.exists(data_dir), "No test dataset") + +test_that("misspelt species are caught with a suggestion", { + problems <- check_species_names( + c(lichen = "Physcia.adscendns", nothing = "x"), data_dir + ) + + expect_match(problems, "Unknown taxon slug.*nothing", all = FALSE) + expect_match(problems, "did you mean Physcia.adscendens", + all = FALSE, fixed = TRUE) + expect_length( + check_species_names(c(lichen = "Physcia.adscendens"), data_dir), + 0L + ) +}) + +test_that("experiment_config() lists every problem at once", { + species <- tempfile(fileext = ".csv") + writeLines(c("taxon,species", "lichen,Physcia.adscendns"), species) + + specs <- list(lichen = modify_stage( + lichen_spec(), "habitat", min_detections = 5L + )) + specs$lichen$stages[[2]]$engine <- "no_such_engine" + + err <- tryCatch( + experiment_config( + "exp_999_test", taxa = "lichen", specs = specs, + species = species, n_bootstraps = 2, + regions = "east", metrics = "no_such_metric", + compare_metrics = "oob_nothing", + data_dir = data_dir + ), + error = conditionMessage + ) + + expect_match(err, "did you mean Physcia.adscendens", fixed = TRUE) + expect_match(err, "has no region(s) east", fixed = TRUE) + expect_match(err, "Unknown metric(s): no_such_metric", fixed = TRUE) + expect_match(err, "Unknown compare_metrics: oob_nothing", + fixed = TRUE) + expect_match(err, "Unknown fitting engine `no_such_engine`", + fixed = TRUE) +}) + +test_that("every species in the template can be fitted", { + template <- file.path( + getOption("sdm.project_root", "."), "1_code", "experiments", + "_template", "species.csv" + ) + + expect_length( + check_species_names(read_species_file(template), data_dir), 0L + ) + # Every row is fitted by some run, and the unedited template + # configures every run + config <- experiment_config( + "exp_999_test", species = template, n_bootstraps = 2, + data_dir = data_dir + ) + expect_setequal(config$taxa, names(standard_specs())) +}) + +test_that("experiment_config() catches a missing species file", { + expect_error( + experiment_config("exp_999_test", taxa = "lichen", + species = "no/such/species.csv", + n_bootstraps = 2, data_dir = data_dir), + "Species file not found" + ) +}) + +test_that("experiment_config() catches a misspelt covariate", { + err <- tryCatch( + experiment_config( + "exp_999_test", taxa = "lichen", n_bootstraps = 2, + species = c(lichen = "Physcia.adscendens"), + stage_models = list( + climate = extend_models("climate_plant_v2_full", "bio155") + ), + data_dir = data_dir + ), + error = conditionMessage + ) + + expect_match(err, "covariate `bio155` is not in the dataset", + fixed = TRUE) + expect_match(err, "did you mean bio15", fixed = TRUE) +}) + +test_that("without `taxa`, the species list sets the runs", { + config <- experiment_config( + "exp_999_test", n_bootstraps = 2, + species = c(lichen = "Physcia.adscendens", + mammal = "Moose_Summer"), + data_dir = data_dir + ) + + # Moose_Summer is in the summer queue only + expect_equal(config$taxa, c("lichen", "mammal_summer")) + expect_equal(names(config$specs), config$taxa) + + # Given `taxa` wins, as before + config <- experiment_config( + "exp_999_test", taxa = "lichen", n_bootstraps = 2, + species = c(lichen = "Physcia.adscendens"), data_dir = data_dir + ) + expect_equal(config$taxa, "lichen") +}) + +test_that("a listed species no run fits is reported", { + queue <- species_catalogue(data_dir) + mammals <- queue[queue$taxon == "mammal", ] + # A mammal never in a modelled tier, if the queue has one + never <- setdiff(mammals$species, + mammals$species[mammals$tier == "modelled"]) + skip_if(length(never) == 0, "Every mammal is modelled somewhere") + + expect_error( + experiment_config( + "exp_999_test", n_bootstraps = 2, + species = c(lichen = "Physcia.adscendens", mammal = never[1]), + data_dir = data_dir + ), + paste("No run fits mammal", never[1]), fixed = TRUE + ) +}) + +test_that("a valid configuration passes, with today's defaults", { + config <- experiment_config( + "exp_999_test", taxa = c("lichen", "bird"), n_bootstraps = 2, + species = c(lichen = "Physcia.adscendens", bird = "AMRO"), + data_dir = data_dir + ) + + expect_null(config$regions) + expect_null(config$metrics) + expect_equal(record_config(config)$compare_metrics, + default_compare_metrics()) + expect_equal(record_config(config)$species_file, "inline") +}) diff --git a/1_code/tests/testthat/test-new_experiment.R b/1_code/tests/testthat/test-new_experiment.R index f7f92c7..46ebc2d 100644 --- a/1_code/tests/testthat/test-new_experiment.R +++ b/1_code/tests/testthat/test-new_experiment.R @@ -25,7 +25,8 @@ fake_repo <- function(existing = list()) { getOption("sdm.project_root", "."), "1_code", "experiments", "_template" ) - file.copy(list.files(source_template, full.names = TRUE), template) + file.copy(list.files(source_template, full.names = TRUE), template, + recursive = TRUE) for (where in names(existing)) { dir.create(file.path(root, where, existing[[where]]), @@ -72,7 +73,8 @@ test_that("new_experiment() scaffolds all three folders", { code <- file.path(root, "1_code/experiments", out$id) expected <- c( - file.path(code, c("run.R", "README.md")), + file.path(code, c("run.R", "README.md", "species.csv", + "hooks/.gitkeep")), file.path(root, "2_pipeline", out$id, c(".gitkeep", "logs/.gitkeep")), file.path(root, "3_output", out$id, @@ -89,6 +91,19 @@ test_that("new_experiment() scaffolds all three folders", { expect_true(any(grepl( 'id = "exp_003_covariate_scale"', run, fixed = TRUE ))) + expect_true(any(grepl( + 'exp_dir <- "1_code/experiments/exp_003_covariate_scale"', run, + fixed = TRUE + ))) + # The species list is copied as is: every species, for the + # experiment to cut down + expect_identical( + readLines(file.path(code, "species.csv")), + readLines(file.path( + getOption("sdm.project_root", "."), "1_code", "experiments", + "_template", "species.csv" + )) + ) expect_true(any(grepl("Experiment 003 - Covariate Scale", run, fixed = TRUE))) expect_true(any(grepl("author: Test Author", run, fixed = TRUE))) diff --git a/1_code/tests/testthat/test-province.R b/1_code/tests/testthat/test-province.R new file mode 100644 index 0000000..38a53cd --- /dev/null +++ b/1_code/tests/testthat/test-province.R @@ -0,0 +1,135 @@ +# --- +# title: Tests - Separate Province Tables +# author: Brendan Casey +# created: 2026-10-14 +# inputs: none; the loaded framework, and the test dataset for +# the last check (skipped when it has no province rows) +# outputs: none; testthat results +# notes: +# - A taxon whose province-wide data are their own table (the +# mammal climate stage): province draws map to region units +# through a key, the province frame carries companion +# responses, and the mammal climate response is v2's. +# - Run through 1_code/tests/run_tests.R, which loads the +# framework and sets `data_dir`. +# --- + +test_that("province draws map to region units through the key", { + province <- data.frame( + survey_unit_id = c("p|a", "p|b", "p|c"), + location_project = c("a", "b", "c") + ) + # Region order differs; "d" has no province row + region <- data.frame( + survey_unit_id = c("r|c", "r|a", "r|d", "r|b"), + location_project = c("c", "a", "d", "b") + ) + + draws <- list( + province$survey_unit_id, + c("p|a", "p|a", "p|c") + ) + mapped <- map_province_draws( + draws, province, region, "location_project" + ) + + # Draw 1: every mapped unit once, in region order, "d" dropped + expect_equal(mapped[[1]], c("r|c", "r|a", "r|b")) + # Repeats kept + expect_equal(mapped[[2]], c("r|c", "r|a", "r|a")) +}) + +test_that("a duplicated province key maps to its first unit", { + province <- data.frame( + survey_unit_id = c("p|a__1", "p|a__2"), + location_project = c("a", "a") + ) + region <- data.frame( + survey_unit_id = "r|a", location_project = "a" + ) + + mapped <- map_province_draws( + list(province$survey_unit_id, "p|a__2"), province, region, + "location_project" + ) + + expect_equal(mapped[[1]], "r|a") + expect_length(mapped[[2]], 0) +}) + +test_that("model_frame attaches companion responses", { + md <- list( + taxon = "x", + covariates = data.frame(survey_unit_id = c("u1", "u2")), + response = data.frame( + survey_unit_id = c("u1", "u2"), + A_Summer = c(1, 0), A_Winter = c(0, 2) + ), + offset = NULL + ) + + frame <- model_frame( + md, "A_Summer", + companions = c(summer = "A_Summer", winter = "A_Winter") + ) + expect_equal(frame$response_winter, c(0, 2)) + expect_error( + model_frame(md, "A_Summer", companions = c(winter = "B_Winter")), + "not a column" + ) +}) + +test_that("province_key needs province draws", { + spec <- mammal_spec() + expect_silent(validate_spec(spec)) + + spec$resample$scope <- NULL + expect_error(validate_spec(spec), "scope` must be \"province\"") +}) + +test_that("the climate response pools seasons as v2 does", { + frame <- data.frame( + survey_unit_id = c("p|a", "p|b", "p|c", "p|d", "p|e__1", + "p|e__2", "p|f", "p|a"), + location_project = c("a", "b", "c", "d", "e", "e", "f", "a"), + response_summer = c(0, NA, NA, 0, 0, 0, 1, 0), + response_winter = c(2, 0, 0, NA, 0, 3, 0, 2), + summer_days = c(50, 0, 30, 0, 40, 40, 20, 50), + winter_days = c(60, 70, 60, 0, 40, 40, 20, 60), + FFP = c(100, 100, 100, 100, 100, 100, NA, 100) + ) + + expected <- c( + 1L, # a: winter density, mean above 0 + 0L, # b: NA summer with 0 days drops out (base R) + NA_integer_, # c: NA summer with days gives NA + NA_integer_, # d: no days in either season (NaN) + 1L, # e: two units of one deployment, pooled + NA_integer_, # and counted once + NA_integer_, # f: no camera climate + 1L # a again: a bootstrap repeat, kept + ) + + expect_equal(mammal_climate_response(NULL, frame), expected) + expect_equal( + mammal_pooled_density(frame)$density[2], + stats::weighted.mean(c(NA, 0), c(0, 70)) + ) +}) + +test_that("the dataset's province rows load", { + skip_if_not(dir.exists(data_dir), "No test dataset") + skip_if( + is.null(province_region(data_dir, "mammal")), + "The test dataset has no mammal province rows" + ) + + md <- build_model_data( + "mammal", data_dir, c("Moose_Summer", "Moose_Winter"), + c("FFP", "location_project"), region = "province" + ) + expect_true(all(startsWith(md$units, "mammal_province|"))) + expect_null(province_region(data_dir, "lichen")) +}) + +# End of script ---- diff --git a/1_code/tests/testthat/test-specs.R b/1_code/tests/testthat/test-specs.R index f3e12c0..4c6d58d 100644 --- a/1_code/tests/testthat/test-specs.R +++ b/1_code/tests/testthat/test-specs.R @@ -16,7 +16,7 @@ test_that("every standard spec validates", { expect_silent(validate_spec(standard_specs()[[key]])) } expect_silent(validate_spec( - mammal_spec(climate_source = "fitted", season = "summer") + mammal_spec(climate_source = "precomputed", season = "summer") )) }) @@ -31,7 +31,9 @@ test_that("one call swaps the habitat method of every run", { # The mammal hurdle keeps its structure; both halves take the # new engine, and v2's GLM table-building is removed with them - habitat <- specs$mammal_summer$stages[[1]] + habitat <- Filter( + function(s) s$name == "habitat", specs$mammal_summer$stages + )[[1]] expect_equal(habitat$selection, "hurdle") expect_equal(habitat$part_selection, "single") expect_equal(habitat$engine, "xgboost") diff --git a/3_output/exp_000_parity_v2/report.md b/3_output/exp_000_parity_v2/report.md index db12d46..2d33f8f 100644 --- a/3_output/exp_000_parity_v2/report.md +++ b/3_output/exp_000_parity_v2/report.md @@ -1,6 +1,6 @@ # exp_000_parity_v2 -Generated 2026-10-08 03:59:22 +Generated 2026-10-14 10:16:53 **Trial run.** 14 focal species, 100 of 100 draws. Its parity numbers show the pipeline ran; they are not a parity read. @@ -29,13 +29,13 @@ stated. Change it there, not here. | vascular_plant | habitat | reproduced | IVW on the prediction grid, the stand-age splines, cutblock convergence, footprint pooling, the south pAspen and the 20-detection skip. Matches v2 to 2e-12 on the full-data draw, north and south. | | vascular_plant | validation | reproduced | v2's seven validation AUCs, in-bag as v2 scores them and out-of-bag as well. South exact; north within 2e-4, a residual v2's own coefficients show too. | | vascular_plant | resampling | reproduced | v2's spatial-block bootstrap, drawn once per species across the province with its 20-detection redraw rule. The same method, not the same draws: v2 is unseeded. | -| mammal_summer | climate | reproduced | v2's precomputed climate prediction, read per species; deployments without one are dropped, as in v2. | +| mammal_summer | climate | reproduced | v2's nine climate GLMs, AICc averaged, unweighted, on v2's own table with seasons pooled, province-wide; carried as plogis(link). v2 fits once, so draw 1 is like for like. | | mammal_summer | habitat | reproduced | v2's hurdle: presence, abundance and total abundance on the full habitat set, with the stand-age splines, calibration and cutblock convergence. Matches v2's published tables to 2e-14, north and south. | -| mammal_summer | resampling | partial | Spatial-block bootstrap. v2 fits the habitat stage once, so only iteration 1, the full data, is like for like. | +| mammal_summer | resampling | partial | Spatial-block bootstrap drawn province-wide and mapped to regions. v2 fits once, so only iteration 1, the full data, is like for like. | | mammal_summer | season | reproduced | One season per spec; run.R runs both, and the gate averages them as v2's `.all` references do. | -| mammal_winter | climate | reproduced | v2's precomputed climate prediction, read per species; deployments without one are dropped, as in v2. | +| mammal_winter | climate | reproduced | v2's nine climate GLMs, AICc averaged, unweighted, on v2's own table with seasons pooled, province-wide; carried as plogis(link). v2 fits once, so draw 1 is like for like. | | mammal_winter | habitat | reproduced | v2's hurdle: presence, abundance and total abundance on the full habitat set, with the stand-age splines, calibration and cutblock convergence. Matches v2's published tables to 2e-14, north and south. | -| mammal_winter | resampling | partial | Spatial-block bootstrap. v2 fits the habitat stage once, so only iteration 1, the full data, is like for like. | +| mammal_winter | resampling | partial | Spatial-block bootstrap drawn province-wide and mapped to regions. v2 fits once, so only iteration 1, the full data, is like for like. | | mammal_winter | season | reproduced | One season per spec; run.R runs both, and the gate averages them as v2's `.all` references do. | | bird | climate | reproduced | v2's 25 candidates, AICc averaged, fitted once on the province-wide draw, unweighted, and carried as exp(link) without the offset, as MuMIn predicts it. | | bird | landcover | reproduced | Staged BIC over v2's groups, weighted by vegw or soilw, always advancing to each group's winner. Raw coefficients are translated onto v2's standardized template by standardize.R, a port checked exact against v2's packaged output. | @@ -83,10 +83,10 @@ _Every taxon and region that ran had a reference to compare against._ | bryophyte | bryophyte | south | - | - | 2 | 100 | climate, habitat | TRUE | TRUE | | lichen | lichen | north | - | - | 2 | 100 | climate, habitat | TRUE | TRUE | | lichen | lichen | south | - | - | 2 | 100 | climate, habitat | TRUE | TRUE | -| mammal | mammal_summer | north | summer | hurdle | 2 | 100 | habitat_presence, habitat_abundance, habitat_total | TRUE | TRUE | -| mammal | mammal_summer | south | summer | hurdle | 2 | 100 | habitat_presence, habitat_abundance, habitat_total | TRUE | TRUE | -| mammal | mammal_winter | north | winter | hurdle | 2 | 100 | habitat_presence, habitat_abundance, habitat_total | TRUE | TRUE | -| mammal | mammal_winter | south | winter | hurdle | 2 | 100 | habitat_presence, habitat_abundance, habitat_total | TRUE | TRUE | +| mammal | mammal_summer | north | summer | hurdle | 2 | 100 | climate, habitat_presence, habitat_abundance, habitat_total | TRUE | TRUE | +| mammal | mammal_summer | south | summer | hurdle | 2 | 100 | climate, habitat_presence, habitat_abundance, habitat_total | TRUE | TRUE | +| mammal | mammal_winter | north | winter | hurdle | 2 | 100 | climate, habitat_presence, habitat_abundance, habitat_total | TRUE | TRUE | +| mammal | mammal_winter | south | winter | hurdle | 2 | 100 | climate, habitat_presence, habitat_abundance, habitat_total | TRUE | TRUE | | mite | mite | north | - | - | 2 | 100 | climate, habitat | TRUE | TRUE | | mite | mite | south | - | - | 2 | 100 | climate, habitat | TRUE | TRUE | | vascular_plant | vascular_plant | north | - | - | 2 | 100 | climate, habitat | TRUE | TRUE | @@ -105,23 +105,25 @@ fixes at a placeholder are left out of the reachable scores. | bird | north | climate | median | 2 | 20 | 100 | 100.000 | 20 | 100.000 | 0.000 | 0.000 | 0.000 | 0 | 1.000 | 1.000 | not gated (trial run) | pass | | bryophyte | north | climate | median | 1 | 19 | 100 | 100.000 | 19 | 100.000 | 0.035 | 0.000 | 1.400 | 6 | 1.000 | 1.062 | not gated (trial run) | pass | | lichen | north | climate | median | 2 | 38 | 100 | 100.000 | 38 | 100.000 | 0.000 | 0.000 | 0.644 | 17 | 0.993 | 0.879 | not gated (trial run) | pass | +| mammal | north | climate | iteration 1 | 2 | 10 | 100 | 100.000 | 10 | 100.000 | 0.000 | 0.000 | 0.000 | 0 | 1.000 | - | not gated (trial run) | pass | | mite | north | climate | median | 2 | 38 | 100 | 100.000 | 38 | 100.000 | 0.000 | 0.000 | 1.950 | 20 | 0.905 | 0.953 | not gated (trial run) | pass | | vascular_plant | north | climate | median | 2 | 38 | 100 | 100.000 | 38 | 100.000 | 0.000 | 0.000 | 0.266 | 24 | 1.000 | 0.970 | not gated (trial run) | pass | | bird | south | climate | median | 2 | 20 | 100 | 100.000 | 20 | 100.000 | 0.000 | 0.000 | 0.000 | 0 | 1.000 | 1.000 | not gated (trial run) | pass | | bryophyte | south | climate | median | 1 | 19 | 100 | 100.000 | 19 | 100.000 | 0.035 | 0.000 | 1.400 | 6 | 1.000 | 1.062 | not gated (trial run) | pass | | lichen | south | climate | median | 2 | 38 | 100 | 100.000 | 38 | 100.000 | 0.000 | 0.000 | 0.644 | 17 | 0.993 | 0.879 | not gated (trial run) | pass | +| mammal | south | climate | iteration 1 | 2 | 10 | 100 | 100.000 | 10 | 100.000 | 0.000 | 0.000 | 0.000 | 0 | 1.000 | - | not gated (trial run) | pass | | mite | south | climate | median | 2 | 38 | 100 | 100.000 | 38 | 100.000 | 0.000 | 0.000 | 1.950 | 20 | 0.905 | 0.953 | not gated (trial run) | pass | | vascular_plant | south | climate | median | 2 | 38 | 100 | 100.000 | 38 | 100.000 | 0.000 | 0.000 | 0.266 | 24 | 1.000 | 0.970 | not gated (trial run) | pass | | bird | north | habitat | median | 2 | 178 | 100 | 100.000 | 168 | 100.000 | 0.000 | 0.000 | 0.000 | 0 | 1.000 | 1.000 | not gated (trial run) | pass | | bryophyte | north | habitat | median | 1 | 94 | 100 | 100.000 | 94 | 100.000 | 0.072 | 0.023 | 0.081 | 0 | 0.997 | 0.911 | not gated (trial run) | pass | | lichen | north | habitat | median | 2 | 188 | 100 | 100.000 | 188 | 100.000 | 0.091 | 0.024 | 0.118 | 0 | 0.999 | 0.988 | not gated (trial run) | pass | -| mammal | north | habitat | iteration 1 | 2 | 448 | 65 | 100.000 | 436 | 100.000 | 0.000 | 0.000 | 0.000 | 0 | 1.000 | - | not gated (trial run) | pass | +| mammal | north | habitat | iteration 1 | 2 | 448 | 69 | 100.000 | 436 | 100.000 | 0.000 | 0.000 | 0.000 | 0 | 1.000 | - | not gated (trial run) | pass | | mite | north | habitat | median | 1 | 94 | 100 | 100.000 | 94 | 100.000 | 0.140 | 0.035 | 0.148 | 0 | 0.999 | 0.993 | not gated (trial run) | pass | | vascular_plant | north | habitat | median | 2 | 188 | 100 | 100.000 | 188 | 100.000 | 0.108 | 0.022 | 0.125 | 0 | 0.999 | 0.950 | not gated (trial run) | pass | | bird | south | habitat | median | 2 | 46 | 100 | 100.000 | 40 | 100.000 | 0.000 | 0.000 | 0.000 | 0 | 1.000 | 1.000 | not gated (trial run) | pass | | bryophyte | south | habitat | median | 1 | 19 | 100 | 100.000 | 19 | 100.000 | 0.047 | 0.021 | 0.060 | 0 | 0.993 | 1.042 | not gated (trial run) | pass | | lichen | south | habitat | median | 2 | 40 | 100 | 100.000 | 40 | 100.000 | 0.088 | 0.029 | 0.278 | 1 | 0.994 | 0.961 | not gated (trial run) | pass | -| mammal | south | habitat | iteration 1 | 2 | 126 | 99 | 100.000 | 120 | 100.000 | 0.000 | 0.000 | 0.000 | 0 | 1.000 | - | not gated (trial run) | pass | +| mammal | south | habitat | iteration 1 | 2 | 126 | 100 | 100.000 | 120 | 100.000 | 0.000 | 0.000 | 0.000 | 0 | 1.000 | - | not gated (trial run) | pass | | mite | south | habitat | median | 2 | 40 | 100 | 100.000 | 40 | 100.000 | 0.068 | 0.024 | 0.464 | 1 | 0.991 | 0.975 | not gated (trial run) | pass | | vascular_plant | south | habitat | median | 2 | 40 | 100 | 100.000 | 40 | 100.000 | 0.074 | 0.027 | 0.184 | 0 | 0.997 | 1.021 | not gated (trial run) | pass | @@ -147,7 +149,7 @@ Median AUC over species, scored on each run's final model: for plants, v2's own | bird | south | 2 | 0.706 | 0.702 | - | - | | bryophyte | south | 2 | 0.676 | 0.664 | 0.676 | 0.664 | | lichen | south | 2 | 0.802 | 0.792 | 0.802 | 0.792 | -| mammal | south | 4 | 0.739 | 0.716 | - | - | +| mammal | south | 4 | 0.740 | 0.715 | - | - | | mite | south | 2 | 0.796 | 0.781 | 0.796 | 0.781 | | vascular_plant | south | 2 | 0.850 | 0.844 | 0.850 | 0.844 | diff --git a/3_output/exp_000_parity_v2/run_record.md b/3_output/exp_000_parity_v2/run_record.md index 229ff92..bf53c78 100644 --- a/3_output/exp_000_parity_v2/run_record.md +++ b/3_output/exp_000_parity_v2/run_record.md @@ -6,26 +6,36 @@ Written by `run_record()`. The result stores this describes are gitignored; this _Volatile. Excluded from cross-run comparison._ -- **git_commit**: 4a8cb32 +- **git_commit**: 49a10153fbed84f24261647611e8b426326ef7de +- **git_uncommitted_code**: none +- **hooks**: none +- **packages**: data.table 1.18.0 - **pipeline_dir**: D:/local_projects/active/sdmMethodsDev/2_pipeline/exp_000_parity_v2 +- **platform**: x86_64-w64-mingw32 - **r_version**: 4.5.0 -- **written_at**: 2026-10-08 09:59:23 +- **session_info**: D:/local_projects/active/sdmMethodsDev/2_pipeline/exp_000_parity_v2/session_info.txt +- **written_at**: 2026-10-14 16:16:53 ## Configuration - **boot_seed**: 20260909.0000 - **changes_from_v2**: none +- **compare_metrics**: oob_auc, oob_deviance_explained, oob_rmse, oob_spearman, oob_calibration_slope - **compared_with**: none (baseline) - **covariate_files**: none -- **data_dir**: published 2026-10-05 (//ABMI-DATA2/science/sdmMethodsDev/0_data/test_dataset/2026-10-05) +- **data_dir**: local (D:/local_projects/active/sdmMethodsDev/0_data/test_dataset) - **engines**: bayesglm, glm - **focal_species**: bird=AMRO, bird=YEWA, bryophyte=Bryum.All, bryophyte=Ceratodon.purpureus, lichen=Cladonia.chlorophaea, lichen=Physcia.adscendens, mammal=Coyote_Summer, mammal=Coyote_Winter, mammal=Moose_Summer, mammal=Moose_Winter, mite=Ceratozetes.gracilis, mite=Trhypochthonius.tectorum, vascular_plant=Galium.boreale, vascular_plant=Vicia.americana -- **jobs_not_ok**: 1 -- **jobs_ok**: 2799 +- **hooks**: none +- **jobs_not_ok**: 0 +- **jobs_ok**: 2800 - **jobs_total**: 2800 +- **metrics**: all registered - **n_bootstraps**: 100 - **plant_bootstrap**: spatial_block +- **regions**: all - **selection**: aic_average, hurdle, ivw_grid, staged_bic +- **species_file**: species.csv - **species_n**: 14 - **stage_models**: spec defaults (v2 candidate sets) - **taxa**: bird, bryophyte, lichen, mammal_summer, mammal_winter, mite, vascular_plant @@ -41,10 +51,10 @@ _Volatile. Excluded from cross-run comparison._ | bryophyte | south | 2 | 100 | 7800 | 5600 | 3200 | | lichen | north | 2 | 100 | 22000 | 5600 | 8600 | | lichen | south | 2 | 100 | 7800 | 5600 | 3200 | -| mammal_summer | north | 2 | 100 | 43450 | 2800 | 9000 | -| mammal_summer | south | 2 | 100 | 13731 | 2786 | 3980 | -| mammal_winter | north | 2 | 100 | 44008 | 2800 | 9000 | -| mammal_winter | south | 2 | 100 | 13800 | 2800 | 4000 | +| mammal_summer | north | 2 | 100 | 44630 | 2800 | 9000 | +| mammal_summer | south | 2 | 100 | 14800 | 2800 | 4000 | +| mammal_winter | north | 2 | 100 | 45098 | 2800 | 9000 | +| mammal_winter | south | 2 | 100 | 14800 | 2800 | 4000 | | mite | north | 2 | 100 | 21800 | 5600 | 8600 | | mite | south | 2 | 100 | 7600 | 5600 | 3200 | | vascular_plant | north | 2 | 100 | 21800 | 5600 | 8600 | @@ -194,62 +204,62 @@ _Volatile. Excluded from cross-run comparison._ | lichen | south | v2val_Full_Joint_Truncated | 200 | 0.8057 | 0.7733 | 0.8261 | 0.8013 | 0.0213 | | lichen | south | v2val_Full_Truncated | 200 | 0.8072 | 0.7724 | 0.8287 | 0.8021 | 0.0232 | | lichen | south | v2val_Landcover | 200 | 0.7520 | 0.7339 | 0.7705 | 0.7526 | 0.0143 | -| mammal_summer | north | insample_auc | 200 | 0.7195 | 0.6076 | 0.8295 | 0.7195 | 0.1048 | -| mammal_summer | north | insample_calibration_slope | 200 | 0.9415 | 0.7950 | 0.9994 | 0.9143 | 0.0790 | -| mammal_summer | north | insample_deviance_explained | 200 | 0.1457 | 0.0267 | 0.2897 | 0.1557 | 0.1242 | -| mammal_summer | north | insample_n | 200 | 2563.5000 | 2538.0000 | 2587.0000 | 2577.0800 | 149.9873 | -| mammal_summer | north | insample_prevalence | 200 | 0.2733 | 0.2209 | 0.3269 | 0.2735 | 0.0486 | -| mammal_summer | north | insample_rmse | 200 | 0.3850 | 0.3230 | 0.4431 | 0.3836 | 0.0575 | -| mammal_summer | north | insample_spearman | 200 | 0.3181 | 0.1570 | 0.4710 | 0.3152 | 0.1473 | -| mammal_summer | north | oob_auc | 198 | 0.7008 | 0.5803 | 0.8199 | 0.7002 | 0.1098 | -| mammal_summer | north | oob_calibration_slope | 198 | 0.8458 | 0.5527 | 0.9867 | 0.7990 | 0.1680 | -| mammal_summer | north | oob_deviance_explained | 198 | 0.0563 | 0.0013 | 0.2662 | 0.1234 | 0.1179 | -| mammal_summer | north | oob_n | 200 | 1489.5000 | 1466.0000 | 1515.0000 | 1475.9200 | 149.9873 | -| mammal_summer | north | oob_prevalence | 198 | 0.2735 | 0.2193 | 0.3287 | 0.2728 | 0.0485 | -| mammal_summer | north | oob_rmse | 198 | 0.3917 | 0.3259 | 0.4480 | 0.3883 | 0.0571 | -| mammal_summer | north | oob_spearman | 198 | 0.2890 | 0.1134 | 0.4545 | 0.2851 | 0.1566 | -| mammal_summer | south | insample_auc | 199 | 0.7546 | 0.6196 | 0.7995 | 0.7107 | 0.0791 | -| mammal_summer | south | insample_calibration_slope | 199 | 0.9613 | 0.8460 | 1.0524 | 0.9550 | 0.0819 | -| mammal_summer | south | insample_deviance_explained | 199 | 0.1647 | 0.0214 | 0.2324 | 0.1205 | 0.0943 | -| mammal_summer | south | insample_n | 199 | 674.0000 | 662.0000 | 689.0000 | 678.4171 | 40.4188 | -| mammal_summer | south | insample_prevalence | 199 | 0.2069 | 0.1750 | 0.6994 | 0.4351 | 0.2552 | -| mammal_summer | south | insample_rmse | 199 | 0.3563 | 0.3391 | 0.4250 | 0.3822 | 0.0389 | -| mammal_summer | south | insample_spearman | 199 | 0.3342 | 0.1592 | 0.3998 | 0.2818 | 0.1044 | -| mammal_summer | south | oob_auc | 197 | 0.7154 | 0.5675 | 0.7903 | 0.6784 | 0.0930 | -| mammal_summer | south | oob_calibration_slope | 197 | 0.7382 | 0.3358 | 1.0724 | 0.7360 | 0.2780 | -| mammal_summer | south | oob_deviance_explained | 197 | 0.0147 | -0.0349 | 0.2068 | 0.0619 | 0.1132 | -| mammal_summer | south | oob_n | 199 | 391.0000 | 376.0000 | 403.0000 | 386.5829 | 40.4188 | -| mammal_summer | south | oob_prevalence | 197 | 0.2300 | 0.1685 | 0.7076 | 0.4357 | 0.2560 | -| mammal_summer | south | oob_rmse | 197 | 0.3897 | 0.3411 | 0.4384 | 0.3914 | 0.0411 | -| mammal_summer | south | oob_spearman | 197 | 0.2707 | 0.0727 | 0.3889 | 0.2325 | 0.1292 | -| mammal_winter | north | insample_auc | 200 | 0.7665 | 0.6675 | 0.8446 | 0.7591 | 0.0814 | -| mammal_winter | north | insample_calibration_slope | 200 | 0.9541 | 0.8712 | 1.0110 | 0.9464 | 0.0554 | -| mammal_winter | north | insample_deviance_explained | 200 | 0.1900 | 0.0702 | 0.3056 | 0.1876 | 0.1090 | -| mammal_winter | north | insample_n | 200 | 2609.5000 | 2585.0000 | 2640.1000 | 2626.3900 | 152.6091 | -| mammal_winter | north | insample_prevalence | 200 | 0.1764 | 0.1455 | 0.2061 | 0.1760 | 0.0269 | -| mammal_winter | north | insample_rmse | 200 | 0.3161 | 0.2973 | 0.3353 | 0.3168 | 0.0164 | -| mammal_winter | north | insample_spearman | 200 | 0.3518 | 0.2111 | 0.4734 | 0.3443 | 0.1214 | -| mammal_winter | north | oob_auc | 198 | 0.7540 | 0.6364 | 0.8417 | 0.7404 | 0.0916 | -| mammal_winter | north | oob_calibration_slope | 198 | 0.8826 | 0.6191 | 1.0325 | 0.8552 | 0.1596 | -| mammal_winter | north | oob_deviance_explained | 198 | 0.0839 | 0.0080 | 0.2921 | 0.1494 | 0.1240 | -| mammal_winter | north | oob_n | 200 | 1517.5000 | 1486.9000 | 1542.0000 | 1500.6100 | 152.6091 | -| mammal_winter | north | oob_prevalence | 198 | 0.1742 | 0.1426 | 0.2060 | 0.1734 | 0.0271 | -| mammal_winter | north | oob_rmse | 198 | 0.3199 | 0.2972 | 0.3413 | 0.3193 | 0.0172 | -| mammal_winter | north | oob_spearman | 198 | 0.3263 | 0.1702 | 0.4645 | 0.3186 | 0.1319 | -| mammal_winter | south | insample_auc | 200 | 0.7315 | 0.6779 | 0.8331 | 0.7557 | 0.0673 | -| mammal_winter | south | insample_calibration_slope | 200 | 0.9493 | 0.8167 | 1.0904 | 0.9489 | 0.1093 | -| mammal_winter | south | insample_deviance_explained | 200 | 0.1130 | 0.0574 | 0.2652 | 0.1552 | 0.0897 | -| mammal_winter | south | insample_n | 200 | 695.0000 | 682.0000 | 709.0000 | 699.1600 | 41.4843 | -| mammal_winter | south | insample_prevalence | 200 | 0.3351 | 0.0822 | 0.6144 | 0.3449 | 0.2590 | -| mammal_winter | south | insample_rmse | 200 | 0.3196 | 0.2155 | 0.4099 | 0.3142 | 0.0927 | -| mammal_winter | south | insample_spearman | 200 | 0.3081 | 0.2774 | 0.3331 | 0.3064 | 0.0216 | -| mammal_winter | south | oob_auc | 198 | 0.7123 | 0.6520 | 0.8075 | 0.7227 | 0.0632 | -| mammal_winter | south | oob_calibration_slope | 198 | 0.7412 | 0.4709 | 1.0531 | 0.7570 | 0.2412 | -| mammal_winter | south | oob_deviance_explained | 198 | 0.0619 | 0.0061 | 0.1945 | 0.0691 | 0.1295 | -| mammal_winter | south | oob_n | 200 | 403.0000 | 389.0000 | 416.0000 | 398.8400 | 41.4843 | -| mammal_winter | south | oob_prevalence | 198 | 0.3423 | 0.0803 | 0.6235 | 0.3480 | 0.2608 | -| mammal_winter | south | oob_rmse | 198 | 0.3325 | 0.2222 | 0.4211 | 0.3238 | 0.0914 | -| mammal_winter | south | oob_spearman | 198 | 0.2687 | 0.2234 | 0.3096 | 0.2658 | 0.0350 | +| mammal_summer | north | insample_auc | 200 | 0.7206 | 0.6073 | 0.8275 | 0.7199 | 0.1037 | +| mammal_summer | north | insample_calibration_slope | 200 | 0.9333 | 0.8042 | 1.0034 | 0.9184 | 0.0737 | +| mammal_summer | north | insample_deviance_explained | 200 | 0.1480 | 0.0289 | 0.2859 | 0.1561 | 0.1228 | +| mammal_summer | north | insample_n | 200 | 2563.0000 | 2533.8000 | 2592.0000 | 2576.6500 | 150.4575 | +| mammal_summer | north | insample_prevalence | 200 | 0.2725 | 0.2204 | 0.3249 | 0.2730 | 0.0484 | +| mammal_summer | north | insample_rmse | 200 | 0.3834 | 0.3237 | 0.4424 | 0.3831 | 0.0573 | +| mammal_summer | north | insample_spearman | 200 | 0.3152 | 0.1569 | 0.4663 | 0.3156 | 0.1458 | +| mammal_summer | north | oob_auc | 198 | 0.7013 | 0.5801 | 0.8185 | 0.6988 | 0.1111 | +| mammal_summer | north | oob_calibration_slope | 198 | 0.8306 | 0.5424 | 1.0026 | 0.7914 | 0.1828 | +| mammal_summer | north | oob_deviance_explained | 198 | 0.0490 | 0.0001 | 0.2657 | 0.1224 | 0.1231 | +| mammal_summer | north | oob_n | 200 | 1490.0000 | 1461.0000 | 1519.2000 | 1476.3500 | 150.4575 | +| mammal_summer | north | oob_prevalence | 198 | 0.2730 | 0.2184 | 0.3295 | 0.2737 | 0.0489 | +| mammal_summer | north | oob_rmse | 198 | 0.3937 | 0.3270 | 0.4495 | 0.3892 | 0.0573 | +| mammal_summer | north | oob_spearman | 198 | 0.2888 | 0.1104 | 0.4538 | 0.2829 | 0.1588 | +| mammal_summer | south | insample_auc | 200 | 0.7143 | 0.6234 | 0.8003 | 0.7116 | 0.0795 | +| mammal_summer | south | insample_calibration_slope | 200 | 0.9678 | 0.8533 | 1.0836 | 0.9703 | 0.0972 | +| mammal_summer | south | insample_deviance_explained | 200 | 0.1050 | 0.0210 | 0.2307 | 0.1209 | 0.0962 | +| mammal_summer | south | insample_n | 200 | 673.0000 | 656.0000 | 688.0000 | 676.8100 | 41.1119 | +| mammal_summer | south | insample_prevalence | 200 | 0.4377 | 0.1739 | 0.6994 | 0.4364 | 0.2559 | +| mammal_summer | south | insample_rmse | 200 | 0.3856 | 0.3369 | 0.4249 | 0.3819 | 0.0397 | +| mammal_summer | south | insample_spearman | 200 | 0.2831 | 0.1540 | 0.4021 | 0.2804 | 0.1068 | +| mammal_summer | south | oob_auc | 198 | 0.6840 | 0.5687 | 0.7848 | 0.6787 | 0.0888 | +| mammal_summer | south | oob_calibration_slope | 198 | 0.7858 | 0.3758 | 1.1071 | 0.7808 | 0.3016 | +| mammal_summer | south | oob_deviance_explained | 198 | 0.0149 | -0.0449 | 0.1982 | 0.0506 | 0.1180 | +| mammal_summer | south | oob_n | 200 | 392.0000 | 377.0000 | 409.0000 | 388.1900 | 41.1119 | +| mammal_summer | south | oob_prevalence | 198 | 0.4435 | 0.1688 | 0.7070 | 0.4370 | 0.2547 | +| mammal_summer | south | oob_rmse | 198 | 0.4000 | 0.3424 | 0.4363 | 0.3918 | 0.0402 | +| mammal_summer | south | oob_spearman | 198 | 0.2487 | 0.0883 | 0.3805 | 0.2371 | 0.1207 | +| mammal_winter | north | insample_auc | 200 | 0.7631 | 0.6695 | 0.8444 | 0.7591 | 0.0815 | +| mammal_winter | north | insample_calibration_slope | 200 | 0.9426 | 0.8647 | 1.0007 | 0.9383 | 0.0524 | +| mammal_winter | north | insample_deviance_explained | 200 | 0.1850 | 0.0710 | 0.3049 | 0.1879 | 0.1095 | +| mammal_winter | north | insample_n | 200 | 2608.5000 | 2582.7000 | 2636.3000 | 2624.0150 | 152.9481 | +| mammal_winter | north | insample_prevalence | 200 | 0.1736 | 0.1452 | 0.2056 | 0.1751 | 0.0265 | +| mammal_winter | north | insample_rmse | 200 | 0.3168 | 0.2977 | 0.3358 | 0.3163 | 0.0164 | +| mammal_winter | north | insample_spearman | 200 | 0.3499 | 0.2133 | 0.4719 | 0.3437 | 0.1211 | +| mammal_winter | north | oob_auc | 198 | 0.7508 | 0.6345 | 0.8383 | 0.7396 | 0.0914 | +| mammal_winter | north | oob_calibration_slope | 198 | 0.8705 | 0.6189 | 1.0232 | 0.8459 | 0.1527 | +| mammal_winter | north | oob_deviance_explained | 198 | 0.1140 | 0.0135 | 0.2898 | 0.1503 | 0.1222 | +| mammal_winter | north | oob_n | 200 | 1518.5000 | 1490.7000 | 1544.3000 | 1502.9850 | 152.9481 | +| mammal_winter | north | oob_prevalence | 198 | 0.1704 | 0.1417 | 0.2099 | 0.1751 | 0.0279 | +| mammal_winter | north | oob_rmse | 198 | 0.3214 | 0.3003 | 0.3419 | 0.3202 | 0.0170 | +| mammal_winter | north | oob_spearman | 198 | 0.3306 | 0.1685 | 0.4623 | 0.3186 | 0.1324 | +| mammal_winter | south | insample_auc | 200 | 0.7409 | 0.6805 | 0.8293 | 0.7530 | 0.0642 | +| mammal_winter | south | insample_calibration_slope | 200 | 0.9499 | 0.8259 | 1.0604 | 0.9468 | 0.0933 | +| mammal_winter | south | insample_deviance_explained | 200 | 0.1214 | 0.0605 | 0.2553 | 0.1521 | 0.0855 | +| mammal_winter | south | insample_n | 200 | 694.0000 | 678.0000 | 714.0000 | 698.7250 | 42.4394 | +| mammal_winter | south | insample_prevalence | 200 | 0.3435 | 0.0812 | 0.6126 | 0.3457 | 0.2594 | +| mammal_winter | south | insample_rmse | 200 | 0.3191 | 0.2161 | 0.4094 | 0.3143 | 0.0922 | +| mammal_winter | south | insample_spearman | 200 | 0.3070 | 0.2741 | 0.3331 | 0.3039 | 0.0236 | +| mammal_winter | south | oob_auc | 198 | 0.7149 | 0.6499 | 0.8120 | 0.7247 | 0.0657 | +| mammal_winter | south | oob_calibration_slope | 198 | 0.7685 | 0.4709 | 1.0860 | 0.7887 | 0.2523 | +| mammal_winter | south | oob_deviance_explained | 198 | 0.0646 | 0.0219 | 0.2122 | 0.0789 | 0.1401 | +| mammal_winter | south | oob_n | 200 | 404.0000 | 384.0000 | 420.0000 | 399.2750 | 42.4394 | +| mammal_winter | south | oob_prevalence | 198 | 0.3300 | 0.0776 | 0.6210 | 0.3467 | 0.2598 | +| mammal_winter | south | oob_rmse | 198 | 0.3288 | 0.2190 | 0.4203 | 0.3226 | 0.0922 | +| mammal_winter | south | oob_spearman | 198 | 0.2649 | 0.2157 | 0.3172 | 0.2667 | 0.0419 | | mite | north | insample_auc | 200 | 0.7415 | 0.6711 | 0.7964 | 0.7353 | 0.0572 | | mite | north | insample_calibration_slope | 200 | 0.9839 | 0.8742 | 1.0308 | 0.9642 | 0.0621 | | mite | north | insample_deviance_explained | 200 | 0.0588 | -0.0563 | 0.1372 | 0.0443 | 0.0839 | diff --git a/3_output/exp_000_parity_v2/tables/coverage.csv b/3_output/exp_000_parity_v2/tables/coverage.csv index 30d8940..5f8c72b 100644 --- a/3_output/exp_000_parity_v2/tables/coverage.csv +++ b/3_output/exp_000_parity_v2/tables/coverage.csv @@ -5,10 +5,10 @@ bryophyte,bryophyte,north,,,2,100,"climate, habitat",TRUE,TRUE bryophyte,bryophyte,south,,,2,100,"climate, habitat",TRUE,TRUE lichen,lichen,north,,,2,100,"climate, habitat",TRUE,TRUE lichen,lichen,south,,,2,100,"climate, habitat",TRUE,TRUE -mammal,mammal_summer,north,summer,hurdle,2,100,"habitat_presence, habitat_abundance, habitat_total",TRUE,TRUE -mammal,mammal_summer,south,summer,hurdle,2,100,"habitat_presence, habitat_abundance, habitat_total",TRUE,TRUE -mammal,mammal_winter,north,winter,hurdle,2,100,"habitat_presence, habitat_abundance, habitat_total",TRUE,TRUE -mammal,mammal_winter,south,winter,hurdle,2,100,"habitat_presence, habitat_abundance, habitat_total",TRUE,TRUE +mammal,mammal_summer,north,summer,hurdle,2,100,"climate, habitat_presence, habitat_abundance, habitat_total",TRUE,TRUE +mammal,mammal_summer,south,summer,hurdle,2,100,"climate, habitat_presence, habitat_abundance, habitat_total",TRUE,TRUE +mammal,mammal_winter,north,winter,hurdle,2,100,"climate, habitat_presence, habitat_abundance, habitat_total",TRUE,TRUE +mammal,mammal_winter,south,winter,hurdle,2,100,"climate, habitat_presence, habitat_abundance, habitat_total",TRUE,TRUE mite,mite,north,,,2,100,"climate, habitat",TRUE,TRUE mite,mite,south,,,2,100,"climate, habitat",TRUE,TRUE vascular_plant,vascular_plant,north,,,2,100,"climate, habitat",TRUE,TRUE diff --git a/3_output/exp_000_parity_v2/tables/grid_summary.csv b/3_output/exp_000_parity_v2/tables/grid_summary.csv index 31c4e7b..9770a2b 100644 --- a/3_output/exp_000_parity_v2/tables/grid_summary.csv +++ b/3_output/exp_000_parity_v2/tables/grid_summary.csv @@ -447,266 +447,266 @@ lichen,lichen,south,,,Physcia.adscendens,Wellsites,100,0.041450461300727,0.02893 lichen,lichen,south,,,Physcia.adscendens,Crop,100,0.00916378733877362,0.00751086734379391,0.0107940857677616,0.00923975055880215,0.0091924707651885,0.00153574702124211 lichen,lichen,south,,,Physcia.adscendens,TameP,100,0.0265119152472038,0.0202324904660954,0.0303829572907274,0.026713869603247,0.0260971202538315,0.00452685074013171 lichen,lichen,south,,,Physcia.adscendens,RoughP,100,0.0384136847737998,0.0304612756475431,0.0449729740444777,0.039320400519205,0.0381938851106205,0.00624657883043015 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Decid,100,0.0483594827063645,0.0414118481331626,0.0563982542519368,0.0487558878166635,0.0485336895616105,0.0068092864354131 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Mixedwood,100,0.0351561471368664,0.0242025397384364,0.0463572816733345,0.0339606541972421,0.0355995378172595,0.00894981356273383 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Pine,100,0.0256013795193124,0.01824239977447,0.0313975443070661,0.0258942184497686,0.0253239847649179,0.00500141300051215 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Spruce,100,0.0158309723048582,0.0102588646061362,0.0218278913443854,0.0155269078679997,0.0161450148400433,0.00428664625822814 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,TreedBog,100,0.029434754078122,0.0249943179821213,0.0359303843195688,0.0299192022319042,0.0300046246220914,0.00447375399845672 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,TreedFen,100,0.0291157898857012,0.023775298386277,0.0353418695585902,0.0299192022319042,0.0295256783465423,0.00461968212252635 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,TreedSwamp,100,0.0312737750968453,0.0185619561651247,0.0439052060299924,0.0299192022319042,0.0308756010457884,0.0095324449415456 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,GrassHerb,100,0.0785972684793487,0.0587194800671309,0.108200584386506,0.0701423452426464,0.0800041123333062,0.0207203837913138 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Shrub,100,0.0638763830262266,0.0432418463524077,0.0810865803379858,0.0701423452426464,0.063477882385419,0.0148042032995085 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,GraminoidFen,100,0.0601441339822926,0.0304644977939886,0.0812721117490118,0.0320501190239091,0.0560416896335662,0.0208268772049053 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Marsh,100,0.0601441339822926,0.0304644977939886,0.0812721117490118,0.0320501190239091,0.0560416896335662,0.0208268772049053 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,ShrubbyBog,100,0.0189501510379441,0.00806039782961353,0.0341486287892669,0.0320501190239091,0.0206606382909829,0.0098600640304262 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,ShrubbyFen,100,0.0189501510379441,0.00806039782961353,0.0341486287892669,0.0320501190239091,0.0206606382909829,0.0098600640304262 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,ShrubbySwamp,100,0.0259568256179573,0.0156535396414799,0.0346912354382728,0.0320501190239091,0.0257712580266269,0.00788133710983602 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCDecidR,100,0.0589128268649123,0.0328811565492362,0.0937139933204344,0.0664247062431653,0.0613991568761192,0.0266909524675824 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCDecid1,100,0.101547292339766,0.0530167833013339,0.217810899949766,0.0664247062431653,0.124360210593978,0.0671243003813008 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCDecid2,100,0.0516020670687248,0.0151952152365976,0.0833007528895771,0.0664247062431653,0.0483529287950538,0.0254242491196635 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCDecid3,100,0.124673154645494,0.0530167833013339,0.163457117688787,0.0664247062431653,0.1108234847745,0.0450815627910776 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCDecid4,100,0.124673154645494,0.0530167833013339,0.163457117688787,0.0664247062431653,0.1108234847745,0.0450815627910776 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCMixedwoodR,100,3.72855409894537e-07,5.47842003511913e-08,0.0833007528895771,0.0664247062431653,0.0316713892885261,0.0359620322643694 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCMixedwood1,100,0.0646375446939071,0.035385035599797,0.126200797875627,0.0664247062431653,0.0730497283726891,0.0427794927811525 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCMixedwood2,100,0.0697287476043638,0.0309758891666172,0.169160650963475,0.0664247062431653,0.0884099329763265,0.0629283756685299 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCMixedwood3,100,0.124673154645494,0.0530167833013339,0.163457117688787,0.0664247062431653,0.1108234847745,0.0450815627910776 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCMixedwood4,100,0.124673154645494,0.0530167833013339,0.163457117688787,0.0664247062431653,0.1108234847745,0.0450815627910776 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCPineR,100,0.0484442595993501,0.0268995847238305,0.126785183996014,0.0418434478038768,0.0699726382745438,0.0455943713218502 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCPine1,100,0.0311079332735461,0.0157825531999667,0.0484169484849554,0.0418434478038768,0.0321502261803672,0.0147557755697887 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCPine2,100,0.124673154645494,0.0290689292212467,0.163457117688787,0.0418434478038768,0.0986304776454289,0.0573377553796652 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCPine3,100,0.124673154645494,0.0290689292212467,0.163457117688787,0.0418434478038768,0.0986304776454289,0.0573377553796652 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCPine4,100,0.124673154645494,0.0290689292212467,0.163457117688787,0.0418434478038768,0.0986304776454289,0.0573377553796652 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCSpruceR,100,0.081353803296149,0.0286783678552215,0.251217184909871,0.0418434478038768,0.118495066582276,0.0999479881036481 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCSpruce1,100,0.0447267336414446,9.36619670623034e-08,0.11798015194896,0.0418434478038768,0.0600166873448887,0.0721591677501457 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCSpruce2,100,0.0341282849673548,0.0159969116763001,0.0684462472561841,0.0418434478038768,0.0388186316634564,0.0233437171584887 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCSpruce3,100,0.125435704711867,0.0290708823093577,0.163457117688787,0.0418434478038768,0.101910273358532,0.0552214662302606 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCSpruce4,100,0.125435704711867,0.0290708823093577,0.163457117688787,0.0418434478038768,0.101910273358532,0.0552214662302606 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,HardLin,100,0.134990802687838,0.0688384772973166,0.179271577228808,0.140140987080189,0.128733675612959,0.0416464065610382 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,EnSoftLin,100,0.118353918404146,0.0954236011149053,0.162724311460711,0.108633567141907,0.123770329636941,0.0290009369428649 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,EnSeismic,100,0.101524417191545,0.0557851312646505,0.128492659433118,0.108633567141907,0.0960208287352638,0.0276153535004726 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,TrSoftLin,100,0.109556511052275,0.074868503218827,0.138196677657058,0.108633567141907,0.108683190638727,0.0262063538664031 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Urban,100,0.134990802687838,0.0688384772973166,0.179271577228808,0.140140987080189,0.128733675612959,0.0416464065610382 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Rural,100,0.134990802687838,0.0688384772973166,0.179271577228808,0.140140987080189,0.128733675612959,0.0416464065610382 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Industrial,100,0.134990802687838,0.0688384772973166,0.179271577228808,0.140140987080189,0.128733675612959,0.0416464065610382 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Well,100,0.123932314686558,0.0272975036658561,0.157271661675091,0.140140987080189,0.101655069287838,0.0517767730968891 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Crop,100,0.144356413835438,0.124028100048918,0.171958352945825,0.140140987080189,0.145015373881083,0.0203269226002348 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,TameP,100,0.150075585048021,0.123690046738055,0.186290627573863,0.140140987080189,0.153250281462701,0.0315293417132034 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,RoughP,100,0.154457694577525,0.123272295574508,0.241433921965233,0.140140987080189,0.169721367317023,0.0528643192921944 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Decid,100,0.259683336062034,0.20040089603673,0.308115883796319,0.250263196314734,0.25885090723883,0.0399240895557718 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Mixedwood,100,0.266815619343161,0.223268823520599,0.331002812217121,0.267778296796379,0.271295953435411,0.0448343257574875 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Pine,100,0.193275209507897,0.158156669442869,0.236297391459018,0.191627552662267,0.196778655118468,0.032077588564201 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Spruce,100,0.207185711469881,0.179589123539344,0.271159768784067,0.211937623492895,0.21784119003314,0.0388202562370106 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,TreedBog,100,0.134081498222746,0.107266654105978,0.168729928250215,0.13349759896445,0.138026661208827,0.0251390931398608 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,TreedFen,100,0.141137834842585,0.108255999395942,0.178998737834139,0.140460621172556,0.142474787516157,0.0266951374397913 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,TreedSwamp,100,0.219247637103862,0.142913252569802,0.296984827983815,0.219333664647888,0.217029488517284,0.0572474110799337 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,GrassHerb,100,0.233779407970901,0.179658973644467,0.290895834233095,0.243208536186666,0.236158263337651,0.0440926138807196 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Shrub,100,0.228800145651465,0.168046386406582,0.285014692915752,0.23362253489809,0.230550509439829,0.0484696728373614 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,GraminoidFen,100,0.272357486633059,0.193854615700907,0.342585121230796,0.272335942420037,0.271568823896367,0.0616419937372103 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Marsh,100,0.272357486633059,0.193854615700907,0.342585121230796,0.272335942420037,0.271568823896367,0.0616419937372103 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,ShrubbyBog,100,0.122420233859063,0.0968324948760909,0.167849994236879,0.123758665516073,0.130011225728449,0.0306549088294796 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,ShrubbyFen,100,0.122420233859063,0.0968324948760909,0.167849994236879,0.123758665516073,0.130011225728449,0.0306549088294796 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,ShrubbySwamp,100,0.188984303082597,0.147799307235341,0.231957691360083,0.188745324781336,0.189163454469278,0.0360756816949773 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCDecidR,100,0.208148613906902,0.148920973268902,0.271209916525661,0.204735734181689,0.212888184869838,0.0557692461001884 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCDecid1,100,0.216312801865899,0.151922543756141,0.283020771778373,0.204735734181689,0.219708287260622,0.0602045063836374 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCDecid2,100,0.196838655533488,0.103824327627087,0.2670608013824,0.204735734181689,0.192340230430844,0.0648420072528742 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCDecid3,100,0.200807768687866,0.155412851644233,0.2670608013824,0.204735734181689,0.207869309241703,0.0479696940544398 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCDecid4,100,0.200807768687866,0.155412851644233,0.2670608013824,0.204735734181689,0.207869309241703,0.0479696940544398 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCMixedwoodR,100,0.216312801865899,0.161387117679579,0.371359266565987,0.204735734181689,0.24607735043091,0.113690007470323 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCMixedwood1,100,0.198119002508505,0.0859178071374664,0.271209916525661,0.204735734181689,0.193759391020952,0.0702532635271893 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCMixedwood2,100,0.216312801865899,0.151922543756141,0.346809770627561,0.204735734181689,0.237419231826105,0.0857578713266533 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCMixedwood3,100,0.200807768687866,0.155412851644233,0.2670608013824,0.204735734181689,0.207869309241703,0.0479696940544398 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCMixedwood4,100,0.200807768687866,0.155412851644233,0.2670608013824,0.204735734181689,0.207869309241703,0.0479696940544398 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCPineR,100,0.254152254742842,0.156060977727918,0.365873926915736,0.252728055432474,0.268708347177854,0.10627938175414 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCPine1,100,0.252854977473957,0.177661862261277,0.344755518429091,0.252728055432474,0.25774051080654,0.0747419615882058 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCPine2,100,0.228635901145074,0.16437932467335,0.334320875739281,0.252728055432474,0.239193469814565,0.0670494299755326 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCPine3,100,0.228635901145074,0.16437932467335,0.334320875739281,0.252728055432474,0.239193469814565,0.0670494299755326 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCPine4,100,0.228635901145074,0.16437932467335,0.334320875739281,0.252728055432474,0.239193469814565,0.0670494299755326 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCSpruceR,100,0.203037467918197,0.0346185576882622,0.323878851998171,0.251426565070253,0.198222392835578,0.105391997266464 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCSpruce1,100,0.25916697849522,0.16869487511716,0.578387793790354,0.251426565070253,0.310557928020038,0.175763151783799 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCSpruce2,100,0.249428067418612,0.167313438388596,0.35114039405543,0.251426565070253,0.257706896350914,0.0860457984925452 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCSpruce3,100,0.211915394741454,0.156952491802639,0.323878851998171,0.251426565070253,0.229843810637933,0.0662743610640749 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCSpruce4,100,0.211915394741454,0.156952491802639,0.323878851998171,0.251426565070253,0.229843810637933,0.0662743610640749 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,HardLin,100,0.118973489036885,0.0595749883009149,0.194302195597634,0.111539574120126,0.126165319844306,0.055822474852709 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,EnSoftLin,100,0.252528455338498,0.183191640704092,0.322319883170126,0.245765939186683,0.252222858480715,0.0502239936189235 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,EnSeismic,100,0.406111934925129,0.261849785971992,0.508770353912488,0.40014515763882,0.398820978452058,0.0946884676442976 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,TrSoftLin,100,0.197381027195335,0.138492827864977,0.268193709302946,0.191095088735685,0.200661161915687,0.0536417738670849 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Urban,100,0.118973489036885,0.0595749883009149,0.194302195597634,0.111539574120126,0.126165319844306,0.055822474852709 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Rural,100,0.118973489036885,0.0595749883009149,0.194302195597634,0.111539574120126,0.126165319844306,0.055822474852709 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Industrial,100,0.118973489036885,0.0595749883009149,0.194302195597634,0.111539574120126,0.126165319844306,0.055822474852709 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Well,100,0.150115813653559,0.0641884565559437,0.266317718578671,0.135502134019886,0.160468025351514,0.0806608570334136 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Crop,100,0.178308894527329,0.144256206301652,0.219215382703309,0.177273110573243,0.179797087755398,0.0305000177721645 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,TameP,100,0.0968784190339813,0.0639905020753798,0.160190119712042,0.0952839991630323,0.104890040231061,0.0378313565075219 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,RoughP,100,0.276331603956952,0.155829543505443,0.406895267796159,0.274174686267773,0.275451935045609,0.0898210684407084 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,ClaySub,99,0.533150220736053,0.325398622218069,0.726178963551552,0.555220139484688,0.523909857063885,0.149581249796729 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,Other,99,0.533150220736053,0.325398622218069,0.726178963551552,0.555220139484688,0.523909857063885,0.149581249796729 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,RapidDrain,99,0.583205101528209,0.405737655337897,0.811750562735225,0.555220139484688,0.599967859266422,0.156816619058576 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,Loamy,99,0.566488786814424,0.418909848560532,0.773010386676219,0.585624995059312,0.5772796136132,0.136844885972705 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,SandyLoam,99,0.752069587260714,0.516043348758857,0.875033459295435,0.779017339675273,0.725432489757725,0.133477209151085 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,ThinBreak,99,0.539120407928289,0.37746390423711,0.752938136678661,0.555220139484688,0.55347164428064,0.141540115041928 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,Blowout,99,0.539120407928289,0.37746390423711,0.752938136678661,0.555220139484688,0.55347164428064,0.141540115041928 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,Urban,99,0.560458380581449,0.30584035412618,0.836101653751614,0.487723056831294,0.559830353099832,0.181793550917155 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,Rural,99,0.560458380581449,0.30584035412618,0.836101653751614,0.487723056831294,0.559830353099832,0.181793550917155 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,Industrial,99,0.560458380581449,0.30584035412618,0.836101653751614,0.487723056831294,0.559830353099832,0.181793550917155 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,Well,99,0.483140156808882,0.243543850628091,0.695425018660143,0.487723056831294,0.475755787805985,0.177988341945675 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,Mine,99,0.560458380581449,0.30584035412618,0.836101653751614,0.487723056831294,0.559830353099832,0.181793550917155 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,EnSoftLin,99,0.551988691379397,0.369166654777916,0.839252755172912,0.551988691379397,0.573743395435802,0.186447804813419 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,EnSeismic,99,0.551988691379397,0.369166654777916,0.839252755172912,0.551988691379397,0.573743395435802,0.186447804813419 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,HardLin,99,0.560458380581449,0.30584035412618,0.836101653751614,0.487723056831294,0.559830353099832,0.181793550917155 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,TrSoftLin,99,0.551421057241242,0.374412403426174,0.739303345816835,0.551988691379397,0.55118788794327,0.141256298238092 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,Crop,99,0.637656208587839,0.49529558797905,0.836193078886696,0.664605920258043,0.648907108351841,0.124336335473079 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,RoughP,99,0.628211923070028,0.486701783432612,0.831663290081914,0.664605920258043,0.640720467792209,0.128184887356401 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,TameP,99,0.658995748672994,0.49529558797905,0.836648596529824,0.664605920258043,0.655771301073678,0.122976004048341 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,WetlandMargin,99,0.581770385638187,0.329231355346863,0.799241213154381,0.585444465824865,0.567320654622147,0.182695488693111 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,ClaySub,100,0.0754973526043263,0.0458239666984705,0.12443371523533,0.0784267337005647,0.0820991546821132,0.0340720594732481 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,Other,100,0.0754973526043263,0.0458239666984705,0.12443371523533,0.0784267337005647,0.0820991546821132,0.0340720594732481 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,RapidDrain,100,0.0898193728889103,0.0544321614391112,0.169124236200161,0.0784267337005647,0.103184466735601,0.0508037176402887 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,Loamy,100,0.0754973526043263,0.046969411504197,0.113688607850989,0.0784267337005647,0.0770023896862594,0.0256298204133886 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,SandyLoam,100,0.0585506232943164,0.0209656837086591,0.10982498396462,0.0784267337005647,0.0632575848289767,0.0357931302993329 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,ThinBreak,100,0.0690440751872111,0.0455668700101314,0.107640018556426,0.0784267337005647,0.0744155436126283,0.0265990345899489 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,Blowout,100,0.0690440751872111,0.0455668700101314,0.107640018556426,0.0784267337005647,0.0744155436126283,0.0265990345899489 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,Urban,100,0.0149189755497271,1.15377844073514e-08,0.0687677956339382,0.00922653651061864,0.0265742538392963,0.0307190996810526 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,Rural,100,0.0149189755497271,1.15377844073514e-08,0.0687677956339382,0.00922653651061864,0.0265742538392963,0.0307190996810526 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,Industrial,100,0.0149189755497271,1.15377844073514e-08,0.0687677956339382,0.00922653651061864,0.0265742538392963,0.0307190996810526 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,Well,100,9.70256091755441e-09,3.34243329444256e-09,0.0120656517213894,0.00922653651061864,0.00344041021256161,0.00587883591068888 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,Mine,100,0.0149189755497271,1.15377844073514e-08,0.0687677956339382,0.00922653651061864,0.0265742538392963,0.0307190996810526 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,EnSoftLin,100,0.0613925660634633,0.0228914216364576,0.172980551432417,0.036098691627639,0.0870022146949501,0.0861898400742218 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,EnSeismic,100,0.0613925660634633,0.0228914216364576,0.172980551432417,0.036098691627639,0.0870022146949501,0.0861898400742218 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,HardLin,100,0.0149189755497271,1.15377844073514e-08,0.0687677956339382,0.00922653651061864,0.0265742538392963,0.0307190996810526 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,TrSoftLin,100,0.0312751876831008,0.0153840077110305,0.0582148277417413,0.036098691627639,0.033797894109043,0.0164856945190874 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,Crop,100,0.0290831580423508,0.0168202953913674,0.0444969076233006,0.0270020288489522,0.0300026599427121,0.0108015110559459 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,RoughP,100,0.027060425084569,0.0155717202882352,0.0495357718432256,0.0270020288489522,0.0307206786257607,0.0145473793590478 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,TameP,100,0.021246637642403,0.0100067327177243,0.0385937143721166,0.0270020288489522,0.0228743404925687,0.0113061261626493 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,WetlandMargin,100,1.64157963789091e-08,6.9138159991845e-09,4.35015387623389e-08,2.33837497296015e-08,2.47066699086432e-08,2.77102602578868e-08 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Decid,100,0.0441801422120102,0.0380443781286064,0.0520501930616528,0.0449203614522765,0.0442927353342687,0.00565566738018957 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Mixedwood,100,0.0274521702294942,0.0189460705705972,0.0370713341339322,0.0273241474240622,0.0279182603040639,0.00812137273402431 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Pine,100,0.0169691221092524,0.0115590337125378,0.0224752565704855,0.0176556438960936,0.0172309771750957,0.00445515073415181 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Spruce,100,0.0139342200911224,0.00902317716707155,0.021131550722022,0.0137961282417973,0.014494044040736,0.00469965157007278 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,TreedBog,100,0.0285152872075479,0.0235989018745811,0.0331700061908687,0.0286355651253948,0.0283020176999104,0.00384196970987737 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,TreedFen,100,0.031376327610073,0.0245484737335901,0.0382003227179276,0.0316720599445966,0.0314299542025266,0.00536226751347378 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,TreedSwamp,100,0.0574237874651995,0.0397626701657425,0.0828631811624413,0.0562955719457925,0.0599362255184132,0.0167451860422155 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,GrassHerb,100,0.0752855436869807,0.0571656823514431,0.0999643819729603,0.0783755759784833,0.0767959247473038,0.0165509676115107 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Shrub,100,0.0655325202828486,0.0489750303062312,0.0896679789819056,0.0693588214886089,0.0666932900053699,0.0165528227085852 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,GraminoidFen,100,0.0836102304373164,0.0591663531030278,0.105067520949907,0.0834692933781883,0.0833141459476071,0.0178600396315725 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Marsh,100,0.0836102304373164,0.0591663531030278,0.105067520949907,0.0834692933781883,0.0833141459476071,0.0178600396315725 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,ShrubbyBog,100,0.0148994403771645,0.00859347071369135,0.0237749328750818,0.0174941754511015,0.016542315636732,0.00707394415904985 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,ShrubbyFen,100,0.0148994403771645,0.00859347071369135,0.0237749328750818,0.0174941754511015,0.016542315636732,0.00707394415904985 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,ShrubbySwamp,100,0.0188813430167508,0.010188524815852,0.0266391080315766,0.0195830589922788,0.0191847427790687,0.00688884835074752 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCDecidR,100,0.0609689167321641,0.0435880046934829,0.0880996260162498,0.0634232741924905,0.0644983779591182,0.0227600380888077 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCDecid1,100,0.061455414319186,0.0425169152093179,0.0866492071396139,0.0634232741924905,0.0637468043569909,0.0179989199651142 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCDecid2,100,0.0579259475476142,0.0401946833462241,0.0793308999711383,0.0634232741924905,0.058758319370857,0.0184541471423022 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCDecid3,100,0.065812281901988,0.0458682120231148,0.110469048876007,0.0634232741924905,0.0712673981021456,0.0271165639483053 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCDecid4,100,0.065812281901988,0.0458682120231148,0.110469048876007,0.0634232741924905,0.0712673981021456,0.0271165639483053 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCMixedwoodR,100,0.0639052442071173,0.0444166028245725,0.193772909813427,0.0634232741924905,0.0944426818735319,0.120672566074976 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCMixedwood1,100,0.0578930475597618,0.0363803353576038,0.0858500841121438,0.0634232741924905,0.059437673794497,0.0247051319421999 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCMixedwood2,100,0.0603769675159346,0.0404602333638686,0.0869107653927138,0.0634232741924905,0.0629634759930984,0.0235475845209569 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCMixedwood3,100,0.065812281901988,0.0458682120231148,0.110469048876007,0.0634232741924905,0.0712673981021456,0.0271165639483053 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCMixedwood4,100,0.065812281901988,0.0458682120231148,0.110469048876007,0.0634232741924905,0.0712673981021456,0.0271165639483053 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCPineR,100,0.0454372615554885,0.0264426704342878,0.0772816673703056,0.0429017164110987,0.0517831160467498,0.0304806139233005 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCPine1,100,0.0423136080966415,0.0239610769127983,0.0636460703981842,0.0429017164110987,0.0428987693194692,0.0150418536389454 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCPine2,100,0.0454372615554885,0.0264426704342878,0.109551752803063,0.0429017164110987,0.0559509823014347,0.0321162472627193 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCPine3,100,0.0454372615554885,0.0264426704342878,0.109551752803063,0.0429017164110987,0.0559509823014347,0.0321162472627193 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCPine4,100,0.0454372615554885,0.0264426704342878,0.109551752803063,0.0429017164110987,0.0559509823014347,0.0321162472627193 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCSpruceR,100,0.0266248861652583,0.00858510635952236,0.0545668890367623,0.0309648973086927,0.0291857521558944,0.0179714992004611 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCSpruce1,100,0.0281605702410256,0.0140888139413565,0.0563493928549161,0.0309648973086927,0.0333225413660826,0.025211436347249 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCSpruce2,100,0.0290021079818168,0.0164189449870767,0.0554061952615034,0.0309648973086927,0.0323564703801675,0.015870938516246 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCSpruce3,100,0.0311963025795263,0.0169063198874443,0.109551752803063,0.0309648973086927,0.0450380735939417,0.0364654586371936 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCSpruce4,100,0.0311963025795263,0.0169063198874443,0.109551752803063,0.0309648973086927,0.0450380735939417,0.0364654586371936 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,HardLin,100,0.120838067045032,0.0966706219158405,0.233279953194392,0.118992631310027,0.138264579666678,0.0568468515414428 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,EnSoftLin,100,0.118011413545915,0.0907733798379286,0.170690965008075,0.0980871465772408,0.125038186709424,0.0331086015314007 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,EnSeismic,100,0.0815667321881285,0.0339416726680318,0.111518791819418,0.0980871465772408,0.0751979348594535,0.0310447867439705 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,TrSoftLin,100,0.0948122622235059,0.0641855678678306,0.115586769105764,0.0980871465772408,0.0923907466893072,0.0200531728209402 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Urban,100,0.120838067045032,0.0966706219158405,0.233279953194392,0.118992631310027,0.138264579666678,0.0568468515414428 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Rural,100,0.120838067045032,0.0966706219158405,0.233279953194392,0.118992631310027,0.138264579666678,0.0568468515414428 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Industrial,100,0.120838067045032,0.0966706219158405,0.233279953194392,0.118992631310027,0.138264579666678,0.0568468515414428 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Well,100,0.113022652466482,0.0550316081237054,0.146052807954379,0.118992631310027,0.112237080323506,0.0393792993179918 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Crop,100,0.117003736392723,0.0957174518415361,0.141471068423912,0.118992631310027,0.118180575509835,0.0185889175708889 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,TameP,100,0.121933470860515,0.0966169367673692,0.152182327713371,0.118992631310027,0.123392801114276,0.0222839504856315 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,RoughP,100,0.11398472893091,0.0919934519147151,0.144547924767592,0.118992631310027,0.116389735223143,0.0219274565125492 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Decid,100,0.107018576225334,0.0781860408956421,0.14724878101086,0.111664402452981,0.110593112833016,0.0265339937333045 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Mixedwood,100,0.0600249041780712,0.0387186048969526,0.0952931739407444,0.0655577834788831,0.0636107827695861,0.0232553473916604 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Pine,100,0.0396309480042169,0.0273930089386423,0.0574164498277552,0.0434917161194809,0.0419977730497561,0.0117903714686515 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Spruce,100,0.063519837765127,0.0423944034638921,0.0958282320714583,0.0695392350566296,0.0673039082819413,0.0222054985527112 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,TreedBog,100,0.0250513458868803,0.0163856844345025,0.0384141777296862,0.0324223167296191,0.0265136281869465,0.00897335198181729 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,TreedFen,100,0.0378782204941214,0.0250286657300626,0.0552489917228343,0.0324223167296191,0.0398379940113467,0.0113063709718298 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,TreedSwamp,100,0.127507185499595,0.0840026514669348,0.214189723172413,0.138924945537604,0.138839239739068,0.0500243329966069 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,GrassHerb,100,0.063210452889291,0.0449333207050848,0.0991932646492706,0.0674988002326724,0.0676950424601493,0.0214785935294802 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Shrub,100,0.0807462892653911,0.0437672839247167,0.111985367190735,0.0845204524428601,0.080530874970254,0.0264685695503026 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,GraminoidFen,100,0.0582488597559558,0.033628554365253,0.0903221835370631,0.0715579377130805,0.0607985591346197,0.024022251134648 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Marsh,100,0.0582488597559558,0.033628554365253,0.0903221835370631,0.0715579377130805,0.0607985591346197,0.024022251134648 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,ShrubbyBog,100,0.054101177976491,0.0303167705666363,0.0873301625562935,0.0715579377130805,0.0583031673175539,0.0215763558023682 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,ShrubbyFen,100,0.054101177976491,0.0303167705666363,0.0873301625562935,0.0715579377130805,0.0583031673175539,0.0215763558023682 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,ShrubbySwamp,100,0.0872069283399591,0.0610661624483727,0.127347246867895,0.0715579377130805,0.0905479325592781,0.0285730238747674 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCDecidR,100,0.0911593424841253,0.0584453778398794,0.13941689617098,0.0987996864776002,0.0995308542397944,0.0386189224128256 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCDecid1,100,0.0894786978425815,0.0594106751140486,0.148900197266955,0.0987996864776002,0.101588455377209,0.0487235225290337 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCDecid2,100,0.0911593424841253,0.0594106751140486,0.15100586688468,0.0987996864776002,0.100885170210963,0.0397086172985588 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCDecid3,100,0.0814502525610403,0.0484438488115814,0.124155346530608,0.0987996864776002,0.0834536368550272,0.0302337191183088 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCDecid4,100,0.0814502525610403,0.0484438488115814,0.124155346530608,0.0987996864776002,0.0834536368550272,0.0302337191183088 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCMixedwoodR,100,0.0789271332438918,5.50952067758724e-08,0.124155346530608,0.0987996864776002,0.0670668536263169,0.0486505402997848 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCMixedwood1,100,0.0908997403596427,0.0553451656837641,0.150772645901323,0.0987996864776002,0.105620598942897,0.0672187184321156 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCMixedwood2,100,0.0801127444470547,0.0123987938171872,0.124155346530608,0.0987996864776002,0.0760184068054095,0.0412743660068957 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCMixedwood3,100,0.0814502525610403,0.0484438488115814,0.124155346530608,0.0987996864776002,0.0834536368550272,0.0302337191183088 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCMixedwood4,100,0.0814502525610403,0.0484438488115814,0.124155346530608,0.0987996864776002,0.0834536368550272,0.0302337191183088 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCPineR,100,0.0763878373107279,0.0187048867953609,0.122719355362655,0.0987996864776002,0.0740963478990904,0.0395541797117844 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCPine1,100,0.0890734500932516,0.0483951778064103,0.127573277136019,0.0987996864776002,0.0896864933149464,0.035082662488274 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCPine2,100,0.0775870517260786,0.0435952763542857,0.120847942970241,0.0987996864776002,0.0801633467749005,0.0318064195179464 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCPine3,100,0.0775870517260786,0.0435952763542857,0.120847942970241,0.0987996864776002,0.0801633467749005,0.0318064195179464 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCPine4,100,0.0775870517260786,0.0435952763542857,0.120847942970241,0.0987996864776002,0.0801633467749005,0.0318064195179464 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCSpruceR,100,0.0873268123435719,8.73027675440557e-08,0.151670225506166,0.0987996864776002,0.0870060531575045,0.0555939709321528 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCSpruce1,100,0.109990762983568,0.0690528235829929,0.31601607625292,0.0987996864776002,0.156194351721973,0.129793292503217 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCSpruce2,100,0.0986353171316303,0.0574876224706808,0.159276841473816,0.0987996864776002,0.104909161020079,0.0462071228373226 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCSpruce3,100,0.08769898759399,0.0492870615774235,0.151670225506166,0.0987996864776002,0.0953999877264437,0.044423714215896 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCSpruce4,100,0.08769898759399,0.0492870615774235,0.151670225506166,0.0987996864776002,0.0953999877264437,0.044423714215896 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,HardLin,100,0.0594218708037509,0.0188566375615111,0.0853868289899408,0.0681539858129174,0.0560611678925488,0.0321190931437238 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,EnSoftLin,100,0.0511883484980616,0.0216773916021179,0.0851363710135712,0.0630037477010183,0.0527068863002357,0.024694185086149 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,EnSeismic,100,0.0582948031389333,0.035603305847707,0.0973901385067103,0.0630037477010183,0.0629353106319518,0.0268064665557228 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,TrSoftLin,100,0.063916768024785,0.040499656095095,0.111378571528245,0.0630037477010183,0.0729447152028905,0.0338934466777903 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Urban,100,0.0594218708037509,0.0188566375615111,0.0853868289899408,0.0681539858129174,0.0560611678925488,0.0321190931437238 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Rural,100,0.0594218708037509,0.0188566375615111,0.0853868289899408,0.0681539858129174,0.0560611678925488,0.0321190931437238 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Industrial,100,0.0594218708037509,0.0188566375615111,0.0853868289899408,0.0681539858129174,0.0560611678925488,0.0321190931437238 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Well,100,0.0639396671010574,0.0223565955416337,0.106420658756765,0.0681539858129174,0.0680570215149711,0.0468812951102975 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Crop,100,0.0639396671010574,0.0442256259217382,0.0860637424530528,0.0681539858129174,0.0645352579754965,0.0177221739780754 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,TameP,100,0.0602267341786738,0.0381802904243015,0.0865450094961094,0.0681539858129174,0.0613572901726841,0.0208547596132734 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,RoughP,100,0.0827058600533914,0.0499961758784742,0.221213338882194,0.0681539858129174,0.111786841032919,0.0666586431819001 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,ClaySub,100,0.695946156395264,0.52844388716267,0.840819628284833,0.714170375192759,0.689030492066084,0.111837539280104 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,Other,100,0.695946156395264,0.52844388716267,0.840819628284833,0.714170375192759,0.689030492066084,0.111837539280104 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,RapidDrain,100,0.711243081707945,0.540957520143654,0.858699360593556,0.714170375192759,0.704411537350655,0.116399110698907 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,Loamy,100,0.703435420804858,0.558124707867744,0.840535376692541,0.714170375192759,0.696820761951072,0.1056316539274 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,SandyLoam,100,0.765898474195991,0.625128237557598,0.89519023310151,0.714170375192759,0.75925104168454,0.109119980300251 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,ThinBreak,100,0.702729894924194,0.532784100477285,0.845582463576447,0.714170375192759,0.696498363080483,0.112209043947277 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,Blowout,100,0.702729894924194,0.532784100477285,0.845582463576447,0.714170375192759,0.696498363080483,0.112209043947277 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,Urban,100,0.775789255850362,0.636241543237387,0.8824519427391,0.757773549137013,0.758182230379237,0.11436905098169 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,Rural,100,0.775789255850362,0.636241543237387,0.8824519427391,0.757773549137013,0.758182230379237,0.11436905098169 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,Industrial,100,0.775789255850362,0.636241543237387,0.8824519427391,0.757773549137013,0.758182230379237,0.11436905098169 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,Well,100,0.723172291301919,0.535354887502062,0.873697343957099,0.757773549137013,0.711240341537646,0.127789436959912 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,Mine,100,0.775789255850362,0.636241543237387,0.8824519427391,0.757773549137013,0.758182230379237,0.11436905098169 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,EnSoftLin,100,0.656946705814415,0.471714470156675,0.856438129969767,0.628634628150965,0.649753443360378,0.158071932609515 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,EnSeismic,100,0.656946705814415,0.471714470156675,0.856438129969767,0.628634628150965,0.649753443360378,0.158071932609515 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,HardLin,100,0.775789255850362,0.636241543237387,0.8824519427391,0.757773549137013,0.758182230379237,0.11436905098169 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,TrSoftLin,100,0.629322984636201,0.467535885878004,0.797966928943662,0.628634628150965,0.630495467901194,0.127898159927544 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,Crop,100,0.763625172444686,0.625233314007204,0.873697343957099,0.757773549137013,0.748637157301014,0.0964348772482102 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,RoughP,100,0.757503251016383,0.560574525811163,0.873697343957099,0.757773549137013,0.737968673107066,0.105962302473259 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,TameP,100,0.761837237003966,0.650402585362162,0.873697343957099,0.757773549137013,0.753861584328303,0.0927946141177057 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,WetlandMargin,100,0.888816791069685,0.745819337087173,0.963175043297249,0.870880735779777,0.861580803343261,0.103911677681727 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,ClaySub,100,0.00901907292334418,0.00395896603109892,0.0224763592865055,0.0141945654676911,0.0111116719857896,0.00736053858000641 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,Other,100,0.00901907292334418,0.00395896603109892,0.0224763592865055,0.0141945654676911,0.0111116719857896,0.00736053858000641 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,RapidDrain,100,0.0113976501352768,0.0043465212562151,0.0256046970444095,0.0141945654676911,0.0143241968881526,0.0105633476997002 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,Loamy,100,0.00983864381384956,0.00427177362560075,0.023102750553041,0.0141945654676911,0.0121120043194343,0.00771436958661508 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,SandyLoam,100,0.0121871060388674,0.00530943283352528,0.0296467650966089,0.0141945654676911,0.0157507471339046,0.0113623076859402 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,ThinBreak,100,0.0208105854240252,0.0085593099510179,0.0392579304751684,0.0141945654676911,0.023314514621981,0.0150362529640607 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,Blowout,100,0.0208105854240252,0.0085593099510179,0.0392579304751684,0.0141945654676911,0.023314514621981,0.0150362529640607 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,Urban,100,7.75895576152824e-10,3.44432500305338e-10,2.78333509246948e-09,9.67698723675452e-10,0.000235799040000245,0.00120591748931771 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,Rural,100,7.75895576152824e-10,3.44432500305338e-10,2.78333509246948e-09,9.67698723675452e-10,0.000235799040000245,0.00120591748931771 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,Industrial,100,7.75895576152824e-10,3.44432500305338e-10,2.78333509246948e-09,9.67698723675452e-10,0.000235799040000245,0.00120591748931771 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,Well,100,7.87419219794934e-10,2.99392953216905e-10,2.57405477170408e-09,8.90552084803996e-10,0.00023579898080937,0.00120591750100836 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,Mine,100,7.75895576152824e-10,3.44432500305338e-10,2.78333509246948e-09,9.67698723675452e-10,0.000235799040000245,0.00120591748931771 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,EnSoftLin,100,0.0248424631931178,0.00460836849545691,0.0921001582461561,0.0367985674989132,0.0407755679608486,0.05743250715751 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,EnSeismic,100,0.0248424631931178,0.00460836849545691,0.0921001582461561,0.0367985674989132,0.0407755679608486,0.05743250715751 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,HardLin,100,7.75895576152824e-10,3.44432500305338e-10,2.78333509246948e-09,9.67698723675452e-10,0.000235799040000245,0.00120591748931771 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,TrSoftLin,100,0.00249469387501362,0.000379040108724366,0.00760478473172621,0.00374705166332369,0.00327937656633436,0.0029084108249803 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,Crop,100,0.00404295592923784,0.0019251990644438,0.00822025781058035,0.00559204853976238,0.00486637506500823,0.0030617350571807 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,RoughP,100,0.00644153277046238,0.00252423910713441,0.0162105994287237,0.00988095151082199,0.00796460823501309,0.00629421417448711 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,TameP,100,0.00270840599523607,0.00110079046436486,0.00696658084994184,0.00325978967361389,0.00337440291115855,0.00230092417924499 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,WetlandMargin,100,1.86844798686809e-09,9.42942331037853e-10,4.67968633112177e-09,2.06303834717273e-09,2.71691235950851e-09,2.7090235036235e-09 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Decid,100,0.0483763416738863,0.0398894212907587,0.0568569851760146,0.0487558878166638,0.0483216105119213,0.00669095239158589 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Mixedwood,100,0.0335172747893251,0.0253716086487457,0.0456040294151578,0.0339606541972422,0.0347963364140876,0.00843176389394203 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Pine,100,0.025870288524537,0.0180402366793177,0.0315402338066319,0.0258942184497687,0.0253858737473197,0.00544880892070037 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Spruce,100,0.0159024384560565,0.00993370175485552,0.0232561687568493,0.0155269078679998,0.0162100675056413,0.00508560259672311 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,TreedBog,100,0.0293696822831939,0.0244312213786079,0.035053969579329,0.0299192022319043,0.0297291958646359,0.00451750087141942 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,TreedFen,100,0.0294401444709132,0.0237169789847303,0.0352046857452316,0.0299192022319043,0.0293990399416308,0.00472126499199311 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,TreedSwamp,100,0.0304196848425835,0.0233372640368331,0.0435425680565179,0.0299192022319043,0.0315143491357096,0.00979911451639928 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,GrassHerb,100,0.0765617442195729,0.0565164630845311,0.103140308013521,0.0701423452426467,0.0779725306387943,0.0169830885981569 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Shrub,100,0.0640641049505637,0.0439410291195738,0.0874253129094311,0.0701423452426467,0.064749882959758,0.0170543610746251 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,GraminoidFen,100,0.0593930450638716,0.0308313631519059,0.0820371463623189,0.0320501190239092,0.0567492844264685,0.0210420624846225 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Marsh,100,0.0593930450638716,0.0308313631519059,0.0820371463623189,0.0320501190239092,0.0567492844264685,0.0210420624846225 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,ShrubbyBog,100,0.0195059045649605,0.00807597974258628,0.032793286037659,0.0320501190239092,0.0203880483383981,0.0100210971308265 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,ShrubbyFen,100,0.0195059045649605,0.00807597974258628,0.032793286037659,0.0320501190239092,0.0203880483383981,0.0100210971308265 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,ShrubbySwamp,100,0.0256054767053046,0.0145868707573014,0.0358087482415989,0.0320501190239092,0.0258032045083992,0.00824769061937043 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCDecidR,100,0.0602314540179271,0.0353373884170156,0.088834315420329,0.0664247062431656,0.0613956941510392,0.0256979920724621 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCDecid1,100,0.0926568477190087,0.0487135576770351,0.222003838291162,0.0664247062431656,0.122733450443789,0.0709076430071781 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCDecid2,100,0.0451278028988825,0.0107776766627869,0.0783191231143041,0.0664247062431656,0.0450504342354182,0.0273571052745323 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCDecid3,100,0.101444047914825,0.0487135576770351,0.166332436518029,0.0664247062431656,0.106441911631444,0.0461350042135457 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCDecid4,100,0.101444047914825,0.0487135576770351,0.166332436518029,0.0664247062431656,0.106441911631444,0.0461350042135457 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCMixedwoodR,100,0.012934236752073,9.03372503768465e-08,0.0783191231143041,0.0664247062431656,0.0329289049502976,0.0353554964465799 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCMixedwood1,100,0.064949758120954,0.0344224857538416,0.12351939387085,0.0664247062431656,0.0719813846898619,0.0418204112380406 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCMixedwood2,100,0.0722220568938883,0.0377502989532589,0.137756404129837,0.0664247062431656,0.0825034738329284,0.0500862469458366 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCMixedwood3,100,0.101444047914825,0.0487135576770351,0.166332436518029,0.0664247062431656,0.106441911631444,0.0461350042135457 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCMixedwood4,100,0.101444047914825,0.0487135576770351,0.166332436518029,0.0664247062431656,0.106441911631444,0.0461350042135457 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCPineR,100,0.0495927124772698,0.0238135359936229,0.145327051886078,0.041843447803877,0.0756559829215557,0.0654123104927328 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCPine1,100,0.03506390299764,0.0170968329993216,0.0487274161876905,0.041843447803877,0.0341527461993234,0.0127128618520659 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCPine2,100,0.0790855769776833,0.0262934912171496,0.166332436518029,0.041843447803877,0.092796196873904,0.0579189413282626 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCPine3,100,0.0790855769776833,0.0262934912171496,0.166332436518029,0.041843447803877,0.092796196873904,0.0579189413282626 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCPine4,100,0.0790855769776833,0.0262934912171496,0.166332436518029,0.041843447803877,0.092796196873904,0.0579189413282626 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCSpruceR,100,0.0746888085600797,0.0323813640725937,0.216301377211896,0.041843447803877,0.111356428194121,0.113975721852952 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCSpruce1,100,0.0477462671690115,0.0127378541862894,0.148689507912764,0.041843447803877,0.072984309356014,0.0833210412569373 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCSpruce2,100,0.0397302608696762,0.0126878443312949,0.0764729333335468,0.041843447803877,0.0423100444951685,0.0237471193823885 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCSpruce3,100,0.100537562445983,0.0349774912261607,0.166332436518029,0.041843447803877,0.0999813239145242,0.0525924526029354 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,CCSpruce4,100,0.100537562445983,0.0349774912261607,0.166332436518029,0.041843447803877,0.0999813239145242,0.0525924526029354 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,HardLin,100,0.137562711966662,0.0790156188558482,0.181621562906498,0.14014098708019,0.13806256080768,0.0525616101103766 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,EnSoftLin,100,0.115977447267055,0.0920391647558405,0.150721668124623,0.108633567141907,0.120835671573637,0.0254283875597403 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,EnSeismic,100,0.101992217397031,0.0625188290095038,0.129399122502523,0.108633567141907,0.0989818004583902,0.0256324844162361 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,TrSoftLin,100,0.109828682011614,0.0897358921152103,0.149261584014634,0.108633567141907,0.115541501590469,0.026556947508339 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Urban,100,0.137562711966662,0.0790156188558482,0.181621562906498,0.14014098708019,0.13806256080768,0.0525616101103766 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Rural,100,0.137562711966662,0.0790156188558482,0.181621562906498,0.14014098708019,0.13806256080768,0.0525616101103766 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Industrial,100,0.137562711966662,0.0790156188558482,0.181621562906498,0.14014098708019,0.13806256080768,0.0525616101103766 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Well,100,0.123909833378985,0.0271897886988298,0.162292231460633,0.14014098708019,0.108393980905858,0.0539294431784415 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,Crop,100,0.144791889720452,0.119540580794591,0.174827461158003,0.14014098708019,0.145259428581304,0.0217632505978687 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,TameP,100,0.145581027715794,0.119540580794591,0.191083061763293,0.14014098708019,0.153358901468639,0.0390279509769612 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,RoughP,100,0.149025848710316,0.119811400922346,0.223712853233691,0.14014098708019,0.163638344006032,0.0508390518644565 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Decid,100,0.240019218461993,0.203175379952148,0.288939783677506,0.250263196314733,0.245681623832781,0.0386105273824557 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Mixedwood,100,0.262295346891807,0.207997955952666,0.32119517582802,0.267778296796378,0.26639054445169,0.0471832892248065 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Pine,100,0.186398868567176,0.143677737647131,0.237367013524222,0.191627552662266,0.188702044891747,0.0376033748197877 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Spruce,100,0.205218930131629,0.154923711037661,0.262403855127144,0.211937623492894,0.210886855349248,0.0428368846434047 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,TreedBog,100,0.127919000214836,0.0995436563402518,0.163871177444874,0.13349759896445,0.130948511987664,0.0262087908625052 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,TreedFen,100,0.136824142333737,0.104292709627804,0.176326612226722,0.140460621172555,0.138829420494205,0.0282843227699685 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,TreedSwamp,100,0.207538079904233,0.145207355788809,0.27124987818069,0.219333664647888,0.207486766625489,0.0535834023744268 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,GrassHerb,100,0.22356865459654,0.179475959587724,0.292515687393163,0.243208536186666,0.233296294803696,0.0452548481167061 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Shrub,100,0.22705093523844,0.163348810992694,0.300697514416668,0.23362253489809,0.232089286898537,0.053385028699192 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,GraminoidFen,100,0.258808220795066,0.187651928795646,0.350350105101791,0.272335942420037,0.263900382321318,0.0633382433348152 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Marsh,100,0.258808220795066,0.187651928795646,0.350350105101791,0.272335942420037,0.263900382321318,0.0633382433348152 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,ShrubbyBog,100,0.115665176978393,0.085128094742074,0.167182177637953,0.123758665516073,0.121344319956994,0.0360477252952091 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,ShrubbyFen,100,0.115665176978393,0.085128094742074,0.167182177637953,0.123758665516073,0.121344319956994,0.0360477252952091 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,ShrubbySwamp,100,0.180118805638808,0.127612832912588,0.234243636623912,0.188745324781336,0.181095546371362,0.0427781181392449 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCDecidR,100,0.209444589626503,0.151531402005145,0.273723583674672,0.204735734181689,0.216667999142862,0.070635814495866 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCDecid1,100,0.209444589626503,0.145690430754765,0.26549460982574,0.204735734181689,0.207657502566283,0.0490837834364421 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCDecid2,100,0.203718580527138,0.129828630199959,0.251525664679968,0.204735734181689,0.192698343515708,0.0522789450495831 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCDecid3,100,0.204569091612891,0.150348386642298,0.251525664679968,0.204735734181689,0.200908717783713,0.0394943109237779 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCDecid4,100,0.204569091612891,0.150348386642298,0.251525664679968,0.204735734181689,0.200908717783713,0.0394943109237779 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCMixedwoodR,100,0.210330474348428,0.153188635476085,0.308391035242472,0.204735734181689,0.247034826593274,0.149137821145041 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCMixedwood1,100,0.201048634616138,0.118928403069143,0.251525664679968,0.204735734181689,0.187050173191857,0.0632609901994437 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCMixedwood2,100,0.210121113303379,0.153188635476085,0.285094016921618,0.204735734181689,0.225361711186994,0.0880910421620997 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCMixedwood3,100,0.204569091612891,0.150348386642298,0.251525664679968,0.204735734181689,0.200908717783713,0.0394943109237779 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCMixedwood4,100,0.204569091612891,0.150348386642298,0.251525664679968,0.204735734181689,0.200908717783713,0.0394943109237779 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCPineR,100,0.240610019943424,0.169557191754887,0.335126284313014,0.252728055432474,0.246735643696155,0.0666979815396054 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCPine1,100,0.240610019943424,0.169557191754887,0.317067901172578,0.252728055432474,0.241929922949003,0.0605835633635272 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCPine2,100,0.226610862453347,0.16633261409041,0.313984461405726,0.252728055432474,0.233340345125574,0.0584680537308074 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCPine3,100,0.226610862453347,0.16633261409041,0.313984461405726,0.252728055432474,0.233340345125574,0.0584680537308074 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCPine4,100,0.226610862453347,0.16633261409041,0.313984461405726,0.252728055432474,0.233340345125574,0.0584680537308074 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCSpruceR,100,0.217558781386108,0.0347756315816252,0.330186617851412,0.251426565070253,0.209665737730715,0.106744842659019 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCSpruce1,100,0.249194088385742,0.166660020907171,0.471192885074628,0.251426565070253,0.292250045501902,0.163862448965067 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCSpruce2,100,0.237424742990381,0.161661169750931,0.378813645220973,0.251426565070253,0.258479669610987,0.100788739585177 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCSpruce3,100,0.219712059572248,0.153136913461243,0.330186617851412,0.251426565070253,0.233560384379523,0.0763368016580404 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,CCSpruce4,100,0.219712059572248,0.153136913461243,0.330186617851412,0.251426565070253,0.233560384379523,0.0763368016580404 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,HardLin,100,0.108698327828023,0.0465114943177607,0.176247288106262,0.111539574120126,0.106648603388019,0.051044253205058 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,EnSoftLin,100,0.243085023173799,0.173847555733158,0.320506862938923,0.245765939186682,0.244500605706871,0.0596177015379293 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,EnSeismic,100,0.388449253236289,0.266501390713632,0.516765214618719,0.400145157638819,0.389182800490769,0.094689514110549 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,TrSoftLin,100,0.188039391208597,0.12277602181474,0.262634424518356,0.191095088735685,0.190494842480142,0.0554879802781455 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Urban,100,0.108698327828023,0.0465114943177607,0.176247288106262,0.111539574120126,0.106648603388019,0.051044253205058 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Rural,100,0.108698327828023,0.0465114943177607,0.176247288106262,0.111539574120126,0.106648603388019,0.051044253205058 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Industrial,100,0.108698327828023,0.0465114943177607,0.176247288106262,0.111539574120126,0.106648603388019,0.051044253205058 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Well,100,0.123690281485447,0.0431318632056611,0.239588633345073,0.135502134019885,0.130739023805624,0.0806813742228003 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,Crop,100,0.171914958506185,0.132808874437954,0.216753643282233,0.177273110573242,0.17363809992798,0.0326431070075813 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,TameP,100,0.095736652179098,0.0587267620079873,0.152362329799822,0.095283999163032,0.0992581704036899,0.0365296350144877 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,RoughP,100,0.255472812764853,0.163063318006015,0.350688908060417,0.274174686267773,0.255647833172299,0.076001889172638 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,ClaySub,100,0.533287835473067,0.337091215795514,0.674214552653838,0.555220139484692,0.522978285277454,0.128642476978465 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,Other,100,0.533287835473067,0.337091215795514,0.674214552653838,0.555220139484692,0.522978285277454,0.128642476978465 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,RapidDrain,100,0.579323609692961,0.412529306222181,0.756882536944729,0.555220139484692,0.585144758094389,0.141067440625842 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,Loamy,100,0.580241380270824,0.403705189111786,0.722826149896002,0.585624995059316,0.573389558574618,0.121634990610465 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,SandyLoam,100,0.751202117049153,0.561864061182436,0.878072935136269,0.779017339675275,0.738268406258906,0.111389350740941 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,ThinBreak,100,0.55468262636062,0.387810341544162,0.727784914271261,0.555220139484692,0.560093722921092,0.127810954600842 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,Blowout,100,0.55468262636062,0.387810341544162,0.727784914271261,0.555220139484692,0.560093722921092,0.127810954600842 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,Urban,100,0.528182868334466,0.349670886421608,0.748470263227758,0.487723056831298,0.542010095493207,0.159210566535779 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,Rural,100,0.528182868334466,0.349670886421608,0.748470263227758,0.487723056831298,0.542010095493207,0.159210566535779 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,Industrial,100,0.528182868334466,0.349670886421608,0.748470263227758,0.487723056831298,0.542010095493207,0.159210566535779 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,Well,100,0.461984373859166,0.239745688188146,0.712027178253091,0.487723056831298,0.468320630861291,0.16215914070283 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,Mine,100,0.528182868334466,0.349670886421608,0.748470263227758,0.487723056831298,0.542010095493207,0.159210566535779 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,EnSoftLin,100,0.558264524860134,0.352896251214147,0.730085462289024,0.551988691379401,0.553909582116567,0.157517652406738 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,EnSeismic,100,0.558264524860134,0.352896251214147,0.730085462289024,0.551988691379401,0.553909582116567,0.157517652406738 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,HardLin,100,0.528182868334466,0.349670886421608,0.748470263227758,0.487723056831298,0.542010095493207,0.159210566535779 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,TrSoftLin,100,0.55341410580104,0.380052085820553,0.706878939269257,0.551988691379401,0.550867256168873,0.127554988594625 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,Crop,100,0.666523142455711,0.492208920255737,0.787108433458363,0.664605920258047,0.652345599258276,0.110636776854014 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,RoughP,100,0.665433722907965,0.457506640161808,0.787108433458363,0.664605920258047,0.645022708704481,0.116461352215836 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,TameP,100,0.67128093211634,0.492208920255737,0.787108433458363,0.664605920258047,0.654656152845771,0.11010067446273 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,WetlandMargin,100,0.586168608363953,0.361111274829301,0.788567274306412,0.585444465824869,0.57754493700043,0.157188093314973 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,ClaySub,100,0.0767239576273883,0.0474977119628015,0.113480604625688,0.0784267337005646,0.0774692814274093,0.0262896607907176 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,Other,100,0.0767239576273883,0.0474977119628015,0.113480604625688,0.0784267337005646,0.0774692814274093,0.0262896607907176 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,RapidDrain,100,0.08200149457636,0.0528613800096666,0.152844063712585,0.0784267337005646,0.0951523527989561,0.0463238821637706 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,Loamy,100,0.0738299491102142,0.0489866528047458,0.106818587398297,0.0784267337005646,0.0745739381943143,0.0232826810587571 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,SandyLoam,100,0.0564878528987113,0.0206196319967615,0.0928370892417494,0.0784267337005646,0.0586769805598734,0.0318375450408808 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,ThinBreak,100,0.0775020553038078,0.0500616263397111,0.107974156552814,0.0784267337005646,0.0782166750796376,0.0244728811537213 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,Blowout,100,0.0775020553038078,0.0500616263397111,0.107974156552814,0.0784267337005646,0.0782166750796376,0.0244728811537213 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,Urban,100,0.0131531224275559,5.33142026821767e-09,0.0545745730047247,0.00922653651061865,0.0226252746720229,0.0272069300752551 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,Rural,100,0.0131531224275559,5.33142026821767e-09,0.0545745730047247,0.00922653651061865,0.0226252746720229,0.0272069300752551 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,Industrial,100,0.0131531224275559,5.33142026821767e-09,0.0545745730047247,0.00922653651061865,0.0226252746720229,0.0272069300752551 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,Well,100,6.2940213604982e-09,2.81768019690376e-09,0.00923318173608037,0.00922653651061865,0.00326501781019985,0.00750277973128531 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,Mine,100,0.0131531224275559,5.33142026821767e-09,0.0545745730047247,0.00922653651061865,0.0226252746720229,0.0272069300752551 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,EnSoftLin,100,0.0710928441162692,0.01954771629794,0.193402018418337,0.036098691627639,0.0952567693433893,0.0966476294302459 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,EnSeismic,100,0.0710928441162692,0.01954771629794,0.193402018418337,0.036098691627639,0.0952567693433893,0.0966476294302459 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,HardLin,100,0.0131531224275559,5.33142026821767e-09,0.0545745730047247,0.00922653651061865,0.0226252746720229,0.0272069300752551 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,TrSoftLin,100,0.0276322022284876,0.0123928496283907,0.0455122993182532,0.036098691627639,0.0292778287992828,0.0149107601284826 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,Crop,100,0.0275436117264271,0.016298033126281,0.0408369787869045,0.0270020288489522,0.0291348999249318,0.0106865716838931 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,RoughP,100,0.0275514213538021,0.0157806075619722,0.0452278826358776,0.0270020288489522,0.0295748413923245,0.0108781656430521 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,TameP,100,0.0213415409877546,0.00969769599811815,0.0346769051598787,0.0270020288489522,0.0215658988594981,0.00979987769467476 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,WetlandMargin,100,1.323180684423e-08,6.48322499465484e-09,3.93441798102181e-08,2.33837497296009e-08,2.00454102373436e-08,1.84880166229445e-08 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Decid,100,0.0444561602432516,0.037100006621029,0.0506313929279378,0.0449203614522768,0.0443402313884692,0.00534156706192387 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Mixedwood,100,0.0261238689429193,0.0181005588458572,0.0380198050510442,0.0273241474240624,0.0265039273343915,0.00766908112360962 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Pine,100,0.017462026824314,0.0133021705913876,0.022541027258153,0.0176556438960937,0.017768213720847,0.0038041220082187 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Spruce,100,0.0125930411026172,0.0084262697837341,0.019816610999655,0.0137961282417974,0.0131964430065491,0.00436144531763343 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,TreedBog,100,0.0286213492805131,0.022115295008687,0.0347885961021513,0.028635565125395,0.028575936803779,0.00447682949694337 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,TreedFen,100,0.030075387250229,0.0243750078825827,0.0383287580241218,0.0316720599445968,0.0307574083614567,0.00543234513641382 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,TreedSwamp,100,0.0538817961329222,0.034386590465475,0.0826910624457981,0.0562955719457929,0.0565941024764403,0.0181869741104341 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,GrassHerb,100,0.078173184418061,0.0566161535227547,0.105302547958747,0.0783755759784838,0.0796141443487759,0.0182757885745762 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Shrub,100,0.0655581493166911,0.04643607711841,0.0950514877217542,0.0693588214886093,0.0691706993270102,0.0176932930689596 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,GraminoidFen,100,0.0843529215333654,0.0621530651309933,0.111381066191143,0.0834692933781887,0.0843657480515468,0.0207112748144593 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Marsh,100,0.0843529215333654,0.0621530651309933,0.111381066191143,0.0834692933781887,0.0843657480515468,0.0207112748144593 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,ShrubbyBog,100,0.0166734933948774,0.00909035537231827,0.025448076696349,0.0174941754511016,0.0169001418386709,0.00679355366306236 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,ShrubbyFen,100,0.0166734933948774,0.00909035537231827,0.025448076696349,0.0174941754511016,0.0169001418386709,0.00679355366306236 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,ShrubbySwamp,100,0.0193725369870357,0.0117788468950098,0.030604829961507,0.0195830589922788,0.0198875512388377,0.00669507071664844 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCDecidR,100,0.0642220617983857,0.0449861194124719,0.0867405322392189,0.063423274192491,0.0670154530774801,0.0259648270142619 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCDecid1,100,0.0640890826736864,0.0437457491093014,0.0827378565096553,0.063423274192491,0.063709487620192,0.0180687814496852 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCDecid2,100,0.0600399818305216,0.0353164373465796,0.0809715112975173,0.063423274192491,0.0588030271673895,0.0194884164658998 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCDecid3,100,0.065843509896461,0.046651378528195,0.106369517230159,0.063423274192491,0.0695303096379644,0.023216539737732 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCDecid4,100,0.065843509896461,0.046651378528195,0.106369517230159,0.063423274192491,0.0695303096379644,0.023216539737732 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCMixedwoodR,100,0.0645185414023919,0.0447645039543755,0.106616066138459,0.063423274192491,0.089466777587602,0.121717877042489 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCMixedwood1,100,0.0601176211560154,0.0378261854923728,0.0809715112975173,0.063423274192491,0.0602680534444554,0.0180922555643358 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCMixedwood2,100,0.0637261257096893,0.0378261854923728,0.0866990015006504,0.063423274192491,0.0679379589332609,0.0387077007024671 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCMixedwood3,100,0.065843509896461,0.046651378528195,0.106369517230159,0.063423274192491,0.0695303096379644,0.023216539737732 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCMixedwood4,100,0.065843509896461,0.046651378528195,0.106369517230159,0.063423274192491,0.0695303096379644,0.023216539737732 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCPineR,100,0.0437542113251143,0.0262331530100579,0.069410738418215,0.0429017164110991,0.0471291697050635,0.0201573791760323 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCPine1,100,0.0410867424750988,0.0256330650491266,0.0639435643337463,0.0429017164110991,0.0424953275007428,0.0139281727054407 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCPine2,100,0.044958542436542,0.0289432608676509,0.091484202234522,0.0429017164110991,0.0524346205913024,0.0273176831993445 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCPine3,100,0.044958542436542,0.0289432608676509,0.091484202234522,0.0429017164110991,0.0524346205913024,0.0273176831993445 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCPine4,100,0.044958542436542,0.0289432608676509,0.091484202234522,0.0429017164110991,0.0524346205913024,0.0273176831993445 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCSpruceR,100,0.0315800332135219,0.0134635582610135,0.058282755881521,0.030964897308693,0.0352898096814781,0.0238165546816501 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCSpruce1,100,0.0308825280851181,0.0120925032534051,0.0560803081824228,0.030964897308693,0.0322245392824585,0.0185553889868395 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCSpruce2,100,0.0324661690525685,0.0163909514561722,0.056284262613807,0.030964897308693,0.0356209018469371,0.0177540921019994 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCSpruce3,100,0.0369204072572823,0.0163909514561722,0.091484202234522,0.030964897308693,0.0442542908589931,0.0307260051732227 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,CCSpruce4,100,0.0369204072572823,0.0163909514561722,0.091484202234522,0.030964897308693,0.0442542908589931,0.0307260051732227 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,HardLin,100,0.119996638285634,0.0909614070603904,0.220524917117628,0.118992631310028,0.137996135143596,0.0665820223965668 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,EnSoftLin,100,0.122673086423153,0.0861472947521106,0.168405693867256,0.0980871465772413,0.12371992337013,0.0321284346617133 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,EnSeismic,100,0.0622178137940698,0.0328627528700011,0.105133332248024,0.0980871465772413,0.0665476603239618,0.029499006638478 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,TrSoftLin,100,0.0892003632360302,0.0618165561944981,0.122277091488608,0.0980871465772413,0.0922389123590745,0.0268440716822624 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Urban,100,0.119996638285634,0.0909614070603904,0.220524917117628,0.118992631310028,0.137996135143596,0.0665820223965668 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Rural,100,0.119996638285634,0.0909614070603904,0.220524917117628,0.118992631310028,0.137996135143596,0.0665820223965668 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Industrial,100,0.119996638285634,0.0909614070603904,0.220524917117628,0.118992631310028,0.137996135143596,0.0665820223965668 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Well,100,0.112386014801604,0.056651220945175,0.144906741332509,0.118992631310028,0.1114735703638,0.0444144027795331 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,Crop,100,0.119827981421536,0.0965342122483277,0.139292184612742,0.118992631310028,0.11870232363679,0.0173322679703853 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,TameP,100,0.120609153765577,0.0965342122483277,0.145168372752322,0.118992631310028,0.121796161869388,0.0227520131689065 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,RoughP,100,0.119074813530677,0.0957136035674711,0.147322500341392,0.118992631310028,0.119814857778302,0.022564765908845 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Decid,100,0.107243899372008,0.0845520330158888,0.157702689505775,0.11166440245298,0.114317143174846,0.0276163886512792 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Mixedwood,100,0.0649640228851936,0.0431159406082413,0.0997367262284479,0.0655577834788829,0.0682604376250072,0.0222794531877773 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Pine,100,0.0433681025472427,0.0320819449412026,0.0606120594198091,0.0434917161194807,0.0443698291231618,0.0115536377580468 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Spruce,100,0.0677531352721661,0.0477438363095222,0.101548927940217,0.0695392350566295,0.0720252586633572,0.0216299730257051 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,TreedBog,100,0.0259169154169619,0.0181387930034633,0.0452201104948205,0.0324223167296191,0.0280121552348723,0.0103606250720451 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,TreedFen,100,0.0387669395896791,0.027336567525976,0.0570485281185731,0.0324223167296191,0.0408051400151834,0.0115504644345017 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,TreedSwamp,100,0.128606270468312,0.0981022904693136,0.196845506524735,0.138924945537604,0.140348385022712,0.0414176471697624 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,GrassHerb,100,0.0632411252470609,0.0414444797200237,0.0871164175500363,0.0674988002326723,0.0650191141172866,0.0222757737094382 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Shrub,100,0.0853860238226719,0.047895333833239,0.124847144923538,0.0845204524428601,0.0844079266578619,0.0293533616988448 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,GraminoidFen,100,0.065764433924027,0.0397899875184043,0.101814032733513,0.0715579377130804,0.0673759105765978,0.022003294767371 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Marsh,100,0.065764433924027,0.0397899875184043,0.101814032733513,0.0715579377130804,0.0673759105765978,0.022003294767371 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,ShrubbyBog,100,0.0549426614094726,0.0368508397242025,0.0925614168105729,0.0715579377130804,0.0592130598331129,0.0210000537376886 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,ShrubbyFen,100,0.0549426614094726,0.0368508397242025,0.0925614168105729,0.0715579377130804,0.0592130598331129,0.0210000537376886 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,ShrubbySwamp,100,0.0946938397223873,0.0621037317867852,0.13438372834748,0.0715579377130804,0.0978309581105182,0.0306720693913251 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCDecidR,100,0.0952511866345784,0.0619555615597258,0.160443838952529,0.0987996864776,0.10532515191505,0.0439374587684689 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCDecid1,100,0.0994200408390495,0.0648432917133283,0.163686081123677,0.0987996864776,0.109443228135002,0.0428548799774937 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCDecid2,100,0.0994200408390495,0.0641033253365466,0.160443838952529,0.0987996864776,0.110240178073763,0.050532952229048 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCDecid3,100,0.0812502184496068,0.0504146686928513,0.142149311541224,0.0987996864776,0.0879406109921444,0.0328638706481359 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCDecid4,100,0.0812502184496068,0.0504146686928513,0.142149311541224,0.0987996864776,0.0879406109921444,0.0328638706481359 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCMixedwoodR,100,0.0794124939180167,6.97946200895998e-08,0.142149311541224,0.0987996864776,0.0718702410895562,0.0512261878056619 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCMixedwood1,100,0.0938325786477807,0.0592700712115359,0.152653182838955,0.0987996864776,0.105009340782573,0.0522028585351533 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCMixedwood2,100,0.0806869100683688,0.0273374488009315,0.142149311541224,0.0987996864776,0.083436544752264,0.0396228882166925 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCMixedwood3,100,0.0812502184496068,0.0504146686928513,0.142149311541224,0.0987996864776,0.0879406109921444,0.0328638706481359 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCMixedwood4,100,0.0812502184496068,0.0504146686928513,0.142149311541224,0.0987996864776,0.0879406109921444,0.0328638706481359 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCPineR,100,0.0739377004823517,0.0231115545004555,0.142151432987828,0.0987996864776,0.0758527409396893,0.0431538538443742 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCPine1,100,0.0842926472859167,0.0499932471533785,0.158341328760154,0.0987996864776,0.0970819859317556,0.0433477388901912 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCPine2,100,0.0755452911044962,0.0455926997237236,0.142151432987828,0.0987996864776,0.0831231488839342,0.0362893143971574 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCPine3,100,0.0755452911044962,0.0455926997237236,0.142151432987828,0.0987996864776,0.0831231488839342,0.0362893143971574 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCPine4,100,0.0755452911044962,0.0455926997237236,0.142151432987828,0.0987996864776,0.0831231488839342,0.0362893143971574 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCSpruceR,100,0.0834077165838304,1.50975224436868e-07,0.147982945460932,0.0987996864776,0.0892409483831851,0.0574673781841595 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCSpruce1,100,0.122165399956484,0.0692740788764443,0.295617312473175,0.0987996864776,0.150311505632027,0.102001413949861 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCSpruce2,100,0.0984937381029724,0.061007712241052,0.161641044437261,0.0987996864776,0.114452119198554,0.0645400699949785 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCSpruce3,100,0.081680638722937,0.048346244109403,0.146280761242648,0.0987996864776,0.0957402213122577,0.045754620314565 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,CCSpruce4,100,0.081680638722937,0.048346244109403,0.146280761242648,0.0987996864776,0.0957402213122577,0.045754620314565 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,HardLin,100,0.0622005584964046,0.0155708690123517,0.0942842940530228,0.0681539858129173,0.0576086988588489,0.0330756668249431 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,EnSoftLin,100,0.0495456921417529,0.0279910090746474,0.0837640384969403,0.0630037477010182,0.0522440560032208,0.0214309285636471 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,EnSeismic,100,0.062255880812458,0.0419857552645485,0.10269867960262,0.0630037477010182,0.0679608909405817,0.0254764787867254 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,TrSoftLin,100,0.0671954089649426,0.040823576274068,0.122879877205894,0.0630037477010182,0.0754061507754859,0.0312146223869496 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Urban,100,0.0622005584964046,0.0155708690123517,0.0942842940530228,0.0681539858129173,0.0576086988588489,0.0330756668249431 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Rural,100,0.0622005584964046,0.0155708690123517,0.0942842940530228,0.0681539858129173,0.0576086988588489,0.0330756668249431 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Industrial,100,0.0622005584964046,0.0155708690123517,0.0942842940530228,0.0681539858129173,0.0576086988588489,0.0330756668249431 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Well,100,0.0664403051505919,0.0160957886518951,0.109353255031191,0.0681539858129173,0.0725534483476616,0.0579923729970705 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,Crop,100,0.0642251847114456,0.0439655343954587,0.0860185805182751,0.0681539858129173,0.0647114538836377,0.0171485903754233 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,TameP,100,0.0610874685123159,0.0364198176167285,0.0860095867040654,0.0681539858129173,0.0604546536098686,0.0196815232845098 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,RoughP,100,0.0927152604237154,0.0560381811573657,0.20382439023557,0.0681539858129173,0.112644985094648,0.0573647826032872 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,ClaySub,100,0.724846700762001,0.557935400928003,0.846108860281145,0.714170375192759,0.710061894176626,0.108263213621852 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,Other,100,0.724846700762001,0.557935400928003,0.846108860281145,0.714170375192759,0.710061894176626,0.108263213621852 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,RapidDrain,100,0.727012018410659,0.581135049341875,0.841708065747562,0.714170375192759,0.713996225476568,0.106238294250961 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,Loamy,100,0.722769384879788,0.557935400928003,0.841708065747562,0.714170375192759,0.708416576650525,0.105452969185458 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,SandyLoam,100,0.75753113636802,0.618868453827049,0.880323891978106,0.714170375192759,0.754881311727094,0.103887712803517 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,ThinBreak,100,0.725646118258459,0.581135049341875,0.841708065747562,0.714170375192759,0.711315695212958,0.106036553015519 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,Blowout,100,0.725646118258459,0.581135049341875,0.841708065747562,0.714170375192759,0.711315695212958,0.106036553015519 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,Urban,100,0.771150614234565,0.635112870123813,0.900123066075508,0.757773549137013,0.772412096253203,0.104051036924218 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,Rural,100,0.771150614234565,0.635112870123813,0.900123066075508,0.757773549137013,0.772412096253203,0.104051036924218 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,Industrial,100,0.771150614234565,0.635112870123813,0.900123066075508,0.757773549137013,0.772412096253203,0.104051036924218 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,Well,100,0.73053933465079,0.520656388756287,0.881383501451225,0.757773549137013,0.705168731628601,0.12954610394624 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,Mine,100,0.771150614234565,0.635112870123813,0.900123066075508,0.757773549137013,0.772412096253203,0.104051036924218 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,EnSoftLin,100,0.659789232660266,0.49152975716811,0.83667543927716,0.628634628150965,0.663448860647529,0.135064085580063 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,EnSeismic,100,0.659789232660266,0.49152975716811,0.83667543927716,0.628634628150965,0.663448860647529,0.135064085580063 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,HardLin,100,0.771150614234565,0.635112870123813,0.900123066075508,0.757773549137013,0.772412096253203,0.104051036924218 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,TrSoftLin,100,0.644308396670432,0.479795836836892,0.793084685623838,0.628634628150965,0.631767133713847,0.125947396308499 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,Crop,100,0.766204147935822,0.62680767682252,0.882893305948486,0.757773549137013,0.761913142309583,0.088945077844005 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,RoughP,100,0.75460470620176,0.591581030632153,0.871913328847719,0.757773549137013,0.734380657549024,0.113501385049794 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,TameP,100,0.76986130521123,0.636663229892197,0.880347374752368,0.757773549137013,0.763626680392289,0.0907760343687918 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,WetlandMargin,100,0.881552869370832,0.722938913473091,0.953885216330375,0.870880735779777,0.85812287515435,0.0938528715533264 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,ClaySub,100,0.00896286822999175,0.00378717946942229,0.0216131354950192,0.0141945654676909,0.0109669977464623,0.00748234410228791 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,Other,100,0.00896286822999175,0.00378717946942229,0.0216131354950192,0.0141945654676909,0.0109669977464623,0.00748234410228791 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,RapidDrain,100,0.0114649622370839,0.00393503010001971,0.0251821725003416,0.0141945654676909,0.013361491442824,0.00835790305733527 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,Loamy,100,0.00974383110419694,0.00387507252285811,0.0215346314941135,0.0141945654676909,0.0112054099874938,0.00674568345668748 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,SandyLoam,100,0.0118169017807942,0.00429242136888576,0.0265031593781332,0.0141945654676909,0.0137045818268432,0.00836831673962997 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,ThinBreak,100,0.0211593752752655,0.0089821287006722,0.04901760697848,0.0141945654676909,0.0257381311100531,0.0171990610230458 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,Blowout,100,0.0211593752752655,0.0089821287006722,0.04901760697848,0.0141945654676909,0.0257381311100531,0.0171990610230458 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,Urban,100,9.10445054594609e-10,3.41328651364901e-10,3.02584446675333e-09,9.67698723675421e-10,6.84191152650917e-05,0.000351514790870777 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,Rural,100,9.10445054594609e-10,3.41328651364901e-10,3.02584446675333e-09,9.67698723675421e-10,6.84191152650917e-05,0.000351514790870777 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,Industrial,100,9.10445054594609e-10,3.41328651364901e-10,3.02584446675333e-09,9.67698723675421e-10,6.84191152650917e-05,0.000351514790870777 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,Well,100,7.85489291790109e-10,3.25119032095615e-10,1.9815670788834e-09,8.90552084804154e-10,6.84189160531857e-05,0.000351514830035491 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,Mine,100,9.10445054594609e-10,3.41328651364901e-10,3.02584446675333e-09,9.67698723675421e-10,6.84191152650917e-05,0.000351514790870777 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,EnSoftLin,100,0.0245065397818223,0.00242784926810093,0.0719626476462709,0.0367985674989128,0.0330796146926706,0.0351002616230795 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,EnSeismic,100,0.0245065397818223,0.00242784926810093,0.0719626476462709,0.0367985674989128,0.0330796146926706,0.0351002616230795 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,HardLin,100,9.10445054594609e-10,3.41328651364901e-10,3.02584446675333e-09,9.67698723675421e-10,6.84191152650917e-05,0.000351514790870777 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,TrSoftLin,100,0.00259596123294914,0.000871696328205532,0.00615496447092323,0.00374705166332367,0.00318857103583577,0.00255912170526302 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,Crop,100,0.00457581035458638,0.00185228013185688,0.00871465794862247,0.00559204853976232,0.00496456100961841,0.00302535819148605 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,RoughP,100,0.00603817061316427,0.00187960976151414,0.0164936877904284,0.0098809515108219,0.00739721970400612,0.005226987342806 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,TameP,100,0.00287699997509836,0.00120990672802957,0.00793131683825911,0.00325978967361386,0.00365708128385357,0.00272803609332312 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,WetlandMargin,100,1.89440605216493e-09,7.57113954147194e-10,4.83536488020359e-09,2.06303834717179e-09,2.43843084589693e-09,1.95460507177187e-09 mite,mite,north,,,Ceratozetes.gracilis,WhiteSpruce,100,0.07938259004759,0.0719527394304755,0.0875845851932171,0.0783644708180673,0.0799208933542465,0.00624681141859307 mite,mite,north,,,Ceratozetes.gracilis,Pine,100,0.0571490439101831,0.0530628627411147,0.0629520262365782,0.0561087495328157,0.0576497982383193,0.00368213926355443 mite,mite,north,,,Ceratozetes.gracilis,Deciduous,100,0.121114071952757,0.110635619991547,0.13539009654291,0.120032888034155,0.122127477972057,0.00906642761340119 diff --git a/3_output/exp_000_parity_v2/tables/metric_summary.csv b/3_output/exp_000_parity_v2/tables/metric_summary.csv index ecfde92..d5de5e8 100644 --- a/3_output/exp_000_parity_v2/tables/metric_summary.csv +++ b/3_output/exp_000_parity_v2/tables/metric_summary.csv @@ -279,118 +279,118 @@ lichen,lichen,south,,,Physcia.adscendens,oob_v2val_Full,99,0.814451917130489,0.7 lichen,lichen,south,,,Physcia.adscendens,oob_v2val_Full_Truncated,99,0.814451917130489,0.796076806462607,0.831534146727906,,0.814363415003148,0.0130791932968188 lichen,lichen,south,,,Physcia.adscendens,oob_v2val_Full_Joint,99,0.81242151354924,0.791924831211599,0.828123603004887,,0.811221663413056,0.0135783263054518 lichen,lichen,south,,,Physcia.adscendens,oob_v2val_Full_Joint_Truncated,99,0.81242151354924,0.791924831211599,0.828123603004887,,0.811225340674517,0.0135803458787664 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,insample_auc,100,0.823632636160737,0.814781230564576,0.831408128980217,0.820911148938545,0.823675923645461,0.00684650079811751 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,insample_deviance_explained,100,0.279791843738347,0.261697566054768,0.292547470573807,0.272167552780268,0.279268031798116,0.0123653985651821 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,insample_rmse,100,0.325947657311819,0.321499471172476,0.331112157543476,0.327783381237672,0.326268421037373,0.0036956317406385 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,insample_spearman,100,0.462005079284434,0.44910358192168,0.473619794940904,0.457622037926128,0.46163002365065,0.0098966631801109 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,insample_calibration_slope,100,0.974719951819278,0.940060183795489,1.01331111594891,0.987210797047211,0.976909872220407,0.0281027915235889 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,insample_prevalence,100,0.225366262673778,0.220070866141732,0.230634088644114,0.22526523562793,0.225265109443517,0.00465839538219897 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,insample_n,100,2564.5,2538.9,2588.1,4053,2577.75,150.218989976284 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,oob_auc,99,0.808896640826873,0.79100198897739,0.827548123693069,,0.80902368196587,0.0125632085653662 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,oob_deviance_explained,99,0.243486713148957,0.193079357602948,0.27747427555585,,0.236970758173865,0.0412490070366592 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,oob_rmse,99,0.331823647228338,0.323085219723366,0.340284753308317,,0.3315507150309,0.00655753714958091 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,oob_spearman,99,0.4394124176263,0.417905127967147,0.465739312197084,,0.440143399202189,0.0180559739989518 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,oob_calibration_slope,99,0.93389114249429,0.850341206695487,1.01749958453456,,0.935850683714827,0.065644693475144 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,oob_prevalence,99,0.224915824915825,0.215935838563886,0.234336783397599,,0.225278900992558,0.00808238984604556 -mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,oob_n,100,1488.5,1464.9,1514.1,0,1475.25,150.218989976284 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,insample_auc,100,0.61475601886175,0.603679345102737,0.627353730564579,0.614800455189874,0.615250750441242,0.00876252475511229 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,insample_deviance_explained,100,0.032556283657145,0.0243296542272488,0.039523255842162,0.0327130199155846,0.0322154935907449,0.00570900014754448 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,insample_rmse,100,0.440780068007451,0.437909796174194,0.443705369273563,0.440494520447441,0.44087693597343,0.00236282948666457 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,insample_spearman,100,0.169333701727882,0.148930684100538,0.188146271759874,0.166867112583273,0.168773334408033,0.014769928213376 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,insample_calibration_slope,100,0.847565289100029,0.777129508768977,0.945716349335048,0.901041907949805,0.851701384066199,0.0618664233849605 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,insample_prevalence,100,0.321224265570394,0.314927013046706,0.329608505982353,0.321243523316062,0.321726728813055,0.00570583516475525 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,insample_n,100,2562.5,2537.8,2585.2,4053,2576.41,150.50897453023 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,oob_auc,99,0.59059424536098,0.573979407712817,0.607160496358349,,0.591330408828981,0.012632611891188 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,oob_deviance_explained,99,0.0121132026875843,-0.00570566280736071,0.0241511023697404,,0.00982507363553751,0.0137967405537802 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,oob_rmse,99,0.445175541404113,0.440470485230338,0.448886295412126,,0.44502064145941,0.00386271327949588 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,oob_spearman,99,0.130783814760799,0.101891242189749,0.155800550653179,,0.130108564329046,0.020060042519966 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,oob_calibration_slope,99,0.643049641195582,0.517287601824814,0.821946111710213,,0.662172316955322,0.120921477359268 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,oob_prevalence,99,0.321309285237141,0.306674120479106,0.332295711996362,,0.320409946399226,0.00981841561898746 -mammal,mammal_summer,north,summer,hurdle,Moose_Summer,oob_n,100,1490.5,1467.8,1515.2,0,1476.59,150.50897453023 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,insample_auc,99,0.635001427076396,0.606104262619653,0.657438718250968,0.634240369366988,0.633224007711147,0.0193628357203509 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,insample_deviance_explained,99,0.026440006660178,0.0180120188346803,0.0396931836083979,0.0264112493368064,0.0275214846978343,0.00892330854894092 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,insample_rmse,99,0.421280362710698,0.414044874992987,0.426531332439636,0.421286846656193,0.420762330869611,0.00502970729821291 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,insample_spearman,99,0.180460025029286,0.146567810345527,0.214594440680258,0.175369074792115,0.180194171413398,0.028556375742863 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,insample_calibration_slope,99,0.913890378466184,0.820802523135607,1.01427035726621,1.00531761653027,0.920827083468546,0.0834520299703545 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,insample_prevalence,99,0.690265486725664,0.677825953587333,0.705306871830739,0.691079812206573,0.690722612448517,0.0106422104740515 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,insample_n,99,675,663,685.4,1065,678.252525252525,40.2928928341685 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,oob_auc,98,0.591021454838246,0.551176084882234,0.62061474225386,,0.58902610051224,0.0277690972455097 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,oob_deviance_explained,98,-0.000836625554455406,-0.0331073359910842,0.015317940018266,,-0.0142653121864615,0.0767808279247067 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,oob_rmse,98,0.429820967393949,0.42171561998268,0.442398381954269,,0.431263519646702,0.008314809734625 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,oob_spearman,98,0.110037755841753,0.0541619409153014,0.160705561767885,,0.109089508895746,0.0419990186736059 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,oob_calibration_slope,98,0.602514054060098,0.232115218957736,0.865596324101407,,0.568328155159037,0.247381950253688 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,oob_prevalence,98,0.69250645994832,0.665551467442457,0.71439588688946,,0.69177626225944,0.0183678991223671 -mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,oob_n,99,390,379.6,402,0,386.747474747475,40.2928928341685 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,insample_auc,100,0.786511166542812,0.769504152294664,0.804093618520575,0.786049924840508,0.78746671701533,0.0135173426522638 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,insample_deviance_explained,100,0.21040174777469,0.184564736775399,0.242108898186897,0.205216892616348,0.212563634530583,0.0219479248585407 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,insample_rmse,100,0.343613620192013,0.334680149069424,0.351940369457748,0.345153854253612,0.344018831580113,0.00644192047176435 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,insample_spearman,100,0.38143205353404,0.353940396364084,0.410801187004518,0.381089371361329,0.382473611072285,0.0208009959103412 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,insample_calibration_slope,100,0.984182683497037,0.904891334217879,1.06778036362132,0.986324497785332,0.988817389149015,0.0647894673241446 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,insample_prevalence,100,0.182355333743735,0.168834754036547,0.195212578714749,0.182159624413146,0.182097557830322,0.00986022769722504 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,insample_n,100,674,660.9,691.1,1065,678.58,40.7453734127625 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,oob_auc,99,0.766688006828852,0.728742672765759,0.797692233844295,,0.766798974533161,0.0253241527196272 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,oob_deviance_explained,99,0.160410765184501,-0.0354375437473713,0.21724314670011,,0.13739310872143,0.0906860074411385 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,oob_rmse,99,0.350823352193687,0.337300969232687,0.368279364518069,,0.35201170608914,0.0122095118789058 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,oob_spearman,99,0.356733670694178,0.298765552548449,0.401384960970594,,0.354627831294653,0.0368668870982284 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,oob_calibration_slope,99,0.887224345060846,0.665784227384331,1.20908224079875,,0.901971450944314,0.19515811645007 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,oob_prevalence,99,0.181578947368421,0.160028119078184,0.205617057719676,,0.182250919667934,0.0169141196650609 -mammal,mammal_summer,south,summer,hurdle,Moose_Summer,oob_n,100,391,373.9,404.1,0,386.42,40.7453734127625 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,insample_auc,100,0.840215938467687,0.83092115117171,0.849787411243392,0.839440907842931,0.839669355484163,0.00698601958736087 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,insample_deviance_explained,100,0.295983987195951,0.279264150591629,0.313078108968483,0.294409607036222,0.295680875808017,0.0131751486193903 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,insample_rmse,100,0.300488510882096,0.296257654339334,0.306287986982839,0.30069616131153,0.300981068877169,0.00414792565565438 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,insample_spearman,100,0.464310589597897,0.45159625601271,0.478558931582411,0.463240577826878,0.464694557266924,0.0101849701273848 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,insample_calibration_slope,100,0.972671886418681,0.93201508456542,1.02152533613636,0.982050704865843,0.975156802790362,0.0334477030664325 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,insample_prevalence,100,0.201888432073211,0.197431539212855,0.208159731764965,0.201356917858008,0.202485373586251,0.0045069479986566 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,insample_n,100,2609,2585,2637.1,4127,2624.81,152.9496976133 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,oob_auc,99,0.83021391101824,0.813671018298189,0.846159487988212,,0.830184785139306,0.0136971622845655 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,oob_deviance_explained,99,0.273408193489382,0.229352594121753,0.302275051480294,,0.265737070933002,0.0415956641165299 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,oob_rmse,99,0.304113428131756,0.295100478568192,0.311362926507058,,0.303610679034069,0.00725152029521637 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,oob_spearman,99,0.448324145179786,0.427614830976871,0.47442464641456,,0.448358238154674,0.018797957397872 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,oob_calibration_slope,99,0.943994670763352,0.852757151203258,1.04492756918031,,0.941961322964594,0.0753885220594678 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,oob_prevalence,99,0.200259235255995,0.189775542999326,0.208169005020511,,0.199411477918602,0.00778894989308338 -mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,oob_n,100,1518,1489.9,1542,0,1502.19,152.9496976133 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,insample_auc,100,0.67821310892138,0.663270440192927,0.695381764999782,0.668767750495832,0.678446035955246,0.0124541268014995 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,insample_deviance_explained,100,0.0786332530404745,0.0665566425621889,0.0944247372062849,0.0713753243188524,0.0795171549028447,0.0109910722359914 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,insample_rmse,100,0.332560139164442,0.327709275223946,0.336999200640023,0.332910274284271,0.332573560250602,0.00403872831084821 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,insample_spearman,100,0.223831736114593,0.205313082589125,0.244481495615383,0.211955551417089,0.22398144430408,0.0152979471818789 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,insample_calibration_slope,100,0.917861855484528,0.84216777747185,0.991483856313351,0.94812085347111,0.917671809987021,0.0580754453980805 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,insample_prevalence,100,0.149990410433449,0.143887902486318,0.155025771607528,0.148776350860189,0.149556674913188,0.00428400474177712 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,insample_n,100,2610,2586.7,2642.1,4127,2627.97,153.021715960695 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,oob_auc,99,0.650339977044136,0.626283007371369,0.674110216571647,,0.650601001952465,0.0195432697044396 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,oob_deviance_explained,99,0.0437348310014123,-0.0244103779855468,0.070888637861963,,0.0329829742284183,0.0426626562802078 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,oob_rmse,99,0.335213172770143,0.328331589790136,0.342856337727437,,0.335001854069548,0.00684548448504912 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,oob_spearman,99,0.189218625360862,0.156934768956714,0.217963256910742,,0.18874779877311,0.0243350933657214 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,oob_calibration_slope,99,0.765031662278474,0.564319012355523,0.978273019865697,,0.768379952207126,0.174136095858455 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,oob_prevalence,99,0.146603900470746,0.138208405767461,0.157249189724539,,0.147431774997664,0.00743974221077868 -mammal,mammal_winter,north,winter,hurdle,Moose_Winter,oob_n,100,1517,1484.9,1540.3,0,1499.03,153.021715960695 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,insample_auc,100,0.691441153027097,0.672240387183235,0.706212764410456,0.689082366331686,0.690562117093511,0.013224030335971 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,insample_deviance_explained,100,0.067567099805677,0.0560696673854207,0.0822942032238073,0.0670547674954939,0.0687384650860427,0.0107595907570248 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,insample_rmse,100,0.40682238205156,0.401865860344406,0.410771399612378,0.406888363248781,0.406494036967596,0.00369707913198616 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,insample_spearman,100,0.299224586077027,0.269355636443703,0.329699476853509,0.296933239332004,0.299798019263556,0.0224654844995897 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,insample_calibration_slope,100,0.992627010851258,0.920219152025468,1.09520751465362,1.03306130883064,0.999110545445596,0.0672957533751347 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,insample_prevalence,100,0.603312544835565,0.587842923920052,0.618115052537349,0.604735883424408,0.603077490652041,0.012677090847138 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,insample_n,100,695,682,708.1,1098,699.19,41.3864992869936 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,oob_auc,99,0.664455665867854,0.639651542814446,0.690741654078452,,0.664981037722417,0.0209964276277022 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,oob_deviance_explained,99,0.0446136452786412,0.00619180902025967,0.0633330011439923,,0.0346613411644444,0.0480802066560122 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,oob_rmse,99,0.413228278046299,0.406653224893685,0.424163637351331,,0.414378421096751,0.00688416789861981 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,oob_spearman,99,0.261506753321649,0.206422225163459,0.30115717511071,,0.256292271922553,0.0374825646865808 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,oob_calibration_slope,99,0.817106846468162,0.635134217999263,1.06374115196959,,0.843268239144903,0.196769999721646 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,oob_prevalence,99,0.6075,0.580323143750253,0.635349582154713,,0.607575707802926,0.0220264746278348 -mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,oob_n,100,403,389.9,416,0,398.81,41.3864992869936 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,insample_auc,100,0.820968430916565,0.801436993738574,0.84112134496232,0.819927852628077,0.82091181727687,0.0184559534681181 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,insample_deviance_explained,100,0.239038869889767,0.21013927948378,0.277280840586616,0.234870911391229,0.241677933946565,0.0308121103886393 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,insample_rmse,100,0.222369754770973,0.210112146550844,0.231243986964309,0.223331046901821,0.221987917660098,0.00767650273954978 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,insample_spearman,100,0.312958724627694,0.294454461225426,0.33498073496289,0.313170610278462,0.313004953935226,0.0185636031576594 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,insample_calibration_slope,100,0.883711120332416,0.748929633856669,1.03639244285071,0.926166715024209,0.898788757716915,0.120010897496752 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,insample_prevalence,100,0.0873239436619718,0.0790283819028552,0.0939931596468854,0.0874316939890711,0.0868006891304234,0.00618273855118738 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,insample_n,100,696,681,709.1,1098,699.13,41.7903993130895 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,oob_auc,99,0.783524426719841,0.742644818301232,0.814561651279978,,0.780432466500186,0.0291483446368632 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,oob_deviance_explained,99,0.154109596533938,-0.020676255891081,0.207600467554891,,0.103607468201795,0.17022756036237 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,oob_rmse,99,0.232970422089708,0.214829244725729,0.249813879153275,,0.233231327846636,0.0131440394319924 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,oob_spearman,99,0.278122302398705,0.235207424325587,0.312315920243565,,0.2752488104388,0.0295350486822564 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,oob_calibration_slope,99,0.631374792805361,0.431072836936144,0.94521686736471,,0.670756173356288,0.251404463462431 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,oob_prevalence,99,0.0876288659793814,0.0760837438423646,0.101576468326444,,0.0885136759760043,0.0105262059033636 -mammal,mammal_winter,south,winter,hurdle,Moose_Winter,oob_n,100,402,388.9,417,0,398.87,41.7903993130895 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,insample_auc,100,0.822810789330024,0.815604492419304,0.830429853628498,0.820911148938545,0.822944949558785,0.00606674649691594 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,insample_deviance_explained,100,0.277207819560539,0.263160158859993,0.292543238196494,0.272167552780268,0.278283809503352,0.0111108917882852 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,insample_rmse,100,0.326052737828645,0.32149423790565,0.33095413535273,0.327783381237672,0.32599583596128,0.00368060388877473 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,insample_spearman,100,0.460692654211815,0.44969759792448,0.473346352461843,0.457622037926128,0.460399198120541,0.00944921924292411 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,insample_calibration_slope,100,0.97529203720707,0.935173533434548,1.01400867208107,0.987210797047213,0.976209802905984,0.0305743821177023 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,insample_prevalence,100,0.224453926277481,0.219182866099434,0.230986907818958,0.22526523562793,0.224976563222905,0.00510112896547638 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,insample_n,100,2563,2534,2597.1,4053,2577.16,150.759204620407 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,oob_auc,99,0.810135045677214,0.794970810602448,0.821944357852199,,0.809030099101725,0.0110872169095942 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,oob_deviance_explained,99,0.25047666177894,0.191202924917333,0.270614944851623,,0.241203568163816,0.0340669348770295 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,oob_rmse,99,0.33139543060562,0.323567532609613,0.341730667982327,,0.332310401115346,0.00653564877448147 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,oob_spearman,99,0.43987491123139,0.417423685313865,0.456767797679778,,0.440335474375724,0.0164954001227547 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,oob_calibration_slope,99,0.93528692278846,0.866521668775654,1.03534349764393,,0.940676737829847,0.0698920255267155 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,oob_prevalence,99,0.226666666666667,0.215553872249123,0.235682245923583,,0.225757202244221,0.00879116862358667 +mammal,mammal_summer,north,summer,hurdle,Coyote_Summer,oob_n,100,1490,1455.9,1519,0,1475.84,150.759204620407 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,insample_auc,100,0.617823659773893,0.604516461927476,0.628631984104545,0.614800455189874,0.616761421956133,0.00976461911412697 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,insample_deviance_explained,100,0.0339007601849096,0.0263291456871497,0.0411865445449927,0.0327130199155846,0.0338470815855061,0.00590480373410536 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,insample_rmse,100,0.440098883977437,0.436773824908189,0.443507057723098,0.440494520447441,0.440195434279996,0.00260674728840036 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,insample_spearman,100,0.171090521534784,0.151756054904972,0.191125957843702,0.166867112583273,0.170769863932452,0.0165377875969792 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,insample_calibration_slope,100,0.862620525925969,0.785904553568306,0.929330397763158,0.901041907949805,0.860616692026413,0.0567629344145344 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,insample_prevalence,100,0.321379105658884,0.313118678099039,0.32747089089485,0.321243523316062,0.320995837958434,0.00543819599829418 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,insample_n,100,2563,2529.7,2590,4053,2576.14,150.912913545354 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,oob_auc,99,0.587164113714545,0.575785249795828,0.602772089491671,,0.588544884866055,0.0118624732002253 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,oob_deviance_explained,99,0.00941342548467339,-0.0113456994627058,0.0212189142748623,,0.00365463752946092,0.0285589934088055 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,oob_rmse,99,0.445979456559874,0.441133750532269,0.451237214396611,,0.446085804901711,0.00399332058557931 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,oob_spearman,99,0.123552665294692,0.104987161471936,0.148701374303894,,0.125552101964063,0.0193334513110719 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,oob_calibration_slope,99,0.623262330214489,0.486464288129371,0.79813783646821,,0.64206509854773,0.131390315162463 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,oob_prevalence,99,0.32093023255814,0.310137682812696,0.335204634013701,,0.321625209670558,0.00938803656163787 +mammal,mammal_summer,north,summer,hurdle,Moose_Summer,oob_n,100,1490,1463,1523.3,0,1476.86,150.912913545354 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,insample_auc,100,0.634803626920007,0.610741049343285,0.659568088455996,0.634240369366988,0.634082021387329,0.0187312850260846 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,insample_deviance_explained,100,0.0268537685303301,0.0136370124227746,0.0360182027991281,0.0264112493368064,0.0265401401134712,0.00821049982235054 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,insample_rmse,100,0.421343164086666,0.412941154813685,0.426956982356335,0.421286846656193,0.420927162399406,0.0053194051947628 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,insample_spearman,100,0.177042516370344,0.140168137320052,0.208031522005358,0.175369074792115,0.176681554648772,0.027769724052955 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,insample_calibration_slope,100,0.930769914251995,0.828835110850488,1.02041903967435,1.00531761653027,0.931665113681384,0.103164053676605 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,insample_prevalence,100,0.691966302585771,0.675848411374144,0.704604898153285,0.691079812206573,0.691448661048787,0.0109642868999095 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,insample_n,100,673.5,656.9,688,1065,677.12,41.1381707287659 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,oob_auc,99,0.593302071901294,0.555606698839408,0.631596864379755,,0.594121369515836,0.0284487401095002 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,oob_deviance_explained,99,0.00167749626514002,-0.0312688826497954,0.0210232836375135,,-0.0018300940011778,0.0216894957947076 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,oob_rmse,99,0.429319612116644,0.420013892677394,0.442995031469919,,0.430000986649372,0.00967827861885187 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,oob_spearman,99,0.123577048603096,0.0638608841974166,0.185337239143426,,0.123742676866493,0.0455334039236521 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,oob_calibration_slope,99,0.614566809243851,0.271513543349618,0.962299633403937,,0.663342568156952,0.354991571099046 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,oob_prevalence,99,0.68922305764411,0.667516563686444,0.717487373737374,,0.690443767298512,0.0190067113179566 +mammal,mammal_summer,south,summer,hurdle,Coyote_Summer,oob_n,100,391.5,377,408.1,0,387.88,41.1381707287659 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,insample_auc,100,0.787690117673898,0.771436480192896,0.809309626343477,0.786049924840508,0.789101667022636,0.0144627028915113 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,insample_deviance_explained,100,0.212783610773222,0.189398377824156,0.24970872966567,0.205216892616348,0.215210696559118,0.0233048334633376 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,insample_rmse,100,0.342745823483948,0.332893220047825,0.352205612166313,0.345153854253612,0.342787531936551,0.00763367886994782 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,insample_spearman,100,0.38311141955892,0.35872962599101,0.412057895508837,0.381089371361329,0.38408981732609,0.021041559607854 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,insample_calibration_slope,100,1.00502450004044,0.919106790872982,1.10389313687256,0.986324497785332,1.00900370313612,0.073000637812748 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,insample_prevalence,100,0.181213304242976,0.16931124481683,0.193242748670553,0.182159624413146,0.1813884269001,0.00970600845483194 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,insample_n,100,673,656,687,1065,676.5,41.2904224960098 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,oob_auc,99,0.762726897689769,0.729852829462147,0.793903597567736,,0.76325672049753,0.0245923418315769 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,oob_deviance_explained,99,0.157636554945407,-0.0836464338728246,0.212430956837577,,0.103055409587575,0.148245743536554 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,oob_rmse,99,0.353114380707358,0.337535935185427,0.370309554293426,,0.353577373542241,0.0141202985297347 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,oob_spearman,99,0.351376520734879,0.305802718576665,0.394924173339093,,0.350475263099306,0.0354755375647713 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,oob_calibration_slope,99,0.880822190314007,0.701215458314887,1.11978866872403,,0.898247685807426,0.170285815878653 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,oob_prevalence,99,0.183937823834197,0.162551889048302,0.20412389402934,,0.183585513697277,0.0168514518524878 +mammal,mammal_summer,south,summer,hurdle,Moose_Summer,oob_n,100,392,378,409,0,388.5,41.2904224960098 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,insample_auc,100,0.839944335636909,0.833129244062843,0.846877928092264,0.839440907842931,0.839903650262047,0.00557448428091706 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,insample_deviance_explained,100,0.296995113178883,0.281525374495306,0.311893449414862,0.294409607036222,0.296599293219049,0.0116652336293437 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,insample_rmse,100,0.300499882275642,0.295833966076858,0.304631426626969,0.30069616131153,0.300415954176532,0.00372827934781279 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,insample_spearman,100,0.46354248081478,0.452838720844845,0.476591779892728,0.463240577826878,0.463959331097095,0.00876549944099825 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,insample_calibration_slope,100,0.97175989961261,0.926421017162756,1.01129202766862,0.982050704865843,0.969658384093209,0.0316044627210243 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,insample_prevalence,100,0.200762631770266,0.195056052388284,0.207241284063151,0.201356917858008,0.20110706012536,0.00495248242885654 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,insample_n,100,2610.5,2583.9,2635.4,4127,2624.95,153.227128897931 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,oob_auc,99,0.831029603512896,0.81638193246376,0.841207806047271,,0.82947103945533,0.0104993912656751 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,oob_deviance_explained,99,0.273331011480722,0.226098852163728,0.296854758358736,,0.26623438036081,0.0362605084940545 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,oob_rmse,99,0.304588319831167,0.298159431290531,0.312787640387932,,0.304592354418792,0.0062215492462921 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,oob_spearman,99,0.450418797594334,0.430445722013538,0.467555101233984,,0.449194780082029,0.0162195108617758 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,oob_calibration_slope,99,0.934985520317015,0.842864089175499,1.05059228507324,,0.938935723649935,0.0822801797665789 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,oob_prevalence,99,0.202396804260985,0.190791881947389,0.212116747505983,,0.201754415558426,0.00860820797564151 +mammal,mammal_winter,north,winter,hurdle,Coyote_Winter,oob_n,100,1516.5,1491.6,1543.1,0,1502.05,153.227128897931 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,insample_auc,100,0.678197918411972,0.665286889295939,0.694066931578504,0.668767750495832,0.678269544703103,0.0109022501433117 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,insample_deviance_explained,100,0.078764822965353,0.0667247599709064,0.0928651291194507,0.0713753243188526,0.0792447728756145,0.00990794753046059 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,insample_rmse,100,0.332612319076884,0.326053956016172,0.337555008786784,0.332910274284271,0.332116606926772,0.00417566718680944 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,insample_spearman,100,0.223109017091458,0.208311428633986,0.241879354924964,0.211955551417089,0.223497723500791,0.0133619691564904 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,insample_calibration_slope,100,0.90993381105068,0.843355715875938,0.971039593627399,0.94812085347111,0.906876589660905,0.0501939181524883 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,insample_prevalence,100,0.148988991393271,0.142791523040489,0.155042075040598,0.148776350860189,0.14900574066188,0.00435550601282433 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,insample_n,100,2607.5,2579.9,2636.3,4127,2623.08,153.434793836326 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,oob_auc,99,0.649835304939499,0.625968627025412,0.675980028281669,,0.649644583547716,0.0187591905157571 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,oob_deviance_explained,99,0.0448055742879738,-0.0192592576785819,0.0711297656797476,,0.0344492987291663,0.0396578239199 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,oob_rmse,99,0.335414416833275,0.326677465909169,0.345975819454917,,0.335904113195915,0.00691390612050746 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,oob_spearman,99,0.188451199695192,0.158329944764903,0.221090693490017,,0.188026932145756,0.0230355958512011 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,oob_calibration_slope,99,0.755035262977712,0.572633380560262,0.932087775754081,,0.752810742407195,0.150295974349525 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,oob_prevalence,99,0.148344370860927,0.13811034885192,0.159079260891067,,0.148379045808284,0.00749888277960263 +mammal,mammal_winter,north,winter,hurdle,Moose_Winter,oob_n,100,1519.5,1490.7,1547.1,0,1503.92,153.434793836326 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,insample_auc,100,0.691860154744063,0.673399859370792,0.707179926658962,0.689082366331686,0.691031930239558,0.0137368192252231 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,insample_deviance_explained,100,0.0712261845284293,0.0551638038016942,0.0827931406770291,0.0670547674954939,0.069950224449978,0.0110427022312813 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,insample_rmse,100,0.406137347574509,0.401064172687698,0.410869575222684,0.406888363248781,0.406100303915382,0.00391010475798105 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,insample_spearman,100,0.300780638606997,0.266977262661948,0.328732636137794,0.296933239332004,0.299764109444291,0.0250658010950387 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,insample_calibration_slope,100,0.985668081828159,0.916592232776453,1.08039954168495,1.03306130883064,0.991898780050405,0.0613366816178454 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,insample_prevalence,100,0.603518323431888,0.5921562961536,0.615725590955807,0.604735883424408,0.604333184676431,0.00966206037483909 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,insample_n,100,694,680,711.1,1098,698.61,42.2460075216008 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,oob_auc,99,0.666483084185681,0.629277175802089,0.696251238522722,,0.666114898851002,0.0260240745790743 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,oob_deviance_explained,99,0.0435849516946,0.0207654478426379,0.0670740807151237,,0.0429180554693262,0.020298558724014 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,oob_rmse,99,0.413505421147088,0.405642815336104,0.423644031983321,,0.413998810474046,0.00703733664587233 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,oob_spearman,99,0.255001496110693,0.202000789808166,0.30841016152208,,0.256044893111881,0.0453638110703008 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,oob_calibration_slope,99,0.821758140786532,0.60847955923513,1.08499931480205,,0.848381394443559,0.221405394377051 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,oob_prevalence,99,0.606818181818182,0.585368087439596,0.627036015574302,,0.605470510142013,0.0168888593720911 +mammal,mammal_winter,south,winter,hurdle,Coyote_Winter,oob_n,100,404,386.9,418,0,399.39,42.2460075216008 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,insample_auc,100,0.816132437243359,0.793027210122563,0.839891485159624,0.819927852628077,0.815064369637998,0.0184138082012775 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,insample_deviance_explained,100,0.238608557676136,0.192427834756519,0.275475481888245,0.234870911391229,0.234185520824636,0.0309204873336235 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,insample_rmse,100,0.222302279540357,0.213322277622788,0.232484607860269,0.223331046901821,0.222577453639969,0.00761519200232921 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,insample_spearman,100,0.311490577535659,0.28144482212174,0.333163787212016,0.313170610278462,0.3079720848098,0.0213353963561159 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,insample_calibration_slope,100,0.896355121235575,0.785722197229858,1.03802533490544,0.926166715024209,0.90161901752608,0.0980362756411245 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,insample_prevalence,100,0.0873778188255215,0.0786459588190596,0.0943185693491371,0.0874316939890711,0.0870905780266154,0.00645265199125227 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,insample_n,100,694,676.9,715,1098,698.84,42.8444968355436 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,oob_auc,99,0.783600673844576,0.743422439497955,0.820159238909239,,0.783355925249662,0.0326069741602243 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,oob_deviance_explained,99,0.166305015954698,0.0484018355235042,0.229420651991623,,0.114850383082815,0.190843738046014 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,oob_rmse,99,0.232095087526286,0.213008139702699,0.246901580144276,,0.231225376674549,0.0132701556688799 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,oob_spearman,99,0.277427083199447,0.231627852405583,0.322176236387874,,0.277423688617959,0.0351558301822877 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,oob_calibration_slope,99,0.719706525261824,0.383010199885907,1.0841225850557,,0.72899681169363,0.267836950445254 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,oob_prevalence,99,0.0876288659793814,0.0753768844221106,0.103636247116124,,0.0879823881637428,0.0112620602476394 +mammal,mammal_winter,south,winter,hurdle,Moose_Winter,oob_n,100,404,383,421.1,0,399.16,42.8444968355436 mite,mite,north,,,Ceratozetes.gracilis,insample_auc,100,0.792129407675089,0.784999351238186,0.798007653851505,0.791183069113511,0.791906840547393,0.00544399156186335 mite,mite,north,,,Ceratozetes.gracilis,insample_deviance_explained,100,0.122009919458204,0.105506517747849,0.147733814877035,0.125527507164404,0.125043569305553,0.0170734779462856 mite,mite,north,,,Ceratozetes.gracilis,insample_rmse,100,0.364379787990174,0.359996548653916,0.368133999109812,0.364308079016483,0.364109394241196,0.00321218794005143 diff --git a/3_output/exp_000_parity_v2/tables/parity_summary.csv b/3_output/exp_000_parity_v2/tables/parity_summary.csv index f4a9719..adbb798 100644 --- a/3_output/exp_000_parity_v2/tables/parity_summary.csv +++ b/3_output/exp_000_parity_v2/tables/parity_summary.csv @@ -2,22 +2,24 @@ taxon,region,stage,comparison,species,terms_compared,min_draws,in_band_pct,reach bird,north,climate,median,2,20,100,100,20,100,0,0,2.21e-11,0,1,1,not gated (trial run),pass bryophyte,north,climate,median,1,19,100,100,19,100,0.035,0,1.4,6,1,1.062,not gated (trial run),pass lichen,north,climate,median,2,38,100,100,38,100,0,0,0.644,17,0.993,0.879,not gated (trial run),pass +mammal,north,climate,iteration 1,2,10,100,100,10,100,0,0,7.02e-14,0,1,,not gated (trial run),pass mite,north,climate,median,2,38,100,100,38,100,0,0,1.95,20,0.905,0.953,not gated (trial run),pass vascular_plant,north,climate,median,2,38,100,100,38,100,0,0,0.266,24,1,0.97,not gated (trial run),pass bird,south,climate,median,2,20,100,100,20,100,0,0,2.21e-11,0,1,1,not gated (trial run),pass bryophyte,south,climate,median,1,19,100,100,19,100,0.035,0,1.4,6,1,1.062,not gated (trial run),pass lichen,south,climate,median,2,38,100,100,38,100,0,0,0.644,17,0.993,0.879,not gated (trial run),pass +mammal,south,climate,iteration 1,2,10,100,100,10,100,0,0,7.02e-14,0,1,,not gated (trial run),pass mite,south,climate,median,2,38,100,100,38,100,0,0,1.95,20,0.905,0.953,not gated (trial run),pass vascular_plant,south,climate,median,2,38,100,100,38,100,0,0,0.266,24,1,0.97,not gated (trial run),pass bird,north,habitat,median,2,178,100,100,168,100,0,0,2.71e-11,0,1,1,not gated (trial run),pass bryophyte,north,habitat,median,1,94,100,100,94,100,0.072,0.023,0.0808,0,0.997,0.911,not gated (trial run),pass lichen,north,habitat,median,2,188,100,100,188,100,0.091,0.024,0.118,0,0.999,0.988,not gated (trial run),pass -mammal,north,habitat,iteration 1,2,448,65,100,436,100,0,0,9.77e-15,0,1,,not gated (trial run),pass +mammal,north,habitat,iteration 1,2,448,69,100,436,100,0,0,9.77e-15,0,1,,not gated (trial run),pass mite,north,habitat,median,1,94,100,100,94,100,0.14,0.035,0.148,0,0.999,0.993,not gated (trial run),pass vascular_plant,north,habitat,median,2,188,100,100,188,100,0.108,0.022,0.125,0,0.999,0.95,not gated (trial run),pass bird,south,habitat,median,2,46,100,100,40,100,0,0,5.61e-11,0,1,1,not gated (trial run),pass bryophyte,south,habitat,median,1,19,100,100,19,100,0.047,0.021,0.0601,0,0.993,1.042,not gated (trial run),pass lichen,south,habitat,median,2,40,100,100,40,100,0.088,0.029,0.278,1,0.994,0.961,not gated (trial run),pass -mammal,south,habitat,iteration 1,2,126,99,100,120,100,0,0,5.33e-15,0,1,,not gated (trial run),pass +mammal,south,habitat,iteration 1,2,126,100,100,120,100,0,0,5.33e-15,0,1,,not gated (trial run),pass mite,south,habitat,median,2,40,100,100,40,100,0.068,0.024,0.464,1,0.991,0.975,not gated (trial run),pass vascular_plant,south,habitat,median,2,40,100,100,40,100,0.074,0.027,0.184,0,0.997,1.021,not gated (trial run),pass diff --git a/README.md b/README.md index 9da5cb1..6ce8b37 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ sdmMethodsDev/ │ ├── harness/ # the shared pipeline; names no taxon or method │ ├── methods/ # engines, selection rules, resampling, metrics │ ├── modules/ # one v2 spec per taxon; _shared/ for common code -│ ├── experiments/ # one folder per question; _template/, _shared/ +│ ├── experiments/ # one folder per question; _template/ │ ├── _setup/ # 00-08: build, check and publish the dataset │ ├── tests/ # contract tests and the store comparison │ └── _scratch.R # dated workspace for exploration @@ -78,36 +78,34 @@ up by the name the spec gives. %% ---------------------------- %% Taxon as configuration, not codebase %% ---------------------------- -%%{init: {"themeVariables": {"edgeLabelBackground": "#ffffff"}}}%% +%%{init: { + "look": "classic", + "themeVariables": {"edgeLabelBackground": "#ffffff"}, + "themeCSS": ".node rect, .node polygon, .node circle, .node path, .edgeLabel, .label-container { filter: none !important; box-shadow: none !important; }" +}}%% flowchart TD - subgraph fw["framework: one pipeline, taxon as configuration"] - direction TB - PARAM["response · family · offset · regions · stages · resampling"] - PARAM --> SP["plant-group specs
bryophytes · lichens · mites · vascular plants"] - PARAM --> SM["mammal specs
summer · winter"] - PARAM --> SB["bird spec"] - SP --> H["1_code/harness/"] - SM --> H - SB --> H - MT["1_code/methods/
engines · selection rules · resampling · metrics"] - MT -.->|"looked up by name"| H - H -->|"the same result files
for every taxon and method"| R["result stores"] - end + PARAM["response · family · offset · model type · model selection"] + PARAM --> SP["plants spec.R"] + PARAM --> SM["mammals spec.R"] + PARAM --> SB["birds spec.R"] + SP --> H["1_code/harness/"] + SM --> H + SB --> H + H -->|"select species, covariates,
model specification, etc."| R["result object"] %% ---------------------------- %% Styles %% ---------------------------- - %% Containers - style fw fill:none,stroke:#2D415B,stroke-width:1px + %% Connection lines + linkStyle default stroke-width:1px %% Configuration style PARAM fill:#ffffff,stroke:#2D415B,stroke-width:1px,stroke-dasharray: 5 5 style SP fill:#A3B4C7,stroke:#2D415B,stroke-width:1px style SM fill:#A3B4C7,stroke:#2D415B,stroke-width:1px style SB fill:#A3B4C7,stroke:#2D415B,stroke-width:1px - style MT fill:#A3B4C7,stroke:#2D415B,stroke-width:1px,stroke-dasharray: 5 5 %% Shared pipeline style H fill:#A4A88780,stroke:#C8A02C,stroke-width:4px @@ -123,35 +121,40 @@ v2.0 output. Every later experiment changes one thing and is compared with exp_000, so a difference is a difference in method rather than in plumbing. Because the dataset is frozen, results stay comparable. +An experiment edits only its own folder: its species list, the spec +changes in its `run.R`, and any methods of its own in `hooks/`. The +harness, methods and taxon modules stay a fixed, tested back-end. + ```mermaid %% ---------------------------- %% Everything comparable to one v2 baseline %% ---------------------------- -%%{init: {"themeVariables": {"edgeLabelBackground": "#ffffff"}}}%% +%%{init: { + "look": "classic", + "themeVariables": {"edgeLabelBackground": "#ffffff"}, + "themeCSS": ".node rect, .node polygon, .node circle, .node path, .edgeLabel, .label-container { filter: none !important; box-shadow: none !important; }" +}}%% flowchart TD - subgraph fw["framework: everything comparable to one v2 baseline"] - direction TB - D["0_data/test_dataset/"] --> M["1_code/modules/[taxon]/
v2 specs"] - V2["0_data/v2_results/
published v2.0"] - M --> E0["exp_000_parity_v2"] - M --> E1["exp_001"] - M --> E2["exp_002"] - V2 -->|"parity check"| E0 - E0 --> O0["3_output/exp_000_parity_v2/"] - E1 --> O1["3_output/exp_001/"] - E2 --> O2["3_output/exp_002/"] - O0 --> C["compare_experiments()"] - O1 --> C - O2 --> C - end + D["0_data/test_dataset/"] --> M["1_code/modules/[taxon]/
v2 specs"] + V2["0_data/v2_results/
published v2.0"] + M --> E0["exp_000_parity_v2"] + M --> E1["exp_001"] + M --> E2["exp_002"] + V2 -->|"parity check"| E0 + E0 --> O0["3_output/exp_000_parity_v2/"] + E1 --> O1["3_output/exp_001/"] + E2 --> O2["3_output/exp_002/"] + O0 --> C["compare_experiments()"] + O1 --> C + O2 --> C %% ---------------------------- %% Styles %% ---------------------------- - %% Container - style fw fill:none,stroke:#2D415B,stroke-width:1px + %% Connection lines + linkStyle default stroke-width:.75px %% Highlighted borders style D fill:#A4A88780,stroke:#B8860B,stroke-width:4px diff --git a/docs/example_experiment.md b/docs/example_experiment.md index 131d461..e9ca638 100644 --- a/docs/example_experiment.md +++ b/docs/example_experiment.md @@ -59,7 +59,9 @@ Then drop `dry_run = TRUE` to create them. You get: ``` 1_code/experiments// ├── README.md # the question; from _template/ -└── run.R # the decisions; id, title, author, date filled in +├── run.R # the decisions; id, title, author, date filled in +├── species.csv # every species a run can fit; cut it down +└── hooks/ # empty; methods of the experiment's own 2_pipeline// └── logs/ 3_output// @@ -68,8 +70,10 @@ Then drop `dry_run = TRUE` to create them. You get: └── report.md # a stub ``` -Leave `id` in `run.R` as it is. The rest of this page fills in -those two files and adds one script beside them. +Leave `id` and `exp_dir` in `run.R` as they are. The rest of this +page fills in `README.md`, `species.csv` and `run.R`, and adds one +script beside them. Nothing outside the experiment's folder is +edited. ## Step 2: Write down the question @@ -85,7 +89,13 @@ relative to exp_000 and what does not. exp_002's: | Climate stage | v2's candidates (plants 58, birds 25), model-averaged | The same candidates, each with the nine soil terms added | | Climate → habitat | Carried as `Climate` | The same; the carried prediction now includes soil | | Habitat stage | v2 | v2, unchanged | -| Species, draws, seed | `parity_check`, 20260909 | The same | +| Species, draws, seed | exp_000's (`species.csv`, two per taxon), 20260909 | The same | + +To keep exp_000's species, replace the rows of your `species.csv` +with the plant and bird rows of +`1_code/experiments/exp_000_parity_v2/species.csv`. The rows left +also decide which runs are fitted, so mammal rows would add the +mammal runs. Write down the reasoning for each choice while you still have it. exp_002 records three: @@ -220,7 +230,9 @@ bird_soil[1] ``` To build a candidate set from covariates alone instead, see -`models_from_covariates()` in `1_code/harness/model_sets.R`. +`models_from_covariates()` in `1_code/harness/model_sets.R`. To +take covariates out of a set, `drop_candidates()` and +`strip_terms()` sit beside it. ## Step 5: Configure the run @@ -232,7 +244,7 @@ comparison is like for like: config <- experiment_config( id = "exp_002_soilgrids", taxa = c(plant_taxa, "bird"), - species = "parity_check", # exp_000's species + species = file.path(exp_dir, "species.csv"), # exp_000's n_bootstraps = 5, # 5 checks; 100 for a result seed = 20260909, # exp_000's seed stage_models = stage_models, # the change @@ -241,7 +253,10 @@ config <- experiment_config( ) ``` -`soil_file` is the CSV from Step 3. exp_002's `run.R` also runs +exp_002 names `taxa` because it predates runs being read off the +species list; a new experiment can leave `taxa` out, and its +`species.csv` then sets the runs. `soil_file` is the CSV from +Step 3. exp_002's `run.R` also runs the extraction script when that file is missing, so one `source()` call does everything: @@ -259,9 +274,11 @@ if (!file.exists(soil_file)) { Delete the CSV to extract again. `experiment_config()` checks the covariate file (Step 3's -contract), the runs named in `taxa`, and the draw count before -anything is fitted, and prints what it will do. A problem with any -of them stops it here, not hours in. +contract), the species names, the runs, the draw count, each spec, +and every covariate the formulas name (here, that each `sg_` column +is in `soil_file`) before anything is fitted, and prints what it +will do. It lists every problem at once, so one run shows all there +is to fix, here and not hours in. ## Step 6: Run it at 5 draws @@ -345,7 +362,7 @@ lower RMSE; and a calibration slope closer to 1. A difference is worth reading only if it is consistent across species (`share_better`) and large relative to the bands. **A 5-draw run is a pipeline check, not a result.** Its bands are -unstable, and `parity_check` is two species per taxon. Say so +unstable, and exp_000's species list is two species per taxon. Say so wherever you report it, as exp_002's README does. ## Step 9: Run at 100 draws and write it up @@ -370,8 +387,11 @@ When the 5-draw run is clean: ## Checklist -- [ ] Scaffolded with `new_experiment()`; `id` in `run.R` - unchanged. +- [ ] Scaffolded with `new_experiment()`; `id` and `exp_dir` in + `run.R` unchanged. +- [ ] `species.csv` cut down to the species fitted (exp_000's for + a like-for-like comparison). +- [ ] Nothing edited outside the experiment's folder. - [ ] Question, design table, and reasoning in the README before the first run. - [ ] One change relative to exp_000; species, seed, and draws diff --git a/docs/example_experiment_xgboost.md b/docs/example_experiment_xgboost.md index 1e3a0b2..63bdbab 100644 --- a/docs/example_experiment_xgboost.md +++ b/docs/example_experiment_xgboost.md @@ -35,7 +35,9 @@ You need: not need it. - **The engine registered.** `xgboost` already is, in `1_code/methods/engines/xgboost.R`. To test a method that is not - registered yet, add it first; see the + registered yet, register it in the experiment's own `hooks/` + folder, loaded by `load_framework(hooks = )`; the experiment then + names it like a built-in one. See the [methods README][methods]. - **exp_000 tables at the draw count you will run**, as in the [SoilGrids example](example_experiment.md#before-you-start). @@ -64,7 +66,7 @@ exp_001's design table: | Climate stage | v2: GLM candidates, AICc model averaging (mammals: v2's precomputed prediction) | The same | | Habitat stage (each spec's `habitat_stage`; birds: landcover) | v2: GLM candidates; IVW averaging (plants), staged BIC (birds), or the hurdle with best-by-AICc halves (mammals); v2's post-processing | `xgboost` with the `single` rule, on every covariate v2's habitat candidates use, `Climate` included | | Climate → habitat | Carried as `Climate` | The same | -| Species, draws, seed | `parity_check`, 20260909 | The same | +| Species, draws, seed | exp_000's (`species.csv`, two per taxon), 20260909 | The same | And the reasoning behind it: @@ -99,6 +101,10 @@ validate_spec(spec, stop_on_error = FALSE) #> Choose a rule that does not, such as `single`." ``` +`modify_stage(lichen_spec(), "habitat", engine = "xgboost")`, the +recorded way to change a stage in an experiment, stops with the +same sentence straight away. + So changing the method means changing three things together: the engine, the rule, and the candidate set the rule chooses from. Then v2 steps that read coefficients have to go too. @@ -241,7 +247,7 @@ matches exp_000: config <- experiment_config( id = "exp_001_xgboost", taxa = names(specs), - species = "parity_check", # exp_000's species + species = file.path(exp_dir, "species.csv"), # exp_000's n_bootstraps = 5, # 5 checks; 100 for a result seed = 20260909, # exp_000's seed specs = specs, # the change @@ -257,7 +263,8 @@ step translates bird landcover coefficients onto v2's habitat template, and a tree has no coefficients to translate. To try the change on fewer taxa first, narrow `taxa`, for example -`taxa = c("lichen", "bird")`. +`taxa = c("lichen", "bird")`. In a new experiment, leave `taxa` out +and cut `species.csv` instead: its rows decide the runs. Run the whole script: @@ -294,8 +301,8 @@ Keep in mind: calibrated tables; this run's grid predictions are not calibrated. -As always, a 5-draw run on `parity_check` is a pipeline check, -not a result. +As always, a 5-draw run on exp_000's two species per taxon is a +pipeline check, not a result. ## Step 9: Tune, run at 100 draws, and write it up diff --git a/docs/framework_design.md b/docs/framework_design.md index 6853be7..57e64fe 100644 --- a/docs/framework_design.md +++ b/docs/framework_design.md @@ -98,9 +98,10 @@ requires a separate pipeline. │ ├── _shared/ # plant_group.R; standard_specs.R │ └── /spec.R # that taxon's v2 configuration ├── experiments/ -│ ├── _shared/ # focal-species sets │ ├── _template/ # the starting point for a new one -│ └── exp_NNN_*/run.R # configuration, then run_experiment() +│ └── exp_NNN_*/ # run.R: configuration, then +│ # run_experiment(); species.csv; +│ # hooks/ for its own methods └── tests/ # contract tests; store comparison ``` @@ -109,6 +110,14 @@ particular method: it looks methods up by the name a spec gives. A module contributes only a spec. An experiment changes parts of a spec, or names different methods in it. +**An experiment edits only its own folder.** The harness, methods +and modules are a fixed, tested back-end; everything an experiment +varies is an `experiment_config()` argument, a spec change made in +its `run.R`, its `species.csv`, or a method in its `hooks/`. The +defaults are v2's: `experiment_config()`'s argument defaults hold +the run-level ones, `standard_specs()` the model-level ones, and an +experiment passes only what it changes. + There are six taxon modules: `bryophytes`, `lichens`, `soil_mites`, `vascular_plants`, `mammals` and `birds`. The first four share the v2 pipeline shape in `_shared/plant_group.R`, so four copies cannot @@ -128,6 +137,47 @@ inverse-variance averaging, say - stops with a sentence rather than silently returning nothing hours into a run. `check_engine()` tests a new engine against the contract on synthetic data. +A method one experiment is testing lives in that experiment's +`hooks/` folder rather than here. `load_framework(hooks = )` +sources the hook files after the built-in methods, and each +parallel worker sources them too. A hook registers with the same +`register_*()` calls and meets the same contracts, but may not +reuse a name already registered: that would change the name's +meaning for everything else in the session. A metric declares +which way is better (`direction`), so comparisons read a new +metric correctly. + +### Spec changes are recorded + +An experiment changes a v2 spec with `replace_stage_method()`, +`modify_stage()`, `modify_region()` or `modify_resample()`, never +by editing a module. Each validates the spec and appends the change +to `models_overridden` and `v2_coverage`, so the stores' +`meta.json` and the run record say what differs from v2. Stage +fields cover the engine, rule and its settings, `ic`, +`min_detections`, scope, weighting, carrying and grid constants; +resample settings include the spatial block cut points +(`long_breaks`, `lat_breaks`, defaulting to the v2 plant grid). + +### Configuration is checked before anything runs + +`experiment_config()` collects every problem and stops once with +the list: species names against the dataset's species queue (with +close matches), runs, draw counts, regions, metrics, every spec, +and every covariate the formulas name. The covariate check +resolves names with the same code the run uses +(`region_request()`, `province_request()`, +`covariate_columns_needed()`), so it cannot disagree with the +run. + +### Provenance + +`run_record.md` gives every setting and change from v2, and, in +its volatile Provenance section, the full git commit, how many +files under `1_code/` differ from it, R version, platform, +modelling package versions, and each hook file with its md5. +`2_pipeline//session_info.txt` holds the full `sessionInfo()`. + The rules talk to an engine only through its interface: coefficients come from `engine$coef()`, computed once per fit, and every rule's result carries a `predict()` for its final model. That is what lets @@ -276,9 +326,13 @@ holds; see below): ```r stage_models <- list( - # the v2 set, plus topography in every candidate - "climate" = extend_models( - "climate_plant_v2_full", c("elevation", "slope") + # the v2 set, plus AHM in every candidate + "climate" = extend_models("climate_plant_v2_full", "AHM"), + # the v2 set without the candidates that use space + "lichen.climate" = drop_candidates( + "climate_plant_v2_full", + c("Easting", "Northing", "Easting2", "Northing2", + "EastingNorthing") ), # or built from covariates alone "mite.climate" = models_from_covariates( @@ -287,7 +341,10 @@ stage_models <- list( ) ``` -Keys are `taxon.stage`, or `stage` for every taxon. `form` decides the +Keys are `taxon.stage`, or `stage` for every taxon; a `taxon.stage` +key wins over a `stage` key. `drop_candidates()` removes every +candidate that uses a term; `strip_terms()` keeps every candidate, +less the term. `form` decides the shape of a covariate comparison — `single`, `each`, `ladder` or `all_subsets`, the last capped because ten covariates is 1,023 models per species per draw. Swapping the set changes the covariates automatically: @@ -296,11 +353,16 @@ the v2 plant climate set loads 18, a two-covariate ladder loads 2. An override is recorded in `meta.json`, so a run that did not fit the v2 candidate sets cannot be mistaken for one that did. -Species are selected the same way, by a named vector whose names are -taxa; names may repeat and an unnamed entry applies to every taxon. +Species are selected by the experiment's `species.csv`, one row per +species (`taxon,species,note`). The template lists every species a +run can fit; an experiment deletes the rows it does not fit. The +rows left also set the runs: with no `taxa`, only runs that fit a +listed species are fitted, judged by the run's taxon, tier and +(for mammals) season. Names are checked against the dataset's +species queue. A named vector works too: ```r -focal_species <- c( +species <- c( lichen = "Alectoria.sarmentosa", lichen = "Bryoria.fremontii", mite = "Oppiella.nova" @@ -492,7 +554,18 @@ It sits on a regularization axis. A Bayesian framework comparison needs abundance.) `covariate_files` lets an experiment add covariates the frozen dataset does not hold, from its own files; `exp_002_soilgrids` adds SoilGrids at 0–5 cm from the - sciSpatialR catalogue to the climate stage. Still to come: + sciSpatialR catalogue to the climate stage. +6a. **Built (2026-10-13).** Experiment-local configuration: each + experiment's own `species.csv` (replacing the shared focal-species + sets) and `hooks/`; `modify_stage()`, `modify_region()` and + `modify_resample()`; `regions`, `metrics` and `compare_metrics` in + `experiment_config()`, which now checks species, specs and + covariates before anything runs and reports every problem at + once; runs read off the species list; settable spatial block cut + points; `drop_candidates()` and `strip_terms()`; fuller + provenance. Checked against the previous code at 5 draws for the + 14 parity species: every coefficient, metric and grid prediction + identical. Still to come: `mgcv` GAMs and `brms` as method files, the topography and remote-sensing covariates, and spatial prediction onto the 1 km grid. diff --git a/docs/getting_started.md b/docs/getting_started.md index c3ec248..4b2783d 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -40,6 +40,8 @@ models or reporting products. | **Grid** | A prediction matrix with one row per habitat type, each row a pure stand of that type. Predicting onto it lets methods be compared. | | **Out-of-bag (oob)** | Scored on the survey units a draw left out. `insample` scores the units the draw fitted. | | **Parity** | Whether a run reproduces v2. Distributional, not numerical: v2 sets no seed, so two v2 runs differ. | +| **Species list** | An experiment's `species.csv` (`taxon,species,note`): the species it fits. The rows also decide which runs it fits. | +| **Hook** | A file in an experiment's `hooks/` folder that registers a method of the experiment's own, or defines a function a spec field names. | ## Setup @@ -88,8 +90,7 @@ you. ### Check that the setup works First, load the framework. This sources the harness, every -method, every taxon module, and the shared species sets. It reads -no data. +method, and every taxon module. It reads no data. ```r source("1_code/harness/harness.R") @@ -204,10 +205,9 @@ The full argument list and output file definitions are in the | Folder | Holds | Edit it to | | --- | --- | --- | | `1_code/harness/` | The shared pipeline: configure, fit, store, summarize, compare. Names no taxon and no method | Change how every experiment runs (review as such) | -| `1_code/methods/` | Engines, selection rules, resampling schemes, metrics; each file registers one method | Add a method | +| `1_code/methods/` | Engines, selection rules, resampling schemes, metrics; each file registers one method | Add a method every experiment can use | | `1_code/modules//` | Each taxon's v2 spec; `_shared/` holds the plant-group builder and `standard_specs()` | Record what v2 does for a taxon | -| `1_code/experiments/exp_NNN_*/` | One folder per question; `run.R` holds the decisions | Ask a question | -| `1_code/experiments/_shared/` | Named focal-species sets | Add a reusable species set | +| `1_code/experiments/exp_NNN_*/` | One folder per question: `run.R` holds the decisions, `species.csv` the species, `hooks/` any methods of its own | Ask a question. The only folder an experiment edits | | `1_code/tests/` | Contract tests and the store comparison | Add a check | | `1_code/_setup/` | One-off scripts that build and publish the dataset | Rebuild or extend the dataset (maintainers) | | `0_data/test_dataset/` | The dataset as `_setup/` builds it; runs read the published copy | Nothing; rebuilt by `_setup/` | @@ -234,22 +234,19 @@ configuration. Check three settings: | Setting | Quick check | Full run | | --- | --- | --- | -| `species` | `"one_each"` (one species per taxon) or `"parity_check"` (two per taxon; the committed value) | `NULL`, every species | -| `n_bootstraps` | `5` (the committed value) | `100` | +| `species` | A shorter vector, e.g. `c(lichen = "Physcia.adscendens", bird = "AMRO")` | `NULL`, every species. Committed: `file.path(exp_dir, "species.csv")`, two per taxon | +| `n_bootstraps` | `5` | `100` (the committed value) | | `workers` | 4 | Up to the cores you can spare (committed: 12) | -The named species sets are: - -```r -names(focal_species_sets()) -#> [1] "full" "parity_check" "plants_only" "one_each" -``` +exp_000's `species.csv` holds the two species per taxon the parity +test is read on. Its `note` column says why each was chosen. Later +experiments copy it for a like-for-like comparison. Shorten a trial by cutting species, not draws. A 10-90% band read -from five draws is unstable. On a 24-core machine, the 14 -`parity_check` species at 5 draws took about 6 minutes on one -worker and about 1.6 minutes on 12. A full run of every species -at 100 draws is days of compute. +from five draws is unstable. On a 24-core machine, exp_000's 14 +species at 5 draws took about 6 minutes on one worker and about 2 +minutes on 12. A full run of every species at 100 draws is days of +compute. Parallel workers load the framework from disk when the run starts, so do not edit files in `1_code/` while one is starting. @@ -275,7 +272,8 @@ in section 2 of `run.R`. | --- | --- | | `2_pipeline/exp_000_parity_v2///` | Result stores: `coefficients.csv`, `metrics.csv`, `grid_predictions.csv`, `meta.json` | | `2_pipeline/exp_000_parity_v2/run_log.csv` | Every species, region, and draw: `ok`, or why not | -| `3_output/exp_000_parity_v2/run_record.md` | What ran: species, draws, seed, data versions | +| `2_pipeline/exp_000_parity_v2/session_info.txt` | The full `sessionInfo()` of the run | +| `3_output/exp_000_parity_v2/run_record.md` | What ran: species, draws, seed, data versions, every change from v2, and provenance (full commit, uncommitted code, packages, hooks) | | `3_output/exp_000_parity_v2/report.md` | What each spec reproduces of v2, parity by stage, and model fit | | `3_output/exp_000_parity_v2/tables/parity_summary.csv` | The gate table: one row per taxon, region, and stage | | `3_output/exp_000_parity_v2/tables/parity_terms.csv` | Term-by-term comparison (gitignored) | @@ -360,31 +358,63 @@ new_experiment("covariate scale") It takes the next free number across `1_code/experiments/`, `2_pipeline/`, and `3_output/`, and creates: -- `1_code/experiments//`: `run.R` and `README.md` from - `_template/`, with id, title, author, and date filled in. +- `1_code/experiments//`: `run.R`, `README.md`, `species.csv` + and an empty `hooks/` from `_template/`, with id, title, author, + and date filled in. - `2_pipeline//logs/`: for intermediates (gitignored). - `3_output//`: `figures/`, `tables/`, and a stub `report.md`. -Leave `id` in `run.R` as it is. Write the question and design in -the experiment's `README.md` before you run anything. +Leave `id` and `exp_dir` in `run.R` as they are. Write the question +and design in the experiment's `README.md` before you run +anything. + +`run.R` loads the framework and the experiment's hooks, makes any +spec changes, calls `experiment_config()`, then `run_experiment()`. + +**Edit only the experiment's folder.** Every change below is made +in its `run.R`, its `species.csv`, or its `hooks/`. The harness, +methods and modules are shared by every experiment and are not +edited for one. + +### Choose the species: `species.csv` + +The template's `species.csv` lists every species a run can fit, +one row each, as `taxon,species,note`. Delete the rows you do not +want; use `note` to say why a species is in. The rows left set +both the species and the runs: with no `taxa` in `run.R`, only +runs that fit a listed species are run, so a `Moose_Summer` row +runs `mammal_summer` and not `mammal_winter`. Three species is +three rows: + +``` +taxon,species,note +lichen,Physcia.adscendens, +bird,AMRO, +mammal,Moose_Winter, +``` + +For a like-for-like comparison with exp_000, copy its rows from +`1_code/experiments/exp_000_parity_v2/species.csv`. Names are +checked against the dataset's species queue +(`species_catalogue(test_dataset_dir())`); a misspelt one stops +the run with close matches suggested. -`run.R` loads the framework, calls `experiment_config()`, then -`run_experiment()`. +If `run.R` does give `taxa`, it wins: a run in `taxa` whose taxon +has no rows fits every species of that taxon, and a note says so. ### 1. Different candidate formulas: `stage_models` The covariates a run loads are read off its formulas, so this is -also how you try new covariates. +also how you try new covariates, or drop them. ```r config <- experiment_config( id = "exp_003_covariate_scale", - taxa = c("bryophyte", "lichen", "mite", "vascular_plant"), - species = "parity_check", + species = file.path(exp_dir, "species.csv"), n_bootstraps = 5, stage_models = list( - # every v2 plant climate candidate, plus elevation - climate = extend_models("climate_plant_v2_full", "elevation"), + # every v2 plant climate candidate, plus AHM + climate = extend_models("climate_plant_v2_full", "AHM"), # mites only: a ladder built from three covariates mite.climate = models_from_covariates( c("MAP", "FFP", "CMD"), @@ -395,6 +425,28 @@ config <- experiment_config( ) ``` +Keys are `stage` (every run with that stage) or `taxon.stage` (one +taxon). To remove covariates from a set there are two helpers, +which answer different questions: + +```r +spatial <- c("Easting", "Northing", "Easting2", "Northing2", + "EastingNorthing") + +# Drop every candidate that uses a spatial term (58 -> 28) +drop_candidates("climate_plant_v2_full", spatial) + +# Keep every candidate, less the spatial terms; duplicates merge +strip_terms("climate_plant_v2_full", spatial) +``` + +Habitat candidates are set per region; change them with +`modify_region(spec, "north", habitat_models = ...)`. See what a +taxon holds with `available_covariates(test_dataset_dir(), +"lichen", "north")`: availability differs by taxon, and +`experiment_config()` stops on a covariate a run does not have, +suggesting close matches. + A covariate the dataset does not hold comes from the experiment. Write a script in the experiment's folder that saves a CSV to `2_pipeline//inputs/`, one row per `survey_unit_id`, and name @@ -417,13 +469,33 @@ specs <- lapply( config <- experiment_config( id = "exp_003_covariate_scale", specs = specs, - species = "parity_check", + species = file.path(exp_dir, "species.csv"), n_bootstraps = 5 ) ``` -To change one stage by hand instead, edit the spec list directly, -for example `specs$bird$stages[[2]]$selection <- "single"`. +Any other setting of a spec changes the same way, with a helper +that validates the spec at once and records the change in the run +record: + +```r +specs <- standard_specs() + +# A stage's fields: engine, rule, ic, min_detections, scope, +# weighting, carrying, grid_constants, ... +specs$lichen <- modify_stage(specs$lichen, "climate", ic = "AIC") +specs$lichen <- modify_stage(specs$lichen, "habitat", + min_detections = 10L) + +# A region's fields: filter, grid, habitat_models, constants +specs$lichen <- modify_region(specs$lichen, "north", + filter = ~ nr == "Boreal") +``` + +Module arguments are set the same way, in `run.R`: +`bryophyte_spec(use_protocol = FALSE)`, +`mammal_spec(climate_source = "fitted")`. Never edit a module for +an experiment. `validate_spec()` checks the combination before anything runs. A rule that needs what the engine cannot give, such as standard @@ -431,20 +503,55 @@ errors for inverse-variance averaging, stops the run with a sentence that says so. [`exp_001_xgboost`][exp001] is the worked example. -### 3. A new method +### 3. A new method: `hooks/` + +Put it in the experiment's `hooks/` folder. Copy the matching +template from `1_code/methods/_templates/`, fill it in, and +register it there with `register_engine()`, `register_selection()`, +`register_resampler()` or `register_metric()`. `run.R` loads it: -Copy the matching template from `1_code/methods/_templates/` into -the right folder of `1_code/methods/`, fill it in, and register -it. Check an engine with `check_engine()`, confirm it appears in -`list_methods()`, then name it in a spec: +```r +load_framework(hooks = file.path(exp_dir, "hooks")) +``` + +Check an engine with `check_engine()`, confirm it appears in +`list_methods()`, then name it in a spec like a built-in one: ```r -check_engine("glm") # or check_engine(engine_mine()) +check_engine("my_engine") list_methods("engine") ``` -The engine, selection, resampler, and metrics, and a -checklist, are in the [methods README][methods]. +Parallel workers load the hooks too, and the run record lists each +hook file with its md5. A hook may not reuse a registered name. +Give a metric a `direction` (`"lower"` for an error) so the +comparison with exp_000 reads it the right way. A method that +proves itself moves to `1_code/methods/` so every experiment can +use it. The contracts and a checklist are in the +[methods README][methods]. + +### 4. Regions, resampling and metrics + +```r +specs$lichen <- modify_resample( + specs$lichen, scheme = "spatial_cv", replace = TRUE, + lat_breaks = c(48, 50, 52, 54, 56, 58, 61) # finer blocks +) + +config <- experiment_config( + id = "exp_003_covariate_scale", + specs = specs, + species = file.path(exp_dir, "species.csv"), + n_bootstraps = 5, # with spatial_cv: the folds + regions = "north", # fit one region + metrics = c("auc", "rmse"), # scored per draw + compare_metrics = "oob_auc", # compared with exp_000 + unit_predictions = "oob" # per-unit predictions too +) +``` + +`replace = TRUE` drops the old scheme's settings, which the new one +does not take. Block cut points default to the v2 plant grid. ### Run it and read the comparison @@ -455,6 +562,11 @@ an exp_000 run at the same number of draws. source("1_code/experiments/exp_003_covariate_scale/run.R") ``` +`experiment_config()` checks everything before anything is fitted +and lists every problem at once: species names, runs, regions, +metrics, each spec, and every covariate the formulas name. Fix the +list, run again. + Outputs go to `2_pipeline//` and `3_output//`, laid out as for exp_000. `run_experiment()` prints a headline per taxon, region, and metric (the median difference from exp_000, and the @@ -615,9 +727,9 @@ Rscript 1_code/tests/run_tests.R **2. Confirm the v2 results are unchanged.** Draws are seeded per species and made in order, so draws 1-5 of a 5-draw run match -draws 1-5 of a 100-draw run. Run the `parity_check` species at 5 -draws to a scratch folder (set `pipeline_dir` and `out_dir` in -`experiment_config()`), then compare: +draws 1-5 of a 100-draw run. Run exp_000's configuration with its +`species.csv` at 5 draws to a scratch folder (set `pipeline_dir` +and `out_dir` in `experiment_config()`), then compare: ```sh Rscript 1_code/tests/compare_stores.R \ @@ -645,6 +757,11 @@ tools is in the [tests README][tests]. `harness/`, `methods/`, or `modules/` as a change to every experiment: include passing tests and a `compare_stores.R` result. +- **Experiments edit only their own folder:** `run.R`, + `species.csv`, `hooks/`, and any scripts beside them. A change + that needs `harness/`, `methods/` or `modules/` is a change to + every experiment; make it separately, with tests and a + `compare_stores.R` result. - **Commits from `3_output/`:** commit only runs that count, every species at 100 draws, and check `run_record.md` says so. diff --git a/docs/mammals_climate_design.md b/docs/mammals_climate_design.md new file mode 100644 index 0000000..04f8ff4 --- /dev/null +++ b/docs/mammals_climate_design.md @@ -0,0 +1,420 @@ +# Mammal climate stage: design + +Phase 2 of porting the v2.0 mammal climate stage. It builds on +[mammals_climate_inventory.md](mammals_climate_inventory.md). This is +a proposal only; no code or data has changed. + +Revised 2026-10-14: the climate stage fits on v2's own training data +(the 2022 combined SpTable), added to the test dataset as +province-wide mammal rows (option 1). This replaces the earlier plan +to merge the two regional files. + +**Status, 2026-10-14:** + +- Implemented and verified on a local rebuild of the dataset (see + 7.3 and the session summary). +- Not yet published: the new dataset version and the pin move in + `harness/data_source.R` wait for approval. Until then the default + mammal spec cannot run on the pinned `2026-10-05` dataset, which + has no province rows. + +## 1. Decisions taken + +| # | Question | Decision | +| --- | --- | --- | +| D1 | Parity target | ABMIexploreR's published outputs (`0_data/v2_results/abmiexplorer/v2_results.csv`) | +| D2 | Climate training data | v2's own: `R Dataset SpTable for ABMI North and South mammal coefficients 2022.RData`, added to the test dataset by `_setup` (section 3) | +| D3 | Seasons | As the original code: one climate model per species, seasons pooled | +| D4 | Draws and staging | As birds and plants: a province-wide climate stage, refitted on each province-wide draw, carried as `Climate` into the regional habitat stage | + +`mammal_spec()` defaults to `climate_source = "fitted"`, so +`standard_specs()` and `exp_000_parity_v2` use it with no config edit. +`"precomputed"` stays available for regression checks. + +## 2. Shape + +```text +province frame: v2's 2022 table, one row per deployment + └─ province draw (spatial blocks, draw 1 = all data) + └─ climate stage: 9 binomial GLMs on pooled presence, + AICc full average, unweighted + └─ carry plogis(link) as `Climate` + ├─ north habitat hurdle (draw mapped to north units) + └─ south habitat hurdle (draw mapped to south units) +``` + +This is the bird and plant shape. Three things differ, each because of +how the mammal data are laid out: + +- the climate data are a separate table from the habitat data; +- the response pools two seasons; +- v2 drops habitat rows with no climate prediction. + +## 3. The data: v2's climate table in the test dataset + +### 3.1 Why this file + +The climate stage reads this file (`climate/01_prepare-data.R:50-55`), +and it holds all 4,927 deployments v2 predicted for. The 2024 regional +files the test dataset is built from lack 120 of them: 107 Grassland +and 13 Boreal sites, dropped by regional habitat filters the climate +stage never applied. The file sits at +`G:/Shared drives/ABMI Mammals/Results/Density/`, dated 2024-10-09. + +It was written by `north-models/archive/03 Prepare Data for +Prediction.R:370`. That script blanks *summer* densities when +`WinterDays < 10`, a bug the 2026 regional scripts fixed. v2's climate +fit includes the bug, and so will the port, through the data. + +### 3.2 What `_setup` adds + +The new rows sit under a new covariate key, `mammal_province`, beside +`mammal_north` and `mammal_south`. Existing rows and files do not +change. + +| File | Adds | From | +| --- | --- | --- | +| `mammal.csv` | 5,435 rows, `survey_unit_id` = `mammal_province\|`. The 4 duplicated deployments get the suffix convention `_setup/01` already uses for regional duplicates | The 2022 table's 78 species-season density columns, renamed to harness names through `species_queue.csv` `source_name`. Species with no `mammal.csv` column are left out; columns with no 2022 species are NA | +| `sites.csv` | The same 5,435 units: `location`, `project`, `lured`, `summer_days`, `winter_days`, `lat`, `long`, `nr`, `nsr`, `luf`, `in_mammal = TRUE`, `region = "province"` | `location`, `project`, `Lured`, `SummerDays`, `WinterDays`, `Lat`, `Long`, `NR`, `NSR`, `LUF` | +| `covariates.csv` | The same units, key `mammal_province`: `location_project`, `FFP`, `MAP`, `TD`, `CMD`, `EMT`, `WetlandMargin`, `Water`; other columns NA | Climate from `abmi-camera-climate_2023.Rdata`, joined **as `01_prepare-data.R:81-106` does**: strip `_YYYY`, then `CMU-` and `NWSAR-`, one row per `location`, left join on `location`. `WetlandMargin` and `Water` from the 2022 table (the region filters read them) | +| `lookup/covariate_columns.csv` | Rows for `mammal_province` | `_setup/01` | +| `lookup/dataset_manifest.csv` | `mammal,province,mammal.csv,,mammal_province,,,` | `_setup/07` | + +The table keeps raw densities and days. Pooling to presence is model +logic, so it stays in the module (3.4 and 4.2), where an experiment +can change it. + +### 3.3 `_setup` changes + +| Script | Change | +| --- | --- | +| `00_mirror_setup_inputs.R` | Add the 2022 table to the `abmi_mammals` paths, so it is mirrored to ABMI-DATA2 | +| `01_harmonize_model_ready_v2.R` | New section: read the 2022 table and the camera climate, write the province rows (3.2). Reuses `read_camera_climate()`, which already applies v2's join (year suffix, then `CMU-` and `NWSAR-` stripped, one row per location) | +| `02_validate_test_dataset.R` | Trace province rows back to the 2022 table and camera file. Check that the deployments with a non-NA pooled Moose response equal `mammal_climate_predictions.csv`'s 4,927 | +| `07_harmonize_lookups.R` | The manifest row | +| `08_publish_datasets.R` | No change; run it to publish a new version | +| `harness/data_source.R` | Move the `test_dataset` pin to the new version (one line), after the checks in 7.3 pass | + +**Before publishing**, regenerate locally and compare with the +published `2026-10-05` copy. Every file except the four above must be +byte-identical, and in those four every existing row must be +unchanged. This guards against re-harmonizing other taxa from drifted +sources (see the bird data-version memory). Runs read the local copy +through `SDM_TEST_DATASET` until the new version is published. + +## 4. Where logic lives + +### 4.1 Harness (taxon-agnostic), three changes + +Each is off unless the manifest or a spec sets the new item. Bird and +plant data and specs set none, so their code paths are unchanged. + +**H1. A province frame from the manifest.** + +- *Why:* `province_context()` (`harness/run_model.R:398`) builds the + frame with `region = NULL`, which reads covariate key `mammal`, and + that has no rows. The climate table is a separate province dataset. +- *What:* if the taxon's manifest has a `province` row, + `province_context()` builds with `region = "province"`. Otherwise it + keeps `region = NULL`, as birds and plants do today. The + configuration check (`check_spec_covariates()`, + `harness/spec.R:447-459`) checks province covariates the same way. + Any code that lists a taxon's regions from the manifest must skip + `province`; the implementation will search for such uses. + +**H2. Province draws mapped to regions by key.** New spec field +`province_key = "location_project"`. + +- *Why:* bird and plant regions share `survey_unit_id`s with the + province frame, so a draw is filtered per region by id + (`run_model.R:1009-1010`). Mammal province and regional units have + different ids for the same deployment. +- *What:* when `province_key` is set, `fit_species()` + (`run_model.R:610`) builds a lookup from each region unit to one + province unit: the first with the same key. Each drawn province unit + expands to the region units that map to it, keeping repeats. + - At draw 1 every mapped region unit appears exactly once, so the + regional frames at draw 1 equal today's, less units with no + province row. + - Units with no province row are never drawn. These are the 42 + test-dataset deployments missing from v2's table. Dropping them is + v2's `inner_join` on `location_project`. + +**H3. Paired response columns.** New spec field +`response_companions = function(species)`, returning a named vector +of other response columns the province frame should carry. + +- *Why:* the pooled response needs both seasons, but a frame holds + one species column (`model_frame()`, `harness/data_load.R:802`). +- *What:* `province_context()` also loads the companion columns. + `model_frame()` gets an optional `companions` argument and attaches + each as `response_`, e.g. `response_summer`. Only the + province frame uses it. + +**Found during implementation** (also taxon-agnostic; no effect +on birds or plants, whose specs leave `weight_column` NULL and set +no `province_key`): + +- `province_request()` no longer asks for the spec's + `weight_column`. Province frames are built unweighted, and the + mammal climate rows have no `wt_summer`. +- `fit_species()` fits province stages on the province draw, not on + the region's mapped copy (part of H2; the two are the same object + when no key is set). +- `check_spec_covariates()` says plainly when a spec needs a + province table that the dataset lacks. + +**Not changed:** + +- selection (`aic_average` already does MuMIn's full average); +- the glm engine (its AICc and `na.omit` match MuMIn and `glm`); +- `carry_values()`. Regional climate covariates equal the province + values at every deployment with a v2 prediction (checked: no + regional row with a v2 prediction lacks climate, and matched values + differ by 0). So computing `Climate` on regional rows gives v2's + joined prediction. +- the result store and the comparison scripts. + +### 4.2 Mammal module (`1_code/modules/mammals/`) + +New file `climate.R`, a port of `01_prepare-data.R` and the regional +join: + +| Function | Ports | Does | +| --- | --- | --- | +| `mammal_season_pair(species)` | `01:128-152` | `Moose_Summer` gives `c(summer = "Moose_Summer", winter = "Moose_Winter")`. The spec's `response_companions` | +| `mammal_pooled_density(summer, winter, summer_days, winter_days)` | `01:156-166` | Base `weighted.mean(c(s, w), c(sd, wd))` per unit. Base R drops zero-day seasons; data.table's version does not, and would lose 97 of v2's Moose sites | +| `mammal_climate_response(values, frame)` | `01:156-175` | The climate stage's `response_transform`. Pools seasons per `location_project` across distinct units (v2's `group_by`, which merges the 4 duplicated deployments). Presence = pooled > 0, on the first unit's rows; NA on other units sharing that deployment, and where pooled density or `FFP` is NA. Bootstrap repeats are kept. `glm`'s `na.omit` drops the NA rows, giving v2's 4,927 rows at draw 1 | +| `mammal_climate_available(frame, species, data_dir, spec)` | north `03:359-362`, south `03:329-332` | The spec's `species_frame` when fitted. Keeps regional rows whose `location_project` has a non-NA pooled response in the province data, as v2's `inner_join` + `filter(!is.na(Climate))` does. Recomputes the response, since lure correction and scaling depend on the rows kept, as `mammal_precomputed_climate()` does | + +`spec.R` changes: + +- `climate_source = c("fitted", "precomputed")`: fitted is the + default. +- The climate stage becomes: + + ```r + list( + name = "climate", + models = "climate_bird_mammal_v2", + engine = "glm", + selection = "aic_average", + ic = "AICc", + scope = "province", + weighted = FALSE, + response_transform = mammal_climate_response, + carry_as = "Climate" + ) + ``` + + `family` stays the spec's `"binomial"`. `carry_scale` stays + `"response"` (`plogis`), matching v2's 0-1 site predictions. +- New fields when fitted: `province_key = "location_project"`, + `response_companions = mammal_season_pair`, + `species_frame = mammal_climate_available`, and + `resample$scope = "province"`. +- The habitat stage is unchanged: `carry_from = "climate"`, Climate in + presence only, `abundance_drop = "Climate"`. +- `v2_coverage$climate` becomes "reproduced"; `v2_coverage$resampling` + notes province draws; `notes` is updated. + +`hurdle.R` keeps `mammal_precomputed_climate()` for the non-default +path. + +### 4.3 Not changed + +The bird and plant modules, the `exp_000_parity_v2` config, READMEs, +and vignettes. + +## 5. Interface + +### 5.1 New spec fields (validated in `harness/spec.R`) + +| Field | Type | Default | Used by | +| --- | --- | --- | --- | +| `province_key` | character, a column in both province and region frames | absent: today's behaviour | H2 | +| `response_companions` | `function(species)` returning a named character vector | absent | H3 | + +H1 is driven by the manifest, not a spec field. + +### 5.2 Module functions + +| Function | Arguments | Returns | +| --- | --- | --- | +| `mammal_season_pair` | `species` | Named character, `summer`, `winter` | +| `mammal_pooled_density` | four numeric vectors | Numeric; NA as base `weighted.mean` gives | +| `mammal_climate_response` | `values`, `frame` | Integer 0/1 with NA, one per row | +| `mammal_climate_available` | `frame`, `species`, `data_dir`, `spec` | The frame, rows kept, response recomputed | + +These follow the existing `response_transform` and `species_frame` +signatures (`harness/data_load.R:935-957`, `run_model.R:662-664`). + +## 6. Config (what an experiment can change) + +| Setting | v2 default | Changed with | +| --- | --- | --- | +| Candidate set | `climate_bird_mammal_v2` (9) | `stage_models = list("mammal.climate" = …)`, `extend_models()`, `drop_candidates()` | +| Engine, selection, `ic` | `glm`, `aic_average`, `AICc` | `modify_stage(spec, "climate", …)`, `replace_stage_method()` | +| Weighting | unweighted | `modify_stage(spec, "climate", weighted = TRUE)` | +| Climate covariates | FFP, MAP, CMD, TD | The candidate set; `covariate_files` (keyed on `survey_unit_id`, so province ids) for new ones | +| Species | every mammal | `species.csv` | +| Draws | spatial block, province scope, 20 detections | `modify_resample()` | +| Source of `Climate` | fitted | `mammal_spec(climate_source = "precomputed")` | + +Fixed in the module: the pooled-season response and dropping +deployments with no climate response. These define v2's data, not the +model. + +## 7. Hand-off, outputs, and parity + +### 7.1 Hand-off and outputs + +- **Hand-off:** `carry_values()` applies the averaged coefficients to + the regional `fitting` and `frame` rows, then `plogis`. The result + is the `Climate` column for the hurdle (`run_model.R:1027-1051`). +- **Stores:** `2_pipeline////coefficients.csv` gets + stage `climate` rows (Intercept, FFP, MAP, CMD, TD; full average, + absent terms 0). + - They are written in both regions and both season runs, identical + at draw 1. + - `01_compare_to_v2.R` already maps them to region `all`, part + `presence`, and averages seasons. +- **Not produced:** v2's per-species climate AUC and 1 km map + (spatial prediction is deferred). + +### 7.2 Seasons and draws + +At draw 1, summer and winter runs fit the same model: same rows, same +response. Later draws differ, because each season run seeds its own +species name (`species_seed()`). v2 has no climate bootstrap, so +nothing to match. Sharing draws would need a seeding change; I propose +not making it. **Confidence: Moderate.** This is a choice, not a v2 +fact. + +### 7.3 Parity plan + +All at draw 1, against ABMIexploreR, with the existing mammal +numerical rule (every term within 1e-6, +`exp_000_parity_v2/README.md:105`): + +- (a) climate coefficients (Intercept, FFP, MAP, CMD, TD), Moose and + Coyote; +- (b) habitat presence, abundance, and total tables, north and south, + seasons averaged, same species. + +Internal checks: + +- (c) summer and winter runs give identical climate coefficients; +- (d) regional rows kept equal the `"precomputed"` path's rows; +- (e) carried `Climate` against `lookup/mammal_climate_predictions.csv`, + within 1e-6 on every kept regional row; +- (f) the province rows give 4,927 deployments with a non-NA pooled + response for Moose and Coyote. + +**Result (2026-10-14, local rebuild):** pass. + +- (a) Climate, 10 of 10 terms; max difference 7.0e-14. +- (b) Habitat, north 436 of 436 reachable terms (9.8e-15), south + 120 of 120 (5.3e-15). The precomputed path gives the same. +- (c) Identical (difference 0). +- (d) Identical rows in all four run and region pairs; habitat + coefficients within 1e-14. +- (e) Within 5.2e-15 on every deployment with a v2 prediction. +- (f) 39 of 39 species (`_setup/02`). + +**Expected result (before the run):** pass on (a) and (b). + +- With v2's training data, rows, response, and model all match, so + (a) should agree to numerical precision. +- If (a) and (e) pass, the habitat stage sees v2's rows and v2's + `Climate`, so (b) should stay within 1e-6. It matches to 1e-15 + under `"precomputed"` today. + +**Confidence: Moderate.** Residual risks: + +- `aic_average` against `MuMIn::model.avg` at 1e-6. The weights and + AICc are the same formulas, but untested at this tolerance for + mammals. +- `glm` convergence differences across R versions. +- The precision of v2's prediction CSV. + +The report gives, per term: v2 value, harness value, absolute and +relative difference, pass or fail, and the cause of any failure. + +### 7.4 Runs + +| Run | What | Notes | +| --- | --- | --- | +| Data | `_setup/00` (mirror the file), `01`, `07`, `02` locally; compare with published `2026-10-05` (3.3) | Writes `0_data/test_dataset/` locally. Nothing published yet | +| Parity | Scratchpad driver, `run_experiment()` with its own `id` and pipeline and output directories in the scratchpad. `taxa = c("mammal_summer", "mammal_winter")`, Moose and Coyote, `n_bootstraps = 1`, `baseline = NULL`, `SDM_TEST_DATASET` = local copy | Nothing written to `2_pipeline/exp_000_parity_v2/` | +| Draw plumbing | Same, `n_bootstraps = 3`, Moose summer | Province draws reach both regions. Not scored | +| Regression | Five seeded draws of `mammal_spec(climate_source = "precomputed")`, before and after, compared with `1_code/tests/compare_stores.R` | Expected identical; province rows are never read by regional frames | +| Birds and plants | `validate_spec()` on every `standard_specs()` entry; `Rscript 1_code/tests/run_tests.R`; parse checks | A five-draw bird and plant run only with your approval | +| Tests | `1_code/tests/testthat/`: H1 (manifest province row read), H2 (draw 1 maps each region unit once; repeats kept; unmatched units dropped), H3 (companions attached), `mammal_pooled_density()` against base `weighted.mean` | | +| Publish | `_setup/08`, then move the pin in `data_source.R` | **Only after you approve the results.** A published version cannot be overwritten | + +## 8. Consequences to accept + +1. A new published test dataset version. Earlier runs stay + reproducible on the old pin. +2. Mammal draws 2 and up change (province blocks instead of regional + blocks). Draw 1 does not. +3. `exp_000_parity_v2`'s committed `report.md` and `run_record.md` are + stale for mammals until a full rerun. +4. Mammal runs fit nine extra small GLMs per species and draw. +5. v2's winter-days bug stays in the climate fit, deliberately, for + parity. + +## 9. Suggested improvements (not in scope) + +- Rebuild the climate table from the current density data with the + bug fixed, as an experiment. +- Score the climate stage's own AUC, as v2's `All Species Climate + AUC.csv` does. +- Share province draws across the two season runs (7.2). +- The 1 km climate prediction with v2's 99th-percentile cap + (`make-species-predictions.R:485-505`), once spatial prediction is + in scope. + +## 10. Files that will change + +| File | Change | +| --- | --- | +| `1_code/_setup/00_mirror_setup_inputs.R` | Mirror the 2022 table | +| `1_code/_setup/01_harmonize_model_ready_v2.R` | Province rows (3.2) | +| `1_code/_setup/02_validate_test_dataset.R` | Province-row checks | +| `1_code/_setup/07_harmonize_lookups.R` | Manifest row; grid check skips a row with no grid | +| `1_code/harness/run_model.R` | H1 in `province_context()`, H2 in `fit_species()`, H3 loading | +| `1_code/harness/data_load.R` | `model_frame()` companions | +| `1_code/harness/spec.R` | `spec_fields()`, `validate_spec()`; `check_spec_covariates()` province check | +| `1_code/harness/data_source.R` | Pin bump, after publishing | +| `1_code/modules/mammals/climate.R` | New (4.2) | +| `1_code/modules/mammals/spec.R` | Default, climate stage, new fields, coverage text | +| `1_code/tests/testthat/` | Tests in 7.4 | + +READMEs and docs to update afterwards: + +- `1_code/_setup/README.md`: new input, province rows; +- `1_code/modules/README.md`: mammals rows, `climate.R`; +- `1_code/harness/README.md`: `province_key`, `response_companions`, + the manifest `province` row; +- `exp_000_parity_v2/README.md`: coverage table and parity status; +- `docs/taxon_quirks.md`: "Climate is external" and the winter-days + bug; +- `docs/framework_design.md`: decision 3. + +## 11. Open items for the mammals lead + +- `TODO: confirm with mammals lead`: whether the climate stage should + keep reading the 2022 table, now that the regional scripts no + longer write a combined file, and whether the winter-days bug is + known. +- `TODO: confirm with mammals lead`: whether v2's climate models for + species groups (`Deer`, `Wolves, Coyotes and Allies`, `Voles, Mice + and Allies`) are used downstream. +- `TODO: confirm with mammals lead`: package versions of the v2 + climate run (MuMIn 1.48.11 here). +- `TODO: confirm with mammals lead`: the south density CSV named in + `south-models/02_process-data-files.R` + (`Results/Density/Deployments/Archive/abmi-cmu-bg-nwsar_all-years_density_wide_2022-01-13.csv`) + is no longer on G:. diff --git a/docs/mammals_climate_inventory.md b/docs/mammals_climate_inventory.md new file mode 100644 index 0000000..75396c1 --- /dev/null +++ b/docs/mammals_climate_inventory.md @@ -0,0 +1,299 @@ +# Mammal climate stage: inventory + +Phase 1 of porting the v2.0 mammal climate stage into the shared +harness, so mammals follow the bird and plant pattern: a +province-wide climate stage fitted in the run, carried as `Climate` +into the regional habitat models. Read-only; nothing in the repository +was changed. + +- **Source:** `ABbiodiversity/MammalModels` at `2874dec` + (2026-09-01), `1_code/2_habitat-modeling/climate/`. Read from a + shallow clone in a session scratchpad; never run. +- **Date:** 2026-10-14. +- **Checks run:** read-only R scripts (data.table only, no model + fits) that rebuild v2's training set from v2's own inputs and from + the test dataset, and compare both with v2's outputs. + +## 1. The source climate stage + +### 1.1 Run order + +| Order | File | Does | +| --- | --- | --- | +| 1 | `01_prepare-data.R` | Joins camera climate to the 2022 SpTable, pools seasons into one presence per deployment and species, saves `d_model` | +| 2 | `02_run-models.R` | Sources `00_models.R`; fits nine binomial GLMs per species, averages by AICc, writes coefficients, site predictions, AUC, and maps | +| - | `00_models.R` | Defines `modelsclimate`, the nine formulas | + +Nothing else is sourced. The climate scripts are **not** in +`0_data/v2_scripts/mammals/`, which holds only the `2024/` folder. +Those 21 files match `2874dec` blob for blob (checked 2026-10-14). + +### 1.2 Inputs + +| Input | Path in source | Made by | Reachable now | +| --- | --- | --- | --- | +| Density table `d` | `G:/Shared drives/ABMI Mammals/Results/Density/R Dataset SpTable for ABMI North and South mammal coefficients 2022.RData` (`01:50-55`) | Upstream density pipeline, outside this folder. `TODO: confirm with mammals lead` which script | Yes (dated 2024-10-09) | +| Camera climate `camera.climate` | `S:/sc/AB_data_v2023/sites/processed/climate/abmi-camera-climate_2023.Rdata` (`01:58-62`) | ABMI climate extraction | Yes (dated 2025-12-17); already mirrored for `_setup/01` | +| kgrid | `S:/sc/AB_data_v2023/kgrid/kgrid_2.2.Rdata` (`02:57-59`) | ABMI | Not checked. Used only for maps and the 1 km prediction | +| Provincial boundary | `G:/.../Data/Spatial/AB Provincial Boundary/lpr_000b21a_e.shp` (`02:62-66`) | Statistics Canada | Not checked. Maps only | + +The 2022 SpTable is a different file from the two 2024 regional +SpTables that `_setup/01` builds the test dataset from +(`_setup/01_harmonize_model_ready_v2.R:106-115`). + +### 1.3 Response + +Built in `01_prepare-data.R:128-180`: + +1. Pivot every `Summer` and `Winter` density column + to long, with `days` = `SummerDays` or `WinterDays`. +2. Per `location_project` and species: `weighted.mean(density, + w = days)`, then `present = full_density_km2 > 0`. +3. Drop rows where the mean is NA or `FFP` is NA. + +Three details matter for a faithful port: + +- **Zero-day seasons are ignored.** Base `weighted.mean()` drops + zero-weight elements, so a season with 0 days and NA density does + not make the mean NA. 97 of v2's Moose sites depend on this. + (data.table's grouped `weighted.mean` does not drop them; a port + must not use it.) +- **No lure correction, no scaling, no effort filter.** The response + is raw 0/1. There is no `seas_days > 10`, `WetlandMargin`, or + `Water` filter; every deployment with a density and climate is in. +- **Duplicate `location_project` rows are pooled.** `d` has 5,435 + rows and 5,431 unique `location_project`. Grouping pools + duplicates into one weighted mean, giving 4,931 Moose rows for + 4,927 deployments. + +### 1.4 Climate covariates + +`01_prepare-data.R:81-106`. `camera.climate` rownames are +`location_YYYY`. The year suffix and `CMU-` / `NWSAR-` prefixes are +stripped, and one row is kept per `location`. The join is on +`location` only (normals do not vary by year). The SpTable's own +`AHM, PET, FFP, MAP, MAT, MCMT, MWMT` are dropped and replaced by +`FFP, MAP, TD, CMD, EMT` from the camera file. `EMT` is carried but no +formula uses it. + +### 1.5 Model form + +| Element | Value | Source | +| --- | --- | --- | +| Candidates | `present ~ 1`, `~ FFP`, `~ MAP`, `~ CMD`, `~ TD`, `~ TD + FFP`, `~ MAP + CMD`, `~ TD + FFP + CMD`, `~ MAP + FFP + TD + CMD` | `00:29-44` | +| Engine | `glm(f, data, family = binomial)`, defaults otherwise | `02:97-101` | +| Weights, offset | None | `02:100` | +| Transformations | None | `00:29-44` | +| Failed fits | Dropped silently (`try()`, then `keep()`) | `02:99-104` | +| Averaging | `MuMIn::model.avg(models, rank = "AICc")`; full average (absent terms zero) for coefficients | `02:105`, `02:213-216` | +| Site prediction | `plogis(predict(avg, type = "link", newdata = sp_data))`; MuMIn's default `full = TRUE` | `02:131-153` | +| Hyperparameters, seeds, bootstrap, CV | None. One fit per species on all data | `02:113-118` | + +### 1.6 Fitting scope + +- One model per species, **seasons pooled**, **province-wide** (north + and south together), no regional split. +- All 39 species and species groups in the SpTable, including + `Deer`, `Wolves, Coyotes and Allies`, and `Voles, Mice and Allies`. + ABMIexploreR publishes 21 of them. + +### 1.7 Outputs and how they are consumed + +| Output | Written at | Consumed by | +| --- | --- | --- | +| `Coefficients/ Climate Coefficients.RData`: `avg_coef` (term, full-average coef) and `climate_auc_sp` | `02:213-230` | `make-species-predictions.R:485-505`: `plogis(kgrid %*% coef)` per 1 km cell, capped at the 99th percentile; then `PA_adj = plogis(climate_logit + logit(PA_v))` | +| `Predictions/All Species Climate Predictions.csv`: one row per `location_project`, one probability column per species (full common names) | `02:131-153, 258` | North `03_basic-models.R:83-87, 128-129, 289-295, 328-331, 359-362` and south `88-92, 127-128, 237-264, 298-301, 329-332`: pivoted long, `inner_join` on `location_project`, `filter(!is.na(Climate), !is.na(Count))`, fitted as the `Climate` covariate in the presence models only. South first converts names to PascalCase | +| `Predictions/All Species Climate AUC.csv` | `02:163-190, 260-267` | Reporting only | +| `Prediction Maps/ Climate Predictions.png` | `02:233-252` | Reporting only | + +In the habitat stage, `Climate` enters presence only. Abundance drops +it (north `03:494-496`), and the stored `Climate` presence value is +`plogis(coef)` of its slope (north `03:581-584`). The harness already +ports all of this (`modules/mammals/hurdle.R`). + +### 1.8 Paths and packages + +- Hard-coded: `G:/Shared drives/ABMI Mammals/`, `S:/sc/AB_data_v2023/`, + and a relative `2_pipeline/climate/`. Marked `!!` to update each + cycle (`01:49, 57`). +- Packages: `tidyverse`, `sf` (maps), `MuMIn` (averaging). Installed + here: MuMIn 1.48.11, R 4.5.0. The v2 run's versions are not recorded. + `TODO: confirm with mammals lead`. + +## 2. Checks against v2's outputs + +| Check | Result | +| --- | --- | +| v2 coefficient file (Moose) against ABMIexploreR `v2_results.csv` | Identical on all five terms. The two v2 references agree | +| Prediction CSV sites in the 2022 SpTable | 4,927 of 4,927 | +| Training set rebuilt from v2's inputs (Moose), with base `weighted.mean` | 4,931 rows; matches the CSV's 4,927 deployments plus the 4 pooled duplicates | +| Same, from the test dataset (both regions) | 4,849 unique deployments. **120 of v2's sites are absent, 42 sites are extra** | +| Test dataset north rows against v2, matched sites | 4,195 matched; presence and days agree on all | +| Test dataset south rows against v2, matched sites | 1,125 matched; presence agrees on all but one (Coyote); **days disagree at 110** | +| Overlapping deployments (in both regions) | 516; climate agrees on all; days disagree at 103; Moose densities at 196 | +| Climate values, test dataset against camera file | Identical where matched (max difference 0) | + +**Consequence.** Fitted on the test dataset, the climate stage cannot +match v2's coefficients to 1e-6, because its training set is not v2's. +The differences are in the data, not the method. + +## 3. Target repository + +### 3.1 What exists + +| Piece | Location | State | +| --- | --- | --- | +| Province-scope climate stage | birds `modules/birds/spec.R:71-89`; plants `modules/_shared/plant_group.R:115-125` | `scope = "province"`, `weighted = FALSE`, `aic_average`, `carry_as = "Climate"` | +| Province draws | birds `modules/birds/spec.R:106-113` | `resample$scope = "province"`, filtered per region | +| Province fitting | `harness/run_model.R:398-470` (`province_context()`, `province_request()`), `610-748` (`fit_species()`, `fit_province_stages()`) | Frame built with `region = NULL`, no weight column | +| Carry | `harness/run_model.R:948-972` (`carry_values()`), `1027-1051` | `carry_scale`: response, link, exp | +| Averaging | `methods/selection/aic_average.R` | MuMIn-style full average; predictor averages link predictions | +| Candidate set | `harness/model_sets.R:35-45` `climate_bird_mammal_v2` | The same nine formulas as `00_models.R`, null included | +| Mammal climate today | `modules/mammals/spec.R:141-165`; `modules/mammals/hurdle.R:59-88` | Default `"precomputed"` reads v2's CSV (`lookup/mammal_climate_predictions.csv`). `"fitted"` is region-scope, per season, season-weighted, lure-corrected (`mammal_pa_response`); marked `partial` | +| v2 references | `0_data/v2_results/abmiexplorer/v2_results.csv` (21 species, `mammal,all,climate,presence,all`); `test_dataset/lookup/mammal_climate_predictions.csv` | Both present | +| Comparison | `experiments/exp_000_parity_v2/01_compare_to_v2.R:152-172` | Climate rows mapped to region `all`, part `presence` | + +### 3.2 Climate covariates in `0_data/` + +No `manifest.md` files exist in `0_data/`; the manifest is +`test_dataset/lookup/dataset_manifest.csv` plus `covariate_columns.csv`. + +| Covariate | Mammal north | Mammal south | Source | +| --- | --- | --- | --- | +| FFP, MAP, TD, CMD | Yes | Yes | Camera climate, joined by `_setup/01:263-277, 1037-1070` | +| EMT | Yes | Yes | Same; unused by v2 formulas | +| `summer_days`, `winter_days` | Yes | Yes | `sites.csv` | +| Per-season density | Yes | Yes | `mammal.csv`, `_Summer`, `_Winter` | + +Nothing the formulas need is missing. What is missing is v2's exact +training set (section 2). + +### 3.3 Where the climate output plugs in + +With the province stage, `carry_values()` writes `Climate` to each +regional frame (`run_model.R:1027-1031`). The hurdle stage reads it +with `carry_from = "climate"` (`modules/mammals/spec.R:247-248`), +replacing `mammal_precomputed_climate()` (`hurdle.R:59-88`), which +today attaches the CSV value and drops rows without one. + +## 4. Mapping table + +| Source element | Source location | Target location | Mechanism | Notes | +| --- | --- | --- | --- | --- | +| Nine candidate formulas | `00_models.R:29-44` | `harness/model_sets.R:35-45` (`climate_bird_mammal_v2`) | Config default (existing set) | Already identical; no change | +| Density input (2022 SpTable) | `01_prepare-data.R:50-55, 69` | `test_dataset/mammal.csv` (2024 regional SpTables) | Harness data layer | Different file; see section 2 and Q1 | +| Camera climate join on location | `01_prepare-data.R:58-62, 74-106` | `test_dataset/covariates.csv` via `_setup/01:263-277, 1037-1070` | Existing data | Values identical where matched | +| Drop deployments without climate | `01_prepare-data.R:108-117, 169` | Province frame: rows with NA climate | Engine `na.action` or module response helper | Design decides where | +| Season pooling: `weighted.mean(density, days)`, zero-day seasons ignored | `01_prepare-data.R:128-166` | New mammal climate response helper in `modules/mammals/` | Module function | Needs both season columns of one species; the harness gives a stage one column (Q3) | +| `present = mean > 0`, raw 0/1 | `01_prepare-data.R:167-175` | Same helper | Module function | No lure correction, unlike today's `"fitted"` | +| No filters (effort, wetland, water) | `01_prepare-data.R:156-170` | Province frame, `region_filter = NULL` | Harness (existing) | Already how province frames are built | +| One row per `location_project`; duplicates pooled | `01_prepare-data.R:156-160` | Province frame for a region-keyed taxon | Harness change (Q2) | 512-516 deployments sit in both regional files | +| Province-wide, seasons pooled, one fit per species | `02_run-models.R:113-118` | Stage `scope = "province"` | Spec field | Both season runs would fit the same model (Q3) | +| `glm(..., family = binomial)`, unweighted | `02_run-models.R:97-101` | `engine = "glm"`, `family = "binomial"`, `weighted = FALSE` | Spec fields | Today's `"fitted"` is weighted | +| Failed fits dropped | `02_run-models.R:99-104` | `harness/selection.R` `fit_candidates()` | Harness (existing) | Confirm it drops rather than stops (Phase 2) | +| `model.avg(rank = "AICc")`, full average | `02_run-models.R:105, 213-216` | `selection = "aic_average"`, `ic = "AICc"` | Registered rule | Bird parity already relies on it | +| Site prediction `plogis(link)` | `02_run-models.R:131-153` | `carry_values()`, `carry_scale = "response"` | Spec field (default) | Matches the CSV's 0-1 scale | +| Regional join `inner_join` + `filter(!is.na(Climate))` | north `03:359-362`; south `03:329-332` | `carry_from = "climate"` in the hurdle stage | Spec field (existing) | Carry gives every regional row a value; v2 dropped rows without one. Rows without climate must still drop (Q4) | +| Coefficient file `avg_coef` | `02_run-models.R:213-230` | `2_pipeline////coefficients.csv`, stage `climate` | Harness (existing) | Written once per region, identical values | +| Per-species AUC | `02_run-models.R:163-190` | Metrics: `insample_auc` | Harness (existing) | Province-stage metrics: confirm they are scored (Phase 2) | +| Site predictions CSV | `02_run-models.R:258` | `unit_predictions = "all"` (optional) | Config | Not needed for the hand-off; used for parity check (b) | +| kgrid prediction, 99th percentile cap, maps | `02_run-models.R:194-254`; `make-species-predictions.R:485-505` | None | Out of scope | Spatial prediction is deferred | +| Hard-coded paths | `01:40-64`; `02:46-77` | `harness/data_source.R`, `test_dataset_dir()` | Repo convention | No new paths needed if Q1 is answered "test dataset" | + +## 5. Differences from the bird and plant climate stages + +| Aspect | Mammals (v2) | Birds (v2) | Plants (v2) | +| --- | --- | --- | --- | +| Candidates | 9 (`climate_bird_mammal_v2`) | 25 (+ linear and quadratic space) | 58 (`climate_plant_v2_full`) | +| Engine | `glm` | `glm` | `bayesglm`, `maxit = 250` | +| Weights | None | None | (see plant spec) | +| Response | Presence pooled over two seasons | Count with QPAD offset | Detection | +| Carry scale | Probability (`plogis`) | `exp(link)`, no offset | Probability | +| Draws | Fitted once, no bootstrap | 100 stored draws | 100 spatial-block draws | +| Data frame | Union of two regional files (needs a harness change) | One province file | One province file | +| Seasons | Two harness runs share one climate model | None | None | + +## 6. Missing inputs + +- v2's exact climate training set (2022 SpTable based). Not in the + test dataset; 120 sites absent and 42 extra. Reachable on G: and S: + for a scratchpad-only check. +- Nothing else. Every formula covariate is present. + +## 7. Open questions + +**Resolved 2026-10-14** (see +[mammals_climate_design.md](mammals_climate_design.md)): + +- Q1: parity is against ABMIexploreR, and the climate stage fits on + v2's 2022 table, added to the test dataset. +- Q2: replaced by that table. +- Q3: seasons are pooled, one model per species. +- Q6: the table was written by `north-models/archive/03 Prepare Data + for Prediction.R:370`, from + `all-projects_all-years_wide-density-for-habitat-modeling.csv`. +- Q7: draws are province-wide. + +The questions below are kept as first written. + +**Q1. Parity target.** The test dataset cannot reproduce v2's +training set. Options: + +- (a) Fit on the test dataset as the shipped default. In the parity + run, check the method separately on v2's own training set, rebuilt + in the scratchpad from the 2022 SpTable (exact, 1e-6), and report + the test-dataset differences with their cause. **Recommended.** +- (b) Add v2's climate training set to the published dataset. Needs a + `_setup` change and a new dataset version, against the "_setup is + fixed" rule. +- (c) Accept a looser tolerance on test-dataset fits only. + +**Q2. Province frame for mammals.** Covariates are keyed +`mammal_north` and `mammal_south`. A province frame needs both, +de-duplicated to one row per `location_project`. North values match +v2 everywhere; south days do not at 110 sites. Proposed rule: union +the region frames and keep the first region in manifest order (north) +for shared deployments. This is a taxon-agnostic harness change. + +**Q3. Seasons.** v2 fits one climate model per species; the harness +runs `mammal_summer` and `mammal_winter` separately, and a stage sees +one response column. The pooled response needs both season columns +and both day counts. Phase 2 will propose how a stage response helper +reads the paired season column (e.g. loading both columns for the +province frame), and confirm the two runs fit identical models. + +**Q4. Rows without climate.** v2 drops regional rows with no climate +prediction before the habitat fit. With a fitted stage, rows with NA +climate covariates get NA `Climate`. Phase 2 will confirm these rows +are dropped and not imputed, so the habitat stage sees the rows v2 +saw (on the test dataset, minus the section 2 differences). + +**Q5. Species groups.** v2 fits climate for groups (`Deer`, +`Wolves, Coyotes and Allies`, `Voles, Mice and Allies`) that the +habitat stage excludes or does not model. Out of scope unless they +are in the harness species queue. `TODO: confirm with mammals lead` +whether group climate models are used downstream. + +**Q6. Upstream density script.** Which script writes the 2022 +combined SpTable, and why the climate stage uses it rather than the +2024 regional tables the habitat stage uses. `TODO: confirm with +mammals lead`. + +**Q7. Draws.** v2 fits climate once. Following birds and plants, the +harness refits it per draw (draw 1 is the full data). Province-scope +draws (`resample$scope = "province"`) are needed so both regions share +one climate fit per draw. Draws 2 and up then differ from today's +mammal draws; draw 1 does not. + +## 8. Notes for later phases + +- `mammal_spec()` default changes from `"precomputed"` to `"fitted"`. + `"precomputed"` stays for the regression check. +- The source glosses CMD as "PET - MAP, >= 0". That is inaccurate + (`docs/framework_design.md:497-500`) but does not affect the port: + CMD is read, not derived. +- Sections needing updates after Phase 3: `1_code/modules/README.md` + (mammals row), `experiments/exp_000_parity_v2/README.md` (coverage + table), `docs/taxon_quirks.md` ("Climate is external" row), + `docs/framework_design.md` (decision 3), and `harness/README.md` if + the province frame changes. diff --git a/temp/experiment_config_design.md b/temp/experiment_config_design.md new file mode 100644 index 0000000..42547ed --- /dev/null +++ b/temp/experiment_config_design.md @@ -0,0 +1,554 @@ +# Experiment configuration design + +Phase 2 of the experiment-configuration work. It builds on the +existing R API and closes the leaks found in +`temp/experiment_config_inventory.md`. + +**Status: approved and implemented** (uncommitted, on `development`). +This document now describes the code as built. Section 15 lists what +changed after approval and why; section 12 gives the verification +results. + +## Decisions already made + +| Decision | Source | +| --- | --- | +| Configuration stays in R: `experiment_config()` in each `run.R`. No YAML. | Session, option A | +| Each experiment holds its own species list in its own folder. | Session | +| The shared reference for valid species names is the dataset's `lookup/species_queue.csv`; no new shared file. | Phase 1 review, answer 2 | +| `focal_species_sets()` and `experiments/_shared/focal_species.R` are removed. | Phase 1 review, answer 3 | +| Users can change, from their own experiment: methods (L1), species (L2), any spec setting (L3), regions and metrics (L5), spatial block cut points (L6), and comparison metrics and metric direction (L7). | Phase 1 review, answer 1 | +| L8 (an experiment-supplied prediction grid) is not designed. Spatial prediction is deferred. | Memory; inventory | +| L9 and L10 (derived covariates, named sets) need documentation only. | Inventory | +| The template's `species.csv` lists every species a run can fit, as `taxon,species,note` with empty notes; an experiment deletes the rows it doesn't fit. | After implementation (section 15) | +| Without `taxa`, the species list sets the runs, so users edit `species.csv` alone. | After implementation (section 15) | + +## The rule this design enforces + +> A user edits files in `1_code/experiments//` only. +> Everything else is the fixed back-end. Anything an experiment can +> change is either an argument to `experiment_config()`, a change to +> a spec made in `run.R`, or a function in the experiment's `hooks/`. + +## 1. Experiment folder layout + +``` +1_code/experiments/exp_NNN_short_description/ + run.R # entry point: load, configure, run + README.md # question, design, status, result + species.csv # the species this experiment fits; also sets the runs + hooks/ # optional: the experiment's own methods (*.R) + NN_*.R # optional: step scripts and input builders + utils/ # optional: helpers only this experiment uses +``` + +`run.R` keeps its current numbered sections, with one section per +kind of decision so a reviewer can find each change quickly: + +```r +# 1. Setup ---- +## 1.1 Load the framework and this experiment's hooks ---- +exp_dir <- "1_code/experiments/exp_NNN_short_description" +source("1_code/harness/harness.R") +load_framework(hooks = file.path(exp_dir, "hooks")) + +## 1.2 Spec changes ---- +# What this experiment changes in the v2 specs, if anything +specs <- standard_specs() +specs$lichen <- modify_stage(specs$lichen, "habitat", ic = "BIC") + +## 1.3 Configure the run ---- +config <- experiment_config( + id = "exp_NNN_short_description", + specs = specs, + # Sets the species and, with no `taxa`, the runs + species = file.path(exp_dir, "species.csv"), + ... +) + +# 2. Run ---- +results <- run_experiment(config, ...) +``` + +**`species.csv`**: columns `taxon`, `species` and `note`, one row +per species. `taxon` is the data slug (`bryophyte`, `lichen`, +`mite`, `vascular_plant`, `mammal`, `bird`), as in the former named +vectors. Mammal species carry their season (`Moose_Summer`). `note` +is free text (why a species was chosen) and is ignored by the +harness; other extra columns are ignored too. + +- **The template's file lists every species a run can fit** (1,117 + of the 1,140 in the dataset's species queue), sorted + alphabetically within each taxon, with empty notes. A new + experiment deletes the rows it doesn't fit. The 23 mammals left + out are in the `ua` tier only, which no mammal run fits. +- **The rows left set the runs.** When `run.R` passes no `taxa`, + `experiment_config()` runs only the runs that fit a listed + species (`runs_for_species()`): a `Moose_Summer` row runs + `mammal_summer` but not `mammal_winter`. Users edit + `species.csv` alone. +- **Given `taxa` still wins.** exp_000, exp_001 and exp_002 pass + `taxa`, so they run exactly as before. A run in `taxa` whose + taxon has no rows fits every species of that taxon, with a note + saying so; leaving `taxa` out avoids it. +- **Every species, deliberately:** `species = NULL` and no `taxa` + runs every run on every species, with no file. + +**Alternatives considered** + +- *Spec changes in a separate `specs.R` that `run.R` sources.* This + is cleaner for a large change, but adds a second file for the + common one-line case. The design allows it (users may add any + file to their folder) but doesn't require it. +- *`config.R` holding a list, read by a generic runner.* Rejected: + `run.R` already is that, and a second layer adds no isolation. + +## 2. Defaults + +The defaults stay where they are, and the design states that they +are the defaults file: + +- `experiment_config()` argument defaults (`harness/experiment.R`) + hold the run-level v2 defaults: seed, draws, plant bootstrap, + baseline, and the new arguments below, each defaulting to today's + behaviour. +- `standard_specs()` (`modules/_shared/standard_specs.R`) and the + taxon spec functions hold the model-level v2 defaults. + +An experiment passes only what it changes. An argument left out +falls back to the default, so exp_000 needs no overrides beyond its +`id`, `taxa`, `species`, `n_bootstraps`, and `workers`. (A new +experiment needs no `taxa` either; see section 1.) + +**New `experiment_config()` arguments**, all defaulting to today's +behaviour: + +| Argument | Default | Sets | Closes | +| --- | --- | --- | --- | +| `species` | `NULL` | Now also a path to a `species.csv`. Inline named vectors still work. Set names (`"parity_check"`) no longer exist. | L2 | +| `taxa` (new meaning when `NULL`) | `NULL` | With a species list: only the runs that fit a listed species. Without one: every run, as before. | Section 15 | +| `regions` | `NULL` (every region each spec defines) | Regions to fit, e.g. `"north"` | L5 | +| `metrics` | `NULL` (every registered metric) | Metrics scored per draw | L5 | +| `compare_metrics` | `NULL` (today's five `oob_` metrics) | Metrics in `comparison_metrics.csv` and the console headline | L7 | + +The hook files are not an `experiment_config()` argument. They have +to be loaded before specs are built (a spec can name a hook engine), +so they go to `load_framework()`. `experiment_config()` reads them +from the session and records them. + +**Alternative considered:** *a separate `harness/defaults.R` list +merged with overrides.* Rejected. It would duplicate the argument +defaults and the spec functions, and the two copies could drift. R +function defaults already give "specify only what changes" and are +documented in one roxygen block. **This departs from common practice** +in projects built on the `config` package, which keep defaults in a +YAML `default:` block. It suits this team, who write R and whose +settings include formulas and functions. + +## 3. Swappable components: registry and hooks + +### 3.1 The registry (unchanged) + +Built-in methods stay in `1_code/methods/` and are selected by name +in a spec or a config. The registry already has four kinds: + +| Kind | Built-in names | Selected by | +| --- | --- | --- | +| engine | `glm`, `bayesglm`, `xgboost` | stage `engine` | +| selection | `aic_average`, `aic_best`, `hurdle`, `ivw_grid`, `single`, `staged_bic` | stage `selection` | +| resampler | `precomputed`, `spatial_block`, `spatial_cv` | spec `resample$scheme` | +| metric | `auc`, `calibration_slope`, `deviance_explained`, `n`, `prevalence`, `rmse`, `spearman` | config `metrics` (new) | + +`list_methods()` prints them. + +### 3.2 Hooks + +A hook is an R file in the experiment's `hooks/` folder. It does +either or both of the following: + +1. Registers a method with the existing `register_engine()`, + `register_selection()`, `register_resampler()`, or + `register_metric()`. +2. Defines a plain function that a spec field names, such as a + `response_transform`, `row_filter`, `post_process`, or mammal + `abundance_response`. These functions are set with + `modify_stage()` (section 4). + +**Loading.** `load_framework(project_root = ".", hooks = NULL)` +gains a `hooks` argument: a folder, or a vector of `.R` files. After +the harness, methods, and modules load, it sources every `.R` file +in the folder in file-name order. It stores the normalized paths in +`options(sdm.hook_files = ...)`. `run_jobs()` passes those paths to +each worker, and `load_worker()` sources them after its own +`load_framework()`. Without that, a method registered in the main +session is unknown in a parallel run (inventory, L1). + +**Clash rule.** While hooks are loading, registering a name that is +already registered is an error naming the file and the existing +method. A hook can't silently replace a built-in, which would change +every other experiment's meaning of that name within the session. +Re-sourcing the same hook file in the same session is allowed. + +**Interfaces.** These are the registry's existing contracts, stated +here so a hook author needs only this document and +`1_code/methods/_templates/`. + +| Kind | Register with | Must take | Must return | +| --- | --- | --- | --- | +| Engine | `register_engine(list(name, description, capabilities, fit, predict, coef?, ic?))` | `fit(formula, data, family, weights, offset, control)`; `predict(fit, newdata, type, se)` | `fit`: `list(fit, ok, message)`, never raises. `predict`: one number per row. `coef(fit)`: data frame `term`, `estimate`, `se`, if `"coefficients"` is declared. `ic(fit, type)`: a number, if `"ic"` is declared. | +| Selection rule | `register_selection(name, fn, description, requires)` | `models, base, data, engine, family` (plus `weights, offset, ic, control`; extra arguments are filled from stage fields of the same name) | `selection_result()` | +| Resampler | `register_resampler(name, fn, description)` | `frame, iterations, seed, context` (extra arguments are filled from `spec$resample`) | A list of survey-unit-id vectors, one per iteration | +| Metric | `register_metric(name, fn, description, direction)` | `observed, predicted` (response scale) | One number, or `NA` | +| Spec function | none | The signature of the field it fills; see `harness/run_model.R` | As the field requires | + +`check_engine()` (`harness/engines.R:176`) tests a hook engine +against the contract, as it does for built-ins. + +**Metric direction (L7).** `register_metric()` gains +`direction = NULL`, one of `"higher"`, `"lower"`, `"toward_one"`, or +`"none"`. `better_direction()` (`harness/compare.R:38`) reads the +registered direction first and falls back to today's table, so +built-in metrics need no edit. Loading a hook metric without a +`direction` gives a warning, because it would otherwise be treated +as "higher is better". + +**Alternatives considered** + +- *Auto-load `1_code/experiments//hooks/` from the id.* Rejected + as implicit: the hook files would be loaded by a call that doesn't + name them. An explicit `hooks =` path is one line and easy to + read. +- *Pass functions directly in specs, with no registration.* This + already works for selection rules (`stage$selection` may be a + function) and spec function fields. It doesn't work for engines, + resamplers, or metrics, which are looked up by name, and it + records only `"custom"` in the run record. Registration gives a + name for the record and for validation. +- *Let hooks override built-ins.* Rejected; see the clash rule. + +## 4. Spec changes inside the experiment (L3) + +Three helpers in `harness/spec.R`. Each one changes a spec the +experiment holds, validates it, and records the change so it +appears in `experiment_changes()`, `meta.json`, and the run record. + +| Helper | Changes | Example | +| --- | --- | --- | +| `modify_stage(spec, stage, ...)` | Fields of one named stage. `NULL` removes a field. | `modify_stage(spec, "habitat", min_detections = 10L)` | +| `modify_region(spec, region, ...)` | Fields of one region | `modify_region(spec, "north", filter = ~ nr == "Boreal")` | +| `modify_resample(spec, ..., replace = FALSE)` | `spec$resample`. With `replace = TRUE`, the old settings are dropped except `scope`, for switching scheme. | `modify_resample(spec, scheme = "spatial_cv", replace = TRUE)` | + +Each helper: + +1. Stops if the stage or region isn't in the spec, listing the ones + that are. +2. Sets the fields and runs `validate_spec()`, so a misspelt field + (unknown fields are errors) fails here, in `run.R`, with the + experiment's line in the traceback. +3. Appends `": = "` to `models_overridden` + (the existing record of changes), and marks the stage's + `v2_coverage` entry `"not reproduced"` with the change as its + note, as `replace_stage_method()` already does. + +They apply across taxa the same way `replace_stage_method()` does: + +```r +specs <- lapply(standard_specs(), modify_resample, + scheme = "spatial_cv", replace = TRUE) +``` + +Module argument changes (`bryophyte_spec(use_protocol = FALSE)`, +`mammal_spec(climate_source = "fitted")`) stay as they are: +called in `run.R`, passed in `specs`. + +**Recording a module argument change.** `experiment_changes()` lists +spec differences from `standard_specs()` only through +`models_overridden`. A spec built with a non-default module argument +already records it in `notes` and `protocol_is_v2`. No further change +is proposed. + +**Alternatives considered** + +- *Document hand edits of the spec list (`spec$stages[[2]]$ic <- + "BIC"`).* Rejected: the change isn't recorded, is fragile to stage + order, and gives no validation until run time. +- *One `modify_spec()` with a path argument.* Rejected as harder to + read than three named helpers. + +## 5. Spatial block cut points (L6) + +`resampler_spatial_block()` and `resampler_spatial_cv()` gain +`long_breaks` and `lat_breaks` arguments that default to today's +values, and pass them to `spatial_blocks()`. They become `resample` +settings, set with `modify_resample()`: + +```r +modify_resample(spec, scheme = "spatial_cv", replace = TRUE, + lat_breaks = seq(49, 60, by = 1)) +``` + +The defaults move from `spatial_blocks()` (`harness/resample.R:143`) +into one function, `v2_block_breaks()`, which all three read, so +there is one copy. Results with the defaults are unchanged. These +are `methods/` and `harness/` changes; no module changes. + +## 6. Validation + +`experiment_config()` collects every problem and stops once with the +full list, as `validate_spec()` already does. All checks run before +any fitting. As built, the species list is read and checked first, +because the runs can depend on it; then the runs, draws, covariate +files, regions and metrics, specs, and covariates. An unknown run in +`taxa` still stops at once, since everything after depends on it. + +| Check | New? | Message names | +| --- | --- | --- | +| `taxa` are runs in `specs` | Existing | Unknown runs; available runs | +| With no `taxa`: every listed species is fitted by some run (its taxon, the run's tier and, for mammals, season) | New (section 15) | Each species no run fits | +| `n_bootstraps` is a whole number of 1 or more, and is within the stored draws for `precomputed` | Existing | | +| Species file exists, has `taxon` and `species` columns, and has no blank or duplicate rows | New | The path; the missing column | +| Every species exists in `species_queue.csv` for its taxon | New, moved from `prepare_spec()` | Each unknown name, with up to three close matches (`agrep`) | +| Every `taxon` in the species file is a known slug | New | | +| `regions` exist in every selected spec | New | Spec and its regions | +| `metrics` and `compare_metrics` are registered metrics | New | Unknown name; registered names | +| Each spec passes `validate_spec()` after `stage_models` are applied | New here, moved from `prepare_spec()` | As now | +| Every covariate a spec's formulas name exists in the dataset for that taxon and region (after `term_map()`), or in `covariate_files`, or is derivable | New, moved from load time | Each missing name per run and stage, with close matches | +| `covariate_files` exist and are keyed on `survey_unit_id` | Existing | | +| Hook files source cleanly and don't clash with registered names | New, at `load_framework()` | File, name, existing method | + +`prepare_spec()` keeps its own checks, because `run_specs()` can be +called without `experiment_config()` (tests do this). For a valid +config the checks are only repeated, and results don't change. + +**Risk in the covariate check.** It must reproduce the harness's +name resolution exactly: aliases, `supplied_columns`, carried +columns, block suffixes, and derived columns. A wrong check blocks +valid runs. As built, it shares code with the run rather than +copying it: `region_request()` and `province_request()` were +extracted from `prepare_region()` and `province_context()` in +`run_model.R`, and `covariate_columns_needed()` from +`load_covariates()` in `data_load.R`. Both the check and the run +call them. It passes on all three experiments and every taxon, takes +about a second, and suggests close matches (`bio155`: "did you mean +bio15?"). + +## 7. Provenance + +`run_record.md`, in the committed file: + +- **Provenance section** (volatile, so it doesn't affect run-to-run + comparisons): `written_at`, full `git_commit`, + `git_uncommitted_code` (how many files under `1_code/` differ + from `HEAD`, untracked included), `r_version`, `platform`, + versions of the modelling packages loaded in the main session + (`data.table`, `arm`, `xgboost`, `mgcv`, `MuMIn`; packages loaded + only in parallel workers don't appear), each hook file with its + md5, and the path of `session_info.txt`. +- **Configuration section** (stable): today's keys, plus `regions`, + `metrics`, `compare_metrics`, `species_file` (base name) with + `species_n`, and `hooks` (base names). Changes made with + `modify_*()` already appear under `changes_from_v2`. + +In `2_pipeline//`, which is gitignored: `session_info.txt` +holds the full `sessionInfo()` output, written once fitting +finishes. `run_log.csv` and each store's `meta.json` are unchanged, +except that `meta.json` lists hook names. + +**Alternative considered:** *`sessioninfo::session_info()`.* It is +installed and gives a richer listing, but base `sessionInfo()` is +enough and adds no dependency to the harness. + +## 8. Experiment template + +`1_code/experiments/_template/`: + +| File | Change | +| --- | --- | +| `run.R` | Reorganized into the sections in section 1: hooks in `load_framework()`, a spec-changes section with one example per helper in the header notes, and `species = file.path(exp_dir, "species.csv")`. No `taxa` line: the species list sets the runs. The header lists the levers from least to most invasive, now ending in `hooks/` rather than "a new method in `1_code/methods/`". | +| `README.md` | New rows: species file, hooks, spec changes. | +| `species.csv` | Every species a run can fit (1,117 rows), `taxon,species,note`, notes empty. The experiment deletes the rows it doesn't fit. For a like-for-like comparison, keep exp_000's species (in `exp_000_parity_v2/species.csv`). | +| `hooks/.gitkeep` | Empty folder, so the layout is visible. | + +`new_experiment()` copies the template recursively and fills +placeholders in every file. It needed one fix: it listed files +without `all.files = TRUE`, so `hooks/.gitkeep` (a dot file) never +reached a new experiment. A test now checks that `species.csv` and +`hooks/.gitkeep` arrive. + +## 9. Module boundary + +**No module changes.** Every module setting an experiment can +reasonably vary is reached by one of these: + +- Module function arguments (`use_protocol`, `climate_source`, + `tier`, `season`, `bootstrap`), called in `run.R`. +- `modify_stage()`, `modify_region()`, or `modify_resample()` on the + spec the module returns. +- `replace_stage_method()` (existing). +- A replacement function in `hooks/`, set with `modify_stage()`, for + logic inside a module function (e.g. the mammal 99% winsorizing + cap). + +Modules stay the record of v2. Taxon leads review only changes to +their v2 configuration. + +## 10. Output layout + +Unchanged: + +- Intermediates go to `2_pipeline//`: result stores, + `run_log.csv`, `inputs/`, and the new `session_info.txt`. +- Deliverables go to `3_output//`: `tables/`, `figures/`, + `report.md`, `run_record.md`. + +**Correction to the brief.** `.gitignore` doesn't keep all of +`tables/*.csv`. It keeps `run_record.md`, `report.md`, and five named +tables: `parity_summary`, `coverage`, `metric_summary`, +`grid_summary`, and `v2_self_agreement_summary`. +`comparison_metrics.csv` and `coefficient_summary.csv` stay ignored, +as the comment there explains. The design writes no new table, so +`.gitignore` needs no change. + +## 11. Migration + +| Experiment | Change | +| --- | --- | +| exp_000_parity_v2 | Added `species.csv` (the former `parity_check`, 14 rows), with a `note` column holding each species' detections and prevalence from the deleted file. The selection criteria moved to a comment in `run.R`. `run.R` sets `species` to the file; no other change. | +| exp_001_xgboost | Added `species.csv` (same 14 rows, no notes) and an `exp_dir` line; `run.R` otherwise unchanged. `replace_stage_method()` stays. | +| exp_002_soilgrids | Added `species.csv` (same 14 rows, no notes) and an `exp_dir` line; `run.R` otherwise unchanged. | +| `_template/` | As in section 8. | +| `experiments/_shared/` | `focal_species.R` deleted. The folder goes with it, because git doesn't track empty folders. `load_framework()` already tolerates a missing folder; the code says so in a comment. | + +All three migrated; none was left unchanged. All three keep their +explicit `taxa`, so their runs are unchanged. + +## 12. Parity and verification plan (Phases 3 and 5) + +**Reduced parity run (approved and run).** This follows the repository's established check +(`1_code/tests/compare_stores.R`; memory: five-draw regression +check). + +| Setting | Value | +| --- | --- | +| Config | exp_000's, run as a script from the scratchpad, not by editing `run.R` | +| Species | `parity_check`, all seven runs (14 species) | +| Draws | 5, seeded `20260909` | +| Workers | 12 | +| Output | `pipeline_dir = 2_pipeline/refactor_baseline/stores`; `out_dir = 2_pipeline/refactor_baseline/output`, so nothing in exp_000's `3_output/` is touched | +| Steps | None (`01`-`03` compare with v2 and plot; not needed for regression) | +| Baseline comparison | Off (`baseline = NULL`) | +| Also saved | `commit.txt` (`afdfe3b` plus a clean or dirty flag) and `session_info.txt` | +| Time taken | 1.9 minutes | + +After the refactor, the same script runs against the new code into +`2_pipeline/refactor_after/` (species from `species.csv`). +`compare_stores()` compares `coefficients`, `metrics`, and +`grid_predictions` per run and region. The summary tables are +compared with `all.equal()`. + +**Tolerance:** exact (`0`), because the run is seeded and the code +path is unchanged. + +**Result: pass.** Baseline at `afdfe3b`; all 140 jobs `ok` on both +sides. All 14 run-and-region stores match on coefficients, metrics +and grid predictions with a largest absolute difference of 0; +`coverage`, `metric_summary`, `grid_summary` and +`coefficient_summary` are identical; `meta.json` differs only by +the new `hooks` field. Re-run after the last fitting-path edits +with the same result. The later changes in section 15 touch only +how the species list and runs are chosen; exp_000 passes `taxa`, so +its run is unaffected, and no further parity run was asked for. + +**Phase 5 also covers:** + +- `Rscript 1_code/tests/run_tests.R`, plus new tests for hooks + (including the clash rule), `species.csv`, `modify_*()`, + `regions`, `metrics`, `compare_metrics`, metric direction, block + breaks, and each new validation message. +- A scratch experiment created with `new_experiment()`: one lichen + species, 3 to 5 draws, `workers = 2` (so hooks are proven in + workers). It uses a hook engine (a renamed `glm` wrapper) and a + hook metric with `direction = "lower"`, `modify_stage()`, and + `spatial_cv` with custom `lat_breaks`. It is then deleted with its + `2_pipeline/` and `3_output/` folders. +- Validation checks: a misspelt covariate, an unknown engine, a + missing species file, a misspelt species, and a hook name clash. +- `git status --porcelain` and `git check-ignore -v` on outputs. + +**Results:** every test passes. The scratch experiment ran with +two lichen species on two workers (16 jobs `ok`), proving hooks +load in workers; the run record listed the hook, its md5 and every +change. A run with `regions = "north"` and `metrics = c("auc", +"mae")` produced only those. Each validation error gave a clear +message. From `3_output//`, only `run_record.md`, `report.md`, +the allowed tables and scaffold `.gitkeep` files surfaced; nothing +from `2_pipeline/`, including `session_info.txt`. The scratch +folders were deleted. + +## 13. Where this departs from common R practice + +1. **Defaults as function arguments, not a config file** + (section 2). Most configurable R pipelines (`config`, `targets` + with YAML) keep settings as data. Here settings include formulas + and functions, and the users write R. +2. **Sourced files and a global registry, not a package.** This is + the repository's existing pattern. Hooks extend it rather than + introducing `pkgload` or an R package for the back-end. A package + would give namespacing and `R CMD check`, at the cost of a build + step for every taxon lead. +3. **Hooks registered into a session-wide registry**, rather than + passed as arguments. The clash rule stops one experiment's hook + from changing a built-in for others in the same session. + +## 14. Files changed + +| File | Change | +| --- | --- | +| `1_code/harness/harness.R` | `load_framework(hooks = )`; hook loading with the clash rule | +| `1_code/harness/registry.R` | Clash check while loading hooks; `direction` on `register_metric()` | +| `1_code/harness/experiment.R` | New arguments; collected validation; runs derived from the species list (`runs_for_species()`); `session_info.txt`; record fields; headline uses `compare_metrics` | +| `1_code/harness/run_model.R` | Pass hook files to workers; `region_request()` and `province_request()` extracted, unchanged in behaviour; `hooks` in `meta.json` | +| `1_code/harness/data_load.R` | `covariate_columns_needed()` extracted from `load_covariates()`, unchanged in behaviour | +| `1_code/harness/new_experiment.R` | Copies dot files (`all.files = TRUE`), so `hooks/.gitkeep` arrives | +| `1_code/harness/species.R` | `read_species_file()`; config-time name check with close matches | +| `1_code/harness/spec.R` | `modify_stage()`, `modify_region()`, `modify_resample()`; config-time covariate check | +| `1_code/harness/compare.R` | Registered direction first; `compare_metrics` | +| `1_code/harness/resample.R` | `v2_block_breaks()` | +| `1_code/harness/run_record.R` | Provenance fields | +| `1_code/methods/resampling/spatial_block.R`, `spatial_cv.R` | `long_breaks` and `lat_breaks` settings | +| `1_code/experiments/_shared/focal_species.R` | Deleted | +| `1_code/experiments/_template/` | `run.R` (no `taxa`), `README.md`, `species.csv` (every fittable species), `hooks/.gitkeep` | +| `1_code/experiments/exp_000…`, `exp_001…`, `exp_002…` | `species.csv`; `species =` line in `run.R` | +| `1_code/tests/testthat/` | New `test-experiment_config.R`; `test-new_experiment.R` extended | +| `1_code/modules/` | **None** | + +## Assumptions + +- A hook file is trusted code: it is sourced, not sandboxed. +- `species_queue.csv` in the pinned dataset is the authority for + species names. A species missing from it can't be run, as today. +- `git` is on the PATH where runs happen. When it isn't, the record + writes `-`, as now. +- Existing committed `run_record.md` files will differ in their + Configuration section the next time each experiment runs (new + keys). Their results won't. + +**Confidence: High** that this closes L1 to L3 and L5 to L7 without +changing results: the parity run matched exactly. The config-time +covariate check, rated Moderate in the proposal, is now High: it +calls the same name-resolution code as the run and passed on every +taxon. + +## 15. Revisions after approval + +| Revision | Why | +| --- | --- | +| The template's `species.csv` lists every species a run can fit (1,117 rows, `taxon,species,note`, notes empty) instead of exp_000's 14. Users delete the rows they don't fit. | Requested after implementation: users choose from the full list rather than starting from exp_000's set. | +| A `use` column (TRUE/FALSE per row) was tried and removed. | The requested format is `taxon,species,note`, cut down by deleting rows; a selection column was not needed. | +| Without `taxa`, `experiment_config()` runs only the runs that fit a listed species, and the template drops its `taxa` line. | Before, a taxon with no rows in the list meant "every species" for any run still in `taxa`. Cutting the template to three lichens while `taxa` named all seven runs would have fitted over 1,000 other species. Deriving the runs makes `species.csv` the one file to edit. | +| A listed species that no run can fit is an error. | Found while deriving runs: 23 mammals are `ua` tier only, which the mammal runs never fit. They were also removed from the template. | +| `new_experiment()` copies dot files. | `hooks/.gitkeep` was not reaching new experiments. | + +Not changed: an explicit `taxa` keeps its old meaning, so existing +experiments run as before. Making an explicit `taxa` with an +unlisted run an error was offered and not chosen. diff --git a/temp/experiment_config_inventory.md b/temp/experiment_config_inventory.md new file mode 100644 index 0000000..df279d1 --- /dev/null +++ b/temp/experiment_config_inventory.md @@ -0,0 +1,237 @@ +# Experiment configuration inventory + +Phase 1 of the experiment-configuration work. Read-only scan of +`1_code/harness/`, `1_code/methods/`, `1_code/modules/` (including +`_shared/`), and `1_code/experiments/`, as of commit `afdfe3b` +(branch `development`). `1_code/_setup/` was read only to classify it +as fixed. + +**Agreed direction (this session):** keep R configuration +(`experiment_config()` in each `run.R`); no YAML. Each experiment +holds its own species list and its own spec changes. `_shared/` +holds only reference material. + +## Summary + +Most experiment-variable settings can already be set from an +experiment's own folder: through `experiment_config()` arguments, +`stage_models`, `covariate_files`, a changed `specs` list, or +`steps`. Ten items still need an edit to a shared file. They are +listed as **leaks** in section 4 and marked `LEAK` in the table. + +Status key for the "Settable now?" column: + +- **cfg**: an `experiment_config()` or `run_experiment()` argument. +- **sm**: the `stage_models` argument. +- **cf**: the `covariate_files` argument. +- **spec**: a changed spec passed as `specs`. Today this means + `replace_stage_method()`, or a hand edit of the spec list in + `run.R`. A hand edit works but is not recorded as a change (see + leak L3). +- **step**: a `steps` script in the experiment folder. +- **LEAK**: can be changed only by editing a shared file. + +Line references are `file:line` from the repository root. + +## 1. Inventory + +### 1.1 Data selection + +| Component | Current location | Current value or behaviour | Type | Settable now? | Proposed mechanism | Notes | +| --- | --- | --- | --- | --- | --- | --- | +| Runs (taxa) included | `1_code/harness/experiment.R:69` | `NULL` = every run in `standard_specs()` | list | cfg (`taxa`) | config list | Run names are fixed in `modules/_shared/standard_specs.R:33-45`. | +| Focal species | `1_code/harness/experiment.R:70`, `:162` | Name of a shared set, an inline named vector, or `NULL` for all | list | cfg, but the named sets are shared | Per-experiment `species.csv` (`taxon,species`) | **LEAK L2.** The named sets live in `1_code/experiments/_shared/focal_species.R:32-124`. | +| Named species sets | `1_code/experiments/_shared/focal_species.R:59-122` | `full`, `parity_check`, `plants_only`, `one_each` | list | LEAK | Move each experiment's set into its own `species.csv` | exp_000, exp_001, exp_002, and `_template` all name `parity_check`. | +| Valid species names (reference) | `1_code/harness/species.R:33-66` reads `lookup/species_queue.csv` in the published dataset | One queue for every taxon, region, tier, and season | reference | Read-only | Keep as the shared reference list | Already exists in the data. `resolve_species()` (`species.R:157`) checks names, but only inside `prepare_spec()` (`run_model.R:188`), after configuration. | +| Regions run | `1_code/harness/run_model.R:68`, `:191` | `run_specs(regions = NULL)` = every region in the spec | list | **LEAK** | config list (`regions`) | **L5.** `run_specs()` takes `regions`, but `experiment_config()` and `run_experiment()` (`experiment.R:281-291`) never pass it. | +| Region definition (filters) | `modules/_shared/plant_group.R:98`, `:104`; `modules/birds/spec.R:49`, `:59`; `modules/mammals/spec.R:200-204`, `:221-225` | v2 filters, e.g. `~ nr != "Grassland"`, `~ summer_days > 10 & WetlandMargin != 1` | value (formula) | spec (hand edit) | Spec-change helper (L3) | v2 parity depends on these; changing them is an experiment. | +| Mammal tier | `modules/mammals/spec.R:144` | `"modelled"` | value | spec (`mammal_spec(tier = )`) | Already a module argument | Leave as is. | +| Mammal season | `modules/_shared/standard_specs.R:39-40` | Both seasons, as separate runs | value | cfg (`taxa`) | Already settable | | +| Response definition | `plant_group.R:91` (`"detection"`); `birds/spec.R:39` (`"identity"`); `mammals/spec.R:174` (`mammal_pa_response`) | v2 transforms | value or function | spec (hand edit) | Spec-change helper (L3); a custom function goes in `hooks/` | | +| Mammal abundance winsorizing | `modules/mammals/spec.R:113` | Cap at the 99th percentile | value inside a function | LEAK (in the module) | Keep fixed; to vary it, write a replacement `abundance_response` in `hooks/` | A v2 port. See section 3. | +| Minimum detections to fit a stage | `plant_group.R:167`; harness check at `run_model.R:824-830` | `20L` for the plant habitat stage | value | spec (hand edit) | Spec-change helper | The v2 "20-detection skip". | +| Minimum detections to redraw a bootstrap | `plant_group.R:183`; `mammals/spec.R:272`; used at `methods/resampling/spatial_block.R:131-152` | `20L` | value | spec (hand edit) | Spec-change helper | | +| Data source and version | `1_code/harness/data_source.R:33-38` | Published `test_dataset` and `v2_results`, both `2026-10-05` | value | cfg (`data_dir`); environment variables `SDM_TEST_DATASET` and `SDM_V2_RESULTS` | Keep | Should stay pinned for the team. An experiment can point `data_dir` elsewhere, but its results then don't compare with the baseline. | +| Temporal subsetting (survey years) | none | Not implemented; no year filter exists in the harness | n/a | Possible only through a region `filter` | Spec-change helper on `regions$$filter` | No year-based selection exists today; flagged, not designed. | +| Spatial or temporal thinning | none | Not implemented | n/a | n/a | Out of scope unless asked | Would be a new resampler or a filter, added through `hooks/`. | + +### 1.2 Covariates + +| Component | Current location | Current value or behaviour | Type | Settable now? | Proposed mechanism | Notes | +| --- | --- | --- | --- | --- | --- | --- | +| Candidate formulas per stage | `1_code/harness/model_sets.R:31-295` (v2 sets); applied by `spec.R:391-421` | v2 sets, spliced verbatim | list | sm (`stage_models`, keyed `stage` or `taxon.stage`) | Already settable | Inline formulas or `extend_models()` work. A new *named* set needs a shared edit to `model_sets.R`; inline avoids that. | +| Added covariate terms | `model_sets.R:318` (`extend_models()`), `:349` (`models_from_covariates()`) | Helpers | function | sm | Already settable | exp_002 is the worked example. | +| Covariates not in the dataset | `harness/data_load.R:388-505` | Extra CSVs keyed on `survey_unit_id` | list | cf | Already settable | Checked at config time (`experiment.R:148-158`). | +| Named covariate groups | `harness/covariate_sets.R:39-72` | `climate_v2`, `topography`, `remote_sensing`, and others | list | Shared registry | Use inline vectors in the experiment | Minor. A new named group needs a shared edit; inline vectors avoid it. | +| Derived covariates | `harness/covariate_sets.R:92` (`derived_covariates()`) | Fixed derivations | function | **LEAK** | Supply derived columns through `covariate_files` | **L9, minor.** A new derivation needs a harness edit. The workaround is a CSV written by an experiment script. | +| Transformations and interactions | Inside formula text | e.g. `I()`, `poly()`, `a:b` | value | sm | Already settable | | +| Buffer radii and scales | none | Fixed by `_setup/` harmonization | n/a | cf (an experiment extracts its own) | Already settable | Fixed in the published dataset by design (memory: `_setup` fixed). | +| Landcover and footprint groupings | `plant_group.R:153-156` (post-process), `:690-693` (`coef_adjust_plant_veg` overlap); `birds/standardize.R` | v2 ports | function | No | Keep fixed | Section 3. | +| Plant Protocol term | `plant_group.R:64-82`, `:137-149` | Fitted for bryophytes and lichens, as v2 | value | spec (`bryophyte_spec(use_protocol = )`) | Already a module argument | Recorded in `notes` and `protocol_is_v2`. | +| Offsets | Bird QPAD offset in formula (`run_model.R:723-726`); `birds/spec.R:88` `carry_offset = FALSE` | v2 | value | spec (hand edit) | Keep fixed | Section 3 (memory: bird climate carry has no offset). | +| Region-level held constants | `plant_group.R:109` (`paspen = 0`); `mammals/spec.R:234`, `:262` (`seas_days = 100`, `Climate = 0`) | v2 | value | spec (hand edit) | Spec-change helper | They change what the grid predictions mean; record it if changed. | + +### 1.3 Model structure + +| Component | Current location | Current value or behaviour | Type | Settable now? | Proposed mechanism | Notes | +| --- | --- | --- | --- | --- | --- | --- | +| Engine per stage | `plant_group.R:117`, `:130` (`bayesglm`); `birds/spec.R:74`, `:94` (`glm`); `mammals/spec.R:157`, `:242` (`glm`) | v2 | value (registry name) | spec (`replace_stage_method(engine = )`) | Already settable | exp_001 is the worked example. | +| Built-in engines | `1_code/methods/engines/` | `glm`, `bayesglm`, `xgboost` | registry | Shared | Experiment-local `hooks/` | **LEAK L1.** A new engine has to go in `methods/`. | +| Engine settings (`control`) | `plant_group.R:120`, `:133` (`maxit = 250`); `methods/engines/bayesglm.R:50-51` (default 250); `methods/engines/xgboost.R:78-81` | v2 and untuned xgboost defaults | list | spec (`replace_stage_method(control = )`) | Already settable for a replaced stage; spec-change helper otherwise | xgboost's own `seed = 20260909L` (`xgboost.R:81`) is separate from the experiment seed. See section 5. | +| Selection rule per stage | `plant_group.R:118` (`aic_average`), `:131` (`ivw_grid`); `birds/spec.R:75`, `:95` (`staged_bic`); `mammals/spec.R:244-245` (`hurdle`, `aic_best`) | v2 | value (registry name) | spec (`replace_stage_method(selection = )`) | Already settable | | +| Built-in selection rules | `1_code/methods/selection/` | `aic_average`, `aic_best`, `hurdle`, `ivw_grid`, `single`, `staged_bic` | registry | Shared | `hooks/` | **LEAK L1.** | +| Information criterion | `plant_group.R:119`, `:132`; `birds/spec.R:76`, `:96`; default `run_model.R:757` | `AICc`, or `BIC` for the bird landcover stage | value | spec (hand edit) | Spec-change helper | Validated against `AIC`, `AICc`, and `BIC` (`spec.R:207-210`). | +| Rule settings | `methods/selection/staged_bic.R:37` (`threshold = 2`, `always_advance`); `ivw_grid.R:50` (`head_terms`); `hurdle.R:64-69` | v2 | value | spec (hand edit; stage fields named like rule arguments) | Spec-change helper | `validate_spec()` already accepts rule arguments as stage fields (`spec.R:160-162`). | +| Hierarchy (climate stage carried into habitat) | `carry_as`, `carry_from`, `carry_scale`, `carry_offset` in each spec; `run_model.R:884-907` | v2: province climate carried as `Climate` | value | spec (hand edit) | Spec-change helper | Structural. Changing it is a real experiment; record it. | +| Stage scope (province or region) | `plant_group.R:122`; `birds/spec.R:78`; logic at `run_model.R:364-412` | v2 | value | spec (hand edit) | Spec-change helper | | +| Weighting | `birds/spec.R:56`, `:65`, `:79`; `mammals/spec.R:188`; `run_model.R:834-839` | v2 | value | spec (hand edit) | Spec-change helper | | +| Error family | `plant_group.R:92`; `birds/spec.R:40`; `mammals/spec.R:187`, `:256` | v2 | value | spec (hand edit) | Spec-change helper | | +| Post-processing (v2 ports) | `plant_group.R:153-156`; `mammals/spec.R:266` | v2 | function list | Removed by `replace_stage_method()` | Keep fixed | Section 3. | +| Mammal climate source | `mammals/spec.R:142` | `"precomputed"` (v2) | value | spec (`mammal_spec(climate_source = )`) | Already a module argument | | +| Plant bootstrap source | `harness/experiment.R:76` | `"spatial_block"`; `"v2_ids"` replays v2's draws | value | cfg (`plant_bootstrap`) | Already settable | | + +### 1.4 Evaluation + +| Component | Current location | Current value or behaviour | Type | Settable now? | Proposed mechanism | Notes | +| --- | --- | --- | --- | --- | --- | --- | +| Resampling scheme | `plant_group.R:174-186`; `birds/spec.R:106-113`; `mammals/spec.R:270-274`; dispatch at `harness/resample.R:186-209` | `spatial_block` (plants, mammals); `precomputed` (birds) | value (registry name) | spec (hand edit) | Spec-change helper | `spatial_cv` is registered (`methods/resampling/spatial_cv.R`). | +| Built-in resamplers | `1_code/methods/resampling/` | `precomputed`, `spatial_block`, `spatial_cv` | registry | Shared | `hooks/` | **LEAK L1.** | +| Number of draws or folds | `harness/experiment.R:71`; `spatial_cv.R:92` (folds = `length(iterations)`) | 100 | value | cfg (`n_bootstraps`) | Already settable | For `spatial_cv`, `n_bootstraps` is the fold count. | +| Spatial block cut points | `harness/resample.R:143-144` | Longitude `-121, -116, -112, -109`; latitude `48, 51, 54, 57, 61` (v2 plant grid) | value | **LEAK** | Resample setting (`long_breaks`, `lat_breaks`) passed through the resampler | **L6.** Neither `resampler_spatial_block()` (`spatial_block.R:124-128`) nor `resampler_spatial_cv()` (`spatial_cv.R:87-90`) passes breaks, so the block size for CV can't be changed. | +| Redraw attempts | `spatial_block.R:127` | `max_attempts = 100L` | value | spec (hand edit of `resample`) | Spec-change helper | | +| Seed | `harness/experiment.R:72`; `resample.R:35-37`, `:59-76` | `20260909L`, per-species derived | value | cfg (`seed`) | Already settable | | +| Metrics computed | `harness/eval_metrics.R:34-42`; `run_model.R:71` | Every registered metric | list | **LEAK** | config list (`metrics`) | **L5.** `run_specs(metrics = )` exists but isn't passed from `experiment_config()`. | +| Built-in metrics | `1_code/methods/metrics/` | `auc`, `calibration_slope`, `deviance_explained`, `n`, `prevalence`, `rmse`, `spearman` | registry | Shared | `hooks/` | **LEAK L1.** | +| Metric direction ("better") | `harness/compare.R:38-46` | Hard-coded by name; anything unknown is "higher" | function | **LEAK** | Declare direction when registering a metric | **L7.** A hook metric where lower is better would be reported backwards. | +| Comparison metrics | `harness/compare.R:97-100`; console headline at `harness/experiment.R:488-490` | Five `oob_` metrics; headline shows three | list | **LEAK** | config list (`compare_metrics`) | **L7.** | +| Comparison baseline | `harness/experiment.R:79` | `"exp_000_parity_v2"` | value | cfg (`baseline`) | Already settable | | +| Summary statistics | `harness/summarise.R:178-179` | median and 10-90% band | value | No | Keep fixed | Every comparison and gate reads these; changing them breaks comparability. | +| v2 validation AUCs (plants) | `plant_group.R:233`; `run_model.R:1065-1075` | `plant_v2_validation` | function | Dropped by `replace_stage_method()` | Keep fixed | Section 3. | +| Parity thresholds | `experiments/exp_000_parity_v2/utils/parity_targets.R:80-88` | 1e-6, 90%, 0.25, 0.75 to 1.33 | value | Already experiment-local | n/a | Good existing pattern. | +| Coefficient translation (birds) | `harness/experiment.R:271`; template `run.R:78-81` | `translate = list(bird = bird_habitat_translation)` | function | cfg (`run_experiment(translate = )`) | Already settable | exp_001 omits it, which is correct for xgboost (no coefficients). | + +### 1.5 Prediction + +| Component | Current location | Current value or behaviour | Type | Settable now? | Proposed mechanism | Notes | +| --- | --- | --- | --- | --- | --- | --- | +| Prediction grid | `plant_group.R:99`, `:105`; `birds/spec.R:52`, `:62`; `mammals/spec.R:205`, `:226`; loader `harness/predict_grid.R:35-64` | One habitat-type grid per region, read only from `/lookup/` | value | spec, but only grids already in the dataset | **LEAK** if an experiment needs its own grid | **L8.** The loader can't read a grid from the experiment folder or `2_pipeline//inputs/`. Spatial prediction is deferred (memory), so this is flagged, not designed. | +| Grid rows and columns dropped | `mammals/spec.R:208-209`, `:227` | v2 | value | spec (hand edit) | Spec-change helper | | +| Grid constants (current or reference scenario) | `plant_group.R:109`, `:137-144`; `mammals/spec.R:262` | `Climate = 0`, `paspen = 0`, `Protocol = "New"`, `seas_days = 100` | value | spec (hand edit) | Spec-change helper | The closest thing to a "scenario" today. | +| Resolution, extent, spatial maps | none | Not implemented (habitat-type grids only) | n/a | n/a | Out of scope | Deferred (memory: spatial prediction deferred). | +| Per-unit predictions written | `harness/experiment.R:78` | `"none"` | value | cfg (`unit_predictions`) | Already settable | | + +### 1.6 Run control + +| Component | Current location | Current value or behaviour | Type | Settable now? | Proposed mechanism | Notes | +| --- | --- | --- | --- | --- | --- | --- | +| Workers | `harness/experiment.R:77` | `1L` (experiments set 12) | value | cfg (`workers`) | Already settable | | +| Worker start-up | `harness/run_model.R:505-516` (`load_worker()`) | Each worker sources the framework only | code path | n/a | Also source the experiment's `hooks/` | **L1 blocker.** Without this, a hook method registered in the main session is unknown to the workers. | +| Fit or re-summarize | `harness/experiment.R:269` | `fit = TRUE` | value | cfg (`run_experiment(fit = )`) | Already settable | | +| Extra stages (steps) | `harness/experiment.R:270`, `:522-539` | `script_step()` runs scripts from the experiment folder | function | step | Already settable | | +| Output paths | `harness/experiment.R:82-84`, `:203-205` | `2_pipeline//`, `3_output//` | value | cfg | Already settable | | +| Run record | `harness/run_record.R:171-290`; `experiment.R:354-410` | Settings, coverage, metrics; provenance holds short commit, R version, time | function | Fixed | Add session info, full commit and dirty flag, and hook files | **L4.** | +| Scaffolding | `harness/new_experiment.R:135`, `:278` | Copies every file in `_template/` and fills placeholders | function | n/a | Template gains `species.csv` and `hooks/` | Copies recursively, so new template files are picked up without a code change. Check the placeholder fill on a CSV. | +| Which stages run | Spec `stages` list | All stages | list | spec (hand edit) | Spec-change helper | e.g. dropping the climate stage. | + +## 2. Values that look variable but should stay fixed + +| Item | Location | Why fixed | +| --- | --- | --- | +| Data cleaning and harmonization | `1_code/_setup/` | Defines the frozen v2 test dataset that every experiment and the baseline share. Experiments add data through `covariate_files` (memory: `_setup` fixed). | +| Published dataset versions | `harness/data_source.R:33-38` | Pinned so every experiment reads the same data. Change only as a maintainer release. | +| v2 candidate formulas | `harness/model_sets.R:31-295` | Verbatim v2 record. Experiments replace them through `stage_models` and don't edit them. | +| Bird climate `carry_offset = FALSE` and `carry_scale = "exp"` | `modules/birds/spec.R:86-88` | Matches how v2 carries `exp(link)` without the QPAD offset. Verified against v2 (memory: bird climate carry has no offset). | +| Plant post-processing (age splines, cutblock convergence, pAspen, footprint pooling, overlap constants, cutblock recovery weights) | `modules/_shared/plant_group.R:153-156`, `:379-1027` | Ports of v2 code, checked to 2e-12. They are part of the v2 model, not tuning knobs. | +| Mammal hurdle tables, lure ratio, 99% winsorizing, spline settings, held constants | `modules/mammals/hurdle.R`; `mammals/spec.R:113` | Ports of v2, matched to 2e-14. | +| Bird habitat translation | `modules/birds/standardize.R` | Port of v2's `08.PackageCoefficients.R`, checked exact. | +| Summary statistics (median, 10-90% band) | `harness/summarise.R:178-179` | Every comparison and gate reads them. | +| Metric definitions | `methods/metrics/*.R` | A changed definition breaks comparison with every past run. Add a new metric; don't redefine one. | +| Per-species seed derivation | `harness/resample.R:59-76` | Keeps draws stable when species are added or removed, and parallel equal to serial. | +| Iteration 1 = full data | `methods/resampling/spatial_block.R:79-81` | v2 behaviour; the comparison reads `boot1`. | +| Registry contracts (core arguments) | `harness/registry.R:152-279` | The interface hooks implement; changing them breaks every method. | + +## 3. Hard-coded items in `modules/` and `_shared/` + +Taxon leads own these files. The table lists what an experiment +could reasonably want to vary, and whether that is possible without +editing the module. + +| Item | Location | Possible without a module edit? | How | +| --- | --- | --- | --- | +| Region filters | `plant_group.R:98`, `:104`; `birds/spec.R:49`, `:59`; `mammals/spec.R:200-225` | Yes | Change the spec in the experiment (spec-change helper) | +| Engine, rule, IC, `control` | All spec files | Yes | `replace_stage_method()` or spec-change helper | +| `min_detections` (stage and resample) | `plant_group.R:167`, `:183`; `mammals/spec.R:272` | Yes | Spec-change helper | +| Resampling scheme | All spec files | Yes | Spec-change helper on `resample` | +| Grid constants | `plant_group.R:109`, `:137-144`; `mammals/spec.R:234`, `:262` | Yes | Spec-change helper | +| Protocol term (plants) | `plant_group.R:45`, `:64-82` | Yes | Module argument `use_protocol` | +| Mammal climate source, tier | `mammals/spec.R:142-144` | Yes | Module arguments | +| Mammal winsorizing cap (0.99) | `mammals/spec.R:113`, inside `mammal_agp_response()` | Only by replacing the function | A replacement `abundance_response` in `hooks/`, set with the spec-change helper | +| Plant post-process constants | `plant_group.R:690-693`, `:1009` | Only by replacing the function | A replacement `post_process` in `hooks/`. Not expected; these are v2. | +| `standard_specs()` run list | `modules/_shared/standard_specs.R:33-45` | Yes | Pass a custom `specs` list | +| Named species sets | `experiments/_shared/focal_species.R:32-124` | No | **L2.** Moves to each experiment's `species.csv`. | + +No module needs to change to accept configuration. Every module +setting above can already be overridden on the spec object the +module returns. What's missing is a tidy, recorded way to do it +(L3). **Expected module changes in Phase 4: none.** + +## 4. Leaks, confirmed or corrected + +The four leaks from the plan, plus six more found in the scan. + +| # | Leak | Status | Shared file a user would edit today | Fix direction for Phase 2 | +| --- | --- | --- | --- | --- | +| L1 | New engine, rule, resampler, or metric | **Confirmed** | `1_code/methods/` | `hooks/*.R` in the experiment, registered with the existing `register_*()` functions. Also sourced in workers (`run_model.R:505-516`); today a method registered in the main session is missing from the workers. | +| L2 | Species selection | **Confirmed** | `experiments/_shared/focal_species.R` | Per-experiment `species.csv`. The shared reference list already exists as the dataset's `lookup/species_queue.csv`, read by `species_catalogue()`, so no new shared file is needed. Check names at config time rather than at `prepare_spec()`. | +| L3 | Spec changes other than a method swap | **Corrected: possible but unrecorded** | None strictly. Users can edit the spec list in `run.R`, but nothing documents it, it isn't recorded by `experiment_changes()`, and editing the module is the obvious path | `modify_stage()` / `modify_spec()` helpers that set fields, run `validate_spec()`, and append to `models_overridden`. | +| L4 | Provenance lacks session info and hooks | **Confirmed** | `harness/run_record.R:196-218` (harness, not a user edit) | Add session info, full commit, dirty flag, and hook files and checksums. | +| L5 | Regions and metrics subsets | **New** | `harness/experiment.R` | Pass `regions` and `metrics` through `experiment_config()` to `run_specs()`, which already accepts them. | +| L6 | Spatial block cut points | **New** | `harness/resample.R:143-144` | Let `spatial_block` and `spatial_cv` accept `long_breaks` and `lat_breaks` as `resample` settings, with the current values as defaults. This is a `methods/` change, not a module change. | +| L7 | Comparison metrics and metric direction | **New** | `harness/compare.R:38-46`, `:97-100`; `experiment.R:488-490` | Optional `direction` on `register_metric()`, defaulting to today's table; `compare_metrics` in config. | +| L8 | Experiment-supplied prediction grid | **New, deferred** | `harness/predict_grid.R:35-64` | Flag only. Spatial prediction is deferred. | +| L9 | New derived covariates | **New, minor** | `harness/covariate_sets.R:92` | Already possible through `covariate_files`; document it, no code. | +| L10 | Named model or covariate sets | **New, minor** | `harness/model_sets.R`, `harness/covariate_sets.R` | Already possible inline; document it, no code. | + +## 5. Issues noticed (not fixed) + +1. **xgboost seed is not the experiment seed.** + `methods/engines/xgboost.R:81` defaults `seed = 20260909L`, so + changing the experiment's `seed` doesn't change xgboost's + subsampling or validation split. +2. **Species names are checked late.** Names are checked in + `prepare_spec()` (`run_model.R:188`), after the config prints, + rather than in `experiment_config()`. With 12 workers that is + still before any fitting, but the error appears later than other + config errors. +3. **`validate_spec()` runs late.** It runs only in `prepare_spec()` + (`run_model.R:176`), not in `experiment_config()`. A bad spec + passed through `specs` is caught at run start, not at config time. +4. **`spatial_cv` reports out-of-bag metrics under the `oob_` + prefix**, the same as bootstrap out-of-bag rows. Comparing a + spatial-CV experiment with the bootstrap baseline mixes two kinds + of hold-out. This is a comparability caveat, not a bug. +5. **`run_record` shows `boot_seed` as `20260909.0000`**, because + `fmt_value()` formats a double seed with four decimals. This is + cosmetic. +6. **`1_code/_scratch.R` is tracked.** It is outside the experiment + structure. Not checked further. + +## 6. README and vignette sections that will need updating later + +These are listed only; Phase 4 won't edit them. + +- `1_code/experiments/README.md`: the `_shared/` row, and "Adding an + experiment" steps 1-3 (species file, hooks, spec changes). +- `1_code/harness/README.md`: the `experiment_config()` argument + table (new arguments), and the Files table if new helpers are + added. +- `1_code/methods/README.md`: a note that experiment-local methods + go in `hooks/`, and the metric `direction` field if added. +- `docs/getting_started.md`: around lines 237-250 (`species` and + `focal_species_sets()`), plus the "test a change" walkthroughs. +- `docs/example_experiment.md` (line 235) and + `docs/example_experiment_xgboost.md` (line 244): + `species = "parity_check"`. +- `1_code/tests/README.md` line 22: the `parity_check` reference. +- exp_000, exp_001, exp_002 `README.md`: species rows in the design + tables.