From 681f3ca72ca0823fc5ec5ddefaa173818288edba Mon Sep 17 00:00:00 2001 From: patnr Date: Tue, 27 Oct 2026 14:29:18 +0100 Subject: [PATCH 1/6] Add a benchmark of the update schemes on MiniRes twin experiments benchmarks/minires/run.py runs each method on many twin experiments of a waterflood (that of ert's test-data/ert/waterflood: 1000 m five-spot, producers on BHP, 40 steps of which 16 are history), at 25 x 25 and 50 x 50 cells, with the log-permeability field and two fluid scalars (sor, vo) as unknowns. Each truth is drawn from the prior; each method is scored on the history misfit, the forecast of the unseen steps, and the field and scalars against the truth, separately. The scores are kept in results/.json and RESULTS.md is generated from them. One method to begin with (ES-MDA); more follow. A slow smoke test runs one tiny experiment per method. Co-Authored-By: Claude Opus 5.5 --- benchmarks/minires/RESULTS.md | 62 ++++ benchmarks/minires/cases.py | 133 ++++++++ benchmarks/minires/methods.py | 19 ++ benchmarks/minires/results/coarse.json | 435 +++++++++++++++++++++++++ benchmarks/minires/results/fine.json | 207 ++++++++++++ benchmarks/minires/run.py | 200 ++++++++++++ benchmarks/minires/scores.py | 50 +++ pyproject.toml | 1 + tests/test_benchmark_minires.py | 23 ++ 9 files changed, 1130 insertions(+) create mode 100644 benchmarks/minires/RESULTS.md create mode 100644 benchmarks/minires/cases.py create mode 100644 benchmarks/minires/methods.py create mode 100644 benchmarks/minires/results/coarse.json create mode 100644 benchmarks/minires/results/fine.json create mode 100644 benchmarks/minires/run.py create mode 100644 benchmarks/minires/scores.py create mode 100644 tests/test_benchmark_minires.py diff --git a/benchmarks/minires/RESULTS.md b/benchmarks/minires/RESULTS.md new file mode 100644 index 00000000..61934e1e --- /dev/null +++ b/benchmarks/minires/RESULTS.md @@ -0,0 +1,62 @@ +# MiniRes benchmark: results + +Generated by `run.py`; do not edit by hand. See `cases.py` for the cases, `methods.py` for the +methods and their settings, and `scores.py` for what is scored. Each cell is the mean over +the twin experiments (truths), ± its standard error. Rows are grouped by the question they +address, not ranked: which case and which score matters is for the reader to judge. + +## Case `coarse`: 25 x 25 cells + +20 truths, 50 members; history of 16 of 40 steps, observed at steps [4, 8, 12, 16]. PET at `b731be7`, 2026-10-27. + +### Data and forecast + +| | misfit | RMSE forecast WWCT | simulations | +|---|---|---|---| +| prior | 91.7 ± 4.2 | 0.221 ± 0.021 | | +| *Iterating* | | | | +| ES-MDA, 4 x 4 | 1.68 ± 0.083 | 0.0847 ± 0.013 | 250 | + +### Field (log K) + +| | RMSE | corr | spread | +|---|---|---|---| +| prior | 0.783 ± 0.019 | 0.0201 ± 0.027 | 0.8 ± 0.0028 | +| *Iterating* | | | | +| ES-MDA, 4 x 4 | 0.829 ± 0.016 | 0.205 ± 0.037 | 0.526 ± 0.0041 | + +### Scalars + +| | sor error | sor spread | vo error | vo spread | +|---|---|---|---|---| +| prior | 0.0314 ± 0.0058 | 0.0383 ± 0.00077 | 0.676 ± 0.15 | 1.02 ± 0.023 | +| *Iterating* | | | | | +| ES-MDA, 4 x 4 | 0.0349 ± 0.0059 | 0.0221 ± 0.0007 | 0.602 ± 0.094 | 0.469 ± 0.012 | + +## Case `fine`: 50 x 50 cells + +8 truths, 50 members; history of 16 of 40 steps, observed at steps [4, 8, 12, 16]. PET at `b731be7`, 2026-10-27. + +### Data and forecast + +| | misfit | RMSE forecast WWCT | simulations | +|---|---|---|---| +| prior | 97 ± 7.6 | 0.243 ± 0.029 | | +| *Iterating* | | | | +| ES-MDA, 4 x 4 | 1.9 ± 0.21 | 0.0867 ± 0.015 | 250 | + +### Field (log K) + +| | RMSE | corr | spread | +|---|---|---|---| +| prior | 0.815 ± 0.022 | -0.0673 ± 0.03 | 0.793 ± 0.0038 | +| *Iterating* | | | | +| ES-MDA, 4 x 4 | 0.869 ± 0.035 | 0.179 ± 0.059 | 0.511 ± 0.0037 | + +### Scalars + +| | sor error | sor spread | vo error | vo spread | +|---|---|---|---|---| +| prior | 0.0282 ± 0.008 | 0.0402 ± 0.0012 | 0.583 ± 0.12 | 1.03 ± 0.035 | +| *Iterating* | | | | | +| ES-MDA, 4 x 4 | 0.0326 ± 0.01 | 0.0253 ± 0.001 | 0.356 ± 0.14 | 0.47 ± 0.015 | diff --git a/benchmarks/minires/cases.py b/benchmarks/minires/cases.py new file mode 100644 index 00000000..5d495e4e --- /dev/null +++ b/benchmarks/minires/cases.py @@ -0,0 +1,133 @@ +"""The cases: one waterflood, at two resolutions, observed over its history and scored on its forecast. + +The reservoir is that of ert's ``test-data/ert/waterflood`` (equinor/ert#14598), so that the +two can be set side by side: a 1000 x 1000 x 20 m box in metric units (m, day, bar, mD, cP), +one injector at the centre on rate, four producers towards the corners on bottom-hole +pressure, 40 report steps of 60 days. The producers' rates, and when they water out, are +therefore outcomes of the permeability around them. + +The unknowns are the log-permeability field and two scalars of the fluid, the residual oil +saturation ``sor`` and the oil viscosity ``vo``. Each twin experiment draws its truth from +the same prior as the ensemble, simulates it to the end, and perturbs the history part into +observations; the forecast part is what the posterior is scored against, unseen. +""" + +from dataclasses import dataclass + +import numpy as np + +from misc.sampling import random_stream +from misc.structures.layout import StateLayout +from pipt.misc_tools.extract_tools import extract_prior_info +from simulator.minires import MiniRes + +LX = 1000.0 # extent [m] +DZ = 20.0 # thickness [m]: MiniRes is areal, so the rates are per unit thickness + +DT = 60.0 # report step [days] +NSTEPS = 40 +HISTORY = 16 # the observed steps, of which every fourth is assimilated +OBS_STEPS = list(range(4, HISTORY + 1, 4)) +FORECAST_STEPS = list(range(HISTORY + 1, NSTEPS + 1)) + +PRODUCERS = ["PROD1", "PROD2", "PROD3", "PROD4"] +INJECTOR = "INJ" +DATATYPES = [f"WWCT:{w}" for w in PRODUCERS] + [f"WOPR:{w}" for w in PRODUCERS] + [f"WBHP:{INJECTOR}"] + +# Observation error, as (relative, absolute): the larger applies. As ert's case. +OBS_ERROR = {"WWCT": (0.05, 0.02), "WOPR": (0.10, 5.0), "WBHP": (0.01, 1.0)} + +# The prior. log K about log(200 mD), sd 0.8, spherical variogram of range RANGE metres. +LOGK_MEAN, LOGK_SD, RANGE = np.log(200.0), 0.8, 300.0 +SCALARS = { + # mean sd limits + "sor": (0.22, 0.04, [0.05, 0.40]), + "vo": (5.0, 1.0, [1.5, 10.0]), +} + + +@dataclass(frozen=True) +class Case: + name: str + n: int # cells per side + truths: int # the number of twin experiments run by default + + @property + def dx(self): + return LX / self.n + + def model(self) -> dict: + """``[simulator.model]``, less what the state supplies (K, and the scalars' values).""" + wells = [{"name": INJECTOR, "xy": [LX / 2, LX / 2], "rate": 1000.0 / DZ, "rw": 0.15}] + for name, (fx, fy) in zip(PRODUCERS, [(0.15, 0.15), (0.85, 0.15), (0.15, 0.85), (0.85, 0.85)]): + wells.append({"name": name, "xy": [fx * LX, fy * LX], "bhp": 250.0, "rw": 0.15}) + return { + "Lx": LX, "Ly": LX, "Nx": self.n, "Ny": self.n, + "por": 0.2, "ct": 1e-5, "cdarcy": 0.008527, + "fluid": {"vw": 1.0, "vo": SCALARS["vo"][0], "swc": 0.2, "sor": SCALARS["sor"][0]}, + "wells": wells, + } + + def simulator(self, reportpoint) -> dict: + """The ``[simulator]`` section, reporting at ``reportpoint``.""" + return { + "dt": DT, "reporttype": "steps", "reportpoint": list(reportpoint), + "datatype": DATATYPES, "parallel": 1, "p0": 300.0, "s0": 0.2, + "model": self.model(), + } + + def ensemble(self, ne: int, seed: int, mean_file: str) -> dict: + """The ``[ensemble]`` section; ``mean_file`` is where :meth:`write_prior_mean` put the field's mean.""" + ens = { + "ne": ne, "seed": seed, "disable_tqdm": True, "save_prior": False, + "state": ["permx", *SCALARS], + "prior_permx": { + "vario": "sph", "mean": mean_file, "var": LOGK_SD**2, + "range": RANGE / self.dx, "aniso": 1.0, "angle": 0.0, "grid": [self.n, self.n, 1], + }, + } + for name, (mean, sd, limits) in SCALARS.items(): + ens[f"prior_{name}"] = {"mean": mean, "var": sd**2, "limits": limits} + return ens + + def write_prior_mean(self, path: str) -> None: + np.savez(path, permx=np.full(self.n**2, LOGK_MEAN)) + + def run(self, state: dict) -> np.ndarray: + """Simulate one member to the end: ``(NSTEPS, len(DATATYPES))``, or ``nan`` if it failed.""" + sim = MiniRes(self.simulator(range(1, NSTEPS + 1))) + records = sim.run_fwd_sim(state, 0) + if records is False: + return np.full((NSTEPS, len(DATATYPES)), np.nan) + return np.array([[row[d][0] for d in DATATYPES] for row in records]) + + +CASES = {c.name: c for c in [Case("coarse", 25, truths=20), Case("fine", 50, truths=8)]} + + +@dataclass +class Twin: + """One twin experiment: a truth, its full response, and the observations of its history.""" + + truth: dict # the true state, {variable: values} + response: np.ndarray # (NSTEPS, len(DATATYPES)), noiseless + obs: np.ndarray # (len(OBS_STEPS), len(DATATYPES)), perturbed + std: np.ndarray # the observation error, same shape + + +def twin(case: Case, seed: int, mean_file: str) -> Twin: + """Draw a truth from the prior, simulate it, and perturb the history into observations.""" + keys = case.ensemble(1, seed, mean_file) + enX, layout = StateLayout.from_prior_info(extract_prior_info(keys), 1, rng=random_stream(seed), save=False) + truth = {k: v[:, 0] for k, v in layout.to_dict(enX).items()} + response = case.run(truth) + + exact = response[np.array(OBS_STEPS) - 1] + rel, ab = np.array([OBS_ERROR[d.split(":")[0]] for d in DATATYPES]).T + std = np.maximum(rel * np.abs(exact), ab) + obs = exact + std * np.random.default_rng(seed).standard_normal(exact.shape) + # A water cut outside [0, 1], or a negative rate, would be unphysical + quantity = [d.split(":")[0] for d in DATATYPES] + lower = np.where(np.isin(quantity, ["WWCT", "WOPR"]), 0.0, -np.inf) + upper = np.where(np.isin(quantity, ["WWCT"]), 1.0, np.inf) + return Twin(truth, response, obs.clip(lower, upper), std) diff --git a/benchmarks/minires/methods.py b/benchmarks/minires/methods.py new file mode 100644 index 00000000..8e0050c7 --- /dev/null +++ b/benchmarks/minires/methods.py @@ -0,0 +1,19 @@ +"""The methods compared: each a label, the question it belongs to, and its ``[dataassim]`` settings. + +The settings are PET's own, as a config would give them. Unless a row says it is tuned, +they are defaults or the textbook choice, the same for every case. +""" + +COMMON = {"energy": 99.0, "obsname": "steps"} + +METHODS = [ + { + "label": "ES-MDA, 4 x 4", + "group": "Iterating", + "dataassim": { + "scheme": "esmda", + "analysis": "approx", + "mda": {"tot_assim_steps": 4, "inflation_param": [4, 4, 4, 4]}, + }, + }, +] diff --git a/benchmarks/minires/results/coarse.json b/benchmarks/minires/results/coarse.json new file mode 100644 index 00000000..e32f9ec8 --- /dev/null +++ b/benchmarks/minires/results/coarse.json @@ -0,0 +1,435 @@ +{ + "meta": { + "truths": 20, + "ne": 50, + "commit": "b731be7", + "date": "2026-10-27" + }, + "methods": { + "ES-MDA, 4 x 4": { + "group": "Iterating", + "runs": [ + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250 + ], + "prior": { + "misfit": [ + 62.59639601598487, + 123.05196241421295, + 93.22506871276596, + 68.11519772266308, + 84.25907621454918, + 96.40961602542615, + 80.82025957645489, + 66.07005377237218, + 101.54560412060758, + 85.36695599509565, + 84.85681317502099, + 86.77696301351816, + 99.53319335321834, + 105.47695612554558, + 97.09159986755921, + 132.8728233554532, + 71.05869753024243, + 89.23927313744124, + 85.03146870297716, + 119.6601737296945 + ], + "forecast_wwct_rmse": [ + 0.1696800381197985, + 0.19932798619750214, + 0.18030727564909557, + 0.1687946346333842, + 0.20334516472452202, + 0.22090711830163232, + 0.08119475966762553, + 0.09937498257293596, + 0.156574677706107, + 0.19141035949790325, + 0.17710425467402569, + 0.284394647956727, + 0.36722815919437385, + 0.3633631450880431, + 0.0871894186024358, + 0.3796182871950613, + 0.3304530925059456, + 0.2110413313020751, + 0.21456969178190033, + 0.3404644384560941 + ], + "logk_rmse": [ + 0.8861115737970015, + 0.7138871566140654, + 0.7311114337240311, + 0.794578500774992, + 0.7398491140881406, + 0.7151208051832781, + 0.6651517080794027, + 0.66548804681917, + 0.8119847333173705, + 0.9808250445467559, + 0.87589763060281, + 0.7567644686285704, + 0.7664685244639458, + 0.6734759186020055, + 0.720284226310797, + 0.8370809938580748, + 0.8168980923865478, + 0.8378640493333555, + 0.7818831719048845, + 0.8986200732688832 + ], + "logk_corr": [ + 0.1246439649411075, + 0.041320044934769304, + -0.07031257032538189, + 0.006587399183903993, + -0.04834291429886706, + 0.10247516110712568, + 0.05891645085620727, + -0.050468555461754085, + 0.02976338891111694, + -0.002938995933889871, + 0.29384176709012366, + -0.02703682627000836, + 0.0006754033719746696, + 0.10968951060279891, + 0.016901619123219427, + 0.021257305690592972, + -0.1962829661167955, + -0.24074858187770273, + 0.021991764306852377, + 0.20930361234310244 + ], + "logk_spread": [ + 0.8020730046602623, + 0.8081223960932921, + 0.7992709069836804, + 0.80689275565137, + 0.8087260800099462, + 0.776756049914733, + 0.8148302010845695, + 0.7914120818443737, + 0.7742225610152146, + 0.7844666810750466, + 0.7961890309583656, + 0.8095081632860182, + 0.8135303891787904, + 0.8102967016940247, + 0.7932683248624264, + 0.8200563852133691, + 0.7847055424944857, + 0.807270751436195, + 0.7996489668967339, + 0.7994297351777555 + ], + "sor_error": [ + 0.022517471776421466, + 0.03231193249221098, + 0.039741926229656194, + 0.025226634560968048, + 0.02143169540386905, + 0.03331620304428154, + 0.09985184524027202, + 0.014347419674480971, + 0.019789320113076436, + 0.007200528938781425, + 0.0151715061165347, + 0.0006871729371782098, + 0.06846730526575548, + 0.0054726541204520585, + 0.0157363589256444, + 0.04325964194518711, + 0.013508305935118675, + 0.0143029506873453, + 0.07642224738364964, + 0.05846501566022749 + ], + "sor_spread": [ + 0.037483552502312445, + 0.04443805479855501, + 0.03989209833172408, + 0.03959439780626146, + 0.0373651192814694, + 0.042707108594228896, + 0.03811974321077451, + 0.040700857005014156, + 0.03690223223461561, + 0.039779572774622934, + 0.03532836375446026, + 0.033973758704551923, + 0.031136092849106147, + 0.03541348194730274, + 0.04453781527474917, + 0.03976759448274524, + 0.03409458530957603, + 0.03832621780740705, + 0.040225680565728845, + 0.036738472671362744 + ], + "vo_error": [ + 0.0795937058366567, + 0.11494164476815705, + 1.2568667117930428, + 2.278635530475472, + 0.06830189446609758, + 0.48168370560906393, + 0.3255283041707937, + 0.3060383274058749, + 2.3252256889911536, + 1.1003292963327223, + 0.7792432407050995, + 0.6149525736686261, + 0.12998468061398327, + 0.25109279552830355, + 0.9197588811212274, + 0.020342999691860264, + 0.8290998138473791, + 0.7684392614247813, + 0.3509682114471895, + 0.5242844172918861 + ], + "vo_spread": [ + 0.8414351702422311, + 1.1505420162407765, + 1.0526275960392726, + 1.0154658807892343, + 0.9312057839633537, + 1.0379686315534349, + 1.0571140014594451, + 0.9524395430187967, + 1.1211433903472134, + 1.1029772445921056, + 1.093164220791091, + 0.916497074696293, + 0.8558348454302246, + 1.1186998269669504, + 0.9675067812016532, + 0.8692545869158198, + 1.1936075973977103, + 1.0932252757164844, + 0.9942898361540824, + 1.0722824850643657 + ] + }, + "posterior": { + "misfit": [ + 1.7915037779691652, + 2.2713669028286536, + 1.8834786117549918, + 2.090805229678032, + 1.1295683733463626, + 1.5319714581707664, + 1.6925341525835529, + 2.4210688161316978, + 1.6140742962228585, + 2.0084947181443713, + 1.0498687751961777, + 1.3468499072267313, + 1.7293690104066557, + 1.4096521907680108, + 1.751932889654661, + 1.6288938332347815, + 1.6163310719644617, + 1.3170131524801687, + 2.063243605642272, + 1.2410507890572322 + ], + "forecast_wwct_rmse": [ + 0.05971978267733749, + 0.0808441584343274, + 0.054870103971857195, + 0.020196350753962533, + 0.08954900332708025, + 0.15004744903977274, + 0.02610636709537744, + 0.03962336486488827, + 0.14900681526379084, + 0.029354166192413246, + 0.11220849448933627, + 0.09251367787219592, + 0.1435614254468246, + 0.14793067332321716, + 0.020800996084311543, + 0.14903290369134609, + 0.20415621483726862, + 0.042416725562170625, + 0.023560599138257407, + 0.057783205512547835 + ], + "logk_rmse": [ + 0.8824439139576858, + 0.822063773293608, + 0.8540000151824455, + 0.7516720232705129, + 0.8688289790482115, + 0.752273683144322, + 0.7795559576511735, + 0.7330022922807709, + 0.8306284359302947, + 0.9618180860590525, + 0.9344464608196765, + 0.7724232059018321, + 0.7774633689451728, + 0.7390113473071941, + 0.9269369245875531, + 0.8701089376191667, + 0.7080916379514234, + 0.8720653496895182, + 0.8687236528230379, + 0.875091198747223 + ], + "logk_corr": [ + 0.25834218504251655, + 0.36279578635627274, + -0.05802734806026723, + 0.43356714101280563, + 0.0034891235638091728, + 0.3847164599548598, + -0.05461168372186331, + 0.05718933569329724, + 0.15105488147046053, + 0.23884564661082452, + 0.27778129351489306, + 0.3187360527273105, + 0.3049188976536502, + 0.24155223944941812, + -0.0802551922530325, + 0.24353644135792624, + 0.4634555931257992, + 0.17273082104692686, + 0.06343747962801316, + 0.3113389454842998 + ], + "logk_spread": [ + 0.522249966247683, + 0.540217777799695, + 0.5141898880744636, + 0.5289528638194563, + 0.5315106616983306, + 0.5513444137534439, + 0.5302982140159008, + 0.48017895527902377, + 0.525259019825193, + 0.5067893095530761, + 0.5375047252407087, + 0.528625947683071, + 0.5308990634930119, + 0.540715534695878, + 0.4887122300859935, + 0.5468420419487485, + 0.5202390856526595, + 0.5259851679065075, + 0.5200908716297254, + 0.5468977109777547 + ], + "sor_error": [ + 0.01242897046793881, + 0.007020330782374878, + 0.047914213709490866, + 0.035926265134937574, + 0.021847906141799156, + 0.039723233217432014, + 0.0902211768789665, + 0.00871855022369819, + 0.0510359365610053, + 0.025308113721619202, + 0.012287415158729442, + 0.03542601243845572, + 0.0455292790176694, + 0.01502380677411888, + 0.008522647777007175, + 0.011827804865997882, + 0.050489931534770854, + 0.015445523475865142, + 0.08187491233282074, + 0.0818302613398329 + ], + "sor_spread": [ + 0.02717638356695815, + 0.027602498128945885, + 0.02107191973113316, + 0.02197966357926537, + 0.025661374266385168, + 0.027201555576436658, + 0.019458937710670098, + 0.02135285375076458, + 0.023787370520650386, + 0.02065226479580053, + 0.020494958643270837, + 0.023107897316852952, + 0.016337631269547778, + 0.020991029418424994, + 0.019468587320449687, + 0.01804112096054823, + 0.019210575899859954, + 0.024390711960314428, + 0.02247391444950562, + 0.02084013419112236 + ], + "vo_error": [ + 0.03804462855224422, + 0.8481518045198975, + 0.16070951795245136, + 1.4688631729855812, + 0.7074955697458254, + 0.7252109230335972, + 1.1564938697091711, + 1.0103751305913296, + 0.42109246974746384, + 0.8756191775895035, + 0.3414625893296339, + 0.011628446583380914, + 0.3082926601920537, + 0.4690300502695832, + 0.16576944581815933, + 0.28675349211516643, + 0.2804027382195349, + 0.6759925548024706, + 0.832009762336237, + 1.2517213326608125 + ], + "vo_spread": [ + 0.35458573457832043, + 0.6151094823234938, + 0.48466101503718517, + 0.49436843608716186, + 0.4625746264241777, + 0.4164524395187772, + 0.44974258301004216, + 0.418418042169793, + 0.47536342927143194, + 0.47965508750314906, + 0.5311379838391072, + 0.5262123953623323, + 0.4651828711707321, + 0.48260504021484807, + 0.40232643824700026, + 0.47759285081393976, + 0.4836767500165765, + 0.41983718068368453, + 0.5090848968830625, + 0.436561033186781 + ] + } + } + } +} diff --git a/benchmarks/minires/results/fine.json b/benchmarks/minires/results/fine.json new file mode 100644 index 00000000..5b2c80b8 --- /dev/null +++ b/benchmarks/minires/results/fine.json @@ -0,0 +1,207 @@ +{ + "meta": { + "truths": 8, + "ne": 50, + "commit": "b731be7", + "date": "2026-10-27" + }, + "methods": { + "ES-MDA, 4 x 4": { + "group": "Iterating", + "runs": [ + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250 + ], + "prior": { + "misfit": [ + 76.6857082028925, + 118.48859535370278, + 125.88068474763797, + 88.2290594753218, + 120.13401879681457, + 71.19485589384213, + 93.81381539692943, + 81.58416139246368 + ], + "forecast_wwct_rmse": [ + 0.26855951538628003, + 0.3710015080260862, + 0.2823521538778289, + 0.258549119921913, + 0.29548286093911674, + 0.12655814500299795, + 0.1914157610497057, + 0.14985517995795167 + ], + "logk_rmse": [ + 0.862081806419521, + 0.860747552427497, + 0.8814335507422228, + 0.8509947054495131, + 0.8418541928350743, + 0.7313398522920637, + 0.7298452045306284, + 0.7594852129514473 + ], + "logk_corr": [ + -0.016450365365537036, + -0.14702663951406622, + -0.08504505935224134, + 0.05992274845787901, + -0.09942851429124891, + -0.1345454128985617, + 0.03618745715727143, + -0.15229398835963354 + ], + "logk_spread": [ + 0.7966141389511963, + 0.7989257773797538, + 0.802150283708243, + 0.8046550340635544, + 0.796685800426243, + 0.778724626542447, + 0.7930365195482185, + 0.7752310117323408 + ], + "sor_error": [ + 0.034576287836562164, + 0.07750867354115154, + 0.001919478814194464, + 0.032093631315579296, + 0.012772658013380705, + 0.02436307210664651, + 0.024252339511175364, + 0.01789218898357159 + ], + "sor_spread": [ + 0.03823958904277204, + 0.036130397433477894, + 0.042194407622493546, + 0.04271449501189857, + 0.04399000371765547, + 0.042128076892931896, + 0.034526552731212405, + 0.04152268006305935 + ], + "vo_error": [ + 1.2156756183526172, + 0.7487784775528086, + 0.4811975872810086, + 0.006578025623270811, + 0.6644893701416903, + 0.5505227441668827, + 0.5861382992353326, + 0.4140336539118179 + ], + "vo_spread": [ + 1.0749710722220533, + 0.9922316067216777, + 0.9701440255463729, + 0.9782281932782648, + 1.2395528142331786, + 0.9280871834029033, + 1.0583006464310203, + 0.9696631777900684 + ] + }, + "posterior": { + "misfit": [ + 1.7586002638548632, + 1.6432050233265005, + 1.5160408697861272, + 2.0559294530130785, + 1.2869140809373079, + 3.027528895886259, + 1.4305449247215989, + 2.4806472042872483 + ], + "forecast_wwct_rmse": [ + 0.10136091135761618, + 0.11952149263063845, + 0.15071776244995205, + 0.05344330706244768, + 0.10367678383331855, + 0.02118676573268266, + 0.053962667123366934, + 0.08954923981177185 + ], + "logk_rmse": [ + 0.8859951573293843, + 0.8713868436864755, + 1.056508625792122, + 0.8797782765686962, + 0.8456485203491287, + 0.7011339063549485, + 0.8234989229187442, + 0.890078268092921 + ], + "logk_corr": [ + 0.2084463063545232, + 0.3728317974927288, + -0.0011005938573501331, + 0.2172488313855666, + 0.19041945517327907, + 0.40478049567559793, + 0.1211384952462299, + -0.08525050268305748 + ], + "logk_spread": [ + 0.5219040967180381, + 0.5199596820897883, + 0.5082064296977429, + 0.5142558803160056, + 0.5107465569817028, + 0.5149595619641594, + 0.511392726832032, + 0.48784121835244887 + ], + "sor_error": [ + 0.09121866544834878, + 0.01893216870187392, + 0.014112300998051247, + 0.016500630962240248, + 0.017672623084962785, + 0.054857265618507234, + 0.0029021147594167906, + 0.04479968384056643 + ], + "sor_spread": [ + 0.020477009159431698, + 0.03074139066894728, + 0.02407800669889662, + 0.02536904922400518, + 0.025173870551999027, + 0.02717380058434338, + 0.023638497218210222, + 0.025459340174916863 + ], + "vo_error": [ + 0.46307121162638865, + 0.3163276673395954, + 0.08173338371291106, + 0.07587775277478759, + 0.04112571641791085, + 0.023688484061209714, + 0.7656534005058386, + 1.0804223803609805 + ], + "vo_spread": [ + 0.49871325980136555, + 0.4977008032115977, + 0.4073832878443032, + 0.4692456640668884, + 0.3971130490654576, + 0.49358896781283396, + 0.495497635963307, + 0.5019287799827108 + ] + } + } + } +} diff --git a/benchmarks/minires/run.py b/benchmarks/minires/run.py new file mode 100644 index 00000000..0e649410 --- /dev/null +++ b/benchmarks/minires/run.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +"""Run the methods on the cases' twin experiments, and write the tables in RESULTS.md. + + python run.py # every method, every case, the default number of truths + python run.py --case coarse # one case + python run.py --method "ES-MDA" # the methods whose label contains this + python run.py --truths 2 --ne 10 # a quick check (not written to results/) + +Each (method, truth) is one twin experiment, run in its own process and its own +temporary directory, so that what PET writes there (results, logs) goes nowhere. +The scores of each are kept in ``results/.json``; RESULTS.md is generated from those. +A run of a subset of the methods replaces their entries and leaves the others'. +""" + +import os + +for _var in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS"): + os.environ.setdefault(_var, "1") # the parallelism is over experiments, not within them + +import argparse +import contextlib +import io +import json +import logging +import subprocess +import tempfile +from concurrent.futures import ProcessPoolExecutor +from datetime import date +from pathlib import Path + +import numpy as np +import pandas as pd + +from cases import CASES, DATATYPES, HISTORY, NSTEPS, OBS_STEPS, SCALARS, twin +from input_output.config import normalize +from methods import COMMON, METHODS +from pipt.update_schemes.registry import get_scheme +from scores import scores +from simulator.minires import MiniRes + +HERE = Path(__file__).parent +TRUTH_SEED, ENSEMBLE_SEED = 1000, 2000 # truth i is drawn with TRUTH_SEED + i + + +class CountingMiniRes(MiniRes): + """MiniRes, counting the simulations the method asks for.""" + + runs = 0 + + def run_fwd_sim(self, state, member_index=0, **kwargs): + self.runs += 1 + return super().run_fwd_sim(state, member_index, **kwargs) + + +def experiment(case_name: str, method: dict, i: int, ne: int) -> dict: + """One twin experiment: the prior's and the posterior's scores, and the simulations spent.""" + case = CASES[case_name] + with ( + tempfile.TemporaryDirectory() as tmp, + contextlib.chdir(tmp), + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + case.write_prior_mean("priormean.npz") + tw = twin(case, TRUTH_SEED + i, "priormean.npz") + + index = pd.Index(OBS_STEPS, name="steps") + pd.DataFrame(tw.obs, index=index, columns=DATATYPES).to_csv("data.csv") + variance = [[str(["abs", float(s**2)]) for s in row] for row in tw.std] + pd.DataFrame(variance, index=index, columns=DATATYPES).to_csv("var.csv") + + keys_da, keys_sim, keys_en = normalize( + {**COMMON, **method["dataassim"], "data": "data.csv", "datavar": "var.csv", "savefolder": "Results"}, + case.simulator(OBS_STEPS), + case.ensemble(ne, ENSEMBLE_SEED + i, "priormean.npz"), + ) + sim = CountingMiniRes(keys_sim) + scheme = get_scheme(keys_da["scheme"], keys_da["analysis"])(keys_da, keys_en, sim) + idX = scheme.ensemble.idX + prior = np.array(scheme.ensemble.enX, copy=True) + scheme.run_assimilation() + posterior = np.array(scheme.ensemble.enX) + + out = {"runs": sim.runs} + for label, enX in (("prior", prior), ("posterior", posterior)): + states = {k: enX[a:b] for k, (a, b) in idX.items()} + members = [{k: v[:, j] for k, v in states.items()} for j in range(enX.shape[1])] + out[label] = scores(states, np.array([case.run(m) for m in members]), tw) + return out + + +# ---------------------------------------------------------------------- +# Tables +# ---------------------------------------------------------------------- +TABLES = { + "Data and forecast": {"misfit": "misfit", "forecast_wwct_rmse": "RMSE forecast WWCT", "runs": "simulations"}, + "Field (log K)": {"logk_rmse": "RMSE", "logk_corr": "corr", "logk_spread": "spread"}, + "Scalars": {f"{n}_{s}": f"{n} {s}" for n in SCALARS for s in ("error", "spread")}, +} + + +def cell(values) -> str: + """Mean, and its standard error over the truths.""" + values = np.asarray(values, dtype=float) + if len(values) == 1: + return f"{values[0]:.3g}" + return f"{values.mean():.3g} ± {values.std(ddof=1) / np.sqrt(len(values)):.2g}" + + +def render(results: dict) -> str: + lines = [ + "# MiniRes benchmark: results", + "", + "Generated by `run.py`; do not edit by hand. See `cases.py` for the cases, `methods.py` for the", + "methods and their settings, and `scores.py` for what is scored. Each cell is the mean over", + "the twin experiments (truths), ± its standard error. Rows are grouped by the question they", + "address, not ranked: which case and which score matters is for the reader to judge.", + ] + for case_name, res in results.items(): + meta, runs = res["meta"], res["methods"] + case = CASES[case_name] + lines += [ + "", + f"## Case `{case_name}`: {case.n} x {case.n} cells", + "", + f"{meta['truths']} truths, {meta['ne']} members; history of {HISTORY} of {NSTEPS} steps, " + f"observed at steps {OBS_STEPS}. PET at `{meta['commit']}`, {meta['date']}.", + ] + any_method = next(iter(runs.values())) + for title, columns in TABLES.items(): + lines += ["", f"### {title}", "", "| | " + " | ".join(columns.values()) + " |"] + lines += ["|---" * (len(columns) + 1) + "|"] + prior = [cell(any_method["prior"][k]) if k != "runs" else "" for k in columns] + lines += ["| prior | " + " | ".join(prior) + " |"] + group = None + for label, r in runs.items(): + if r["group"] != group: + group = r["group"] + lines += [f"| *{group}* |" + " |" * len(columns)] + post = [cell(r["posterior"][k]) if k != "runs" else f"{np.mean(r['runs']):.0f}" for k in columns] + lines += [f"| {label} | " + " | ".join(post) + " |"] + return "\n".join(lines) + "\n" + + +def commit() -> str: + def git(*args): + return subprocess.run(["git", *args], cwd=HERE, capture_output=True, text=True).stdout.strip() + + # The last commit of PET's code, which is what the numbers depend on (not of the benchmark itself) + src = str(HERE.parents[1] / "src") + return git("log", "-1", "--format=%h", "--", src) + ("-dirty" if git("status", "--porcelain", "--", src) else "") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--case", choices=list(CASES), action="append", help="default: all") + parser.add_argument("--method", action="append", help="label substring; default: all") + parser.add_argument("--truths", type=int, help="default: the case's own") + parser.add_argument("--ne", type=int, default=50) + parser.add_argument("--workers", type=int, default=os.cpu_count()) + args = parser.parse_args() + + methods = [m for m in METHODS if not args.method or any(s in m["label"] for s in args.method)] + quick = args.truths is not None and args.truths < min(CASES[c].truths for c in args.case or CASES) + + for case_name in args.case or CASES: + truths = args.truths or CASES[case_name].truths + jobs = [(m, i) for m in methods for i in range(truths)] + print(f"{case_name}: {len(methods)} method(s) x {truths} truths ...", flush=True) + with ProcessPoolExecutor(args.workers, initializer=logging.disable, initargs=(logging.WARNING,)) as pool: + done = list(pool.map(experiment, *zip(*[(case_name, m, i, args.ne) for m, i in jobs]))) + + res_file = HERE / "results" / f"{case_name}.json" + meta = {"truths": truths, "ne": args.ne, "commit": commit(), "date": str(date.today())} + old = json.loads(res_file.read_text()) if res_file.exists() else None + compatible = old and (old["meta"]["truths"], old["meta"]["ne"]) == (truths, args.ne) + results = {"meta": meta, "methods": dict(old["methods"]) if compatible else {}} + for m in methods: + mine = [d for (mm, _), d in zip(jobs, done) if mm is m] + results["methods"][m["label"]] = { + "group": m["group"], + "runs": [d["runs"] for d in mine], + **{s: {k: [d[s][k] for d in mine] for k in mine[0][s]} for s in ("prior", "posterior")}, + } + results["methods"] = {m["label"]: results["methods"][m["label"]] for m in METHODS if m["label"] in results["methods"]} + + if quick: + print(render({case_name: results})) + continue + res_file.parent.mkdir(exist_ok=True) + res_file.write_text(json.dumps(results, indent=1) + "\n") + + if not quick: + everything = {c: json.loads((HERE / "results" / f"{c}.json").read_text()) for c in CASES if (HERE / "results" / f"{c}.json").exists()} + (HERE / "RESULTS.md").write_text(render(everything)) + print("Wrote RESULTS.md") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/minires/scores.py b/benchmarks/minires/scores.py new file mode 100644 index 00000000..a2f3cd71 --- /dev/null +++ b/benchmarks/minires/scores.py @@ -0,0 +1,50 @@ +"""What an ensemble is scored on: the data it was shown, the unknowns, and the forecast it was not shown. + +The misfit is what an assimilation minimises, so it says least about it: an ensemble can be +made to fit its history while moving away from the truth. Hence the other scores, which a +twin experiment can compute because it knows the truth. +""" + +import numpy as np + +from cases import DATATYPES, FORECAST_STEPS, OBS_STEPS, SCALARS, Twin + +WWCT = [i for i, d in enumerate(DATATYPES) if d.startswith("WWCT")] + + +def scores(states: dict, responses: np.ndarray, twin: Twin) -> dict: + """Score one ensemble. + + Parameters + ---------- + states : dict + ``{variable: (size, ne)}``, the ensemble's unknowns. + responses : ndarray + ``(ne, NSTEPS, len(DATATYPES))``, each member simulated to the end. + twin : Twin + The truth and its observations. + """ + out = {} + + # The history: mean squared normalized misfit, over members and observations + history = responses[:, np.array(OBS_STEPS) - 1] + out["misfit"] = float(np.nanmean(((history - twin.obs) / twin.std) ** 2)) + + # The forecast: RMS error of the ensemble mean's water cut, over the unseen steps + forecast = np.nanmean(responses[:, np.array(FORECAST_STEPS) - 1][..., WWCT], axis=0) + truth = twin.response[np.array(FORECAST_STEPS) - 1][:, WWCT] + out["forecast_wwct_rmse"] = float(np.sqrt(np.mean((forecast - truth) ** 2))) + + # The field: error and correlation of the ensemble mean, and the spread left + logk, true_logk = states["permx"], twin.truth["permx"] + mean = logk.mean(axis=1) + out["logk_rmse"] = float(np.sqrt(np.mean((mean - true_logk) ** 2))) + out["logk_corr"] = float(np.corrcoef(mean, true_logk)[0, 1]) + out["logk_spread"] = float(logk.std(axis=1, ddof=1).mean()) + + # The scalars: error of the ensemble mean, and the spread left + for name in SCALARS: + values = states[name][0] + out[f"{name}_error"] = float(abs(values.mean() - twin.truth[name][0])) + out[f"{name}_spread"] = float(values.std(ddof=1)) + return out diff --git a/pyproject.toml b/pyproject.toml index 259dd419..40841cf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,4 +108,5 @@ ignore = [ "src/misc/grdecl.py" = ["E722", "E741", "F821"] # vendored parser; F821 is dead Python-2-only code "src/misc/ecl.py" = ["E722", "E741"] "src/misc/grid/sector.py" = ["E722", "E741"] +"benchmarks/minires/run.py" = ["E402"] # pins the BLAS threads before numpy is imported "src/misc/grid/unstruct.py" = ["F841"] # untested legacy grid parser; allocations look WIP, not dead code diff --git a/tests/test_benchmark_minires.py b/tests/test_benchmark_minires.py new file mode 100644 index 00000000..80d7f4cf --- /dev/null +++ b/tests/test_benchmark_minires.py @@ -0,0 +1,23 @@ +"""The MiniRes benchmark (benchmarks/minires) still runs: one tiny twin experiment per method.""" + +import sys +from pathlib import Path + +import numpy as np +import pytest + +pytest.importorskip("minires", reason="pip install PET[minires]") + +sys.path.insert(0, str(Path(__file__).parents[1] / "benchmarks" / "minires")) +from methods import METHODS # noqa: E402 +from run import experiment # noqa: E402 + + +@pytest.mark.slow +@pytest.mark.parametrize("method", METHODS, ids=[m["label"] for m in METHODS]) +def test_a_twin_experiment_runs_and_scores(method): + out = experiment("coarse", method, 0, ne=6) + assert out["runs"] > 0 + for stage in ("prior", "posterior"): + assert all(np.isfinite(v) for v in out[stage].values()), out[stage] + assert out["posterior"]["misfit"] < out["prior"]["misfit"] From 1d3680cbc25663ee856e5960528f661d0932e1de Mon Sep 17 00:00:00 2001 From: patnr Date: Tue, 27 Oct 2026 15:09:40 +0100 Subject: [PATCH 2/6] Benchmark: one table per case Co-Authored-By: Claude Opus 5.5 --- benchmarks/minires/RESULTS.md | 56 +++++++---------------------------- benchmarks/minires/run.py | 34 +++++++++++---------- 2 files changed, 28 insertions(+), 62 deletions(-) diff --git a/benchmarks/minires/RESULTS.md b/benchmarks/minires/RESULTS.md index 61934e1e..99153fa9 100644 --- a/benchmarks/minires/RESULTS.md +++ b/benchmarks/minires/RESULTS.md @@ -9,54 +9,18 @@ address, not ranked: which case and which score matters is for the reader to jud 20 truths, 50 members; history of 16 of 40 steps, observed at steps [4, 8, 12, 16]. PET at `b731be7`, 2026-10-27. -### Data and forecast - -| | misfit | RMSE forecast WWCT | simulations | -|---|---|---|---| -| prior | 91.7 ± 4.2 | 0.221 ± 0.021 | | -| *Iterating* | | | | -| ES-MDA, 4 x 4 | 1.68 ± 0.083 | 0.0847 ± 0.013 | 250 | - -### Field (log K) - -| | RMSE | corr | spread | -|---|---|---|---| -| prior | 0.783 ± 0.019 | 0.0201 ± 0.027 | 0.8 ± 0.0028 | -| *Iterating* | | | | -| ES-MDA, 4 x 4 | 0.829 ± 0.016 | 0.205 ± 0.037 | 0.526 ± 0.0041 | - -### Scalars - -| | sor error | sor spread | vo error | vo spread | -|---|---|---|---|---| -| prior | 0.0314 ± 0.0058 | 0.0383 ± 0.00077 | 0.676 ± 0.15 | 1.02 ± 0.023 | -| *Iterating* | | | | | -| ES-MDA, 4 x 4 | 0.0349 ± 0.0059 | 0.0221 ± 0.0007 | 0.602 ± 0.094 | 0.469 ± 0.012 | +| | misfit | RMSE forecast WWCT | logK RMSE | logK corr | logK spread | sor error | sor spread | vo error | vo spread | simulations | +|---|---|---|---|---|---|---|---|---|---|---| +| prior | 91.7 ± 4.2 | 0.221 ± 0.021 | 0.783 ± 0.019 | 0.0201 ± 0.027 | 0.8 ± 0.0028 | 0.0314 ± 0.0058 | 0.0383 ± 0.00077 | 0.676 ± 0.15 | 1.02 ± 0.023 | | +| *Iterating* | | | | | | | | | | | +| ES-MDA, 4 x 4 | 1.68 ± 0.083 | 0.0847 ± 0.013 | 0.829 ± 0.016 | 0.205 ± 0.037 | 0.526 ± 0.0041 | 0.0349 ± 0.0059 | 0.0221 ± 0.0007 | 0.602 ± 0.094 | 0.469 ± 0.012 | 250 | ## Case `fine`: 50 x 50 cells 8 truths, 50 members; history of 16 of 40 steps, observed at steps [4, 8, 12, 16]. PET at `b731be7`, 2026-10-27. -### Data and forecast - -| | misfit | RMSE forecast WWCT | simulations | -|---|---|---|---| -| prior | 97 ± 7.6 | 0.243 ± 0.029 | | -| *Iterating* | | | | -| ES-MDA, 4 x 4 | 1.9 ± 0.21 | 0.0867 ± 0.015 | 250 | - -### Field (log K) - -| | RMSE | corr | spread | -|---|---|---|---| -| prior | 0.815 ± 0.022 | -0.0673 ± 0.03 | 0.793 ± 0.0038 | -| *Iterating* | | | | -| ES-MDA, 4 x 4 | 0.869 ± 0.035 | 0.179 ± 0.059 | 0.511 ± 0.0037 | - -### Scalars - -| | sor error | sor spread | vo error | vo spread | -|---|---|---|---|---| -| prior | 0.0282 ± 0.008 | 0.0402 ± 0.0012 | 0.583 ± 0.12 | 1.03 ± 0.035 | -| *Iterating* | | | | | -| ES-MDA, 4 x 4 | 0.0326 ± 0.01 | 0.0253 ± 0.001 | 0.356 ± 0.14 | 0.47 ± 0.015 | +| | misfit | RMSE forecast WWCT | logK RMSE | logK corr | logK spread | sor error | sor spread | vo error | vo spread | simulations | +|---|---|---|---|---|---|---|---|---|---|---| +| prior | 97 ± 7.6 | 0.243 ± 0.029 | 0.815 ± 0.022 | -0.0673 ± 0.03 | 0.793 ± 0.0038 | 0.0282 ± 0.008 | 0.0402 ± 0.0012 | 0.583 ± 0.12 | 1.03 ± 0.035 | | +| *Iterating* | | | | | | | | | | | +| ES-MDA, 4 x 4 | 1.9 ± 0.21 | 0.0867 ± 0.015 | 0.869 ± 0.035 | 0.179 ± 0.059 | 0.511 ± 0.0037 | 0.0326 ± 0.01 | 0.0253 ± 0.001 | 0.356 ± 0.14 | 0.47 ± 0.015 | 250 | diff --git a/benchmarks/minires/run.py b/benchmarks/minires/run.py index 0e649410..c81ae683 100644 --- a/benchmarks/minires/run.py +++ b/benchmarks/minires/run.py @@ -92,10 +92,14 @@ def experiment(case_name: str, method: dict, i: int, ne: int) -> dict: # ---------------------------------------------------------------------- # Tables # ---------------------------------------------------------------------- -TABLES = { - "Data and forecast": {"misfit": "misfit", "forecast_wwct_rmse": "RMSE forecast WWCT", "runs": "simulations"}, - "Field (log K)": {"logk_rmse": "RMSE", "logk_corr": "corr", "logk_spread": "spread"}, - "Scalars": {f"{n}_{s}": f"{n} {s}" for n in SCALARS for s in ("error", "spread")}, +COLUMNS = { + "misfit": "misfit", + "forecast_wwct_rmse": "RMSE forecast WWCT", + "logk_rmse": "logK RMSE", + "logk_corr": "logK corr", + "logk_spread": "logK spread", + **{f"{n}_{s}": f"{n} {s}" for n in SCALARS for s in ("error", "spread")}, + "runs": "simulations", } @@ -127,18 +131,16 @@ def render(results: dict) -> str: f"observed at steps {OBS_STEPS}. PET at `{meta['commit']}`, {meta['date']}.", ] any_method = next(iter(runs.values())) - for title, columns in TABLES.items(): - lines += ["", f"### {title}", "", "| | " + " | ".join(columns.values()) + " |"] - lines += ["|---" * (len(columns) + 1) + "|"] - prior = [cell(any_method["prior"][k]) if k != "runs" else "" for k in columns] - lines += ["| prior | " + " | ".join(prior) + " |"] - group = None - for label, r in runs.items(): - if r["group"] != group: - group = r["group"] - lines += [f"| *{group}* |" + " |" * len(columns)] - post = [cell(r["posterior"][k]) if k != "runs" else f"{np.mean(r['runs']):.0f}" for k in columns] - lines += [f"| {label} | " + " | ".join(post) + " |"] + lines += ["", "| | " + " | ".join(COLUMNS.values()) + " |", "|---" * (len(COLUMNS) + 1) + "|"] + prior = [cell(any_method["prior"][k]) if k != "runs" else "" for k in COLUMNS] + lines += ["| prior | " + " | ".join(prior) + " |"] + group = None + for label, r in runs.items(): + if r["group"] != group: + group = r["group"] + lines += [f"| *{group}* |" + " |" * len(COLUMNS)] + post = [cell(r["posterior"][k]) if k != "runs" else f"{np.mean(r['runs']):.0f}" for k in COLUMNS] + lines += [f"| {label} | " + " | ".join(post) + " |"] return "\n".join(lines) + "\n" From 279e1a5d71b1044415c5c019a5ea3ccfeca86f5b Mon Sep 17 00:00:00 2001 From: patnr Date: Tue, 27 Oct 2026 15:12:01 +0100 Subject: [PATCH 3/6] Benchmark: spell out the method's label, drop the group rows Co-Authored-By: Claude Opus 5.5 --- benchmarks/minires/RESULTS.md | 10 ++++------ benchmarks/minires/methods.py | 5 ++--- benchmarks/minires/results/coarse.json | 3 +-- benchmarks/minires/results/fine.json | 3 +-- benchmarks/minires/run.py | 9 ++------- 5 files changed, 10 insertions(+), 20 deletions(-) diff --git a/benchmarks/minires/RESULTS.md b/benchmarks/minires/RESULTS.md index 99153fa9..07da3a25 100644 --- a/benchmarks/minires/RESULTS.md +++ b/benchmarks/minires/RESULTS.md @@ -2,8 +2,8 @@ Generated by `run.py`; do not edit by hand. See `cases.py` for the cases, `methods.py` for the methods and their settings, and `scores.py` for what is scored. Each cell is the mean over -the twin experiments (truths), ± its standard error. Rows are grouped by the question they -address, not ranked: which case and which score matters is for the reader to judge. +the twin experiments (truths), ± its standard error. The rows are not ranked: which case +and which score matters is for the reader to judge. ## Case `coarse`: 25 x 25 cells @@ -12,8 +12,7 @@ address, not ranked: which case and which score matters is for the reader to jud | | misfit | RMSE forecast WWCT | logK RMSE | logK corr | logK spread | sor error | sor spread | vo error | vo spread | simulations | |---|---|---|---|---|---|---|---|---|---|---| | prior | 91.7 ± 4.2 | 0.221 ± 0.021 | 0.783 ± 0.019 | 0.0201 ± 0.027 | 0.8 ± 0.0028 | 0.0314 ± 0.0058 | 0.0383 ± 0.00077 | 0.676 ± 0.15 | 1.02 ± 0.023 | | -| *Iterating* | | | | | | | | | | | -| ES-MDA, 4 x 4 | 1.68 ± 0.083 | 0.0847 ± 0.013 | 0.829 ± 0.016 | 0.205 ± 0.037 | 0.526 ± 0.0041 | 0.0349 ± 0.0059 | 0.0221 ± 0.0007 | 0.602 ± 0.094 | 0.469 ± 0.012 | 250 | +| ES-MDA (4 steps, α = 4) | 1.68 ± 0.083 | 0.0847 ± 0.013 | 0.829 ± 0.016 | 0.205 ± 0.037 | 0.526 ± 0.0041 | 0.0349 ± 0.0059 | 0.0221 ± 0.0007 | 0.602 ± 0.094 | 0.469 ± 0.012 | 250 | ## Case `fine`: 50 x 50 cells @@ -22,5 +21,4 @@ address, not ranked: which case and which score matters is for the reader to jud | | misfit | RMSE forecast WWCT | logK RMSE | logK corr | logK spread | sor error | sor spread | vo error | vo spread | simulations | |---|---|---|---|---|---|---|---|---|---|---| | prior | 97 ± 7.6 | 0.243 ± 0.029 | 0.815 ± 0.022 | -0.0673 ± 0.03 | 0.793 ± 0.0038 | 0.0282 ± 0.008 | 0.0402 ± 0.0012 | 0.583 ± 0.12 | 1.03 ± 0.035 | | -| *Iterating* | | | | | | | | | | | -| ES-MDA, 4 x 4 | 1.9 ± 0.21 | 0.0867 ± 0.015 | 0.869 ± 0.035 | 0.179 ± 0.059 | 0.511 ± 0.0037 | 0.0326 ± 0.01 | 0.0253 ± 0.001 | 0.356 ± 0.14 | 0.47 ± 0.015 | 250 | +| ES-MDA (4 steps, α = 4) | 1.9 ± 0.21 | 0.0867 ± 0.015 | 0.869 ± 0.035 | 0.179 ± 0.059 | 0.511 ± 0.0037 | 0.0326 ± 0.01 | 0.0253 ± 0.001 | 0.356 ± 0.14 | 0.47 ± 0.015 | 250 | diff --git a/benchmarks/minires/methods.py b/benchmarks/minires/methods.py index 8e0050c7..c916a6ca 100644 --- a/benchmarks/minires/methods.py +++ b/benchmarks/minires/methods.py @@ -1,4 +1,4 @@ -"""The methods compared: each a label, the question it belongs to, and its ``[dataassim]`` settings. +"""The methods compared: each a label and its ``[dataassim]`` settings. The settings are PET's own, as a config would give them. Unless a row says it is tuned, they are defaults or the textbook choice, the same for every case. @@ -8,8 +8,7 @@ METHODS = [ { - "label": "ES-MDA, 4 x 4", - "group": "Iterating", + "label": "ES-MDA (4 steps, α = 4)", "dataassim": { "scheme": "esmda", "analysis": "approx", diff --git a/benchmarks/minires/results/coarse.json b/benchmarks/minires/results/coarse.json index e32f9ec8..46940c66 100644 --- a/benchmarks/minires/results/coarse.json +++ b/benchmarks/minires/results/coarse.json @@ -6,8 +6,7 @@ "date": "2026-10-27" }, "methods": { - "ES-MDA, 4 x 4": { - "group": "Iterating", + "ES-MDA (4 steps, \u03b1 = 4)": { "runs": [ 250, 250, diff --git a/benchmarks/minires/results/fine.json b/benchmarks/minires/results/fine.json index 5b2c80b8..9dca35fa 100644 --- a/benchmarks/minires/results/fine.json +++ b/benchmarks/minires/results/fine.json @@ -6,8 +6,7 @@ "date": "2026-10-27" }, "methods": { - "ES-MDA, 4 x 4": { - "group": "Iterating", + "ES-MDA (4 steps, \u03b1 = 4)": { "runs": [ 250, 250, diff --git a/benchmarks/minires/run.py b/benchmarks/minires/run.py index c81ae683..81b81646 100644 --- a/benchmarks/minires/run.py +++ b/benchmarks/minires/run.py @@ -117,8 +117,8 @@ def render(results: dict) -> str: "", "Generated by `run.py`; do not edit by hand. See `cases.py` for the cases, `methods.py` for the", "methods and their settings, and `scores.py` for what is scored. Each cell is the mean over", - "the twin experiments (truths), ± its standard error. Rows are grouped by the question they", - "address, not ranked: which case and which score matters is for the reader to judge.", + "the twin experiments (truths), ± its standard error. The rows are not ranked: which case", + "and which score matters is for the reader to judge.", ] for case_name, res in results.items(): meta, runs = res["meta"], res["methods"] @@ -134,11 +134,7 @@ def render(results: dict) -> str: lines += ["", "| | " + " | ".join(COLUMNS.values()) + " |", "|---" * (len(COLUMNS) + 1) + "|"] prior = [cell(any_method["prior"][k]) if k != "runs" else "" for k in COLUMNS] lines += ["| prior | " + " | ".join(prior) + " |"] - group = None for label, r in runs.items(): - if r["group"] != group: - group = r["group"] - lines += [f"| *{group}* |" + " |" * len(COLUMNS)] post = [cell(r["posterior"][k]) if k != "runs" else f"{np.mean(r['runs']):.0f}" for k in COLUMNS] lines += [f"| {label} | " + " | ".join(post) + " |"] return "\n".join(lines) + "\n" @@ -180,7 +176,6 @@ def main(): for m in methods: mine = [d for (mm, _), d in zip(jobs, done) if mm is m] results["methods"][m["label"]] = { - "group": m["group"], "runs": [d["runs"] for d in mine], **{s: {k: [d[s][k] for d in mine] for k in mine[0][s]} for s in ("prior", "posterior")}, } From a106928fff2559faee1ce032ed60d4c1563dcd78 Mon Sep 17 00:00:00 2001 From: patnr Date: Tue, 27 Oct 2026 15:26:19 +0100 Subject: [PATCH 4/6] Benchmark: describe the case on its own terms Co-Authored-By: Claude Opus 5.5 --- benchmarks/minires/cases.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/benchmarks/minires/cases.py b/benchmarks/minires/cases.py index 5d495e4e..e2c170ee 100644 --- a/benchmarks/minires/cases.py +++ b/benchmarks/minires/cases.py @@ -1,7 +1,6 @@ """The cases: one waterflood, at two resolutions, observed over its history and scored on its forecast. -The reservoir is that of ert's ``test-data/ert/waterflood`` (equinor/ert#14598), so that the -two can be set side by side: a 1000 x 1000 x 20 m box in metric units (m, day, bar, mD, cP), +The reservoir is a 1000 x 1000 x 20 m box in metric units (m, day, bar, mD, cP), with one injector at the centre on rate, four producers towards the corners on bottom-hole pressure, 40 report steps of 60 days. The producers' rates, and when they water out, are therefore outcomes of the permeability around them. @@ -34,7 +33,7 @@ INJECTOR = "INJ" DATATYPES = [f"WWCT:{w}" for w in PRODUCERS] + [f"WOPR:{w}" for w in PRODUCERS] + [f"WBHP:{INJECTOR}"] -# Observation error, as (relative, absolute): the larger applies. As ert's case. +# Observation error, as (relative, absolute): the larger applies. OBS_ERROR = {"WWCT": (0.05, 0.02), "WOPR": (0.10, 5.0), "WBHP": (0.01, 1.0)} # The prior. log K about log(200 mD), sd 0.8, spherical variogram of range RANGE metres. From 06512d5708819ee097905be4b09ae1136baedac8 Mon Sep 17 00:00:00 2001 From: patnr Date: Tue, 27 Oct 2026 15:47:58 +0100 Subject: [PATCH 5/6] Benchmark: ES-MDA with adaptive localization, at two thresholds The localization's grid is filled in from the case, since the methods are shared between cases. Co-Authored-By: Claude Opus 5.5 --- benchmarks/minires/RESULTS.md | 4 + benchmarks/minires/methods.py | 18 + benchmarks/minires/results/coarse.json | 848 +++++++++++++++++++++++++ benchmarks/minires/results/fine.json | 392 ++++++++++++ benchmarks/minires/run.py | 5 +- 5 files changed, 1266 insertions(+), 1 deletion(-) diff --git a/benchmarks/minires/RESULTS.md b/benchmarks/minires/RESULTS.md index 07da3a25..fef2a1ff 100644 --- a/benchmarks/minires/RESULTS.md +++ b/benchmarks/minires/RESULTS.md @@ -13,6 +13,8 @@ and which score matters is for the reader to judge. |---|---|---|---|---|---|---|---|---|---|---| | prior | 91.7 ± 4.2 | 0.221 ± 0.021 | 0.783 ± 0.019 | 0.0201 ± 0.027 | 0.8 ± 0.0028 | 0.0314 ± 0.0058 | 0.0383 ± 0.00077 | 0.676 ± 0.15 | 1.02 ± 0.023 | | | ES-MDA (4 steps, α = 4) | 1.68 ± 0.083 | 0.0847 ± 0.013 | 0.829 ± 0.016 | 0.205 ± 0.037 | 0.526 ± 0.0041 | 0.0349 ± 0.0059 | 0.0221 ± 0.0007 | 0.602 ± 0.094 | 0.469 ± 0.012 | 250 | +| ES-MDA (4 steps, α = 4), adaptive localization (universal threshold) | 29.6 ± 1.7 | 0.146 ± 0.02 | 0.779 ± 0.019 | 0.122 ± 0.024 | 0.773 ± 0.0031 | 0.0302 ± 0.0052 | 0.0339 ± 0.001 | 0.531 ± 0.08 | 0.894 ± 0.025 | 250 | +| ES-MDA (4 steps, α = 4), adaptive localization (3σ threshold) | 17.7 ± 0.91 | 0.123 ± 0.018 | 0.777 ± 0.019 | 0.167 ± 0.024 | 0.752 ± 0.0029 | 0.0253 ± 0.0053 | 0.0329 ± 0.00071 | 0.43 ± 0.083 | 0.773 ± 0.022 | 250 | ## Case `fine`: 50 x 50 cells @@ -22,3 +24,5 @@ and which score matters is for the reader to judge. |---|---|---|---|---|---|---|---|---|---|---| | prior | 97 ± 7.6 | 0.243 ± 0.029 | 0.815 ± 0.022 | -0.0673 ± 0.03 | 0.793 ± 0.0038 | 0.0282 ± 0.008 | 0.0402 ± 0.0012 | 0.583 ± 0.12 | 1.03 ± 0.035 | | | ES-MDA (4 steps, α = 4) | 1.9 ± 0.21 | 0.0867 ± 0.015 | 0.869 ± 0.035 | 0.179 ± 0.059 | 0.511 ± 0.0037 | 0.0326 ± 0.01 | 0.0253 ± 0.001 | 0.356 ± 0.14 | 0.47 ± 0.015 | 250 | +| ES-MDA (4 steps, α = 4), adaptive localization (universal threshold) | 47.7 ± 5.4 | 0.2 ± 0.026 | 0.806 ± 0.021 | 0.0307 ± 0.042 | 0.777 ± 0.0026 | 0.0328 ± 0.0082 | 0.0364 ± 0.00088 | 0.476 ± 0.15 | 0.926 ± 0.039 | 250 | +| ES-MDA (4 steps, α = 4), adaptive localization (3σ threshold) | 20.2 ± 3.5 | 0.144 ± 0.021 | 0.794 ± 0.019 | 0.151 ± 0.04 | 0.745 ± 0.0018 | 0.0358 ± 0.0099 | 0.0355 ± 0.0013 | 0.495 ± 0.12 | 0.766 ± 0.035 | 250 | diff --git a/benchmarks/minires/methods.py b/benchmarks/minires/methods.py index c916a6ca..9cdbf92c 100644 --- a/benchmarks/minires/methods.py +++ b/benchmarks/minires/methods.py @@ -15,4 +15,22 @@ "mda": {"tot_assim_steps": 4, "inflation_param": [4, 4, 4, 4]}, }, }, + { + "label": "ES-MDA (4 steps, α = 4), adaptive localization (universal threshold)", + "dataassim": { + "scheme": "esmda", + "analysis": "approx", + "mda": {"tot_assim_steps": 4, "inflation_param": [4, 4, 4, 4]}, + "localization": {"name": "autoadaloc", "threshold": "universal", "type": "hard"}, + }, + }, + { + "label": "ES-MDA (4 steps, α = 4), adaptive localization (3σ threshold)", + "dataassim": { + "scheme": "esmda", + "analysis": "approx", + "mda": {"tot_assim_steps": 4, "inflation_param": [4, 4, 4, 4]}, + "localization": {"name": "autoadaloc", "threshold": "adaptive", "cutoff": 3.0, "type": "hard"}, + }, + }, ] diff --git a/benchmarks/minires/results/coarse.json b/benchmarks/minires/results/coarse.json index 46940c66..496e1658 100644 --- a/benchmarks/minires/results/coarse.json +++ b/benchmarks/minires/results/coarse.json @@ -429,6 +429,854 @@ 0.436561033186781 ] } + }, + "ES-MDA (4 steps, \u03b1 = 4), adaptive localization (universal threshold)": { + "runs": [ + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250 + ], + "prior": { + "misfit": [ + 62.59639601598487, + 123.05196241421295, + 93.22506871276596, + 68.11519772266308, + 84.25907621454918, + 96.40961602542615, + 80.82025957645489, + 66.07005377237218, + 101.54560412060758, + 85.36695599509565, + 84.85681317502099, + 86.77696301351816, + 99.53319335321834, + 105.47695612554558, + 97.09159986755921, + 132.8728233554532, + 71.05869753024243, + 89.23927313744124, + 85.03146870297716, + 119.6601737296945 + ], + "forecast_wwct_rmse": [ + 0.1696800381197985, + 0.19932798619750214, + 0.18030727564909557, + 0.1687946346333842, + 0.20334516472452202, + 0.22090711830163232, + 0.08119475966762553, + 0.09937498257293596, + 0.156574677706107, + 0.19141035949790325, + 0.17710425467402569, + 0.284394647956727, + 0.36722815919437385, + 0.3633631450880431, + 0.0871894186024358, + 0.3796182871950613, + 0.3304530925059456, + 0.2110413313020751, + 0.21456969178190033, + 0.3404644384560941 + ], + "logk_rmse": [ + 0.8861115737970015, + 0.7138871566140654, + 0.7311114337240311, + 0.794578500774992, + 0.7398491140881406, + 0.7151208051832781, + 0.6651517080794027, + 0.66548804681917, + 0.8119847333173705, + 0.9808250445467559, + 0.87589763060281, + 0.7567644686285704, + 0.7664685244639458, + 0.6734759186020055, + 0.720284226310797, + 0.8370809938580748, + 0.8168980923865478, + 0.8378640493333555, + 0.7818831719048845, + 0.8986200732688832 + ], + "logk_corr": [ + 0.1246439649411075, + 0.041320044934769304, + -0.07031257032538189, + 0.006587399183903993, + -0.04834291429886706, + 0.10247516110712568, + 0.05891645085620727, + -0.050468555461754085, + 0.02976338891111694, + -0.002938995933889871, + 0.29384176709012366, + -0.02703682627000836, + 0.0006754033719746696, + 0.10968951060279891, + 0.016901619123219427, + 0.021257305690592972, + -0.1962829661167955, + -0.24074858187770273, + 0.021991764306852377, + 0.20930361234310244 + ], + "logk_spread": [ + 0.8020730046602623, + 0.8081223960932921, + 0.7992709069836804, + 0.80689275565137, + 0.8087260800099462, + 0.776756049914733, + 0.8148302010845695, + 0.7914120818443737, + 0.7742225610152146, + 0.7844666810750466, + 0.7961890309583656, + 0.8095081632860182, + 0.8135303891787904, + 0.8102967016940247, + 0.7932683248624264, + 0.8200563852133691, + 0.7847055424944857, + 0.807270751436195, + 0.7996489668967339, + 0.7994297351777555 + ], + "sor_error": [ + 0.022517471776421466, + 0.03231193249221098, + 0.039741926229656194, + 0.025226634560968048, + 0.02143169540386905, + 0.03331620304428154, + 0.09985184524027202, + 0.014347419674480971, + 0.019789320113076436, + 0.007200528938781425, + 0.0151715061165347, + 0.0006871729371782098, + 0.06846730526575548, + 0.0054726541204520585, + 0.0157363589256444, + 0.04325964194518711, + 0.013508305935118675, + 0.0143029506873453, + 0.07642224738364964, + 0.05846501566022749 + ], + "sor_spread": [ + 0.037483552502312445, + 0.04443805479855501, + 0.03989209833172408, + 0.03959439780626146, + 0.0373651192814694, + 0.042707108594228896, + 0.03811974321077451, + 0.040700857005014156, + 0.03690223223461561, + 0.039779572774622934, + 0.03532836375446026, + 0.033973758704551923, + 0.031136092849106147, + 0.03541348194730274, + 0.04453781527474917, + 0.03976759448274524, + 0.03409458530957603, + 0.03832621780740705, + 0.040225680565728845, + 0.036738472671362744 + ], + "vo_error": [ + 0.0795937058366567, + 0.11494164476815705, + 1.2568667117930428, + 2.278635530475472, + 0.06830189446609758, + 0.48168370560906393, + 0.3255283041707937, + 0.3060383274058749, + 2.3252256889911536, + 1.1003292963327223, + 0.7792432407050995, + 0.6149525736686261, + 0.12998468061398327, + 0.25109279552830355, + 0.9197588811212274, + 0.020342999691860264, + 0.8290998138473791, + 0.7684392614247813, + 0.3509682114471895, + 0.5242844172918861 + ], + "vo_spread": [ + 0.8414351702422311, + 1.1505420162407765, + 1.0526275960392726, + 1.0154658807892343, + 0.9312057839633537, + 1.0379686315534349, + 1.0571140014594451, + 0.9524395430187967, + 1.1211433903472134, + 1.1029772445921056, + 1.093164220791091, + 0.916497074696293, + 0.8558348454302246, + 1.1186998269669504, + 0.9675067812016532, + 0.8692545869158198, + 1.1936075973977103, + 1.0932252757164844, + 0.9942898361540824, + 1.0722824850643657 + ] + }, + "posterior": { + "misfit": [ + 28.98245516831326, + 26.640489138168373, + 36.520244247318246, + 25.122768810703654, + 21.216709372324775, + 30.60775813217205, + 19.482372567334508, + 25.67004701759868, + 41.083636183696534, + 32.24540819205011, + 22.726198603541512, + 23.885579377951625, + 29.913611463117938, + 47.35922409302765, + 32.66534780271965, + 34.130099768273446, + 20.509555424454422, + 35.57100734740797, + 37.676965687168206, + 20.122780371709954 + ], + "forecast_wwct_rmse": [ + 0.1192960037103914, + 0.1308908346417886, + 0.09907894684552583, + 0.07399029057084992, + 0.05214737077369033, + 0.1518223788784104, + 0.045138096634286994, + 0.058733273345566106, + 0.11340622239910982, + 0.10273718778403114, + 0.08790194871376793, + 0.21165315145732744, + 0.2999997075340914, + 0.324983096083049, + 0.048915238769594, + 0.2785914918377289, + 0.26532850629321136, + 0.14872427399524982, + 0.10046988163578885, + 0.2140438974295937 + ], + "logk_rmse": [ + 0.8723265243168787, + 0.71558859869194, + 0.730118775334099, + 0.7744032118294121, + 0.7534755384302392, + 0.6960993757861955, + 0.676586173858072, + 0.6677888464207973, + 0.8114819143704914, + 0.9863397861098405, + 0.8690046265556238, + 0.7507953229389861, + 0.751231641050578, + 0.6741407384767325, + 0.7144435500589994, + 0.8153532007294195, + 0.8080383392387986, + 0.8217986173998528, + 0.7839218745484915, + 0.9087651652717483 + ], + "logk_corr": [ + 0.21562760690959415, + 0.2677830782076458, + -0.0037820079968811744, + 0.18645483751578662, + 0.0027023384642244175, + 0.25493431358918395, + 0.0578894479949427, + -0.015622867852202288, + 0.09644721953782456, + -0.017412765562904182, + 0.2718251463619918, + 0.14588112125158204, + 0.15803154147202222, + 0.13196023908437082, + 0.14787187910986407, + 0.20201346548047264, + 0.023832888313686466, + -0.03306003848951207, + 0.08239081460611448, + 0.26418458662936783 + ], + "logk_spread": [ + 0.7846822827172081, + 0.7778963647302004, + 0.7703632243128027, + 0.7829707729197906, + 0.7849494316087203, + 0.7480418079562016, + 0.7748883077566656, + 0.764856480748323, + 0.7549566571544074, + 0.7632484937536197, + 0.7618369620829599, + 0.778456426488579, + 0.7812320412437296, + 0.7865212542431416, + 0.7642426557022188, + 0.7944431026927856, + 0.7520150081875574, + 0.790131212983226, + 0.7803296828055041, + 0.7552168915495385 + ], + "sor_error": [ + 0.022517471776421466, + 0.015473119551599163, + 0.040856022442424145, + 0.012634147423579545, + 0.02143169540386905, + 0.0468378138041686, + 0.073097981068167, + 0.0035472004384132505, + 0.019789320113076436, + 0.008405533616043959, + 0.026359081768927733, + 0.012749755875471253, + 0.06155557675451226, + 0.01666633967260489, + 0.0157363589256444, + 0.0025461771761337015, + 0.029184789140362127, + 0.030228933951816056, + 0.07642224738364964, + 0.06875093506945718 + ], + "sor_spread": [ + 0.037483552502312445, + 0.03586172804839687, + 0.03881952159155224, + 0.03496513620081496, + 0.0373651192814694, + 0.03589946968605953, + 0.03243347845859705, + 0.03282075753671645, + 0.03690223223461561, + 0.03217489630803584, + 0.030029512516125827, + 0.030854686895663647, + 0.02666634485684442, + 0.026905139849190813, + 0.04453781527474917, + 0.03227310839905577, + 0.02978050010591354, + 0.03383968526731101, + 0.040225680565728845, + 0.02889499733467563 + ], + "vo_error": [ + 0.0795937058366567, + 0.32753056262982927, + 1.1495119807289225, + 1.0842503388391727, + 0.1775751663566414, + 0.31846819315149233, + 0.4830259580320333, + 0.6649818100018461, + 0.8768466997370292, + 0.8015920293728342, + 0.5444248646991197, + 0.968023476398228, + 0.4932374992712232, + 0.13004694059325228, + 0.9197588811212274, + 0.04161467508678918, + 0.8290998138473791, + 0.2596047776628927, + 0.3509682114471895, + 0.12341210352477638 + ], + "vo_spread": [ + 0.8414351702422311, + 0.9294268808251006, + 1.0109015474730771, + 0.8569868527044845, + 0.8043347141221031, + 0.9709225047072543, + 0.803994714877551, + 0.8039474776964238, + 0.9235853530202788, + 0.9353061995774441, + 0.8106538289540101, + 0.8208281690344851, + 0.6596616731777826, + 0.8799817301935094, + 0.9675067812016532, + 0.8367163823628767, + 1.1936075973977103, + 1.0012886084537571, + 0.9942898361540824, + 0.8378905220788019 + ] + } + }, + "ES-MDA (4 steps, \u03b1 = 4), adaptive localization (3\u03c3 threshold)": { + "runs": [ + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250 + ], + "prior": { + "misfit": [ + 62.59639601598487, + 123.05196241421295, + 93.22506871276596, + 68.11519772266308, + 84.25907621454918, + 96.40961602542615, + 80.82025957645489, + 66.07005377237218, + 101.54560412060758, + 85.36695599509565, + 84.85681317502099, + 86.77696301351816, + 99.53319335321834, + 105.47695612554558, + 97.09159986755921, + 132.8728233554532, + 71.05869753024243, + 89.23927313744124, + 85.03146870297716, + 119.6601737296945 + ], + "forecast_wwct_rmse": [ + 0.1696800381197985, + 0.19932798619750214, + 0.18030727564909557, + 0.1687946346333842, + 0.20334516472452202, + 0.22090711830163232, + 0.08119475966762553, + 0.09937498257293596, + 0.156574677706107, + 0.19141035949790325, + 0.17710425467402569, + 0.284394647956727, + 0.36722815919437385, + 0.3633631450880431, + 0.0871894186024358, + 0.3796182871950613, + 0.3304530925059456, + 0.2110413313020751, + 0.21456969178190033, + 0.3404644384560941 + ], + "logk_rmse": [ + 0.8861115737970015, + 0.7138871566140654, + 0.7311114337240311, + 0.794578500774992, + 0.7398491140881406, + 0.7151208051832781, + 0.6651517080794027, + 0.66548804681917, + 0.8119847333173705, + 0.9808250445467559, + 0.87589763060281, + 0.7567644686285704, + 0.7664685244639458, + 0.6734759186020055, + 0.720284226310797, + 0.8370809938580748, + 0.8168980923865478, + 0.8378640493333555, + 0.7818831719048845, + 0.8986200732688832 + ], + "logk_corr": [ + 0.1246439649411075, + 0.041320044934769304, + -0.07031257032538189, + 0.006587399183903993, + -0.04834291429886706, + 0.10247516110712568, + 0.05891645085620727, + -0.050468555461754085, + 0.02976338891111694, + -0.002938995933889871, + 0.29384176709012366, + -0.02703682627000836, + 0.0006754033719746696, + 0.10968951060279891, + 0.016901619123219427, + 0.021257305690592972, + -0.1962829661167955, + -0.24074858187770273, + 0.021991764306852377, + 0.20930361234310244 + ], + "logk_spread": [ + 0.8020730046602623, + 0.8081223960932921, + 0.7992709069836804, + 0.80689275565137, + 0.8087260800099462, + 0.776756049914733, + 0.8148302010845695, + 0.7914120818443737, + 0.7742225610152146, + 0.7844666810750466, + 0.7961890309583656, + 0.8095081632860182, + 0.8135303891787904, + 0.8102967016940247, + 0.7932683248624264, + 0.8200563852133691, + 0.7847055424944857, + 0.807270751436195, + 0.7996489668967339, + 0.7994297351777555 + ], + "sor_error": [ + 0.022517471776421466, + 0.03231193249221098, + 0.039741926229656194, + 0.025226634560968048, + 0.02143169540386905, + 0.03331620304428154, + 0.09985184524027202, + 0.014347419674480971, + 0.019789320113076436, + 0.007200528938781425, + 0.0151715061165347, + 0.0006871729371782098, + 0.06846730526575548, + 0.0054726541204520585, + 0.0157363589256444, + 0.04325964194518711, + 0.013508305935118675, + 0.0143029506873453, + 0.07642224738364964, + 0.05846501566022749 + ], + "sor_spread": [ + 0.037483552502312445, + 0.04443805479855501, + 0.03989209833172408, + 0.03959439780626146, + 0.0373651192814694, + 0.042707108594228896, + 0.03811974321077451, + 0.040700857005014156, + 0.03690223223461561, + 0.039779572774622934, + 0.03532836375446026, + 0.033973758704551923, + 0.031136092849106147, + 0.03541348194730274, + 0.04453781527474917, + 0.03976759448274524, + 0.03409458530957603, + 0.03832621780740705, + 0.040225680565728845, + 0.036738472671362744 + ], + "vo_error": [ + 0.0795937058366567, + 0.11494164476815705, + 1.2568667117930428, + 2.278635530475472, + 0.06830189446609758, + 0.48168370560906393, + 0.3255283041707937, + 0.3060383274058749, + 2.3252256889911536, + 1.1003292963327223, + 0.7792432407050995, + 0.6149525736686261, + 0.12998468061398327, + 0.25109279552830355, + 0.9197588811212274, + 0.020342999691860264, + 0.8290998138473791, + 0.7684392614247813, + 0.3509682114471895, + 0.5242844172918861 + ], + "vo_spread": [ + 0.8414351702422311, + 1.1505420162407765, + 1.0526275960392726, + 1.0154658807892343, + 0.9312057839633537, + 1.0379686315534349, + 1.0571140014594451, + 0.9524395430187967, + 1.1211433903472134, + 1.1029772445921056, + 1.093164220791091, + 0.916497074696293, + 0.8558348454302246, + 1.1186998269669504, + 0.9675067812016532, + 0.8692545869158198, + 1.1936075973977103, + 1.0932252757164844, + 0.9942898361540824, + 1.0722824850643657 + ] + }, + "posterior": { + "misfit": [ + 13.267772053463455, + 15.437561753727605, + 14.485685909676786, + 18.175236552524925, + 15.076190748607745, + 18.12992003502848, + 12.135785660314555, + 18.57977550672438, + 21.620314854150553, + 23.3079025623003, + 20.27494158293104, + 12.932899696772923, + 25.920114613259063, + 23.153973785318804, + 15.210569727401854, + 16.481277212000528, + 14.165528581878588, + 21.05715721006404, + 21.78011234783619, + 13.488065341648424 + ], + "forecast_wwct_rmse": [ + 0.07022273177785818, + 0.09194662627330161, + 0.06212161734296693, + 0.06627431995368546, + 0.05134904926815731, + 0.11940725252713395, + 0.047747544355780644, + 0.05695603921908428, + 0.07142505192722504, + 0.11097819849913194, + 0.07763756327850505, + 0.14108939356919567, + 0.284429081277272, + 0.29289495772820623, + 0.0565097349500302, + 0.24167015462672892, + 0.2304022470851044, + 0.11727310895049929, + 0.08504591212081876, + 0.189196185593185 + ], + "logk_rmse": [ + 0.8679279223386982, + 0.7237561828895215, + 0.7443142070801838, + 0.7532854937001076, + 0.7489361875683163, + 0.6987707229401375, + 0.6796246190692058, + 0.667805766476269, + 0.8050363255300188, + 0.9770506304942479, + 0.8632667251725071, + 0.7200711181542674, + 0.7153300533690374, + 0.6881153286836268, + 0.7222495931641263, + 0.8041332304573007, + 0.8139710942532719, + 0.8139107807131648, + 0.8083971474135555, + 0.9189074697566602 + ], + "logk_corr": [ + 0.23459615275438572, + 0.2808822427029181, + 0.055030134248308486, + 0.295336904950608, + 0.0796043145384268, + 0.2679556627341797, + 0.05355566818845626, + 0.04692596176084399, + 0.16804171876937202, + 0.0675169916814269, + 0.295654016550157, + 0.27954744835273765, + 0.313568297927656, + 0.10389786312204491, + 0.15992844398864067, + 0.22625847577332173, + 0.036060660822467616, + 0.06921056577805025, + 0.04449802115831381, + 0.2586312223090657 + ], + "logk_spread": [ + 0.7679382969733638, + 0.7525152448339636, + 0.7467668101405357, + 0.7620787310025784, + 0.7736416605064841, + 0.7316398101108343, + 0.7547818455525891, + 0.7427159498962903, + 0.7376792638721524, + 0.7363810486997412, + 0.7450360377412326, + 0.7626556688825075, + 0.7633806152539336, + 0.7652487752765477, + 0.7438380423527782, + 0.7639675983699045, + 0.7347133062422383, + 0.7639101318400171, + 0.7548934499882485, + 0.7363629187148393 + ], + "sor_error": [ + 0.005384660911569916, + 0.015473119551599163, + 0.011842897248335427, + 0.0030350558441404607, + 0.02143169540386905, + 0.05315410496862322, + 0.073097981068167, + 0.003054470425185668, + 0.0008432498608358652, + 0.013509732813183212, + 0.01638208716611189, + 0.01818228503616151, + 0.048976900444017424, + 0.01666633967260489, + 0.006227998041014066, + 0.0025461771761337015, + 0.029184789140362127, + 0.030228933951816056, + 0.07219609395981247, + 0.06364506843186313 + ], + "sor_spread": [ + 0.0352043640351695, + 0.03586172804839687, + 0.03471079504888823, + 0.0346767305214905, + 0.0373651192814694, + 0.03995676905357032, + 0.03243347845859705, + 0.03394580288584681, + 0.03186858080542909, + 0.03255264731899628, + 0.033019403682582285, + 0.03093368532649966, + 0.026403985763609188, + 0.026905139849190813, + 0.03079535355605519, + 0.03227310839905577, + 0.02978050010591354, + 0.03383968526731101, + 0.03333631061871692, + 0.031345808539444804 + ], + "vo_error": [ + 0.10032514113068469, + 0.21495627430715292, + 0.14247533750074926, + 1.5848246660167584, + 0.06623293877634673, + 0.02140197617553241, + 0.4025140122937678, + 0.6190737491068372, + 0.6401675850068917, + 0.4627300394987177, + 0.29793320946093527, + 0.36455922110120476, + 0.3384575696211929, + 0.32492396736897966, + 0.2249327631281206, + 0.050809847724706536, + 1.0158473893062059, + 0.5506478132511905, + 0.543037642747799, + 0.6412731036436385 + ], + "vo_spread": [ + 0.7720810352415833, + 0.8682864791123279, + 0.7479073999428244, + 0.9028242686934165, + 0.774250154128175, + 0.8232569148080826, + 0.8168676337145327, + 0.6530370192609716, + 0.724160974378875, + 0.8029605977909464, + 0.7303524250065353, + 0.7121160764455419, + 0.6800066901337405, + 0.7204622210709786, + 0.627156625287774, + 0.7382627558228716, + 1.0513215129686218, + 0.8123752691550642, + 0.8589440666013441, + 0.640084880584695 + ] + } } } } diff --git a/benchmarks/minires/results/fine.json b/benchmarks/minires/results/fine.json index 9dca35fa..4bb2b7ef 100644 --- a/benchmarks/minires/results/fine.json +++ b/benchmarks/minires/results/fine.json @@ -201,6 +201,398 @@ 0.5019287799827108 ] } + }, + "ES-MDA (4 steps, \u03b1 = 4), adaptive localization (universal threshold)": { + "runs": [ + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250 + ], + "prior": { + "misfit": [ + 76.6857082028925, + 118.48859535370278, + 125.88068474763797, + 88.2290594753218, + 120.13401879681457, + 71.19485589384213, + 93.81381539692943, + 81.58416139246368 + ], + "forecast_wwct_rmse": [ + 0.26855951538628003, + 0.3710015080260862, + 0.2823521538778289, + 0.258549119921913, + 0.29548286093911674, + 0.12655814500299795, + 0.1914157610497057, + 0.14985517995795167 + ], + "logk_rmse": [ + 0.862081806419521, + 0.860747552427497, + 0.8814335507422228, + 0.8509947054495131, + 0.8418541928350743, + 0.7313398522920637, + 0.7298452045306284, + 0.7594852129514473 + ], + "logk_corr": [ + -0.016450365365537036, + -0.14702663951406622, + -0.08504505935224134, + 0.05992274845787901, + -0.09942851429124891, + -0.1345454128985617, + 0.03618745715727143, + -0.15229398835963354 + ], + "logk_spread": [ + 0.7966141389511963, + 0.7989257773797538, + 0.802150283708243, + 0.8046550340635544, + 0.796685800426243, + 0.778724626542447, + 0.7930365195482185, + 0.7752310117323408 + ], + "sor_error": [ + 0.034576287836562164, + 0.07750867354115154, + 0.001919478814194464, + 0.032093631315579296, + 0.012772658013380705, + 0.02436307210664651, + 0.024252339511175364, + 0.01789218898357159 + ], + "sor_spread": [ + 0.03823958904277204, + 0.036130397433477894, + 0.042194407622493546, + 0.04271449501189857, + 0.04399000371765547, + 0.042128076892931896, + 0.034526552731212405, + 0.04152268006305935 + ], + "vo_error": [ + 1.2156756183526172, + 0.7487784775528086, + 0.4811975872810086, + 0.006578025623270811, + 0.6644893701416903, + 0.5505227441668827, + 0.5861382992353326, + 0.4140336539118179 + ], + "vo_spread": [ + 1.0749710722220533, + 0.9922316067216777, + 0.9701440255463729, + 0.9782281932782648, + 1.2395528142331786, + 0.9280871834029033, + 1.0583006464310203, + 0.9696631777900684 + ] + }, + "posterior": { + "misfit": [ + 43.98481629093686, + 63.26576800653424, + 16.403032434581824, + 49.92134319001806, + 56.52855548885102, + 54.76046089839836, + 37.25089801428513, + 59.43203588343219 + ], + "forecast_wwct_rmse": [ + 0.2342654221080745, + 0.30895162973246715, + 0.19125144856985649, + 0.24596817563736845, + 0.2608621619694931, + 0.11753050095989737, + 0.10936033579091593, + 0.13343951715344343 + ], + "logk_rmse": [ + 0.8558831824300456, + 0.853825322415626, + 0.8485116190422426, + 0.8497174645365726, + 0.8319631297956654, + 0.7334196168904353, + 0.7186299175160985, + 0.7565442539458359 + ], + "logk_corr": [ + 0.0706289461461521, + -0.015182531158826765, + 0.1846489927531531, + 0.08322038621946326, + 0.015746298835186814, + -0.11701363824161891, + 0.16101369658300574, + -0.13769711934421738 + ], + "logk_spread": [ + 0.7773033562068479, + 0.7836358551891279, + 0.773903132784877, + 0.7868724110092317, + 0.7849087614624058, + 0.774736154430035, + 0.7711451615421901, + 0.7660264629958794 + ], + "sor_error": [ + 0.034576287836562164, + 0.07750867354115154, + 0.008051585132393263, + 0.04049998206309252, + 0.009663579743787204, + 0.049730858866636696, + 0.024252339511175364, + 0.01789218898357159 + ], + "sor_spread": [ + 0.03823958904277204, + 0.036130397433477894, + 0.03597912078866424, + 0.03565228580033325, + 0.03339204695281016, + 0.035572269582703465, + 0.034526552731212405, + 0.04152268006305935 + ], + "vo_error": [ + 1.2156756183526172, + 0.05453730094539999, + 0.12601795862373066, + 0.006578025623270811, + 0.7226942750278376, + 0.6839099226751779, + 0.5861382992353326, + 0.4140336539118179 + ], + "vo_spread": [ + 1.0749710722220533, + 0.8806052253116802, + 0.7855653005285126, + 0.9782281932782648, + 0.8337608602007149, + 0.8299789276712952, + 1.0583006464310203, + 0.9696631777900684 + ] + } + }, + "ES-MDA (4 steps, \u03b1 = 4), adaptive localization (3\u03c3 threshold)": { + "runs": [ + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250 + ], + "prior": { + "misfit": [ + 76.6857082028925, + 118.48859535370278, + 125.88068474763797, + 88.2290594753218, + 120.13401879681457, + 71.19485589384213, + 93.81381539692943, + 81.58416139246368 + ], + "forecast_wwct_rmse": [ + 0.26855951538628003, + 0.3710015080260862, + 0.2823521538778289, + 0.258549119921913, + 0.29548286093911674, + 0.12655814500299795, + 0.1914157610497057, + 0.14985517995795167 + ], + "logk_rmse": [ + 0.862081806419521, + 0.860747552427497, + 0.8814335507422228, + 0.8509947054495131, + 0.8418541928350743, + 0.7313398522920637, + 0.7298452045306284, + 0.7594852129514473 + ], + "logk_corr": [ + -0.016450365365537036, + -0.14702663951406622, + -0.08504505935224134, + 0.05992274845787901, + -0.09942851429124891, + -0.1345454128985617, + 0.03618745715727143, + -0.15229398835963354 + ], + "logk_spread": [ + 0.7966141389511963, + 0.7989257773797538, + 0.802150283708243, + 0.8046550340635544, + 0.796685800426243, + 0.778724626542447, + 0.7930365195482185, + 0.7752310117323408 + ], + "sor_error": [ + 0.034576287836562164, + 0.07750867354115154, + 0.001919478814194464, + 0.032093631315579296, + 0.012772658013380705, + 0.02436307210664651, + 0.024252339511175364, + 0.01789218898357159 + ], + "sor_spread": [ + 0.03823958904277204, + 0.036130397433477894, + 0.042194407622493546, + 0.04271449501189857, + 0.04399000371765547, + 0.042128076892931896, + 0.034526552731212405, + 0.04152268006305935 + ], + "vo_error": [ + 1.2156756183526172, + 0.7487784775528086, + 0.4811975872810086, + 0.006578025623270811, + 0.6644893701416903, + 0.5505227441668827, + 0.5861382992353326, + 0.4140336539118179 + ], + "vo_spread": [ + 1.0749710722220533, + 0.9922316067216777, + 0.9701440255463729, + 0.9782281932782648, + 1.2395528142331786, + 0.9280871834029033, + 1.0583006464310203, + 0.9696631777900684 + ] + }, + "posterior": { + "misfit": [ + 19.73792784596038, + 16.31644729282754, + 10.601250359591925, + 14.41826185679646, + 18.675066923927876, + 14.923363048269408, + 24.31984936323121, + 42.41184840844644 + ], + "forecast_wwct_rmse": [ + 0.1910663542153361, + 0.2005244163369256, + 0.1683861568128584, + 0.14784199263657483, + 0.20602046702041743, + 0.04023239094936644, + 0.09932670705726565, + 0.09979149857329582 + ], + "logk_rmse": [ + 0.8545716581457826, + 0.841788433576811, + 0.841683867783141, + 0.8186629079250293, + 0.7938198016340672, + 0.7223265439968245, + 0.7222728858138057, + 0.7565736401115633 + ], + "logk_corr": [ + 0.1102656975145053, + 0.17073177539765103, + 0.24952298347830223, + 0.2700452445797318, + 0.21783716186282134, + 0.0945573953457165, + 0.18491156179419574, + -0.08671293154984779 + ], + "logk_spread": [ + 0.7514130116256484, + 0.7480175695313376, + 0.7419574636627498, + 0.7390278988385468, + 0.7494010556775629, + 0.7420782756808155, + 0.7479245196431874, + 0.7382799188189086 + ], + "sor_error": [ + 0.07195871670575449, + 0.07750867354115154, + 0.001919478814194464, + 0.03510535501525666, + 0.009663579743787204, + 0.047561704502575075, + 0.025058834206631242, + 0.017559747631029854 + ], + "sor_spread": [ + 0.030243799222457072, + 0.036130397433477894, + 0.042194407622493546, + 0.036523583835101285, + 0.03339204695281016, + 0.03611578136434504, + 0.03226302642807711, + 0.037378038423759266 + ], + "vo_error": [ + 0.8343723377232175, + 0.24671580983949326, + 0.18825285397403935, + 0.5990556188003646, + 0.24061972191810543, + 0.10840506852377096, + 0.712827345951081, + 1.0306230518055113 + ], + "vo_spread": [ + 0.9135488735443291, + 0.8513361179765446, + 0.6024468368747081, + 0.7770145278046457, + 0.7249841558766282, + 0.7419569255377249, + 0.6934846538571406, + 0.8226826895749983 + ] + } } } } diff --git a/benchmarks/minires/run.py b/benchmarks/minires/run.py index 81b81646..0fedee05 100644 --- a/benchmarks/minires/run.py +++ b/benchmarks/minires/run.py @@ -69,8 +69,11 @@ def experiment(case_name: str, method: dict, i: int, ne: int) -> dict: variance = [[str(["abs", float(s**2)]) for s in row] for row in tw.std] pd.DataFrame(variance, index=index, columns=DATATYPES).to_csv("var.csv") + dataassim = {**COMMON, **method["dataassim"], "data": "data.csv", "datavar": "var.csv", "savefolder": "Results"} + if "localization" in dataassim: # the grid is the case's, not the method's + dataassim["localization"] = {"field": [case.n, case.n], **dataassim["localization"]} keys_da, keys_sim, keys_en = normalize( - {**COMMON, **method["dataassim"], "data": "data.csv", "datavar": "var.csv", "savefolder": "Results"}, + dataassim, case.simulator(OBS_STEPS), case.ensemble(ne, ENSEMBLE_SEED + i, "priormean.npz"), ) From d9b9decb0f8a945be6ff11cfb229e9edbcc84782 Mon Sep 17 00:00:00 2001 From: patnr Date: Tue, 27 Oct 2026 15:50:47 +0100 Subject: [PATCH 6/6] Benchmark smoke test: allow a localization to leave a tiny ensemble unchanged Co-Authored-By: Claude Opus 5.5 --- tests/test_benchmark_minires.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_benchmark_minires.py b/tests/test_benchmark_minires.py index 80d7f4cf..18ce4980 100644 --- a/tests/test_benchmark_minires.py +++ b/tests/test_benchmark_minires.py @@ -20,4 +20,5 @@ def test_a_twin_experiment_runs_and_scores(method): assert out["runs"] > 0 for stage in ("prior", "posterior"): assert all(np.isfinite(v) for v in out[stage].values()), out[stage] - assert out["posterior"]["misfit"] < out["prior"]["misfit"] + # Not strictly: with so few members, a localization may find every correlation to be noise + assert out["posterior"]["misfit"] <= out["prior"]["misfit"]