diff --git a/benchmarks/minires/RESULTS.md b/benchmarks/minires/RESULTS.md index fef2a1f..a4f0acd 100644 --- a/benchmarks/minires/RESULTS.md +++ b/benchmarks/minires/RESULTS.md @@ -7,22 +7,24 @@ and which score matters is for the reader to judge. ## Case `coarse`: 25 x 25 cells -20 truths, 50 members; history of 16 of 40 steps, observed at steps [4, 8, 12, 16]. PET at `b731be7`, 2026-10-27. +20 truths, 50 members; history of 16 of 40 steps, observed at steps [4, 8, 12, 16]. PET at `2fe3aba`, 2026-10-28. -| | misfit | RMSE forecast WWCT | logK RMSE | logK corr | logK spread | sor error | sor spread | vo error | vo spread | simulations | -|---|---|---|---|---|---|---|---|---|---|---| -| prior | 91.7 ± 4.2 | 0.221 ± 0.021 | 0.783 ± 0.019 | 0.0201 ± 0.027 | 0.8 ± 0.0028 | 0.0314 ± 0.0058 | 0.0383 ± 0.00077 | 0.676 ± 0.15 | 1.02 ± 0.023 | | -| ES-MDA (4 steps, α = 4) | 1.68 ± 0.083 | 0.0847 ± 0.013 | 0.829 ± 0.016 | 0.205 ± 0.037 | 0.526 ± 0.0041 | 0.0349 ± 0.0059 | 0.0221 ± 0.0007 | 0.602 ± 0.094 | 0.469 ± 0.012 | 250 | -| ES-MDA (4 steps, α = 4), adaptive localization (universal threshold) | 29.6 ± 1.7 | 0.146 ± 0.02 | 0.779 ± 0.019 | 0.122 ± 0.024 | 0.773 ± 0.0031 | 0.0302 ± 0.0052 | 0.0339 ± 0.001 | 0.531 ± 0.08 | 0.894 ± 0.025 | 250 | -| ES-MDA (4 steps, α = 4), adaptive localization (3σ threshold) | 17.7 ± 0.91 | 0.123 ± 0.018 | 0.777 ± 0.019 | 0.167 ± 0.024 | 0.752 ± 0.0029 | 0.0253 ± 0.0053 | 0.0329 ± 0.00071 | 0.43 ± 0.083 | 0.773 ± 0.022 | 250 | +| | misfit | RMSE forecast WWCT | logK RMSE | logK corr | logK spread | sor error | sor spread | simulations | +|---|---|---|---|---|---|---|---|---| +| prior | 99.2 ± 5 | 0.242 ± 0.023 | 0.78 ± 0.026 | 0.0299 ± 0.036 | 0.802 ± 0.0037 | 0.0314 ± 0.0058 | 0.0383 ± 0.00077 | | +| ES-MDA (4 steps, α = 4) | 1.71 ± 0.095 | 0.092 ± 0.015 | 0.763 ± 0.029 | 0.299 ± 0.043 | 0.494 ± 0.0058 | 0.0362 ± 0.0058 | 0.0212 ± 0.00071 | 250 | +| ES-MDA (4 steps, α = 4), adaptive localization (universal threshold) | 20.6 ± 1.9 | 0.153 ± 0.023 | 0.758 ± 0.025 | 0.21 ± 0.031 | 0.744 ± 0.0038 | 0.0257 ± 0.0049 | 0.0327 ± 0.00083 | 250 | +| ES-MDA (4 steps, α = 4), adaptive localization (3σ threshold) | 13.5 ± 1.2 | 0.124 ± 0.019 | 0.75 ± 0.024 | 0.262 ± 0.032 | 0.719 ± 0.0042 | 0.0288 ± 0.0051 | 0.0316 ± 0.00071 | 250 | +| ES-MDA (4 steps, α = 4), distance localization (Gaspari-Cohn, 300 m) | 8.96 ± 0.96 | 0.101 ± 0.017 | 0.718 ± 0.023 | 0.337 ± 0.039 | 0.678 ± 0.0035 | 0.0303 ± 0.0046 | 0.0236 ± 0.00071 | 250 | ## Case `fine`: 50 x 50 cells -8 truths, 50 members; history of 16 of 40 steps, observed at steps [4, 8, 12, 16]. PET at `b731be7`, 2026-10-27. +8 truths, 50 members; history of 16 of 40 steps, observed at steps [4, 8, 12, 16]. PET at `2fe3aba`, 2026-10-28. -| | misfit | RMSE forecast WWCT | logK RMSE | logK corr | logK spread | sor error | sor spread | vo error | vo spread | simulations | -|---|---|---|---|---|---|---|---|---|---|---| -| prior | 97 ± 7.6 | 0.243 ± 0.029 | 0.815 ± 0.022 | -0.0673 ± 0.03 | 0.793 ± 0.0038 | 0.0282 ± 0.008 | 0.0402 ± 0.0012 | 0.583 ± 0.12 | 1.03 ± 0.035 | | -| ES-MDA (4 steps, α = 4) | 1.9 ± 0.21 | 0.0867 ± 0.015 | 0.869 ± 0.035 | 0.179 ± 0.059 | 0.511 ± 0.0037 | 0.0326 ± 0.01 | 0.0253 ± 0.001 | 0.356 ± 0.14 | 0.47 ± 0.015 | 250 | -| ES-MDA (4 steps, α = 4), adaptive localization (universal threshold) | 47.7 ± 5.4 | 0.2 ± 0.026 | 0.806 ± 0.021 | 0.0307 ± 0.042 | 0.777 ± 0.0026 | 0.0328 ± 0.0082 | 0.0364 ± 0.00088 | 0.476 ± 0.15 | 0.926 ± 0.039 | 250 | -| ES-MDA (4 steps, α = 4), adaptive localization (3σ threshold) | 20.2 ± 3.5 | 0.144 ± 0.021 | 0.794 ± 0.019 | 0.151 ± 0.04 | 0.745 ± 0.0018 | 0.0358 ± 0.0099 | 0.0355 ± 0.0013 | 0.495 ± 0.12 | 0.766 ± 0.035 | 250 | +| | misfit | RMSE forecast WWCT | logK RMSE | logK corr | logK spread | sor error | sor spread | simulations | +|---|---|---|---|---|---|---|---|---| +| prior | 99 ± 5.2 | 0.247 ± 0.035 | 0.817 ± 0.029 | -0.0719 ± 0.047 | 0.793 ± 0.0058 | 0.0282 ± 0.008 | 0.0402 ± 0.0012 | | +| ES-MDA (4 steps, α = 4) | 1.83 ± 0.16 | 0.0984 ± 0.02 | 0.793 ± 0.059 | 0.355 ± 0.07 | 0.489 ± 0.0055 | 0.0308 ± 0.01 | 0.0231 ± 0.00081 | 250 | +| ES-MDA (4 steps, α = 4), adaptive localization (universal threshold) | 33.3 ± 3.6 | 0.173 ± 0.039 | 0.793 ± 0.027 | 0.153 ± 0.032 | 0.752 ± 0.0046 | 0.0306 ± 0.0089 | 0.0342 ± 0.0024 | 250 | +| ES-MDA (4 steps, α = 4), adaptive localization (3σ threshold) | 16.4 ± 2.4 | 0.158 ± 0.032 | 0.761 ± 0.03 | 0.29 ± 0.043 | 0.711 ± 0.0039 | 0.0335 ± 0.0086 | 0.0317 ± 0.0014 | 250 | +| ES-MDA (4 steps, α = 4), distance localization (Gaspari-Cohn, 300 m) | 12.4 ± 2.4 | 0.112 ± 0.029 | 0.732 ± 0.036 | 0.402 ± 0.047 | 0.675 ± 0.0048 | 0.0338 ± 0.011 | 0.0249 ± 0.00089 | 250 | diff --git a/benchmarks/minires/cases.py b/benchmarks/minires/cases.py index e2c170e..96b767a 100644 --- a/benchmarks/minires/cases.py +++ b/benchmarks/minires/cases.py @@ -5,8 +5,8 @@ pressure, 40 report steps of 60 days. The producers' rates, and when they water out, are therefore outcomes of the permeability around them. -The unknowns are the log-permeability field and two scalars of the fluid, the residual oil -saturation ``sor`` and the oil viscosity ``vo``. Each twin experiment draws its truth from +The unknowns are the log-permeability field and a scalar of the fluid, the residual oil +saturation ``sor``; the oil viscosity is known. Each twin experiment draws its truth from the same prior as the ensemble, simulates it to the end, and perturbs the history part into observations; the forecast part is what the posterior is scored against, unseen. """ @@ -36,12 +36,12 @@ # Observation error, as (relative, absolute): the larger applies. OBS_ERROR = {"WWCT": (0.05, 0.02), "WOPR": (0.10, 5.0), "WBHP": (0.01, 1.0)} -# The prior. log K about log(200 mD), sd 0.8, spherical variogram of range RANGE metres. +# The prior. log K about log(200 mD), sd 0.8, with a smooth variogram of range RANGE metres: +# PET's "exp" is exp(-3 (h/range)^1.9), i.e. all but Gaussian. LOGK_MEAN, LOGK_SD, RANGE = np.log(200.0), 0.8, 300.0 SCALARS = { # mean sd limits "sor": (0.22, 0.04, [0.05, 0.40]), - "vo": (5.0, 1.0, [1.5, 10.0]), } @@ -63,7 +63,7 @@ def model(self) -> dict: return { "Lx": LX, "Ly": LX, "Nx": self.n, "Ny": self.n, "por": 0.2, "ct": 1e-5, "cdarcy": 0.008527, - "fluid": {"vw": 1.0, "vo": SCALARS["vo"][0], "swc": 0.2, "sor": SCALARS["sor"][0]}, + "fluid": {"vw": 1.0, "vo": 5.0, "swc": 0.2, "sor": SCALARS["sor"][0]}, "wells": wells, } @@ -81,7 +81,7 @@ def ensemble(self, ne: int, seed: int, mean_file: str) -> dict: "ne": ne, "seed": seed, "disable_tqdm": True, "save_prior": False, "state": ["permx", *SCALARS], "prior_permx": { - "vario": "sph", "mean": mean_file, "var": LOGK_SD**2, + "vario": "exp", "mean": mean_file, "var": LOGK_SD**2, "range": RANGE / self.dx, "aniso": 1.0, "angle": 0.0, "grid": [self.n, self.n, 1], }, } @@ -89,6 +89,16 @@ def ensemble(self, ne: int, seed: int, mean_file: str) -> dict: ens[f"prior_{name}"] = {"mean": mean, "var": sd**2, "limits": limits} return ens + def distance_entries(self, radius: float) -> list: + """Distance-localization entries: a taper of ``radius`` metres on the field, at each datum's well.""" + cells = {w["name"]: [int(w["xy"][0] // self.dx), int(w["xy"][1] // self.dx)] for w in self.model()["wells"]} + return [ + {"x": x, "y": y, "z": 0, "radius": int(round(radius / self.dx)), "z_range": ":", "aniso": 1.0, + "rotation": 0.0, "data_type": d, "time": "*", "param": "permx"} + for d in DATATYPES + for x, y in [cells[d.split(":")[1]]] + ] + def write_prior_mean(self, path: str) -> None: np.savez(path, permx=np.full(self.n**2, LOGK_MEAN)) diff --git a/benchmarks/minires/methods.py b/benchmarks/minires/methods.py index 9cdbf92..725d063 100644 --- a/benchmarks/minires/methods.py +++ b/benchmarks/minires/methods.py @@ -1,7 +1,9 @@ """The methods compared: each a label and its ``[dataassim]`` settings. -The settings are PET's own, as a config would give them. Unless a row says it is tuned, -they are defaults or the textbook choice, the same for every case. +The settings are PET's own, as a config would give them, except that a distance localization +gives its ``radius`` in metres: ``run.py`` places a taper of that radius at each datum's well, +on the case's grid. Unless a row says it is tuned, the settings are defaults or the textbook +choice, the same for every case. """ COMMON = {"energy": 99.0, "obsname": "steps"} @@ -33,4 +35,14 @@ "localization": {"name": "autoadaloc", "threshold": "adaptive", "cutoff": 3.0, "type": "hard"}, }, }, + { + # The radius, in metres, is the prior's correlation range; the scalars have no position + "label": "ES-MDA (4 steps, α = 4), distance localization (Gaspari-Cohn, 300 m)", + "dataassim": { + "scheme": "esmda", + "analysis": "approx", + "mda": {"tot_assim_steps": 4, "inflation_param": [4, 4, 4, 4]}, + "localization": {"name": "distance_loc", "taper_func": "gc", "radius": 300.0, "unlocalized": ["sor"]}, + }, + }, ] diff --git a/benchmarks/minires/results/coarse.json b/benchmarks/minires/results/coarse.json index 496e165..e2ec54e 100644 --- a/benchmarks/minires/results/coarse.json +++ b/benchmarks/minires/results/coarse.json @@ -2,8 +2,8 @@ "meta": { "truths": 20, "ne": 50, - "commit": "b731be7", - "date": "2026-10-27" + "commit": "2fe3aba", + "date": "2026-10-28" }, "methods": { "ES-MDA (4 steps, \u03b1 = 4)": { @@ -31,114 +31,114 @@ ], "prior": { "misfit": [ - 62.59639601598487, - 123.05196241421295, - 93.22506871276596, - 68.11519772266308, - 84.25907621454918, - 96.40961602542615, - 80.82025957645489, - 66.07005377237218, - 101.54560412060758, - 85.36695599509565, - 84.85681317502099, - 86.77696301351816, - 99.53319335321834, - 105.47695612554558, - 97.09159986755921, - 132.8728233554532, - 71.05869753024243, - 89.23927313744124, - 85.03146870297716, - 119.6601737296945 + 55.73130842543591, + 137.14238354916625, + 114.09722293210099, + 80.09194178027617, + 100.13094358370643, + 97.29077092226981, + 98.42023772548586, + 69.81606779513308, + 111.41697224955796, + 100.60865367469653, + 108.24239909317613, + 94.86648311914568, + 99.34679448299906, + 122.17070986099917, + 87.12229784800878, + 143.00490875234343, + 79.37897246996903, + 83.12456366160681, + 78.02166026525214, + 123.34403717242958 ], "forecast_wwct_rmse": [ - 0.1696800381197985, - 0.19932798619750214, - 0.18030727564909557, - 0.1687946346333842, - 0.20334516472452202, - 0.22090711830163232, - 0.08119475966762553, - 0.09937498257293596, - 0.156574677706107, - 0.19141035949790325, - 0.17710425467402569, - 0.284394647956727, - 0.36722815919437385, - 0.3633631450880431, - 0.0871894186024358, - 0.3796182871950613, - 0.3304530925059456, - 0.2110413313020751, - 0.21456969178190033, - 0.3404644384560941 + 0.13897683761752685, + 0.1644551925369494, + 0.22364858922210806, + 0.16059248472538515, + 0.23169390875527782, + 0.250722037254747, + 0.11006553183502107, + 0.11062601385209776, + 0.20960001594202213, + 0.3606088558974006, + 0.42336953116755854, + 0.28085894101043396, + 0.36709674181770213, + 0.19153361579123543, + 0.11373896423020613, + 0.4458542897751092, + 0.3385492979897476, + 0.15519526317619853, + 0.24633298519037847, + 0.3131563966724724 ], "logk_rmse": [ - 0.8861115737970015, - 0.7138871566140654, - 0.7311114337240311, - 0.794578500774992, - 0.7398491140881406, - 0.7151208051832781, - 0.6651517080794027, - 0.66548804681917, - 0.8119847333173705, - 0.9808250445467559, - 0.87589763060281, - 0.7567644686285704, - 0.7664685244639458, - 0.6734759186020055, - 0.720284226310797, - 0.8370809938580748, - 0.8168980923865478, - 0.8378640493333555, - 0.7818831719048845, - 0.8986200732688832 + 0.9030002922081395, + 0.6916503406718245, + 0.7458963667545276, + 0.7942733987048187, + 0.7255402569241036, + 0.6843648645613214, + 0.5911643495111129, + 0.6115742353212735, + 0.8627357173488694, + 0.995519683257674, + 0.9043422688969299, + 0.7452539557908949, + 0.7679352095793981, + 0.6241604000749895, + 0.6763021667284895, + 0.8808756305104981, + 0.8298916872293414, + 0.8533907019277677, + 0.776420030217678, + 0.9455142870126433 ], "logk_corr": [ - 0.1246439649411075, - 0.041320044934769304, - -0.07031257032538189, - 0.006587399183903993, - -0.04834291429886706, - 0.10247516110712568, - 0.05891645085620727, - -0.050468555461754085, - 0.02976338891111694, - -0.002938995933889871, - 0.29384176709012366, - -0.02703682627000836, - 0.0006754033719746696, - 0.10968951060279891, - 0.016901619123219427, - 0.021257305690592972, - -0.1962829661167955, - -0.24074858187770273, - 0.021991764306852377, - 0.20930361234310244 + 0.17536033195822878, + 0.08389854939744265, + -0.07318927784229992, + -0.007091421098255735, + -0.009191201373457553, + 0.1448913178629544, + 0.0663367297108044, + -0.08681360834337648, + 0.03882316929790062, + -0.02061380122927239, + 0.4381110963976184, + -0.061586935202472354, + -0.011091452878049378, + 0.10685651020097005, + 0.06460589093159294, + 0.00027436399107725034, + -0.2653042196494431, + -0.23086368472884924, + -0.057321487594945576, + 0.30095066827477357 ], "logk_spread": [ - 0.8020730046602623, - 0.8081223960932921, - 0.7992709069836804, - 0.80689275565137, - 0.8087260800099462, - 0.776756049914733, - 0.8148302010845695, - 0.7914120818443737, - 0.7742225610152146, - 0.7844666810750466, - 0.7961890309583656, - 0.8095081632860182, - 0.8135303891787904, - 0.8102967016940247, - 0.7932683248624264, - 0.8200563852133691, - 0.7847055424944857, - 0.807270751436195, - 0.7996489668967339, - 0.7994297351777555 + 0.800422185348705, + 0.8144912198836424, + 0.801565572040535, + 0.8036901679878548, + 0.8129623320326442, + 0.7693921943052282, + 0.8197128074236573, + 0.7909572075734426, + 0.767560560610762, + 0.7859507280281789, + 0.8024323212253139, + 0.8154986762353057, + 0.8229151733189514, + 0.8182018204578533, + 0.7944666204083238, + 0.8243284096941285, + 0.7810301012315071, + 0.8031221061828192, + 0.7986416601975875, + 0.8060063739422594 ], "sor_error": [ 0.022517471776421466, @@ -183,250 +183,162 @@ 0.03832621780740705, 0.040225680565728845, 0.036738472671362744 - ], - "vo_error": [ - 0.0795937058366567, - 0.11494164476815705, - 1.2568667117930428, - 2.278635530475472, - 0.06830189446609758, - 0.48168370560906393, - 0.3255283041707937, - 0.3060383274058749, - 2.3252256889911536, - 1.1003292963327223, - 0.7792432407050995, - 0.6149525736686261, - 0.12998468061398327, - 0.25109279552830355, - 0.9197588811212274, - 0.020342999691860264, - 0.8290998138473791, - 0.7684392614247813, - 0.3509682114471895, - 0.5242844172918861 - ], - "vo_spread": [ - 0.8414351702422311, - 1.1505420162407765, - 1.0526275960392726, - 1.0154658807892343, - 0.9312057839633537, - 1.0379686315534349, - 1.0571140014594451, - 0.9524395430187967, - 1.1211433903472134, - 1.1029772445921056, - 1.093164220791091, - 0.916497074696293, - 0.8558348454302246, - 1.1186998269669504, - 0.9675067812016532, - 0.8692545869158198, - 1.1936075973977103, - 1.0932252757164844, - 0.9942898361540824, - 1.0722824850643657 ] }, "posterior": { "misfit": [ - 1.7915037779691652, - 2.2713669028286536, - 1.8834786117549918, - 2.090805229678032, - 1.1295683733463626, - 1.5319714581707664, - 1.6925341525835529, - 2.4210688161316978, - 1.6140742962228585, - 2.0084947181443713, - 1.0498687751961777, - 1.3468499072267313, - 1.7293690104066557, - 1.4096521907680108, - 1.751932889654661, - 1.6288938332347815, - 1.6163310719644617, - 1.3170131524801687, - 2.063243605642272, - 1.2410507890572322 + 2.0885833660525073, + 1.1654320317046258, + 0.9702194998824529, + 2.3208560618726404, + 1.2361134014712045, + 1.6891209821455535, + 2.2232831707192116, + 1.796979385718127, + 2.0649680065755938, + 2.0614858269468703, + 1.1152923950743063, + 1.502634626369418, + 2.355658500859111, + 1.7633468191130381, + 2.2172134573767854, + 1.51265190182608, + 1.7790794358023003, + 1.5114020490249673, + 1.439070674150037, + 1.4034460277208465 ], "forecast_wwct_rmse": [ - 0.05971978267733749, - 0.0808441584343274, - 0.054870103971857195, - 0.020196350753962533, - 0.08954900332708025, - 0.15004744903977274, - 0.02610636709537744, - 0.03962336486488827, - 0.14900681526379084, - 0.029354166192413246, - 0.11220849448933627, - 0.09251367787219592, - 0.1435614254468246, - 0.14793067332321716, - 0.020800996084311543, - 0.14903290369134609, - 0.20415621483726862, - 0.042416725562170625, - 0.023560599138257407, - 0.057783205512547835 + 0.026237060916436992, + 0.08934127872680576, + 0.14325672918140142, + 0.016298164134063296, + 0.1379451086050211, + 0.13690932768536895, + 0.023888310676841988, + 0.06291358676695905, + 0.06038691760435827, + 0.2007961613477072, + 0.19477675224450022, + 0.07074683102036071, + 0.10622367238071827, + 0.04351786222296672, + 0.02456106215086734, + 0.2060176666702341, + 0.18287690824131142, + 0.024951076473496737, + 0.043720933397057675, + 0.04382204088423731 ], "logk_rmse": [ - 0.8824439139576858, - 0.822063773293608, - 0.8540000151824455, - 0.7516720232705129, - 0.8688289790482115, - 0.752273683144322, - 0.7795559576511735, - 0.7330022922807709, - 0.8306284359302947, - 0.9618180860590525, - 0.9344464608196765, - 0.7724232059018321, - 0.7774633689451728, - 0.7390113473071941, - 0.9269369245875531, - 0.8701089376191667, - 0.7080916379514234, - 0.8720653496895182, - 0.8687236528230379, - 0.875091198747223 + 1.0095852320618797, + 0.6152638659505486, + 0.7809881218763406, + 0.6408310746450001, + 0.9306532673707145, + 0.625503838600714, + 0.6332009935860087, + 0.5992924031899522, + 0.7723896809541831, + 0.9107798501604581, + 0.9579392922994596, + 0.6702508946977567, + 0.678378426665037, + 0.623924541253469, + 0.7795931706626656, + 0.8929828507016606, + 0.7504193571098152, + 0.7391751675405658, + 0.810558087793525, + 0.8451281957524109 ], "logk_corr": [ - 0.25834218504251655, - 0.36279578635627274, - -0.05802734806026723, - 0.43356714101280563, - 0.0034891235638091728, - 0.3847164599548598, - -0.05461168372186331, - 0.05718933569329724, - 0.15105488147046053, - 0.23884564661082452, - 0.27778129351489306, - 0.3187360527273105, - 0.3049188976536502, - 0.24155223944941812, - -0.0802551922530325, - 0.24353644135792624, - 0.4634555931257992, - 0.17273082104692686, - 0.06343747962801316, - 0.3113389454842998 + 0.055859966143787944, + 0.536314175916031, + 0.04820912581255484, + 0.5956582938915032, + -0.054825428747305546, + 0.4752180939436844, + 0.18185183977716227, + 0.22808256070410615, + 0.216443014066933, + 0.42036363629199963, + 0.3559214369177022, + 0.463420340720018, + 0.4825757011001998, + 0.3907157513693261, + -0.0470420146371292, + 0.2639871794585886, + 0.44627438750856885, + 0.35845199922830157, + 0.23158073030886603, + 0.3335310365894146 ], "logk_spread": [ - 0.522249966247683, - 0.540217777799695, - 0.5141898880744636, - 0.5289528638194563, - 0.5315106616983306, - 0.5513444137534439, - 0.5302982140159008, - 0.48017895527902377, - 0.525259019825193, - 0.5067893095530761, - 0.5375047252407087, - 0.528625947683071, - 0.5308990634930119, - 0.540715534695878, - 0.4887122300859935, - 0.5468420419487485, - 0.5202390856526595, - 0.5259851679065075, - 0.5200908716297254, - 0.5468977109777547 + 0.47826087235958037, + 0.5039881389444623, + 0.5101990831444814, + 0.4760906911041851, + 0.540459601171268, + 0.5099650810097389, + 0.4864629263080873, + 0.448238027775582, + 0.45282678503007917, + 0.4927171044775145, + 0.5067202253466933, + 0.487713253802094, + 0.5260599218003307, + 0.5035769787764132, + 0.4729719817078039, + 0.5369714235328467, + 0.46316576101553125, + 0.4835748750698791, + 0.4731097649493036, + 0.5199918046141441 ], "sor_error": [ - 0.01242897046793881, - 0.007020330782374878, - 0.047914213709490866, - 0.035926265134937574, - 0.021847906141799156, - 0.039723233217432014, - 0.0902211768789665, - 0.00871855022369819, - 0.0510359365610053, - 0.025308113721619202, - 0.012287415158729442, - 0.03542601243845572, - 0.0455292790176694, - 0.01502380677411888, - 0.008522647777007175, - 0.011827804865997882, - 0.050489931534770854, - 0.015445523475865142, - 0.08187491233282074, - 0.0818302613398329 + 0.0340754601605652, + 0.006894746546100977, + 0.05155507562123027, + 0.024870316078611954, + 0.06053796926515098, + 0.08536400091136284, + 0.06946671651122452, + 0.022295676189764302, + 0.002075560487546868, + 0.005178513469931617, + 0.014597273634646774, + 0.00825598811300407, + 0.04366839933115879, + 0.038066518435756785, + 0.006239339135063859, + 0.06403906637225834, + 0.05984337891762295, + 0.015567823509614803, + 0.046709184304735835, + 0.0651597451125937 ], "sor_spread": [ - 0.02717638356695815, - 0.027602498128945885, - 0.02107191973113316, - 0.02197966357926537, - 0.025661374266385168, - 0.027201555576436658, - 0.019458937710670098, - 0.02135285375076458, - 0.023787370520650386, - 0.02065226479580053, - 0.020494958643270837, - 0.023107897316852952, - 0.016337631269547778, - 0.020991029418424994, - 0.019468587320449687, - 0.01804112096054823, - 0.019210575899859954, - 0.024390711960314428, - 0.02247391444950562, - 0.02084013419112236 - ], - "vo_error": [ - 0.03804462855224422, - 0.8481518045198975, - 0.16070951795245136, - 1.4688631729855812, - 0.7074955697458254, - 0.7252109230335972, - 1.1564938697091711, - 1.0103751305913296, - 0.42109246974746384, - 0.8756191775895035, - 0.3414625893296339, - 0.011628446583380914, - 0.3082926601920537, - 0.4690300502695832, - 0.16576944581815933, - 0.28675349211516643, - 0.2804027382195349, - 0.6759925548024706, - 0.832009762336237, - 1.2517213326608125 - ], - "vo_spread": [ - 0.35458573457832043, - 0.6151094823234938, - 0.48466101503718517, - 0.49436843608716186, - 0.4625746264241777, - 0.4164524395187772, - 0.44974258301004216, - 0.418418042169793, - 0.47536342927143194, - 0.47965508750314906, - 0.5311379838391072, - 0.5262123953623323, - 0.4651828711707321, - 0.48260504021484807, - 0.40232643824700026, - 0.47759285081393976, - 0.4836767500165765, - 0.41983718068368453, - 0.5090848968830625, - 0.436561033186781 + 0.019254135478676965, + 0.02890171877008459, + 0.024260149037147138, + 0.023829433461926706, + 0.01984781147703395, + 0.025562575556420566, + 0.015678706344469873, + 0.024736362053217168, + 0.021717966916272462, + 0.021347806049122873, + 0.020632448018023268, + 0.018143135311159058, + 0.01673539264505068, + 0.021389155676282648, + 0.019711740952739778, + 0.019257474607845784, + 0.01800471395362484, + 0.021757589608573543, + 0.020649234990537464, + 0.02296018612342055 ] } }, @@ -455,114 +367,114 @@ ], "prior": { "misfit": [ - 62.59639601598487, - 123.05196241421295, - 93.22506871276596, - 68.11519772266308, - 84.25907621454918, - 96.40961602542615, - 80.82025957645489, - 66.07005377237218, - 101.54560412060758, - 85.36695599509565, - 84.85681317502099, - 86.77696301351816, - 99.53319335321834, - 105.47695612554558, - 97.09159986755921, - 132.8728233554532, - 71.05869753024243, - 89.23927313744124, - 85.03146870297716, - 119.6601737296945 + 55.73130842543591, + 137.14238354916625, + 114.09722293210099, + 80.09194178027617, + 100.13094358370643, + 97.29077092226981, + 98.42023772548586, + 69.81606779513308, + 111.41697224955796, + 100.60865367469653, + 108.24239909317613, + 94.86648311914568, + 99.34679448299906, + 122.17070986099917, + 87.12229784800878, + 143.00490875234343, + 79.37897246996903, + 83.12456366160681, + 78.02166026525214, + 123.34403717242958 ], "forecast_wwct_rmse": [ - 0.1696800381197985, - 0.19932798619750214, - 0.18030727564909557, - 0.1687946346333842, - 0.20334516472452202, - 0.22090711830163232, - 0.08119475966762553, - 0.09937498257293596, - 0.156574677706107, - 0.19141035949790325, - 0.17710425467402569, - 0.284394647956727, - 0.36722815919437385, - 0.3633631450880431, - 0.0871894186024358, - 0.3796182871950613, - 0.3304530925059456, - 0.2110413313020751, - 0.21456969178190033, - 0.3404644384560941 + 0.13897683761752685, + 0.1644551925369494, + 0.22364858922210806, + 0.16059248472538515, + 0.23169390875527782, + 0.250722037254747, + 0.11006553183502107, + 0.11062601385209776, + 0.20960001594202213, + 0.3606088558974006, + 0.42336953116755854, + 0.28085894101043396, + 0.36709674181770213, + 0.19153361579123543, + 0.11373896423020613, + 0.4458542897751092, + 0.3385492979897476, + 0.15519526317619853, + 0.24633298519037847, + 0.3131563966724724 ], "logk_rmse": [ - 0.8861115737970015, - 0.7138871566140654, - 0.7311114337240311, - 0.794578500774992, - 0.7398491140881406, - 0.7151208051832781, - 0.6651517080794027, - 0.66548804681917, - 0.8119847333173705, - 0.9808250445467559, - 0.87589763060281, - 0.7567644686285704, - 0.7664685244639458, - 0.6734759186020055, - 0.720284226310797, - 0.8370809938580748, - 0.8168980923865478, - 0.8378640493333555, - 0.7818831719048845, - 0.8986200732688832 + 0.9030002922081395, + 0.6916503406718245, + 0.7458963667545276, + 0.7942733987048187, + 0.7255402569241036, + 0.6843648645613214, + 0.5911643495111129, + 0.6115742353212735, + 0.8627357173488694, + 0.995519683257674, + 0.9043422688969299, + 0.7452539557908949, + 0.7679352095793981, + 0.6241604000749895, + 0.6763021667284895, + 0.8808756305104981, + 0.8298916872293414, + 0.8533907019277677, + 0.776420030217678, + 0.9455142870126433 ], "logk_corr": [ - 0.1246439649411075, - 0.041320044934769304, - -0.07031257032538189, - 0.006587399183903993, - -0.04834291429886706, - 0.10247516110712568, - 0.05891645085620727, - -0.050468555461754085, - 0.02976338891111694, - -0.002938995933889871, - 0.29384176709012366, - -0.02703682627000836, - 0.0006754033719746696, - 0.10968951060279891, - 0.016901619123219427, - 0.021257305690592972, - -0.1962829661167955, - -0.24074858187770273, - 0.021991764306852377, - 0.20930361234310244 + 0.17536033195822878, + 0.08389854939744265, + -0.07318927784229992, + -0.007091421098255735, + -0.009191201373457553, + 0.1448913178629544, + 0.0663367297108044, + -0.08681360834337648, + 0.03882316929790062, + -0.02061380122927239, + 0.4381110963976184, + -0.061586935202472354, + -0.011091452878049378, + 0.10685651020097005, + 0.06460589093159294, + 0.00027436399107725034, + -0.2653042196494431, + -0.23086368472884924, + -0.057321487594945576, + 0.30095066827477357 ], "logk_spread": [ - 0.8020730046602623, - 0.8081223960932921, - 0.7992709069836804, - 0.80689275565137, - 0.8087260800099462, - 0.776756049914733, - 0.8148302010845695, - 0.7914120818443737, - 0.7742225610152146, - 0.7844666810750466, - 0.7961890309583656, - 0.8095081632860182, - 0.8135303891787904, - 0.8102967016940247, - 0.7932683248624264, - 0.8200563852133691, - 0.7847055424944857, - 0.807270751436195, - 0.7996489668967339, - 0.7994297351777555 + 0.800422185348705, + 0.8144912198836424, + 0.801565572040535, + 0.8036901679878548, + 0.8129623320326442, + 0.7693921943052282, + 0.8197128074236573, + 0.7909572075734426, + 0.767560560610762, + 0.7859507280281789, + 0.8024323212253139, + 0.8154986762353057, + 0.8229151733189514, + 0.8182018204578533, + 0.7944666204083238, + 0.8243284096941285, + 0.7810301012315071, + 0.8031221061828192, + 0.7986416601975875, + 0.8060063739422594 ], "sor_error": [ 0.022517471776421466, @@ -607,254 +519,502 @@ 0.03832621780740705, 0.040225680565728845, 0.036738472671362744 - ], - "vo_error": [ - 0.0795937058366567, - 0.11494164476815705, - 1.2568667117930428, - 2.278635530475472, - 0.06830189446609758, - 0.48168370560906393, - 0.3255283041707937, - 0.3060383274058749, - 2.3252256889911536, - 1.1003292963327223, - 0.7792432407050995, - 0.6149525736686261, - 0.12998468061398327, - 0.25109279552830355, - 0.9197588811212274, - 0.020342999691860264, - 0.8290998138473791, - 0.7684392614247813, - 0.3509682114471895, - 0.5242844172918861 - ], - "vo_spread": [ - 0.8414351702422311, - 1.1505420162407765, - 1.0526275960392726, - 1.0154658807892343, - 0.9312057839633537, - 1.0379686315534349, - 1.0571140014594451, - 0.9524395430187967, - 1.1211433903472134, - 1.1029772445921056, - 1.093164220791091, - 0.916497074696293, - 0.8558348454302246, - 1.1186998269669504, - 0.9675067812016532, - 0.8692545869158198, - 1.1936075973977103, - 1.0932252757164844, - 0.9942898361540824, - 1.0722824850643657 ] }, "posterior": { "misfit": [ - 28.98245516831326, - 26.640489138168373, - 36.520244247318246, - 25.122768810703654, - 21.216709372324775, - 30.60775813217205, - 19.482372567334508, - 25.67004701759868, - 41.083636183696534, - 32.24540819205011, - 22.726198603541512, - 23.885579377951625, - 29.913611463117938, - 47.35922409302765, - 32.66534780271965, - 34.130099768273446, - 20.509555424454422, - 35.57100734740797, - 37.676965687168206, - 20.122780371709954 + 17.50917010529437, + 11.74871309442787, + 12.149368994587348, + 20.294679874981828, + 18.95154166180228, + 25.680281858104312, + 19.85933442930282, + 17.136280000861866, + 28.736827585510806, + 17.877278794397352, + 15.844309795540536, + 21.149319009302797, + 20.701288980959824, + 52.282373206533336, + 17.548298155175242, + 26.820224291203697, + 14.61600127508221, + 18.926225469767978, + 21.376037046645635, + 13.759219732434847 ], "forecast_wwct_rmse": [ - 0.1192960037103914, - 0.1308908346417886, - 0.09907894684552583, - 0.07399029057084992, - 0.05214737077369033, - 0.1518223788784104, - 0.045138096634286994, - 0.058733273345566106, - 0.11340622239910982, - 0.10273718778403114, - 0.08790194871376793, - 0.21165315145732744, - 0.2999997075340914, - 0.324983096083049, - 0.048915238769594, - 0.2785914918377289, - 0.26532850629321136, - 0.14872427399524982, - 0.10046988163578885, - 0.2140438974295937 + 0.0694106697605296, + 0.05415272702540509, + 0.09273426620557147, + 0.0496128412858447, + 0.08292212031698931, + 0.1635512454984648, + 0.04148463948011642, + 0.08765009772642587, + 0.13137302925434036, + 0.28598796723039027, + 0.38076885768722357, + 0.17489655010499397, + 0.2601459023982242, + 0.1683726934496219, + 0.028105860621296563, + 0.35255976227631525, + 0.2314002497619012, + 0.07514380858504477, + 0.16565003772972445, + 0.16290241722856824 ], "logk_rmse": [ - 0.8723265243168787, - 0.71558859869194, - 0.730118775334099, - 0.7744032118294121, - 0.7534755384302392, - 0.6960993757861955, - 0.676586173858072, - 0.6677888464207973, - 0.8114819143704914, - 0.9863397861098405, - 0.8690046265556238, - 0.7507953229389861, - 0.751231641050578, - 0.6741407384767325, - 0.7144435500589994, - 0.8153532007294195, - 0.8080383392387986, - 0.8217986173998528, - 0.7839218745484915, - 0.9087651652717483 + 0.9139833091720232, + 0.6733468990903894, + 0.6797588826581427, + 0.7650059004106244, + 0.7057868443938224, + 0.6402085972817783, + 0.5807471021098367, + 0.6207680058662918, + 0.8779485289571911, + 0.9648743804948781, + 0.859936828100289, + 0.7129291032363654, + 0.7103633322203634, + 0.6273374799291627, + 0.692606682581193, + 0.8416180194392416, + 0.8054092446246804, + 0.8380772965986087, + 0.769677882905029, + 0.8843482076956343 ], "logk_corr": [ - 0.21562760690959415, - 0.2677830782076458, - -0.0037820079968811744, - 0.18645483751578662, - 0.0027023384642244175, - 0.25493431358918395, - 0.0578894479949427, - -0.015622867852202288, - 0.09644721953782456, - -0.017412765562904182, - 0.2718251463619918, - 0.14588112125158204, - 0.15803154147202222, - 0.13196023908437082, - 0.14787187910986407, - 0.20201346548047264, - 0.023832888313686466, - -0.03306003848951207, - 0.08239081460611448, - 0.26418458662936783 + 0.13204686061760704, + 0.3947081430654889, + 0.28184966708689163, + 0.22593288791746738, + 0.19188228469042343, + 0.3642087794735481, + 0.196612970222793, + 0.040573295856300984, + 0.16358271975809358, + 0.23524206520373594, + 0.4450086588881791, + 0.248364527663096, + 0.38334565686580746, + 0.08991999651350059, + 0.055049720214639566, + 0.1971498588680997, + 0.09322382223474213, + -0.10006865569507159, + 0.18509327799141215, + 0.38551923595014775 ], "logk_spread": [ - 0.7846822827172081, - 0.7778963647302004, - 0.7703632243128027, - 0.7829707729197906, - 0.7849494316087203, - 0.7480418079562016, - 0.7748883077566656, - 0.764856480748323, - 0.7549566571544074, - 0.7632484937536197, - 0.7618369620829599, - 0.778456426488579, - 0.7812320412437296, - 0.7865212542431416, - 0.7642426557022188, - 0.7944431026927856, - 0.7520150081875574, - 0.790131212983226, - 0.7803296828055041, - 0.7552168915495385 + 0.7509727666569086, + 0.7389169283263051, + 0.7332056604499859, + 0.7564790622434648, + 0.7666588622176295, + 0.7179540417672294, + 0.7537032906833814, + 0.7356830720855357, + 0.7150770015265326, + 0.7222130136699209, + 0.7346971765122607, + 0.770118212094501, + 0.7553991411661566, + 0.77109939624642, + 0.7377673090964448, + 0.7563854001725645, + 0.7353155618842638, + 0.760028974638137, + 0.7356964004817917, + 0.7280633362450745 ], "sor_error": [ 0.022517471776421466, - 0.015473119551599163, - 0.040856022442424145, - 0.012634147423579545, + 0.013748003169458789, + 0.02553879031717257, + 0.00675449372589304, 0.02143169540386905, - 0.0468378138041686, - 0.073097981068167, - 0.0035472004384132505, + 0.039057777011483163, + 0.07279764745732842, + 0.007539815323715782, + 0.008065337933816419, + 0.0024176032500323186, + 0.019365171343103893, + 0.0009603636487416634, + 0.0515922475358937, + 0.008218650006591777, + 0.009214399900861758, + 0.015212583004793029, + 0.03885490348458667, + 0.02620376094610105, + 0.06658815468347357, + 0.05846501566022749 + ], + "sor_spread": [ + 0.037483552502312445, + 0.03886501623625229, + 0.03213078731429525, + 0.03434224731736044, + 0.0373651192814694, + 0.035465047256918646, + 0.024428905349836695, + 0.03199366530895278, + 0.030583932536353886, + 0.029768147335002552, + 0.03317045026838399, + 0.031021880346690164, + 0.026894602944169302, + 0.02892019067705685, + 0.03308951623373754, + 0.03376030854817762, + 0.030067130118487332, + 0.03180457106274826, + 0.03576774385822216, + 0.036738472671362744 + ] + } + }, + "ES-MDA (4 steps, \u03b1 = 4), adaptive localization (3\u03c3 threshold)": { + "runs": [ + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250 + ], + "prior": { + "misfit": [ + 55.73130842543591, + 137.14238354916625, + 114.09722293210099, + 80.09194178027617, + 100.13094358370643, + 97.29077092226981, + 98.42023772548586, + 69.81606779513308, + 111.41697224955796, + 100.60865367469653, + 108.24239909317613, + 94.86648311914568, + 99.34679448299906, + 122.17070986099917, + 87.12229784800878, + 143.00490875234343, + 79.37897246996903, + 83.12456366160681, + 78.02166026525214, + 123.34403717242958 + ], + "forecast_wwct_rmse": [ + 0.13897683761752685, + 0.1644551925369494, + 0.22364858922210806, + 0.16059248472538515, + 0.23169390875527782, + 0.250722037254747, + 0.11006553183502107, + 0.11062601385209776, + 0.20960001594202213, + 0.3606088558974006, + 0.42336953116755854, + 0.28085894101043396, + 0.36709674181770213, + 0.19153361579123543, + 0.11373896423020613, + 0.4458542897751092, + 0.3385492979897476, + 0.15519526317619853, + 0.24633298519037847, + 0.3131563966724724 + ], + "logk_rmse": [ + 0.9030002922081395, + 0.6916503406718245, + 0.7458963667545276, + 0.7942733987048187, + 0.7255402569241036, + 0.6843648645613214, + 0.5911643495111129, + 0.6115742353212735, + 0.8627357173488694, + 0.995519683257674, + 0.9043422688969299, + 0.7452539557908949, + 0.7679352095793981, + 0.6241604000749895, + 0.6763021667284895, + 0.8808756305104981, + 0.8298916872293414, + 0.8533907019277677, + 0.776420030217678, + 0.9455142870126433 + ], + "logk_corr": [ + 0.17536033195822878, + 0.08389854939744265, + -0.07318927784229992, + -0.007091421098255735, + -0.009191201373457553, + 0.1448913178629544, + 0.0663367297108044, + -0.08681360834337648, + 0.03882316929790062, + -0.02061380122927239, + 0.4381110963976184, + -0.061586935202472354, + -0.011091452878049378, + 0.10685651020097005, + 0.06460589093159294, + 0.00027436399107725034, + -0.2653042196494431, + -0.23086368472884924, + -0.057321487594945576, + 0.30095066827477357 + ], + "logk_spread": [ + 0.800422185348705, + 0.8144912198836424, + 0.801565572040535, + 0.8036901679878548, + 0.8129623320326442, + 0.7693921943052282, + 0.8197128074236573, + 0.7909572075734426, + 0.767560560610762, + 0.7859507280281789, + 0.8024323212253139, + 0.8154986762353057, + 0.8229151733189514, + 0.8182018204578533, + 0.7944666204083238, + 0.8243284096941285, + 0.7810301012315071, + 0.8031221061828192, + 0.7986416601975875, + 0.8060063739422594 + ], + "sor_error": [ + 0.022517471776421466, + 0.03231193249221098, + 0.039741926229656194, + 0.025226634560968048, + 0.02143169540386905, + 0.03331620304428154, + 0.09985184524027202, + 0.014347419674480971, 0.019789320113076436, - 0.008405533616043959, - 0.026359081768927733, - 0.012749755875471253, - 0.06155557675451226, - 0.01666633967260489, + 0.007200528938781425, + 0.0151715061165347, + 0.0006871729371782098, + 0.06846730526575548, + 0.0054726541204520585, 0.0157363589256444, - 0.0025461771761337015, - 0.029184789140362127, - 0.030228933951816056, + 0.04325964194518711, + 0.013508305935118675, + 0.0143029506873453, 0.07642224738364964, - 0.06875093506945718 + 0.05846501566022749 ], "sor_spread": [ 0.037483552502312445, - 0.03586172804839687, - 0.03881952159155224, - 0.03496513620081496, + 0.04443805479855501, + 0.03989209833172408, + 0.03959439780626146, 0.0373651192814694, - 0.03589946968605953, - 0.03243347845859705, - 0.03282075753671645, + 0.042707108594228896, + 0.03811974321077451, + 0.040700857005014156, 0.03690223223461561, - 0.03217489630803584, - 0.030029512516125827, - 0.030854686895663647, - 0.02666634485684442, - 0.026905139849190813, + 0.039779572774622934, + 0.03532836375446026, + 0.033973758704551923, + 0.031136092849106147, + 0.03541348194730274, 0.04453781527474917, - 0.03227310839905577, - 0.02978050010591354, - 0.03383968526731101, + 0.03976759448274524, + 0.03409458530957603, + 0.03832621780740705, 0.040225680565728845, - 0.02889499733467563 + 0.036738472671362744 + ] + }, + "posterior": { + "misfit": [ + 10.043811674861, + 7.470366672098372, + 16.947515099642487, + 12.576527535240997, + 12.074796478495633, + 9.884746017155067, + 21.839200685333523, + 12.229406609236033, + 15.126319676774585, + 7.60601097845596, + 7.384866774128411, + 14.549789637870788, + 9.629093940356464, + 29.177374946406495, + 11.800836563264632, + 8.568711736846819, + 14.552428235277148, + 15.18677753885384, + 20.10011627663674, + 13.054132167977574 + ], + "forecast_wwct_rmse": [ + 0.03651588284095516, + 0.036807956272454215, + 0.11817700431248518, + 0.03243839805417268, + 0.05306464835391156, + 0.11273072857620949, + 0.041733353734786025, + 0.07189109616820276, + 0.08614009370387285, + 0.23802229978218395, + 0.32389498564461694, + 0.17813859631902107, + 0.20563194471510815, + 0.10542340796710721, + 0.017831046014373516, + 0.24373157981063903, + 0.1777017268436892, + 0.059964383578569834, + 0.1655145531216571, + 0.17217833552646575 ], - "vo_error": [ - 0.0795937058366567, - 0.32753056262982927, - 1.1495119807289225, - 1.0842503388391727, - 0.1775751663566414, - 0.31846819315149233, - 0.4830259580320333, - 0.6649818100018461, - 0.8768466997370292, - 0.8015920293728342, - 0.5444248646991197, - 0.968023476398228, - 0.4932374992712232, - 0.13004694059325228, - 0.9197588811212274, - 0.04161467508678918, - 0.8290998138473791, - 0.2596047776628927, - 0.3509682114471895, - 0.12341210352477638 + "logk_rmse": [ + 0.9003059641169615, + 0.661857413922882, + 0.685339831732933, + 0.7467026440305587, + 0.7083603762393746, + 0.5974020989638117, + 0.5796192462499683, + 0.6350135081194609, + 0.8749897333099601, + 0.9377375372935375, + 0.8465128149128245, + 0.702094546579062, + 0.7144253263283685, + 0.6097563990324145, + 0.6889330335811429, + 0.8527167314405492, + 0.7595744820771717, + 0.8257259575481716, + 0.7694067405985381, + 0.8958454174017089 + ], + "logk_corr": [ + 0.2083224501561341, + 0.40899052580490364, + 0.29240468756199006, + 0.30932228657120703, + 0.2034549280209954, + 0.49243598006546563, + 0.2127629749796669, + -0.016009999572817412, + 0.15713547429941502, + 0.3600399547187669, + 0.4709986267508911, + 0.32100664194619427, + 0.370569664374933, + 0.23666075718352672, + 0.11439391055337295, + 0.25100034892608475, + 0.3017259118878392, + -0.06033284605856338, + 0.23809536754025978, + 0.36476351214033725 ], - "vo_spread": [ - 0.8414351702422311, - 0.9294268808251006, - 1.0109015474730771, - 0.8569868527044845, - 0.8043347141221031, - 0.9709225047072543, - 0.803994714877551, - 0.8039474776964238, - 0.9235853530202788, - 0.9353061995774441, - 0.8106538289540101, - 0.8208281690344851, - 0.6596616731777826, - 0.8799817301935094, - 0.9675067812016532, - 0.8367163823628767, - 1.1936075973977103, - 1.0012886084537571, - 0.9942898361540824, - 0.8378905220788019 + "logk_spread": [ + 0.7362145653208392, + 0.7095711880465798, + 0.7139205106667695, + 0.7267609661361856, + 0.7437829962464569, + 0.6913843457404499, + 0.7467024799812989, + 0.7029788260180346, + 0.6892183995726269, + 0.687951244144651, + 0.7171103964979874, + 0.7403780543256681, + 0.7305490014166837, + 0.7384057326658688, + 0.7044073739402128, + 0.7364333700593182, + 0.7037137969548939, + 0.7288249708313016, + 0.7093941546475815, + 0.7126167564198802 + ], + "sor_error": [ + 0.01083702582091503, + 0.019391193914448573, + 0.006317432563716863, + 0.014499038170179696, + 0.021394493143824328, + 0.04735487192343735, + 0.07461410181265452, + 0.021956103120090026, + 0.008065337933816419, + 0.00983940589343868, + 0.017726549957747778, + 0.0009603636487416634, + 0.05950818960841972, + 0.024875358044620655, + 0.019566955072877812, + 0.013391555923388643, + 0.03885490348458667, + 0.0303795661826623, + 0.06415581840396722, + 0.07212494361527871 + ], + "sor_spread": [ + 0.03536617334886471, + 0.03815659402665314, + 0.03237289579636989, + 0.036323109816549445, + 0.029237301219728434, + 0.03469263897072999, + 0.028404152163439965, + 0.032293952022595294, + 0.030583932536353886, + 0.027457465610068794, + 0.029423558616983387, + 0.031021880346690164, + 0.02552162490064567, + 0.030575636230374566, + 0.033134462514235005, + 0.031066167728477957, + 0.030067130118487332, + 0.032754302007591524, + 0.03441999727236118, + 0.02872498699432848 ] } }, - "ES-MDA (4 steps, \u03b1 = 4), adaptive localization (3\u03c3 threshold)": { + "ES-MDA (4 steps, \u03b1 = 4), distance localization (Gaspari-Cohn, 300 m)": { "runs": [ 250, 250, @@ -879,114 +1039,114 @@ ], "prior": { "misfit": [ - 62.59639601598487, - 123.05196241421295, - 93.22506871276596, - 68.11519772266308, - 84.25907621454918, - 96.40961602542615, - 80.82025957645489, - 66.07005377237218, - 101.54560412060758, - 85.36695599509565, - 84.85681317502099, - 86.77696301351816, - 99.53319335321834, - 105.47695612554558, - 97.09159986755921, - 132.8728233554532, - 71.05869753024243, - 89.23927313744124, - 85.03146870297716, - 119.6601737296945 + 55.73130842543591, + 137.14238354916625, + 114.09722293210099, + 80.09194178027617, + 100.13094358370643, + 97.29077092226981, + 98.42023772548586, + 69.81606779513308, + 111.41697224955796, + 100.60865367469653, + 108.24239909317613, + 94.86648311914568, + 99.34679448299906, + 122.17070986099917, + 87.12229784800878, + 143.00490875234343, + 79.37897246996903, + 83.12456366160681, + 78.02166026525214, + 123.34403717242958 ], "forecast_wwct_rmse": [ - 0.1696800381197985, - 0.19932798619750214, - 0.18030727564909557, - 0.1687946346333842, - 0.20334516472452202, - 0.22090711830163232, - 0.08119475966762553, - 0.09937498257293596, - 0.156574677706107, - 0.19141035949790325, - 0.17710425467402569, - 0.284394647956727, - 0.36722815919437385, - 0.3633631450880431, - 0.0871894186024358, - 0.3796182871950613, - 0.3304530925059456, - 0.2110413313020751, - 0.21456969178190033, - 0.3404644384560941 + 0.13897683761752685, + 0.1644551925369494, + 0.22364858922210806, + 0.16059248472538515, + 0.23169390875527782, + 0.250722037254747, + 0.11006553183502107, + 0.11062601385209776, + 0.20960001594202213, + 0.3606088558974006, + 0.42336953116755854, + 0.28085894101043396, + 0.36709674181770213, + 0.19153361579123543, + 0.11373896423020613, + 0.4458542897751092, + 0.3385492979897476, + 0.15519526317619853, + 0.24633298519037847, + 0.3131563966724724 ], "logk_rmse": [ - 0.8861115737970015, - 0.7138871566140654, - 0.7311114337240311, - 0.794578500774992, - 0.7398491140881406, - 0.7151208051832781, - 0.6651517080794027, - 0.66548804681917, - 0.8119847333173705, - 0.9808250445467559, - 0.87589763060281, - 0.7567644686285704, - 0.7664685244639458, - 0.6734759186020055, - 0.720284226310797, - 0.8370809938580748, - 0.8168980923865478, - 0.8378640493333555, - 0.7818831719048845, - 0.8986200732688832 + 0.9030002922081395, + 0.6916503406718245, + 0.7458963667545276, + 0.7942733987048187, + 0.7255402569241036, + 0.6843648645613214, + 0.5911643495111129, + 0.6115742353212735, + 0.8627357173488694, + 0.995519683257674, + 0.9043422688969299, + 0.7452539557908949, + 0.7679352095793981, + 0.6241604000749895, + 0.6763021667284895, + 0.8808756305104981, + 0.8298916872293414, + 0.8533907019277677, + 0.776420030217678, + 0.9455142870126433 ], "logk_corr": [ - 0.1246439649411075, - 0.041320044934769304, - -0.07031257032538189, - 0.006587399183903993, - -0.04834291429886706, - 0.10247516110712568, - 0.05891645085620727, - -0.050468555461754085, - 0.02976338891111694, - -0.002938995933889871, - 0.29384176709012366, - -0.02703682627000836, - 0.0006754033719746696, - 0.10968951060279891, - 0.016901619123219427, - 0.021257305690592972, - -0.1962829661167955, - -0.24074858187770273, - 0.021991764306852377, - 0.20930361234310244 + 0.17536033195822878, + 0.08389854939744265, + -0.07318927784229992, + -0.007091421098255735, + -0.009191201373457553, + 0.1448913178629544, + 0.0663367297108044, + -0.08681360834337648, + 0.03882316929790062, + -0.02061380122927239, + 0.4381110963976184, + -0.061586935202472354, + -0.011091452878049378, + 0.10685651020097005, + 0.06460589093159294, + 0.00027436399107725034, + -0.2653042196494431, + -0.23086368472884924, + -0.057321487594945576, + 0.30095066827477357 ], "logk_spread": [ - 0.8020730046602623, - 0.8081223960932921, - 0.7992709069836804, - 0.80689275565137, - 0.8087260800099462, - 0.776756049914733, - 0.8148302010845695, - 0.7914120818443737, - 0.7742225610152146, - 0.7844666810750466, - 0.7961890309583656, - 0.8095081632860182, - 0.8135303891787904, - 0.8102967016940247, - 0.7932683248624264, - 0.8200563852133691, - 0.7847055424944857, - 0.807270751436195, - 0.7996489668967339, - 0.7994297351777555 + 0.800422185348705, + 0.8144912198836424, + 0.801565572040535, + 0.8036901679878548, + 0.8129623320326442, + 0.7693921943052282, + 0.8197128074236573, + 0.7909572075734426, + 0.767560560610762, + 0.7859507280281789, + 0.8024323212253139, + 0.8154986762353057, + 0.8229151733189514, + 0.8182018204578533, + 0.7944666204083238, + 0.8243284096941285, + 0.7810301012315071, + 0.8031221061828192, + 0.7986416601975875, + 0.8060063739422594 ], "sor_error": [ 0.022517471776421466, @@ -1031,250 +1191,162 @@ 0.03832621780740705, 0.040225680565728845, 0.036738472671362744 - ], - "vo_error": [ - 0.0795937058366567, - 0.11494164476815705, - 1.2568667117930428, - 2.278635530475472, - 0.06830189446609758, - 0.48168370560906393, - 0.3255283041707937, - 0.3060383274058749, - 2.3252256889911536, - 1.1003292963327223, - 0.7792432407050995, - 0.6149525736686261, - 0.12998468061398327, - 0.25109279552830355, - 0.9197588811212274, - 0.020342999691860264, - 0.8290998138473791, - 0.7684392614247813, - 0.3509682114471895, - 0.5242844172918861 - ], - "vo_spread": [ - 0.8414351702422311, - 1.1505420162407765, - 1.0526275960392726, - 1.0154658807892343, - 0.9312057839633537, - 1.0379686315534349, - 1.0571140014594451, - 0.9524395430187967, - 1.1211433903472134, - 1.1029772445921056, - 1.093164220791091, - 0.916497074696293, - 0.8558348454302246, - 1.1186998269669504, - 0.9675067812016532, - 0.8692545869158198, - 1.1936075973977103, - 1.0932252757164844, - 0.9942898361540824, - 1.0722824850643657 ] }, "posterior": { "misfit": [ - 13.267772053463455, - 15.437561753727605, - 14.485685909676786, - 18.175236552524925, - 15.076190748607745, - 18.12992003502848, - 12.135785660314555, - 18.57977550672438, - 21.620314854150553, - 23.3079025623003, - 20.27494158293104, - 12.932899696772923, - 25.920114613259063, - 23.153973785318804, - 15.210569727401854, - 16.481277212000528, - 14.165528581878588, - 21.05715721006404, - 21.78011234783619, - 13.488065341648424 + 5.872912294046541, + 9.760212147247584, + 4.142389097841648, + 14.065647301092092, + 9.533446273054542, + 5.33545612914502, + 9.159417361031155, + 7.146798888351072, + 9.78803096529259, + 7.799859298744522, + 21.99802182740366, + 5.7765885705378945, + 5.640943460561466, + 4.50770033940414, + 9.217924450361934, + 11.706344349782274, + 5.134145596994169, + 6.999482894203293, + 14.049981338227354, + 11.580386384810089 ], "forecast_wwct_rmse": [ - 0.07022273177785818, - 0.09194662627330161, - 0.06212161734296693, - 0.06627431995368546, - 0.05134904926815731, - 0.11940725252713395, - 0.047747544355780644, - 0.05695603921908428, - 0.07142505192722504, - 0.11097819849913194, - 0.07763756327850505, - 0.14108939356919567, - 0.284429081277272, - 0.29289495772820623, - 0.0565097349500302, - 0.24167015462672892, - 0.2304022470851044, - 0.11727310895049929, - 0.08504591212081876, - 0.189196185593185 + 0.03116360109244235, + 0.0688509607947753, + 0.09489281911200144, + 0.02087603453940527, + 0.06653615195638304, + 0.0880467738825718, + 0.048381912478828434, + 0.05167183402069209, + 0.07525159253085338, + 0.1917853724113286, + 0.2907654837092412, + 0.12053935755888233, + 0.21301267560487738, + 0.034499033068549706, + 0.02126054822207605, + 0.2224248052515908, + 0.189755631932766, + 0.030990557757969217, + 0.08534552991048118, + 0.07955564981299015 ], "logk_rmse": [ - 0.8679279223386982, - 0.7237561828895215, - 0.7443142070801838, - 0.7532854937001076, - 0.7489361875683163, - 0.6987707229401375, - 0.6796246190692058, - 0.667805766476269, - 0.8050363255300188, - 0.9770506304942479, - 0.8632667251725071, - 0.7200711181542674, - 0.7153300533690374, - 0.6881153286836268, - 0.7222495931641263, - 0.8041332304573007, - 0.8139710942532719, - 0.8139107807131648, - 0.8083971474135555, - 0.9189074697566602 + 0.9109599116361329, + 0.5821291039230777, + 0.6803114149985898, + 0.7482627401340514, + 0.7308307699794252, + 0.5893472418429595, + 0.5679069388539125, + 0.6067588610756502, + 0.7270848323530817, + 0.8797057063004347, + 0.8084600673968704, + 0.6389601136109743, + 0.6907048769942459, + 0.6390601290784483, + 0.7179600818506133, + 0.8221855726450429, + 0.6687552447686687, + 0.7369912428953564, + 0.7412197542240301, + 0.876135221917863 ], "logk_corr": [ - 0.23459615275438572, - 0.2808822427029181, - 0.055030134248308486, - 0.295336904950608, - 0.0796043145384268, - 0.2679556627341797, - 0.05355566818845626, - 0.04692596176084399, - 0.16804171876937202, - 0.0675169916814269, - 0.295654016550157, - 0.27954744835273765, - 0.313568297927656, - 0.10389786312204491, - 0.15992844398864067, - 0.22625847577332173, - 0.036060660822467616, - 0.06921056577805025, - 0.04449802115831381, - 0.2586312223090657 + 0.19481102715406304, + 0.5622339137243039, + 0.24875904144222666, + 0.3213621431386836, + 0.15130248271491187, + 0.5328344999561665, + 0.27800419722297703, + 0.09730043041210114, + 0.356910038889509, + 0.4955810922980395, + 0.5080092828765016, + 0.520591791424669, + 0.4000146300410843, + 0.2011901324385226, + -0.043680950967257395, + 0.3173225375651303, + 0.5441665827217969, + 0.41737844107716277, + 0.16404931842589024, + 0.4752268466627501 ], "logk_spread": [ - 0.7679382969733638, - 0.7525152448339636, - 0.7467668101405357, - 0.7620787310025784, - 0.7736416605064841, - 0.7316398101108343, - 0.7547818455525891, - 0.7427159498962903, - 0.7376792638721524, - 0.7363810486997412, - 0.7450360377412326, - 0.7626556688825075, - 0.7633806152539336, - 0.7652487752765477, - 0.7438380423527782, - 0.7639675983699045, - 0.7347133062422383, - 0.7639101318400171, - 0.7548934499882485, - 0.7363629187148393 + 0.6792343944480258, + 0.6780843970513413, + 0.6660876663211538, + 0.6931965509803897, + 0.703671252838271, + 0.6647350544314715, + 0.6843162406370682, + 0.6657733904352006, + 0.6464098607791859, + 0.6626403580809692, + 0.6864681504763143, + 0.6879082358379959, + 0.6873444047059852, + 0.6978213145182485, + 0.6690919127540157, + 0.7055719640565118, + 0.6586141007373031, + 0.6712279012741247, + 0.6715177645989872, + 0.6880810029003315 ], "sor_error": [ - 0.005384660911569916, - 0.015473119551599163, - 0.011842897248335427, - 0.0030350558441404607, - 0.02143169540386905, - 0.05315410496862322, - 0.073097981068167, - 0.003054470425185668, - 0.0008432498608358652, - 0.013509732813183212, - 0.01638208716611189, - 0.01818228503616151, - 0.048976900444017424, - 0.01666633967260489, - 0.006227998041014066, - 0.0025461771761337015, - 0.029184789140362127, - 0.030228933951816056, - 0.07219609395981247, - 0.06364506843186313 + 0.020512574796947486, + 8.629505614302091e-05, + 0.03220785204732271, + 0.03139683612815336, + 0.03479405959223986, + 0.06195220014923014, + 0.07528360914305543, + 0.027633453728901303, + 0.009618336221596357, + 0.005066687861410291, + 0.011202787353997629, + 0.005606304740979756, + 0.036589912991799034, + 0.03719725412744146, + 0.017970153810306894, + 0.04835154611020909, + 0.04268146579179474, + 0.009320341853781944, + 0.05394103203492423, + 0.04499004432366052 ], "sor_spread": [ - 0.0352043640351695, - 0.03586172804839687, - 0.03471079504888823, - 0.0346767305214905, - 0.0373651192814694, - 0.03995676905357032, - 0.03243347845859705, - 0.03394580288584681, - 0.03186858080542909, - 0.03255264731899628, - 0.033019403682582285, - 0.03093368532649966, - 0.026403985763609188, - 0.026905139849190813, - 0.03079535355605519, - 0.03227310839905577, - 0.02978050010591354, - 0.03383968526731101, - 0.03333631061871692, - 0.031345808539444804 - ], - "vo_error": [ - 0.10032514113068469, - 0.21495627430715292, - 0.14247533750074926, - 1.5848246660167584, - 0.06623293877634673, - 0.02140197617553241, - 0.4025140122937678, - 0.6190737491068372, - 0.6401675850068917, - 0.4627300394987177, - 0.29793320946093527, - 0.36455922110120476, - 0.3384575696211929, - 0.32492396736897966, - 0.2249327631281206, - 0.050809847724706536, - 1.0158473893062059, - 0.5506478132511905, - 0.543037642747799, - 0.6412731036436385 - ], - "vo_spread": [ - 0.7720810352415833, - 0.8682864791123279, - 0.7479073999428244, - 0.9028242686934165, - 0.774250154128175, - 0.8232569148080826, - 0.8168676337145327, - 0.6530370192609716, - 0.724160974378875, - 0.8029605977909464, - 0.7303524250065353, - 0.7121160764455419, - 0.6800066901337405, - 0.7204622210709786, - 0.627156625287774, - 0.7382627558228716, - 1.0513215129686218, - 0.8123752691550642, - 0.8589440666013441, - 0.640084880584695 + 0.023497954519388336, + 0.031042047334775582, + 0.024771778264633726, + 0.027946838870727204, + 0.02435771338592363, + 0.027321659789190305, + 0.01928546991017437, + 0.027046320806711006, + 0.02402132071374579, + 0.020621332117500606, + 0.023362246411809062, + 0.021078029526446357, + 0.01825201802972675, + 0.02143900335383341, + 0.0197830098741639, + 0.022255674958951113, + 0.0226298526427787, + 0.02350407582082401, + 0.024343814607067027, + 0.02547370348779486 ] } } diff --git a/benchmarks/minires/results/fine.json b/benchmarks/minires/results/fine.json index 4bb2b7e..625b4e0 100644 --- a/benchmarks/minires/results/fine.json +++ b/benchmarks/minires/results/fine.json @@ -2,8 +2,8 @@ "meta": { "truths": 8, "ne": 50, - "commit": "b731be7", - "date": "2026-10-27" + "commit": "2fe3aba", + "date": "2026-10-28" }, "methods": { "ES-MDA (4 steps, \u03b1 = 4)": { @@ -19,54 +19,54 @@ ], "prior": { "misfit": [ - 76.6857082028925, - 118.48859535370278, - 125.88068474763797, - 88.2290594753218, - 120.13401879681457, - 71.19485589384213, - 93.81381539692943, - 81.58416139246368 + 83.71402762058042, + 107.61890411418219, + 125.14064250627528, + 110.66203860697256, + 101.93100063114483, + 85.76017237388646, + 87.83533829394854, + 89.42457713848351 ], "forecast_wwct_rmse": [ - 0.26855951538628003, - 0.3710015080260862, - 0.2823521538778289, - 0.258549119921913, - 0.29548286093911674, - 0.12655814500299795, - 0.1914157610497057, - 0.14985517995795167 + 0.33244379856295336, + 0.35907318127804927, + 0.34804875233563376, + 0.1637096371898429, + 0.3033915416178762, + 0.11458051725316307, + 0.17579949404873707, + 0.1773781325181814 ], "logk_rmse": [ - 0.862081806419521, - 0.860747552427497, - 0.8814335507422228, - 0.8509947054495131, - 0.8418541928350743, - 0.7313398522920637, - 0.7298452045306284, - 0.7594852129514473 + 0.8855016648839699, + 0.8530833548392485, + 0.9212598298068488, + 0.864300835997857, + 0.8414569700680161, + 0.7180021414805273, + 0.697586501355747, + 0.7546547886507916 ], "logk_corr": [ - -0.016450365365537036, - -0.14702663951406622, - -0.08504505935224134, - 0.05992274845787901, - -0.09942851429124891, - -0.1345454128985617, - 0.03618745715727143, - -0.15229398835963354 + 0.029471349797890893, + -0.21469617916841383, + -0.15687698669515146, + 0.1198606189316185, + -0.09586911968660838, + -0.21418648930482656, + 0.08605399460726655, + -0.12912255923473992 ], "logk_spread": [ - 0.7966141389511963, - 0.7989257773797538, - 0.802150283708243, - 0.8046550340635544, - 0.796685800426243, - 0.778724626542447, - 0.7930365195482185, - 0.7752310117323408 + 0.7977330561076191, + 0.797654926446505, + 0.8043079567371398, + 0.81323700681211, + 0.7998730530333449, + 0.7675351934931004, + 0.7910079818271508, + 0.768669638631306 ], "sor_error": [ 0.034576287836562164, @@ -87,118 +87,78 @@ 0.042128076892931896, 0.034526552731212405, 0.04152268006305935 - ], - "vo_error": [ - 1.2156756183526172, - 0.7487784775528086, - 0.4811975872810086, - 0.006578025623270811, - 0.6644893701416903, - 0.5505227441668827, - 0.5861382992353326, - 0.4140336539118179 - ], - "vo_spread": [ - 1.0749710722220533, - 0.9922316067216777, - 0.9701440255463729, - 0.9782281932782648, - 1.2395528142331786, - 0.9280871834029033, - 1.0583006464310203, - 0.9696631777900684 ] }, "posterior": { "misfit": [ - 1.7586002638548632, - 1.6432050233265005, - 1.5160408697861272, - 2.0559294530130785, - 1.2869140809373079, - 3.027528895886259, - 1.4305449247215989, - 2.4806472042872483 + 2.087888991664033, + 1.7715218972859401, + 1.5522280073471029, + 1.9708977557842724, + 0.979899022723122, + 1.8382797311730046, + 2.5472662325736444, + 1.85554036178876 ], "forecast_wwct_rmse": [ - 0.10136091135761618, - 0.11952149263063845, - 0.15071776244995205, - 0.05344330706244768, - 0.10367678383331855, - 0.02118676573268266, - 0.053962667123366934, - 0.08954923981177185 + 0.13594017943024403, + 0.07876148445329076, + 0.20078393816056184, + 0.11943140044467951, + 0.10983210586153257, + 0.02211752892860109, + 0.031240937282136238, + 0.08885154885393515 ], "logk_rmse": [ - 0.8859951573293843, - 0.8713868436864755, - 1.056508625792122, - 0.8797782765686962, - 0.8456485203491287, - 0.7011339063549485, - 0.8234989229187442, - 0.890078268092921 + 1.0193752923495825, + 0.9283723032554442, + 0.9923032530008867, + 0.7720405717370357, + 0.6656510452711139, + 0.5634845298110588, + 0.7208660414492185, + 0.6783362840228697 ], "logk_corr": [ - 0.2084463063545232, - 0.3728317974927288, - -0.0011005938573501331, - 0.2172488313855666, - 0.19041945517327907, - 0.40478049567559793, - 0.1211384952462299, - -0.08525050268305748 + 0.06812597526231985, + 0.4001198249361263, + 0.088897107154293, + 0.4859559378538481, + 0.561360406022182, + 0.5776136010976756, + 0.2673299633030705, + 0.38805598231268257 ], "logk_spread": [ - 0.5219040967180381, - 0.5199596820897883, - 0.5082064296977429, - 0.5142558803160056, - 0.5107465569817028, - 0.5149595619641594, - 0.511392726832032, - 0.48784121835244887 + 0.5051830506283718, + 0.4946350748798657, + 0.513532814532144, + 0.486847378861202, + 0.4922883583729772, + 0.46407150065969927, + 0.4814345759391401, + 0.4776898836174607 ], "sor_error": [ - 0.09121866544834878, - 0.01893216870187392, - 0.014112300998051247, - 0.016500630962240248, - 0.017672623084962785, - 0.054857265618507234, - 0.0029021147594167906, - 0.04479968384056643 + 0.07740000324633711, + 0.012750408690228698, + 0.006063865966548454, + 0.016762823090693296, + 0.06111939181492995, + 0.054189219244542675, + 0.014180209115922993, + 0.0036854753499510062 ], "sor_spread": [ - 0.020477009159431698, - 0.03074139066894728, - 0.02407800669889662, - 0.02536904922400518, - 0.025173870551999027, - 0.02717380058434338, - 0.023638497218210222, - 0.025459340174916863 - ], - "vo_error": [ - 0.46307121162638865, - 0.3163276673395954, - 0.08173338371291106, - 0.07587775277478759, - 0.04112571641791085, - 0.023688484061209714, - 0.7656534005058386, - 1.0804223803609805 - ], - "vo_spread": [ - 0.49871325980136555, - 0.4977008032115977, - 0.4073832878443032, - 0.4692456640668884, - 0.3971130490654576, - 0.49358896781283396, - 0.495497635963307, - 0.5019287799827108 + 0.019905065952168347, + 0.024536697552813156, + 0.02581454091392486, + 0.024126386308034403, + 0.022823724259765296, + 0.021609537908483616, + 0.02049279811949448, + 0.02580167441060449 ] } }, @@ -215,54 +175,54 @@ ], "prior": { "misfit": [ - 76.6857082028925, - 118.48859535370278, - 125.88068474763797, - 88.2290594753218, - 120.13401879681457, - 71.19485589384213, - 93.81381539692943, - 81.58416139246368 + 83.71402762058042, + 107.61890411418219, + 125.14064250627528, + 110.66203860697256, + 101.93100063114483, + 85.76017237388646, + 87.83533829394854, + 89.42457713848351 ], "forecast_wwct_rmse": [ - 0.26855951538628003, - 0.3710015080260862, - 0.2823521538778289, - 0.258549119921913, - 0.29548286093911674, - 0.12655814500299795, - 0.1914157610497057, - 0.14985517995795167 + 0.33244379856295336, + 0.35907318127804927, + 0.34804875233563376, + 0.1637096371898429, + 0.3033915416178762, + 0.11458051725316307, + 0.17579949404873707, + 0.1773781325181814 ], "logk_rmse": [ - 0.862081806419521, - 0.860747552427497, - 0.8814335507422228, - 0.8509947054495131, - 0.8418541928350743, - 0.7313398522920637, - 0.7298452045306284, - 0.7594852129514473 + 0.8855016648839699, + 0.8530833548392485, + 0.9212598298068488, + 0.864300835997857, + 0.8414569700680161, + 0.7180021414805273, + 0.697586501355747, + 0.7546547886507916 ], "logk_corr": [ - -0.016450365365537036, - -0.14702663951406622, - -0.08504505935224134, - 0.05992274845787901, - -0.09942851429124891, - -0.1345454128985617, - 0.03618745715727143, - -0.15229398835963354 + 0.029471349797890893, + -0.21469617916841383, + -0.15687698669515146, + 0.1198606189316185, + -0.09586911968660838, + -0.21418648930482656, + 0.08605399460726655, + -0.12912255923473992 ], "logk_spread": [ - 0.7966141389511963, - 0.7989257773797538, - 0.802150283708243, - 0.8046550340635544, - 0.796685800426243, - 0.778724626542447, - 0.7930365195482185, - 0.7752310117323408 + 0.7977330561076191, + 0.797654926446505, + 0.8043079567371398, + 0.81323700681211, + 0.7998730530333449, + 0.7675351934931004, + 0.7910079818271508, + 0.768669638631306 ], "sor_error": [ 0.034576287836562164, @@ -283,122 +243,238 @@ 0.042128076892931896, 0.034526552731212405, 0.04152268006305935 - ], - "vo_error": [ - 1.2156756183526172, - 0.7487784775528086, - 0.4811975872810086, - 0.006578025623270811, - 0.6644893701416903, - 0.5505227441668827, - 0.5861382992353326, - 0.4140336539118179 - ], - "vo_spread": [ - 1.0749710722220533, - 0.9922316067216777, - 0.9701440255463729, - 0.9782281932782648, - 1.2395528142331786, - 0.9280871834029033, - 1.0583006464310203, - 0.9696631777900684 ] }, "posterior": { "misfit": [ - 43.98481629093686, - 63.26576800653424, - 16.403032434581824, - 49.92134319001806, - 56.52855548885102, - 54.76046089839836, - 37.25089801428513, - 59.43203588343219 + 27.459286191375057, + 47.49960392596989, + 43.43672596599238, + 25.78496499616186, + 22.177076092376243, + 21.61079262901964, + 40.342719301752965, + 38.10519548151111 ], "forecast_wwct_rmse": [ - 0.2342654221080745, - 0.30895162973246715, - 0.19125144856985649, - 0.24596817563736845, - 0.2608621619694931, - 0.11753050095989737, - 0.10936033579091593, - 0.13343951715344343 + 0.3052882305737283, + 0.2902710414744202, + 0.30322142928992407, + 0.0700951586965657, + 0.17074515889591413, + 0.03769183040079784, + 0.0868744387168838, + 0.119838112334906 ], "logk_rmse": [ - 0.8558831824300456, - 0.853825322415626, - 0.8485116190422426, - 0.8497174645365726, - 0.8319631297956654, - 0.7334196168904353, - 0.7186299175160985, - 0.7565442539458359 + 0.8571481140374041, + 0.8298506114009864, + 0.9021107937148806, + 0.8414108333393979, + 0.787994844041064, + 0.7033265858128059, + 0.6996884535343812, + 0.7204408870401566 ], "logk_corr": [ - 0.0706289461461521, - -0.015182531158826765, - 0.1846489927531531, - 0.08322038621946326, - 0.015746298835186814, - -0.11701363824161891, - 0.16101369658300574, - -0.13769711934421738 + 0.25223824947891804, + 0.11959385271477609, + 0.10532421753866748, + 0.26828427699169877, + 0.24244422984360972, + 0.03679714857588492, + 0.05544959462475698, + 0.1457653410353589 ], "logk_spread": [ - 0.7773033562068479, - 0.7836358551891279, - 0.773903132784877, - 0.7868724110092317, - 0.7849087614624058, - 0.774736154430035, - 0.7711451615421901, - 0.7660264629958794 + 0.7547819139915253, + 0.764281163982213, + 0.7694563417139216, + 0.7530827294974993, + 0.752655650585289, + 0.7341841599605483, + 0.7523634824636932, + 0.7322776237938455 + ], + "sor_error": [ + 0.0732732748469957, + 0.05240158372942216, + 0.014473495984371504, + 0.032093631315579296, + 0.012772658013380705, + 0.04843192048666639, + 0.003998672968880318, + 0.007147662135787658 + ], + "sor_spread": [ + 0.024228381620746316, + 0.02929756399120832, + 0.03426436609312096, + 0.04271449501189857, + 0.04399000371765547, + 0.03573525306687847, + 0.029712380227363305, + 0.033804622999118746 + ] + } + }, + "ES-MDA (4 steps, \u03b1 = 4), adaptive localization (3\u03c3 threshold)": { + "runs": [ + 250, + 250, + 250, + 250, + 250, + 250, + 250, + 250 + ], + "prior": { + "misfit": [ + 83.71402762058042, + 107.61890411418219, + 125.14064250627528, + 110.66203860697256, + 101.93100063114483, + 85.76017237388646, + 87.83533829394854, + 89.42457713848351 + ], + "forecast_wwct_rmse": [ + 0.33244379856295336, + 0.35907318127804927, + 0.34804875233563376, + 0.1637096371898429, + 0.3033915416178762, + 0.11458051725316307, + 0.17579949404873707, + 0.1773781325181814 + ], + "logk_rmse": [ + 0.8855016648839699, + 0.8530833548392485, + 0.9212598298068488, + 0.864300835997857, + 0.8414569700680161, + 0.7180021414805273, + 0.697586501355747, + 0.7546547886507916 + ], + "logk_corr": [ + 0.029471349797890893, + -0.21469617916841383, + -0.15687698669515146, + 0.1198606189316185, + -0.09586911968660838, + -0.21418648930482656, + 0.08605399460726655, + -0.12912255923473992 + ], + "logk_spread": [ + 0.7977330561076191, + 0.797654926446505, + 0.8043079567371398, + 0.81323700681211, + 0.7998730530333449, + 0.7675351934931004, + 0.7910079818271508, + 0.768669638631306 ], "sor_error": [ 0.034576287836562164, 0.07750867354115154, - 0.008051585132393263, - 0.04049998206309252, - 0.009663579743787204, - 0.049730858866636696, + 0.001919478814194464, + 0.032093631315579296, + 0.012772658013380705, + 0.02436307210664651, 0.024252339511175364, 0.01789218898357159 ], "sor_spread": [ 0.03823958904277204, 0.036130397433477894, - 0.03597912078866424, - 0.03565228580033325, - 0.03339204695281016, - 0.035572269582703465, + 0.042194407622493546, + 0.04271449501189857, + 0.04399000371765547, + 0.042128076892931896, 0.034526552731212405, 0.04152268006305935 + ] + }, + "posterior": { + "misfit": [ + 7.576562580693318, + 28.56168462308427, + 14.674385816404078, + 14.214432691162935, + 8.858532483741195, + 15.986500587100108, + 20.110402038564477, + 20.916076273757202 + ], + "forecast_wwct_rmse": [ + 0.238735630871627, + 0.2787405334280098, + 0.24610851120064797, + 0.062121845553735594, + 0.1862542948104578, + 0.05694448922094695, + 0.09969050397322719, + 0.09390919182263398 ], - "vo_error": [ - 1.2156756183526172, - 0.05453730094539999, - 0.12601795862373066, - 0.006578025623270811, - 0.7226942750278376, - 0.6839099226751779, - 0.5861382992353326, - 0.4140336539118179 - ], - "vo_spread": [ - 1.0749710722220533, - 0.8806052253116802, - 0.7855653005285126, - 0.9782281932782648, - 0.8337608602007149, - 0.8299789276712952, - 1.0583006464310203, - 0.9696631777900684 + "logk_rmse": [ + 0.849492490882655, + 0.8276965298509716, + 0.8728287855552486, + 0.8099077382667329, + 0.6813078844769732, + 0.6872206891731132, + 0.6842997689068038, + 0.6786173938407646 + ], + "logk_corr": [ + 0.2853557354983551, + 0.20107271136331875, + 0.23995732703319517, + 0.4016933348479066, + 0.5377530848617101, + 0.19156597285802981, + 0.19106358375016677, + 0.2735937387482794 + ], + "logk_spread": [ + 0.7153473103953384, + 0.7190752124887306, + 0.7285843124636022, + 0.7137488692262325, + 0.706464204096582, + 0.7036599341420884, + 0.7113523519451919, + 0.6918258912871229 + ], + "sor_error": [ + 0.07022760877854434, + 0.04052604212134303, + 0.014473495984371504, + 0.01911973122961208, + 0.059326431302224286, + 0.046940009172157265, + 0.002074795699886278, + 0.015676865789199385 + ], + "sor_spread": [ + 0.026922842364982876, + 0.03178359062236804, + 0.03426436609312096, + 0.030042719457920343, + 0.0316515589711606, + 0.03577467930798504, + 0.026426507893684656, + 0.03681332998402922 ] } }, - "ES-MDA (4 steps, \u03b1 = 4), adaptive localization (3\u03c3 threshold)": { + "ES-MDA (4 steps, \u03b1 = 4), distance localization (Gaspari-Cohn, 300 m)": { "runs": [ 250, 250, @@ -411,54 +487,54 @@ ], "prior": { "misfit": [ - 76.6857082028925, - 118.48859535370278, - 125.88068474763797, - 88.2290594753218, - 120.13401879681457, - 71.19485589384213, - 93.81381539692943, - 81.58416139246368 + 83.71402762058042, + 107.61890411418219, + 125.14064250627528, + 110.66203860697256, + 101.93100063114483, + 85.76017237388646, + 87.83533829394854, + 89.42457713848351 ], "forecast_wwct_rmse": [ - 0.26855951538628003, - 0.3710015080260862, - 0.2823521538778289, - 0.258549119921913, - 0.29548286093911674, - 0.12655814500299795, - 0.1914157610497057, - 0.14985517995795167 + 0.33244379856295336, + 0.35907318127804927, + 0.34804875233563376, + 0.1637096371898429, + 0.3033915416178762, + 0.11458051725316307, + 0.17579949404873707, + 0.1773781325181814 ], "logk_rmse": [ - 0.862081806419521, - 0.860747552427497, - 0.8814335507422228, - 0.8509947054495131, - 0.8418541928350743, - 0.7313398522920637, - 0.7298452045306284, - 0.7594852129514473 + 0.8855016648839699, + 0.8530833548392485, + 0.9212598298068488, + 0.864300835997857, + 0.8414569700680161, + 0.7180021414805273, + 0.697586501355747, + 0.7546547886507916 ], "logk_corr": [ - -0.016450365365537036, - -0.14702663951406622, - -0.08504505935224134, - 0.05992274845787901, - -0.09942851429124891, - -0.1345454128985617, - 0.03618745715727143, - -0.15229398835963354 + 0.029471349797890893, + -0.21469617916841383, + -0.15687698669515146, + 0.1198606189316185, + -0.09586911968660838, + -0.21418648930482656, + 0.08605399460726655, + -0.12912255923473992 ], "logk_spread": [ - 0.7966141389511963, - 0.7989257773797538, - 0.802150283708243, - 0.8046550340635544, - 0.796685800426243, - 0.778724626542447, - 0.7930365195482185, - 0.7752310117323408 + 0.7977330561076191, + 0.797654926446505, + 0.8043079567371398, + 0.81323700681211, + 0.7998730530333449, + 0.7675351934931004, + 0.7910079818271508, + 0.768669638631306 ], "sor_error": [ 0.034576287836562164, @@ -479,118 +555,78 @@ 0.042128076892931896, 0.034526552731212405, 0.04152268006305935 - ], - "vo_error": [ - 1.2156756183526172, - 0.7487784775528086, - 0.4811975872810086, - 0.006578025623270811, - 0.6644893701416903, - 0.5505227441668827, - 0.5861382992353326, - 0.4140336539118179 - ], - "vo_spread": [ - 1.0749710722220533, - 0.9922316067216777, - 0.9701440255463729, - 0.9782281932782648, - 1.2395528142331786, - 0.9280871834029033, - 1.0583006464310203, - 0.9696631777900684 ] }, "posterior": { "misfit": [ - 19.73792784596038, - 16.31644729282754, - 10.601250359591925, - 14.41826185679646, - 18.675066923927876, - 14.923363048269408, - 24.31984936323121, - 42.41184840844644 + 14.054084030408678, + 5.67256340049648, + 26.396639787059705, + 15.966895747282154, + 6.870021528589728, + 13.77645219987654, + 8.98801250519199, + 7.696143710523182 ], "forecast_wwct_rmse": [ - 0.1910663542153361, - 0.2005244163369256, - 0.1683861568128584, - 0.14784199263657483, - 0.20602046702041743, - 0.04023239094936644, - 0.09932670705726565, - 0.09979149857329582 + 0.23248626365002303, + 0.1144822616203722, + 0.23922503696666128, + 0.048137530729955795, + 0.11350371686243653, + 0.04653962803027561, + 0.05165483055594074, + 0.0500080797204279 ], "logk_rmse": [ - 0.8545716581457826, - 0.841788433576811, - 0.841683867783141, - 0.8186629079250293, - 0.7938198016340672, - 0.7223265439968245, - 0.7222728858138057, - 0.7565736401115633 + 0.8140031271823935, + 0.7542810438026765, + 0.9067512121032111, + 0.8012628486306604, + 0.6591673311929274, + 0.6595634135828213, + 0.6254686287741698, + 0.6328064151671196 ], "logk_corr": [ - 0.1102656975145053, - 0.17073177539765103, - 0.24952298347830223, - 0.2700452445797318, - 0.21783716186282134, - 0.0945573953457165, - 0.18491156179419574, - -0.08671293154984779 + 0.39163468195861567, + 0.48262045285847566, + 0.14255777094162791, + 0.3923775722567384, + 0.5900323239019307, + 0.302642012322515, + 0.4394896115936328, + 0.4727725970660957 ], "logk_spread": [ - 0.7514130116256484, - 0.7480175695313376, - 0.7419574636627498, - 0.7390278988385468, - 0.7494010556775629, - 0.7420782756808155, - 0.7479245196431874, - 0.7382799188189086 + 0.6783620310054811, + 0.6825387335549483, + 0.6963477718753457, + 0.6730762722811527, + 0.6868982554361088, + 0.6555029293912891, + 0.6712207831881537, + 0.6596455901676358 ], "sor_error": [ - 0.07195871670575449, - 0.07750867354115154, - 0.001919478814194464, - 0.03510535501525666, - 0.009663579743787204, - 0.047561704502575075, - 0.025058834206631242, - 0.017559747631029854 + 0.08468561180173223, + 0.04123235875045539, + 0.014579486481314935, + 0.011229915480184233, + 0.0706076541901823, + 0.03845312701127737, + 0.007013168754631344, + 0.0028549457720886406 ], "sor_spread": [ - 0.030243799222457072, - 0.036130397433477894, - 0.042194407622493546, - 0.036523583835101285, - 0.03339204695281016, - 0.03611578136434504, - 0.03226302642807711, - 0.037378038423759266 - ], - "vo_error": [ - 0.8343723377232175, - 0.24671580983949326, - 0.18825285397403935, - 0.5990556188003646, - 0.24061972191810543, - 0.10840506852377096, - 0.712827345951081, - 1.0306230518055113 - ], - "vo_spread": [ - 0.9135488735443291, - 0.8513361179765446, - 0.6024468368747081, - 0.7770145278046457, - 0.7249841558766282, - 0.7419569255377249, - 0.6934846538571406, - 0.8226826895749983 + 0.020065968944044228, + 0.026320953892348006, + 0.02648588749379577, + 0.02581430404350884, + 0.025839201349871582, + 0.026353968364454406, + 0.021619191010545505, + 0.02640300400763263 ] } } diff --git a/benchmarks/minires/run.py b/benchmarks/minires/run.py index 0fedee0..c836302 100644 --- a/benchmarks/minires/run.py +++ b/benchmarks/minires/run.py @@ -4,7 +4,11 @@ python run.py # every method, every case, the default number of truths python run.py --case coarse # one case python run.py --method "ES-MDA" # the methods whose label contains this - python run.py --truths 2 --ne 10 # a quick check (not written to results/) + python run.py --quick # a check while developing: the coarse case, a few truths + +A run of fewer truths than the case's own is a check, printed and not written to results/: +its truths are the first of the official run's, so it is printed next to the official scores +on those same truths, and any difference is due to the code (the experiments are seeded). Each (method, truth) is one twin experiment, run in its own process and its own temporary directory, so that what PET writes there (results, logs) goes nowhere. @@ -40,6 +44,7 @@ HERE = Path(__file__).parent TRUTH_SEED, ENSEMBLE_SEED = 1000, 2000 # truth i is drawn with TRUTH_SEED + i +QUICK = {"case": ["coarse"], "truths": 5} class CountingMiniRes(MiniRes): @@ -70,8 +75,12 @@ def experiment(case_name: str, method: dict, i: int, ne: int) -> dict: pd.DataFrame(variance, index=index, columns=DATATYPES).to_csv("var.csv") dataassim = {**COMMON, **method["dataassim"], "data": "data.csv", "datavar": "var.csv", "savefolder": "Results"} - if "localization" in dataassim: # the grid is the case's, not the method's - dataassim["localization"] = {"field": [case.n, case.n], **dataassim["localization"]} + if "localization" in dataassim: # the grid and the wells are the case's, not the method's + loc = {"field": [case.n, case.n], **dataassim["localization"]} + if loc["name"] == "distance_loc": + loc["field"] = [1, case.n, case.n] + loc["entries"] = [{"taper": loc["taper_func"], **e} for e in case.distance_entries(loc.pop("radius"))] + dataassim["localization"] = loc keys_da, keys_sim, keys_en = normalize( dataassim, case.simulator(OBS_STEPS), @@ -114,6 +123,25 @@ def cell(values) -> str: return f"{values.mean():.3g} ± {values.std(ddof=1) / np.sqrt(len(values)):.2g}" +def render_case(title: str, res: dict) -> list: + meta, runs = res["meta"], res["methods"] + lines = [ + "", + title, + "", + f"{meta['truths']} truths, {meta['ne']} members; history of {HISTORY} of {NSTEPS} steps, " + f"observed at steps {OBS_STEPS}. PET at `{meta['commit']}`, {meta['date']}.", + ] + any_method = next(iter(runs.values())) + lines += ["", "| | " + " | ".join(COLUMNS.values()) + " |", "|---" * (len(COLUMNS) + 1) + "|"] + prior = [cell(any_method["prior"][k]) if k != "runs" else "" for k in COLUMNS] + lines += ["| prior | " + " | ".join(prior) + " |"] + for label, r in runs.items(): + post = [cell(r["posterior"][k]) if k != "runs" else f"{np.mean(r['runs']):.0f}" for k in COLUMNS] + lines += [f"| {label} | " + " | ".join(post) + " |"] + return lines + + def render(results: dict) -> str: lines = [ "# MiniRes benchmark: results", @@ -124,25 +152,21 @@ def render(results: dict) -> str: "and which score matters is for the reader to judge.", ] for case_name, res in results.items(): - meta, runs = res["meta"], res["methods"] case = CASES[case_name] - lines += [ - "", - f"## Case `{case_name}`: {case.n} x {case.n} cells", - "", - f"{meta['truths']} truths, {meta['ne']} members; history of {HISTORY} of {NSTEPS} steps, " - f"observed at steps {OBS_STEPS}. PET at `{meta['commit']}`, {meta['date']}.", - ] - any_method = next(iter(runs.values())) - lines += ["", "| | " + " | ".join(COLUMNS.values()) + " |", "|---" * (len(COLUMNS) + 1) + "|"] - prior = [cell(any_method["prior"][k]) if k != "runs" else "" for k in COLUMNS] - lines += ["| prior | " + " | ".join(prior) + " |"] - for label, r in runs.items(): - post = [cell(r["posterior"][k]) if k != "runs" else f"{np.mean(r['runs']):.0f}" for k in COLUMNS] - lines += [f"| {label} | " + " | ".join(post) + " |"] + lines += render_case(f"## Case `{case_name}`: {case.n} x {case.n} cells", res) return "\n".join(lines) + "\n" +def first_truths(res: dict, truths: int, labels) -> dict: + """The official results ``res``, of the methods ``labels``, cut down to their first ``truths``.""" + + def cut(v): + return {k: cut(w) for k, w in v.items()} if isinstance(v, dict) else v[:truths] + + methods = {label: cut(r) for label, r in res["methods"].items() if label in labels} + return {"meta": {**res["meta"], "truths": truths}, "methods": methods} + + def commit() -> str: def git(*args): return subprocess.run(["git", *args], cwd=HERE, capture_output=True, text=True).stdout.strip() @@ -159,7 +183,11 @@ def main(): parser.add_argument("--truths", type=int, help="default: the case's own") parser.add_argument("--ne", type=int, default=50) parser.add_argument("--workers", type=int, default=os.cpu_count()) + parser.add_argument("--quick", action="store_true", help=f"default to {QUICK}") args = parser.parse_args() + if args.quick: + args.case = args.case or QUICK["case"] + args.truths = args.truths or QUICK["truths"] methods = [m for m in METHODS if not args.method or any(s in m["label"] for s in args.method)] quick = args.truths is not None and args.truths < min(CASES[c].truths for c in args.case or CASES) @@ -185,7 +213,11 @@ def main(): results["methods"] = {m["label"]: results["methods"][m["label"]] for m in METHODS if m["label"] in results["methods"]} if quick: - print(render({case_name: results})) + print("\n".join(render_case(f"## Case `{case_name}`: this run", results))) + if old and old["meta"]["ne"] == args.ne and old["meta"]["truths"] >= truths: + official = first_truths(old, truths, results["methods"]) + print("\n".join(render_case(f"## Case `{case_name}`: official, on the same truths", official))) + print() continue res_file.parent.mkdir(exist_ok=True) res_file.write_text(json.dumps(results, indent=1) + "\n")