From 9c254f4a63cd9a6da23a67d4f9afd2a4d6c3465f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 01:08:05 +0000 Subject: [PATCH 1/2] Fail assert_unbiased on a deterministic estimator that misses the truth A study whose estimates never varied has a Monte Carlo standard error of zero, and bias_t reported that as a t statistic of zero, so an estimator returning 5.0 for a truth of 2.0 passed assert_unbiased. So did a single replicate off by 1e6, because sampling_sd is zero below two replicates. bias_t is now infinite for a constant that misses the truth and zero only for one that hits it, and assert_unbiased refuses studies of fewer than two replicates, as se_ratio_tolerance and assert_narrower already do. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011pXaJ5DLex7BrdUyfTqjL4 --- CHANGELOG.md | 9 ++++++++ src/simcheck/gates.py | 11 ++++++++++ src/simcheck/results.py | 13 ++++++++---- tests/test_negative.py | 47 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77eb29a..1f7ca54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- `assert_unbiased` passed a deterministic estimator however far it missed the + truth, because a zero Monte Carlo standard error was reported as a bias t + statistic of zero. `bias_t` is now infinite for a constant that misses the + truth, and zero only for one that hits it. +- `assert_unbiased` raises `ValueError` on a single-replicate study instead of + passing it, as `se_ratio_tolerance` and `assert_narrower` already did. + ## [0.1.0] - 2026-08-09 ### Added diff --git a/src/simcheck/gates.py b/src/simcheck/gates.py index 23a35ba..175c92f 100644 --- a/src/simcheck/gates.py +++ b/src/simcheck/gates.py @@ -458,14 +458,25 @@ def assert_unbiased( bias too small for the study to resolve does not fail, and the study can be made more demanding simply by running more replicates. + An estimator that returns the same value on every replicate has no Monte + Carlo noise, so any difference from the truth is resolved exactly and fails. + Args: result: A completed Monte Carlo study. label: Included in the failure message. sigmas: How many Monte Carlo standard errors of slack to allow. Raises: + ValueError: If the study has fewer than two replicates. One replicate has + no estimable spread, so it cannot say whether its distance from the + truth is bias or noise. AssertionError: If the bias t statistic exceeds the gate. """ + if result.reps < 2: + raise ValueError( + f"{label or 'this study'}: a single replicate has no spread, so its " + "distance from the truth cannot be told apart from noise" + ) if not abs(result.bias_t) < sigmas: raise AssertionError( f"{label}: bias {result.bias:+.6f} is {result.bias_t:+.2f} Monte " diff --git a/src/simcheck/results.py b/src/simcheck/results.py index fb7a369..bbe41d5 100644 --- a/src/simcheck/results.py +++ b/src/simcheck/results.py @@ -22,6 +22,7 @@ from __future__ import annotations +import math from dataclasses import dataclass import numpy as np @@ -168,11 +169,15 @@ def mc_se(self) -> float: def bias_t(self) -> float: """Bias in units of its own Monte Carlo standard error. - A t statistic for the null that the estimator is unbiased. Zero when the - estimator is deterministic, in which case there is no sampling variation - to test against. + A t statistic for the null that the estimator is unbiased. When the + estimator did not vary, there is no noise for a bias to hide in: the t + statistic is zero if every estimate equals the truth and infinite + otherwise. Reporting zero for a constant that misses the truth would + certify a biased estimator as unbiased because it is *also* deterministic. """ - return self.bias / self.mc_se if self.mc_se else 0.0 + if self.mc_se: + return self.bias / self.mc_se + return math.copysign(math.inf, self.bias) if self.bias else 0.0 @property def coverage(self) -> float: diff --git a/tests/test_negative.py b/tests/test_negative.py index 96d3194..b3107cf 100644 --- a/tests/test_negative.py +++ b/tests/test_negative.py @@ -222,6 +222,53 @@ def test_assert_unbiased_fails_on_a_biased_estimator(): assert_unbiased(_study(bias=0.5, sd=0.1), "biased") +def _constant_study(value: float, truth: float, reps: int = 400) -> MonteCarloResult: + """An estimator that returns ``value`` on every replicate. + + Parameters + ---------- + value + The estimate every replicate reports. + truth + The true value. + reps + Replicates. + + Returns + ------- + MonteCarloResult + The constructed study. + """ + return MonteCarloResult( + estimates=np.full(reps, value), + standard_errors=np.full(reps, 0.1), + covered=None, + rejected=None, + truth=truth, + ) + + +def test_assert_unbiased_fails_on_a_deterministic_biased_estimator(): + """A constant that misses the truth is biased, with no noise to excuse it. + + The Monte Carlo standard error is exactly zero here, and the gate used to + read that as a t statistic of zero -- so returning 5 for a truth of 2 passed. + """ + with pytest.raises(AssertionError, match=r"\+inf Monte"): + assert_unbiased(_constant_study(5.0, truth=2.0), "constant, wrong") + + +def test_assert_unbiased_passes_a_deterministic_exact_estimator(): + """A constant that equals the truth is unbiased, and must not fire.""" + assert_unbiased(_constant_study(2.0, truth=2.0), "constant, right") + + +def test_assert_unbiased_rejects_a_single_replicate(): + """One draw cannot separate bias from noise, however far off it lands.""" + with pytest.raises(ValueError, match="single replicate"): + assert_unbiased(_constant_study(1e6, truth=0.0, reps=1), "one draw") + + def test_assert_unbiased_resolves_smaller_bias_with_more_replicates(): """The gate must tighten as the study grows, without any edit. From 1624af8c93e8bc7e218fe94e5db083d2735944b8 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 01:11:24 +0000 Subject: [PATCH 2/2] Raise, not assert bias, when identical estimates miss the truth Zero observed spread does not imply a deterministic estimator: 400 draws of a Bernoulli(0.001) estimator are all zero about two times in three, and the previous commit reported that unbiased estimator as infinitely biased. assert_unbiased now raises ValueError in that case, explaining that the study cannot tell a deterministic bias from a discrete estimator that did not vary, and bias_t is NaN rather than infinite. A biased constant still cannot pass. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011pXaJ5DLex7BrdUyfTqjL4 --- CHANGELOG.md | 6 ++++-- src/simcheck/gates.py | 20 +++++++++++++++----- src/simcheck/results.py | 12 +++++++----- tests/test_negative.py | 23 +++++++++++++++++++---- 4 files changed, 45 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f7ca54..0a00df6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `assert_unbiased` passed a deterministic estimator however far it missed the truth, because a zero Monte Carlo standard error was reported as a bias t - statistic of zero. `bias_t` is now infinite for a constant that misses the - truth, and zero only for one that hits it. + statistic of zero. Identical estimates that miss the truth now raise + `ValueError`, since the study cannot tell a deterministic bias from a + discrete estimator that happened not to vary; `bias_t` is NaN for them and + zero only when they hit the truth. - `assert_unbiased` raises `ValueError` on a single-replicate study instead of passing it, as `se_ratio_tolerance` and `assert_narrower` already did. diff --git a/src/simcheck/gates.py b/src/simcheck/gates.py index 175c92f..70585d3 100644 --- a/src/simcheck/gates.py +++ b/src/simcheck/gates.py @@ -458,8 +458,10 @@ def assert_unbiased( bias too small for the study to resolve does not fail, and the study can be made more demanding simply by running more replicates. - An estimator that returns the same value on every replicate has no Monte - Carlo noise, so any difference from the truth is resolved exactly and fails. + A study whose estimates are all equal passes only if they equal the truth. + If they miss it, the study has no spread to measure the miss against, and + it raises rather than passing: the estimator may be deterministic and + biased, or discrete and unlucky, and identical estimates cannot say which. Args: result: A completed Monte Carlo study. @@ -467,9 +469,9 @@ def assert_unbiased( sigmas: How many Monte Carlo standard errors of slack to allow. Raises: - ValueError: If the study has fewer than two replicates. One replicate has - no estimable spread, so it cannot say whether its distance from the - truth is bias or noise. + ValueError: If the study has fewer than two replicates, or if its + estimates are all equal and differ from the truth. Either way it has + no spread against which to tell bias from noise. AssertionError: If the bias t statistic exceeds the gate. """ if result.reps < 2: @@ -477,6 +479,14 @@ def assert_unbiased( f"{label or 'this study'}: a single replicate has no spread, so its " "distance from the truth cannot be told apart from noise" ) + if not result.mc_se and result.bias: + raise ValueError( + f"{label or 'this study'}: all {result.reps} estimates equal " + f"{result.bias + result.truth:.6g}, missing the truth by " + f"{result.bias:+.6g}, with no spread to measure that against. A " + "deterministic estimator is biased by exactly that much; a discrete " + "one may need more replicates before it varies." + ) if not abs(result.bias_t) < sigmas: raise AssertionError( f"{label}: bias {result.bias:+.6f} is {result.bias_t:+.2f} Monte " diff --git a/src/simcheck/results.py b/src/simcheck/results.py index bbe41d5..b2ce97e 100644 --- a/src/simcheck/results.py +++ b/src/simcheck/results.py @@ -170,14 +170,16 @@ def bias_t(self) -> float: """Bias in units of its own Monte Carlo standard error. A t statistic for the null that the estimator is unbiased. When the - estimator did not vary, there is no noise for a bias to hide in: the t - statistic is zero if every estimate equals the truth and infinite - otherwise. Reporting zero for a constant that misses the truth would - certify a biased estimator as unbiased because it is *also* deterministic. + estimates did not vary it is zero if every one equals the truth and NaN + otherwise. NaN, not zero: zero would certify a constant that misses the + truth as unbiased. And NaN, not infinity: identical estimates may come + from a deterministic estimator, whose miss is a bias, or from a discrete + one that happened not to vary in this study, whose miss may be noise the + study was too small to see. The study cannot tell which. """ if self.mc_se: return self.bias / self.mc_se - return math.copysign(math.inf, self.bias) if self.bias else 0.0 + return math.nan if self.bias else 0.0 @property def coverage(self) -> float: diff --git a/tests/test_negative.py b/tests/test_negative.py index b3107cf..3f7051b 100644 --- a/tests/test_negative.py +++ b/tests/test_negative.py @@ -248,14 +248,29 @@ def _constant_study(value: float, truth: float, reps: int = 400) -> MonteCarloRe ) -def test_assert_unbiased_fails_on_a_deterministic_biased_estimator(): - """A constant that misses the truth is biased, with no noise to excuse it. +def test_assert_unbiased_does_not_pass_a_constant_that_misses_the_truth(): + """Identical estimates that miss the truth must not pass. The Monte Carlo standard error is exactly zero here, and the gate used to read that as a t statistic of zero -- so returning 5 for a truth of 2 passed. + It raises rather than asserting bias, because the same study arises from a + discrete unbiased estimator that happened not to vary. """ - with pytest.raises(AssertionError, match=r"\+inf Monte"): - assert_unbiased(_constant_study(5.0, truth=2.0), "constant, wrong") + study = _constant_study(5.0, truth=2.0) + assert np.isnan(study.bias_t) + with pytest.raises(ValueError, match="no spread"): + assert_unbiased(study, "constant, wrong") + + +def test_assert_unbiased_does_not_call_an_unvarying_discrete_estimator_biased(): + """An unbiased Bernoulli estimator that never varied is not reported biased. + + With a success probability of 0.001, all 400 draws are zero about two times + in three. The estimator is unbiased; the study is too small to show it. + """ + study = _constant_study(0.0, truth=0.001) + with pytest.raises(ValueError, match="more replicates"): + assert_unbiased(study, "rare Bernoulli") def test_assert_unbiased_passes_a_deterministic_exact_estimator():