diff --git a/CHANGELOG.md b/CHANGELOG.md index 77eb29a..0a00df6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- `assert_unbiased` passed a deterministic estimator however far it missed the + truth, because a zero Monte Carlo standard error was reported as a bias t + statistic of zero. Identical estimates that miss the truth now raise + `ValueError`, since the study cannot tell a deterministic bias from a + discrete estimator that happened not to vary; `bias_t` is NaN for them and + zero only when they hit the truth. +- `assert_unbiased` raises `ValueError` on a single-replicate study instead of + passing it, as `se_ratio_tolerance` and `assert_narrower` already did. + ## [0.1.0] - 2026-08-09 ### Added diff --git a/src/simcheck/gates.py b/src/simcheck/gates.py index 23a35ba..70585d3 100644 --- a/src/simcheck/gates.py +++ b/src/simcheck/gates.py @@ -458,14 +458,35 @@ def assert_unbiased( bias too small for the study to resolve does not fail, and the study can be made more demanding simply by running more replicates. + A study whose estimates are all equal passes only if they equal the truth. + If they miss it, the study has no spread to measure the miss against, and + it raises rather than passing: the estimator may be deterministic and + biased, or discrete and unlucky, and identical estimates cannot say which. + Args: result: A completed Monte Carlo study. label: Included in the failure message. sigmas: How many Monte Carlo standard errors of slack to allow. Raises: + ValueError: If the study has fewer than two replicates, or if its + estimates are all equal and differ from the truth. Either way it has + no spread against which to tell bias from noise. AssertionError: If the bias t statistic exceeds the gate. """ + if result.reps < 2: + raise ValueError( + f"{label or 'this study'}: a single replicate has no spread, so its " + "distance from the truth cannot be told apart from noise" + ) + if not result.mc_se and result.bias: + raise ValueError( + f"{label or 'this study'}: all {result.reps} estimates equal " + f"{result.bias + result.truth:.6g}, missing the truth by " + f"{result.bias:+.6g}, with no spread to measure that against. A " + "deterministic estimator is biased by exactly that much; a discrete " + "one may need more replicates before it varies." + ) if not abs(result.bias_t) < sigmas: raise AssertionError( f"{label}: bias {result.bias:+.6f} is {result.bias_t:+.2f} Monte " diff --git a/src/simcheck/results.py b/src/simcheck/results.py index fb7a369..b2ce97e 100644 --- a/src/simcheck/results.py +++ b/src/simcheck/results.py @@ -22,6 +22,7 @@ from __future__ import annotations +import math from dataclasses import dataclass import numpy as np @@ -168,11 +169,17 @@ def mc_se(self) -> float: def bias_t(self) -> float: """Bias in units of its own Monte Carlo standard error. - A t statistic for the null that the estimator is unbiased. Zero when the - estimator is deterministic, in which case there is no sampling variation - to test against. + A t statistic for the null that the estimator is unbiased. When the + estimates did not vary it is zero if every one equals the truth and NaN + otherwise. NaN, not zero: zero would certify a constant that misses the + truth as unbiased. And NaN, not infinity: identical estimates may come + from a deterministic estimator, whose miss is a bias, or from a discrete + one that happened not to vary in this study, whose miss may be noise the + study was too small to see. The study cannot tell which. """ - return self.bias / self.mc_se if self.mc_se else 0.0 + if self.mc_se: + return self.bias / self.mc_se + return math.nan if self.bias else 0.0 @property def coverage(self) -> float: diff --git a/tests/test_negative.py b/tests/test_negative.py index 96d3194..3f7051b 100644 --- a/tests/test_negative.py +++ b/tests/test_negative.py @@ -222,6 +222,68 @@ def test_assert_unbiased_fails_on_a_biased_estimator(): assert_unbiased(_study(bias=0.5, sd=0.1), "biased") +def _constant_study(value: float, truth: float, reps: int = 400) -> MonteCarloResult: + """An estimator that returns ``value`` on every replicate. + + Parameters + ---------- + value + The estimate every replicate reports. + truth + The true value. + reps + Replicates. + + Returns + ------- + MonteCarloResult + The constructed study. + """ + return MonteCarloResult( + estimates=np.full(reps, value), + standard_errors=np.full(reps, 0.1), + covered=None, + rejected=None, + truth=truth, + ) + + +def test_assert_unbiased_does_not_pass_a_constant_that_misses_the_truth(): + """Identical estimates that miss the truth must not pass. + + The Monte Carlo standard error is exactly zero here, and the gate used to + read that as a t statistic of zero -- so returning 5 for a truth of 2 passed. + It raises rather than asserting bias, because the same study arises from a + discrete unbiased estimator that happened not to vary. + """ + study = _constant_study(5.0, truth=2.0) + assert np.isnan(study.bias_t) + with pytest.raises(ValueError, match="no spread"): + assert_unbiased(study, "constant, wrong") + + +def test_assert_unbiased_does_not_call_an_unvarying_discrete_estimator_biased(): + """An unbiased Bernoulli estimator that never varied is not reported biased. + + With a success probability of 0.001, all 400 draws are zero about two times + in three. The estimator is unbiased; the study is too small to show it. + """ + study = _constant_study(0.0, truth=0.001) + with pytest.raises(ValueError, match="more replicates"): + assert_unbiased(study, "rare Bernoulli") + + +def test_assert_unbiased_passes_a_deterministic_exact_estimator(): + """A constant that equals the truth is unbiased, and must not fire.""" + assert_unbiased(_constant_study(2.0, truth=2.0), "constant, right") + + +def test_assert_unbiased_rejects_a_single_replicate(): + """One draw cannot separate bias from noise, however far off it lands.""" + with pytest.raises(ValueError, match="single replicate"): + assert_unbiased(_constant_study(1e6, truth=0.0, reps=1), "one draw") + + def test_assert_unbiased_resolves_smaller_bias_with_more_replicates(): """The gate must tighten as the study grows, without any edit.