From 6adc1a114c88645eb978ef88694094f2e250f423 Mon Sep 17 00:00:00 2001 From: Gaurav Gandhi Date: Sat, 29 Aug 2026 11:23:04 +0530 Subject: [PATCH] fix(evaluation): reject num_samples=0 in JudgeModelOptions at construction time parallelism_limit already has ge=1; num_samples never did, despite being the same kind of count-like field in the same class. num_samples=0 is never an intentional value anywhere in this codebase, and silently causes LlmAsJudge.evaluate_invocations to drop the affected invocation from results entirely with no error and no NOT_EVALUATED marker. --- src/google/adk/evaluation/eval_metrics.py | 1 + tests/unittests/evaluation/test_eval_config.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/src/google/adk/evaluation/eval_metrics.py b/src/google/adk/evaluation/eval_metrics.py index ea8df5b5e93..f46329d74e2 100644 --- a/src/google/adk/evaluation/eval_metrics.py +++ b/src/google/adk/evaluation/eval_metrics.py @@ -97,6 +97,7 @@ class JudgeModelOptions(EvalBaseModel): num_samples: int = Field( default=5, + ge=1, description=( "The number of times to sample the model for each invocation" " evaluation. Given that models tend to have certain degree of" diff --git a/tests/unittests/evaluation/test_eval_config.py b/tests/unittests/evaluation/test_eval_config.py index d0f0a9c4f44..d09b74d3ed0 100644 --- a/tests/unittests/evaluation/test_eval_config.py +++ b/tests/unittests/evaluation/test_eval_config.py @@ -178,6 +178,17 @@ def test_eval_metric_criterion_survives_json_round_trip(): assert criterion.judge_model_options.judge_model == "my-judge" +def test_judge_model_options_rejects_zero_num_samples(): + """num_samples=0 must be rejected, matching parallelism_limit's own ge=1. + + A zero-sample judge configuration is never a legitimate value -- it causes + LlmAsJudge.evaluate_invocations to silently drop the invocation from the + aggregated result with no error and no NOT_EVALUATED marker. + """ + with pytest.raises(ValidationError): + JudgeModelOptions(num_samples=0) + + def test_eval_config_dump_preserves_concrete_criterion_fields(): """Criteria values keep their subclass fields, and plain thresholds survive.""" eval_config = EvalConfig(