From 62ecca20c10ac25e2b637f2a60916e346f9dc719 Mon Sep 17 00:00:00 2001 From: Subidh Khanal Date: Tue, 27 Oct 2026 23:14:44 +0530 Subject: [PATCH 1/3] fix(backtests): keep a pipeline order's zero confidence instead of reading it as 0.75 pipeline_output_to_decision read an order's confidence as float(value or 0.75), so confidence 0 became 0.75, passed the 0.3 floor and traded. Only a missing or empty confidence now takes the 0.75 default. --- .../infrastructure/llm/pipeline_runner.py | 8 ++++++- .../test_portfolio_manager_move.py | 22 +++++++++++++++++++ .../llm/test_pipeline_runner.py | 14 ++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/dashboard/backend/infrastructure/llm/pipeline_runner.py b/dashboard/backend/infrastructure/llm/pipeline_runner.py index 92252f46..882d7637 100644 --- a/dashboard/backend/infrastructure/llm/pipeline_runner.py +++ b/dashboard/backend/infrastructure/llm/pipeline_runner.py @@ -247,7 +247,13 @@ def pipeline_output_to_decision(parsed: Dict[str, Any]) -> Optional[Dict[str, An { "action": side, "symbol": order.get("symbol"), - "confidence": float(order.get("confidence", 0.75) or 0.75), + # Default only a missing confidence: 0 is the model + # saying it is not confident, not an absent value. + "confidence": float( + 0.75 + if order.get("confidence") in (None, "") + else order["confidence"] + ), "reasoning": order.get("reason") or order.get("rationale") or "", diff --git a/dashboard/backend/tests/backtesting/test_portfolio_manager_move.py b/dashboard/backend/tests/backtesting/test_portfolio_manager_move.py index 1627274f..19ff440a 100644 --- a/dashboard/backend/tests/backtesting/test_portfolio_manager_move.py +++ b/dashboard/backend/tests/backtesting/test_portfolio_manager_move.py @@ -377,6 +377,28 @@ def test_strict_pipeline_accepts_empty_orders_as_model_hold(): assert pm.llm_decisions == 1 +def test_pipeline_order_with_zero_confidence_does_not_trade(): + """A pipeline order the model gave confidence 0 stays under the 0.3 floor.""" + pm = CanonicalPortfolioManager(100000) + reply = {"orders": [{"symbol": "AAPL", "side": "buy", "qty": 10, "confidence": 0}]} + client = _FakeClient(_FakeResp(json.dumps(reply), _FakeUsage(3, 2))) + + result = pm.make_trading_decision_with_llm( + _llm_state(), + client, + pipeline=[ + { + "label": "Trading instruction", + "prompt": "Return the orders for this bar.", + "outputFormat": '{"orders": []}', + } + ], + strict_llm=True, + ) + + assert result == {"actions": []} + + @pytest.mark.parametrize( "action", [ diff --git a/dashboard/backend/tests/infrastructure/llm/test_pipeline_runner.py b/dashboard/backend/tests/infrastructure/llm/test_pipeline_runner.py index 6264a08e..2d6da5fc 100644 --- a/dashboard/backend/tests/infrastructure/llm/test_pipeline_runner.py +++ b/dashboard/backend/tests/infrastructure/llm/test_pipeline_runner.py @@ -101,6 +101,20 @@ def test_pipeline_output_to_decision_orders(): assert decision["actions"][0]["position_size"] == 10 +def test_pipeline_order_with_zero_confidence_keeps_it(): + """``confidence: 0`` is the model saying it is not confident at all; only + a missing confidence takes the 0.75 default.""" + decision = pipeline_output_to_decision( + {"orders": [{"symbol": "AAPL", "side": "buy", "qty": 10, "confidence": 0}]} + ) + assert decision["actions"][0]["confidence"] == 0.0 + + decision = pipeline_output_to_decision( + {"orders": [{"symbol": "AAPL", "side": "buy", "qty": 10}]} + ) + assert decision["actions"][0]["confidence"] == 0.75 + + def test_pipeline_output_to_decision_actions_passthrough(): parsed = { "actions": [ From adeea77b63ea9ed7e4a54a6adfa79df375cbcbca Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Thu, 29 Oct 2026 22:49:15 -0400 Subject: [PATCH 2/3] fix(backtests): read every LLM action's confidence by one 0-1 rule Review of the zero-confidence fix found the same "not confident, but it still trades" shape, and crash variants, around it: - [] / {} / "high" confidence raised out of pipeline_output_to_decision, losing the step's usage and aborting a paid run after its calls were charged; NaN, Infinity and 20 cleared the 0.3 floor (nan < 0.3 is False). - risk_actions hard-coded confidence 0.8, so a zero-confidence stop-loss sold; size_pct 0 read as 1.0 (the whole position), "50%" raised. - an `actions` reply reached the gate's {confidence:.0%} raw, so "0.9" and null raised. - int(qty) turned "10.5", NaN and 0.6 into 0 (auto-size) and raised OverflowError on Infinity. read_confidence (validator.py) is now the one rule for the gate and every pipeline envelope: a missing confidence takes the path's default, a number from 0 to 1 (or a numeric string) is kept, anything else skips the action. The converter hands unusable confidences and every qty on as written, so the gate skips and logs what the model sent. The three defaults (0.5, 0.75, 0.8) are named in one place but kept apart: unifying would resize every default-instruction pipeline order. Co-Authored-By: Claude Opus 5.5 --- dashboard/backend/SECURITY.md | 2 +- .../domain/backtesting/decision_tape.py | 4 +- .../domain/backtesting/portfolio_manager.py | 16 +++- .../infrastructure/llm/pipeline_runner.py | 50 +++++++--- .../backend/infrastructure/llm/validator.py | 63 ++++++++++++- .../test_portfolio_manager_move.py | 70 +++++++++++++- .../domain/backtesting/test_decision_tape.py | 4 +- .../backtesting/test_decision_tape_engine.py | 14 +-- .../llm/test_pipeline_runner.py | 94 +++++++++++++++++-- dashboard/backend/tests/test_llm_validator.py | 35 ++++++- 10 files changed, 313 insertions(+), 39 deletions(-) diff --git a/dashboard/backend/SECURITY.md b/dashboard/backend/SECURITY.md index 307df967..0561d0fb 100644 --- a/dashboard/backend/SECURITY.md +++ b/dashboard/backend/SECURITY.md @@ -120,7 +120,7 @@ It applies these constraints inline instead: | Action count | Capped at one action per universe symbol (`len(allowed_symbols)`); excess entries ignored, or the whole batch rejected on a strict run | | Per-order size | `position_size > MAX_ORDER_SHARES` (10k, same constant as the validator) skipped; unparseable/non-finite sizes skipped | | Cash bound | Buys must fit in available cash; sells limited to held shares | -| Confidence floor | Actions with confidence < 0.3 skipped | +| Confidence floor | Read by `read_confidence` (`infrastructure/llm/validator.py`): a missing confidence takes the path's default (0.5; a pipeline's `orders` 0.75, `risk_actions` 0.8), any other value that is not a number from 0 to 1 (a bool, text, NaN, `25`) skips the action, and actions below 0.3 are skipped | A custom `strategy_prompt` replaces only the *strategy* section of the prompt; the market snapshot, symbol allowlist, and strict JSON output contract are diff --git a/dashboard/backend/domain/backtesting/decision_tape.py b/dashboard/backend/domain/backtesting/decision_tape.py index b0228059..e6fc0ceb 100644 --- a/dashboard/backend/domain/backtesting/decision_tape.py +++ b/dashboard/backend/domain/backtesting/decision_tape.py @@ -113,8 +113,8 @@ def _intent_order(entry: Any) -> Dict[str, Any]: Not ``_pick``: the gate in ``make_trading_decision_with_llm`` reads each field with a default, so dropping a key changes the outcome -- a present - ``confidence: null`` raises in the gate's format string (the bar falls back - to rule-based) while an absent one reads as 0.5 and trades. So: + ``confidence: "high"`` skips the action (``read_confidence``) while an + absent one reads as 0.5 and trades. So: - an absent field stays absent, and a ``null`` one stays ``null``; - a string ``reasoning`` is cut to ``MAX_REASONING_CHARS`` (the gate only diff --git a/dashboard/backend/domain/backtesting/portfolio_manager.py b/dashboard/backend/domain/backtesting/portfolio_manager.py index 5245dcaf..17e70b01 100644 --- a/dashboard/backend/domain/backtesting/portfolio_manager.py +++ b/dashboard/backend/domain/backtesting/portfolio_manager.py @@ -36,9 +36,11 @@ from dashboard.backend.domain.backtesting.constants import INITIAL_CAPITAL from dashboard.backend.infrastructure.llm.validator import ( DJIA_30, + LLM_ACTION_DEFAULT_CONFIDENCE, MAX_ORDER_SHARES, create_prompt, normalize_symbol, + read_confidence, ) from dashboard.backend.domain.trading.portfolio import ( append_equity_record as _append_equity_record, @@ -869,12 +871,20 @@ def request_decision(*, recovery: bool): for llm_action in llm_actions: symbol = normalize_symbol(str(llm_action.get("symbol") or "")) action_type = llm_action.get("action", "hold").lower() - confidence = llm_action.get("confidence", 0.5) + raw_confidence = llm_action.get("confidence") + confidence = read_confidence( + raw_confidence, LLM_ACTION_DEFAULT_CONFIDENCE + ) reasoning = llm_action.get("reasoning", "") - + + if confidence is None: + print(f"\n Processing: {symbol} ({action_type.upper()}, conf={raw_confidence!r})") + print(f" ⚠️ {symbol}: Skip (confidence {raw_confidence!r} is not a number from 0 to 1)") + continue + print(f"\n Processing: {symbol} ({action_type.upper()}, conf={confidence:.0%})") print(f" Reasoning: {reasoning[:60]}...") - + # Skip low-confidence decisions if confidence < 0.3: print(f" ⏸️ Skipping (confidence {confidence:.0%} too low)") diff --git a/dashboard/backend/infrastructure/llm/pipeline_runner.py b/dashboard/backend/infrastructure/llm/pipeline_runner.py index 882d7637..174e30f8 100644 --- a/dashboard/backend/infrastructure/llm/pipeline_runner.py +++ b/dashboard/backend/infrastructure/llm/pipeline_runner.py @@ -14,6 +14,7 @@ import copy import json +import math import os import re from typing import Any, Dict, List, Optional, Tuple @@ -32,6 +33,12 @@ is_transient_provider_failure, ) from dashboard.backend.infrastructure.llm.execution.log_safe import log_safe_token +from dashboard.backend.infrastructure.llm.validator import ( + PIPELINE_ORDER_DEFAULT_CONFIDENCE, + PIPELINE_RISK_DEFAULT_CONFIDENCE, + model_number, + read_confidence, +) # Owned by domain/agents/defaults.py and re-exported here, where the engine, # the route and the tests have always imported them from. @@ -239,25 +246,27 @@ def pipeline_output_to_decision(parsed: Dict[str, Any]) -> Optional[Dict[str, An qty = order.get( "qty", order.get("quantity", order.get("position_size", 0)) ) - try: - position_size = int(qty) - except (TypeError, ValueError): - position_size = 0 + raw_confidence = order.get("confidence") + confidence = read_confidence( + raw_confidence, PIPELINE_ORDER_DEFAULT_CONFIDENCE + ) normalized.append( { "action": side, "symbol": order.get("symbol"), - # Default only a missing confidence: 0 is the model - # saying it is not confident, not an absent value. - "confidence": float( - 0.75 - if order.get("confidence") in (None, "") - else order["confidence"] + # An unusable confidence is handed on as written: the + # gate's read_confidence skips the order and logs it. + "confidence": ( + raw_confidence if confidence is None else confidence ), "reasoning": order.get("reason") or order.get("rationale") or "", - "position_size": position_size, + # As written, too. The gate coerces the size, skips one + # it cannot read and sizes a missing or zero one from + # confidence; int() here turned "10.5" and NaN into 0, + # a request to auto-size, and raised on Infinity. + "position_size": qty, "stop_loss_price": order.get("stop_loss_price"), "take_profit_price": order.get("take_profit_price"), } @@ -286,12 +295,27 @@ def pipeline_output_to_decision(parsed: Dict[str, Any]) -> Optional[Dict[str, An if action_type in ("buy", "sell", "hold") else "hold" ) - size_pct = float(risk.get("size_pct", 1.0) or 1.0) + # A missing size sells the whole position. One the model gave + # must be a positive number: 0 asks to sell nothing, and a + # malformed size is not read as "all of it" (the same rule as + # read_confidence), so either one holds. + size_pct = model_number(risk.get("size_pct"), 1.0) + if side == "sell" and ( + size_pct is None + or not (math.isfinite(size_pct) and size_pct > 0) + ): + side = "hold" + raw_confidence = risk.get("confidence") + confidence = read_confidence( + raw_confidence, PIPELINE_RISK_DEFAULT_CONFIDENCE + ) normalized.append( { "action": side, "symbol": risk.get("symbol"), - "confidence": 0.8, + "confidence": ( + raw_confidence if confidence is None else confidence + ), "reasoning": risk.get("reason") or risk.get("rationale") or action_type, diff --git a/dashboard/backend/infrastructure/llm/validator.py b/dashboard/backend/infrastructure/llm/validator.py index 371f6f11..742856ba 100644 --- a/dashboard/backend/infrastructure/llm/validator.py +++ b/dashboard/backend/infrastructure/llm/validator.py @@ -13,6 +13,7 @@ import json import logging +import math import re from typing import Dict, Any, List, Optional, Tuple from enum import Enum @@ -82,6 +83,66 @@ def normalize_symbol(symbol: str) -> str: return trimmed.upper() +# What an action's confidence reads as when the model leaves it out, per +# backtest decision path. Kept apart on purpose: the default pipeline +# instruction asks for no confidence, so the orders it produces carry 0.75 and +# one without a qty is sized from it. Moving it to 0.5 would shrink those buys +# by a third and stop new runs comparing with old ones. +LLM_ACTION_DEFAULT_CONFIDENCE = 0.5 # the gate, for an ``actions`` reply +PIPELINE_ORDER_DEFAULT_CONFIDENCE = 0.75 # a pipeline's ``orders`` +PIPELINE_RISK_DEFAULT_CONFIDENCE = 0.8 # a pipeline's ``risk_actions`` + + +def model_number(value: Any, default: Optional[float] = None) -> Optional[float]: + """A number a model wrote into its JSON reply, as a float. + + A missing value (``None``, or a string of only whitespace) is ``default``. + A numeric string such as ``"0.4"`` is read as its number, as ``float()`` + always read it on these paths. Anything else that is not a number -- a + bool, text such as ``"high"``, a list or an object -- is ``None``: + ``int(True) == 1`` is not what a model meant. The result is not range + checked and can be NaN or infinite; an int too large for a float reads + as infinite rather than raising ``OverflowError``. + """ + if value is None or (isinstance(value, str) and not value.strip()): + return default + if isinstance(value, bool): + return None + if isinstance(value, int): + try: + return float(value) + except OverflowError: + return math.inf if value > 0 else -math.inf + if isinstance(value, (float, str)): + try: + return float(value) + except ValueError: + return None + return None + + +def _confidence_in_range(confidence: float) -> bool: + return math.isfinite(confidence) and 0.0 <= confidence <= 1.0 + + +def read_confidence(value: Any, default: float) -> Optional[float]: + """An action's 0-1 confidence, or ``None`` when the action must be skipped. + + One rule for the backtest gate and for every pipeline envelope it reads: + a confidence the model left out takes ``default``, and one it gave must be + a number from 0 to 1. ``0`` is the model saying it is not confident, not a + missing value. Anything else it gave -- a bool, ``"high"``, a list, NaN, + infinity, or a number off the scale such as ``25`` -- is ``None``: reading + a scale into it (``25`` as 25%) or a default over it would trade on a + confidence the model never stated. The range is the one + ``LLMTradingDecision`` enforces for external agents. + """ + number = model_number(value, default) + if number is None or not _confidence_in_range(number): + return None + return number + + class TradingAction(str, Enum): """Allowed trading actions""" BUY = "buy" @@ -128,7 +189,7 @@ def validate_confidence(cls, v): """Confidence must be 0.0 to 1.0""" if not isinstance(v, (int, float)): raise ValueError(f"Confidence must be numeric, got {type(v)}") - if not 0.0 <= v <= 1.0: + if not _confidence_in_range(v): raise ValueError(f"Confidence {v} out of range [0.0, 1.0]") return float(v) diff --git a/dashboard/backend/tests/backtesting/test_portfolio_manager_move.py b/dashboard/backend/tests/backtesting/test_portfolio_manager_move.py index 19ff440a..534c27b3 100644 --- a/dashboard/backend/tests/backtesting/test_portfolio_manager_move.py +++ b/dashboard/backend/tests/backtesting/test_portfolio_manager_move.py @@ -377,12 +377,12 @@ def test_strict_pipeline_accepts_empty_orders_as_model_hold(): assert pm.llm_decisions == 1 -def test_pipeline_order_with_zero_confidence_does_not_trade(): - """A pipeline order the model gave confidence 0 stays under the 0.3 floor.""" +def _paid_pipeline_decision(reply): + """One strict pipeline bar on a fail-closed (billed) client, which + re-raises any error that is not a transient provider failure.""" pm = CanonicalPortfolioManager(100000) - reply = {"orders": [{"symbol": "AAPL", "side": "buy", "qty": 10, "confidence": 0}]} client = _FakeClient(_FakeResp(json.dumps(reply), _FakeUsage(3, 2))) - + client.fail_closed = True result = pm.make_trading_decision_with_llm( _llm_state(), client, @@ -395,8 +395,70 @@ def test_pipeline_order_with_zero_confidence_does_not_trade(): ], strict_llm=True, ) + return pm, result + + +def test_pipeline_order_with_zero_confidence_does_not_trade(): + """A pipeline order the model gave confidence 0 stays under the 0.3 floor, + and the bar still counts as one the model drove.""" + pm, result = _paid_pipeline_decision( + {"orders": [{"symbol": "AAPL", "side": "buy", "qty": 10, "confidence": 0}]} + ) + + assert result == {"actions": []} + assert pm.llm_calls == 1 + assert pm.llm_decisions == 1 + + +@pytest.mark.parametrize( + "confidence", + [[], {}, float("nan"), float("inf"), 20, -0.1, True, False, "high", + pytest.param(10**400, id="int-past-float")], +) +def test_pipeline_order_with_unusable_confidence_is_skipped_not_aborted(confidence, capsys): + """``[]``/``{}``/``"high"`` raised out of the converter and aborted a paid + run after its calls were charged; NaN, Infinity and 20 cleared the 0.3 + floor (``nan < 0.3`` is False) and bought. Each is now one skipped order + on a bar the model still drove.""" + pm, result = _paid_pipeline_decision( + {"orders": [{"symbol": "AAPL", "side": "buy", "qty": 10, "confidence": confidence}]} + ) assert result == {"actions": []} + assert pm.llm_calls == 1 + assert pm.llm_decisions == 1 + assert "is not a number from 0 to 1" in capsys.readouterr().out + + +@pytest.mark.parametrize( + "confidence, shares", + [("0.9", [5]), (None, [5]), ("0.2", [])], +) +def test_pipeline_actions_reply_confidence_is_read_not_formatted_raw(confidence, shares): + """An ``actions`` reply reached the gate's ``{confidence:.0%}`` as + written: "0.9" raised ValueError and null TypeError, aborting a paid run. + A numeric string is now read as its number and null as the 0.5 default.""" + pm, result = _paid_pipeline_decision( + {"actions": [{"symbol": "AAPL", "action": "buy", "position_size": 5, + "confidence": confidence, "reasoning": "x"}]} + ) + + assert [a["shares"] for a in result["actions"]] == shares + assert pm.llm_decisions == 1 + + +@pytest.mark.parametrize("qty", ["10.5", float("nan"), float("inf")]) +def test_pipeline_order_with_unreadable_qty_is_skipped_not_auto_sized(qty, capsys): + """The converter turned "10.5" and NaN into 0, which the gate auto-sizes + from confidence, and raised OverflowError on Infinity (aborting a paid + run). The gate now sees the size the model wrote and skips it.""" + pm, result = _paid_pipeline_decision( + {"orders": [{"symbol": "AAPL", "side": "buy", "qty": qty, "confidence": 0.9}]} + ) + + assert result == {"actions": []} + assert pm.llm_decisions == 1 + assert "unparseable position_size" in capsys.readouterr().out @pytest.mark.parametrize( diff --git a/dashboard/backend/tests/domain/backtesting/test_decision_tape.py b/dashboard/backend/tests/domain/backtesting/test_decision_tape.py index e2bccc3d..4b204139 100644 --- a/dashboard/backend/tests/domain/backtesting/test_decision_tape.py +++ b/dashboard/backend/tests/domain/backtesting/test_decision_tape.py @@ -74,8 +74,8 @@ def _one_order(entry): def test_intent_keeps_present_null_fields_and_omits_absent_ones(): - """The gate reads each field with a default: a null confidence raises in - its format string, an absent one reads as 0.5. Dropping null is not exact.""" + """The gate reads each field with a default, so the tape keeps a field's + presence exactly: a null stays null and an absent field stays absent.""" intent = _one_order({"symbol": "AAPL", "action": "buy", "confidence": None, "reasoning": None}) assert intent["orders"][0] == {"symbol": "AAPL", "action": "buy", "confidence": None, "reasoning": None} assert tape.orders_for_replay(intent) == [intent["orders"][0]] diff --git a/dashboard/backend/tests/domain/backtesting/test_decision_tape_engine.py b/dashboard/backend/tests/domain/backtesting/test_decision_tape_engine.py index 04b1cd41..173500a8 100644 --- a/dashboard/backend/tests/domain/backtesting/test_decision_tape_engine.py +++ b/dashboard/backend/tests/domain/backtesting/test_decision_tape_engine.py @@ -327,10 +327,11 @@ def test_intent_replay_through_the_gate_reproduces_an_llm_run(monkeypatch, store def test_intent_replay_keeps_null_and_non_finite_fields(monkeypatch, store): - """Finding 3: ``confidence: null`` makes the gate raise (the bar falls back - to rule-based) while an absent confidence reads as 0.5 and trades; an - ``Infinity`` position_size is skipped while ``null`` becomes a - confidence-sized buy. The tape must hand the replay the same values.""" + """Finding 3: a ``null`` reasoning makes the gate raise (the bar falls back + to rule-based) while an absent one trades; an ``Infinity`` position_size + is skipped while ``null`` becomes a confidence-sized buy. ``confidence: + null`` reads as the 0.5 default, as an absent one does. The tape must hand + the replay the same values.""" script = [ json.dumps({"actions": [{"symbol": "AAPL", "action": "buy", "position_size": 3, "confidence": None, "reasoning": "null conf"}]}), json.dumps({"actions": [{"symbol": "MSFT", "action": "buy", "position_size": float("inf"), "confidence": 0.9, "reasoning": "inf size"}]}), @@ -341,9 +342,10 @@ def test_intent_replay_keeps_null_and_non_finite_fields(monkeypatch, store): _bt, first, first_curve = _run(monkeypatch, run_id="agent_intent_edge", llm_client=_PipelineStub(script), pipeline=pipeline) tape = trace_service.load_decision_tape("agent_intent_edge") - assert tape[0]["decision"]["driver"] == "llm_fallback" + assert tape[0]["decision"]["driver"] == "llm" assert tape[0]["decision"]["intent"]["orders"][0]["confidence"] is None - assert tape[0]["decision"]["gate_rewrote"] is False # a fallback is not a rewrite + assert tape[0]["decision"]["actions"][0]["confidence"] == 0.5 + assert tape[0]["decision"]["gate_rewrote"] is False assert tape[1]["decision"]["intent"]["orders"][0]["position_size"] is None assert tape[1]["decision"]["intent"]["orders"][0]["raw"] == {"position_size": "Infinity"} assert tape[2]["decision"]["driver"] == "llm_fallback" # reasoning[:60] on None diff --git a/dashboard/backend/tests/infrastructure/llm/test_pipeline_runner.py b/dashboard/backend/tests/infrastructure/llm/test_pipeline_runner.py index 2d6da5fc..3b708aee 100644 --- a/dashboard/backend/tests/infrastructure/llm/test_pipeline_runner.py +++ b/dashboard/backend/tests/infrastructure/llm/test_pipeline_runner.py @@ -24,6 +24,10 @@ trading_day_key, _build_step_prompt, ) +from dashboard.backend.infrastructure.llm.validator import ( + LLM_ACTION_DEFAULT_CONFIDENCE, + read_confidence, +) class _PipelineResponse: @@ -101,18 +105,60 @@ def test_pipeline_output_to_decision_orders(): assert decision["actions"][0]["position_size"] == 10 -def test_pipeline_order_with_zero_confidence_keeps_it(): +@pytest.mark.parametrize( + "fields, expected", + [ + ({"confidence": 0}, 0.0), + ({"confidence": 0.0}, 0.0), + ({"confidence": "0"}, 0.0), + ({"confidence": "0.4"}, 0.4), + ({}, 0.75), + ({"confidence": None}, 0.75), + ({"confidence": ""}, 0.75), + ], +) +def test_pipeline_order_confidence_defaults_only_when_missing(fields, expected): """``confidence: 0`` is the model saying it is not confident at all; only - a missing confidence takes the 0.75 default.""" + a missing one (absent, null or empty) takes the 0.75 default.""" + order = {"symbol": "AAPL", "side": "buy", "qty": 10, **fields} + decision = pipeline_output_to_decision({"orders": [order]}) + assert decision["actions"][0]["confidence"] == expected + + +@pytest.mark.parametrize( + "confidence", + [[], {}, False, True, "high", float("nan"), float("inf"), 20, -0.1, + pytest.param(10**400, id="int-past-float")], +) +def test_pipeline_order_unusable_confidence_is_handed_on_as_written(confidence): + """The converter raised on ``[]``/``{}`` and ``"high"``, losing the step's + usage and aborting a paid run, and read ``false`` as 0.75. It now hands + the value on unchanged, for the gate to skip the order and log it.""" decision = pipeline_output_to_decision( - {"orders": [{"symbol": "AAPL", "side": "buy", "qty": 10, "confidence": 0}]} + {"orders": [{"symbol": "AAPL", "side": "buy", "qty": 10, "confidence": confidence}]} ) - assert decision["actions"][0]["confidence"] == 0.0 + handed_on = decision["actions"][0]["confidence"] + assert handed_on is confidence + assert read_confidence(handed_on, LLM_ACTION_DEFAULT_CONFIDENCE) is None + +@pytest.mark.parametrize( + "qty", + ["10.5", float("nan"), float("inf"), pytest.param(10**400, id="int-past-float"), 0.6], +) +def test_pipeline_order_qty_is_handed_on_as_written(qty): + """``int(qty)`` turned "10.5", NaN and 0.6 into 0, which the gate reads as + "size it for me", and raised OverflowError on Infinity. The gate coerces + and checks the size itself.""" decision = pipeline_output_to_decision( - {"orders": [{"symbol": "AAPL", "side": "buy", "qty": 10}]} + {"orders": [{"symbol": "AAPL", "side": "buy", "qty": qty}]} ) - assert decision["actions"][0]["confidence"] == 0.75 + assert decision["actions"][0]["position_size"] is qty + + +def test_pipeline_order_without_qty_still_asks_the_gate_to_size_it(): + decision = pipeline_output_to_decision({"orders": [{"symbol": "AAPL", "side": "buy"}]}) + assert decision["actions"][0]["position_size"] == 0 def test_pipeline_output_to_decision_actions_passthrough(): @@ -157,6 +203,42 @@ def test_pipeline_output_to_decision_risk_actions(): ] +def test_pipeline_risk_action_keeps_the_models_confidence(): + """``risk_actions`` hard-coded confidence 0.8, so a stop-loss the model + gave confidence 0 still cleared the 0.3 floor and sold.""" + decision = pipeline_output_to_decision( + { + "risk_actions": [ + {"symbol": "AAPL", "action": "stop_loss", "confidence": 0}, + {"symbol": "MSFT", "action": "stop_loss"}, + {"symbol": "JPM", "action": "stop_loss", "confidence": "high"}, + ] + } + ) + assert [a["confidence"] for a in decision["actions"]] == [0.0, 0.8, "high"] + + +@pytest.mark.parametrize( + "size_pct", [0, 0.0, "0", -0.5, "50%", float("nan"), float("inf"), True, []] +) +def test_pipeline_risk_action_size_it_cannot_use_holds(size_pct): + """``float(size_pct or 1.0)`` read 0 as 1.0, the whole position, and + raised on "50%" and Infinity. A size the model gave must be positive.""" + decision = pipeline_output_to_decision( + {"risk_actions": [{"symbol": "AAPL", "action": "trail", "size_pct": size_pct}]} + ) + assert decision["actions"][0]["action"] == "hold" + + +@pytest.mark.parametrize("fields", [{}, {"size_pct": None}, {"size_pct": ""}]) +def test_pipeline_risk_action_without_a_size_sells_the_position(fields): + decision = pipeline_output_to_decision( + {"risk_actions": [{"symbol": "AAPL", "action": "trail", **fields}]} + ) + assert decision["actions"][0]["action"] == "sell" + assert decision["actions"][0]["position_size"] == 100 + + @pytest.mark.parametrize( "parsed", [{"actions": []}, {"orders": []}, {"risk_actions": []}], diff --git a/dashboard/backend/tests/test_llm_validator.py b/dashboard/backend/tests/test_llm_validator.py index 4c2b6f3d..ca6f2570 100644 --- a/dashboard/backend/tests/test_llm_validator.py +++ b/dashboard/backend/tests/test_llm_validator.py @@ -16,7 +16,9 @@ from dashboard.backend.infrastructure.llm.validator import ( validate_llm_response, TradingAction, - create_safe_prompt + create_safe_prompt, + model_number, + read_confidence, ) @@ -454,3 +456,34 @@ def test_e2e_valid_buy_to_sell(self, portfolio_state, current_prices): if __name__ == "__main__": pytest.main([__file__, "-v"]) + + +class TestReadConfidence: + """read_confidence: one confidence rule for the backtest gate and every + pipeline envelope it reads.""" + + @pytest.mark.parametrize("value", [None, "", " "]) + def test_missing_confidence_takes_the_default(self, value): + assert read_confidence(value, 0.75) == 0.75 + + @pytest.mark.parametrize( + "value, expected", + [(0, 0.0), (0.0, 0.0), ("0", 0.0), (0.3, 0.3), (" 0.4 ", 0.4), (1, 1.0)], + ) + def test_a_number_from_0_to_1_is_kept(self, value, expected): + assert read_confidence(value, 0.75) == expected + + @pytest.mark.parametrize( + "value", + [False, True, "high", "50%", [], {}, [0.5], float("nan"), "NaN", + float("inf"), float("-inf"), 25, 1.5, -0.1, + pytest.param(10**400, id="int-past-float")], + ) + def test_anything_else_skips_the_action(self, value): + assert read_confidence(value, 0.75) is None + + def test_model_number_reads_an_oversized_int_as_infinite(self): + assert model_number(10**400) == float("inf") + assert model_number(-(10**400)) == float("-inf") + assert model_number(True) is None + assert model_number(None, 1.0) == 1.0 From 16dd8fffeedc039b194f1754f06cb269be1365a0 Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Thu, 29 Oct 2026 22:59:41 -0400 Subject: [PATCH 3/3] test(llm): raise the post-parse loop error with a non-string action A numeric-string confidence ("0.9") is now read by the 0-1 rule instead of crashing the processing loop, so the H6 test that relied on that crash raises through action.lower() instead. It still pins the same rule: the call is billed but is not counted as a decision. Co-Authored-By: Claude Opus 5.5 --- dashboard/backend/tests/llm/test_backtest_harness.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dashboard/backend/tests/llm/test_backtest_harness.py b/dashboard/backend/tests/llm/test_backtest_harness.py index e277d5e7..8ea4e051 100644 --- a/dashboard/backend/tests/llm/test_backtest_harness.py +++ b/dashboard/backend/tests/llm/test_backtest_harness.py @@ -656,7 +656,9 @@ def test_filtered_actions_still_count_as_a_decision(): def test_post_parse_exception_is_billed_but_not_a_decision(): # Valid JSON with a non-empty actions list, but a field has the wrong type - # (confidence as a string "0.9"), so the per-action processing loop raises. + # (action as a number, which has no .lower()), so the per-action processing + # loop raises. (This used confidence "0.9" until the gate learned to read + # numeric strings, which made that a usable confidence.) # The blanket except converts the whole step to a pure rule-based fallback — # the returned actions are the rule-based engine's, not the model's — so it # must NOT count toward model coverage even though parsing "succeeded". @@ -664,7 +666,7 @@ def test_post_parse_exception_is_billed_but_not_a_decision(): # 100% coverage while trading fully rule-based, defeating the H6 guard.) pm = bha.PortfolioManager(100000) resp_text = json.dumps({"actions": [ - {"symbol": "AAPL", "action": "buy", "confidence": "0.9", + {"symbol": "AAPL", "action": 7, "confidence": 0.9, "reasoning": "strong", "position_size": 10}, ]}) client = _FakeClient(_FakeResponse(resp_text, usage=_FakeUsage(10, 5)))