Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dashboard/backend/SECURITY.md
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ It applies these constraints inline instead:
| Action count | Capped at one action per universe symbol (`len(allowed_symbols)`); excess entries ignored, or the whole batch rejected on a strict run |
| Per-order size | `position_size > MAX_ORDER_SHARES` (10k, same constant as the validator) skipped; unparseable/non-finite sizes skipped |
| Cash bound | Buys must fit in available cash; sells limited to held shares |
| Confidence floor | Actions with confidence < 0.3 skipped |
| Confidence floor | Read by `read_confidence` (`infrastructure/llm/validator.py`): a missing confidence takes the path's default (0.5; a pipeline's `orders` 0.75, `risk_actions` 0.8), any other value that is not a number from 0 to 1 (a bool, text, NaN, `25`) skips the action, and actions below 0.3 are skipped |

A custom `strategy_prompt` replaces only the *strategy* section of the prompt;
the market snapshot, symbol allowlist, and strict JSON output contract are
Expand Down
4 changes: 2 additions & 2 deletions dashboard/backend/domain/backtesting/decision_tape.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,8 @@ def _intent_order(entry: Any) -> Dict[str, Any]:

Not ``_pick``: the gate in ``make_trading_decision_with_llm`` reads each
field with a default, so dropping a key changes the outcome -- a present
``confidence: null`` raises in the gate's format string (the bar falls back
to rule-based) while an absent one reads as 0.5 and trades. So:
``confidence: "high"`` skips the action (``read_confidence``) while an
absent one reads as 0.5 and trades. So:

- an absent field stays absent, and a ``null`` one stays ``null``;
- a string ``reasoning`` is cut to ``MAX_REASONING_CHARS`` (the gate only
Expand Down
16 changes: 13 additions & 3 deletions dashboard/backend/domain/backtesting/portfolio_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,11 @@
from dashboard.backend.domain.backtesting.constants import INITIAL_CAPITAL
from dashboard.backend.infrastructure.llm.validator import (
DJIA_30,
LLM_ACTION_DEFAULT_CONFIDENCE,
MAX_ORDER_SHARES,
create_prompt,
normalize_symbol,
read_confidence,
)
from dashboard.backend.domain.trading.portfolio import (
append_equity_record as _append_equity_record,
Expand Down Expand Up @@ -869,12 +871,20 @@ def request_decision(*, recovery: bool):
for llm_action in llm_actions:
symbol = normalize_symbol(str(llm_action.get("symbol") or ""))
action_type = llm_action.get("action", "hold").lower()
confidence = llm_action.get("confidence", 0.5)
raw_confidence = llm_action.get("confidence")
confidence = read_confidence(
raw_confidence, LLM_ACTION_DEFAULT_CONFIDENCE
)
reasoning = llm_action.get("reasoning", "")


if confidence is None:
print(f"\n Processing: {symbol} ({action_type.upper()}, conf={raw_confidence!r})")
print(f" ⚠️ {symbol}: Skip (confidence {raw_confidence!r} is not a number from 0 to 1)")
continue

print(f"\n Processing: {symbol} ({action_type.upper()}, conf={confidence:.0%})")
print(f" Reasoning: {reasoning[:60]}...")

# Skip low-confidence decisions
if confidence < 0.3:
print(f" ⏸️ Skipping (confidence {confidence:.0%} too low)")
Expand Down
46 changes: 38 additions & 8 deletions dashboard/backend/infrastructure/llm/pipeline_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import copy
import json
import math
import os
import re
from typing import Any, Dict, List, Optional, Tuple
Expand All @@ -32,6 +33,12 @@
is_transient_provider_failure,
)
from dashboard.backend.infrastructure.llm.execution.log_safe import log_safe_token
from dashboard.backend.infrastructure.llm.validator import (
PIPELINE_ORDER_DEFAULT_CONFIDENCE,
PIPELINE_RISK_DEFAULT_CONFIDENCE,
model_number,
read_confidence,
)

# Owned by domain/agents/defaults.py and re-exported here, where the engine,
# the route and the tests have always imported them from.
Expand Down Expand Up @@ -239,19 +246,27 @@ def pipeline_output_to_decision(parsed: Dict[str, Any]) -> Optional[Dict[str, An
qty = order.get(
"qty", order.get("quantity", order.get("position_size", 0))
)
try:
position_size = int(qty)
except (TypeError, ValueError):
position_size = 0
raw_confidence = order.get("confidence")
confidence = read_confidence(
raw_confidence, PIPELINE_ORDER_DEFAULT_CONFIDENCE
)
normalized.append(
{
"action": side,
"symbol": order.get("symbol"),
"confidence": float(order.get("confidence", 0.75) or 0.75),
# An unusable confidence is handed on as written: the
# gate's read_confidence skips the order and logs it.
"confidence": (
raw_confidence if confidence is None else confidence
),
"reasoning": order.get("reason")
or order.get("rationale")
or "",
"position_size": position_size,
# As written, too. The gate coerces the size, skips one
# it cannot read and sizes a missing or zero one from
# confidence; int() here turned "10.5" and NaN into 0,
# a request to auto-size, and raised on Infinity.
"position_size": qty,
"stop_loss_price": order.get("stop_loss_price"),
"take_profit_price": order.get("take_profit_price"),
}
Expand Down Expand Up @@ -280,12 +295,27 @@ def pipeline_output_to_decision(parsed: Dict[str, Any]) -> Optional[Dict[str, An
if action_type in ("buy", "sell", "hold")
else "hold"
)
size_pct = float(risk.get("size_pct", 1.0) or 1.0)
# A missing size sells the whole position. One the model gave
# must be a positive number: 0 asks to sell nothing, and a
# malformed size is not read as "all of it" (the same rule as
# read_confidence), so either one holds.
size_pct = model_number(risk.get("size_pct"), 1.0)
if side == "sell" and (
size_pct is None
or not (math.isfinite(size_pct) and size_pct > 0)
):
side = "hold"
raw_confidence = risk.get("confidence")
confidence = read_confidence(
raw_confidence, PIPELINE_RISK_DEFAULT_CONFIDENCE
)
normalized.append(
{
"action": side,
"symbol": risk.get("symbol"),
"confidence": 0.8,
"confidence": (
raw_confidence if confidence is None else confidence
),
"reasoning": risk.get("reason")
or risk.get("rationale")
or action_type,
Expand Down
63 changes: 62 additions & 1 deletion dashboard/backend/infrastructure/llm/validator.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

import json
import logging
import math
import re
from typing import Dict, Any, List, Optional, Tuple
from enum import Enum
Expand Down Expand Up @@ -82,6 +83,66 @@ def normalize_symbol(symbol: str) -> str:
return trimmed.upper()


# What an action's confidence reads as when the model leaves it out, per
# backtest decision path. Kept apart on purpose: the default pipeline
# instruction asks for no confidence, so the orders it produces carry 0.75 and
# one without a qty is sized from it. Moving it to 0.5 would shrink those buys
# by a third and stop new runs comparing with old ones.
LLM_ACTION_DEFAULT_CONFIDENCE = 0.5 # the gate, for an ``actions`` reply
PIPELINE_ORDER_DEFAULT_CONFIDENCE = 0.75 # a pipeline's ``orders``
PIPELINE_RISK_DEFAULT_CONFIDENCE = 0.8 # a pipeline's ``risk_actions``


def model_number(value: Any, default: Optional[float] = None) -> Optional[float]:
"""A number a model wrote into its JSON reply, as a float.

A missing value (``None``, or a string of only whitespace) is ``default``.
A numeric string such as ``"0.4"`` is read as its number, as ``float()``
always read it on these paths. Anything else that is not a number -- a
bool, text such as ``"high"``, a list or an object -- is ``None``:
``int(True) == 1`` is not what a model meant. The result is not range
checked and can be NaN or infinite; an int too large for a float reads
as infinite rather than raising ``OverflowError``.
"""
if value is None or (isinstance(value, str) and not value.strip()):
return default
if isinstance(value, bool):
return None
if isinstance(value, int):
try:
return float(value)
except OverflowError:
return math.inf if value > 0 else -math.inf
if isinstance(value, (float, str)):
try:
return float(value)
except ValueError:
return None
return None


def _confidence_in_range(confidence: float) -> bool:
return math.isfinite(confidence) and 0.0 <= confidence <= 1.0


def read_confidence(value: Any, default: float) -> Optional[float]:
"""An action's 0-1 confidence, or ``None`` when the action must be skipped.

One rule for the backtest gate and for every pipeline envelope it reads:
a confidence the model left out takes ``default``, and one it gave must be
a number from 0 to 1. ``0`` is the model saying it is not confident, not a
missing value. Anything else it gave -- a bool, ``"high"``, a list, NaN,
infinity, or a number off the scale such as ``25`` -- is ``None``: reading
a scale into it (``25`` as 25%) or a default over it would trade on a
confidence the model never stated. The range is the one
``LLMTradingDecision`` enforces for external agents.
"""
number = model_number(value, default)
if number is None or not _confidence_in_range(number):
return None
return number


class TradingAction(str, Enum):
"""Allowed trading actions"""
BUY = "buy"
Expand Down Expand Up @@ -128,7 +189,7 @@ def validate_confidence(cls, v):
"""Confidence must be 0.0 to 1.0"""
if not isinstance(v, (int, float)):
raise ValueError(f"Confidence must be numeric, got {type(v)}")
if not 0.0 <= v <= 1.0:
if not _confidence_in_range(v):
raise ValueError(f"Confidence {v} out of range [0.0, 1.0]")
return float(v)

Expand Down
84 changes: 84 additions & 0 deletions dashboard/backend/tests/backtesting/test_portfolio_manager_move.py
Original file line number Diff line number Diff line change
Expand Up @@ -377,6 +377,90 @@ def test_strict_pipeline_accepts_empty_orders_as_model_hold():
assert pm.llm_decisions == 1


def _paid_pipeline_decision(reply):
"""One strict pipeline bar on a fail-closed (billed) client, which
re-raises any error that is not a transient provider failure."""
pm = CanonicalPortfolioManager(100000)
client = _FakeClient(_FakeResp(json.dumps(reply), _FakeUsage(3, 2)))
client.fail_closed = True
result = pm.make_trading_decision_with_llm(
_llm_state(),
client,
pipeline=[
{
"label": "Trading instruction",
"prompt": "Return the orders for this bar.",
"outputFormat": '{"orders": []}',
}
],
strict_llm=True,
)
return pm, result


def test_pipeline_order_with_zero_confidence_does_not_trade():
"""A pipeline order the model gave confidence 0 stays under the 0.3 floor,
and the bar still counts as one the model drove."""
pm, result = _paid_pipeline_decision(
{"orders": [{"symbol": "AAPL", "side": "buy", "qty": 10, "confidence": 0}]}
)

assert result == {"actions": []}
assert pm.llm_calls == 1
assert pm.llm_decisions == 1


@pytest.mark.parametrize(
"confidence",
[[], {}, float("nan"), float("inf"), 20, -0.1, True, False, "high",
pytest.param(10**400, id="int-past-float")],
)
def test_pipeline_order_with_unusable_confidence_is_skipped_not_aborted(confidence, capsys):
"""``[]``/``{}``/``"high"`` raised out of the converter and aborted a paid
run after its calls were charged; NaN, Infinity and 20 cleared the 0.3
floor (``nan < 0.3`` is False) and bought. Each is now one skipped order
on a bar the model still drove."""
pm, result = _paid_pipeline_decision(
{"orders": [{"symbol": "AAPL", "side": "buy", "qty": 10, "confidence": confidence}]}
)

assert result == {"actions": []}
assert pm.llm_calls == 1
assert pm.llm_decisions == 1
assert "is not a number from 0 to 1" in capsys.readouterr().out


@pytest.mark.parametrize(
"confidence, shares",
[("0.9", [5]), (None, [5]), ("0.2", [])],
)
def test_pipeline_actions_reply_confidence_is_read_not_formatted_raw(confidence, shares):
"""An ``actions`` reply reached the gate's ``{confidence:.0%}`` as
written: "0.9" raised ValueError and null TypeError, aborting a paid run.
A numeric string is now read as its number and null as the 0.5 default."""
pm, result = _paid_pipeline_decision(
{"actions": [{"symbol": "AAPL", "action": "buy", "position_size": 5,
"confidence": confidence, "reasoning": "x"}]}
)

assert [a["shares"] for a in result["actions"]] == shares
assert pm.llm_decisions == 1


@pytest.mark.parametrize("qty", ["10.5", float("nan"), float("inf")])
def test_pipeline_order_with_unreadable_qty_is_skipped_not_auto_sized(qty, capsys):
"""The converter turned "10.5" and NaN into 0, which the gate auto-sizes
from confidence, and raised OverflowError on Infinity (aborting a paid
run). The gate now sees the size the model wrote and skips it."""
pm, result = _paid_pipeline_decision(
{"orders": [{"symbol": "AAPL", "side": "buy", "qty": qty, "confidence": 0.9}]}
)

assert result == {"actions": []}
assert pm.llm_decisions == 1
assert "unparseable position_size" in capsys.readouterr().out


@pytest.mark.parametrize(
"action",
[
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,8 @@ def _one_order(entry):


def test_intent_keeps_present_null_fields_and_omits_absent_ones():
"""The gate reads each field with a default: a null confidence raises in
its format string, an absent one reads as 0.5. Dropping null is not exact."""
"""The gate reads each field with a default, so the tape keeps a field's
presence exactly: a null stays null and an absent field stays absent."""
intent = _one_order({"symbol": "AAPL", "action": "buy", "confidence": None, "reasoning": None})
assert intent["orders"][0] == {"symbol": "AAPL", "action": "buy", "confidence": None, "reasoning": None}
assert tape.orders_for_replay(intent) == [intent["orders"][0]]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -327,10 +327,11 @@ def test_intent_replay_through_the_gate_reproduces_an_llm_run(monkeypatch, store


def test_intent_replay_keeps_null_and_non_finite_fields(monkeypatch, store):
"""Finding 3: ``confidence: null`` makes the gate raise (the bar falls back
to rule-based) while an absent confidence reads as 0.5 and trades; an
``Infinity`` position_size is skipped while ``null`` becomes a
confidence-sized buy. The tape must hand the replay the same values."""
"""Finding 3: a ``null`` reasoning makes the gate raise (the bar falls back
to rule-based) while an absent one trades; an ``Infinity`` position_size
is skipped while ``null`` becomes a confidence-sized buy. ``confidence:
null`` reads as the 0.5 default, as an absent one does. The tape must hand
the replay the same values."""
script = [
json.dumps({"actions": [{"symbol": "AAPL", "action": "buy", "position_size": 3, "confidence": None, "reasoning": "null conf"}]}),
json.dumps({"actions": [{"symbol": "MSFT", "action": "buy", "position_size": float("inf"), "confidence": 0.9, "reasoning": "inf size"}]}),
Expand All @@ -341,9 +342,10 @@ def test_intent_replay_keeps_null_and_non_finite_fields(monkeypatch, store):
_bt, first, first_curve = _run(monkeypatch, run_id="agent_intent_edge", llm_client=_PipelineStub(script), pipeline=pipeline)
tape = trace_service.load_decision_tape("agent_intent_edge")

assert tape[0]["decision"]["driver"] == "llm_fallback"
assert tape[0]["decision"]["driver"] == "llm"
assert tape[0]["decision"]["intent"]["orders"][0]["confidence"] is None
assert tape[0]["decision"]["gate_rewrote"] is False # a fallback is not a rewrite
assert tape[0]["decision"]["actions"][0]["confidence"] == 0.5
assert tape[0]["decision"]["gate_rewrote"] is False
assert tape[1]["decision"]["intent"]["orders"][0]["position_size"] is None
assert tape[1]["decision"]["intent"]["orders"][0]["raw"] == {"position_size": "Infinity"}
assert tape[2]["decision"]["driver"] == "llm_fallback" # reasoning[:60] on None
Expand Down
Loading
Loading