From ea98c5215c09cdd9521a0e14ed2330091f47c27e Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Fri, 30 Oct 2026 20:01:44 -0400 Subject: [PATCH 1/5] fix(backtests): require sign-in for hosted-runtime backtests POST /backtest/run only 401'd guests on the pipeline runtime. A guest (X-Session-Id alone) could clone the AI Hedge Fund agent and launch a run billed to the operator's OPENROUTER_API_KEY with no Credits reservation; the per-owner slot and rate-limit key were a session id the guest mints itself. Refuse a guest on any non-pipeline runtime, before the window bound and readiness checks, so a guest cannot learn whether a runtime is armed. Co-Authored-By: Claude Opus 5.5 --- dashboard/backend/api/routers/backtests.py | 28 +++++++--- .../backend/tests/test_backtest_cancel.py | 5 ++ .../backend/tests/test_backtests_router.py | 56 +++++++++++++++++++ 3 files changed, 81 insertions(+), 8 deletions(-) diff --git a/dashboard/backend/api/routers/backtests.py b/dashboard/backend/api/routers/backtests.py index a81623ebd..c766c20b6 100644 --- a/dashboard/backend/api/routers/backtests.py +++ b/dashboard/backend/api/routers/backtests.py @@ -4121,6 +4121,26 @@ def run_backtest_endpoint( except ValueError as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc + from dashboard.backend.api.dependencies import _optional_user + + optional_user = _optional_user( + request, + request.headers.get("authorization") or request.headers.get("Authorization"), + ) + user_id = optional_user["id"] if optional_user else None + + # A hosted runtime bills the operator's own provider key with no Credits + # reservation, and a guest's per-owner slot and rate-limit key are a + # session id the guest mints itself, so neither bounds a guest. Refused + # first, before the window bound and the readiness checks, so a guest + # learns nothing about whether the runtime is armed here. Keyed on + # ``!= PIPELINE`` so a runtime added later is gated by default. + if runtime_type != PIPELINE_RUNTIME_TYPE and user_id is None: + raise HTTPException( + status_code=401, + detail="Sign in before running a hosted agent backtest.", + ) + decision_source_was_explicit = decision_source is not None profile, resolved_decision_source = _resolve_market_profile_request( data_source, @@ -4310,14 +4330,6 @@ def run_backtest_endpoint( else: print(f" Assets: default DJIA ({len(DJIA_30)})", flush=True) - from dashboard.backend.api.dependencies import _optional_user - - optional_user = _optional_user( - request, - request.headers.get("authorization") or request.headers.get("Authorization"), - ) - user_id = optional_user["id"] if optional_user else None - # Mint the id before constructing the signed worker handoff. It is both the # client-visible run identity and part of the handoff's tamper boundary. live_run_id = f"agent_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}" diff --git a/dashboard/backend/tests/test_backtest_cancel.py b/dashboard/backend/tests/test_backtest_cancel.py index 42e395154..df9bd312f 100644 --- a/dashboard/backend/tests/test_backtest_cancel.py +++ b/dashboard/backend/tests/test_backtest_cancel.py @@ -594,6 +594,11 @@ def test_backtest_run_refuses_an_over_long_hosted_window(client, monkeypatch): json={}, headers=headers, ).json()["agent"] + # Signed in after the guest clone: a guest is refused 401 before the window. + monkeypatch.setattr( + "dashboard.backend.api.dependencies._optional_user", + lambda *_args, **_kwargs: {"id": 7}, + ) response = client.post( "/backtest/run", diff --git a/dashboard/backend/tests/test_backtests_router.py b/dashboard/backend/tests/test_backtests_router.py index 3da2fd982..5c09c4c12 100644 --- a/dashboard/backend/tests/test_backtests_router.py +++ b/dashboard/backend/tests/test_backtests_router.py @@ -1004,6 +1004,58 @@ def _stub_hosted_runtime_installed(monkeypatch): monkeypatch.setattr(bt, "runtime_unavailable_reason", lambda: None) +def _sign_in(monkeypatch, user_id=7): + """Sign the caller in from here on; an agent cloned earlier stays guest-owned. + + Clone-as-guest-then-sign-in is a real user order, and the browser session + still matches the agent's owner, so access holds. + """ + monkeypatch.setattr( + "dashboard.backend.api.dependencies._optional_user", + lambda *_args, **_kwargs: {"id": user_id}, + ) + + +def test_guest_cannot_start_a_hosted_backtest(client, monkeypatch): + """A hosted run bills the operator's OpenRouter key with no reservation. + + The refusal comes before the readiness checks: with OPENROUTER_API_KEY + unset and the runtime missing, a guest still gets 401, never the 503 that + would tell it how this deployment is configured. + """ + spy = _Spy() + monkeypatch.setattr(bt, "run_backtest_background", spy) + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + monkeypatch.setattr( + bt.agent_credential_store, + "get_secret", + lambda agent_id, credential_name: "user-financial-datasets-test-key", + ) + headers = {"X-Session-Id": str(uuid.uuid4())} + cloned = client.post( + "/api/v1/agents/marketplace/ai-hedge-fund/clone", + json={}, + headers=headers, + ) + assert cloned.status_code == 200, cloned.text + + response = client.post( + "/backtest/run", + json={ + "start_date": "2026-05-01", + "end_date": "2026-05-02", + "decision_source": "llm", + "agent_id": cloned.json()["agent"]["agent_id"], + }, + headers=headers, + ) + + assert response.status_code == 401, response.text + assert "Sign in" in response.text + assert "OpenRouter" not in response.text + assert spy.calls == 0 + + def test_backtest_run_dispatches_ai_hedge_fund_runtime(client, monkeypatch): spy = _Spy() monkeypatch.setattr(bt, "run_backtest_background", spy) @@ -1023,6 +1075,7 @@ def test_backtest_run_dispatches_ai_hedge_fund_runtime(client, monkeypatch): ) assert cloned.status_code == 200 agent = cloned.json()["agent"] + _sign_in(monkeypatch) response = client.post( "/backtest/run", @@ -1061,6 +1114,7 @@ def test_ai_hedge_fund_backtest_requires_owned_agent_credential(client, monkeypa json={}, headers=headers, ).json()["agent"] + _sign_in(monkeypatch) monkeypatch.setattr( bt.agent_credential_store, "get_secret", @@ -1119,6 +1173,7 @@ def test_ai_hedge_fund_backtest_rejects_run_when_runtime_not_installed( json={}, headers=headers, ).json()["agent"] + _sign_in(monkeypatch) response = client.post( "/backtest/run", @@ -1373,6 +1428,7 @@ def test_ai_hedge_fund_requires_openrouter_not_direct_openai(client, monkeypatch json={}, headers=headers, ).json()["agent"] + _sign_in(monkeypatch) response = client.post( "/backtest/run", From bbe1d8920b231783278d04801b5be56ace566682 Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Fri, 30 Oct 2026 20:48:10 -0400 Subject: [PATCH 2/5] feat(backtests): bill AI Hedge Fund runs to the caller's ATL Credits Upstream calls its model inside its own interpreter, so ATL meters whole decisions: the route signs a platform-credits handoff (BYOK is a 422, a balance that cannot hold one decision a 402), and the child reserves a per-decision ceiling and settles at the token usage the bridge reports via langchain-core's get_usage_metadata_callback. Usage above the ceiling is capped, never recorded as overage. Unreported usage bills the ceiling on a completed or timed-out decision and is released on a crash before the usage scope opened. The existing finalize_run releases holds a killed child leaves, and the timeout card reads the spend like a pipeline run's. The Docker build now fails if the venv's langchain-core lacks the meter. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- Dockerfile | 5 + dashboard/backend/api/routers/backtests.py | 101 +++- .../backend/domain/backtesting/engine.py | 7 +- .../infrastructure/ai_hedge_fund/adapter.py | 71 ++- .../infrastructure/ai_hedge_fund/billing.py | 204 ++++++++ .../infrastructure/ai_hedge_fund/bridge.py | 67 ++- .../infrastructure/ai_hedge_fund/__init__.py | 0 .../ai_hedge_fund/test_billing.py | 457 ++++++++++++++++++ .../tests/test_admin_analytics_frontend.py | 2 +- .../backend/tests/test_analytics_frontend.py | 2 +- .../test_backtest_comparison_frontend.py | 2 +- .../backend/tests/test_backtests_router.py | 118 ++++- .../backend/tests/test_frontend_fast_boot.py | 2 +- dashboard/frontend/app.html | 2 +- dashboard/frontend/app.js | 4 +- dashboard/scripts/backtest_hourly_agent.py | 22 +- docs/architecture/backend-internals.md | 6 +- docs/architecture/configuration.md | 2 +- 19 files changed, 1039 insertions(+), 37 deletions(-) create mode 100644 dashboard/backend/infrastructure/ai_hedge_fund/billing.py create mode 100644 dashboard/backend/tests/infrastructure/ai_hedge_fund/__init__.py create mode 100644 dashboard/backend/tests/infrastructure/ai_hedge_fund/test_billing.py diff --git a/CLAUDE.md b/CLAUDE.md index 8360d4b62..35a044f51 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -59,7 +59,7 @@ No linter or formatter is configured. **LLM billing** - **LLM backtest billing — the live path (no env var of its own).** `POST /backtest/run` with `decision_source='llm'` on the pipeline runtime is sign-in-only (401) and needs `billing_mode` = `byok` | `platform_credits` (422). The worker gets a signed, secret-free handoff on **stdin**; never put a raw key in argv or env. `infrastructure/llm/execution/service.py` bills **per call by usage** (reserve, settle, release). `attempt_index` spans repeats and failover: select a lane by `provider_id`, never `attempt_index = 1`. Settlement fails closed (`BILLING_FAILED`); BYOK never touches the ledger. -- ⚠ **Only two paths spend Credits:** that layer and research runs (`api/routers/research.py`). Every other LLM call bills operator keys unreserved: leaderboard entries (CommonStack), Discord `/ask`, Trading Algo chat and backtest (`/api/algo/*`), the AI Hedge Fund bridge (`OPENROUTER_API_KEY`). `make_llm_client()` without an integration prefers `COMMONSTACK_API_KEY`, so the Trading Algo backtest hits CommonStack though `execute_algo` checks only `ANTHROPIC_API_KEY`. +- ⚠ **Only three paths spend Credits:** that layer, research runs (`api/routers/research.py`) and AI Hedge Fund decisions (`ai_hedge_fund/billing.py`). Every other LLM call bills operator keys unreserved: leaderboard entries (CommonStack), Discord `/ask`, Trading Algo chat and backtest (`/api/algo/*`). `make_llm_client()` without an integration prefers `COMMONSTACK_API_KEY`, so the Trading Algo backtest hits CommonStack though `execute_algo` checks only `ANTHROPIC_API_KEY`. - ⚠ **Pinned is not deterministic.** Temperature 0 is a request, not an outcome: providers are not deterministic and each bar's prompt embeds the previous bar's answer, so one different draw carries to the end of the run. UI copy never says "deterministic" (`formatBacktestSampling`; `test_backtest_sampling_row.py`). Measure the spread with `python dashboard/scripts/diff_backtest_runs.py ` (exit 2 on a config mismatch unless `--allow-mismatch`). - Ledger tables, launch preflight (402/403), mid-run `INSUFFICIENT_CREDITS`, the welcome grant, the parent-side run lifecycle and pinned sampling per catalog model. → `backend-internals.md` § LLM billing internals - `ATL_STRIPE_TEST_BILLING_ENABLED` (strict opt-in) + `STRIPE_SECRET_KEY` (`sk_test_` only), `STRIPE_WEBHOOK_SECRET`, `PUBLIC_APP_URL`: Stripe **Test Mode** Credits purchases. ⚠ The purchase ledger rides `USERS_DATABASE_URL`. → `configuration.md` diff --git a/Dockerfile b/Dockerfile index 53aa33201..d7544388c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,10 @@ RUN pip install --no-cache-dir --no-deps -r requirements.txt \ # venv must run on 3.12 even though ATL itself stays on 3.13. uv installs # a managed 3.12 under /opt/uv-python; the venv keeps an absolute reference # to it, so uninstalling the uv build tool afterwards is safe. +# +# The second import is the bridge's Credits meter (bridge.py:_usage_counter). +# Missing, every decision still runs but bills its reservation ceiling, so an +# upstream dependency drift fails the build instead of overcharging quietly. ENV UV_PYTHON_INSTALL_DIR=/opt/uv-python RUN pip install --no-cache-dir uv \ && uv python install 3.12 \ @@ -28,6 +32,7 @@ RUN pip install --no-cache-dir uv \ && uv pip install --python /opt/ai-hedge-fund-venv \ -r requirements-ai-hedge-fund.txt \ && /opt/ai-hedge-fund-venv/bin/python -c "from src.main import run_hedge_fund" \ + && /opt/ai-hedge-fund-venv/bin/python -c "from langchain_core.callbacks import get_usage_metadata_callback" \ && pip uninstall -y uv # Copy dashboard application diff --git a/dashboard/backend/api/routers/backtests.py b/dashboard/backend/api/routers/backtests.py index c766c20b6..0d244f9cb 100644 --- a/dashboard/backend/api/routers/backtests.py +++ b/dashboard/backend/api/routers/backtests.py @@ -157,10 +157,15 @@ normalize_runtime_type, ) from dashboard.backend.infrastructure.ai_hedge_fund.adapter import ( + DEFAULT_MODEL_NAME as AI_HEDGE_FUND_MODEL_NAME, AiHedgeFundConfigurationError, resolve_step_timeout_seconds, runtime_unavailable_reason, ) +from dashboard.backend.infrastructure.ai_hedge_fund.billing import ( + PROVIDER_ID as AI_HEDGE_FUND_PROVIDER_ID, + decision_ceiling_micro as ai_hedge_fund_decision_ceiling_micro, +) from dashboard.backend.domain.backtesting.constants import ( INITIAL_CAPITAL, MAX_BACKTEST_INITIAL_CAPITAL, @@ -3179,6 +3184,47 @@ def _enforce_platform_credits_balance( return claim +def _enforce_ai_hedge_fund_credits(user_id: int, decision_ceiling_micro: int) -> None: + """Refuse an AI Hedge Fund run that cannot hold even its first decision. + + One decision, not the window: the per-decision hold in the child is the + gate (a run that drains mid-way stops with ``INSUFFICIENT_CREDITS`` like a + pipeline run), and the ceiling is deliberately generous, so a whole-window + multiple of it would refuse accounts that can afford the run. 403 for a + restricted account. Fail-open on a failed read, for the reason + ``_enforce_platform_credits_balance`` gives: the hold still refuses. + """ + try: + balance = credits_service.get_balance(int(user_id)) + except Exception as exc: # noqa: BLE001 - the reservation remains the gate + print( + "WARNING: credits.preflight_balance_unavailable " + f"category={type(exc).__name__}", + flush=True, + ) + return + if balance.account_status == "restricted": + raise HTTPException( + status_code=403, + detail=LLMExecutionError.account_restricted( + balance.restriction_reason, + int(balance.outstanding_credits_micro or 0), + ).safe_message, + ) + available = int(balance.total_available_micro) + if available < decision_ceiling_micro: + raise HTTPException( + status_code=402, + detail=( + "AI Hedge Fund holds up to " + f"{_format_credits(decision_ceiling_micro, round_up=True)} ATL " + "Credits per trading day while it decides, and your balance is " + f"{_format_credits(available)} Credits. Add Credits on the " + "Credits page, or select fewer analysts or tickers." + ), + ) + + def _enforce_pipeline_llm_window( runtime_type: str, decision_source: str, @@ -4129,12 +4175,13 @@ def run_backtest_endpoint( ) user_id = optional_user["id"] if optional_user else None - # A hosted runtime bills the operator's own provider key with no Credits - # reservation, and a guest's per-owner slot and rate-limit key are a - # session id the guest mints itself, so neither bounds a guest. Refused - # first, before the window bound and the readiness checks, so a guest - # learns nothing about whether the runtime is armed here. Keyed on - # ``!= PIPELINE`` so a runtime added later is gated by default. + # A hosted runtime calls a platform-managed model, billed to the caller's + # ATL Credits, which a guest does not have; and a guest's per-owner slot + # and rate-limit key are a session id the guest mints itself, so neither + # bounds a guest. Refused first, before the window bound and the readiness + # checks, so a guest learns nothing about whether the runtime is armed + # here. Keyed on ``!= PIPELINE`` so a runtime added later is gated by + # default. if runtime_type != PIPELINE_RUNTIME_TYPE and user_id is None: raise HTTPException( status_code=401, @@ -4162,6 +4209,13 @@ def run_backtest_endpoint( status_code=422, detail="AI Hedge Fund requires decision_source='llm'.", ) + if billing_mode is BillingMode.BYOK: + # The model is platform-managed, so there is no key of the + # caller's to bill; refused rather than silently switched lanes. + raise HTTPException( + status_code=422, + detail="AI Hedge Fund runs on ATL Credits only; it has no BYOK option.", + ) # Before the credential lookup, which decrypts agent-owner material: a # window this deployment refuses is refused whether or not the agent is # configured, and there is no reason to touch the secret store to say @@ -4450,6 +4504,41 @@ def run_backtest_endpoint( "universe_selection": universe_selection, }, ) + elif runtime_type == AI_HEDGE_FUND_RUNTIME_TYPE: + # Signed in (the 401 above) and on Credits only (the 422 above). The + # handoff carries no model client here: the child turns it into + # per-decision holds (`infrastructure/ai_hedge_fund/billing.py`), and + # it is what threads the lane into the parent's `finalize_run` and the + # timeout card's spend lookup, exactly as for a pipeline run. + billing_mode = BillingMode.PLATFORM_CREDITS + provider_id = AI_HEDGE_FUND_PROVIDER_ID + provider_ids = (provider_id,) + model = AI_HEDGE_FUND_MODEL_NAME + _enforce_ai_hedge_fund_credits( + int(user_id), + ai_hedge_fund_decision_ceiling_micro( + model, + (runtime_config or {}).get("analysts"), + len(selected_assets or ()) or len(profile.symbols), + ), + ) + execution_handoff_payload = create_execution_handoff( + user_id=int(user_id), + run_id=live_run_id, + billing_mode=billing_mode, + provider_id=provider_id, + provider_ids=provider_ids, + model_id=model, + prompt_metadata={ + "start_date": start_date, + "end_date": end_date, + "runtime_type": runtime_type, + "runtime_config": runtime_config, + "data_source": data_source, + "assets": selected_assets, + "universe_selection": universe_selection, + }, + ) refusal = _try_acquire_backtest_slot( live_run_id=live_run_id, diff --git a/dashboard/backend/domain/backtesting/engine.py b/dashboard/backend/domain/backtesting/engine.py index 986fe4632..6cf95a847 100644 --- a/dashboard/backend/domain/backtesting/engine.py +++ b/dashboard/backend/domain/backtesting/engine.py @@ -291,6 +291,7 @@ def __init__( runtime_type: str = DEFAULT_RUNTIME_TYPE, runtime_config: Optional[Dict] = None, execution_client: Any = None, + hosted_billing: Any = None, stock_pool: Optional[str] = None, pool_mode: Optional[str] = None, universe_selection: Optional[Dict] = None, @@ -350,7 +351,11 @@ def __init__( RuntimeDispatcher( self.runtime_type, self.runtime_config, - runtime=AiHedgeFundRuntime(self.runtime_config), + # The hosted runtime's Credits metering, built by the child + # from the signed handoff; None off the API path. + runtime=AiHedgeFundRuntime( + self.runtime_config, billing=hosted_billing + ), ) if self.runtime_type == AI_HEDGE_FUND_RUNTIME_TYPE else None diff --git a/dashboard/backend/infrastructure/ai_hedge_fund/adapter.py b/dashboard/backend/infrastructure/ai_hedge_fund/adapter.py index e43297f7d..01e20e156 100644 --- a/dashboard/backend/infrastructure/ai_hedge_fund/adapter.py +++ b/dashboard/backend/infrastructure/ai_hedge_fund/adapter.py @@ -25,6 +25,11 @@ AgentRuntimeError, normalize_runtime_config, ) +from dashboard.backend.infrastructure.ai_hedge_fund.billing import ( + HostedDecisionBilling, + decision_ceiling_micro, + parse_usage, +) from dashboard.backend.infrastructure.llm.validator import ( MAX_ORDER_SHARES, actions_to_executable, @@ -84,8 +89,23 @@ class AiHedgeFundRuntimeError(AgentRuntimeError): Deliberately *transient* by default: the backtest engine holds the step and keeps the run alive for these, because a timeout or a bad upstream response on one day says nothing about the next. + + ``usage`` is what the bridge reported spending before it failed (None when + it reported nothing) and ``timed_out`` marks a decision killed mid-run; + together they decide what the failed decision is billed (``billing.py``). """ + def __init__( + self, + message: str = "", + *, + usage: Optional[tuple[int, int]] = None, + timed_out: bool = False, + ): + super().__init__(message) + self.usage = usage + self.timed_out = timed_out + class AiHedgeFundConfigurationError( AiHedgeFundRuntimeError, AgentRuntimeConfigurationError @@ -260,7 +280,8 @@ def run(self, payload: Dict[str, Any], *, timeout_seconds: int) -> Dict[str, Any ) except subprocess.TimeoutExpired as exc: raise AiHedgeFundRuntimeError( - f"AI Hedge Fund runtime timed out after {timeout_seconds} seconds" + f"AI Hedge Fund runtime timed out after {timeout_seconds} seconds", + timed_out=True, ) from exc except OSError as exc: raise AiHedgeFundRuntimeError( @@ -271,8 +292,17 @@ def run(self, payload: Dict[str, Any], *, timeout_seconds: int) -> Dict[str, Any detail = _redact_runtime_error( result.stderr or result.stdout or "unknown error", environment ) + # The bridge writes its usage before a failing exit; absent, + # it died before opening its usage scope. + try: + reported = json.loads(output_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + reported = None raise AiHedgeFundRuntimeError( - f"AI Hedge Fund runtime failed with code {result.returncode}: {detail}" + f"AI Hedge Fund runtime failed with code {result.returncode}: {detail}", + usage=parse_usage( + reported.get("usage") if isinstance(reported, dict) else None + ), ) try: output = json.loads(output_path.read_text(encoding="utf-8")) @@ -296,11 +326,15 @@ def __init__( *, runner: Optional[AiHedgeFundSubprocessRunner] = None, environment: Optional[Mapping[str, str]] = None, + billing: Optional[HostedDecisionBilling] = None, ): self.config = normalize_runtime_config( AI_HEDGE_FUND_RUNTIME_TYPE, config or {} ) self.runner = runner or AiHedgeFundSubprocessRunner() + # None only off the API path (the operator's own CLI run, tests): every + # `POST /backtest/run` launch signs a handoff the child turns into this. + self.billing = billing self.environment = dict(os.environ if environment is None else environment) # Hosted model/provider selection is platform-owned. Reuse ATL's # registered OpenRouter model instead of accepting agent config or an @@ -776,6 +810,37 @@ def output_to_atl_actions( actions, _traces = cls.output_to_atl_actions_with_trace(output, context) return actions + def _run_billed(self, payload: Dict[str, Any]) -> Dict[str, Any]: + """Run one upstream decision, held and settled on the caller's Credits. + + Settled before the output is mapped: an output ATL then refuses still + spent its model calls. A configuration error raised before the bridge + starts is released like any no-call failure. + """ + if self.billing is None: + return self.runner.run(payload, timeout_seconds=self.timeout_seconds) + ceiling = decision_ceiling_micro( + self.model_name, self.config.get("analysts"), len(payload["tickers"]) + ) + reservation_id = self.billing.reserve(ceiling) + try: + output = self.runner.run(payload, timeout_seconds=self.timeout_seconds) + except AiHedgeFundRuntimeError as exc: + self.billing.close( + reservation_id, + ceiling_micro=ceiling, + usage=exc.usage, + outcome="timed_out" if exc.timed_out else "failed", + ) + raise + self.billing.close( + reservation_id, + ceiling_micro=ceiling, + usage=parse_usage(output.get("usage")), + outcome="completed", + ) + return output + def decide(self, context: AgentRuntimeContext) -> Dict[str, list[Dict[str, Any]]]: decision_day = context.timestamp.date().isoformat() if DEFAULT_DECISION_INTERVAL == "daily" and decision_day == self._last_decision_day: @@ -792,7 +857,7 @@ def decide(self, context: AgentRuntimeContext) -> Dict[str, list[Dict[str, Any]] # hourly bar retries upstream -- so one bad day costs bars-per-day full # timeouts and API spend for no new information. self._last_decision_day = decision_day - output = self.runner.run(payload, timeout_seconds=self.timeout_seconds) + output = self._run_billed(payload) self.calls += 1 actions, traces = self.output_to_atl_actions_with_trace(output, context) cutoff = context.latest_market_date_before_decision diff --git a/dashboard/backend/infrastructure/ai_hedge_fund/billing.py b/dashboard/backend/infrastructure/ai_hedge_fund/billing.py new file mode 100644 index 000000000..ec05d5299 --- /dev/null +++ b/dashboard/backend/infrastructure/ai_hedge_fund/billing.py @@ -0,0 +1,204 @@ +"""ATL Credits metering for the hosted AI Hedge Fund runtime. + +Upstream makes its model calls inside its own interpreter, so ATL cannot meter +them one by one the way ``infrastructure/llm/execution/service.py`` does. The +unit here is one decision (one bridge subprocess): reserve a ceiling before it, +settle at the token usage the bridge reports after it. The run's reservations +share the run id of the signed execution handoff, so the existing +``finalize_run`` (child and parent) releases any hold a killed child leaves. +""" + +from __future__ import annotations + +from typing import Any, Mapping, Optional, Sequence + +from dashboard.backend.domain.agents.runtime import AI_HEDGE_FUND_ANALYSTS +from dashboard.backend.domain.credits.repository_common import ( + CreditAccountRestrictedStoreError, + InsufficientCreditsError, +) +from dashboard.backend.infrastructure.llm.execution.errors import ( + ExecutionErrorCategory, + LLMExecutionError, +) +from dashboard.backend.infrastructure.llm.token_cost import ( + credits_micro_for_usd, + estimate_cost_usd, +) + +PROVIDER_ID = "openrouter" + +# Per-call budget behind a decision's reservation ceiling. Generous on purpose: +# a ceiling below the real spend is capped at settlement (the operator eats the +# rest) rather than recorded as an overage, which would restrict the account +# for ATL's own estimate. Every selected analyst plus the portfolio manager is +# counted as one call per ticker, though several analysts are rule-based. +CEILING_INPUT_TOKENS_PER_CALL = 8_000 +CEILING_OUTPUT_TOKENS_PER_CALL = 2_000 + + +def decision_ceiling_micro( + model: str, analysts: Optional[Sequence[str]], ticker_count: int +) -> int: + """The most one decision is held for, in Credit micro-units. + + One formula for the launch preflight and the per-decision hold, so the + route cannot admit a run whose first decision the child then refuses. + No analysts configured means upstream runs all of them. + """ + analyst_count = len(analysts or AI_HEDGE_FUND_ANALYSTS) + calls = (analyst_count + 1) * max(int(ticker_count), 1) + return credits_micro_for_usd( + estimate_cost_usd( + model, + calls * CEILING_INPUT_TOKENS_PER_CALL, + calls * CEILING_OUTPUT_TOKENS_PER_CALL, + ) + ) + + +def parse_usage(value: Any) -> Optional[tuple[int, int]]: + """``(input_tokens, output_tokens)`` from the bridge's report, else None.""" + if not isinstance(value, Mapping): + return None + counts = [] + for key in ("input_tokens", "output_tokens"): + count = value.get(key) + if isinstance(count, bool) or not isinstance(count, int) or count < 0: + return None + counts.append(count) + return counts[0], counts[1] + + +class HostedDecisionBilling: + """Reserve, settle and release one run's per-decision Credit holds.""" + + def __init__(self, *, credits: Any, user_id: int, run_id: str, model_id: str): + self.credits = credits + self.user_id = int(user_id) + self.run_id = str(run_id) + self.model_id = model_id + self._next_call_index = 0 + + @classmethod + def from_handoff(cls, *, credits: Any, handoff: Any) -> "HostedDecisionBilling": + return cls( + credits=credits, + user_id=handoff.user_id, + run_id=handoff.run_id, + model_id=handoff.model_id, + ) + + def reserve(self, amount_micro: int) -> str: + """Hold one decision's ceiling; the run stops when it cannot.""" + call_index = self._next_call_index + self._next_call_index += 1 + try: + reservation = self.credits.reserve_llm_credits( + user_id=self.user_id, + run_id=self.run_id, + call_index=call_index, + amount_micro=int(amount_micro), + provider_id=PROVIDER_ID, + ) + except CreditAccountRestrictedStoreError as exc: + try: + balance = self.credits.get_balance(self.user_id) + reason = balance.restriction_reason + outstanding_micro = balance.outstanding_credits_micro + except Exception: # noqa: BLE001 - keep the safe fallback + reason = None + outstanding_micro = 0 + raise LLMExecutionError.account_restricted( + reason, outstanding_micro + ) from exc + except InsufficientCreditsError as exc: + raise LLMExecutionError( + ExecutionErrorCategory.INSUFFICIENT_CREDITS + ) from exc + except Exception as exc: # noqa: BLE001 - store errors stay internal + raise LLMExecutionError(ExecutionErrorCategory.BILLING_FAILED) from exc + if reservation.status != "open": + raise LLMExecutionError(ExecutionErrorCategory.BILLING_FAILED) + return reservation.reservation_id + + def close( + self, + reservation_id: str, + *, + ceiling_micro: int, + usage: Optional[tuple[int, int]], + outcome: str, + ) -> int: + """Settle or release one decision's hold; returns the micro charged. + + ``outcome`` is ``completed``, ``timed_out`` or ``failed``. Reported + usage is billed whatever the outcome. Without it the charge depends on + what the missing report means: a completed decision whose usage went + unreported is a broken meter, and a timed-out one ran its model calls + until it was killed, so both settle at the ceiling; any other failure + exited before the bridge's usage scope opened (an import or start-up + crash), so it made no model call and is released. + """ + if usage is not None: + input_tokens, output_tokens = usage + uncapped = credits_micro_for_usd( + estimate_cost_usd(self.model_id, input_tokens, output_tokens) + ) + basis = "usage" + if outcome == "completed" and input_tokens + output_tokens == 0: + # Legitimate only when every selected analyst is rule-based and + # upstream's portfolio manager pre-filled every ticker as a + # hold (it skips its model call then). Anything else is a meter + # that stopped counting (a provider omitting usage, a + # langchain-core change), so it is released but never silently. + print( + "WARNING: ai_hedge_fund.usage_zero completed decision " + "reported no model tokens", + flush=True, + ) + elif outcome in {"completed", "timed_out"}: + input_tokens = output_tokens = None + uncapped = int(ceiling_micro) + basis = "ceiling" + if outcome == "completed": + print( + "ERROR: ai_hedge_fund.usage_unreported " + "billing at the decision ceiling", + flush=True, + ) + else: + input_tokens = output_tokens = None + uncapped = 0 + basis = "no_model_call" + charged = min(uncapped, int(ceiling_micro)) + if uncapped > charged: + print( + "WARNING: ai_hedge_fund.usage_above_ceiling " + f"uncapped_micro={uncapped} ceiling_micro={ceiling_micro}", + flush=True, + ) + try: + if charged <= 0: + self.credits.release_llm_credits(reservation_id, reason=basis) + return 0 + settlement = self.credits.settle_llm_credits( + reservation_id, + actual_micro=charged, + evidence={ + "source": "ai_hedge_fund", + "model_id": self.model_id, + "provider_id": PROVIDER_ID, + "outcome": outcome, + "basis": basis, + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "ceiling_micro": int(ceiling_micro), + "uncapped_micro": uncapped, + }, + ) + except Exception as exc: # noqa: BLE001 - store errors stay internal + raise LLMExecutionError(ExecutionErrorCategory.BILLING_FAILED) from exc + if settlement.status != "settled": + raise LLMExecutionError(ExecutionErrorCategory.BILLING_FAILED) + return charged diff --git a/dashboard/backend/infrastructure/ai_hedge_fund/bridge.py b/dashboard/backend/infrastructure/ai_hedge_fund/bridge.py index 250fc6d7a..b9a2f1bcd 100644 --- a/dashboard/backend/infrastructure/ai_hedge_fund/bridge.py +++ b/dashboard/backend/infrastructure/ai_hedge_fund/bridge.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import contextlib import json from pathlib import Path @@ -49,6 +50,35 @@ def disabled_load_dotenv(*_args, **_kwargs) -> bool: setattr(dotenv_main, "load_dotenv", disabled_load_dotenv) +def _usage_counter(): + """A context manager that totals every model call's tokens, or ``None``. + + LangChain's ``get_usage_metadata_callback`` registers an inheritable + context-var handler, so it sees the calls LangGraph runs on its worker + threads too (the analysts fan out in parallel). Absent in an older + langchain-core; the caller then reports no usage and ATL bills the + decision's reservation ceiling instead. + """ + try: + from langchain_core.callbacks import get_usage_metadata_callback + except ImportError: + return None + return get_usage_metadata_callback() + + +def _usage_totals(counter) -> dict | None: + by_model = getattr(counter, "usage_metadata", None) + if not isinstance(by_model, dict): + return None + totals = {"input_tokens": 0, "output_tokens": 0} + for usage in by_model.values(): + for key in totals: + value = usage.get(key) if isinstance(usage, dict) else None + if isinstance(value, int) and not isinstance(value, bool) and value > 0: + totals[key] += value + return totals + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--input", required=True) @@ -63,21 +93,36 @@ def main() -> None: payload = json.loads(Path(args.input).read_text(encoding="utf-8")) model_name, model_provider = _managed_model_from_payload(payload) - result = run_hedge_fund( - tickers=payload["tickers"], - start_date=payload["start_date"], - end_date=payload["end_date"], - portfolio=payload["portfolio"], - show_reasoning=bool(payload.get("show_reasoning", False)), - selected_analysts=list(payload.get("selected_analysts") or []), - model_name=model_name, - model_provider=model_provider, - ) - Path(args.output).write_text( + counter = _usage_counter() + output_path = Path(args.output) + handler = None + try: + with counter if counter is not None else contextlib.nullcontext() as handler: + result = run_hedge_fund( + tickers=payload["tickers"], + start_date=payload["start_date"], + end_date=payload["end_date"], + portfolio=payload["portfolio"], + show_reasoning=bool(payload.get("show_reasoning", False)), + selected_analysts=list(payload.get("selected_analysts") or []), + model_name=model_name, + model_provider=model_provider, + ) + except BaseException: + # A failed decision may still have spent model calls; ATL bills what + # was used, so report it before the non-zero exit. Usage only: there + # is no decision to read. + output_path.write_text( + json.dumps({"usage": _usage_totals(handler)}), + encoding="utf-8", + ) + raise + output_path.write_text( json.dumps( { "decisions": result.get("decisions"), "analyst_signals": result.get("analyst_signals"), + "usage": _usage_totals(handler), } ), encoding="utf-8", diff --git a/dashboard/backend/tests/infrastructure/ai_hedge_fund/__init__.py b/dashboard/backend/tests/infrastructure/ai_hedge_fund/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dashboard/backend/tests/infrastructure/ai_hedge_fund/test_billing.py b/dashboard/backend/tests/infrastructure/ai_hedge_fund/test_billing.py new file mode 100644 index 000000000..f92ac70a3 --- /dev/null +++ b/dashboard/backend/tests/infrastructure/ai_hedge_fund/test_billing.py @@ -0,0 +1,457 @@ +"""ATL Credits metering of the hosted AI Hedge Fund runtime. + +Against a real SQLite Credits store, so the holds, debits and releases asserted +here are the ledger's own, not a fake's idea of them. +""" + +from __future__ import annotations + +import io +import json +import sqlite3 +import subprocess +import sys +import types +from datetime import date, datetime, timezone + +import pytest + +from dashboard.backend.domain.agents.runtime import ( + AI_HEDGE_FUND_ANALYSTS, + AgentRuntimeContext, +) +from dashboard.backend.domain.credits.repository import CreditsStore +from dashboard.backend.domain.credits.service import CreditsService +from dashboard.backend.infrastructure.ai_hedge_fund import bridge +from dashboard.backend.infrastructure.ai_hedge_fund.adapter import ( + DEFAULT_MODEL_NAME, + DEFAULT_MODEL_PROVIDER, + AiHedgeFundRuntime, + AiHedgeFundRuntimeError, + AiHedgeFundSubprocessRunner, +) +from dashboard.backend.infrastructure.ai_hedge_fund.billing import ( + HostedDecisionBilling, + decision_ceiling_micro, +) +from dashboard.backend.infrastructure.llm.execution.errors import ( + ExecutionErrorCategory, + LLMExecutionError, +) +from dashboard.backend.infrastructure.llm.token_cost import ( + credits_micro_for_usd, + estimate_cost_usd, +) + +FUNDED_USER = 1 +EMPTY_USER = 2 +RUN_ID = "agent_20260501_000000_ahfbill1" +SYMBOLS = ["AAPL", "MSFT", "IBM", "JPM", "DIS"] +CEILING = decision_ceiling_micro(DEFAULT_MODEL_NAME, ["ben_graham"], len(SYMBOLS)) + + +def _credits(tmp_path) -> CreditsService: + path = tmp_path / "credits.db" + with sqlite3.connect(path) as conn: + conn.execute( + """ + CREATE TABLE users ( + id INTEGER PRIMARY KEY, + email TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + role TEXT NOT NULL, + created_at TEXT NOT NULL + ) + """ + ) + conn.executemany( + """ + INSERT INTO users (id, email, display_name, password_hash, role, created_at) + VALUES (?, ?, ?, 'unused', 'user', '2026-08-13T00:00:00+00:00') + """, + [ + (FUNDED_USER, "funded@example.com", "Funded"), + (EMPTY_USER, "empty@example.com", "Empty"), + ], + ) + service = CreditsService(store=CreditsStore(path)) + assert service.grant_default_signup_credits(FUNDED_USER) + return service + + +def _runtime(credits, runner, *, user_id=FUNDED_USER): + return AiHedgeFundRuntime( + {"analysts": ["ben_graham"]}, + runner=runner, + environment={}, + billing=HostedDecisionBilling( + credits=credits, + user_id=user_id, + run_id=RUN_ID, + model_id=DEFAULT_MODEL_NAME, + ), + ) + + +def _context(day=1): + return AgentRuntimeContext( + timestamp=datetime(2026, 5, day, 14, tzinfo=timezone.utc), + backtest_start_date="2026-05-01", + symbols=list(SYMBOLS), + cash=1_000.0, + total_equity=1_000.0, + positions={}, + entry_prices={}, + current_prices={symbol: 10.0 for symbol in SYMBOLS}, + latest_market_date_before_decision=date(2026, 4, day + 20), + ) + + +class _Runner: + """Answers each decision with the next scripted output or exception.""" + + def __init__(self, *outcomes): + self.outcomes = list(outcomes) + self.calls = 0 + + def run(self, payload, *, timeout_seconds): + self.calls += 1 + outcome = self.outcomes.pop(0) + if isinstance(outcome, BaseException): + raise outcome + return outcome + + +def _rows(tmp_path): + """Every hold of the run: (call_index, status, reserved, settled, evidence).""" + with sqlite3.connect(tmp_path / "credits.db") as conn: + rows = conn.execute( + """ + SELECT call_index, status, reserved_micro, settled_micro, evidence_json + FROM credit_llm_reservations WHERE run_id = ? ORDER BY call_index + """, + (RUN_ID,), + ).fetchall() + return [ + ( + row[0], + row[1], + row[2], + row[3], + json.loads(row[4]) if row[4] else None, + ) + for row in rows + ] + + +def _usage_micro(input_tokens, output_tokens): + return credits_micro_for_usd( + estimate_cost_usd(DEFAULT_MODEL_NAME, input_tokens, output_tokens) + ) + + +def test_ceiling_counts_every_analyst_when_none_are_configured(): + assert decision_ceiling_micro(DEFAULT_MODEL_NAME, None, 5) == decision_ceiling_micro( + DEFAULT_MODEL_NAME, list(AI_HEDGE_FUND_ANALYSTS), 5 + ) + # Priced, not free: a $0 ceiling would hold nothing and settle nothing. + assert CEILING > 0 + + +def test_each_decision_settles_at_its_reported_usage(tmp_path): + credits = _credits(tmp_path) + runner = _Runner( + {"decisions": {}, "usage": {"input_tokens": 12_000, "output_tokens": 3_000}}, + {"decisions": {}, "usage": {"input_tokens": 4_000, "output_tokens": 1_000}}, + ) + runtime = _runtime(credits, runner) + + runtime.decide(_context(1)) + runtime.decide(_context(2)) + + rows = _rows(tmp_path) + assert [(index, status) for index, status, *_ in rows] == [ + (0, "settled"), + (1, "settled"), + ] + assert [row[2] for row in rows] == [CEILING, CEILING] + assert [row[3] for row in rows] == [ + _usage_micro(12_000, 3_000), + _usage_micro(4_000, 1_000), + ] + assert rows[0][4]["basis"] == "usage" + assert rows[0][4]["source"] == "ai_hedge_fund" + spent, calls = credits.sum_run_llm_spend(FUNDED_USER, RUN_ID) + assert (spent, calls) == (rows[0][3] + rows[1][3], 2) + + +def test_unreported_usage_on_a_completed_decision_bills_the_ceiling_loudly( + tmp_path, capsys +): + credits = _credits(tmp_path) + runtime = _runtime(credits, _Runner({"decisions": {}, "usage": None})) + + runtime.decide(_context()) + + [(_, status, _, settled, evidence)] = _rows(tmp_path) + assert (status, settled, evidence["basis"]) == ("settled", CEILING, "ceiling") + assert "ERROR: ai_hedge_fund.usage_unreported" in capsys.readouterr().out + + +def test_usage_above_the_ceiling_is_capped_rather_than_recorded_as_overage( + tmp_path, capsys +): + credits = _credits(tmp_path) + runtime = _runtime( + credits, + _Runner( + { + "decisions": {}, + "usage": {"input_tokens": 50_000_000, "output_tokens": 0}, + } + ), + ) + + runtime.decide(_context()) + + [(_, status, _, settled, evidence)] = _rows(tmp_path) + assert (status, settled) == ("settled", CEILING) + assert evidence["uncapped_micro"] > CEILING + # No overage, so the account is not restricted over ATL's own estimate. + assert credits.get_balance(FUNDED_USER).account_status == "active" + assert "WARNING: ai_hedge_fund.usage_above_ceiling" in capsys.readouterr().out + + +@pytest.mark.parametrize( + "error, status, settled", + [ + # Killed mid-run: its model calls ran until the kill. + (AiHedgeFundRuntimeError("timed out", timed_out=True), "settled", CEILING), + # Failed after reporting what it had spent. + ( + AiHedgeFundRuntimeError("exit 1", usage=(10_000, 2_000)), + "settled", + _usage_micro(10_000, 2_000), + ), + # Failed before its usage scope opened: no model call was made. + (AiHedgeFundRuntimeError("exit 1"), "released", 0), + ], +) +def test_a_failed_decision_is_billed_by_what_its_failure_shows( + tmp_path, error, status, settled +): + credits = _credits(tmp_path) + runtime = _runtime(credits, _Runner(error)) + + with pytest.raises(AiHedgeFundRuntimeError): + runtime.decide(_context()) + + [(_, row_status, _, row_settled, _)] = _rows(tmp_path) + assert (row_status, row_settled) == (status, settled) + + +def test_an_empty_balance_stops_the_run_before_upstream_is_called(tmp_path): + credits = _credits(tmp_path) + runner = _Runner({"decisions": {}}) + runtime = _runtime(credits, runner, user_id=EMPTY_USER) + + with pytest.raises(LLMExecutionError) as excinfo: + runtime.decide(_context()) + + # Not an AgentRuntimeError, so the engine does not hold the step and carry + # on: it reaches the child's `llm.run_failed` line like a pipeline run's. + assert excinfo.value.category is ExecutionErrorCategory.INSUFFICIENT_CREDITS + assert runner.calls == 0 + assert _rows(tmp_path) == [] + + +def test_runner_carries_the_bridges_usage_out_of_a_failed_exit(monkeypatch, tmp_path): + python = tmp_path / "python" + python.write_text("") + python.chmod(0o755) + + def fake_run(command, **kwargs): + output = command[command.index("--output") + 1] + with open(output, "w", encoding="utf-8") as handle: + json.dump({"usage": {"input_tokens": 7, "output_tokens": 3}}, handle) + return subprocess.CompletedProcess(command, 1, stdout="", stderr="boom") + + monkeypatch.setattr(subprocess, "run", fake_run) + runner = AiHedgeFundSubprocessRunner({"AI_HEDGE_FUND_PYTHON": str(python)}) + + with pytest.raises(AiHedgeFundRuntimeError) as excinfo: + runner.run({"tickers": ["AAPL"]}, timeout_seconds=5) + + assert excinfo.value.usage == (7, 3) + assert excinfo.value.timed_out is False + + +def _bridge_args(monkeypatch, tmp_path): + input_path = tmp_path / "input.json" + output_path = tmp_path / "output.json" + input_path.write_text( + json.dumps( + { + "tickers": ["AAPL"], + "start_date": "2026-02-03", + "end_date": "2026-05-04", + "portfolio": {"cash": 1000}, + "model_name": DEFAULT_MODEL_NAME, + "model_provider": DEFAULT_MODEL_PROVIDER, + } + ), + encoding="utf-8", + ) + monkeypatch.setattr( + sys, + "argv", + ["bridge.py", "--input", str(input_path), "--output", str(output_path)], + ) + monkeypatch.setattr(bridge, "_disable_dotenv_loading", lambda: None) + return output_path + + +def _fake_upstream(monkeypatch, run_hedge_fund): + """Install a fake upstream and a fake langchain-core usage counter. + + The counter mirrors ``get_usage_metadata_callback``: a context manager + whose handler accumulates ``usage_metadata`` per model name. The real one + was exercised against upstream's pinned langchain stack while writing this. + """ + fake_main = types.ModuleType("src.main") + fake_main.run_hedge_fund = run_hedge_fund + fake_src = types.ModuleType("src") + fake_src.main = fake_main + monkeypatch.setitem(sys.modules, "src", fake_src) + monkeypatch.setitem(sys.modules, "src.main", fake_main) + + handler = types.SimpleNamespace(usage_metadata={}) + + class _Counter: + def __enter__(self): + return handler + + def __exit__(self, *exc): + return False + + callbacks = types.ModuleType("langchain_core.callbacks") + callbacks.get_usage_metadata_callback = _Counter + core = types.ModuleType("langchain_core") + core.callbacks = callbacks + monkeypatch.setitem(sys.modules, "langchain_core", core) + monkeypatch.setitem(sys.modules, "langchain_core.callbacks", callbacks) + return handler + + +def test_bridge_reports_summed_usage_with_the_decisions(monkeypatch, tmp_path): + output_path = _bridge_args(monkeypatch, tmp_path) + + def run_hedge_fund(**_kwargs): + handler.usage_metadata.update( + { + DEFAULT_MODEL_NAME: {"input_tokens": 900, "output_tokens": 90}, + "other-model": {"input_tokens": 100, "output_tokens": 10}, + } + ) + return {"decisions": {}, "analyst_signals": {}} + + handler = _fake_upstream(monkeypatch, run_hedge_fund) + + bridge.main() + + output = json.loads(output_path.read_text(encoding="utf-8")) + assert output["usage"] == {"input_tokens": 1000, "output_tokens": 100} + assert output["decisions"] == {} + + +def test_bridge_reports_usage_spent_before_a_failure(monkeypatch, tmp_path): + output_path = _bridge_args(monkeypatch, tmp_path) + + def run_hedge_fund(**_kwargs): + handler.usage_metadata[DEFAULT_MODEL_NAME] = { + "input_tokens": 50, + "output_tokens": 5, + } + raise RuntimeError("upstream exploded") + + handler = _fake_upstream(monkeypatch, run_hedge_fund) + + with pytest.raises(RuntimeError): + bridge.main() + + assert json.loads(output_path.read_text(encoding="utf-8")) == { + "usage": {"input_tokens": 50, "output_tokens": 5} + } + + +def test_bridge_without_a_usage_counter_reports_none(monkeypatch, tmp_path): + output_path = _bridge_args(monkeypatch, tmp_path) + _fake_upstream(monkeypatch, lambda **_kwargs: {"decisions": {}}) + # An older langchain-core: the import fails, so ATL bills the ceiling. + monkeypatch.setitem(sys.modules, "langchain_core.callbacks", None) + + bridge.main() + + assert json.loads(output_path.read_text(encoding="utf-8"))["usage"] is None + + +def test_worker_meters_a_hosted_handoff_instead_of_driving_a_model_client( + monkeypatch, +): + """The child turns the route's handoff into per-decision holds. + + No model client: the engine would read its `sampling_record` and stamp a + pinned-sampling policy on a run whose model it never called. + """ + from dashboard.backend.infrastructure.llm.execution.handoff import ( + create_execution_handoff, + ) + from dashboard.scripts import backtest_hourly_agent as cli + + token = create_execution_handoff( + user_id=FUNDED_USER, + run_id=RUN_ID, + billing_mode="platform_credits", + provider_id="openrouter", + model_id=DEFAULT_MODEL_NAME, + ) + monkeypatch.setattr( + sys, "stdin", io.TextIOWrapper(io.BytesIO(token.encode("ascii"))) + ) + monkeypatch.setattr( + sys, + "argv", + [ + "backtest_hourly_agent", + "--execution-handoff-stdin", + "--run-id", RUN_ID, + "--model", DEFAULT_MODEL_NAME, + "--runtime-type", "ai_hedge_fund", + "--decision-source", "llm", + "--start", "2026-05-01", + "--end", "2026-05-04", + ], + ) + captured = {} + + class _Stop(Exception): + pass + + def _engine(*_args, **kwargs): + captured.update(kwargs) + raise _Stop + + monkeypatch.setattr(cli, "HourlyBacktester", _engine) + + with pytest.raises(_Stop): + cli.main() + + assert captured["execution_client"] is None + billing = captured["hosted_billing"] + assert isinstance(billing, HostedDecisionBilling) + assert (billing.user_id, billing.run_id, billing.model_id) == ( + FUNDED_USER, + RUN_ID, + DEFAULT_MODEL_NAME, + ) diff --git a/dashboard/backend/tests/test_admin_analytics_frontend.py b/dashboard/backend/tests/test_admin_analytics_frontend.py index c06695ec6..79d5218ff 100644 --- a/dashboard/backend/tests/test_admin_analytics_frontend.py +++ b/dashboard/backend/tests/test_admin_analytics_frontend.py @@ -162,7 +162,7 @@ def test_app_lifecycle_and_cache_versions_are_wired(): # Lockstep owner for the console's bumped tags and the /admin page's pins: # every bump edits this test in the same change (Global Constraints). assert 'styles.css?v=160' in APP_HTML - assert 'app.js?v=163' in APP_HTML + assert 'app.js?v=164' in APP_HTML assert 'js/admin-tabs.js?v=12' in APP_HTML for tag in ( 'href="admin.css?v=14"', diff --git a/dashboard/backend/tests/test_analytics_frontend.py b/dashboard/backend/tests/test_analytics_frontend.py index f50dd5593..f3e847220 100644 --- a/dashboard/backend/tests/test_analytics_frontend.py +++ b/dashboard/backend/tests/test_analytics_frontend.py @@ -41,7 +41,7 @@ def _function_body(source: str, signature: str) -> str: def test_analytics_script_loads_between_app_and_page_scripts(): - app_at = APP_HTML.index('') + app_at = APP_HTML.index('') analytics_at = APP_HTML.index( '' ) diff --git a/dashboard/backend/tests/test_backtest_comparison_frontend.py b/dashboard/backend/tests/test_backtest_comparison_frontend.py index 29b4518e9..4421193da 100644 --- a/dashboard/backend/tests/test_backtest_comparison_frontend.py +++ b/dashboard/backend/tests/test_backtest_comparison_frontend.py @@ -191,7 +191,7 @@ def test_exact_raw_ties_mark_every_tied_series_best(): def test_comparison_script_and_semantic_table_ship_before_app(): helper = '' - app = '' + app = '' assert 'href="styles.css?v=160"' in APP_HTML assert APP_HTML.index(helper) < APP_HTML.index(app) for element_id in ( diff --git a/dashboard/backend/tests/test_backtests_router.py b/dashboard/backend/tests/test_backtests_router.py index 5c09c4c12..57d15670b 100644 --- a/dashboard/backend/tests/test_backtests_router.py +++ b/dashboard/backend/tests/test_backtests_router.py @@ -28,6 +28,8 @@ UnsupportedExecutionModel, ) import dashboard.backend.api.routers.backtests as bt +from dashboard.backend.infrastructure.llm.execution.models import BillingMode +from dashboard.backend.infrastructure.llm.providers import openrouter from dashboard.backend.tests._fake_child import FakeChild @@ -1017,7 +1019,7 @@ def _sign_in(monkeypatch, user_id=7): def test_guest_cannot_start_a_hosted_backtest(client, monkeypatch): - """A hosted run bills the operator's OpenRouter key with no reservation. + """A hosted run bills ATL Credits, which a guest does not have. The refusal comes before the readiness checks: with OPENROUTER_API_KEY unset and the runtime missing, a guest still gets 401, never the 503 that @@ -1056,9 +1058,120 @@ def test_guest_cannot_start_a_hosted_backtest(client, monkeypatch): assert spy.calls == 0 +def _post_ai_hedge_fund_run(client, monkeypatch, credits, **body): + """Launch the cloned AI Hedge Fund agent as a signed-in caller. + + Returns the response, the worker spy and the handoffs the route signed. + """ + spy = _Spy() + handoffs = [] + monkeypatch.setattr(bt, "run_backtest_background", spy) + monkeypatch.setattr(bt, "credits_service", credits) + monkeypatch.setattr( + bt, "create_execution_handoff", lambda **kw: handoffs.append(kw) or "h" + ) + monkeypatch.setenv("OPENROUTER_API_KEY", "platform-openrouter-test-key") + _stub_hosted_runtime_installed(monkeypatch) + monkeypatch.setattr( + bt.agent_credential_store, + "get_secret", + lambda agent_id, credential_name: "user-financial-datasets-test-key", + ) + headers = {"X-Session-Id": str(uuid.uuid4())} + agent = client.post( + "/api/v1/agents/marketplace/ai-hedge-fund/clone", + json={}, + headers=headers, + ).json()["agent"] + _sign_in(monkeypatch) + response = client.post( + "/backtest/run", + json={ + "start_date": "2026-05-01", + "end_date": "2026-05-02", + "decision_source": "llm", + "agent_id": agent["agent_id"], + **body, + }, + headers=headers, + ) + return response, spy, handoffs + + +def test_ai_hedge_fund_run_bills_credits_through_a_signed_handoff( + client, monkeypatch +): + """The hosted run is a platform-credits run in everything but its client. + + The handoff names the managed model, so the child's `--model` check holds, + and the lane reaches the worker, so the parent's `finalize_run` and the + timeout card's spend lookup both see a Credits run. + """ + response, spy, handoffs = _post_ai_hedge_fund_run( + client, monkeypatch, _FakeBalanceCredits(8_000_000) + ) + + assert response.status_code == 200, response.text + assert response.json()["billing_mode"] == "platform_credits" + assert response.json()["provider_id"] == "openrouter" + assert len(handoffs) == 1 + handoff = handoffs[0] + assert handoff["user_id"] == 7 + assert handoff["run_id"] == response.json()["run_id"] + assert handoff["billing_mode"] is BillingMode.PLATFORM_CREDITS + assert handoff["provider_ids"] == ("openrouter",) + assert handoff["model_id"] == openrouter.DEFAULT_MODEL + assert spy.last_kwargs["execution_handoff_payload"] == "h" + assert spy.last_kwargs["billing_mode"] == "platform_credits" + assert spy.last_kwargs["model"] == handoff["model_id"] + + +def test_ai_hedge_fund_run_that_cannot_hold_one_decision_is_refused_with_402( + client, monkeypatch +): + credits = _FakeBalanceCredits(1_000) + + response, spy, handoffs = _post_ai_hedge_fund_run(client, monkeypatch, credits) + + assert response.status_code == 402, response.text + detail = response.json()["detail"] + assert "per trading day" in detail + assert "your balance is 0.00 Credits" in detail + assert credits.balance_reads == [7] + assert handoffs == [] + assert spy.calls == 0 + + +def test_ai_hedge_fund_restricted_account_is_refused_with_403(client, monkeypatch): + credits = _FakeBalanceCredits( + 50_000_000, status="restricted", reason="refund_reconciliation" + ) + + response, spy, handoffs = _post_ai_hedge_fund_run(client, monkeypatch, credits) + + assert response.status_code == 403 + assert "paused for payment refund review" in response.json()["detail"] + assert handoffs == [] + assert spy.calls == 0 + + +def test_ai_hedge_fund_refuses_byok_instead_of_switching_lanes(client, monkeypatch): + credits = _FakeBalanceCredits(8_000_000) + + response, spy, handoffs = _post_ai_hedge_fund_run( + client, monkeypatch, credits, billing_mode="byok" + ) + + assert response.status_code == 422 + assert "ATL Credits only" in response.json()["detail"] + assert credits.balance_reads == [] + assert spy.calls == 0 + + def test_backtest_run_dispatches_ai_hedge_fund_runtime(client, monkeypatch): spy = _Spy() monkeypatch.setattr(bt, "run_backtest_background", spy) + monkeypatch.setattr(bt, "credits_service", _FakeBalanceCredits(8_000_000)) monkeypatch.setenv("OPENROUTER_API_KEY", "platform-openrouter-test-key") _stub_hosted_runtime_installed(monkeypatch) monkeypatch.setattr( @@ -1100,7 +1213,8 @@ def test_backtest_run_dispatches_ai_hedge_fund_runtime(client, monkeypatch): spy.last_kwargs["financial_datasets_api_key"] == "user-financial-datasets-test-key" ) - assert spy.last_kwargs["model"] is None + # The caller's model is ignored; the platform-managed one runs and bills. + assert spy.last_kwargs["model"] == openrouter.DEFAULT_MODEL assert spy.last_kwargs["pipeline"] is None diff --git a/dashboard/backend/tests/test_frontend_fast_boot.py b/dashboard/backend/tests/test_frontend_fast_boot.py index 88a36b412..f69431e71 100644 --- a/dashboard/backend/tests/test_frontend_fast_boot.py +++ b/dashboard/backend/tests/test_frontend_fast_boot.py @@ -194,7 +194,7 @@ def test_cache_busters_bumped(): # the next bump, so the exact one looks like the broken guard and gets # "fixed" by loosening it. That collision has already cost this repo one # round of follow-ups (#347/#348). - assert "app.js?v=163" in APP_HTML + assert "app.js?v=164" in APP_HTML assert "js/agent-editor.js?v=33" in APP_HTML assert "styles.css?v=160" in APP_HTML assert "js/leaderboard.js?v=38" in APP_HTML diff --git a/dashboard/frontend/app.html b/dashboard/frontend/app.html index 230189a5f..39d200dfc 100644 --- a/dashboard/frontend/app.html +++ b/dashboard/frontend/app.html @@ -2516,7 +2516,7 @@

Refund Credits purchase

- + diff --git a/dashboard/frontend/app.js b/dashboard/frontend/app.js index 104d28d84..009adb0c5 100644 --- a/dashboard/frontend/app.js +++ b/dashboard/frontend/app.js @@ -742,7 +742,7 @@ window.AGENT_SHELF_LABELS = MARKET_LABELS; * * The AI Hedge Fund runtime's Nemotron is deliberately absent: it is a property * of a hosted runtime, not a user choice, and syncBacktestModelFieldMode - * already renders that case as "AI Hedge Fund — hosted runtime". */ + * already renders that case as "AI Hedge Fund — hosted runtime, ...". */ const SUPPORTED_MODELS = [ { slug: 'anthropic/claude-haiku-4-5', label: 'Claude Haiku 4.5', vendor: 'anthropic' }, { slug: 'anthropic/claude-sonnet-4-6', label: 'Claude Sonnet 4.6', vendor: 'anthropic' }, @@ -2578,7 +2578,7 @@ function syncBacktestModelFieldMode() { if (readonly) { readonly.hidden = modelIsEditable; readonly.textContent = isHostedRuntime - ? 'AI Hedge Fund — hosted runtime' + ? 'AI Hedge Fund — hosted runtime, billed to your ATL Credits' : 'Rule-based — simulated practice data, no AI involved'; } if (billingGroup) billingGroup.hidden = isHostedRuntime || isRuleBased; diff --git a/dashboard/scripts/backtest_hourly_agent.py b/dashboard/scripts/backtest_hourly_agent.py index 1825d5b51..035f5a2f5 100644 --- a/dashboard/scripts/backtest_hourly_agent.py +++ b/dashboard/scripts/backtest_hourly_agent.py @@ -190,6 +190,9 @@ from dashboard.backend.domain.backtesting.engine import HourlyBacktester from dashboard.backend.domain.backtesting.decision_tape import trace_lifecycle from dashboard.backend.domain.backtesting.partial_results import LiveProgressHeartbeat +from dashboard.backend.infrastructure.ai_hedge_fund.billing import ( + HostedDecisionBilling, +) from dashboard.backend.infrastructure.llm.execution.handoff import ( ExecutionHandoffError, consume_execution_handoff, @@ -402,15 +405,25 @@ def main(): execution_service = None execution_client = None + hosted_billing = None if execution_handoff is not None: + # Built for either runtime: its `finalize_run` below releases whatever + # hold the run leaves open, a hosted decision's included. execution_service = LLMExecutionService( providers=get_model_provider_service(), credits=credits_service, ) - execution_client = AnthropicCompatibleExecutionClient( - execution_service=execution_service, - handoff=execution_handoff, - ) + if args.runtime_type == AI_HEDGE_FUND_RUNTIME_TYPE: + # Upstream calls its model in its own interpreter, so the handoff + # meters whole decisions rather than driving a model client. + hosted_billing = HostedDecisionBilling.from_handoff( + credits=credits_service, handoff=execution_handoff + ) + else: + execution_client = AnthropicCompatibleExecutionClient( + execution_service=execution_service, + handoff=execution_handoff, + ) session_id = args.session_id @@ -555,6 +568,7 @@ def _flush_tape(): runtime_type=args.runtime_type, runtime_config=runtime_config, execution_client=execution_client, + hosted_billing=hosted_billing, launched_at=args.launched_at, # All three are module-scope constants rather than reads taken here, # on purpose. They describe an interval that ended before the engine diff --git a/docs/architecture/backend-internals.md b/docs/architecture/backend-internals.md index d1c8dfd93..40cbc3d7c 100644 --- a/docs/architecture/backend-internals.md +++ b/docs/architecture/backend-internals.md @@ -34,7 +34,11 @@ Expands the `CLAUDE.md` rule "LLM backtest billing, the live path" (`POST /backt Reservations go to `credit_llm_reservations` and settled usage to `credit_llm_usage_entries` (`domain/credits/repository_postgres.py`); never `credit_ledger_entries`, whose `entry_type` CHECK admits only purchase / refund / admin_grant_assign / admin_grant_reclaim. One reservation is one provider attempt (unique on user, run, `call_index`, `attempt_index`). The handoff is built by `create_execution_handoff` (`infrastructure/llm/execution/handoff.py`); BYOK's `finalize_run` returns `[]` and never touches the ledger. Design: `docs/superpowers/plans/2026-08-24-unified-llm-execution-layer.md`. -Research runs reserve `RESEARCH_RUN_ESTIMATE_USD_CENTS` (default 100) and settle at agent-reported usage, else the estimate. Other paths billed to the operator's keys rather than the user's Credits: leaderboard entries through their `dashboard/config/leaderboard.json` `integration`, Discord `/ask` (`domain/chat/service.py`), Trading Algo chat and backtest (`/api/algo/*`; the backtest is hourly Claude decisions, timed out at 60 minutes in `domain/backtesting/algo_service.py`), the AI Hedge Fund bridge (`OPENROUTER_API_KEY`) and the switched-off Robinhood path. +Research runs reserve `RESEARCH_RUN_ESTIMATE_USD_CENTS` (default 100) and settle at agent-reported usage, else the estimate. + +AI Hedge Fund runs (`infrastructure/ai_hedge_fund/billing.py`) are sign-in-only and Credits-only (BYOK is a 422). The route signs a platform-credits handoff naming the managed OpenRouter model, refuses a balance that cannot hold one decision (402; restricted 403), and the child meters whole decisions, because upstream calls its model inside its own interpreter: one reservation per decision (`call_index` counts decisions) at a ceiling of `(analysts + 1) x tickers` calls of 8000 in / 2000 out tokens, settled at the token usage `bridge.py` reports through langchain-core's `get_usage_metadata_callback` (it sees LangGraph's parallel analyst threads). Usage above the ceiling is capped, not recorded as overage, so ATL's estimate never restricts an account. Without a report: a completed decision settles at the ceiling and logs `ERROR: ai_hedge_fund.usage_unreported`, a timed-out one settles at the ceiling (its calls ran until the kill), and any other failure is released (it died before the usage scope opened). A completed decision reporting zero tokens is released with `WARNING: ai_hedge_fund.usage_zero`. An empty balance mid-run raises `INSUFFICIENT_CREDITS` out of the engine, as for a pipeline run, and the existing `finalize_run` (child and parent) releases a hold a killed child leaves. + +Other paths billed to the operator's keys rather than the user's Credits: leaderboard entries through their `dashboard/config/leaderboard.json` `integration`, Discord `/ask` (`domain/chat/service.py`), Trading Algo chat and backtest (`/api/algo/*`; the backtest is hourly Claude decisions, timed out at 60 minutes in `domain/backtesting/algo_service.py`) and the switched-off Robinhood path. ### Pinned sampling policy diff --git a/docs/architecture/configuration.md b/docs/architecture/configuration.md index 5f697f205..1f987ddea 100644 --- a/docs/architecture/configuration.md +++ b/docs/architecture/configuration.md @@ -69,7 +69,7 @@ Strict opt-in; default off; THIS ONE SPENDS MONEY. **Rule: CLAUDE.md.** The `app ## LLM execution and timeouts ### `ANTHROPIC_API_KEY` / `COMMONSTACK_API_KEY` / `OPENROUTER_API_KEY` -Operator-owned LLM keys; secrets; set by: operator. **Rule: CLAUDE.md** (every LLM call outside the billed layer bills these keys with no reservation). `make_llm_client()` with no integration prefers `COMMONSTACK_API_KEY` (one gateway key for several vendors) over `ANTHROPIC_API_KEY`; OpenRouter is opt-in, never auto-selected. So the Trading Algo backtest hits CommonStack when its key is set, although `execute_algo` checks only `ANTHROPIC_API_KEY`. `OPENROUTER_API_KEY` also backs the AI Hedge Fund bridge (`api/routers/backtests.py` answers 503 when unset) and gates a one-time migration enabling the legacy OpenRouter platform lane (`domain/model_providers/repository.py`). Readers: `infrastructure/llm/providers/` (`__init__.py`, `commonstack.py`, `anthropic_native.py`, `openrouter.py`), `domain/chat/service.py`, `domain/backtesting/algo_service.py`. For `OPENAI_API_KEY` and `DEEPSEEK_API_KEY` see the unread section. +Operator-owned LLM keys; secrets; set by: operator. **Rule: CLAUDE.md** (every LLM call outside the billed layer bills these keys with no reservation). `make_llm_client()` with no integration prefers `COMMONSTACK_API_KEY` (one gateway key for several vendors) over `ANTHROPIC_API_KEY`; OpenRouter is opt-in, never auto-selected. So the Trading Algo backtest hits CommonStack when its key is set, although `execute_algo` checks only `ANTHROPIC_API_KEY`. `OPENROUTER_API_KEY` also backs the AI Hedge Fund bridge (`api/routers/backtests.py` answers 503 when unset; the operator pays OpenRouter, the caller's Credits are debited per decision, `backend-internals.md` § Ledger tables and who is billed) and gates a one-time migration enabling the legacy OpenRouter platform lane (`domain/model_providers/repository.py`). Readers: `infrastructure/llm/providers/` (`__init__.py`, `commonstack.py`, `anthropic_native.py`, `openrouter.py`), `domain/chat/service.py`, `domain/backtesting/algo_service.py`. For `OPENAI_API_KEY` and `DEEPSEEK_API_KEY` see the unread section. ### `LLM_PROVIDER_READ_TIMEOUT_SECONDS` Optional; default 180; range 30-600 (rejects a dropped or doubled digit; junk warns and falls back, never raises). Read deadline of ONE physical provider attempt, which is the whole-generation deadline because providers do not stream; connect 8 s, write and pool 60 s. **Rule: CLAUDE.md.** Reader: `infrastructure/llm/http_policy.py` (`_parse_provider_read_timeout`, `PROVIDER_READ_TIMEOUT_SECONDS`, `provider_http_timeout()`); patch that global in tests, not a re-export. Set by: operator. From 87bdbbe901458fcca83c7b402c19ba44bee72c28 Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Fri, 30 Oct 2026 21:00:26 -0400 Subject: [PATCH 3/5] fix(ai-hedge-fund): fail closed when a decision's usage is unknown A failed decision with no usage report was released, so an OOM SIGKILL after model calls (or an output write that raised outside the handler) made the decision free. The bridge now writes a started marker before upstream can call the model and its final output inside the try; the adapter bills the ceiling for any failure past the marker without a readable report, and for a clean exit whose output it cannot read. Only a bridge that left no file at all, so made no call, is released, and billing treats any outcome other than "failed" as billable. Co-Authored-By: Claude Opus 5.5 --- .../infrastructure/ai_hedge_fund/adapter.py | 36 ++++++--- .../infrastructure/ai_hedge_fund/billing.py | 24 +++--- .../infrastructure/ai_hedge_fund/bridge.py | 29 ++++--- .../ai_hedge_fund/test_billing.py | 79 ++++++++++++++++--- docs/architecture/backend-internals.md | 2 +- 5 files changed, 123 insertions(+), 47 deletions(-) diff --git a/dashboard/backend/infrastructure/ai_hedge_fund/adapter.py b/dashboard/backend/infrastructure/ai_hedge_fund/adapter.py index 01e20e156..4e21cf23c 100644 --- a/dashboard/backend/infrastructure/ai_hedge_fund/adapter.py +++ b/dashboard/backend/infrastructure/ai_hedge_fund/adapter.py @@ -91,7 +91,9 @@ class AiHedgeFundRuntimeError(AgentRuntimeError): on one day says nothing about the next. ``usage`` is what the bridge reported spending before it failed (None when - it reported nothing) and ``timed_out`` marks a decision killed mid-run; + it reported nothing) and ``killed`` marks a decision that stopped after it + could call the model but left no usage report (a timeout, a signal, a + crash past the bridge's started marker); together they decide what the failed decision is billed (``billing.py``). """ @@ -100,11 +102,11 @@ def __init__( message: str = "", *, usage: Optional[tuple[int, int]] = None, - timed_out: bool = False, + killed: bool = False, ): super().__init__(message) self.usage = usage - self.timed_out = timed_out + self.killed = killed class AiHedgeFundConfigurationError( @@ -281,7 +283,7 @@ def run(self, payload: Dict[str, Any], *, timeout_seconds: int) -> Dict[str, Any except subprocess.TimeoutExpired as exc: raise AiHedgeFundRuntimeError( f"AI Hedge Fund runtime timed out after {timeout_seconds} seconds", - timed_out=True, + killed=True, ) from exc except OSError as exc: raise AiHedgeFundRuntimeError( @@ -292,27 +294,37 @@ def run(self, payload: Dict[str, Any], *, timeout_seconds: int) -> Dict[str, Any detail = _redact_runtime_error( result.stderr or result.stdout or "unknown error", environment ) - # The bridge writes its usage before a failing exit; absent, - # it died before opening its usage scope. + # Fail closed. The bridge writes a started marker before + # upstream can call the model and its usage before a failing + # exit, so: no file = died before any call (released); a file + # without a readable usage report = an OOM SIGKILL, a crash + # mid-write, a missing meter (billed the ceiling). + started = output_path.exists() try: reported = json.loads(output_path.read_text(encoding="utf-8")) except (OSError, ValueError): reported = None + usage = parse_usage( + reported.get("usage") if isinstance(reported, dict) else None + ) raise AiHedgeFundRuntimeError( f"AI Hedge Fund runtime failed with code {result.returncode}: {detail}", - usage=parse_usage( - reported.get("usage") if isinstance(reported, dict) else None - ), + usage=usage, + killed=started and usage is None, ) + # A clean exit ran upstream to the end, so an unreadable output + # spent an unknown amount: billed the ceiling, never released. try: output = json.loads(output_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise AiHedgeFundOutputError( - "AI Hedge Fund runtime did not produce valid JSON output" + "AI Hedge Fund runtime did not produce valid JSON output", + killed=True, ) from exc if not isinstance(output, dict): raise AiHedgeFundOutputError( - "AI Hedge Fund runtime output must be a JSON object" + "AI Hedge Fund runtime output must be a JSON object", + killed=True, ) return output @@ -830,7 +842,7 @@ def _run_billed(self, payload: Dict[str, Any]) -> Dict[str, Any]: reservation_id, ceiling_micro=ceiling, usage=exc.usage, - outcome="timed_out" if exc.timed_out else "failed", + outcome="killed" if exc.killed else "failed", ) raise self.billing.close( diff --git a/dashboard/backend/infrastructure/ai_hedge_fund/billing.py b/dashboard/backend/infrastructure/ai_hedge_fund/billing.py index ec05d5299..b348035b0 100644 --- a/dashboard/backend/infrastructure/ai_hedge_fund/billing.py +++ b/dashboard/backend/infrastructure/ai_hedge_fund/billing.py @@ -132,13 +132,13 @@ def close( ) -> int: """Settle or release one decision's hold; returns the micro charged. - ``outcome`` is ``completed``, ``timed_out`` or ``failed``. Reported - usage is billed whatever the outcome. Without it the charge depends on - what the missing report means: a completed decision whose usage went - unreported is a broken meter, and a timed-out one ran its model calls - until it was killed, so both settle at the ceiling; any other failure - exited before the bridge's usage scope opened (an import or start-up - crash), so it made no model call and is released. + ``outcome`` is ``completed``, ``killed`` or ``failed``. Reported usage + is billed whatever the outcome. Without it the charge fails closed: a + completed decision whose usage went unreported is a broken meter, and a + killed one (a timeout, a signal, any failure past the bridge's started + marker) could have called the model, so both settle at the ceiling. + Only ``failed`` is released, which the adapter reports solely when the + bridge left no file at all, i.e. died before any model call. """ if usage is not None: input_tokens, output_tokens = usage @@ -157,7 +157,11 @@ def close( "reported no model tokens", flush=True, ) - elif outcome in {"completed", "timed_out"}: + elif outcome == "failed": + input_tokens = output_tokens = None + uncapped = 0 + basis = "no_model_call" + else: input_tokens = output_tokens = None uncapped = int(ceiling_micro) basis = "ceiling" @@ -167,10 +171,6 @@ def close( "billing at the decision ceiling", flush=True, ) - else: - input_tokens = output_tokens = None - uncapped = 0 - basis = "no_model_call" charged = min(uncapped, int(ceiling_micro)) if uncapped > charged: print( diff --git a/dashboard/backend/infrastructure/ai_hedge_fund/bridge.py b/dashboard/backend/infrastructure/ai_hedge_fund/bridge.py index b9a2f1bcd..e90d7983e 100644 --- a/dashboard/backend/infrastructure/ai_hedge_fund/bridge.py +++ b/dashboard/backend/infrastructure/ai_hedge_fund/bridge.py @@ -95,6 +95,11 @@ def main() -> None: model_name, model_provider = _managed_model_from_payload(payload) counter = _usage_counter() output_path = Path(args.output) + # Written before upstream can make a model call. A failed exit that leaves + # only this marker (an OOM SIGKILL, a crash the handler below never saw) + # spent an unknown amount, and ATL bills it the ceiling; no file at all + # means the bridge died before any call was possible. + output_path.write_text(json.dumps({"started": True}), encoding="utf-8") handler = None try: with counter if counter is not None else contextlib.nullcontext() as handler: @@ -108,25 +113,27 @@ def main() -> None: model_name=model_name, model_provider=model_provider, ) + # Inside the try, so an output that fails to serialize still leaves + # the usage report below rather than only the marker. + output_path.write_text( + json.dumps( + { + "decisions": result.get("decisions"), + "analyst_signals": result.get("analyst_signals"), + "usage": _usage_totals(handler), + } + ), + encoding="utf-8", + ) except BaseException: # A failed decision may still have spent model calls; ATL bills what # was used, so report it before the non-zero exit. Usage only: there # is no decision to read. output_path.write_text( - json.dumps({"usage": _usage_totals(handler)}), + json.dumps({"started": True, "usage": _usage_totals(handler)}), encoding="utf-8", ) raise - output_path.write_text( - json.dumps( - { - "decisions": result.get("decisions"), - "analyst_signals": result.get("analyst_signals"), - "usage": _usage_totals(handler), - } - ), - encoding="utf-8", - ) if __name__ == "__main__": diff --git a/dashboard/backend/tests/infrastructure/ai_hedge_fund/test_billing.py b/dashboard/backend/tests/infrastructure/ai_hedge_fund/test_billing.py index f92ac70a3..622d08f46 100644 --- a/dashboard/backend/tests/infrastructure/ai_hedge_fund/test_billing.py +++ b/dashboard/backend/tests/infrastructure/ai_hedge_fund/test_billing.py @@ -26,6 +26,7 @@ from dashboard.backend.infrastructure.ai_hedge_fund.adapter import ( DEFAULT_MODEL_NAME, DEFAULT_MODEL_PROVIDER, + AiHedgeFundOutputError, AiHedgeFundRuntime, AiHedgeFundRuntimeError, AiHedgeFundSubprocessRunner, @@ -226,8 +227,9 @@ def test_usage_above_the_ceiling_is_capped_rather_than_recorded_as_overage( @pytest.mark.parametrize( "error, status, settled", [ - # Killed mid-run: its model calls ran until the kill. - (AiHedgeFundRuntimeError("timed out", timed_out=True), "settled", CEILING), + # Killed mid-run (timeout, signal, crash past the started marker): its + # model calls may have run, so it fails closed at the ceiling. + (AiHedgeFundRuntimeError("timed out", killed=True), "settled", CEILING), # Failed after reporting what it had spent. ( AiHedgeFundRuntimeError("exit 1", usage=(10_000, 2_000)), @@ -266,25 +268,62 @@ def test_an_empty_balance_stops_the_run_before_upstream_is_called(tmp_path): assert _rows(tmp_path) == [] -def test_runner_carries_the_bridges_usage_out_of_a_failed_exit(monkeypatch, tmp_path): +def _runner_exiting(monkeypatch, tmp_path, returncode, written): + """A runner whose bridge leaves ``written`` (text, or None for no file).""" python = tmp_path / "python" python.write_text("") python.chmod(0o755) def fake_run(command, **kwargs): - output = command[command.index("--output") + 1] - with open(output, "w", encoding="utf-8") as handle: - json.dump({"usage": {"input_tokens": 7, "output_tokens": 3}}, handle) - return subprocess.CompletedProcess(command, 1, stdout="", stderr="boom") + if written is not None: + output = command[command.index("--output") + 1] + with open(output, "w", encoding="utf-8") as handle: + handle.write(written) + return subprocess.CompletedProcess(command, returncode, stdout="", stderr="boom") monkeypatch.setattr(subprocess, "run", fake_run) - runner = AiHedgeFundSubprocessRunner({"AI_HEDGE_FUND_PYTHON": str(python)}) + return AiHedgeFundSubprocessRunner({"AI_HEDGE_FUND_PYTHON": str(python)}) + + +@pytest.mark.parametrize( + "returncode, written, usage, killed", + [ + # The bridge's handler reported what it spent before exiting. + ( + 1, + json.dumps({"started": True, "usage": {"input_tokens": 7, "output_tokens": 3}}), + (7, 3), + False, + ), + # An OOM SIGKILL after the marker: no report, spend unknown. + (-9, json.dumps({"started": True}), None, True), + # Killed mid-write: a truncated file is still proof it started. + (-9, '{"started": tr', None, True), + # No file: died before upstream could call the model. + (1, None, None, False), + ], +) +def test_a_failed_exit_fails_closed_once_the_bridge_has_started( + monkeypatch, tmp_path, returncode, written, usage, killed +): + runner = _runner_exiting(monkeypatch, tmp_path, returncode, written) with pytest.raises(AiHedgeFundRuntimeError) as excinfo: runner.run({"tickers": ["AAPL"]}, timeout_seconds=5) - assert excinfo.value.usage == (7, 3) - assert excinfo.value.timed_out is False + assert (excinfo.value.usage, excinfo.value.killed) == (usage, killed) + + +@pytest.mark.parametrize("written", ["not json", "[]"]) +def test_an_unreadable_clean_exit_bills_the_ceiling(monkeypatch, tmp_path, written): + # A clean exit ran upstream to the end; refusing its output must not make + # the decision free. + runner = _runner_exiting(monkeypatch, tmp_path, 0, written) + + with pytest.raises(AiHedgeFundOutputError) as excinfo: + runner.run({"tickers": ["AAPL"]}, timeout_seconds=5) + + assert excinfo.value.killed is True def _bridge_args(monkeypatch, tmp_path): @@ -381,10 +420,28 @@ def run_hedge_fund(**_kwargs): bridge.main() assert json.loads(output_path.read_text(encoding="utf-8")) == { - "usage": {"input_tokens": 50, "output_tokens": 5} + "started": True, + "usage": {"input_tokens": 50, "output_tokens": 5}, } +def test_bridge_marks_itself_started_before_upstream_can_call_the_model( + monkeypatch, tmp_path +): + output_path = _bridge_args(monkeypatch, tmp_path) + seen = {} + + def run_hedge_fund(**_kwargs): + seen["at_call"] = json.loads(output_path.read_text(encoding="utf-8")) + return {"decisions": {}} + + _fake_upstream(monkeypatch, run_hedge_fund) + + bridge.main() + + assert seen["at_call"] == {"started": True} + + def test_bridge_without_a_usage_counter_reports_none(monkeypatch, tmp_path): output_path = _bridge_args(monkeypatch, tmp_path) _fake_upstream(monkeypatch, lambda **_kwargs: {"decisions": {}}) diff --git a/docs/architecture/backend-internals.md b/docs/architecture/backend-internals.md index 40cbc3d7c..9adbd92e8 100644 --- a/docs/architecture/backend-internals.md +++ b/docs/architecture/backend-internals.md @@ -36,7 +36,7 @@ Reservations go to `credit_llm_reservations` and settled usage to `credit_llm_us Research runs reserve `RESEARCH_RUN_ESTIMATE_USD_CENTS` (default 100) and settle at agent-reported usage, else the estimate. -AI Hedge Fund runs (`infrastructure/ai_hedge_fund/billing.py`) are sign-in-only and Credits-only (BYOK is a 422). The route signs a platform-credits handoff naming the managed OpenRouter model, refuses a balance that cannot hold one decision (402; restricted 403), and the child meters whole decisions, because upstream calls its model inside its own interpreter: one reservation per decision (`call_index` counts decisions) at a ceiling of `(analysts + 1) x tickers` calls of 8000 in / 2000 out tokens, settled at the token usage `bridge.py` reports through langchain-core's `get_usage_metadata_callback` (it sees LangGraph's parallel analyst threads). Usage above the ceiling is capped, not recorded as overage, so ATL's estimate never restricts an account. Without a report: a completed decision settles at the ceiling and logs `ERROR: ai_hedge_fund.usage_unreported`, a timed-out one settles at the ceiling (its calls ran until the kill), and any other failure is released (it died before the usage scope opened). A completed decision reporting zero tokens is released with `WARNING: ai_hedge_fund.usage_zero`. An empty balance mid-run raises `INSUFFICIENT_CREDITS` out of the engine, as for a pipeline run, and the existing `finalize_run` (child and parent) releases a hold a killed child leaves. +AI Hedge Fund runs (`infrastructure/ai_hedge_fund/billing.py`) are sign-in-only and Credits-only (BYOK is a 422). The route signs a platform-credits handoff naming the managed OpenRouter model, refuses a balance that cannot hold one decision (402; restricted 403), and the child meters whole decisions, because upstream calls its model inside its own interpreter: one reservation per decision (`call_index` counts decisions) at a ceiling of `(analysts + 1) x tickers` calls of 8000 in / 2000 out tokens, settled at the token usage `bridge.py` reports through langchain-core's `get_usage_metadata_callback` (it sees LangGraph's parallel analyst threads). Usage above the ceiling is capped, not recorded as overage, so ATL's estimate never restricts an account. Without a report it fails closed: the bridge writes a `{"started": true}` marker before upstream can call the model, so a completed decision (which also logs `ERROR: ai_hedge_fund.usage_unreported`), a timeout, and any failed exit that left the marker but no readable usage (an OOM SIGKILL, a crash mid-write) settle at the ceiling; only a failure that left no file at all (died before any call) is released. A completed decision reporting zero tokens is released with `WARNING: ai_hedge_fund.usage_zero`. An empty balance mid-run raises `INSUFFICIENT_CREDITS` out of the engine, as for a pipeline run, and the existing `finalize_run` (child and parent) releases a hold a killed child leaves. Other paths billed to the operator's keys rather than the user's Credits: leaderboard entries through their `dashboard/config/leaderboard.json` `integration`, Discord `/ask` (`domain/chat/service.py`), Trading Algo chat and backtest (`/api/algo/*`; the backtest is hourly Claude decisions, timed out at 60 minutes in `domain/backtesting/algo_service.py`) and the switched-off Robinhood path. From c1649f3d6de4c0a00cb3c45c2aeec6aca816c9b7 Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Fri, 30 Oct 2026 21:42:05 -0400 Subject: [PATCH 4/5] fix(ai-hedge-fund): close the review's billing holes - Bill the ceiling on every escape from a decision, SystemExit from a cancel's SIGTERM included, instead of leaving the hold for finalize_run to release for free. - Meter calls, not just tokens: the bridge's own inheritable callback hook reports calls and unmetered_calls, so calls whose usage went missing bill the ceiling (ERROR, relayed live) and only zero calls is free. - Admit AI Hedge Fund runs through the shared Credits admission at their window ceiling, so they enter the in-flight ledger with pipeline runs. - Share the reserve-error mapping with LLMExecutionService. - Run the ownership check before the guest 401, so it cannot reveal that a stranger's agent is hosted. - Record the Credits lane on hosted runs and in the modal's launch config, and stop a guest's hosted launch client-side. Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 4 +- dashboard/backend/api/routers/backtests.py | 130 ++++++---- .../backend/domain/backtesting/engine.py | 7 + .../infrastructure/ai_hedge_fund/adapter.py | 16 +- .../infrastructure/ai_hedge_fund/billing.py | 149 ++++++------ .../infrastructure/ai_hedge_fund/bridge.py | 148 ++++++++---- .../infrastructure/llm/execution/service.py | 82 +++++-- .../ai_hedge_fund/test_billing.py | 226 ++++++++++++++---- .../llm/test_provider_retry_timeout_policy.py | 2 +- .../backend/tests/test_agent_runs_metadata.py | 30 +++ .../tests/test_ai_hedge_fund_frontend.py | 30 +++ .../backend/tests/test_backtests_router.py | 71 +++++- dashboard/frontend/app.js | 15 +- docs/architecture/backend-internals.md | 2 +- 14 files changed, 673 insertions(+), 239 deletions(-) diff --git a/Dockerfile b/Dockerfile index d7544388c..7481cf83e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,7 @@ RUN pip install --no-cache-dir --no-deps -r requirements.txt \ # a managed 3.12 under /opt/uv-python; the venv keeps an absolute reference # to it, so uninstalling the uv build tool afterwards is safe. # -# The second import is the bridge's Credits meter (bridge.py:_usage_counter). +# The second import is the bridge's Credits meter (bridge.py:_usage_meter). # Missing, every decision still runs but bills its reservation ceiling, so an # upstream dependency drift fails the build instead of overcharging quietly. ENV UV_PYTHON_INSTALL_DIR=/opt/uv-python @@ -32,7 +32,7 @@ RUN pip install --no-cache-dir uv \ && uv pip install --python /opt/ai-hedge-fund-venv \ -r requirements-ai-hedge-fund.txt \ && /opt/ai-hedge-fund-venv/bin/python -c "from src.main import run_hedge_fund" \ - && /opt/ai-hedge-fund-venv/bin/python -c "from langchain_core.callbacks import get_usage_metadata_callback" \ + && /opt/ai-hedge-fund-venv/bin/python -c "from langchain_core.callbacks import BaseCallbackHandler; from langchain_core.tracers.context import register_configure_hook" \ && pip uninstall -y uv # Copy dashboard application diff --git a/dashboard/backend/api/routers/backtests.py b/dashboard/backend/api/routers/backtests.py index 0d244f9cb..a98b0a4e1 100644 --- a/dashboard/backend/api/routers/backtests.py +++ b/dashboard/backend/api/routers/backtests.py @@ -43,7 +43,7 @@ # ...and the bounded observational store calls (`_run_with_deadline`): the # timeout arm's spend lookup and the trace label. from threading import Thread as _DeadlineThread -from typing import Any, Dict, List, Literal, NamedTuple, Optional, Tuple +from typing import Any, Dict, List, Literal, NamedTuple, Optional, Sequence, Tuple import pytz from datetime import datetime, timedelta @@ -164,6 +164,7 @@ ) from dashboard.backend.infrastructure.ai_hedge_fund.billing import ( PROVIDER_ID as AI_HEDGE_FUND_PROVIDER_ID, + decision_calls as ai_hedge_fund_decision_calls, decision_ceiling_micro as ai_hedge_fund_decision_ceiling_micro, ) from dashboard.backend.domain.backtesting.constants import ( @@ -3052,6 +3053,9 @@ class _CreditsClaim(NamedTuple): # outside the lock (it is a store call). A run missing here started after # the read and is counted as having spent nothing yet. spent_by_run: Dict[str, int] + # False for a runtime whose model is platform-managed (AI Hedge Fund): + # the refusal then cannot suggest the caller's own API key. + byok_available: bool = True def _owner_platform_credit_slots(user_id: int) -> List[Tuple[str, int]]: @@ -3103,6 +3107,11 @@ def _credits_shortfall( f"{running} running backtest{'s' if running != 1 else ''} " f"{'are' if running != 1 else 'is'} expected to use" ) + if not claim.byok_available: + return message + ( + ". Add Credits on the Credits page, shorten the date range, or " + "select fewer analysts or tickers." + ) return message + ( ". Add Credits on the Credits page, shorten the date range, choose a " "lower-cost model, or use your own API key." @@ -3148,6 +3157,24 @@ def _enforce_platform_credits_balance( ) if estimate is None: return None + return _admit_platform_credits( + user_id=user_id, model_id=model_id, estimate=estimate + ) + + +def _admit_platform_credits( + *, + user_id: int, + model_id: str, + estimate: _PlatformCreditsEstimate, + byok_available: bool = True, +) -> Optional[_CreditsClaim]: + """The balance half of every ATL Credits preflight, whatever priced the run. + + 403 for a restricted account, 402 when the balance less the owner's other + in-flight Credits runs cannot cover ``estimate``; else the claim the slot + ledger re-checks and records. None (let through) on a failed balance read. + """ with _backtest_slots_lock: in_flight = _owner_platform_credit_slots(user_id) try: @@ -3177,6 +3204,7 @@ def _enforce_platform_credits_balance( calls=estimate.calls, model_id=model_id, spent_by_run=spent_by_run, + byok_available=byok_available, ) shortfall = _credits_shortfall(claim, in_flight) if shortfall is not None: @@ -3184,45 +3212,39 @@ def _enforce_platform_credits_balance( return claim -def _enforce_ai_hedge_fund_credits(user_id: int, decision_ceiling_micro: int) -> None: - """Refuse an AI Hedge Fund run that cannot hold even its first decision. - - One decision, not the window: the per-decision hold in the child is the - gate (a run that drains mid-way stops with ``INSUFFICIENT_CREDITS`` like a - pipeline run), and the ceiling is deliberately generous, so a whole-window - multiple of it would refuse accounts that can afford the run. 403 for a - restricted account. Fail-open on a failed read, for the reason - ``_enforce_platform_credits_balance`` gives: the hold still refuses. +def _enforce_ai_hedge_fund_credits( + *, + user_id: int, + model_id: str, + analysts: Optional[Sequence[str]], + ticker_count: int, + start_date: str, + end_date: str, + market: object = None, +) -> Optional[_CreditsClaim]: + """Refuse an AI Hedge Fund run its owner's Credits cannot hold to the end. + + Priced at its ceiling: one decision per trading session, each held at the + child's own per-decision ceiling (``billing.decision_ceiling_micro``), so + the claim recorded on the slot is the most the run can draw while it is in + flight. The window bound (``MAX_AI_HEDGE_FUND_TRADING_DAYS``) keeps that + a few Credits at most. Through the shared admission, so pipeline and + AI Hedge Fund launches on one balance count each other and cannot each + pass alone and then starve together. """ - try: - balance = credits_service.get_balance(int(user_id)) - except Exception as exc: # noqa: BLE001 - the reservation remains the gate - print( - "WARNING: credits.preflight_balance_unavailable " - f"category={type(exc).__name__}", - flush=True, - ) - return - if balance.account_status == "restricted": - raise HTTPException( - status_code=403, - detail=LLMExecutionError.account_restricted( - balance.restriction_reason, - int(balance.outstanding_credits_micro or 0), - ).safe_message, - ) - available = int(balance.total_available_micro) - if available < decision_ceiling_micro: - raise HTTPException( - status_code=402, - detail=( - "AI Hedge Fund holds up to " - f"{_format_credits(decision_ceiling_micro, round_up=True)} ATL " - "Credits per trading day while it decides, and your balance is " - f"{_format_credits(available)} Credits. Add Credits on the " - "Credits page, or select fewer analysts or tickers." - ), - ) + sessions = _estimated_trading_sessions(start_date, end_date, market=market) + if sessions <= 0: + return None + return _admit_platform_credits( + user_id=user_id, + model_id=model_id, + estimate=_PlatformCreditsEstimate( + calls=sessions * ai_hedge_fund_decision_calls(analysts, ticker_count), + needed_micro=sessions + * ai_hedge_fund_decision_ceiling_micro(model_id, analysts, ticker_count), + ), + byok_available=False, + ) def _enforce_pipeline_llm_window( @@ -3371,7 +3393,11 @@ def text(self) -> str: ) -_RELAYED_CHILD_LINE_PREFIX = "ERROR: llm." +# Child lines echoed to the service log live (``_drain_stream``): model +# execution failures, and the hosted AI Hedge Fund meter's alarms +# (``infrastructure/ai_hedge_fund/billing.py``), whose fail-closed charge must +# reach the log, not only an end-of-run dump that can elide it. +_RELAYED_CHILD_LINE_PREFIXES = ("ERROR: llm.", "ERROR: ai_hedge_fund.") # What a non-owner reads for a run a model-execution failure stopped: its # safe message names account facts (an empty balance, a refund review). _LLM_FAILURE_PUBLIC_ERROR = "Backtest stopped: the model run could not continue." @@ -3425,7 +3451,7 @@ def _without_relayed_lines(text: str) -> str: return "".join( line for line in text.splitlines(keepends=True) - if not line.startswith(_RELAYED_CHILD_LINE_PREFIX) + if not line.startswith(_RELAYED_CHILD_LINE_PREFIXES) ) @@ -3444,7 +3470,7 @@ def _drain_stream( try: for line in iter(stream.readline, ""): capture.feed(line) - if line.startswith(_RELAYED_CHILD_LINE_PREFIX): + if line.startswith(_RELAYED_CHILD_LINE_PREFIXES): # Echoed live, not left to the capture: the timeout path never # dumps it, a normal exit dumps it only when the run ends, and # a long run's middle is elided. Redacted like the dump, since @@ -4183,6 +4209,11 @@ def run_backtest_endpoint( # here. Keyed on ``!= PIPELINE`` so a runtime added later is gated by # default. if runtime_type != PIPELINE_RUNTIME_TYPE and user_id is None: + # Ownership first: the runtime type above is read off any agent id, so + # a 401 here for a stranger's agent would tell a guest that agent is a + # hosted one. A guest's own agent passes, and a stranger's gets the + # same 404/403 the session resolver gives any agent id it cannot run. + _resolve_backtest_session(request, agent_id) raise HTTPException( status_code=401, detail="Sign in before running a hosted agent backtest.", @@ -4514,13 +4545,14 @@ def run_backtest_endpoint( provider_id = AI_HEDGE_FUND_PROVIDER_ID provider_ids = (provider_id,) model = AI_HEDGE_FUND_MODEL_NAME - _enforce_ai_hedge_fund_credits( - int(user_id), - ai_hedge_fund_decision_ceiling_micro( - model, - (runtime_config or {}).get("analysts"), - len(selected_assets or ()) or len(profile.symbols), - ), + credits_claim = _enforce_ai_hedge_fund_credits( + user_id=int(user_id), + model_id=model, + analysts=(runtime_config or {}).get("analysts"), + ticker_count=len(selected_assets or ()) or len(profile.symbols), + start_date=start_date, + end_date=end_date, + market=profile.market, ) execution_handoff_payload = create_execution_handoff( user_id=int(user_id), diff --git a/dashboard/backend/domain/backtesting/engine.py b/dashboard/backend/domain/backtesting/engine.py index 6cf95a847..d064b529b 100644 --- a/dashboard/backend/domain/backtesting/engine.py +++ b/dashboard/backend/domain/backtesting/engine.py @@ -334,6 +334,7 @@ def __init__( # Model id; defaults to the gateway-appropriate slug (CommonStack vs native). self.model = model or default_model_name() self.execution_client = execution_client + self.hosted_billing = hosted_billing self.live_run_id = (live_run_id or "").strip() or None self.owner_user_id = int(owner_user_id) if owner_user_id is not None else None self.progress_file = (progress_file or "").strip() or None @@ -1845,6 +1846,12 @@ def _agent_run_metadata(self) -> Dict: llm_execution = summary() if llm_execution is not None: meta["llm_execution"] = llm_execution.model_dump(mode="json") + hosted_billing = getattr(self, "hosted_billing", None) + if hosted_billing is not None: + # A hosted run bills Credits without an execution client, so no + # `llm_execution` names its lane; the results panel reads these. + meta["billing_mode"] = hosted_billing.billing_mode + meta["provider_id"] = hosted_billing.provider_id if self.prompt_adaptations: meta["prompt_adaptations"] = self.prompt_adaptations if self.initial_pipeline is not None: diff --git a/dashboard/backend/infrastructure/ai_hedge_fund/adapter.py b/dashboard/backend/infrastructure/ai_hedge_fund/adapter.py index 4e21cf23c..dac815959 100644 --- a/dashboard/backend/infrastructure/ai_hedge_fund/adapter.py +++ b/dashboard/backend/infrastructure/ai_hedge_fund/adapter.py @@ -26,6 +26,7 @@ normalize_runtime_config, ) from dashboard.backend.infrastructure.ai_hedge_fund.billing import ( + DecisionUsage, HostedDecisionBilling, decision_ceiling_micro, parse_usage, @@ -101,7 +102,7 @@ def __init__( self, message: str = "", *, - usage: Optional[tuple[int, int]] = None, + usage: Optional[DecisionUsage] = None, killed: bool = False, ): super().__init__(message) @@ -828,6 +829,14 @@ def _run_billed(self, payload: Dict[str, Any]) -> Dict[str, Any]: Settled before the output is mapped: an output ATL then refuses still spent its model calls. A configuration error raised before the bridge starts is released like any no-call failure. + + Every other escape is billed the ceiling, ``BaseException`` included: + a cancel or the parent's run timeout reaches this child as SIGTERM, + which ``_exit_on_sigterm`` turns into ``SystemExit`` inside + ``subprocess.run`` (that kills the bridge, which shares the child's + process group and gets the same signal anyway, before it can report), + so the decision's calls may have run. Left open, the hold would reach + ``finalize_run`` and be released for free. """ if self.billing is None: return self.runner.run(payload, timeout_seconds=self.timeout_seconds) @@ -845,6 +854,11 @@ def _run_billed(self, payload: Dict[str, Any]) -> Dict[str, Any]: outcome="killed" if exc.killed else "failed", ) raise + except BaseException: + self.billing.close( + reservation_id, ceiling_micro=ceiling, usage=None, outcome="killed" + ) + raise self.billing.close( reservation_id, ceiling_micro=ceiling, diff --git a/dashboard/backend/infrastructure/ai_hedge_fund/billing.py b/dashboard/backend/infrastructure/ai_hedge_fund/billing.py index b348035b0..5f4b316f8 100644 --- a/dashboard/backend/infrastructure/ai_hedge_fund/billing.py +++ b/dashboard/backend/infrastructure/ai_hedge_fund/billing.py @@ -10,17 +10,16 @@ from __future__ import annotations -from typing import Any, Mapping, Optional, Sequence +from typing import Any, Mapping, NamedTuple, Optional, Sequence from dashboard.backend.domain.agents.runtime import AI_HEDGE_FUND_ANALYSTS -from dashboard.backend.domain.credits.repository_common import ( - CreditAccountRestrictedStoreError, - InsufficientCreditsError, -) from dashboard.backend.infrastructure.llm.execution.errors import ( ExecutionErrorCategory, LLMExecutionError, ) +from dashboard.backend.infrastructure.llm.execution.service import ( + reserve_platform_credits, +) from dashboard.backend.infrastructure.llm.token_cost import ( credits_micro_for_usd, estimate_cost_usd, @@ -37,17 +36,24 @@ CEILING_OUTPUT_TOKENS_PER_CALL = 2_000 +def decision_calls(analysts: Optional[Sequence[str]], ticker_count: int) -> int: + """The model calls one decision's ceiling budgets for. + + No analysts configured means upstream runs all of them. + """ + analyst_count = len(analysts or AI_HEDGE_FUND_ANALYSTS) + return (analyst_count + 1) * max(int(ticker_count), 1) + + def decision_ceiling_micro( model: str, analysts: Optional[Sequence[str]], ticker_count: int ) -> int: """The most one decision is held for, in Credit micro-units. One formula for the launch preflight and the per-decision hold, so the - route cannot admit a run whose first decision the child then refuses. - No analysts configured means upstream runs all of them. + route cannot admit a run whose decisions the child then refuses. """ - analyst_count = len(analysts or AI_HEDGE_FUND_ANALYSTS) - calls = (analyst_count + 1) * max(int(ticker_count), 1) + calls = decision_calls(analysts, ticker_count) return credits_micro_for_usd( estimate_cost_usd( model, @@ -57,22 +63,38 @@ def decision_ceiling_micro( ) -def parse_usage(value: Any) -> Optional[tuple[int, int]]: - """``(input_tokens, output_tokens)`` from the bridge's report, else None.""" +class DecisionUsage(NamedTuple): + """What the bridge's meter counted over one decision.""" + + input_tokens: int + output_tokens: int + # Model calls that ended, and those of them that reported no usage. + calls: int + unmetered_calls: int + + +def parse_usage(value: Any) -> Optional[DecisionUsage]: + """The bridge's usage report, else None (absent, malformed, inconsistent).""" if not isinstance(value, Mapping): return None counts = [] - for key in ("input_tokens", "output_tokens"): + for key in DecisionUsage._fields: count = value.get(key) if isinstance(count, bool) or not isinstance(count, int) or count < 0: return None counts.append(count) - return counts[0], counts[1] + usage = DecisionUsage(*counts) + if usage.unmetered_calls > usage.calls: + return None + return usage class HostedDecisionBilling: """Reserve, settle and release one run's per-decision Credit holds.""" + billing_mode = "platform_credits" + provider_id = PROVIDER_ID + def __init__(self, *, credits: Any, user_id: int, run_id: str, model_id: str): self.credits = credits self.user_id = int(user_id) @@ -93,31 +115,14 @@ def reserve(self, amount_micro: int) -> str: """Hold one decision's ceiling; the run stops when it cannot.""" call_index = self._next_call_index self._next_call_index += 1 - try: - reservation = self.credits.reserve_llm_credits( - user_id=self.user_id, - run_id=self.run_id, - call_index=call_index, - amount_micro=int(amount_micro), - provider_id=PROVIDER_ID, - ) - except CreditAccountRestrictedStoreError as exc: - try: - balance = self.credits.get_balance(self.user_id) - reason = balance.restriction_reason - outstanding_micro = balance.outstanding_credits_micro - except Exception: # noqa: BLE001 - keep the safe fallback - reason = None - outstanding_micro = 0 - raise LLMExecutionError.account_restricted( - reason, outstanding_micro - ) from exc - except InsufficientCreditsError as exc: - raise LLMExecutionError( - ExecutionErrorCategory.INSUFFICIENT_CREDITS - ) from exc - except Exception as exc: # noqa: BLE001 - store errors stay internal - raise LLMExecutionError(ExecutionErrorCategory.BILLING_FAILED) from exc + reservation = reserve_platform_credits( + self.credits, + user_id=self.user_id, + run_id=self.run_id, + call_index=call_index, + amount_micro=int(amount_micro), + provider_id=PROVIDER_ID, + ) if reservation.status != "open": raise LLMExecutionError(ExecutionErrorCategory.BILLING_FAILED) return reservation.reservation_id @@ -127,42 +132,30 @@ def close( reservation_id: str, *, ceiling_micro: int, - usage: Optional[tuple[int, int]], + usage: Optional[DecisionUsage], outcome: str, ) -> int: """Settle or release one decision's hold; returns the micro charged. - ``outcome`` is ``completed``, ``killed`` or ``failed``. Reported usage - is billed whatever the outcome. Without it the charge fails closed: a - completed decision whose usage went unreported is a broken meter, and a - killed one (a timeout, a signal, any failure past the bridge's started - marker) could have called the model, so both settle at the ceiling. - Only ``failed`` is released, which the adapter reports solely when the - bridge left no file at all, i.e. died before any model call. + ``outcome`` is ``completed``, ``killed`` or ``failed``. The charge + fails closed whenever the spend is unknown, settling at the ceiling: + + - no usage report on a ``completed`` or ``killed`` decision (an + unreadable output, a timeout, a signal, a crash past the bridge's + started marker): its model calls may have run; + - a report with unmetered calls: those calls ran and their provider + (or a langchain change) dropped the usage, which is a broken meter. + + Released only when nothing was spent: a report of zero calls (every + analyst rule-based, every ticker pre-filled), or ``failed`` with no + report, which the adapter says solely when the bridge left no file at + all, i.e. died before any model call. Otherwise the metered tokens. """ - if usage is not None: - input_tokens, output_tokens = usage - uncapped = credits_micro_for_usd( - estimate_cost_usd(self.model_id, input_tokens, output_tokens) - ) - basis = "usage" - if outcome == "completed" and input_tokens + output_tokens == 0: - # Legitimate only when every selected analyst is rule-based and - # upstream's portfolio manager pre-filled every ticker as a - # hold (it skips its model call then). Anything else is a meter - # that stopped counting (a provider omitting usage, a - # langchain-core change), so it is released but never silently. - print( - "WARNING: ai_hedge_fund.usage_zero completed decision " - "reported no model tokens", - flush=True, - ) - elif outcome == "failed": - input_tokens = output_tokens = None + input_tokens = output_tokens = None + if usage is None and outcome == "failed": uncapped = 0 basis = "no_model_call" - else: - input_tokens = output_tokens = None + elif usage is None: uncapped = int(ceiling_micro) basis = "ceiling" if outcome == "completed": @@ -171,6 +164,24 @@ def close( "billing at the decision ceiling", flush=True, ) + elif usage.unmetered_calls: + uncapped = int(ceiling_micro) + basis = "ceiling" + print( + "ERROR: ai_hedge_fund.usage_unmetered " + f"calls={usage.calls} unmetered_calls={usage.unmetered_calls} " + "billing at the decision ceiling", + flush=True, + ) + elif usage.calls == 0: + uncapped = 0 + basis = "no_model_call" + else: + input_tokens, output_tokens = usage.input_tokens, usage.output_tokens + uncapped = credits_micro_for_usd( + estimate_cost_usd(self.model_id, input_tokens, output_tokens) + ) + basis = "usage" charged = min(uncapped, int(ceiling_micro)) if uncapped > charged: print( @@ -193,6 +204,10 @@ def close( "basis": basis, "input_tokens": input_tokens, "output_tokens": output_tokens, + "calls": usage.calls if usage is not None else None, + "unmetered_calls": ( + usage.unmetered_calls if usage is not None else None + ), "ceiling_micro": int(ceiling_micro), "uncapped_micro": uncapped, }, diff --git a/dashboard/backend/infrastructure/ai_hedge_fund/bridge.py b/dashboard/backend/infrastructure/ai_hedge_fund/bridge.py index e90d7983e..6ddb00906 100644 --- a/dashboard/backend/infrastructure/ai_hedge_fund/bridge.py +++ b/dashboard/backend/infrastructure/ai_hedge_fund/bridge.py @@ -4,7 +4,9 @@ import argparse import contextlib +import contextvars import json +import threading from pathlib import Path MANAGED_MODEL_NAME = "nvidia/nemotron-3-nano-30b-a3b" @@ -50,33 +52,83 @@ def disabled_load_dotenv(*_args, **_kwargs) -> bool: setattr(dotenv_main, "load_dotenv", disabled_load_dotenv) -def _usage_counter(): - """A context manager that totals every model call's tokens, or ``None``. - - LangChain's ``get_usage_metadata_callback`` registers an inheritable - context-var handler, so it sees the calls LangGraph runs on its worker - threads too (the analysts fan out in parallel). Absent in an older - langchain-core; the caller then reports no usage and ATL bills the - decision's reservation ceiling instead. - """ +def _reported_usage(response) -> dict | None: + """One model call's ``input_tokens``/``output_tokens``, or None if unreported.""" try: - from langchain_core.callbacks import get_usage_metadata_callback - except ImportError: + usage = response.generations[0][0].message.usage_metadata + except (AttributeError, IndexError, TypeError): return None - return get_usage_metadata_callback() - - -def _usage_totals(counter) -> dict | None: - by_model = getattr(counter, "usage_metadata", None) - if not isinstance(by_model, dict): + if not isinstance(usage, dict): return None - totals = {"input_tokens": 0, "output_tokens": 0} - for usage in by_model.values(): - for key in totals: - value = usage.get(key) if isinstance(usage, dict) else None - if isinstance(value, int) and not isinstance(value, bool) and value > 0: - totals[key] += value - return totals + counts = {} + for key in ("input_tokens", "output_tokens"): + value = usage.get(key) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + return None + counts[key] = value + return counts + + +def _usage_meter(): + """A context manager yielding a meter of every model call (None without one). + + The meter counts calls as well as tokens. That is what tells a decision + that made no model call (every analyst rule-based and every ticker + pre-filled as a hold) from one whose calls came back without usage (a + provider or langchain change): the first is free, the second a broken meter + ATL bills at the decision ceiling. Registered the way langchain-core's own + ``get_usage_metadata_callback`` is, as an inheritable context-var hook, so + it sees the calls LangGraph runs on its worker threads too (the analysts + fan out in parallel). Without that hook API it yields None and ATL bills + the ceiling. + """ + try: + from langchain_core.callbacks import BaseCallbackHandler + from langchain_core.tracers.context import register_configure_hook + except ImportError: + return contextlib.nullcontext() + + class UsageMeter(BaseCallbackHandler): + def __init__(self) -> None: + super().__init__() + self._lock = threading.Lock() + self._totals = { + "input_tokens": 0, + "output_tokens": 0, + "calls": 0, + "unmetered_calls": 0, + } + + def on_llm_end(self, response, **_kwargs) -> None: + usage = _reported_usage(response) + with self._lock: + self._totals["calls"] += 1 + if usage is None: + self._totals["unmetered_calls"] += 1 + return + for key, value in usage.items(): + self._totals[key] += value + + def report(self) -> dict: + with self._lock: + return dict(self._totals) + + @contextlib.contextmanager + def metered(): + meter = UsageMeter() + meter_var = contextvars.ContextVar("atl_usage_meter", default=None) + register_configure_hook(meter_var, inheritable=True) + token = meter_var.set(meter) + try: + yield meter + finally: + meter_var.reset(token) + + return metered() + + +def _usage_report(meter) -> dict | None: + return meter.report() if meter is not None else None def main() -> None: @@ -93,16 +145,14 @@ def main() -> None: payload = json.loads(Path(args.input).read_text(encoding="utf-8")) model_name, model_provider = _managed_model_from_payload(payload) - counter = _usage_counter() output_path = Path(args.output) # Written before upstream can make a model call. A failed exit that leaves # only this marker (an OOM SIGKILL, a crash the handler below never saw) # spent an unknown amount, and ATL bills it the ceiling; no file at all # means the bridge died before any call was possible. output_path.write_text(json.dumps({"started": True}), encoding="utf-8") - handler = None - try: - with counter if counter is not None else contextlib.nullcontext() as handler: + with _usage_meter() as meter: + try: result = run_hedge_fund( tickers=payload["tickers"], start_date=payload["start_date"], @@ -113,27 +163,27 @@ def main() -> None: model_name=model_name, model_provider=model_provider, ) - # Inside the try, so an output that fails to serialize still leaves - # the usage report below rather than only the marker. - output_path.write_text( - json.dumps( - { - "decisions": result.get("decisions"), - "analyst_signals": result.get("analyst_signals"), - "usage": _usage_totals(handler), - } - ), - encoding="utf-8", - ) - except BaseException: - # A failed decision may still have spent model calls; ATL bills what - # was used, so report it before the non-zero exit. Usage only: there - # is no decision to read. - output_path.write_text( - json.dumps({"started": True, "usage": _usage_totals(handler)}), - encoding="utf-8", - ) - raise + # Inside the try, so an output that fails to serialize still + # leaves the usage report below rather than only the marker. + output_path.write_text( + json.dumps( + { + "decisions": result.get("decisions"), + "analyst_signals": result.get("analyst_signals"), + "usage": _usage_report(meter), + } + ), + encoding="utf-8", + ) + except BaseException: + # A failed decision may still have spent model calls; ATL bills + # what was used, so report it before the non-zero exit. Usage + # only: there is no decision to read. + output_path.write_text( + json.dumps({"started": True, "usage": _usage_report(meter)}), + encoding="utf-8", + ) + raise if __name__ == "__main__": diff --git a/dashboard/backend/infrastructure/llm/execution/service.py b/dashboard/backend/infrastructure/llm/execution/service.py index 90e3185f4..dcf56e746 100644 --- a/dashboard/backend/infrastructure/llm/execution/service.py +++ b/dashboard/backend/infrastructure/llm/execution/service.py @@ -8,7 +8,10 @@ import time from collections.abc import Callable, Iterator -from dashboard.backend.domain.credits.models import LLMSettlementResult +from dashboard.backend.domain.credits.models import ( + LLMReservation, + LLMSettlementResult, +) from dashboard.backend.domain.credits.service import CreditsService from dashboard.backend.domain.credits.repository_common import ( CreditAccountRestrictedStoreError, @@ -85,6 +88,50 @@ ) +def reserve_platform_credits( + credits: CreditsService, + *, + user_id: int, + run_id: str, + call_index: int, + amount_micro: int, + provider_id: str, + attempt_index: int = 0, +) -> LLMReservation: + """Hold one call's ceiling, mapping a refused hold to the run's error. + + The one reserve-error contract for every Credits path (this service and the + hosted AI Hedge Fund meter): a restricted account names its reason and the + amount owed, an empty balance is INSUFFICIENT_CREDITS, and any other store + failure stays internal as BILLING_FAILED. The ``status`` of the returned + hold is the caller's to check, since only the caller can release it. + """ + try: + return credits.reserve_llm_credits( + user_id=user_id, + run_id=run_id, + call_index=call_index, + attempt_index=attempt_index, + provider_id=provider_id, + amount_micro=amount_micro, + ) + except CreditAccountRestrictedStoreError as exc: + try: + balance = credits.get_balance(user_id) + reason = balance.restriction_reason + outstanding_micro = balance.outstanding_credits_micro + except Exception: # noqa: BLE001 - keep the safe fallback + reason = None + outstanding_micro = 0 + raise LLMExecutionError.account_restricted( + reason, outstanding_micro + ) from exc + except InsufficientCreditsError as exc: + raise LLMExecutionError(ExecutionErrorCategory.INSUFFICIENT_CREDITS) from exc + except Exception as exc: # noqa: BLE001 - store errors stay internal + raise LLMExecutionError(ExecutionErrorCategory.BILLING_FAILED) from exc + + def _report_provider_attempt_failed( request: LLMExecutionRequest, attempt_index: int, @@ -382,30 +429,15 @@ def _execute_platform( try: reserved_micro = _reservation_ceiling_micro(request, pricing_snapshot) if reserved_micro > 0: - try: - reservation = self.credits.reserve_llm_credits( - user_id=request.user_id, - run_id=request.run_id, - call_index=request.call_index, - attempt_index=attempt_index, - provider_id=request.provider_id, - amount_micro=reserved_micro, - ) - except CreditAccountRestrictedStoreError as exc: - try: - balance = self.credits.get_balance(request.user_id) - reason = balance.restriction_reason - outstanding_micro = balance.outstanding_credits_micro - except Exception: # noqa: BLE001 - keep the safe fallback - reason = None - outstanding_micro = 0 - raise LLMExecutionError.account_restricted( - reason, outstanding_micro - ) from exc - except InsufficientCreditsError as exc: - raise LLMExecutionError( - ExecutionErrorCategory.INSUFFICIENT_CREDITS - ) from exc + reservation = reserve_platform_credits( + self.credits, + user_id=request.user_id, + run_id=request.run_id, + call_index=request.call_index, + attempt_index=attempt_index, + provider_id=request.provider_id, + amount_micro=reserved_micro, + ) reservation_id = reservation.reservation_id self._platform_runs.add(request.run_id) if reservation.status != "open": diff --git a/dashboard/backend/tests/infrastructure/ai_hedge_fund/test_billing.py b/dashboard/backend/tests/infrastructure/ai_hedge_fund/test_billing.py index 622d08f46..458b63214 100644 --- a/dashboard/backend/tests/infrastructure/ai_hedge_fund/test_billing.py +++ b/dashboard/backend/tests/infrastructure/ai_hedge_fund/test_billing.py @@ -32,8 +32,10 @@ AiHedgeFundSubprocessRunner, ) from dashboard.backend.infrastructure.ai_hedge_fund.billing import ( + DecisionUsage, HostedDecisionBilling, decision_ceiling_micro, + parse_usage, ) from dashboard.backend.infrastructure.llm.execution.errors import ( ExecutionErrorCategory, @@ -146,6 +148,16 @@ def _rows(tmp_path): ] +def _usage(input_tokens, output_tokens, *, calls=1, unmetered_calls=0): + """The bridge's usage report, as it lands in its output file.""" + return { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "calls": calls, + "unmetered_calls": unmetered_calls, + } + + def _usage_micro(input_tokens, output_tokens): return credits_micro_for_usd( estimate_cost_usd(DEFAULT_MODEL_NAME, input_tokens, output_tokens) @@ -163,8 +175,8 @@ def test_ceiling_counts_every_analyst_when_none_are_configured(): def test_each_decision_settles_at_its_reported_usage(tmp_path): credits = _credits(tmp_path) runner = _Runner( - {"decisions": {}, "usage": {"input_tokens": 12_000, "output_tokens": 3_000}}, - {"decisions": {}, "usage": {"input_tokens": 4_000, "output_tokens": 1_000}}, + {"decisions": {}, "usage": _usage(12_000, 3_000, calls=6)}, + {"decisions": {}, "usage": _usage(4_000, 1_000, calls=2)}, ) runtime = _runtime(credits, runner) @@ -200,6 +212,81 @@ def test_unreported_usage_on_a_completed_decision_bills_the_ceiling_loudly( assert "ERROR: ai_hedge_fund.usage_unreported" in capsys.readouterr().out +def test_calls_that_reported_no_usage_bill_the_ceiling_loudly(tmp_path, capsys): + """A meter that counts calls but no tokens is broken, never free. + + Zero tokens alone cannot say whether the model was called; the call count + can, so a provider (or a langchain change) that drops usage is billed the + ceiling and named in an ERROR the parent relays to the service log. + """ + credits = _credits(tmp_path) + runtime = _runtime( + credits, + _Runner({"decisions": {}, "usage": _usage(0, 0, calls=3, unmetered_calls=3)}), + ) + + runtime.decide(_context()) + + [(_, status, _, settled, evidence)] = _rows(tmp_path) + assert (status, settled, evidence["basis"]) == ("settled", CEILING, "ceiling") + assert (evidence["calls"], evidence["unmetered_calls"]) == (3, 3) + assert "ERROR: ai_hedge_fund.usage_unmetered calls=3 unmetered_calls=3" in ( + capsys.readouterr().out + ) + + +def test_a_decision_that_made_no_model_call_is_released_quietly(tmp_path, capsys): + # Every analyst rule-based and every ticker pre-filled: nothing was spent. + credits = _credits(tmp_path) + runtime = _runtime(credits, _Runner({"decisions": {}, "usage": _usage(0, 0, calls=0)})) + + runtime.decide(_context()) + + [(_, status, _, settled, _)] = _rows(tmp_path) + assert (status, settled) == ("released", 0) + assert "ai_hedge_fund." not in capsys.readouterr().out + + +@pytest.mark.parametrize( + "value", + [ + None, + {"input_tokens": 1, "output_tokens": 1}, # no call counts: an old bridge + _usage(1, 1, calls=1, unmetered_calls=2), # more unmetered than calls + _usage(1, -1), + _usage(True, 1), + ], +) +def test_a_malformed_usage_report_is_no_report(value): + assert parse_usage(value) is None + + +@pytest.mark.parametrize( + "interrupt", + [ + # A cancel or the parent's run timeout: SIGTERM, which the child's + # handler turns into SystemExit inside subprocess.run. + SystemExit(143), + KeyboardInterrupt(), + # Anything else past the runner's own errors, e.g. decoding output. + UnicodeDecodeError("utf-8", b"\xff", 0, 1, "invalid start byte"), + ], +) +def test_an_interrupted_decision_bills_the_ceiling_instead_of_leaving_the_hold( + tmp_path, interrupt +): + """Left open, the hold would reach `finalize_run` and be released free.""" + credits = _credits(tmp_path) + runtime = _runtime(credits, _Runner(interrupt)) + + with pytest.raises(type(interrupt)): + runtime.decide(_context()) + + [(_, status, _, settled, evidence)] = _rows(tmp_path) + assert (status, settled) == ("settled", CEILING) + assert (evidence["outcome"], evidence["basis"]) == ("killed", "ceiling") + + def test_usage_above_the_ceiling_is_capped_rather_than_recorded_as_overage( tmp_path, capsys ): @@ -209,7 +296,7 @@ def test_usage_above_the_ceiling_is_capped_rather_than_recorded_as_overage( _Runner( { "decisions": {}, - "usage": {"input_tokens": 50_000_000, "output_tokens": 0}, + "usage": _usage(50_000_000, 0), } ), ) @@ -232,10 +319,18 @@ def test_usage_above_the_ceiling_is_capped_rather_than_recorded_as_overage( (AiHedgeFundRuntimeError("timed out", killed=True), "settled", CEILING), # Failed after reporting what it had spent. ( - AiHedgeFundRuntimeError("exit 1", usage=(10_000, 2_000)), + AiHedgeFundRuntimeError( + "exit 1", usage=DecisionUsage(10_000, 2_000, 4, 0) + ), "settled", _usage_micro(10_000, 2_000), ), + # Failed after calls whose usage went unreported: still the ceiling. + ( + AiHedgeFundRuntimeError("exit 1", usage=DecisionUsage(0, 0, 2, 2)), + "settled", + CEILING, + ), # Failed before its usage scope opened: no model call was made. (AiHedgeFundRuntimeError("exit 1"), "released", 0), ], @@ -291,8 +386,8 @@ def fake_run(command, **kwargs): # The bridge's handler reported what it spent before exiting. ( 1, - json.dumps({"started": True, "usage": {"input_tokens": 7, "output_tokens": 3}}), - (7, 3), + json.dumps({"started": True, "usage": _usage(7, 3)}), + DecisionUsage(7, 3, 1, 0), False, ), # An OOM SIGKILL after the marker: no report, spend unknown. @@ -352,11 +447,14 @@ def _bridge_args(monkeypatch, tmp_path): def _fake_upstream(monkeypatch, run_hedge_fund): - """Install a fake upstream and a fake langchain-core usage counter. - - The counter mirrors ``get_usage_metadata_callback``: a context manager - whose handler accumulates ``usage_metadata`` per model name. The real one - was exercised against upstream's pinned langchain stack while writing this. + """Install a fake upstream and fake langchain-core callback hooks. + + The fakes stand in for the two langchain-core names the meter uses: a + ``BaseCallbackHandler`` base, and ``register_configure_hook``, which here + records the meter's context var. The returned function reads that var, so + a fake model call reaches the active meter the way a real call's callback + manager does. The real pair, worker threads included, was exercised + against upstream's pinned langchain stack while writing this. """ fake_main = types.ModuleType("src.main") fake_main.run_hedge_fund = run_hedge_fund @@ -365,63 +463,111 @@ def _fake_upstream(monkeypatch, run_hedge_fund): monkeypatch.setitem(sys.modules, "src", fake_src) monkeypatch.setitem(sys.modules, "src.main", fake_main) - handler = types.SimpleNamespace(usage_metadata={}) - - class _Counter: - def __enter__(self): - return handler - - def __exit__(self, *exc): - return False - + hooks = [] callbacks = types.ModuleType("langchain_core.callbacks") - callbacks.get_usage_metadata_callback = _Counter + callbacks.BaseCallbackHandler = type("BaseCallbackHandler", (), {}) + context = types.ModuleType("langchain_core.tracers.context") + context.register_configure_hook = ( + lambda var, inheritable: hooks.append((var, inheritable)) + ) + tracers = types.ModuleType("langchain_core.tracers") + tracers.context = context core = types.ModuleType("langchain_core") core.callbacks = callbacks - monkeypatch.setitem(sys.modules, "langchain_core", core) - monkeypatch.setitem(sys.modules, "langchain_core.callbacks", callbacks) - return handler + core.tracers = tracers + for name, module in ( + ("langchain_core", core), + ("langchain_core.callbacks", callbacks), + ("langchain_core.tracers", tracers), + ("langchain_core.tracers.context", context), + ): + monkeypatch.setitem(sys.modules, name, module) + + def active_meter(): + var, inheritable = hooks[-1] + # Inheritable, or the analysts' worker-thread calls go unseen. + assert inheritable is True + return var.get() + + return active_meter + + +def _model_call(meter, usage_metadata): + """One finished chat-model call as langchain hands it to ``on_llm_end``.""" + message = types.SimpleNamespace(usage_metadata=usage_metadata) + meter.on_llm_end( + types.SimpleNamespace(generations=[[types.SimpleNamespace(message=message)]]) + ) def test_bridge_reports_summed_usage_with_the_decisions(monkeypatch, tmp_path): output_path = _bridge_args(monkeypatch, tmp_path) def run_hedge_fund(**_kwargs): - handler.usage_metadata.update( - { - DEFAULT_MODEL_NAME: {"input_tokens": 900, "output_tokens": 90}, - "other-model": {"input_tokens": 100, "output_tokens": 10}, - } - ) + meter = active_meter() + _model_call(meter, {"input_tokens": 900, "output_tokens": 90}) + _model_call(meter, {"input_tokens": 100, "output_tokens": 10}) return {"decisions": {}, "analyst_signals": {}} - handler = _fake_upstream(monkeypatch, run_hedge_fund) + active_meter = _fake_upstream(monkeypatch, run_hedge_fund) bridge.main() output = json.loads(output_path.read_text(encoding="utf-8")) - assert output["usage"] == {"input_tokens": 1000, "output_tokens": 100} + assert output["usage"] == _usage(1000, 100, calls=2) assert output["decisions"] == {} +@pytest.mark.parametrize( + "usage_metadata", + [None, {}, {"input_tokens": 5}, {"input_tokens": 5, "output_tokens": "1"}], +) +def test_bridge_counts_a_call_without_usage_as_unmetered( + monkeypatch, tmp_path, usage_metadata +): + output_path = _bridge_args(monkeypatch, tmp_path) + + def run_hedge_fund(**_kwargs): + meter = active_meter() + _model_call(meter, {"input_tokens": 900, "output_tokens": 90}) + _model_call(meter, usage_metadata) + return {"decisions": {}} + + active_meter = _fake_upstream(monkeypatch, run_hedge_fund) + + bridge.main() + + assert json.loads(output_path.read_text(encoding="utf-8"))["usage"] == _usage( + 900, 90, calls=2, unmetered_calls=1 + ) + + +def test_bridge_reports_zero_calls_when_upstream_made_none(monkeypatch, tmp_path): + output_path = _bridge_args(monkeypatch, tmp_path) + _fake_upstream(monkeypatch, lambda **_kwargs: {"decisions": {}}) + + bridge.main() + + assert json.loads(output_path.read_text(encoding="utf-8"))["usage"] == _usage( + 0, 0, calls=0 + ) + + def test_bridge_reports_usage_spent_before_a_failure(monkeypatch, tmp_path): output_path = _bridge_args(monkeypatch, tmp_path) def run_hedge_fund(**_kwargs): - handler.usage_metadata[DEFAULT_MODEL_NAME] = { - "input_tokens": 50, - "output_tokens": 5, - } + _model_call(active_meter(), {"input_tokens": 50, "output_tokens": 5}) raise RuntimeError("upstream exploded") - handler = _fake_upstream(monkeypatch, run_hedge_fund) + active_meter = _fake_upstream(monkeypatch, run_hedge_fund) with pytest.raises(RuntimeError): bridge.main() assert json.loads(output_path.read_text(encoding="utf-8")) == { "started": True, - "usage": {"input_tokens": 50, "output_tokens": 5}, + "usage": _usage(50, 5), } @@ -442,11 +588,11 @@ def run_hedge_fund(**_kwargs): assert seen["at_call"] == {"started": True} -def test_bridge_without_a_usage_counter_reports_none(monkeypatch, tmp_path): +def test_bridge_without_the_callback_hook_api_reports_none(monkeypatch, tmp_path): output_path = _bridge_args(monkeypatch, tmp_path) _fake_upstream(monkeypatch, lambda **_kwargs: {"decisions": {}}) # An older langchain-core: the import fails, so ATL bills the ceiling. - monkeypatch.setitem(sys.modules, "langchain_core.callbacks", None) + monkeypatch.setitem(sys.modules, "langchain_core.tracers.context", None) bridge.main() diff --git a/dashboard/backend/tests/infrastructure/llm/test_provider_retry_timeout_policy.py b/dashboard/backend/tests/infrastructure/llm/test_provider_retry_timeout_policy.py index 35aadf7e9..840c09807 100644 --- a/dashboard/backend/tests/infrastructure/llm/test_provider_retry_timeout_policy.py +++ b/dashboard/backend/tests/infrastructure/llm/test_provider_retry_timeout_policy.py @@ -556,6 +556,6 @@ def test_attempt_failure_line_is_relay_safe(capsys): # A run id that is not a plain identifier is dropped, never echoed. assert " run=- " in second for line in (first, second): - assert line.startswith(backtests._RELAYED_CHILD_LINE_PREFIX) + assert line.startswith(backtests._RELAYED_CHILD_LINE_PREFIXES) assert backtests._redact_credentials(line) == line assert SECRET not in line diff --git a/dashboard/backend/tests/test_agent_runs_metadata.py b/dashboard/backend/tests/test_agent_runs_metadata.py index 3ffec8b51..274553069 100644 --- a/dashboard/backend/tests/test_agent_runs_metadata.py +++ b/dashboard/backend/tests/test_agent_runs_metadata.py @@ -287,3 +287,33 @@ def test_engine_records_unreadable_confidence_counts(): backtester.unusable_confidence = {} assert "unusable_confidence" not in backtester._agent_run_metadata() + + +def test_engine_names_a_hosted_runs_credits_lane(): + """A hosted run bills Credits with no execution client to report it. + + Without these the results panel read Billing "—" for a run that charged + the user, so the engine names the lane off the hosted meter itself. + """ + import dashboard.backend.domain.backtesting.engine as engine_mod + from dashboard.backend.infrastructure.ai_hedge_fund.billing import ( + HostedDecisionBilling, + ) + + backtester = engine_mod.HourlyBacktester.__new__(engine_mod.HourlyBacktester) + backtester.prompt_adaptations = [] + backtester.initial_pipeline = None + backtester.pipeline = None + backtester.symbols = ["AAPL"] + backtester.data_source = "alpaca" + backtester.use_llm = False + backtester.hosted_billing = HostedDecisionBilling( + credits=None, user_id=7, run_id="agent_x", model_id="m" + ) + + meta = backtester._agent_run_metadata() + + assert (meta["billing_mode"], meta["provider_id"]) == ( + "platform_credits", + "openrouter", + ) diff --git a/dashboard/backend/tests/test_ai_hedge_fund_frontend.py b/dashboard/backend/tests/test_ai_hedge_fund_frontend.py index 8b96226d7..fd3924fa0 100644 --- a/dashboard/backend/tests/test_ai_hedge_fund_frontend.py +++ b/dashboard/backend/tests/test_ai_hedge_fund_frontend.py @@ -268,3 +268,33 @@ def test_editor_asset_cache_bust_advances_with_its_source(): assert editor_version >= 27 assert styles_version >= 87 + + +def _run_backtest_body() -> str: + from dashboard.backend.tests._frontend_source import fn_body, strip_comments + + return strip_comments(fn_body("async function runBacktest(")) + + +def test_a_guest_is_asked_to_sign_in_before_a_hosted_run_starts(): + """The route answers a guest's hosted run 401 (it bills ATL Credits). + + Checked before `activateAgent`, so the agent is never marked running and + the progress panel never starts for a launch the server always refuses. + """ + body = _run_backtest_body() + guard = body.index("!== 'pipeline' && !isSignedIn()") + assert guard < body.index("await activateAgent(activeAgent)") + assert "Sign in to run a hosted agent backtest" in body[guard:] + + +def test_a_hosted_run_records_its_credits_lane_for_the_results_panel(): + """The modal sends no billing_mode for a hosted run, which bills Credits. + + Left to `selectedBillingMode` (set for pipeline runs only), the results + panel read Billing "—" for a run that charged the user. + """ + body = _run_backtest_body() + launch = body[body.index("const launchConfigBase") :] + billing = launch[launch.index("billingMode:") : launch.index("providerId:")] + assert "isHostedRuntime ? 'platform_credits'" in billing diff --git a/dashboard/backend/tests/test_backtests_router.py b/dashboard/backend/tests/test_backtests_router.py index 57d15670b..1f5d35e82 100644 --- a/dashboard/backend/tests/test_backtests_router.py +++ b/dashboard/backend/tests/test_backtests_router.py @@ -1058,6 +1058,38 @@ def test_guest_cannot_start_a_hosted_backtest(client, monkeypatch): assert spy.calls == 0 +def test_guest_cannot_learn_that_a_strangers_agent_is_hosted(client, monkeypatch): + """The ownership answer comes before the guest 401. + + The runtime type is read off any agent id, so a 401 for a stranger's + agent would tell a guest the agent is a hosted one; it gets the same 403 + a stranger's agent of any runtime gets. + """ + spy = _Spy() + monkeypatch.setattr(bt, "run_backtest_background", spy) + cloned = client.post( + "/api/v1/agents/marketplace/ai-hedge-fund/clone", + json={}, + headers={"X-Session-Id": str(uuid.uuid4())}, + ) + assert cloned.status_code == 200, cloned.text + + response = client.post( + "/backtest/run", + json={ + "start_date": "2026-05-01", + "end_date": "2026-05-02", + "decision_source": "llm", + "agent_id": cloned.json()["agent"]["agent_id"], + }, + headers={"X-Session-Id": str(uuid.uuid4())}, + ) + + assert response.status_code == 403, response.text + assert response.json()["detail"] == "Not your agent" + assert spy.calls == 0 + + def _post_ai_hedge_fund_run(client, monkeypatch, credits, **body): """Launch the cloned AI Hedge Fund agent as a signed-in caller. @@ -1126,7 +1158,7 @@ def test_ai_hedge_fund_run_bills_credits_through_a_signed_handoff( assert spy.last_kwargs["model"] == handoff["model_id"] -def test_ai_hedge_fund_run_that_cannot_hold_one_decision_is_refused_with_402( +def test_ai_hedge_fund_run_its_balance_cannot_hold_is_refused_with_402( client, monkeypatch ): credits = _FakeBalanceCredits(1_000) @@ -1135,13 +1167,48 @@ def test_ai_hedge_fund_run_that_cannot_hold_one_decision_is_refused_with_402( assert response.status_code == 402, response.text detail = response.json()["detail"] - assert "per trading day" in detail assert "your balance is 0.00 Credits" in detail + # The model is platform-managed: no "use your own API key" remedy. + assert "select fewer analysts or tickers" in detail + assert "API key" not in detail assert credits.balance_reads == [7] assert handoffs == [] assert spy.calls == 0 +def test_ai_hedge_fund_run_enters_the_in_flight_credits_ledger(client, monkeypatch): + """Admitted at its window ceiling, so the owner's next launch counts it.""" + response, _spy, _handoffs = _post_ai_hedge_fund_run( + client, monkeypatch, _FakeBalanceCredits(8_000_000) + ) + + assert response.status_code == 200, response.text + run_id = response.json()["run_id"] + [(admitted_run, estimate)] = bt._owner_platform_credit_slots(7) + assert admitted_run == run_id + assert estimate > 0 + + +def test_ai_hedge_fund_launch_counts_the_owners_running_credits_runs( + client, monkeypatch +): + # Alone it fits in 8 Credits; beside a sibling still expected to spend + # almost all of them, both would start and then starve together. + _admit_running_platform_slot("agent_sibling", user_id=7, estimate=7_999_000) + + class _Credits(_FakeBalanceCredits): + def sum_run_llm_spend(self, user_id, run_id): + return (0, 0) + + response, spy, handoffs = _post_ai_hedge_fund_run( + client, monkeypatch, _Credits(8_000_000) + ) + + assert response.status_code == 402, response.text + assert "1 running backtest is expected to use" in response.json()["detail"] + assert handoffs == [] and spy.calls == 0 + + def test_ai_hedge_fund_restricted_account_is_refused_with_403(client, monkeypatch): credits = _FakeBalanceCredits( 50_000_000, status="restricted", reason="refund_reconciliation" diff --git a/dashboard/frontend/app.js b/dashboard/frontend/app.js index 009adb0c5..c424e66fb 100644 --- a/dashboard/frontend/app.js +++ b/dashboard/frontend/app.js @@ -10943,6 +10943,12 @@ async function runBacktest() { alert('Please create or select an agent first.'); return; } + if ((activeAgent.runtime_type || 'pipeline') !== 'pipeline' && !isSignedIn()) { + // The route refuses a guest's hosted run with 401 (it bills ATL + // Credits); say so before the agent and progress panel start. + showModalError('Sign in to run a hosted agent backtest: it is billed to your ATL Credits.'); + return; + } await activateAgent(activeAgent); const isHostedRuntime = (activeAgent.runtime_type || 'pipeline') !== 'pipeline'; @@ -11047,8 +11053,13 @@ async function runBacktest() { fill_policy: 'next_source_bar_open', }, decisionSource, - billingMode: isRuleBasedDecision ? null : selectedBillingMode, - providerId: isRuleBasedDecision || selectedBillingMode === 'platform_credits' + // A hosted runtime bills ATL Credits server-side (the route refuses + // BYOK), though the modal sends no billing_mode for it. + billingMode: isRuleBasedDecision + ? null + : (isHostedRuntime ? 'platform_credits' : selectedBillingMode), + providerId: isRuleBasedDecision || isHostedRuntime + || selectedBillingMode === 'platform_credits' ? null : selectedProviderId, marketDataLabel: isIFind diff --git a/docs/architecture/backend-internals.md b/docs/architecture/backend-internals.md index 9adbd92e8..9d8f0709f 100644 --- a/docs/architecture/backend-internals.md +++ b/docs/architecture/backend-internals.md @@ -36,7 +36,7 @@ Reservations go to `credit_llm_reservations` and settled usage to `credit_llm_us Research runs reserve `RESEARCH_RUN_ESTIMATE_USD_CENTS` (default 100) and settle at agent-reported usage, else the estimate. -AI Hedge Fund runs (`infrastructure/ai_hedge_fund/billing.py`) are sign-in-only and Credits-only (BYOK is a 422). The route signs a platform-credits handoff naming the managed OpenRouter model, refuses a balance that cannot hold one decision (402; restricted 403), and the child meters whole decisions, because upstream calls its model inside its own interpreter: one reservation per decision (`call_index` counts decisions) at a ceiling of `(analysts + 1) x tickers` calls of 8000 in / 2000 out tokens, settled at the token usage `bridge.py` reports through langchain-core's `get_usage_metadata_callback` (it sees LangGraph's parallel analyst threads). Usage above the ceiling is capped, not recorded as overage, so ATL's estimate never restricts an account. Without a report it fails closed: the bridge writes a `{"started": true}` marker before upstream can call the model, so a completed decision (which also logs `ERROR: ai_hedge_fund.usage_unreported`), a timeout, and any failed exit that left the marker but no readable usage (an OOM SIGKILL, a crash mid-write) settle at the ceiling; only a failure that left no file at all (died before any call) is released. A completed decision reporting zero tokens is released with `WARNING: ai_hedge_fund.usage_zero`. An empty balance mid-run raises `INSUFFICIENT_CREDITS` out of the engine, as for a pipeline run, and the existing `finalize_run` (child and parent) releases a hold a killed child leaves. +AI Hedge Fund runs (`infrastructure/ai_hedge_fund/billing.py`) are sign-in-only and Credits-only (BYOK is a 422). A guest's 401 comes after the agent ownership check, so a stranger's agent answers the same 403/404 as any other and the 401 cannot reveal that it is hosted. The route signs a platform-credits handoff naming the managed OpenRouter model and admits the run through the same `_admit_platform_credits` as a pipeline run, priced at its ceiling (one decision per trading session, each at the per-decision ceiling below), so the slot ledger counts it against the owner's other in-flight Credits runs and vice versa (402; restricted 403). The child meters whole decisions, because upstream calls its model inside its own interpreter: one reservation per decision (`call_index` counts decisions) at a ceiling of `(analysts + 1) x tickers` calls of 8000 in / 2000 out tokens, reserved through the shared `reserve_platform_credits` (`infrastructure/llm/execution/service.py`). `bridge.py` meters every model call with an inheritable langchain-core callback hook (it sees LangGraph's parallel analyst threads) and reports tokens plus `calls` and `unmetered_calls`, which is what tells *no call* from *calls without usage*. Settled at the reported tokens; usage above the ceiling is capped, not recorded as overage, so ATL's estimate never restricts an account. Released only when nothing was spent: zero calls reported, or a failure that left no bridge file at all (died before any call). Every unknown spend settles at the ceiling: unmetered calls (`ERROR: ai_hedge_fund.usage_unmetered`), a completed decision without a report (`ERROR: ai_hedge_fund.usage_unreported`), a timeout, a failed exit that left the bridge's `{"started": true}` marker but no readable usage (an OOM SIGKILL, a crash mid-write), and any other escape from the decision, `SystemExit` from a cancel's SIGTERM included. The parent relays `ERROR: ai_hedge_fund.` lines to the service log live, like `ERROR: llm.`. An empty balance mid-run raises `INSUFFICIENT_CREDITS` out of the engine, as for a pipeline run, and the existing `finalize_run` (child and parent) releases a hold only a SIGKILLed child can leave (the child is killed before it settles; the bridge shares its process group and dies with it). The run's metadata records `billing_mode`/`provider_id`, since no execution client reports them. Other paths billed to the operator's keys rather than the user's Credits: leaderboard entries through their `dashboard/config/leaderboard.json` `integration`, Discord `/ask` (`domain/chat/service.py`), Trading Algo chat and backtest (`/api/algo/*`; the backtest is hourly Claude decisions, timed out at 60 minutes in `domain/backtesting/algo_service.py`) and the switched-off Robinhood path. From f818cf9d64f38aa03e3e57bd238e40aec15b6cbb Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Fri, 30 Oct 2026 22:27:01 -0400 Subject: [PATCH 5/5] docs(lab): AI Hedge Fund needs sign-in and bills ATL Credits Six user-facing pages still said the hosted runtime needed no sign-in and spent no Credits. Add an "AI Hedge Fund runs" section to Credits & Billing: per-decision holds, full-hold charge when usage is unknown or a decision is cut short, and the whole-window launch check. Co-Authored-By: Claude Opus 5.5 --- docs/source/lab/accounts.rst | 5 +++-- docs/source/lab/credits_billing.rst | 31 +++++++++++++++++++++++++---- docs/source/lab/getting_started.rst | 11 +++++----- docs/source/lab/key_features.rst | 4 ++-- docs/source/lab/marketplace.rst | 7 ++++--- docs/source/lab/operating_modes.rst | 2 +- 6 files changed, 43 insertions(+), 17 deletions(-) diff --git a/docs/source/lab/accounts.rst b/docs/source/lab/accounts.rst index 6a080da24..45199ea60 100644 --- a/docs/source/lab/accounts.rst +++ b/docs/source/lab/accounts.rst @@ -4,8 +4,9 @@ Accounts and Profile An account is **optional** for rule-based backtests, which run without one. Backtests where an LLM you pick makes the decisions need you to be signed in and to choose how the AI usage is billed (see :doc:`credits_billing`). The -hosted AI Hedge Fund runtime is the exception: it runs on a model the platform -manages and needs neither. Signing in adds a few things: +hosted AI Hedge Fund runtime needs you to be signed in too: it runs on a model +the platform manages and is always paid for with ATL Credits. Signing in adds a +few things: - **Persistent agents.** Agents you register under **My Agents** stay tied to your account, and their API keys keep working across devices and sessions. diff --git a/docs/source/lab/credits_billing.rst b/docs/source/lab/credits_billing.rst index 6ecf20b3e..1ef0f6278 100644 --- a/docs/source/lab/credits_billing.rst +++ b/docs/source/lab/credits_billing.rst @@ -9,9 +9,10 @@ decision, plus any post-trade review — and those calls cost money. Before such backtest can start you sign in and choose who pays for them: **your own API key**, or **ATL Credits**. Rule-based backtests make no model calls, need no account and cost nothing. The hosted AI Hedge Fund runtime runs on a model the -platform manages: it needs no sign-in, shows no billing choice and spends no -Credits. :doc:`Research Agents ` runs are paid for with ATL -Credits too (see :ref:`research-credits`). +platform manages: it needs you to sign in, shows no billing choice and is always +paid for with ATL Credits (see :ref:`ai-hedge-fund-credits`). +:doc:`Research Agents ` runs are paid for with ATL Credits too +(see :ref:`research-credits`). Open the page from the account menu: click your avatar, then **Credits & Billing**. It has three tabs: **API Keys**, **Credits** and **Activity**. @@ -44,7 +45,8 @@ two options. If neither option is available to you yet, the dialog offers **Go to API Keys** so you can add and verify a key. The switch only appears for LLM-driven runs; -rule-based runs and the hosted AI Hedge Fund runtime do not show it. +rule-based runs do not show it, and neither does the hosted AI Hedge Fund +runtime, which always uses ATL Credits. Use your own API key @@ -117,6 +119,27 @@ charged what the agent reports it used, or the held amount if it reports nothing; a run that fails releases its hold and costs nothing. While the run is in progress its hold is not available to spend on backtests. +.. _ai-hedge-fund-credits: + +AI Hedge Fund runs +~~~~~~~~~~~~~~~~~~ + +The hosted AI Hedge Fund runtime is billed per decision rather than per call, +because each decision makes several model calls (one per analyst and one for +the portfolio manager, for every asset). Before each decision ATL holds the +most it could cost, then charges the input and output tokens its model calls +report and releases the rest. A decision that makes no model calls costs +nothing. + +When ATL cannot tell what a decision used, it charges the full hold rather than +nothing: if a call reports no usage, or the decision is cut short by a timeout, +a crash or a **Cancel** while it is running. + +Before the run starts, your balance must cover the most the whole window could +cost: one decision per trading day, each at its maximum. Other backtests you +have running on ATL Credits are set aside first, as above. What you are charged +is still each decision's reported usage, which is usually far less. + Welcome Credits --------------- diff --git a/docs/source/lab/getting_started.rst b/docs/source/lab/getting_started.rst index 4f1fa5025..4939c9159 100644 --- a/docs/source/lab/getting_started.rst +++ b/docs/source/lab/getting_started.rst @@ -19,9 +19,10 @@ signed in and to choose how the model calls are paid for, in the same dialog: **Provider** where offered, and choose the **Model for this run**; if no key is saved and Credits are unavailable, **Go to API Keys** takes you to where keys are stored. See :doc:`credits_billing` for how each option is charged. Runs where -no AI model is involved (rule-based decisions) work without signing in. So does -the hosted AI Hedge Fund runtime: its model is managed by the platform, so it -shows no billing choice and spends no Credits. +no AI model is involved (rule-based decisions) work without signing in. The +hosted AI Hedge Fund runtime needs you to sign in too, but shows no billing +choice: its model is managed by the platform and is always paid for with ATL +Credits. A run does not start pricing hours the moment you click. The card names the stage it is in while it gets there — ``Loading market data``, then @@ -131,8 +132,8 @@ Accounts (optional) You can try rule-based backtests without signing in. Creating an account persists the agents you register, is required for any backtest driven by an AI -model you pick (together with an AI billing choice, above — the hosted AI Hedge -Fund runtime is the exception), and lets you link Discord. Neither dashboard +model you pick (together with an AI billing choice, above) and for the hosted AI +Hedge Fund runtime, and lets you link Discord. Neither dashboard leaderboard takes user submissions, so your dashboard backtests never appear on one. See :doc:`accounts` to sign up and manage your profile, and :doc:`credits_billing` for the AI billing options. diff --git a/docs/source/lab/key_features.rst b/docs/source/lab/key_features.rst index 20d4e1153..2b51c79ad 100644 --- a/docs/source/lab/key_features.rst +++ b/docs/source/lab/key_features.rst @@ -8,6 +8,6 @@ Key Features - **Performance metrics** — Final portfolio value, cumulative return, max drawdown, Sharpe ratio, and equity-curve comparison views. - **Community templates** — Add ready-made agent templates from the **Community** tab to your own workspace and customize them. - **Two leaderboards** — A Competition board and a Live Trading board rank a curated roster of baselines and models. Neither accepts user or agent submissions. (Runs started through the Agent API v2 are listed separately, on the public ``GET /api/v2/leaderboard`` — see :doc:`agent_api`.) -- **Credits and BYOK** — Backtests driven by an LLM you pick need you to sign in and choose how AI calls are paid for: your own provider key (BYOK) or ATL Credits. The hosted AI Hedge Fund runtime needs neither. Stripe purchases are Test Mode only, with no real money. See :doc:`credits_billing`. +- **Credits and BYOK** — Backtests driven by an LLM you pick need you to sign in and choose how AI calls are paid for: your own provider key (BYOK) or ATL Credits. The hosted AI Hedge Fund runtime also needs you to sign in, and always uses ATL Credits. Stripe purchases are Test Mode only, with no real money. See :doc:`credits_billing`. - **Agent API** — Connect external agents and drive runs step by step through the API (:doc:`agent_api`, :doc:`external_agents`). -- **Optional accounts** — Sign in to persist your agents and link Discord. Signing in is required for backtests driven by an LLM you pick (not for the hosted AI Hedge Fund runtime). +- **Optional accounts** — Sign in to persist your agents and link Discord. Signing in is required for backtests driven by an LLM you pick and for the hosted AI Hedge Fund runtime. diff --git a/docs/source/lab/marketplace.rst b/docs/source/lab/marketplace.rst index bb5a05808..2144273c8 100644 --- a/docs/source/lab/marketplace.rst +++ b/docs/source/lab/marketplace.rst @@ -183,9 +183,10 @@ workbench. See :doc:`research_agents`. Adding a template does not mean you can run it. Backtests driven by a model you pick need you to be signed in and to choose how the AI is paid for — your own key or ATL Credits (see :doc:`credits_billing`). Rule-based - backtests run signed out, and so does the hosted AI Hedge Fund template: it - runs on a model the platform manages, so there is no billing choice, but it - needs a Financial Datasets API key saved on the agent before it will run. + backtests run signed out. The hosted AI Hedge Fund template needs you to be + signed in: it runs on a model the platform manages and is always paid for + with ATL Credits, so there is no billing choice, and it also needs a + Financial Datasets API key saved on the agent before it will run. Neither leaderboard takes submissions, so your own copy is never ranked on either one. diff --git a/docs/source/lab/operating_modes.rst b/docs/source/lab/operating_modes.rst index 4787bc490..9fd142886 100644 --- a/docs/source/lab/operating_modes.rst +++ b/docs/source/lab/operating_modes.rst @@ -2,7 +2,7 @@ Operating Modes =============== **Backtesting** - Evaluate agents on historical market data and compare results to market baselines (buy-and-hold and, for US runs, the Dow index). Rule-based runs work signed out; runs driven by an LLM you pick need you to sign in and choose how the AI calls are paid for (your own API key, or ATL Credits) — see :doc:`credits_billing`. The hosted AI Hedge Fund runtime runs on a platform-managed model and needs neither. + Evaluate agents on historical market data and compare results to market baselines (buy-and-hold and, for US runs, the Dow index). Rule-based runs work signed out; runs driven by an LLM you pick need you to sign in and choose how the AI calls are paid for (your own API key, or ATL Credits) — see :doc:`credits_billing`. The hosted AI Hedge Fund runtime runs on a platform-managed model: it needs you to sign in and is always paid for with ATL Credits. **Paper trading** Not available. The dashboard shows the **Paper Trading** tab greyed out, and the Lab does not run agents against a broker's paper account.