diff --git a/docs/DC90-MANAGED-CAPABILITY-SAFE-FIXTURES-2026-04-27.md b/docs/DC90-MANAGED-CAPABILITY-SAFE-FIXTURES-2026-04-27.md index fcb4fb7d..37156e25 100644 --- a/docs/DC90-MANAGED-CAPABILITY-SAFE-FIXTURES-2026-04-27.md +++ b/docs/DC90-MANAGED-CAPABILITY-SAFE-FIXTURES-2026-04-27.md @@ -271,7 +271,7 @@ These are the next lowest-risk managed rails because they use public/synthetic i 6. Verify cleanup through Rhumb-managed `agent.get_status` until the provider returns `upstream_status=404`, because the local direct E2B key can return an ambiguous 404/no-access response for hosted-managed sandboxes. 7. Stop immediately if create fails or no `sandboxID` is returned; never create a second sandbox in the same run. -- Existing cleanup pattern: `scripts/runtime_review_e2b_depth11_20260403.py` already creates, status-checks, and directly deletes Rhumb/direct E2B sandboxes. The DC90 helper intentionally avoids direct-control duplication and uses a 10-second TTL plus managed cleanup verification for the hosted dogfood path. +- The March–April runtime-review E2B script that created, status-checked, and directly deleted Rhumb/direct E2B sandboxes has been removed. The DC90 helper uses a 10-second TTL plus managed cleanup verification for the hosted dogfood path. - Future code-exec payload, after lifecycle cleanup is proven: ```json diff --git a/scripts/publish_runtime_review_pair.py b/scripts/publish_runtime_review_pair.py deleted file mode 100755 index be39231e..00000000 --- a/scripts/publish_runtime_review_pair.py +++ /dev/null @@ -1,253 +0,0 @@ -#!/usr/bin/env python3 -"""Publish one runtime-backed evidence+review pair to the public trust surface. - -Why it exists: -- Runtime reruns can finish successfully while the public trust surface stays stale. -- Keel / Pedro loops need a repeatable, idempotent way to publish the finished pass. -- The publish path should work from the linked Railway production context without - copy-pasting ad hoc curl blobs. - -Usage example: - railway run -s rhumb-api python3 scripts/publish_runtime_review_pair.py \ - --service replicate \ - --headline "Replicate: current-pass rerun confirms ai.generate_text parity through Rhumb Resolve" \ - --summary "Fresh current-pass runtime rerun passed for Replicate ai.generate_text through Rhumb Resolve. Managed and direct executions both created successful predictions and matched on final succeeded state and exact text output." \ - --evidence-title "Replicate current-pass runtime rerun parity check via Rhumb Resolve" \ - --evidence-summary "Fresh current-pass runtime rerun passed for Replicate ai.generate_text through Rhumb Resolve. Managed and direct executions both created successful predictions and matched on final succeeded state and exact text output." \ - --source-ref runtime-review:replicate:20260329T231345Z \ - --source-batch-id runtime-review:replicate:20260329T231345Z \ - --reviewed-at 2026-03-29T23:13:55.412121752Z \ - --fresh-until 2026-04-28T23:13:55.412121752Z \ - --reviewer-agent-id 06b352f9-05f7-4064-9432-c9c8de858276 \ - --agent-id 06b352f9-05f7-4064-9432-c9c8de858276 \ - --run-id exec_c512c7fc9e4c4d35a5f2a675eb71e97c \ - --tag runtime_review --tag replicate --tag ai.generate_text --tag current_pass --tag phase3 \ - --raw-payload-file artifacts/runtime-review-pass-20260329T231345Z-pdl-replicate-current.json -""" - -from __future__ import annotations - -import argparse -import json -import os -import sys -from typing import Any -from urllib.error import HTTPError -from urllib.parse import quote -from urllib.request import Request, urlopen - -DEFAULT_CREATED_BY = "pedro_runtime_review_loop" -DEFAULT_CONFIDENCE = 0.99 -DEFAULT_REVIEWER_LABEL = "Pedro / Keel runtime review loop" - - -def _require_env(name: str) -> str: - value = os.environ.get(name) - if not value: - raise RuntimeError(f"Missing required environment variable: {name}") - return value - - -def _base_url() -> str: - return _require_env("SUPABASE_URL").rstrip("/") + "/rest/v1" - - -def _headers(*, return_representation: bool = False) -> dict[str, str]: - key = _require_env("SUPABASE_SERVICE_ROLE_KEY") - headers = { - "apikey": key, - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - } - headers["Prefer"] = "return=representation" if return_representation else "return=minimal" - return headers - - -def _request(method: str, path: str, *, payload: Any | None = None, return_representation: bool = False) -> Any: - body = None - if payload is not None: - body = json.dumps(payload).encode("utf-8") - req = Request( - _base_url() + path, - data=body, - headers=_headers(return_representation=return_representation), - method=method, - ) - try: - with urlopen(req, timeout=60) as response: - raw = response.read().decode("utf-8") - if not raw: - return None - return json.loads(raw) - except HTTPError as exc: - detail = exc.read().decode("utf-8", errors="replace") - raise RuntimeError(f"{method} {path} failed with HTTP {exc.code}: {detail}") from exc - - -def _fetch_one(path: str) -> dict[str, Any] | None: - data = _request("GET", path) - if isinstance(data, list): - return data[0] if data else None - if isinstance(data, dict): - return data - return None - - -def _insert_returning(table: str, payload: dict[str, Any]) -> dict[str, Any]: - data = _request("POST", f"/{table}", payload=payload, return_representation=True) - if isinstance(data, list) and data: - return data[0] - if isinstance(data, dict): - return data - raise RuntimeError(f"Insert into {table} returned no row") - - -def _load_optional_json(path: str | None) -> Any: - if not path: - return {} - with open(path, "r", encoding="utf-8") as handle: - return json.load(handle) - - -def _existing_evidence(service: str, source_ref: str) -> dict[str, Any] | None: - return _fetch_one( - "/evidence_records" - f"?service_slug=eq.{quote(service, safe='-_')}" - f"&source_ref=eq.{quote(source_ref, safe='-_:')}" - "&limit=1" - "&select=id,service_slug,source_ref" - ) - - -def _existing_review(service: str, source_batch_id: str) -> dict[str, Any] | None: - return _fetch_one( - "/service_reviews" - f"?service_slug=eq.{quote(service, safe='-_')}" - f"&source_batch_id=eq.{quote(source_batch_id, safe='-_:')}" - "&limit=1" - "&select=id,service_slug,source_batch_id,evidence_refs_json" - ) - - -def _existing_link(review_id: str, evidence_id: str) -> dict[str, Any] | None: - return _fetch_one( - "/review_evidence_links" - f"?review_id=eq.{quote(review_id, safe='-_')}" - f"&evidence_record_id=eq.{quote(evidence_id, safe='-_')}" - "&limit=1" - "&select=review_id,evidence_record_id,role" - ) - - -def parse_args(argv: list[str]) -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--service", required=True) - parser.add_argument("--headline", required=True) - parser.add_argument("--summary", required=True) - parser.add_argument("--evidence-title", required=True) - parser.add_argument("--evidence-summary", required=True) - parser.add_argument("--source-ref", required=True) - parser.add_argument("--source-batch-id", required=True) - parser.add_argument("--reviewed-at", required=True) - parser.add_argument("--fresh-until", required=True) - parser.add_argument("--review-type", default="manual") - parser.add_argument("--review-status", default="published") - parser.add_argument("--reviewer-label", default=DEFAULT_REVIEWER_LABEL) - parser.add_argument("--reviewer-agent-id") - parser.add_argument("--agent-id") - parser.add_argument("--run-id") - parser.add_argument("--confidence", type=float, default=DEFAULT_CONFIDENCE) - parser.add_argument("--source-type", default="runtime_verified") - parser.add_argument("--evidence-kind", default="runtime_review") - parser.add_argument("--created-by", default=DEFAULT_CREATED_BY) - parser.add_argument("--tag", action="append", default=[]) - parser.add_argument("--raw-payload-file") - parser.add_argument("--normalized-payload-file") - parser.add_argument("--link-role", default="primary") - return parser.parse_args(argv) - - -def main(argv: list[str] | None = None) -> int: - args = parse_args(argv or sys.argv[1:]) - - raw_payload = _load_optional_json(args.raw_payload_file) - normalized_payload = _load_optional_json(args.normalized_payload_file) - - evidence = _existing_evidence(args.service, args.source_ref) - evidence_state = "reused" - if evidence is None: - evidence = _insert_returning( - "evidence_records", - { - "service_slug": args.service, - "source_type": args.source_type, - "source_ref": args.source_ref, - "evidence_kind": args.evidence_kind, - "title": args.evidence_title, - "summary": args.evidence_summary, - "raw_payload_json": raw_payload, - "normalized_payload_json": normalized_payload, - "observed_at": args.reviewed_at, - "fresh_until": args.fresh_until, - "confidence": args.confidence, - "agent_id": args.agent_id, - "run_id": args.run_id, - "created_by": args.created_by, - }, - ) - evidence_state = "inserted" - - evidence_id = str(evidence["id"]) - - review = _existing_review(args.service, args.source_batch_id) - review_state = "reused" - if review is None: - review = _insert_returning( - "service_reviews", - { - "service_slug": args.service, - "review_type": args.review_type, - "review_status": args.review_status, - "headline": args.headline, - "summary": args.summary, - "confidence": args.confidence, - "reviewed_at": args.reviewed_at, - "reviewer_agent_id": args.reviewer_agent_id, - "reviewer_label": args.reviewer_label, - "source_batch_id": args.source_batch_id, - "fresh_until": args.fresh_until, - "evidence_count": 1, - "evidence_refs_json": [evidence_id], - "tags_json": args.tag, - "highest_trust_source": args.source_type, - }, - ) - review_state = "inserted" - - review_id = str(review["id"]) - - link = _existing_link(review_id, evidence_id) - link_state = "reused" - if link is None: - link = _insert_returning( - "review_evidence_links", - { - "review_id": review_id, - "evidence_record_id": evidence_id, - "role": args.link_role, - }, - ) - link_state = "inserted" - - result = { - "service": args.service, - "evidence": {"state": evidence_state, "id": evidence_id, "source_ref": args.source_ref}, - "review": {"state": review_state, "id": review_id, "source_batch_id": args.source_batch_id}, - "link": {"state": link_state, "review_id": review_id, "evidence_record_id": evidence_id, "role": args.link_role}, - } - print(json.dumps(result, indent=2, sort_keys=True)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/runtime_review_algolia_depth10_20260402.py b/scripts/runtime_review_algolia_depth10_20260402.py deleted file mode 100644 index f03019e9..00000000 --- a/scripts/runtime_review_algolia_depth10_20260402.py +++ /dev/null @@ -1,326 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -import uuid -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -SERVICE_SLUG = "algolia" -CAPABILITY_ID = "search.autocomplete" -INDEX_NAME = "rhumb_test" -QUERY = "runtime test" -POST_GRANT_PROPAGATION_DELAY_SECONDS = 5 -ESTIMATE_AUTH_RETRY_ATTEMPTS = 4 -ESTIMATE_AUTH_RETRY_DELAY_SECONDS = 5 - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _looks_like_invalid_key(body: Any) -> bool: - if not isinstance(body, dict): - return False - detail = str(body.get("detail") or "").lower() - error = str(body.get("error") or "").lower() - return "invalid or expired rhumb api key" in detail or "invalid or expired rhumb api key" in error - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request(method, url, headers=headers, params=params, json=json_body) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label in {"🟒 Runtime-verified", "πŸ§ͺ Tester-generated"}: - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - short = uuid.uuid4().hex[:8] - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-algolia-depth10.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-02-algolia-depth10.json" - org_id = f"org_runtime_review_algolia_{stamp.lower()}_{short}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "capability_id": CAPABILITY_ID, - "index_name": INDEX_NAME, - "query": QUERY, - "review_kind": "current_depth10", - "organization_id": org_id, - "artifact_path": str(artifact_path.relative_to(REPO_ROOT)), - "started_at": started_at.replace(microsecond=0).isoformat().replace("+00:00", "Z"), - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-algolia-{stamp}", - starter_credits_cents=5000, - ) - payload["wallet_bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-algolia-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Algolia depth-10 runtime pass", - tags=["runtime_review", "phase3", "current_pass", SERVICE_SLUG, "depth10"], - ) - payload["review_agent"] = {"agent_id": agent_id, "organization_id": org_id} - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = {"status": "success", "access_id": access_id} - payload["post_grant_delay_seconds"] = POST_GRANT_PROPAGATION_DELAY_SECONDS - _json_dump(artifact_path, payload) - - await asyncio.sleep(POST_GRANT_PROPAGATION_DELAY_SECONDS) - - api_headers = {"X-Rhumb-Key": api_key} - algolia_app_id = os.environ["RHUMB_CREDENTIAL_ALGOLIA_APP_ID"] - algolia_api_key = os.environ["RHUMB_CREDENTIAL_ALGOLIA_API_KEY"] - direct_headers = { - "X-Algolia-Application-Id": algolia_app_id, - "X-Algolia-API-Key": algolia_api_key, - "Content-Type": "application/json", - } - direct_url = f"https://{algolia_app_id}-dsn.algolia.net/1/indexes/{INDEX_NAME}/query" - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - - estimate_attempts = 0 - estimate = HttpResult(status_code=0, body=None) - while estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS: - estimate_attempts += 1 - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - if not ( - estimate.status_code == 401 - and _looks_like_invalid_key(estimate.body) - and estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS - ): - break - await asyncio.sleep(ESTIMATE_AUTH_RETRY_DELAY_SECONDS) - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body={"indexName": INDEX_NAME, "query": QUERY}, - timeout=120.0, - ) - direct = await _request_json( - "POST", - direct_url, - headers=direct_headers, - json_body={"query": QUERY}, - timeout=120.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - rhumb_hits = rhumb_upstream.get("hits") if isinstance(rhumb_upstream, dict) else [] - direct_hits = direct.body.get("hits") if isinstance(direct.body, dict) else [] - rhumb_top = rhumb_hits[0] if isinstance(rhumb_hits, list) and rhumb_hits else None - direct_top = direct_hits[0] if isinstance(direct_hits, list) and direct_hits else None - observed_at = _iso_now() - fresh_until = ( - datetime.fromisoformat(observed_at.replace("Z", "+00:00")) + timedelta(days=30) - ).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "provider_used": rhumb_payload.get("provider_used"), - "upstream_status": rhumb_payload.get("upstream_status"), - "execution_id": rhumb_payload.get("execution_id"), - "rhumb_nb_hits": rhumb_upstream.get("nbHits") if isinstance(rhumb_upstream, dict) else None, - "direct_nb_hits": direct.body.get("nbHits") if isinstance(direct.body, dict) else None, - "rhumb_query": rhumb_upstream.get("query") if isinstance(rhumb_upstream, dict) else None, - "direct_query": direct.body.get("query") if isinstance(direct.body, dict) else None, - "rhumb_top": { - "objectID": (rhumb_top or {}).get("objectID"), - "name": (rhumb_top or {}).get("name"), - }, - "direct_top": { - "objectID": (direct_top or {}).get("objectID"), - "name": (direct_top or {}).get("name"), - }, - "nb_hits_match": (rhumb_upstream.get("nbHits") if isinstance(rhumb_upstream, dict) else None) == (direct.body.get("nbHits") if isinstance(direct.body, dict) else None), - "query_match": (rhumb_upstream.get("query") if isinstance(rhumb_upstream, dict) else None) == (direct.body.get("query") if isinstance(direct.body, dict) else None), - "top_object_id_match": (rhumb_top or {}).get("objectID") == (direct_top or {}).get("objectID"), - "top_name_match": (rhumb_top or {}).get("name") == (direct_top or {}).get("name"), - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["nb_hits_match"] - and comparison["query_match"] - and comparison["top_object_id_match"] - and comparison["top_name_match"] - ) else "fail" - - payload.update( - { - "reviewed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": { - "status_code": estimate.status_code, - "body": estimate.body, - "attempts": estimate_attempts, - }, - "rhumb_execute": asdict(execute), - "direct": asdict(direct), - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Algolia depth-10 runtime pass failed; inspect artifact for parity details") - - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "Algolia: depth-10 runtime review confirms search.autocomplete parity through Rhumb Resolve", - "--summary", "Fresh depth-10 runtime review passed for Algolia search.autocomplete through Rhumb Resolve. Managed and direct executions matched on nbHits, echoed query, and the top hit objectID and name for the same live query against the same index.", - "--evidence-title", "Algolia depth-10 runtime review parity check via Rhumb Resolve", - "--evidence-summary", "Fresh depth-10 runtime review passed for Algolia search.autocomplete through Rhumb Resolve. Managed and direct executions matched on nbHits, echoed query, and the top hit objectID and name for the same live query against the same index.", - "--source-ref", f"runtime-review:algolia:{stamp}", - "--source-batch-id", f"runtime-review:algolia:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", rhumb_payload.get("execution_id") or "", - "--tag", "runtime_review", - "--tag", SERVICE_SLUG, - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--tag", "depth10", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["publication"] = publication - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["disable_agent"] = { - "status_code": 200 if disabled else 404, - "body": {"status": "success" if disabled else "not_found", "agent_id": agent_id}, - } - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_algolia_depth5_20260330.py b/scripts/runtime_review_algolia_depth5_20260330.py deleted file mode 100644 index 7f708e5a..00000000 --- a/scripts/runtime_review_algolia_depth5_20260330.py +++ /dev/null @@ -1,289 +0,0 @@ -#!/usr/bin/env python3 -"""Algolia current-depth5 runtime review pass β€” 2026-03-30.""" -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -SERVICE_SLUG = "algolia" -CAPABILITY_ID = "search.autocomplete" -INDEX_NAME = "rhumb_test" -QUERY = "rhumb" - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request(method, url, headers=headers, params=params, json=json_body) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-algolia-current-depth5.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-30-algolia-depth5.json" - org_id = f"org_runtime_review_algolia_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "capability_id": CAPABILITY_ID, - "index_name": INDEX_NAME, - "query": QUERY, - "review_kind": "current_depth5", - "organization_id": org_id, - "artifact_path": str(artifact_path), - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-algolia-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-algolia-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Algolia depth-5 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "algolia"], - ) - payload["review_agent"] = {"agent_id": agent_id, "organization_id": org_id} - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = {"status": "success", "access_id": access_id} - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - algolia_app_id = os.environ["RHUMB_CREDENTIAL_ALGOLIA_APP_ID"] - algolia_api_key = os.environ["RHUMB_CREDENTIAL_ALGOLIA_API_KEY"] - direct_headers = { - "X-Algolia-Application-Id": algolia_app_id, - "X-Algolia-API-Key": algolia_api_key, - "Content-Type": "application/json", - } - direct_url = f"https://{algolia_app_id}-dsn.algolia.net/1/indexes/{INDEX_NAME}/query" - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body={"indexName": INDEX_NAME, "query": QUERY}, - timeout=120.0, - ) - direct = await _request_json( - "POST", - direct_url, - headers=direct_headers, - json_body={"query": QUERY}, - timeout=120.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - rhumb_hits = rhumb_upstream.get("hits") if isinstance(rhumb_upstream, dict) else [] - direct_hits = direct.body.get("hits") if isinstance(direct.body, dict) else [] - rhumb_top = rhumb_hits[0] if isinstance(rhumb_hits, list) and rhumb_hits else None - direct_top = direct_hits[0] if isinstance(direct_hits, list) and direct_hits else None - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_provider_used": rhumb_payload.get("provider_used"), - "rhumb_upstream_status": rhumb_payload.get("upstream_status"), - "rhumb_nb_hits": rhumb_upstream.get("nbHits") if isinstance(rhumb_upstream, dict) else None, - "direct_nb_hits": direct.body.get("nbHits") if isinstance(direct.body, dict) else None, - "rhumb_top": { - "objectID": (rhumb_top or {}).get("objectID"), - "name": (rhumb_top or {}).get("name"), - }, - "direct_top": { - "objectID": (direct_top or {}).get("objectID"), - "name": (direct_top or {}).get("name"), - }, - "nb_hits_match": (rhumb_upstream.get("nbHits") if isinstance(rhumb_upstream, dict) else None) == (direct.body.get("nbHits") if isinstance(direct.body, dict) else None), - "top_object_id_match": (rhumb_top or {}).get("objectID") == (direct_top or {}).get("objectID"), - "top_name_match": (rhumb_top or {}).get("name") == (direct_top or {}).get("name"), - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["nb_hits_match"] - and comparison["top_object_id_match"] - and comparison["top_name_match"] - ) else "fail" - - payload.update({ - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": {"status": estimate.status_code, "data": estimate.body}, - "rhumb": {"status": execute.status_code, "data": execute.body}, - "direct": {"status": direct.status_code, "data": direct.body}, - "comparison": comparison, - "verdict": verdict, - }) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Algolia depth-5 runtime pass failed; inspect artifact for parity details") - - execution_id = rhumb_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "Algolia: current-depth rerun confirms search.autocomplete parity through Rhumb Resolve again", - "--summary", "Fresh current-depth runtime rerun passed for Algolia search.autocomplete through Rhumb Resolve. Managed and direct executions matched on nbHits plus the top hit objectID and name for the same live query against the same index, lifting Algolia another layer above the callable review floor.", - "--evidence-title", "Algolia current-depth runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh current-depth runtime rerun passed for Algolia search.autocomplete through Rhumb Resolve. Managed and direct executions matched on nbHits plus the top hit objectID and name for the same live query against the same index.", - "--source-ref", f"runtime-review:algolia:{stamp}", - "--source-batch-id", f"runtime-review:algolia:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "algolia", - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "service_slug": SERVICE_SLUG, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_apify_depth10_20260402.py b/scripts/runtime_review_apify_depth10_20260402.py deleted file mode 100644 index 92a14b4a..00000000 --- a/scripts/runtime_review_apify_depth10_20260402.py +++ /dev/null @@ -1,338 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import datetime, timedelta, timezone -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -SERVICE_SLUG = "apify" -CAPABILITY_ID = "scrape.extract" -APIFY_RUN_URL = "https://api.apify.com/v2/acts/apify~website-content-crawler/runs" -TARGET_URL = "https://example.com" -CRAWL_INPUT = { - "startUrls": [{"url": TARGET_URL}], - "maxCrawlDepth": 0, - "maxCrawlPages": 1, -} - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -async def _fetch_apify_dataset_item(api_token: str, dataset_id: str) -> HttpResult: - return await _request_json( - "GET", - f"https://api.apify.com/v2/datasets/{dataset_id}/items", - headers={"Authorization": f"Bearer {api_token}"}, - params={"clean": "true", "limit": 1}, - timeout=120.0, - ) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label in {"🟒 Runtime-verified", "πŸ§ͺ Tester-generated"}: - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [ - str(REPO_ROOT / "packages/api/.venv/bin/python"), - str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), - *args, - ], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - "publish_runtime_review_pair.py failed " - f"({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -async def main() -> None: - started_at = datetime.now(tz=timezone.utc) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-apify-depth10.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-02-apify-depth10.json" - org_id = f"org_runtime_review_apify_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "capability_id": CAPABILITY_ID, - "review_kind": "current_depth10", - "organization_id": org_id, - "artifact_path": str(artifact_path.relative_to(REPO_ROOT)), - "target_url": TARGET_URL, - "crawl_input": CRAWL_INPUT, - "started_at": started_at.replace(microsecond=0).isoformat().replace("+00:00", "Z"), - } - _json_dump(artifact_path, payload) - - apify_token = os.environ["RHUMB_CREDENTIAL_APIFY_API_TOKEN"] - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-apify-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-apify-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Apify depth-10 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "apify", "depth10"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = { - "status": "success", - "access_id": access_id, - } - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body=CRAWL_INPUT, - timeout=180.0, - ) - direct = await _request_json( - "POST", - APIFY_RUN_URL, - headers={"Authorization": f"Bearer {apify_token}"}, - params={"waitForFinish": 60}, - json_body=CRAWL_INPUT, - timeout=180.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_run = (rhumb_payload.get("upstream_response") or {}).get("data") if isinstance(rhumb_payload, dict) else {} - direct_run = ((direct.body or {}).get("data")) if isinstance(direct.body, dict) else {} - - rhumb_dataset_id = (rhumb_run or {}).get("defaultDatasetId") - direct_dataset_id = (direct_run or {}).get("defaultDatasetId") - - rhumb_dataset = await _fetch_apify_dataset_item(apify_token, rhumb_dataset_id) if rhumb_dataset_id else HttpResult(status_code=0, body=None) - direct_dataset = await _fetch_apify_dataset_item(apify_token, direct_dataset_id) if direct_dataset_id else HttpResult(status_code=0, body=None) - - rhumb_sample = rhumb_dataset.body[0] if rhumb_dataset.status_code == 200 and isinstance(rhumb_dataset.body, list) and rhumb_dataset.body else None - direct_sample = direct_dataset.body[0] if direct_dataset.status_code == 200 and isinstance(direct_dataset.body, list) and direct_dataset.body else None - - sampled_fields = { - "url": ((rhumb_sample or {}).get("url"), (direct_sample or {}).get("url")), - "metadata.title": (((rhumb_sample or {}).get("metadata") or {}).get("title"), ((direct_sample or {}).get("metadata") or {}).get("title")), - "markdown": ((rhumb_sample or {}).get("markdown"), (direct_sample or {}).get("markdown")), - } - checks = {field: left == right for field, (left, right) in sampled_fields.items()} - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_provider_used": rhumb_payload.get("provider_used"), - "rhumb_upstream_status": rhumb_payload.get("upstream_status"), - "rhumb_execution_id": rhumb_payload.get("execution_id"), - "rhumb_run_id": (rhumb_run or {}).get("id"), - "direct_run_id": (direct_run or {}).get("id"), - "rhumb_dataset_id": rhumb_dataset_id, - "direct_dataset_id": direct_dataset_id, - "sampled_fields": list(sampled_fields.keys()), - "checks": checks, - "all_matched": all(checks.values()), - "rhumb_sample": { - "url": (rhumb_sample or {}).get("url"), - "metadata": {"title": ((rhumb_sample or {}).get("metadata") or {}).get("title")}, - "markdown": (rhumb_sample or {}).get("markdown"), - }, - "direct_sample": { - "url": (direct_sample or {}).get("url"), - "metadata": {"title": ((direct_sample or {}).get("metadata") or {}).get("title")}, - "markdown": (direct_sample or {}).get("markdown"), - }, - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 201 - and rhumb_dataset.status_code == 200 - and direct_dataset.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 201 - and comparison["all_matched"] - ) else "fail" - - payload.update( - { - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": {"status": estimate.status_code, "data": estimate.body}, - "rhumb": {"status": execute.status_code, "data": execute.body}, - "direct": {"status": direct.status_code, "data": direct.body}, - "rhumb_dataset": {"status": rhumb_dataset.status_code, "data": rhumb_dataset.body}, - "direct_dataset": {"status": direct_dataset.status_code, "data": direct_dataset.body}, - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Apify depth-10 runtime pass failed; inspect artifact for parity details") - - execution_id = rhumb_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "Apify: depth-10 runtime review confirms scrape.extract parity through Rhumb Resolve", - "--summary", "Fresh depth-10 runtime review passed for Apify scrape.extract through Rhumb Resolve. Managed and direct executions matched exactly on URL, metadata.title, and markdown output for the same one-page website-content-crawler target.", - "--evidence-title", "Apify depth-10 runtime review parity check via Rhumb Resolve", - "--evidence-summary", "Fresh depth-10 runtime review passed for Apify scrape.extract through Rhumb Resolve. Managed and direct executions matched exactly on URL, metadata.title, and markdown output for the same one-page website-content-crawler target.", - "--source-ref", f"runtime-review:apify:{stamp}", - "--source-batch-id", f"runtime-review:apify:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "apify", - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--tag", "depth10", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "service_slug": SERVICE_SLUG, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_apify_depth4_20260330.py b/scripts/runtime_review_apify_depth4_20260330.py deleted file mode 100644 index 1c9de119..00000000 --- a/scripts/runtime_review_apify_depth4_20260330.py +++ /dev/null @@ -1,331 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -import uuid -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -APIFY_RUN_URL = "https://api.apify.com/v2/acts/apify~website-content-crawler/runs" -TARGET_URL = "https://example.com" -CRAWL_INPUT = { - "startUrls": [{"url": TARGET_URL}], - "maxCrawlDepth": 0, - "maxCrawlPages": 1, -} - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -async def _fetch_apify_dataset_item(api_token: str, dataset_id: str) -> HttpResult: - return await _request_json( - "GET", - f"https://api.apify.com/v2/datasets/{dataset_id}/items", - headers={"Authorization": f"Bearer {api_token}"}, - params={"clean": "true", "limit": 1}, - timeout=120.0, - ) - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - short = uuid.uuid4().hex[:8] - run_id = f"runtime-review-{stamp}-{short}" - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-apify-current-depth4.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-30-apify-depth4.json" - org_id = f"org_runtime_review_apify_{stamp.lower()}_{short}" - - payload: dict[str, Any] = { - "run_started_at": started_at.isoformat(), - "run_id": run_id, - "organization_id": org_id, - "target_url": TARGET_URL, - "crawl_input": CRAWL_INPUT, - } - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-apify-{stamp}", - starter_credits_cents=5000, - ) - payload["wallet_bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-apify-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Apify depth-4 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "apify"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, "apify") - payload["grant_result"] = { - "service": "apify", - "status_code": 200, - "body": {"status": "success", "access_id": access_id}, - } - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - apify_token = os.environ["RHUMB_CREDENTIAL_APIFY_API_TOKEN"] - - try: - counts_before = await _fetch_public_review_stats("apify") - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/scrape.extract/execute/estimate", - headers=api_headers, - params={"provider": "apify", "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/scrape.extract/execute", - headers=api_headers, - params={"provider": "apify", "credential_mode": "rhumb_managed"}, - json_body=CRAWL_INPUT, - timeout=180.0, - ) - direct = await _request_json( - "POST", - APIFY_RUN_URL, - headers={"Authorization": f"Bearer {apify_token}"}, - params={"waitForFinish": 60}, - json_body=CRAWL_INPUT, - timeout=180.0, - ) - - rhumb_run = (((execute.body or {}).get("data") or {}).get("upstream_response") or {}).get("data") if isinstance(execute.body, dict) else None - direct_run = ((direct.body or {}).get("data")) if isinstance(direct.body, dict) else None - rhumb_dataset_id = (rhumb_run or {}).get("defaultDatasetId") - direct_dataset_id = (direct_run or {}).get("defaultDatasetId") - - rhumb_dataset = await _fetch_apify_dataset_item(apify_token, rhumb_dataset_id) if rhumb_dataset_id else HttpResult(status_code=0, body=None) - direct_dataset = await _fetch_apify_dataset_item(apify_token, direct_dataset_id) if direct_dataset_id else HttpResult(status_code=0, body=None) - - rhumb_sample = rhumb_dataset.body[0] if rhumb_dataset.status_code == 200 and isinstance(rhumb_dataset.body, list) and rhumb_dataset.body else None - direct_sample = direct_dataset.body[0] if direct_dataset.status_code == 200 and isinstance(direct_dataset.body, list) and direct_dataset.body else None - - sampled_fields = { - "url": ((rhumb_sample or {}).get("url"), (direct_sample or {}).get("url")), - "metadata.title": (((rhumb_sample or {}).get("metadata") or {}).get("title"), ((direct_sample or {}).get("metadata") or {}).get("title")), - "markdown": ((rhumb_sample or {}).get("markdown"), (direct_sample or {}).get("markdown")), - } - checks = {field: left == right for field, (left, right) in sampled_fields.items()} - observed_at = _iso_now() - execution_id = (((execute.body or {}).get("data") or {}).get("execution_id")) if isinstance(execute.body, dict) else None - upstream_status = (((execute.body or {}).get("data") or {}).get("upstream_status")) if isinstance(execute.body, dict) else None - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 201 - and rhumb_dataset.status_code == 200 - and direct_dataset.status_code == 200 - and all(checks.values()) - ) else "fail" - - payload["apify_depth4_pass"] = { - "service_slug": "apify", - "provider": "apify", - "capability_id": "scrape.extract", - "counts_before": counts_before, - "estimate": asdict(estimate), - "rhumb_execute": asdict(execute), - "direct_control": asdict(direct), - "rhumb_dataset": asdict(rhumb_dataset), - "direct_dataset": asdict(direct_dataset), - "rhumb_extracted": { - "run_id": (rhumb_run or {}).get("id"), - "dataset_id": rhumb_dataset_id, - "sample": rhumb_sample, - }, - "direct_extracted": { - "run_id": (direct_run or {}).get("id"), - "dataset_id": direct_dataset_id, - "sample": direct_sample, - }, - "parity": { - "sampled_fields": list(sampled_fields.keys()), - "rhumb_sample": { - "url": (rhumb_sample or {}).get("url"), - "metadata": {"title": ((rhumb_sample or {}).get("metadata") or {}).get("title")}, - "markdown": (rhumb_sample or {}).get("markdown"), - }, - "direct_sample": { - "url": (direct_sample or {}).get("url"), - "metadata": {"title": ((direct_sample or {}).get("metadata") or {}).get("title")}, - "markdown": (direct_sample or {}).get("markdown"), - }, - "checks": checks, - "all_matched": all(checks.values()), - "execution_id": execution_id, - "provider_used": (((execute.body or {}).get("data") or {}).get("provider_used")) if isinstance(execute.body, dict) else None, - "upstream_status": upstream_status, - }, - "observed_at": observed_at, - "verdict": verdict, - } - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Apify depth-4 runtime pass failed; inspect artifact for execute vs direct-control details") - - reviewed_at = observed_at - fresh_until = ( - datetime.fromisoformat(reviewed_at.replace("Z", "+00:00")) + timedelta(days=30) - ).replace(microsecond=0).isoformat().replace("+00:00", "Z") - publication = _run_publish( - [ - "--service", "apify", - "--headline", "Apify: current-depth rerun confirms scrape.extract parity through Rhumb Resolve", - "--summary", "Fresh current-depth runtime rerun passed for Apify scrape.extract through Rhumb Resolve. Managed and direct executions matched exactly on URL, metadata.title, and markdown output for the same one-page website-content-crawler target.", - "--evidence-title", "Apify current-depth runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh current-depth runtime rerun passed for Apify scrape.extract through Rhumb Resolve. Managed and direct executions matched exactly on URL, metadata.title, and markdown output for the same one-page website-content-crawler target.", - "--source-ref", f"runtime-review:apify:{stamp}", - "--source-batch-id", f"runtime-review:apify:{stamp}", - "--reviewed-at", reviewed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id or "", - "--tag", "runtime_review", - "--tag", "apify", - "--tag", "scrape.extract", - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["apify_depth4_pass"]["evidence_id"] = ((publication.get("evidence") or {}).get("id")) - payload["apify_depth4_pass"]["review_id"] = ((publication.get("review") or {}).get("id")) - payload["apify_depth4_pass"]["counts_after"] = await _fetch_public_review_stats("apify") - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["disable_agent"] = { - "status_code": 200 if disabled else 404, - "body": {"status": "success" if disabled else "not_found", "agent_id": agent_id}, - } - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_apify_depth5_20260330.py b/scripts/runtime_review_apify_depth5_20260330.py deleted file mode 100644 index 42812e38..00000000 --- a/scripts/runtime_review_apify_depth5_20260330.py +++ /dev/null @@ -1,331 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -import uuid -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -APIFY_RUN_URL = "https://api.apify.com/v2/acts/apify~website-content-crawler/runs" -TARGET_URL = "https://example.com" -CRAWL_INPUT = { - "startUrls": [{"url": TARGET_URL}], - "maxCrawlDepth": 0, - "maxCrawlPages": 1, -} - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -async def _fetch_apify_dataset_item(api_token: str, dataset_id: str) -> HttpResult: - return await _request_json( - "GET", - f"https://api.apify.com/v2/datasets/{dataset_id}/items", - headers={"Authorization": f"Bearer {api_token}"}, - params={"clean": "true", "limit": 1}, - timeout=120.0, - ) - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - short = uuid.uuid4().hex[:8] - run_id = f"runtime-review-{stamp}-{short}" - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-apify-current-depth5.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-30-apify-depth5.json" - org_id = f"org_runtime_review_apify_{stamp.lower()}_{short}" - - payload: dict[str, Any] = { - "run_started_at": started_at.isoformat(), - "run_id": run_id, - "organization_id": org_id, - "target_url": TARGET_URL, - "crawl_input": CRAWL_INPUT, - } - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-apify-{stamp}", - starter_credits_cents=5000, - ) - payload["wallet_bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-apify-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Apify depth-5 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "apify"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, "apify") - payload["grant_result"] = { - "service": "apify", - "status_code": 200, - "body": {"status": "success", "access_id": access_id}, - } - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - apify_token = os.environ["RHUMB_CREDENTIAL_APIFY_API_TOKEN"] - - try: - counts_before = await _fetch_public_review_stats("apify") - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/scrape.extract/execute/estimate", - headers=api_headers, - params={"provider": "apify", "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/scrape.extract/execute", - headers=api_headers, - params={"provider": "apify", "credential_mode": "rhumb_managed"}, - json_body=CRAWL_INPUT, - timeout=180.0, - ) - direct = await _request_json( - "POST", - APIFY_RUN_URL, - headers={"Authorization": f"Bearer {apify_token}"}, - params={"waitForFinish": 60}, - json_body=CRAWL_INPUT, - timeout=180.0, - ) - - rhumb_run = (((execute.body or {}).get("data") or {}).get("upstream_response") or {}).get("data") if isinstance(execute.body, dict) else None - direct_run = ((direct.body or {}).get("data")) if isinstance(direct.body, dict) else None - rhumb_dataset_id = (rhumb_run or {}).get("defaultDatasetId") - direct_dataset_id = (direct_run or {}).get("defaultDatasetId") - - rhumb_dataset = await _fetch_apify_dataset_item(apify_token, rhumb_dataset_id) if rhumb_dataset_id else HttpResult(status_code=0, body=None) - direct_dataset = await _fetch_apify_dataset_item(apify_token, direct_dataset_id) if direct_dataset_id else HttpResult(status_code=0, body=None) - - rhumb_sample = rhumb_dataset.body[0] if rhumb_dataset.status_code == 200 and isinstance(rhumb_dataset.body, list) and rhumb_dataset.body else None - direct_sample = direct_dataset.body[0] if direct_dataset.status_code == 200 and isinstance(direct_dataset.body, list) and direct_dataset.body else None - - sampled_fields = { - "url": ((rhumb_sample or {}).get("url"), (direct_sample or {}).get("url")), - "metadata.title": (((rhumb_sample or {}).get("metadata") or {}).get("title"), ((direct_sample or {}).get("metadata") or {}).get("title")), - "markdown": ((rhumb_sample or {}).get("markdown"), (direct_sample or {}).get("markdown")), - } - checks = {field: left == right for field, (left, right) in sampled_fields.items()} - observed_at = _iso_now() - execution_id = (((execute.body or {}).get("data") or {}).get("execution_id")) if isinstance(execute.body, dict) else None - upstream_status = (((execute.body or {}).get("data") or {}).get("upstream_status")) if isinstance(execute.body, dict) else None - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 201 - and rhumb_dataset.status_code == 200 - and direct_dataset.status_code == 200 - and all(checks.values()) - ) else "fail" - - payload["apify_depth5_pass"] = { - "service_slug": "apify", - "provider": "apify", - "capability_id": "scrape.extract", - "counts_before": counts_before, - "estimate": asdict(estimate), - "rhumb_execute": asdict(execute), - "direct_control": asdict(direct), - "rhumb_dataset": asdict(rhumb_dataset), - "direct_dataset": asdict(direct_dataset), - "rhumb_extracted": { - "run_id": (rhumb_run or {}).get("id"), - "dataset_id": rhumb_dataset_id, - "sample": rhumb_sample, - }, - "direct_extracted": { - "run_id": (direct_run or {}).get("id"), - "dataset_id": direct_dataset_id, - "sample": direct_sample, - }, - "parity": { - "sampled_fields": list(sampled_fields.keys()), - "rhumb_sample": { - "url": (rhumb_sample or {}).get("url"), - "metadata": {"title": ((rhumb_sample or {}).get("metadata") or {}).get("title")}, - "markdown": (rhumb_sample or {}).get("markdown"), - }, - "direct_sample": { - "url": (direct_sample or {}).get("url"), - "metadata": {"title": ((direct_sample or {}).get("metadata") or {}).get("title")}, - "markdown": (direct_sample or {}).get("markdown"), - }, - "checks": checks, - "all_matched": all(checks.values()), - "execution_id": execution_id, - "provider_used": (((execute.body or {}).get("data") or {}).get("provider_used")) if isinstance(execute.body, dict) else None, - "upstream_status": upstream_status, - }, - "observed_at": observed_at, - "verdict": verdict, - } - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Apify depth-5 runtime pass failed; inspect artifact for execute vs direct-control details") - - reviewed_at = observed_at - fresh_until = ( - datetime.fromisoformat(reviewed_at.replace("Z", "+00:00")) + timedelta(days=30) - ).replace(microsecond=0).isoformat().replace("+00:00", "Z") - publication = _run_publish( - [ - "--service", "apify", - "--headline", "Apify: current-depth rerun confirms scrape.extract parity through Rhumb Resolve again", - "--summary", "Fresh current-depth runtime rerun passed for Apify scrape.extract through Rhumb Resolve. Managed and direct executions matched exactly on URL, metadata.title, and markdown output for the same one-page website-content-crawler target.", - "--evidence-title", "Apify current-depth runtime rerun parity check via Rhumb Resolve again", - "--evidence-summary", "Fresh current-depth runtime rerun passed for Apify scrape.extract through Rhumb Resolve. Managed and direct executions matched exactly on URL, metadata.title, and markdown output for the same one-page website-content-crawler target.", - "--source-ref", f"runtime-review:apify:{stamp}", - "--source-batch-id", f"runtime-review:apify:{stamp}", - "--reviewed-at", reviewed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id or "", - "--tag", "runtime_review", - "--tag", "apify", - "--tag", "scrape.extract", - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["apify_depth5_pass"]["evidence_id"] = ((publication.get("evidence") or {}).get("id")) - payload["apify_depth5_pass"]["review_id"] = ((publication.get("review") or {}).get("id")) - payload["apify_depth5_pass"]["counts_after"] = await _fetch_public_review_stats("apify") - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["disable_agent"] = { - "status_code": 200 if disabled else 404, - "body": {"status": "success" if disabled else "not_found", "agent_id": agent_id}, - } - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_apollo_depth11_20260403.py b/scripts/runtime_review_apollo_depth11_20260403.py deleted file mode 100644 index b0fc8d57..00000000 --- a/scripts/runtime_review_apollo_depth11_20260403.py +++ /dev/null @@ -1,332 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -import uuid -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -APOLLO_URL = "https://api.apollo.io/api/v1/people/match" -SERVICE_SLUG = "apollo" -CAPABILITY_ID = "data.enrich_person" -REQUEST_BODY = {"email": "tim@apple.com"} -POST_GRANT_PROPAGATION_DELAY_SECONDS = 5 -ESTIMATE_AUTH_RETRY_ATTEMPTS = 4 -ESTIMATE_AUTH_RETRY_DELAY_SECONDS = 5 - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _looks_like_invalid_key(body: Any) -> bool: - if not isinstance(body, dict): - return False - detail = str(body.get("detail") or "").lower() - error = str(body.get("error") or "").lower() - return "invalid or expired rhumb api key" in detail or "invalid or expired rhumb api key" in error - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request(method, url, headers=headers, params=params, json=json_body) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label in {"🟒 Runtime-verified", "πŸ§ͺ Tester-generated"}: - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _extract_person_fields(person: Any) -> dict[str, Any]: - person_data = person if isinstance(person, dict) else {} - organization = person_data.get("organization") - organization_data = organization if isinstance(organization, dict) else {} - return { - "name": person_data.get("name"), - "title": person_data.get("title"), - "organization_name": organization_data.get("name"), - "linkedin_url": person_data.get("linkedin_url"), - "email_status": person_data.get("email_status"), - } - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - short = uuid.uuid4().hex[:8] - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-apollo-depth11.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-03-apollo-depth11.json" - org_id = f"org_runtime_review_apollo_{stamp.lower()}_{short}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "capability_id": CAPABILITY_ID, - "request_body": REQUEST_BODY, - "review_kind": "current_depth11", - "organization_id": org_id, - "artifact_path": str(artifact_path.relative_to(REPO_ROOT)), - "started_at": started_at.replace(microsecond=0).isoformat().replace("+00:00", "Z"), - "post_grant_delay_seconds": POST_GRANT_PROPAGATION_DELAY_SECONDS, - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-apollo-{stamp}", - starter_credits_cents=5000, - ) - payload["wallet_bootstrap"] = bootstrap - _json_dump(artifact_path, payload) - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-apollo-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Apollo depth-11 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "apollo", "depth11"], - ) - payload["review_agent"] = {"agent_id": agent_id, "organization_id": org_id} - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = {"status": "success", "access_id": access_id} - _json_dump(artifact_path, payload) - - await asyncio.sleep(POST_GRANT_PROPAGATION_DELAY_SECONDS) - - api_headers = {"X-Rhumb-Key": api_key} - direct_headers = { - "X-Api-Key": os.environ["RHUMB_CREDENTIAL_APOLLO_API_KEY"], - "Content-Type": "application/json", - } - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - - estimate_attempts = 0 - estimate = HttpResult(status_code=0, body=None) - while estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS: - estimate_attempts += 1 - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - if not ( - estimate.status_code == 401 - and _looks_like_invalid_key(estimate.body) - and estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS - ): - break - await asyncio.sleep(ESTIMATE_AUTH_RETRY_DELAY_SECONDS) - - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body=REQUEST_BODY, - timeout=120.0, - ) - direct = await _request_json( - "POST", - APOLLO_URL, - headers=direct_headers, - json_body=REQUEST_BODY, - timeout=120.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - rhumb_person = rhumb_upstream.get("person") if isinstance(rhumb_upstream, dict) else None - direct_person = direct.body.get("person") if isinstance(direct.body, dict) else None - managed_fields = _extract_person_fields(rhumb_person) - direct_fields = _extract_person_fields(direct_person) - field_matches = { - field_name: managed_fields[field_name] == direct_fields[field_name] - for field_name in managed_fields - } - - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - field_name: { - "managed": managed_fields[field_name], - "direct": direct_fields[field_name], - "match": field_matches[field_name], - } - for field_name in managed_fields - } - comparison.update( - { - "all_fields_match": all(field_matches.values()), - "provider_used": rhumb_payload.get("provider_used"), - "upstream_status": rhumb_payload.get("upstream_status"), - "execution_id": rhumb_payload.get("execution_id"), - } - ) - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["all_fields_match"] - ) else "fail" - - payload.update( - { - "reviewed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": { - "status_code": estimate.status_code, - "body": estimate.body, - "attempts": estimate_attempts, - }, - "rhumb_execute": asdict(execute), - "direct_control": asdict(direct), - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Apollo depth-11 runtime pass failed; inspect artifact for parity details") - - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "Apollo: depth-11 runtime review confirms data.enrich_person parity through Rhumb Resolve", - "--summary", "Fresh depth-11 runtime review passed for Apollo data.enrich_person through Rhumb Resolve. Managed and direct executions matched on name, title, organization name, LinkedIn URL, and email status for the same live email input.", - "--evidence-title", "Apollo depth-11 runtime review parity check via Rhumb Resolve", - "--evidence-summary", "Fresh depth-11 runtime review passed for Apollo data.enrich_person through Rhumb Resolve. Managed and direct executions matched on name, title, organization name, LinkedIn URL, and email status for the same live email input.", - "--source-ref", f"runtime-review:apollo:{stamp}", - "--source-batch-id", f"runtime-review:apollo:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", rhumb_payload.get("execution_id") or "", - "--tag", "runtime_review", - "--tag", "apollo", - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--tag", "depth11", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["publication"] = publication - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["disable_agent"] = { - "status_code": 200 if disabled else 404, - "body": {"status": "success" if disabled else "not_found", "agent_id": agent_id}, - } - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_apollo_depth5_20260330.py b/scripts/runtime_review_apollo_depth5_20260330.py deleted file mode 100644 index d4168ab4..00000000 --- a/scripts/runtime_review_apollo_depth5_20260330.py +++ /dev/null @@ -1,342 +0,0 @@ -#!/usr/bin/env python3 -"""Apollo current-depth5 runtime review pass - 2026-03-30.""" -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -APOLLO_URL = "https://api.apollo.io/api/v1/people/match" -SERVICE_SLUG = "apollo" -CAPABILITY_ID = "data.enrich_person" -REQUEST_BODY = {"email": "tim@apple.com"} - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json( - "GET", - f"{BASE_URL}/services/{service_slug}/reviews", - timeout=60.0, - ) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [ - str(REPO_ROOT / "packages/api/.venv/bin/python"), - str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), - *args, - ], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - "publish_runtime_review_pair.py failed " - f"({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _extract_person_fields(person: Any) -> dict[str, Any]: - person_data = person if isinstance(person, dict) else {} - organization = person_data.get("organization") - organization_data = organization if isinstance(organization, dict) else {} - return { - "name": person_data.get("name"), - "title": person_data.get("title"), - "organization_name": organization_data.get("name"), - "linkedin_url": person_data.get("linkedin_url"), - "email_status": person_data.get("email_status"), - } - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-apollo-current-depth5.json" - publication_path = ( - REPO_ROOT / "artifacts/runtime-review-publication-2026-03-30-apollo-depth5.json" - ) - org_id = f"org_runtime_review_apollo_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "capability_id": CAPABILITY_ID, - "request_body": REQUEST_BODY, - "review_kind": "current_depth5", - "organization_id": org_id, - "artifact_path": str(artifact_path), - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-apollo-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-apollo-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Apollo depth-5 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "apollo"], - ) - payload["review_agent"] = {"agent_id": agent_id, "organization_id": org_id} - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = {"status": "success", "access_id": access_id} - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - direct_headers = { - "X-Api-Key": os.environ["RHUMB_CREDENTIAL_APOLLO_API_KEY"], - "Content-Type": "application/json", - } - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body=REQUEST_BODY, - timeout=120.0, - ) - direct = await _request_json( - "POST", - APOLLO_URL, - headers=direct_headers, - json_body=REQUEST_BODY, - timeout=120.0, - ) - - rhumb_payload = ( - ((execute.body or {}).get("data") or {}) - if isinstance(execute.body, dict) - else {} - ) - rhumb_upstream = ( - (rhumb_payload.get("upstream_response") or {}) - if isinstance(rhumb_payload, dict) - else {} - ) - rhumb_person = rhumb_upstream.get("person") if isinstance(rhumb_upstream, dict) else None - direct_person = direct.body.get("person") if isinstance(direct.body, dict) else None - managed_fields = _extract_person_fields(rhumb_person) - direct_fields = _extract_person_fields(direct_person) - field_matches = { - field_name: managed_fields[field_name] == direct_fields[field_name] - for field_name in managed_fields - } - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = ( - (reviewed_at_dt + timedelta(days=30)) - .replace(microsecond=0) - .isoformat() - .replace("+00:00", "Z") - ) - - comparison: dict[str, Any] = { - field_name: { - "managed": managed_fields[field_name], - "direct": direct_fields[field_name], - "match": field_matches[field_name], - } - for field_name in managed_fields - } - comparison["all_fields_match"] = all(field_matches.values()) - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["all_fields_match"] - ) else "fail" - - payload.update( - { - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": {"status": estimate.status_code, "data": estimate.body}, - "rhumb": {"status": execute.status_code, "data": execute.body}, - "direct": {"status": direct.status_code, "data": direct.body}, - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError( - "Apollo depth-5 runtime pass failed; inspect artifact for parity details" - ) - - execution_id = rhumb_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", - ( - "Apollo: current-depth rerun confirms data.enrich_person parity through " - "Rhumb Resolve again" - ), - "--summary", - ( - "Fresh current-depth runtime rerun passed for Apollo data.enrich_person " - "through Rhumb Resolve again. Managed and direct executions matched on " - "name, title, organization_name, linkedin_url, and email_status for the " - "same live email input." - ), - "--evidence-title", - "Apollo current-depth runtime rerun parity check via Rhumb Resolve again", - "--evidence-summary", - ( - "Fresh current-depth runtime rerun passed for Apollo data.enrich_person " - "through Rhumb Resolve again. Managed and direct executions matched on " - "name, title, organization_name, linkedin_url, and email_status for the " - "same live email input." - ), - "--source-ref", f"runtime-review:apollo:{stamp}", - "--source-batch-id", f"runtime-review:apollo:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "apollo", - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "service_slug": SERVICE_SLUG, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_brave_depth11_20260403.py b/scripts/runtime_review_brave_depth11_20260403.py deleted file mode 100644 index 6601a2e6..00000000 --- a/scripts/runtime_review_brave_depth11_20260403.py +++ /dev/null @@ -1,348 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -import uuid -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -BRAVE_BASE_URL = "https://api.search.brave.com/res/v1/web/search" -PUBLIC_SERVICE_SLUG = "brave-search-api" -CANONICAL_PUBLISH_SLUG = "brave-search" -CAPABILITY_ID = "search.query" -QUERY = "LLM observability tools" -RESULT_COUNT = 5 -POST_GRANT_PROPAGATION_DELAY_SECONDS = 5 -ESTIMATE_AUTH_RETRY_ATTEMPTS = 4 -ESTIMATE_AUTH_RETRY_DELAY_SECONDS = 5 - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _looks_like_invalid_key(body: Any) -> bool: - if not isinstance(body, dict): - return False - detail = str(body.get("detail") or "").lower() - error = str(body.get("error") or "").lower() - return "invalid or expired rhumb api key" in detail or "invalid or expired rhumb api key" in error - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request(method, url, headers=headers, params=params, json=json_body) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label in {"🟒 Runtime-verified", "πŸ§ͺ Tester-generated"}: - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _web_results(body: Any) -> list[dict[str, Any]]: - if not isinstance(body, dict): - return [] - web = body.get("web") - if not isinstance(web, dict): - return [] - results = web.get("results") - return results if isinstance(results, list) else [] - - -def _top_result(results: list[dict[str, Any]]) -> dict[str, Any] | None: - if not results: - return None - item = results[0] - if not isinstance(item, dict): - return None - return { - "title": item.get("title"), - "url": item.get("url"), - "description": item.get("description"), - } - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - short = uuid.uuid4().hex[:8] - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-brave-depth11.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-03-brave-depth11.json" - org_id = f"org_runtime_review_brave_{stamp.lower()}_{short}" - - payload: dict[str, Any] = { - "provider": PUBLIC_SERVICE_SLUG, - "canonical_publish_slug": CANONICAL_PUBLISH_SLUG, - "capability_id": CAPABILITY_ID, - "query": QUERY, - "num_results": RESULT_COUNT, - "review_kind": "current_depth11", - "organization_id": org_id, - "artifact_path": str(artifact_path.relative_to(REPO_ROOT)), - "started_at": started_at.replace(microsecond=0).isoformat().replace("+00:00", "Z"), - "post_grant_delay_seconds": POST_GRANT_PROPAGATION_DELAY_SECONDS, - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-brave-{stamp}", - starter_credits_cents=5000, - ) - payload["wallet_bootstrap"] = bootstrap - _json_dump(artifact_path, payload) - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-brave-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Brave Search depth-11 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "brave-search", "depth11"], - ) - payload["review_agent"] = {"agent_id": agent_id, "organization_id": org_id} - - access_id = await store.grant_service_access(agent_id, CANONICAL_PUBLISH_SLUG) - payload["grant_access"] = {"status": "success", "access_id": access_id} - _json_dump(artifact_path, payload) - - await asyncio.sleep(POST_GRANT_PROPAGATION_DELAY_SECONDS) - - api_headers = {"X-Rhumb-Key": api_key} - brave_token = os.environ["RHUMB_CREDENTIAL_BRAVE_SEARCH_API_KEY"] - - try: - counts_before = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - - estimate_attempts = 0 - estimate = HttpResult(status_code=0, body=None) - while estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS: - estimate_attempts += 1 - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": PUBLIC_SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - if not ( - estimate.status_code == 401 - and _looks_like_invalid_key(estimate.body) - and estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS - ): - break - await asyncio.sleep(ESTIMATE_AUTH_RETRY_DELAY_SECONDS) - - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": PUBLIC_SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body={"query": QUERY, "numResults": RESULT_COUNT}, - timeout=120.0, - ) - direct = await _request_json( - "GET", - BRAVE_BASE_URL, - headers={ - "Accept": "application/json", - "X-Subscription-Token": brave_token, - }, - params={"q": QUERY, "count": RESULT_COUNT}, - timeout=60.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - rhumb_results = _web_results(rhumb_upstream) - direct_results = _web_results(direct.body) - rhumb_top = _top_result(rhumb_results) - direct_top = _top_result(direct_results) - rhumb_top3_urls = [row.get("url") for row in rhumb_results[:3] if isinstance(row, dict)] - direct_top3_urls = [row.get("url") for row in direct_results[:3] if isinstance(row, dict)] - - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "provider_used": rhumb_payload.get("provider_used"), - "upstream_status": rhumb_payload.get("upstream_status"), - "execution_id": rhumb_payload.get("execution_id"), - "rhumb_count": len(rhumb_results), - "direct_count": len(direct_results), - "rhumb_top": rhumb_top, - "direct_top": direct_top, - "rhumb_top3_urls": rhumb_top3_urls, - "direct_top3_urls": direct_top3_urls, - "result_count_match": len(rhumb_results) == len(direct_results) == RESULT_COUNT, - "top_title_match": (rhumb_top or {}).get("title") == (direct_top or {}).get("title"), - "top_url_match": (rhumb_top or {}).get("url") == (direct_top or {}).get("url"), - "top_description_match": (rhumb_top or {}).get("description") == (direct_top or {}).get("description"), - "top3_urls_match": rhumb_top3_urls == direct_top3_urls, - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == CANONICAL_PUBLISH_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["result_count_match"] - and comparison["top_title_match"] - and comparison["top_url_match"] - and comparison["top_description_match"] - and comparison["top3_urls_match"] - ) else "fail" - - payload.update( - { - "reviewed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": { - "status_code": estimate.status_code, - "body": estimate.body, - "attempts": estimate_attempts, - }, - "rhumb_execute": asdict(execute), - "direct": asdict(direct), - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Brave Search depth-11 runtime pass failed; inspect artifact for parity details") - - publication = _run_publish( - [ - "--service", CANONICAL_PUBLISH_SLUG, - "--headline", "Brave Search: depth-11 runtime review confirms search.query parity through Rhumb Resolve", - "--summary", "Fresh depth-11 runtime review passed for Brave Search search.query through Rhumb Resolve. Managed and direct executions matched on result count, top result fields, and top-3 URL ordering for the same live query.", - "--evidence-title", "Brave Search depth-11 runtime review parity check via Rhumb Resolve", - "--evidence-summary", "Fresh depth-11 runtime review passed for Brave Search search.query through Rhumb Resolve. Managed and direct executions matched on result count, top result fields, and top-3 URL ordering for the same live query.", - "--source-ref", f"runtime-review:brave-search:{stamp}", - "--source-batch-id", f"runtime-review:brave-search:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", rhumb_payload.get("execution_id") or "", - "--tag", "runtime_review", - "--tag", "brave-search", - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--tag", "depth11", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["publication"] = publication - payload["counts_after"] = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["disable_agent"] = { - "status_code": 200 if disabled else 404, - "body": {"status": "success" if disabled else "not_found", "agent_id": agent_id}, - } - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_brave_depth5_20260330.py b/scripts/runtime_review_brave_depth5_20260330.py deleted file mode 100644 index 5ff2b5c3..00000000 --- a/scripts/runtime_review_brave_depth5_20260330.py +++ /dev/null @@ -1,322 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -BRAVE_BASE_URL = "https://api.search.brave.com/res/v1/web/search" -QUERY = "LLM observability tools" -RESULT_COUNT = 5 -PUBLIC_SERVICE_SLUG = "brave-search-api" -CANONICAL_PUBLISH_SLUG = "brave-search" - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _web_results(body: Any) -> list[dict[str, Any]]: - if not isinstance(body, dict): - return [] - web = body.get("web") - if not isinstance(web, dict): - return [] - results = web.get("results") - return results if isinstance(results, list) else [] - - -def _top_result(results: list[dict[str, Any]]) -> dict[str, Any] | None: - if not results: - return None - item = results[0] - if not isinstance(item, dict): - return None - return { - "title": item.get("title"), - "url": item.get("url"), - "description": item.get("description"), - } - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-brave-current-depth5.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-30-brave-depth5.json" - org_id = f"org_runtime_review_brave_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": PUBLIC_SERVICE_SLUG, - "canonical_publish_slug": CANONICAL_PUBLISH_SLUG, - "query": QUERY, - "numResults": RESULT_COUNT, - "review_kind": "current_depth5", - "organization_id": org_id, - "artifact_path": str(artifact_path), - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-brave-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-brave-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Brave Search depth-5 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "brave-search"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, CANONICAL_PUBLISH_SLUG) - payload["grant_access"] = { - "status": "success", - "access_id": access_id, - } - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - brave_token = os.environ["RHUMB_CREDENTIAL_BRAVE_SEARCH_API_KEY"] - - try: - counts_before = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/search.query/execute/estimate", - headers=api_headers, - params={"provider": PUBLIC_SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/search.query/execute", - headers=api_headers, - params={"provider": PUBLIC_SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body={"query": QUERY, "numResults": RESULT_COUNT}, - timeout=120.0, - ) - direct = await _request_json( - "GET", - BRAVE_BASE_URL, - headers={ - "Accept": "application/json", - "X-Subscription-Token": brave_token, - }, - params={"q": QUERY, "count": RESULT_COUNT}, - timeout=60.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - rhumb_results = _web_results(rhumb_upstream) - direct_results = _web_results(direct.body) - rhumb_top = _top_result(rhumb_results) - direct_top = _top_result(direct_results) - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_count": len(rhumb_results), - "direct_count": len(direct_results), - "rhumb_top": rhumb_top, - "direct_top": direct_top, - "parity": { - "result_count_match": len(rhumb_results) == len(direct_results) == RESULT_COUNT, - "top_title_match": (rhumb_top or {}).get("title") == (direct_top or {}).get("title"), - "top_url_match": (rhumb_top or {}).get("url") == (direct_top or {}).get("url"), - }, - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == CANONICAL_PUBLISH_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["parity"]["result_count_match"] - and comparison["parity"]["top_title_match"] - and comparison["parity"]["top_url_match"] - ) else "fail" - - payload.update( - { - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": {"status": estimate.status_code, "data": estimate.body}, - "rhumb": { - "status": execute.status_code, - "data": execute.body, - }, - "direct": { - "status": direct.status_code, - "data": direct.body, - }, - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Brave Search depth-5 runtime pass failed; inspect artifact for parity details") - - execution_id = rhumb_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", CANONICAL_PUBLISH_SLUG, - "--headline", "Brave Search: current-depth rerun confirms search.query parity through Rhumb Resolve again", - "--summary", "Fresh current-depth runtime rerun passed for Brave Search search.query through Rhumb Resolve. Managed and direct executions matched on result count, top title, and top URL for the same live query, lifting the public runtime-backed review stack another layer.", - "--evidence-title", "Brave Search current-depth runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh current-depth runtime rerun passed for Brave Search search.query through Rhumb Resolve. Managed and direct executions matched on result count, top title, and top URL for the same live query.", - "--source-ref", f"runtime-review:brave-search:{stamp}", - "--source-batch-id", f"runtime-review:brave-search:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "brave-search", - "--tag", "search.query", - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "canonical_service_slug": CANONICAL_PUBLISH_SLUG, - "public_service_route": PUBLIC_SERVICE_SLUG, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_brave_depth6_20260331.py b/scripts/runtime_review_brave_depth6_20260331.py deleted file mode 100644 index 858675d9..00000000 --- a/scripts/runtime_review_brave_depth6_20260331.py +++ /dev/null @@ -1,322 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -BRAVE_BASE_URL = "https://api.search.brave.com/res/v1/web/search" -QUERY = "LLM observability tools" -RESULT_COUNT = 5 -PUBLIC_SERVICE_SLUG = "brave-search-api" -CANONICAL_PUBLISH_SLUG = "brave-search" - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _web_results(body: Any) -> list[dict[str, Any]]: - if not isinstance(body, dict): - return [] - web = body.get("web") - if not isinstance(web, dict): - return [] - results = web.get("results") - return results if isinstance(results, list) else [] - - -def _top_result(results: list[dict[str, Any]]) -> dict[str, Any] | None: - if not results: - return None - item = results[0] - if not isinstance(item, dict): - return None - return { - "title": item.get("title"), - "url": item.get("url"), - "description": item.get("description"), - } - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-brave-current-depth6.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-31-brave-depth6.json" - org_id = f"org_runtime_review_brave_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": PUBLIC_SERVICE_SLUG, - "canonical_publish_slug": CANONICAL_PUBLISH_SLUG, - "query": QUERY, - "numResults": RESULT_COUNT, - "review_kind": "current_depth6", - "organization_id": org_id, - "artifact_path": str(artifact_path), - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-brave-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-brave-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Brave Search depth-6 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "brave-search"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, CANONICAL_PUBLISH_SLUG) - payload["grant_access"] = { - "status": "success", - "access_id": access_id, - } - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - brave_token = os.environ["RHUMB_CREDENTIAL_BRAVE_SEARCH_API_KEY"] - - try: - counts_before = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/search.query/execute/estimate", - headers=api_headers, - params={"provider": PUBLIC_SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/search.query/execute", - headers=api_headers, - params={"provider": PUBLIC_SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body={"query": QUERY, "numResults": RESULT_COUNT}, - timeout=120.0, - ) - direct = await _request_json( - "GET", - BRAVE_BASE_URL, - headers={ - "Accept": "application/json", - "X-Subscription-Token": brave_token, - }, - params={"q": QUERY, "count": RESULT_COUNT}, - timeout=60.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - rhumb_results = _web_results(rhumb_upstream) - direct_results = _web_results(direct.body) - rhumb_top = _top_result(rhumb_results) - direct_top = _top_result(direct_results) - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_count": len(rhumb_results), - "direct_count": len(direct_results), - "rhumb_top": rhumb_top, - "direct_top": direct_top, - "parity": { - "result_count_match": len(rhumb_results) == len(direct_results) == RESULT_COUNT, - "top_title_match": (rhumb_top or {}).get("title") == (direct_top or {}).get("title"), - "top_url_match": (rhumb_top or {}).get("url") == (direct_top or {}).get("url"), - }, - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == CANONICAL_PUBLISH_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["parity"]["result_count_match"] - and comparison["parity"]["top_title_match"] - and comparison["parity"]["top_url_match"] - ) else "fail" - - payload.update( - { - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": {"status": estimate.status_code, "data": estimate.body}, - "rhumb": { - "status": execute.status_code, - "data": execute.body, - }, - "direct": { - "status": direct.status_code, - "data": direct.body, - }, - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Brave Search depth-6 runtime pass failed; inspect artifact for parity details") - - execution_id = rhumb_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", CANONICAL_PUBLISH_SLUG, - "--headline", "Brave Search: depth-6 rerun confirms search.query parity through Rhumb Resolve again", - "--summary", "Fresh depth-6 runtime rerun passed for Brave Search search.query through Rhumb Resolve. Managed and direct executions matched on result count, top title, and top URL for the same live query, lifting the public runtime-backed review stack from 5 to 6.", - "--evidence-title", "Brave Search depth-6 runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh depth-6 runtime rerun passed for Brave Search search.query through Rhumb Resolve. Managed and direct executions matched on result count, top title, and top URL for the same live query.", - "--source-ref", f"runtime-review:brave-search:{stamp}", - "--source-batch-id", f"runtime-review:brave-search:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "brave-search", - "--tag", "search.query", - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "canonical_service_slug": CANONICAL_PUBLISH_SLUG, - "public_service_route": PUBLIC_SERVICE_SLUG, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_e2b_depth11_20260403.py b/scripts/runtime_review_e2b_depth11_20260403.py deleted file mode 100644 index a8032c9c..00000000 --- a/scripts/runtime_review_e2b_depth11_20260403.py +++ /dev/null @@ -1,449 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -import uuid -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -E2B_BASE_URL = "https://api.e2b.app" -SERVICE_SLUG = "e2b" -CREATE_CAPABILITY_ID = "agent.spawn" -STATUS_CAPABILITY_ID = "agent.get_status" -POST_GRANT_PROPAGATION_DELAY_SECONDS = 5 -ESTIMATE_AUTH_RETRY_ATTEMPTS = 4 -ESTIMATE_AUTH_RETRY_DELAY_SECONDS = 5 -REVIEW_TEMPLATE = { - "templateID": "base", - "timeout": 300, -} - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -RUNTIME_BACKED_TRUST_LABELS = {"🟒 Runtime-verified", "πŸ§ͺ Tester-generated"} - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _looks_like_invalid_key(body: Any) -> bool: - if not isinstance(body, dict): - return False - detail = str(body.get("detail") or "").lower() - error = str(body.get("error") or "").lower() - return "invalid or expired rhumb api key" in detail or "invalid or expired rhumb api key" in error - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -async def _delete_direct_sandbox(api_key: str, sandbox_id: str | None) -> int | None: - if not sandbox_id: - return None - result = await _request_json( - "DELETE", - f"{E2B_BASE_URL}/sandboxes/{sandbox_id}", - headers={"X-API-Key": api_key}, - timeout=60.0, - ) - return result.status_code - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label in RUNTIME_BACKED_TRUST_LABELS: - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _summarize_create(result: HttpResult) -> dict[str, Any]: - body = result.body if isinstance(result.body, dict) else {} - data = (body.get("data") or {}) if isinstance(body, dict) else {} - upstream = (data.get("upstream_response") or {}) if isinstance(data, dict) else {} - return { - "status_code": result.status_code, - "execution_id": data.get("execution_id"), - "provider_used": data.get("provider_used"), - "credential_mode": data.get("credential_mode"), - "upstream_status": data.get("upstream_status"), - "sandbox": { - "sandboxID": upstream.get("sandboxID"), - "templateID": upstream.get("templateID"), - "alias": upstream.get("alias"), - "envdVersion": upstream.get("envdVersion"), - }, - "raw": result.body, - } - - -def _summarize_status(result: HttpResult) -> dict[str, Any]: - body = result.body if isinstance(result.body, dict) else {} - data = (body.get("data") or {}) if isinstance(body, dict) else {} - upstream = (data.get("upstream_response") or {}) if isinstance(data, dict) else {} - return { - "status_code": result.status_code, - "execution_id": data.get("execution_id"), - "provider_used": data.get("provider_used"), - "credential_mode": data.get("credential_mode"), - "upstream_status": data.get("upstream_status"), - "sandbox": { - "sandboxID": upstream.get("sandboxID"), - "templateID": upstream.get("templateID"), - "alias": upstream.get("alias"), - "state": upstream.get("state"), - "envdVersion": upstream.get("envdVersion"), - "cpuCount": upstream.get("cpuCount"), - "memoryMB": upstream.get("memoryMB"), - }, - "raw": result.body, - } - - -def _summarize_direct_create(result: HttpResult) -> dict[str, Any]: - body = result.body if isinstance(result.body, dict) else {} - return { - "status_code": result.status_code, - "sandbox": { - "sandboxID": body.get("sandboxID"), - "templateID": body.get("templateID"), - "alias": body.get("alias"), - "envdVersion": body.get("envdVersion"), - }, - "raw": result.body, - } - - -def _summarize_direct_status(result: HttpResult) -> dict[str, Any]: - body = result.body if isinstance(result.body, dict) else {} - return { - "status_code": result.status_code, - "sandbox": { - "sandboxID": body.get("sandboxID"), - "templateID": body.get("templateID"), - "alias": body.get("alias"), - "state": body.get("state"), - "envdVersion": body.get("envdVersion"), - "cpuCount": body.get("cpuCount"), - "memoryMB": body.get("memoryMB"), - }, - "raw": result.body, - } - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - short = uuid.uuid4().hex[:8] - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-e2b-depth11.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-03-e2b-depth11.json" - org_id = f"org_runtime_review_e2b_{stamp.lower()}_{short}" - review_template = { - **REVIEW_TEMPLATE, - "metadata": { - "source": "rhumb-runtime-review", - "service": SERVICE_SLUG, - "stamp": stamp, - }, - } - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "create_capability_id": CREATE_CAPABILITY_ID, - "status_capability_id": STATUS_CAPABILITY_ID, - "review_kind": "current_depth11", - "organization_id": org_id, - "artifact_path": str(artifact_path.relative_to(REPO_ROOT)), - "started_at": started_at.replace(microsecond=0).isoformat().replace("+00:00", "Z"), - "review_template": review_template, - } - _json_dump(artifact_path, payload) - - e2b_api_key = os.environ["RHUMB_CREDENTIAL_E2B_API_KEY"] - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-e2b-{stamp}", - starter_credits_cents=5000, - ) - payload["wallet_bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-e2b-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for E2B depth-11 runtime pass", - tags=["runtime_review", "phase3", "current_pass", SERVICE_SLUG, "depth11"], - ) - payload["review_agent"] = {"agent_id": agent_id, "organization_id": org_id} - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = {"status": "success", "access_id": access_id} - payload["post_grant_delay_seconds"] = POST_GRANT_PROPAGATION_DELAY_SECONDS - _json_dump(artifact_path, payload) - - await asyncio.sleep(POST_GRANT_PROPAGATION_DELAY_SECONDS) - - api_headers = {"X-Rhumb-Key": api_key} - rhumb_sandbox_id: str | None = None - direct_sandbox_id: str | None = None - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - - estimate_attempts = 0 - estimate = HttpResult(status_code=0, body=None) - while estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS: - estimate_attempts += 1 - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CREATE_CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - if not ( - estimate.status_code == 401 - and _looks_like_invalid_key(estimate.body) - and estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS - ): - break - await asyncio.sleep(ESTIMATE_AUTH_RETRY_DELAY_SECONDS) - - rhumb_create = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CREATE_CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body=review_template, - timeout=120.0, - ) - rhumb_create_summary = _summarize_create(rhumb_create) - rhumb_sandbox_id = (rhumb_create_summary.get("sandbox") or {}).get("sandboxID") - - rhumb_status = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{STATUS_CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body={"sandboxId": rhumb_sandbox_id}, - timeout=120.0, - ) - rhumb_status_summary = _summarize_status(rhumb_status) - - direct_create = await _request_json( - "POST", - f"{E2B_BASE_URL}/sandboxes", - headers={"X-API-Key": e2b_api_key}, - json_body=review_template, - timeout=120.0, - ) - direct_create_summary = _summarize_direct_create(direct_create) - direct_sandbox_id = (direct_create_summary.get("sandbox") or {}).get("sandboxID") - - direct_status = await _request_json( - "GET", - f"{E2B_BASE_URL}/sandboxes/{direct_sandbox_id}", - headers={"X-API-Key": e2b_api_key}, - timeout=120.0, - ) - direct_status_summary = _summarize_direct_status(direct_status) - - comparison = { - "create_template_match": ( - rhumb_create_summary["sandbox"]["alias"] == direct_create_summary["sandbox"]["alias"] - and rhumb_create_summary["sandbox"]["templateID"] == direct_create_summary["sandbox"]["templateID"] - ), - "status_template_match": ( - rhumb_status_summary["sandbox"]["alias"] == direct_status_summary["sandbox"]["alias"] - and rhumb_status_summary["sandbox"]["templateID"] == direct_status_summary["sandbox"]["templateID"] - ), - "status_state_match": rhumb_status_summary["sandbox"]["state"] == direct_status_summary["sandbox"]["state"], - "envd_version_match": rhumb_status_summary["sandbox"]["envdVersion"] == direct_status_summary["sandbox"]["envdVersion"], - "compute_shape_match": ( - rhumb_status_summary["sandbox"]["cpuCount"] == direct_status_summary["sandbox"]["cpuCount"] - and rhumb_status_summary["sandbox"]["memoryMB"] == direct_status_summary["sandbox"]["memoryMB"] - ), - } - - observed_at = _iso_now() - fresh_until = ( - datetime.fromisoformat(observed_at.replace("Z", "+00:00")) + timedelta(days=30) - ).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - verdict = "pass" if ( - estimate.status_code == 200 - and rhumb_create.status_code == 200 - and rhumb_status.status_code == 200 - and rhumb_create_summary["upstream_status"] == 201 - and rhumb_status_summary["upstream_status"] == 200 - and direct_create.status_code == 201 - and direct_status.status_code == 200 - and rhumb_create_summary["provider_used"] == SERVICE_SLUG - and rhumb_status_summary["provider_used"] == SERVICE_SLUG - and all(comparison.values()) - ) else "fail" - - payload.update( - { - "reviewed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": { - "status_code": estimate.status_code, - "body": estimate.body, - "attempts": estimate_attempts, - }, - "rhumb_create": rhumb_create_summary, - "rhumb_status": rhumb_status_summary, - "direct_create": direct_create_summary, - "direct_status": direct_status_summary, - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("E2B depth-11 runtime pass failed; inspect artifact for parity details") - - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "E2B: depth-11 runtime review confirms sandbox parity through Rhumb Resolve", - "--summary", "Fresh depth-11 runtime review passed for E2B agent.spawn plus agent.get_status through Rhumb Resolve. Managed and direct executions matched on template alias, internal template id, running state, envd version, and sandbox compute shape.", - "--evidence-title", "E2B depth-11 runtime review parity check via Rhumb Resolve", - "--evidence-summary", "Fresh depth-11 runtime review passed for E2B agent.spawn plus agent.get_status through Rhumb Resolve. Managed and direct executions matched on template alias, internal template id, running state, envd version, and sandbox compute shape.", - "--source-ref", f"runtime-review:e2b:{stamp}", - "--source-batch-id", f"runtime-review:e2b:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", rhumb_create_summary.get("execution_id") or "", - "--tag", "runtime_review", - "--tag", SERVICE_SLUG, - "--tag", CREATE_CAPABILITY_ID, - "--tag", STATUS_CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--tag", "depth11", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["publication"] = publication - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - payload.setdefault("cleanup", {}) - payload["cleanup"]["rhumb_sandbox_delete_status"] = await _delete_direct_sandbox(e2b_api_key, rhumb_sandbox_id) - payload["cleanup"]["direct_sandbox_delete_status"] = await _delete_direct_sandbox(e2b_api_key, direct_sandbox_id) - disabled = await store.disable_agent(agent_id) - payload["disable_agent"] = { - "status_code": 200 if disabled else 404, - "body": {"status": "success" if disabled else "not_found", "agent_id": agent_id}, - } - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_e2b_depth4_20260330.py b/scripts/runtime_review_e2b_depth4_20260330.py deleted file mode 100644 index fef8a60d..00000000 --- a/scripts/runtime_review_e2b_depth4_20260330.py +++ /dev/null @@ -1,399 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -import uuid -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -E2B_BASE_URL = "https://api.e2b.app" -REVIEW_TEMPLATE = { - "templateID": "base", - "timeout": 300, -} - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -async def _delete_direct_sandbox(api_key: str, sandbox_id: str | None) -> int | None: - if not sandbox_id: - return None - result = await _request_json( - "DELETE", - f"{E2B_BASE_URL}/sandboxes/{sandbox_id}", - headers={"X-API-Key": api_key}, - timeout=60.0, - ) - return result.status_code - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - short = uuid.uuid4().hex[:8] - metadata = { - "source": "rhumb-runtime-review", - "service": "e2b", - "stamp": stamp, - } - review_template = {**REVIEW_TEMPLATE, "metadata": metadata} - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-e2b-current-depth4.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-30-e2b-depth4.json" - org_id = f"org_runtime_review_e2b_{stamp.lower()}_{short}" - - payload: dict[str, Any] = { - "service": "e2b", - "review_kind": "current_depth4", - "reviewed_at": None, - "fresh_until": None, - "review_template": review_template, - "temp_org": { - "org_id": org_id, - "balance_usd_cents": 5000, - }, - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-e2b-{stamp}", - starter_credits_cents=5000, - ) - payload["wallet_bootstrap"] = bootstrap - _json_dump(artifact_path, payload) - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-e2b-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for E2B depth-4 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "e2b"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, "e2b") - payload["grant_result"] = { - "service": "e2b", - "status_code": 200, - "body": {"status": "success", "access_id": access_id}, - } - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - e2b_api_key = os.environ["RHUMB_CREDENTIAL_E2B_API_KEY"] - - rhumb_sandbox_id: str | None = None - direct_sandbox_id: str | None = None - try: - counts_before = await _fetch_public_review_stats("e2b") - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/agent.spawn/execute/estimate", - headers=api_headers, - params={"provider": "e2b", "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - rhumb_create = await _request_json( - "POST", - f"{BASE_URL}/capabilities/agent.spawn/execute", - headers=api_headers, - params={"provider": "e2b", "credential_mode": "rhumb_managed"}, - json_body=review_template, - timeout=120.0, - ) - - rhumb_create_data = ((rhumb_create.body or {}).get("data") or {}) if isinstance(rhumb_create.body, dict) else {} - rhumb_create_upstream = (rhumb_create_data.get("upstream_response") or {}) if isinstance(rhumb_create_data, dict) else {} - rhumb_sandbox_id = rhumb_create_upstream.get("sandboxID") - - rhumb_status = await _request_json( - "POST", - f"{BASE_URL}/capabilities/agent.get_status/execute", - headers=api_headers, - params={"provider": "e2b", "credential_mode": "rhumb_managed"}, - json_body={"sandboxId": rhumb_sandbox_id}, - timeout=120.0, - ) - - direct_create = await _request_json( - "POST", - f"{E2B_BASE_URL}/sandboxes", - headers={"X-API-Key": e2b_api_key}, - json_body=review_template, - timeout=120.0, - ) - direct_create_body = direct_create.body if isinstance(direct_create.body, dict) else {} - direct_sandbox_id = direct_create_body.get("sandboxID") - - direct_status = await _request_json( - "GET", - f"{E2B_BASE_URL}/sandboxes/{direct_sandbox_id}", - headers={"X-API-Key": e2b_api_key}, - timeout=120.0, - ) - direct_status_body = direct_status.body if isinstance(direct_status.body, dict) else {} - - rhumb_status_data = ((rhumb_status.body or {}).get("data") or {}) if isinstance(rhumb_status.body, dict) else {} - rhumb_status_upstream = (rhumb_status_data.get("upstream_response") or {}) if isinstance(rhumb_status_data, dict) else {} - - rhumb_create_summary = { - "status_code": rhumb_create.status_code, - "execution_id": rhumb_create_data.get("execution_id"), - "provider_used": rhumb_create_data.get("provider_used"), - "credential_mode": rhumb_create_data.get("credential_mode"), - "upstream_status": rhumb_create_data.get("upstream_status"), - "sandbox": { - "sandboxID": rhumb_create_upstream.get("sandboxID"), - "templateID": rhumb_create_upstream.get("templateID"), - "alias": rhumb_create_upstream.get("alias"), - "envdVersion": rhumb_create_upstream.get("envdVersion"), - }, - "raw": rhumb_create.body, - } - rhumb_status_summary = { - "status_code": rhumb_status.status_code, - "execution_id": rhumb_status_data.get("execution_id"), - "provider_used": rhumb_status_data.get("provider_used"), - "credential_mode": rhumb_status_data.get("credential_mode"), - "upstream_status": rhumb_status_data.get("upstream_status"), - "sandbox": { - "sandboxID": rhumb_status_upstream.get("sandboxID"), - "templateID": rhumb_status_upstream.get("templateID"), - "alias": rhumb_status_upstream.get("alias"), - "state": rhumb_status_upstream.get("state"), - "envdVersion": rhumb_status_upstream.get("envdVersion"), - "cpuCount": rhumb_status_upstream.get("cpuCount"), - "memoryMB": rhumb_status_upstream.get("memoryMB"), - }, - "raw": rhumb_status.body, - } - direct_create_summary = { - "status_code": direct_create.status_code, - "sandbox": { - "sandboxID": direct_create_body.get("sandboxID"), - "templateID": direct_create_body.get("templateID"), - "alias": direct_create_body.get("alias"), - "envdVersion": direct_create_body.get("envdVersion"), - }, - "raw": direct_create.body, - } - direct_status_summary = { - "status_code": direct_status.status_code, - "sandbox": { - "sandboxID": direct_status_body.get("sandboxID"), - "templateID": direct_status_body.get("templateID"), - "alias": direct_status_body.get("alias"), - "state": direct_status_body.get("state"), - "envdVersion": direct_status_body.get("envdVersion"), - "cpuCount": direct_status_body.get("cpuCount"), - "memoryMB": direct_status_body.get("memoryMB"), - }, - "raw": direct_status.body, - } - - comparison = { - "create_template_match": ( - rhumb_create_summary["sandbox"]["alias"] == direct_create_summary["sandbox"]["alias"] - and rhumb_create_summary["sandbox"]["templateID"] == direct_create_summary["sandbox"]["templateID"] - ), - "status_template_match": ( - rhumb_status_summary["sandbox"]["alias"] == direct_status_summary["sandbox"]["alias"] - and rhumb_status_summary["sandbox"]["templateID"] == direct_status_summary["sandbox"]["templateID"] - ), - "status_state_match": rhumb_status_summary["sandbox"]["state"] == direct_status_summary["sandbox"]["state"], - "envd_version_match": rhumb_status_summary["sandbox"]["envdVersion"] == direct_status_summary["sandbox"]["envdVersion"], - "compute_shape_match": ( - rhumb_status_summary["sandbox"]["cpuCount"] == direct_status_summary["sandbox"]["cpuCount"] - and rhumb_status_summary["sandbox"]["memoryMB"] == direct_status_summary["sandbox"]["memoryMB"] - ), - } - - verdict = "pass" if ( - estimate.status_code == 200 - and rhumb_create.status_code == 200 - and rhumb_status.status_code == 200 - and rhumb_create_summary["upstream_status"] == 201 - and rhumb_status_summary["upstream_status"] == 200 - and direct_create.status_code == 201 - and direct_status.status_code == 200 - and all(comparison.values()) - ) else "fail" - - observed_at = _iso_now() - fresh_until = ( - datetime.fromisoformat(observed_at.replace("Z", "+00:00")) + timedelta(days=30) - ).replace(microsecond=0).isoformat().replace("+00:00", "Z") - payload.update( - { - "reviewed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": asdict(estimate), - "rhumb_create": rhumb_create_summary, - "rhumb_status": rhumb_status_summary, - "direct_create": direct_create_summary, - "direct_status": direct_status_summary, - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("E2B depth-4 runtime pass failed; inspect artifact for parity details") - - publication = _run_publish( - [ - "--service", "e2b", - "--headline", "E2B: current-depth rerun confirms sandbox parity through Rhumb Resolve", - "--summary", "Fresh current-depth runtime rerun passed for E2B agent.spawn plus agent.get_status through Rhumb Resolve. Managed and direct executions matched on template alias and internal template id, running state, envd version, and sandbox compute shape.", - "--evidence-title", "E2B current-depth runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh current-depth runtime rerun passed for E2B agent.spawn plus agent.get_status through Rhumb Resolve. Managed and direct executions matched on template alias and internal template id, running state, envd version, and sandbox compute shape.", - "--source-ref", f"runtime-review:e2b:{stamp}", - "--source-batch-id", f"runtime-review:e2b:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", rhumb_create_summary["execution_id"] or "", - "--tag", "runtime_review", - "--tag", "e2b", - "--tag", "agent.spawn", - "--tag", "agent.get_status", - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["publication"] = publication - payload["counts_after"] = await _fetch_public_review_stats("e2b") - _json_dump(artifact_path, payload) - finally: - payload.setdefault("cleanup", {}) - payload["cleanup"]["rhumb_sandbox_delete_status"] = await _delete_direct_sandbox(e2b_api_key, rhumb_sandbox_id) - payload["cleanup"]["direct_sandbox_delete_status"] = await _delete_direct_sandbox(e2b_api_key, direct_sandbox_id) - disabled = await store.disable_agent(agent_id) - payload["disable_agent"] = { - "status_code": 200 if disabled else 404, - "body": {"status": "success" if disabled else "not_found", "agent_id": agent_id}, - } - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_e2b_depth5_20260330.py b/scripts/runtime_review_e2b_depth5_20260330.py deleted file mode 100644 index 8694b69d..00000000 --- a/scripts/runtime_review_e2b_depth5_20260330.py +++ /dev/null @@ -1,399 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -import uuid -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -E2B_BASE_URL = "https://api.e2b.app" -REVIEW_TEMPLATE = { - "templateID": "base", - "timeout": 300, -} - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -async def _delete_direct_sandbox(api_key: str, sandbox_id: str | None) -> int | None: - if not sandbox_id: - return None - result = await _request_json( - "DELETE", - f"{E2B_BASE_URL}/sandboxes/{sandbox_id}", - headers={"X-API-Key": api_key}, - timeout=60.0, - ) - return result.status_code - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - short = uuid.uuid4().hex[:8] - metadata = { - "source": "rhumb-runtime-review", - "service": "e2b", - "stamp": stamp, - } - review_template = {**REVIEW_TEMPLATE, "metadata": metadata} - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-e2b-current-depth5.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-30-e2b-depth5.json" - org_id = f"org_runtime_review_e2b_{stamp.lower()}_{short}" - - payload: dict[str, Any] = { - "service": "e2b", - "review_kind": "current_depth5", - "reviewed_at": None, - "fresh_until": None, - "review_template": review_template, - "temp_org": { - "org_id": org_id, - "balance_usd_cents": 5000, - }, - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-e2b-{stamp}", - starter_credits_cents=5000, - ) - payload["wallet_bootstrap"] = bootstrap - _json_dump(artifact_path, payload) - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-e2b-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for E2B depth-5 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "e2b"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, "e2b") - payload["grant_result"] = { - "service": "e2b", - "status_code": 200, - "body": {"status": "success", "access_id": access_id}, - } - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - e2b_api_key = os.environ["RHUMB_CREDENTIAL_E2B_API_KEY"] - - rhumb_sandbox_id: str | None = None - direct_sandbox_id: str | None = None - try: - counts_before = await _fetch_public_review_stats("e2b") - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/agent.spawn/execute/estimate", - headers=api_headers, - params={"provider": "e2b", "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - rhumb_create = await _request_json( - "POST", - f"{BASE_URL}/capabilities/agent.spawn/execute", - headers=api_headers, - params={"provider": "e2b", "credential_mode": "rhumb_managed"}, - json_body=review_template, - timeout=120.0, - ) - - rhumb_create_data = ((rhumb_create.body or {}).get("data") or {}) if isinstance(rhumb_create.body, dict) else {} - rhumb_create_upstream = (rhumb_create_data.get("upstream_response") or {}) if isinstance(rhumb_create_data, dict) else {} - rhumb_sandbox_id = rhumb_create_upstream.get("sandboxID") - - rhumb_status = await _request_json( - "POST", - f"{BASE_URL}/capabilities/agent.get_status/execute", - headers=api_headers, - params={"provider": "e2b", "credential_mode": "rhumb_managed"}, - json_body={"sandboxId": rhumb_sandbox_id}, - timeout=120.0, - ) - - direct_create = await _request_json( - "POST", - f"{E2B_BASE_URL}/sandboxes", - headers={"X-API-Key": e2b_api_key}, - json_body=review_template, - timeout=120.0, - ) - direct_create_body = direct_create.body if isinstance(direct_create.body, dict) else {} - direct_sandbox_id = direct_create_body.get("sandboxID") - - direct_status = await _request_json( - "GET", - f"{E2B_BASE_URL}/sandboxes/{direct_sandbox_id}", - headers={"X-API-Key": e2b_api_key}, - timeout=120.0, - ) - direct_status_body = direct_status.body if isinstance(direct_status.body, dict) else {} - - rhumb_status_data = ((rhumb_status.body or {}).get("data") or {}) if isinstance(rhumb_status.body, dict) else {} - rhumb_status_upstream = (rhumb_status_data.get("upstream_response") or {}) if isinstance(rhumb_status_data, dict) else {} - - rhumb_create_summary = { - "status_code": rhumb_create.status_code, - "execution_id": rhumb_create_data.get("execution_id"), - "provider_used": rhumb_create_data.get("provider_used"), - "credential_mode": rhumb_create_data.get("credential_mode"), - "upstream_status": rhumb_create_data.get("upstream_status"), - "sandbox": { - "sandboxID": rhumb_create_upstream.get("sandboxID"), - "templateID": rhumb_create_upstream.get("templateID"), - "alias": rhumb_create_upstream.get("alias"), - "envdVersion": rhumb_create_upstream.get("envdVersion"), - }, - "raw": rhumb_create.body, - } - rhumb_status_summary = { - "status_code": rhumb_status.status_code, - "execution_id": rhumb_status_data.get("execution_id"), - "provider_used": rhumb_status_data.get("provider_used"), - "credential_mode": rhumb_status_data.get("credential_mode"), - "upstream_status": rhumb_status_data.get("upstream_status"), - "sandbox": { - "sandboxID": rhumb_status_upstream.get("sandboxID"), - "templateID": rhumb_status_upstream.get("templateID"), - "alias": rhumb_status_upstream.get("alias"), - "state": rhumb_status_upstream.get("state"), - "envdVersion": rhumb_status_upstream.get("envdVersion"), - "cpuCount": rhumb_status_upstream.get("cpuCount"), - "memoryMB": rhumb_status_upstream.get("memoryMB"), - }, - "raw": rhumb_status.body, - } - direct_create_summary = { - "status_code": direct_create.status_code, - "sandbox": { - "sandboxID": direct_create_body.get("sandboxID"), - "templateID": direct_create_body.get("templateID"), - "alias": direct_create_body.get("alias"), - "envdVersion": direct_create_body.get("envdVersion"), - }, - "raw": direct_create.body, - } - direct_status_summary = { - "status_code": direct_status.status_code, - "sandbox": { - "sandboxID": direct_status_body.get("sandboxID"), - "templateID": direct_status_body.get("templateID"), - "alias": direct_status_body.get("alias"), - "state": direct_status_body.get("state"), - "envdVersion": direct_status_body.get("envdVersion"), - "cpuCount": direct_status_body.get("cpuCount"), - "memoryMB": direct_status_body.get("memoryMB"), - }, - "raw": direct_status.body, - } - - comparison = { - "create_template_match": ( - rhumb_create_summary["sandbox"]["alias"] == direct_create_summary["sandbox"]["alias"] - and rhumb_create_summary["sandbox"]["templateID"] == direct_create_summary["sandbox"]["templateID"] - ), - "status_template_match": ( - rhumb_status_summary["sandbox"]["alias"] == direct_status_summary["sandbox"]["alias"] - and rhumb_status_summary["sandbox"]["templateID"] == direct_status_summary["sandbox"]["templateID"] - ), - "status_state_match": rhumb_status_summary["sandbox"]["state"] == direct_status_summary["sandbox"]["state"], - "envd_version_match": rhumb_status_summary["sandbox"]["envdVersion"] == direct_status_summary["sandbox"]["envdVersion"], - "compute_shape_match": ( - rhumb_status_summary["sandbox"]["cpuCount"] == direct_status_summary["sandbox"]["cpuCount"] - and rhumb_status_summary["sandbox"]["memoryMB"] == direct_status_summary["sandbox"]["memoryMB"] - ), - } - - verdict = "pass" if ( - estimate.status_code == 200 - and rhumb_create.status_code == 200 - and rhumb_status.status_code == 200 - and rhumb_create_summary["upstream_status"] == 201 - and rhumb_status_summary["upstream_status"] == 200 - and direct_create.status_code == 201 - and direct_status.status_code == 200 - and all(comparison.values()) - ) else "fail" - - observed_at = _iso_now() - fresh_until = ( - datetime.fromisoformat(observed_at.replace("Z", "+00:00")) + timedelta(days=30) - ).replace(microsecond=0).isoformat().replace("+00:00", "Z") - payload.update( - { - "reviewed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": asdict(estimate), - "rhumb_create": rhumb_create_summary, - "rhumb_status": rhumb_status_summary, - "direct_create": direct_create_summary, - "direct_status": direct_status_summary, - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("E2B depth-5 runtime pass failed; inspect artifact for parity details") - - publication = _run_publish( - [ - "--service", "e2b", - "--headline", "E2B: current-depth rerun confirms sandbox parity through Rhumb Resolve", - "--summary", "Fresh current-depth runtime rerun passed for E2B agent.spawn plus agent.get_status through Rhumb Resolve. Managed and direct executions matched on template alias and internal template id, running state, envd version, and sandbox compute shape.", - "--evidence-title", "E2B current-depth runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh current-depth runtime rerun passed for E2B agent.spawn plus agent.get_status through Rhumb Resolve. Managed and direct executions matched on template alias and internal template id, running state, envd version, and sandbox compute shape.", - "--source-ref", f"runtime-review:e2b:{stamp}", - "--source-batch-id", f"runtime-review:e2b:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", rhumb_create_summary["execution_id"] or "", - "--tag", "runtime_review", - "--tag", "e2b", - "--tag", "agent.spawn", - "--tag", "agent.get_status", - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["publication"] = publication - payload["counts_after"] = await _fetch_public_review_stats("e2b") - _json_dump(artifact_path, payload) - finally: - payload.setdefault("cleanup", {}) - payload["cleanup"]["rhumb_sandbox_delete_status"] = await _delete_direct_sandbox(e2b_api_key, rhumb_sandbox_id) - payload["cleanup"]["direct_sandbox_delete_status"] = await _delete_direct_sandbox(e2b_api_key, direct_sandbox_id) - disabled = await store.disable_agent(agent_id) - payload["disable_agent"] = { - "status_code": 200 if disabled else 404, - "body": {"status": "success" if disabled else "not_found", "agent_id": agent_id}, - } - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_exa_depth10_20260402.py b/scripts/runtime_review_exa_depth10_20260402.py deleted file mode 100644 index abd1b97b..00000000 --- a/scripts/runtime_review_exa_depth10_20260402.py +++ /dev/null @@ -1,344 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import datetime, timedelta, timezone -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -EXA_URL = "https://api.exa.ai/search" -SERVICE_SLUG = "exa" -CAPABILITY_ID = "search.query" -QUERY = "best AI agent observability tools" -NUM_RESULTS = 3 - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _get_secret(item_title: str) -> str: - result = subprocess.run( - [ - "sop", - "item", - "get", - item_title, - "--vault", - "OpenClaw Agents", - "--fields", - "credential", - "--reveal", - ], - capture_output=True, - text=True, - timeout=30, - check=False, - ) - if result.returncode != 0 or not result.stdout.strip(): - raise RuntimeError(f"Unable to load secret: {item_title}\n{result.stderr}") - return result.stdout.strip() - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label in {"🟒 Runtime-verified", "πŸ§ͺ Tester-generated"}: - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [ - str(REPO_ROOT / "packages/api/.venv/bin/python"), - str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), - *args, - ], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - "publish_runtime_review_pair.py failed " - f"({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -async def main() -> None: - started_at = datetime.now(tz=timezone.utc) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-exa-depth10.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-02-exa-depth10.json" - org_id = f"org_runtime_review_exa_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "capability_id": CAPABILITY_ID, - "query": QUERY, - "num_results": NUM_RESULTS, - "review_kind": "current_depth10", - "organization_id": org_id, - "artifact_path": str(artifact_path.relative_to(REPO_ROOT)), - "started_at": started_at.replace(microsecond=0).isoformat().replace("+00:00", "Z"), - } - _json_dump(artifact_path, payload) - - exa_api_key = os.environ.get("RHUMB_CREDENTIAL_EXA_API_KEY") or _get_secret("Exa API Key") - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-exa-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-exa-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Exa depth-10 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "exa", "depth10"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = { - "status": "success", - "access_id": access_id, - } - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - direct_headers = { - "x-api-key": exa_api_key, - "Content-Type": "application/json", - "User-Agent": "Mozilla/5.0 (compatible; RhumbRuntimeReview/1.0)", - } - direct_body = { - "query": QUERY, - "numResults": NUM_RESULTS, - } - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body={ - "query": QUERY, - "numResults": NUM_RESULTS, - }, - timeout=120.0, - ) - direct = await _request_json( - "POST", - EXA_URL, - headers=direct_headers, - json_body=direct_body, - timeout=120.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - rhumb_results = rhumb_upstream.get("results") if isinstance(rhumb_upstream, dict) else [] - direct_results = direct.body.get("results") if isinstance(direct.body, dict) else [] - rhumb_top = rhumb_results[0] if isinstance(rhumb_results, list) and rhumb_results else None - direct_top = direct_results[0] if isinstance(direct_results, list) and direct_results else None - rhumb_top3_urls = [row.get("url") for row in rhumb_results[:3] if isinstance(row, dict)] if isinstance(rhumb_results, list) else [] - direct_top3_urls = [row.get("url") for row in direct_results[:3] if isinstance(row, dict)] if isinstance(direct_results, list) else [] - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_provider_used": rhumb_payload.get("provider_used"), - "rhumb_upstream_status": rhumb_payload.get("upstream_status"), - "rhumb_count": len(rhumb_results) if isinstance(rhumb_results, list) else None, - "direct_count": len(direct_results) if isinstance(direct_results, list) else None, - "rhumb_top": { - "title": (rhumb_top or {}).get("title"), - "url": (rhumb_top or {}).get("url"), - }, - "direct_top": { - "title": (direct_top or {}).get("title"), - "url": (direct_top or {}).get("url"), - }, - "rhumb_top3_urls": rhumb_top3_urls, - "direct_top3_urls": direct_top3_urls, - "result_count_match": isinstance(rhumb_results, list) and isinstance(direct_results, list) and len(rhumb_results) == len(direct_results), - "top_title_match": (rhumb_top or {}).get("title") == (direct_top or {}).get("title"), - "top_url_match": (rhumb_top or {}).get("url") == (direct_top or {}).get("url"), - "top3_urls_match": rhumb_top3_urls == direct_top3_urls, - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["result_count_match"] - and comparison["top_title_match"] - and comparison["top_url_match"] - and comparison["top3_urls_match"] - ) else "fail" - - payload.update( - { - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": {"status": estimate.status_code, "data": estimate.body}, - "rhumb": {"status": execute.status_code, "data": execute.body}, - "direct": {"status": direct.status_code, "data": direct.body}, - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Exa depth-10 runtime pass failed; inspect artifact for parity details") - - execution_id = rhumb_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "Exa: depth-10 runtime review confirms search.query parity through Rhumb Resolve", - "--summary", "Fresh depth-10 runtime review passed for Exa search.query through Rhumb Resolve. Managed and direct executions matched on result count, top title, top URL, and top-3 URL ordering for the same live search query.", - "--evidence-title", "Exa depth-10 runtime review parity check via Rhumb Resolve", - "--evidence-summary", "Fresh depth-10 runtime review passed for Exa search.query through Rhumb Resolve. Managed and direct executions matched on result count, top title, top URL, and top-3 URL ordering for the same live search query.", - "--source-ref", f"runtime-review:exa:{stamp}", - "--source-batch-id", f"runtime-review:exa:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "exa", - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--tag", "depth10", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "service_slug": SERVICE_SLUG, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_exa_depth11_20260403.py b/scripts/runtime_review_exa_depth11_20260403.py deleted file mode 100644 index 67f828a2..00000000 --- a/scripts/runtime_review_exa_depth11_20260403.py +++ /dev/null @@ -1,359 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -import uuid -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -EXA_URL = "https://api.exa.ai/search" -SERVICE_SLUG = "exa" -CAPABILITY_ID = "search.query" -QUERY = "best AI agent observability tools" -NUM_RESULTS = 3 -POST_GRANT_PROPAGATION_DELAY_SECONDS = 5 -ESTIMATE_AUTH_RETRY_ATTEMPTS = 4 -ESTIMATE_AUTH_RETRY_DELAY_SECONDS = 5 - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _looks_like_invalid_key(body: Any) -> bool: - if not isinstance(body, dict): - return False - detail = str(body.get("detail") or "").lower() - error = str(body.get("error") or "").lower() - return "invalid or expired rhumb api key" in detail or "invalid or expired rhumb api key" in error - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request(method, url, headers=headers, params=params, json=json_body) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _get_secret(item_title: str) -> str: - result = subprocess.run( - [ - "sop", - "item", - "get", - item_title, - "--vault", - "OpenClaw Agents", - "--fields", - "credential", - "--reveal", - ], - capture_output=True, - text=True, - timeout=30, - check=False, - ) - if result.returncode != 0 or not result.stdout.strip(): - raise RuntimeError(f"Unable to load secret: {item_title}\n{result.stderr}") - return result.stdout.strip() - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label in {"🟒 Runtime-verified", "πŸ§ͺ Tester-generated"}: - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - short = uuid.uuid4().hex[:8] - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-exa-depth11.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-03-exa-depth11.json" - org_id = f"org_runtime_review_exa_{stamp.lower()}_{short}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "capability_id": CAPABILITY_ID, - "query": QUERY, - "num_results": NUM_RESULTS, - "review_kind": "current_depth11", - "organization_id": org_id, - "artifact_path": str(artifact_path.relative_to(REPO_ROOT)), - "started_at": started_at.replace(microsecond=0).isoformat().replace("+00:00", "Z"), - "post_grant_delay_seconds": POST_GRANT_PROPAGATION_DELAY_SECONDS, - } - _json_dump(artifact_path, payload) - - exa_api_key = os.environ.get("RHUMB_CREDENTIAL_EXA_API_KEY") or _get_secret("Exa API Key") - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-exa-{stamp}", - starter_credits_cents=5000, - ) - payload["wallet_bootstrap"] = bootstrap - _json_dump(artifact_path, payload) - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-exa-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Exa depth-11 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "exa", "depth11"], - ) - payload["review_agent"] = {"agent_id": agent_id, "organization_id": org_id} - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = {"status": "success", "access_id": access_id} - _json_dump(artifact_path, payload) - - await asyncio.sleep(POST_GRANT_PROPAGATION_DELAY_SECONDS) - - api_headers = {"X-Rhumb-Key": api_key} - direct_headers = { - "x-api-key": exa_api_key, - "Content-Type": "application/json", - "User-Agent": "Mozilla/5.0 (compatible; RhumbRuntimeReview/1.0)", - } - direct_body = { - "query": QUERY, - "numResults": NUM_RESULTS, - } - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - - estimate_attempts = 0 - estimate = HttpResult(status_code=0, body=None) - while estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS: - estimate_attempts += 1 - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - if not ( - estimate.status_code == 401 - and _looks_like_invalid_key(estimate.body) - and estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS - ): - break - await asyncio.sleep(ESTIMATE_AUTH_RETRY_DELAY_SECONDS) - - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body={ - "query": QUERY, - "numResults": NUM_RESULTS, - }, - timeout=120.0, - ) - direct = await _request_json( - "POST", - EXA_URL, - headers=direct_headers, - json_body=direct_body, - timeout=120.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - rhumb_results = rhumb_upstream.get("results") if isinstance(rhumb_upstream, dict) else [] - direct_results = direct.body.get("results") if isinstance(direct.body, dict) else [] - rhumb_top = rhumb_results[0] if isinstance(rhumb_results, list) and rhumb_results else None - direct_top = direct_results[0] if isinstance(direct_results, list) and direct_results else None - rhumb_top3_urls = [row.get("url") for row in rhumb_results[:3] if isinstance(row, dict)] if isinstance(rhumb_results, list) else [] - direct_top3_urls = [row.get("url") for row in direct_results[:3] if isinstance(row, dict)] if isinstance(direct_results, list) else [] - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "provider_used": rhumb_payload.get("provider_used"), - "upstream_status": rhumb_payload.get("upstream_status"), - "execution_id": rhumb_payload.get("execution_id"), - "rhumb_count": len(rhumb_results) if isinstance(rhumb_results, list) else None, - "direct_count": len(direct_results) if isinstance(direct_results, list) else None, - "rhumb_top": { - "title": (rhumb_top or {}).get("title"), - "url": (rhumb_top or {}).get("url"), - }, - "direct_top": { - "title": (direct_top or {}).get("title"), - "url": (direct_top or {}).get("url"), - }, - "rhumb_top3_urls": rhumb_top3_urls, - "direct_top3_urls": direct_top3_urls, - "result_count_match": isinstance(rhumb_results, list) and isinstance(direct_results, list) and len(rhumb_results) == len(direct_results), - "top_title_match": (rhumb_top or {}).get("title") == (direct_top or {}).get("title"), - "top_url_match": (rhumb_top or {}).get("url") == (direct_top or {}).get("url"), - "top3_urls_match": rhumb_top3_urls == direct_top3_urls, - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["result_count_match"] - and comparison["top_title_match"] - and comparison["top_url_match"] - and comparison["top3_urls_match"] - ) else "fail" - - payload.update( - { - "reviewed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": { - "status_code": estimate.status_code, - "body": estimate.body, - "attempts": estimate_attempts, - }, - "rhumb_execute": asdict(execute), - "direct": asdict(direct), - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Exa depth-11 runtime pass failed; inspect artifact for parity details") - - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "Exa: depth-11 runtime review confirms search.query parity through Rhumb Resolve", - "--summary", "Fresh depth-11 runtime review passed for Exa search.query through Rhumb Resolve. Managed and direct executions matched on result count, top title, top URL, and top-3 URL ordering for the same live search query.", - "--evidence-title", "Exa depth-11 runtime review parity check via Rhumb Resolve", - "--evidence-summary", "Fresh depth-11 runtime review passed for Exa search.query through Rhumb Resolve. Managed and direct executions matched on result count, top title, top URL, and top-3 URL ordering for the same live search query.", - "--source-ref", f"runtime-review:exa:{stamp}", - "--source-batch-id", f"runtime-review:exa:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", rhumb_payload.get("execution_id") or "", - "--tag", "runtime_review", - "--tag", "exa", - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--tag", "depth11", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["publication"] = publication - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["disable_agent"] = { - "status_code": 200 if disabled else 404, - "body": {"status": "success" if disabled else "not_found", "agent_id": agent_id}, - } - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_exa_depth5_20260330.py b/scripts/runtime_review_exa_depth5_20260330.py deleted file mode 100644 index ee019f62..00000000 --- a/scripts/runtime_review_exa_depth5_20260330.py +++ /dev/null @@ -1,317 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -EXA_URL = "https://api.exa.ai/search" -SERVICE_SLUG = "exa" -CAPABILITY_ID = "search.query" -QUERY = "best AI agent observability tools" -NUM_RESULTS = 3 - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [ - str(REPO_ROOT / "packages/api/.venv/bin/python"), - str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), - *args, - ], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - "publish_runtime_review_pair.py failed " - f"({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-exa-current-depth5.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-30-exa-depth5.json" - org_id = f"org_runtime_review_exa_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "capability_id": CAPABILITY_ID, - "query": QUERY, - "num_results": NUM_RESULTS, - "review_kind": "current_depth5", - "organization_id": org_id, - "artifact_path": str(artifact_path), - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-exa-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-exa-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Exa depth-5 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "exa"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = { - "status": "success", - "access_id": access_id, - } - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - direct_headers = { - "x-api-key": os.environ["RHUMB_CREDENTIAL_EXA_API_KEY"], - "Content-Type": "application/json", - "User-Agent": "Mozilla/5.0 (compatible; RhumbRuntimeReview/1.0)", - } - direct_body = { - "query": QUERY, - "numResults": NUM_RESULTS, - } - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body={ - "query": QUERY, - "numResults": NUM_RESULTS, - }, - timeout=120.0, - ) - direct = await _request_json( - "POST", - EXA_URL, - headers=direct_headers, - json_body=direct_body, - timeout=120.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - rhumb_results = rhumb_upstream.get("results") if isinstance(rhumb_upstream, dict) else [] - direct_results = direct.body.get("results") if isinstance(direct.body, dict) else [] - rhumb_top = rhumb_results[0] if isinstance(rhumb_results, list) and rhumb_results else None - direct_top = direct_results[0] if isinstance(direct_results, list) and direct_results else None - rhumb_top3_urls = [row.get("url") for row in rhumb_results[:3] if isinstance(row, dict)] if isinstance(rhumb_results, list) else [] - direct_top3_urls = [row.get("url") for row in direct_results[:3] if isinstance(row, dict)] if isinstance(direct_results, list) else [] - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_provider_used": rhumb_payload.get("provider_used"), - "rhumb_upstream_status": rhumb_payload.get("upstream_status"), - "rhumb_count": len(rhumb_results) if isinstance(rhumb_results, list) else None, - "direct_count": len(direct_results) if isinstance(direct_results, list) else None, - "rhumb_top": { - "title": (rhumb_top or {}).get("title"), - "url": (rhumb_top or {}).get("url"), - }, - "direct_top": { - "title": (direct_top or {}).get("title"), - "url": (direct_top or {}).get("url"), - }, - "rhumb_top3_urls": rhumb_top3_urls, - "direct_top3_urls": direct_top3_urls, - "result_count_match": isinstance(rhumb_results, list) and isinstance(direct_results, list) and len(rhumb_results) == len(direct_results), - "top_title_match": (rhumb_top or {}).get("title") == (direct_top or {}).get("title"), - "top_url_match": (rhumb_top or {}).get("url") == (direct_top or {}).get("url"), - "top3_urls_match": rhumb_top3_urls == direct_top3_urls, - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["result_count_match"] - and comparison["top_title_match"] - and comparison["top_url_match"] - and comparison["top3_urls_match"] - ) else "fail" - - payload.update( - { - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": {"status": estimate.status_code, "data": estimate.body}, - "rhumb": {"status": execute.status_code, "data": execute.body}, - "direct": {"status": direct.status_code, "data": direct.body}, - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Exa depth-5 runtime pass failed; inspect artifact for parity details") - - execution_id = rhumb_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "Exa: current-depth rerun confirms search.query parity through Rhumb Resolve again", - "--summary", "Fresh current-depth runtime rerun passed for Exa search.query through Rhumb Resolve. Managed and direct executions matched on result count, top title, top URL, and top-3 URL ordering for the same live search query.", - "--evidence-title", "Exa current-depth runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh current-depth runtime rerun passed for Exa search.query through Rhumb Resolve. Managed and direct executions matched on result count, top title, top URL, and top-3 URL ordering for the same live search query.", - "--source-ref", f"runtime-review:exa:{stamp}", - "--source-batch-id", f"runtime-review:exa:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "exa", - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "service_slug": SERVICE_SLUG, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_firecrawl_depth11_20260403.py b/scripts/runtime_review_firecrawl_depth11_20260403.py deleted file mode 100644 index d9f01bd1..00000000 --- a/scripts/runtime_review_firecrawl_depth11_20260403.py +++ /dev/null @@ -1,339 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -import uuid -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -FIRECRAWL_URL = "https://api.firecrawl.dev/v1/scrape" -SERVICE_SLUG = "firecrawl" -CAPABILITY_ID = "scrape.extract" -TARGET_URL = "https://example.com" -FORMATS = ["markdown"] -POST_GRANT_PROPAGATION_DELAY_SECONDS = 5 -ESTIMATE_AUTH_RETRY_ATTEMPTS = 4 -ESTIMATE_AUTH_RETRY_DELAY_SECONDS = 5 - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _looks_like_invalid_key(body: Any) -> bool: - if not isinstance(body, dict): - return False - detail = str(body.get("detail") or "").lower() - error = str(body.get("error") or "").lower() - return "invalid or expired rhumb api key" in detail or "invalid or expired rhumb api key" in error - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request(method, url, headers=headers, params=params, json=json_body) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label in {"🟒 Runtime-verified", "πŸ§ͺ Tester-generated"}: - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _firecrawl_data(body: Any) -> dict[str, Any]: - if not isinstance(body, dict): - return {} - data = body.get("data") - return data if isinstance(data, dict) else {} - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - short = uuid.uuid4().hex[:8] - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-firecrawl-depth11.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-03-firecrawl-depth11.json" - org_id = f"org_runtime_review_firecrawl_{stamp.lower()}_{short}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "capability_id": CAPABILITY_ID, - "target_url": TARGET_URL, - "formats": FORMATS, - "review_kind": "current_depth11", - "organization_id": org_id, - "artifact_path": str(artifact_path.relative_to(REPO_ROOT)), - "started_at": started_at.replace(microsecond=0).isoformat().replace("+00:00", "Z"), - "post_grant_delay_seconds": POST_GRANT_PROPAGATION_DELAY_SECONDS, - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-firecrawl-{stamp}", - starter_credits_cents=5000, - ) - payload["wallet_bootstrap"] = bootstrap - _json_dump(artifact_path, payload) - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-firecrawl-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Firecrawl depth-11 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "firecrawl", "depth11"], - ) - payload["review_agent"] = {"agent_id": agent_id, "organization_id": org_id} - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = {"status": "success", "access_id": access_id} - _json_dump(artifact_path, payload) - - await asyncio.sleep(POST_GRANT_PROPAGATION_DELAY_SECONDS) - - api_headers = {"X-Rhumb-Key": api_key} - firecrawl_key = os.environ["RHUMB_CREDENTIAL_FIRECRAWL_API_KEY"] - direct_headers = { - "Authorization": f"Bearer {firecrawl_key}", - "Content-Type": "application/json", - } - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - - estimate_attempts = 0 - estimate = HttpResult(status_code=0, body=None) - while estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS: - estimate_attempts += 1 - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - if not ( - estimate.status_code == 401 - and _looks_like_invalid_key(estimate.body) - and estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS - ): - break - await asyncio.sleep(ESTIMATE_AUTH_RETRY_DELAY_SECONDS) - - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body={"url": TARGET_URL, "formats": FORMATS}, - timeout=120.0, - ) - - direct = await _request_json( - "POST", - FIRECRAWL_URL, - headers=direct_headers, - json_body={"url": TARGET_URL, "formats": FORMATS}, - timeout=120.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - rhumb_data = _firecrawl_data(rhumb_upstream) - direct_data = _firecrawl_data(direct.body) - - rhumb_markdown = rhumb_data.get("markdown") if isinstance(rhumb_data, dict) else None - direct_markdown = direct_data.get("markdown") if isinstance(direct_data, dict) else None - rhumb_metadata = rhumb_data.get("metadata") if isinstance(rhumb_data, dict) else {} - direct_metadata = direct_data.get("metadata") if isinstance(direct_data, dict) else {} - - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "provider_used": rhumb_payload.get("provider_used"), - "upstream_status": rhumb_payload.get("upstream_status"), - "execution_id": rhumb_payload.get("execution_id"), - "rhumb_success": rhumb_upstream.get("success") if isinstance(rhumb_upstream, dict) else None, - "direct_success": direct.body.get("success") if isinstance(direct.body, dict) else None, - "rhumb_title": (rhumb_metadata or {}).get("title") if isinstance(rhumb_metadata, dict) else None, - "direct_title": (direct_metadata or {}).get("title") if isinstance(direct_metadata, dict) else None, - "rhumb_source_url": (rhumb_metadata or {}).get("sourceURL") if isinstance(rhumb_metadata, dict) else None, - "direct_source_url": (direct_metadata or {}).get("sourceURL") if isinstance(direct_metadata, dict) else None, - "rhumb_has_markdown": bool(rhumb_markdown), - "direct_has_markdown": bool(direct_markdown), - "markdown_prefix_match": (rhumb_markdown or "")[:120] == (direct_markdown or "")[:120], - "success_match": (rhumb_upstream.get("success") if isinstance(rhumb_upstream, dict) else None) - == (direct.body.get("success") if isinstance(direct.body, dict) else None) - == True, - "title_match": ((rhumb_metadata or {}).get("title") if isinstance(rhumb_metadata, dict) else None) - == ((direct_metadata or {}).get("title") if isinstance(direct_metadata, dict) else None), - "source_url_match": ((rhumb_metadata or {}).get("sourceURL") if isinstance(rhumb_metadata, dict) else None) - == ((direct_metadata or {}).get("sourceURL") if isinstance(direct_metadata, dict) else None), - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["success_match"] - and comparison["title_match"] - and comparison["source_url_match"] - and comparison["rhumb_has_markdown"] - and comparison["direct_has_markdown"] - and comparison["markdown_prefix_match"] - ) else "fail" - - payload.update( - { - "reviewed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": { - "status_code": estimate.status_code, - "body": estimate.body, - "attempts": estimate_attempts, - }, - "rhumb_execute": asdict(execute), - "direct_control": asdict(direct), - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Firecrawl depth-11 runtime pass failed; inspect artifact for parity details") - - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "Firecrawl: depth-11 runtime review confirms scrape.extract parity through Rhumb Resolve", - "--summary", "Fresh depth-11 runtime review passed for Firecrawl scrape.extract through Rhumb Resolve. Managed and direct executions matched on scrape success, metadata title, source URL, and markdown output for the same live target URL.", - "--evidence-title", "Firecrawl depth-11 runtime review parity check via Rhumb Resolve", - "--evidence-summary", "Fresh depth-11 runtime review passed for Firecrawl scrape.extract through Rhumb Resolve. Managed and direct executions matched on scrape success, metadata title, source URL, and markdown output.", - "--source-ref", f"runtime-review:firecrawl:{stamp}", - "--source-batch-id", f"runtime-review:firecrawl:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", rhumb_payload.get("execution_id") or "", - "--tag", "runtime_review", - "--tag", "firecrawl", - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--tag", "depth11", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["publication"] = publication - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["disable_agent"] = { - "status_code": 200 if disabled else 404, - "body": {"status": "success" if disabled else "not_found", "agent_id": agent_id}, - } - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_firecrawl_depth5_20260330.py b/scripts/runtime_review_firecrawl_depth5_20260330.py deleted file mode 100644 index 950c5b29..00000000 --- a/scripts/runtime_review_firecrawl_depth5_20260330.py +++ /dev/null @@ -1,299 +0,0 @@ -#!/usr/bin/env python3 -"""Firecrawl current-depth5 runtime review pass β€” 2026-03-30.""" -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -FIRECRAWL_URL = "https://api.firecrawl.dev/v1/scrape" -SERVICE_SLUG = "firecrawl" -CAPABILITY_ID = "scrape.extract" -TARGET_URL = "https://example.com" - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request(method, url, headers=headers, params=params, json=json_body) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-firecrawl-current-depth5.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-30-firecrawl-depth5.json" - org_id = f"org_runtime_review_firecrawl_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "capability_id": CAPABILITY_ID, - "target_url": TARGET_URL, - "review_kind": "current_depth5", - "organization_id": org_id, - "artifact_path": str(artifact_path), - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-firecrawl-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-firecrawl-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Firecrawl depth-5 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "firecrawl"], - ) - payload["review_agent"] = {"agent_id": agent_id, "organization_id": org_id} - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = {"status": "success", "access_id": access_id} - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - - # Direct Firecrawl control - firecrawl_key = os.environ["RHUMB_CREDENTIAL_FIRECRAWL_API_KEY"] - direct_headers = { - "Authorization": f"Bearer {firecrawl_key}", - "Content-Type": "application/json", - } - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - - # 1. Estimate - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - - # 2. Rhumb-managed execute - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body={"url": TARGET_URL, "formats": ["markdown"]}, - timeout=120.0, - ) - - # 3. Direct Firecrawl control - direct = await _request_json( - "POST", - FIRECRAWL_URL, - headers=direct_headers, - json_body={"url": TARGET_URL, "formats": ["markdown"]}, - timeout=120.0, - ) - - # Parse results - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - - # Firecrawl response shape: {success, data: {markdown, metadata: {title, ...}}} - rhumb_data = rhumb_upstream.get("data") if isinstance(rhumb_upstream, dict) else {} - direct_data = direct.body.get("data") if isinstance(direct.body, dict) else {} - - rhumb_success = rhumb_upstream.get("success") if isinstance(rhumb_upstream, dict) else None - direct_success = direct.body.get("success") if isinstance(direct.body, dict) else None - - rhumb_title = ((rhumb_data or {}).get("metadata") or {}).get("title") if isinstance(rhumb_data, dict) else None - direct_title = ((direct_data or {}).get("metadata") or {}).get("title") if isinstance(direct_data, dict) else None - - rhumb_has_markdown = bool((rhumb_data or {}).get("markdown")) if isinstance(rhumb_data, dict) else False - direct_has_markdown = bool((direct_data or {}).get("markdown")) if isinstance(direct_data, dict) else False - - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_provider_used": rhumb_payload.get("provider_used"), - "rhumb_upstream_status": rhumb_payload.get("upstream_status"), - "rhumb_success": rhumb_success, - "direct_success": direct_success, - "rhumb_title": rhumb_title, - "direct_title": direct_title, - "rhumb_has_markdown": rhumb_has_markdown, - "direct_has_markdown": direct_has_markdown, - "success_match": rhumb_success == direct_success == True, - "title_match": rhumb_title == direct_title, - "markdown_match": rhumb_has_markdown and direct_has_markdown, - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["success_match"] - and comparison["title_match"] - and comparison["markdown_match"] - ) else "fail" - - payload.update({ - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": {"status": estimate.status_code, "data": estimate.body}, - "rhumb": {"status": execute.status_code, "data": execute.body}, - "direct": {"status": direct.status_code, "data": direct.body}, - "comparison": comparison, - "verdict": verdict, - }) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Firecrawl depth-5 runtime pass failed; inspect artifact for parity details") - - execution_id = rhumb_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "Firecrawl: current-depth rerun confirms scrape.extract parity through Rhumb Resolve again", - "--summary", "Fresh current-depth runtime rerun passed for Firecrawl scrape.extract through Rhumb Resolve. Managed and direct executions matched on scrape success, page title, and markdown presence for the same target URL, lifting Firecrawl another layer above the callable review floor.", - "--evidence-title", "Firecrawl current-depth runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh current-depth runtime rerun passed for Firecrawl scrape.extract through Rhumb Resolve. Managed and direct executions matched on scrape success, page title, and markdown presence.", - "--source-ref", f"runtime-review:firecrawl:{stamp}", - "--source-batch-id", f"runtime-review:firecrawl:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "firecrawl", - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "service_slug": SERVICE_SLUG, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_firecrawl_depth6_20260331.py b/scripts/runtime_review_firecrawl_depth6_20260331.py deleted file mode 100644 index fc4db17a..00000000 --- a/scripts/runtime_review_firecrawl_depth6_20260331.py +++ /dev/null @@ -1,299 +0,0 @@ -#!/usr/bin/env python3 -"""Firecrawl current-depth6 runtime review pass β€” 2026-03-31.""" -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -FIRECRAWL_URL = "https://api.firecrawl.dev/v1/scrape" -SERVICE_SLUG = "firecrawl" -CAPABILITY_ID = "scrape.extract" -TARGET_URL = "https://example.com" - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request(method, url, headers=headers, params=params, json=json_body) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-firecrawl-current-depth6.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-31-firecrawl-depth6.json" - org_id = f"org_runtime_review_firecrawl_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "capability_id": CAPABILITY_ID, - "target_url": TARGET_URL, - "review_kind": "current_depth6", - "organization_id": org_id, - "artifact_path": str(artifact_path), - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-firecrawl-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-firecrawl-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Firecrawl depth-6 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "firecrawl"], - ) - payload["review_agent"] = {"agent_id": agent_id, "organization_id": org_id} - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = {"status": "success", "access_id": access_id} - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - - # Direct Firecrawl control - firecrawl_key = os.environ["RHUMB_CREDENTIAL_FIRECRAWL_API_KEY"] - direct_headers = { - "Authorization": f"Bearer {firecrawl_key}", - "Content-Type": "application/json", - } - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - - # 1. Estimate - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - - # 2. Rhumb-managed execute - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body={"url": TARGET_URL, "formats": ["markdown"]}, - timeout=120.0, - ) - - # 3. Direct Firecrawl control - direct = await _request_json( - "POST", - FIRECRAWL_URL, - headers=direct_headers, - json_body={"url": TARGET_URL, "formats": ["markdown"]}, - timeout=120.0, - ) - - # Parse results - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - - # Firecrawl response shape: {success, data: {markdown, metadata: {title, ...}}} - rhumb_data = rhumb_upstream.get("data") if isinstance(rhumb_upstream, dict) else {} - direct_data = direct.body.get("data") if isinstance(direct.body, dict) else {} - - rhumb_success = rhumb_upstream.get("success") if isinstance(rhumb_upstream, dict) else None - direct_success = direct.body.get("success") if isinstance(direct.body, dict) else None - - rhumb_title = ((rhumb_data or {}).get("metadata") or {}).get("title") if isinstance(rhumb_data, dict) else None - direct_title = ((direct_data or {}).get("metadata") or {}).get("title") if isinstance(direct_data, dict) else None - - rhumb_has_markdown = bool((rhumb_data or {}).get("markdown")) if isinstance(rhumb_data, dict) else False - direct_has_markdown = bool((direct_data or {}).get("markdown")) if isinstance(direct_data, dict) else False - - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_provider_used": rhumb_payload.get("provider_used"), - "rhumb_upstream_status": rhumb_payload.get("upstream_status"), - "rhumb_success": rhumb_success, - "direct_success": direct_success, - "rhumb_title": rhumb_title, - "direct_title": direct_title, - "rhumb_has_markdown": rhumb_has_markdown, - "direct_has_markdown": direct_has_markdown, - "success_match": rhumb_success == direct_success == True, - "title_match": rhumb_title == direct_title, - "markdown_match": rhumb_has_markdown and direct_has_markdown, - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["success_match"] - and comparison["title_match"] - and comparison["markdown_match"] - ) else "fail" - - payload.update({ - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": {"status": estimate.status_code, "data": estimate.body}, - "rhumb": {"status": execute.status_code, "data": execute.body}, - "direct": {"status": direct.status_code, "data": direct.body}, - "comparison": comparison, - "verdict": verdict, - }) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Firecrawl depth-6 runtime pass failed; inspect artifact for parity details") - - execution_id = rhumb_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "Firecrawl: current-depth rerun confirms scrape.extract parity through Rhumb Resolve at depth 6", - "--summary", "Fresh current-depth runtime rerun passed for Firecrawl scrape.extract through Rhumb Resolve. Managed and direct executions matched on scrape success, page title, and markdown presence for the same target URL, lifting Firecrawl from claim-safe depth 5 to 6 in the callable review rotation.", - "--evidence-title", "Firecrawl current-depth runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh current-depth runtime rerun passed for Firecrawl scrape.extract through Rhumb Resolve. Managed and direct executions matched on scrape success, page title, and markdown presence.", - "--source-ref", f"runtime-review:firecrawl:{stamp}", - "--source-batch-id", f"runtime-review:firecrawl:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "firecrawl", - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "service_slug": SERVICE_SLUG, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_github_depth10_20260402.py b/scripts/runtime_review_github_depth10_20260402.py deleted file mode 100644 index 2bac0223..00000000 --- a/scripts/runtime_review_github_depth10_20260402.py +++ /dev/null @@ -1,363 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -import uuid -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -SERVICE_SLUG = "github" -CAPABILITY_ID = "social.get_profile" -TARGET_USERNAME = "torvalds" -POST_GRANT_PROPAGATION_DELAY_SECONDS = 5 -ESTIMATE_AUTH_RETRY_ATTEMPTS = 4 -ESTIMATE_AUTH_RETRY_DELAY_SECONDS = 5 -DIRECT_URL = f"https://api.github.com/users/{TARGET_USERNAME}" -PARITY_FIELDS = [ - "login", - "id", - "type", - "site_admin", - "name", - "company", - "blog", - "location", - "public_repos", - "followers", - "following", - "created_at", -] - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _looks_like_invalid_key(body: Any) -> bool: - if not isinstance(body, dict): - return False - detail = str(body.get("detail") or "").lower() - error = str(body.get("error") or "").lower() - return "invalid or expired rhumb api key" in detail or "invalid or expired rhumb api key" in error - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request(method, url, headers=headers, params=params, json=json_body) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label in {"🟒 Runtime-verified", "πŸ§ͺ Tester-generated"}: - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _get_github_token() -> str | None: - for env_key in ("RHUMB_CREDENTIAL_GITHUB_TOKEN", "GITHUB_TOKEN", "GH_TOKEN"): - value = os.environ.get(env_key) - if value: - return value - - commands = [ - [ - "sop", - "item", - "get", - "GitHub PAT (supertrained)", - "--vault", - "OpenClaw Agents", - "--fields", - "credential", - "--reveal", - ], - ["gh", "auth", "token"], - ] - for cmd in commands: - try: - proc = subprocess.run(cmd, capture_output=True, text=True, timeout=30, check=False) - except Exception: - continue - if proc.returncode == 0 and proc.stdout.strip(): - return proc.stdout.strip() - return None - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - short = uuid.uuid4().hex[:8] - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-github-depth10.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-02-github-depth10.json" - org_id = f"org_runtime_review_github_{stamp.lower()}_{short}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "capability_id": CAPABILITY_ID, - "target_username": TARGET_USERNAME, - "direct_url": DIRECT_URL, - "review_kind": "current_depth10", - "organization_id": org_id, - "artifact_path": str(artifact_path.relative_to(REPO_ROOT)), - "started_at": started_at.replace(microsecond=0).isoformat().replace("+00:00", "Z"), - "parity_fields": PARITY_FIELDS, - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-github-{stamp}", - starter_credits_cents=5000, - ) - payload["wallet_bootstrap"] = bootstrap - _json_dump(artifact_path, payload) - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-github-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for GitHub depth-10 runtime pass", - tags=["runtime_review", "phase3", "current_pass", SERVICE_SLUG, "depth10"], - ) - payload["review_agent"] = {"agent_id": agent_id, "organization_id": org_id} - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = {"status": "success", "access_id": access_id} - payload["post_grant_delay_seconds"] = POST_GRANT_PROPAGATION_DELAY_SECONDS - _json_dump(artifact_path, payload) - - await asyncio.sleep(POST_GRANT_PROPAGATION_DELAY_SECONDS) - - api_headers = {"X-Rhumb-Key": api_key} - github_token = _get_github_token() - direct_headers = { - "Accept": "application/vnd.github+json", - "User-Agent": "rhumb-runtime-review/1.0", - } - if github_token: - direct_headers["Authorization"] = f"Bearer {github_token}" - payload["direct_auth_mode"] = "token" if github_token else "unauthenticated_public_read" - _json_dump(artifact_path, payload) - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - - estimate_attempts = 0 - estimate = HttpResult(status_code=0, body=None) - while estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS: - estimate_attempts += 1 - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "byo"}, - timeout=60.0, - ) - if not ( - estimate.status_code == 401 - and _looks_like_invalid_key(estimate.body) - and estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS - ): - break - await asyncio.sleep(ESTIMATE_AUTH_RETRY_DELAY_SECONDS) - - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - json_body={ - "provider": SERVICE_SLUG, - "credential_mode": "byo", - "method": "GET", - "path": f"/users/{TARGET_USERNAME}", - "interface": "runtime_review", - }, - timeout=120.0, - ) - direct = await _request_json( - "GET", - DIRECT_URL, - headers=direct_headers, - timeout=120.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - - rhumb_fields = {field: rhumb_upstream.get(field) if isinstance(rhumb_upstream, dict) else None for field in PARITY_FIELDS} - direct_fields = {field: direct.body.get(field) if isinstance(direct.body, dict) else None for field in PARITY_FIELDS} - field_matches = {field: rhumb_fields.get(field) == direct_fields.get(field) for field in PARITY_FIELDS} - - observed_at = _iso_now() - fresh_until = ( - datetime.fromisoformat(observed_at.replace("Z", "+00:00")) + timedelta(days=30) - ).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "provider_used": rhumb_payload.get("provider_used"), - "upstream_status": rhumb_payload.get("upstream_status"), - "execution_id": rhumb_payload.get("execution_id"), - "rhumb_fields": rhumb_fields, - "direct_fields": direct_fields, - "field_matches": field_matches, - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 200 - and all(field_matches.values()) - ) else "fail" - - payload.update( - { - "reviewed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": { - "status_code": estimate.status_code, - "body": estimate.body, - "attempts": estimate_attempts, - }, - "rhumb_execute": asdict(execute), - "direct": asdict(direct), - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("GitHub depth-10 runtime pass failed; inspect artifact for parity details") - - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "GitHub: depth-10 runtime review confirms social.get_profile parity through Rhumb Resolve", - "--summary", "Fresh depth-10 runtime review passed for GitHub social.get_profile through Rhumb Resolve. Managed and direct executions matched on identity, company/location metadata, follow counts, and account timestamps for the same live public profile.", - "--evidence-title", "GitHub depth-10 runtime review parity check via Rhumb Resolve", - "--evidence-summary", "Fresh depth-10 runtime review passed for GitHub social.get_profile through Rhumb Resolve. Managed and direct executions matched on identity, company/location metadata, follow counts, and account timestamps for the same live public profile.", - "--source-ref", f"runtime-review:github:{stamp}", - "--source-batch-id", f"runtime-review:github:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", rhumb_payload.get("execution_id") or "", - "--tag", "runtime_review", - "--tag", SERVICE_SLUG, - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--tag", "depth10", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["publication"] = publication - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["disable_agent"] = { - "status_code": 200 if disabled else 404, - "body": {"status": "success" if disabled else "not_found", "agent_id": agent_id}, - } - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_github_depth3_20260331.py b/scripts/runtime_review_github_depth3_20260331.py deleted file mode 100644 index cda87016..00000000 --- a/scripts/runtime_review_github_depth3_20260331.py +++ /dev/null @@ -1,287 +0,0 @@ -#!/usr/bin/env python3 -"""GitHub current-depth3 runtime review pass β€” 2026-03-31.""" -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -GITHUB_URL = "https://api.github.com/users/supertrained" -SERVICE_SLUG = "github" -CAPABILITY_ID = "social.get_profile" -TARGET_USERNAME = "supertrained" - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request(method, url, headers=headers, params=params, json=json_body) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-github-depth3.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-31-github-depth3.json" - org_id = f"org_runtime_review_github_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "capability_id": CAPABILITY_ID, - "target_username": TARGET_USERNAME, - "review_kind": "current_depth3", - "organization_id": org_id, - "artifact_path": str(artifact_path), - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-github-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-github-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for GitHub depth-3 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "github"], - ) - payload["review_agent"] = {"agent_id": agent_id, "organization_id": org_id} - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = {"status": "success", "access_id": access_id} - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - direct_headers = { - "Accept": "application/vnd.github+json", - "User-Agent": "rhumb-runtime-review/1.0", - } - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "byo"}, - timeout=60.0, - ) - - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - json_body={ - "provider": SERVICE_SLUG, - "credential_mode": "byo", - "method": "GET", - "path": f"/users/{TARGET_USERNAME}", - "interface": "runtime_review", - }, - timeout=120.0, - ) - - direct = await _request_json( - "GET", - GITHUB_URL, - headers=direct_headers, - timeout=120.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_provider_used": rhumb_payload.get("provider_used"), - "rhumb_upstream_status": rhumb_payload.get("upstream_status"), - "rhumb_login": rhumb_upstream.get("login") if isinstance(rhumb_upstream, dict) else None, - "direct_login": direct.body.get("login") if isinstance(direct.body, dict) else None, - "rhumb_public_repos": rhumb_upstream.get("public_repos") if isinstance(rhumb_upstream, dict) else None, - "direct_public_repos": direct.body.get("public_repos") if isinstance(direct.body, dict) else None, - "rhumb_created_at": rhumb_upstream.get("created_at") if isinstance(rhumb_upstream, dict) else None, - "direct_created_at": direct.body.get("created_at") if isinstance(direct.body, dict) else None, - "rhumb_updated_at": rhumb_upstream.get("updated_at") if isinstance(rhumb_upstream, dict) else None, - "direct_updated_at": direct.body.get("updated_at") if isinstance(direct.body, dict) else None, - "login_match": (rhumb_upstream.get("login") if isinstance(rhumb_upstream, dict) else None) == (direct.body.get("login") if isinstance(direct.body, dict) else None), - "public_repos_match": (rhumb_upstream.get("public_repos") if isinstance(rhumb_upstream, dict) else None) == (direct.body.get("public_repos") if isinstance(direct.body, dict) else None), - "created_at_match": (rhumb_upstream.get("created_at") if isinstance(rhumb_upstream, dict) else None) == (direct.body.get("created_at") if isinstance(direct.body, dict) else None), - "updated_at_match": (rhumb_upstream.get("updated_at") if isinstance(rhumb_upstream, dict) else None) == (direct.body.get("updated_at") if isinstance(direct.body, dict) else None), - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["login_match"] - and comparison["public_repos_match"] - and comparison["created_at_match"] - and comparison["updated_at_match"] - ) else "fail" - - payload.update({ - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": {"status": estimate.status_code, "data": estimate.body}, - "rhumb": {"status": execute.status_code, "data": execute.body}, - "direct": {"status": direct.status_code, "data": direct.body}, - "comparison": comparison, - "verdict": verdict, - }) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("GitHub depth-3 runtime pass failed; inspect artifact for parity details") - - execution_id = rhumb_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "GitHub: current-depth rerun confirms social.get_profile parity through Rhumb Resolve at depth 3", - "--summary", "Fresh current-depth runtime rerun passed for GitHub social.get_profile through Rhumb Resolve. Rhumb and direct GitHub control matched on login identity, public repo count, and account timestamps for the same public profile, lifting GitHub from claim-safe depth 2 to 3 in the callable rotation.", - "--evidence-title", "GitHub current-depth runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh current-depth runtime rerun passed for GitHub social.get_profile through Rhumb Resolve. Rhumb and direct GitHub control matched on login identity, public repo count, and account timestamps for the same public profile.", - "--source-ref", f"runtime-review:github:{stamp}", - "--source-batch-id", f"runtime-review:github:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "github", - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "service_slug": SERVICE_SLUG, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_github_depth4_20260401.py b/scripts/runtime_review_github_depth4_20260401.py deleted file mode 100644 index 3893efc2..00000000 --- a/scripts/runtime_review_github_depth4_20260401.py +++ /dev/null @@ -1,218 +0,0 @@ -#!/usr/bin/env python3 -""" -GitHub depth-4 runtime review pass. - -Previous passes used social.get_profile on supertrained via v1 execute. -This pass uses the proxy directly on GET /users/octocat to prove -a different target profile and broader response parity. -""" - -import json -import os -import subprocess -import sys -from datetime import datetime, timezone - -API_BASE = os.environ.get("RHUMB_API_BASE", "https://api.rhumb.dev") -TARGET_USER = "octocat" - - -def get_admin_key(): - key = os.environ.get("RHUMB_ADMIN_KEY") - if key: - return key - try: - result = subprocess.run( - ["sop", "item", "get", "Rhumb Admin Secret (Railway)", "--vault", "OpenClaw Agents", - "--fields", "credential", "--reveal"], - capture_output=True, text=True, timeout=30 - ) - if result.returncode == 0 and result.stdout.strip(): - return result.stdout.strip() - except Exception: - pass - raise RuntimeError("No admin key available") - - -def get_github_token(): - token = os.environ.get("RHUMB_CREDENTIAL_GITHUB_TOKEN") - if token: - return token - try: - result = subprocess.run( - ["sop", "item", "get", "GitHub PAT (supertrained)", "--vault", "OpenClaw Agents", - "--fields", "credential", "--reveal"], - capture_output=True, text=True, timeout=30 - ) - if result.returncode == 0 and result.stdout.strip(): - return result.stdout.strip() - except Exception: - pass - # Try gh cli token - try: - result = subprocess.run(["gh", "auth", "token"], capture_output=True, text=True, timeout=10) - if result.returncode == 0 and result.stdout.strip(): - return result.stdout.strip() - except Exception: - pass - raise RuntimeError("No GitHub token available") - - -def api(method, path, headers=None, json_body=None): - import urllib.request - url = f"{API_BASE}{path}" - data = json.dumps(json_body).encode() if json_body else None - req = urllib.request.Request(url, data=data, method=method) - req.add_header("Content-Type", "application/json") - if headers: - for k, v in headers.items(): - if v is not None: - req.add_header(k, str(v)) - try: - with urllib.request.urlopen(req, timeout=30) as resp: - return resp.status, json.loads(resp.read()) - except urllib.error.HTTPError as e: - body = e.read() - try: - return e.code, json.loads(body) - except Exception: - return e.code, {"raw": body.decode(errors="replace")} - - -def main(): - ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") - admin_key = get_admin_key() - github_token = get_github_token() - admin_headers = {"X-Rhumb-Admin-Key": admin_key} - - # 1. Create temp agent + grant github access - print("Creating temp agent...") - _, agent_resp = api("POST", "/v1/admin/agents", admin_headers, { - "organization_id": "org_rhumb_internal", - "name": f"github-depth4-reviewer-{ts}", - }) - agent_id = agent_resp.get("agent_id") - api_key = agent_resp.get("api_key") - print(f"Agent: {agent_id}") - - if not api_key: - print(f"ERROR: No API key. Response: {json.dumps(agent_resp, indent=2)}") - return 1 - - print("Granting github access...") - grant_status, grant_resp = api("POST", f"/v1/admin/agents/{agent_id}/grant-access", admin_headers, { - "service": "github" - }) - print(f"Grant: {grant_status} - {grant_resp.get('status', grant_resp.get('error', 'unknown'))}") - if grant_status not in (200, 201): - print(f"Grant failed: {json.dumps(grant_resp, indent=2)}") - return 1 - - agent_headers = {"X-Rhumb-Key": api_key} - - # 2. Managed: GET /users/octocat via proxy - print(f"\n--- Managed: GET /users/{TARGET_USER} via Rhumb proxy ---") - managed_status, managed_resp = api("POST", "/v1/proxy/", agent_headers, { - "service": "github", - "method": "GET", - "path": f"/users/{TARGET_USER}" - }) - print(f"HTTP status: {managed_status}") - - managed_body = managed_resp.get("body", {}) - managed_fields = { - "login": managed_body.get("login"), - "id": managed_body.get("id"), - "type": managed_body.get("type"), - "site_admin": managed_body.get("site_admin"), - "name": managed_body.get("name"), - "company": managed_body.get("company"), - "blog": managed_body.get("blog"), - "location": managed_body.get("location"), - "public_repos": managed_body.get("public_repos"), - "created_at": managed_body.get("created_at"), - } - print(f"Managed fields: {json.dumps(managed_fields, indent=2)}") - - # 3. Direct: GET /users/octocat - print(f"\n--- Direct: GET https://api.github.com/users/{TARGET_USER} ---") - import urllib.request - direct_req = urllib.request.Request( - f"https://api.github.com/users/{TARGET_USER}", - method="GET" - ) - direct_req.add_header("Authorization", f"Bearer {github_token}") - direct_req.add_header("Accept", "application/vnd.github+json") - direct_req.add_header("User-Agent", "rhumb-runtime-review") - try: - with urllib.request.urlopen(direct_req, timeout=30) as resp: - direct_status = resp.status - direct_data = json.loads(resp.read()) - except urllib.error.HTTPError as e: - direct_status = e.code - direct_data = json.loads(e.read()) - - direct_fields = { - "login": direct_data.get("login"), - "id": direct_data.get("id"), - "type": direct_data.get("type"), - "site_admin": direct_data.get("site_admin"), - "name": direct_data.get("name"), - "company": direct_data.get("company"), - "blog": direct_data.get("blog"), - "location": direct_data.get("location"), - "public_repos": direct_data.get("public_repos"), - "created_at": direct_data.get("created_at"), - } - print(f"Direct status: {direct_status}") - print(f"Direct fields: {json.dumps(direct_fields, indent=2)}") - - # 4. Parity check - print("\n--- Parity check ---") - parity_ok = True - for field in ["login", "id", "type", "site_admin", "name", "company", "blog", "location", "public_repos", "created_at"]: - mv = managed_fields.get(field) - dv = direct_fields.get(field) - match = mv == dv - print(f" {field}: managed={mv} direct={dv} match={match}") - if not match: - parity_ok = False - - # 5. Build artifact - artifact = { - "pass_id": f"github-depth4-{ts}", - "provider": "github", - "capability": f"GET /users/{TARGET_USER} (safe read)", - "timestamp": ts, - "managed": { - "http_status": managed_status, - "fields": managed_fields, - }, - "direct": { - "http_status": direct_status, - "fields": direct_fields - }, - "parity": parity_ok, - "parity_fields_checked": list(managed_fields.keys()), - "agent_id": agent_id, - "org_id": "org_rhumb_internal" - } - - out = f"artifacts/runtime-review-pass-{ts}-github-depth4.json" - if len(sys.argv) > 2 and sys.argv[1] == "--json-out": - out = sys.argv[2] - os.makedirs(os.path.dirname(out) or ".", exist_ok=True) - with open(out, "w") as f: - json.dump(artifact, f, indent=2) - print(f"\nArtifact: {out}") - print(f"Parity: {'PASS' if parity_ok else 'FAIL'}") - - # 6. Disable temp agent - api("PUT", f"/v1/admin/agents/{agent_id}", admin_headers, {"enabled": False}) - print(f"Agent {agent_id} disabled") - - return 0 if parity_ok else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/runtime_review_google_ai_depth11_20260403.py b/scripts/runtime_review_google_ai_depth11_20260403.py deleted file mode 100755 index 0d8cab0a..00000000 --- a/scripts/runtime_review_google_ai_depth11_20260403.py +++ /dev/null @@ -1,348 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -import uuid -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any -from urllib.parse import quote - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -GOOGLE_BASE_URL = "https://generativelanguage.googleapis.com/v1beta" -SERVICE_SLUG = "google-ai" -CAPABILITY_ID = "ai.generate_text" -MODEL = "gemini-2.5-flash" -SENTINEL = "RHUMB_GOOGLE_AI_20260403_DEPTH11" -INPUT_BODY = { - "model": MODEL, - "contents": [ - { - "parts": [ - { - "text": f"Reply with exactly: {SENTINEL}", - } - ] - } - ], - "generationConfig": {"temperature": 0}, -} -POST_GRANT_PROPAGATION_DELAY_SECONDS = 5 -ESTIMATE_AUTH_RETRY_ATTEMPTS = 4 -ESTIMATE_AUTH_RETRY_DELAY_SECONDS = 5 - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _looks_like_invalid_key(body: Any) -> bool: - if not isinstance(body, dict): - return False - detail = str(body.get("detail") or "").lower() - error = str(body.get("error") or "").lower() - return "invalid or expired rhumb api key" in detail or "invalid or expired rhumb api key" in error - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request(method, url, headers=headers, params=params, json=json_body) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label in {"🟒 Runtime-verified", "πŸ§ͺ Tester-generated"}: - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _extract_text(body: Any) -> str | None: - if not isinstance(body, dict): - return None - candidates = body.get("candidates") - if not isinstance(candidates, list) or not candidates: - return None - candidate = candidates[0] - if not isinstance(candidate, dict): - return None - content = candidate.get("content") - if not isinstance(content, dict): - return None - parts = content.get("parts") - if not isinstance(parts, list) or not parts: - return None - part = parts[0] - if not isinstance(part, dict): - return None - text = part.get("text") - return text if isinstance(text, str) else None - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - short = uuid.uuid4().hex[:8] - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-google-ai-depth11.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-03-google-ai-depth11.json" - org_id = f"org_runtime_review_google_ai_{stamp.lower()}_{short}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "capability_id": CAPABILITY_ID, - "model": MODEL, - "sentinel": SENTINEL, - "review_kind": "current_depth11", - "organization_id": org_id, - "artifact_path": str(artifact_path.relative_to(REPO_ROOT)), - "started_at": started_at.replace(microsecond=0).isoformat().replace("+00:00", "Z"), - "post_grant_delay_seconds": POST_GRANT_PROPAGATION_DELAY_SECONDS, - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-google-ai-{stamp}", - starter_credits_cents=5000, - ) - payload["wallet_bootstrap"] = bootstrap - _json_dump(artifact_path, payload) - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-google-ai-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Google AI depth-11 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "google-ai", "depth11"], - ) - payload["review_agent"] = {"agent_id": agent_id, "organization_id": org_id} - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = {"status": "success", "access_id": access_id} - _json_dump(artifact_path, payload) - - await asyncio.sleep(POST_GRANT_PROPAGATION_DELAY_SECONDS) - - api_headers = {"X-Rhumb-Key": api_key} - google_api_key = os.environ["RHUMB_CREDENTIAL_GOOGLE_AI_API_KEY"] - direct_url = f"{GOOGLE_BASE_URL}/models/{quote(MODEL, safe='')}:generateContent" - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - - estimate_attempts = 0 - estimate = HttpResult(status_code=0, body=None) - while estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS: - estimate_attempts += 1 - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - if not ( - estimate.status_code == 401 - and _looks_like_invalid_key(estimate.body) - and estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS - ): - break - await asyncio.sleep(ESTIMATE_AUTH_RETRY_DELAY_SECONDS) - - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body=INPUT_BODY, - timeout=120.0, - ) - direct = await _request_json( - "POST", - direct_url, - headers={"Content-Type": "application/json"}, - params={"key": google_api_key}, - json_body={ - "contents": INPUT_BODY["contents"], - "generationConfig": INPUT_BODY["generationConfig"], - }, - timeout=120.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - rhumb_text = _extract_text(rhumb_upstream) - direct_text = _extract_text(direct.body) - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "provider_used": rhumb_payload.get("provider_used"), - "upstream_status": rhumb_payload.get("upstream_status"), - "execution_id": rhumb_payload.get("execution_id"), - "rhumb_model_version": rhumb_upstream.get("modelVersion") if isinstance(rhumb_upstream, dict) else None, - "direct_model_version": direct.body.get("modelVersion") if isinstance(direct.body, dict) else None, - "rhumb_text": rhumb_text, - "direct_text": direct_text, - "text_match": rhumb_text == direct_text, - "sentinel_match": rhumb_text == SENTINEL == direct_text, - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["text_match"] - and comparison["sentinel_match"] - ) else "fail" - - payload.update( - { - "reviewed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": { - "status_code": estimate.status_code, - "body": estimate.body, - "attempts": estimate_attempts, - }, - "rhumb_execute": asdict(execute), - "direct": asdict(direct), - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Google AI depth-11 runtime pass failed; inspect artifact for parity details") - - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "Google AI: depth-11 runtime review confirms ai.generate_text parity through Rhumb Resolve", - "--summary", "Fresh depth-11 runtime review passed for Google AI ai.generate_text through Rhumb Resolve. Managed and direct executions matched on exact output text for the same Gemini 2.5 Flash prompt.", - "--evidence-title", "Google AI depth-11 runtime review parity check via Rhumb Resolve", - "--evidence-summary", "Fresh depth-11 runtime review passed for Google AI ai.generate_text through Rhumb Resolve. Managed and direct executions matched on exact output text for the same Gemini 2.5 Flash prompt.", - "--source-ref", f"runtime-review:google-ai:{stamp}", - "--source-batch-id", f"runtime-review:google-ai:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", rhumb_payload.get("execution_id") or "", - "--tag", "runtime_review", - "--tag", "google-ai", - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--tag", "depth11", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["publication"] = publication - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["disable_agent"] = { - "status_code": 200 if disabled else 404, - "body": {"status": "success" if disabled else "not_found", "agent_id": agent_id}, - } - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_google_ai_depth5_20260330.py b/scripts/runtime_review_google_ai_depth5_20260330.py deleted file mode 100644 index fc22f57a..00000000 --- a/scripts/runtime_review_google_ai_depth5_20260330.py +++ /dev/null @@ -1,330 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any -from urllib.parse import quote - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -GOOGLE_BASE_URL = "https://generativelanguage.googleapis.com/v1beta" -SERVICE_SLUG = "google-ai" -MODEL = "gemini-2.5-flash" -SENTINEL = "RHUMB_GOOGLE_AI_20260330_DEPTH5" -INPUT_BODY = { - "model": MODEL, - "contents": [ - { - "parts": [ - { - "text": f"Reply with exactly: {SENTINEL}", - } - ] - } - ], - "generationConfig": {"temperature": 0}, -} - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _extract_text(body: Any) -> str | None: - if not isinstance(body, dict): - return None - candidates = body.get("candidates") - if not isinstance(candidates, list) or not candidates: - return None - candidate = candidates[0] - if not isinstance(candidate, dict): - return None - content = candidate.get("content") - if not isinstance(content, dict): - return None - parts = content.get("parts") - if not isinstance(parts, list) or not parts: - return None - part = parts[0] - if not isinstance(part, dict): - return None - text = part.get("text") - return text if isinstance(text, str) else None - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-google-ai-current-depth5.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-30-google-ai-depth5.json" - org_id = f"org_runtime_review_google_ai_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "model": MODEL, - "sentinel": SENTINEL, - "review_kind": "current_depth5", - "organization_id": org_id, - "artifact_path": str(artifact_path), - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-google-ai-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-google-ai-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Google AI depth-5 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "google-ai"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = { - "status": "success", - "access_id": access_id, - } - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - google_api_key = os.environ["RHUMB_CREDENTIAL_GOOGLE_AI_API_KEY"] - direct_url = f"{GOOGLE_BASE_URL}/models/{quote(MODEL, safe='')}:generateContent" - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/ai.generate_text/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/ai.generate_text/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body=INPUT_BODY, - timeout=120.0, - ) - direct = await _request_json( - "POST", - direct_url, - headers={"Content-Type": "application/json"}, - params={"key": google_api_key}, - json_body={ - "contents": INPUT_BODY["contents"], - "generationConfig": INPUT_BODY["generationConfig"], - }, - timeout=120.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - rhumb_text = _extract_text(rhumb_upstream) - direct_text = _extract_text(direct.body) - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_provider_used": rhumb_payload.get("provider_used"), - "rhumb_upstream_status": rhumb_payload.get("upstream_status"), - "rhumb_model_version": rhumb_upstream.get("modelVersion") if isinstance(rhumb_upstream, dict) else None, - "direct_model_version": direct.body.get("modelVersion") if isinstance(direct.body, dict) else None, - "rhumb_text": rhumb_text, - "direct_text": direct_text, - "text_match": rhumb_text == direct_text, - "sentinel_match": rhumb_text == SENTINEL == direct_text, - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["text_match"] - and comparison["sentinel_match"] - ) else "fail" - - payload.update( - { - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": {"status": estimate.status_code, "data": estimate.body}, - "rhumb": { - "status": execute.status_code, - "data": execute.body, - }, - "direct": { - "status": direct.status_code, - "data": direct.body, - }, - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Google AI depth-5 runtime pass failed; inspect artifact for parity details") - - execution_id = rhumb_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "Google AI: current-depth rerun confirms ai.generate_text parity through Rhumb Resolve again", - "--summary", "Fresh current-depth runtime rerun passed for Google AI ai.generate_text through Rhumb Resolve. Managed and direct executions matched on exact output text for the same Gemini 2.5 Flash prompt, lifting the public runtime-backed review stack another layer.", - "--evidence-title", "Google AI current-depth runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh current-depth runtime rerun passed for Google AI ai.generate_text through Rhumb Resolve. Managed and direct executions matched on exact output text for the same Gemini 2.5 Flash prompt.", - "--source-ref", f"runtime-review:google-ai:{stamp}", - "--source-batch-id", f"runtime-review:google-ai:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "google-ai", - "--tag", "ai.generate_text", - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "service_slug": SERVICE_SLUG, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_google_ai_depth6_20260331.py b/scripts/runtime_review_google_ai_depth6_20260331.py deleted file mode 100644 index c3d112a6..00000000 --- a/scripts/runtime_review_google_ai_depth6_20260331.py +++ /dev/null @@ -1,330 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any -from urllib.parse import quote - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -GOOGLE_BASE_URL = "https://generativelanguage.googleapis.com/v1beta" -SERVICE_SLUG = "google-ai" -MODEL = "gemini-2.5-flash" -SENTINEL = "RHUMB_GOOGLE_AI_20260331_DEPTH6" -INPUT_BODY = { - "model": MODEL, - "contents": [ - { - "parts": [ - { - "text": f"Reply with exactly: {SENTINEL}", - } - ] - } - ], - "generationConfig": {"temperature": 0}, -} - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _extract_text(body: Any) -> str | None: - if not isinstance(body, dict): - return None - candidates = body.get("candidates") - if not isinstance(candidates, list) or not candidates: - return None - candidate = candidates[0] - if not isinstance(candidate, dict): - return None - content = candidate.get("content") - if not isinstance(content, dict): - return None - parts = content.get("parts") - if not isinstance(parts, list) or not parts: - return None - part = parts[0] - if not isinstance(part, dict): - return None - text = part.get("text") - return text if isinstance(text, str) else None - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-google-ai-current-depth6.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-31-google-ai-depth6.json" - org_id = f"org_runtime_review_google_ai_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "model": MODEL, - "sentinel": SENTINEL, - "review_kind": "current_depth6", - "organization_id": org_id, - "artifact_path": str(artifact_path), - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-google-ai-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-google-ai-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Google AI depth-6 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "google-ai"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = { - "status": "success", - "access_id": access_id, - } - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - google_api_key = os.environ["RHUMB_CREDENTIAL_GOOGLE_AI_API_KEY"] - direct_url = f"{GOOGLE_BASE_URL}/models/{quote(MODEL, safe='')}:generateContent" - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/ai.generate_text/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/ai.generate_text/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body=INPUT_BODY, - timeout=120.0, - ) - direct = await _request_json( - "POST", - direct_url, - headers={"Content-Type": "application/json"}, - params={"key": google_api_key}, - json_body={ - "contents": INPUT_BODY["contents"], - "generationConfig": INPUT_BODY["generationConfig"], - }, - timeout=120.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - rhumb_text = _extract_text(rhumb_upstream) - direct_text = _extract_text(direct.body) - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_provider_used": rhumb_payload.get("provider_used"), - "rhumb_upstream_status": rhumb_payload.get("upstream_status"), - "rhumb_model_version": rhumb_upstream.get("modelVersion") if isinstance(rhumb_upstream, dict) else None, - "direct_model_version": direct.body.get("modelVersion") if isinstance(direct.body, dict) else None, - "rhumb_text": rhumb_text, - "direct_text": direct_text, - "text_match": rhumb_text == direct_text, - "sentinel_match": rhumb_text == SENTINEL == direct_text, - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["text_match"] - and comparison["sentinel_match"] - ) else "fail" - - payload.update( - { - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": {"status": estimate.status_code, "data": estimate.body}, - "rhumb": { - "status": execute.status_code, - "data": execute.body, - }, - "direct": { - "status": direct.status_code, - "data": direct.body, - }, - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Google AI depth-6 runtime pass failed; inspect artifact for parity details") - - execution_id = rhumb_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "Google AI: depth-6 freshness rerun confirms ai.generate_text parity through Rhumb Resolve", - "--summary", "Fresh depth-6 runtime rerun passed for Google AI ai.generate_text through Rhumb Resolve. Managed and direct executions matched on exact output text for the same Gemini 2.5 Flash prompt, lifting the public runtime-backed review stack from depth 5 to depth 6.", - "--evidence-title", "Google AI depth-6 runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh depth-6 runtime rerun passed for Google AI ai.generate_text through Rhumb Resolve. Managed and direct executions matched on exact output text for the same Gemini 2.5 Flash prompt.", - "--source-ref", f"runtime-review:google-ai:{stamp}", - "--source-batch-id", f"runtime-review:google-ai:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "google-ai", - "--tag", "ai.generate_text", - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "service_slug": SERVICE_SLUG, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_pdl_fix_verify_20260401.py b/scripts/runtime_review_pdl_fix_verify_20260401.py deleted file mode 100644 index 7986a47c..00000000 --- a/scripts/runtime_review_pdl_fix_verify_20260401.py +++ /dev/null @@ -1,233 +0,0 @@ -#!/usr/bin/env python3 -"""Production PDL fix-verification rerun via funded temp-org rail. - -Run with the linked Railway production environment and the repo venv, e.g. - railway run packages/api/.venv/bin/python scripts/runtime_review_pdl_fix_verify_20260401.py - -Why this version exists: -- Mission 0 needs a real canonical-path rerun after commit 94c8df8. -- Admin-created agents under org_rhumb_internal can hit a Rhumb x402 billing gate before - reaching PDL, which hides the provider result. -- A funded temp org plus a short propagation delay after access grant gives a clean Phase 3 rail. -""" - -from __future__ import annotations - -import asyncio -import json -import os -import sys -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = os.environ.get("RHUMB_API_BASE", "https://api.rhumb.dev/v1") -PROVIDER = "people-data-labs" -CAPABILITY_ID = "data.enrich_person" -PROFILE_URL = "https://www.linkedin.com/in/satyanadella/" -POST_GRANT_PROPAGATION_DELAY_SECONDS = 5 -ESTIMATE_AUTH_RETRY_ATTEMPTS = 4 -ESTIMATE_AUTH_RETRY_DELAY_SECONDS = 5 - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> tuple[int, Any]: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request(method, url, headers=headers, params=params, json=json_body) - try: - return response.status_code, response.json() - except Exception: - return response.status_code, response.text - - -def _looks_like_invalid_key(body: Any) -> bool: - if not isinstance(body, dict): - return False - detail = str(body.get("detail") or "").lower() - error = str(body.get("error") or "").lower() - return "invalid or expired rhumb api key" in detail or "invalid or expired rhumb api key" in error - - -def _sample_fields(payload: dict[str, Any] | None) -> dict[str, Any]: - payload = payload or {} - return { - "full_name": payload.get("full_name"), - "job_title": payload.get("job_title"), - "job_company_name": payload.get("job_company_name"), - "linkedin_url": payload.get("linkedin_url"), - } - - -def _write_artifact(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -async def main() -> int: - stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-pdl-fix-verify-20260401b.json" - org_id = f"org_runtime_review_pdl_fixverify_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": PROVIDER, - "capability_id": CAPABILITY_ID, - "review_kind": "fix_verify", - "started_at": datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z"), - "artifact_path": str(artifact_path.relative_to(REPO_ROOT)), - "input": {"profile": PROFILE_URL}, - "organization_id": org_id, - "post_grant_delay_seconds": POST_GRANT_PROPAGATION_DELAY_SECONDS, - } - _write_artifact(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"pdl-fix-verify-{stamp}", - starter_credits_cents=5000, - ) - payload["wallet_bootstrap"] = bootstrap - _write_artifact(artifact_path, payload) - - agent_id, api_key = await store.register_agent( - name=f"pdl-fix-verify-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for PDL fix-verification rerun", - tags=["runtime_review", "phase3", "fix_verify", "people-data-labs"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - _write_artifact(artifact_path, payload) - - access_id = await store.grant_service_access(agent_id, PROVIDER) - payload["grant_access"] = {"status": "success", "access_id": access_id} - _write_artifact(artifact_path, payload) - - # Freshly-created grants can race the auth/read path; wait briefly so the estimate call - # does not flap with a transient invalid-key/permission miss. - await asyncio.sleep(POST_GRANT_PROPAGATION_DELAY_SECONDS) - - api_headers = {"X-Rhumb-Key": api_key} - - try: - estimate_attempts = 0 - estimate_status = 0 - estimate_body: Any = None - while estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS: - estimate_attempts += 1 - estimate_status, estimate_body = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": PROVIDER, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - if not ( - estimate_status == 401 - and _looks_like_invalid_key(estimate_body) - and estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS - ): - break - await asyncio.sleep(ESTIMATE_AUTH_RETRY_DELAY_SECONDS) - execute_status, execute_body = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": PROVIDER, "credential_mode": "rhumb_managed"}, - json_body={"profile": PROFILE_URL}, - timeout=120.0, - ) - direct_status, direct_body = await _request_json( - "GET", - "https://api.peopledatalabs.com/v5/person/enrich", - headers={"X-Api-Key": os.environ["RHUMB_CREDENTIAL_PDL_API_KEY"]}, - params={"profile": PROFILE_URL}, - timeout=60.0, - ) - - rhumb_data = ((execute_body or {}).get("data") or {}) if isinstance(execute_body, dict) else {} - rhumb_upstream = (rhumb_data.get("upstream_response") or {}) if isinstance(rhumb_data, dict) else {} - rhumb_payload = rhumb_upstream.get("data") if isinstance(rhumb_upstream.get("data"), dict) else rhumb_upstream - direct_payload = direct_body.get("data") if isinstance(direct_body, dict) and isinstance(direct_body.get("data"), dict) else direct_body - - rhumb_sample = _sample_fields(rhumb_payload if isinstance(rhumb_payload, dict) else {}) - direct_sample = _sample_fields(direct_payload if isinstance(direct_payload, dict) else {}) - rhumb_error_message = ((rhumb_upstream.get("error") or {}).get("message")) if isinstance(rhumb_upstream, dict) else None - direct_error_message = ((direct_body.get("error") or {}).get("message")) if isinstance(direct_body, dict) else None - control_quota_blocked = direct_status == 402 - parity_ok = False - if direct_status == 200: - parity_ok = rhumb_sample == direct_sample - elif control_quota_blocked: - parity_ok = ( - rhumb_data.get("upstream_status") == direct_status - and rhumb_error_message == direct_error_message - and rhumb_data.get("provider_used") == PROVIDER - ) - - payload.update( - { - "estimate": { - "status": estimate_status, - "attempts": estimate_attempts, - "data": estimate_body, - }, - "rhumb_execute": {"status": execute_status, "data": execute_body}, - "direct_control": {"status": direct_status, "data": direct_body}, - "parity": { - "sampled_keys": list(rhumb_sample.keys()), - "rhumb_sample": rhumb_sample, - "direct_sample": direct_sample, - "provider_used": rhumb_data.get("provider_used"), - "execution_id": rhumb_data.get("execution_id"), - "upstream_status": rhumb_data.get("upstream_status"), - "same_error_message": rhumb_error_message == direct_error_message, - "all_matched": parity_ok, - }, - "control_quota_blocked": control_quota_blocked, - "observed_at": datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z"), - "verdict": "pass" - if ( - estimate_status == 200 - and execute_status == 200 - and rhumb_data.get("provider_used") == PROVIDER - and parity_ok - ) - else "fail", - } - ) - _write_artifact(artifact_path, payload) - finally: - payload["agent_disabled"] = bool(await store.disable_agent(agent_id)) - _write_artifact(artifact_path, payload) - - if payload.get("verdict") != "pass": - raise RuntimeError("PDL fix-verify failed; inspect artifact") - - print(json.dumps(payload, indent=2)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(asyncio.run(main())) diff --git a/scripts/runtime_review_pdl_unstructured_20260330.py b/scripts/runtime_review_pdl_unstructured_20260330.py deleted file mode 100644 index a2bac7ce..00000000 --- a/scripts/runtime_review_pdl_unstructured_20260330.py +++ /dev/null @@ -1,421 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -import uuid -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -TEXT_FILENAME = "runtime-review-unstructured-depth4.txt" -TEXT_CONTENT = ( - "Rhumb Runtime Review\n\n" - "Unstructured should parse this short sample into a Title and a NarrativeText block." -) -PDL_PROFILE = "https://www.linkedin.com/in/satyanadella/" - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - data: dict[str, Any] | list[tuple[str, str]] | None = None, - files: Any | None = None, - timeout: float = 60.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - data=data, - files=files, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - data = body.get("data") - if isinstance(data, dict): - reviews = data.get("reviews") or [] - elif isinstance(data, list): - reviews = data - runtime_backed = 0 - published = 0 - if isinstance(reviews, list): - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - if source_type == "runtime_verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews") - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -async def main() -> None: - skip_pdl = os.environ.get("SKIP_PDL", "").strip().lower() in {"1", "true", "yes", "on"} - skip_unstructured = os.environ.get("SKIP_UNSTRUCTURED", "").strip().lower() in {"1", "true", "yes", "on"} - - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - short = uuid.uuid4().hex[:8] - run_id = f"runtime-review-{stamp}-{short}" - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-pdl-unstructured-depth4.json" - pdl_pub_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-30-pdl-fix-verify.json" - unstructured_pub_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-30-unstructured-depth4.json" - org_id = f"org_runtime_review_{stamp.lower()}_{short}" - - payload: dict[str, Any] = { - "run_started_at": started_at.isoformat(), - "run_id": run_id, - "organization_id": org_id, - } - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-{stamp}", - starter_credits_cents=5000, - ) - payload["wallet_bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for PDL fix-verify + Unstructured depth4 runtime pass", - tags=["runtime_review", "phase3", "current_pass"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - grant_results: list[dict[str, Any]] = [] - for service in ["people-data-labs", "unstructured"]: - access_id = await store.grant_service_access(agent_id, service) - grant_results.append( - { - "service": service, - "status_code": 200, - "body": {"status": "success", "access_id": access_id}, - } - ) - payload["grant_results"] = grant_results - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - - try: - # Mission 0 β€” PDL fix verify using canonical slug only. - if skip_pdl: - payload["mission0_pdl"] = { - "status": "skipped", - "reason": "SKIP_PDL=1", - } - else: - pdl_before = await _fetch_public_review_stats("people-data-labs") - pdl_estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/data.enrich_person/execute/estimate", - headers=api_headers, - params={"provider": "people-data-labs", "credential_mode": "rhumb_managed"}, - ) - pdl_execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/data.enrich_person/execute", - headers=api_headers, - params={"provider": "people-data-labs", "credential_mode": "rhumb_managed"}, - json_body={"profile": PDL_PROFILE}, - ) - pdl_direct = await _request_json( - "GET", - "https://api.peopledatalabs.com/v5/person/enrich", - headers={"X-Api-Key": os.environ["RHUMB_CREDENTIAL_PDL_API_KEY"]}, - params={"profile": PDL_PROFILE}, - ) - - pdl_rhumb_data = (((pdl_execute.body or {}).get("data") or {}).get("upstream_response") or {}).get("data") if isinstance(pdl_execute.body, dict) else None - pdl_direct_data = ((pdl_direct.body or {}).get("data")) if isinstance(pdl_direct.body, dict) else None - sampled_keys = ["full_name", "job_title", "job_company_name", "linkedin_url"] - pdl_checks = { - key: (pdl_rhumb_data or {}).get(key) == (pdl_direct_data or {}).get(key) - for key in sampled_keys - } - control_quota_blocked = ( - pdl_direct.status_code == 402 - and isinstance(pdl_direct.body, dict) - and "account maximum" in json.dumps(pdl_direct.body).lower() - ) - pdl_observed_at = _iso_now() - - verdict = "fail" - if pdl_execute.status_code == 200 and pdl_direct.status_code == 200 and all(pdl_checks.values()): - verdict = "pass" - elif pdl_execute.status_code == 200 and control_quota_blocked: - verdict = "pass_control_blocked" - - payload["mission0_pdl"] = { - "service_slug": "people-data-labs", - "provider": "people-data-labs", - "capability_id": "data.enrich_person", - "run_id": f"{run_id}-pdl", - "input": {"profile": PDL_PROFILE}, - "counts_before": pdl_before, - "estimate": asdict(pdl_estimate), - "rhumb_execute": asdict(pdl_execute), - "direct_control": asdict(pdl_direct), - "control_quota_blocked": control_quota_blocked, - "parity": { - "sampled_keys": sampled_keys, - "rhumb_sample": {key: (pdl_rhumb_data or {}).get(key) for key in sampled_keys}, - "direct_sample": {key: (pdl_direct_data or {}).get(key) for key in sampled_keys}, - "checks": pdl_checks, - "all_matched": all(pdl_checks.values()), - "provider_used": (((pdl_execute.body or {}).get("data") or {}).get("provider_used")) if isinstance(pdl_execute.body, dict) else None, - "execution_id": (((pdl_execute.body or {}).get("data") or {}).get("execution_id")) if isinstance(pdl_execute.body, dict) else None, - "upstream_status": (((pdl_execute.body or {}).get("data") or {}).get("upstream_status")) if isinstance(pdl_execute.body, dict) else None, - }, - "observed_at": pdl_observed_at, - "verdict": verdict, - } - _json_dump(artifact_path, payload) - - if verdict == "fail": - raise RuntimeError("PDL fix-verify rerun failed; inspect artifact for execute vs direct-control details") - - if verdict == "pass": - pdl_reviewed_at = pdl_observed_at - pdl_fresh_until = ( - datetime.fromisoformat(pdl_reviewed_at.replace("Z", "+00:00")) + timedelta(days=30) - ).replace(microsecond=0).isoformat().replace("+00:00", "Z") - pdl_publish = _run_publish( - [ - "--service", "people-data-labs", - "--headline", "People Data Labs: fix-verify rerun confirms data.enrich_person parity through Rhumb Resolve", - "--summary", "Fresh post-fix runtime rerun passed for canonical People Data Labs data.enrich_person through Rhumb Resolve. Managed and direct executions matched on full_name, job_title, job_company_name, and linkedin_url for the same live LinkedIn input.", - "--evidence-title", "People Data Labs fix-verify runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh post-fix runtime rerun passed for canonical People Data Labs data.enrich_person through Rhumb Resolve. Managed and direct executions matched on full_name, job_title, job_company_name, and linkedin_url for the same live LinkedIn input.", - "--source-ref", f"runtime-review:people-data-labs:{stamp}", - "--source-batch-id", f"runtime-review:people-data-labs:{stamp}", - "--reviewed-at", pdl_reviewed_at, - "--fresh-until", pdl_fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", payload["mission0_pdl"]["parity"]["execution_id"] or "", - "--tag", "runtime_review", - "--tag", "people-data-labs", - "--tag", "data.enrich_person", - "--tag", "current_pass", - "--tag", "phase3", - "--tag", "fix_verify", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - pdl_pub_path, - ) - payload["mission0_pdl"]["evidence_id"] = ((pdl_publish.get("evidence") or {}).get("id")) - payload["mission0_pdl"]["review_id"] = ((pdl_publish.get("review") or {}).get("id")) - else: - payload["mission0_pdl"]["publication_skipped_reason"] = ( - "direct control immediately hit provider account max after the managed pass; logged for internal verification only" - ) - payload["mission0_pdl"]["counts_after"] = await _fetch_public_review_stats("people-data-labs") - _json_dump(artifact_path, payload) - - # Mission 1 β€” next weakest callable provider, fresh ordering => unstructured. - if skip_unstructured: - payload["mission1_unstructured"] = { - "status": "skipped", - "reason": "SKIP_UNSTRUCTURED=1", - } - else: - un_before = await _fetch_public_review_stats("unstructured") - un_estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/document.parse/execute/estimate", - headers=api_headers, - params={"provider": "unstructured", "credential_mode": "rhumb_managed"}, - ) - un_execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/document.parse/execute", - headers=api_headers, - params={"provider": "unstructured", "credential_mode": "rhumb_managed"}, - json_body={ - "strategy": "fast", - "files": [ - { - "filename": TEXT_FILENAME, - "text": TEXT_CONTENT, - "content_type": "text/plain", - } - ], - }, - timeout=120.0, - ) - un_direct = await _request_json( - "POST", - "https://api.unstructuredapp.io/general/v0/general", - headers={"unstructured-api-key": os.environ["RHUMB_CREDENTIAL_UNSTRUCTURED_API_KEY"]}, - data={"strategy": "fast"}, - files={"files": (TEXT_FILENAME, TEXT_CONTENT.encode("utf-8"), "text/plain")}, - timeout=120.0, - ) - - rhumb_elements = (((un_execute.body or {}).get("data") or {}).get("upstream_response")) if isinstance(un_execute.body, dict) else None - direct_elements = un_direct.body if isinstance(un_direct.body, list) else [] - rhumb_types = [row.get("type") for row in rhumb_elements or [] if isinstance(row, dict)] - direct_types = [row.get("type") for row in direct_elements or [] if isinstance(row, dict)] - un_observed_at = _iso_now() - - payload["mission1_unstructured"] = { - "service_slug": "unstructured", - "provider": "unstructured", - "capability_id": "document.parse", - "run_id": f"{run_id}-unstructured", - "counts_before": un_before, - "input_shape": { - "filename": TEXT_FILENAME, - "strategy": "fast", - }, - "estimate": asdict(un_estimate), - "rhumb_execute": asdict(un_execute), - "direct_control": asdict(un_direct), - "parity": { - "rhumb_types": rhumb_types, - "direct_types": direct_types, - "rhumb_count": len(rhumb_elements or []), - "direct_count": len(direct_elements or []), - "types_match": rhumb_types == direct_types, - "count_match": len(rhumb_elements or []) == len(direct_elements or []), - "provider_used": (((un_execute.body or {}).get("data") or {}).get("provider_used")) if isinstance(un_execute.body, dict) else None, - "execution_id": (((un_execute.body or {}).get("data") or {}).get("execution_id")) if isinstance(un_execute.body, dict) else None, - "upstream_status": (((un_execute.body or {}).get("data") or {}).get("upstream_status")) if isinstance(un_execute.body, dict) else None, - }, - "observed_at": un_observed_at, - "verdict": "pass" if un_execute.status_code == 200 and un_direct.status_code == 200 and rhumb_types == direct_types and len(rhumb_elements or []) == len(direct_elements or []) else "fail", - } - _json_dump(artifact_path, payload) - - if payload["mission1_unstructured"]["verdict"] != "pass": - raise RuntimeError("Unstructured depth4 runtime pass failed; inspect artifact for execute vs direct-control details") - - un_reviewed_at = un_observed_at - un_fresh_until = ( - datetime.fromisoformat(un_reviewed_at.replace("Z", "+00:00")) + timedelta(days=30) - ).replace(microsecond=0).isoformat().replace("+00:00", "Z") - un_publish = _run_publish( - [ - "--service", "unstructured", - "--headline", "Unstructured: current-depth rerun confirms document.parse parity through Rhumb Resolve", - "--summary", "Fresh current-depth runtime rerun passed for Unstructured document.parse through Rhumb Resolve. Managed and direct executions matched exactly on element types and element count for the same text file input.", - "--evidence-title", "Unstructured current-depth runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh current-depth runtime rerun passed for Unstructured document.parse through Rhumb Resolve. Managed and direct executions matched exactly on element types and element count for the same text file input.", - "--source-ref", f"runtime-review:unstructured:{stamp}", - "--source-batch-id", f"runtime-review:unstructured:{stamp}", - "--reviewed-at", un_reviewed_at, - "--fresh-until", un_fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", payload["mission1_unstructured"]["parity"]["execution_id"] or "", - "--tag", "runtime_review", - "--tag", "unstructured", - "--tag", "document.parse", - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - unstructured_pub_path, - ) - payload["mission1_unstructured"]["evidence_id"] = ((un_publish.get("evidence") or {}).get("id")) - payload["mission1_unstructured"]["review_id"] = ((un_publish.get("review") or {}).get("id")) - payload["mission1_unstructured"]["counts_after"] = await _fetch_public_review_stats("unstructured") - _json_dump(artifact_path, payload) - - finally: - disabled = await store.disable_agent(agent_id) - payload["disable_agent"] = { - "status_code": 200 if disabled else 404, - "body": {"status": "success" if disabled else "not_found", "agent_id": agent_id}, - } - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_replicate_depth10_20260402.py b/scripts/runtime_review_replicate_depth10_20260402.py deleted file mode 100644 index 6176c264..00000000 --- a/scripts/runtime_review_replicate_depth10_20260402.py +++ /dev/null @@ -1,394 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -import uuid -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -REPLICATE_BASE_URL = "https://api.replicate.com/v1" -SERVICE_SLUG = "replicate" -CAPABILITY_ID = "ai.generate_text" -MODEL_VERSION = "5a6809ca6288247d06daf6365557e5e429063f32a21146b2a807c682652136b8" -SYSTEM_PROMPT = "You are a precise assistant." -MAX_DIRECT_CREATE_ATTEMPTS = 3 -DIRECT_RETRY_AFTER_SECONDS = 10 -POLL_TIMEOUT_SECONDS = 120.0 - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -async def _poll_prediction(api_token: str, prediction_id: str, *, timeout_seconds: float = POLL_TIMEOUT_SECONDS) -> HttpResult: - deadline = asyncio.get_running_loop().time() + timeout_seconds - last: HttpResult | None = None - while True: - last = await _request_json( - "GET", - f"{REPLICATE_BASE_URL}/predictions/{prediction_id}", - headers={"Authorization": f"Bearer {api_token}"}, - timeout=60.0, - ) - status = last.body.get("status") if isinstance(last.body, dict) else None - if status in {"succeeded", "failed", "canceled"}: - return last - if asyncio.get_running_loop().time() >= deadline: - return last - await asyncio.sleep(2.0) - - -async def _create_direct_prediction( - api_token: str, - payload: dict[str, Any], - *, - max_attempts: int = MAX_DIRECT_CREATE_ATTEMPTS, -) -> tuple[HttpResult, list[dict[str, Any]]]: - attempts: list[dict[str, Any]] = [] - last: HttpResult | None = None - for attempt in range(1, max_attempts + 1): - last = await _request_json( - "POST", - f"{REPLICATE_BASE_URL}/predictions", - headers={"Authorization": f"Bearer {api_token}"}, - json_body=payload, - timeout=120.0, - ) - attempts.append( - { - "attempt": attempt, - "status_code": last.status_code, - "body": last.body, - } - ) - if last.status_code != 429: - return last, attempts - retry_after = DIRECT_RETRY_AFTER_SECONDS - if isinstance(last.body, dict): - retry_after = int(last.body.get("retry_after") or last.body.get("retryAfter") or retry_after) - await asyncio.sleep(float(retry_after)) - assert last is not None - return last, attempts - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label in {"🟒 Runtime-verified", "πŸ§ͺ Tester-generated"}: - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [ - str(REPO_ROOT / "packages/api/.venv/bin/python"), - str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), - *args, - ], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _flatten_output(value: Any) -> str | None: - if isinstance(value, str): - return value.strip() - if isinstance(value, list): - parts = [chunk for chunk in value if isinstance(chunk, str)] - return "".join(parts).strip() - return None - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - short = uuid.uuid4().hex[:8] - expected_output = f"REPLICATE_DEPTH10_OK_{stamp}" - prompt = f"Reply with exactly {expected_output}" - model_input = { - "prompt": prompt, - "system_prompt": SYSTEM_PROMPT, - "max_new_tokens": 20, - "temperature": 0.1, - } - execute_body = { - "version": MODEL_VERSION, - "input": model_input, - } - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-replicate-depth10.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-02-replicate-depth10.json" - org_id = f"org_runtime_review_replicate_{stamp.lower()}_{short}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "capability_id": CAPABILITY_ID, - "review_kind": "current_depth10", - "started_at": started_at.replace(microsecond=0).isoformat().replace("+00:00", "Z"), - "artifact_path": str(artifact_path.relative_to(REPO_ROOT)), - "organization_id": org_id, - "execute_body": execute_body, - "expected_output": expected_output, - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-replicate-{stamp}", - starter_credits_cents=5000, - ) - payload["wallet_bootstrap"] = bootstrap - _json_dump(artifact_path, payload) - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-replicate-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Replicate depth-10 runtime pass", - tags=["runtime_review", "phase3", "current_pass", SERVICE_SLUG, "depth10"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = { - "service": SERVICE_SLUG, - "status": "success", - "access_id": access_id, - } - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - replicate_token = os.environ["RHUMB_CREDENTIAL_REPLICATE_API_TOKEN"] - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body=execute_body, - timeout=120.0, - ) - - execute_data = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (execute_data.get("upstream_response") or {}) if isinstance(execute_data, dict) else {} - rhumb_prediction_id = rhumb_upstream.get("id") - rhumb_final = ( - await _poll_prediction(replicate_token, rhumb_prediction_id) - if rhumb_prediction_id - else HttpResult(status_code=0, body=None) - ) - - direct_create, direct_attempts = await _create_direct_prediction( - replicate_token, - execute_body, - max_attempts=MAX_DIRECT_CREATE_ATTEMPTS, - ) - direct_create_body = direct_create.body if isinstance(direct_create.body, dict) else {} - direct_prediction_id = direct_create_body.get("id") - direct_final = ( - await _poll_prediction(replicate_token, direct_prediction_id) - if direct_prediction_id - else HttpResult(status_code=0, body=None) - ) - - rhumb_final_body = rhumb_final.body if isinstance(rhumb_final.body, dict) else {} - direct_final_body = direct_final.body if isinstance(direct_final.body, dict) else {} - rhumb_output = _flatten_output(rhumb_final_body.get("output")) - direct_output = _flatten_output(direct_final_body.get("output")) - - comparison = { - "prompt": prompt, - "rhumb_prediction_id": rhumb_prediction_id, - "direct_prediction_id": direct_prediction_id, - "rhumb_status": rhumb_final_body.get("status"), - "direct_status": direct_final_body.get("status"), - "rhumb_output": rhumb_output, - "direct_output": direct_output, - "status_match": rhumb_final_body.get("status") == direct_final_body.get("status"), - "output_match": rhumb_output == direct_output == expected_output, - "provider_used": execute_data.get("provider_used"), - "upstream_status": execute_data.get("upstream_status"), - "execution_id": execute_data.get("execution_id"), - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and execute_data.get("provider_used") == SERVICE_SLUG - and execute_data.get("upstream_status") == 201 - and rhumb_final.status_code == 200 - and direct_create.status_code == 201 - and direct_final.status_code == 200 - and comparison["status_match"] - and comparison["output_match"] - and comparison["rhumb_status"] == "succeeded" - ) else "fail" - - observed_at = _iso_now() - fresh_until = ( - datetime.fromisoformat(observed_at.replace("Z", "+00:00")) + timedelta(days=30) - ).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - payload.update( - { - "reviewed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": asdict(estimate), - "rhumb_execute": asdict(execute), - "rhumb_final": asdict(rhumb_final), - "direct_create_attempts": direct_attempts, - "direct_create": asdict(direct_create), - "direct_final": asdict(direct_final), - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Replicate depth-10 runtime pass failed; inspect artifact for parity details") - - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "Replicate: depth-10 runtime review confirms ai.generate_text parity through Rhumb Resolve", - "--summary", "Fresh depth-10 runtime review passed for Replicate ai.generate_text through Rhumb Resolve. Managed and direct executions both created successful predictions and matched on final succeeded state and exact text output for the same pinned model version.", - "--evidence-title", "Replicate depth-10 runtime review parity check via Rhumb Resolve", - "--evidence-summary", "Fresh depth-10 runtime review passed for Replicate ai.generate_text through Rhumb Resolve. Managed and direct executions both created successful predictions and matched on final succeeded state and exact text output for the same pinned model version.", - "--source-ref", f"runtime-review:replicate:{stamp}", - "--source-batch-id", f"runtime-review:replicate:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execute_data.get("execution_id") or "", - "--tag", "runtime_review", - "--tag", SERVICE_SLUG, - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--tag", "depth10", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["publication"] = publication - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["disable_agent"] = { - "status_code": 200 if disabled else 404, - "body": {"status": "success" if disabled else "not_found", "agent_id": agent_id}, - } - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_replicate_depth4_20260330.py b/scripts/runtime_review_replicate_depth4_20260330.py deleted file mode 100644 index 2d232b6a..00000000 --- a/scripts/runtime_review_replicate_depth4_20260330.py +++ /dev/null @@ -1,370 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -import uuid -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -REPLICATE_BASE_URL = "https://api.replicate.com/v1" -MODEL_VERSION = "5a6809ca6288247d06daf6365557e5e429063f32a21146b2a807c682652136b8" -SYSTEM_PROMPT = "You are a precise assistant." - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -async def _poll_prediction(api_token: str, prediction_id: str, *, timeout_seconds: float = 120.0) -> HttpResult: - deadline = asyncio.get_running_loop().time() + timeout_seconds - last: HttpResult | None = None - while True: - last = await _request_json( - "GET", - f"{REPLICATE_BASE_URL}/predictions/{prediction_id}", - headers={"Authorization": f"Bearer {api_token}"}, - timeout=60.0, - ) - status = last.body.get("status") if isinstance(last.body, dict) else None - if status in {"succeeded", "failed", "canceled"}: - return last - if asyncio.get_running_loop().time() >= deadline: - return last - await asyncio.sleep(2.0) - - -async def _create_direct_prediction( - api_token: str, - payload: dict[str, Any], - *, - max_attempts: int = 3, -) -> tuple[HttpResult, list[dict[str, Any]]]: - attempts: list[dict[str, Any]] = [] - last: HttpResult | None = None - for attempt in range(1, max_attempts + 1): - last = await _request_json( - "POST", - f"{REPLICATE_BASE_URL}/predictions", - headers={"Authorization": f"Bearer {api_token}"}, - json_body=payload, - timeout=120.0, - ) - attempts.append({ - "attempt": attempt, - "status_code": last.status_code, - "body": last.body, - }) - if last.status_code != 429: - return last, attempts - retry_after = 10 - if isinstance(last.body, dict): - retry_after = int(last.body.get("retry_after") or last.body.get("retryAfter") or retry_after) - await asyncio.sleep(float(retry_after)) - assert last is not None - return last, attempts - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _flatten_output(value: Any) -> str | None: - if isinstance(value, str): - return value.strip() - if isinstance(value, list): - parts = [chunk for chunk in value if isinstance(chunk, str)] - return "".join(parts).strip() - return None - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - short = uuid.uuid4().hex[:8] - prompt = f"Reply with exactly REPLICATE_CURRENT_DEPTH4_OK_{stamp}" - model_input = { - "prompt": prompt, - "system_prompt": SYSTEM_PROMPT, - "max_new_tokens": 20, - "temperature": 0.1, - } - execute_body = { - "version": MODEL_VERSION, - "input": model_input, - } - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-replicate-current-depth4.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-30-replicate-depth4.json" - org_id = f"org_runtime_review_replicate_{stamp.lower()}_{short}" - - payload: dict[str, Any] = { - "service": "replicate", - "review_kind": "current_depth4", - "reviewed_at": None, - "fresh_until": None, - "temp_org": { - "org_id": org_id, - "balance_usd_cents": 5000, - }, - "execute_body": execute_body, - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-replicate-{stamp}", - starter_credits_cents=5000, - ) - payload["wallet_bootstrap"] = bootstrap - _json_dump(artifact_path, payload) - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-replicate-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Replicate depth-4 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "replicate"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, "replicate") - payload["grant_result"] = { - "service": "replicate", - "status_code": 200, - "body": {"status": "success", "access_id": access_id}, - } - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - replicate_token = os.environ["RHUMB_CREDENTIAL_REPLICATE_API_TOKEN"] - - try: - counts_before = await _fetch_public_review_stats("replicate") - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/ai.generate_text/execute/estimate", - headers=api_headers, - params={"provider": "replicate", "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/ai.generate_text/execute", - headers=api_headers, - params={"provider": "replicate", "credential_mode": "rhumb_managed"}, - json_body=execute_body, - timeout=120.0, - ) - - execute_data = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (execute_data.get("upstream_response") or {}) if isinstance(execute_data, dict) else {} - rhumb_prediction_id = rhumb_upstream.get("id") - rhumb_final = await _poll_prediction(replicate_token, rhumb_prediction_id) if rhumb_prediction_id else HttpResult(status_code=0, body=None) - - direct_create, direct_attempts = await _create_direct_prediction( - replicate_token, - execute_body, - max_attempts=3, - ) - direct_create_body = direct_create.body if isinstance(direct_create.body, dict) else {} - direct_prediction_id = direct_create_body.get("id") - direct_final = await _poll_prediction(replicate_token, direct_prediction_id) if direct_prediction_id else HttpResult(status_code=0, body=None) - - rhumb_final_body = rhumb_final.body if isinstance(rhumb_final.body, dict) else {} - direct_final_body = direct_final.body if isinstance(direct_final.body, dict) else {} - rhumb_output = _flatten_output(rhumb_final_body.get("output")) - direct_output = _flatten_output(direct_final_body.get("output")) - - comparison = { - "prompt": prompt, - "rhumb_prediction_id": rhumb_prediction_id, - "direct_prediction_id": direct_prediction_id, - "rhumb_status": rhumb_final_body.get("status"), - "direct_status": direct_final_body.get("status"), - "rhumb_output": rhumb_output, - "direct_output": direct_output, - "status_match": rhumb_final_body.get("status") == direct_final_body.get("status"), - "output_match": rhumb_output == direct_output == f"REPLICATE_CURRENT_DEPTH4_OK_{stamp}", - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and execute_data.get("upstream_status") == 201 - and rhumb_final.status_code == 200 - and direct_create.status_code == 201 - and direct_final.status_code == 200 - and comparison["status_match"] - and comparison["output_match"] - and comparison["rhumb_status"] == "succeeded" - ) else "fail" - - observed_at = _iso_now() - fresh_until = ( - datetime.fromisoformat(observed_at.replace("Z", "+00:00")) + timedelta(days=30) - ).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - payload.update( - { - "reviewed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": asdict(estimate), - "rhumb_execute": asdict(execute), - "rhumb_final": asdict(rhumb_final), - "direct_create_attempts": direct_attempts, - "direct_create": asdict(direct_create), - "direct_final": asdict(direct_final), - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Replicate depth-4 runtime pass failed; inspect artifact for parity details") - - publication = _run_publish( - [ - "--service", "replicate", - "--headline", "Replicate: current-depth rerun confirms ai.generate_text parity through Rhumb Resolve", - "--summary", "Fresh current-depth runtime rerun passed for Replicate ai.generate_text through Rhumb Resolve. Managed and direct executions both created successful predictions and matched on final succeeded state and exact text output for the same pinned model version.", - "--evidence-title", "Replicate current-depth runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh current-depth runtime rerun passed for Replicate ai.generate_text through Rhumb Resolve. Managed and direct executions both created successful predictions and matched on final succeeded state and exact text output for the same pinned model version.", - "--source-ref", f"runtime-review:replicate:{stamp}", - "--source-batch-id", f"runtime-review:replicate:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execute_data.get("execution_id") or "", - "--tag", "runtime_review", - "--tag", "replicate", - "--tag", "ai.generate_text", - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["publication"] = publication - payload["counts_after"] = await _fetch_public_review_stats("replicate") - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["disable_agent"] = { - "status_code": 200 if disabled else 404, - "body": {"status": "success" if disabled else "not_found", "agent_id": agent_id}, - } - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_replicate_depth5_20260331.py b/scripts/runtime_review_replicate_depth5_20260331.py deleted file mode 100644 index 160bf522..00000000 --- a/scripts/runtime_review_replicate_depth5_20260331.py +++ /dev/null @@ -1,370 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -import uuid -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -REPLICATE_BASE_URL = "https://api.replicate.com/v1" -MODEL_VERSION = "5a6809ca6288247d06daf6365557e5e429063f32a21146b2a807c682652136b8" -SYSTEM_PROMPT = "You are a precise assistant." - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -async def _poll_prediction(api_token: str, prediction_id: str, *, timeout_seconds: float = 120.0) -> HttpResult: - deadline = asyncio.get_running_loop().time() + timeout_seconds - last: HttpResult | None = None - while True: - last = await _request_json( - "GET", - f"{REPLICATE_BASE_URL}/predictions/{prediction_id}", - headers={"Authorization": f"Bearer {api_token}"}, - timeout=60.0, - ) - status = last.body.get("status") if isinstance(last.body, dict) else None - if status in {"succeeded", "failed", "canceled"}: - return last - if asyncio.get_running_loop().time() >= deadline: - return last - await asyncio.sleep(2.0) - - -async def _create_direct_prediction( - api_token: str, - payload: dict[str, Any], - *, - max_attempts: int = 3, -) -> tuple[HttpResult, list[dict[str, Any]]]: - attempts: list[dict[str, Any]] = [] - last: HttpResult | None = None - for attempt in range(1, max_attempts + 1): - last = await _request_json( - "POST", - f"{REPLICATE_BASE_URL}/predictions", - headers={"Authorization": f"Bearer {api_token}"}, - json_body=payload, - timeout=120.0, - ) - attempts.append({ - "attempt": attempt, - "status_code": last.status_code, - "body": last.body, - }) - if last.status_code != 429: - return last, attempts - retry_after = 10 - if isinstance(last.body, dict): - retry_after = int(last.body.get("retry_after") or last.body.get("retryAfter") or retry_after) - await asyncio.sleep(float(retry_after)) - assert last is not None - return last, attempts - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _flatten_output(value: Any) -> str | None: - if isinstance(value, str): - return value.strip() - if isinstance(value, list): - parts = [chunk for chunk in value if isinstance(chunk, str)] - return "".join(parts).strip() - return None - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - short = uuid.uuid4().hex[:8] - prompt = f"Reply with exactly REPLICATE_CURRENT_DEPTH5_OK_{stamp}" - model_input = { - "prompt": prompt, - "system_prompt": SYSTEM_PROMPT, - "max_new_tokens": 20, - "temperature": 0.1, - } - execute_body = { - "version": MODEL_VERSION, - "input": model_input, - } - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-replicate-current-depth5.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-31-replicate-depth5.json" - org_id = f"org_runtime_review_replicate_{stamp.lower()}_{short}" - - payload: dict[str, Any] = { - "service": "replicate", - "review_kind": "current_depth5", - "reviewed_at": None, - "fresh_until": None, - "temp_org": { - "org_id": org_id, - "balance_usd_cents": 5000, - }, - "execute_body": execute_body, - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-replicate-{stamp}", - starter_credits_cents=5000, - ) - payload["wallet_bootstrap"] = bootstrap - _json_dump(artifact_path, payload) - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-replicate-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Replicate depth-5 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "replicate"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, "replicate") - payload["grant_result"] = { - "service": "replicate", - "status_code": 200, - "body": {"status": "success", "access_id": access_id}, - } - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - replicate_token = os.environ["RHUMB_CREDENTIAL_REPLICATE_API_TOKEN"] - - try: - counts_before = await _fetch_public_review_stats("replicate") - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/ai.generate_text/execute/estimate", - headers=api_headers, - params={"provider": "replicate", "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/ai.generate_text/execute", - headers=api_headers, - params={"provider": "replicate", "credential_mode": "rhumb_managed"}, - json_body=execute_body, - timeout=120.0, - ) - - execute_data = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (execute_data.get("upstream_response") or {}) if isinstance(execute_data, dict) else {} - rhumb_prediction_id = rhumb_upstream.get("id") - rhumb_final = await _poll_prediction(replicate_token, rhumb_prediction_id) if rhumb_prediction_id else HttpResult(status_code=0, body=None) - - direct_create, direct_attempts = await _create_direct_prediction( - replicate_token, - execute_body, - max_attempts=3, - ) - direct_create_body = direct_create.body if isinstance(direct_create.body, dict) else {} - direct_prediction_id = direct_create_body.get("id") - direct_final = await _poll_prediction(replicate_token, direct_prediction_id) if direct_prediction_id else HttpResult(status_code=0, body=None) - - rhumb_final_body = rhumb_final.body if isinstance(rhumb_final.body, dict) else {} - direct_final_body = direct_final.body if isinstance(direct_final.body, dict) else {} - rhumb_output = _flatten_output(rhumb_final_body.get("output")) - direct_output = _flatten_output(direct_final_body.get("output")) - - comparison = { - "prompt": prompt, - "rhumb_prediction_id": rhumb_prediction_id, - "direct_prediction_id": direct_prediction_id, - "rhumb_status": rhumb_final_body.get("status"), - "direct_status": direct_final_body.get("status"), - "rhumb_output": rhumb_output, - "direct_output": direct_output, - "status_match": rhumb_final_body.get("status") == direct_final_body.get("status"), - "output_match": rhumb_output == direct_output == f"REPLICATE_CURRENT_DEPTH5_OK_{stamp}", - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and execute_data.get("upstream_status") == 201 - and rhumb_final.status_code == 200 - and direct_create.status_code == 201 - and direct_final.status_code == 200 - and comparison["status_match"] - and comparison["output_match"] - and comparison["rhumb_status"] == "succeeded" - ) else "fail" - - observed_at = _iso_now() - fresh_until = ( - datetime.fromisoformat(observed_at.replace("Z", "+00:00")) + timedelta(days=30) - ).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - payload.update( - { - "reviewed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": asdict(estimate), - "rhumb_execute": asdict(execute), - "rhumb_final": asdict(rhumb_final), - "direct_create_attempts": direct_attempts, - "direct_create": asdict(direct_create), - "direct_final": asdict(direct_final), - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Replicate depth-5 runtime pass failed; inspect artifact for parity details") - - publication = _run_publish( - [ - "--service", "replicate", - "--headline", "Replicate: current-depth rerun confirms ai.generate_text parity through Rhumb Resolve", - "--summary", "Fresh current-depth runtime rerun passed for Replicate ai.generate_text through Rhumb Resolve. Managed and direct executions both created successful predictions and matched on final succeeded state and exact text output for the same pinned model version.", - "--evidence-title", "Replicate current-depth runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh current-depth runtime rerun passed for Replicate ai.generate_text through Rhumb Resolve. Managed and direct executions both created successful predictions and matched on final succeeded state and exact text output for the same pinned model version.", - "--source-ref", f"runtime-review:replicate:{stamp}", - "--source-batch-id", f"runtime-review:replicate:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execute_data.get("execution_id") or "", - "--tag", "runtime_review", - "--tag", "replicate", - "--tag", "ai.generate_text", - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["publication"] = publication - payload["counts_after"] = await _fetch_public_review_stats("replicate") - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["disable_agent"] = { - "status_code": 200 if disabled else 404, - "body": {"status": "success" if disabled else "not_found", "agent_id": agent_id}, - } - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_slack_depth10_20260403.py b/scripts/runtime_review_slack_depth10_20260403.py deleted file mode 100644 index 91bbad9f..00000000 --- a/scripts/runtime_review_slack_depth10_20260403.py +++ /dev/null @@ -1,357 +0,0 @@ -#!/usr/bin/env python3 -"""Slack current-depth runtime review pass β€” 2026-04-03. - -Slack remains the sole weakest callable provider in the live public audit at depth 9. -This pass re-verifies the safe, non-mutating auth identity read (`POST /api/auth.test`) -through the Rhumb proxy and compares it directly against Slack. -""" -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -PROVIDER = "slack" -PUBLIC_SERVICE_SLUG = "slack" -METHOD = "POST" -PATH = "/api/auth.test" -DIRECT_URL = "https://slack.com/api/auth.test" -SLACK_ENV_VAR = "RHUMB_CREDENTIAL_SLACK_BOT_TOKEN" -SLACK_1PASSWORD_ITEM = "Slack - TeamSuper Bot Token" -POST_GRANT_PROPAGATION_DELAY_SECONDS = 5 -EXECUTE_AUTH_RETRY_ATTEMPTS = 4 -EXECUTE_AUTH_RETRY_DELAY_SECONDS = 5 -RUNTIME_BACKED_TRUST_LABELS = {"🟒 Runtime-verified", "πŸ§ͺ Tester-generated"} - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _looks_like_invalid_key(body: Any) -> bool: - if not isinstance(body, dict): - return False - detail = str(body.get("detail") or "").lower() - error = str(body.get("error") or "").lower() - return "invalid or expired rhumb api key" in detail or "invalid or expired rhumb api key" in error - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - content: bytes | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - content=content, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label in RUNTIME_BACKED_TRUST_LABELS: - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _require_slack_token() -> tuple[str, str]: - env_value = os.environ.get(SLACK_ENV_VAR, "").strip() - if env_value: - return env_value, "environment" - - proc = subprocess.run( - [ - "sop", - "item", - "get", - SLACK_1PASSWORD_ITEM, - "--vault", - "OpenClaw Agents", - "--fields", - "credential", - "--reveal", - ], - cwd=REPO_ROOT, - capture_output=True, - text=True, - timeout=20, - check=False, - ) - if proc.returncode == 0 and proc.stdout.strip(): - return proc.stdout.strip(), "1password" - - raise RuntimeError( - f"Slack credential unavailable via {SLACK_ENV_VAR} or 1Password item {SLACK_1PASSWORD_ITEM!r}" - ) - - -def _pick_fields(payload: dict[str, Any] | None) -> dict[str, Any]: - payload = payload or {} - return { - "ok": payload.get("ok"), - "url": payload.get("url"), - "team": payload.get("team"), - "team_id": payload.get("team_id"), - "user": payload.get("user"), - "user_id": payload.get("user_id"), - "bot_id": payload.get("bot_id"), - "is_enterprise_install": payload.get("is_enterprise_install"), - } - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-slack-depth10.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-03-slack-depth10.json" - org_id = f"org_runtime_review_slack_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": PROVIDER, - "method": METHOD, - "path": PATH, - "review_kind": "current_depth10", - "organization_id": org_id, - "artifact_path": str(artifact_path), - } - _json_dump(artifact_path, payload) - - slack_token, token_source = _require_slack_token() - payload["direct_credential_source"] = token_source - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-slack-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-slack-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Slack depth-10 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "slack"], - ) - payload["review_agent"] = {"agent_id": agent_id, "organization_id": org_id} - - access_id = await store.grant_service_access(agent_id, PROVIDER) - payload["grant_access"] = {"status": "success", "access_id": access_id} - payload["post_grant_delay_seconds"] = POST_GRANT_PROPAGATION_DELAY_SECONDS - _json_dump(artifact_path, payload) - - await asyncio.sleep(POST_GRANT_PROPAGATION_DELAY_SECONDS) - - try: - counts_before = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - current_depth = int(counts_before.get("runtime_backed_reviews") or 0) - target_depth = current_depth + 1 - - execute_attempts = 0 - execute = HttpResult(status_code=0, body=None) - while execute_attempts < EXECUTE_AUTH_RETRY_ATTEMPTS: - execute_attempts += 1 - execute = await _request_json( - "POST", - f"{BASE_URL}/proxy/", - headers={"X-Rhumb-Key": api_key}, - json_body={ - "service": PROVIDER, - "method": METHOD, - "path": PATH, - }, - timeout=120.0, - ) - if not ( - execute.status_code == 401 - and _looks_like_invalid_key(execute.body) - and execute_attempts < EXECUTE_AUTH_RETRY_ATTEMPTS - ): - break - await asyncio.sleep(EXECUTE_AUTH_RETRY_DELAY_SECONDS) - - direct = await _request_json( - "POST", - DIRECT_URL, - headers={ - "Authorization": f"Bearer {slack_token}", - "Content-Type": "application/x-www-form-urlencoded", - "User-Agent": "rhumb-runtime-review/1.0", - }, - content=b"", - timeout=60.0, - ) - - execute_payload = execute.body if isinstance(execute.body, dict) else {} - rhumb_body = execute_payload.get("body") if isinstance(execute_payload.get("body"), dict) else {} - rhumb_fields = _pick_fields(rhumb_body) - direct_fields = _pick_fields(direct.body if isinstance(direct.body, dict) else {}) - - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_fields": rhumb_fields, - "direct_fields": direct_fields, - "parity": rhumb_fields == direct_fields, - } - - verdict = "pass" if ( - execute.status_code == 200 - and direct.status_code == 200 - and comparison["parity"] - ) else "fail" - - payload.update( - { - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "target_depth": target_depth, - "execute_attempts": execute_attempts, - "rhumb": {"status": execute.status_code, "data": execute.body}, - "direct": {"status": direct.status_code, "data": direct.body}, - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Slack depth-10 runtime pass failed; inspect artifact for parity details") - - execution_id = execute_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", PROVIDER, - "--headline", f"Slack: depth-{target_depth} rerun confirms auth.test parity through Rhumb Resolve", - "--summary", f"Fresh depth-{target_depth} runtime rerun passed for Slack auth.test through Rhumb Resolve. The safe proxy auth identity read matched direct Slack control on workspace URL/name, workspace id, bot user identity, bot id, and enterprise-install flag.", - "--evidence-title", f"Slack depth-{target_depth} runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", f"Fresh depth-{target_depth} runtime rerun passed for Slack auth.test through Rhumb Resolve. The safe proxy auth identity read matched direct Slack control on normalized workspace and bot identity fields.", - "--source-ref", f"runtime-review:slack:{stamp}", - "--source-batch-id", f"runtime-review:slack:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "slack", - "--tag", "auth.test", - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "service_slug": PROVIDER, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_slack_depth4_20260401.py b/scripts/runtime_review_slack_depth4_20260401.py deleted file mode 100644 index 54e7f6cc..00000000 --- a/scripts/runtime_review_slack_depth4_20260401.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python3 -""" -Slack depth-4 runtime review pass. - -Previous passes (depth 1-3) all used POST /api/auth.test and verified -team/user/bot identity fields. This pass uses the same safe-read endpoint -but expands the parity assertion surface to include url, team_id, -is_enterprise_install, and the full set of identity fields β€” proving -the proxy continues to pass through a complete, unmodified Slack response. -""" - -import json -import os -import subprocess -import sys -from datetime import datetime, timezone - -API_BASE = os.environ.get("RHUMB_API_BASE", "https://api.rhumb.dev") - - -def get_admin_key(): - key = os.environ.get("RHUMB_ADMIN_KEY") - if key: - return key - try: - result = subprocess.run( - ["sop", "item", "get", "Rhumb Admin Secret (Railway)", "--vault", "OpenClaw Agents", - "--fields", "credential", "--reveal"], - capture_output=True, text=True, timeout=30 - ) - if result.returncode == 0 and result.stdout.strip(): - return result.stdout.strip() - except Exception: - pass - raise RuntimeError("No admin key available") - - -def get_slack_token(): - token = os.environ.get("RHUMB_CREDENTIAL_SLACK_BOT_TOKEN") - if token: - return token - try: - result = subprocess.run( - ["sop", "item", "get", "Slack - TeamSuper Bot Token", "--vault", "OpenClaw Agents", - "--fields", "credential", "--reveal"], - capture_output=True, text=True, timeout=30 - ) - if result.returncode == 0 and result.stdout.strip(): - return result.stdout.strip() - except Exception: - pass - raise RuntimeError("No Slack bot token available") - - -def api(method, path, headers=None, json_body=None): - import urllib.request - url = f"{API_BASE}{path}" - data = json.dumps(json_body).encode() if json_body else None - req = urllib.request.Request(url, data=data, method=method) - req.add_header("Content-Type", "application/json") - if headers: - for k, v in headers.items(): - if v is not None: - req.add_header(k, str(v)) - try: - with urllib.request.urlopen(req, timeout=30) as resp: - return resp.status, json.loads(resp.read()) - except urllib.error.HTTPError as e: - body = e.read() - try: - return e.code, json.loads(body) - except Exception: - return e.code, {"raw": body.decode(errors="replace")} - - -def main(): - ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") - admin_key = get_admin_key() - slack_token = get_slack_token() - admin_headers = {"X-Rhumb-Admin-Key": admin_key} - - # 1. Create temp agent - print("Creating temp agent...") - _, agent_resp = api("POST", "/v1/admin/agents", admin_headers, { - "organization_id": "org_rhumb_internal", - "name": f"slack-depth4-reviewer-{ts}", - }) - agent_id = agent_resp.get("agent_id") - api_key = agent_resp.get("api_key") - print(f"Agent: {agent_id}") - - if not api_key: - print(f"ERROR: No API key returned. Response: {json.dumps(agent_resp, indent=2)}") - return 1 - - # 2. Grant service access to slack - print("Granting slack access...") - grant_status, grant_resp = api("POST", f"/v1/admin/agents/{agent_id}/grant-access", admin_headers, { - "service": "slack" - }) - print(f"Grant: {grant_status} - {grant_resp.get('status', grant_resp.get('error', 'unknown'))}") - if grant_status not in (200, 201): - print(f"Grant failed: {json.dumps(grant_resp, indent=2)}") - return 1 - - agent_headers = {"X-Rhumb-Key": api_key} - - # 3. Managed execution: POST /api/auth.test via Rhumb proxy - print("\n--- Managed: POST /api/auth.test via Rhumb proxy ---") - managed_status, managed_resp = api("POST", "/v1/proxy/", agent_headers, { - "service": "slack", - "method": "POST", - "path": "/api/auth.test" - }) - print(f"HTTP status: {managed_status}") - - # Proxy returns body in the "body" key - managed_body = managed_resp.get("body", {}) - execution_id = managed_resp.get("execution_id", "unknown") - managed_latency = managed_resp.get("latency_ms") - managed_upstream_latency = managed_resp.get("upstream_latency_ms") - - managed_fields = { - "ok": managed_body.get("ok"), - "url": managed_body.get("url"), - "team": managed_body.get("team"), - "team_id": managed_body.get("team_id"), - "user": managed_body.get("user"), - "user_id": managed_body.get("user_id"), - "bot_id": managed_body.get("bot_id"), - "is_enterprise_install": managed_body.get("is_enterprise_install"), - } - print(f"Managed fields: {json.dumps(managed_fields, indent=2)}") - - # 4. Direct Slack control: auth.test - print("\n--- Direct: POST https://slack.com/api/auth.test ---") - import urllib.request - direct_req = urllib.request.Request( - "https://slack.com/api/auth.test", - method="POST", - data=b"" - ) - direct_req.add_header("Authorization", f"Bearer {slack_token}") - direct_req.add_header("Content-Type", "application/x-www-form-urlencoded") - try: - with urllib.request.urlopen(direct_req, timeout=30) as resp: - direct_status = resp.status - direct_data = json.loads(resp.read()) - except urllib.error.HTTPError as e: - direct_status = e.code - direct_data = json.loads(e.read()) - - direct_fields = { - "ok": direct_data.get("ok"), - "url": direct_data.get("url"), - "team": direct_data.get("team"), - "team_id": direct_data.get("team_id"), - "user": direct_data.get("user"), - "user_id": direct_data.get("user_id"), - "bot_id": direct_data.get("bot_id"), - "is_enterprise_install": direct_data.get("is_enterprise_install"), - } - print(f"Direct status: {direct_status}") - print(f"Direct fields: {json.dumps(direct_fields, indent=2)}") - - # 5. Parity check β€” expanded assertion surface - print("\n--- Parity check (expanded) ---") - parity_ok = True - for field in ["ok", "url", "team", "team_id", "user", "user_id", "bot_id", "is_enterprise_install"]: - mv = managed_fields.get(field) - dv = direct_fields.get(field) - match = mv == dv - print(f" {field}: managed={mv} direct={dv} match={match}") - if not match: - parity_ok = False - - # 6. Build artifact - artifact = { - "pass_id": f"slack-depth4-{ts}", - "provider": "slack", - "capability": "auth.test (expanded parity surface)", - "timestamp": ts, - "managed": { - "http_status": managed_status, - "fields": managed_fields, - "execution_id": execution_id, - "latency_ms": managed_latency, - "upstream_latency_ms": managed_upstream_latency, - }, - "direct": { - "http_status": direct_status, - "fields": direct_fields - }, - "parity": parity_ok, - "parity_fields_checked": ["ok", "url", "team", "team_id", "user", "user_id", "bot_id", "is_enterprise_install"], - "agent_id": agent_id, - "org_id": "org_rhumb_internal" - } - - out = f"artifacts/runtime-review-pass-{ts}-slack-depth4.json" - if len(sys.argv) > 2 and sys.argv[1] == "--json-out": - out = sys.argv[2] - os.makedirs(os.path.dirname(out) or ".", exist_ok=True) - with open(out, "w") as f: - json.dump(artifact, f, indent=2) - print(f"\nArtifact: {out}") - print(f"Parity: {'PASS' if parity_ok else 'FAIL'}") - - # 7. Disable temp agent - api("PUT", f"/v1/admin/agents/{agent_id}", admin_headers, {"enabled": False}) - print(f"Agent {agent_id} disabled") - - return 0 if parity_ok else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/runtime_review_slack_depth8_20260402.py b/scripts/runtime_review_slack_depth8_20260402.py deleted file mode 100644 index 241d3e6f..00000000 --- a/scripts/runtime_review_slack_depth8_20260402.py +++ /dev/null @@ -1,330 +0,0 @@ -#!/usr/bin/env python3 -"""Slack current-depth runtime review pass β€” 2026-04-02. - -Slack remains the sole weakest callable provider in the live public audit at depth 7. -This pass re-verifies the safe, non-mutating auth identity read (`POST /api/auth.test`) -through the Rhumb proxy and compares it directly against Slack. -""" -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -PROVIDER = "slack" -PUBLIC_SERVICE_SLUG = "slack" -METHOD = "POST" -PATH = "/api/auth.test" -DIRECT_URL = "https://slack.com/api/auth.test" -SLACK_ENV_VAR = "RHUMB_CREDENTIAL_SLACK_BOT_TOKEN" -SLACK_1PASSWORD_ITEM = "Slack - TeamSuper Bot Token" -RUNTIME_BACKED_TRUST_LABELS = {"🟒 Runtime-verified", "πŸ§ͺ Tester-generated"} - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - content: bytes | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - content=content, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label in RUNTIME_BACKED_TRUST_LABELS: - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _require_slack_token() -> tuple[str, str]: - env_value = os.environ.get(SLACK_ENV_VAR, "").strip() - if env_value: - return env_value, "environment" - - proc = subprocess.run( - [ - "sop", - "item", - "get", - SLACK_1PASSWORD_ITEM, - "--vault", - "OpenClaw Agents", - "--fields", - "credential", - "--reveal", - ], - cwd=REPO_ROOT, - capture_output=True, - text=True, - timeout=20, - check=False, - ) - if proc.returncode == 0 and proc.stdout.strip(): - return proc.stdout.strip(), "1password" - - raise RuntimeError( - f"Slack credential unavailable via {SLACK_ENV_VAR} or 1Password item {SLACK_1PASSWORD_ITEM!r}" - ) - - -def _pick_fields(payload: dict[str, Any] | None) -> dict[str, Any]: - payload = payload or {} - return { - "ok": payload.get("ok"), - "url": payload.get("url"), - "team": payload.get("team"), - "team_id": payload.get("team_id"), - "user": payload.get("user"), - "user_id": payload.get("user_id"), - "bot_id": payload.get("bot_id"), - "is_enterprise_install": payload.get("is_enterprise_install"), - } - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-slack-depth8.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-02-slack-depth8.json" - org_id = f"org_runtime_review_slack_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": PROVIDER, - "method": METHOD, - "path": PATH, - "review_kind": "current_depth8", - "organization_id": org_id, - "artifact_path": str(artifact_path), - } - _json_dump(artifact_path, payload) - - slack_token, token_source = _require_slack_token() - payload["direct_credential_source"] = token_source - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-slack-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-slack-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Slack depth-8 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "slack"], - ) - payload["review_agent"] = {"agent_id": agent_id, "organization_id": org_id} - - access_id = await store.grant_service_access(agent_id, PROVIDER) - payload["grant_access"] = {"status": "success", "access_id": access_id} - _json_dump(artifact_path, payload) - - try: - counts_before = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - current_depth = int(counts_before.get("runtime_backed_reviews") or 0) - target_depth = current_depth + 1 - - execute = await _request_json( - "POST", - f"{BASE_URL}/proxy/", - headers={"X-Rhumb-Key": api_key}, - json_body={ - "service": PROVIDER, - "method": METHOD, - "path": PATH, - }, - timeout=120.0, - ) - direct = await _request_json( - "POST", - DIRECT_URL, - headers={ - "Authorization": f"Bearer {slack_token}", - "Content-Type": "application/x-www-form-urlencoded", - "User-Agent": "rhumb-runtime-review/1.0", - }, - content=b"", - timeout=60.0, - ) - - execute_payload = execute.body if isinstance(execute.body, dict) else {} - rhumb_body = execute_payload.get("body") if isinstance(execute_payload.get("body"), dict) else {} - rhumb_fields = _pick_fields(rhumb_body) - direct_fields = _pick_fields(direct.body if isinstance(direct.body, dict) else {}) - - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_fields": rhumb_fields, - "direct_fields": direct_fields, - "parity": rhumb_fields == direct_fields, - } - - verdict = "pass" if ( - execute.status_code == 200 - and direct.status_code == 200 - and comparison["parity"] - ) else "fail" - - payload.update( - { - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "target_depth": target_depth, - "rhumb": {"status": execute.status_code, "data": execute.body}, - "direct": {"status": direct.status_code, "data": direct.body}, - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Slack depth-8 runtime pass failed; inspect artifact for parity details") - - execution_id = execute_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", PROVIDER, - "--headline", f"Slack: depth-{target_depth} rerun confirms auth.test parity through Rhumb Resolve", - "--summary", f"Fresh depth-{target_depth} runtime rerun passed for Slack auth.test through Rhumb Resolve. The safe proxy auth identity read matched direct Slack control on workspace URL/name, workspace id, bot user identity, bot id, and enterprise-install flag.", - "--evidence-title", f"Slack depth-{target_depth} runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", f"Fresh depth-{target_depth} runtime rerun passed for Slack auth.test through Rhumb Resolve. The safe proxy auth identity read matched direct Slack control on normalized workspace and bot identity fields.", - "--source-ref", f"runtime-review:slack:{stamp}", - "--source-batch-id", f"runtime-review:slack:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "slack", - "--tag", "auth.test", - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "service_slug": PROVIDER, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_slack_depth9_20260402.py b/scripts/runtime_review_slack_depth9_20260402.py deleted file mode 100644 index 986733f6..00000000 --- a/scripts/runtime_review_slack_depth9_20260402.py +++ /dev/null @@ -1,330 +0,0 @@ -#!/usr/bin/env python3 -"""Slack current-depth runtime review pass β€” 2026-04-02. - -Slack remains the sole weakest callable provider in the live public audit at depth 8. -This pass re-verifies the safe, non-mutating auth identity read (`POST /api/auth.test`) -through the Rhumb proxy and compares it directly against Slack. -""" -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -PROVIDER = "slack" -PUBLIC_SERVICE_SLUG = "slack" -METHOD = "POST" -PATH = "/api/auth.test" -DIRECT_URL = "https://slack.com/api/auth.test" -SLACK_ENV_VAR = "RHUMB_CREDENTIAL_SLACK_BOT_TOKEN" -SLACK_1PASSWORD_ITEM = "Slack - TeamSuper Bot Token" -RUNTIME_BACKED_TRUST_LABELS = {"🟒 Runtime-verified", "πŸ§ͺ Tester-generated"} - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - content: bytes | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - content=content, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label in RUNTIME_BACKED_TRUST_LABELS: - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _require_slack_token() -> tuple[str, str]: - env_value = os.environ.get(SLACK_ENV_VAR, "").strip() - if env_value: - return env_value, "environment" - - proc = subprocess.run( - [ - "sop", - "item", - "get", - SLACK_1PASSWORD_ITEM, - "--vault", - "OpenClaw Agents", - "--fields", - "credential", - "--reveal", - ], - cwd=REPO_ROOT, - capture_output=True, - text=True, - timeout=20, - check=False, - ) - if proc.returncode == 0 and proc.stdout.strip(): - return proc.stdout.strip(), "1password" - - raise RuntimeError( - f"Slack credential unavailable via {SLACK_ENV_VAR} or 1Password item {SLACK_1PASSWORD_ITEM!r}" - ) - - -def _pick_fields(payload: dict[str, Any] | None) -> dict[str, Any]: - payload = payload or {} - return { - "ok": payload.get("ok"), - "url": payload.get("url"), - "team": payload.get("team"), - "team_id": payload.get("team_id"), - "user": payload.get("user"), - "user_id": payload.get("user_id"), - "bot_id": payload.get("bot_id"), - "is_enterprise_install": payload.get("is_enterprise_install"), - } - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-slack-depth9.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-02-slack-depth9.json" - org_id = f"org_runtime_review_slack_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": PROVIDER, - "method": METHOD, - "path": PATH, - "review_kind": "current_depth9", - "organization_id": org_id, - "artifact_path": str(artifact_path), - } - _json_dump(artifact_path, payload) - - slack_token, token_source = _require_slack_token() - payload["direct_credential_source"] = token_source - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-slack-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-slack-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Slack depth-9 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "slack"], - ) - payload["review_agent"] = {"agent_id": agent_id, "organization_id": org_id} - - access_id = await store.grant_service_access(agent_id, PROVIDER) - payload["grant_access"] = {"status": "success", "access_id": access_id} - _json_dump(artifact_path, payload) - - try: - counts_before = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - current_depth = int(counts_before.get("runtime_backed_reviews") or 0) - target_depth = current_depth + 1 - - execute = await _request_json( - "POST", - f"{BASE_URL}/proxy/", - headers={"X-Rhumb-Key": api_key}, - json_body={ - "service": PROVIDER, - "method": METHOD, - "path": PATH, - }, - timeout=120.0, - ) - direct = await _request_json( - "POST", - DIRECT_URL, - headers={ - "Authorization": f"Bearer {slack_token}", - "Content-Type": "application/x-www-form-urlencoded", - "User-Agent": "rhumb-runtime-review/1.0", - }, - content=b"", - timeout=60.0, - ) - - execute_payload = execute.body if isinstance(execute.body, dict) else {} - rhumb_body = execute_payload.get("body") if isinstance(execute_payload.get("body"), dict) else {} - rhumb_fields = _pick_fields(rhumb_body) - direct_fields = _pick_fields(direct.body if isinstance(direct.body, dict) else {}) - - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_fields": rhumb_fields, - "direct_fields": direct_fields, - "parity": rhumb_fields == direct_fields, - } - - verdict = "pass" if ( - execute.status_code == 200 - and direct.status_code == 200 - and comparison["parity"] - ) else "fail" - - payload.update( - { - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "target_depth": target_depth, - "rhumb": {"status": execute.status_code, "data": execute.body}, - "direct": {"status": direct.status_code, "data": direct.body}, - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Slack depth-9 runtime pass failed; inspect artifact for parity details") - - execution_id = execute_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", PROVIDER, - "--headline", f"Slack: depth-{target_depth} rerun confirms auth.test parity through Rhumb Resolve", - "--summary", f"Fresh depth-{target_depth} runtime rerun passed for Slack auth.test through Rhumb Resolve. The safe proxy auth identity read matched direct Slack control on workspace URL/name, workspace id, bot user identity, bot id, and enterprise-install flag.", - "--evidence-title", f"Slack depth-{target_depth} runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", f"Fresh depth-{target_depth} runtime rerun passed for Slack auth.test through Rhumb Resolve. The safe proxy auth identity read matched direct Slack control on normalized workspace and bot identity fields.", - "--source-ref", f"runtime-review:slack:{stamp}", - "--source-batch-id", f"runtime-review:slack:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "slack", - "--tag", "auth.test", - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "service_slug": PROVIDER, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_stripe_depth10_20260403.py b/scripts/runtime_review_stripe_depth10_20260403.py deleted file mode 100644 index 126ea3ec..00000000 --- a/scripts/runtime_review_stripe_depth10_20260403.py +++ /dev/null @@ -1,403 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -import uuid -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -SERVICE_SLUG = "stripe" -CAPABILITY_ID = "wallet.get_balance" -METHOD = "GET" -PATH = "/v1/balance" -DIRECT_URL = "https://api.stripe.com/v1/balance" -STRIPE_ENV_VAR = "RHUMB_CREDENTIAL_STRIPE_API_KEY" -STRIPE_1PASSWORD_ITEM = "Stripe Test Secret Key (Rhumb)" -POST_GRANT_PROPAGATION_DELAY_SECONDS = 5 -ESTIMATE_AUTH_RETRY_ATTEMPTS = 4 -ESTIMATE_AUTH_RETRY_DELAY_SECONDS = 5 - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _looks_like_invalid_key(body: Any) -> bool: - if not isinstance(body, dict): - return False - detail = str(body.get("detail") or "").lower() - error = str(body.get("error") or "").lower() - return "invalid or expired rhumb api key" in detail or "invalid or expired rhumb api key" in error - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request(method, url, headers=headers, params=params, json=json_body) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -RUNTIME_BACKED_TRUST_LABELS = {"🟒 Runtime-verified", "πŸ§ͺ Tester-generated"} - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label in RUNTIME_BACKED_TRUST_LABELS: - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _require_stripe_key() -> tuple[str, str]: - env_value = os.environ.get(STRIPE_ENV_VAR, "").strip() - if env_value: - return env_value, "environment" - - proc = subprocess.run( - [ - "sop", - "item", - "get", - STRIPE_1PASSWORD_ITEM, - "--vault", - "OpenClaw Agents", - "--fields", - "credential", - "--reveal", - ], - cwd=REPO_ROOT, - capture_output=True, - text=True, - timeout=20, - check=False, - ) - if proc.returncode == 0 and proc.stdout.strip(): - return proc.stdout.strip(), "1password" - - raise RuntimeError( - f"Stripe credential unavailable via {STRIPE_ENV_VAR} or 1Password item {STRIPE_1PASSWORD_ITEM!r}" - ) - - -def _find_currency_row(rows: Any, currency: str = "usd") -> dict[str, Any]: - if isinstance(rows, list): - for row in rows: - if isinstance(row, dict) and row.get("currency") == currency: - return row - for row in rows: - if isinstance(row, dict): - return row - return {} - - -def _pick_fields(payload: dict[str, Any] | None) -> dict[str, Any]: - payload = payload or {} - available = _find_currency_row(payload.get("available")) - pending = _find_currency_row(payload.get("pending")) - refund_prefunding = payload.get("refund_and_dispute_prefunding") - if not isinstance(refund_prefunding, dict): - refund_prefunding = {} - refund_available = _find_currency_row(refund_prefunding.get("available")) - refund_pending = _find_currency_row(refund_prefunding.get("pending")) - return { - "object": payload.get("object"), - "livemode": payload.get("livemode"), - "available": { - "currency": available.get("currency"), - "amount": available.get("amount"), - "card_amount": ((available.get("source_types") or {}).get("card")) if isinstance(available.get("source_types"), dict) else None, - }, - "pending": { - "currency": pending.get("currency"), - "amount": pending.get("amount"), - "card_amount": ((pending.get("source_types") or {}).get("card")) if isinstance(pending.get("source_types"), dict) else None, - }, - "refund_and_dispute_prefunding": { - "available": { - "currency": refund_available.get("currency"), - "amount": refund_available.get("amount"), - }, - "pending": { - "currency": refund_pending.get("currency"), - "amount": refund_pending.get("amount"), - }, - }, - } - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - short = uuid.uuid4().hex[:8] - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-stripe-depth10.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-03-stripe-depth10.json" - org_id = f"org_runtime_review_stripe_{stamp.lower()}_{short}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "capability_id": CAPABILITY_ID, - "method": METHOD, - "path": PATH, - "review_kind": "current_depth10", - "organization_id": org_id, - "artifact_path": str(artifact_path.relative_to(REPO_ROOT)), - "started_at": started_at.replace(microsecond=0).isoformat().replace("+00:00", "Z"), - } - _json_dump(artifact_path, payload) - - stripe_api_key, key_source = _require_stripe_key() - payload["direct_credential_source"] = key_source - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-stripe-{stamp}", - starter_credits_cents=5000, - ) - payload["wallet_bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-stripe-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Stripe depth-10 runtime pass", - tags=["runtime_review", "phase3", "current_pass", SERVICE_SLUG, "depth10"], - ) - payload["review_agent"] = {"agent_id": agent_id, "organization_id": org_id} - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = {"status": "success", "access_id": access_id} - payload["post_grant_delay_seconds"] = POST_GRANT_PROPAGATION_DELAY_SECONDS - _json_dump(artifact_path, payload) - - await asyncio.sleep(POST_GRANT_PROPAGATION_DELAY_SECONDS) - - api_headers = {"X-Rhumb-Key": api_key} - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - - estimate_attempts = 0 - estimate = HttpResult(status_code=0, body=None) - while estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS: - estimate_attempts += 1 - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "byo"}, - timeout=60.0, - ) - if not ( - estimate.status_code == 401 - and _looks_like_invalid_key(estimate.body) - and estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS - ): - break - await asyncio.sleep(ESTIMATE_AUTH_RETRY_DELAY_SECONDS) - - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - json_body={ - "provider": SERVICE_SLUG, - "credential_mode": "byo", - "method": METHOD, - "path": PATH, - "interface": "runtime_review", - }, - timeout=120.0, - ) - direct = await _request_json( - "GET", - DIRECT_URL, - headers={ - "Authorization": f"Bearer {stripe_api_key}", - "User-Agent": "rhumb-runtime-review/1.0", - }, - timeout=120.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - rhumb_fields = _pick_fields(rhumb_upstream if isinstance(rhumb_upstream, dict) else {}) - direct_fields = _pick_fields(direct.body if isinstance(direct.body, dict) else {}) - field_matches = { - "object_match": rhumb_fields.get("object") == direct_fields.get("object"), - "livemode_match": rhumb_fields.get("livemode") == direct_fields.get("livemode"), - "available_match": rhumb_fields.get("available") == direct_fields.get("available"), - "pending_match": rhumb_fields.get("pending") == direct_fields.get("pending"), - "refund_prefunding_match": rhumb_fields.get("refund_and_dispute_prefunding") == direct_fields.get("refund_and_dispute_prefunding"), - } - - observed_at = _iso_now() - fresh_until = ( - datetime.fromisoformat(observed_at.replace("Z", "+00:00")) + timedelta(days=30) - ).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "provider_used": rhumb_payload.get("provider_used"), - "upstream_status": rhumb_payload.get("upstream_status"), - "execution_id": rhumb_payload.get("execution_id"), - "rhumb_fields": rhumb_fields, - "direct_fields": direct_fields, - "field_matches": field_matches, - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 200 - and all(field_matches.values()) - ) else "fail" - - payload.update( - { - "reviewed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": { - "status_code": estimate.status_code, - "body": estimate.body, - "attempts": estimate_attempts, - }, - "rhumb_execute": asdict(execute), - "direct": asdict(direct), - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Stripe depth-10 runtime pass failed; inspect artifact for parity details") - - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "Stripe: depth-10 runtime review confirms wallet.get_balance parity through Rhumb Resolve", - "--summary", "Fresh depth-10 runtime review passed for Stripe wallet.get_balance through Rhumb Resolve. Managed and direct executions matched on balance object, livemode, available funds, pending funds, and refund/dispute prefunding amounts.", - "--evidence-title", "Stripe depth-10 runtime review parity check via Rhumb Resolve", - "--evidence-summary", "Fresh depth-10 runtime review passed for Stripe wallet.get_balance through Rhumb Resolve. Managed and direct executions matched on balance object, livemode, available funds, pending funds, and refund/dispute prefunding amounts.", - "--source-ref", f"runtime-review:stripe:{stamp}", - "--source-batch-id", f"runtime-review:stripe:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", rhumb_payload.get("execution_id") or "", - "--tag", "runtime_review", - "--tag", SERVICE_SLUG, - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--tag", "depth10", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["publication"] = publication - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["disable_agent"] = { - "status_code": 200 if disabled else 404, - "body": {"status": "success" if disabled else "not_found", "agent_id": agent_id}, - } - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_stripe_depth4_20260331.py b/scripts/runtime_review_stripe_depth4_20260331.py deleted file mode 100644 index 35072eec..00000000 --- a/scripts/runtime_review_stripe_depth4_20260331.py +++ /dev/null @@ -1,310 +0,0 @@ -#!/usr/bin/env python3 -"""Stripe current-depth4 runtime review pass β€” 2026-03-31.""" -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -PROVIDER = "stripe" -PUBLIC_SERVICE_SLUG = "stripe" -DIRECT_URL = "https://api.stripe.com/v1/account" -STRIPE_ENV_VAR = "RHUMB_CREDENTIAL_STRIPE_API_KEY" -STRIPE_1PASSWORD_ITEM = "Stripe Test Secret Key (Rhumb)" - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request(method, url, headers=headers, params=params, json=json_body) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _require_stripe_key() -> tuple[str, str]: - env_value = os.environ.get(STRIPE_ENV_VAR, "").strip() - if env_value: - return env_value, "environment" - - proc = subprocess.run( - [ - "sop", - "item", - "get", - STRIPE_1PASSWORD_ITEM, - "--vault", - "OpenClaw Agents", - "--fields", - "credential", - "--reveal", - ], - cwd=REPO_ROOT, - capture_output=True, - text=True, - timeout=20, - check=False, - ) - if proc.returncode == 0 and proc.stdout.strip(): - return proc.stdout.strip(), "1password" - - raise RuntimeError( - f"Stripe credential unavailable via {STRIPE_ENV_VAR} or 1Password item {STRIPE_1PASSWORD_ITEM!r}" - ) - - -def _pick_fields(payload: dict[str, Any] | None) -> dict[str, Any]: - payload = payload or {} - settings = payload.get("settings") if isinstance(payload.get("settings"), dict) else {} - branding = settings.get("branding") if isinstance(settings.get("branding"), dict) else {} - return { - "id": payload.get("id"), - "country": payload.get("country"), - "default_currency": payload.get("default_currency"), - "charges_enabled": payload.get("charges_enabled"), - "payouts_enabled": payload.get("payouts_enabled"), - "details_submitted": payload.get("details_submitted"), - "business_type": payload.get("business_type"), - "branding_icon": branding.get("icon"), - } - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-stripe-depth4.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-31-stripe-depth4.json" - org_id = f"org_runtime_review_stripe_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": PROVIDER, - "capability_mode": "proxy_safe_read", - "path": "/v1/account", - "review_kind": "current_depth4", - "organization_id": org_id, - "artifact_path": str(artifact_path), - } - _json_dump(artifact_path, payload) - - stripe_api_key, key_source = _require_stripe_key() - payload["direct_credential_source"] = key_source - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-stripe-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-stripe-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Stripe depth-4 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "stripe"], - ) - payload["review_agent"] = {"agent_id": agent_id, "organization_id": org_id} - - access_id = await store.grant_service_access(agent_id, PROVIDER) - payload["grant_access"] = {"status": "success", "access_id": access_id} - _json_dump(artifact_path, payload) - - try: - counts_before = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - - proxy = await _request_json( - "POST", - f"{BASE_URL}/proxy/", - headers={"X-Rhumb-Key": api_key}, - json_body={ - "service": PROVIDER, - "method": "GET", - "path": "/v1/account", - }, - timeout=120.0, - ) - - direct = await _request_json( - "GET", - DIRECT_URL, - headers={ - "Authorization": f"Bearer {stripe_api_key}", - "User-Agent": "rhumb-runtime-review/1.0", - }, - timeout=120.0, - ) - - proxy_body = proxy.body if isinstance(proxy.body, dict) else {} - rhumb_upstream = proxy_body.get("body") if isinstance(proxy_body.get("body"), dict) else {} - rhumb_fields = _pick_fields(rhumb_upstream) - direct_fields = _pick_fields(direct.body if isinstance(direct.body, dict) else {}) - - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_fields": rhumb_fields, - "direct_fields": direct_fields, - "parity": rhumb_fields == direct_fields, - } - - verdict = "pass" if ( - proxy.status_code == 200 - and proxy_body.get("status_code") == 200 - and direct.status_code == 200 - and comparison["parity"] - ) else "fail" - - payload.update( - { - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "rhumb": {"status": proxy.status_code, "data": proxy.body}, - "direct": {"status": direct.status_code, "data": direct.body}, - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Stripe depth-4 runtime pass failed; inspect artifact for parity details") - - publication = _run_publish( - [ - "--service", PROVIDER, - "--headline", "Stripe: current-depth rerun confirms safe account-read parity through Rhumb Resolve at depth 4", - "--summary", "Fresh current-depth runtime rerun passed for Stripe through Rhumb Resolve on the safe non-mutating GET /v1/account path. Rhumb proxy execution and direct Stripe control matched on account identity, country, currency, and account-state fields, lifting Stripe from claim-safe depth 3 to 4 in the callable rotation.", - "--evidence-title", "Stripe current-depth runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh current-depth runtime rerun passed for Stripe through Rhumb Resolve on the safe non-mutating GET /v1/account path. Rhumb proxy execution matched direct Stripe control on account identity, country, currency, and account-state fields.", - "--source-ref", f"runtime-review:stripe:{stamp}", - "--source-batch-id", f"runtime-review:stripe:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--tag", "runtime_review", - "--tag", "stripe", - "--tag", "proxy.safe_read", - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "service_slug": PROVIDER, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - } - payload["counts_after"] = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_stripe_depth8_20260402.py b/scripts/runtime_review_stripe_depth8_20260402.py deleted file mode 100644 index f0882a64..00000000 --- a/scripts/runtime_review_stripe_depth8_20260402.py +++ /dev/null @@ -1,368 +0,0 @@ -#!/usr/bin/env python3 -"""Stripe current-depth runtime review pass β€” 2026-04-02. - -Mission 1 requires a real capability execute through Rhumb Resolve, not just a raw -proxy pass. This rail uses the existing BYO execute path for Stripe on a safe, -non-mutating wallet balance read. -""" -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -PROVIDER = "stripe" -PUBLIC_SERVICE_SLUG = "stripe" -CAPABILITY_ID = "wallet.get_balance" -METHOD = "GET" -PATH = "/v1/balance" -DIRECT_URL = "https://api.stripe.com/v1/balance" -STRIPE_ENV_VAR = "RHUMB_CREDENTIAL_STRIPE_API_KEY" -STRIPE_1PASSWORD_ITEM = "Stripe Test Secret Key (Rhumb)" - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request(method, url, headers=headers, params=params, json=json_body) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -RUNTIME_BACKED_TRUST_LABELS = {"🟒 Runtime-verified", "πŸ§ͺ Tester-generated"} - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label in RUNTIME_BACKED_TRUST_LABELS: - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _require_stripe_key() -> tuple[str, str]: - env_value = os.environ.get(STRIPE_ENV_VAR, "").strip() - if env_value: - return env_value, "environment" - - proc = subprocess.run( - [ - "sop", - "item", - "get", - STRIPE_1PASSWORD_ITEM, - "--vault", - "OpenClaw Agents", - "--fields", - "credential", - "--reveal", - ], - cwd=REPO_ROOT, - capture_output=True, - text=True, - timeout=20, - check=False, - ) - if proc.returncode == 0 and proc.stdout.strip(): - return proc.stdout.strip(), "1password" - - raise RuntimeError( - f"Stripe credential unavailable via {STRIPE_ENV_VAR} or 1Password item {STRIPE_1PASSWORD_ITEM!r}" - ) - - -def _find_currency_row(rows: Any, currency: str = "usd") -> dict[str, Any]: - if isinstance(rows, list): - for row in rows: - if isinstance(row, dict) and row.get("currency") == currency: - return row - for row in rows: - if isinstance(row, dict): - return row - return {} - - -def _pick_fields(payload: dict[str, Any] | None) -> dict[str, Any]: - payload = payload or {} - available = _find_currency_row(payload.get("available")) - pending = _find_currency_row(payload.get("pending")) - refund_prefunding = payload.get("refund_and_dispute_prefunding") - if not isinstance(refund_prefunding, dict): - refund_prefunding = {} - refund_available = _find_currency_row(refund_prefunding.get("available")) - refund_pending = _find_currency_row(refund_prefunding.get("pending")) - return { - "object": payload.get("object"), - "livemode": payload.get("livemode"), - "available": { - "currency": available.get("currency"), - "amount": available.get("amount"), - "card_amount": ((available.get("source_types") or {}).get("card")) if isinstance(available.get("source_types"), dict) else None, - }, - "pending": { - "currency": pending.get("currency"), - "amount": pending.get("amount"), - "card_amount": ((pending.get("source_types") or {}).get("card")) if isinstance(pending.get("source_types"), dict) else None, - }, - "refund_and_dispute_prefunding": { - "available": { - "currency": refund_available.get("currency"), - "amount": refund_available.get("amount"), - }, - "pending": { - "currency": refund_pending.get("currency"), - "amount": refund_pending.get("amount"), - }, - }, - } - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-stripe-depth8.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-02-stripe-depth8.json" - org_id = f"org_runtime_review_stripe_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": PROVIDER, - "capability_id": CAPABILITY_ID, - "method": METHOD, - "path": PATH, - "review_kind": "current_depth8", - "organization_id": org_id, - "artifact_path": str(artifact_path), - } - _json_dump(artifact_path, payload) - - stripe_api_key, key_source = _require_stripe_key() - payload["direct_credential_source"] = key_source - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-stripe-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-stripe-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Stripe depth-8 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "stripe"], - ) - payload["review_agent"] = {"agent_id": agent_id, "organization_id": org_id} - - access_id = await store.grant_service_access(agent_id, PROVIDER) - payload["grant_access"] = {"status": "success", "access_id": access_id} - _json_dump(artifact_path, payload) - - try: - counts_before = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - current_depth = int(counts_before.get("runtime_backed_reviews") or 0) - target_depth = current_depth + 1 - - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers={"X-Rhumb-Key": api_key}, - params={"provider": PROVIDER, "credential_mode": "byo"}, - timeout=60.0, - ) - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers={"X-Rhumb-Key": api_key}, - json_body={ - "provider": PROVIDER, - "credential_mode": "byo", - "method": METHOD, - "path": PATH, - }, - timeout=120.0, - ) - direct = await _request_json( - "GET", - DIRECT_URL, - headers={ - "Authorization": f"Bearer {stripe_api_key}", - "User-Agent": "rhumb-runtime-review/1.0", - }, - timeout=120.0, - ) - - execute_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (execute_payload.get("upstream_response") or {}) if isinstance(execute_payload, dict) else {} - rhumb_fields = _pick_fields(rhumb_upstream if isinstance(rhumb_upstream, dict) else {}) - direct_fields = _pick_fields(direct.body if isinstance(direct.body, dict) else {}) - - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_fields": rhumb_fields, - "direct_fields": direct_fields, - "parity": rhumb_fields == direct_fields, - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and execute_payload.get("provider_used") == PROVIDER - and execute_payload.get("upstream_status") == 200 - and comparison["parity"] - ) else "fail" - - payload.update( - { - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "target_depth": target_depth, - "estimate": {"status": estimate.status_code, "data": estimate.body}, - "rhumb": {"status": execute.status_code, "data": execute.body}, - "direct": {"status": direct.status_code, "data": direct.body}, - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Stripe depth-8 runtime pass failed; inspect artifact for parity details") - - execution_id = execute_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", PROVIDER, - "--headline", f"Stripe: depth-{target_depth} rerun confirms wallet.get_balance parity through Rhumb Resolve", - "--summary", f"Fresh depth-{target_depth} runtime rerun passed for Stripe wallet.get_balance through Rhumb Resolve. The execute-route BYO call on GET /v1/balance matched direct Stripe control on balance object, livemode, available funds, pending funds, and refund/dispute prefunding amounts.", - "--evidence-title", f"Stripe depth-{target_depth} runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", f"Fresh depth-{target_depth} runtime rerun passed for Stripe wallet.get_balance through Rhumb Resolve. The execute-route BYO call on GET /v1/balance matched direct Stripe control on the normalized balance fields.", - "--source-ref", f"runtime-review:stripe:{stamp}", - "--source-batch-id", f"runtime-review:stripe:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "stripe", - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "service_slug": PROVIDER, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_tavily_depth11_20260403.py b/scripts/runtime_review_tavily_depth11_20260403.py deleted file mode 100644 index e71fbd31..00000000 --- a/scripts/runtime_review_tavily_depth11_20260403.py +++ /dev/null @@ -1,351 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -import uuid -from dataclasses import asdict, dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -TAVILY_URL = "https://api.tavily.com/search" -SERVICE_SLUG = "tavily" -CAPABILITY_ID = "search.query" -QUERY = "LLM observability tools" -SEARCH_DEPTH = "basic" -MAX_RESULTS = 5 -POST_GRANT_PROPAGATION_DELAY_SECONDS = 5 -ESTIMATE_AUTH_RETRY_ATTEMPTS = 4 -ESTIMATE_AUTH_RETRY_DELAY_SECONDS = 5 - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _looks_like_invalid_key(body: Any) -> bool: - if not isinstance(body, dict): - return False - detail = str(body.get("detail") or "").lower() - error = str(body.get("error") or "").lower() - return "invalid or expired rhumb api key" in detail or "invalid or expired rhumb api key" in error - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request(method, url, headers=headers, params=params, json=json_body) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label in {"🟒 Runtime-verified", "πŸ§ͺ Tester-generated"}: - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _results(body: Any) -> list[dict[str, Any]]: - if not isinstance(body, dict): - return [] - results = body.get("results") - return results if isinstance(results, list) else [] - - -def _top_result(results: list[dict[str, Any]]) -> dict[str, Any] | None: - if not results: - return None - item = results[0] - if not isinstance(item, dict): - return None - return { - "title": item.get("title"), - "url": item.get("url"), - "content": item.get("content"), - "score": item.get("score"), - } - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - short = uuid.uuid4().hex[:8] - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-tavily-depth11.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-03-tavily-depth11.json" - org_id = f"org_runtime_review_tavily_{stamp.lower()}_{short}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "capability_id": CAPABILITY_ID, - "query": QUERY, - "search_depth": SEARCH_DEPTH, - "max_results": MAX_RESULTS, - "review_kind": "current_depth11", - "organization_id": org_id, - "artifact_path": str(artifact_path.relative_to(REPO_ROOT)), - "started_at": started_at.replace(microsecond=0).isoformat().replace("+00:00", "Z"), - "post_grant_delay_seconds": POST_GRANT_PROPAGATION_DELAY_SECONDS, - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-tavily-{stamp}", - starter_credits_cents=5000, - ) - payload["wallet_bootstrap"] = bootstrap - _json_dump(artifact_path, payload) - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-tavily-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Tavily depth-11 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "tavily", "depth11"], - ) - payload["review_agent"] = {"agent_id": agent_id, "organization_id": org_id} - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = {"status": "success", "access_id": access_id} - _json_dump(artifact_path, payload) - - await asyncio.sleep(POST_GRANT_PROPAGATION_DELAY_SECONDS) - - api_headers = {"X-Rhumb-Key": api_key} - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - - estimate_attempts = 0 - estimate = HttpResult(status_code=0, body=None) - while estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS: - estimate_attempts += 1 - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - if not ( - estimate.status_code == 401 - and _looks_like_invalid_key(estimate.body) - and estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS - ): - break - await asyncio.sleep(ESTIMATE_AUTH_RETRY_DELAY_SECONDS) - - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body={ - "query": QUERY, - "search_depth": SEARCH_DEPTH, - "max_results": MAX_RESULTS, - }, - timeout=120.0, - ) - direct = await _request_json( - "POST", - TAVILY_URL, - headers={"Content-Type": "application/json"}, - json_body={ - "api_key": os.environ["RHUMB_CREDENTIAL_TAVILY_API_KEY"], - "query": QUERY, - "search_depth": SEARCH_DEPTH, - "max_results": MAX_RESULTS, - }, - timeout=120.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - rhumb_results = _results(rhumb_upstream) - direct_results = _results(direct.body) - rhumb_top = _top_result(rhumb_results) - direct_top = _top_result(direct_results) - rhumb_top3_urls = [row.get("url") for row in rhumb_results[:3] if isinstance(row, dict)] - direct_top3_urls = [row.get("url") for row in direct_results[:3] if isinstance(row, dict)] - - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "provider_used": rhumb_payload.get("provider_used"), - "upstream_status": rhumb_payload.get("upstream_status"), - "execution_id": rhumb_payload.get("execution_id"), - "rhumb_count": len(rhumb_results), - "direct_count": len(direct_results), - "rhumb_top": rhumb_top, - "direct_top": direct_top, - "rhumb_top3_urls": rhumb_top3_urls, - "direct_top3_urls": direct_top3_urls, - "result_count_match": len(rhumb_results) == len(direct_results) == MAX_RESULTS, - "top_title_match": (rhumb_top or {}).get("title") == (direct_top or {}).get("title"), - "top_url_match": (rhumb_top or {}).get("url") == (direct_top or {}).get("url"), - "top_content_match": (rhumb_top or {}).get("content") == (direct_top or {}).get("content"), - "top3_urls_match": rhumb_top3_urls == direct_top3_urls, - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["result_count_match"] - and comparison["top_title_match"] - and comparison["top_url_match"] - and comparison["top_content_match"] - and comparison["top3_urls_match"] - ) else "fail" - - payload.update( - { - "reviewed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": { - "status_code": estimate.status_code, - "body": estimate.body, - "attempts": estimate_attempts, - }, - "rhumb_execute": asdict(execute), - "direct": asdict(direct), - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Tavily depth-11 runtime pass failed; inspect artifact for parity details") - - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "Tavily: depth-11 runtime review confirms search.query parity through Rhumb Resolve", - "--summary", "Fresh depth-11 runtime review passed for Tavily search.query through Rhumb Resolve. Managed and direct executions matched on result count, top result fields, and top-3 URL ordering for the same live query.", - "--evidence-title", "Tavily depth-11 runtime review parity check via Rhumb Resolve", - "--evidence-summary", "Fresh depth-11 runtime review passed for Tavily search.query through Rhumb Resolve. Managed and direct executions matched on result count, top result fields, and top-3 URL ordering for the same live query.", - "--source-ref", f"runtime-review:tavily:{stamp}", - "--source-batch-id", f"runtime-review:tavily:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", rhumb_payload.get("execution_id") or "", - "--tag", "runtime_review", - "--tag", "tavily", - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--tag", "depth11", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["publication"] = publication - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["disable_agent"] = { - "status_code": 200 if disabled else 404, - "body": {"status": "success" if disabled else "not_found", "agent_id": agent_id}, - } - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_tavily_depth5_20260330.py b/scripts/runtime_review_tavily_depth5_20260330.py deleted file mode 100644 index b4d86b53..00000000 --- a/scripts/runtime_review_tavily_depth5_20260330.py +++ /dev/null @@ -1,307 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -TAVILY_URL = "https://api.tavily.com/search" -SERVICE_SLUG = "tavily" -CAPABILITY_ID = "search.query" -QUERY = "best AI agent observability tools" -SEARCH_DEPTH = "basic" -MAX_RESULTS = 5 - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-tavily-current-depth5.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-30-tavily-depth5.json" - org_id = f"org_runtime_review_tavily_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "capability_id": CAPABILITY_ID, - "query": QUERY, - "search_depth": SEARCH_DEPTH, - "max_results": MAX_RESULTS, - "review_kind": "current_depth5", - "organization_id": org_id, - "artifact_path": str(artifact_path), - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-tavily-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-tavily-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Tavily depth-5 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "tavily"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = { - "status": "success", - "access_id": access_id, - } - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - direct_headers = {"Content-Type": "application/json"} - direct_body = { - "api_key": os.environ["RHUMB_CREDENTIAL_TAVILY_API_KEY"], - "query": QUERY, - "search_depth": SEARCH_DEPTH, - "max_results": MAX_RESULTS, - } - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body={ - "query": QUERY, - "search_depth": SEARCH_DEPTH, - "max_results": MAX_RESULTS, - }, - timeout=120.0, - ) - direct = await _request_json( - "POST", - TAVILY_URL, - headers=direct_headers, - json_body=direct_body, - timeout=120.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - rhumb_results = rhumb_upstream.get("results") if isinstance(rhumb_upstream, dict) else [] - direct_results = direct.body.get("results") if isinstance(direct.body, dict) else [] - rhumb_top = rhumb_results[0] if isinstance(rhumb_results, list) and rhumb_results else None - direct_top = direct_results[0] if isinstance(direct_results, list) and direct_results else None - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_provider_used": rhumb_payload.get("provider_used"), - "rhumb_upstream_status": rhumb_payload.get("upstream_status"), - "rhumb_count": len(rhumb_results) if isinstance(rhumb_results, list) else None, - "direct_count": len(direct_results) if isinstance(direct_results, list) else None, - "rhumb_top": { - "title": (rhumb_top or {}).get("title"), - "url": (rhumb_top or {}).get("url"), - }, - "direct_top": { - "title": (direct_top or {}).get("title"), - "url": (direct_top or {}).get("url"), - }, - "result_count_match": isinstance(rhumb_results, list) and isinstance(direct_results, list) and len(rhumb_results) == len(direct_results), - "top_title_match": (rhumb_top or {}).get("title") == (direct_top or {}).get("title"), - "top_url_match": (rhumb_top or {}).get("url") == (direct_top or {}).get("url"), - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["result_count_match"] - and comparison["top_title_match"] - and comparison["top_url_match"] - ) else "fail" - - payload.update( - { - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": {"status": estimate.status_code, "data": estimate.body}, - "rhumb": {"status": execute.status_code, "data": execute.body}, - "direct": {"status": direct.status_code, "data": direct.body}, - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Tavily depth-5 runtime pass failed; inspect artifact for parity details") - - execution_id = rhumb_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "Tavily: current-depth rerun confirms search.query parity through Rhumb Resolve again", - "--summary", "Fresh current-depth runtime rerun passed for Tavily search.query through Rhumb Resolve. Managed and direct executions matched on result count, top title, and top URL for the same live search query, lifting Tavily another layer above the callable review floor.", - "--evidence-title", "Tavily current-depth runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh current-depth runtime rerun passed for Tavily search.query through Rhumb Resolve. Managed and direct executions matched on result count, top title, and top URL for the same live search query.", - "--source-ref", f"runtime-review:tavily:{stamp}", - "--source-batch-id", f"runtime-review:tavily:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "tavily", - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "service_slug": SERVICE_SLUG, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_twilio_depth10_20260403.py b/scripts/runtime_review_twilio_depth10_20260403.py deleted file mode 100644 index 05f1b9f1..00000000 --- a/scripts/runtime_review_twilio_depth10_20260403.py +++ /dev/null @@ -1,369 +0,0 @@ -#!/usr/bin/env python3 -"""Twilio current-depth runtime review pass β€” 2026-04-03. - -Twilio is the stalest provider in the live weakest callable bucket after Stripe -moved to depth 10. This pass re-verifies the safe BYO `phone.lookup` path through -Rhumb Resolve against direct Twilio Lookup v2 and publishes a fresh runtime-backed -review pair. -""" -from __future__ import annotations - -import asyncio -import base64 -import json -import os -import subprocess -import sys -import uuid -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -PROVIDER = "twilio" -CAPABILITY_ID = "phone.lookup" -NUMBER = "+14155552671" -LOOKUP_FIELDS = "line_type_intelligence" -TWILIO_LOOKUP_URL = f"https://lookups.twilio.com/v2/PhoneNumbers/{NUMBER}" -PUBLIC_SERVICE_SLUG = "twilio" -CANONICAL_PUBLISH_SLUG = "twilio" -POST_GRANT_PROPAGATION_DELAY_SECONDS = 5 -ESTIMATE_AUTH_RETRY_ATTEMPTS = 4 -ESTIMATE_AUTH_RETRY_DELAY_SECONDS = 5 -RUNTIME_BACKED_TRUST_LABELS = {"🟒 Runtime-verified", "πŸ§ͺ Tester-generated"} - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _looks_like_invalid_key(body: Any) -> bool: - if not isinstance(body, dict): - return False - detail = str(body.get("detail") or "").lower() - error = str(body.get("error") or "").lower() - return "invalid or expired rhumb api key" in detail or "invalid or expired rhumb api key" in error - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label in RUNTIME_BACKED_TRUST_LABELS: - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _pick_fields(payload: dict[str, Any] | None) -> dict[str, Any]: - payload = payload or {} - line_type = payload.get("line_type_intelligence") - if not isinstance(line_type, dict): - line_type = {} - return { - "phone_number": payload.get("phone_number"), - "valid": payload.get("valid"), - "country_code": payload.get("country_code"), - "calling_country_code": payload.get("calling_country_code"), - "national_format": payload.get("national_format"), - "line_type_intelligence": { - "type": line_type.get("type"), - "carrier_name": line_type.get("carrier_name"), - "error_code": line_type.get("error_code"), - }, - } - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - short = uuid.uuid4().hex[:8] - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-twilio-depth10.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-03-twilio-depth10.json" - org_id = f"org_runtime_review_twilio_{stamp.lower()}_{short}" - - payload: dict[str, Any] = { - "provider": PROVIDER, - "canonical_publish_slug": CANONICAL_PUBLISH_SLUG, - "capability_id": CAPABILITY_ID, - "number": NUMBER, - "fields": LOOKUP_FIELDS, - "review_kind": "current_depth10", - "organization_id": org_id, - "artifact_path": str(artifact_path.relative_to(REPO_ROOT)), - "started_at": started_at.replace(microsecond=0).isoformat().replace("+00:00", "Z"), - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-twilio-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-twilio-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Twilio depth-10 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "twilio", "depth10"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, CANONICAL_PUBLISH_SLUG) - payload["grant_access"] = { - "status": "success", - "access_id": access_id, - } - payload["post_grant_delay_seconds"] = POST_GRANT_PROPAGATION_DELAY_SECONDS - _json_dump(artifact_path, payload) - - await asyncio.sleep(POST_GRANT_PROPAGATION_DELAY_SECONDS) - - api_headers = {"X-Rhumb-Key": api_key} - twilio_basic_auth = os.environ["RHUMB_CREDENTIAL_TWILIO_BASIC_AUTH"] - basic_header = "Basic " + base64.b64encode(twilio_basic_auth.encode("utf-8")).decode("ascii") - - try: - counts_before = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - estimate_attempts = 0 - estimate = HttpResult(status_code=0, body=None) - while estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS: - estimate_attempts += 1 - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": PROVIDER, "credential_mode": "byo"}, - timeout=60.0, - ) - if not ( - estimate.status_code == 401 - and _looks_like_invalid_key(estimate.body) - and estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS - ): - break - await asyncio.sleep(ESTIMATE_AUTH_RETRY_DELAY_SECONDS) - - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": PROVIDER, "credential_mode": "byo"}, - json_body={ - "method": "GET", - "path": f"/v2/PhoneNumbers/{NUMBER}", - "params": {"Fields": LOOKUP_FIELDS}, - }, - timeout=120.0, - ) - direct = await _request_json( - "GET", - TWILIO_LOOKUP_URL, - headers={"Authorization": basic_header}, - params={"Fields": LOOKUP_FIELDS}, - timeout=60.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - rhumb_fields = _pick_fields(rhumb_upstream if isinstance(rhumb_upstream, dict) else {}) - direct_fields = _pick_fields(direct.body if isinstance(direct.body, dict) else {}) - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "provider_used": rhumb_payload.get("provider_used"), - "upstream_status": rhumb_payload.get("upstream_status"), - "execution_id": rhumb_payload.get("execution_id"), - "rhumb_fields": rhumb_fields, - "direct_fields": direct_fields, - "parity": rhumb_fields == direct_fields, - } - - estimate_pattern = (((estimate.body or {}).get("data") or {}).get("endpoint_pattern")) if isinstance(estimate.body, dict) else None - catalog_note = None - if isinstance(estimate_pattern, str) and "carrier" in estimate_pattern.lower() and LOOKUP_FIELDS != "carrier": - catalog_note = { - "estimate_endpoint_pattern": estimate_pattern, - "runtime_field_used": LOOKUP_FIELDS, - "observation": "Twilio Lookup v2 accepted Fields=line_type_intelligence; estimate still advertises the deprecated carrier field.", - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == CANONICAL_PUBLISH_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["parity"] - ) else "fail" - - payload.update( - { - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": { - "status": estimate.status_code, - "attempts": estimate_attempts, - "data": estimate.body, - }, - "rhumb": { - "status": execute.status_code, - "data": execute.body, - }, - "direct": { - "status": direct.status_code, - "data": direct.body, - }, - "comparison": comparison, - "catalog_note": catalog_note, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Twilio depth-10 runtime pass failed; inspect artifact for parity details") - - execution_id = rhumb_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", CANONICAL_PUBLISH_SLUG, - "--headline", "Twilio: depth-10 runtime review confirms phone.lookup parity through Rhumb Resolve", - "--summary", "Fresh depth-10 runtime review passed for Twilio phone.lookup through Rhumb Resolve. BYO execution and direct Twilio Lookup v2 returned the same normalized phone record for the same number using Fields=line_type_intelligence.", - "--evidence-title", "Twilio depth-10 runtime review parity check via Rhumb Resolve", - "--evidence-summary", "Fresh depth-10 runtime review passed for Twilio phone.lookup through Rhumb Resolve. BYO execution matched direct Twilio Lookup v2 on phone_number, validity, country metadata, and line_type_intelligence payload.", - "--source-ref", f"runtime-review:twilio:{stamp}", - "--source-batch-id", f"runtime-review:twilio:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "twilio", - "--tag", "phone.lookup", - "--tag", "current_pass", - "--tag", "phase3", - "--tag", "depth10", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "canonical_service_slug": CANONICAL_PUBLISH_SLUG, - "public_service_route": PUBLIC_SERVICE_SLUG, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_twilio_depth3_20260331.py b/scripts/runtime_review_twilio_depth3_20260331.py deleted file mode 100644 index 9053d475..00000000 --- a/scripts/runtime_review_twilio_depth3_20260331.py +++ /dev/null @@ -1,324 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import base64 -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -PROVIDER = "twilio" -CAPABILITY_ID = "phone.lookup" -NUMBER = "+14155552671" -LOOKUP_FIELDS = "line_type_intelligence" -TWILIO_LOOKUP_URL = f"https://lookups.twilio.com/v2/PhoneNumbers/{NUMBER}" -PUBLIC_SERVICE_SLUG = "twilio" -CANONICAL_PUBLISH_SLUG = "twilio" - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _pick_fields(payload: dict[str, Any] | None) -> dict[str, Any]: - payload = payload or {} - line_type = payload.get("line_type_intelligence") - if not isinstance(line_type, dict): - line_type = {} - return { - "phone_number": payload.get("phone_number"), - "valid": payload.get("valid"), - "country_code": payload.get("country_code"), - "calling_country_code": payload.get("calling_country_code"), - "national_format": payload.get("national_format"), - "line_type_intelligence": { - "type": line_type.get("type"), - "carrier_name": line_type.get("carrier_name"), - "error_code": line_type.get("error_code"), - }, - } - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-twilio-depth3.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-31-twilio-depth3.json" - org_id = f"org_runtime_review_twilio_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": PROVIDER, - "canonical_publish_slug": CANONICAL_PUBLISH_SLUG, - "capability_id": CAPABILITY_ID, - "number": NUMBER, - "fields": LOOKUP_FIELDS, - "review_kind": "current_depth3", - "organization_id": org_id, - "artifact_path": str(artifact_path), - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-twilio-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-twilio-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Twilio depth-3 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "twilio"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, CANONICAL_PUBLISH_SLUG) - payload["grant_access"] = { - "status": "success", - "access_id": access_id, - } - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - twilio_basic_auth = os.environ["RHUMB_CREDENTIAL_TWILIO_BASIC_AUTH"] - basic_header = "Basic " + base64.b64encode(twilio_basic_auth.encode("utf-8")).decode("ascii") - - try: - counts_before = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": PROVIDER, "credential_mode": "byo"}, - timeout=60.0, - ) - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": PROVIDER, "credential_mode": "byo"}, - json_body={ - "method": "GET", - "path": f"/v2/PhoneNumbers/{NUMBER}", - "params": {"Fields": LOOKUP_FIELDS}, - }, - timeout=120.0, - ) - direct = await _request_json( - "GET", - TWILIO_LOOKUP_URL, - headers={"Authorization": basic_header}, - params={"Fields": LOOKUP_FIELDS}, - timeout=60.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - rhumb_fields = _pick_fields(rhumb_upstream if isinstance(rhumb_upstream, dict) else {}) - direct_fields = _pick_fields(direct.body if isinstance(direct.body, dict) else {}) - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_fields": rhumb_fields, - "direct_fields": direct_fields, - "parity": rhumb_fields == direct_fields, - } - - estimate_pattern = (((estimate.body or {}).get("data") or {}).get("endpoint_pattern")) if isinstance(estimate.body, dict) else None - catalog_note = None - if isinstance(estimate_pattern, str) and "carrier" in estimate_pattern.lower() and LOOKUP_FIELDS != "carrier": - catalog_note = { - "estimate_endpoint_pattern": estimate_pattern, - "runtime_field_used": LOOKUP_FIELDS, - "observation": "Twilio Lookup v2 accepted Fields=line_type_intelligence; estimate still advertises the deprecated carrier field.", - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == CANONICAL_PUBLISH_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["parity"] - ) else "fail" - - payload.update( - { - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": {"status": estimate.status_code, "data": estimate.body}, - "rhumb": { - "status": execute.status_code, - "data": execute.body, - }, - "direct": { - "status": direct.status_code, - "data": direct.body, - }, - "comparison": comparison, - "catalog_note": catalog_note, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Twilio depth-3 runtime pass failed; inspect artifact for parity details") - - execution_id = rhumb_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", CANONICAL_PUBLISH_SLUG, - "--headline", "Twilio: depth-3 rerun confirms phone.lookup parity through Rhumb Resolve", - "--summary", "Fresh depth-3 runtime rerun passed for Twilio phone.lookup through Rhumb Resolve. Managed BYO execution and direct Twilio Lookup v2 returned the same normalized phone record for the same number using Fields=line_type_intelligence.", - "--evidence-title", "Twilio depth-3 runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh depth-3 runtime rerun passed for Twilio phone.lookup through Rhumb Resolve. Managed BYO execution matched direct Twilio Lookup v2 on phone_number, validity, country metadata, and line_type_intelligence payload.", - "--source-ref", f"runtime-review:twilio:{stamp}", - "--source-batch-id", f"runtime-review:twilio:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "twilio", - "--tag", "phone.lookup", - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "canonical_service_slug": CANONICAL_PUBLISH_SLUG, - "public_service_route": PUBLIC_SERVICE_SLUG, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_twilio_depth7_20260402.py b/scripts/runtime_review_twilio_depth7_20260402.py deleted file mode 100644 index a470f029..00000000 --- a/scripts/runtime_review_twilio_depth7_20260402.py +++ /dev/null @@ -1,357 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import base64 -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -PROVIDER = "twilio" -CAPABILITY_ID = "phone.lookup" -NUMBER = "+14155552671" -LOOKUP_FIELDS = "line_type_intelligence" -TWILIO_LOOKUP_URL = f"https://lookups.twilio.com/v2/PhoneNumbers/{NUMBER}" -PUBLIC_SERVICE_SLUG = "twilio" -CANONICAL_PUBLISH_SLUG = "twilio" -POST_GRANT_PROPAGATION_DELAY_SECONDS = 5 -ESTIMATE_AUTH_RETRY_ATTEMPTS = 4 -ESTIMATE_AUTH_RETRY_DELAY_SECONDS = 5 - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _looks_like_invalid_key(body: Any) -> bool: - if not isinstance(body, dict): - return False - detail = str(body.get("detail") or "").lower() - error = str(body.get("error") or "").lower() - return "invalid or expired rhumb api key" in detail or "invalid or expired rhumb api key" in error - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _pick_fields(payload: dict[str, Any] | None) -> dict[str, Any]: - payload = payload or {} - line_type = payload.get("line_type_intelligence") - if not isinstance(line_type, dict): - line_type = {} - return { - "phone_number": payload.get("phone_number"), - "valid": payload.get("valid"), - "country_code": payload.get("country_code"), - "calling_country_code": payload.get("calling_country_code"), - "national_format": payload.get("national_format"), - "line_type_intelligence": { - "type": line_type.get("type"), - "carrier_name": line_type.get("carrier_name"), - "error_code": line_type.get("error_code"), - }, - } - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-twilio-depth7.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-02-twilio-depth7.json" - org_id = f"org_runtime_review_twilio_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": PROVIDER, - "canonical_publish_slug": CANONICAL_PUBLISH_SLUG, - "capability_id": CAPABILITY_ID, - "number": NUMBER, - "fields": LOOKUP_FIELDS, - "review_kind": "current_depth7", - "organization_id": org_id, - "artifact_path": str(artifact_path), - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-twilio-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-twilio-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Twilio depth-7 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "twilio"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, CANONICAL_PUBLISH_SLUG) - payload["grant_access"] = { - "status": "success", - "access_id": access_id, - } - payload["post_grant_delay_seconds"] = POST_GRANT_PROPAGATION_DELAY_SECONDS - _json_dump(artifact_path, payload) - - # Fresh agent access and auth can race the estimate read path; mirror the - # PDL harness fix so this rail does not false-fail when execute/direct parity - # is already green. - await asyncio.sleep(POST_GRANT_PROPAGATION_DELAY_SECONDS) - - api_headers = {"X-Rhumb-Key": api_key} - twilio_basic_auth = os.environ["RHUMB_CREDENTIAL_TWILIO_BASIC_AUTH"] - basic_header = "Basic " + base64.b64encode(twilio_basic_auth.encode("utf-8")).decode("ascii") - - try: - counts_before = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - estimate_attempts = 0 - estimate = HttpResult(status_code=0, body=None) - while estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS: - estimate_attempts += 1 - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": PROVIDER, "credential_mode": "byo"}, - timeout=60.0, - ) - if not ( - estimate.status_code == 401 - and _looks_like_invalid_key(estimate.body) - and estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS - ): - break - await asyncio.sleep(ESTIMATE_AUTH_RETRY_DELAY_SECONDS) - - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": PROVIDER, "credential_mode": "byo"}, - json_body={ - "method": "GET", - "path": f"/v2/PhoneNumbers/{NUMBER}", - "params": {"Fields": LOOKUP_FIELDS}, - }, - timeout=120.0, - ) - direct = await _request_json( - "GET", - TWILIO_LOOKUP_URL, - headers={"Authorization": basic_header}, - params={"Fields": LOOKUP_FIELDS}, - timeout=60.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - rhumb_fields = _pick_fields(rhumb_upstream if isinstance(rhumb_upstream, dict) else {}) - direct_fields = _pick_fields(direct.body if isinstance(direct.body, dict) else {}) - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_fields": rhumb_fields, - "direct_fields": direct_fields, - "parity": rhumb_fields == direct_fields, - } - - estimate_pattern = (((estimate.body or {}).get("data") or {}).get("endpoint_pattern")) if isinstance(estimate.body, dict) else None - catalog_note = None - if isinstance(estimate_pattern, str) and "carrier" in estimate_pattern.lower() and LOOKUP_FIELDS != "carrier": - catalog_note = { - "estimate_endpoint_pattern": estimate_pattern, - "runtime_field_used": LOOKUP_FIELDS, - "observation": "Twilio Lookup v2 accepted Fields=line_type_intelligence; estimate still advertises the deprecated carrier field.", - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == CANONICAL_PUBLISH_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["parity"] - ) else "fail" - - payload.update( - { - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": { - "status": estimate.status_code, - "attempts": estimate_attempts, - "data": estimate.body, - }, - "rhumb": { - "status": execute.status_code, - "data": execute.body, - }, - "direct": { - "status": direct.status_code, - "data": direct.body, - }, - "comparison": comparison, - "catalog_note": catalog_note, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Twilio depth-7 runtime pass failed; inspect artifact for parity details") - - execution_id = rhumb_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", CANONICAL_PUBLISH_SLUG, - "--headline", "Twilio: depth-7 rerun confirms phone.lookup parity through Rhumb Resolve", - "--summary", "Fresh depth-7 runtime rerun passed for Twilio phone.lookup through Rhumb Resolve. Managed BYO execution and direct Twilio Lookup v2 returned the same normalized phone record for the same number using Fields=line_type_intelligence.", - "--evidence-title", "Twilio depth-7 runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh depth-7 runtime rerun passed for Twilio phone.lookup through Rhumb Resolve. Managed BYO execution matched direct Twilio Lookup v2 on phone_number, validity, country metadata, and line_type_intelligence payload.", - "--source-ref", f"runtime-review:twilio:{stamp}", - "--source-batch-id", f"runtime-review:twilio:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "twilio", - "--tag", "phone.lookup", - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "canonical_service_slug": CANONICAL_PUBLISH_SLUG, - "public_service_route": PUBLIC_SERVICE_SLUG, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_twilio_depth8_20260402.py b/scripts/runtime_review_twilio_depth8_20260402.py deleted file mode 100644 index a7f12fa7..00000000 --- a/scripts/runtime_review_twilio_depth8_20260402.py +++ /dev/null @@ -1,362 +0,0 @@ -#!/usr/bin/env python3 -"""Twilio current-depth runtime review pass β€” 2026-04-02. - -Twilio is the stalest provider in the live weakest callable bucket after Stripe -moved to depth 9. This pass re-verifies the safe BYO `phone.lookup` path through -Rhumb Resolve against direct Twilio Lookup v2 and publishes a fresh runtime-backed -review pair. -""" -from __future__ import annotations - -import asyncio -import base64 -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -PROVIDER = "twilio" -CAPABILITY_ID = "phone.lookup" -NUMBER = "+14155552671" -LOOKUP_FIELDS = "line_type_intelligence" -TWILIO_LOOKUP_URL = f"https://lookups.twilio.com/v2/PhoneNumbers/{NUMBER}" -PUBLIC_SERVICE_SLUG = "twilio" -CANONICAL_PUBLISH_SLUG = "twilio" -POST_GRANT_PROPAGATION_DELAY_SECONDS = 5 -ESTIMATE_AUTH_RETRY_ATTEMPTS = 4 -ESTIMATE_AUTH_RETRY_DELAY_SECONDS = 5 -RUNTIME_BACKED_TRUST_LABELS = {"🟒 Runtime-verified", "πŸ§ͺ Tester-generated"} - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _looks_like_invalid_key(body: Any) -> bool: - if not isinstance(body, dict): - return False - detail = str(body.get("detail") or "").lower() - error = str(body.get("error") or "").lower() - return "invalid or expired rhumb api key" in detail or "invalid or expired rhumb api key" in error - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label in RUNTIME_BACKED_TRUST_LABELS: - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [str(REPO_ROOT / "packages/api/.venv/bin/python"), str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), *args], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"publish_runtime_review_pair.py failed ({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _pick_fields(payload: dict[str, Any] | None) -> dict[str, Any]: - payload = payload or {} - line_type = payload.get("line_type_intelligence") - if not isinstance(line_type, dict): - line_type = {} - return { - "phone_number": payload.get("phone_number"), - "valid": payload.get("valid"), - "country_code": payload.get("country_code"), - "calling_country_code": payload.get("calling_country_code"), - "national_format": payload.get("national_format"), - "line_type_intelligence": { - "type": line_type.get("type"), - "carrier_name": line_type.get("carrier_name"), - "error_code": line_type.get("error_code"), - }, - } - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-twilio-depth8.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-02-twilio-depth8.json" - org_id = f"org_runtime_review_twilio_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": PROVIDER, - "canonical_publish_slug": CANONICAL_PUBLISH_SLUG, - "capability_id": CAPABILITY_ID, - "number": NUMBER, - "fields": LOOKUP_FIELDS, - "review_kind": "current_depth8", - "organization_id": org_id, - "artifact_path": str(artifact_path), - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-twilio-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-twilio-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Twilio depth-8 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "twilio"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, CANONICAL_PUBLISH_SLUG) - payload["grant_access"] = { - "status": "success", - "access_id": access_id, - } - payload["post_grant_delay_seconds"] = POST_GRANT_PROPAGATION_DELAY_SECONDS - _json_dump(artifact_path, payload) - - await asyncio.sleep(POST_GRANT_PROPAGATION_DELAY_SECONDS) - - api_headers = {"X-Rhumb-Key": api_key} - twilio_basic_auth = os.environ["RHUMB_CREDENTIAL_TWILIO_BASIC_AUTH"] - basic_header = "Basic " + base64.b64encode(twilio_basic_auth.encode("utf-8")).decode("ascii") - - try: - counts_before = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - estimate_attempts = 0 - estimate = HttpResult(status_code=0, body=None) - while estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS: - estimate_attempts += 1 - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": PROVIDER, "credential_mode": "byo"}, - timeout=60.0, - ) - if not ( - estimate.status_code == 401 - and _looks_like_invalid_key(estimate.body) - and estimate_attempts < ESTIMATE_AUTH_RETRY_ATTEMPTS - ): - break - await asyncio.sleep(ESTIMATE_AUTH_RETRY_DELAY_SECONDS) - - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": PROVIDER, "credential_mode": "byo"}, - json_body={ - "method": "GET", - "path": f"/v2/PhoneNumbers/{NUMBER}", - "params": {"Fields": LOOKUP_FIELDS}, - }, - timeout=120.0, - ) - direct = await _request_json( - "GET", - TWILIO_LOOKUP_URL, - headers={"Authorization": basic_header}, - params={"Fields": LOOKUP_FIELDS}, - timeout=60.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_upstream = (rhumb_payload.get("upstream_response") or {}) if isinstance(rhumb_payload, dict) else {} - rhumb_fields = _pick_fields(rhumb_upstream if isinstance(rhumb_upstream, dict) else {}) - direct_fields = _pick_fields(direct.body if isinstance(direct.body, dict) else {}) - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_fields": rhumb_fields, - "direct_fields": direct_fields, - "parity": rhumb_fields == direct_fields, - } - - estimate_pattern = (((estimate.body or {}).get("data") or {}).get("endpoint_pattern")) if isinstance(estimate.body, dict) else None - catalog_note = None - if isinstance(estimate_pattern, str) and "carrier" in estimate_pattern.lower() and LOOKUP_FIELDS != "carrier": - catalog_note = { - "estimate_endpoint_pattern": estimate_pattern, - "runtime_field_used": LOOKUP_FIELDS, - "observation": "Twilio Lookup v2 accepted Fields=line_type_intelligence; estimate still advertises the deprecated carrier field.", - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == CANONICAL_PUBLISH_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["parity"] - ) else "fail" - - payload.update( - { - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": { - "status": estimate.status_code, - "attempts": estimate_attempts, - "data": estimate.body, - }, - "rhumb": { - "status": execute.status_code, - "data": execute.body, - }, - "direct": { - "status": direct.status_code, - "data": direct.body, - }, - "comparison": comparison, - "catalog_note": catalog_note, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Twilio depth-8 runtime pass failed; inspect artifact for parity details") - - execution_id = rhumb_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", CANONICAL_PUBLISH_SLUG, - "--headline", "Twilio: depth-8 rerun confirms phone.lookup parity through Rhumb Resolve again", - "--summary", "Fresh depth-8 runtime rerun passed for Twilio phone.lookup through Rhumb Resolve. BYO execution and direct Twilio Lookup v2 returned the same normalized phone record for the same number using Fields=line_type_intelligence.", - "--evidence-title", "Twilio depth-8 runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh depth-8 runtime rerun passed for Twilio phone.lookup through Rhumb Resolve. BYO execution matched direct Twilio Lookup v2 on phone_number, validity, country metadata, and line_type_intelligence payload.", - "--source-ref", f"runtime-review:twilio:{stamp}", - "--source-batch-id", f"runtime-review:twilio:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "twilio", - "--tag", "phone.lookup", - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "canonical_service_slug": CANONICAL_PUBLISH_SLUG, - "public_service_route": PUBLIC_SERVICE_SLUG, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(PUBLIC_SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_unstructured_depth10_20260402.py b/scripts/runtime_review_unstructured_depth10_20260402.py deleted file mode 100644 index b22258ee..00000000 --- a/scripts/runtime_review_unstructured_depth10_20260402.py +++ /dev/null @@ -1,349 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -SERVICE_SLUG = "unstructured" -CAPABILITY_ID = "document.parse" -TEXT_FILENAME = "runtime-review-unstructured-depth10.txt" -TEXT_CONTENT = ( - "Rhumb Runtime Review\n\n" - "Unstructured should parse this short sample into a Title and a NarrativeText block." -) -DIRECT_URL = "https://api.unstructuredapp.io/general/v0/general" -UNSTRUCTURED_SECRET_ITEM = "Unstructured API Key" - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _get_secret(item_title: str) -> str: - result = subprocess.run( - [ - "sop", - "item", - "get", - item_title, - "--vault", - "OpenClaw Agents", - "--fields", - "credential", - "--reveal", - ], - capture_output=True, - text=True, - timeout=30, - check=False, - ) - if result.returncode != 0 or not result.stdout.strip(): - raise RuntimeError(f"Unable to load secret: {item_title}\n{result.stderr}") - return result.stdout.strip() - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - data: dict[str, Any] | list[tuple[str, str]] | None = None, - files: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - data=data, - files=files, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label in {"🟒 Runtime-verified", "πŸ§ͺ Tester-generated"}: - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [ - str(REPO_ROOT / "packages/api/.venv/bin/python"), - str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), - *args, - ], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - "publish_runtime_review_pair.py failed " - f"({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -def _extract_elements(raw: Any) -> list[dict[str, Any]]: - if isinstance(raw, list): - return [row for row in raw if isinstance(row, dict)] - return [] - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-unstructured-depth10.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-04-02-unstructured-depth10.json" - org_id = f"org_runtime_review_unstructured_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "capability_id": CAPABILITY_ID, - "review_kind": "current_depth10", - "organization_id": org_id, - "artifact_path": str(artifact_path.relative_to(REPO_ROOT)), - "input_shape": { - "filename": TEXT_FILENAME, - "strategy": "fast", - }, - "started_at": started_at.replace(microsecond=0).isoformat().replace("+00:00", "Z"), - } - _json_dump(artifact_path, payload) - - unstructured_api_key = os.environ.get("RHUMB_CREDENTIAL_UNSTRUCTURED_API_KEY") or _get_secret(UNSTRUCTURED_SECRET_ITEM) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-unstructured-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-unstructured-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Unstructured depth-10 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "unstructured", "depth10"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = { - "status": "success", - "access_id": access_id, - } - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body={ - "strategy": "fast", - "files": [ - { - "filename": TEXT_FILENAME, - "text": TEXT_CONTENT, - "content_type": "text/plain", - } - ], - }, - timeout=120.0, - ) - direct = await _request_json( - "POST", - DIRECT_URL, - headers={"unstructured-api-key": unstructured_api_key}, - data={"strategy": "fast"}, - files={"files": (TEXT_FILENAME, TEXT_CONTENT.encode("utf-8"), "text/plain")}, - timeout=120.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_elements = _extract_elements(rhumb_payload.get("upstream_response") if isinstance(rhumb_payload, dict) else []) - direct_elements = _extract_elements(direct.body) - rhumb_types = [row.get("type") for row in rhumb_elements] - direct_types = [row.get("type") for row in direct_elements] - rhumb_texts = [row.get("text") for row in rhumb_elements] - direct_texts = [row.get("text") for row in direct_elements] - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_provider_used": rhumb_payload.get("provider_used"), - "rhumb_upstream_status": rhumb_payload.get("upstream_status"), - "rhumb_count": len(rhumb_elements), - "direct_count": len(direct_elements), - "rhumb_types": rhumb_types, - "direct_types": direct_types, - "rhumb_texts": rhumb_texts, - "direct_texts": direct_texts, - "count_match": len(rhumb_elements) == len(direct_elements), - "types_match": rhumb_types == direct_types, - "texts_match": rhumb_texts == direct_texts, - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["count_match"] - and comparison["types_match"] - and comparison["texts_match"] - ) else "fail" - - payload.update( - { - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": {"status": estimate.status_code, "data": estimate.body}, - "rhumb": {"status": execute.status_code, "data": execute.body}, - "direct": {"status": direct.status_code, "data": direct.body}, - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Unstructured depth-10 runtime pass failed; inspect artifact for parity details") - - execution_id = rhumb_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "Unstructured: depth-10 runtime review confirms document.parse parity through Rhumb Resolve", - "--summary", "Fresh depth-10 runtime review passed for Unstructured document.parse through Rhumb Resolve. Managed and direct executions matched exactly on parsed element count, element type ordering, and extracted text for the same sample file.", - "--evidence-title", "Unstructured depth-10 runtime review parity check via Rhumb Resolve", - "--evidence-summary", "Fresh depth-10 runtime review passed for Unstructured document.parse through Rhumb Resolve. Managed and direct executions matched exactly on parsed element count, element type ordering, and extracted text for the same sample file.", - "--source-ref", f"runtime-review:unstructured:{stamp}", - "--source-batch-id", f"runtime-review:unstructured:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "unstructured", - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--tag", "depth10", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "service_slug": SERVICE_SLUG, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/runtime_review_unstructured_depth5_20260330.py b/scripts/runtime_review_unstructured_depth5_20260330.py deleted file mode 100644 index 3bb7b7c9..00000000 --- a/scripts/runtime_review_unstructured_depth5_20260330.py +++ /dev/null @@ -1,309 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import httpx - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "packages/api")) - -from db.client import get_supabase_client # noqa: E402 -from schemas.agent_identity import AgentIdentityStore # noqa: E402 -from services.billing_bootstrap import ensure_org_billing_bootstrap # noqa: E402 - -BASE_URL = "https://api.rhumb.dev/v1" -SERVICE_SLUG = "unstructured" -CAPABILITY_ID = "document.parse" -TEXT_FILENAME = "runtime-review-unstructured-depth5.txt" -TEXT_CONTENT = ( - "Rhumb Runtime Review\n\n" - "Unstructured should parse this short sample into a Title and a NarrativeText block." -) -DIRECT_URL = "https://api.unstructuredapp.io/general/v0/general" - - -@dataclass -class HttpResult: - status_code: int - body: Any - - -def _json_dump(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _iso_now() -> str: - return datetime.now(tz=UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -async def _request_json( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - json_body: Any | None = None, - data: dict[str, Any] | list[tuple[str, str]] | None = None, - files: Any | None = None, - timeout: float = 120.0, -) -> HttpResult: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.request( - method, - url, - headers=headers, - params=params, - json=json_body, - data=data, - files=files, - ) - try: - body: Any = response.json() - except Exception: - body = response.text - return HttpResult(status_code=response.status_code, body=body) - - -def _extract_review_stats(body: Any) -> dict[str, int | None]: - reviews = [] - if isinstance(body, dict): - maybe_reviews = body.get("reviews") - if isinstance(maybe_reviews, list): - reviews = maybe_reviews - else: - data = body.get("data") - if isinstance(data, dict) and isinstance(data.get("reviews"), list): - reviews = data.get("reviews") or [] - runtime_backed = 0 - published = 0 - for review in reviews: - if not isinstance(review, dict): - continue - if review.get("status") == "published" or review.get("review_status") == "published": - published += 1 - source_type = review.get("source_type") or review.get("sourceType") - trust_label = review.get("trust_label") or review.get("trustLabel") - if source_type == "runtime_verified" or trust_label == "🟒 Runtime-verified": - runtime_backed += 1 - return { - "published_reviews": published or (len(reviews) if isinstance(reviews, list) else None), - "runtime_backed_reviews": runtime_backed, - } - - -async def _fetch_public_review_stats(service_slug: str) -> dict[str, int | None]: - result = await _request_json("GET", f"{BASE_URL}/services/{service_slug}/reviews", timeout=60.0) - if result.status_code != 200: - return {"published_reviews": None, "runtime_backed_reviews": None} - return _extract_review_stats(result.body) - - -def _run_publish(args: list[str], output_path: Path) -> dict[str, Any]: - env = os.environ.copy() - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = str(REPO_ROOT / "packages/api") + (f":{existing}" if existing else "") - proc = subprocess.run( - [ - str(REPO_ROOT / "packages/api/.venv/bin/python"), - str(REPO_ROOT / "scripts/publish_runtime_review_pair.py"), - *args, - ], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - "publish_runtime_review_pair.py failed " - f"({proc.returncode})\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - payload = json.loads(proc.stdout) - _json_dump(output_path, payload) - return payload - - -async def main() -> None: - started_at = datetime.now(tz=UTC) - stamp = started_at.strftime("%Y%m%dT%H%M%SZ") - artifact_path = REPO_ROOT / f"artifacts/runtime-review-pass-{stamp}-unstructured-current-depth5.json" - publication_path = REPO_ROOT / "artifacts/runtime-review-publication-2026-03-30-unstructured-depth5.json" - org_id = f"org_runtime_review_unstructured_{stamp.lower()}" - - payload: dict[str, Any] = { - "provider": SERVICE_SLUG, - "capability_id": CAPABILITY_ID, - "review_kind": "current_depth5", - "organization_id": org_id, - "artifact_path": str(artifact_path), - "input_shape": { - "filename": TEXT_FILENAME, - "strategy": "fast", - }, - } - _json_dump(artifact_path, payload) - - supabase = await get_supabase_client() - store = AgentIdentityStore(supabase) - - bootstrap = await ensure_org_billing_bootstrap( - org_id, - name=f"runtime-review-unstructured-{stamp}", - starter_credits_cents=5000, - ) - payload["bootstrap"] = bootstrap - - agent_id, api_key = await store.register_agent( - name=f"runtime-review-unstructured-{stamp}", - organization_id=org_id, - rate_limit_qpm=100, - description="Temp review agent for Unstructured depth-5 runtime pass", - tags=["runtime_review", "phase3", "current_pass", "unstructured"], - ) - payload["review_agent"] = { - "agent_id": agent_id, - "organization_id": org_id, - } - - access_id = await store.grant_service_access(agent_id, SERVICE_SLUG) - payload["grant_access"] = { - "status": "success", - "access_id": access_id, - } - _json_dump(artifact_path, payload) - - api_headers = {"X-Rhumb-Key": api_key} - - try: - counts_before = await _fetch_public_review_stats(SERVICE_SLUG) - estimate = await _request_json( - "GET", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute/estimate", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - timeout=60.0, - ) - execute = await _request_json( - "POST", - f"{BASE_URL}/capabilities/{CAPABILITY_ID}/execute", - headers=api_headers, - params={"provider": SERVICE_SLUG, "credential_mode": "rhumb_managed"}, - json_body={ - "strategy": "fast", - "files": [ - { - "filename": TEXT_FILENAME, - "text": TEXT_CONTENT, - "content_type": "text/plain", - } - ], - }, - timeout=120.0, - ) - direct = await _request_json( - "POST", - DIRECT_URL, - headers={"unstructured-api-key": os.environ["RHUMB_CREDENTIAL_UNSTRUCTURED_API_KEY"]}, - data={"strategy": "fast"}, - files={"files": (TEXT_FILENAME, TEXT_CONTENT.encode("utf-8"), "text/plain")}, - timeout=120.0, - ) - - rhumb_payload = ((execute.body or {}).get("data") or {}) if isinstance(execute.body, dict) else {} - rhumb_elements = rhumb_payload.get("upstream_response") if isinstance(rhumb_payload, dict) else [] - direct_elements = direct.body if isinstance(direct.body, list) else [] - rhumb_types = [row.get("type") for row in rhumb_elements or [] if isinstance(row, dict)] - direct_types = [row.get("type") for row in direct_elements or [] if isinstance(row, dict)] - observed_at = _iso_now() - reviewed_at_dt = datetime.fromisoformat(observed_at.replace("Z", "+00:00")) - fresh_until = (reviewed_at_dt + timedelta(days=30)).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - comparison = { - "rhumb_provider_used": rhumb_payload.get("provider_used"), - "rhumb_upstream_status": rhumb_payload.get("upstream_status"), - "rhumb_count": len(rhumb_elements or []), - "direct_count": len(direct_elements or []), - "rhumb_types": rhumb_types, - "direct_types": direct_types, - "count_match": len(rhumb_elements or []) == len(direct_elements or []), - "types_match": rhumb_types == direct_types, - } - - verdict = "pass" if ( - estimate.status_code == 200 - and execute.status_code == 200 - and direct.status_code == 200 - and rhumb_payload.get("provider_used") == SERVICE_SLUG - and rhumb_payload.get("upstream_status") == 200 - and comparison["count_match"] - and comparison["types_match"] - ) else "fail" - - payload.update( - { - "observed_at": observed_at, - "fresh_until": fresh_until, - "counts_before": counts_before, - "estimate": {"status": estimate.status_code, "data": estimate.body}, - "rhumb": {"status": execute.status_code, "data": execute.body}, - "direct": {"status": direct.status_code, "data": direct.body}, - "comparison": comparison, - "verdict": verdict, - } - ) - _json_dump(artifact_path, payload) - - if verdict != "pass": - raise RuntimeError("Unstructured depth-5 runtime pass failed; inspect artifact for parity details") - - execution_id = rhumb_payload.get("execution_id") or "" - publication = _run_publish( - [ - "--service", SERVICE_SLUG, - "--headline", "Unstructured: current-depth rerun confirms document.parse parity through Rhumb Resolve again", - "--summary", "Fresh current-depth runtime rerun passed for Unstructured document.parse through Rhumb Resolve. Managed and direct executions matched exactly on element types and element count for the same text file input.", - "--evidence-title", "Unstructured current-depth runtime rerun parity check via Rhumb Resolve", - "--evidence-summary", "Fresh current-depth runtime rerun passed for Unstructured document.parse through Rhumb Resolve. Managed and direct executions matched exactly on element types and element count for the same text file input.", - "--source-ref", f"runtime-review:unstructured:{stamp}", - "--source-batch-id", f"runtime-review:unstructured:{stamp}", - "--reviewed-at", observed_at, - "--fresh-until", fresh_until, - "--reviewer-agent-id", agent_id, - "--agent-id", agent_id, - "--run-id", execution_id, - "--tag", "runtime_review", - "--tag", "unstructured", - "--tag", CAPABILITY_ID, - "--tag", "current_pass", - "--tag", "phase3", - "--raw-payload-file", str(artifact_path.relative_to(REPO_ROOT)), - ], - publication_path, - ) - payload["published"] = { - "service_slug": SERVICE_SLUG, - "evidence_id": (publication.get("evidence") or {}).get("id"), - "review_id": (publication.get("review") or {}).get("id"), - "execution_id": execution_id, - } - payload["counts_after"] = await _fetch_public_review_stats(SERVICE_SLUG) - _json_dump(artifact_path, payload) - finally: - disabled = await store.disable_agent(agent_id) - payload["agent_disabled"] = bool(disabled) - _json_dump(artifact_path, payload) - - -if __name__ == "__main__": - asyncio.run(main())