diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index eec7db5..25c7c13 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -230,9 +230,9 @@ pub struct ModelConf { /// Billing weights per token component; None = every component at 1.0. #[serde(default)] pub token_rate: Option, - /// Anthropic prompt caching: mark the system prompt and the latest user - /// turn as cache breakpoints, so a multi-turn conversation's prefix bills - /// at the cache-read rate. Pair with `token_rate`. + /// Anthropic prompt caching: mark the system prompt and latest user turn where the client + /// placed none (an `anthropic/*` Responses model on OpenRouter gets a top-level + /// `cache_control`). Pair with `token_rate`. #[serde(default)] pub prompt_cache: bool, /// Models tried in order when this one fails upstream (5xx, 429, 401-403, connection diff --git a/crates/engines/src/claude_engine.rs b/crates/engines/src/claude_engine.rs index 95cf14c..ea1f37f 100644 --- a/crates/engines/src/claude_engine.rs +++ b/crates/engines/src/claude_engine.rs @@ -13,6 +13,12 @@ use crate::transport::{UpstreamBody, UpstreamRequest}; const DEFAULT_MAX_TOKENS: i64 = 1024; const THINKING_DEFAULT_MAX_TOKENS: i64 = 16384; +const MAX_BREAKPOINTS: usize = 4; +const BEDROCK_REJECTED_BETAS: [&str; 3] = [ + "advisor-tool-2026-03-01", + "advanced-tool-use-2025-11-20", + "prompt-caching-scope-2026-01-05", +]; base_engine!(ClaudeEngine); @@ -20,11 +26,26 @@ impl ClaudeEngine { /// The Messages body. On Bedrock the model rides in the invoke path and /// streaming in the URL, and the API version is a body field. fn build_body(&mut self) -> GResult> { - let system_text = self.base.system_text(); + let param = self.base.param()?; + let protocol = param.protocol; + let converse = protocol == gw_consts::Protocol::AwsConverse; + let served_as_named = param.fallback_from.is_none(); + let in_place = self.base.request.preserve_anthropic_wire + && !converse + && (served_as_named || !claude_before_5(¶m.model_name)); + let mut system_text = if in_place { + String::new() + } else { + self.base.system_text() + }; let mut messages: Vec = Vec::new(); let (mut system_parts, mut marked) = (Vec::new(), false); for m in std::mem::take(&mut self.base.request.message) { if m.role == gw_consts::role::SYSTEM { + if in_place { + messages.extend(system_turn(m, served_as_named)); + continue; + } match m.parts { Some(Value::Array(parts)) => { marked |= parts.iter().any(|part| part.get("cache_control").is_some()); @@ -77,33 +98,33 @@ impl ClaudeEngine { push_turn(&mut messages, role, content); } - let prompt_cache = self.base.request.prompt_cache; - if prompt_cache - && let Some(last) = messages.last_mut() - && last["role"] == "user" - { - last["content"] = Value::Array(cached_blocks(last["content"].take())); - } let param = self.base.param()?; - let converse = param.protocol == gw_consts::Protocol::AwsConverse; let anthropic_fields = !converse || crate::converse::claude_model(¶m.model_name); let bedrock = matches!( - param.protocol, + protocol, gw_consts::Protocol::AwsAnthropic | gw_consts::Protocol::AwsConverse ); let mut body = Map::new(); - if param.protocol == gw_consts::Protocol::AwsAnthropic { + if protocol == gw_consts::Protocol::AwsAnthropic { body.insert("anthropic_version".into(), "bedrock-2023-05-31".into()); - if let Some(betas) = self.base.request.anthropic_beta.as_deref() { - body.insert( - "anthropic_beta".into(), - Value::Array(betas.split(',').map(|b| Value::from(b.trim())).collect()), - ); - } } else if !bedrock { body.insert("model".into(), param.model_name.clone().into()); body.insert("stream".into(), self.base.request.stream.into()); } + if bedrock + && anthropic_fields + && let Some(betas) = self.base.request.anthropic_beta.as_deref() + { + let betas: Vec = betas + .split(',') + .map(str::trim) + .filter(|b| !BEDROCK_REJECTED_BETAS.contains(b)) + .map(Value::from) + .collect(); + if !betas.is_empty() { + body.insert("anthropic_beta".into(), Value::Array(betas)); + } + } body.insert("messages".into(), Value::Array(messages)); let dialect = gw_protocol::reasoning::anthropic_thinking_dialect(¶m.model_name); let mut max_tokens = @@ -115,22 +136,15 @@ impl ClaudeEngine { let mut client_cap = false; // 4.7+ rejects the sampling knobs outright; 4.6 only once thinking is engaged let mut sampling_rejected = dialect == ThinkingDialect::AdaptiveSummarized; - let mut native_system = marked.then(|| { - Value::Array( - system_parts - .into_iter() - .map(|part| match part { - Value::String(text) => { - object([("type", "text".into()), ("text", text.into())]) - } - part => part, - }) - .collect(), - ) - }); + let (mut native_system, mut top_system) = (None, None); if let Some(gw_models::TypedParams::Chat(p)) = self.base.take_typed() { // the native system-block array keeps the client's cache_control (and ttl) - native_system = p.system_blocks.filter(Value::is_array).or(native_system); + native_system = p.system_blocks.filter(Value::is_array); + if in_place { + system_text = p.system.unwrap_or_default(); + } else { + top_system = p.system; + } if let Some(mt) = p.max_tokens { max_tokens = mt; client_cap = true; @@ -194,12 +208,32 @@ impl ClaudeEngine { self.base.output_cap = max_tokens; } let mut raw = self.base.take_raw(); + let messages_long = marked && breakpoints(body.get("messages")).1; + let folded = system_parts.into_iter().map(|part| match part { + Value::String(text) => object([("type", "text".into()), ("text", text.into())]), + Value::Object(mut block) => { + if messages_long && block.get("cache_control").is_some_and(|c| !is_long_ttl(c)) { + block.remove("cache_control"); + } + Value::Object(block) + } + part => part, + }); let system = match native_system { - Some(blocks) if prompt_cache => Some(Value::Array(cached_blocks(blocks))), - Some(blocks) => Some(blocks), - None if system_text.is_empty() => None, - None if prompt_cache => Some(Value::Array(cached_blocks(Value::String(system_text)))), - None => Some(Value::String(system_text)), + Some(Value::Array(mut blocks)) => { + blocks.extend(folded); + Some(Value::Array(blocks)) + } + _ if marked => Some(Value::Array( + top_system + .map(|text| content_blocks(Value::String(text))) + .unwrap_or_default() + .into_iter() + .chain(folded) + .collect(), + )), + _ if system_text.is_empty() => None, + _ => Some(Value::String(system_text)), }; if let Some(system) = system { body.insert("system".into(), system); @@ -227,6 +261,9 @@ impl ClaudeEngine { } } crate::base::merge_raw_extras_owned(&mut body, raw); + if self.base.request.prompt_cache { + place_breakpoints(&mut body); + } Ok(body) } @@ -846,16 +883,88 @@ fn push_turn(messages: &mut Vec, role: &str, content: Value) { messages.push(Value::Object(msg)); } +fn system_turn(m: gw_models::ChatMsg, served_as_named: bool) -> Option { + let mut turn = m.extra; + let content = m.parts.unwrap_or(Value::String(m.content)); + let empty = match &content { + Value::String(text) => text.is_empty(), + Value::Array(blocks) => blocks.is_empty(), + _ => false, + }; + if !served_as_named && turn.remove("output_config").is_some() && turn.is_empty() && empty { + return None; + } + turn.insert("role".into(), m.role.into()); + turn.insert("content".into(), content); + Some(Value::Object(turn)) +} + +fn claude_before_5(model: &str) -> bool { + model + .find("claude") + .and_then(|i| { + model[i..] + .split(|c: char| !c.is_ascii_digit()) + .find(|s| !s.is_empty()) + }) + .and_then(|major| major.parse::().ok()) + .is_some_and(|major| major < 5) +} + +fn place_breakpoints(body: &mut Map) { + let top = body.get("cache_control"); + let top_long = top.is_some_and(is_long_ttl); + let (tool_marks, _) = breakpoints(body.get("tools")); + let (system_marks, _) = breakpoints(body.get("system")); + let (message_marks, messages_long) = breakpoints(body.get("messages")); + let mut slots = MAX_BREAKPOINTS + .saturating_sub(tool_marks + system_marks + message_marks + usize::from(top.is_some())); + if slots > 0 + && message_marks == 0 + && top.is_none() + && let Some(Value::Array(turns)) = body.get_mut("messages") + && let Some(last) = turns.iter_mut().rfind(|t| t["role"] != "system") + && last["role"] == "user" + { + cache_last_block(&mut last["content"]); + slots -= 1; + } + if slots > 0 + && system_marks == 0 + && !messages_long + && !top_long + && let Some(system) = body.get_mut("system") + { + cache_last_block(system); + } +} + +fn breakpoints(blocks: Option<&Value>) -> (usize, bool) { + let (mut count, mut long) = (0, false); + for block in blocks.and_then(Value::as_array).into_iter().flatten() { + if let Some(control) = block.get("cache_control") { + count += 1; + long |= is_long_ttl(control); + } + let (nested, nested_long) = breakpoints(block.get("content")); + count += nested; + long |= nested_long; + } + (count, long) +} + +fn is_long_ttl(control: &Value) -> bool { + control["ttl"].as_str().is_some_and(|ttl| ttl != "5m") +} + /// Content as blocks with a prompt-cache breakpoint on the last one (a breakpoint -/// caches everything before it); a client-placed breakpoint wins (it may carry a ttl). -fn cached_blocks(content: Value) -> Vec { - let mut blocks = content_blocks(content); +/// caches everything before it). +fn cache_last_block(content: &mut Value) { + let mut blocks = content_blocks(content.take()); if let Some(Value::Object(block)) = blocks.last_mut() { - block - .entry("cache_control") - .or_insert_with(|| json!({"type": "ephemeral"})); + block.insert("cache_control".into(), json!({"type": "ephemeral"})); } - blocks + *content = Value::Array(blocks); } /// Tool definitions in the anthropic wire shape: OpenAI-shaped defs are @@ -980,6 +1089,42 @@ mod tests { assert!(!up.headers.iter().any(|(k, _)| *k == "anthropic-beta")); } + #[test] + fn bedrock_drops_the_betas_it_rejects_and_converse_carries_the_rest_for_claude() { + let betas = + "context-management-2025-06-27,advisor-tool-2026-03-01,advanced-tool-use-2025-11-20"; + for (protocol, model, want) in [ + ( + Protocol::AwsAnthropic, + "us.anthropic.claude-opus-5-5", + Some(json!(["context-management-2025-06-27"])), + ), + ( + Protocol::AwsConverse, + "global.anthropic.claude-opus-5-5", + Some(json!(["context-management-2025-06-27"])), + ), + (Protocol::AwsConverse, "us.amazon.nova-lite-v1:0", None), + ] { + let mut r = base_req(); + r.anthropic_beta = Some(betas.to_owned()); + r.model_param_v2 = Some(ModelParamV2::with_name(protocol, model)); + let mut e = ClaudeEngine::new(r, Arc::new(MockTransport)); + let body = e.build_body().unwrap(); + assert_eq!(body.get("anthropic_beta").cloned(), want, "{model}"); + if protocol == Protocol::AwsConverse { + let out = crate::converse::request(body, model); + assert_eq!( + out.get("additionalModelRequestFields") + .and_then(|extra| extra.get("anthropic_beta")) + .cloned(), + want, + "{model}" + ); + } + } + } + #[test] fn openai_tools_without_description_or_parameters_normalize_cleanly() { let tools = normalize_tools_anthropic(json!([ diff --git a/crates/engines/src/families.rs b/crates/engines/src/families.rs index c4b8534..99ec201 100644 --- a/crates/engines/src/families.rs +++ b/crates/engines/src/families.rs @@ -1211,6 +1211,13 @@ impl ResponsesEngine { &mut body, gw_protocol::reasoning::EffortWire::Responses, ); + if self.base.request.prompt_cache + && self.base.wire_kind() == crate::generation::OPENROUTER_KIND + && model.trim_start_matches('~').starts_with("anthropic/") + { + body.entry("cache_control") + .or_insert_with(|| object([("type", "ephemeral".into())])); + } body.insert("model".to_owned(), model.into()); self.base.output_cap = body .get("max_output_tokens") diff --git a/crates/engines/src/generation.rs b/crates/engines/src/generation.rs index ded339f..55be141 100644 --- a/crates/engines/src/generation.rs +++ b/crates/engines/src/generation.rs @@ -9,7 +9,7 @@ use serde_json::{Value, json}; use crate::base::{parse_json_reply, versioned_url}; use crate::transport::{Transport, UpstreamRequest}; -const OPENROUTER_KIND: &str = "openrouter"; +pub(crate) const OPENROUTER_KIND: &str = "openrouter"; /// Waits before each lookup of an abandoned stream's recorded usage; OpenRouter records one in about 5 s. const LOOKUP_DELAYS: [Duration; 4] = [ Duration::from_secs(3), diff --git a/crates/engines/tests/request_construction.rs b/crates/engines/tests/request_construction.rs index 2f2d65c..0e7f2af 100644 --- a/crates/engines/tests/request_construction.rs +++ b/crates/engines/tests/request_construction.rs @@ -1380,7 +1380,338 @@ async fn anthropic_prompt_cache_marks_system_and_latest_user_turn() { assert_eq!(b["messages"][0]["content"], "hello"); } +fn native_system_turn_req(protocol: Protocol, model: &str) -> GatewayRequest { + let mut env = ChatMsg::text("system", "env block"); + env.extra.insert( + "output_config".into(), + serde_json::json!({"effort": "medium"}), + ); + let mut effort = ChatMsg::text("system", ""); + effort.parts = Some(serde_json::json!([])); + effort.extra.insert( + "output_config".into(), + serde_json::json!({"effort": "high"}), + ); + let mut reminder = ChatMsg::text("system", "reminder"); + reminder.parts = Some(serde_json::json!([ + {"type": "text", "text": "reminder", "cache_control": {"type": "ephemeral"}} + ])); + let mut req = chat_req(protocol, model); + req.message = vec![ + ChatMsg::text("user", "hi"), + env, + ChatMsg::text("assistant", "ok"), + effort, + ChatMsg::text("user", "again"), + reminder, + ]; + req.preserve_anthropic_wire = true; + req.model_param_v2.as_mut().unwrap().typed = Some(TypedParams::Chat(ChatParams { + system: Some("sys".into()), + system_blocks: Some(serde_json::json!([ + {"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}} + ])), + ..Default::default() + })); + req +} + +#[tokio::test] +async fn native_system_turns_stay_in_place_on_the_anthropic_wires() { + let am = Protocol::AnthropicMessages; + for (protocol, model, swapped) in [ + (am, "claude-opus-5-5", false), + (Protocol::AwsAnthropic, "anthropic.claude-opus-5-5", false), + (am, "~anthropic/claude-opus-latest", true), + (am, "MiniMax-M2", true), + (am, "claude-sonnet-5", true), + ] { + let t = RecordingTransport::new(CLAUDE_OK); + let mut req = native_system_turn_req(protocol, model); + req.model_param_v2.as_mut().unwrap().fallback_from = + swapped.then(|| "claude-opus-5-5".into()); + ClaudeEngine::new(req, t.clone()).run().await.unwrap(); + let b = t.body_json(); + let mut want = serde_json::json!([ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "env block", "output_config": {"effort": "medium"}}, + {"role": "assistant", "content": "ok"}, + {"role": "system", "content": [], "output_config": {"effort": "high"}}, + {"role": "user", "content": "again"}, + {"role": "system", "content": [ + {"type": "text", "text": "reminder", "cache_control": {"type": "ephemeral"}} + ]}, + ]); + if swapped { + let turns = want.as_array_mut().unwrap(); + turns.retain(|turn| turn["content"] != serde_json::json!([])); + for turn in turns { + turn.as_object_mut().unwrap().remove("output_config"); + } + } + assert_eq!(b["messages"], want, "{protocol:?} {model}"); + assert_eq!( + b["system"], + serde_json::json!([{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]), + "{protocol:?} {model}" + ); + } +} + +#[tokio::test] +async fn native_system_turns_fold_into_the_system_prompt_on_converse_or_a_swapped_model() { + let am = Protocol::AnthropicMessages; + for (protocol, model, swapped) in [ + (Protocol::AwsConverse, "us.amazon.nova-lite-v1:0", false), + (am, "claude-sonnet-4-6", true), + (am, "anthropic/claude-sonnet-4.6", true), + (am, "anthropic/claude-opus-4", true), + ] { + let reply = if protocol == Protocol::AwsConverse { + CONVERSE_OK + } else { + CLAUDE_OK + }; + let t = RecordingTransport::new(reply); + let mut req = native_system_turn_req(protocol, model); + req.model_param_v2.as_mut().unwrap().fallback_from = + swapped.then(|| "claude-opus-5-5".into()); + ClaudeEngine::new(req, t.clone()).run().await.unwrap(); + let b = t.body_json(); + let roles: Vec<&str> = b["messages"] + .as_array() + .unwrap() + .iter() + .map(|m| m["role"].as_str().unwrap()) + .collect(); + assert_eq!(roles, ["user", "assistant", "user"], "{b}"); + assert_eq!(system_texts(&b), ["sys", "env block", "reminder"], "{b}"); + } +} + +#[tokio::test] +async fn a_cache_marked_system_turn_keeps_the_top_level_system_text_on_converse() { + let t = RecordingTransport::new(CONVERSE_OK); + let mut env = ChatMsg::text("system", "env block"); + env.parts = Some(serde_json::json!([ + {"type": "text", "text": "env block", "cache_control": {"type": "ephemeral"}} + ])); + let mut req = chat_req(Protocol::AwsConverse, "us.anthropic.claude-opus-5-5"); + req.preserve_anthropic_wire = true; + req.message = vec![ChatMsg::text("user", "hi"), env]; + req.model_param_v2.as_mut().unwrap().typed = Some(TypedParams::Chat(ChatParams { + system: Some("sys".into()), + ..Default::default() + })); + ClaudeEngine::new(req, t.clone()).run().await.unwrap(); + let b = t.body_json(); + assert_eq!(system_texts(&b), ["sys", "env block"], "{b}"); +} + +#[tokio::test] +async fn a_folded_system_turn_never_puts_a_5m_breakpoint_before_a_1h_one_on_converse() { + let t = RecordingTransport::new(CONVERSE_OK); + let mut user = ChatMsg::text("user", "remember ALPHA"); + user.parts = Some( + serde_json::json!([{"type": "text", "text": "remember ALPHA", + "cache_control": {"type": "ephemeral", "ttl": "1h"}}]), + ); + let mut rule = ChatMsg::text("system", "one word"); + rule.parts = Some(serde_json::json!([{"type": "text", "text": "one word", + "cache_control": {"type": "ephemeral"}}])); + let mut req = chat_req(Protocol::AwsConverse, "global.anthropic.claude-opus-5-5"); + req.preserve_anthropic_wire = true; + req.message = vec![ + user, + rule, + ChatMsg::text("assistant", "noted"), + ChatMsg::text("user", "the word?"), + ]; + req.model_param_v2.as_mut().unwrap().typed = Some(TypedParams::Chat(ChatParams { + system: Some("terse".into()), + ..Default::default() + })); + ClaudeEngine::new(req, t.clone()).run().await.unwrap(); + let b = t.body_json(); + assert_eq!(system_texts(&b), ["terse", "one word"], "{b}"); + assert!(!b["system"].to_string().contains("cachePoint"), "{b}"); + assert!(b["messages"][0].to_string().contains("1h"), "{b}"); +} + +#[tokio::test] +async fn prompt_cache_fills_only_the_breakpoints_the_client_left_out() { + let five = serde_json::json!({"type": "ephemeral"}); + let hour = serde_json::json!({"type": "ephemeral", "ttl": "1h"}); + let text = |text: &str, control: Option<&Value>| { + let mut block = serde_json::json!({"type": "text", "text": text}); + if let Some(control) = control { + block["cache_control"] = control.clone(); + } + block + }; + let tool = |name: &str| serde_json::json!({"name": name, "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral"}}); + let tool_result = serde_json::json!({"type": "tool_result", "tool_use_id": "tu", + "content": [{"type": "text", "text": "r", "cache_control": {"type": "ephemeral"}}]}); + let (f, h) = (Some(&five), Some(&hour)); + let q = || vec![text("q", None)]; + #[rustfmt::skip] + let cases = [ + ("system only", Some(vec![text("sys", f)]), None, q(), None, 2, true, true), + ("system 1h", Some(vec![text("sys", h)]), None, q(), None, 2, true, true), + ("system and message", Some(vec![text("sys", f)]), None, vec![text("q", f)], + None, 2, true, true), + ("message only", None, None, vec![text("q", f)], None, 2, true, true), + ("message 1h", None, None, vec![text("q", h)], None, 1, true, false), + ("tool_result", None, None, vec![tool_result, text("q", None)], None, 2, false, true), + ("top-level", None, None, q(), f, 2, false, true), + ("top-level 1h", None, None, q(), h, 1, false, false), + ("four taken", Some(vec![text("a", f), text("b", f), text("c", f)]), + Some(vec![tool("t")]), q(), None, 4, false, true), + ("one slot left", None, Some(vec![tool("t1"), tool("t2"), tool("t3")]), + q(), None, 4, true, false), + ]; + for (name, system, tools, last, top, total, user_marked, system_marked) in cases { + let t = RecordingTransport::new(CLAUDE_OK); + let mut turn = ChatMsg::text("user", ""); + turn.parts = Some(Value::Array(last)); + let mut req = chat_req(Protocol::AnthropicMessages, "claude-opus-5-5"); + req.preserve_anthropic_wire = true; + req.prompt_cache = true; + req.message = vec![ + ChatMsg::text("user", "hi"), + ChatMsg::text("assistant", "ok"), + turn, + ]; + let param = req.model_param_v2.as_mut().unwrap(); + if let Some(top) = top { + param.raw = serde_json::json!({"cache_control": top}); + } + param.typed = Some(TypedParams::Chat(ChatParams { + system: Some("sys".into()), + system_blocks: system.map(Value::Array), + tools: tools.map(Value::Array), + ..Default::default() + })); + ClaudeEngine::new(req, t.clone()).run().await.unwrap(); + let b = t.body_json(); + let marked = |v: &Value| { + v.as_array() + .and_then(|blocks| blocks.last()) + .is_some_and(|block| block.get("cache_control").is_some()) + }; + assert_eq!( + b.to_string().matches("cache_control").count(), + total, + "{name}: {b}" + ); + assert_eq!( + marked(&b["messages"][2]["content"]), + user_marked, + "{name}: {b}" + ); + assert_eq!(marked(&b["system"]), system_marked, "{name}: {b}"); + let mut ttls = Vec::new(); + for key in ["tools", "system", "messages"] { + collect_ttls(&b[key], &mut ttls); + } + assert!( + !ttls.windows(2).any(|w| w == ["5m", "1h"]), + "{name}: {ttls:?}" + ); + } +} + +fn system_texts(b: &Value) -> Vec<&str> { + b["system"] + .as_array() + .into_iter() + .flatten() + .filter_map(|block| block["text"].as_str()) + .collect() +} + +fn collect_ttls(v: &Value, out: &mut Vec<&'static str>) { + for block in v.as_array().into_iter().flatten() { + if let Some(control) = block.get("cache_control") { + out.push(if control["ttl"] == "1h" { "1h" } else { "5m" }); + } + collect_ttls(&block["content"], out); + } +} + +#[tokio::test] +async fn prompt_cache_marks_the_last_user_turn_ahead_of_a_trailing_system_turn() { + let t = RecordingTransport::new(CLAUDE_OK); + let mut req = chat_req(Protocol::AnthropicMessages, "claude-opus-5-5"); + req.preserve_anthropic_wire = true; + req.prompt_cache = true; + req.message = vec![ + ChatMsg::text("user", "hello"), + ChatMsg::text("system", "reminder"), + ]; + req.model_param_v2.as_mut().unwrap().typed = Some(TypedParams::Chat(ChatParams { + system: Some("be brief".into()), + ..Default::default() + })); + ClaudeEngine::new(req, t.clone()).run().await.unwrap(); + let b = t.body_json(); + assert_eq!( + b["system"], + serde_json::json!([{"type": "text", "text": "be brief", "cache_control": {"type": "ephemeral"}}]) + ); + assert_eq!( + b["messages"], + serde_json::json!([ + {"role": "user", "content": [ + {"type": "text", "text": "hello", "cache_control": {"type": "ephemeral"}} + ]}, + {"role": "system", "content": "reminder"}, + ]) + ); +} + +#[tokio::test] +async fn openrouter_anthropic_responses_carry_a_top_level_cache_control() { + use gw_models::Account; + + let ephemeral = serde_json::json!({"type": "ephemeral"}); + let hour = serde_json::json!({"type": "ephemeral", "ttl": "1h"}); + let (e, own) = (Some(&ephemeral), Some(&hour)); + #[rustfmt::skip] + let cases = [ + ("anthropic/claude-opus-5.5", "openrouter", true, None, e), + ("~anthropic/claude-opus-latest", "openrouter", true, None, e), + ("anthropic/claude-opus-5.5", "openrouter", true, own, own), + ("anthropic/claude-opus-5.5", "openrouter", false, None, None), + ("openai/gpt-6-sol", "openrouter", true, None, None), + ("anthropic/claude-opus-5.5", "openai", true, None, None), + ]; + for (model, kind, prompt_cache, client, want) in cases { + let t = RecordingTransport::new(RESPONSES_OK); + let mut req = GatewayRequest { + preserve_responses_wire: true, + prompt_cache, + account: Some(Arc::new(Account { + name: kind.into(), + kind: kind.into(), + ..Default::default() + })), + model_param_v2: Some(ModelParamV2::with_name(Protocol::Responses, model)), + ..Default::default() + }; + let mut raw = serde_json::json!({"input": [{"role": "user", "content": "hi"}]}); + if let Some(client) = client { + raw["cache_control"] = client.clone(); + } + req.model_param_v2.as_mut().unwrap().raw = raw; + ResponsesEngine::new(req, t.clone()).run().await.unwrap(); + let b = t.body_json(); + assert_eq!(b.get("cache_control"), want, "{model} {prompt_cache} {b}"); + } +} + const RESPONSES_OK: &str = r#"{"id":"r","object":"response","model":"gpt-6-astra","status":"completed","output":[],"usage":{"input_tokens":1,"output_tokens":1}}"#; +const CONVERSE_OK: &str = r#"{"output":{"message":{"role":"assistant","content":[{"text":"ok"}]}},"stopReason":"end_turn","usage":{"inputTokens":3,"outputTokens":1}}"#; const CLAUDE_OK: &str = r#"{"model":"claude-test","content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1}}"#; const OPENAI_OK: &str = r#"{"model":"gpt","choices":[{"message":{"content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}"#; @@ -2020,9 +2351,7 @@ async fn bedrock_claude_signs_the_messages_body_without_model_or_stream() { #[tokio::test] async fn converse_request_lands_on_the_converse_path_in_converse_shape() { - let t = RecordingTransport::new( - r#"{"output":{"message":{"role":"assistant","content":[{"text":"ok"}]}},"stopReason":"end_turn","usage":{"inputTokens":3,"outputTokens":1}}"#, - ); + let t = RecordingTransport::new(CONVERSE_OK); let mut req = chat_req(Protocol::AwsConverse, "us.meta.llama3-3-70b-instruct-v1:0"); if let Some(p) = req.model_param_v2.as_mut() { p.typed = Some(TypedParams::Chat(ChatParams { @@ -2131,6 +2460,7 @@ async fn native_system_blocks_keep_the_clients_cache_control() { r#"{"id":"m","type":"message","role":"assistant","model":"claude","content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1}}"#, ); let mut req = chat_req(Protocol::AnthropicMessages, "claude-haiku-4-5"); + req.message.retain(|m| m.role != "system"); req.prompt_cache = true; if let Some(p) = req.model_param_v2.as_mut() { p.typed = Some(TypedParams::Chat(ChatParams { @@ -2179,9 +2509,7 @@ async fn a_participant_name_reaches_the_openai_wire() { #[tokio::test] async fn a_marked_chat_system_stays_plain_for_a_non_claude_converse_model() { - let t = RecordingTransport::new( - r#"{"output":{"message":{"role":"assistant","content":[{"text":"ok"}]}},"stopReason":"end_turn","usage":{"inputTokens":3,"outputTokens":1}}"#, - ); + let t = RecordingTransport::new(CONVERSE_OK); let mut req = chat_req(Protocol::AwsConverse, "us.xai.grok-4.6"); req.message[0].parts = Some(serde_json::json!([ {"type": "text", "text": "be brief", "cache_control": {"type": "ephemeral"}} diff --git a/crates/models/src/request.rs b/crates/models/src/request.rs index 615f308..e33c60e 100644 --- a/crates/models/src/request.rs +++ b/crates/models/src/request.rs @@ -143,7 +143,7 @@ pub struct BatchItem { /// Domain types referenced by `GatewayRequest`. Per-vendor long-tail fields /// ride in `raw`/`extra` passthroughs rather than being individually modeled. pub mod domain { - use serde_json::Value; + use serde_json::{Map, Value}; use crate::params::TypedParams; @@ -215,6 +215,8 @@ pub mod domain { /// array: signed thinking, encrypted reasoning) in emission order. #[serde(default, skip_serializing_if = "Option::is_none")] pub reasoning_details: Option, + #[serde(default, skip_serializing_if = "Map::is_empty")] + pub extra: Map, } impl ChatMsg { diff --git a/crates/protocol/src/anthropic.rs b/crates/protocol/src/anthropic.rs index 9fa76a7..e28ec45 100644 --- a/crates/protocol/src/anthropic.rs +++ b/crates/protocol/src/anthropic.rs @@ -53,6 +53,8 @@ pub struct InMessage { pub role: String, /// string or [{type:"text"|"image"|"tool_result"|...}] blocks pub content: Value, + #[serde(flatten)] + pub extra: Map, } #[derive(Debug, Clone, Default, Serialize, Deserialize)] diff --git a/crates/views/src/lib.rs b/crates/views/src/lib.rs index 86e4bc4..08eafb4 100644 --- a/crates/views/src/lib.rs +++ b/crates/views/src/lib.rs @@ -3110,6 +3110,7 @@ fn chat_request(body: ChatCompletionRequest) -> (Vec, ModelParamV2) { name: m.name, reasoning_content: m.reasoning_content, reasoning_details: m.reasoning_details.map(Value::Array), + extra: serde_json::Map::new(), } }) .collect(); @@ -3589,15 +3590,19 @@ async fn messages( message: body .messages .into_iter() - .map(|m| match m.content { - Value::String(s) => ChatMsg::text(m.role, s), - Value::Array(blocks) => { - let text = gw_protocol::anthropic::blocks_text(&blocks); - let mut msg = ChatMsg::text(m.role, text); - msg.parts = Some(Value::Array(blocks)); - msg - } - _ => ChatMsg::text(m.role, String::new()), + .map(|m| { + let (text, parts) = match m.content { + Value::String(s) => (s, None), + Value::Array(blocks) => ( + gw_protocol::anthropic::blocks_text(&blocks), + Some(Value::Array(blocks)), + ), + _ => (String::new(), None), + }; + let mut msg = ChatMsg::text(m.role, text); + msg.parts = parts; + msg.extra = m.extra; + msg }) .collect(), model_param_v2: Some(param), @@ -5139,13 +5144,23 @@ mod tests { } } - #[derive(Debug, Default)] - struct RecordingResponses { + #[derive(Debug)] + struct Recording { + reply: &'static [u8], seen: std::sync::Mutex>, } + impl Recording { + fn new(reply: &'static [u8]) -> Arc { + Arc::new(Self { + reply, + seen: std::sync::Mutex::new(None), + }) + } + } + #[async_trait::async_trait] - impl gw_engines::transport::Transport for RecordingResponses { + impl gw_engines::transport::Transport for Recording { async fn send( &self, req: gw_engines::transport::UpstreamRequest, @@ -5154,7 +5169,7 @@ mod tests { Ok(gw_engines::transport::UpstreamResponse { status: 200, body: gw_engines::transport::UpstreamBody::Json(bytes::Bytes::from_static( - br#"{"id":"resp_1","model":"gpt-5-responses","status":"completed","output":[{"type":"message","content":[{"type":"output_text","text":"ok"}]}],"usage":{"input_tokens":3,"output_tokens":1}}"#, + self.reply, )), headers: Default::default(), }) @@ -5297,7 +5312,9 @@ mod tests { #[tokio::test] async fn messages_system_blocks_become_instructions_on_a_responses_model() { - let transport = Arc::new(RecordingResponses::default()); + let transport = Recording::new( + br#"{"id":"resp_1","model":"gpt-5-responses","status":"completed","output":[{"type":"message","content":[{"type":"output_text","text":"ok"}]}],"usage":{"input_tokens":3,"output_tokens":1}}"#, + ); let (router, _) = retained_view_app(transport.clone()); let request = Request::builder() .method("POST") @@ -5315,6 +5332,41 @@ mod tests { assert!(sent.get("system").is_none(), "{sent}"); } + #[tokio::test] + async fn messages_system_turns_reach_the_anthropic_upstream_in_place() { + let yaml = "listen: {host: h, port: 1}\nmodels: [{name: claude-opus-5-5, protocol: anthropic-messages}]\naccounts: [{name: a1, provider: anthropic, protocols: [anthropic-messages]}]\naccess_keys: [{ak: k, product: p, qps: 100, daily_token_quota: 100000}]"; + let cfg = Arc::new(GatewayConfig::from_yaml(yaml).unwrap()); + let state = Arc::new(GatewayState::from_config(&cfg)); + let transport = Recording::new( + br#"{"model":"claude-opus-5-5","content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn","usage":{"input_tokens":3,"output_tokens":1}}"#, + ); + let router = app(AppState::new(cfg, state, transport.clone())); + let request = Request::builder() + .method("POST") + .uri("/v1/messages") + .header("content-type", "application/json") + .header("x-api-key", "k") + .body(Body::from( + r#"{"model":"claude-opus-5-5","max_tokens":32,"system":[{"type":"text","text":"sys","cache_control":{"type":"ephemeral"}}],"messages":[{"role":"user","content":"hi"},{"role":"system","content":"env","output_config":{"effort":"medium"}},{"role":"assistant","content":[{"type":"text","text":"ok"}]},{"role":"user","content":"again"},{"role":"system","content":[{"type":"text","text":"reminder","cache_control":{"type":"ephemeral"}}]}]}"#, + )) + .unwrap(); + let response = router.oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let sent = transport.seen.lock().unwrap().take().unwrap(); + assert_eq!( + sent["messages"], + json!([ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "env", "output_config": {"effort": "medium"}}, + {"role": "assistant", "content": [{"type": "text", "text": "ok"}]}, + {"role": "user", "content": "again"}, + {"role": "system", "content": [ + {"type": "text", "text": "reminder", "cache_control": {"type": "ephemeral"}} + ]}, + ]) + ); + } + #[tokio::test] async fn retained_tts_decode_failure_uses_view_status() { let (router, store) = retained_view_app(Arc::new(InvalidTtsPayload)); diff --git a/docs/api.md b/docs/api.md index 5f15dd6..e1ed524 100644 --- a/docs/api.md +++ b/docs/api.md @@ -174,7 +174,7 @@ a cost. | Method | Path | Notes | |--------|------|-------| -| POST | `/v1/messages` | streaming + non-streaming; the client's `anthropic-beta` header is forwarded to Anthropic-wire upstreams (as the `anthropic_beta` body list on Bedrock) | +| POST | `/v1/messages` | streaming + non-streaming; the client's `anthropic-beta` header is forwarded to Anthropic-wire upstreams (as the `anthropic_beta` body list on Bedrock, minus the betas Bedrock rejects) | `/v1/messages` works on both Anthropic-protocol models and OpenAI-protocol models — the gateway converts between the two, including the streaming event diff --git a/docs/configuration.md b/docs/configuration.md index bdd4a17..40a3f3c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -136,7 +136,7 @@ models: audio_completion: 8.33 # audio output tokens (default = completion weight) long_context: {threshold_tokens: 200000, prompt_weight: 2.0, completion_weight: 1.5} # optional tier past a prompt size batch_discount: 0.5 # optional: /v1/batches items at this fraction of the price; must be finite and in (0.0, 1.0] - prompt_cache: true # anthropic-messages only: prompt-cache breakpoints + prompt_cache: true # prompt-cache breakpoints (Anthropic wires; anthropic/* on OpenRouter Responses) fallback_models: [gpt-4o-mini] # optional: tried in order on an upstream 5xx / 429 / 401-403 refusal / connection failure / no healthy account (rejected at load when unknown, self or duplicate) variants: # optional weighted canary split, sticky per user - {model: gpt-4o, weight: 90} # self-reference keeps a share here @@ -162,23 +162,36 @@ generated seconds, a `search` model's queries (one per call) — and adds to `cost_micros` next to any token cost; the count lands in the ledger's `billed_units` and the `/admin/usage` aggregates. An account's `cost_unit_price_micros` is the vendor side of the -same unit, for margin. `prompt_cache` (anthropic-messages models) -marks the system prompt and the latest user turn as Anthropic prompt-cache -breakpoints, so each turn of a conversation re-reads its prefix at the -cache-read rate; it is per model and off by default because a long one-shot -prompt would pay the cache-write premium for nothing. A `/v1/messages` client -that sends `system` as blocks with its own `cache_control` (including -`ttl: 1h`) keeps them as sent; the gateway only marks the last system block -when that block carries no breakpoint of its own. A `/v1/chat/completions` -system message whose content parts carry `cache_control` keeps them on an -Anthropic-wire model the same way. On an openai-chat model -the client's own breakpoints on system blocks, message blocks and -`tool_result` ride through as OpenAI content parts, which OpenRouter honors -for Anthropic models; a breakpoint on a tool definition does not (Gemini's -compatible endpoint rejects the field), and a later breakpoint covers the -tools anyway. `variants` splits a -public name across other declared same-protocol models (one level): -entitlement and the per-(AK, model) daily counter judge the public name, +same unit, for margin. `prompt_cache` (anthropic-messages models) marks the +system prompt and the latest user turn as Anthropic prompt-cache breakpoints, +so each turn of a conversation re-reads its prefix at the cache-read rate; it +is per model and off by default because a long one-shot prompt would pay the +cache-write premium for nothing. Client breakpoints win: the gateway marks the +latest user turn only when the client put no breakpoint on a message block +(`tool_result` content included) and sent no top-level `cache_control`, and +the system prompt only when the client left it unmarked and no 1-hour +breakpoint follows it. The turn goes first, and neither is added past +Anthropic's four breakpoints. A `/v1/messages` client's breakpoints, `ttl: 1h` +included, ride through as sent. On `anthropic-messages` and `aws-anthropic` +models its mid-conversation `role: "system"` messages stay in place with their +`output_config` and `clear_at`. They are folded into the end of `system` on +`aws-converse`, and when the gateway serves a pre-5 Claude model in place of +the one the client named (`fallback_models`, `variants`, a tenant's +`fallback_model`), since those models reject a system turn; any other model +served in place of the named one keeps them in place without their +`output_config`, which only some models accept, and drops a turn that carried +nothing else. On a `responses` model served by an OpenRouter account, +`prompt_cache` on an `anthropic/*` model adds the top-level +`cache_control: {type: ephemeral}` that OpenRouter needs to cache it, unless +the client sent its own. A `/v1/chat/completions` system message whose content +parts carry `cache_control` keeps them on an Anthropic-wire model the same +way. On an openai-chat model the client's own breakpoints on system blocks, +message blocks and `tool_result` ride through as OpenAI content parts, which +OpenRouter honors for Anthropic models; a breakpoint on a tool definition does +not (Gemini's compatible endpoint rejects the field), and a later breakpoint +covers the tools anyway. `variants` splits a public name +across other declared same-protocol models (one level): entitlement and the +per-(AK, model) daily counter judge the public name, billing prices the served variant, and the response echoes the requested name. Selection hashes the effective user, so a user sticks to one backend across the fleet; a realtime session picks its variant once at the handshake diff --git a/docs/examples.md b/docs/examples.md index 35466ad..ee7ec15 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -249,11 +249,14 @@ captured from the real client and replays in the live matrix (`agents` group). **Claude Code** — the Anthropic wire, so the gateway model must speak it natively or by conversion. Claude Code sends a system-block array with `cache_control` points, its tool set, `thinking` (with `display: omitted`), -`context_management`, and its user id in `metadata.user_id`; it enables beta -features through the `anthropic-beta` header, which the gateway forwards to -Anthropic-wire upstreams (the header for `kind: anthropic`, the -`anthropic_beta` body list for `aws-anthropic`). Point it at the gateway with -the access key as the API key: +`context_management`, mid-conversation `role: "system"` messages (the +request's last breakpoint often sits on one), and its user id in +`metadata.user_id`; it enables beta features through the `anthropic-beta` +header, which the gateway forwards to Anthropic-wire upstreams (the header for +`kind: anthropic`, the `anthropic_beta` body list for `aws-anthropic` and for +Claude on `aws-converse`, minus the advisor-tool, advanced-tool-use and +prompt-caching-scope betas Bedrock rejects). Point it at the gateway with the +access key as the API key: ```bash export ANTHROPIC_BASE_URL=https://gw.example.com @@ -289,7 +292,10 @@ env_key = "GW_API_KEY" # export GW_API_KEY=ak-... wire_api = "responses" ``` -Codex tries a WebSocket first and falls back to HTTPS on its own. +Codex tries a WebSocket first and falls back to HTTPS on its own. An +`anthropic/*` model on an OpenRouter Responses account caches nothing unless +the request carries a top-level `cache_control`; set `prompt_cache: true` on +that model and the gateway adds it. **VS Code chat (GitHub Copilot)** — *Chat: Manage Language Models* → *Add Models* → *Custom Endpoint*: pick the API type (Chat Completions, Responses diff --git a/docs/providers.md b/docs/providers.md index 0252666..43cde29 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -91,8 +91,8 @@ usage); the rest are marked non-streaming below and always answer buffered: | `dashscope` | Alibaba Qwen (native) | `https://dashscope-intl.aliyuncs.com` | `Bearer`; streams via `X-DashScope-SSE` + `incremental_output` | | `anthropic-messages` | any Anthropic-compatible endpoint (e.g. MiniMax) | vendor's `/anthropic` base | `x-api-key`; some report `input_tokens` only in `message_delta` — handled | | `ernie` | Baidu Ernie (Wenxin) | `https://aip.baidubce.com` | a `bce-v3/…` key goes as `Bearer`, a legacy token as the `access_token` query param (non-streaming); Qianfan's OpenAI-compatible `https://qianfan.baidubce.com/v2` also works as `kind: openai` + `endpoint` | -| `aws-anthropic` | Anthropic Claude on AWS Bedrock | `https://bedrock-runtime..amazonaws.com` | SigV4 (see below); model name = the Bedrock model id (`anthropic.claude-…`, `us.anthropic.claude-…`); the full Messages engine (system, tools, thinking dialects by generation, prompt-cache breakpoints, signed reasoning) on the InvokeModel wire — `anthropic_version` in the body, the client's `anthropic-beta` header as the `anthropic_beta` list, model and streaming in the path; streams via InvokeModelWithResponseStream (EventStream frames decoded into the same event sequence) | -| `aws-converse` | any model on AWS Bedrock via the Converse API | `https://bedrock-runtime..amazonaws.com` | SigV4 or API key (see below); model name = the Bedrock model id or inference profile (`eu.amazon.nova-micro-v1:0`, `us.meta.llama3-3-70b-instruct-v1:0`, `mistral.pixtral-large-2502-v1:0`, `anthropic.claude-…`); the Messages engine transcoded to Converse — system, tools + tool results, images, thinking replay, prompt-cache points (Claude takes them on system, messages and tools with their TTL, Nova on system and messages at the default TTL; other families get none; OpenAI GPT, Grok and Kimi ids refuse an image inside a tool result, so there its blocks from the first image on, documents aside, follow it as sibling blocks, while Llama 4 and the rest keep the image nested) — and back (buffered and `converse-stream`); Claude reasoning knobs ride in `additionalModelRequestFields`, other passthrough extras too; on an `openai.gpt-` or `xai.grok-` id the requested effort becomes that family's `reasoning_config` enum instead — which takes `max` from every generation but never `minimal` — and Bedrock's own refusal of `temperature`/`topP` for those ids drops them from `inferenceConfig` (live: `us.`/`global.openai.gpt-6-astra`, `…gpt-5.6-luna`, and `us.xai.grok-4.6`, which rejects both knobs the same way, takes `none`…`max`, and returns its reasoning as `reasoningContent.redactedContent` — surfaced as a `redacted_thinking` block; Converse also rejects `cachePoint` on Grok and never reports a cache read, so price that route without a cache weight) | +| `aws-anthropic` | Anthropic Claude on AWS Bedrock | `https://bedrock-runtime..amazonaws.com` | SigV4 (see below); model name = the Bedrock model id (`anthropic.claude-…`, `us.anthropic.claude-…`); the full Messages engine (system, tools, thinking dialects by generation, prompt-cache breakpoints, signed reasoning) on the InvokeModel wire — `anthropic_version` in the body, the client's `anthropic-beta` header as the `anthropic_beta` list (minus the advisor-tool, advanced-tool-use and prompt-caching-scope betas Bedrock rejects), model and streaming in the path; streams via InvokeModelWithResponseStream (EventStream frames decoded into the same event sequence) | +| `aws-converse` | any model on AWS Bedrock via the Converse API | `https://bedrock-runtime..amazonaws.com` | SigV4 or API key (see below); model name = the Bedrock model id or inference profile (`eu.amazon.nova-micro-v1:0`, `us.meta.llama3-3-70b-instruct-v1:0`, `mistral.pixtral-large-2502-v1:0`, `anthropic.claude-…`); the Messages engine transcoded to Converse — system, tools + tool results, images, thinking replay, prompt-cache points (Claude takes them on system, messages and tools with their TTL, Nova on system and messages at the default TTL; other families get none; OpenAI GPT, Grok and Kimi ids refuse an image inside a tool result, so there its blocks from the first image on, documents aside, follow it as sibling blocks, while Llama 4 and the rest keep the image nested) — and back (buffered and `converse-stream`); Claude reasoning knobs ride in `additionalModelRequestFields`, other passthrough extras too, and for Claude the filtered `anthropic_beta` list with them, which `context_management` needs; on an `openai.gpt-` or `xai.grok-` id the requested effort becomes that family's `reasoning_config` enum instead — which takes `max` from every generation but never `minimal` — and Bedrock's own refusal of `temperature`/`topP` for those ids drops them from `inferenceConfig` (live: `us.`/`global.openai.gpt-6-astra`, `…gpt-5.6-luna`, and `us.xai.grok-4.6`, which rejects both knobs the same way, takes `none`…`max`, and returns its reasoning as `reasoningContent.redactedContent` — surfaced as a `redacted_thinking` block; Converse also rejects `cachePoint` on Grok and never reports a cache read, so price that route without a cache weight) | | `aws-embed` | Titan / Cohere embeddings on AWS Bedrock | `https://bedrock-runtime..amazonaws.com` | SigV4 or API-key Bearer (see below); model name = the Bedrock model id; Titan `{inputText}` → `{embedding, inputTextTokenCount}` takes exactly one input per call (a batch is refused with 400; `dimensions` forwarded when the client sends it), Cohere `{texts, input_type: search_document}` → `{embeddings}` (v4: `{embeddings: {float}}`, `dimensions` forwarded as `output_dimension`) in one call, also on a `us.`/`global.` inference profile; answered in the OpenAI `/v1/embeddings` list shape, usage from Bedrock's `x-amzn-bedrock-*-token-count` headers | | `aws-llama` | Meta Llama on AWS Bedrock | `https://bedrock-runtime..amazonaws.com` | SigV4 (see below); model name = the Bedrock model id or inference profile (`meta.llama3-8b-instruct-v1:0`, `us.meta.llama3-3-70b-instruct-v1:0`, `us.meta.llama4-scout-17b-instruct-v1:0`); the conversation is rendered into the Llama 3 (or Llama 4) chat template; usage from the token-count headers / invocation metrics, else the body counts | | `minimax-v1` | MiniMax legacy v1 (`abab*`) | `https://api.minimax.chat` | `Bearer` (non-streaming); kept for existing accounts — the vendor has retired it for new ones; new integrations should use MiniMax's OpenAI-/Anthropic-compatible endpoints |