From c588582da169e75b45a0b00666ba8dd77920825f Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 26 Oct 2026 10:38:02 +0800 Subject: [PATCH 01/10] fix: keep Messages system turns and client cache breakpoints; cache OpenRouter anthropic Responses Claude Code sends mid-conversation role:system messages, and the request's message-level cache breakpoint sits on the trailing one. ClaudeEngine folded them into the system prompt and dropped them whenever the top-level system was a block array, so the model lost them and only the system prompt cached. On anthropic-messages and aws-anthropic they now stay in place with their output_config and clear_at; Converse appends them to the system blocks. prompt_cache now adds gw breakpoints only when the client placed none, which keeps a request within Anthropic's four breakpoints and the client's TTL order. OpenRouter's Responses API caches an anthropic/* model only under a top-level cache_control; prompt_cache on such a model now adds it unless the client did. --- crates/config/src/lib.rs | 5 +- crates/engines/src/claude_engine.rs | 111 ++++++--- crates/engines/src/families.rs | 7 + crates/engines/src/generation.rs | 2 +- crates/engines/tests/request_construction.rs | 238 +++++++++++++++++++ crates/models/src/request.rs | 4 +- crates/protocol/src/anthropic.rs | 2 + crates/views/src/lib.rs | 80 +++++-- docs/configuration.md | 21 +- docs/examples.md | 9 +- 10 files changed, 415 insertions(+), 64 deletions(-) diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index eec7db55..970aabf0 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -231,8 +231,9 @@ pub struct ModelConf { #[serde(default)] pub token_rate: Option, /// Anthropic prompt caching: mark the system prompt and the latest user - /// turn as cache breakpoints, so a multi-turn conversation's prefix bills - /// at the cache-read rate. Pair with `token_rate`. + /// turn as cache breakpoints unless the client placed any (an `anthropic/*` + /// Responses model on OpenRouter gets the top-level `cache_control` instead), + /// so a conversation's prefix bills at the cache-read rate. Pair with `token_rate`. #[serde(default)] pub prompt_cache: bool, /// Models tried in order when this one fails upstream (5xx, 429, 401-403, connection diff --git a/crates/engines/src/claude_engine.rs b/crates/engines/src/claude_engine.rs index 95cf14ca..67853a36 100644 --- a/crates/engines/src/claude_engine.rs +++ b/crates/engines/src/claude_engine.rs @@ -20,11 +20,22 @@ impl ClaudeEngine { /// The Messages body. On Bedrock the model rides in the invoke path and /// streaming in the URL, and the API version is a body field. fn build_body(&mut self) -> GResult> { - let system_text = self.base.system_text(); + let protocol = self.base.param()?.protocol; + let converse = protocol == gw_consts::Protocol::AwsConverse; + let in_place = self.base.request.preserve_anthropic_wire && !converse; + let mut system_text = if in_place { + String::new() + } else { + self.base.system_text() + }; let mut messages: Vec = Vec::new(); let (mut system_parts, mut marked) = (Vec::new(), false); for m in std::mem::take(&mut self.base.request.message) { if m.role == gw_consts::role::SYSTEM { + if in_place { + messages.push(system_turn(m)); + continue; + } match m.parts { Some(Value::Array(parts)) => { marked |= parts.iter().any(|part| part.get("cache_control").is_some()); @@ -77,22 +88,14 @@ impl ClaudeEngine { push_turn(&mut messages, role, content); } - let prompt_cache = self.base.request.prompt_cache; - if prompt_cache - && let Some(last) = messages.last_mut() - && last["role"] == "user" - { - last["content"] = Value::Array(cached_blocks(last["content"].take())); - } let param = self.base.param()?; - let converse = param.protocol == gw_consts::Protocol::AwsConverse; let anthropic_fields = !converse || crate::converse::claude_model(¶m.model_name); let bedrock = matches!( - param.protocol, + protocol, gw_consts::Protocol::AwsAnthropic | gw_consts::Protocol::AwsConverse ); let mut body = Map::new(); - if param.protocol == gw_consts::Protocol::AwsAnthropic { + if protocol == gw_consts::Protocol::AwsAnthropic { body.insert("anthropic_version".into(), "bedrock-2023-05-31".into()); if let Some(betas) = self.base.request.anthropic_beta.as_deref() { body.insert( @@ -115,22 +118,13 @@ impl ClaudeEngine { let mut client_cap = false; // 4.7+ rejects the sampling knobs outright; 4.6 only once thinking is engaged let mut sampling_rejected = dialect == ThinkingDialect::AdaptiveSummarized; - let mut native_system = marked.then(|| { - Value::Array( - system_parts - .into_iter() - .map(|part| match part { - Value::String(text) => { - object([("type", "text".into()), ("text", text.into())]) - } - part => part, - }) - .collect(), - ) - }); + let mut native_system = None; if let Some(gw_models::TypedParams::Chat(p)) = self.base.take_typed() { // the native system-block array keeps the client's cache_control (and ttl) - native_system = p.system_blocks.filter(Value::is_array).or(native_system); + native_system = p.system_blocks.filter(Value::is_array); + if in_place && let Some(system) = p.system { + system_text = system; + } if let Some(mt) = p.max_tokens { max_tokens = mt; client_cap = true; @@ -194,12 +188,18 @@ impl ClaudeEngine { self.base.output_cap = max_tokens; } let mut raw = self.base.take_raw(); + let folded = system_parts.into_iter().map(|part| match part { + Value::String(text) => object([("type", "text".into()), ("text", text.into())]), + part => part, + }); let system = match native_system { - Some(blocks) if prompt_cache => Some(Value::Array(cached_blocks(blocks))), - Some(blocks) => Some(blocks), - None if system_text.is_empty() => None, - None if prompt_cache => Some(Value::Array(cached_blocks(Value::String(system_text)))), - None => Some(Value::String(system_text)), + Some(Value::Array(mut blocks)) => { + blocks.extend(folded); + Some(Value::Array(blocks)) + } + _ if marked => Some(Value::Array(folded.collect())), + _ if system_text.is_empty() => None, + _ => Some(Value::String(system_text)), }; if let Some(system) = system { body.insert("system".into(), system); @@ -227,6 +227,9 @@ impl ClaudeEngine { } } crate::base::merge_raw_extras_owned(&mut body, raw); + if self.base.request.prompt_cache && !has_breakpoint(&body) { + place_breakpoints(&mut body); + } Ok(body) } @@ -846,16 +849,50 @@ fn push_turn(messages: &mut Vec, role: &str, content: Value) { messages.push(Value::Object(msg)); } +fn system_turn(m: gw_models::ChatMsg) -> Value { + let mut turn = m.extra; + turn.insert("role".into(), m.role.into()); + turn.insert( + "content".into(), + m.parts.unwrap_or(Value::String(m.content)), + ); + Value::Object(turn) +} + +fn has_breakpoint(body: &Map) -> bool { + fn marked(blocks: Option<&Value>) -> bool { + blocks.and_then(Value::as_array).is_some_and(|blocks| { + blocks + .iter() + .any(|b| b.get("cache_control").is_some() || marked(b.get("content"))) + }) + } + body.contains_key("cache_control") + || ["tools", "system", "messages"] + .into_iter() + .any(|key| marked(body.get(key))) +} + +fn place_breakpoints(body: &mut Map) { + if let Some(system) = body.get_mut("system") { + cache_last_block(system); + } + if let Some(Value::Array(turns)) = body.get_mut("messages") + && let Some(last) = turns.iter_mut().rfind(|t| t["role"] != "system") + && last["role"] == "user" + { + cache_last_block(&mut last["content"]); + } +} + /// Content as blocks with a prompt-cache breakpoint on the last one (a breakpoint -/// caches everything before it); a client-placed breakpoint wins (it may carry a ttl). -fn cached_blocks(content: Value) -> Vec { - let mut blocks = content_blocks(content); +/// caches everything before it). +fn cache_last_block(content: &mut Value) { + let mut blocks = content_blocks(content.take()); if let Some(Value::Object(block)) = blocks.last_mut() { - block - .entry("cache_control") - .or_insert_with(|| json!({"type": "ephemeral"})); + block.insert("cache_control".into(), json!({"type": "ephemeral"})); } - blocks + *content = Value::Array(blocks); } /// Tool definitions in the anthropic wire shape: OpenAI-shaped defs are diff --git a/crates/engines/src/families.rs b/crates/engines/src/families.rs index c4b8534f..99ec201e 100644 --- a/crates/engines/src/families.rs +++ b/crates/engines/src/families.rs @@ -1211,6 +1211,13 @@ impl ResponsesEngine { &mut body, gw_protocol::reasoning::EffortWire::Responses, ); + if self.base.request.prompt_cache + && self.base.wire_kind() == crate::generation::OPENROUTER_KIND + && model.trim_start_matches('~').starts_with("anthropic/") + { + body.entry("cache_control") + .or_insert_with(|| object([("type", "ephemeral".into())])); + } body.insert("model".to_owned(), model.into()); self.base.output_cap = body .get("max_output_tokens") diff --git a/crates/engines/src/generation.rs b/crates/engines/src/generation.rs index ded339f9..55be1416 100644 --- a/crates/engines/src/generation.rs +++ b/crates/engines/src/generation.rs @@ -9,7 +9,7 @@ use serde_json::{Value, json}; use crate::base::{parse_json_reply, versioned_url}; use crate::transport::{Transport, UpstreamRequest}; -const OPENROUTER_KIND: &str = "openrouter"; +pub(crate) const OPENROUTER_KIND: &str = "openrouter"; /// Waits before each lookup of an abandoned stream's recorded usage; OpenRouter records one in about 5 s. const LOOKUP_DELAYS: [Duration; 4] = [ Duration::from_secs(3), diff --git a/crates/engines/tests/request_construction.rs b/crates/engines/tests/request_construction.rs index 2f2d65cf..66bbb98f 100644 --- a/crates/engines/tests/request_construction.rs +++ b/crates/engines/tests/request_construction.rs @@ -1380,6 +1380,243 @@ async fn anthropic_prompt_cache_marks_system_and_latest_user_turn() { assert_eq!(b["messages"][0]["content"], "hello"); } +fn native_system_turn_req(protocol: Protocol, model: &str) -> GatewayRequest { + let mut env = ChatMsg::text("system", "env block"); + env.extra.insert( + "output_config".into(), + serde_json::json!({"effort": "medium"}), + ); + let mut effort = ChatMsg::text("system", ""); + effort.parts = Some(serde_json::json!([])); + effort.extra.insert( + "output_config".into(), + serde_json::json!({"effort": "high"}), + ); + let mut reminder = ChatMsg::text("system", "reminder"); + reminder.parts = Some(serde_json::json!([ + {"type": "text", "text": "reminder", "cache_control": {"type": "ephemeral"}} + ])); + let mut req = chat_req(protocol, model); + req.message = vec![ + ChatMsg::text("user", "hi"), + env, + ChatMsg::text("assistant", "ok"), + effort, + ChatMsg::text("user", "again"), + reminder, + ]; + req.preserve_anthropic_wire = true; + req.model_param_v2.as_mut().unwrap().typed = Some(TypedParams::Chat(ChatParams { + system: Some("sys".into()), + system_blocks: Some(serde_json::json!([ + {"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}} + ])), + ..Default::default() + })); + req +} + +#[tokio::test] +async fn native_system_turns_stay_in_place_on_the_anthropic_wires() { + for (protocol, model) in [ + (Protocol::AnthropicMessages, "claude-opus-5-5"), + (Protocol::AwsAnthropic, "anthropic.claude-opus-5-5"), + ] { + let t = RecordingTransport::new(CLAUDE_OK); + let req = native_system_turn_req(protocol, model); + ClaudeEngine::new(req, t.clone()).run().await.unwrap(); + let b = t.body_json(); + assert_eq!( + b["messages"], + serde_json::json!([ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "env block", "output_config": {"effort": "medium"}}, + {"role": "assistant", "content": "ok"}, + {"role": "system", "content": [], "output_config": {"effort": "high"}}, + {"role": "user", "content": "again"}, + {"role": "system", "content": [ + {"type": "text", "text": "reminder", "cache_control": {"type": "ephemeral"}} + ]}, + ]), + "{protocol:?}" + ); + assert_eq!( + b["system"], + serde_json::json!([{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]), + "{protocol:?}" + ); + } +} + +#[tokio::test] +async fn native_system_turns_fold_into_the_system_prompt_on_converse() { + let t = RecordingTransport::new( + r#"{"output":{"message":{"role":"assistant","content":[{"text":"ok"}]}},"stopReason":"end_turn","usage":{"inputTokens":3,"outputTokens":1}}"#, + ); + let req = native_system_turn_req(Protocol::AwsConverse, "us.amazon.nova-lite-v1:0"); + ClaudeEngine::new(req, t.clone()).run().await.unwrap(); + let b = t.body_json(); + let roles: Vec<&str> = b["messages"] + .as_array() + .unwrap() + .iter() + .map(|m| m["role"].as_str().unwrap()) + .collect(); + assert_eq!(roles, ["user", "assistant", "user"], "{b}"); + let system: Vec<&str> = b["system"] + .as_array() + .unwrap() + .iter() + .filter_map(|block| block["text"].as_str()) + .collect(); + assert_eq!(system, ["sys", "env block", "reminder"], "{b}"); +} + +#[tokio::test] +async fn prompt_cache_adds_no_breakpoint_beside_the_clients() { + let breakpoints = |b: &Value| b.to_string().matches("cache_control").count(); + let t = RecordingTransport::new(CLAUDE_OK); + let mut req = native_system_turn_req(Protocol::AnthropicMessages, "claude-opus-5-5"); + req.prompt_cache = true; + if let Some(TypedParams::Chat(p)) = req.model_param_v2.as_mut().unwrap().typed.as_mut() { + p.system_blocks = Some(serde_json::json!([ + {"type": "text", "text": "billing"}, + {"type": "text", "text": "core", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + {"type": "text", "text": "sys", "cache_control": {"type": "ephemeral", "ttl": "1h"}} + ])); + p.tools = Some(serde_json::json!([ + {"name": "t", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral", "ttl": "1h"}} + ])); + } + ClaudeEngine::new(req, t.clone()).run().await.unwrap(); + let b = t.body_json(); + assert_eq!(breakpoints(&b), 4, "{b}"); + assert_eq!(b["messages"][4]["content"], "again", "{b}"); + + let mut tool_result = ChatMsg::text("user", ""); + tool_result.parts = Some( + serde_json::json!([{"type": "tool_result", "tool_use_id": "tu", + "content": [{"type": "text", "text": "r", "cache_control": {"type": "ephemeral"}}]}]), + ); + for (raw, tools, last) in [ + ( + serde_json::json!({"cache_control": {"type": "ephemeral"}}), + None, + None, + ), + ( + Value::Null, + Some( + serde_json::json!([{"name": "t", "input_schema": {"type": "object"}, + "cache_control": {"type": "ephemeral"}}]), + ), + None, + ), + (Value::Null, None, Some(tool_result)), + ] { + let t = RecordingTransport::new(CLAUDE_OK); + let mut req = chat_req(Protocol::AnthropicMessages, "claude-opus-5-5"); + req.prompt_cache = true; + req.message.extend(last); + let param = req.model_param_v2.as_mut().unwrap(); + param.raw = raw; + param.typed = Some(TypedParams::Chat(ChatParams { + tools, + ..Default::default() + })); + ClaudeEngine::new(req, t.clone()).run().await.unwrap(); + let b = t.body_json(); + assert_eq!(breakpoints(&b), 1, "{b}"); + assert_eq!(b["system"], "be brief", "{b}"); + } +} + +#[tokio::test] +async fn prompt_cache_marks_the_last_user_turn_ahead_of_a_trailing_system_turn() { + let t = RecordingTransport::new(CLAUDE_OK); + let mut req = chat_req(Protocol::AnthropicMessages, "claude-opus-5-5"); + req.preserve_anthropic_wire = true; + req.prompt_cache = true; + req.message = vec![ + ChatMsg::text("user", "hello"), + ChatMsg::text("system", "reminder"), + ]; + req.model_param_v2.as_mut().unwrap().typed = Some(TypedParams::Chat(ChatParams { + system: Some("be brief".into()), + ..Default::default() + })); + ClaudeEngine::new(req, t.clone()).run().await.unwrap(); + let b = t.body_json(); + assert_eq!( + b["system"], + serde_json::json!([{"type": "text", "text": "be brief", "cache_control": {"type": "ephemeral"}}]) + ); + assert_eq!( + b["messages"], + serde_json::json!([ + {"role": "user", "content": [ + {"type": "text", "text": "hello", "cache_control": {"type": "ephemeral"}} + ]}, + {"role": "system", "content": "reminder"}, + ]) + ); +} + +#[tokio::test] +async fn openrouter_anthropic_responses_carry_a_top_level_cache_control() { + use gw_models::Account; + + let ephemeral = serde_json::json!({"type": "ephemeral"}); + let own = serde_json::json!({"type": "ephemeral", "ttl": "1h"}); + for (model, kind, prompt_cache, client, want) in [ + ( + "anthropic/claude-opus-5.5", + "openrouter", + true, + None, + Some(&ephemeral), + ), + ( + "~anthropic/claude-opus-latest", + "openrouter", + true, + None, + Some(&ephemeral), + ), + ( + "anthropic/claude-opus-5.5", + "openrouter", + true, + Some(&own), + Some(&own), + ), + ("anthropic/claude-opus-5.5", "openrouter", false, None, None), + ("openai/gpt-6-sol", "openrouter", true, None, None), + ("anthropic/claude-opus-5.5", "openai", true, None, None), + ] { + let t = RecordingTransport::new(RESPONSES_OK); + let mut req = GatewayRequest { + preserve_responses_wire: true, + prompt_cache, + account: Some(Arc::new(Account { + name: kind.into(), + kind: kind.into(), + ..Default::default() + })), + model_param_v2: Some(ModelParamV2::with_name(Protocol::Responses, model)), + ..Default::default() + }; + let mut raw = serde_json::json!({"input": [{"role": "user", "content": "hi"}]}); + if let Some(client) = client { + raw["cache_control"] = client.clone(); + } + req.model_param_v2.as_mut().unwrap().raw = raw; + ResponsesEngine::new(req, t.clone()).run().await.unwrap(); + let b = t.body_json(); + assert_eq!(b.get("cache_control"), want, "{model} {prompt_cache} {b}"); + } +} + const RESPONSES_OK: &str = r#"{"id":"r","object":"response","model":"gpt-6-astra","status":"completed","output":[],"usage":{"input_tokens":1,"output_tokens":1}}"#; const CLAUDE_OK: &str = r#"{"model":"claude-test","content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1}}"#; const OPENAI_OK: &str = r#"{"model":"gpt","choices":[{"message":{"content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}"#; @@ -2131,6 +2368,7 @@ async fn native_system_blocks_keep_the_clients_cache_control() { r#"{"id":"m","type":"message","role":"assistant","model":"claude","content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1}}"#, ); let mut req = chat_req(Protocol::AnthropicMessages, "claude-haiku-4-5"); + req.message.retain(|m| m.role != "system"); req.prompt_cache = true; if let Some(p) = req.model_param_v2.as_mut() { p.typed = Some(TypedParams::Chat(ChatParams { diff --git a/crates/models/src/request.rs b/crates/models/src/request.rs index 615f308b..e33c60e2 100644 --- a/crates/models/src/request.rs +++ b/crates/models/src/request.rs @@ -143,7 +143,7 @@ pub struct BatchItem { /// Domain types referenced by `GatewayRequest`. Per-vendor long-tail fields /// ride in `raw`/`extra` passthroughs rather than being individually modeled. pub mod domain { - use serde_json::Value; + use serde_json::{Map, Value}; use crate::params::TypedParams; @@ -215,6 +215,8 @@ pub mod domain { /// array: signed thinking, encrypted reasoning) in emission order. #[serde(default, skip_serializing_if = "Option::is_none")] pub reasoning_details: Option, + #[serde(default, skip_serializing_if = "Map::is_empty")] + pub extra: Map, } impl ChatMsg { diff --git a/crates/protocol/src/anthropic.rs b/crates/protocol/src/anthropic.rs index 9fa76a79..e28ec452 100644 --- a/crates/protocol/src/anthropic.rs +++ b/crates/protocol/src/anthropic.rs @@ -53,6 +53,8 @@ pub struct InMessage { pub role: String, /// string or [{type:"text"|"image"|"tool_result"|...}] blocks pub content: Value, + #[serde(flatten)] + pub extra: Map, } #[derive(Debug, Clone, Default, Serialize, Deserialize)] diff --git a/crates/views/src/lib.rs b/crates/views/src/lib.rs index 86e4bc40..08eafb48 100644 --- a/crates/views/src/lib.rs +++ b/crates/views/src/lib.rs @@ -3110,6 +3110,7 @@ fn chat_request(body: ChatCompletionRequest) -> (Vec, ModelParamV2) { name: m.name, reasoning_content: m.reasoning_content, reasoning_details: m.reasoning_details.map(Value::Array), + extra: serde_json::Map::new(), } }) .collect(); @@ -3589,15 +3590,19 @@ async fn messages( message: body .messages .into_iter() - .map(|m| match m.content { - Value::String(s) => ChatMsg::text(m.role, s), - Value::Array(blocks) => { - let text = gw_protocol::anthropic::blocks_text(&blocks); - let mut msg = ChatMsg::text(m.role, text); - msg.parts = Some(Value::Array(blocks)); - msg - } - _ => ChatMsg::text(m.role, String::new()), + .map(|m| { + let (text, parts) = match m.content { + Value::String(s) => (s, None), + Value::Array(blocks) => ( + gw_protocol::anthropic::blocks_text(&blocks), + Some(Value::Array(blocks)), + ), + _ => (String::new(), None), + }; + let mut msg = ChatMsg::text(m.role, text); + msg.parts = parts; + msg.extra = m.extra; + msg }) .collect(), model_param_v2: Some(param), @@ -5139,13 +5144,23 @@ mod tests { } } - #[derive(Debug, Default)] - struct RecordingResponses { + #[derive(Debug)] + struct Recording { + reply: &'static [u8], seen: std::sync::Mutex>, } + impl Recording { + fn new(reply: &'static [u8]) -> Arc { + Arc::new(Self { + reply, + seen: std::sync::Mutex::new(None), + }) + } + } + #[async_trait::async_trait] - impl gw_engines::transport::Transport for RecordingResponses { + impl gw_engines::transport::Transport for Recording { async fn send( &self, req: gw_engines::transport::UpstreamRequest, @@ -5154,7 +5169,7 @@ mod tests { Ok(gw_engines::transport::UpstreamResponse { status: 200, body: gw_engines::transport::UpstreamBody::Json(bytes::Bytes::from_static( - br#"{"id":"resp_1","model":"gpt-5-responses","status":"completed","output":[{"type":"message","content":[{"type":"output_text","text":"ok"}]}],"usage":{"input_tokens":3,"output_tokens":1}}"#, + self.reply, )), headers: Default::default(), }) @@ -5297,7 +5312,9 @@ mod tests { #[tokio::test] async fn messages_system_blocks_become_instructions_on_a_responses_model() { - let transport = Arc::new(RecordingResponses::default()); + let transport = Recording::new( + br#"{"id":"resp_1","model":"gpt-5-responses","status":"completed","output":[{"type":"message","content":[{"type":"output_text","text":"ok"}]}],"usage":{"input_tokens":3,"output_tokens":1}}"#, + ); let (router, _) = retained_view_app(transport.clone()); let request = Request::builder() .method("POST") @@ -5315,6 +5332,41 @@ mod tests { assert!(sent.get("system").is_none(), "{sent}"); } + #[tokio::test] + async fn messages_system_turns_reach_the_anthropic_upstream_in_place() { + let yaml = "listen: {host: h, port: 1}\nmodels: [{name: claude-opus-5-5, protocol: anthropic-messages}]\naccounts: [{name: a1, provider: anthropic, protocols: [anthropic-messages]}]\naccess_keys: [{ak: k, product: p, qps: 100, daily_token_quota: 100000}]"; + let cfg = Arc::new(GatewayConfig::from_yaml(yaml).unwrap()); + let state = Arc::new(GatewayState::from_config(&cfg)); + let transport = Recording::new( + br#"{"model":"claude-opus-5-5","content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn","usage":{"input_tokens":3,"output_tokens":1}}"#, + ); + let router = app(AppState::new(cfg, state, transport.clone())); + let request = Request::builder() + .method("POST") + .uri("/v1/messages") + .header("content-type", "application/json") + .header("x-api-key", "k") + .body(Body::from( + r#"{"model":"claude-opus-5-5","max_tokens":32,"system":[{"type":"text","text":"sys","cache_control":{"type":"ephemeral"}}],"messages":[{"role":"user","content":"hi"},{"role":"system","content":"env","output_config":{"effort":"medium"}},{"role":"assistant","content":[{"type":"text","text":"ok"}]},{"role":"user","content":"again"},{"role":"system","content":[{"type":"text","text":"reminder","cache_control":{"type":"ephemeral"}}]}]}"#, + )) + .unwrap(); + let response = router.oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let sent = transport.seen.lock().unwrap().take().unwrap(); + assert_eq!( + sent["messages"], + json!([ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "env", "output_config": {"effort": "medium"}}, + {"role": "assistant", "content": [{"type": "text", "text": "ok"}]}, + {"role": "user", "content": "again"}, + {"role": "system", "content": [ + {"type": "text", "text": "reminder", "cache_control": {"type": "ephemeral"}} + ]}, + ]) + ); + } + #[tokio::test] async fn retained_tts_decode_failure_uses_view_status() { let (router, store) = retained_view_app(Arc::new(InvalidTtsPayload)); diff --git a/docs/configuration.md b/docs/configuration.md index bdd4a174..b1b60ab7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -136,7 +136,7 @@ models: audio_completion: 8.33 # audio output tokens (default = completion weight) long_context: {threshold_tokens: 200000, prompt_weight: 2.0, completion_weight: 1.5} # optional tier past a prompt size batch_discount: 0.5 # optional: /v1/batches items at this fraction of the price; must be finite and in (0.0, 1.0] - prompt_cache: true # anthropic-messages only: prompt-cache breakpoints + prompt_cache: true # prompt-cache breakpoints (Anthropic wires; anthropic/* on OpenRouter Responses) fallback_models: [gpt-4o-mini] # optional: tried in order on an upstream 5xx / 429 / 401-403 refusal / connection failure / no healthy account (rejected at load when unknown, self or duplicate) variants: # optional weighted canary split, sticky per user - {model: gpt-4o, weight: 90} # self-reference keeps a share here @@ -166,12 +166,19 @@ same unit, for margin. `prompt_cache` (anthropic-messages models) marks the system prompt and the latest user turn as Anthropic prompt-cache breakpoints, so each turn of a conversation re-reads its prefix at the cache-read rate; it is per model and off by default because a long one-shot -prompt would pay the cache-write premium for nothing. A `/v1/messages` client -that sends `system` as blocks with its own `cache_control` (including -`ttl: 1h`) keeps them as sent; the gateway only marks the last system block -when that block carries no breakpoint of its own. A `/v1/chat/completions` -system message whose content parts carry `cache_control` keeps them on an -Anthropic-wire model the same way. On an openai-chat model +prompt would pay the cache-write premium for nothing. Client breakpoints win: +when the request already carries any `cache_control` (top-level, on a tool, a +system block or a message block), the gateway adds none, so the request stays +within Anthropic's four breakpoints and the client's TTL order. A +`/v1/messages` client's breakpoints, `ttl: 1h` included, ride through as sent, +and on `anthropic-messages` and `aws-anthropic` models its mid-conversation +`role: "system"` messages stay in place with their `output_config` and +`clear_at`; on `aws-converse` they join the end of `system`. On a `responses` model served by an OpenRouter account, +`prompt_cache` on an `anthropic/*` model adds the top-level +`cache_control: {type: ephemeral}` that OpenRouter needs to cache it, unless +the client sent its own. A `/v1/chat/completions` system message whose +content parts carry `cache_control` keeps them on an Anthropic-wire model the +same way. On an openai-chat model the client's own breakpoints on system blocks, message blocks and `tool_result` ride through as OpenAI content parts, which OpenRouter honors for Anthropic models; a breakpoint on a tool definition does not (Gemini's diff --git a/docs/examples.md b/docs/examples.md index 35466ade..c32a7a60 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -249,7 +249,9 @@ captured from the real client and replays in the live matrix (`agents` group). **Claude Code** — the Anthropic wire, so the gateway model must speak it natively or by conversion. Claude Code sends a system-block array with `cache_control` points, its tool set, `thinking` (with `display: omitted`), -`context_management`, and its user id in `metadata.user_id`; it enables beta +`context_management`, mid-conversation `role: "system"` messages (the +request's last breakpoint often sits on one), and its user id in +`metadata.user_id`; it enables beta features through the `anthropic-beta` header, which the gateway forwards to Anthropic-wire upstreams (the header for `kind: anthropic`, the `anthropic_beta` body list for `aws-anthropic`). Point it at the gateway with @@ -289,7 +291,10 @@ env_key = "GW_API_KEY" # export GW_API_KEY=ak-... wire_api = "responses" ``` -Codex tries a WebSocket first and falls back to HTTPS on its own. +Codex tries a WebSocket first and falls back to HTTPS on its own. An +`anthropic/*` model on an OpenRouter Responses account caches nothing unless +the request carries a top-level `cache_control`; set `prompt_cache: true` on +that model and the gateway adds it. **VS Code chat (GitHub Copilot)** — *Chat: Manage Language Models* → *Add Models* → *Custom Endpoint*: pick the API type (Chat Completions, Responses From 39de02675b095e2c3dcf90825e714a950f408549 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 26 Oct 2026 10:56:15 +0800 Subject: [PATCH 02/10] fix: let prompt_cache fill the breakpoints a client left out within four and TTL order A client that marks only its system prompt lost the gateway's last-user-turn breakpoint, so its conversation stopped caching. The gateway now marks the latest user turn unless the client marked a message block or sent a top-level cache_control, and the system prompt unless the client marked it or a 1h breakpoint follows it, turn first, never past four. --- crates/config/src/lib.rs | 2 +- crates/engines/src/claude_engine.rs | 59 ++++-- crates/engines/tests/request_construction.rs | 199 +++++++++++++++---- docs/configuration.md | 8 +- 4 files changed, 206 insertions(+), 62 deletions(-) diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 970aabf0..181e97a2 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -231,7 +231,7 @@ pub struct ModelConf { #[serde(default)] pub token_rate: Option, /// Anthropic prompt caching: mark the system prompt and the latest user - /// turn as cache breakpoints unless the client placed any (an `anthropic/*` + /// turn as cache breakpoints where the client placed none (an `anthropic/*` /// Responses model on OpenRouter gets the top-level `cache_control` instead), /// so a conversation's prefix bills at the cache-read rate. Pair with `token_rate`. #[serde(default)] diff --git a/crates/engines/src/claude_engine.rs b/crates/engines/src/claude_engine.rs index 67853a36..50669975 100644 --- a/crates/engines/src/claude_engine.rs +++ b/crates/engines/src/claude_engine.rs @@ -13,6 +13,7 @@ use crate::transport::{UpstreamBody, UpstreamRequest}; const DEFAULT_MAX_TOKENS: i64 = 1024; const THINKING_DEFAULT_MAX_TOKENS: i64 = 16384; +const MAX_BREAKPOINTS: usize = 4; base_engine!(ClaudeEngine); @@ -227,7 +228,7 @@ impl ClaudeEngine { } } crate::base::merge_raw_extras_owned(&mut body, raw); - if self.base.request.prompt_cache && !has_breakpoint(&body) { + if self.base.request.prompt_cache { place_breakpoints(&mut body); } Ok(body) @@ -859,30 +860,50 @@ fn system_turn(m: gw_models::ChatMsg) -> Value { Value::Object(turn) } -fn has_breakpoint(body: &Map) -> bool { - fn marked(blocks: Option<&Value>) -> bool { - blocks.and_then(Value::as_array).is_some_and(|blocks| { - blocks - .iter() - .any(|b| b.get("cache_control").is_some() || marked(b.get("content"))) - }) - } - body.contains_key("cache_control") - || ["tools", "system", "messages"] - .into_iter() - .any(|key| marked(body.get(key))) -} - fn place_breakpoints(body: &mut Map) { - if let Some(system) = body.get_mut("system") { - cache_last_block(system); - } - if let Some(Value::Array(turns)) = body.get_mut("messages") + let top = body.get("cache_control"); + let top_long = top.is_some_and(is_long_ttl); + let (tool_marks, _) = breakpoints(body.get("tools")); + let (system_marks, _) = breakpoints(body.get("system")); + let (message_marks, messages_long) = breakpoints(body.get("messages")); + let mut slots = MAX_BREAKPOINTS + .saturating_sub(tool_marks + system_marks + message_marks + usize::from(top.is_some())); + if slots > 0 + && message_marks == 0 + && top.is_none() + && let Some(Value::Array(turns)) = body.get_mut("messages") && let Some(last) = turns.iter_mut().rfind(|t| t["role"] != "system") && last["role"] == "user" { cache_last_block(&mut last["content"]); + slots -= 1; } + if slots > 0 + && system_marks == 0 + && !messages_long + && !top_long + && let Some(system) = body.get_mut("system") + { + cache_last_block(system); + } +} + +fn breakpoints(blocks: Option<&Value>) -> (usize, bool) { + let (mut count, mut long) = (0, false); + for block in blocks.and_then(Value::as_array).into_iter().flatten() { + if let Some(control) = block.get("cache_control") { + count += 1; + long |= is_long_ttl(control); + } + let (nested, nested_long) = breakpoints(block.get("content")); + count += nested; + long |= nested_long; + } + (count, long) +} + +fn is_long_ttl(control: &Value) -> bool { + control["ttl"].as_str().is_some_and(|ttl| ttl != "5m") } /// Content as blocks with a prompt-cache breakpoint on the last one (a breakpoint diff --git a/crates/engines/tests/request_construction.rs b/crates/engines/tests/request_construction.rs index 66bbb98f..29461607 100644 --- a/crates/engines/tests/request_construction.rs +++ b/crates/engines/tests/request_construction.rs @@ -1473,61 +1473,182 @@ async fn native_system_turns_fold_into_the_system_prompt_on_converse() { } #[tokio::test] -async fn prompt_cache_adds_no_breakpoint_beside_the_clients() { - let breakpoints = |b: &Value| b.to_string().matches("cache_control").count(); - let t = RecordingTransport::new(CLAUDE_OK); - let mut req = native_system_turn_req(Protocol::AnthropicMessages, "claude-opus-5-5"); - req.prompt_cache = true; - if let Some(TypedParams::Chat(p)) = req.model_param_v2.as_mut().unwrap().typed.as_mut() { - p.system_blocks = Some(serde_json::json!([ - {"type": "text", "text": "billing"}, - {"type": "text", "text": "core", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, - {"type": "text", "text": "sys", "cache_control": {"type": "ephemeral", "ttl": "1h"}} - ])); - p.tools = Some(serde_json::json!([ - {"name": "t", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral", "ttl": "1h"}} - ])); - } - ClaudeEngine::new(req, t.clone()).run().await.unwrap(); - let b = t.body_json(); - assert_eq!(breakpoints(&b), 4, "{b}"); - assert_eq!(b["messages"][4]["content"], "again", "{b}"); - - let mut tool_result = ChatMsg::text("user", ""); - tool_result.parts = Some( - serde_json::json!([{"type": "tool_result", "tool_use_id": "tu", - "content": [{"type": "text", "text": "r", "cache_control": {"type": "ephemeral"}}]}]), - ); - for (raw, tools, last) in [ +async fn prompt_cache_fills_only_the_breakpoints_the_client_left_out() { + let five = serde_json::json!({"type": "ephemeral"}); + let hour = serde_json::json!({"type": "ephemeral", "ttl": "1h"}); + let text = |text: &str, control: Option<&Value>| { + let mut block = serde_json::json!({"type": "text", "text": text}); + if let Some(control) = control { + block["cache_control"] = control.clone(); + } + block + }; + let tool = |name: &str| serde_json::json!({"name": name, "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral"}}); + let tool_result = serde_json::json!({"type": "tool_result", "tool_use_id": "tu", + "content": [{"type": "text", "text": "r", "cache_control": {"type": "ephemeral"}}]}); + let cases = [ ( - serde_json::json!({"cache_control": {"type": "ephemeral"}}), + "system only", + Some(vec![text("sys", Some(&five))]), None, + vec![text("q", None)], None, + 2, + true, + true, ), ( - Value::Null, - Some( - serde_json::json!([{"name": "t", "input_schema": {"type": "object"}, - "cache_control": {"type": "ephemeral"}}]), - ), + "system 1h", + Some(vec![text("sys", Some(&hour))]), + None, + vec![text("q", None)], None, + 2, + true, + true, ), - (Value::Null, None, Some(tool_result)), - ] { + ( + "system and message", + Some(vec![text("sys", Some(&five))]), + None, + vec![text("q", Some(&five))], + None, + 2, + true, + true, + ), + ( + "message only", + None, + None, + vec![text("q", Some(&five))], + None, + 2, + true, + true, + ), + ( + "message 1h", + None, + None, + vec![text("q", Some(&hour))], + None, + 1, + true, + false, + ), + ( + "tool_result", + None, + None, + vec![tool_result, text("q", None)], + None, + 2, + false, + true, + ), + ( + "top-level", + None, + None, + vec![text("q", None)], + Some(&five), + 2, + false, + true, + ), + ( + "top-level 1h", + None, + None, + vec![text("q", None)], + Some(&hour), + 1, + false, + false, + ), + ( + "four taken", + Some(vec![ + text("a", Some(&five)), + text("b", Some(&five)), + text("c", Some(&five)), + ]), + Some(vec![tool("t")]), + vec![text("q", None)], + None, + 4, + false, + true, + ), + ( + "one slot left", + None, + Some(vec![tool("t1"), tool("t2"), tool("t3")]), + vec![text("q", None)], + None, + 4, + true, + false, + ), + ]; + for (name, system, tools, last, top, total, user_marked, system_marked) in cases { let t = RecordingTransport::new(CLAUDE_OK); + let mut turn = ChatMsg::text("user", ""); + turn.parts = Some(Value::Array(last)); let mut req = chat_req(Protocol::AnthropicMessages, "claude-opus-5-5"); + req.preserve_anthropic_wire = true; req.prompt_cache = true; - req.message.extend(last); + req.message = vec![ + ChatMsg::text("user", "hi"), + ChatMsg::text("assistant", "ok"), + turn, + ]; let param = req.model_param_v2.as_mut().unwrap(); - param.raw = raw; + if let Some(top) = top { + param.raw = serde_json::json!({"cache_control": top}); + } param.typed = Some(TypedParams::Chat(ChatParams { - tools, + system: Some("sys".into()), + system_blocks: system.map(Value::Array), + tools: tools.map(Value::Array), ..Default::default() })); ClaudeEngine::new(req, t.clone()).run().await.unwrap(); let b = t.body_json(); - assert_eq!(breakpoints(&b), 1, "{b}"); - assert_eq!(b["system"], "be brief", "{b}"); + let marked = |v: &Value| { + v.as_array() + .and_then(|blocks| blocks.last()) + .is_some_and(|block| block.get("cache_control").is_some()) + }; + assert_eq!( + b.to_string().matches("cache_control").count(), + total, + "{name}: {b}" + ); + assert_eq!( + marked(&b["messages"][2]["content"]), + user_marked, + "{name}: {b}" + ); + assert_eq!(marked(&b["system"]), system_marked, "{name}: {b}"); + let mut ttls = Vec::new(); + for key in ["tools", "system", "messages"] { + collect_ttls(&b[key], &mut ttls); + } + assert!( + !ttls.windows(2).any(|w| w == ["5m", "1h"]), + "{name}: {ttls:?}" + ); + } +} + +fn collect_ttls(v: &Value, out: &mut Vec<&'static str>) { + for block in v.as_array().into_iter().flatten() { + if let Some(control) = block.get("cache_control") { + out.push(if control["ttl"] == "1h" { "1h" } else { "5m" }); + } + collect_ttls(&block["content"], out); } } diff --git a/docs/configuration.md b/docs/configuration.md index b1b60ab7..1a1f385f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -167,9 +167,11 @@ marks the system prompt and the latest user turn as Anthropic prompt-cache breakpoints, so each turn of a conversation re-reads its prefix at the cache-read rate; it is per model and off by default because a long one-shot prompt would pay the cache-write premium for nothing. Client breakpoints win: -when the request already carries any `cache_control` (top-level, on a tool, a -system block or a message block), the gateway adds none, so the request stays -within Anthropic's four breakpoints and the client's TTL order. A +the gateway marks the latest user turn only when the client put no breakpoint +on a message block (`tool_result` content included) and sent no top-level +`cache_control`, and the system prompt only when the client left it unmarked +and no 1-hour breakpoint follows it. The turn goes first, and neither is added +past Anthropic's four breakpoints. A `/v1/messages` client's breakpoints, `ttl: 1h` included, ride through as sent, and on `anthropic-messages` and `aws-anthropic` models its mid-conversation `role: "system"` messages stay in place with their `output_config` and From 61f7b176a31cf67c8d3a535a271d0c54dd080465 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 26 Oct 2026 11:49:13 +0800 Subject: [PATCH 03/10] fix: keep the top-level system text and the client's TTL order when Converse folds system turns A cache-marked system turn replaced a string top-level system on aws-converse, and a folded 5m breakpoint landed ahead of a 1h message breakpoint, which Bedrock rejects. Both reproduced live on Bedrock. --- crates/engines/src/claude_engine.rs | 29 ++++-- crates/engines/tests/request_construction.rs | 92 ++++++++++++++++++-- 2 files changed, 109 insertions(+), 12 deletions(-) diff --git a/crates/engines/src/claude_engine.rs b/crates/engines/src/claude_engine.rs index 50669975..e297e516 100644 --- a/crates/engines/src/claude_engine.rs +++ b/crates/engines/src/claude_engine.rs @@ -21,9 +21,10 @@ impl ClaudeEngine { /// The Messages body. On Bedrock the model rides in the invoke path and /// streaming in the URL, and the API version is a body field. fn build_body(&mut self) -> GResult> { - let protocol = self.base.param()?.protocol; + let param = self.base.param()?; + let (protocol, served_as_named) = (param.protocol, param.fallback_from.is_none()); let converse = protocol == gw_consts::Protocol::AwsConverse; - let in_place = self.base.request.preserve_anthropic_wire && !converse; + let in_place = self.base.request.preserve_anthropic_wire && !converse && served_as_named; let mut system_text = if in_place { String::new() } else { @@ -119,12 +120,14 @@ impl ClaudeEngine { let mut client_cap = false; // 4.7+ rejects the sampling knobs outright; 4.6 only once thinking is engaged let mut sampling_rejected = dialect == ThinkingDialect::AdaptiveSummarized; - let mut native_system = None; + let (mut native_system, mut top_system) = (None, None); if let Some(gw_models::TypedParams::Chat(p)) = self.base.take_typed() { // the native system-block array keeps the client's cache_control (and ttl) native_system = p.system_blocks.filter(Value::is_array); - if in_place && let Some(system) = p.system { - system_text = system; + if in_place { + system_text = p.system.unwrap_or_default(); + } else { + top_system = p.system; } if let Some(mt) = p.max_tokens { max_tokens = mt; @@ -189,8 +192,15 @@ impl ClaudeEngine { self.base.output_cap = max_tokens; } let mut raw = self.base.take_raw(); + let messages_long = marked && breakpoints(body.get("messages")).1; let folded = system_parts.into_iter().map(|part| match part { Value::String(text) => object([("type", "text".into()), ("text", text.into())]), + Value::Object(mut block) => { + if messages_long && block.get("cache_control").is_some_and(|c| !is_long_ttl(c)) { + block.remove("cache_control"); + } + Value::Object(block) + } part => part, }); let system = match native_system { @@ -198,7 +208,14 @@ impl ClaudeEngine { blocks.extend(folded); Some(Value::Array(blocks)) } - _ if marked => Some(Value::Array(folded.collect())), + _ if marked => Some(Value::Array( + top_system + .map(|text| content_blocks(Value::String(text))) + .unwrap_or_default() + .into_iter() + .chain(folded) + .collect(), + )), _ if system_text.is_empty() => None, _ => Some(Value::String(system_text)), }; diff --git a/crates/engines/tests/request_construction.rs b/crates/engines/tests/request_construction.rs index 29461607..44bec6f1 100644 --- a/crates/engines/tests/request_construction.rs +++ b/crates/engines/tests/request_construction.rs @@ -1449,27 +1449,107 @@ async fn native_system_turns_stay_in_place_on_the_anthropic_wires() { } #[tokio::test] -async fn native_system_turns_fold_into_the_system_prompt_on_converse() { +async fn native_system_turns_fold_into_the_system_prompt_on_converse_or_a_swapped_model() { + let converse_ok = r#"{"output":{"message":{"role":"assistant","content":[{"text":"ok"}]}},"stopReason":"end_turn","usage":{"inputTokens":3,"outputTokens":1}}"#; + for (protocol, model, requested, reply) in [ + ( + Protocol::AwsConverse, + "us.amazon.nova-lite-v1:0", + None, + converse_ok, + ), + ( + Protocol::AnthropicMessages, + "claude-sonnet-4-6", + Some("claude-opus-5-5"), + CLAUDE_OK, + ), + ] { + let t = RecordingTransport::new(reply); + let mut req = native_system_turn_req(protocol, model); + req.model_param_v2.as_mut().unwrap().fallback_from = requested.map(str::to_owned); + ClaudeEngine::new(req, t.clone()).run().await.unwrap(); + let b = t.body_json(); + let roles: Vec<&str> = b["messages"] + .as_array() + .unwrap() + .iter() + .map(|m| m["role"].as_str().unwrap()) + .collect(); + assert_eq!(roles, ["user", "assistant", "user"], "{b}"); + let system: Vec<&str> = b["system"] + .as_array() + .unwrap() + .iter() + .filter_map(|block| block["text"].as_str()) + .collect(); + assert_eq!(system, ["sys", "env block", "reminder"], "{b}"); + } +} + +#[tokio::test] +async fn a_cache_marked_system_turn_keeps_the_top_level_system_text_on_converse() { let t = RecordingTransport::new( r#"{"output":{"message":{"role":"assistant","content":[{"text":"ok"}]}},"stopReason":"end_turn","usage":{"inputTokens":3,"outputTokens":1}}"#, ); - let req = native_system_turn_req(Protocol::AwsConverse, "us.amazon.nova-lite-v1:0"); + let mut env = ChatMsg::text("system", "env block"); + env.parts = Some(serde_json::json!([ + {"type": "text", "text": "env block", "cache_control": {"type": "ephemeral"}} + ])); + let mut req = chat_req(Protocol::AwsConverse, "us.anthropic.claude-opus-5-5"); + req.preserve_anthropic_wire = true; + req.message = vec![ChatMsg::text("user", "hi"), env]; + req.model_param_v2.as_mut().unwrap().typed = Some(TypedParams::Chat(ChatParams { + system: Some("sys".into()), + ..Default::default() + })); ClaudeEngine::new(req, t.clone()).run().await.unwrap(); let b = t.body_json(); - let roles: Vec<&str> = b["messages"] + let system: Vec<&str> = b["system"] .as_array() .unwrap() .iter() - .map(|m| m["role"].as_str().unwrap()) + .filter_map(|block| block["text"].as_str()) .collect(); - assert_eq!(roles, ["user", "assistant", "user"], "{b}"); + assert_eq!(system, ["sys", "env block"], "{b}"); +} + +#[tokio::test] +async fn a_folded_system_turn_never_puts_a_5m_breakpoint_before_a_1h_one_on_converse() { + let t = RecordingTransport::new( + r#"{"output":{"message":{"role":"assistant","content":[{"text":"ok"}]}},"stopReason":"end_turn","usage":{"inputTokens":3,"outputTokens":1}}"#, + ); + let mut user = ChatMsg::text("user", "remember ALPHA"); + user.parts = Some( + serde_json::json!([{"type": "text", "text": "remember ALPHA", + "cache_control": {"type": "ephemeral", "ttl": "1h"}}]), + ); + let mut rule = ChatMsg::text("system", "one word"); + rule.parts = Some(serde_json::json!([{"type": "text", "text": "one word", + "cache_control": {"type": "ephemeral"}}])); + let mut req = chat_req(Protocol::AwsConverse, "global.anthropic.claude-opus-5-5"); + req.preserve_anthropic_wire = true; + req.message = vec![ + user, + rule, + ChatMsg::text("assistant", "noted"), + ChatMsg::text("user", "the word?"), + ]; + req.model_param_v2.as_mut().unwrap().typed = Some(TypedParams::Chat(ChatParams { + system: Some("terse".into()), + ..Default::default() + })); + ClaudeEngine::new(req, t.clone()).run().await.unwrap(); + let b = t.body_json(); let system: Vec<&str> = b["system"] .as_array() .unwrap() .iter() .filter_map(|block| block["text"].as_str()) .collect(); - assert_eq!(system, ["sys", "env block", "reminder"], "{b}"); + assert_eq!(system, ["terse", "one word"], "{b}"); + assert!(!b["system"].to_string().contains("cachePoint"), "{b}"); + assert!(b["messages"][0].to_string().contains("1h"), "{b}"); } #[tokio::test] From 5fe0144596631062e0dd3ac6b63c1f433aedc84a Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 26 Oct 2026 12:04:05 +0800 Subject: [PATCH 04/10] fix: fold system turns only when a fallback or variant serves a pre-5 Claude model 61f7b17 also folded them on every model swap, which put the variant alias route (claude-opus-5-5 to ~anthropic/claude-opus-latest) back on system-only caching. Pre-5 Claude models reject a mid-conversation system turn; 5.x and non-Claude Anthropic-wire vendors take it. --- crates/engines/src/claude_engine.rs | 15 ++++-- crates/engines/tests/request_construction.rs | 23 ++++++--- docs/configuration.md | 51 ++++++++++---------- 3 files changed, 55 insertions(+), 34 deletions(-) diff --git a/crates/engines/src/claude_engine.rs b/crates/engines/src/claude_engine.rs index e297e516..33dac0fb 100644 --- a/crates/engines/src/claude_engine.rs +++ b/crates/engines/src/claude_engine.rs @@ -4,7 +4,9 @@ use gw_models::{GResult, GatewayResponse}; use gw_protocol::object; -use gw_protocol::reasoning::{ThinkingDialect, is_thinking_block}; +use gw_protocol::reasoning::{ + ThinkingDialect, anthropic_thinking_dialect, anthropic_thinks_by_default, is_thinking_block, +}; use serde_json::{Map, Value, json}; use crate::base::base_engine; @@ -22,9 +24,11 @@ impl ClaudeEngine { /// streaming in the URL, and the API version is a body field. fn build_body(&mut self) -> GResult> { let param = self.base.param()?; - let (protocol, served_as_named) = (param.protocol, param.fallback_from.is_none()); + let protocol = param.protocol; let converse = protocol == gw_consts::Protocol::AwsConverse; - let in_place = self.base.request.preserve_anthropic_wire && !converse && served_as_named; + let in_place = self.base.request.preserve_anthropic_wire + && !converse + && (param.fallback_from.is_none() || !claude_before_5(¶m.model_name)); let mut system_text = if in_place { String::new() } else { @@ -877,6 +881,11 @@ fn system_turn(m: gw_models::ChatMsg) -> Value { Value::Object(turn) } +fn claude_before_5(model: &str) -> bool { + model.contains("claude") + && !anthropic_thinks_by_default(anthropic_thinking_dialect(model), model) +} + fn place_breakpoints(body: &mut Map) { let top = body.get("cache_control"); let top_long = top.is_some_and(is_long_ttl); diff --git a/crates/engines/tests/request_construction.rs b/crates/engines/tests/request_construction.rs index 44bec6f1..8874af52 100644 --- a/crates/engines/tests/request_construction.rs +++ b/crates/engines/tests/request_construction.rs @@ -1418,12 +1418,23 @@ fn native_system_turn_req(protocol: Protocol, model: &str) -> GatewayRequest { #[tokio::test] async fn native_system_turns_stay_in_place_on_the_anthropic_wires() { - for (protocol, model) in [ - (Protocol::AnthropicMessages, "claude-opus-5-5"), - (Protocol::AwsAnthropic, "anthropic.claude-opus-5-5"), + for (protocol, model, requested) in [ + (Protocol::AnthropicMessages, "claude-opus-5-5", None), + (Protocol::AwsAnthropic, "anthropic.claude-opus-5-5", None), + ( + Protocol::AnthropicMessages, + "~anthropic/claude-opus-latest", + Some("claude-opus-5-5"), + ), + ( + Protocol::AnthropicMessages, + "MiniMax-M2", + Some("claude-opus-5-5"), + ), ] { let t = RecordingTransport::new(CLAUDE_OK); - let req = native_system_turn_req(protocol, model); + let mut req = native_system_turn_req(protocol, model); + req.model_param_v2.as_mut().unwrap().fallback_from = requested.map(str::to_owned); ClaudeEngine::new(req, t.clone()).run().await.unwrap(); let b = t.body_json(); assert_eq!( @@ -1438,12 +1449,12 @@ async fn native_system_turns_stay_in_place_on_the_anthropic_wires() { {"type": "text", "text": "reminder", "cache_control": {"type": "ephemeral"}} ]}, ]), - "{protocol:?}" + "{protocol:?} {model}" ); assert_eq!( b["system"], serde_json::json!([{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]), - "{protocol:?}" + "{protocol:?} {model}" ); } } diff --git a/docs/configuration.md b/docs/configuration.md index 1a1f385f..039dec10 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -162,31 +162,32 @@ generated seconds, a `search` model's queries (one per call) — and adds to `cost_micros` next to any token cost; the count lands in the ledger's `billed_units` and the `/admin/usage` aggregates. An account's `cost_unit_price_micros` is the vendor side of the -same unit, for margin. `prompt_cache` (anthropic-messages models) -marks the system prompt and the latest user turn as Anthropic prompt-cache -breakpoints, so each turn of a conversation re-reads its prefix at the -cache-read rate; it is per model and off by default because a long one-shot -prompt would pay the cache-write premium for nothing. Client breakpoints win: -the gateway marks the latest user turn only when the client put no breakpoint -on a message block (`tool_result` content included) and sent no top-level -`cache_control`, and the system prompt only when the client left it unmarked -and no 1-hour breakpoint follows it. The turn goes first, and neither is added -past Anthropic's four breakpoints. A -`/v1/messages` client's breakpoints, `ttl: 1h` included, ride through as sent, -and on `anthropic-messages` and `aws-anthropic` models its mid-conversation -`role: "system"` messages stay in place with their `output_config` and -`clear_at`; on `aws-converse` they join the end of `system`. On a `responses` model served by an OpenRouter account, -`prompt_cache` on an `anthropic/*` model adds the top-level -`cache_control: {type: ephemeral}` that OpenRouter needs to cache it, unless -the client sent its own. A `/v1/chat/completions` system message whose -content parts carry `cache_control` keeps them on an Anthropic-wire model the -same way. On an openai-chat model -the client's own breakpoints on system blocks, message blocks and -`tool_result` ride through as OpenAI content parts, which OpenRouter honors -for Anthropic models; a breakpoint on a tool definition does not (Gemini's -compatible endpoint rejects the field), and a later breakpoint covers the -tools anyway. `variants` splits a -public name across other declared same-protocol models (one level): +same unit, for margin. `prompt_cache` (anthropic-messages models) marks the +system prompt and the latest user turn as Anthropic prompt-cache breakpoints, +so each turn of a conversation re-reads its prefix at the cache-read rate; it +is per model and off by default because a long one-shot prompt would pay the +cache-write premium for nothing. Client breakpoints win: the gateway marks the +latest user turn only when the client put no breakpoint on a message block +(`tool_result` content included) and sent no top-level `cache_control`, and +the system prompt only when the client left it unmarked and no 1-hour +breakpoint follows it. The turn goes first, and neither is added past +Anthropic's four breakpoints. A `/v1/messages` client's breakpoints, `ttl: 1h` +included, ride through as sent. On `anthropic-messages` and `aws-anthropic` +models its mid-conversation `role: "system"` messages stay in place with their +`output_config` and `clear_at`; they join the end of `system` on +`aws-converse`, and when a fallback or variant serves a pre-5 Claude model in +place of the one the client named, since those models reject the turn. On a +`responses` model served by an OpenRouter account, `prompt_cache` on an +`anthropic/*` model adds the top-level `cache_control: {type: ephemeral}` that +OpenRouter needs to cache it, unless the client sent its own. A +`/v1/chat/completions` system message whose content parts carry +`cache_control` keeps them on an Anthropic-wire model the same way. On an +openai-chat model the client's own breakpoints on system blocks, message +blocks and `tool_result` ride through as OpenAI content parts, which +OpenRouter honors for Anthropic models; a breakpoint on a tool definition does +not (Gemini's compatible endpoint rejects the field), and a later breakpoint +covers the tools anyway. `variants` splits a public name across other declared +same-protocol models (one level): entitlement and the per-(AK, model) daily counter judge the public name, billing prices the served variant, and the response echoes the requested name. Selection hashes the effective user, so a user sticks to one backend From de41d924aa28b70f184319ae82f13f0cfd174dd2 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 26 Oct 2026 12:15:00 +0800 Subject: [PATCH 05/10] docs: name the swaps that fold system turns and rewrap the prompt_cache paragraph --- docs/configuration.md | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 039dec10..b1d2ad66 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -174,21 +174,22 @@ breakpoint follows it. The turn goes first, and neither is added past Anthropic's four breakpoints. A `/v1/messages` client's breakpoints, `ttl: 1h` included, ride through as sent. On `anthropic-messages` and `aws-anthropic` models its mid-conversation `role: "system"` messages stay in place with their -`output_config` and `clear_at`; they join the end of `system` on -`aws-converse`, and when a fallback or variant serves a pre-5 Claude model in -place of the one the client named, since those models reject the turn. On a -`responses` model served by an OpenRouter account, `prompt_cache` on an -`anthropic/*` model adds the top-level `cache_control: {type: ephemeral}` that -OpenRouter needs to cache it, unless the client sent its own. A -`/v1/chat/completions` system message whose content parts carry -`cache_control` keeps them on an Anthropic-wire model the same way. On an -openai-chat model the client's own breakpoints on system blocks, message -blocks and `tool_result` ride through as OpenAI content parts, which -OpenRouter honors for Anthropic models; a breakpoint on a tool definition does -not (Gemini's compatible endpoint rejects the field), and a later breakpoint -covers the tools anyway. `variants` splits a public name across other declared -same-protocol models (one level): -entitlement and the per-(AK, model) daily counter judge the public name, +`output_config` and `clear_at`. They are folded into the end of `system` on +`aws-converse`, and when the gateway serves a pre-5 Claude model in place of +the one the client named (`fallback_models`, `variants`, a tenant's +`fallback_model`), since those models reject a system turn and its per-turn +effort. On a `responses` model served by an OpenRouter account, +`prompt_cache` on an `anthropic/*` model adds the top-level +`cache_control: {type: ephemeral}` that OpenRouter needs to cache it, unless +the client sent its own. A `/v1/chat/completions` system message whose +content parts carry `cache_control` keeps them on an Anthropic-wire model the +same way. On an openai-chat model the client's own breakpoints on system +blocks, message blocks and `tool_result` ride through as OpenAI content parts, +which OpenRouter honors for Anthropic models; a breakpoint on a tool +definition does not (Gemini's compatible endpoint rejects the field), and a +later breakpoint covers the tools anyway. `variants` splits a public name +across other declared same-protocol models (one level): entitlement and the +per-(AK, model) daily counter judge the public name, billing prices the served variant, and the response echoes the requested name. Selection hashes the effective user, so a user sticks to one backend across the fleet; a realtime session picks its variant once at the handshake From 544236b8878e0eed434423bc1528e9639d88aa2a Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 26 Oct 2026 12:59:19 +0800 Subject: [PATCH 06/10] fix: fold system turns for any pre-5 Claude id, and drop a swapped turn's per-turn effort The pre-5 check rode on the thinking dialect, which only parses Anthropic's hyphenated ids: OpenRouter's anthropic/claude-sonnet-4.6, -opus-4.7, -haiku-4.5, -opus-4 and -sonnet-4 counted as 5.x, so a variant to one of them sent the client's system turns and came back 424. The check now reads the major version that follows "claude". A turn's output_config is shaped for the model the client named; Sonnet 5 takes a mid-conversation system turn but rejects per-turn effort, so a swap to it failed too. A swapped request now keeps 5.x system turns in place without their output_config. --- crates/engines/src/claude_engine.rs | 25 +++++++---- crates/engines/tests/request_construction.rs | 47 ++++++++++++++------ docs/configuration.md | 19 ++++---- 3 files changed, 60 insertions(+), 31 deletions(-) diff --git a/crates/engines/src/claude_engine.rs b/crates/engines/src/claude_engine.rs index 33dac0fb..1a06bba1 100644 --- a/crates/engines/src/claude_engine.rs +++ b/crates/engines/src/claude_engine.rs @@ -4,9 +4,7 @@ use gw_models::{GResult, GatewayResponse}; use gw_protocol::object; -use gw_protocol::reasoning::{ - ThinkingDialect, anthropic_thinking_dialect, anthropic_thinks_by_default, is_thinking_block, -}; +use gw_protocol::reasoning::{ThinkingDialect, is_thinking_block}; use serde_json::{Map, Value, json}; use crate::base::base_engine; @@ -26,9 +24,10 @@ impl ClaudeEngine { let param = self.base.param()?; let protocol = param.protocol; let converse = protocol == gw_consts::Protocol::AwsConverse; + let served_as_named = param.fallback_from.is_none(); let in_place = self.base.request.preserve_anthropic_wire && !converse - && (param.fallback_from.is_none() || !claude_before_5(¶m.model_name)); + && (served_as_named || !claude_before_5(¶m.model_name)); let mut system_text = if in_place { String::new() } else { @@ -39,7 +38,7 @@ impl ClaudeEngine { for m in std::mem::take(&mut self.base.request.message) { if m.role == gw_consts::role::SYSTEM { if in_place { - messages.push(system_turn(m)); + messages.push(system_turn(m, served_as_named)); continue; } match m.parts { @@ -871,8 +870,11 @@ fn push_turn(messages: &mut Vec, role: &str, content: Value) { messages.push(Value::Object(msg)); } -fn system_turn(m: gw_models::ChatMsg) -> Value { +fn system_turn(m: gw_models::ChatMsg, served_as_named: bool) -> Value { let mut turn = m.extra; + if !served_as_named { + turn.remove("output_config"); + } turn.insert("role".into(), m.role.into()); turn.insert( "content".into(), @@ -882,8 +884,15 @@ fn system_turn(m: gw_models::ChatMsg) -> Value { } fn claude_before_5(model: &str) -> bool { - model.contains("claude") - && !anthropic_thinks_by_default(anthropic_thinking_dialect(model), model) + model + .find("claude") + .and_then(|i| { + model[i..] + .split(|c: char| !c.is_ascii_digit()) + .find(|s| !s.is_empty()) + }) + .and_then(|major| major.parse::().ok()) + .is_some_and(|major| major < 5) } fn place_breakpoints(body: &mut Map) { diff --git a/crates/engines/tests/request_construction.rs b/crates/engines/tests/request_construction.rs index 8874af52..16763c76 100644 --- a/crates/engines/tests/request_construction.rs +++ b/crates/engines/tests/request_construction.rs @@ -1431,26 +1431,33 @@ async fn native_system_turns_stay_in_place_on_the_anthropic_wires() { "MiniMax-M2", Some("claude-opus-5-5"), ), + ( + Protocol::AnthropicMessages, + "claude-sonnet-5", + Some("claude-opus-5-5"), + ), ] { let t = RecordingTransport::new(CLAUDE_OK); let mut req = native_system_turn_req(protocol, model); req.model_param_v2.as_mut().unwrap().fallback_from = requested.map(str::to_owned); ClaudeEngine::new(req, t.clone()).run().await.unwrap(); let b = t.body_json(); - assert_eq!( - b["messages"], - serde_json::json!([ - {"role": "user", "content": "hi"}, - {"role": "system", "content": "env block", "output_config": {"effort": "medium"}}, - {"role": "assistant", "content": "ok"}, - {"role": "system", "content": [], "output_config": {"effort": "high"}}, - {"role": "user", "content": "again"}, - {"role": "system", "content": [ - {"type": "text", "text": "reminder", "cache_control": {"type": "ephemeral"}} - ]}, - ]), - "{protocol:?} {model}" - ); + let mut want = serde_json::json!([ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "env block", "output_config": {"effort": "medium"}}, + {"role": "assistant", "content": "ok"}, + {"role": "system", "content": [], "output_config": {"effort": "high"}}, + {"role": "user", "content": "again"}, + {"role": "system", "content": [ + {"type": "text", "text": "reminder", "cache_control": {"type": "ephemeral"}} + ]}, + ]); + if requested.is_some() { + for turn in want.as_array_mut().unwrap() { + turn.as_object_mut().unwrap().remove("output_config"); + } + } + assert_eq!(b["messages"], want, "{protocol:?} {model}"); assert_eq!( b["system"], serde_json::json!([{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]), @@ -1475,6 +1482,18 @@ async fn native_system_turns_fold_into_the_system_prompt_on_converse_or_a_swappe Some("claude-opus-5-5"), CLAUDE_OK, ), + ( + Protocol::AnthropicMessages, + "anthropic/claude-sonnet-4.6", + Some("claude-opus-5-5"), + CLAUDE_OK, + ), + ( + Protocol::AnthropicMessages, + "anthropic/claude-opus-4", + Some("claude-opus-5-5"), + CLAUDE_OK, + ), ] { let t = RecordingTransport::new(reply); let mut req = native_system_turn_req(protocol, model); diff --git a/docs/configuration.md b/docs/configuration.md index b1d2ad66..859f1a4e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -177,15 +177,16 @@ models its mid-conversation `role: "system"` messages stay in place with their `output_config` and `clear_at`. They are folded into the end of `system` on `aws-converse`, and when the gateway serves a pre-5 Claude model in place of the one the client named (`fallback_models`, `variants`, a tenant's -`fallback_model`), since those models reject a system turn and its per-turn -effort. On a `responses` model served by an OpenRouter account, -`prompt_cache` on an `anthropic/*` model adds the top-level -`cache_control: {type: ephemeral}` that OpenRouter needs to cache it, unless -the client sent its own. A `/v1/chat/completions` system message whose -content parts carry `cache_control` keeps them on an Anthropic-wire model the -same way. On an openai-chat model the client's own breakpoints on system -blocks, message blocks and `tool_result` ride through as OpenAI content parts, -which OpenRouter honors for Anthropic models; a breakpoint on a tool +`fallback_model`), since those models reject a system turn; any other model +served in place of the named one keeps them in place without their +`output_config`, which only some models accept. On a `responses` model served +by an OpenRouter account, `prompt_cache` on an `anthropic/*` model adds the +top-level `cache_control: {type: ephemeral}` that OpenRouter needs to cache +it, unless the client sent its own. A `/v1/chat/completions` system message +whose content parts carry `cache_control` keeps them on an Anthropic-wire +model the same way. On an openai-chat model the client's own breakpoints on +system blocks, message blocks and `tool_result` ride through as OpenAI content +parts, which OpenRouter honors for Anthropic models; a breakpoint on a tool definition does not (Gemini's compatible endpoint rejects the field), and a later breakpoint covers the tools anyway. `variants` splits a public name across other declared same-protocol models (one level): entitlement and the From e17dc860d4ff57a4cca95cba89138010fd13bacf Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 26 Oct 2026 12:59:59 +0800 Subject: [PATCH 07/10] review: the prompt_cache doc back to its three lines --- crates/config/src/lib.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 181e97a2..25c7c132 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -230,10 +230,9 @@ pub struct ModelConf { /// Billing weights per token component; None = every component at 1.0. #[serde(default)] pub token_rate: Option, - /// Anthropic prompt caching: mark the system prompt and the latest user - /// turn as cache breakpoints where the client placed none (an `anthropic/*` - /// Responses model on OpenRouter gets the top-level `cache_control` instead), - /// so a conversation's prefix bills at the cache-read rate. Pair with `token_rate`. + /// Anthropic prompt caching: mark the system prompt and latest user turn where the client + /// placed none (an `anthropic/*` Responses model on OpenRouter gets a top-level + /// `cache_control`). Pair with `token_rate`. #[serde(default)] pub prompt_cache: bool, /// Models tried in order when this one fails upstream (5xx, 429, 401-403, connection From bf1f6e1840cb9ad858ed9497986dae20132db7dc Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 26 Oct 2026 13:33:41 +0800 Subject: [PATCH 08/10] fix: drop the betas Bedrock rejects, and carry the rest to Claude on Converse Bedrock answers advisor-tool-2026-03-01, advanced-tool-use-2025-11-20 and prompt-caching-scope-2026-01-05 with a 400, and Claude Code sends the first two on every request, so each Bedrock call failed once before the client retried with its tools undeferred. Without that beta Bedrock still honors defer_loading. Converse got no betas at all, so it refused the client's context_management; the filtered list now rides in additionalModelRequestFields for Claude models, which is what Bedrock needs to accept it. --- crates/engines/src/claude_engine.rs | 61 ++++++++++++++++++++++++++--- docs/api.md | 2 +- docs/examples.md | 11 +++--- docs/providers.md | 4 +- 4 files changed, 64 insertions(+), 14 deletions(-) diff --git a/crates/engines/src/claude_engine.rs b/crates/engines/src/claude_engine.rs index 1a06bba1..8938f8e2 100644 --- a/crates/engines/src/claude_engine.rs +++ b/crates/engines/src/claude_engine.rs @@ -14,6 +14,11 @@ use crate::transport::{UpstreamBody, UpstreamRequest}; const DEFAULT_MAX_TOKENS: i64 = 1024; const THINKING_DEFAULT_MAX_TOKENS: i64 = 16384; const MAX_BREAKPOINTS: usize = 4; +const BEDROCK_REJECTED_BETAS: [&str; 3] = [ + "advisor-tool-2026-03-01", + "advanced-tool-use-2025-11-20", + "prompt-caching-scope-2026-01-05", +]; base_engine!(ClaudeEngine); @@ -102,16 +107,24 @@ impl ClaudeEngine { let mut body = Map::new(); if protocol == gw_consts::Protocol::AwsAnthropic { body.insert("anthropic_version".into(), "bedrock-2023-05-31".into()); - if let Some(betas) = self.base.request.anthropic_beta.as_deref() { - body.insert( - "anthropic_beta".into(), - Value::Array(betas.split(',').map(|b| Value::from(b.trim())).collect()), - ); - } } else if !bedrock { body.insert("model".into(), param.model_name.clone().into()); body.insert("stream".into(), self.base.request.stream.into()); } + if bedrock + && anthropic_fields + && let Some(betas) = self.base.request.anthropic_beta.as_deref() + { + let betas: Vec = betas + .split(',') + .map(str::trim) + .filter(|b| !BEDROCK_REJECTED_BETAS.contains(b)) + .map(Value::from) + .collect(); + if !betas.is_empty() { + body.insert("anthropic_beta".into(), Value::Array(betas)); + } + } body.insert("messages".into(), Value::Array(messages)); let dialect = gw_protocol::reasoning::anthropic_thinking_dialect(¶m.model_name); let mut max_tokens = @@ -1073,6 +1086,42 @@ mod tests { assert!(!up.headers.iter().any(|(k, _)| *k == "anthropic-beta")); } + #[test] + fn bedrock_drops_the_betas_it_rejects_and_converse_carries_the_rest_for_claude() { + let betas = + "context-management-2025-06-27,advisor-tool-2026-03-01,advanced-tool-use-2025-11-20"; + for (protocol, model, want) in [ + ( + Protocol::AwsAnthropic, + "us.anthropic.claude-opus-5-5", + Some(json!(["context-management-2025-06-27"])), + ), + ( + Protocol::AwsConverse, + "global.anthropic.claude-opus-5-5", + Some(json!(["context-management-2025-06-27"])), + ), + (Protocol::AwsConverse, "us.amazon.nova-lite-v1:0", None), + ] { + let mut r = base_req(); + r.anthropic_beta = Some(betas.to_owned()); + r.model_param_v2 = Some(ModelParamV2::with_name(protocol, model)); + let mut e = ClaudeEngine::new(r, Arc::new(MockTransport)); + let body = e.build_body().unwrap(); + assert_eq!(body.get("anthropic_beta").cloned(), want, "{model}"); + if protocol == Protocol::AwsConverse { + let out = crate::converse::request(body, model); + assert_eq!( + out.get("additionalModelRequestFields") + .and_then(|extra| extra.get("anthropic_beta")) + .cloned(), + want, + "{model}" + ); + } + } + } + #[test] fn openai_tools_without_description_or_parameters_normalize_cleanly() { let tools = normalize_tools_anthropic(json!([ diff --git a/docs/api.md b/docs/api.md index 5f15dd6c..e1ed524b 100644 --- a/docs/api.md +++ b/docs/api.md @@ -174,7 +174,7 @@ a cost. | Method | Path | Notes | |--------|------|-------| -| POST | `/v1/messages` | streaming + non-streaming; the client's `anthropic-beta` header is forwarded to Anthropic-wire upstreams (as the `anthropic_beta` body list on Bedrock) | +| POST | `/v1/messages` | streaming + non-streaming; the client's `anthropic-beta` header is forwarded to Anthropic-wire upstreams (as the `anthropic_beta` body list on Bedrock, minus the betas Bedrock rejects) | `/v1/messages` works on both Anthropic-protocol models and OpenAI-protocol models — the gateway converts between the two, including the streaming event diff --git a/docs/examples.md b/docs/examples.md index c32a7a60..ee7ec15f 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -251,11 +251,12 @@ natively or by conversion. Claude Code sends a system-block array with `cache_control` points, its tool set, `thinking` (with `display: omitted`), `context_management`, mid-conversation `role: "system"` messages (the request's last breakpoint often sits on one), and its user id in -`metadata.user_id`; it enables beta -features through the `anthropic-beta` header, which the gateway forwards to -Anthropic-wire upstreams (the header for `kind: anthropic`, the -`anthropic_beta` body list for `aws-anthropic`). Point it at the gateway with -the access key as the API key: +`metadata.user_id`; it enables beta features through the `anthropic-beta` +header, which the gateway forwards to Anthropic-wire upstreams (the header for +`kind: anthropic`, the `anthropic_beta` body list for `aws-anthropic` and for +Claude on `aws-converse`, minus the advisor-tool, advanced-tool-use and +prompt-caching-scope betas Bedrock rejects). Point it at the gateway with the +access key as the API key: ```bash export ANTHROPIC_BASE_URL=https://gw.example.com diff --git a/docs/providers.md b/docs/providers.md index 02526664..43cde29f 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -91,8 +91,8 @@ usage); the rest are marked non-streaming below and always answer buffered: | `dashscope` | Alibaba Qwen (native) | `https://dashscope-intl.aliyuncs.com` | `Bearer`; streams via `X-DashScope-SSE` + `incremental_output` | | `anthropic-messages` | any Anthropic-compatible endpoint (e.g. MiniMax) | vendor's `/anthropic` base | `x-api-key`; some report `input_tokens` only in `message_delta` — handled | | `ernie` | Baidu Ernie (Wenxin) | `https://aip.baidubce.com` | a `bce-v3/…` key goes as `Bearer`, a legacy token as the `access_token` query param (non-streaming); Qianfan's OpenAI-compatible `https://qianfan.baidubce.com/v2` also works as `kind: openai` + `endpoint` | -| `aws-anthropic` | Anthropic Claude on AWS Bedrock | `https://bedrock-runtime..amazonaws.com` | SigV4 (see below); model name = the Bedrock model id (`anthropic.claude-…`, `us.anthropic.claude-…`); the full Messages engine (system, tools, thinking dialects by generation, prompt-cache breakpoints, signed reasoning) on the InvokeModel wire — `anthropic_version` in the body, the client's `anthropic-beta` header as the `anthropic_beta` list, model and streaming in the path; streams via InvokeModelWithResponseStream (EventStream frames decoded into the same event sequence) | -| `aws-converse` | any model on AWS Bedrock via the Converse API | `https://bedrock-runtime..amazonaws.com` | SigV4 or API key (see below); model name = the Bedrock model id or inference profile (`eu.amazon.nova-micro-v1:0`, `us.meta.llama3-3-70b-instruct-v1:0`, `mistral.pixtral-large-2502-v1:0`, `anthropic.claude-…`); the Messages engine transcoded to Converse — system, tools + tool results, images, thinking replay, prompt-cache points (Claude takes them on system, messages and tools with their TTL, Nova on system and messages at the default TTL; other families get none; OpenAI GPT, Grok and Kimi ids refuse an image inside a tool result, so there its blocks from the first image on, documents aside, follow it as sibling blocks, while Llama 4 and the rest keep the image nested) — and back (buffered and `converse-stream`); Claude reasoning knobs ride in `additionalModelRequestFields`, other passthrough extras too; on an `openai.gpt-` or `xai.grok-` id the requested effort becomes that family's `reasoning_config` enum instead — which takes `max` from every generation but never `minimal` — and Bedrock's own refusal of `temperature`/`topP` for those ids drops them from `inferenceConfig` (live: `us.`/`global.openai.gpt-6-astra`, `…gpt-5.6-luna`, and `us.xai.grok-4.6`, which rejects both knobs the same way, takes `none`…`max`, and returns its reasoning as `reasoningContent.redactedContent` — surfaced as a `redacted_thinking` block; Converse also rejects `cachePoint` on Grok and never reports a cache read, so price that route without a cache weight) | +| `aws-anthropic` | Anthropic Claude on AWS Bedrock | `https://bedrock-runtime..amazonaws.com` | SigV4 (see below); model name = the Bedrock model id (`anthropic.claude-…`, `us.anthropic.claude-…`); the full Messages engine (system, tools, thinking dialects by generation, prompt-cache breakpoints, signed reasoning) on the InvokeModel wire — `anthropic_version` in the body, the client's `anthropic-beta` header as the `anthropic_beta` list (minus the advisor-tool, advanced-tool-use and prompt-caching-scope betas Bedrock rejects), model and streaming in the path; streams via InvokeModelWithResponseStream (EventStream frames decoded into the same event sequence) | +| `aws-converse` | any model on AWS Bedrock via the Converse API | `https://bedrock-runtime..amazonaws.com` | SigV4 or API key (see below); model name = the Bedrock model id or inference profile (`eu.amazon.nova-micro-v1:0`, `us.meta.llama3-3-70b-instruct-v1:0`, `mistral.pixtral-large-2502-v1:0`, `anthropic.claude-…`); the Messages engine transcoded to Converse — system, tools + tool results, images, thinking replay, prompt-cache points (Claude takes them on system, messages and tools with their TTL, Nova on system and messages at the default TTL; other families get none; OpenAI GPT, Grok and Kimi ids refuse an image inside a tool result, so there its blocks from the first image on, documents aside, follow it as sibling blocks, while Llama 4 and the rest keep the image nested) — and back (buffered and `converse-stream`); Claude reasoning knobs ride in `additionalModelRequestFields`, other passthrough extras too, and for Claude the filtered `anthropic_beta` list with them, which `context_management` needs; on an `openai.gpt-` or `xai.grok-` id the requested effort becomes that family's `reasoning_config` enum instead — which takes `max` from every generation but never `minimal` — and Bedrock's own refusal of `temperature`/`topP` for those ids drops them from `inferenceConfig` (live: `us.`/`global.openai.gpt-6-astra`, `…gpt-5.6-luna`, and `us.xai.grok-4.6`, which rejects both knobs the same way, takes `none`…`max`, and returns its reasoning as `reasoningContent.redactedContent` — surfaced as a `redacted_thinking` block; Converse also rejects `cachePoint` on Grok and never reports a cache read, so price that route without a cache weight) | | `aws-embed` | Titan / Cohere embeddings on AWS Bedrock | `https://bedrock-runtime..amazonaws.com` | SigV4 or API-key Bearer (see below); model name = the Bedrock model id; Titan `{inputText}` → `{embedding, inputTextTokenCount}` takes exactly one input per call (a batch is refused with 400; `dimensions` forwarded when the client sends it), Cohere `{texts, input_type: search_document}` → `{embeddings}` (v4: `{embeddings: {float}}`, `dimensions` forwarded as `output_dimension`) in one call, also on a `us.`/`global.` inference profile; answered in the OpenAI `/v1/embeddings` list shape, usage from Bedrock's `x-amzn-bedrock-*-token-count` headers | | `aws-llama` | Meta Llama on AWS Bedrock | `https://bedrock-runtime..amazonaws.com` | SigV4 (see below); model name = the Bedrock model id or inference profile (`meta.llama3-8b-instruct-v1:0`, `us.meta.llama3-3-70b-instruct-v1:0`, `us.meta.llama4-scout-17b-instruct-v1:0`); the conversation is rendered into the Llama 3 (or Llama 4) chat template; usage from the token-count headers / invocation metrics, else the body counts | | `minimax-v1` | MiniMax legacy v1 (`abab*`) | `https://api.minimax.chat` | `Bearer` (non-streaming); kept for existing accounts — the vendor has retired it for new ones; new integrations should use MiniMax's OpenAI-/Anthropic-compatible endpoints | From 1153e8b33755674eaa392d91a147a2cdd103b666 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 26 Oct 2026 13:45:23 +0800 Subject: [PATCH 09/10] fix: skip a swapped system turn that dropping output_config leaves empty An effort-only system turn ({"content": [], "output_config": ...}) loses its only field when the request is swapped, and Anthropic refuses the empty turn ("system content must contain at least one block"), so a swap to Sonnet 5 returned 424. Such a turn is now skipped; a client's own empty turn still passes through as sent. --- crates/engines/src/claude_engine.rs | 21 +++++++++++--------- crates/engines/tests/request_construction.rs | 4 +++- docs/configuration.md | 21 ++++++++++---------- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/crates/engines/src/claude_engine.rs b/crates/engines/src/claude_engine.rs index 8938f8e2..ea1f37fe 100644 --- a/crates/engines/src/claude_engine.rs +++ b/crates/engines/src/claude_engine.rs @@ -43,7 +43,7 @@ impl ClaudeEngine { for m in std::mem::take(&mut self.base.request.message) { if m.role == gw_consts::role::SYSTEM { if in_place { - messages.push(system_turn(m, served_as_named)); + messages.extend(system_turn(m, served_as_named)); continue; } match m.parts { @@ -883,17 +883,20 @@ fn push_turn(messages: &mut Vec, role: &str, content: Value) { messages.push(Value::Object(msg)); } -fn system_turn(m: gw_models::ChatMsg, served_as_named: bool) -> Value { +fn system_turn(m: gw_models::ChatMsg, served_as_named: bool) -> Option { let mut turn = m.extra; - if !served_as_named { - turn.remove("output_config"); + let content = m.parts.unwrap_or(Value::String(m.content)); + let empty = match &content { + Value::String(text) => text.is_empty(), + Value::Array(blocks) => blocks.is_empty(), + _ => false, + }; + if !served_as_named && turn.remove("output_config").is_some() && turn.is_empty() && empty { + return None; } turn.insert("role".into(), m.role.into()); - turn.insert( - "content".into(), - m.parts.unwrap_or(Value::String(m.content)), - ); - Value::Object(turn) + turn.insert("content".into(), content); + Some(Value::Object(turn)) } fn claude_before_5(model: &str) -> bool { diff --git a/crates/engines/tests/request_construction.rs b/crates/engines/tests/request_construction.rs index 16763c76..55a69cc1 100644 --- a/crates/engines/tests/request_construction.rs +++ b/crates/engines/tests/request_construction.rs @@ -1453,7 +1453,9 @@ async fn native_system_turns_stay_in_place_on_the_anthropic_wires() { ]}, ]); if requested.is_some() { - for turn in want.as_array_mut().unwrap() { + let turns = want.as_array_mut().unwrap(); + turns.retain(|turn| turn["content"] != serde_json::json!([])); + for turn in turns { turn.as_object_mut().unwrap().remove("output_config"); } } diff --git a/docs/configuration.md b/docs/configuration.md index 859f1a4e..40a3f3cf 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -179,16 +179,17 @@ models its mid-conversation `role: "system"` messages stay in place with their the one the client named (`fallback_models`, `variants`, a tenant's `fallback_model`), since those models reject a system turn; any other model served in place of the named one keeps them in place without their -`output_config`, which only some models accept. On a `responses` model served -by an OpenRouter account, `prompt_cache` on an `anthropic/*` model adds the -top-level `cache_control: {type: ephemeral}` that OpenRouter needs to cache -it, unless the client sent its own. A `/v1/chat/completions` system message -whose content parts carry `cache_control` keeps them on an Anthropic-wire -model the same way. On an openai-chat model the client's own breakpoints on -system blocks, message blocks and `tool_result` ride through as OpenAI content -parts, which OpenRouter honors for Anthropic models; a breakpoint on a tool -definition does not (Gemini's compatible endpoint rejects the field), and a -later breakpoint covers the tools anyway. `variants` splits a public name +`output_config`, which only some models accept, and drops a turn that carried +nothing else. On a `responses` model served by an OpenRouter account, +`prompt_cache` on an `anthropic/*` model adds the top-level +`cache_control: {type: ephemeral}` that OpenRouter needs to cache it, unless +the client sent its own. A `/v1/chat/completions` system message whose content +parts carry `cache_control` keeps them on an Anthropic-wire model the same +way. On an openai-chat model the client's own breakpoints on system blocks, +message blocks and `tool_result` ride through as OpenAI content parts, which +OpenRouter honors for Anthropic models; a breakpoint on a tool definition does +not (Gemini's compatible endpoint rejects the field), and a later breakpoint +covers the tools anyway. `variants` splits a public name across other declared same-protocol models (one level): entitlement and the per-(AK, model) daily counter judge the public name, billing prices the served variant, and the response echoes the requested From 29b8cdaaa59e15b977218f9d7f5d21505a8c72a4 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 26 Oct 2026 13:59:36 +0800 Subject: [PATCH 10/10] review: one row per test case, one Converse reply fixture, one system-text helper --- crates/engines/tests/request_construction.rs | 273 +++++-------------- 1 file changed, 65 insertions(+), 208 deletions(-) diff --git a/crates/engines/tests/request_construction.rs b/crates/engines/tests/request_construction.rs index 55a69cc1..0e7f2afd 100644 --- a/crates/engines/tests/request_construction.rs +++ b/crates/engines/tests/request_construction.rs @@ -1418,28 +1418,18 @@ fn native_system_turn_req(protocol: Protocol, model: &str) -> GatewayRequest { #[tokio::test] async fn native_system_turns_stay_in_place_on_the_anthropic_wires() { - for (protocol, model, requested) in [ - (Protocol::AnthropicMessages, "claude-opus-5-5", None), - (Protocol::AwsAnthropic, "anthropic.claude-opus-5-5", None), - ( - Protocol::AnthropicMessages, - "~anthropic/claude-opus-latest", - Some("claude-opus-5-5"), - ), - ( - Protocol::AnthropicMessages, - "MiniMax-M2", - Some("claude-opus-5-5"), - ), - ( - Protocol::AnthropicMessages, - "claude-sonnet-5", - Some("claude-opus-5-5"), - ), + let am = Protocol::AnthropicMessages; + for (protocol, model, swapped) in [ + (am, "claude-opus-5-5", false), + (Protocol::AwsAnthropic, "anthropic.claude-opus-5-5", false), + (am, "~anthropic/claude-opus-latest", true), + (am, "MiniMax-M2", true), + (am, "claude-sonnet-5", true), ] { let t = RecordingTransport::new(CLAUDE_OK); let mut req = native_system_turn_req(protocol, model); - req.model_param_v2.as_mut().unwrap().fallback_from = requested.map(str::to_owned); + req.model_param_v2.as_mut().unwrap().fallback_from = + swapped.then(|| "claude-opus-5-5".into()); ClaudeEngine::new(req, t.clone()).run().await.unwrap(); let b = t.body_json(); let mut want = serde_json::json!([ @@ -1452,7 +1442,7 @@ async fn native_system_turns_stay_in_place_on_the_anthropic_wires() { {"type": "text", "text": "reminder", "cache_control": {"type": "ephemeral"}} ]}, ]); - if requested.is_some() { + if swapped { let turns = want.as_array_mut().unwrap(); turns.retain(|turn| turn["content"] != serde_json::json!([])); for turn in turns { @@ -1470,36 +1460,22 @@ async fn native_system_turns_stay_in_place_on_the_anthropic_wires() { #[tokio::test] async fn native_system_turns_fold_into_the_system_prompt_on_converse_or_a_swapped_model() { - let converse_ok = r#"{"output":{"message":{"role":"assistant","content":[{"text":"ok"}]}},"stopReason":"end_turn","usage":{"inputTokens":3,"outputTokens":1}}"#; - for (protocol, model, requested, reply) in [ - ( - Protocol::AwsConverse, - "us.amazon.nova-lite-v1:0", - None, - converse_ok, - ), - ( - Protocol::AnthropicMessages, - "claude-sonnet-4-6", - Some("claude-opus-5-5"), - CLAUDE_OK, - ), - ( - Protocol::AnthropicMessages, - "anthropic/claude-sonnet-4.6", - Some("claude-opus-5-5"), - CLAUDE_OK, - ), - ( - Protocol::AnthropicMessages, - "anthropic/claude-opus-4", - Some("claude-opus-5-5"), - CLAUDE_OK, - ), + let am = Protocol::AnthropicMessages; + for (protocol, model, swapped) in [ + (Protocol::AwsConverse, "us.amazon.nova-lite-v1:0", false), + (am, "claude-sonnet-4-6", true), + (am, "anthropic/claude-sonnet-4.6", true), + (am, "anthropic/claude-opus-4", true), ] { + let reply = if protocol == Protocol::AwsConverse { + CONVERSE_OK + } else { + CLAUDE_OK + }; let t = RecordingTransport::new(reply); let mut req = native_system_turn_req(protocol, model); - req.model_param_v2.as_mut().unwrap().fallback_from = requested.map(str::to_owned); + req.model_param_v2.as_mut().unwrap().fallback_from = + swapped.then(|| "claude-opus-5-5".into()); ClaudeEngine::new(req, t.clone()).run().await.unwrap(); let b = t.body_json(); let roles: Vec<&str> = b["messages"] @@ -1509,21 +1485,13 @@ async fn native_system_turns_fold_into_the_system_prompt_on_converse_or_a_swappe .map(|m| m["role"].as_str().unwrap()) .collect(); assert_eq!(roles, ["user", "assistant", "user"], "{b}"); - let system: Vec<&str> = b["system"] - .as_array() - .unwrap() - .iter() - .filter_map(|block| block["text"].as_str()) - .collect(); - assert_eq!(system, ["sys", "env block", "reminder"], "{b}"); + assert_eq!(system_texts(&b), ["sys", "env block", "reminder"], "{b}"); } } #[tokio::test] async fn a_cache_marked_system_turn_keeps_the_top_level_system_text_on_converse() { - let t = RecordingTransport::new( - r#"{"output":{"message":{"role":"assistant","content":[{"text":"ok"}]}},"stopReason":"end_turn","usage":{"inputTokens":3,"outputTokens":1}}"#, - ); + let t = RecordingTransport::new(CONVERSE_OK); let mut env = ChatMsg::text("system", "env block"); env.parts = Some(serde_json::json!([ {"type": "text", "text": "env block", "cache_control": {"type": "ephemeral"}} @@ -1537,20 +1505,12 @@ async fn a_cache_marked_system_turn_keeps_the_top_level_system_text_on_converse( })); ClaudeEngine::new(req, t.clone()).run().await.unwrap(); let b = t.body_json(); - let system: Vec<&str> = b["system"] - .as_array() - .unwrap() - .iter() - .filter_map(|block| block["text"].as_str()) - .collect(); - assert_eq!(system, ["sys", "env block"], "{b}"); + assert_eq!(system_texts(&b), ["sys", "env block"], "{b}"); } #[tokio::test] async fn a_folded_system_turn_never_puts_a_5m_breakpoint_before_a_1h_one_on_converse() { - let t = RecordingTransport::new( - r#"{"output":{"message":{"role":"assistant","content":[{"text":"ok"}]}},"stopReason":"end_turn","usage":{"inputTokens":3,"outputTokens":1}}"#, - ); + let t = RecordingTransport::new(CONVERSE_OK); let mut user = ChatMsg::text("user", "remember ALPHA"); user.parts = Some( serde_json::json!([{"type": "text", "text": "remember ALPHA", @@ -1573,13 +1533,7 @@ async fn a_folded_system_turn_never_puts_a_5m_breakpoint_before_a_1h_one_on_conv })); ClaudeEngine::new(req, t.clone()).run().await.unwrap(); let b = t.body_json(); - let system: Vec<&str> = b["system"] - .as_array() - .unwrap() - .iter() - .filter_map(|block| block["text"].as_str()) - .collect(); - assert_eq!(system, ["terse", "one word"], "{b}"); + assert_eq!(system_texts(&b), ["terse", "one word"], "{b}"); assert!(!b["system"].to_string().contains("cachePoint"), "{b}"); assert!(b["messages"][0].to_string().contains("1h"), "{b}"); } @@ -1598,111 +1552,23 @@ async fn prompt_cache_fills_only_the_breakpoints_the_client_left_out() { let tool = |name: &str| serde_json::json!({"name": name, "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral"}}); let tool_result = serde_json::json!({"type": "tool_result", "tool_use_id": "tu", "content": [{"type": "text", "text": "r", "cache_control": {"type": "ephemeral"}}]}); + let (f, h) = (Some(&five), Some(&hour)); + let q = || vec![text("q", None)]; + #[rustfmt::skip] let cases = [ - ( - "system only", - Some(vec![text("sys", Some(&five))]), - None, - vec![text("q", None)], - None, - 2, - true, - true, - ), - ( - "system 1h", - Some(vec![text("sys", Some(&hour))]), - None, - vec![text("q", None)], - None, - 2, - true, - true, - ), - ( - "system and message", - Some(vec![text("sys", Some(&five))]), - None, - vec![text("q", Some(&five))], - None, - 2, - true, - true, - ), - ( - "message only", - None, - None, - vec![text("q", Some(&five))], - None, - 2, - true, - true, - ), - ( - "message 1h", - None, - None, - vec![text("q", Some(&hour))], - None, - 1, - true, - false, - ), - ( - "tool_result", - None, - None, - vec![tool_result, text("q", None)], - None, - 2, - false, - true, - ), - ( - "top-level", - None, - None, - vec![text("q", None)], - Some(&five), - 2, - false, - true, - ), - ( - "top-level 1h", - None, - None, - vec![text("q", None)], - Some(&hour), - 1, - false, - false, - ), - ( - "four taken", - Some(vec![ - text("a", Some(&five)), - text("b", Some(&five)), - text("c", Some(&five)), - ]), - Some(vec![tool("t")]), - vec![text("q", None)], - None, - 4, - false, - true, - ), - ( - "one slot left", - None, - Some(vec![tool("t1"), tool("t2"), tool("t3")]), - vec![text("q", None)], - None, - 4, - true, - false, - ), + ("system only", Some(vec![text("sys", f)]), None, q(), None, 2, true, true), + ("system 1h", Some(vec![text("sys", h)]), None, q(), None, 2, true, true), + ("system and message", Some(vec![text("sys", f)]), None, vec![text("q", f)], + None, 2, true, true), + ("message only", None, None, vec![text("q", f)], None, 2, true, true), + ("message 1h", None, None, vec![text("q", h)], None, 1, true, false), + ("tool_result", None, None, vec![tool_result, text("q", None)], None, 2, false, true), + ("top-level", None, None, q(), f, 2, false, true), + ("top-level 1h", None, None, q(), h, 1, false, false), + ("four taken", Some(vec![text("a", f), text("b", f), text("c", f)]), + Some(vec![tool("t")]), q(), None, 4, false, true), + ("one slot left", None, Some(vec![tool("t1"), tool("t2"), tool("t3")]), + q(), None, 4, true, false), ]; for (name, system, tools, last, top, total, user_marked, system_marked) in cases { let t = RecordingTransport::new(CLAUDE_OK); @@ -1755,6 +1621,15 @@ async fn prompt_cache_fills_only_the_breakpoints_the_client_left_out() { } } +fn system_texts(b: &Value) -> Vec<&str> { + b["system"] + .as_array() + .into_iter() + .flatten() + .filter_map(|block| block["text"].as_str()) + .collect() +} + fn collect_ttls(v: &Value, out: &mut Vec<&'static str>) { for block in v.as_array().into_iter().flatten() { if let Some(control) = block.get("cache_control") { @@ -1800,33 +1675,18 @@ async fn openrouter_anthropic_responses_carry_a_top_level_cache_control() { use gw_models::Account; let ephemeral = serde_json::json!({"type": "ephemeral"}); - let own = serde_json::json!({"type": "ephemeral", "ttl": "1h"}); - for (model, kind, prompt_cache, client, want) in [ - ( - "anthropic/claude-opus-5.5", - "openrouter", - true, - None, - Some(&ephemeral), - ), - ( - "~anthropic/claude-opus-latest", - "openrouter", - true, - None, - Some(&ephemeral), - ), - ( - "anthropic/claude-opus-5.5", - "openrouter", - true, - Some(&own), - Some(&own), - ), + let hour = serde_json::json!({"type": "ephemeral", "ttl": "1h"}); + let (e, own) = (Some(&ephemeral), Some(&hour)); + #[rustfmt::skip] + let cases = [ + ("anthropic/claude-opus-5.5", "openrouter", true, None, e), + ("~anthropic/claude-opus-latest", "openrouter", true, None, e), + ("anthropic/claude-opus-5.5", "openrouter", true, own, own), ("anthropic/claude-opus-5.5", "openrouter", false, None, None), ("openai/gpt-6-sol", "openrouter", true, None, None), ("anthropic/claude-opus-5.5", "openai", true, None, None), - ] { + ]; + for (model, kind, prompt_cache, client, want) in cases { let t = RecordingTransport::new(RESPONSES_OK); let mut req = GatewayRequest { preserve_responses_wire: true, @@ -1851,6 +1711,7 @@ async fn openrouter_anthropic_responses_carry_a_top_level_cache_control() { } const RESPONSES_OK: &str = r#"{"id":"r","object":"response","model":"gpt-6-astra","status":"completed","output":[],"usage":{"input_tokens":1,"output_tokens":1}}"#; +const CONVERSE_OK: &str = r#"{"output":{"message":{"role":"assistant","content":[{"text":"ok"}]}},"stopReason":"end_turn","usage":{"inputTokens":3,"outputTokens":1}}"#; const CLAUDE_OK: &str = r#"{"model":"claude-test","content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1}}"#; const OPENAI_OK: &str = r#"{"model":"gpt","choices":[{"message":{"content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}"#; @@ -2490,9 +2351,7 @@ async fn bedrock_claude_signs_the_messages_body_without_model_or_stream() { #[tokio::test] async fn converse_request_lands_on_the_converse_path_in_converse_shape() { - let t = RecordingTransport::new( - r#"{"output":{"message":{"role":"assistant","content":[{"text":"ok"}]}},"stopReason":"end_turn","usage":{"inputTokens":3,"outputTokens":1}}"#, - ); + let t = RecordingTransport::new(CONVERSE_OK); let mut req = chat_req(Protocol::AwsConverse, "us.meta.llama3-3-70b-instruct-v1:0"); if let Some(p) = req.model_param_v2.as_mut() { p.typed = Some(TypedParams::Chat(ChatParams { @@ -2650,9 +2509,7 @@ async fn a_participant_name_reaches_the_openai_wire() { #[tokio::test] async fn a_marked_chat_system_stays_plain_for_a_non_claude_converse_model() { - let t = RecordingTransport::new( - r#"{"output":{"message":{"role":"assistant","content":[{"text":"ok"}]}},"stopReason":"end_turn","usage":{"inputTokens":3,"outputTokens":1}}"#, - ); + let t = RecordingTransport::new(CONVERSE_OK); let mut req = chat_req(Protocol::AwsConverse, "us.xai.grok-4.6"); req.message[0].parts = Some(serde_json::json!([ {"type": "text", "text": "be brief", "cache_control": {"type": "ephemeral"}}