From 27a6ba4231fe69cdeed35abd7f7ad5f4f43dec75 Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Sun, 25 Oct 2026 11:29:55 +0800 Subject: [PATCH 1/4] Routing rules rewrite reasoning instead of extended thinking `set.thinking` (bool) is replaced by `set.reasoning`: `off`, or one of `low` / `medium` / `high` / `xhigh` / `max`. A level only changes requests that already have reasoning on; `off` also removes the effort. Requests without reasoning are left untouched. - tw-dialect: read, set and remove the reasoning effort in each format's raw body (`params::reasoning_on` / `reasoning_effort` / `set_reasoning_effort` / `remove_reasoning_effort`); `disable_thinking` now also drops Anthropic's `output_config.effort` (and the same inside Bedrock's `additionalModelRequestFields`); `think::nearest` and `think::effort_rejection` tell what to resend when an upstream refuses an effort. - tw-engine: `SetAction.reasoning: Option`. - tw-api: `RuleRewrite.reasoning` (`RuleReasoning`), `SetField::Reasoning`, `AttemptView.effort` (`EffortChange`, `EffortReason`); control API 54. - Config manual updated. Co-Authored-By: Claude Opus 5.5 --- crates/tw-api/src/lib.rs | 65 +- crates/tw-config/tests/manual/schema.rs | 16 +- crates/tw-control/src/dryrun.rs | 4 +- crates/tw-control/src/routes.rs | 24 +- crates/tw-control/tests/dryrun.rs | 21 + crates/tw-control/tests/handover.rs | 2 +- crates/tw-control/tests/in_flight.rs | 1 + crates/tw-control/tests/key_limits.rs | 2 + crates/tw-control/tests/live_state.rs | 1 + crates/tw-control/tests/replay.rs | 1 + crates/tw-control/tests/route_stats.rs | 2 +- crates/tw-control/tests/routes.rs | 44 + crates/tw-control/tests/upstream_limits.rs | 1 + crates/tw-dialect/src/params.rs | 780 +++++++++++++++++- crates/tw-dialect/src/think.rs | 159 ++++ crates/tw-engine/src/engine.rs | 159 +++- crates/tw-engine/src/lib.rs | 4 +- crates/tw-gateway/src/forward.rs | 45 +- crates/tw-gateway/src/server.rs | 5 + crates/tw-gateway/src/server/pipeline/hop.rs | 1 + crates/tw-gateway/src/server/pipeline/idle.rs | 2 + crates/tw-gateway/src/translate.rs | 12 +- crates/tw-gateway/tests/alias_ws.rs | 4 +- crates/tw-gateway/tests/passthrough.rs | 2 +- crates/tw-gateway/tests/routing_facts.rs | 2 +- crates/tw-observe/src/bus.rs | 2 + crates/tw-store/src/db.rs | 2 + crates/tw-store/src/recorder.rs | 14 + docs/config.md | 2 +- docs/config.zh-CN.md | 2 +- 30 files changed, 1319 insertions(+), 62 deletions(-) diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs index 5dea51e0..4e5cde26 100644 --- a/crates/tw-api/src/lib.rs +++ b/crates/tw-api/src/lib.rs @@ -502,7 +502,8 @@ slug_enum! { /// 换模型，整个 prompt cache 作废 Model = "model", MaxTokens = "max_tokens", - Thinking = "thinking", + /// 推理：值是 [`RuleReasoning`] 的那个词 + Reasoning = "reasoning", /// 以上改写只在新会话开始时应用 OnlyAtSessionStart = "only_at_session_start", } @@ -1074,7 +1075,16 @@ pub const MSG_CODES: &str = include_str!("../msg-codes.txt"); /// [`AttemptView`] 多了 `retry`（这一跳是第几次重试）和 `backoff_ms`（发出之前退避了多久）。 /// 新码：`gw.upstream.stream_ended_early`（流只发了开头的例行事件就收了尾，算在 /// [`FAILED_AFTER_SENDING`] 里） -pub const CONTROL_API_VERSION: u32 = 53; +/// +/// **54 起路由规则改的是推理，不再是扩展思考**：[`RuleRewrite`] 的 `thinking`（布尔）换成 +/// `reasoning`（[`RuleReasoning`]：`off` 关掉推理，或者改成 `low` / `medium` / `high` / `xhigh` / +/// `max` 一档，只改开着推理的请求），[`SetField`] 的 `thinking` 换成 `reasoning`（[`SetView`] 的 +/// 值是那个词）。配置里的 `set.thinking` 不认了。[`AttemptView`] 多了 `effort`（[`EffortChange`]： +/// 这一跳发出去的推理强度和客户端要的不一样，`reason` 是 [`EffortReason`] 的 `rule` / `rejected` / +/// `remembered`）：上游因为推理强度回 400 时，同一家改一档（或者去掉强度）再发一次，是尝试链上 +/// 新的一跳，不算退避重试；记下这一家的这个模型不认那一档，7 天之内直接改。照 53 写的界面 +/// 保存规则时带的 `thinking` 不起作用 +pub const CONTROL_API_VERSION: u32 = 54; /// `/live` 的速度看最近多久（见 [`LiveView::tokens_per_sec`]），毫秒 pub const LIVE_RATE_WINDOW_MS: u64 = 3_000; @@ -2008,6 +2018,11 @@ pub struct AttemptView { /// 这一跳发出去之前退避了多少毫秒：一轮重试的头一跳带着。不算在 `ms` 里。别的没有 #[serde(default, skip_serializing_if = "Option::is_none")] pub backoff_ms: Option, + /// 这一跳发出去的推理强度和客户端要的不一样：规则改写的、上游拒收了改用别的一档重发的、 + /// 记得这个模型不认那一档直接改了的（见 [`EffortChange`]）。一样的没有。被拒收的那一跳 + /// 自己是一跳普通的 `status`（400），重发的那一跳带着 `rejected` + #[serde(default, skip_serializing_if = "Option::is_none")] + pub effort: Option, /// 这一跳走的代理：这家上游配置的代理名（跟随系统的是 `system`），直连是 None。没发出去 /// 的一跳（满着、被拒、转换不了……）是它要走的那个 —— 连不上的那一跳，问题可能就出在 /// 这个代理上。WebSocket 只连直连的上游，一律是 None @@ -2015,6 +2030,29 @@ pub struct AttemptView { pub proxy: Option, } +/// 一跳发出去的推理强度和客户端要的不一样（[`AttemptView::effort`]）。 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct EffortChange { + /// 客户端要的那一档：`minimal` / `low` / `medium` / `high` / `xhigh` / `max` + pub asked: String, + /// 发出去的那一档，同一个词表。null 是去掉了推理强度，按模型的默认 + pub sent: Option, + pub reason: EffortReason, +} + +slug_enum! { + /// 推理强度为什么改了（[`EffortChange::reason`]）。 + pub enum EffortReason { + /// 路由规则改写（`set.reasoning`） + Rule = "rule", + /// 上游拒收了上一跳的那一档（400），同一家改用这一档重发了一次 + Rejected = "rejected", + /// 这一家的这个模型拒收过那一档，记着（7 天），直接改了 + Remembered = "remembered", + } +} + /// 发出去之后才失败的一跳报的码（见 [`AttemptView::sent`]）：上游没在时限内回话、请求在路上 /// 断了、回答读到一半连接断了、流在第一段内容之前报了错。**上游可能已经收下了它、在算了**。 /// @@ -2807,9 +2845,22 @@ pub struct RuleRewrite { pub model: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub max_tokens: Option, - /// 打开或关闭扩展思考 + /// 推理：关掉，或者改成一档。**只改开着推理的请求**，没开推理的（生成标题这类）原样不动 #[serde(default, skip_serializing_if = "Option::is_none")] - pub thinking: Option, + pub reasoning: Option, +} + +slug_enum! { + /// 规则改写的推理（[`RuleRewrite::reasoning`]）。没有 `minimal` + pub enum RuleReasoning { + /// 关掉推理，连同推理强度 + Off = "off", + Low = "low", + Medium = "medium", + High = "high", + XHigh = "xhigh", + Max = "max", + } } /// 规则里的一个条件。 @@ -5949,8 +6000,9 @@ pub struct MismatchView { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[cfg_attr(feature = "ts", derive(ts_rs::TS))] pub struct SetView { - /// `model`（换模型，整个 prompt cache 作废）/ `max_tokens` / `thinking` / - /// `only_at_session_start`（以上改写只在新会话开始时应用，值是 `true`） + /// `model`（换模型，整个 prompt cache 作废）/ `max_tokens` / `reasoning`（值是 + /// [`RuleReasoning`] 的那个词）/ `only_at_session_start`（以上改写只在新会话开始时应用， + /// 值是 `true`） pub field: SetField, pub value: String, } @@ -6929,6 +6981,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } diff --git a/crates/tw-config/tests/manual/schema.rs b/crates/tw-config/tests/manual/schema.rs index 7e51af7f..d433d368 100644 --- a/crates/tw-config/tests/manual/schema.rs +++ b/crates/tw-config/tests/manual/schema.rs @@ -11,7 +11,9 @@ use super::{Def, Kind, Lang, Row, Section, T2, Ty}; use tw_config::proxy::ProxyAuth; use tw_config::*; use tw_engine::rule::When; -use tw_engine::{BalanceBy, Group, GroupType, Member, Pinned, RouteSet, Rule, SetAction}; +use tw_engine::{ + BalanceBy, Group, GroupType, Member, Pinned, RouteSet, Rule, SetAction, SetReasoning, +}; use tw_pricing::{PerMillion, PricingConfig, SheetDef}; const fn t(en: &'static str, zh: &'static str) -> T2 { @@ -86,6 +88,9 @@ fn balance_bys() -> Vec<&'static str> { fn limit_pers() -> Vec<&'static str> { super::fields::() } +fn set_reasonings() -> Vec<&'static str> { + super::fields::() +} /// 同步位置三种写法各自的字段。`target` 是按 `type` 分的枚举，问 serde 要字段的办法问不出 /// 枚举每一种的字段，所以借它的报错：写一个不存在的字段，serde 说出这一种认得的全部字段 @@ -1897,10 +1902,13 @@ pub fn sections() -> Vec
{ t("Replace `max_tokens`.", "替换 `max_tokens`。"), ), row( - "thinking", - Kind::Bool, + "reasoning", + Kind::Enum(set_reasonings), Def::Unset, - t("Turn extended thinking on or off.", "开启或关闭扩展思考。"), + t( + "Reasoning. `off` turns it off and removes the reasoning effort; a level changes the effort. Only requests that already have reasoning on are changed; requests without reasoning (such as title generation) are left as they are.", + "推理。`off` 关闭推理，连同推理强度一起去掉；写一档则改为该档强度。只改写已开启推理的请求，未开启推理的请求（如生成标题）保持不变。", + ), ), row( "only_at_session_start", diff --git a/crates/tw-control/src/dryrun.rs b/crates/tw-control/src/dryrun.rs index 705c78b3..74550076 100644 --- a/crates/tw-control/src/dryrun.rs +++ b/crates/tw-control/src/dryrun.rs @@ -405,8 +405,8 @@ fn describe(set: &tw_engine::SetAction) -> Vec { if let Some(t) = set.max_tokens { push(SetField::MaxTokens, t.to_string()); } - if let Some(t) = set.thinking { - push(SetField::Thinking, t.to_string()); + if let Some(r) = set.reasoning { + push(SetField::Reasoning, r.slug().to_string()); } if set.only_at_session_start { push(SetField::OnlyAtSessionStart, "true".to_string()); diff --git a/crates/tw-control/src/routes.rs b/crates/tw-control/src/routes.rs index bef80b13..dd2bdd00 100644 --- a/crates/tw-control/src/routes.rs +++ b/crates/tw-control/src/routes.rs @@ -282,7 +282,7 @@ pub(crate) fn to_rule(input: &tw_api::RuleInput, cfg: &tw_config::Config) -> Res .filter(|m| !m.is_empty()) .map(str::to_string), max_tokens: s.max_tokens, - thinking: s.thinking, + reasoning: s.reasoning.map(set_reasoning), only_at_session_start: false, }; (!a.is_empty()).then_some(a) @@ -811,7 +811,7 @@ pub(crate) fn rule_view(r: &Rule, n: tw_engine::RuleNotes) -> tw_api::RuleView { .map(|s| tw_api::RuleRewrite { model: s.model.clone(), max_tokens: s.max_tokens, - thinking: s.thinking, + reasoning: s.reasoning.map(rule_reasoning), }), catch_all: n.catch_all, phase_two: n.phase_two, @@ -819,6 +819,26 @@ pub(crate) fn rule_view(r: &Rule, n: tw_engine::RuleNotes) -> tw_api::RuleView { } } +/// 规则改写的推理：界面的词 → 配置里的 +fn set_reasoning(r: tw_api::RuleReasoning) -> tw_engine::SetReasoning { + use tw_api::RuleReasoning as R; + use tw_dialect::ir::Effort; + use tw_engine::SetReasoning as S; + match r { + R::Off => S::Off, + R::Low => S::Level(Effort::Low), + R::Medium => S::Level(Effort::Medium), + R::High => S::Level(Effort::High), + R::XHigh => S::Level(Effort::XHigh), + R::Max => S::Level(Effort::Max), + } +} + +/// 反过来。两边的词是同一张表（[`tw_engine::SetReasoning::slug`]） +fn rule_reasoning(r: tw_engine::SetReasoning) -> tw_api::RuleReasoning { + tw_api::RuleReasoning::from_slug(r.slug()).unwrap_or(tw_api::RuleReasoning::Low) +} + // ─────────────────────────────────────────────────────────── 共用 fn reserved(name: &str, what: &str) -> Result<(), Msg> { diff --git a/crates/tw-control/tests/dryrun.rs b/crates/tw-control/tests/dryrun.rs index 6bb56c2d..f20f4619 100644 --- a/crates/tw-control/tests/dryrun.rs +++ b/crates/tw-control/tests/dryrun.rs @@ -332,6 +332,27 @@ async fn a_set_only_rule_still_shows_up_as_matched_and_says_what_it_changes() { ); } +/// 推理改写在试算里是一项 `reasoning`，值是配置里的那个词 +#[tokio::test] +async fn a_reasoning_rewrite_shows_as_its_word() { + let (_d, app) = app_with(&CFG.replace( + " set: { model: claude-haiku-4-5 }", + " set: { model: claude-haiku-4-5, reasoning: xhigh }", + )); + let r = run( + &app, + r#"{"model":"claude-sonnet-4-5","input_tokens":300000}"#, + ) + .await; + assert!( + r.set + .iter() + .any(|s| s.field == tw_api::SetField::Reasoning && s.value == "xhigh"), + "{:?}", + r.set + ); +} + #[tokio::test] async fn the_dry_run_changes_nothing_and_sends_nothing() { // 它只算。跑一百遍和跑零遍，对上游和配置文件来说没有区别。 diff --git a/crates/tw-control/tests/handover.rs b/crates/tw-control/tests/handover.rs index e6f2ceab..f7fd20b5 100644 --- a/crates/tw-control/tests/handover.rs +++ b/crates/tw-control/tests/handover.rs @@ -146,7 +146,7 @@ async fn status_reports_the_handover_phase_and_the_draining_count() { let st = status(&s).await; assert_eq!(st.handover, tw_api::HandoverState::Draining); assert_eq!(st.draining, 3); - assert_eq!(st.api_version, 53); + assert_eq!(st.api_version, 54); } /// arm 之后（以及接替者正在接管时）不开始新的账号登录：回调端口不交接。（平常、排空中的 diff --git a/crates/tw-control/tests/in_flight.rs b/crates/tw-control/tests/in_flight.rs index 46de16b3..3256f073 100644 --- a/crates/tw-control/tests/in_flight.rs +++ b/crates/tw-control/tests/in_flight.rs @@ -104,6 +104,7 @@ async fn it_gives_every_running_request_with_what_has_happened_to_it_so_far() { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing: tw_api::Billing::PerToken, diff --git a/crates/tw-control/tests/key_limits.rs b/crates/tw-control/tests/key_limits.rs index 7ff289b7..056c5cf3 100644 --- a/crates/tw-control/tests/key_limits.rs +++ b/crates/tw-control/tests/key_limits.rs @@ -147,6 +147,7 @@ fn request(id: u64, key: &str, at: i64, input: u64) -> Vec { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing: tw_api::Billing::PerToken, @@ -460,6 +461,7 @@ fn hop(outcome: tw_api::AttemptOutcome, code: &str, busy: bool) -> tw_api::Attem pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } diff --git a/crates/tw-control/tests/live_state.rs b/crates/tw-control/tests/live_state.rs index 98e64db2..af4b2d15 100644 --- a/crates/tw-control/tests/live_state.rs +++ b/crates/tw-control/tests/live_state.rs @@ -161,6 +161,7 @@ async fn a_request_still_running_can_be_opened_and_becomes_whole_when_it_ends() pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing: tw_api::Billing::PerToken, diff --git a/crates/tw-control/tests/replay.rs b/crates/tw-control/tests/replay.rs index e439bae4..3e01a109 100644 --- a/crates/tw-control/tests/replay.rs +++ b/crates/tw-control/tests/replay.rs @@ -394,6 +394,7 @@ fn routed(hops: &[(&str, Option<&str>)]) -> Option { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }) .collect(); diff --git a/crates/tw-control/tests/route_stats.rs b/crates/tw-control/tests/route_stats.rs index 2671bf20..6675a834 100644 --- a/crates/tw-control/tests/route_stats.rs +++ b/crates/tw-control/tests/route_stats.rs @@ -52,7 +52,7 @@ routes: deny: 这个项目不用 opus - name: 关掉思考 when: {{ model: claude-sonnet-* }} - set: {{ thinking: false }} + set: {{ reasoning: off }} - name: 从没命中过 when: {{ model: gpt-* }} to: up diff --git a/crates/tw-control/tests/routes.rs b/crates/tw-control/tests/routes.rs index 21e40fb5..f1864a38 100644 --- a/crates/tw-control/tests/routes.rs +++ b/crates/tw-control/tests/routes.rs @@ -400,6 +400,50 @@ async fn a_rule_that_only_rewrites_can_be_saved() { assert!(codex["rules"][0].get("guard").is_none(), "{codex}"); } +/// 推理改写是一个词：存进配置写成 `reasoning: xhigh`，读回来还是那个词；不认识的词 +/// （`minimal` 也是）读不成请求体 +#[tokio::test] +async fn a_reasoning_rewrite_is_saved_as_its_word() { + let b = bed(BASE); + for word in ["off", "xhigh"] { + let (st, v) = call( + &b.app, + "PUT", + "/routes/codex", + json!({ "route": { "name": "codex", "rules": [ + { "name": "推理", "conditions": [], "set": { "reasoning": word } }, + catch_all("主力"), + ]}}), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + assert!( + b.file().contains(&format!("reasoning: {word}")), + "{}", + b.file() + ); + let codex = find(&b.overview().await["routes"], "codex").clone(); + assert_eq!( + codex["rules"][0]["set"], + json!({ "reasoning": word }), + "{codex}" + ); + } + let before = b.file(); + let (st, v) = call( + &b.app, + "PUT", + "/routes/codex", + json!({ "route": { "name": "codex", "rules": [ + { "name": "推理", "conditions": [], "set": { "reasoning": "minimal" } }, + catch_all("主力"), + ]}}), + ) + .await; + assert_eq!(st, StatusCode::UNPROCESSABLE_ENTITY, "{v}"); + assert_eq!(b.file(), before); +} + #[tokio::test] async fn deleting_a_route_sends_its_keys_where_the_user_chose() { let b = bed(BASE); diff --git a/crates/tw-control/tests/upstream_limits.rs b/crates/tw-control/tests/upstream_limits.rs index 6aa965f7..d3f7895a 100644 --- a/crates/tw-control/tests/upstream_limits.rs +++ b/crates/tw-control/tests/upstream_limits.rs @@ -110,6 +110,7 @@ fn attempt(provider: &str, status: u16) -> tw_api::AttemptView { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } diff --git a/crates/tw-dialect/src/params.rs b/crates/tw-dialect/src/params.rs index 78d24b45..50dee2f9 100644 --- a/crates/tw-dialect/src/params.rs +++ b/crates/tw-dialect/src/params.rs @@ -13,7 +13,8 @@ use serde_json::{Map, Value}; -use crate::ir::Dialect; +use crate::ir::{Dialect, Effort}; +use crate::think; /// 客户端写的最大输出 token 数。没写（或者写的不是一个非负整数）是 `None`。 pub fn max_output_tokens(dialect: Dialect, body: &Value) -> Option { @@ -188,9 +189,12 @@ pub fn set_model(dialect: Dialect, body: &mut Value, model: &str) { } } -/// 关掉推理：去掉开推理的那个字段。 +/// 关掉推理：去掉开推理的那个字段，连同推理强度。 /// /// **只做「关掉」这一个方向**：开启要一个预算，而这一层没有一个说得过去的值可以编。 +/// Anthropic 的强度（`output_config.effort`）单独写在思考开关外面，只去掉开关的话自适应的 +/// 模型照样按那一档想，要一起去掉；`output_config` 去空了的整个去掉。Converse 的透传口袋里 +/// 同理。 pub fn disable_thinking(dialect: Dialect, body: &mut Value) { let Some(obj) = body.as_object_mut() else { return; @@ -198,6 +202,7 @@ pub fn disable_thinking(dialect: Dialect, body: &mut Value) { match dialect { Dialect::Anthropic => { obj.remove("thinking"); + remove_output_effort(obj); } Dialect::Chat => { obj.remove("reasoning_effort"); @@ -220,11 +225,369 @@ pub fn disable_thinking(dialect: Dialect, body: &mut Value) { .and_then(Value::as_object_mut) { f.remove("thinking"); + remove_output_effort(f); } } } } +/// 去掉 `output_config.effort`；`output_config` 去空了的整个去掉。返回去没去掉 +fn remove_output_effort(holder: &mut Map) -> bool { + let Some(o) = holder + .get_mut("output_config") + .and_then(Value::as_object_mut) + else { + return false; + }; + let removed = o.remove("effort").is_some(); + if o.is_empty() { + holder.remove("output_config"); + } + removed +} + +// ───────────────────────────────────────────────────────── 推理强度 +// +// 「推理开着」和解码器的判据一样：Anthropic 的 `thinking.type` 是 `enabled` / `adaptive`，或者 +// 只写了 `output_config.effort`；Chat 的 `reasoning_effort` 不是 `none`，或者 GLM 那种 +// `thinking.type: enabled`；Responses 写了 `reasoning`、强度不是 `none`（对话中途的 +// `configuration_update` 最后一个说了算）；Gemini 写了 `thinkingConfig`、预算不是 0；Converse +// 的透传口袋里和 Anthropic 一样。明确关掉的（`disabled`、`none`、预算 0）不算开着。 +// +// 强度写在哪儿：Anthropic 是 `output_config.effort`，按预算思考的（`enabled` + +// `budget_tokens`、没写强度）是预算；Chat 是 `reasoning_effort`；Responses 是 +// `reasoning.effort`；Gemini 写了等级的是 `thinkingLevel`、写了预算的是 `thinkingBudget`； +// Converse 和 Anthropic 一样，在 `additionalModelRequestFields` 里。预算按 +// [`think::effort_of_budget`] 读成一档、按 [`think::budget_of_effort`] 写回去。 + +/// 客户端开着推理吗（判据见上） +pub fn reasoning_on(dialect: Dialect, body: &Value) -> bool { + let Some(obj) = body.as_object() else { + return false; + }; + match dialect { + Dialect::Anthropic => claude_on(obj), + // 和解码器一样：`thinking` 说关掉的以它为准，`reasoning_effort` 认得出的看它，都没有再看 + // `thinking` 是不是开着 + Dialect::Chat => { + let switch = obj + .get("thinking") + .and_then(|t| t.get("type")) + .and_then(Value::as_str); + match str_at(obj, "reasoning_effort").and_then(think::parse_openai) { + _ if switch == Some("disabled") => false, + Some(e) => e.is_some(), + None => switch == Some("enabled"), + } + } + Dialect::Responses => match responses_update(obj).and_then(think::parse_openai) { + Some(e) => e.is_some(), + None => obj + .get("reasoning") + .filter(|r| r.is_object()) + .is_some_and(|r| r.get("effort").and_then(Value::as_str) != Some("none")), + }, + Dialect::Gemini => gemini_thinking(obj).is_some_and(|t| { + either_ref(t, "thinkingBudget", "thinking_budget").and_then(Value::as_i64) != Some(0) + }), + Dialect::Bedrock => obj + .get("additionalModelRequestFields") + .and_then(Value::as_object) + .is_some_and(claude_on), + } +} + +/// 客户端要的推理强度。推理没开、开着却没说强度（自适应的、Gemini 让模型自己定的）是 `None` +pub fn reasoning_effort(dialect: Dialect, body: &Value) -> Option { + if !reasoning_on(dialect, body) { + return None; + } + let obj = body.as_object()?; + match dialect { + Dialect::Anthropic => claude_effort(obj), + Dialect::Chat => str_at(obj, "reasoning_effort") + .and_then(think::parse_openai) + .flatten(), + Dialect::Responses => responses_update(obj) + .or_else(|| obj.get("reasoning").and_then(|r| r.get("effort"))?.as_str()) + .and_then(think::parse_openai) + .flatten(), + Dialect::Gemini => { + let t = gemini_thinking(obj)?; + match either_ref(t, "thinkingLevel", "thinking_level").and_then(Value::as_str) { + Some(l) => think::parse_gemini_level(l), + None => either_ref(t, "thinkingBudget", "thinking_budget") + .and_then(Value::as_i64) + .filter(|b| *b > 0) + .map(|b| think::effort_of_budget(b as u64)), + } + } + Dialect::Bedrock => obj + .get("additionalModelRequestFields") + .and_then(Value::as_object) + .and_then(claude_effort), + } +} + +/// 把推理强度改成 `e`。**只改开着推理的请求**：没开的（生成标题这类辅助请求、明确关掉的） +/// 原样不动，不替客户端打开推理。返回改没改。 +/// +/// 客户端写了强度的改强度；按预算思考的改预算（Anthropic 和 Converse 的预算还要小于最大输出， +/// 压不到 1024 以上的不改）；Gemini 等级、预算都没写的（让模型自己定）不改 —— 不知道这个模型 +/// 认哪一个。Anthropic 自适应、没写强度的补上 `output_config.effort`。Responses 对话中途的 +/// `configuration_update` 一起改：最后一个盖过顶层的。 +pub fn set_reasoning_effort(dialect: Dialect, body: &mut Value, e: Effort) -> bool { + if !reasoning_on(dialect, body) { + return false; + } + let Some(obj) = body.as_object_mut() else { + return false; + }; + match dialect { + Dialect::Anthropic => { + let max = obj.get("max_tokens").and_then(Value::as_u64); + set_claude_effort(obj, e, max) + } + Dialect::Chat => put(obj, "reasoning_effort", think::openai(e)), + Dialect::Responses => { + let mut changed = match obj.get_mut("reasoning") { + Some(Value::Object(r)) => put(r, "effort", think::openai(e)), + Some(_) => false, + None => { + obj.insert("reasoning".into(), serde_json::json!({})); + match obj.get_mut("reasoning") { + Some(Value::Object(r)) => put(r, "effort", think::openai(e)), + _ => false, + } + } + }; + for r in responses_updates(obj) { + if r.contains_key("effort") { + changed |= put(r, "effort", think::openai(e)); + } + } + changed + } + Dialect::Gemini => { + let Some(t) = gemini_thinking_mut(obj) else { + return false; + }; + if let Some(k) = key_of(t, "thinkingLevel", "thinking_level") { + put(t, k, think::gemini_level(e)) + } else if let Some(k) = key_of(t, "thinkingBudget", "thinking_budget") { + let budget = think::budget_of_effort(e); + let before = t.get(k).cloned(); + t.insert(k.into(), Value::from(budget)); + before.as_ref() != t.get(k) + } else { + false + } + } + Dialect::Bedrock => { + let max = obj + .get("inferenceConfig") + .and_then(|c| c.get("maxTokens")) + .and_then(Value::as_u64); + match obj + .get_mut("additionalModelRequestFields") + .and_then(Value::as_object_mut) + { + Some(f) => set_claude_effort(f, e, max), + None => false, + } + } + } +} + +/// 去掉写明的推理强度，按模型的默认想。推理照样开着。返回去没去掉。 +/// +/// 按预算思考的 Anthropic 和 Converse 不动：预算是开思考必须的，去掉它整个请求不合法。 +/// Responses 对话中途的 `configuration_update` 里的也去掉，只剩个空壳的整条去掉。 +pub fn remove_reasoning_effort(dialect: Dialect, body: &mut Value) -> bool { + if !reasoning_on(dialect, body) { + return false; + } + let Some(obj) = body.as_object_mut() else { + return false; + }; + match dialect { + Dialect::Anthropic => remove_output_effort(obj), + Dialect::Chat => obj.remove("reasoning_effort").is_some(), + Dialect::Responses => { + let mut removed = obj + .get_mut("reasoning") + .and_then(Value::as_object_mut) + .is_some_and(|r| r.remove("effort").is_some()); + if let Some(Value::Array(items)) = obj.get_mut("input") { + let before = items.len(); + for item in items.iter_mut() { + if let Some(r) = update_reasoning(item) { + removed |= r.remove("effort").is_some(); + } + } + // 只剩 `type`（和 `id`）的一条没有意思了 + items.retain(|item| { + !(is_update(item) + && item + .get("reasoning") + .is_some_and(|r| r.as_object().is_some_and(Map::is_empty)) + && item.as_object().is_some_and(|o| { + o.keys() + .all(|k| matches!(k.as_str(), "type" | "id" | "reasoning")) + })) + }); + removed |= items.len() != before; + } + removed + } + Dialect::Gemini => { + let Some(t) = gemini_thinking_mut(obj) else { + return false; + }; + let mut removed = false; + for k in ["thinkingLevel", "thinking_level"] { + removed |= t.remove(k).is_some(); + } + for k in ["thinkingBudget", "thinking_budget"] { + if t.get(k).and_then(Value::as_i64).is_some_and(|b| b > 0) { + removed |= t.remove(k).is_some(); + } + } + removed + } + Dialect::Bedrock => obj + .get_mut("additionalModelRequestFields") + .and_then(Value::as_object_mut) + .is_some_and(remove_output_effort), + } +} + +/// Anthropic（和 Converse 背后的 Claude）开着思考吗。`holder` 是装着 `thinking` 的那个对象 +fn claude_on(holder: &Map) -> bool { + match holder + .get("thinking") + .and_then(|t| t.get("type")) + .and_then(Value::as_str) + { + Some("enabled" | "adaptive") => true, + Some(_) => false, + None => output_effort(holder).is_some(), + } +} + +fn output_effort(holder: &Map) -> Option { + holder + .get("output_config") + .and_then(|o| o.get("effort")) + .and_then(Value::as_str) + .and_then(think::parse_anthropic) +} + +/// 按预算思考的预算。`thinking.type` 是 `enabled` 才有 +fn claude_budget(holder: &Map) -> Option { + let t = holder.get("thinking")?; + (t.get("type").and_then(Value::as_str) == Some("enabled")) + .then(|| t.get("budget_tokens").and_then(Value::as_u64)) + .flatten() +} + +fn claude_effort(holder: &Map) -> Option { + output_effort(holder).or_else(|| claude_budget(holder).map(think::effort_of_budget)) +} + +/// 见 [`set_reasoning_effort`]。`max` 是最大输出：预算要小于它 +fn set_claude_effort(holder: &mut Map, e: Effort, max: Option) -> bool { + /// Anthropic 允许的最小思考预算 + const BUDGET_MIN: u64 = 1024; + let effort_written = holder + .get("output_config") + .is_some_and(|o| o.get("effort").is_some()); + if !effort_written && let Some(before) = claude_budget(holder) { + let budget = max.map_or(think::budget_of_effort(e), |m| { + think::budget_of_effort(e).min(m.saturating_sub(1)) + }); + if budget < BUDGET_MIN || budget == before { + return false; + } + if let Some(t) = holder.get_mut("thinking").and_then(Value::as_object_mut) { + t.insert("budget_tokens".into(), Value::from(budget)); + return true; + } + return false; + } + match section(holder, "output_config") { + Some(o) => put(o, "effort", think::anthropic(e)), + None => false, + } +} + +/// 把 `obj[key]` 写成字符串 `value`。返回值变没变 +fn put(obj: &mut Map, key: &str, value: &str) -> bool { + if obj.get(key).and_then(Value::as_str) == Some(value) { + return false; + } + obj.insert(key.into(), Value::String(value.to_string())); + true +} + +fn str_at<'a>(obj: &'a Map, key: &str) -> Option<&'a str> { + obj.get(key).and_then(Value::as_str) +} + +fn is_update(item: &Value) -> bool { + item.get("type").and_then(Value::as_str) == Some("configuration_update") +} + +/// 一条 `configuration_update` 里的 `reasoning` +fn update_reasoning(item: &mut Value) -> Option<&mut Map> { + if !is_update(item) { + return None; + } + item.get_mut("reasoning").and_then(Value::as_object_mut) +} + +/// Responses 对话中途最后一次改的推理强度（`input` 里最后一条带着强度的 +/// `configuration_update`），和解码器一样它盖过顶层的 +fn responses_update(obj: &Map) -> Option<&str> { + obj.get("input")? + .as_array()? + .iter() + .filter(|i| is_update(i)) + .filter_map(|i| i.get("reasoning")?.get("effort")?.as_str()) + .next_back() +} + +/// 每一条 `configuration_update` 里的 `reasoning` +fn responses_updates(obj: &mut Map) -> Vec<&mut Map> { + match obj.get_mut("input") { + Some(Value::Array(items)) => items.iter_mut().filter_map(update_reasoning).collect(), + _ => Vec::new(), + } +} + +/// Gemini 的思考配置：生成参数里的 `thinkingConfig`，两层都认下划线的写法（先认驼峰） +fn gemini_thinking(obj: &Map) -> Option<&Map> { + let g = either_ref(obj, "generationConfig", "generation_config")?; + either(g, "thinkingConfig", "thinking_config")?.as_object() +} + +fn gemini_thinking_mut(obj: &mut Map) -> Option<&mut Map> { + let g = key_of(obj, "generationConfig", "generation_config")?; + let g = obj.get_mut(g)?.as_object_mut()?; + let t = key_of(g, "thinkingConfig", "thinking_config")?; + g.get_mut(t)?.as_object_mut() +} + +/// [`either`] 的对象版 +fn either_ref<'a>(obj: &'a Map, camel: &str, snake: &str) -> Option<&'a Value> { + obj.get(camel).or_else(|| obj.get(snake)) +} + +/// 两个写法里客户端用的那一个键，先认驼峰 +fn key_of<'k>(obj: &Map, camel: &'k str, snake: &'k str) -> Option<&'k str> { + [camel, snake].into_iter().find(|k| obj.contains_key(*k)) +} + /// Gemini 的一个字段：驼峰的在就是它，否则是下划线写法的 fn either<'a>(v: &'a Value, camel: &str, snake: &str) -> Option<&'a Value> { v.get(camel).or_else(|| v.get(snake)) @@ -705,4 +1068,417 @@ mod tests { assert_eq!(v["x"], 1); } } + + #[test] + fn turning_thinking_off_takes_the_effort_with_it() { + // 自适应的 Claude 只去掉开关的话，照样按 `output_config.effort` 那一档想 + let mut v = json!({"thinking": {"type": "adaptive"}, "output_config": {"effort": "xhigh"}}); + disable_thinking(Dialect::Anthropic, &mut v); + assert_eq!(v, json!({})); + let mut v = json!({"thinking": {"type": "adaptive"}, + "output_config": {"effort": "high", "format": {"type": "json_schema"}}}); + disable_thinking(Dialect::Anthropic, &mut v); + assert_eq!( + v, + json!({"output_config": {"format": {"type": "json_schema"}}}) + ); + let mut v = json!({"additionalModelRequestFields": { + "thinking": {"type": "adaptive"}, "output_config": {"effort": "max"}, "top_k": 3}}); + disable_thinking(Dialect::Bedrock, &mut v); + assert_eq!(v, json!({"additionalModelRequestFields": {"top_k": 3}})); + } + + #[test] + fn reasoning_is_on_where_the_decoder_says_it_is() { + let on = [ + ( + Dialect::Anthropic, + json!({"thinking": {"type": "enabled", "budget_tokens": 2048}}), + ), + ( + Dialect::Anthropic, + json!({"thinking": {"type": "adaptive"}}), + ), + ( + Dialect::Anthropic, + json!({"output_config": {"effort": "low"}}), + ), + (Dialect::Chat, json!({"reasoning_effort": "high"})), + (Dialect::Chat, json!({"thinking": {"type": "enabled"}})), + ( + Dialect::Responses, + json!({"reasoning": {"summary": "auto"}}), + ), + ( + Dialect::Responses, + json!({"reasoning": {"effort": "none"}, + "input": [{"type": "configuration_update", "reasoning": {"effort": "high"}}]}), + ), + ( + Dialect::Gemini, + json!({"generationConfig": {"thinkingConfig": {"thinkingBudget": -1}}}), + ), + ( + Dialect::Gemini, + json!({"generation_config": {"thinking_config": {"thinking_level": "low"}}}), + ), + ( + Dialect::Bedrock, + json!({"additionalModelRequestFields": {"thinking": {"type": "adaptive"}}}), + ), + ]; + for (d, v) in on { + assert!(reasoning_on(d, &v), "{d:?}: {v}"); + } + let off = [ + (Dialect::Anthropic, json!({})), + ( + Dialect::Anthropic, + json!({"thinking": {"type": "disabled"}, "output_config": {"effort": "low"}}), + ), + (Dialect::Chat, json!({"reasoning_effort": "none"})), + ( + Dialect::Chat, + json!({"thinking": {"type": "disabled"}, "reasoning_effort": "high"}), + ), + ( + Dialect::Chat, + json!({"thinking": {"type": "enabled"}, "reasoning_effort": "none"}), + ), + (Dialect::Responses, json!({"reasoning": {"effort": "none"}})), + ( + Dialect::Responses, + json!({"reasoning": {"effort": "high"}, + "input": [{"type": "configuration_update", "reasoning": {"effort": "none"}}]}), + ), + ( + Dialect::Gemini, + json!({"generationConfig": {"thinkingConfig": {"thinkingBudget": 0}}}), + ), + ( + Dialect::Gemini, + json!({"generationConfig": {"temperature": 1}}), + ), + ( + Dialect::Bedrock, + json!({"additionalModelRequestFields": {"top_k": 3}}), + ), + (Dialect::Chat, json!([1])), + ]; + for (d, v) in off { + assert!(!reasoning_on(d, &v), "{d:?}: {v}"); + assert_eq!(reasoning_effort(d, &v), None, "{d:?}: {v}"); + } + } + + #[test] + fn the_effort_is_read_from_where_each_format_writes_it() { + let cases = [ + ( + Dialect::Anthropic, + json!({"thinking": {"type": "adaptive"}, "output_config": {"effort": "xhigh"}}), + Some(Effort::XHigh), + ), + ( + Dialect::Anthropic, + json!({"thinking": {"type": "adaptive"}}), + None, + ), + ( + Dialect::Anthropic, + json!({"thinking": {"type": "enabled", "budget_tokens": 10000}}), + Some(Effort::Medium), + ), + ( + Dialect::Chat, + json!({"reasoning_effort": "minimal"}), + Some(Effort::Minimal), + ), + ( + Dialect::Chat, + json!({"thinking": {"type": "enabled"}}), + None, + ), + ( + Dialect::Responses, + json!({"reasoning": {"effort": "medium"}, + "input": [{"type": "configuration_update", "reasoning": {"effort": "high"}}]}), + Some(Effort::High), + ), + ( + Dialect::Responses, + json!({"reasoning": {"effort": "max"}}), + Some(Effort::Max), + ), + ( + Dialect::Gemini, + json!({"generationConfig": {"thinkingConfig": {"thinkingLevel": "HIGH"}}}), + Some(Effort::High), + ), + ( + Dialect::Gemini, + json!({"generationConfig": {"thinkingConfig": {"thinkingBudget": 4000}}}), + Some(Effort::Low), + ), + ( + Dialect::Gemini, + json!({"generationConfig": {"thinkingConfig": {"thinkingBudget": -1}}}), + None, + ), + ( + Dialect::Bedrock, + json!({"additionalModelRequestFields": { + "thinking": {"type": "adaptive"}, "output_config": {"effort": "low"}}}), + Some(Effort::Low), + ), + ]; + for (d, v, want) in cases { + assert_eq!(reasoning_effort(d, &v), want, "{d:?}: {v}"); + } + } + + #[test] + fn setting_the_effort_writes_the_field_the_client_wrote() { + let set = |d, mut v: Value, e| { + let changed = set_reasoning_effort(d, &mut v, e); + (changed, v) + }; + assert_eq!( + set( + Dialect::Anthropic, + json!({"thinking": {"type": "adaptive"}, "output_config": {"effort": "xhigh"}}), + Effort::Max + ), + ( + true, + json!({"thinking": {"type": "adaptive"}, "output_config": {"effort": "max"}}) + ) + ); + // 自适应、没写强度：补上 + assert_eq!( + set( + Dialect::Anthropic, + json!({"thinking": {"type": "adaptive"}}), + Effort::High + ), + ( + true, + json!({"thinking": {"type": "adaptive"}, "output_config": {"effort": "high"}}) + ) + ); + // 按预算的改预算，要小于最大输出 + assert_eq!( + set( + Dialect::Anthropic, + json!({"max_tokens": 20000, "thinking": {"type": "enabled", "budget_tokens": 4000}}), + Effort::Max + ), + ( + true, + json!({"max_tokens": 20000, "thinking": {"type": "enabled", "budget_tokens": 19999}}) + ) + ); + let tiny = + json!({"max_tokens": 1000, "thinking": {"type": "enabled", "budget_tokens": 1024}}); + assert_eq!( + set(Dialect::Anthropic, tiny.clone(), Effort::Max), + (false, tiny) + ); + // 已经是这一档：没改 + let same = json!({"reasoning_effort": "high"}); + assert_eq!( + set(Dialect::Chat, same.clone(), Effort::High), + (false, same) + ); + assert_eq!( + set( + Dialect::Chat, + json!({"thinking": {"type": "enabled"}}), + Effort::Low + ), + ( + true, + json!({"thinking": {"type": "enabled"}, "reasoning_effort": "low"}) + ) + ); + assert_eq!( + set( + Dialect::Responses, + json!({"reasoning": {"effort": "xhigh", "summary": "auto"}, + "input": [{"type": "configuration_update", "reasoning": {"effort": "medium"}}, {"type": "message"}]}), + Effort::Max + ), + ( + true, + json!({"reasoning": {"effort": "max", "summary": "auto"}, + "input": [{"type": "configuration_update", "reasoning": {"effort": "max"}}, {"type": "message"}]}) + ) + ); + assert_eq!( + set( + Dialect::Gemini, + json!({"generationConfig": {"thinkingConfig": {"thinkingLevel": "low"}}}), + Effort::Max + ), + ( + true, + json!({"generationConfig": {"thinkingConfig": {"thinkingLevel": "HIGH"}}}) + ) + ); + assert_eq!( + set( + Dialect::Gemini, + json!({"generation_config": {"thinking_config": {"thinking_budget": 1024}}}), + Effort::High + ), + ( + true, + json!({"generation_config": {"thinking_config": {"thinking_budget": 32000}}}) + ) + ); + // Gemini 让模型自己定的：不知道它认等级还是预算，不改 + let dynamic = json!({"generationConfig": {"thinkingConfig": {"includeThoughts": true}}}); + assert_eq!( + set(Dialect::Gemini, dynamic.clone(), Effort::High), + (false, dynamic) + ); + assert_eq!( + set( + Dialect::Bedrock, + json!({"inferenceConfig": {"maxTokens": 9000}, + "additionalModelRequestFields": {"thinking": {"type": "enabled", "budget_tokens": 2000}}}), + Effort::High + ), + ( + true, + json!({"inferenceConfig": {"maxTokens": 9000}, + "additionalModelRequestFields": {"thinking": {"type": "enabled", "budget_tokens": 8999}}}) + ) + ); + assert_eq!( + set( + Dialect::Bedrock, + json!({"additionalModelRequestFields": { + "thinking": {"type": "adaptive"}, "output_config": {"effort": "xhigh"}}}), + Effort::High + ), + ( + true, + json!({"additionalModelRequestFields": { + "thinking": {"type": "adaptive"}, "output_config": {"effort": "high"}}}) + ) + ); + } + + #[test] + fn a_request_without_reasoning_is_never_given_any() { + for (d, v) in [ + (Dialect::Anthropic, json!({"max_tokens": 10})), + ( + Dialect::Anthropic, + json!({"thinking": {"type": "disabled"}}), + ), + (Dialect::Chat, json!({"reasoning_effort": "none"})), + (Dialect::Responses, json!({"input": "hi"})), + ( + Dialect::Gemini, + json!({"generationConfig": {"thinkingConfig": {"thinkingBudget": 0}}}), + ), + ( + Dialect::Bedrock, + json!({"inferenceConfig": {"maxTokens": 10}}), + ), + ] { + let mut w = v.clone(); + assert!(!set_reasoning_effort(d, &mut w, Effort::Max), "{d:?}"); + assert!(!remove_reasoning_effort(d, &mut w), "{d:?}"); + assert_eq!(w, v, "{d:?}"); + } + } + + #[test] + fn removing_the_effort_leaves_reasoning_on() { + let remove = |d, mut v: Value| { + let changed = remove_reasoning_effort(d, &mut v); + (changed, v) + }; + assert_eq!( + remove( + Dialect::Anthropic, + json!({"thinking": {"type": "adaptive"}, "output_config": {"effort": "xhigh"}}) + ), + (true, json!({"thinking": {"type": "adaptive"}})) + ); + // 预算是开思考必须的：不动 + let budget = json!({"thinking": {"type": "enabled", "budget_tokens": 4000}}); + assert_eq!(remove(Dialect::Anthropic, budget.clone()), (false, budget)); + assert_eq!( + remove( + Dialect::Chat, + json!({"thinking": {"type": "enabled"}, "reasoning_effort": "max"}) + ), + (true, json!({"thinking": {"type": "enabled"}})) + ); + assert_eq!( + remove( + Dialect::Responses, + json!({"reasoning": {"effort": "xhigh", "summary": "auto"}, + "input": [{"type": "configuration_update", "reasoning": {"effort": "high"}}, {"type": "message"}]}) + ), + ( + true, + json!({"reasoning": {"summary": "auto"}, "input": [{"type": "message"}]}) + ) + ); + assert_eq!( + remove( + Dialect::Gemini, + json!({"generationConfig": {"thinkingConfig": {"thinkingLevel": "LOW", "includeThoughts": true}}}) + ), + ( + true, + json!({"generationConfig": {"thinkingConfig": {"includeThoughts": true}}}) + ) + ); + assert_eq!( + remove( + Dialect::Bedrock, + json!({"additionalModelRequestFields": { + "thinking": {"type": "adaptive"}, "output_config": {"effort": "max"}}}) + ), + ( + true, + json!({"additionalModelRequestFields": {"thinking": {"type": "adaptive"}}}) + ) + ); + } + + #[test] + fn what_is_set_is_what_is_read_back() { + let bodies = [ + ( + Dialect::Anthropic, + json!({"thinking": {"type": "adaptive"}}), + ), + (Dialect::Chat, json!({"reasoning_effort": "low"})), + (Dialect::Responses, json!({"reasoning": {}})), + ( + Dialect::Bedrock, + json!({"additionalModelRequestFields": {"thinking": {"type": "adaptive"}}}), + ), + ]; + for (d, v) in bodies { + for e in [ + Effort::Low, + Effort::Medium, + Effort::High, + Effort::XHigh, + Effort::Max, + ] { + let mut w = v.clone(); + set_reasoning_effort(d, &mut w, e); + assert_eq!(reasoning_effort(d, &w), Some(e), "{d:?}: {w}"); + } + } + } } diff --git a/crates/tw-dialect/src/think.rs b/crates/tw-dialect/src/think.rs index 5a012a0f..d3aef9af 100644 --- a/crates/tw-dialect/src/think.rs +++ b/crates/tw-dialect/src/think.rs @@ -163,6 +163,103 @@ pub(crate) fn continues_tool_turn_without_thinking(messages: &[crate::ir::Messag ) } +/// 由低到高的全部档位 +pub const LEVELS: [Effort; 6] = [ + Effort::Minimal, + Effort::Low, + Effort::Medium, + Effort::High, + Effort::XHigh, + Effort::Max, +]; + +/// `levels` 里离 `want` 最近的那一档。一样近的取高的那一档：上游说「只有 high 和 max」时， +/// 要 xhigh 的客户端要的是多想，不是少想。`levels` 是空的是 `None` +pub fn nearest(levels: &[Effort], want: Effort) -> Option { + let rank = |e: Effort| e as i32; + levels + .iter() + .copied() + .min_by_key(|e| ((rank(*e) - rank(want)).abs(), std::cmp::Reverse(rank(*e)))) +} + +/// 推理强度被上游拒了之后，重发时怎么写。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EffortFix { + /// 改成这一档 + Use(Effort), + /// 去掉强度，按模型的默认 + Drop, +} + +/// 上游这个错误是不是因为推理强度：是的话，重发时怎么写。`sent` 是这一跳发出去的那一档。 +/// +/// **只认 400**，而且原话（不分大小写）里要说到 `effort`、或者整词说到发出去的那一档、或者 +/// 整词说到另外至少两档 —— 「请使用 low、high 或 max」这样不提 `effort` 的也认得出。 +/// +/// 重发写哪一档看原话里的档位（不算发出去的那一档）：恰好一档后面紧跟着「or below」「or +/// lower」「以下」的，那一档是上限，就用它；不然取离发出去的那一档最近的（[`nearest`]）； +/// 一档都没说的，去掉强度。 +pub fn effort_rejection(status: u16, error_body: &str, sent: Effort) -> Option { + if status != 400 { + return None; + } + let text = error_body.to_lowercase(); + let found = levels_in(&text); + let others: Vec<(usize, Effort)> = found.iter().copied().filter(|(_, e)| *e != sent).collect(); + let mut distinct: Vec = others.iter().map(|(_, e)| *e).collect(); + distinct.sort(); + distinct.dedup(); + let about_effort = + text.contains("effort") || found.iter().any(|(_, e)| *e == sent) || distinct.len() >= 2; + if !about_effort { + return None; + } + let mut ceilings: Vec = others + .iter() + .filter(|(end, _)| ceiling_after(&text[*end..])) + .map(|(_, e)| *e) + .collect(); + ceilings.sort(); + ceilings.dedup(); + if let [only] = ceilings.as_slice() { + return Some(EffortFix::Use(*only)); + } + Some(match nearest(&distinct, sent) { + Some(e) => EffortFix::Use(e), + None => EffortFix::Drop, + }) +} + +/// 小写的 `text` 里整词出现的档位名：每一处的结尾位置和是哪一档 +fn levels_in(text: &str) -> Vec<(usize, Effort)> { + let word = |c: char| c.is_ascii_alphanumeric() || c == '_'; + let mut out = Vec::new(); + for e in LEVELS { + let name = openai(e); + for (at, _) in text.match_indices(name) { + let end = at + name.len(); + let before = text[..at].chars().next_back(); + let after = text[end..].chars().next(); + if !before.is_some_and(word) && !after.is_some_and(word) { + out.push((end, e)); + } + } + } + out.sort_by_key(|(end, _)| *end); + out +} + +/// 档位名后面紧跟着的是不是「以下」：中间可以隔着引号、括号和空白 +fn ceiling_after(rest: &str) -> bool { + let rest = rest.trim_start_matches(|c: char| { + c.is_whitespace() || matches!(c, '\'' | '"' | '`' | '’' | '”' | '」' | ')' | '）') + }); + ["or below", "or lower", "以下"] + .iter() + .any(|p| rest.starts_with(p)) +} + /// 这个 Gemini 模型用 `thinkingLevel` 而不是 `thinkingBudget` pub fn gemini_uses_level(model: &str) -> bool { let m = model.to_ascii_lowercase(); @@ -226,6 +323,68 @@ mod tests { } } + #[test] + fn the_nearest_level_breaks_ties_upward() { + use Effort::*; + assert_eq!(nearest(&[Low, High, Max], Medium), Some(High)); + assert_eq!(nearest(&[High, Max], XHigh), Some(Max)); + assert_eq!(nearest(&[Low, Medium], Max), Some(Medium)); + assert_eq!(nearest(&[Minimal], High), Some(Minimal)); + assert_eq!(nearest(&[], High), None); + } + + #[test] + fn an_effort_rejection_names_what_to_send_instead() { + use Effort::*; + // Anthropic：列出认的那几档 → 离 xhigh 最近的，high 和 max 一样近，取 max + let anthropic = r#"{"type":"error","error":{"type":"invalid_request_error","message":"The parameter `output_config.effort` specified in the request are not valid: expected `low`, `medium`, `high` or `max`, but got `xhigh` instead."}}"#; + assert_eq!( + effort_rejection(400, anthropic, XHigh), + Some(EffortFix::Use(Max)) + ); + // 「high 或以下」：上限 + let ceiling = "output_config.effort 'xhigh' is not supported when thinking is disabled on this model. Use effort 'high' or below, or enable thinking."; + assert_eq!( + effort_rejection(400, ceiling, XHigh), + Some(EffortFix::Use(High)) + ); + assert_eq!( + effort_rejection(400, "effort max is too high; use medium or lower", Max), + Some(EffortFix::Use(Medium)) + ); + // 不提 effort、也不提发出去的那一档，列了别的几档 + let glm = r#"{"error":{"code":"1210","message":"该模型始终思考，不支持关闭思考；请使用 low、high 或 max。"}}"#; + assert_eq!( + effort_rejection(400, glm, Medium), + Some(EffortFix::Use(High)) + ); + // 说到了强度、没说用哪一档：去掉 + assert_eq!( + effort_rejection(400, "reasoning_effort is not supported by this model", High), + Some(EffortFix::Drop) + ); + assert_eq!( + effort_rejection(400, "Unsupported value: 'xhigh'", XHigh), + Some(EffortFix::Drop) + ); + // 与强度无关的 400、不是 400 的：不管 + assert_eq!( + effort_rejection( + 400, + r#"{"error":{"message":"messages: field required"}}"#, + High + ), + None + ); + assert_eq!(effort_rejection(500, anthropic, XHigh), None); + // 只说到一个别的档位、不是整词的不算 + assert_eq!( + effort_rejection(400, "max_tokens must be at most 8192", High), + None + ); + assert_eq!(effort_rejection(400, "xhighly unusual", High), None); + } + #[test] fn openai_none_turns_reasoning_off() { assert_eq!(parse_openai("none"), Some(None)); diff --git a/crates/tw-engine/src/engine.rs b/crates/tw-engine/src/engine.rs index b8bf476b..c2b8efac 100644 --- a/crates/tw-engine/src/engine.rs +++ b/crates/tw-engine/src/engine.rs @@ -5,6 +5,7 @@ //! 所以引擎只有一条代码路径，用户看到的却是零配置可用。 use serde::{Deserialize, Serialize}; +use tw_dialect::ir::Effort; use tw_types::{Msg, msg}; use crate::facts::RequestFacts; @@ -320,8 +321,12 @@ pub struct SetAction { pub model: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub max_tokens: Option, + /// 推理：`off` 关掉，或者改成一档（`low` / `medium` / `high` / `xhigh` / `max`）。 + /// + /// **只改开着推理的请求**：没开推理的（生成标题这类辅助请求）原样不动，改成一档也不替它 + /// 打开 —— 开推理要多花钱，不该由一条规则悄悄替客户端决定。`off` 连同推理强度一起去掉 #[serde(default, skip_serializing_if = "Option::is_none")] - pub thinking: Option, + pub reasoning: Option, /// 只在新会话开始时应用，跑到一半不动它。 /// /// **长会话场景下这应该是默认值**，但会话识别是 M3 的事 —— @@ -340,14 +345,86 @@ impl SetAction { if other.max_tokens.is_some() { self.max_tokens = other.max_tokens; } - if other.thinking.is_some() { - self.thinking = other.thinking; + if other.reasoning.is_some() { + self.reasoning = other.reasoning; } self.only_at_session_start |= other.only_at_session_start; } pub fn is_empty(&self) -> bool { - self.model.is_none() && self.max_tokens.is_none() && self.thinking.is_none() + self.model.is_none() && self.max_tokens.is_none() && self.reasoning.is_none() + } +} + +/// 规则改写的推理（[`SetAction::reasoning`]）。配置里写成一个词：`off`、`low`、`medium`、 +/// `high`、`xhigh`、`max`。**没有 `minimal`**：只有 OpenAI 认它，写给别家等于 `low`，规则里 +/// 写它只会让人以为比 `low` 还少 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(from = "ReasoningWord", into = "ReasoningWord")] +pub enum SetReasoning { + /// 关掉推理 + Off, + /// 改成这一档 + Level(Effort), +} + +impl SetReasoning { + /// 写在配置里的那个词 + pub fn slug(self) -> &'static str { + ReasoningWord::from(self).slug() + } +} + +/// [`SetReasoning`] 在配置里的写法。不认识的词（`minimal` 也是）读配置时报出来，列出认得的 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +enum ReasoningWord { + Off, + Low, + Medium, + High, + #[serde(rename = "xhigh")] + XHigh, + Max, +} + +impl ReasoningWord { + fn slug(self) -> &'static str { + match self { + Self::Off => "off", + Self::Low => "low", + Self::Medium => "medium", + Self::High => "high", + Self::XHigh => "xhigh", + Self::Max => "max", + } + } +} + +impl From for SetReasoning { + fn from(w: ReasoningWord) -> Self { + match w { + ReasoningWord::Off => Self::Off, + ReasoningWord::Low => Self::Level(Effort::Low), + ReasoningWord::Medium => Self::Level(Effort::Medium), + ReasoningWord::High => Self::Level(Effort::High), + ReasoningWord::XHigh => Self::Level(Effort::XHigh), + ReasoningWord::Max => Self::Level(Effort::Max), + } + } +} + +impl From for ReasoningWord { + fn from(r: SetReasoning) -> Self { + match r { + SetReasoning::Off => Self::Off, + // 构造不出 `minimal`（配置里不认它），真有的话按最接近的 `low` 写 + SetReasoning::Level(Effort::Minimal | Effort::Low) => Self::Low, + SetReasoning::Level(Effort::Medium) => Self::Medium, + SetReasoning::Level(Effort::High) => Self::High, + SetReasoning::Level(Effort::XHigh) => Self::XHigh, + SetReasoning::Level(Effort::Max) => Self::Max, + } } } @@ -1897,7 +1974,7 @@ mod tests { "{}", None, Some(SetAction { - thinking: Some(false), + reasoning: Some(SetReasoning::Off), ..Default::default() }), None, @@ -1918,7 +1995,7 @@ mod tests { ); let d = decision(&e, &facts("x")); assert_eq!(d.candidates, ["a"], "去向是第一条有 to 的"); - assert_eq!(d.set.thinking, Some(false)); + assert_eq!(d.set.reasoning, Some(SetReasoning::Off)); assert_eq!( d.set.model.as_deref(), Some("cheap"), @@ -1926,13 +2003,52 @@ mod tests { ); } + /// `set.reasoning` 是一个词；`minimal` 和不认识的词读配置时就报出来，列出认得的。 + /// 旧的 `thinking` 不认了 + #[test] + fn a_reasoning_rewrite_is_one_word() { + let parse = |y: &str| serde_yaml_ng::from_str::(y); + for (word, want) in [ + ("off", SetReasoning::Off), + ("low", SetReasoning::Level(Effort::Low)), + ("medium", SetReasoning::Level(Effort::Medium)), + ("high", SetReasoning::Level(Effort::High)), + ("xhigh", SetReasoning::Level(Effort::XHigh)), + ("max", SetReasoning::Level(Effort::Max)), + ] { + let s = parse(&format!("reasoning: {word}")).unwrap(); + assert_eq!(s.reasoning, Some(want)); + assert_eq!(want.slug(), word); + assert_eq!( + serde_yaml_ng::to_string(&s).unwrap().trim(), + format!("reasoning: {word}") + ); + } + for bad in ["minimal", "none", "true", "XHIGH"] { + let e = parse(&format!("reasoning: {bad}")).unwrap_err().to_string(); + assert!( + e.contains("reasoning") && e.contains("`xhigh`") && e.contains("`off`"), + "{bad}: {e}" + ); + } + assert!(parse("thinking: false").is_err()); + // 后面的规则盖过前面的 + let mut a = parse("reasoning: off").unwrap(); + a.merge(&parse("reasoning: max").unwrap()); + assert_eq!(a.reasoning, Some(SetReasoning::Level(Effort::Max))); + a.merge(&parse("max_tokens: 9").unwrap()); + assert_eq!(a.reasoning, Some(SetReasoning::Level(Effort::Max))); + assert!(!a.is_empty()); + assert!(!parse("reasoning: off").unwrap().is_empty()); + } + /// 改写是谁加的要记下来：命中了、真的改了东西的才算，没命中的、只写了 /// `only_at_session_start` 的不算。决定去向的那条自己带着改写时也算。 #[test] fn the_decision_names_every_rule_that_added_a_rewrite() { let thinking_off = || { Some(SetAction { - thinking: Some(false), + reasoning: Some(SetReasoning::Off), ..Default::default() }) }; @@ -1988,7 +2104,7 @@ mod tests { "{ provider_would_be: relay }", None, Some(SetAction { - thinking: Some(false), + reasoning: Some(SetReasoning::Off), ..Default::default() }), None, @@ -2082,7 +2198,7 @@ mod tests { "{ provider_would_be: relay }", None, Some(SetAction { - thinking: Some(false), + reasoning: Some(SetReasoning::Off), ..Default::default() }), None, @@ -2092,7 +2208,7 @@ mod tests { ); let d = decision(&e, &facts("x")); assert_eq!(d.matched_rule, "兜底"); - assert_eq!(d.set.thinking, None, "阶段一不该看见阶段二的 set"); + assert_eq!(d.set.reasoning, None, "阶段一不该看见阶段二的 set"); } #[test] @@ -2106,7 +2222,7 @@ mod tests { "{ provider_would_be: relay }", None, Some(SetAction { - thinking: Some(false), + reasoning: Some(SetReasoning::Off), ..Default::default() }), None, @@ -2116,12 +2232,12 @@ mod tests { ); let base = SetAction::default(); match e.phase_two(&facts("x"), "relay", &base).unwrap() { - Outcome2::Proceed { set: s, .. } => assert_eq!(s.thinking, Some(false)), + Outcome2::Proceed { set: s, .. } => assert_eq!(s.reasoning, Some(SetReasoning::Off)), other => panic!("{other:?}"), } // 换一家就不该命中了 —— 这正是故障转移后必须重跑的理由 match e.phase_two(&facts("x"), "official", &base).unwrap() { - Outcome2::Proceed { set: s, .. } => assert_eq!(s.thinking, None), + Outcome2::Proceed { set: s, .. } => assert_eq!(s.reasoning, None), other => panic!("{other:?}"), } } @@ -2139,7 +2255,7 @@ mod tests { "{ provider_would_be: [a, b] }", None, Some(SetAction { - thinking: Some(false), + reasoning: Some(SetReasoning::Off), ..Default::default() }), None, @@ -2148,9 +2264,13 @@ mod tests { ], ); let base = SetAction::default(); - for (p, want) in [("a", Some(false)), ("b", Some(false)), ("c", None)] { + for (p, want) in [ + ("a", Some(SetReasoning::Off)), + ("b", Some(SetReasoning::Off)), + ("c", None), + ] { match e.phase_two(&facts("x"), p, &base).unwrap() { - Outcome2::Proceed { set: s, .. } => assert_eq!(s.thinking, want, "provider={p}"), + Outcome2::Proceed { set: s, .. } => assert_eq!(s.reasoning, want, "provider={p}"), other => panic!("{other:?}"), } } @@ -3012,7 +3132,7 @@ to: ", ), rule( - "{name: 阶段二, when: {provider_would_be: relay}, set: {model: x, thinking: false}}", + "{name: 阶段二, when: {provider_would_be: relay}, set: {model: x, reasoning: off}}", ), rule("{name: 兜底, to: __all__}"), ]; @@ -3040,7 +3160,10 @@ to: Outcome2::Proceed { model, set, .. } => { let a = e.asked_of(&facts("opus"), &d, "relay", model.as_deref()); assert_eq!((a.model.as_str(), a.origin), ("opus", Origin::Pinned)); - assert_eq!((set.thinking, set.max_tokens), (Some(false), Some(10))); + assert_eq!( + (set.reasoning, set.max_tokens), + (Some(SetReasoning::Off), Some(10)) + ); } other => panic!("{other:?}"), } diff --git a/crates/tw-engine/src/lib.rs b/crates/tw-engine/src/lib.rs index f9dab4ca..a2be0d21 100644 --- a/crates/tw-engine/src/lib.rs +++ b/crates/tw-engine/src/lib.rs @@ -10,8 +10,8 @@ pub use catalog::{Catalog, ProviderModels}; pub use engine::{ ALL_UPSTREAMS, Asked, BalanceBy, CATCH_ALL_RULE, DEFAULT_ROUTE, Decision, Engine, Facts, Group, GroupType, Origin, Outcome, Outcome2, Pinned, RESERVED_PREFIX, RouteError, RouteSet, Rule, - RuleNotes, SetAction, Target, balance_factors, has_catch_all, is_builtin_group, notes, - order_by, scalar_name, + RuleNotes, SetAction, SetReasoning, Target, balance_factors, has_catch_all, is_builtin_group, + notes, order_by, scalar_name, }; pub use facts::{ RequestFacts, Text as TextTokens, TokenParts, estimate_parts, estimate_strings, estimate_tokens, diff --git a/crates/tw-gateway/src/forward.rs b/crates/tw-gateway/src/forward.rs index fb491ac0..8337f6d4 100644 --- a/crates/tw-gateway/src/forward.rs +++ b/crates/tw-gateway/src/forward.rs @@ -128,14 +128,16 @@ fn set_params( if let Some(t) = set.max_tokens { params::set_max_output_tokens(client, v, t); } - match set.thinking { - // 开启思考需要一个 budget，而我们没有一个合理的值可以编。 - // **只做「关掉」这一个方向** —— 那是降级场景真正需要的。 - Some(true) => tracing::warn!( - "set.thinking: true is not supported yet (it needs budget_tokens); ignoring it" - ), - Some(false) => params::disable_thinking(client, v), - None => {} + // 推理只改开着推理的请求：没开的不替它打开（开推理要一个预算，也要多花钱），关掉的也 + // 不动 —— 去掉 `reasoning_effort: "none"` 反倒让推理按模型的默认打开了 + match set.reasoning { + Some(tw_engine::SetReasoning::Off) if params::reasoning_on(client, v) => { + params::disable_thinking(client, v) + } + Some(tw_engine::SetReasoning::Level(e)) => { + params::set_reasoning_effort(client, v, e); + } + _ => {} } } @@ -266,7 +268,7 @@ mod tests { let out = apply_set( &b, &tw_engine::SetAction { - thinking: Some(false), + reasoning: Some(tw_engine::SetReasoning::Off), ..Default::default() }, None, @@ -296,7 +298,7 @@ mod tests { let set = tw_engine::SetAction { model: Some("small".into()), max_tokens: Some(512), - thinking: Some(false), + reasoning: Some(tw_engine::SetReasoning::Off), ..Default::default() }; let run = |body: &'static str, d| { @@ -340,12 +342,12 @@ mod tests { fn set( model: Option<&str>, max_tokens: Option, - thinking: Option, + reasoning: Option, ) -> tw_engine::SetAction { tw_engine::SetAction { model: model.map(str::to_string), max_tokens, - thinking, + reasoning, ..Default::default() } } @@ -369,7 +371,11 @@ mod tests { ); let out = apply_set( &Bytes::from(body.clone()), - &set(Some("claude-haiku-5"), Some(4096), Some(false)), + &set( + Some("claude-haiku-5"), + Some(4096), + Some(tw_engine::SetReasoning::Off), + ), Some(Dialect::Anthropic), ); assert_eq!( @@ -394,7 +400,11 @@ mod tests { ); let out = apply_set( &Bytes::from(gemini), - &set(Some("ignored"), Some(512), Some(false)), + &set( + Some("ignored"), + Some(512), + Some(tw_engine::SetReasoning::Off), + ), Some(Dialect::Gemini), ); assert_eq!( @@ -461,6 +471,7 @@ mod tests { "\"thinking\":{\"type\":\"enabled\",\"budget_tokens\":9}", "\"reasoning_effort\":\"high\"", "\"reasoning\":{\"effort\":\"low\"}", + "\"output_config\":{\"effort\":\"xhigh\"}", "\"inferenceConfig\":{\"temperature\":1,\"maxTokens\":3}", "\"additionalModelRequestFields\":{\"thinking\":{\"type\":\"enabled\"},\"k\":1}", "\"tools\":[{\"input_schema\":{\"type\":\"object\",\"properties\":{\"b\":{},\"a\":{}}}}]", @@ -491,7 +502,11 @@ mod tests { let s = set( [None, Some("small"), Some("big")][r.below(3)], [None, Some(9), Some(512)][r.below(3)], - [None, Some(false), Some(true)][r.below(3)], + [ + None, + Some(tw_engine::SetReasoning::Off), + Some(tw_engine::SetReasoning::Level(tw_dialect::ir::Effort::Max)), + ][r.below(3)], ); if s.is_empty() { continue; diff --git a/crates/tw-gateway/src/server.rs b/crates/tw-gateway/src/server.rs index b376472a..d9b605ad 100644 --- a/crates/tw-gateway/src/server.rs +++ b/crates/tw-gateway/src/server.rs @@ -389,6 +389,7 @@ pub(crate) fn hop( pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } @@ -419,6 +420,7 @@ pub(crate) fn hop_busy( pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } @@ -471,6 +473,7 @@ pub(crate) fn hop_waiting( pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } @@ -502,6 +505,7 @@ pub(crate) fn hop_used_up( pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } @@ -526,6 +530,7 @@ pub(crate) fn hop_failed( pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } diff --git a/crates/tw-gateway/src/server/pipeline/hop.rs b/crates/tw-gateway/src/server/pipeline/hop.rs index 2334e34c..5e40ecf1 100644 --- a/crates/tw-gateway/src/server/pipeline/hop.rs +++ b/crates/tw-gateway/src/server/pipeline/hop.rs @@ -1848,6 +1848,7 @@ fn estimated_hop( pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } diff --git a/crates/tw-gateway/src/server/pipeline/idle.rs b/crates/tw-gateway/src/server/pipeline/idle.rs index 71cf82b1..544fd3e3 100644 --- a/crates/tw-gateway/src/server/pipeline/idle.rs +++ b/crates/tw-gateway/src/server/pipeline/idle.rs @@ -101,6 +101,7 @@ pub(super) fn abandoned( pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } @@ -126,6 +127,7 @@ pub(super) fn aborted_hop( pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } diff --git a/crates/tw-gateway/src/translate.rs b/crates/tw-gateway/src/translate.rs index 8a74ee0f..70afc575 100644 --- a/crates/tw-gateway/src/translate.rs +++ b/crates/tw-gateway/src/translate.rs @@ -57,9 +57,15 @@ pub fn apply_set(r: &mut Request, set: &tw_engine::SetAction) { if let Some(t) = set.max_tokens { r.max_tokens = Some(t); } - // 和直通时一样只做「关掉」：去掉推理配置，按模型默认走 - if set.thinking == Some(false) { - r.reasoning = None; + // 和直通时一样只改开着推理的请求：关掉是去掉推理配置；改一档是改强度，预算跟着强度 + // 由编码器折算（客户端写的预算不再作数） + match (set.reasoning, r.reasoning.as_mut()) { + (Some(tw_engine::SetReasoning::Off), Some(re)) if re.enabled => r.reasoning = None, + (Some(tw_engine::SetReasoning::Level(e)), Some(re)) if re.enabled => { + re.effort = Some(e); + re.budget = None; + } + _ => {} } } diff --git a/crates/tw-gateway/tests/alias_ws.rs b/crates/tw-gateway/tests/alias_ws.rs index 7a476802..4b700120 100644 --- a/crates/tw-gateway/tests/alias_ws.rs +++ b/crates/tw-gateway/tests/alias_ws.rs @@ -664,14 +664,14 @@ fn create_reasoning(model: &str) -> WsMsg { } /// 规则的参数改写每一帧照 HTTP 那条路改（Responses 的字段名）：`set.max_tokens` 写成 -/// `max_output_tokens`，`set.thinking: false` 去掉 `reasoning`。条件按这一帧求 —— 按模型附加的 +/// `max_output_tokens`，`set.reasoning: off` 去掉 `reasoning`。条件按这一帧求 —— 按模型附加的 /// 改写、按模型拒绝的规则，升级时还不知道模型，每一帧照样生效 #[tokio::test] async fn rule_parameter_rewrites_and_model_conditions_apply_to_each_frame() { let (up, seen) = upstream().await; let rules = " - { name: Codex 限长, when: { client: codex }, set: { max_tokens: 1234 } } -- { name: Max 不思考, when: { model: gpt-5.1-codex-max }, set: { thinking: false, max_tokens: 99 } } +- { name: Max 不思考, when: { model: gpt-5.1-codex-max }, set: { reasoning: off, max_tokens: 99 } } - { name: 不给, when: { model: banned }, deny: 这个模型不走这里 } - { name: 兜底, to: __all__ } "; diff --git a/crates/tw-gateway/tests/passthrough.rs b/crates/tw-gateway/tests/passthrough.rs index 405b984d..44d2c114 100644 --- a/crates/tw-gateway/tests/passthrough.rs +++ b/crates/tw-gateway/tests/passthrough.rs @@ -1191,7 +1191,7 @@ async fn a_phase_two_rule_is_recomputed_after_failover() { when: serde_yaml_ng::from_str("{ provider_would_be: relay }").unwrap(), to: None, set: Some(tw_engine::SetAction { - thinking: Some(false), + reasoning: Some(tw_engine::SetReasoning::Off), ..Default::default() }), deny: None, diff --git a/crates/tw-gateway/tests/routing_facts.rs b/crates/tw-gateway/tests/routing_facts.rs index 475671ca..5d4fc997 100644 --- a/crates/tw-gateway/tests/routing_facts.rs +++ b/crates/tw-gateway/tests/routing_facts.rs @@ -60,7 +60,7 @@ fn rule( fn thinking_off() -> Option { Some(SetAction { - thinking: Some(false), + reasoning: Some(tw_engine::SetReasoning::Off), ..Default::default() }) } diff --git a/crates/tw-observe/src/bus.rs b/crates/tw-observe/src/bus.rs index 1955b049..3d98229e 100644 --- a/crates/tw-observe/src/bus.rs +++ b/crates/tw-observe/src/bus.rs @@ -512,6 +512,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }, tw_api::AttemptView { @@ -527,6 +528,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }, ], diff --git a/crates/tw-store/src/db.rs b/crates/tw-store/src/db.rs index 683fa0eb..1493a575 100644 --- a/crates/tw-store/src/db.rs +++ b/crates/tw-store/src/db.rs @@ -3639,6 +3639,7 @@ mod cost_state_tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], ..Default::default() @@ -4151,6 +4152,7 @@ mod cost { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } }) diff --git a/crates/tw-store/src/recorder.rs b/crates/tw-store/src/recorder.rs index 8dbbdfb6..00e15b15 100644 --- a/crates/tw-store/src/recorder.rs +++ b/crates/tw-store/src/recorder.rs @@ -995,6 +995,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }, tw_api::AttemptView { @@ -1010,6 +1011,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: Some("机场".into()), }, ], @@ -1078,6 +1080,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing: tw_api::Billing::PerToken, @@ -1115,6 +1118,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }, tw_api::AttemptView { @@ -1130,6 +1134,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }, ], @@ -1186,6 +1191,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing: tw_api::Billing::PerToken, @@ -1317,6 +1323,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing: tw_api::Billing::PerToken, @@ -1646,6 +1653,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing: tw_api::Billing::Free, @@ -1691,6 +1699,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing: tw_api::Billing::Free, @@ -1775,6 +1784,7 @@ mod billing_tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing, @@ -1870,6 +1880,7 @@ mod billing_tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing, @@ -2319,6 +2330,7 @@ mod security_detail_tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing: tw_api::Billing::PerToken, @@ -2617,6 +2629,7 @@ mod translation_tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }) .collect(), @@ -3312,6 +3325,7 @@ mod settle_hook_tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing: tw_api::Billing::PerToken, diff --git a/docs/config.md b/docs/config.md index aba312de..91b7b22a 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1507,7 +1507,7 @@ it. Answers carry the name the client asked for, as with aliases. |---|---|---|---| | `model` | string | — | Send a different model. The prompt cache of the session is lost. | | `max_tokens` | integer | — | Replace `max_tokens`. | -| `thinking` | bool | — | Turn extended thinking on or off. | +| `reasoning` | `off` \| `low` \| `medium` \| `high` \| `xhigh` \| `max` | — | Reasoning. `off` turns it off and removes the reasoning effort; a level changes the effort. Only requests that already have reasoning on are changed; requests without reasoning (such as title generation) are left as they are. | | `only_at_session_start` | bool | `false` | Apply only when a session starts. Recorded and shown; not in effect yet. | diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index 7f8d133c..5a752fe6 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -1096,7 +1096,7 @@ routes: |---|---|---|---| | `model` | 字符串 | — | 换成另一个模型发送。该会话的 prompt cache 随之失效。 | | `max_tokens` | 整数 | — | 替换 `max_tokens`。 | -| `thinking` | 布尔 | — | 开启或关闭扩展思考。 | +| `reasoning` | `off` \| `low` \| `medium` \| `high` \| `xhigh` \| `max` | — | 推理。`off` 关闭推理，连同推理强度一起去掉；写一档则改为该档强度。只改写已开启推理的请求，未开启推理的请求（如生成标题）保持不变。 | | `only_at_session_start` | 布尔 | `false` | 只在会话开始时应用。目前只记录和显示，尚未生效。 | From 66ac96ac8ab4cb7da105dab8619b89404dbd1c33 Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Sun, 25 Oct 2026 11:48:51 +0800 Subject: [PATCH 2/4] Send a refused reasoning effort again with one the upstream takes When an upstream answers 400 because of the reasoning effort (the error mentions `effort`, the level sent, or at least two other levels), the request goes once more to the same upstream as a new attempt: with the ceiling the error names, else the nearest level it lists (ties go up), else without an effort. It is not a backoff retry. If the resend is answered, the gateway remembers per upstream and model which level to use instead, for 7 days, in `effort-memory.json` in the data directory; a resend refused again is handed on and not remembered. Entries of an upstream are dropped when its address, key or protocol changes or it is removed. The attempt chain says on each hop whether the effort sent differs from the one asked, and why (`rule`, `rejected`, `remembered`). Co-Authored-By: Claude Opus 5.5 --- crates/tw-dialect/src/ir.rs | 2 +- crates/tw-gateway/src/effort_memory.rs | 390 +++++++++++++++++ crates/tw-gateway/src/forward.rs | 14 +- crates/tw-gateway/src/lib.rs | 1 + crates/tw-gateway/src/server/pipeline/hop.rs | 212 +++++++++- crates/tw-gateway/src/state.rs | 8 + crates/tw-gateway/tests/effort.rs | 418 +++++++++++++++++++ docs/config.md | 14 + docs/config.zh-CN.md | 4 + 9 files changed, 1052 insertions(+), 11 deletions(-) create mode 100644 crates/tw-gateway/src/effort_memory.rs create mode 100644 crates/tw-gateway/tests/effort.rs diff --git a/crates/tw-dialect/src/ir.rs b/crates/tw-dialect/src/ir.rs index df773db8..1084dc63 100644 --- a/crates/tw-dialect/src/ir.rs +++ b/crates/tw-dialect/src/ir.rs @@ -344,7 +344,7 @@ pub struct Reasoning { pub summary: bool, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Effort { Minimal, Low, diff --git a/crates/tw-gateway/src/effort_memory.rs b/crates/tw-gateway/src/effort_memory.rs new file mode 100644 index 00000000..4224b185 --- /dev/null +++ b/crates/tw-gateway/src/effort_memory.rs @@ -0,0 +1,390 @@ +//! 上游不认的推理强度，记下来。 +//! +//! 各家认的档位不一样：Claude 的有的模型没有 `xhigh`，关着思考的不收 `high` 以上，GLM 这类 +//! 始终思考的模型不收 `medium`。客户端（或者路由规则）写的那一档被上游拒了（400，原话说的是 +//! 推理强度，见 [`tw_dialect::think::effort_rejection`]），同一家改一档、或者去掉强度再发一次。 +//! **重发成了才记**：（上游，发给它的模型名）不认哪一档、改用什么。往后发给它的请求要是还写着 +//! 那一档，发之前直接改，不用每次都先被拒一次。重发又被拒了的不记：原话里碰巧有一个档位名的 +//! 400（「max context length exceeded」）不该让这个模型此后 7 天都换一档。 +//! +//! **7 天后忘掉**：上游可能已经认了，那时再试原来那一档。存在数据目录的 `effort-memory.json` +//! 里，重启之后还在；一家上游的地址、密钥、协议变了，或者删掉了，它的这几条随之作废 —— 那已经 +//! 是另一个上游了。 + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use serde::{Deserialize, Serialize}; +use tw_dialect::ir::Effort; +use tw_dialect::think::{self, EffortFix}; + +/// 存在数据目录（配置文件所在的目录）里的文件名 +pub const FILE: &str = "effort-memory.json"; + +/// 记多久，秒 +pub const KEEP_SECS: u64 = 7 * 24 * 3600; + +/// 最多记多少条。满了丢最早记下的那一条：丢了也只是下次再被拒一次、再重发一次 +const MAX: usize = 1024; + +/// 一条：这一家的这个模型不认 `rejected` 这一档，改用 `use`（`None` 是去掉强度） +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct Entry { + provider: String, + model: String, + /// 记下时这家长什么样（地址、凭据、协议的指纹）。对不上就作废 + identity: String, + rejected: String, + #[serde(rename = "use")] + instead: Option, + /// 记下的时刻，Unix 秒 + at: u64, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +struct File { + entries: Vec, +} + +type Key = (String, String, Effort); + +#[derive(Default)] +struct Inner { + /// 文件在哪儿。没告诉过数据目录（测试里直接建的网关）时只在内存里记 + path: Option, + by: HashMap, u64)>, +} + +/// 见模块说明。**跨重载存活**：配置换了只作废变了的那几家 +#[derive(Default)] +pub struct EffortMemory { + inner: Mutex, + /// 写文件排着队：后写的那一次写的一定是最新的那一份 + writing: Mutex<()>, +} + +/// 决定这几条还作不作数的那几样：地址、凭据、协议。**存指纹，不存原文** —— 这个文件不该多出 +/// 一份密钥 +fn identity(p: &tw_config::Provider) -> String { + let raw = format!( + "{}|{}|{:?}", + p.base_url, + p.credential_identity(), + p.effective_protocol() + ); + blake3::hash(raw.as_bytes()).to_hex().to_string() +} + +fn fresh(at: u64, now: u64) -> bool { + now.saturating_sub(at) < KEEP_SECS +} + +pub fn now_secs() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +impl EffortMemory { + fn lock(&self) -> std::sync::MutexGuard<'_, Inner> { + self.inner.lock().unwrap_or_else(|e| e.into_inner()) + } + + /// 告诉它数据目录在哪儿：读进那里的文件（读不了、读不成的当没有：丢了只是再被拒一次）， + /// 再按 `cfg` 对一遍。往后记下的写回那里 + pub fn set_dir(self: &Arc, dir: PathBuf, cfg: &tw_config::Config) { + let path = dir.join(FILE); + let file: File = std::fs::read(&path) + .ok() + .and_then(|b| serde_json::from_slice(&b).ok()) + .unwrap_or_default(); + { + let mut g = self.lock(); + g.path = Some(path); + g.by.clear(); + for e in file.entries { + let Some(rejected) = parse(&e.rejected) else { + continue; + }; + let instead = match e.instead.as_deref() { + None => None, + Some(s) => match parse(s) { + Some(x) => Some(x), + None => continue, + }, + }; + g.by.insert((e.provider, e.model, rejected), (e.identity, instead, e.at)); + } + } + self.reconcile(cfg); + } + + /// 配置换了之后对一遍：删掉的上游、地址密钥协议变了的上游，它们的几条作废 + pub fn reconcile(self: &Arc, cfg: &tw_config::Config) { + let ids: HashMap<&str, String> = cfg + .providers + .iter() + .map(|p| (p.name.as_str(), identity(p))) + .collect(); + let now = now_secs(); + let changed = { + let mut g = self.lock(); + let before = g.by.len(); + g.by.retain(|(provider, _, _), (id, _, at)| { + ids.get(provider.as_str()) == Some(id) && fresh(*at, now) + }); + g.by.len() != before + }; + if changed { + self.save(); + } + } + + /// 这一家的这个模型有没有记着什么。发之前先问这一句，没有的话不用解请求体 + pub fn has(&self, provider: &str, model: &str) -> bool { + let g = self.lock(); + !g.by.is_empty() && g.by.keys().any(|(p, m, _)| p == provider && m == model) + } + + /// 发给这一家的这个模型的请求写着 `rejected` 这一档：记着该改成什么的话交回来 + pub fn lookup( + &self, + p: &tw_config::Provider, + model: &str, + rejected: Effort, + now: u64, + ) -> Option { + let g = self.lock(); + let (id, instead, at) = g.by.get(&(p.name.clone(), model.to_string(), rejected))?; + (fresh(*at, now) && *id == identity(p)).then_some(match instead { + Some(e) => EffortFix::Use(*e), + None => EffortFix::Drop, + }) + } + + /// 记下：这一家的这个模型不认 `rejected`，改用 `fix` 重发成了。改用的就是它自己的不记 + pub fn note( + self: &Arc, + p: &tw_config::Provider, + model: &str, + rejected: Effort, + fix: EffortFix, + now: u64, + ) { + let instead = match fix { + EffortFix::Use(e) if e == rejected => return, + EffortFix::Use(e) => Some(e), + EffortFix::Drop => None, + }; + { + let mut g = self.lock(); + let key = (p.name.clone(), model.to_string(), rejected); + if g.by.len() >= MAX + && !g.by.contains_key(&key) + && let Some(oldest) = + g.by.iter() + .min_by_key(|(_, (_, _, at))| *at) + .map(|(k, _)| k.clone()) + { + g.by.remove(&oldest); + } + g.by.insert(key, (identity(p), instead, now)); + } + tracing::info!( + provider = %p.name, + model = %model, + rejected = think::openai(rejected), + instead = instead.map_or("none", think::openai), + "remembered a reasoning effort this model does not take" + ); + self.save(); + } + + /// 写回文件。**不在转发那条路上写**：有异步运行时就挪到阻塞线程上 + fn save(self: &Arc) { + let me = self.clone(); + match tokio::runtime::Handle::try_current() { + Ok(h) => { + h.spawn_blocking(move || me.save_now()); + } + Err(_) => me.save_now(), + } + } + + /// 写回文件（原子写）。排着队写，写的时候现取最新的那一份 + pub(crate) fn save_now(&self) { + let _w = self.writing.lock().unwrap_or_else(|e| e.into_inner()); + let (path, text) = { + let g = self.lock(); + let Some(path) = g.path.clone() else { + return; + }; + let now = now_secs(); + let mut entries: Vec = + g.by.iter() + .filter(|(_, (_, _, at))| fresh(*at, now)) + .map(|((provider, model, rejected), (id, instead, at))| Entry { + provider: provider.clone(), + model: model.clone(), + identity: id.clone(), + rejected: think::openai(*rejected).to_string(), + instead: instead.map(|e| think::openai(e).to_string()), + at: *at, + }) + .collect(); + entries + .sort_by(|a, b| (&a.provider, &a.model, a.at).cmp(&(&b.provider, &b.model, b.at))); + match serde_json::to_string_pretty(&File { entries }) { + Ok(t) => (path, t), + Err(_) => return, + } + }; + if let Err(e) = tw_config::store::write_atomic(&path, &text) { + tracing::warn!("the reasoning effort memory could not be written: {e}"); + } + } +} + +/// 存着的档位名。`none` 之类不是一档的不认 +fn parse(s: &str) -> Option { + think::parse_openai(s).flatten() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn provider(name: &str, key: &str) -> tw_config::Provider { + tw_config::Provider { + name: name.into(), + base_url: "https://relay.example.com".into(), + key: Some(key.into()), + ..Default::default() + } + } + + fn config(providers: Vec) -> tw_config::Config { + tw_config::Config { + version: 1, + providers, + ..Default::default() + } + } + + #[test] + fn a_remembered_level_is_replaced_until_it_expires() { + let m = Arc::new(EffortMemory::default()); + let p = provider("relay", "sk-1"); + let now = 1_000_000; + assert!(!m.has("relay", "glm-5")); + m.note( + &p, + "glm-5", + Effort::Medium, + EffortFix::Use(Effort::High), + now, + ); + assert!(m.has("relay", "glm-5")); + assert!(!m.has("relay", "glm-4")); + assert_eq!( + m.lookup(&p, "glm-5", Effort::Medium, now + 60), + Some(EffortFix::Use(Effort::High)) + ); + assert_eq!(m.lookup(&p, "glm-5", Effort::Low, now), None); + // 7 天之后再试原来那一档 + assert_eq!(m.lookup(&p, "glm-5", Effort::Medium, now + KEEP_SECS), None); + // 去掉强度也记得住；改用的就是它自己的不记 + m.note(&p, "glm-5", Effort::Max, EffortFix::Drop, now); + assert_eq!( + m.lookup(&p, "glm-5", Effort::Max, now), + Some(EffortFix::Drop) + ); + m.note(&p, "glm-5", Effort::Low, EffortFix::Use(Effort::Low), now); + assert_eq!(m.lookup(&p, "glm-5", Effort::Low, now), None); + } + + #[test] + fn a_changed_or_removed_upstream_forgets_what_it_refused() { + let m = Arc::new(EffortMemory::default()); + let now = now_secs(); + let (a, b) = (provider("a", "sk-1"), provider("b", "sk-2")); + m.note(&a, "m", Effort::XHigh, EffortFix::Use(Effort::Max), now); + m.note(&b, "m", Effort::XHigh, EffortFix::Use(Effort::Max), now); + // 换了密钥：已经是另一个上游了，查的时候就不作数 + let rekeyed = provider("a", "sk-9"); + assert_eq!(m.lookup(&rekeyed, "m", Effort::XHigh, now), None); + m.reconcile(&config(vec![rekeyed.clone()])); + assert!(!m.has("a", "m"), "密钥变了"); + assert!(!m.has("b", "m"), "删掉了"); + // 地址、协议变了也一样 + m.note(&rekeyed, "m", Effort::XHigh, EffortFix::Drop, now); + let moved = tw_config::Provider { + base_url: "https://other.example.com".into(), + ..rekeyed.clone() + }; + m.reconcile(&config(vec![moved])); + assert!(!m.has("a", "m")); + m.note(&rekeyed, "m", Effort::XHigh, EffortFix::Drop, now); + let reprotocoled = tw_config::Provider { + protocol: Some(tw_config::Protocol::OpenaiChat), + ..rekeyed.clone() + }; + m.reconcile(&config(vec![reprotocoled])); + assert!(!m.has("a", "m")); + // 没变的留着 + m.note(&rekeyed, "m", Effort::XHigh, EffortFix::Drop, now); + m.reconcile(&config(vec![rekeyed.clone()])); + assert!(m.has("a", "m")); + } + + #[test] + fn what_is_remembered_survives_a_restart_in_the_data_directory() { + let dir = tempfile::tempdir().unwrap(); + let p = provider("relay", "sk-1"); + let cfg = config(vec![p.clone()]); + let now = now_secs(); + let m = Arc::new(EffortMemory::default()); + m.set_dir(dir.path().to_path_buf(), &cfg); + m.note( + &p, + "claude-opus-4-6", + Effort::XHigh, + EffortFix::Use(Effort::Max), + now, + ); + m.note(&p, "glm-5", Effort::Medium, EffortFix::Drop, now); + m.save_now(); + let text = std::fs::read_to_string(dir.path().join(FILE)).unwrap(); + assert!(text.contains("\"rejected\": \"xhigh\""), "{text}"); + assert!(text.contains("\"use\": \"max\""), "{text}"); + assert!(!text.contains("sk-1"), "不存密钥：{text}"); + + let again = Arc::new(EffortMemory::default()); + again.set_dir(dir.path().to_path_buf(), &cfg); + assert_eq!( + again.lookup(&p, "claude-opus-4-6", Effort::XHigh, now), + Some(EffortFix::Use(Effort::Max)) + ); + assert_eq!( + again.lookup(&p, "glm-5", Effort::Medium, now), + Some(EffortFix::Drop) + ); + // 读进来时这家已经换了密钥：作废，文件跟着改 + let again = Arc::new(EffortMemory::default()); + again.set_dir( + dir.path().to_path_buf(), + &config(vec![provider("relay", "sk-2")]), + ); + assert!(!again.has("relay", "glm-5")); + again.save_now(); + let text = std::fs::read_to_string(dir.path().join(FILE)).unwrap(); + assert!(!text.contains("glm-5"), "{text}"); + // 读不成的文件当没有 + std::fs::write(dir.path().join(FILE), "not json").unwrap(); + let broken = Arc::new(EffortMemory::default()); + broken.set_dir(dir.path().to_path_buf(), &cfg); + assert!(!broken.has("relay", "glm-5")); + } +} diff --git a/crates/tw-gateway/src/forward.rs b/crates/tw-gateway/src/forward.rs index 8337f6d4..e65380bd 100644 --- a/crates/tw-gateway/src/forward.rs +++ b/crates/tw-gateway/src/forward.rs @@ -97,12 +97,22 @@ pub fn apply_set( } let mut v = before.clone(); set_params(&mut v, set, client); - match crate::splice::rewrite(body, &before, &v) { + rewritten(body, &before, &v) +} + +/// 请求体 `body`（解出来是 `before`）改成了 `after`：只动改了的那几个字段，别的字节一个不动 +/// （见 [`crate::splice`]）。一样的原样交回 +pub(crate) fn rewritten( + body: &Bytes, + before: &serde_json::Value, + after: &serde_json::Value, +) -> Bytes { + match crate::splice::rewrite(body, before, after) { crate::splice::Rewrite::Changed(b) => return Bytes::from(b), crate::splice::Rewrite::Same => return body.clone(), crate::splice::Rewrite::Unusual => {} } - match serde_json::to_vec(&v) { + match serde_json::to_vec(after) { Ok(b) => Bytes::from(b), // 序列化不该失败，但真失败了宁可发原文也不要发半个 body Err(e) => { diff --git a/crates/tw-gateway/src/lib.rs b/crates/tw-gateway/src/lib.rs index 3ddbb9d4..2a766094 100644 --- a/crates/tw-gateway/src/lib.rs +++ b/crates/tw-gateway/src/lib.rs @@ -20,6 +20,7 @@ pub mod client_api; pub mod clientprobe; pub mod content; pub mod count; +pub mod effort_memory; pub mod egress; pub mod ending; pub mod error; diff --git a/crates/tw-gateway/src/server/pipeline/hop.rs b/crates/tw-gateway/src/server/pipeline/hop.rs index 5e40ecf1..10245d91 100644 --- a/crates/tw-gateway/src/server/pipeline/hop.rs +++ b/crates/tw-gateway/src/server/pipeline/hop.rs @@ -24,6 +24,8 @@ use crate::failure::{Cause, Verdict}; use crate::forward; use crate::server::{hop, hop_failed, note_health}; use crate::state::{AppState, Runtime}; +use tw_dialect::ir::Effort; +use tw_dialect::think::EffortFix; use tw_types::msg; /// 接下了这个请求的那一家，和回程要用的东西。 @@ -199,9 +201,17 @@ pub(super) async fn try_upstreams<'a>( // 插件写进来的、这个请求已经报过的命中：每一跳再查一遍，同样的只报一次（见 // `crate::guard::Reported`） let mut reported = crate::guard::Reported::default(); + // 推理强度被上游拒了（见 `crate::effort_memory`）：同一家改一档再发的那一跳要用的 —— 发给 + // 哪一家、被拒那一跳本来要发的那一档、改成什么。**是尝试链上新的一跳**，不算退避重试 + let mut effort_resend: Option<(&String, Effort, EffortFix)> = None; + // 为推理强度重发过的那几家：一家只重发一次 + let mut effort_resent: Vec<&String> = Vec::new(); + // 这一跳发出去的推理强度和客户端要的不一样：那一跳在尝试链上的位置和那一项 + let mut effort_note: Option<(usize, tw_api::EffortChange)> = None; loop { stamp_queued(&mut chain, queued.take()); + stamp_effort(&mut chain, effort_note.take()); // 候选都试过了，最后一跳在内容之前暂时出错：等待期限之内退避一阵，把暂时出错的那几家再 // 试一轮。不重试的，交出去的是最后那个错误 if queue.is_empty() && waiting.is_empty() { @@ -591,10 +601,40 @@ pub(super) async fn try_upstreams<'a>( let model = Some(plugged.model.clone().unwrap_or(sent)).filter(asked_other); let asked = Asked::of(req, reading, &plugged); - let out = match super::heavy(asked.body, || { - prepare(state, req, reading, &asked, provider, &effective_set, id) + // 发给这一家的模型名：推理强度按（上游，它）记（见 `crate::effort_memory`） + let sent_model = effective_set + .model + .clone() + .unwrap_or_else(|| reading.facts.model.clone()); + // 这一跳是推理强度被拒之后的重发：用定好的那一档 + let carried = match effort_resend { + Some((n, pre, fix)) if n == name => { + effort_resend = None; + Some((pre, fix)) + } + _ => None, + }; + // `hop_effort`：这一跳改强度之前要发的那一档（见 `adjust_effort`），没算的是 None + let (out, hop_effort) = match super::heavy(asked.body, || { + prepare(state, req, reading, &asked, provider, &effective_set, id).map(|mut out| { + let effort = adjust_effort( + state, + req, + reading, + &asked, + provider, + &sent_model, + &effective_set, + &mut out, + carried.map(|(_, fix)| fix), + ); + (out, effort) + }) }) { - Ok(out) => out, + Ok((out, (pre, change))) => { + effort_note = change.map(|c| (chain.len(), c)); + (out, pre) + } Err(err) => { chain.push(hop_failed( &provider.name, @@ -705,11 +745,8 @@ pub(super) async fn try_upstreams<'a>( .rewritten .as_ref() .map(|r| (r.body.clone(), ledger.clone())); - // 这一跳接下了的话，回答钩子要的 - let sent_model = effective_set - .model - .clone() - .unwrap_or_else(|| reading.facts.model.clone()); + // 这一跳接下了的话，回答钩子要的：插件改了名的是改过的 + let sent_model = effective_set.model.clone().unwrap_or(sent_model); let (attempt, bridge) = (chain.len(), plugged.bridge); // 这一跳在报文记录里记到哪儿（见 `crate::content`）：尝试链上的第几跳，从 1 数 let tap = crate::content::Tap::new( @@ -865,6 +902,40 @@ pub(super) async fn try_upstreams<'a>( // **最后一家的 4xx 交给客户端**：没有下一家可换了，上游自己的原话 // （「invalid x-api-key」）比我们转述的一句「回了 401」有用得多。 // 5xx 和 429 照旧报我们的话，带着尝试链 + // 上游因为推理强度拒了它（见 `crate::effort_memory`）：这一跳是一跳普通的 400，同一家 + // 改一档（或者去掉强度）再发一次，是尝试链上新的一跳。**一家只重发一次**，再被拒 + // 照常交给客户端。上游好好的，不算失败 + if status == 400 + && reading.generates + && !counting + && !effort_resent.contains(&name) + && let Some((pre, fix)) = effort_refused(req, &out, hop_effort, &head) + { + tap.said(&head); + state.note_proxy_ok(&provider.proxy); + note_health( + &state.bus, + &state.health, + &provider.name, + state.health.record_success(&provider.name), + ); + tracing::info!( + provider = %provider.name, + model = %sent_model, + "the upstream refused the reasoning effort; sending again with another one" + ); + chain.push(hop( + &provider.name, + model.clone(), + tw_api::AttemptOutcome::Status, + status, + hop_started, + )); + effort_resent.push(name); + effort_resend = Some((name, pre, fix)); + queue.push_front(name); + continue; + } let hand_on = last && status < 500 && status != 429; match verdict { // 限流（额度用完不算）：**不是失败**。这一家暂停（见 `crate::throttle`），这个请求 @@ -1023,11 +1094,15 @@ pub(super) async fn try_upstreams<'a>( }); // 不压开头的最后一家：它答没答上（成败的账）要等回程看到第一段内容才知道 let mut unconfirmed = last && hold.is_some() && !held; + // 这一家答上了：响应头是 2xx、不压开头的，或者压着开头等到了内容。推理强度改了一档 + // 重发的，这时才记下这个模型不认原来那一档（见 `crate::effort_memory`） + let mut answered = false; let r = match hold.filter(|_| held) { None => { // 不压开头：响应头就是它的答复，这一家没在限流（探路的过了，连续限流的次数 // 归零） state.throttles.answered(&provider.name, sent_at); + answered = true; r } Some(hold) => { @@ -1064,6 +1139,7 @@ pub(super) async fn try_upstreams<'a>( // 上游，响应头不说明它好了 Some(super::opening::Opening::Go(r)) => { state.throttles.answered(&provider.name, sent_at); + answered = true; r } // 到点了还没有内容：这一跳没有结论 @@ -1225,6 +1301,15 @@ pub(super) async fn try_upstreams<'a>( state.health.record_success(&provider.name), ); } + if answered && let Some((pre, fix)) = carried { + state.effort_memory.note( + provider, + &sent_model, + pre, + fix, + crate::effort_memory::now_secs(), + ); + } chain.push(hop( &provider.name, model.clone(), @@ -1301,6 +1386,7 @@ pub(super) async fn try_upstreams<'a>( } stamp_queued(&mut chain, queued.take()); + stamp_effort(&mut chain, effort_note.take()); stamp_retries(&mut chain, &retries.rounds); // 尝试链走完了，两条路都要发 —— 挂在 RequestFinished 上的话， // 失败那条路就没有尝试链，而那恰恰是最需要看它的时候。 @@ -1455,6 +1541,15 @@ fn stamp_queued(chain: &mut [tw_api::AttemptView], queued: Option<(usize, u64)>) } } +/// 推理强度改过的那一跳：在尝试链上补上改了什么（见 [`tw_api::AttemptView::effort`]） +fn stamp_effort(chain: &mut [tw_api::AttemptView], note: Option<(usize, tw_api::EffortChange)>) { + if let Some((at, change)) = note + && let Some(a) = chain.get_mut(at) + { + a.effort = Some(change); + } +} + /// 发出去了的一跳在这一家用量上限上占着的（见 [`crate::upstream_limits`]）。**丢掉就交给请求号**， /// 这一跳算一个请求、等结算：不管它怎么收场，上游都收到了它。回了 429 的、其实没发出去的调 /// [`Spent::release`] 还回去 @@ -2166,6 +2261,107 @@ fn prepare( }) } +/// 这一跳的推理强度（见 [`crate::effort_memory`]）：规则改过之后（`out` 是按规则改好、要发出去 +/// 的那一份），这一家的这个模型记着不认这一档的，或者这一跳是被拒之后的重发（`carried`），在 +/// 发出去的这一份上改掉。交回改之前要发的那一档，和尝试链上要记的那一项（发出去的和客户端要的 +/// 不一样、又是规则或者这里改的才记）。 +/// +/// **用不着时不解请求体**：没有规则改推理、这一家的这个模型什么都没记着、也不是重发的，什么都 +/// 不做 +#[allow(clippy::too_many_arguments)] +fn adjust_effort( + state: &AppState, + req: &Inbound, + reading: &crate::client_api::Reading, + asked: &Asked<'_>, + provider: &tw_config::Provider, + model: &str, + set: &tw_engine::SetAction, + out: &mut Outbound, + carried: Option, +) -> (Option, Option) { + use tw_dialect::params; + let rule = set.reasoning.is_some(); + if !reading.generates + || (!rule && carried.is_none() && !state.effort_memory.has(&provider.name, model)) + { + return (None, None); + } + let Some(wire) = wire(req, out) else { + return (None, None); + }; + let Ok(before) = serde_json::from_slice::(&out.body) else { + return (None, None); + }; + let pre = params::reasoning_effort(wire, &before); + let mut reason = rule.then_some(tw_api::EffortReason::Rule); + let mut after = before.clone(); + if let Some(p) = pre { + let fix = match carried { + Some(f) => Some((f, tw_api::EffortReason::Rejected)), + None => state + .effort_memory + .lookup(provider, model, p, crate::effort_memory::now_secs()) + .map(|f| (f, tw_api::EffortReason::Remembered)), + }; + if let Some((f, why)) = fix + && apply_fix(wire, &mut after, f) + { + out.body = forward::rewritten(&out.body, &before, &after); + // 去掉、改掉了上游不认的值：发出去的不再是客户端那一份（报文记录另存它） + out.reshaped = true; + reason = Some(why); + } + } + let sent = params::reasoning_effort(wire, &after); + let wanted = match asked.decoded { + Some(Ok(d)) => d + .request + .reasoning + .as_ref() + .filter(|r| r.enabled) + .and_then(tw_dialect::think::effort), + _ => req.api.and_then(|a| { + let v = serde_json::from_slice::(asked.body).ok()?; + params::reasoning_effort(a.dialect(), &v) + }), + }; + let change = match (reason, wanted) { + (Some(reason), Some(w)) if Some(w) != sent => Some(tw_api::EffortChange { + asked: tw_dialect::think::openai(w).to_string(), + sent: sent.map(|e| tw_dialect::think::openai(e).to_string()), + reason, + }), + _ => None, + }; + (pre, change) +} + +/// 照 `fix` 改请求体 `v`（`wire` 格式）。返回改没改 +fn apply_fix(wire: tw_dialect::ir::Dialect, v: &mut serde_json::Value, fix: EffortFix) -> bool { + match fix { + EffortFix::Use(e) => tw_dialect::params::set_reasoning_effort(wire, v, e), + EffortFix::Drop => tw_dialect::params::remove_reasoning_effort(wire, v), + } +} + +/// 上游回的这个 400 是不是因为推理强度（[`tw_dialect::think::effort_rejection`]）：是的话，交回 +/// 记的时候按哪一档记（这一跳改强度之前要发的那一档，`pre`；没算过的就是发出去的那一档）和 +/// 重发时怎么改。照着改了也和发出去的一样的（按预算思考的去不掉强度）不算：重发一遍还是被拒 +fn effort_refused( + req: &Inbound, + out: &Outbound, + pre: Option, + said: &[u8], +) -> Option<(Effort, EffortFix)> { + let wire = wire(req, out)?; + let v = serde_json::from_slice::(&out.body).ok()?; + let sent = tw_dialect::params::reasoning_effort(wire, &v)?; + let fix = tw_dialect::think::effort_rejection(400, &String::from_utf8_lossy(said), sent)?; + let mut tried = v; + apply_fix(wire, &mut tried, fix).then_some((pre.unwrap_or(sent), fix)) +} + /// 这一跳发出去的那一份是什么格式：转换过的是目标格式，直通的是客户端的 fn wire(req: &Inbound, out: &Outbound) -> Option { out.target.or(req.api.map(|a| a.dialect())) diff --git a/crates/tw-gateway/src/state.rs b/crates/tw-gateway/src/state.rs index 4e2009b7..d49a7a2d 100644 --- a/crates/tw-gateway/src/state.rs +++ b/crates/tw-gateway/src/state.rs @@ -262,6 +262,9 @@ pub struct AppState { /// 哪些（上游，模型）拒过转换时自动标的提示缓存断点（见 [`crate::cache_marks`]）。 /// **跨重载存活** pub cache_marks: Arc, + /// 哪些（上游，模型）不认哪一档推理强度、改用什么（见 [`crate::effort_memory`]）。 + /// **跨重载存活**，存在数据目录里 + pub effort_memory: Arc, /// 脚本插件里跨重载存活的那一半：运行时、插件文件在哪儿、计数和日志、编译缓存 /// （见 [`crate::plugin::Plugins`]）。跟着配置换的那一半在 `Runtime::plugins` pub plugins: Arc, @@ -367,6 +370,7 @@ impl AppState { balance: Default::default(), seals: Default::default(), cache_marks: Default::default(), + effort_memory: Default::default(), plugins, swap: Default::default(), ping_every: crate::PING_EVERY, @@ -608,6 +612,8 @@ impl AppState { self.key_limits.configure(&next.config); self.upstream_limits.configure(&next.config); self.throttles.configure(&next.config); + // 删掉的、地址密钥协议变了的上游，记着的推理强度作废 + self.effort_memory.reconcile(&next.config); self.rt.store(Arc::new(next)); self.announce_broken(broken); // 模型汇总马上按新配置重算：删掉、停用的上游的模型必须立刻消失（列表 @@ -644,6 +650,8 @@ impl AppState { /// 告诉网关配置文件在哪个目录：**插件文件的路径相对它**。控制面拿到配置文件的 /// 路径时调；目录变了就把插件重读一遍 pub fn set_config_dir(&self, dir: std::path::PathBuf) { + // 记着的推理强度也存在这个目录里（见 `crate::effort_memory`） + self.effort_memory.set_dir(dir.clone(), &self.config()); if self.plugins.set_dir(dir) { self.reload_plugins(); } diff --git a/crates/tw-gateway/tests/effort.rs b/crates/tw-gateway/tests/effort.rs new file mode 100644 index 00000000..3903ca6f --- /dev/null +++ b/crates/tw-gateway/tests/effort.rs @@ -0,0 +1,418 @@ +//! 推理强度，端到端：规则改写（`set.reasoning`），上游因为推理强度回 400 时同一家改一档再发 +//! 一次、记下来（见 `tw_gateway::effort_memory`）。 + +use std::net::SocketAddr; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use axum::Router; +use axum::extract::State; +use serde_json::{Value, json}; + +/// 一个假上游收到的请求体 +type Seen = Arc>>; + +/// 一个假上游：按到达的顺序依次用 `replies` 回（用完了一直用最后一个），记下收到的请求体 +async fn upstream(replies: Vec<(u16, Value)>) -> (SocketAddr, Seen) { + let seen: Seen = Default::default(); + let app = Router::new() + .fallback(move |State(s): State, body: bytes::Bytes| { + let replies = replies.clone(); + async move { + let mut s = s.lock().unwrap(); + let n = s.len(); + s.push(serde_json::from_slice(&body).unwrap_or(Value::Null)); + let (status, reply) = replies[n.min(replies.len() - 1)].clone(); + axum::response::Response::builder() + .status(status) + .header("content-type", "application/json") + .body(axum::body::Body::from(reply.to_string())) + .unwrap() + } + }) + .with_state(seen.clone()); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(l, app).await.unwrap() }); + (addr, seen) +} + +fn config(up: SocketAddr, protocol: &str, rules: &str) -> tw_config::Config { + let routes = if rules.is_empty() { + String::new() + } else { + format!("routes:\n - name: default\n rules:\n{rules}") + }; + serde_yaml_ng::from_str(&format!( + "version: 1 +clients: + - name: c + key: tw-k +providers: + - name: relay + base_url: http://{up} + key: sk-upstream + protocol: {protocol} +{routes}" + )) + .unwrap() +} + +struct Gw { + addr: SocketAddr, + state: tw_gateway::AppState, + rx: tokio::sync::broadcast::Receiver, +} + +async fn gateway(cfg: tw_config::Config, dir: Option<&std::path::Path>) -> Gw { + let state = tw_gateway::AppState::new(cfg).unwrap(); + if let Some(d) = dir { + state.set_config_dir(d.to_path_buf()); + } + let rx = state.bus.subscribe(); + let addr = tw_gateway::serve(state.clone(), ([127, 0, 0, 1], 0).into()) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + Gw { addr, state, rx } +} + +async fn post(gw: &Gw, path: &str, body: Value) -> (u16, String) { + let resp = reqwest::Client::new() + .post(format!("http://{}{path}", gw.addr)) + .header("content-type", "application/json") + .header("x-api-key", "tw-k") + .header("authorization", "Bearer tw-k") + .body(body.to_string()) + .send() + .await + .unwrap(); + let status = resp.status().as_u16(); + (status, resp.text().await.unwrap()) +} + +/// 这个请求的尝试链 +async fn attempts( + rx: &mut tokio::sync::broadcast::Receiver, +) -> Vec { + loop { + let e = tokio::time::timeout(Duration::from_secs(5), rx.recv()) + .await + .expect("no routing event") + .unwrap(); + if let tw_api::Event::RequestRouted { attempts, .. } = e { + return attempts; + } + } +} + +fn anthropic_ok() -> Value { + json!({"id": "msg_1", "type": "message", "role": "assistant", "model": "claude-opus-4-6", + "content": [{"type": "text", "text": "好"}], "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 1}}) +} + +fn chat_ok() -> Value { + json!({"id": "c1", "object": "chat.completion", "model": "glm-5", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "好"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}) +} + +fn anthropic_error(message: &str) -> Value { + json!({"type": "error", "error": {"type": "invalid_request_error", "message": message}}) +} + +/// 自适应思考、写着 `effort` 的 Claude 请求 +fn claude(effort: &str) -> Value { + json!({ + "model": "claude-opus-4-6", "max_tokens": 2000, + "thinking": {"type": "adaptive"}, "output_config": {"effort": effort}, + "messages": [{"role": "user", "content": "hi"}], + }) +} + +fn effort_of(v: &Value) -> Option<&str> { + v.pointer("/output_config/effort").and_then(Value::as_str) +} + +fn change( + asked: &str, + sent: Option<&str>, + reason: tw_api::EffortReason, +) -> Option { + Some(tw_api::EffortChange { + asked: asked.into(), + sent: sent.map(str::to_string), + reason, + }) +} + +const NOT_VALID: &str = "The parameter `output_config.effort` specified in the request are not valid: expected `low`, `medium`, `high` or `max`, but got `xhigh` instead."; + +/// 上游不认 `xhigh`、列出了认的那几档：同一家改用最近的 `max` 再发，是尝试链上新的一跳；记下来， +/// 下一个请求直接发 `max`。存在数据目录里 +#[tokio::test] +async fn a_refused_effort_is_sent_again_once_with_the_nearest_level_and_remembered() { + let dir = tempfile::tempdir().unwrap(); + let (up, seen) = upstream(vec![ + (400, anthropic_error(NOT_VALID)), + (200, anthropic_ok()), + ]) + .await; + let mut gw = gateway(config(up, "anthropic", ""), Some(dir.path())).await; + let (status, body) = post(&gw, "/v1/messages", claude("xhigh")).await; + assert_eq!(status, 200, "{body}"); + { + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), 2); + assert_eq!(effort_of(&seen[0]), Some("xhigh")); + assert_eq!(effort_of(&seen[1]), Some("max")); + // 别的照旧 + assert_eq!(seen[1]["thinking"], json!({"type": "adaptive"})); + assert_eq!(seen[1]["messages"], seen[0]["messages"]); + } + let chain = attempts(&mut gw.rx).await; + assert_eq!(chain.len(), 2, "{chain:?}"); + assert_eq!(chain[0].outcome, tw_api::AttemptOutcome::Status); + assert_eq!(chain[0].status, Some(400)); + assert_eq!(chain[0].effort, None, "被拒的那一跳发的就是客户端要的"); + assert_eq!(chain[1].outcome, tw_api::AttemptOutcome::Served); + assert_eq!(chain[1].provider, "relay"); + assert_eq!(chain[1].retry, None, "不是退避重试"); + assert_eq!( + chain[1].effort, + change("xhigh", Some("max"), tw_api::EffortReason::Rejected) + ); + + // 记住了：直接发 max，上游只收到一次 + let (status, _) = post(&gw, "/v1/messages", claude("xhigh")).await; + assert_eq!(status, 200); + assert_eq!(seen.lock().unwrap().len(), 3); + assert_eq!(effort_of(&seen.lock().unwrap()[2]), Some("max")); + let chain = attempts(&mut gw.rx).await; + assert_eq!(chain.len(), 1); + assert_eq!( + chain[0].effort, + change("xhigh", Some("max"), tw_api::EffortReason::Remembered) + ); + // 别的档位不受影响 + post(&gw, "/v1/messages", claude("high")).await; + assert_eq!(effort_of(&seen.lock().unwrap()[3]), Some("high")); + assert_eq!(attempts(&mut gw.rx).await[0].effort, None); + + // 存在数据目录里（写文件挪到了阻塞线程上，等它一下） + let file = dir.path().join(tw_gateway::effort_memory::FILE); + let mut text = String::new(); + for _ in 0..50 { + text = std::fs::read_to_string(&file).unwrap_or_default(); + if !text.is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert!( + text.contains("claude-opus-4-6") && text.contains("\"use\": \"max\""), + "{text}" + ); + assert!(!text.contains("sk-upstream"), "{text}"); + + // 换了密钥：已经是另一个上游了，记着的作废，再试原来那一档 + let mut cfg = gw.state.config().as_ref().clone(); + cfg.providers[0].key = Some("sk-other".into()); + gw.state.reload(cfg).unwrap(); + post(&gw, "/v1/messages", claude("xhigh")).await; + assert_eq!(effort_of(&seen.lock().unwrap()[4]), Some("xhigh")); +} + +/// 「high 或以下」是上限：改用 high +#[tokio::test] +async fn a_ceiling_in_the_refusal_is_the_level_sent_again() { + let (up, seen) = upstream(vec![ + ( + 400, + anthropic_error( + "output_config.effort 'xhigh' is not supported when thinking is disabled on this model. Use effort 'high' or below, or enable thinking.", + ), + ), + (200, anthropic_ok()), + ]) + .await; + let mut gw = gateway(config(up, "anthropic", ""), None).await; + let (status, body) = post(&gw, "/v1/messages", claude("xhigh")).await; + assert_eq!(status, 200, "{body}"); + assert_eq!(effort_of(&seen.lock().unwrap()[1]), Some("high")); + let chain = attempts(&mut gw.rx).await; + assert_eq!( + chain[1].effort, + change("xhigh", Some("high"), tw_api::EffortReason::Rejected) + ); +} + +/// 不提 `effort`、只列了几档的（GLM 始终思考的模型，码 1210）：离 medium 最近的 low 和 high +/// 一样近，取 high +#[tokio::test] +async fn levels_listed_without_the_word_effort_are_understood_too() { + let refusal = json!({"error": {"code": "1210", + "message": "该模型始终思考，不支持关闭思考；请使用 low、high 或 max。"}}); + let (up, seen) = upstream(vec![(400, refusal), (200, chat_ok())]).await; + let mut gw = gateway(config(up, "openai-chat", ""), None).await; + let (status, body) = post( + &gw, + "/v1/chat/completions", + json!({"model": "glm-5", "reasoning_effort": "medium", + "messages": [{"role": "user", "content": "hi"}]}), + ) + .await; + assert_eq!(status, 200, "{body}"); + { + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), 2); + assert_eq!(seen[1]["reasoning_effort"], "high"); + } + let chain = attempts(&mut gw.rx).await; + assert_eq!( + chain[1].effort, + change("medium", Some("high"), tw_api::EffortReason::Rejected) + ); +} + +/// 与推理强度无关的 400：原样交给客户端，不重发 +#[tokio::test] +async fn another_400_is_not_sent_again() { + let reply = anthropic_error("messages.0.content: field required"); + let (up, seen) = upstream(vec![(400, reply.clone()), (200, anthropic_ok())]).await; + let mut gw = gateway(config(up, "anthropic", ""), None).await; + let (status, body) = post(&gw, "/v1/messages", claude("xhigh")).await; + assert_eq!(status, 400); + assert_eq!(serde_json::from_str::(&body).unwrap(), reply); + assert_eq!(seen.lock().unwrap().len(), 1); + let chain = attempts(&mut gw.rx).await; + assert_eq!(chain.len(), 1); + assert_eq!(chain[0].outcome, tw_api::AttemptOutcome::Served); +} + +/// 原话里碰巧有发出去的那一档的名字（「max context length」）：照规矩重发一次，再被拒就把那个 +/// 错误交给客户端，**而且不记**：下一个请求照样发 max +#[tokio::test] +async fn a_resend_refused_again_is_handed_on_and_not_remembered() { + let reply = anthropic_error("prompt is too long: max context length exceeded"); + let (up, seen) = upstream(vec![(400, reply.clone())]).await; + let mut gw = gateway(config(up, "anthropic", ""), None).await; + let (status, body) = post(&gw, "/v1/messages", claude("max")).await; + assert_eq!(status, 400); + assert_eq!(serde_json::from_str::(&body).unwrap(), reply); + { + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), 2, "只重发一次"); + assert_eq!(effort_of(&seen[0]), Some("max")); + assert_eq!(effort_of(&seen[1]), None, "说不出用哪一档：去掉强度"); + } + let chain = attempts(&mut gw.rx).await; + assert_eq!(chain.len(), 2); + assert_eq!(chain[1].status, Some(400)); + assert_eq!( + chain[1].effort, + change("max", None, tw_api::EffortReason::Rejected) + ); + assert!(!gw.state.effort_memory.has("relay", "claude-opus-4-6")); + post(&gw, "/v1/messages", claude("max")).await; + let seen = seen.lock().unwrap(); + assert_eq!(effort_of(&seen[2]), Some("max"), "没有记住"); +} + +/// 规则改写推理：改一档只改开着推理的请求，`off` 连强度一起去掉，没开推理的原样不动 +#[tokio::test] +async fn a_rule_rewrites_reasoning_only_where_it_is_on() { + let ok = anthropic_ok(); + let (up, seen) = upstream(vec![(200, ok)]).await; + let rules = + " - { name: 想多点, when: { model: claude-opus-4-6 }, set: { reasoning: xhigh } } + - { name: 别想, when: { model: claude-haiku-4-5 }, set: { reasoning: off } } + - { name: 兜底, to: relay } +"; + let mut gw = gateway(config(up, "anthropic", rules), None).await; + post(&gw, "/v1/messages", claude("high")).await; + assert_eq!(effort_of(&seen.lock().unwrap()[0]), Some("xhigh")); + assert_eq!( + attempts(&mut gw.rx).await[0].effort, + change("high", Some("xhigh"), tw_api::EffortReason::Rule) + ); + // 没开推理的（生成标题这类）：一个字节都不动 + let plain = json!({"model": "claude-opus-4-6", "max_tokens": 50, + "messages": [{"role": "user", "content": "起个标题"}]}); + post(&gw, "/v1/messages", plain.clone()).await; + assert_eq!(seen.lock().unwrap()[1], plain); + assert_eq!(attempts(&mut gw.rx).await[0].effort, None); + // off：开关和强度一起去掉 + let mut haiku = claude("high"); + haiku["model"] = json!("claude-haiku-4-5"); + post(&gw, "/v1/messages", haiku).await; + { + let seen = seen.lock().unwrap(); + assert!(seen[2].get("thinking").is_none(), "{}", seen[2]); + assert!(seen[2].get("output_config").is_none(), "{}", seen[2]); + } + assert_eq!( + attempts(&mut gw.rx).await[0].effort, + change("high", None, tw_api::EffortReason::Rule) + ); +} + +/// 规则定的那一档上游不认：照样改一档重发（自动兼容在规则之后） +#[tokio::test] +async fn a_level_set_by_a_rule_is_fixed_like_any_other() { + let (up, seen) = upstream(vec![ + (400, anthropic_error(NOT_VALID)), + (200, anthropic_ok()), + ]) + .await; + let rules = " - { name: 想多点, set: { reasoning: xhigh } } + - { name: 兜底, to: relay } +"; + let mut gw = gateway(config(up, "anthropic", rules), None).await; + let (status, _) = post(&gw, "/v1/messages", claude("low")).await; + assert_eq!(status, 200); + { + let seen = seen.lock().unwrap(); + assert_eq!(effort_of(&seen[0]), Some("xhigh")); + assert_eq!(effort_of(&seen[1]), Some("max")); + } + let chain = attempts(&mut gw.rx).await; + assert_eq!( + chain[0].effort, + change("low", Some("xhigh"), tw_api::EffortReason::Rule) + ); + assert_eq!( + chain[1].effort, + change("low", Some("max"), tw_api::EffortReason::Rejected) + ); +} + +/// 转换过格式的请求一样：改的是转成上游格式之后的那一份 +#[tokio::test] +async fn a_converted_request_is_fixed_in_the_upstream_format() { + let (up, seen) = upstream(vec![ + (400, anthropic_error(NOT_VALID)), + (200, anthropic_ok()), + ]) + .await; + let mut gw = gateway(config(up, "anthropic", ""), None).await; + let (status, body) = post( + &gw, + "/v1/chat/completions", + json!({"model": "claude-opus-4-6", "reasoning_effort": "xhigh", + "messages": [{"role": "user", "content": "hi"}]}), + ) + .await; + assert_eq!(status, 200, "{body}"); + { + let seen = seen.lock().unwrap(); + assert_eq!(effort_of(&seen[0]), Some("xhigh")); + assert_eq!(effort_of(&seen[1]), Some("max")); + } + let chain = attempts(&mut gw.rx).await; + assert_eq!( + chain[1].effort, + change("xhigh", Some("max"), tw_api::EffortReason::Rejected) + ); +} diff --git a/docs/config.md b/docs/config.md index 91b7b22a..b450f399 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1500,6 +1500,20 @@ read, naming the rule and the value. `set.model` may name an alias; each upstream then receives its own name for it. Answers carry the name the client asked for, as with aliases. +`set.reasoning` writes the effort where the client's format keeps it +(`output_config.effort`, `reasoning_effort`, `reasoning.effort`, Gemini's +`thinkingLevel`, or the thinking budget when the client gave a budget). + +When an upstream answers 400 because of the reasoning effort, whether the +client or a rule chose it, the request is sent once more to the same upstream +with a level the upstream named (the nearest one, or the ceiling it gave), or +without an effort when it named none. The resend is a new attempt in the +request details and does not count as a retry. If it succeeds, the level is +remembered for that upstream and model for 7 days in `effort-memory.json` in +the data directory, so later requests send the working level directly. The +entries of an upstream are forgotten when its address, key or protocol +changes, or when it is removed. + diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index 5a752fe6..7518e693 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -1089,6 +1089,10 @@ routes: `set.model` 可以写别名，每家上游收到各自对它的叫法。和别名一样，回答里的模型名写成客户端请求的名称。 +`set.reasoning` 按客户端的格式写推理强度：`output_config.effort`、`reasoning_effort`、`reasoning.effort`、Gemini 的 `thinkingLevel`；客户端给的是思考预算时改预算。 + +上游因为推理强度回 400 时（无论这一档是客户端写的还是规则改的），请求在同一家上游再发一次：改用上游原话里列出的、离原来那一档最近的一档，原话给了上限的用上限，一档都没列的去掉推理强度。重发在请求详情里是新的一跳，不算重试。重发成功后，这一档按上游和模型记 7 天，存在数据目录的 `effort-memory.json` 里，之后的请求直接发可用的那一档。上游的地址、密钥或协议变了，或者上游被删除，它的记录随之作废。 + From 8066fcc626249a2501517800c71375cf487de45f Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Sun, 25 Oct 2026 11:54:52 +0800 Subject: [PATCH 3/4] Reasoning off also removes the Chat `thinking` switch DeepSeek, GLM and Kimi turn reasoning on in Chat requests with `thinking: {type: "enabled"}`; removing only `reasoning_effort` left reasoning on. `disable_thinking` now removes that switch too. Co-Authored-By: Claude Opus 5.5 --- crates/tw-dialect/src/params.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/crates/tw-dialect/src/params.rs b/crates/tw-dialect/src/params.rs index 50dee2f9..fa014bfe 100644 --- a/crates/tw-dialect/src/params.rs +++ b/crates/tw-dialect/src/params.rs @@ -204,8 +204,17 @@ pub fn disable_thinking(dialect: Dialect, body: &mut Value) { obj.remove("thinking"); remove_output_effort(obj); } + // DeepSeek、GLM、Kimi 的 `thinking.type: enabled` 也是开关：只去掉强度的话推理照样开着 Dialect::Chat => { obj.remove("reasoning_effort"); + if obj + .get("thinking") + .and_then(|t| t.get("type")) + .and_then(Value::as_str) + == Some("enabled") + { + obj.remove("thinking"); + } } Dialect::Responses => { obj.remove("reasoning"); @@ -1088,6 +1097,19 @@ mod tests { assert_eq!(v, json!({"additionalModelRequestFields": {"top_k": 3}})); } + #[test] + fn turning_thinking_off_in_chat_takes_the_glm_switch_too() { + // DeepSeek、GLM、Kimi 的写法：开关在 `thinking`，强度在 `reasoning_effort` + let mut v = json!({"thinking": {"type": "enabled"}, "reasoning_effort": "high", "x": 1}); + disable_thinking(Dialect::Chat, &mut v); + assert_eq!(v, json!({"x": 1})); + let mut v = json!({"thinking": {"type": "enabled"}, "x": 1}); + assert!(reasoning_on(Dialect::Chat, &v)); + disable_thinking(Dialect::Chat, &mut v); + assert_eq!(v, json!({"x": 1})); + assert!(!reasoning_on(Dialect::Chat, &v)); + } + #[test] fn reasoning_is_on_where_the_decoder_says_it_is() { let on = [ From 1376f1bbe5411cae72a195bc08c557d389cd81ff Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:32:20 +0800 Subject: [PATCH 4/4] Reasoning effort fixes: written fields, none for OpenAI off, tighter detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Detection and the resend work on the effort field as written (`params::written_effort`, `replace_reasoning_effort`, `remove_reasoning_effort`), whether or not reasoning is on, so an effort refused with thinking disabled is fixed too. Budgets are not a written effort: budget-only requests are never resent, looked up or remembered. - A refusal must mention effort, reasoning, thinking, 思考 or 推理; verbosity and content-filter 400s listing low/medium/high no longer count. - `off` writes `none` for OpenAI Chat `reasoning_effort` and Responses `reasoning.effort`; an upstream refusing `none` gets the field removed and that is remembered (`think::none_rejection`). - Memory is keyed by upstream, model and the format sent. A remembered fix that is refused is dropped, and so is one whose resend succeeded with the original level. Loading the file no longer reconciles, so safe mode's stand-in configuration cannot wipe it. - The resend keeps the refused hop's slot and probe and goes out at once; the chosen fix and the attempt's effort note are only used up once the hop is actually sent. - `AttemptView.effort.asked` is the level before this change (the client's for `rule`, the refused or replaced one otherwise). - Gemini thinking budgets stay within the model limit (Pro 32768, else 24576) when rewritten and when converted; Bedrock budgets are not rewritten without `inferenceConfig.maxTokens`. - Docs: WebSocket turns are not resent. Co-Authored-By: Claude Opus 5.5 --- crates/tw-api/src/lib.rs | 16 +- crates/tw-config/tests/manual/schema.rs | 4 +- crates/tw-control/src/replay.rs | 2 +- crates/tw-dialect/src/gemini/request.rs | 39 ++- crates/tw-dialect/src/params.rs | 316 +++++++++++++++-- crates/tw-dialect/src/think.rs | 97 +++++- crates/tw-gateway/src/effort_memory.rs | 336 +++++++++++++------ crates/tw-gateway/src/forward.rs | 57 +++- crates/tw-gateway/src/server/pipeline/hop.rs | 199 +++++++---- crates/tw-gateway/src/state.rs | 2 +- crates/tw-gateway/src/ws.rs | 3 +- crates/tw-gateway/tests/alias_ws.rs | 9 +- crates/tw-gateway/tests/effort.rs | 241 ++++++++++++- docs/config.md | 7 +- docs/config.zh-CN.md | 4 +- 15 files changed, 1093 insertions(+), 239 deletions(-) diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs index 4e5cde26..231e0ef2 100644 --- a/crates/tw-api/src/lib.rs +++ b/crates/tw-api/src/lib.rs @@ -2018,9 +2018,9 @@ pub struct AttemptView { /// 这一跳发出去之前退避了多少毫秒：一轮重试的头一跳带着。不算在 `ms` 里。别的没有 #[serde(default, skip_serializing_if = "Option::is_none")] pub backoff_ms: Option, - /// 这一跳发出去的推理强度和客户端要的不一样：规则改写的、上游拒收了改用别的一档重发的、 - /// 记得这个模型不认那一档直接改了的（见 [`EffortChange`]）。一样的没有。被拒收的那一跳 - /// 自己是一跳普通的 `status`（400），重发的那一跳带着 `rejected` + /// 这一跳改了推理强度：规则改写的、上游拒收了改用别的一档重发的、记得这个模型不认那一档直接 + /// 改了的（见 [`EffortChange`]）。改了也和原来一样的没有。被拒收的那一跳自己是一跳普通的 + /// `status`（400），重发的那一跳带着 `rejected` #[serde(default, skip_serializing_if = "Option::is_none")] pub effort: Option, /// 这一跳走的代理：这家上游配置的代理名（跟随系统的是 `system`），直连是 None。没发出去 @@ -2030,13 +2030,17 @@ pub struct AttemptView { pub proxy: Option, } -/// 一跳发出去的推理强度和客户端要的不一样（[`AttemptView::effort`]）。 +/// 一跳改了推理强度（[`AttemptView::effort`]）。 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[cfg_attr(feature = "ts", derive(ts_rs::TS))] pub struct EffortChange { - /// 客户端要的那一档：`minimal` / `low` / `medium` / `high` / `xhigh` / `max` + /// 这一次改之前要发的那一档：`rule` 是客户端要的；`rejected`、`remembered` 是被拒收、被换掉的 + /// 那一档（规则改过之后的）—— 客户端要 high、规则改成 xhigh、被拒后改用 max，这一项是 + /// `xhigh`。词表：`none`（OpenAI 的「关掉推理」）/ `minimal` / `low` / `medium` / `high` / + /// `xhigh` / `max` pub asked: String, - /// 发出去的那一档，同一个词表。null 是去掉了推理强度，按模型的默认 + /// 发出去的那一档，同一个词表（规则 `off` 在 OpenAI 的格式里是 `none`）。null 是去掉了推理 + /// 强度，按模型的默认 pub sent: Option, pub reason: EffortReason, } diff --git a/crates/tw-config/tests/manual/schema.rs b/crates/tw-config/tests/manual/schema.rs index d433d368..fa396ea6 100644 --- a/crates/tw-config/tests/manual/schema.rs +++ b/crates/tw-config/tests/manual/schema.rs @@ -1906,8 +1906,8 @@ pub fn sections() -> Vec
{ Kind::Enum(set_reasonings), Def::Unset, t( - "Reasoning. `off` turns it off and removes the reasoning effort; a level changes the effort. Only requests that already have reasoning on are changed; requests without reasoning (such as title generation) are left as they are.", - "推理。`off` 关闭推理，连同推理强度一起去掉；写一档则改为该档强度。只改写已开启推理的请求，未开启推理的请求（如生成标题）保持不变。", + "Reasoning. `off` turns it off: the reasoning settings and the effort are removed, and in the OpenAI formats the effort is set to `none`. A level changes the effort. Only requests that already have reasoning on are changed; requests without reasoning (such as title generation) are left as they are.", + "推理。`off` 关闭推理：去掉推理设置和推理强度，OpenAI 的格式改为写 `none`。写一档则改为该档强度。只改写已开启推理的请求，未开启推理的请求（如生成标题）保持不变。", ), ), row( diff --git a/crates/tw-control/src/replay.rs b/crates/tw-control/src/replay.rs index bb477123..7e1d39c5 100644 --- a/crates/tw-control/src/replay.rs +++ b/crates/tw-control/src/replay.rs @@ -191,7 +191,7 @@ fn renamed(row: &tw_store::db::RequestRow, raw: Vec, model: &str) -> (bytes: ..Default::default() }; ( - tw_gateway::forward::apply_set(&raw, &set, Some(api.dialect())), + tw_gateway::forward::apply_set(&raw, &set, Some(api.dialect()), model), row.path.clone(), ) } diff --git a/crates/tw-dialect/src/gemini/request.rs b/crates/tw-dialect/src/gemini/request.rs index 80e21ee6..4cbbd559 100644 --- a/crates/tw-dialect/src/gemini/request.rs +++ b/crates/tw-dialect/src/gemini/request.rs @@ -512,8 +512,10 @@ pub fn encode_request(r: &Request, _t: &Target, dropped: &mut Dropped) -> Value t.insert("thinkingLevel".into(), json!(think::gemini_level(e))); } } else { - // -1 是让模型自己决定 - let budget = think::budget(re).map(|b| b as i64).unwrap_or(-1); + // -1 是让模型自己决定。超过这个模型的上限整个请求 400（Claude 那边的预算能到 64000） + let budget = think::budget(re) + .map(|b| b.min(think::gemini_budget_cap(&r.model)) as i64) + .unwrap_or(-1); t.insert("thinkingBudget".into(), json!(budget)); } if re.summary { @@ -842,6 +844,39 @@ mod tests { assert_eq!(dropped, ["thinking"]); } + /// Claude 那边的预算（最多 64000）转过来压在这个模型的上限以内，超过了整个请求 400 + #[test] + fn a_budget_stays_within_the_model_limit() { + let mut r = Request { + model: "gemini-2.5-flash".into(), + reasoning: Some(Reasoning { + enabled: true, + effort: Some(Effort::Max), + budget: None, + summary: false, + }), + ..Default::default() + }; + let (v, _) = encode(&r, Dialect::Anthropic); + assert_eq!( + v["generationConfig"]["thinkingConfig"]["thinkingBudget"], + 24576 + ); + r.model = "gemini-2.5-pro".into(); + r.reasoning.as_mut().unwrap().budget = Some(48000); + let (v, _) = encode(&r, Dialect::Anthropic); + assert_eq!( + v["generationConfig"]["thinkingConfig"]["thinkingBudget"], + 32768 + ); + r.reasoning.as_mut().unwrap().budget = Some(8192); + let (v, _) = encode(&r, Dialect::Anthropic); + assert_eq!( + v["generationConfig"]["thinkingConfig"]["thinkingBudget"], + 8192 + ); + } + #[test] fn snake_case_fields_are_understood_too() { let mut d = Dropped::new(Dialect::Gemini); diff --git a/crates/tw-dialect/src/params.rs b/crates/tw-dialect/src/params.rs index fa014bfe..f94cf1ad 100644 --- a/crates/tw-dialect/src/params.rs +++ b/crates/tw-dialect/src/params.rs @@ -195,6 +195,12 @@ pub fn set_model(dialect: Dialect, body: &mut Value, model: &str) { /// Anthropic 的强度（`output_config.effort`）单独写在思考开关外面，只去掉开关的话自适应的 /// 模型照样按那一档想，要一起去掉；`output_config` 去空了的整个去掉。Converse 的透传口袋里 /// 同理。 +/// +/// **OpenAI 的两种格式写 `none`，不是去掉**：GPT-5 这类模型没写强度时按默认的那一档想，去掉 +/// 等于没关。Chat 的 `reasoning_effort`、Responses 的 `reasoning.effort`（连同对话中途的 +/// `configuration_update`）写成 `none`；Chat 里 DeepSeek、GLM、Kimi 那种 `thinking.type: enabled` +/// 的开关整个去掉，连同强度。上游不认 `none` 的由网关去掉再发（见 +/// [`think::none_rejection`]） pub fn disable_thinking(dialect: Dialect, body: &mut Value) { let Some(obj) = body.as_object_mut() else { return; @@ -206,7 +212,6 @@ pub fn disable_thinking(dialect: Dialect, body: &mut Value) { } // DeepSeek、GLM、Kimi 的 `thinking.type: enabled` 也是开关：只去掉强度的话推理照样开着 Dialect::Chat => { - obj.remove("reasoning_effort"); if obj .get("thinking") .and_then(|t| t.get("type")) @@ -214,10 +219,20 @@ pub fn disable_thinking(dialect: Dialect, body: &mut Value) { == Some("enabled") { obj.remove("thinking"); + obj.remove("reasoning_effort"); + } else if obj.contains_key("reasoning_effort") { + put(obj, "reasoning_effort", "none"); } } Dialect::Responses => { - obj.remove("reasoning"); + if let Some(Value::Object(r)) = obj.get_mut("reasoning") { + put(r, "effort", "none"); + } + for r in responses_updates(obj) { + if r.contains_key("effort") { + put(r, "effort", "none"); + } + } } Dialect::Gemini => { for config in ["generationConfig", "generation_config"] { @@ -268,6 +283,106 @@ fn remove_output_effort(holder: &mut Map) -> bool { // `reasoning.effort`；Gemini 写了等级的是 `thinkingLevel`、写了预算的是 `thinkingBudget`； // Converse 和 Anthropic 一样，在 `additionalModelRequestFields` 里。预算按 // [`think::effort_of_budget`] 读成一档、按 [`think::budget_of_effort`] 写回去。 +// +// **写明的强度**（[`written_effort`]）只算强度字段本身，不管推理开没开、不算预算：上游因为强度 +// 拒收时认的是发出去的那个字段 —— Anthropic「关着思考时不认 `xhigh`」那种 400，`thinking` 是 +// `disabled`、强度照样写着。改它、去掉它（[`replace_reasoning_effort`]、 +// [`remove_reasoning_effort`]）也按字段在不在，用来照上游的话重发。路由规则改推理按「推理开着」 +// 走（[`set_reasoning_effort`]、[`disable_thinking`]）。 + +/// 请求体里写明的推理强度（[`written_effort`]）：一档，或者 OpenAI 的 `none` +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum WrittenEffort { + /// `none`：OpenAI 的「关掉推理」 + Off, + Level(Effort), +} + +impl WrittenEffort { + /// 写在请求体里的那个词，按 OpenAI 的写法：`none`、`minimal` … `max` + pub fn slug(self) -> &'static str { + match self { + Self::Off => "none", + Self::Level(e) => think::openai(e), + } + } + + /// 反过来。不认识的词是 `None` + pub fn parse(s: &str) -> Option { + think::parse_openai(s).map(|e| e.map_or(Self::Off, Self::Level)) + } +} + +/// 请求体里写明的推理强度：Anthropic 的 `output_config.effort`、Chat 的 `reasoning_effort`、 +/// Responses 的 `reasoning.effort`（对话中途的 `configuration_update` 最后一个说了算）、Gemini 的 +/// `thinkingLevel`、Converse 透传口袋里的 `output_config.effort`。**按字段在不在**，不看推理开没 +/// 开；只写了预算的不算。认不出的值是 `None` +pub fn written_effort(dialect: Dialect, body: &Value) -> Option { + let obj = body.as_object()?; + match dialect { + Dialect::Anthropic => output_effort(obj).map(WrittenEffort::Level), + Dialect::Chat => str_at(obj, "reasoning_effort").and_then(WrittenEffort::parse), + Dialect::Responses => responses_update(obj) + .or_else(|| obj.get("reasoning").and_then(|r| r.get("effort"))?.as_str()) + .and_then(WrittenEffort::parse), + Dialect::Gemini => gemini_thinking(obj) + .and_then(|t| either_ref(t, "thinkingLevel", "thinking_level")) + .and_then(Value::as_str) + .and_then(think::parse_gemini_level) + .map(WrittenEffort::Level), + Dialect::Bedrock => obj + .get("additionalModelRequestFields") + .and_then(Value::as_object) + .and_then(output_effort) + .map(WrittenEffort::Level), + } +} + +/// 把写明的推理强度（[`written_effort`]）改成 `e`：**只改已经写着的那个字段**，不看推理开没开， +/// 不补字段、不碰预算。返回改没改 +pub fn replace_reasoning_effort(dialect: Dialect, body: &mut Value, e: Effort) -> bool { + let Some(obj) = body.as_object_mut() else { + return false; + }; + let in_output = |holder: &mut Map| match holder + .get_mut("output_config") + .and_then(Value::as_object_mut) + { + Some(o) if o.contains_key("effort") => put(o, "effort", think::anthropic(e)), + _ => false, + }; + match dialect { + Dialect::Anthropic => in_output(obj), + Dialect::Chat => { + obj.contains_key("reasoning_effort") && put(obj, "reasoning_effort", think::openai(e)) + } + Dialect::Responses => { + let mut changed = match obj.get_mut("reasoning") { + Some(Value::Object(r)) if r.contains_key("effort") => { + put(r, "effort", think::openai(e)) + } + _ => false, + }; + for r in responses_updates(obj) { + if r.contains_key("effort") { + changed |= put(r, "effort", think::openai(e)); + } + } + changed + } + Dialect::Gemini => match gemini_thinking_mut(obj) { + Some(t) => match key_of(t, "thinkingLevel", "thinking_level") { + Some(k) => put(t, k, think::gemini_level(e)), + None => false, + }, + None => false, + }, + Dialect::Bedrock => obj + .get_mut("additionalModelRequestFields") + .and_then(Value::as_object_mut) + .is_some_and(in_output), + } +} /// 客户端开着推理吗（判据见上） pub fn reasoning_on(dialect: Dialect, body: &Value) -> bool { @@ -345,7 +460,11 @@ pub fn reasoning_effort(dialect: Dialect, body: &Value) -> Option { /// 压不到 1024 以上的不改）；Gemini 等级、预算都没写的（让模型自己定）不改 —— 不知道这个模型 /// 认哪一个。Anthropic 自适应、没写强度的补上 `output_config.effort`。Responses 对话中途的 /// `configuration_update` 一起改：最后一个盖过顶层的。 -pub fn set_reasoning_effort(dialect: Dialect, body: &mut Value, e: Effort) -> bool { +/// +/// `model` 是发给上游的模型名，Gemini 的预算按它压在上限以内（[`think::gemini_budget_cap`]）： +/// 超过了整个请求 400。Converse 没写最大输出（`inferenceConfig.maxTokens`）的不改预算：不知道 +/// 压到多少才合法 +pub fn set_reasoning_effort(dialect: Dialect, body: &mut Value, e: Effort, model: &str) -> bool { if !reasoning_on(dialect, body) { return false; } @@ -355,7 +474,7 @@ pub fn set_reasoning_effort(dialect: Dialect, body: &mut Value, e: Effort) -> bo match dialect { Dialect::Anthropic => { let max = obj.get("max_tokens").and_then(Value::as_u64); - set_claude_effort(obj, e, max) + set_claude_effort(obj, e, max, false) } Dialect::Chat => put(obj, "reasoning_effort", think::openai(e)), Dialect::Responses => { @@ -384,7 +503,7 @@ pub fn set_reasoning_effort(dialect: Dialect, body: &mut Value, e: Effort) -> bo if let Some(k) = key_of(t, "thinkingLevel", "thinking_level") { put(t, k, think::gemini_level(e)) } else if let Some(k) = key_of(t, "thinkingBudget", "thinking_budget") { - let budget = think::budget_of_effort(e); + let budget = think::budget_of_effort(e).min(think::gemini_budget_cap(model)); let before = t.get(k).cloned(); t.insert(k.into(), Value::from(budget)); before.as_ref() != t.get(k) @@ -401,21 +520,19 @@ pub fn set_reasoning_effort(dialect: Dialect, body: &mut Value, e: Effort) -> bo .get_mut("additionalModelRequestFields") .and_then(Value::as_object_mut) { - Some(f) => set_claude_effort(f, e, max), + Some(f) => set_claude_effort(f, e, max, true), None => false, } } } } -/// 去掉写明的推理强度，按模型的默认想。推理照样开着。返回去没去掉。 +/// 去掉写明的推理强度（[`written_effort`]，`none` 也算），按模型的默认想。**按字段在不在**，不看 +/// 推理开没开；预算不动（按预算思考的 Anthropic 和 Converse 去掉预算整个请求不合法）。返回去没 +/// 去掉。 /// -/// 按预算思考的 Anthropic 和 Converse 不动：预算是开思考必须的，去掉它整个请求不合法。 /// Responses 对话中途的 `configuration_update` 里的也去掉，只剩个空壳的整条去掉。 pub fn remove_reasoning_effort(dialect: Dialect, body: &mut Value) -> bool { - if !reasoning_on(dialect, body) { - return false; - } let Some(obj) = body.as_object_mut() else { return false; }; @@ -457,11 +574,6 @@ pub fn remove_reasoning_effort(dialect: Dialect, body: &mut Value) -> bool { for k in ["thinkingLevel", "thinking_level"] { removed |= t.remove(k).is_some(); } - for k in ["thinkingBudget", "thinking_budget"] { - if t.get(k).and_then(Value::as_i64).is_some_and(|b| b > 0) { - removed |= t.remove(k).is_some(); - } - } removed } Dialect::Bedrock => obj @@ -504,14 +616,22 @@ fn claude_effort(holder: &Map) -> Option { output_effort(holder).or_else(|| claude_budget(holder).map(think::effort_of_budget)) } -/// 见 [`set_reasoning_effort`]。`max` 是最大输出：预算要小于它 -fn set_claude_effort(holder: &mut Map, e: Effort, max: Option) -> bool { +/// 见 [`set_reasoning_effort`]。`max` 是最大输出：预算要小于它。`need_max`：没写最大输出的不改预算 +fn set_claude_effort( + holder: &mut Map, + e: Effort, + max: Option, + need_max: bool, +) -> bool { /// Anthropic 允许的最小思考预算 const BUDGET_MIN: u64 = 1024; let effort_written = holder .get("output_config") .is_some_and(|o| o.get("effort").is_some()); if !effort_written && let Some(before) = claude_budget(holder) { + if need_max && max.is_none() { + return false; + } let budget = max.map_or(think::budget_of_effort(e), |m| { think::budget_of_effort(e).min(m.saturating_sub(1)) }); @@ -1053,11 +1173,6 @@ mod tests { Dialect::Anthropic, json!({"thinking": {"type": "enabled"}, "x": 1}), ), - (Dialect::Chat, json!({"reasoning_effort": "high", "x": 1})), - ( - Dialect::Responses, - json!({"reasoning": {"effort": "high"}, "x": 1}), - ), ( Dialect::Gemini, json!({"generationConfig": {"thinkingConfig": {}}, "x": 1}), @@ -1078,6 +1193,35 @@ mod tests { } } + /// OpenAI 的两种格式写 `none`：去掉的话 GPT-5 这类模型按默认的那一档想，等于没关 + #[test] + fn turning_thinking_off_in_openai_formats_writes_none() { + let mut v = json!({"reasoning_effort": "high", "x": 1}); + disable_thinking(Dialect::Chat, &mut v); + assert_eq!(v, json!({"reasoning_effort": "none", "x": 1})); + assert!(!reasoning_on(Dialect::Chat, &v)); + let mut v = json!({"reasoning": {"effort": "high", "summary": "auto"}, + "input": [{"type": "configuration_update", "reasoning": {"effort": "xhigh"}}]}); + disable_thinking(Dialect::Responses, &mut v); + assert_eq!( + v, + json!({"reasoning": {"effort": "none", "summary": "auto"}, + "input": [{"type": "configuration_update", "reasoning": {"effort": "none"}}]}) + ); + assert!(!reasoning_on(Dialect::Responses, &v)); + assert_eq!( + written_effort(Dialect::Responses, &v), + Some(WrittenEffort::Off) + ); + // 没写强度、只写了摘要的：照样关 + let mut v = json!({"reasoning": {"summary": "auto"}}); + disable_thinking(Dialect::Responses, &mut v); + assert_eq!( + v, + json!({"reasoning": {"summary": "auto", "effort": "none"}}) + ); + } + #[test] fn turning_thinking_off_takes_the_effort_with_it() { // 自适应的 Claude 只去掉开关的话，照样按 `output_config.effort` 那一档想 @@ -1262,7 +1406,7 @@ mod tests { #[test] fn setting_the_effort_writes_the_field_the_client_wrote() { let set = |d, mut v: Value, e| { - let changed = set_reasoning_effort(d, &mut v, e); + let changed = set_reasoning_effort(d, &mut v, e, "gemini-2.5-pro"); (changed, v) }; assert_eq!( @@ -1412,8 +1556,7 @@ mod tests { ), ] { let mut w = v.clone(); - assert!(!set_reasoning_effort(d, &mut w, Effort::Max), "{d:?}"); - assert!(!remove_reasoning_effort(d, &mut w), "{d:?}"); + assert!(!set_reasoning_effort(d, &mut w, Effort::Max, "m"), "{d:?}"); assert_eq!(w, v, "{d:?}"); } } @@ -1498,9 +1641,128 @@ mod tests { Effort::Max, ] { let mut w = v.clone(); - set_reasoning_effort(d, &mut w, e); + set_reasoning_effort(d, &mut w, e, "m"); assert_eq!(reasoning_effort(d, &w), Some(e), "{d:?}: {w}"); } } } + + /// 写明的强度按字段在不在：关着思考也照样写着（上游正是为它拒收的）；只写了预算的不算 + #[test] + fn the_written_effort_is_the_field_itself() { + let disabled = + json!({"thinking": {"type": "disabled"}, "output_config": {"effort": "xhigh"}}); + assert!(!reasoning_on(Dialect::Anthropic, &disabled)); + assert_eq!(reasoning_effort(Dialect::Anthropic, &disabled), None); + assert_eq!( + written_effort(Dialect::Anthropic, &disabled), + Some(WrittenEffort::Level(Effort::XHigh)) + ); + let mut v = disabled.clone(); + assert!(replace_reasoning_effort( + Dialect::Anthropic, + &mut v, + Effort::High + )); + assert_eq!(v["output_config"]["effort"], "high"); + let mut v = disabled.clone(); + assert!(remove_reasoning_effort(Dialect::Anthropic, &mut v)); + assert_eq!(v, json!({"thinking": {"type": "disabled"}})); + let bedrock = json!({"additionalModelRequestFields": { + "thinking": {"type": "disabled"}, "output_config": {"effort": "max"}}}); + assert_eq!( + written_effort(Dialect::Bedrock, &bedrock), + Some(WrittenEffort::Level(Effort::Max)) + ); + let mut v = bedrock.clone(); + assert!(replace_reasoning_effort( + Dialect::Bedrock, + &mut v, + Effort::High + )); + assert_eq!( + v["additionalModelRequestFields"]["output_config"]["effort"], + "high" + ); + // 只写了预算：不是写明的强度，改它、去掉它都不动 + for (d, budget) in [ + ( + Dialect::Anthropic, + json!({"thinking": {"type": "enabled", "budget_tokens": 4000}}), + ), + ( + Dialect::Gemini, + json!({"generationConfig": {"thinkingConfig": {"thinkingBudget": 4000}}}), + ), + ( + Dialect::Bedrock, + json!({"additionalModelRequestFields": {"thinking": {"type": "enabled", "budget_tokens": 4000}}}), + ), + ] { + assert_eq!(written_effort(d, &budget), None, "{d:?}"); + assert!(reasoning_effort(d, &budget).is_some(), "{d:?}"); + let mut v = budget.clone(); + assert!(!replace_reasoning_effort(d, &mut v, Effort::Max), "{d:?}"); + assert!(!remove_reasoning_effort(d, &mut v), "{d:?}"); + assert_eq!(v, budget, "{d:?}"); + } + // `none` 也是写明的 + let none = json!({"reasoning_effort": "none"}); + assert_eq!( + written_effort(Dialect::Chat, &none), + Some(WrittenEffort::Off) + ); + let mut v = none.clone(); + assert!(remove_reasoning_effort(Dialect::Chat, &mut v)); + assert_eq!(v, json!({})); + assert_eq!(WrittenEffort::Off.slug(), "none"); + assert_eq!(WrittenEffort::parse("none"), Some(WrittenEffort::Off)); + assert_eq!( + WrittenEffort::parse("max"), + Some(WrittenEffort::Level(Effort::Max)) + ); + // 改只改写着的字段，不补 + let mut v = json!({"thinking": {"type": "adaptive"}}); + assert!(!replace_reasoning_effort( + Dialect::Anthropic, + &mut v, + Effort::Max + )); + assert_eq!(v, json!({"thinking": {"type": "adaptive"}})); + let mut v = json!({"thinking": {"type": "enabled"}}); + assert!(!replace_reasoning_effort( + Dialect::Chat, + &mut v, + Effort::Max + )); + } + + /// Gemini 2.5 的预算有上限：Pro 32768，Flash 24576。Converse 没写最大输出的不改预算 + #[test] + fn budgets_stay_within_what_the_model_takes() { + let gemini = json!({"generationConfig": {"thinkingConfig": {"thinkingBudget": 1024}}}); + for (model, e, want) in [ + ("gemini-2.5-pro", Effort::Max, 32768), + ("gemini-2.5-flash", Effort::Max, 24576), + ("gemini-2.5-flash-lite", Effort::XHigh, 24576), + ("gemini-2.5-flash", Effort::Medium, 10000), + ] { + let mut v = gemini.clone(); + assert!(set_reasoning_effort(Dialect::Gemini, &mut v, e, model)); + assert_eq!( + v["generationConfig"]["thinkingConfig"]["thinkingBudget"], want, + "{model}" + ); + } + let bedrock = json!({"additionalModelRequestFields": { + "thinking": {"type": "enabled", "budget_tokens": 2000}}}); + let mut v = bedrock.clone(); + assert!(!set_reasoning_effort( + Dialect::Bedrock, + &mut v, + Effort::Max, + "m" + )); + assert_eq!(v, bedrock); + } } diff --git a/crates/tw-dialect/src/think.rs b/crates/tw-dialect/src/think.rs index d3aef9af..67378f53 100644 --- a/crates/tw-dialect/src/think.rs +++ b/crates/tw-dialect/src/think.rs @@ -194,27 +194,24 @@ pub enum EffortFix { /// 上游这个错误是不是因为推理强度：是的话，重发时怎么写。`sent` 是这一跳发出去的那一档。 /// -/// **只认 400**，而且原话（不分大小写）里要说到 `effort`、或者整词说到发出去的那一档、或者 -/// 整词说到另外至少两档 —— 「请使用 low、high 或 max」这样不提 `effort` 的也认得出。 +/// **只认 400**，原话（不分大小写）里要说到推理（`effort`、`reasoning`、`thinking`、「思考」 +/// 「推理」之一），而且说到 `effort`、或者整词说到发出去的那一档、或者整词说到另外至少两档 —— +/// 「该模型始终思考……请使用 low、high 或 max」这样不提 `effort` 的也认得出；`text.verbosity` +/// 的 low / medium / high、内容审查的 `severity` 不提推理，不算。 /// /// 重发写哪一档看原话里的档位（不算发出去的那一档）：恰好一档后面紧跟着「or below」「or /// lower」「以下」的，那一档是上限，就用它；不然取离发出去的那一档最近的（[`nearest`]）； /// 一档都没说的，去掉强度。 pub fn effort_rejection(status: u16, error_body: &str, sent: Effort) -> Option { - if status != 400 { - return None; - } let text = error_body.to_lowercase(); let found = levels_in(&text); + if !about_effort(status, &text, openai(sent), &found) { + return None; + } let others: Vec<(usize, Effort)> = found.iter().copied().filter(|(_, e)| *e != sent).collect(); let mut distinct: Vec = others.iter().map(|(_, e)| *e).collect(); distinct.sort(); distinct.dedup(); - let about_effort = - text.contains("effort") || found.iter().any(|(_, e)| *e == sent) || distinct.len() >= 2; - if !about_effort { - return None; - } let mut ceilings: Vec = others .iter() .filter(|(end, _)| ceiling_after(&text[*end..])) @@ -231,6 +228,42 @@ pub fn effort_rejection(status: u16, error_body: &str, sent: Effort) -> Option bool { + let text = error_body.to_lowercase(); + about_effort(status, &text, "none", &levels_in(&text)) +} + +/// 见 [`effort_rejection`]。`text` 是小写的原话，`sent` 是发出去的那个词，`found` 是原话里的档位 +fn about_effort(status: u16, text: &str, sent: &str, found: &[(usize, Effort)]) -> bool { + if status != 400 + || !["effort", "reasoning", "thinking", "思考", "推理"] + .iter() + .any(|w| text.contains(w)) + { + return false; + } + let mut others: Vec = found + .iter() + .map(|(_, e)| *e) + .filter(|e| openai(*e) != sent) + .collect(); + others.sort(); + others.dedup(); + text.contains("effort") || whole_word(text, sent) || others.len() >= 2 +} + +/// `word` 在小写的 `text` 里整词出现过 +fn whole_word(text: &str, word: &str) -> bool { + let w = |c: char| c.is_ascii_alphanumeric() || c == '_'; + text.match_indices(word).any(|(at, _)| { + let end = at + word.len(); + !text[..at].chars().next_back().is_some_and(w) && !text[end..].chars().next().is_some_and(w) + }) +} + /// 小写的 `text` 里整词出现的档位名：每一处的结尾位置和是哪一档 fn levels_in(text: &str) -> Vec<(usize, Effort)> { let word = |c: char| c.is_ascii_alphanumeric() || c == '_'; @@ -260,6 +293,15 @@ fn ceiling_after(rest: &str) -> bool { .any(|p| rest.starts_with(p)) } +/// Gemini 2.5 的思考预算最多多少：Pro 是 32768，Flash、Flash-Lite 是 24576。超过了整个请求 400 +pub fn gemini_budget_cap(model: &str) -> u64 { + if model.to_ascii_lowercase().contains("pro") { + 32768 + } else { + 24576 + } +} + /// 这个 Gemini 模型用 `thinkingLevel` 而不是 `thinkingBudget` pub fn gemini_uses_level(model: &str) -> bool { let m = model.to_ascii_lowercase(); @@ -364,9 +406,24 @@ mod tests { Some(EffortFix::Drop) ); assert_eq!( - effort_rejection(400, "Unsupported value: 'xhigh'", XHigh), + effort_rejection(400, "Unsupported value: 'xhigh' for reasoning", XHigh), Some(EffortFix::Drop) ); + // 不说到推理的：不算，哪怕列了好几档、说到了发出去的那一档 + assert_eq!( + effort_rejection(400, "Unsupported value: 'xhigh'", XHigh), + None + ); + assert_eq!( + effort_rejection(400, "prompt is too long: max context length exceeded", Max), + None + ); + let verbosity = r#"{"error":{"message":"Unsupported value: 'low' is not supported with the 'gpt-5-codex' model. Supported values are: 'medium' and 'high'.","type":"invalid_request_error","param":"text.verbosity","code":"unsupported_value"}}"#; + assert_eq!(effort_rejection(400, verbosity, Medium), None); + assert_eq!(effort_rejection(400, verbosity, Low), None); + let azure = r#"{"error":{"message":"The response was filtered due to the prompt triggering Azure OpenAI's content management policy.","type":null,"param":"prompt","code":"content_filter","status":400,"innererror":{"code":"ResponsibleAIPolicyViolation","content_filter_result":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"low"},"sexual":{"filtered":true,"severity":"medium"},"violence":{"filtered":false,"severity":"high"}}}}}"#; + assert_eq!(effort_rejection(400, azure, Medium), None); + assert_eq!(effort_rejection(400, azure, XHigh), None); // 与强度无关的 400、不是 400 的：不管 assert_eq!( effort_rejection( @@ -385,6 +442,24 @@ mod tests { assert_eq!(effort_rejection(400, "xhighly unusual", High), None); } + #[test] + fn a_refused_none_is_recognised_like_a_level() { + assert!(none_rejection( + 400, + "Unsupported value: 'none' is not supported with this model for reasoning.effort. Supported values are: 'low', 'medium', and 'high'." + )); + assert!(none_rejection( + 400, + "该模型始终思考，不支持关闭思考；请使用 low、high 或 max。" + )); + assert!(!none_rejection(400, "Unsupported value: 'none'")); + assert!(!none_rejection( + 500, + "reasoning_effort none is not supported" + )); + assert!(!none_rejection(400, "messages: field required")); + } + #[test] fn openai_none_turns_reasoning_off() { assert_eq!(parse_openai("none"), Some(None)); diff --git a/crates/tw-gateway/src/effort_memory.rs b/crates/tw-gateway/src/effort_memory.rs index 4224b185..8a8d9340 100644 --- a/crates/tw-gateway/src/effort_memory.rs +++ b/crates/tw-gateway/src/effort_memory.rs @@ -1,22 +1,27 @@ //! 上游不认的推理强度，记下来。 //! //! 各家认的档位不一样：Claude 的有的模型没有 `xhigh`，关着思考的不收 `high` 以上，GLM 这类 -//! 始终思考的模型不收 `medium`。客户端（或者路由规则）写的那一档被上游拒了（400，原话说的是 -//! 推理强度，见 [`tw_dialect::think::effort_rejection`]），同一家改一档、或者去掉强度再发一次。 -//! **重发成了才记**：（上游，发给它的模型名）不认哪一档、改用什么。往后发给它的请求要是还写着 -//! 那一档，发之前直接改，不用每次都先被拒一次。重发又被拒了的不记：原话里碰巧有一个档位名的 -//! 400（「max context length exceeded」）不该让这个模型此后 7 天都换一档。 +//! 始终思考的模型不收 `medium`，有的 OpenAI 兼容接口不收 `none`。请求体里写明的那一档（见 +//! [`tw_dialect::params::written_effort`]，客户端写的或者路由规则改的）被上游拒了（400，原话说的是 +//! 推理强度，见 [`tw_dialect::think::effort_rejection`]、[`tw_dialect::think::none_rejection`]）， +//! 同一家改一档、或者去掉强度马上再发一次。**重发成了才记**：（上游，发给它的模型名，发出去的 +//! 格式）不认哪一档、改用什么。往后发给它的请求要是还写着那一档，发之前直接改，不用每次都先被 +//! 拒一次。重发又被拒了的不记；记着的改法自己被拒了、重发成了用的正是原来那一档的，那一条删掉。 +//! 只写了预算的请求不查也不记：预算不是写明的强度。 //! //! **7 天后忘掉**：上游可能已经认了，那时再试原来那一档。存在数据目录的 `effort-memory.json` //! 里，重启之后还在；一家上游的地址、密钥、协议变了，或者删掉了，它的这几条随之作废 —— 那已经 //! 是另一个上游了。 +//! +//! WebSocket 连接上的每一轮不重发也不记：一轮被拒了照原样交给客户端。 use std::collections::HashMap; use std::path::PathBuf; use std::sync::{Arc, Mutex}; use serde::{Deserialize, Serialize}; -use tw_dialect::ir::Effort; +use tw_dialect::ir::{Dialect, Effort}; +use tw_dialect::params::WrittenEffort; use tw_dialect::think::{self, EffortFix}; /// 存在数据目录（配置文件所在的目录）里的文件名 @@ -28,13 +33,16 @@ pub const KEEP_SECS: u64 = 7 * 24 * 3600; /// 最多记多少条。满了丢最早记下的那一条：丢了也只是下次再被拒一次、再重发一次 const MAX: usize = 1024; -/// 一条：这一家的这个模型不认 `rejected` 这一档，改用 `use`（`None` 是去掉强度） +/// 一条：这一家的这个模型（按这种格式发的）不认 `rejected`，改用 `use`（`None` 是去掉强度） #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] struct Entry { provider: String, model: String, + /// 发出去的格式：`anthropic` / `chat` / `responses` / `gemini` / `bedrock` + dialect: String, /// 记下时这家长什么样（地址、凭据、协议的指纹）。对不上就作废 identity: String, + /// 写明的那一档，`none` 也是（[`WrittenEffort::slug`]） rejected: String, #[serde(rename = "use")] instead: Option, @@ -47,7 +55,7 @@ struct File { entries: Vec, } -type Key = (String, String, Effort); +type Key = (String, String, Dialect, WrittenEffort); #[derive(Default)] struct Inner { @@ -87,41 +95,67 @@ pub fn now_secs() -> u64 { .unwrap_or(0) } +const DIALECTS: [(Dialect, &str); 5] = [ + (Dialect::Anthropic, "anthropic"), + (Dialect::Chat, "chat"), + (Dialect::Responses, "responses"), + (Dialect::Gemini, "gemini"), + (Dialect::Bedrock, "bedrock"), +]; + +fn dialect_slug(d: Dialect) -> &'static str { + DIALECTS + .iter() + .find(|(x, _)| *x == d) + .map_or("anthropic", |(_, s)| s) +} + +fn parse_dialect(s: &str) -> Option { + DIALECTS.iter().find(|(_, x)| *x == s).map(|(d, _)| *d) +} + impl EffortMemory { fn lock(&self) -> std::sync::MutexGuard<'_, Inner> { self.inner.lock().unwrap_or_else(|e| e.into_inner()) } - /// 告诉它数据目录在哪儿：读进那里的文件（读不了、读不成的当没有：丢了只是再被拒一次）， - /// 再按 `cfg` 对一遍。往后记下的写回那里 - pub fn set_dir(self: &Arc, dir: PathBuf, cfg: &tw_config::Config) { + /// 告诉它数据目录在哪儿：读进那里的文件（读不了、读不成的当没有：丢了只是再被拒一次）。 + /// 往后记下的写回那里。 + /// + /// **读进来时不对账**：安全模式下在服务的是临时顶上的配置，一家上游都没有，对它对账会把 + /// 记着的全删掉。查的时候照样比这家的身份（[`Self::lookup`]），删掉的、变了的上游的那几条 + /// 等下一次换入配置时清掉（[`Self::reconcile`]） + pub fn set_dir(&self, dir: PathBuf) { let path = dir.join(FILE); let file: File = std::fs::read(&path) .ok() .and_then(|b| serde_json::from_slice(&b).ok()) .unwrap_or_default(); - { - let mut g = self.lock(); - g.path = Some(path); - g.by.clear(); - for e in file.entries { - let Some(rejected) = parse(&e.rejected) else { - continue; - }; - let instead = match e.instead.as_deref() { - None => None, - Some(s) => match parse(s) { - Some(x) => Some(x), - None => continue, - }, - }; - g.by.insert((e.provider, e.model, rejected), (e.identity, instead, e.at)); - } + let mut g = self.lock(); + g.path = Some(path); + g.by.clear(); + for e in file.entries { + let (Some(dialect), Some(rejected)) = + (parse_dialect(&e.dialect), WrittenEffort::parse(&e.rejected)) + else { + continue; + }; + let instead = match e.instead.as_deref() { + None => None, + Some(s) => match think::parse_openai(s).flatten() { + Some(x) => Some(x), + None => continue, + }, + }; + g.by.insert( + (e.provider, e.model, dialect, rejected), + (e.identity, instead, e.at), + ); } - self.reconcile(cfg); } - /// 配置换了之后对一遍：删掉的上游、地址密钥协议变了的上游，它们的几条作废 + /// 换入一份配置之后对一遍：删掉的上游、地址密钥协议变了的上游，它们的几条作废。**只对真的 + /// 配置**：安全模式下临时顶上的那份不走这里（见 [`Self::set_dir`]） pub fn reconcile(self: &Arc, cfg: &tw_config::Config) { let ids: HashMap<&str, String> = cfg .providers @@ -132,7 +166,7 @@ impl EffortMemory { let changed = { let mut g = self.lock(); let before = g.by.len(); - g.by.retain(|(provider, _, _), (id, _, at)| { + g.by.retain(|(provider, ..), (id, _, at)| { ids.get(provider.as_str()) == Some(id) && fresh(*at, now) }); g.by.len() != before @@ -142,45 +176,55 @@ impl EffortMemory { } } - /// 这一家的这个模型有没有记着什么。发之前先问这一句，没有的话不用解请求体 - pub fn has(&self, provider: &str, model: &str) -> bool { + /// 这一家的这个模型（按这种格式发的）有没有记着什么。发之前先问这一句，没有的话不用解请求体 + pub fn has(&self, provider: &str, model: &str, dialect: Dialect) -> bool { let g = self.lock(); - !g.by.is_empty() && g.by.keys().any(|(p, m, _)| p == provider && m == model) + !g.by.is_empty() + && g.by + .keys() + .any(|(p, m, d, _)| p == provider && m == model && *d == dialect) } - /// 发给这一家的这个模型的请求写着 `rejected` 这一档：记着该改成什么的话交回来 + /// 发给这一家的这个模型的请求（`dialect` 格式）写着 `rejected`：记着该改成什么的话交回来 pub fn lookup( &self, p: &tw_config::Provider, model: &str, - rejected: Effort, + dialect: Dialect, + rejected: WrittenEffort, now: u64, ) -> Option { let g = self.lock(); - let (id, instead, at) = g.by.get(&(p.name.clone(), model.to_string(), rejected))?; + let (id, instead, at) = + g.by.get(&(p.name.clone(), model.to_string(), dialect, rejected))?; (fresh(*at, now) && *id == identity(p)).then_some(match instead { Some(e) => EffortFix::Use(*e), None => EffortFix::Drop, }) } - /// 记下：这一家的这个模型不认 `rejected`，改用 `fix` 重发成了。改用的就是它自己的不记 + /// 记下：这一家的这个模型（`dialect` 格式）不认 `rejected`，改用 `fix` 重发成了。改用的就是 + /// 它自己的（上游又认了），那一条删掉 pub fn note( self: &Arc, p: &tw_config::Provider, model: &str, - rejected: Effort, + dialect: Dialect, + rejected: WrittenEffort, fix: EffortFix, now: u64, ) { let instead = match fix { - EffortFix::Use(e) if e == rejected => return, + EffortFix::Use(e) if WrittenEffort::Level(e) == rejected => { + self.forget(&p.name, model, dialect, rejected); + return; + } EffortFix::Use(e) => Some(e), EffortFix::Drop => None, }; { let mut g = self.lock(); - let key = (p.name.clone(), model.to_string(), rejected); + let key = (p.name.clone(), model.to_string(), dialect, rejected); if g.by.len() >= MAX && !g.by.contains_key(&key) && let Some(oldest) = @@ -195,13 +239,31 @@ impl EffortMemory { tracing::info!( provider = %p.name, model = %model, - rejected = think::openai(rejected), - instead = instead.map_or("none", think::openai), + rejected = rejected.slug(), + instead = instead.map_or("(removed)", think::openai), "remembered a reasoning effort this model does not take" ); self.save(); } + /// 删掉一条：记着的改法自己被拒了，或者上游又认原来那一档了 + pub fn forget( + self: &Arc, + provider: &str, + model: &str, + dialect: Dialect, + rejected: WrittenEffort, + ) { + let removed = self + .lock() + .by + .remove(&(provider.to_string(), model.to_string(), dialect, rejected)) + .is_some(); + if removed { + self.save(); + } + } + /// 写回文件。**不在转发那条路上写**：有异步运行时就挪到阻塞线程上 fn save(self: &Arc) { let me = self.clone(); @@ -225,17 +287,26 @@ impl EffortMemory { let mut entries: Vec = g.by.iter() .filter(|(_, (_, _, at))| fresh(*at, now)) - .map(|((provider, model, rejected), (id, instead, at))| Entry { - provider: provider.clone(), - model: model.clone(), - identity: id.clone(), - rejected: think::openai(*rejected).to_string(), - instead: instead.map(|e| think::openai(e).to_string()), - at: *at, - }) + .map( + |((provider, model, dialect, rejected), (id, instead, at))| Entry { + provider: provider.clone(), + model: model.clone(), + dialect: dialect_slug(*dialect).to_string(), + identity: id.clone(), + rejected: rejected.slug().to_string(), + instead: instead.map(|e| think::openai(e).to_string()), + at: *at, + }, + ) .collect(); - entries - .sort_by(|a, b| (&a.provider, &a.model, a.at).cmp(&(&b.provider, &b.model, b.at))); + entries.sort_by(|a, b| { + (&a.provider, &a.model, &a.dialect, &a.rejected).cmp(&( + &b.provider, + &b.model, + &b.dialect, + &b.rejected, + )) + }); match serde_json::to_string_pretty(&File { entries }) { Ok(t) => (path, t), Err(_) => return, @@ -247,15 +318,13 @@ impl EffortMemory { } } -/// 存着的档位名。`none` 之类不是一档的不认 -fn parse(s: &str) -> Option { - think::parse_openai(s).flatten() -} - #[cfg(test)] mod tests { use super::*; + const A: Dialect = Dialect::Anthropic; + const L: fn(Effort) -> WrittenEffort = WrittenEffort::Level; + fn provider(name: &str, key: &str) -> tw_config::Provider { tw_config::Provider { name: name.into(), @@ -278,31 +347,63 @@ mod tests { let m = Arc::new(EffortMemory::default()); let p = provider("relay", "sk-1"); let now = 1_000_000; - assert!(!m.has("relay", "glm-5")); + assert!(!m.has("relay", "glm-5", A)); m.note( &p, "glm-5", - Effort::Medium, + A, + L(Effort::Medium), EffortFix::Use(Effort::High), now, ); - assert!(m.has("relay", "glm-5")); - assert!(!m.has("relay", "glm-4")); + assert!(m.has("relay", "glm-5", A)); + assert!(!m.has("relay", "glm-4", A)); + // 按发出去的格式分开记：同一个模型名换一种格式发是另一回事 + assert!(!m.has("relay", "glm-5", Dialect::Chat)); assert_eq!( - m.lookup(&p, "glm-5", Effort::Medium, now + 60), + m.lookup(&p, "glm-5", Dialect::Chat, L(Effort::Medium), now), + None + ); + assert_eq!( + m.lookup(&p, "glm-5", A, L(Effort::Medium), now + 60), Some(EffortFix::Use(Effort::High)) ); - assert_eq!(m.lookup(&p, "glm-5", Effort::Low, now), None); + assert_eq!(m.lookup(&p, "glm-5", A, L(Effort::Low), now), None); // 7 天之后再试原来那一档 - assert_eq!(m.lookup(&p, "glm-5", Effort::Medium, now + KEEP_SECS), None); - // 去掉强度也记得住；改用的就是它自己的不记 - m.note(&p, "glm-5", Effort::Max, EffortFix::Drop, now); assert_eq!( - m.lookup(&p, "glm-5", Effort::Max, now), + m.lookup(&p, "glm-5", A, L(Effort::Medium), now + KEEP_SECS), + None + ); + // 去掉强度也记得住，`none` 也是一档 + m.note(&p, "glm-5", A, L(Effort::Max), EffortFix::Drop, now); + assert_eq!( + m.lookup(&p, "glm-5", A, L(Effort::Max), now), + Some(EffortFix::Drop) + ); + m.note( + &p, + "gpt-x", + Dialect::Chat, + WrittenEffort::Off, + EffortFix::Drop, + now, + ); + assert_eq!( + m.lookup(&p, "gpt-x", Dialect::Chat, WrittenEffort::Off, now), Some(EffortFix::Drop) ); - m.note(&p, "glm-5", Effort::Low, EffortFix::Use(Effort::Low), now); - assert_eq!(m.lookup(&p, "glm-5", Effort::Low, now), None); + // 重发成了用的正是原来那一档：上游又认了，删掉 + m.note( + &p, + "glm-5", + A, + L(Effort::Medium), + EffortFix::Use(Effort::Medium), + now, + ); + assert_eq!(m.lookup(&p, "glm-5", A, L(Effort::Medium), now), None); + m.forget("relay", "glm-5", A, L(Effort::Max)); + assert_eq!(m.lookup(&p, "glm-5", A, L(Effort::Max), now), None); } #[test] @@ -310,81 +411,122 @@ mod tests { let m = Arc::new(EffortMemory::default()); let now = now_secs(); let (a, b) = (provider("a", "sk-1"), provider("b", "sk-2")); - m.note(&a, "m", Effort::XHigh, EffortFix::Use(Effort::Max), now); - m.note(&b, "m", Effort::XHigh, EffortFix::Use(Effort::Max), now); + let x = L(Effort::XHigh); + m.note(&a, "m", A, x, EffortFix::Use(Effort::Max), now); + m.note(&b, "m", A, x, EffortFix::Use(Effort::Max), now); // 换了密钥：已经是另一个上游了，查的时候就不作数 let rekeyed = provider("a", "sk-9"); - assert_eq!(m.lookup(&rekeyed, "m", Effort::XHigh, now), None); + assert_eq!(m.lookup(&rekeyed, "m", A, x, now), None); m.reconcile(&config(vec![rekeyed.clone()])); - assert!(!m.has("a", "m"), "密钥变了"); - assert!(!m.has("b", "m"), "删掉了"); + assert!(!m.has("a", "m", A), "密钥变了"); + assert!(!m.has("b", "m", A), "删掉了"); // 地址、协议变了也一样 - m.note(&rekeyed, "m", Effort::XHigh, EffortFix::Drop, now); + m.note(&rekeyed, "m", A, x, EffortFix::Drop, now); let moved = tw_config::Provider { base_url: "https://other.example.com".into(), ..rekeyed.clone() }; m.reconcile(&config(vec![moved])); - assert!(!m.has("a", "m")); - m.note(&rekeyed, "m", Effort::XHigh, EffortFix::Drop, now); + assert!(!m.has("a", "m", A)); + m.note(&rekeyed, "m", A, x, EffortFix::Drop, now); let reprotocoled = tw_config::Provider { protocol: Some(tw_config::Protocol::OpenaiChat), ..rekeyed.clone() }; m.reconcile(&config(vec![reprotocoled])); - assert!(!m.has("a", "m")); + assert!(!m.has("a", "m", A)); // 没变的留着 - m.note(&rekeyed, "m", Effort::XHigh, EffortFix::Drop, now); + m.note(&rekeyed, "m", A, x, EffortFix::Drop, now); m.reconcile(&config(vec![rekeyed.clone()])); - assert!(m.has("a", "m")); + assert!(m.has("a", "m", A)); } #[test] fn what_is_remembered_survives_a_restart_in_the_data_directory() { let dir = tempfile::tempdir().unwrap(); let p = provider("relay", "sk-1"); - let cfg = config(vec![p.clone()]); let now = now_secs(); let m = Arc::new(EffortMemory::default()); - m.set_dir(dir.path().to_path_buf(), &cfg); + m.set_dir(dir.path().to_path_buf()); m.note( &p, "claude-opus-4-6", - Effort::XHigh, + A, + L(Effort::XHigh), EffortFix::Use(Effort::Max), now, ); - m.note(&p, "glm-5", Effort::Medium, EffortFix::Drop, now); + m.note( + &p, + "gpt-x", + Dialect::Responses, + WrittenEffort::Off, + EffortFix::Drop, + now, + ); m.save_now(); let text = std::fs::read_to_string(dir.path().join(FILE)).unwrap(); assert!(text.contains("\"rejected\": \"xhigh\""), "{text}"); assert!(text.contains("\"use\": \"max\""), "{text}"); + assert!(text.contains("\"rejected\": \"none\""), "{text}"); + assert!(text.contains("\"dialect\": \"responses\""), "{text}"); assert!(!text.contains("sk-1"), "不存密钥：{text}"); let again = Arc::new(EffortMemory::default()); - again.set_dir(dir.path().to_path_buf(), &cfg); + again.set_dir(dir.path().to_path_buf()); assert_eq!( - again.lookup(&p, "claude-opus-4-6", Effort::XHigh, now), + again.lookup(&p, "claude-opus-4-6", A, L(Effort::XHigh), now), Some(EffortFix::Use(Effort::Max)) ); assert_eq!( - again.lookup(&p, "glm-5", Effort::Medium, now), + again.lookup(&p, "gpt-x", Dialect::Responses, WrittenEffort::Off, now), Some(EffortFix::Drop) ); - // 读进来时这家已经换了密钥：作废，文件跟着改 - let again = Arc::new(EffortMemory::default()); - again.set_dir( - dir.path().to_path_buf(), - &config(vec![provider("relay", "sk-2")]), + // 读进来时这家已经换了密钥：查不到；换入真的配置时清掉 + let rekeyed = provider("relay", "sk-2"); + assert_eq!( + again.lookup(&rekeyed, "claude-opus-4-6", A, L(Effort::XHigh), now), + None ); - assert!(!again.has("relay", "glm-5")); + again.reconcile(&config(vec![rekeyed])); again.save_now(); let text = std::fs::read_to_string(dir.path().join(FILE)).unwrap(); - assert!(!text.contains("glm-5"), "{text}"); + assert!(!text.contains("claude-opus-4-6"), "{text}"); // 读不成的文件当没有 std::fs::write(dir.path().join(FILE), "not json").unwrap(); let broken = Arc::new(EffortMemory::default()); - broken.set_dir(dir.path().to_path_buf(), &cfg); - assert!(!broken.has("relay", "glm-5")); + broken.set_dir(dir.path().to_path_buf()); + assert!(!broken.has("relay", "gpt-x", Dialect::Responses)); + } + + /// 安全模式下网关拿着一份一家上游都没有的临时配置起来：读进文件时不对账，记着的一条不丢 + #[test] + fn safe_mode_does_not_wipe_the_file() { + let dir = tempfile::tempdir().unwrap(); + let p = provider("relay", "sk-1"); + let now = now_secs(); + let m = Arc::new(EffortMemory::default()); + m.set_dir(dir.path().to_path_buf()); + m.note( + &p, + "m", + A, + L(Effort::XHigh), + EffortFix::Use(Effort::Max), + now, + ); + m.save_now(); + let gw = crate::AppState::new(tw_config::Config::default()).unwrap(); + gw.set_config_dir(dir.path().to_path_buf()); + std::thread::sleep(std::time::Duration::from_millis(50)); + let text = std::fs::read_to_string(dir.path().join(FILE)).unwrap(); + assert!(text.contains("\"use\": \"max\""), "{text}"); + assert!(gw.effort_memory.has("relay", "m", A)); + // 修好了，换入真的配置：这家没变，留着 + gw.reload(config(vec![p.clone()])).unwrap(); + assert_eq!( + gw.effort_memory.lookup(&p, "m", A, L(Effort::XHigh), now), + Some(EffortFix::Use(Effort::Max)) + ); } } diff --git a/crates/tw-gateway/src/forward.rs b/crates/tw-gateway/src/forward.rs index e65380bd..72f1422d 100644 --- a/crates/tw-gateway/src/forward.rs +++ b/crates/tw-gateway/src/forward.rs @@ -79,10 +79,14 @@ pub fn gemini_path_with_model(path: &str, model: &str) -> String { /// **只动改了的那几个字段，别的字节一个不动**（见 [`crate::splice`]）：解析再写回的话， /// 每个对象的键都按字母重排了，工具定义、工具参数一起变了样。改出来和原来一样的（规则 /// 写的值客户端本来就是这么写的）原样发。改到的对象里同一个键写了两遍的，才照旧写回。 +/// +/// `model` 是发给上游的模型名：Gemini 的思考预算按它压在上限以内（见 +/// [`tw_dialect::params::set_reasoning_effort`]） pub fn apply_set( body: &Bytes, set: &tw_engine::SetAction, client: Option, + model: &str, ) -> Bytes { if set.is_empty() { return body.clone(); @@ -96,7 +100,7 @@ pub fn apply_set( return body.clone(); } let mut v = before.clone(); - set_params(&mut v, set, client); + set_params(&mut v, set, client, model); rewritten(body, &before, &v) } @@ -129,6 +133,7 @@ fn set_params( v: &mut serde_json::Value, set: &tw_engine::SetAction, client: Option, + model: &str, ) { use tw_dialect::params; let client = client.unwrap_or(tw_dialect::ir::Dialect::Anthropic); @@ -145,7 +150,7 @@ fn set_params( params::disable_thinking(client, v) } Some(tw_engine::SetReasoning::Level(e)) => { - params::set_reasoning_effort(client, v, e); + params::set_reasoning_effort(client, v, e, model); } _ => {} } @@ -251,7 +256,10 @@ mod tests { // 出站直通。cc-switch 那次把缓存命中率从 99% 打到 20%， // 就是因为一个「看起来无害」的重写跑在了每个请求上。 let b = Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#); - assert_eq!(apply_set(&b, &tw_engine::SetAction::default(), None), b); + assert_eq!( + apply_set(&b, &tw_engine::SetAction::default(), None, "m"), + b + ); } #[test] @@ -264,6 +272,7 @@ mod tests { ..Default::default() }, None, + "m", ); let v: serde_json::Value = serde_json::from_slice(&out).unwrap(); assert_eq!(v["model"], "haiku"); @@ -282,11 +291,32 @@ mod tests { ..Default::default() }, None, + "m", ); let v: serde_json::Value = serde_json::from_slice(&out).unwrap(); assert!(v.get("thinking").is_none()); } + /// Gemini 的预算按发给它的模型压在上限以内：规则写 max 不能变成一个必然 400 的 64000 + #[test] + fn a_gemini_budget_from_a_rule_stays_within_the_model_limit() { + let b = Bytes::from_static( + br#"{"contents":[],"generationConfig":{"thinkingConfig":{"thinkingBudget":2048}}}"#, + ); + let s = tw_engine::SetAction { + reasoning: Some(tw_engine::SetReasoning::Level(tw_dialect::ir::Effort::Max)), + ..Default::default() + }; + for (model, want) in [("gemini-2.5-flash", 24576), ("gemini-2.5-pro", 32768)] { + let out = apply_set(&b, &s, Some(tw_dialect::ir::Dialect::Gemini), model); + let v: serde_json::Value = serde_json::from_slice(&out).unwrap(); + assert_eq!( + v["generationConfig"]["thinkingConfig"]["thinkingBudget"], want, + "{model}" + ); + } + } + #[test] fn a_body_we_cannot_parse_is_forwarded_untouched() { // 我们的解析器不认识的东西，上游可能完全认识。 @@ -298,6 +328,7 @@ mod tests { ..Default::default() }, None, + "m", ); assert_eq!(out, b); } @@ -316,6 +347,7 @@ mod tests { &Bytes::from_static(body.as_bytes()), &set, Some(d), + "m", )) .unwrap() }; @@ -327,13 +359,15 @@ mod tests { (v["model"].as_str(), v["max_completion_tokens"].as_u64()), (Some("small"), Some(512)) ); - assert!(v.get("reasoning_effort").is_none() && v.get("max_tokens").is_none()); + // OpenAI 的格式里 `off` 写 `none`：去掉的话按模型默认的那一档想 + assert_eq!(v["reasoning_effort"], "none"); + assert!(v.get("max_tokens").is_none()); let v = run( r#"{"model":"big","reasoning":{"effort":"high"}}"#, Dialect::Responses, ); assert_eq!(v["max_output_tokens"], 512); - assert!(v.get("reasoning").is_none()); + assert_eq!(v["reasoning"], serde_json::json!({"effort": "none"})); let v = run( r#"{"contents":[],"generationConfig":{"thinkingConfig":{"thinkingBudget":9}}}"#, Dialect::Gemini, @@ -365,7 +399,7 @@ mod tests { /// 以前的写法：解析、改、整份写回（键按字母重排） fn whole(body: &[u8], s: &tw_engine::SetAction, d: Option) -> Bytes { let mut v: serde_json::Value = serde_json::from_slice(body).unwrap(); - set_params(&mut v, s, d); + set_params(&mut v, s, d, "m"); Bytes::from(serde_json::to_vec(&v).unwrap()) } @@ -387,6 +421,7 @@ mod tests { Some(tw_engine::SetReasoning::Off), ), Some(Dialect::Anthropic), + "m", ); assert_eq!( std::str::from_utf8(&out).unwrap(), @@ -399,6 +434,7 @@ mod tests { &Bytes::from(format!(r#"{{"model":"m",{tools}}}"#)), &set(None, Some(10), None), Some(Dialect::Anthropic), + "m", ); assert_eq!( std::str::from_utf8(&out).unwrap(), @@ -416,6 +452,7 @@ mod tests { Some(tw_engine::SetReasoning::Off), ), Some(Dialect::Gemini), + "m", ); assert_eq!( std::str::from_utf8(&out).unwrap(), @@ -429,7 +466,7 @@ mod tests { #[test] fn a_set_that_changes_nothing_sends_the_same_bytes() { let b = Bytes::from_static(br#"{ "max_tokens" : 100, "b":{"z":1,"a":2} }"#); - let out = apply_set(&b, &set(None, Some(100), None), None); + let out = apply_set(&b, &set(None, Some(100), None), None, "m"); assert_eq!(out.as_ptr(), b.as_ptr()); } @@ -440,7 +477,7 @@ mod tests { let b = br#"{"model":"a","x":1,"model":"b"}"#; let s = set(Some("c"), None, None); assert_eq!( - apply_set(&Bytes::from_static(b), &s, None), + apply_set(&Bytes::from_static(b), &s, None, "m"), whole(b, &s, None) ); } @@ -522,7 +559,7 @@ mod tests { continue; } let d = Some(dialects[r.below(dialects.len())]); - let out = apply_set(&Bytes::from(body.clone()), &s, d); + let out = apply_set(&Bytes::from(body.clone()), &s, d, "m"); let value = |b: &[u8]| serde_json::from_slice::(b).unwrap(); assert_eq!( value(&out), @@ -531,7 +568,7 @@ mod tests { ); let before = value(body.as_bytes()); let mut after = before.clone(); - set_params(&mut after, &s, d); + set_params(&mut after, &s, d, "m"); if crate::splice::rewrite(body.as_bytes(), &before, &after) == crate::splice::Rewrite::Unusual { diff --git a/crates/tw-gateway/src/server/pipeline/hop.rs b/crates/tw-gateway/src/server/pipeline/hop.rs index 10245d91..cce2b940 100644 --- a/crates/tw-gateway/src/server/pipeline/hop.rs +++ b/crates/tw-gateway/src/server/pipeline/hop.rs @@ -24,7 +24,7 @@ use crate::failure::{Cause, Verdict}; use crate::forward; use crate::server::{hop, hop_failed, note_health}; use crate::state::{AppState, Runtime}; -use tw_dialect::ir::Effort; +use tw_dialect::params::WrittenEffort; use tw_dialect::think::EffortFix; use tw_types::msg; @@ -203,7 +203,11 @@ pub(super) async fn try_upstreams<'a>( let mut reported = crate::guard::Reported::default(); // 推理强度被上游拒了（见 `crate::effort_memory`）：同一家改一档再发的那一跳要用的 —— 发给 // 哪一家、被拒那一跳本来要发的那一档、改成什么。**是尝试链上新的一跳**，不算退避重试 - let mut effort_resend: Option<(&String, Effort, EffortFix)> = None; + let mut effort_resend: Option<(&String, WrittenEffort, EffortFix)> = None; + // 重发的那一跳接着用被拒那一跳在这家占着的位置（和探路的名额）：**马上发**，不排到别的 + // 候选后面、不重新等空位 + let mut effort_hold: Option<(&String, crate::slots::Slot, Option)> = + None; // 为推理强度重发过的那几家：一家只重发一次 let mut effort_resent: Vec<&String> = Vec::new(); // 这一跳发出去的推理强度和客户端要的不一样：那一跳在尝试链上的位置和那一项 @@ -435,13 +439,18 @@ pub(super) async fn try_upstreams<'a>( // 数 token 不排队、不占位置：它不跑模型，一眨眼就回来；为它排在几个长回答后面等上半分钟、 // 最后回一个 429，客户端连上下文还剩多少都看不到。这一家暂时发不出去的，由网关估 let limits = crate::throttle::usage_limits(provider); - let (slot, mut probe) = match granted.take() { - Some((ready, waited)) => { + let held = match effort_hold.take() { + Some((n, slot, probe)) if n == name => Some((slot, probe)), + _ => None, + }; + let (slot, mut probe) = match (held, granted.take()) { + (Some(held), _) => held, + (None, Some((ready, waited))) => { queued = Some((chain.len(), waited.as_millis() as u64)); hop_started = std::time::Instant::now(); (ready.slot, ready.probe) } - None if counting => { + (None, None) if counting => { if state.throttles.peek(&provider.name, limits) != crate::throttle::Peek::Open { chain.push(estimated_hop(provider, None, None, hop_started)); estimated = Some(provider); @@ -449,7 +458,7 @@ pub(super) async fn try_upstreams<'a>( } (crate::slots::Slot::free(), None) } - None => { + (None, None) => { let model = Some(sent.clone()).filter(asked_other); // 这段对话留在这家是为了它的缓存：等它，等不到再换下一家（缓存就丢在这家了）。 // 别的候选发不出去当场跳过，记下来，候选都看过之后再等 @@ -606,16 +615,14 @@ pub(super) async fn try_upstreams<'a>( .model .clone() .unwrap_or_else(|| reading.facts.model.clone()); - // 这一跳是推理强度被拒之后的重发：用定好的那一档 + // 这一跳是推理强度被拒之后的重发：用定好的那一档。**真的发出去了才算用掉**：发之前就失败了 + // 的（取不到凭据、转换不了），下一次轮到这一家时照样用 let carried = match effort_resend { - Some((n, pre, fix)) if n == name => { - effort_resend = None; - Some((pre, fix)) - } + Some((n, pre, fix)) if n == name => Some((pre, fix)), _ => None, }; - // `hop_effort`：这一跳改强度之前要发的那一档（见 `adjust_effort`），没算的是 None - let (out, hop_effort) = match super::heavy(asked.body, || { + // `hop_effort`：这一跳的推理强度改了什么（见 `adjust_effort`） + let (out, mut hop_effort) = match super::heavy(asked.body, || { prepare(state, req, reading, &asked, provider, &effective_set, id).map(|mut out| { let effort = adjust_effort( state, @@ -631,10 +638,7 @@ pub(super) async fn try_upstreams<'a>( (out, effort) }) }) { - Ok((out, (pre, change))) => { - effort_note = change.map(|c| (chain.len(), c)); - (out, pre) - } + Ok(out) => out, Err(err) => { chain.push(hop_failed( &provider.name, @@ -740,7 +744,12 @@ pub(super) async fn try_upstreams<'a>( attempt = attempts.len(), "forwarding" ); - // 这一跳要发出去了：插件改过的话，它收到的就是改过的那一份 + // 这一跳要发出去了：重发定好的那一档用掉了，推理强度改了什么记在这一跳上 + if carried.is_some() { + effort_resend = None; + } + effort_note = hop_effort.change.take().map(|c| (chain.len(), c)); + // 插件改过的话，上游收到的就是改过的那一份 after_plugins = plugged .rewritten .as_ref() @@ -899,18 +908,21 @@ pub(super) async fn try_upstreams<'a>( if !matches!(verdict, Verdict::Failed(Cause::RateLimited { .. })) { to_throttle(state, &provider.name, sent_at, verdict); } - // **最后一家的 4xx 交给客户端**：没有下一家可换了，上游自己的原话 - // （「invalid x-api-key」）比我们转述的一句「回了 401」有用得多。 - // 5xx 和 429 照旧报我们的话，带着尝试链 // 上游因为推理强度拒了它（见 `crate::effort_memory`）：这一跳是一跳普通的 400，同一家 - // 改一档（或者去掉强度）再发一次，是尝试链上新的一跳。**一家只重发一次**，再被拒 - // 照常交给客户端。上游好好的，不算失败 + // 改一档（或者去掉强度）马上再发一次，是尝试链上新的一跳，接着用这一跳占着的位置。 + // **一家只重发一次**，再被拒照常交给客户端。上游好好的，不算失败。记着的改法自己 + // 被拒了：那一条删掉 if status == 400 && reading.generates && !counting && !effort_resent.contains(&name) - && let Some((pre, fix)) = effort_refused(req, &out, hop_effort, &head) + && let Some((wire, pre, fix)) = effort_refused(req, &out, &hop_effort, &head) { + if hop_effort.remembered { + state + .effort_memory + .forget(&provider.name, &sent_model, wire, pre); + } tap.said(&head); state.note_proxy_ok(&provider.proxy); note_health( @@ -933,9 +945,13 @@ pub(super) async fn try_upstreams<'a>( )); effort_resent.push(name); effort_resend = Some((name, pre, fix)); + effort_hold = Some((name, slot, probe.take())); queue.push_front(name); continue; } + // **最后一家的 4xx 交给客户端**：没有下一家可换了，上游自己的原话 + // （「invalid x-api-key」）比我们转述的一句「回了 401」有用得多。 + // 5xx 和 429 照旧报我们的话，带着尝试链 let hand_on = last && status < 500 && status != 429; match verdict { // 限流（额度用完不算）：**不是失败**。这一家暂停（见 `crate::throttle`），这个请求 @@ -1301,10 +1317,14 @@ pub(super) async fn try_upstreams<'a>( state.health.record_success(&provider.name), ); } - if answered && let Some((pre, fix)) = carried { + if answered + && let Some((pre, fix)) = carried + && let Some(wire) = wire(req, &out) + { state.effort_memory.note( provider, &sent_model, + wire, pre, fix, crate::effort_memory::now_secs(), @@ -2058,7 +2078,11 @@ fn prepare( // 参数改写。**只在这里动 body，而且只动被点名的那几个字段** —— // 出站直通说过任何 body 改写都可能是缓存杀手，所以这是 // 一个用户显式要求的例外，不是默认行为。 - let out = forward::apply_set(asked.body, effective_set, client_dialect); + let model = effective_set + .model + .as_deref() + .unwrap_or(&reading.facts.model); + let out = forward::apply_set(asked.body, effective_set, client_dialect, model); if let (Some(tw_dialect::ir::Dialect::Gemini), Some(m)) = (client_dialect, &effective_set.model) { @@ -2261,10 +2285,24 @@ fn prepare( }) } +/// 一跳的推理强度改了什么（见 [`adjust_effort`]）。 +#[derive(Default)] +struct HopEffort { + /// 这里改之前（规则改过之后）请求体里写明的那一档（[`tw_dialect::params::written_effort`]）。 + /// 没算过、没写明的（只写了预算的也是）是 None + pre: Option, + /// 用上了记着的改法 + remembered: bool, + /// 尝试链上这一跳要记的那一项 + change: Option, +} + /// 这一跳的推理强度（见 [`crate::effort_memory`]）：规则改过之后（`out` 是按规则改好、要发出去 /// 的那一份），这一家的这个模型记着不认这一档的，或者这一跳是被拒之后的重发（`carried`），在 -/// 发出去的这一份上改掉。交回改之前要发的那一档，和尝试链上要记的那一项（发出去的和客户端要的 -/// 不一样、又是规则或者这里改的才记）。 +/// 发出去的这一份上改掉写明的那一档。 +/// +/// 尝试链上记的（[`tw_api::EffortChange`]）：这里改了的，`asked` 是改之前写明的那一档；只有规则 +/// 改了的，`asked` 是客户端要的那一档。和发出去的一样的不记。 /// /// **用不着时不解请求体**：没有规则改推理、这一家的这个模型什么都没记着、也不是重发的，什么都 /// 不做 @@ -2279,29 +2317,27 @@ fn adjust_effort( set: &tw_engine::SetAction, out: &mut Outbound, carried: Option, -) -> (Option, Option) { +) -> HopEffort { use tw_dialect::params; let rule = set.reasoning.is_some(); - if !reading.generates - || (!rule && carried.is_none() && !state.effort_memory.has(&provider.name, model)) - { - return (None, None); - } - let Some(wire) = wire(req, out) else { - return (None, None); + let Some(wire) = wire(req, out).filter(|_| reading.generates) else { + return HopEffort::default(); }; + if !rule && carried.is_none() && !state.effort_memory.has(&provider.name, model, wire) { + return HopEffort::default(); + } let Ok(before) = serde_json::from_slice::(&out.body) else { - return (None, None); + return HopEffort::default(); }; - let pre = params::reasoning_effort(wire, &before); - let mut reason = rule.then_some(tw_api::EffortReason::Rule); + let pre = params::written_effort(wire, &before); + let mut fixed = None; let mut after = before.clone(); if let Some(p) = pre { let fix = match carried { Some(f) => Some((f, tw_api::EffortReason::Rejected)), None => state .effort_memory - .lookup(provider, model, p, crate::effort_memory::now_secs()) + .lookup(provider, model, wire, p, crate::effort_memory::now_secs()) .map(|f| (f, tw_api::EffortReason::Remembered)), }; if let Some((f, why)) = fix @@ -2310,56 +2346,77 @@ fn adjust_effort( out.body = forward::rewritten(&out.body, &before, &after); // 去掉、改掉了上游不认的值：发出去的不再是客户端那一份（报文记录另存它） out.reshaped = true; - reason = Some(why); + fixed = Some(why); } } - let sent = params::reasoning_effort(wire, &after); - let wanted = match asked.decoded { - Some(Ok(d)) => d - .request - .reasoning - .as_ref() - .filter(|r| r.enabled) - .and_then(tw_dialect::think::effort), - _ => req.api.and_then(|a| { - let v = serde_json::from_slice::(asked.body).ok()?; - params::reasoning_effort(a.dialect(), &v) - }), + let shown = |v: &serde_json::Value| { + params::written_effort(wire, v) + .map(|w| w.slug()) + .or_else(|| params::reasoning_effort(wire, v).map(tw_dialect::think::openai)) }; - let change = match (reason, wanted) { - (Some(reason), Some(w)) if Some(w) != sent => Some(tw_api::EffortChange { - asked: tw_dialect::think::openai(w).to_string(), - sent: sent.map(|e| tw_dialect::think::openai(e).to_string()), - reason, - }), - _ => None, + let sent = shown(&after); + let from = match fixed { + Some(_) => pre.map(|p| p.slug()), + None if rule => match asked.decoded { + Some(Ok(d)) => d + .request + .reasoning + .as_ref() + .filter(|r| r.enabled) + .and_then(tw_dialect::think::effort) + .map(tw_dialect::think::openai), + _ => req.api.and_then(|a| { + let v = serde_json::from_slice::(asked.body).ok()?; + params::reasoning_effort(a.dialect(), &v).map(tw_dialect::think::openai) + }), + }, + None => None, }; - (pre, change) + let reason = fixed.unwrap_or(tw_api::EffortReason::Rule); + let change = from + .filter(|f| Some(*f) != sent) + .map(|f| tw_api::EffortChange { + asked: f.to_string(), + sent: sent.map(str::to_string), + reason, + }); + HopEffort { + pre, + remembered: fixed == Some(tw_api::EffortReason::Remembered), + change, + } } -/// 照 `fix` 改请求体 `v`（`wire` 格式）。返回改没改 +/// 照 `fix` 改请求体 `v`（`wire` 格式）里写明的那一档。返回改没改 fn apply_fix(wire: tw_dialect::ir::Dialect, v: &mut serde_json::Value, fix: EffortFix) -> bool { match fix { - EffortFix::Use(e) => tw_dialect::params::set_reasoning_effort(wire, v, e), + EffortFix::Use(e) => tw_dialect::params::replace_reasoning_effort(wire, v, e), EffortFix::Drop => tw_dialect::params::remove_reasoning_effort(wire, v), } } -/// 上游回的这个 400 是不是因为推理强度（[`tw_dialect::think::effort_rejection`]）：是的话，交回 -/// 记的时候按哪一档记（这一跳改强度之前要发的那一档，`pre`；没算过的就是发出去的那一档）和 -/// 重发时怎么改。照着改了也和发出去的一样的（按预算思考的去不掉强度）不算：重发一遍还是被拒 +/// 上游回的这个 400 是不是因为写明的推理强度（[`tw_dialect::think::effort_rejection`]，发出去的 +/// 是 `none` 的看 [`tw_dialect::think::none_rejection`]，那时去掉它）：是的话，交回发出去的格式、 +/// 记的时候按哪一档记（这一跳改强度之前写明的那一档；没算过的就是发出去的那一档）和重发时怎么 +/// 改。只写了预算的不算；照着改了也和发出去的一样的不算：重发一遍还是被拒 fn effort_refused( req: &Inbound, out: &Outbound, - pre: Option, + hop: &HopEffort, said: &[u8], -) -> Option<(Effort, EffortFix)> { +) -> Option<(tw_dialect::ir::Dialect, WrittenEffort, EffortFix)> { let wire = wire(req, out)?; let v = serde_json::from_slice::(&out.body).ok()?; - let sent = tw_dialect::params::reasoning_effort(wire, &v)?; - let fix = tw_dialect::think::effort_rejection(400, &String::from_utf8_lossy(said), sent)?; + let sent = tw_dialect::params::written_effort(wire, &v)?; + let said = String::from_utf8_lossy(said); + let fix = match sent { + WrittenEffort::Level(e) => tw_dialect::think::effort_rejection(400, &said, e)?, + WrittenEffort::Off => { + tw_dialect::think::none_rejection(400, &said).then_some(EffortFix::Drop)? + } + }; let mut tried = v; - apply_fix(wire, &mut tried, fix).then_some((pre.unwrap_or(sent), fix)) + apply_fix(wire, &mut tried, fix).then_some((wire, hop.pre.unwrap_or(sent), fix)) } /// 这一跳发出去的那一份是什么格式：转换过的是目标格式，直通的是客户端的 diff --git a/crates/tw-gateway/src/state.rs b/crates/tw-gateway/src/state.rs index d49a7a2d..669ee5f9 100644 --- a/crates/tw-gateway/src/state.rs +++ b/crates/tw-gateway/src/state.rs @@ -651,7 +651,7 @@ impl AppState { /// 路径时调；目录变了就把插件重读一遍 pub fn set_config_dir(&self, dir: std::path::PathBuf) { // 记着的推理强度也存在这个目录里（见 `crate::effort_memory`） - self.effort_memory.set_dir(dir.clone(), &self.config()); + self.effort_memory.set_dir(dir.clone()); if self.plugins.set_dir(dir) { self.reload_plugins(); } diff --git a/crates/tw-gateway/src/ws.rs b/crates/tw-gateway/src/ws.rs index d86f00e3..07f32d75 100644 --- a/crates/tw-gateway/src/ws.rs +++ b/crates/tw-gateway/src/ws.rs @@ -2212,7 +2212,8 @@ fn rewrite(text: String, model: &str, set: &tw_engine::SetAction) -> String { return text; } let body = bytes::Bytes::from(text); - let out = crate::forward::apply_set(&body, &set, Some(tw_dialect::ir::Dialect::Responses)); + let out = + crate::forward::apply_set(&body, &set, Some(tw_dialect::ir::Dialect::Responses), model); String::from_utf8(Vec::from(out)) .unwrap_or_else(|_| String::from_utf8_lossy(&body).into_owned()) } diff --git a/crates/tw-gateway/tests/alias_ws.rs b/crates/tw-gateway/tests/alias_ws.rs index 4b700120..82fa47fb 100644 --- a/crates/tw-gateway/tests/alias_ws.rs +++ b/crates/tw-gateway/tests/alias_ws.rs @@ -664,7 +664,7 @@ fn create_reasoning(model: &str) -> WsMsg { } /// 规则的参数改写每一帧照 HTTP 那条路改（Responses 的字段名）：`set.max_tokens` 写成 -/// `max_output_tokens`，`set.reasoning: off` 去掉 `reasoning`。条件按这一帧求 —— 按模型附加的 +/// `max_output_tokens`，`set.reasoning: off` 把强度写成 `none`。条件按这一帧求 —— 按模型附加的 /// 改写、按模型拒绝的规则，升级时还不知道模型，每一帧照样生效 #[tokio::test] async fn rule_parameter_rewrites_and_model_conditions_apply_to_each_frame() { @@ -695,7 +695,12 @@ async fn rule_parameter_rewrites_and_model_conditions_apply_to_each_frame() { { let seen = seen.lock().unwrap(); assert_eq!(seen[1]["max_output_tokens"], 99, "{:?}", seen[1]); - assert!(seen[1].get("reasoning").is_none(), "{:?}", seen[1]); + assert_eq!( + seen[1]["reasoning"], + json!({"effort": "none"}), + "{:?}", + seen[1] + ); assert_eq!(seen[1]["model"], "gpt-5.1-codex-max"); } diff --git a/crates/tw-gateway/tests/effort.rs b/crates/tw-gateway/tests/effort.rs index 3903ca6f..90985cff 100644 --- a/crates/tw-gateway/tests/effort.rs +++ b/crates/tw-gateway/tests/effort.rs @@ -291,11 +291,12 @@ async fn another_400_is_not_sent_again() { assert_eq!(chain[0].outcome, tw_api::AttemptOutcome::Served); } -/// 原话里碰巧有发出去的那一档的名字（「max context length」）：照规矩重发一次，再被拒就把那个 +/// 原话里碰巧有发出去的那一档的名字（「max context length」），又说到了思考：照规矩重发一次，再被拒就把那个 /// 错误交给客户端，**而且不记**：下一个请求照样发 max #[tokio::test] async fn a_resend_refused_again_is_handed_on_and_not_remembered() { - let reply = anthropic_error("prompt is too long: max context length exceeded"); + let reply = + anthropic_error("prompt is too long for extended thinking: max context length exceeded"); let (up, seen) = upstream(vec![(400, reply.clone())]).await; let mut gw = gateway(config(up, "anthropic", ""), None).await; let (status, body) = post(&gw, "/v1/messages", claude("max")).await; @@ -314,7 +315,11 @@ async fn a_resend_refused_again_is_handed_on_and_not_remembered() { chain[1].effort, change("max", None, tw_api::EffortReason::Rejected) ); - assert!(!gw.state.effort_memory.has("relay", "claude-opus-4-6")); + assert!(!gw.state.effort_memory.has( + "relay", + "claude-opus-4-6", + tw_dialect::ir::Dialect::Anthropic + )); post(&gw, "/v1/messages", claude("max")).await; let seen = seen.lock().unwrap(); assert_eq!(effort_of(&seen[2]), Some("max"), "没有记住"); @@ -384,7 +389,8 @@ async fn a_level_set_by_a_rule_is_fixed_like_any_other() { ); assert_eq!( chain[1].effort, - change("low", Some("max"), tw_api::EffortReason::Rejected) + change("xhigh", Some("max"), tw_api::EffortReason::Rejected), + "被拒的是规则改出来的 xhigh" ); } @@ -416,3 +422,230 @@ async fn a_converted_request_is_fixed_in_the_upstream_format() { change("xhigh", Some("max"), tw_api::EffortReason::Rejected) ); } + +/// 关着思考、写着强度的请求（Anthropic 说「关着思考时不认 xhigh，用 high 或以下」）：按写着的 +/// 字段认，照样改一档重发 +#[tokio::test] +async fn an_effort_written_with_thinking_off_is_fixed_too() { + let (up, seen) = upstream(vec![ + ( + 400, + anthropic_error( + "output_config.effort 'xhigh' is not supported when thinking is disabled on this model. Use effort 'high' or below, or enable thinking.", + ), + ), + (200, anthropic_ok()), + ]) + .await; + let mut gw = gateway(config(up, "anthropic", ""), None).await; + let mut body = claude("xhigh"); + body["thinking"] = json!({"type": "disabled"}); + let (status, text) = post(&gw, "/v1/messages", body).await; + assert_eq!(status, 200, "{text}"); + { + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), 2); + assert_eq!(effort_of(&seen[1]), Some("high")); + assert_eq!(seen[1]["thinking"], json!({"type": "disabled"})); + } + let chain = attempts(&mut gw.rx).await; + assert_eq!( + chain[1].effort, + change("xhigh", Some("high"), tw_api::EffortReason::Rejected) + ); +} + +/// 记着的改法自己被拒了：那一条删掉，照新的原话再改一次，成了记新的；重发成了用的正是原来那一档 +/// 的（上游又认了）：删掉，往后照原样发 +#[tokio::test] +async fn a_remembered_fix_that_is_refused_is_replaced_or_forgotten() { + let ceiling = anthropic_error( + "effort 'max' is not supported when thinking is disabled. Use effort 'high' or below.", + ); + let back = anthropic_error("effort 'max' is not available for this model: use xhigh"); + let (up, seen) = upstream(vec![ + (400, anthropic_error(NOT_VALID)), + (200, anthropic_ok()), + (400, ceiling), + (200, anthropic_ok()), + (200, anthropic_ok()), + ]) + .await; + let mut gw = gateway(config(up, "anthropic", ""), None).await; + let e = |i: usize| effort_of(&seen.lock().unwrap()[i]).map(str::to_string); + // 记下 xhigh → max + post(&gw, "/v1/messages", claude("xhigh")).await; + attempts(&mut gw.rx).await; + // max 自己被拒，上限 high：改记 xhigh → high + let (status, _) = post(&gw, "/v1/messages", claude("xhigh")).await; + assert_eq!(status, 200); + assert_eq!((e(2), e(3)), (Some("max".into()), Some("high".into()))); + let chain = attempts(&mut gw.rx).await; + assert_eq!( + chain[0].effort, + change("xhigh", Some("max"), tw_api::EffortReason::Remembered) + ); + assert_eq!( + chain[1].effort, + change("xhigh", Some("high"), tw_api::EffortReason::Rejected), + "asked 是改之前写明的那一档" + ); + post(&gw, "/v1/messages", claude("xhigh")).await; + assert_eq!(e(4), Some("high".into())); + attempts(&mut gw.rx).await; + + // 另一家：记着 xhigh → max，max 被拒、原话让用 xhigh，重发 xhigh 成了 —— 删掉 + let (up, seen) = upstream(vec![ + (400, anthropic_error(NOT_VALID)), + (200, anthropic_ok()), + (400, back), + (200, anthropic_ok()), + ]) + .await; + let mut gw = gateway(config(up, "anthropic", ""), None).await; + let e = |i: usize| effort_of(&seen.lock().unwrap()[i]).map(str::to_string); + post(&gw, "/v1/messages", claude("xhigh")).await; + attempts(&mut gw.rx).await; + post(&gw, "/v1/messages", claude("xhigh")).await; + assert_eq!((e(2), e(3)), (Some("max".into()), Some("xhigh".into()))); + attempts(&mut gw.rx).await; + assert!(!gw.state.effort_memory.has( + "relay", + "claude-opus-4-6", + tw_dialect::ir::Dialect::Anthropic + )); + post(&gw, "/v1/messages", claude("xhigh")).await; + assert_eq!(e(4), Some("xhigh".into())); +} + +/// 只写了预算的请求没有写明的强度：上游的 400 说到了思考和档位名也不重发、不记 +#[tokio::test] +async fn a_budget_only_request_is_not_an_effort_rejection() { + let reply = anthropic_error( + "thinking.budget_tokens: the max is lower for this model; use low, medium or high effort instead", + ); + let (up, seen) = upstream(vec![(400, reply.clone()), (200, anthropic_ok())]).await; + let gw = gateway(config(up, "anthropic", ""), None).await; + let (status, _) = post( + &gw, + "/v1/messages", + json!({"model": "claude-sonnet-4-5", "max_tokens": 64000, + "thinking": {"type": "enabled", "budget_tokens": 50000}, + "messages": [{"role": "user", "content": "hi"}]}), + ) + .await; + assert_eq!(status, 400); + assert_eq!(seen.lock().unwrap().len(), 1); + assert!(!gw.state.effort_memory.has( + "relay", + "claude-sonnet-4-5", + tw_dialect::ir::Dialect::Anthropic + )); + // 记着这个模型不认 xhigh，预算折算出来正是 xhigh 的请求也不改：预算不是写明的强度 + let p = gw.state.config().providers[0].clone(); + gw.state.effort_memory.note( + &p, + "claude-sonnet-4-5", + tw_dialect::ir::Dialect::Anthropic, + tw_dialect::params::WrittenEffort::Level(tw_dialect::ir::Effort::XHigh), + tw_dialect::think::EffortFix::Use(tw_dialect::ir::Effort::Low), + tw_gateway::effort_memory::now_secs(), + ); + post( + &gw, + "/v1/messages", + json!({"model": "claude-sonnet-4-5", "max_tokens": 64000, + "thinking": {"type": "enabled", "budget_tokens": 50000}, + "messages": [{"role": "user", "content": "hi"}]}), + ) + .await; + assert_eq!(seen.lock().unwrap()[1]["thinking"]["budget_tokens"], 50000); +} + +/// 重发马上发给同一家：排在它后面的候选一个都不碰，这家只许一个并发也不用等 +#[tokio::test] +async fn the_resend_goes_to_the_same_upstream_at_once() { + let (up, seen) = upstream(vec![ + (400, anthropic_error(NOT_VALID)), + (200, anthropic_ok()), + ]) + .await; + let (backup, backup_seen) = upstream(vec![(200, anthropic_ok())]).await; + let cfg: tw_config::Config = serde_yaml_ng::from_str(&format!( + "version: 1 +clients: + - name: c + key: tw-k +providers: + - name: relay + base_url: http://{up} + key: sk-upstream + protocol: anthropic + max_concurrent: 1 + - name: backup + base_url: http://{backup} + key: sk-backup + protocol: anthropic +groups: + - name: both + type: fallback + providers: [relay, backup] +routes: + - name: default + rules: + - {{ name: 兜底, to: both }} +" + )) + .unwrap(); + let mut gw = gateway(cfg, None).await; + let (status, body) = post(&gw, "/v1/messages", claude("xhigh")).await; + assert_eq!(status, 200, "{body}"); + assert_eq!(seen.lock().unwrap().len(), 2); + assert!(backup_seen.lock().unwrap().is_empty()); + let chain = attempts(&mut gw.rx).await; + assert_eq!( + chain + .iter() + .map(|a| a.provider.as_str()) + .collect::>(), + ["relay", "relay"] + ); + assert_eq!(chain[1].queued_ms, None, "没有排队"); +} + +/// `off` 在 OpenAI 的格式里写 `none`（去掉的话 GPT-5 这类按默认的那一档想）；上游不认 `none` +/// 的去掉再发，记下来，往后直接去掉 +#[tokio::test] +async fn off_writes_none_and_an_upstream_that_refuses_it_gets_the_field_removed() { + let refusal = json!({"error": {"message": "Unsupported value: 'none' is not supported with this model for reasoning_effort. Supported values are: 'low', 'medium', and 'high'.", + "type": "invalid_request_error", "param": "reasoning_effort", "code": "unsupported_value"}}); + let (up, seen) = upstream(vec![(400, refusal), (200, chat_ok())]).await; + let rules = " - { name: 别想, set: { reasoning: off } } + - { name: 兜底, to: relay } +"; + let mut gw = gateway(config(up, "openai-chat", rules), None).await; + let ask = json!({"model": "o3-mini", "reasoning_effort": "high", + "messages": [{"role": "user", "content": "hi"}]}); + let (status, body) = post(&gw, "/v1/chat/completions", ask.clone()).await; + assert_eq!(status, 200, "{body}"); + { + let seen = seen.lock().unwrap(); + assert_eq!(seen[0]["reasoning_effort"], "none"); + assert!(seen[1].get("reasoning_effort").is_none(), "{}", seen[1]); + } + let chain = attempts(&mut gw.rx).await; + assert_eq!( + chain[0].effort, + change("high", Some("none"), tw_api::EffortReason::Rule) + ); + assert_eq!( + chain[1].effort, + change("none", None, tw_api::EffortReason::Rejected) + ); + post(&gw, "/v1/chat/completions", ask).await; + assert!(seen.lock().unwrap()[2].get("reasoning_effort").is_none()); + assert_eq!( + attempts(&mut gw.rx).await[0].effort, + change("none", None, tw_api::EffortReason::Remembered) + ); +} diff --git a/docs/config.md b/docs/config.md index b450f399..267f2ba7 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1512,7 +1512,10 @@ request details and does not count as a retry. If it succeeds, the level is remembered for that upstream and model for 7 days in `effort-memory.json` in the data directory, so later requests send the working level directly. The entries of an upstream are forgotten when its address, key or protocol -changes, or when it is removed. +changes, or when it is removed. Requests that only give a thinking budget are +not resent. On WebSocket connections (Codex), a turn refused for its effort is +not resent and nothing is remembered: the refusal reaches the client as it +is. @@ -1521,7 +1524,7 @@ changes, or when it is removed. |---|---|---|---| | `model` | string | — | Send a different model. The prompt cache of the session is lost. | | `max_tokens` | integer | — | Replace `max_tokens`. | -| `reasoning` | `off` \| `low` \| `medium` \| `high` \| `xhigh` \| `max` | — | Reasoning. `off` turns it off and removes the reasoning effort; a level changes the effort. Only requests that already have reasoning on are changed; requests without reasoning (such as title generation) are left as they are. | +| `reasoning` | `off` \| `low` \| `medium` \| `high` \| `xhigh` \| `max` | — | Reasoning. `off` turns it off: the reasoning settings and the effort are removed, and in the OpenAI formats the effort is set to `none`. A level changes the effort. Only requests that already have reasoning on are changed; requests without reasoning (such as title generation) are left as they are. | | `only_at_session_start` | bool | `false` | Apply only when a session starts. Recorded and shown; not in effect yet. | diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index 7518e693..87bc681c 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -1091,7 +1091,7 @@ routes: `set.reasoning` 按客户端的格式写推理强度：`output_config.effort`、`reasoning_effort`、`reasoning.effort`、Gemini 的 `thinkingLevel`；客户端给的是思考预算时改预算。 -上游因为推理强度回 400 时（无论这一档是客户端写的还是规则改的），请求在同一家上游再发一次：改用上游原话里列出的、离原来那一档最近的一档，原话给了上限的用上限，一档都没列的去掉推理强度。重发在请求详情里是新的一跳，不算重试。重发成功后，这一档按上游和模型记 7 天，存在数据目录的 `effort-memory.json` 里，之后的请求直接发可用的那一档。上游的地址、密钥或协议变了，或者上游被删除，它的记录随之作废。 +上游因为推理强度回 400 时（无论这一档是客户端写的还是规则改的），请求在同一家上游再发一次：改用上游原话里列出的、离原来那一档最近的一档，原话给了上限的用上限，一档都没列的去掉推理强度。重发在请求详情里是新的一跳，不算重试。重发成功后，这一档按上游和模型记 7 天，存在数据目录的 `effort-memory.json` 里，之后的请求直接发可用的那一档。上游的地址、密钥或协议变了，或者上游被删除，它的记录随之作废。只给了思考预算的请求不重发。WebSocket 连接（Codex）上，一轮因推理强度被拒时不重发、也不记录，拒绝原样交给客户端。 @@ -1100,7 +1100,7 @@ routes: |---|---|---|---| | `model` | 字符串 | — | 换成另一个模型发送。该会话的 prompt cache 随之失效。 | | `max_tokens` | 整数 | — | 替换 `max_tokens`。 | -| `reasoning` | `off` \| `low` \| `medium` \| `high` \| `xhigh` \| `max` | — | 推理。`off` 关闭推理，连同推理强度一起去掉；写一档则改为该档强度。只改写已开启推理的请求，未开启推理的请求（如生成标题）保持不变。 | +| `reasoning` | `off` \| `low` \| `medium` \| `high` \| `xhigh` \| `max` | — | 推理。`off` 关闭推理：去掉推理设置和推理强度，OpenAI 的格式改为写 `none`。写一档则改为该档强度。只改写已开启推理的请求，未开启推理的请求（如生成标题）保持不变。 | | `only_at_session_start` | 布尔 | `false` | 只在会话开始时应用。目前只记录和显示，尚未生效。 |