diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs index 5dea51e0..231e0ef2 100644 --- a/crates/tw-api/src/lib.rs +++ b/crates/tw-api/src/lib.rs @@ -502,7 +502,8 @@ slug_enum! { /// 换模型，整个 prompt cache 作废 Model = "model", MaxTokens = "max_tokens", - Thinking = "thinking", + /// 推理：值是 [`RuleReasoning`] 的那个词 + Reasoning = "reasoning", /// 以上改写只在新会话开始时应用 OnlyAtSessionStart = "only_at_session_start", } @@ -1074,7 +1075,16 @@ pub const MSG_CODES: &str = include_str!("../msg-codes.txt"); /// [`AttemptView`] 多了 `retry`（这一跳是第几次重试）和 `backoff_ms`（发出之前退避了多久）。 /// 新码：`gw.upstream.stream_ended_early`（流只发了开头的例行事件就收了尾，算在 /// [`FAILED_AFTER_SENDING`] 里） -pub const CONTROL_API_VERSION: u32 = 53; +/// +/// **54 起路由规则改的是推理，不再是扩展思考**：[`RuleRewrite`] 的 `thinking`（布尔）换成 +/// `reasoning`（[`RuleReasoning`]：`off` 关掉推理，或者改成 `low` / `medium` / `high` / `xhigh` / +/// `max` 一档，只改开着推理的请求），[`SetField`] 的 `thinking` 换成 `reasoning`（[`SetView`] 的 +/// 值是那个词）。配置里的 `set.thinking` 不认了。[`AttemptView`] 多了 `effort`（[`EffortChange`]： +/// 这一跳发出去的推理强度和客户端要的不一样，`reason` 是 [`EffortReason`] 的 `rule` / `rejected` / +/// `remembered`）：上游因为推理强度回 400 时，同一家改一档（或者去掉强度）再发一次，是尝试链上 +/// 新的一跳，不算退避重试；记下这一家的这个模型不认那一档，7 天之内直接改。照 53 写的界面 +/// 保存规则时带的 `thinking` 不起作用 +pub const CONTROL_API_VERSION: u32 = 54; /// `/live` 的速度看最近多久（见 [`LiveView::tokens_per_sec`]），毫秒 pub const LIVE_RATE_WINDOW_MS: u64 = 3_000; @@ -2008,6 +2018,11 @@ pub struct AttemptView { /// 这一跳发出去之前退避了多少毫秒：一轮重试的头一跳带着。不算在 `ms` 里。别的没有 #[serde(default, skip_serializing_if = "Option::is_none")] pub backoff_ms: Option, + /// 这一跳改了推理强度：规则改写的、上游拒收了改用别的一档重发的、记得这个模型不认那一档直接 + /// 改了的（见 [`EffortChange`]）。改了也和原来一样的没有。被拒收的那一跳自己是一跳普通的 + /// `status`（400），重发的那一跳带着 `rejected` + #[serde(default, skip_serializing_if = "Option::is_none")] + pub effort: Option, /// 这一跳走的代理：这家上游配置的代理名（跟随系统的是 `system`），直连是 None。没发出去 /// 的一跳（满着、被拒、转换不了……）是它要走的那个 —— 连不上的那一跳，问题可能就出在 /// 这个代理上。WebSocket 只连直连的上游，一律是 None @@ -2015,6 +2030,33 @@ pub struct AttemptView { pub proxy: Option, } +/// 一跳改了推理强度（[`AttemptView::effort`]）。 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct EffortChange { + /// 这一次改之前要发的那一档：`rule` 是客户端要的；`rejected`、`remembered` 是被拒收、被换掉的 + /// 那一档（规则改过之后的）—— 客户端要 high、规则改成 xhigh、被拒后改用 max，这一项是 + /// `xhigh`。词表：`none`（OpenAI 的「关掉推理」）/ `minimal` / `low` / `medium` / `high` / + /// `xhigh` / `max` + pub asked: String, + /// 发出去的那一档，同一个词表（规则 `off` 在 OpenAI 的格式里是 `none`）。null 是去掉了推理 + /// 强度，按模型的默认 + pub sent: Option, + pub reason: EffortReason, +} + +slug_enum! { + /// 推理强度为什么改了（[`EffortChange::reason`]）。 + pub enum EffortReason { + /// 路由规则改写（`set.reasoning`） + Rule = "rule", + /// 上游拒收了上一跳的那一档（400），同一家改用这一档重发了一次 + Rejected = "rejected", + /// 这一家的这个模型拒收过那一档，记着（7 天），直接改了 + Remembered = "remembered", + } +} + /// 发出去之后才失败的一跳报的码（见 [`AttemptView::sent`]）：上游没在时限内回话、请求在路上 /// 断了、回答读到一半连接断了、流在第一段内容之前报了错。**上游可能已经收下了它、在算了**。 /// @@ -2807,9 +2849,22 @@ pub struct RuleRewrite { pub model: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub max_tokens: Option, - /// 打开或关闭扩展思考 + /// 推理：关掉，或者改成一档。**只改开着推理的请求**，没开推理的（生成标题这类）原样不动 #[serde(default, skip_serializing_if = "Option::is_none")] - pub thinking: Option, + pub reasoning: Option, +} + +slug_enum! { + /// 规则改写的推理（[`RuleRewrite::reasoning`]）。没有 `minimal` + pub enum RuleReasoning { + /// 关掉推理，连同推理强度 + Off = "off", + Low = "low", + Medium = "medium", + High = "high", + XHigh = "xhigh", + Max = "max", + } } /// 规则里的一个条件。 @@ -5949,8 +6004,9 @@ pub struct MismatchView { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[cfg_attr(feature = "ts", derive(ts_rs::TS))] pub struct SetView { - /// `model`（换模型，整个 prompt cache 作废）/ `max_tokens` / `thinking` / - /// `only_at_session_start`（以上改写只在新会话开始时应用，值是 `true`） + /// `model`（换模型，整个 prompt cache 作废）/ `max_tokens` / `reasoning`（值是 + /// [`RuleReasoning`] 的那个词）/ `only_at_session_start`（以上改写只在新会话开始时应用， + /// 值是 `true`） pub field: SetField, pub value: String, } @@ -6929,6 +6985,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } diff --git a/crates/tw-config/tests/manual/schema.rs b/crates/tw-config/tests/manual/schema.rs index 7e51af7f..fa396ea6 100644 --- a/crates/tw-config/tests/manual/schema.rs +++ b/crates/tw-config/tests/manual/schema.rs @@ -11,7 +11,9 @@ use super::{Def, Kind, Lang, Row, Section, T2, Ty}; use tw_config::proxy::ProxyAuth; use tw_config::*; use tw_engine::rule::When; -use tw_engine::{BalanceBy, Group, GroupType, Member, Pinned, RouteSet, Rule, SetAction}; +use tw_engine::{ + BalanceBy, Group, GroupType, Member, Pinned, RouteSet, Rule, SetAction, SetReasoning, +}; use tw_pricing::{PerMillion, PricingConfig, SheetDef}; const fn t(en: &'static str, zh: &'static str) -> T2 { @@ -86,6 +88,9 @@ fn balance_bys() -> Vec<&'static str> { fn limit_pers() -> Vec<&'static str> { super::fields::() } +fn set_reasonings() -> Vec<&'static str> { + super::fields::() +} /// 同步位置三种写法各自的字段。`target` 是按 `type` 分的枚举，问 serde 要字段的办法问不出 /// 枚举每一种的字段，所以借它的报错：写一个不存在的字段，serde 说出这一种认得的全部字段 @@ -1897,10 +1902,13 @@ pub fn sections() -> Vec
{ t("Replace `max_tokens`.", "替换 `max_tokens`。"), ), row( - "thinking", - Kind::Bool, + "reasoning", + Kind::Enum(set_reasonings), Def::Unset, - t("Turn extended thinking on or off.", "开启或关闭扩展思考。"), + t( + "Reasoning. `off` turns it off: the reasoning settings and the effort are removed, and in the OpenAI formats the effort is set to `none`. A level changes the effort. Only requests that already have reasoning on are changed; requests without reasoning (such as title generation) are left as they are.", + "推理。`off` 关闭推理：去掉推理设置和推理强度，OpenAI 的格式改为写 `none`。写一档则改为该档强度。只改写已开启推理的请求，未开启推理的请求（如生成标题）保持不变。", + ), ), row( "only_at_session_start", diff --git a/crates/tw-control/src/dryrun.rs b/crates/tw-control/src/dryrun.rs index 705c78b3..74550076 100644 --- a/crates/tw-control/src/dryrun.rs +++ b/crates/tw-control/src/dryrun.rs @@ -405,8 +405,8 @@ fn describe(set: &tw_engine::SetAction) -> Vec { if let Some(t) = set.max_tokens { push(SetField::MaxTokens, t.to_string()); } - if let Some(t) = set.thinking { - push(SetField::Thinking, t.to_string()); + if let Some(r) = set.reasoning { + push(SetField::Reasoning, r.slug().to_string()); } if set.only_at_session_start { push(SetField::OnlyAtSessionStart, "true".to_string()); diff --git a/crates/tw-control/src/replay.rs b/crates/tw-control/src/replay.rs index bb477123..7e1d39c5 100644 --- a/crates/tw-control/src/replay.rs +++ b/crates/tw-control/src/replay.rs @@ -191,7 +191,7 @@ fn renamed(row: &tw_store::db::RequestRow, raw: Vec, model: &str) -> (bytes: ..Default::default() }; ( - tw_gateway::forward::apply_set(&raw, &set, Some(api.dialect())), + tw_gateway::forward::apply_set(&raw, &set, Some(api.dialect()), model), row.path.clone(), ) } diff --git a/crates/tw-control/src/routes.rs b/crates/tw-control/src/routes.rs index bef80b13..dd2bdd00 100644 --- a/crates/tw-control/src/routes.rs +++ b/crates/tw-control/src/routes.rs @@ -282,7 +282,7 @@ pub(crate) fn to_rule(input: &tw_api::RuleInput, cfg: &tw_config::Config) -> Res .filter(|m| !m.is_empty()) .map(str::to_string), max_tokens: s.max_tokens, - thinking: s.thinking, + reasoning: s.reasoning.map(set_reasoning), only_at_session_start: false, }; (!a.is_empty()).then_some(a) @@ -811,7 +811,7 @@ pub(crate) fn rule_view(r: &Rule, n: tw_engine::RuleNotes) -> tw_api::RuleView { .map(|s| tw_api::RuleRewrite { model: s.model.clone(), max_tokens: s.max_tokens, - thinking: s.thinking, + reasoning: s.reasoning.map(rule_reasoning), }), catch_all: n.catch_all, phase_two: n.phase_two, @@ -819,6 +819,26 @@ pub(crate) fn rule_view(r: &Rule, n: tw_engine::RuleNotes) -> tw_api::RuleView { } } +/// 规则改写的推理：界面的词 → 配置里的 +fn set_reasoning(r: tw_api::RuleReasoning) -> tw_engine::SetReasoning { + use tw_api::RuleReasoning as R; + use tw_dialect::ir::Effort; + use tw_engine::SetReasoning as S; + match r { + R::Off => S::Off, + R::Low => S::Level(Effort::Low), + R::Medium => S::Level(Effort::Medium), + R::High => S::Level(Effort::High), + R::XHigh => S::Level(Effort::XHigh), + R::Max => S::Level(Effort::Max), + } +} + +/// 反过来。两边的词是同一张表（[`tw_engine::SetReasoning::slug`]） +fn rule_reasoning(r: tw_engine::SetReasoning) -> tw_api::RuleReasoning { + tw_api::RuleReasoning::from_slug(r.slug()).unwrap_or(tw_api::RuleReasoning::Low) +} + // ─────────────────────────────────────────────────────────── 共用 fn reserved(name: &str, what: &str) -> Result<(), Msg> { diff --git a/crates/tw-control/tests/dryrun.rs b/crates/tw-control/tests/dryrun.rs index 6bb56c2d..f20f4619 100644 --- a/crates/tw-control/tests/dryrun.rs +++ b/crates/tw-control/tests/dryrun.rs @@ -332,6 +332,27 @@ async fn a_set_only_rule_still_shows_up_as_matched_and_says_what_it_changes() { ); } +/// 推理改写在试算里是一项 `reasoning`，值是配置里的那个词 +#[tokio::test] +async fn a_reasoning_rewrite_shows_as_its_word() { + let (_d, app) = app_with(&CFG.replace( + " set: { model: claude-haiku-4-5 }", + " set: { model: claude-haiku-4-5, reasoning: xhigh }", + )); + let r = run( + &app, + r#"{"model":"claude-sonnet-4-5","input_tokens":300000}"#, + ) + .await; + assert!( + r.set + .iter() + .any(|s| s.field == tw_api::SetField::Reasoning && s.value == "xhigh"), + "{:?}", + r.set + ); +} + #[tokio::test] async fn the_dry_run_changes_nothing_and_sends_nothing() { // 它只算。跑一百遍和跑零遍，对上游和配置文件来说没有区别。 diff --git a/crates/tw-control/tests/handover.rs b/crates/tw-control/tests/handover.rs index e6f2ceab..f7fd20b5 100644 --- a/crates/tw-control/tests/handover.rs +++ b/crates/tw-control/tests/handover.rs @@ -146,7 +146,7 @@ async fn status_reports_the_handover_phase_and_the_draining_count() { let st = status(&s).await; assert_eq!(st.handover, tw_api::HandoverState::Draining); assert_eq!(st.draining, 3); - assert_eq!(st.api_version, 53); + assert_eq!(st.api_version, 54); } /// arm 之后（以及接替者正在接管时）不开始新的账号登录：回调端口不交接。（平常、排空中的 diff --git a/crates/tw-control/tests/in_flight.rs b/crates/tw-control/tests/in_flight.rs index 46de16b3..3256f073 100644 --- a/crates/tw-control/tests/in_flight.rs +++ b/crates/tw-control/tests/in_flight.rs @@ -104,6 +104,7 @@ async fn it_gives_every_running_request_with_what_has_happened_to_it_so_far() { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing: tw_api::Billing::PerToken, diff --git a/crates/tw-control/tests/key_limits.rs b/crates/tw-control/tests/key_limits.rs index 7ff289b7..056c5cf3 100644 --- a/crates/tw-control/tests/key_limits.rs +++ b/crates/tw-control/tests/key_limits.rs @@ -147,6 +147,7 @@ fn request(id: u64, key: &str, at: i64, input: u64) -> Vec { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing: tw_api::Billing::PerToken, @@ -460,6 +461,7 @@ fn hop(outcome: tw_api::AttemptOutcome, code: &str, busy: bool) -> tw_api::Attem pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } diff --git a/crates/tw-control/tests/live_state.rs b/crates/tw-control/tests/live_state.rs index 98e64db2..af4b2d15 100644 --- a/crates/tw-control/tests/live_state.rs +++ b/crates/tw-control/tests/live_state.rs @@ -161,6 +161,7 @@ async fn a_request_still_running_can_be_opened_and_becomes_whole_when_it_ends() pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing: tw_api::Billing::PerToken, diff --git a/crates/tw-control/tests/replay.rs b/crates/tw-control/tests/replay.rs index e439bae4..3e01a109 100644 --- a/crates/tw-control/tests/replay.rs +++ b/crates/tw-control/tests/replay.rs @@ -394,6 +394,7 @@ fn routed(hops: &[(&str, Option<&str>)]) -> Option { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }) .collect(); diff --git a/crates/tw-control/tests/route_stats.rs b/crates/tw-control/tests/route_stats.rs index 2671bf20..6675a834 100644 --- a/crates/tw-control/tests/route_stats.rs +++ b/crates/tw-control/tests/route_stats.rs @@ -52,7 +52,7 @@ routes: deny: 这个项目不用 opus - name: 关掉思考 when: {{ model: claude-sonnet-* }} - set: {{ thinking: false }} + set: {{ reasoning: off }} - name: 从没命中过 when: {{ model: gpt-* }} to: up diff --git a/crates/tw-control/tests/routes.rs b/crates/tw-control/tests/routes.rs index 21e40fb5..f1864a38 100644 --- a/crates/tw-control/tests/routes.rs +++ b/crates/tw-control/tests/routes.rs @@ -400,6 +400,50 @@ async fn a_rule_that_only_rewrites_can_be_saved() { assert!(codex["rules"][0].get("guard").is_none(), "{codex}"); } +/// 推理改写是一个词：存进配置写成 `reasoning: xhigh`，读回来还是那个词；不认识的词 +/// （`minimal` 也是）读不成请求体 +#[tokio::test] +async fn a_reasoning_rewrite_is_saved_as_its_word() { + let b = bed(BASE); + for word in ["off", "xhigh"] { + let (st, v) = call( + &b.app, + "PUT", + "/routes/codex", + json!({ "route": { "name": "codex", "rules": [ + { "name": "推理", "conditions": [], "set": { "reasoning": word } }, + catch_all("主力"), + ]}}), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + assert!( + b.file().contains(&format!("reasoning: {word}")), + "{}", + b.file() + ); + let codex = find(&b.overview().await["routes"], "codex").clone(); + assert_eq!( + codex["rules"][0]["set"], + json!({ "reasoning": word }), + "{codex}" + ); + } + let before = b.file(); + let (st, v) = call( + &b.app, + "PUT", + "/routes/codex", + json!({ "route": { "name": "codex", "rules": [ + { "name": "推理", "conditions": [], "set": { "reasoning": "minimal" } }, + catch_all("主力"), + ]}}), + ) + .await; + assert_eq!(st, StatusCode::UNPROCESSABLE_ENTITY, "{v}"); + assert_eq!(b.file(), before); +} + #[tokio::test] async fn deleting_a_route_sends_its_keys_where_the_user_chose() { let b = bed(BASE); diff --git a/crates/tw-control/tests/upstream_limits.rs b/crates/tw-control/tests/upstream_limits.rs index 6aa965f7..d3f7895a 100644 --- a/crates/tw-control/tests/upstream_limits.rs +++ b/crates/tw-control/tests/upstream_limits.rs @@ -110,6 +110,7 @@ fn attempt(provider: &str, status: u16) -> tw_api::AttemptView { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } diff --git a/crates/tw-dialect/src/gemini/request.rs b/crates/tw-dialect/src/gemini/request.rs index 80e21ee6..4cbbd559 100644 --- a/crates/tw-dialect/src/gemini/request.rs +++ b/crates/tw-dialect/src/gemini/request.rs @@ -512,8 +512,10 @@ pub fn encode_request(r: &Request, _t: &Target, dropped: &mut Dropped) -> Value t.insert("thinkingLevel".into(), json!(think::gemini_level(e))); } } else { - // -1 是让模型自己决定 - let budget = think::budget(re).map(|b| b as i64).unwrap_or(-1); + // -1 是让模型自己决定。超过这个模型的上限整个请求 400（Claude 那边的预算能到 64000） + let budget = think::budget(re) + .map(|b| b.min(think::gemini_budget_cap(&r.model)) as i64) + .unwrap_or(-1); t.insert("thinkingBudget".into(), json!(budget)); } if re.summary { @@ -842,6 +844,39 @@ mod tests { assert_eq!(dropped, ["thinking"]); } + /// Claude 那边的预算（最多 64000）转过来压在这个模型的上限以内，超过了整个请求 400 + #[test] + fn a_budget_stays_within_the_model_limit() { + let mut r = Request { + model: "gemini-2.5-flash".into(), + reasoning: Some(Reasoning { + enabled: true, + effort: Some(Effort::Max), + budget: None, + summary: false, + }), + ..Default::default() + }; + let (v, _) = encode(&r, Dialect::Anthropic); + assert_eq!( + v["generationConfig"]["thinkingConfig"]["thinkingBudget"], + 24576 + ); + r.model = "gemini-2.5-pro".into(); + r.reasoning.as_mut().unwrap().budget = Some(48000); + let (v, _) = encode(&r, Dialect::Anthropic); + assert_eq!( + v["generationConfig"]["thinkingConfig"]["thinkingBudget"], + 32768 + ); + r.reasoning.as_mut().unwrap().budget = Some(8192); + let (v, _) = encode(&r, Dialect::Anthropic); + assert_eq!( + v["generationConfig"]["thinkingConfig"]["thinkingBudget"], + 8192 + ); + } + #[test] fn snake_case_fields_are_understood_too() { let mut d = Dropped::new(Dialect::Gemini); diff --git a/crates/tw-dialect/src/ir.rs b/crates/tw-dialect/src/ir.rs index df773db8..1084dc63 100644 --- a/crates/tw-dialect/src/ir.rs +++ b/crates/tw-dialect/src/ir.rs @@ -344,7 +344,7 @@ pub struct Reasoning { pub summary: bool, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Effort { Minimal, Low, diff --git a/crates/tw-dialect/src/params.rs b/crates/tw-dialect/src/params.rs index 78d24b45..f94cf1ad 100644 --- a/crates/tw-dialect/src/params.rs +++ b/crates/tw-dialect/src/params.rs @@ -13,7 +13,8 @@ use serde_json::{Map, Value}; -use crate::ir::Dialect; +use crate::ir::{Dialect, Effort}; +use crate::think; /// 客户端写的最大输出 token 数。没写（或者写的不是一个非负整数）是 `None`。 pub fn max_output_tokens(dialect: Dialect, body: &Value) -> Option { @@ -188,9 +189,18 @@ pub fn set_model(dialect: Dialect, body: &mut Value, model: &str) { } } -/// 关掉推理：去掉开推理的那个字段。 +/// 关掉推理：去掉开推理的那个字段，连同推理强度。 /// /// **只做「关掉」这一个方向**：开启要一个预算，而这一层没有一个说得过去的值可以编。 +/// Anthropic 的强度（`output_config.effort`）单独写在思考开关外面，只去掉开关的话自适应的 +/// 模型照样按那一档想，要一起去掉；`output_config` 去空了的整个去掉。Converse 的透传口袋里 +/// 同理。 +/// +/// **OpenAI 的两种格式写 `none`，不是去掉**：GPT-5 这类模型没写强度时按默认的那一档想，去掉 +/// 等于没关。Chat 的 `reasoning_effort`、Responses 的 `reasoning.effort`（连同对话中途的 +/// `configuration_update`）写成 `none`；Chat 里 DeepSeek、GLM、Kimi 那种 `thinking.type: enabled` +/// 的开关整个去掉，连同强度。上游不认 `none` 的由网关去掉再发（见 +/// [`think::none_rejection`]） pub fn disable_thinking(dialect: Dialect, body: &mut Value) { let Some(obj) = body.as_object_mut() else { return; @@ -198,12 +208,31 @@ pub fn disable_thinking(dialect: Dialect, body: &mut Value) { match dialect { Dialect::Anthropic => { obj.remove("thinking"); + remove_output_effort(obj); } + // DeepSeek、GLM、Kimi 的 `thinking.type: enabled` 也是开关：只去掉强度的话推理照样开着 Dialect::Chat => { - obj.remove("reasoning_effort"); + if obj + .get("thinking") + .and_then(|t| t.get("type")) + .and_then(Value::as_str) + == Some("enabled") + { + obj.remove("thinking"); + obj.remove("reasoning_effort"); + } else if obj.contains_key("reasoning_effort") { + put(obj, "reasoning_effort", "none"); + } } Dialect::Responses => { - obj.remove("reasoning"); + if let Some(Value::Object(r)) = obj.get_mut("reasoning") { + put(r, "effort", "none"); + } + for r in responses_updates(obj) { + if r.contains_key("effort") { + put(r, "effort", "none"); + } + } } Dialect::Gemini => { for config in ["generationConfig", "generation_config"] { @@ -220,11 +249,474 @@ pub fn disable_thinking(dialect: Dialect, body: &mut Value) { .and_then(Value::as_object_mut) { f.remove("thinking"); + remove_output_effort(f); + } + } + } +} + +/// 去掉 `output_config.effort`；`output_config` 去空了的整个去掉。返回去没去掉 +fn remove_output_effort(holder: &mut Map) -> bool { + let Some(o) = holder + .get_mut("output_config") + .and_then(Value::as_object_mut) + else { + return false; + }; + let removed = o.remove("effort").is_some(); + if o.is_empty() { + holder.remove("output_config"); + } + removed +} + +// ───────────────────────────────────────────────────────── 推理强度 +// +// 「推理开着」和解码器的判据一样：Anthropic 的 `thinking.type` 是 `enabled` / `adaptive`，或者 +// 只写了 `output_config.effort`；Chat 的 `reasoning_effort` 不是 `none`，或者 GLM 那种 +// `thinking.type: enabled`；Responses 写了 `reasoning`、强度不是 `none`（对话中途的 +// `configuration_update` 最后一个说了算）；Gemini 写了 `thinkingConfig`、预算不是 0；Converse +// 的透传口袋里和 Anthropic 一样。明确关掉的（`disabled`、`none`、预算 0）不算开着。 +// +// 强度写在哪儿：Anthropic 是 `output_config.effort`，按预算思考的（`enabled` + +// `budget_tokens`、没写强度）是预算；Chat 是 `reasoning_effort`；Responses 是 +// `reasoning.effort`；Gemini 写了等级的是 `thinkingLevel`、写了预算的是 `thinkingBudget`； +// Converse 和 Anthropic 一样，在 `additionalModelRequestFields` 里。预算按 +// [`think::effort_of_budget`] 读成一档、按 [`think::budget_of_effort`] 写回去。 +// +// **写明的强度**（[`written_effort`]）只算强度字段本身，不管推理开没开、不算预算：上游因为强度 +// 拒收时认的是发出去的那个字段 —— Anthropic「关着思考时不认 `xhigh`」那种 400，`thinking` 是 +// `disabled`、强度照样写着。改它、去掉它（[`replace_reasoning_effort`]、 +// [`remove_reasoning_effort`]）也按字段在不在，用来照上游的话重发。路由规则改推理按「推理开着」 +// 走（[`set_reasoning_effort`]、[`disable_thinking`]）。 + +/// 请求体里写明的推理强度（[`written_effort`]）：一档，或者 OpenAI 的 `none` +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum WrittenEffort { + /// `none`：OpenAI 的「关掉推理」 + Off, + Level(Effort), +} + +impl WrittenEffort { + /// 写在请求体里的那个词，按 OpenAI 的写法：`none`、`minimal` … `max` + pub fn slug(self) -> &'static str { + match self { + Self::Off => "none", + Self::Level(e) => think::openai(e), + } + } + + /// 反过来。不认识的词是 `None` + pub fn parse(s: &str) -> Option { + think::parse_openai(s).map(|e| e.map_or(Self::Off, Self::Level)) + } +} + +/// 请求体里写明的推理强度：Anthropic 的 `output_config.effort`、Chat 的 `reasoning_effort`、 +/// Responses 的 `reasoning.effort`（对话中途的 `configuration_update` 最后一个说了算）、Gemini 的 +/// `thinkingLevel`、Converse 透传口袋里的 `output_config.effort`。**按字段在不在**，不看推理开没 +/// 开；只写了预算的不算。认不出的值是 `None` +pub fn written_effort(dialect: Dialect, body: &Value) -> Option { + let obj = body.as_object()?; + match dialect { + Dialect::Anthropic => output_effort(obj).map(WrittenEffort::Level), + Dialect::Chat => str_at(obj, "reasoning_effort").and_then(WrittenEffort::parse), + Dialect::Responses => responses_update(obj) + .or_else(|| obj.get("reasoning").and_then(|r| r.get("effort"))?.as_str()) + .and_then(WrittenEffort::parse), + Dialect::Gemini => gemini_thinking(obj) + .and_then(|t| either_ref(t, "thinkingLevel", "thinking_level")) + .and_then(Value::as_str) + .and_then(think::parse_gemini_level) + .map(WrittenEffort::Level), + Dialect::Bedrock => obj + .get("additionalModelRequestFields") + .and_then(Value::as_object) + .and_then(output_effort) + .map(WrittenEffort::Level), + } +} + +/// 把写明的推理强度（[`written_effort`]）改成 `e`：**只改已经写着的那个字段**，不看推理开没开， +/// 不补字段、不碰预算。返回改没改 +pub fn replace_reasoning_effort(dialect: Dialect, body: &mut Value, e: Effort) -> bool { + let Some(obj) = body.as_object_mut() else { + return false; + }; + let in_output = |holder: &mut Map| match holder + .get_mut("output_config") + .and_then(Value::as_object_mut) + { + Some(o) if o.contains_key("effort") => put(o, "effort", think::anthropic(e)), + _ => false, + }; + match dialect { + Dialect::Anthropic => in_output(obj), + Dialect::Chat => { + obj.contains_key("reasoning_effort") && put(obj, "reasoning_effort", think::openai(e)) + } + Dialect::Responses => { + let mut changed = match obj.get_mut("reasoning") { + Some(Value::Object(r)) if r.contains_key("effort") => { + put(r, "effort", think::openai(e)) + } + _ => false, + }; + for r in responses_updates(obj) { + if r.contains_key("effort") { + changed |= put(r, "effort", think::openai(e)); + } + } + changed + } + Dialect::Gemini => match gemini_thinking_mut(obj) { + Some(t) => match key_of(t, "thinkingLevel", "thinking_level") { + Some(k) => put(t, k, think::gemini_level(e)), + None => false, + }, + None => false, + }, + Dialect::Bedrock => obj + .get_mut("additionalModelRequestFields") + .and_then(Value::as_object_mut) + .is_some_and(in_output), + } +} + +/// 客户端开着推理吗（判据见上） +pub fn reasoning_on(dialect: Dialect, body: &Value) -> bool { + let Some(obj) = body.as_object() else { + return false; + }; + match dialect { + Dialect::Anthropic => claude_on(obj), + // 和解码器一样：`thinking` 说关掉的以它为准，`reasoning_effort` 认得出的看它，都没有再看 + // `thinking` 是不是开着 + Dialect::Chat => { + let switch = obj + .get("thinking") + .and_then(|t| t.get("type")) + .and_then(Value::as_str); + match str_at(obj, "reasoning_effort").and_then(think::parse_openai) { + _ if switch == Some("disabled") => false, + Some(e) => e.is_some(), + None => switch == Some("enabled"), + } + } + Dialect::Responses => match responses_update(obj).and_then(think::parse_openai) { + Some(e) => e.is_some(), + None => obj + .get("reasoning") + .filter(|r| r.is_object()) + .is_some_and(|r| r.get("effort").and_then(Value::as_str) != Some("none")), + }, + Dialect::Gemini => gemini_thinking(obj).is_some_and(|t| { + either_ref(t, "thinkingBudget", "thinking_budget").and_then(Value::as_i64) != Some(0) + }), + Dialect::Bedrock => obj + .get("additionalModelRequestFields") + .and_then(Value::as_object) + .is_some_and(claude_on), + } +} + +/// 客户端要的推理强度。推理没开、开着却没说强度（自适应的、Gemini 让模型自己定的）是 `None` +pub fn reasoning_effort(dialect: Dialect, body: &Value) -> Option { + if !reasoning_on(dialect, body) { + return None; + } + let obj = body.as_object()?; + match dialect { + Dialect::Anthropic => claude_effort(obj), + Dialect::Chat => str_at(obj, "reasoning_effort") + .and_then(think::parse_openai) + .flatten(), + Dialect::Responses => responses_update(obj) + .or_else(|| obj.get("reasoning").and_then(|r| r.get("effort"))?.as_str()) + .and_then(think::parse_openai) + .flatten(), + Dialect::Gemini => { + let t = gemini_thinking(obj)?; + match either_ref(t, "thinkingLevel", "thinking_level").and_then(Value::as_str) { + Some(l) => think::parse_gemini_level(l), + None => either_ref(t, "thinkingBudget", "thinking_budget") + .and_then(Value::as_i64) + .filter(|b| *b > 0) + .map(|b| think::effort_of_budget(b as u64)), + } + } + Dialect::Bedrock => obj + .get("additionalModelRequestFields") + .and_then(Value::as_object) + .and_then(claude_effort), + } +} + +/// 把推理强度改成 `e`。**只改开着推理的请求**：没开的（生成标题这类辅助请求、明确关掉的） +/// 原样不动，不替客户端打开推理。返回改没改。 +/// +/// 客户端写了强度的改强度；按预算思考的改预算（Anthropic 和 Converse 的预算还要小于最大输出， +/// 压不到 1024 以上的不改）；Gemini 等级、预算都没写的（让模型自己定）不改 —— 不知道这个模型 +/// 认哪一个。Anthropic 自适应、没写强度的补上 `output_config.effort`。Responses 对话中途的 +/// `configuration_update` 一起改：最后一个盖过顶层的。 +/// +/// `model` 是发给上游的模型名，Gemini 的预算按它压在上限以内（[`think::gemini_budget_cap`]）： +/// 超过了整个请求 400。Converse 没写最大输出（`inferenceConfig.maxTokens`）的不改预算：不知道 +/// 压到多少才合法 +pub fn set_reasoning_effort(dialect: Dialect, body: &mut Value, e: Effort, model: &str) -> bool { + if !reasoning_on(dialect, body) { + return false; + } + let Some(obj) = body.as_object_mut() else { + return false; + }; + match dialect { + Dialect::Anthropic => { + let max = obj.get("max_tokens").and_then(Value::as_u64); + set_claude_effort(obj, e, max, false) + } + Dialect::Chat => put(obj, "reasoning_effort", think::openai(e)), + Dialect::Responses => { + let mut changed = match obj.get_mut("reasoning") { + Some(Value::Object(r)) => put(r, "effort", think::openai(e)), + Some(_) => false, + None => { + obj.insert("reasoning".into(), serde_json::json!({})); + match obj.get_mut("reasoning") { + Some(Value::Object(r)) => put(r, "effort", think::openai(e)), + _ => false, + } + } + }; + for r in responses_updates(obj) { + if r.contains_key("effort") { + changed |= put(r, "effort", think::openai(e)); + } + } + changed + } + Dialect::Gemini => { + let Some(t) = gemini_thinking_mut(obj) else { + return false; + }; + if let Some(k) = key_of(t, "thinkingLevel", "thinking_level") { + put(t, k, think::gemini_level(e)) + } else if let Some(k) = key_of(t, "thinkingBudget", "thinking_budget") { + let budget = think::budget_of_effort(e).min(think::gemini_budget_cap(model)); + let before = t.get(k).cloned(); + t.insert(k.into(), Value::from(budget)); + before.as_ref() != t.get(k) + } else { + false + } + } + Dialect::Bedrock => { + let max = obj + .get("inferenceConfig") + .and_then(|c| c.get("maxTokens")) + .and_then(Value::as_u64); + match obj + .get_mut("additionalModelRequestFields") + .and_then(Value::as_object_mut) + { + Some(f) => set_claude_effort(f, e, max, true), + None => false, + } + } + } +} + +/// 去掉写明的推理强度（[`written_effort`]，`none` 也算），按模型的默认想。**按字段在不在**，不看 +/// 推理开没开；预算不动（按预算思考的 Anthropic 和 Converse 去掉预算整个请求不合法）。返回去没 +/// 去掉。 +/// +/// Responses 对话中途的 `configuration_update` 里的也去掉，只剩个空壳的整条去掉。 +pub fn remove_reasoning_effort(dialect: Dialect, body: &mut Value) -> bool { + let Some(obj) = body.as_object_mut() else { + return false; + }; + match dialect { + Dialect::Anthropic => remove_output_effort(obj), + Dialect::Chat => obj.remove("reasoning_effort").is_some(), + Dialect::Responses => { + let mut removed = obj + .get_mut("reasoning") + .and_then(Value::as_object_mut) + .is_some_and(|r| r.remove("effort").is_some()); + if let Some(Value::Array(items)) = obj.get_mut("input") { + let before = items.len(); + for item in items.iter_mut() { + if let Some(r) = update_reasoning(item) { + removed |= r.remove("effort").is_some(); + } + } + // 只剩 `type`（和 `id`）的一条没有意思了 + items.retain(|item| { + !(is_update(item) + && item + .get("reasoning") + .is_some_and(|r| r.as_object().is_some_and(Map::is_empty)) + && item.as_object().is_some_and(|o| { + o.keys() + .all(|k| matches!(k.as_str(), "type" | "id" | "reasoning")) + })) + }); + removed |= items.len() != before; + } + removed + } + Dialect::Gemini => { + let Some(t) = gemini_thinking_mut(obj) else { + return false; + }; + let mut removed = false; + for k in ["thinkingLevel", "thinking_level"] { + removed |= t.remove(k).is_some(); } + removed } + Dialect::Bedrock => obj + .get_mut("additionalModelRequestFields") + .and_then(Value::as_object_mut) + .is_some_and(remove_output_effort), } } +/// Anthropic（和 Converse 背后的 Claude）开着思考吗。`holder` 是装着 `thinking` 的那个对象 +fn claude_on(holder: &Map) -> bool { + match holder + .get("thinking") + .and_then(|t| t.get("type")) + .and_then(Value::as_str) + { + Some("enabled" | "adaptive") => true, + Some(_) => false, + None => output_effort(holder).is_some(), + } +} + +fn output_effort(holder: &Map) -> Option { + holder + .get("output_config") + .and_then(|o| o.get("effort")) + .and_then(Value::as_str) + .and_then(think::parse_anthropic) +} + +/// 按预算思考的预算。`thinking.type` 是 `enabled` 才有 +fn claude_budget(holder: &Map) -> Option { + let t = holder.get("thinking")?; + (t.get("type").and_then(Value::as_str) == Some("enabled")) + .then(|| t.get("budget_tokens").and_then(Value::as_u64)) + .flatten() +} + +fn claude_effort(holder: &Map) -> Option { + output_effort(holder).or_else(|| claude_budget(holder).map(think::effort_of_budget)) +} + +/// 见 [`set_reasoning_effort`]。`max` 是最大输出：预算要小于它。`need_max`：没写最大输出的不改预算 +fn set_claude_effort( + holder: &mut Map, + e: Effort, + max: Option, + need_max: bool, +) -> bool { + /// Anthropic 允许的最小思考预算 + const BUDGET_MIN: u64 = 1024; + let effort_written = holder + .get("output_config") + .is_some_and(|o| o.get("effort").is_some()); + if !effort_written && let Some(before) = claude_budget(holder) { + if need_max && max.is_none() { + return false; + } + let budget = max.map_or(think::budget_of_effort(e), |m| { + think::budget_of_effort(e).min(m.saturating_sub(1)) + }); + if budget < BUDGET_MIN || budget == before { + return false; + } + if let Some(t) = holder.get_mut("thinking").and_then(Value::as_object_mut) { + t.insert("budget_tokens".into(), Value::from(budget)); + return true; + } + return false; + } + match section(holder, "output_config") { + Some(o) => put(o, "effort", think::anthropic(e)), + None => false, + } +} + +/// 把 `obj[key]` 写成字符串 `value`。返回值变没变 +fn put(obj: &mut Map, key: &str, value: &str) -> bool { + if obj.get(key).and_then(Value::as_str) == Some(value) { + return false; + } + obj.insert(key.into(), Value::String(value.to_string())); + true +} + +fn str_at<'a>(obj: &'a Map, key: &str) -> Option<&'a str> { + obj.get(key).and_then(Value::as_str) +} + +fn is_update(item: &Value) -> bool { + item.get("type").and_then(Value::as_str) == Some("configuration_update") +} + +/// 一条 `configuration_update` 里的 `reasoning` +fn update_reasoning(item: &mut Value) -> Option<&mut Map> { + if !is_update(item) { + return None; + } + item.get_mut("reasoning").and_then(Value::as_object_mut) +} + +/// Responses 对话中途最后一次改的推理强度（`input` 里最后一条带着强度的 +/// `configuration_update`），和解码器一样它盖过顶层的 +fn responses_update(obj: &Map) -> Option<&str> { + obj.get("input")? + .as_array()? + .iter() + .filter(|i| is_update(i)) + .filter_map(|i| i.get("reasoning")?.get("effort")?.as_str()) + .next_back() +} + +/// 每一条 `configuration_update` 里的 `reasoning` +fn responses_updates(obj: &mut Map) -> Vec<&mut Map> { + match obj.get_mut("input") { + Some(Value::Array(items)) => items.iter_mut().filter_map(update_reasoning).collect(), + _ => Vec::new(), + } +} + +/// Gemini 的思考配置：生成参数里的 `thinkingConfig`，两层都认下划线的写法（先认驼峰） +fn gemini_thinking(obj: &Map) -> Option<&Map> { + let g = either_ref(obj, "generationConfig", "generation_config")?; + either(g, "thinkingConfig", "thinking_config")?.as_object() +} + +fn gemini_thinking_mut(obj: &mut Map) -> Option<&mut Map> { + let g = key_of(obj, "generationConfig", "generation_config")?; + let g = obj.get_mut(g)?.as_object_mut()?; + let t = key_of(g, "thinkingConfig", "thinking_config")?; + g.get_mut(t)?.as_object_mut() +} + +/// [`either`] 的对象版 +fn either_ref<'a>(obj: &'a Map, camel: &str, snake: &str) -> Option<&'a Value> { + obj.get(camel).or_else(|| obj.get(snake)) +} + +/// 两个写法里客户端用的那一个键，先认驼峰 +fn key_of<'k>(obj: &Map, camel: &'k str, snake: &'k str) -> Option<&'k str> { + [camel, snake].into_iter().find(|k| obj.contains_key(*k)) +} + /// Gemini 的一个字段：驼峰的在就是它，否则是下划线写法的 fn either<'a>(v: &'a Value, camel: &str, snake: &str) -> Option<&'a Value> { v.get(camel).or_else(|| v.get(snake)) @@ -681,11 +1173,6 @@ mod tests { Dialect::Anthropic, json!({"thinking": {"type": "enabled"}, "x": 1}), ), - (Dialect::Chat, json!({"reasoning_effort": "high", "x": 1})), - ( - Dialect::Responses, - json!({"reasoning": {"effort": "high"}, "x": 1}), - ), ( Dialect::Gemini, json!({"generationConfig": {"thinkingConfig": {}}, "x": 1}), @@ -705,4 +1192,577 @@ mod tests { assert_eq!(v["x"], 1); } } + + /// OpenAI 的两种格式写 `none`：去掉的话 GPT-5 这类模型按默认的那一档想，等于没关 + #[test] + fn turning_thinking_off_in_openai_formats_writes_none() { + let mut v = json!({"reasoning_effort": "high", "x": 1}); + disable_thinking(Dialect::Chat, &mut v); + assert_eq!(v, json!({"reasoning_effort": "none", "x": 1})); + assert!(!reasoning_on(Dialect::Chat, &v)); + let mut v = json!({"reasoning": {"effort": "high", "summary": "auto"}, + "input": [{"type": "configuration_update", "reasoning": {"effort": "xhigh"}}]}); + disable_thinking(Dialect::Responses, &mut v); + assert_eq!( + v, + json!({"reasoning": {"effort": "none", "summary": "auto"}, + "input": [{"type": "configuration_update", "reasoning": {"effort": "none"}}]}) + ); + assert!(!reasoning_on(Dialect::Responses, &v)); + assert_eq!( + written_effort(Dialect::Responses, &v), + Some(WrittenEffort::Off) + ); + // 没写强度、只写了摘要的：照样关 + let mut v = json!({"reasoning": {"summary": "auto"}}); + disable_thinking(Dialect::Responses, &mut v); + assert_eq!( + v, + json!({"reasoning": {"summary": "auto", "effort": "none"}}) + ); + } + + #[test] + fn turning_thinking_off_takes_the_effort_with_it() { + // 自适应的 Claude 只去掉开关的话，照样按 `output_config.effort` 那一档想 + let mut v = json!({"thinking": {"type": "adaptive"}, "output_config": {"effort": "xhigh"}}); + disable_thinking(Dialect::Anthropic, &mut v); + assert_eq!(v, json!({})); + let mut v = json!({"thinking": {"type": "adaptive"}, + "output_config": {"effort": "high", "format": {"type": "json_schema"}}}); + disable_thinking(Dialect::Anthropic, &mut v); + assert_eq!( + v, + json!({"output_config": {"format": {"type": "json_schema"}}}) + ); + let mut v = json!({"additionalModelRequestFields": { + "thinking": {"type": "adaptive"}, "output_config": {"effort": "max"}, "top_k": 3}}); + disable_thinking(Dialect::Bedrock, &mut v); + assert_eq!(v, json!({"additionalModelRequestFields": {"top_k": 3}})); + } + + #[test] + fn turning_thinking_off_in_chat_takes_the_glm_switch_too() { + // DeepSeek、GLM、Kimi 的写法：开关在 `thinking`，强度在 `reasoning_effort` + let mut v = json!({"thinking": {"type": "enabled"}, "reasoning_effort": "high", "x": 1}); + disable_thinking(Dialect::Chat, &mut v); + assert_eq!(v, json!({"x": 1})); + let mut v = json!({"thinking": {"type": "enabled"}, "x": 1}); + assert!(reasoning_on(Dialect::Chat, &v)); + disable_thinking(Dialect::Chat, &mut v); + assert_eq!(v, json!({"x": 1})); + assert!(!reasoning_on(Dialect::Chat, &v)); + } + + #[test] + fn reasoning_is_on_where_the_decoder_says_it_is() { + let on = [ + ( + Dialect::Anthropic, + json!({"thinking": {"type": "enabled", "budget_tokens": 2048}}), + ), + ( + Dialect::Anthropic, + json!({"thinking": {"type": "adaptive"}}), + ), + ( + Dialect::Anthropic, + json!({"output_config": {"effort": "low"}}), + ), + (Dialect::Chat, json!({"reasoning_effort": "high"})), + (Dialect::Chat, json!({"thinking": {"type": "enabled"}})), + ( + Dialect::Responses, + json!({"reasoning": {"summary": "auto"}}), + ), + ( + Dialect::Responses, + json!({"reasoning": {"effort": "none"}, + "input": [{"type": "configuration_update", "reasoning": {"effort": "high"}}]}), + ), + ( + Dialect::Gemini, + json!({"generationConfig": {"thinkingConfig": {"thinkingBudget": -1}}}), + ), + ( + Dialect::Gemini, + json!({"generation_config": {"thinking_config": {"thinking_level": "low"}}}), + ), + ( + Dialect::Bedrock, + json!({"additionalModelRequestFields": {"thinking": {"type": "adaptive"}}}), + ), + ]; + for (d, v) in on { + assert!(reasoning_on(d, &v), "{d:?}: {v}"); + } + let off = [ + (Dialect::Anthropic, json!({})), + ( + Dialect::Anthropic, + json!({"thinking": {"type": "disabled"}, "output_config": {"effort": "low"}}), + ), + (Dialect::Chat, json!({"reasoning_effort": "none"})), + ( + Dialect::Chat, + json!({"thinking": {"type": "disabled"}, "reasoning_effort": "high"}), + ), + ( + Dialect::Chat, + json!({"thinking": {"type": "enabled"}, "reasoning_effort": "none"}), + ), + (Dialect::Responses, json!({"reasoning": {"effort": "none"}})), + ( + Dialect::Responses, + json!({"reasoning": {"effort": "high"}, + "input": [{"type": "configuration_update", "reasoning": {"effort": "none"}}]}), + ), + ( + Dialect::Gemini, + json!({"generationConfig": {"thinkingConfig": {"thinkingBudget": 0}}}), + ), + ( + Dialect::Gemini, + json!({"generationConfig": {"temperature": 1}}), + ), + ( + Dialect::Bedrock, + json!({"additionalModelRequestFields": {"top_k": 3}}), + ), + (Dialect::Chat, json!([1])), + ]; + for (d, v) in off { + assert!(!reasoning_on(d, &v), "{d:?}: {v}"); + assert_eq!(reasoning_effort(d, &v), None, "{d:?}: {v}"); + } + } + + #[test] + fn the_effort_is_read_from_where_each_format_writes_it() { + let cases = [ + ( + Dialect::Anthropic, + json!({"thinking": {"type": "adaptive"}, "output_config": {"effort": "xhigh"}}), + Some(Effort::XHigh), + ), + ( + Dialect::Anthropic, + json!({"thinking": {"type": "adaptive"}}), + None, + ), + ( + Dialect::Anthropic, + json!({"thinking": {"type": "enabled", "budget_tokens": 10000}}), + Some(Effort::Medium), + ), + ( + Dialect::Chat, + json!({"reasoning_effort": "minimal"}), + Some(Effort::Minimal), + ), + ( + Dialect::Chat, + json!({"thinking": {"type": "enabled"}}), + None, + ), + ( + Dialect::Responses, + json!({"reasoning": {"effort": "medium"}, + "input": [{"type": "configuration_update", "reasoning": {"effort": "high"}}]}), + Some(Effort::High), + ), + ( + Dialect::Responses, + json!({"reasoning": {"effort": "max"}}), + Some(Effort::Max), + ), + ( + Dialect::Gemini, + json!({"generationConfig": {"thinkingConfig": {"thinkingLevel": "HIGH"}}}), + Some(Effort::High), + ), + ( + Dialect::Gemini, + json!({"generationConfig": {"thinkingConfig": {"thinkingBudget": 4000}}}), + Some(Effort::Low), + ), + ( + Dialect::Gemini, + json!({"generationConfig": {"thinkingConfig": {"thinkingBudget": -1}}}), + None, + ), + ( + Dialect::Bedrock, + json!({"additionalModelRequestFields": { + "thinking": {"type": "adaptive"}, "output_config": {"effort": "low"}}}), + Some(Effort::Low), + ), + ]; + for (d, v, want) in cases { + assert_eq!(reasoning_effort(d, &v), want, "{d:?}: {v}"); + } + } + + #[test] + fn setting_the_effort_writes_the_field_the_client_wrote() { + let set = |d, mut v: Value, e| { + let changed = set_reasoning_effort(d, &mut v, e, "gemini-2.5-pro"); + (changed, v) + }; + assert_eq!( + set( + Dialect::Anthropic, + json!({"thinking": {"type": "adaptive"}, "output_config": {"effort": "xhigh"}}), + Effort::Max + ), + ( + true, + json!({"thinking": {"type": "adaptive"}, "output_config": {"effort": "max"}}) + ) + ); + // 自适应、没写强度：补上 + assert_eq!( + set( + Dialect::Anthropic, + json!({"thinking": {"type": "adaptive"}}), + Effort::High + ), + ( + true, + json!({"thinking": {"type": "adaptive"}, "output_config": {"effort": "high"}}) + ) + ); + // 按预算的改预算，要小于最大输出 + assert_eq!( + set( + Dialect::Anthropic, + json!({"max_tokens": 20000, "thinking": {"type": "enabled", "budget_tokens": 4000}}), + Effort::Max + ), + ( + true, + json!({"max_tokens": 20000, "thinking": {"type": "enabled", "budget_tokens": 19999}}) + ) + ); + let tiny = + json!({"max_tokens": 1000, "thinking": {"type": "enabled", "budget_tokens": 1024}}); + assert_eq!( + set(Dialect::Anthropic, tiny.clone(), Effort::Max), + (false, tiny) + ); + // 已经是这一档：没改 + let same = json!({"reasoning_effort": "high"}); + assert_eq!( + set(Dialect::Chat, same.clone(), Effort::High), + (false, same) + ); + assert_eq!( + set( + Dialect::Chat, + json!({"thinking": {"type": "enabled"}}), + Effort::Low + ), + ( + true, + json!({"thinking": {"type": "enabled"}, "reasoning_effort": "low"}) + ) + ); + assert_eq!( + set( + Dialect::Responses, + json!({"reasoning": {"effort": "xhigh", "summary": "auto"}, + "input": [{"type": "configuration_update", "reasoning": {"effort": "medium"}}, {"type": "message"}]}), + Effort::Max + ), + ( + true, + json!({"reasoning": {"effort": "max", "summary": "auto"}, + "input": [{"type": "configuration_update", "reasoning": {"effort": "max"}}, {"type": "message"}]}) + ) + ); + assert_eq!( + set( + Dialect::Gemini, + json!({"generationConfig": {"thinkingConfig": {"thinkingLevel": "low"}}}), + Effort::Max + ), + ( + true, + json!({"generationConfig": {"thinkingConfig": {"thinkingLevel": "HIGH"}}}) + ) + ); + assert_eq!( + set( + Dialect::Gemini, + json!({"generation_config": {"thinking_config": {"thinking_budget": 1024}}}), + Effort::High + ), + ( + true, + json!({"generation_config": {"thinking_config": {"thinking_budget": 32000}}}) + ) + ); + // Gemini 让模型自己定的：不知道它认等级还是预算，不改 + let dynamic = json!({"generationConfig": {"thinkingConfig": {"includeThoughts": true}}}); + assert_eq!( + set(Dialect::Gemini, dynamic.clone(), Effort::High), + (false, dynamic) + ); + assert_eq!( + set( + Dialect::Bedrock, + json!({"inferenceConfig": {"maxTokens": 9000}, + "additionalModelRequestFields": {"thinking": {"type": "enabled", "budget_tokens": 2000}}}), + Effort::High + ), + ( + true, + json!({"inferenceConfig": {"maxTokens": 9000}, + "additionalModelRequestFields": {"thinking": {"type": "enabled", "budget_tokens": 8999}}}) + ) + ); + assert_eq!( + set( + Dialect::Bedrock, + json!({"additionalModelRequestFields": { + "thinking": {"type": "adaptive"}, "output_config": {"effort": "xhigh"}}}), + Effort::High + ), + ( + true, + json!({"additionalModelRequestFields": { + "thinking": {"type": "adaptive"}, "output_config": {"effort": "high"}}}) + ) + ); + } + + #[test] + fn a_request_without_reasoning_is_never_given_any() { + for (d, v) in [ + (Dialect::Anthropic, json!({"max_tokens": 10})), + ( + Dialect::Anthropic, + json!({"thinking": {"type": "disabled"}}), + ), + (Dialect::Chat, json!({"reasoning_effort": "none"})), + (Dialect::Responses, json!({"input": "hi"})), + ( + Dialect::Gemini, + json!({"generationConfig": {"thinkingConfig": {"thinkingBudget": 0}}}), + ), + ( + Dialect::Bedrock, + json!({"inferenceConfig": {"maxTokens": 10}}), + ), + ] { + let mut w = v.clone(); + assert!(!set_reasoning_effort(d, &mut w, Effort::Max, "m"), "{d:?}"); + assert_eq!(w, v, "{d:?}"); + } + } + + #[test] + fn removing_the_effort_leaves_reasoning_on() { + let remove = |d, mut v: Value| { + let changed = remove_reasoning_effort(d, &mut v); + (changed, v) + }; + assert_eq!( + remove( + Dialect::Anthropic, + json!({"thinking": {"type": "adaptive"}, "output_config": {"effort": "xhigh"}}) + ), + (true, json!({"thinking": {"type": "adaptive"}})) + ); + // 预算是开思考必须的：不动 + let budget = json!({"thinking": {"type": "enabled", "budget_tokens": 4000}}); + assert_eq!(remove(Dialect::Anthropic, budget.clone()), (false, budget)); + assert_eq!( + remove( + Dialect::Chat, + json!({"thinking": {"type": "enabled"}, "reasoning_effort": "max"}) + ), + (true, json!({"thinking": {"type": "enabled"}})) + ); + assert_eq!( + remove( + Dialect::Responses, + json!({"reasoning": {"effort": "xhigh", "summary": "auto"}, + "input": [{"type": "configuration_update", "reasoning": {"effort": "high"}}, {"type": "message"}]}) + ), + ( + true, + json!({"reasoning": {"summary": "auto"}, "input": [{"type": "message"}]}) + ) + ); + assert_eq!( + remove( + Dialect::Gemini, + json!({"generationConfig": {"thinkingConfig": {"thinkingLevel": "LOW", "includeThoughts": true}}}) + ), + ( + true, + json!({"generationConfig": {"thinkingConfig": {"includeThoughts": true}}}) + ) + ); + assert_eq!( + remove( + Dialect::Bedrock, + json!({"additionalModelRequestFields": { + "thinking": {"type": "adaptive"}, "output_config": {"effort": "max"}}}) + ), + ( + true, + json!({"additionalModelRequestFields": {"thinking": {"type": "adaptive"}}}) + ) + ); + } + + #[test] + fn what_is_set_is_what_is_read_back() { + let bodies = [ + ( + Dialect::Anthropic, + json!({"thinking": {"type": "adaptive"}}), + ), + (Dialect::Chat, json!({"reasoning_effort": "low"})), + (Dialect::Responses, json!({"reasoning": {}})), + ( + Dialect::Bedrock, + json!({"additionalModelRequestFields": {"thinking": {"type": "adaptive"}}}), + ), + ]; + for (d, v) in bodies { + for e in [ + Effort::Low, + Effort::Medium, + Effort::High, + Effort::XHigh, + Effort::Max, + ] { + let mut w = v.clone(); + set_reasoning_effort(d, &mut w, e, "m"); + assert_eq!(reasoning_effort(d, &w), Some(e), "{d:?}: {w}"); + } + } + } + + /// 写明的强度按字段在不在：关着思考也照样写着（上游正是为它拒收的）；只写了预算的不算 + #[test] + fn the_written_effort_is_the_field_itself() { + let disabled = + json!({"thinking": {"type": "disabled"}, "output_config": {"effort": "xhigh"}}); + assert!(!reasoning_on(Dialect::Anthropic, &disabled)); + assert_eq!(reasoning_effort(Dialect::Anthropic, &disabled), None); + assert_eq!( + written_effort(Dialect::Anthropic, &disabled), + Some(WrittenEffort::Level(Effort::XHigh)) + ); + let mut v = disabled.clone(); + assert!(replace_reasoning_effort( + Dialect::Anthropic, + &mut v, + Effort::High + )); + assert_eq!(v["output_config"]["effort"], "high"); + let mut v = disabled.clone(); + assert!(remove_reasoning_effort(Dialect::Anthropic, &mut v)); + assert_eq!(v, json!({"thinking": {"type": "disabled"}})); + let bedrock = json!({"additionalModelRequestFields": { + "thinking": {"type": "disabled"}, "output_config": {"effort": "max"}}}); + assert_eq!( + written_effort(Dialect::Bedrock, &bedrock), + Some(WrittenEffort::Level(Effort::Max)) + ); + let mut v = bedrock.clone(); + assert!(replace_reasoning_effort( + Dialect::Bedrock, + &mut v, + Effort::High + )); + assert_eq!( + v["additionalModelRequestFields"]["output_config"]["effort"], + "high" + ); + // 只写了预算：不是写明的强度，改它、去掉它都不动 + for (d, budget) in [ + ( + Dialect::Anthropic, + json!({"thinking": {"type": "enabled", "budget_tokens": 4000}}), + ), + ( + Dialect::Gemini, + json!({"generationConfig": {"thinkingConfig": {"thinkingBudget": 4000}}}), + ), + ( + Dialect::Bedrock, + json!({"additionalModelRequestFields": {"thinking": {"type": "enabled", "budget_tokens": 4000}}}), + ), + ] { + assert_eq!(written_effort(d, &budget), None, "{d:?}"); + assert!(reasoning_effort(d, &budget).is_some(), "{d:?}"); + let mut v = budget.clone(); + assert!(!replace_reasoning_effort(d, &mut v, Effort::Max), "{d:?}"); + assert!(!remove_reasoning_effort(d, &mut v), "{d:?}"); + assert_eq!(v, budget, "{d:?}"); + } + // `none` 也是写明的 + let none = json!({"reasoning_effort": "none"}); + assert_eq!( + written_effort(Dialect::Chat, &none), + Some(WrittenEffort::Off) + ); + let mut v = none.clone(); + assert!(remove_reasoning_effort(Dialect::Chat, &mut v)); + assert_eq!(v, json!({})); + assert_eq!(WrittenEffort::Off.slug(), "none"); + assert_eq!(WrittenEffort::parse("none"), Some(WrittenEffort::Off)); + assert_eq!( + WrittenEffort::parse("max"), + Some(WrittenEffort::Level(Effort::Max)) + ); + // 改只改写着的字段，不补 + let mut v = json!({"thinking": {"type": "adaptive"}}); + assert!(!replace_reasoning_effort( + Dialect::Anthropic, + &mut v, + Effort::Max + )); + assert_eq!(v, json!({"thinking": {"type": "adaptive"}})); + let mut v = json!({"thinking": {"type": "enabled"}}); + assert!(!replace_reasoning_effort( + Dialect::Chat, + &mut v, + Effort::Max + )); + } + + /// Gemini 2.5 的预算有上限：Pro 32768，Flash 24576。Converse 没写最大输出的不改预算 + #[test] + fn budgets_stay_within_what_the_model_takes() { + let gemini = json!({"generationConfig": {"thinkingConfig": {"thinkingBudget": 1024}}}); + for (model, e, want) in [ + ("gemini-2.5-pro", Effort::Max, 32768), + ("gemini-2.5-flash", Effort::Max, 24576), + ("gemini-2.5-flash-lite", Effort::XHigh, 24576), + ("gemini-2.5-flash", Effort::Medium, 10000), + ] { + let mut v = gemini.clone(); + assert!(set_reasoning_effort(Dialect::Gemini, &mut v, e, model)); + assert_eq!( + v["generationConfig"]["thinkingConfig"]["thinkingBudget"], want, + "{model}" + ); + } + let bedrock = json!({"additionalModelRequestFields": { + "thinking": {"type": "enabled", "budget_tokens": 2000}}}); + let mut v = bedrock.clone(); + assert!(!set_reasoning_effort( + Dialect::Bedrock, + &mut v, + Effort::Max, + "m" + )); + assert_eq!(v, bedrock); + } } diff --git a/crates/tw-dialect/src/think.rs b/crates/tw-dialect/src/think.rs index 5a012a0f..67378f53 100644 --- a/crates/tw-dialect/src/think.rs +++ b/crates/tw-dialect/src/think.rs @@ -163,6 +163,145 @@ pub(crate) fn continues_tool_turn_without_thinking(messages: &[crate::ir::Messag ) } +/// 由低到高的全部档位 +pub const LEVELS: [Effort; 6] = [ + Effort::Minimal, + Effort::Low, + Effort::Medium, + Effort::High, + Effort::XHigh, + Effort::Max, +]; + +/// `levels` 里离 `want` 最近的那一档。一样近的取高的那一档：上游说「只有 high 和 max」时， +/// 要 xhigh 的客户端要的是多想，不是少想。`levels` 是空的是 `None` +pub fn nearest(levels: &[Effort], want: Effort) -> Option { + let rank = |e: Effort| e as i32; + levels + .iter() + .copied() + .min_by_key(|e| ((rank(*e) - rank(want)).abs(), std::cmp::Reverse(rank(*e)))) +} + +/// 推理强度被上游拒了之后，重发时怎么写。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EffortFix { + /// 改成这一档 + Use(Effort), + /// 去掉强度，按模型的默认 + Drop, +} + +/// 上游这个错误是不是因为推理强度：是的话，重发时怎么写。`sent` 是这一跳发出去的那一档。 +/// +/// **只认 400**，原话（不分大小写）里要说到推理（`effort`、`reasoning`、`thinking`、「思考」 +/// 「推理」之一），而且说到 `effort`、或者整词说到发出去的那一档、或者整词说到另外至少两档 —— +/// 「该模型始终思考……请使用 low、high 或 max」这样不提 `effort` 的也认得出；`text.verbosity` +/// 的 low / medium / high、内容审查的 `severity` 不提推理，不算。 +/// +/// 重发写哪一档看原话里的档位（不算发出去的那一档）：恰好一档后面紧跟着「or below」「or +/// lower」「以下」的，那一档是上限，就用它；不然取离发出去的那一档最近的（[`nearest`]）； +/// 一档都没说的，去掉强度。 +pub fn effort_rejection(status: u16, error_body: &str, sent: Effort) -> Option { + let text = error_body.to_lowercase(); + let found = levels_in(&text); + if !about_effort(status, &text, openai(sent), &found) { + return None; + } + let others: Vec<(usize, Effort)> = found.iter().copied().filter(|(_, e)| *e != sent).collect(); + let mut distinct: Vec = others.iter().map(|(_, e)| *e).collect(); + distinct.sort(); + distinct.dedup(); + let mut ceilings: Vec = others + .iter() + .filter(|(end, _)| ceiling_after(&text[*end..])) + .map(|(_, e)| *e) + .collect(); + ceilings.sort(); + ceilings.dedup(); + if let [only] = ceilings.as_slice() { + return Some(EffortFix::Use(*only)); + } + Some(match nearest(&distinct, sent) { + Some(e) => EffortFix::Use(e), + None => EffortFix::Drop, + }) +} + +/// 发出去的是 `none`（OpenAI 的「关掉推理」，见 [`crate::params::disable_thinking`]），上游这个 +/// 错误是不是因为它：判据和 [`effort_rejection`] 一样，`none` 当作发出去的那一档。是的话重发时 +/// 去掉这个字段 +pub fn none_rejection(status: u16, error_body: &str) -> bool { + let text = error_body.to_lowercase(); + about_effort(status, &text, "none", &levels_in(&text)) +} + +/// 见 [`effort_rejection`]。`text` 是小写的原话，`sent` 是发出去的那个词，`found` 是原话里的档位 +fn about_effort(status: u16, text: &str, sent: &str, found: &[(usize, Effort)]) -> bool { + if status != 400 + || !["effort", "reasoning", "thinking", "思考", "推理"] + .iter() + .any(|w| text.contains(w)) + { + return false; + } + let mut others: Vec = found + .iter() + .map(|(_, e)| *e) + .filter(|e| openai(*e) != sent) + .collect(); + others.sort(); + others.dedup(); + text.contains("effort") || whole_word(text, sent) || others.len() >= 2 +} + +/// `word` 在小写的 `text` 里整词出现过 +fn whole_word(text: &str, word: &str) -> bool { + let w = |c: char| c.is_ascii_alphanumeric() || c == '_'; + text.match_indices(word).any(|(at, _)| { + let end = at + word.len(); + !text[..at].chars().next_back().is_some_and(w) && !text[end..].chars().next().is_some_and(w) + }) +} + +/// 小写的 `text` 里整词出现的档位名：每一处的结尾位置和是哪一档 +fn levels_in(text: &str) -> Vec<(usize, Effort)> { + let word = |c: char| c.is_ascii_alphanumeric() || c == '_'; + let mut out = Vec::new(); + for e in LEVELS { + let name = openai(e); + for (at, _) in text.match_indices(name) { + let end = at + name.len(); + let before = text[..at].chars().next_back(); + let after = text[end..].chars().next(); + if !before.is_some_and(word) && !after.is_some_and(word) { + out.push((end, e)); + } + } + } + out.sort_by_key(|(end, _)| *end); + out +} + +/// 档位名后面紧跟着的是不是「以下」：中间可以隔着引号、括号和空白 +fn ceiling_after(rest: &str) -> bool { + let rest = rest.trim_start_matches(|c: char| { + c.is_whitespace() || matches!(c, '\'' | '"' | '`' | '’' | '”' | '」' | ')' | '）') + }); + ["or below", "or lower", "以下"] + .iter() + .any(|p| rest.starts_with(p)) +} + +/// Gemini 2.5 的思考预算最多多少：Pro 是 32768，Flash、Flash-Lite 是 24576。超过了整个请求 400 +pub fn gemini_budget_cap(model: &str) -> u64 { + if model.to_ascii_lowercase().contains("pro") { + 32768 + } else { + 24576 + } +} + /// 这个 Gemini 模型用 `thinkingLevel` 而不是 `thinkingBudget` pub fn gemini_uses_level(model: &str) -> bool { let m = model.to_ascii_lowercase(); @@ -226,6 +365,101 @@ mod tests { } } + #[test] + fn the_nearest_level_breaks_ties_upward() { + use Effort::*; + assert_eq!(nearest(&[Low, High, Max], Medium), Some(High)); + assert_eq!(nearest(&[High, Max], XHigh), Some(Max)); + assert_eq!(nearest(&[Low, Medium], Max), Some(Medium)); + assert_eq!(nearest(&[Minimal], High), Some(Minimal)); + assert_eq!(nearest(&[], High), None); + } + + #[test] + fn an_effort_rejection_names_what_to_send_instead() { + use Effort::*; + // Anthropic：列出认的那几档 → 离 xhigh 最近的，high 和 max 一样近，取 max + let anthropic = r#"{"type":"error","error":{"type":"invalid_request_error","message":"The parameter `output_config.effort` specified in the request are not valid: expected `low`, `medium`, `high` or `max`, but got `xhigh` instead."}}"#; + assert_eq!( + effort_rejection(400, anthropic, XHigh), + Some(EffortFix::Use(Max)) + ); + // 「high 或以下」：上限 + let ceiling = "output_config.effort 'xhigh' is not supported when thinking is disabled on this model. Use effort 'high' or below, or enable thinking."; + assert_eq!( + effort_rejection(400, ceiling, XHigh), + Some(EffortFix::Use(High)) + ); + assert_eq!( + effort_rejection(400, "effort max is too high; use medium or lower", Max), + Some(EffortFix::Use(Medium)) + ); + // 不提 effort、也不提发出去的那一档，列了别的几档 + let glm = r#"{"error":{"code":"1210","message":"该模型始终思考，不支持关闭思考；请使用 low、high 或 max。"}}"#; + assert_eq!( + effort_rejection(400, glm, Medium), + Some(EffortFix::Use(High)) + ); + // 说到了强度、没说用哪一档：去掉 + assert_eq!( + effort_rejection(400, "reasoning_effort is not supported by this model", High), + Some(EffortFix::Drop) + ); + assert_eq!( + effort_rejection(400, "Unsupported value: 'xhigh' for reasoning", XHigh), + Some(EffortFix::Drop) + ); + // 不说到推理的：不算，哪怕列了好几档、说到了发出去的那一档 + assert_eq!( + effort_rejection(400, "Unsupported value: 'xhigh'", XHigh), + None + ); + assert_eq!( + effort_rejection(400, "prompt is too long: max context length exceeded", Max), + None + ); + let verbosity = r#"{"error":{"message":"Unsupported value: 'low' is not supported with the 'gpt-5-codex' model. Supported values are: 'medium' and 'high'.","type":"invalid_request_error","param":"text.verbosity","code":"unsupported_value"}}"#; + assert_eq!(effort_rejection(400, verbosity, Medium), None); + assert_eq!(effort_rejection(400, verbosity, Low), None); + let azure = r#"{"error":{"message":"The response was filtered due to the prompt triggering Azure OpenAI's content management policy.","type":null,"param":"prompt","code":"content_filter","status":400,"innererror":{"code":"ResponsibleAIPolicyViolation","content_filter_result":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"low"},"sexual":{"filtered":true,"severity":"medium"},"violence":{"filtered":false,"severity":"high"}}}}}"#; + assert_eq!(effort_rejection(400, azure, Medium), None); + assert_eq!(effort_rejection(400, azure, XHigh), None); + // 与强度无关的 400、不是 400 的：不管 + assert_eq!( + effort_rejection( + 400, + r#"{"error":{"message":"messages: field required"}}"#, + High + ), + None + ); + assert_eq!(effort_rejection(500, anthropic, XHigh), None); + // 只说到一个别的档位、不是整词的不算 + assert_eq!( + effort_rejection(400, "max_tokens must be at most 8192", High), + None + ); + assert_eq!(effort_rejection(400, "xhighly unusual", High), None); + } + + #[test] + fn a_refused_none_is_recognised_like_a_level() { + assert!(none_rejection( + 400, + "Unsupported value: 'none' is not supported with this model for reasoning.effort. Supported values are: 'low', 'medium', and 'high'." + )); + assert!(none_rejection( + 400, + "该模型始终思考，不支持关闭思考；请使用 low、high 或 max。" + )); + assert!(!none_rejection(400, "Unsupported value: 'none'")); + assert!(!none_rejection( + 500, + "reasoning_effort none is not supported" + )); + assert!(!none_rejection(400, "messages: field required")); + } + #[test] fn openai_none_turns_reasoning_off() { assert_eq!(parse_openai("none"), Some(None)); diff --git a/crates/tw-engine/src/engine.rs b/crates/tw-engine/src/engine.rs index b8bf476b..c2b8efac 100644 --- a/crates/tw-engine/src/engine.rs +++ b/crates/tw-engine/src/engine.rs @@ -5,6 +5,7 @@ //! 所以引擎只有一条代码路径，用户看到的却是零配置可用。 use serde::{Deserialize, Serialize}; +use tw_dialect::ir::Effort; use tw_types::{Msg, msg}; use crate::facts::RequestFacts; @@ -320,8 +321,12 @@ pub struct SetAction { pub model: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub max_tokens: Option, + /// 推理：`off` 关掉，或者改成一档（`low` / `medium` / `high` / `xhigh` / `max`）。 + /// + /// **只改开着推理的请求**：没开推理的（生成标题这类辅助请求）原样不动，改成一档也不替它 + /// 打开 —— 开推理要多花钱，不该由一条规则悄悄替客户端决定。`off` 连同推理强度一起去掉 #[serde(default, skip_serializing_if = "Option::is_none")] - pub thinking: Option, + pub reasoning: Option, /// 只在新会话开始时应用，跑到一半不动它。 /// /// **长会话场景下这应该是默认值**，但会话识别是 M3 的事 —— @@ -340,14 +345,86 @@ impl SetAction { if other.max_tokens.is_some() { self.max_tokens = other.max_tokens; } - if other.thinking.is_some() { - self.thinking = other.thinking; + if other.reasoning.is_some() { + self.reasoning = other.reasoning; } self.only_at_session_start |= other.only_at_session_start; } pub fn is_empty(&self) -> bool { - self.model.is_none() && self.max_tokens.is_none() && self.thinking.is_none() + self.model.is_none() && self.max_tokens.is_none() && self.reasoning.is_none() + } +} + +/// 规则改写的推理（[`SetAction::reasoning`]）。配置里写成一个词：`off`、`low`、`medium`、 +/// `high`、`xhigh`、`max`。**没有 `minimal`**：只有 OpenAI 认它，写给别家等于 `low`，规则里 +/// 写它只会让人以为比 `low` 还少 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(from = "ReasoningWord", into = "ReasoningWord")] +pub enum SetReasoning { + /// 关掉推理 + Off, + /// 改成这一档 + Level(Effort), +} + +impl SetReasoning { + /// 写在配置里的那个词 + pub fn slug(self) -> &'static str { + ReasoningWord::from(self).slug() + } +} + +/// [`SetReasoning`] 在配置里的写法。不认识的词（`minimal` 也是）读配置时报出来，列出认得的 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +enum ReasoningWord { + Off, + Low, + Medium, + High, + #[serde(rename = "xhigh")] + XHigh, + Max, +} + +impl ReasoningWord { + fn slug(self) -> &'static str { + match self { + Self::Off => "off", + Self::Low => "low", + Self::Medium => "medium", + Self::High => "high", + Self::XHigh => "xhigh", + Self::Max => "max", + } + } +} + +impl From for SetReasoning { + fn from(w: ReasoningWord) -> Self { + match w { + ReasoningWord::Off => Self::Off, + ReasoningWord::Low => Self::Level(Effort::Low), + ReasoningWord::Medium => Self::Level(Effort::Medium), + ReasoningWord::High => Self::Level(Effort::High), + ReasoningWord::XHigh => Self::Level(Effort::XHigh), + ReasoningWord::Max => Self::Level(Effort::Max), + } + } +} + +impl From for ReasoningWord { + fn from(r: SetReasoning) -> Self { + match r { + SetReasoning::Off => Self::Off, + // 构造不出 `minimal`（配置里不认它），真有的话按最接近的 `low` 写 + SetReasoning::Level(Effort::Minimal | Effort::Low) => Self::Low, + SetReasoning::Level(Effort::Medium) => Self::Medium, + SetReasoning::Level(Effort::High) => Self::High, + SetReasoning::Level(Effort::XHigh) => Self::XHigh, + SetReasoning::Level(Effort::Max) => Self::Max, + } } } @@ -1897,7 +1974,7 @@ mod tests { "{}", None, Some(SetAction { - thinking: Some(false), + reasoning: Some(SetReasoning::Off), ..Default::default() }), None, @@ -1918,7 +1995,7 @@ mod tests { ); let d = decision(&e, &facts("x")); assert_eq!(d.candidates, ["a"], "去向是第一条有 to 的"); - assert_eq!(d.set.thinking, Some(false)); + assert_eq!(d.set.reasoning, Some(SetReasoning::Off)); assert_eq!( d.set.model.as_deref(), Some("cheap"), @@ -1926,13 +2003,52 @@ mod tests { ); } + /// `set.reasoning` 是一个词；`minimal` 和不认识的词读配置时就报出来，列出认得的。 + /// 旧的 `thinking` 不认了 + #[test] + fn a_reasoning_rewrite_is_one_word() { + let parse = |y: &str| serde_yaml_ng::from_str::(y); + for (word, want) in [ + ("off", SetReasoning::Off), + ("low", SetReasoning::Level(Effort::Low)), + ("medium", SetReasoning::Level(Effort::Medium)), + ("high", SetReasoning::Level(Effort::High)), + ("xhigh", SetReasoning::Level(Effort::XHigh)), + ("max", SetReasoning::Level(Effort::Max)), + ] { + let s = parse(&format!("reasoning: {word}")).unwrap(); + assert_eq!(s.reasoning, Some(want)); + assert_eq!(want.slug(), word); + assert_eq!( + serde_yaml_ng::to_string(&s).unwrap().trim(), + format!("reasoning: {word}") + ); + } + for bad in ["minimal", "none", "true", "XHIGH"] { + let e = parse(&format!("reasoning: {bad}")).unwrap_err().to_string(); + assert!( + e.contains("reasoning") && e.contains("`xhigh`") && e.contains("`off`"), + "{bad}: {e}" + ); + } + assert!(parse("thinking: false").is_err()); + // 后面的规则盖过前面的 + let mut a = parse("reasoning: off").unwrap(); + a.merge(&parse("reasoning: max").unwrap()); + assert_eq!(a.reasoning, Some(SetReasoning::Level(Effort::Max))); + a.merge(&parse("max_tokens: 9").unwrap()); + assert_eq!(a.reasoning, Some(SetReasoning::Level(Effort::Max))); + assert!(!a.is_empty()); + assert!(!parse("reasoning: off").unwrap().is_empty()); + } + /// 改写是谁加的要记下来：命中了、真的改了东西的才算，没命中的、只写了 /// `only_at_session_start` 的不算。决定去向的那条自己带着改写时也算。 #[test] fn the_decision_names_every_rule_that_added_a_rewrite() { let thinking_off = || { Some(SetAction { - thinking: Some(false), + reasoning: Some(SetReasoning::Off), ..Default::default() }) }; @@ -1988,7 +2104,7 @@ mod tests { "{ provider_would_be: relay }", None, Some(SetAction { - thinking: Some(false), + reasoning: Some(SetReasoning::Off), ..Default::default() }), None, @@ -2082,7 +2198,7 @@ mod tests { "{ provider_would_be: relay }", None, Some(SetAction { - thinking: Some(false), + reasoning: Some(SetReasoning::Off), ..Default::default() }), None, @@ -2092,7 +2208,7 @@ mod tests { ); let d = decision(&e, &facts("x")); assert_eq!(d.matched_rule, "兜底"); - assert_eq!(d.set.thinking, None, "阶段一不该看见阶段二的 set"); + assert_eq!(d.set.reasoning, None, "阶段一不该看见阶段二的 set"); } #[test] @@ -2106,7 +2222,7 @@ mod tests { "{ provider_would_be: relay }", None, Some(SetAction { - thinking: Some(false), + reasoning: Some(SetReasoning::Off), ..Default::default() }), None, @@ -2116,12 +2232,12 @@ mod tests { ); let base = SetAction::default(); match e.phase_two(&facts("x"), "relay", &base).unwrap() { - Outcome2::Proceed { set: s, .. } => assert_eq!(s.thinking, Some(false)), + Outcome2::Proceed { set: s, .. } => assert_eq!(s.reasoning, Some(SetReasoning::Off)), other => panic!("{other:?}"), } // 换一家就不该命中了 —— 这正是故障转移后必须重跑的理由 match e.phase_two(&facts("x"), "official", &base).unwrap() { - Outcome2::Proceed { set: s, .. } => assert_eq!(s.thinking, None), + Outcome2::Proceed { set: s, .. } => assert_eq!(s.reasoning, None), other => panic!("{other:?}"), } } @@ -2139,7 +2255,7 @@ mod tests { "{ provider_would_be: [a, b] }", None, Some(SetAction { - thinking: Some(false), + reasoning: Some(SetReasoning::Off), ..Default::default() }), None, @@ -2148,9 +2264,13 @@ mod tests { ], ); let base = SetAction::default(); - for (p, want) in [("a", Some(false)), ("b", Some(false)), ("c", None)] { + for (p, want) in [ + ("a", Some(SetReasoning::Off)), + ("b", Some(SetReasoning::Off)), + ("c", None), + ] { match e.phase_two(&facts("x"), p, &base).unwrap() { - Outcome2::Proceed { set: s, .. } => assert_eq!(s.thinking, want, "provider={p}"), + Outcome2::Proceed { set: s, .. } => assert_eq!(s.reasoning, want, "provider={p}"), other => panic!("{other:?}"), } } @@ -3012,7 +3132,7 @@ to: ", ), rule( - "{name: 阶段二, when: {provider_would_be: relay}, set: {model: x, thinking: false}}", + "{name: 阶段二, when: {provider_would_be: relay}, set: {model: x, reasoning: off}}", ), rule("{name: 兜底, to: __all__}"), ]; @@ -3040,7 +3160,10 @@ to: Outcome2::Proceed { model, set, .. } => { let a = e.asked_of(&facts("opus"), &d, "relay", model.as_deref()); assert_eq!((a.model.as_str(), a.origin), ("opus", Origin::Pinned)); - assert_eq!((set.thinking, set.max_tokens), (Some(false), Some(10))); + assert_eq!( + (set.reasoning, set.max_tokens), + (Some(SetReasoning::Off), Some(10)) + ); } other => panic!("{other:?}"), } diff --git a/crates/tw-engine/src/lib.rs b/crates/tw-engine/src/lib.rs index f9dab4ca..a2be0d21 100644 --- a/crates/tw-engine/src/lib.rs +++ b/crates/tw-engine/src/lib.rs @@ -10,8 +10,8 @@ pub use catalog::{Catalog, ProviderModels}; pub use engine::{ ALL_UPSTREAMS, Asked, BalanceBy, CATCH_ALL_RULE, DEFAULT_ROUTE, Decision, Engine, Facts, Group, GroupType, Origin, Outcome, Outcome2, Pinned, RESERVED_PREFIX, RouteError, RouteSet, Rule, - RuleNotes, SetAction, Target, balance_factors, has_catch_all, is_builtin_group, notes, - order_by, scalar_name, + RuleNotes, SetAction, SetReasoning, Target, balance_factors, has_catch_all, is_builtin_group, + notes, order_by, scalar_name, }; pub use facts::{ RequestFacts, Text as TextTokens, TokenParts, estimate_parts, estimate_strings, estimate_tokens, diff --git a/crates/tw-gateway/src/effort_memory.rs b/crates/tw-gateway/src/effort_memory.rs new file mode 100644 index 00000000..8a8d9340 --- /dev/null +++ b/crates/tw-gateway/src/effort_memory.rs @@ -0,0 +1,532 @@ +//! 上游不认的推理强度，记下来。 +//! +//! 各家认的档位不一样：Claude 的有的模型没有 `xhigh`，关着思考的不收 `high` 以上，GLM 这类 +//! 始终思考的模型不收 `medium`，有的 OpenAI 兼容接口不收 `none`。请求体里写明的那一档（见 +//! [`tw_dialect::params::written_effort`]，客户端写的或者路由规则改的）被上游拒了（400，原话说的是 +//! 推理强度，见 [`tw_dialect::think::effort_rejection`]、[`tw_dialect::think::none_rejection`]）， +//! 同一家改一档、或者去掉强度马上再发一次。**重发成了才记**：（上游，发给它的模型名，发出去的 +//! 格式）不认哪一档、改用什么。往后发给它的请求要是还写着那一档，发之前直接改，不用每次都先被 +//! 拒一次。重发又被拒了的不记；记着的改法自己被拒了、重发成了用的正是原来那一档的，那一条删掉。 +//! 只写了预算的请求不查也不记：预算不是写明的强度。 +//! +//! **7 天后忘掉**：上游可能已经认了，那时再试原来那一档。存在数据目录的 `effort-memory.json` +//! 里，重启之后还在；一家上游的地址、密钥、协议变了，或者删掉了，它的这几条随之作废 —— 那已经 +//! 是另一个上游了。 +//! +//! WebSocket 连接上的每一轮不重发也不记：一轮被拒了照原样交给客户端。 + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use serde::{Deserialize, Serialize}; +use tw_dialect::ir::{Dialect, Effort}; +use tw_dialect::params::WrittenEffort; +use tw_dialect::think::{self, EffortFix}; + +/// 存在数据目录（配置文件所在的目录）里的文件名 +pub const FILE: &str = "effort-memory.json"; + +/// 记多久，秒 +pub const KEEP_SECS: u64 = 7 * 24 * 3600; + +/// 最多记多少条。满了丢最早记下的那一条：丢了也只是下次再被拒一次、再重发一次 +const MAX: usize = 1024; + +/// 一条：这一家的这个模型（按这种格式发的）不认 `rejected`，改用 `use`（`None` 是去掉强度） +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct Entry { + provider: String, + model: String, + /// 发出去的格式：`anthropic` / `chat` / `responses` / `gemini` / `bedrock` + dialect: String, + /// 记下时这家长什么样（地址、凭据、协议的指纹）。对不上就作废 + identity: String, + /// 写明的那一档，`none` 也是（[`WrittenEffort::slug`]） + rejected: String, + #[serde(rename = "use")] + instead: Option, + /// 记下的时刻，Unix 秒 + at: u64, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +struct File { + entries: Vec, +} + +type Key = (String, String, Dialect, WrittenEffort); + +#[derive(Default)] +struct Inner { + /// 文件在哪儿。没告诉过数据目录（测试里直接建的网关）时只在内存里记 + path: Option, + by: HashMap, u64)>, +} + +/// 见模块说明。**跨重载存活**：配置换了只作废变了的那几家 +#[derive(Default)] +pub struct EffortMemory { + inner: Mutex, + /// 写文件排着队：后写的那一次写的一定是最新的那一份 + writing: Mutex<()>, +} + +/// 决定这几条还作不作数的那几样：地址、凭据、协议。**存指纹，不存原文** —— 这个文件不该多出 +/// 一份密钥 +fn identity(p: &tw_config::Provider) -> String { + let raw = format!( + "{}|{}|{:?}", + p.base_url, + p.credential_identity(), + p.effective_protocol() + ); + blake3::hash(raw.as_bytes()).to_hex().to_string() +} + +fn fresh(at: u64, now: u64) -> bool { + now.saturating_sub(at) < KEEP_SECS +} + +pub fn now_secs() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +const DIALECTS: [(Dialect, &str); 5] = [ + (Dialect::Anthropic, "anthropic"), + (Dialect::Chat, "chat"), + (Dialect::Responses, "responses"), + (Dialect::Gemini, "gemini"), + (Dialect::Bedrock, "bedrock"), +]; + +fn dialect_slug(d: Dialect) -> &'static str { + DIALECTS + .iter() + .find(|(x, _)| *x == d) + .map_or("anthropic", |(_, s)| s) +} + +fn parse_dialect(s: &str) -> Option { + DIALECTS.iter().find(|(_, x)| *x == s).map(|(d, _)| *d) +} + +impl EffortMemory { + fn lock(&self) -> std::sync::MutexGuard<'_, Inner> { + self.inner.lock().unwrap_or_else(|e| e.into_inner()) + } + + /// 告诉它数据目录在哪儿：读进那里的文件（读不了、读不成的当没有：丢了只是再被拒一次）。 + /// 往后记下的写回那里。 + /// + /// **读进来时不对账**：安全模式下在服务的是临时顶上的配置，一家上游都没有，对它对账会把 + /// 记着的全删掉。查的时候照样比这家的身份（[`Self::lookup`]），删掉的、变了的上游的那几条 + /// 等下一次换入配置时清掉（[`Self::reconcile`]） + pub fn set_dir(&self, dir: PathBuf) { + let path = dir.join(FILE); + let file: File = std::fs::read(&path) + .ok() + .and_then(|b| serde_json::from_slice(&b).ok()) + .unwrap_or_default(); + let mut g = self.lock(); + g.path = Some(path); + g.by.clear(); + for e in file.entries { + let (Some(dialect), Some(rejected)) = + (parse_dialect(&e.dialect), WrittenEffort::parse(&e.rejected)) + else { + continue; + }; + let instead = match e.instead.as_deref() { + None => None, + Some(s) => match think::parse_openai(s).flatten() { + Some(x) => Some(x), + None => continue, + }, + }; + g.by.insert( + (e.provider, e.model, dialect, rejected), + (e.identity, instead, e.at), + ); + } + } + + /// 换入一份配置之后对一遍：删掉的上游、地址密钥协议变了的上游，它们的几条作废。**只对真的 + /// 配置**：安全模式下临时顶上的那份不走这里（见 [`Self::set_dir`]） + pub fn reconcile(self: &Arc, cfg: &tw_config::Config) { + let ids: HashMap<&str, String> = cfg + .providers + .iter() + .map(|p| (p.name.as_str(), identity(p))) + .collect(); + let now = now_secs(); + let changed = { + let mut g = self.lock(); + let before = g.by.len(); + g.by.retain(|(provider, ..), (id, _, at)| { + ids.get(provider.as_str()) == Some(id) && fresh(*at, now) + }); + g.by.len() != before + }; + if changed { + self.save(); + } + } + + /// 这一家的这个模型（按这种格式发的）有没有记着什么。发之前先问这一句，没有的话不用解请求体 + pub fn has(&self, provider: &str, model: &str, dialect: Dialect) -> bool { + let g = self.lock(); + !g.by.is_empty() + && g.by + .keys() + .any(|(p, m, d, _)| p == provider && m == model && *d == dialect) + } + + /// 发给这一家的这个模型的请求（`dialect` 格式）写着 `rejected`：记着该改成什么的话交回来 + pub fn lookup( + &self, + p: &tw_config::Provider, + model: &str, + dialect: Dialect, + rejected: WrittenEffort, + now: u64, + ) -> Option { + let g = self.lock(); + let (id, instead, at) = + g.by.get(&(p.name.clone(), model.to_string(), dialect, rejected))?; + (fresh(*at, now) && *id == identity(p)).then_some(match instead { + Some(e) => EffortFix::Use(*e), + None => EffortFix::Drop, + }) + } + + /// 记下：这一家的这个模型（`dialect` 格式）不认 `rejected`，改用 `fix` 重发成了。改用的就是 + /// 它自己的（上游又认了），那一条删掉 + pub fn note( + self: &Arc, + p: &tw_config::Provider, + model: &str, + dialect: Dialect, + rejected: WrittenEffort, + fix: EffortFix, + now: u64, + ) { + let instead = match fix { + EffortFix::Use(e) if WrittenEffort::Level(e) == rejected => { + self.forget(&p.name, model, dialect, rejected); + return; + } + EffortFix::Use(e) => Some(e), + EffortFix::Drop => None, + }; + { + let mut g = self.lock(); + let key = (p.name.clone(), model.to_string(), dialect, rejected); + if g.by.len() >= MAX + && !g.by.contains_key(&key) + && let Some(oldest) = + g.by.iter() + .min_by_key(|(_, (_, _, at))| *at) + .map(|(k, _)| k.clone()) + { + g.by.remove(&oldest); + } + g.by.insert(key, (identity(p), instead, now)); + } + tracing::info!( + provider = %p.name, + model = %model, + rejected = rejected.slug(), + instead = instead.map_or("(removed)", think::openai), + "remembered a reasoning effort this model does not take" + ); + self.save(); + } + + /// 删掉一条：记着的改法自己被拒了，或者上游又认原来那一档了 + pub fn forget( + self: &Arc, + provider: &str, + model: &str, + dialect: Dialect, + rejected: WrittenEffort, + ) { + let removed = self + .lock() + .by + .remove(&(provider.to_string(), model.to_string(), dialect, rejected)) + .is_some(); + if removed { + self.save(); + } + } + + /// 写回文件。**不在转发那条路上写**：有异步运行时就挪到阻塞线程上 + fn save(self: &Arc) { + let me = self.clone(); + match tokio::runtime::Handle::try_current() { + Ok(h) => { + h.spawn_blocking(move || me.save_now()); + } + Err(_) => me.save_now(), + } + } + + /// 写回文件（原子写）。排着队写，写的时候现取最新的那一份 + pub(crate) fn save_now(&self) { + let _w = self.writing.lock().unwrap_or_else(|e| e.into_inner()); + let (path, text) = { + let g = self.lock(); + let Some(path) = g.path.clone() else { + return; + }; + let now = now_secs(); + let mut entries: Vec = + g.by.iter() + .filter(|(_, (_, _, at))| fresh(*at, now)) + .map( + |((provider, model, dialect, rejected), (id, instead, at))| Entry { + provider: provider.clone(), + model: model.clone(), + dialect: dialect_slug(*dialect).to_string(), + identity: id.clone(), + rejected: rejected.slug().to_string(), + instead: instead.map(|e| think::openai(e).to_string()), + at: *at, + }, + ) + .collect(); + entries.sort_by(|a, b| { + (&a.provider, &a.model, &a.dialect, &a.rejected).cmp(&( + &b.provider, + &b.model, + &b.dialect, + &b.rejected, + )) + }); + match serde_json::to_string_pretty(&File { entries }) { + Ok(t) => (path, t), + Err(_) => return, + } + }; + if let Err(e) = tw_config::store::write_atomic(&path, &text) { + tracing::warn!("the reasoning effort memory could not be written: {e}"); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const A: Dialect = Dialect::Anthropic; + const L: fn(Effort) -> WrittenEffort = WrittenEffort::Level; + + fn provider(name: &str, key: &str) -> tw_config::Provider { + tw_config::Provider { + name: name.into(), + base_url: "https://relay.example.com".into(), + key: Some(key.into()), + ..Default::default() + } + } + + fn config(providers: Vec) -> tw_config::Config { + tw_config::Config { + version: 1, + providers, + ..Default::default() + } + } + + #[test] + fn a_remembered_level_is_replaced_until_it_expires() { + let m = Arc::new(EffortMemory::default()); + let p = provider("relay", "sk-1"); + let now = 1_000_000; + assert!(!m.has("relay", "glm-5", A)); + m.note( + &p, + "glm-5", + A, + L(Effort::Medium), + EffortFix::Use(Effort::High), + now, + ); + assert!(m.has("relay", "glm-5", A)); + assert!(!m.has("relay", "glm-4", A)); + // 按发出去的格式分开记：同一个模型名换一种格式发是另一回事 + assert!(!m.has("relay", "glm-5", Dialect::Chat)); + assert_eq!( + m.lookup(&p, "glm-5", Dialect::Chat, L(Effort::Medium), now), + None + ); + assert_eq!( + m.lookup(&p, "glm-5", A, L(Effort::Medium), now + 60), + Some(EffortFix::Use(Effort::High)) + ); + assert_eq!(m.lookup(&p, "glm-5", A, L(Effort::Low), now), None); + // 7 天之后再试原来那一档 + assert_eq!( + m.lookup(&p, "glm-5", A, L(Effort::Medium), now + KEEP_SECS), + None + ); + // 去掉强度也记得住，`none` 也是一档 + m.note(&p, "glm-5", A, L(Effort::Max), EffortFix::Drop, now); + assert_eq!( + m.lookup(&p, "glm-5", A, L(Effort::Max), now), + Some(EffortFix::Drop) + ); + m.note( + &p, + "gpt-x", + Dialect::Chat, + WrittenEffort::Off, + EffortFix::Drop, + now, + ); + assert_eq!( + m.lookup(&p, "gpt-x", Dialect::Chat, WrittenEffort::Off, now), + Some(EffortFix::Drop) + ); + // 重发成了用的正是原来那一档：上游又认了，删掉 + m.note( + &p, + "glm-5", + A, + L(Effort::Medium), + EffortFix::Use(Effort::Medium), + now, + ); + assert_eq!(m.lookup(&p, "glm-5", A, L(Effort::Medium), now), None); + m.forget("relay", "glm-5", A, L(Effort::Max)); + assert_eq!(m.lookup(&p, "glm-5", A, L(Effort::Max), now), None); + } + + #[test] + fn a_changed_or_removed_upstream_forgets_what_it_refused() { + let m = Arc::new(EffortMemory::default()); + let now = now_secs(); + let (a, b) = (provider("a", "sk-1"), provider("b", "sk-2")); + let x = L(Effort::XHigh); + m.note(&a, "m", A, x, EffortFix::Use(Effort::Max), now); + m.note(&b, "m", A, x, EffortFix::Use(Effort::Max), now); + // 换了密钥：已经是另一个上游了，查的时候就不作数 + let rekeyed = provider("a", "sk-9"); + assert_eq!(m.lookup(&rekeyed, "m", A, x, now), None); + m.reconcile(&config(vec![rekeyed.clone()])); + assert!(!m.has("a", "m", A), "密钥变了"); + assert!(!m.has("b", "m", A), "删掉了"); + // 地址、协议变了也一样 + m.note(&rekeyed, "m", A, x, EffortFix::Drop, now); + let moved = tw_config::Provider { + base_url: "https://other.example.com".into(), + ..rekeyed.clone() + }; + m.reconcile(&config(vec![moved])); + assert!(!m.has("a", "m", A)); + m.note(&rekeyed, "m", A, x, EffortFix::Drop, now); + let reprotocoled = tw_config::Provider { + protocol: Some(tw_config::Protocol::OpenaiChat), + ..rekeyed.clone() + }; + m.reconcile(&config(vec![reprotocoled])); + assert!(!m.has("a", "m", A)); + // 没变的留着 + m.note(&rekeyed, "m", A, x, EffortFix::Drop, now); + m.reconcile(&config(vec![rekeyed.clone()])); + assert!(m.has("a", "m", A)); + } + + #[test] + fn what_is_remembered_survives_a_restart_in_the_data_directory() { + let dir = tempfile::tempdir().unwrap(); + let p = provider("relay", "sk-1"); + let now = now_secs(); + let m = Arc::new(EffortMemory::default()); + m.set_dir(dir.path().to_path_buf()); + m.note( + &p, + "claude-opus-4-6", + A, + L(Effort::XHigh), + EffortFix::Use(Effort::Max), + now, + ); + m.note( + &p, + "gpt-x", + Dialect::Responses, + WrittenEffort::Off, + EffortFix::Drop, + now, + ); + m.save_now(); + let text = std::fs::read_to_string(dir.path().join(FILE)).unwrap(); + assert!(text.contains("\"rejected\": \"xhigh\""), "{text}"); + assert!(text.contains("\"use\": \"max\""), "{text}"); + assert!(text.contains("\"rejected\": \"none\""), "{text}"); + assert!(text.contains("\"dialect\": \"responses\""), "{text}"); + assert!(!text.contains("sk-1"), "不存密钥：{text}"); + + let again = Arc::new(EffortMemory::default()); + again.set_dir(dir.path().to_path_buf()); + assert_eq!( + again.lookup(&p, "claude-opus-4-6", A, L(Effort::XHigh), now), + Some(EffortFix::Use(Effort::Max)) + ); + assert_eq!( + again.lookup(&p, "gpt-x", Dialect::Responses, WrittenEffort::Off, now), + Some(EffortFix::Drop) + ); + // 读进来时这家已经换了密钥：查不到；换入真的配置时清掉 + let rekeyed = provider("relay", "sk-2"); + assert_eq!( + again.lookup(&rekeyed, "claude-opus-4-6", A, L(Effort::XHigh), now), + None + ); + again.reconcile(&config(vec![rekeyed])); + again.save_now(); + let text = std::fs::read_to_string(dir.path().join(FILE)).unwrap(); + assert!(!text.contains("claude-opus-4-6"), "{text}"); + // 读不成的文件当没有 + std::fs::write(dir.path().join(FILE), "not json").unwrap(); + let broken = Arc::new(EffortMemory::default()); + broken.set_dir(dir.path().to_path_buf()); + assert!(!broken.has("relay", "gpt-x", Dialect::Responses)); + } + + /// 安全模式下网关拿着一份一家上游都没有的临时配置起来：读进文件时不对账，记着的一条不丢 + #[test] + fn safe_mode_does_not_wipe_the_file() { + let dir = tempfile::tempdir().unwrap(); + let p = provider("relay", "sk-1"); + let now = now_secs(); + let m = Arc::new(EffortMemory::default()); + m.set_dir(dir.path().to_path_buf()); + m.note( + &p, + "m", + A, + L(Effort::XHigh), + EffortFix::Use(Effort::Max), + now, + ); + m.save_now(); + let gw = crate::AppState::new(tw_config::Config::default()).unwrap(); + gw.set_config_dir(dir.path().to_path_buf()); + std::thread::sleep(std::time::Duration::from_millis(50)); + let text = std::fs::read_to_string(dir.path().join(FILE)).unwrap(); + assert!(text.contains("\"use\": \"max\""), "{text}"); + assert!(gw.effort_memory.has("relay", "m", A)); + // 修好了，换入真的配置：这家没变，留着 + gw.reload(config(vec![p.clone()])).unwrap(); + assert_eq!( + gw.effort_memory.lookup(&p, "m", A, L(Effort::XHigh), now), + Some(EffortFix::Use(Effort::Max)) + ); + } +} diff --git a/crates/tw-gateway/src/forward.rs b/crates/tw-gateway/src/forward.rs index fb491ac0..72f1422d 100644 --- a/crates/tw-gateway/src/forward.rs +++ b/crates/tw-gateway/src/forward.rs @@ -79,10 +79,14 @@ pub fn gemini_path_with_model(path: &str, model: &str) -> String { /// **只动改了的那几个字段，别的字节一个不动**（见 [`crate::splice`]）：解析再写回的话， /// 每个对象的键都按字母重排了，工具定义、工具参数一起变了样。改出来和原来一样的（规则 /// 写的值客户端本来就是这么写的）原样发。改到的对象里同一个键写了两遍的，才照旧写回。 +/// +/// `model` 是发给上游的模型名：Gemini 的思考预算按它压在上限以内（见 +/// [`tw_dialect::params::set_reasoning_effort`]） pub fn apply_set( body: &Bytes, set: &tw_engine::SetAction, client: Option, + model: &str, ) -> Bytes { if set.is_empty() { return body.clone(); @@ -96,13 +100,23 @@ pub fn apply_set( return body.clone(); } let mut v = before.clone(); - set_params(&mut v, set, client); - match crate::splice::rewrite(body, &before, &v) { + set_params(&mut v, set, client, model); + rewritten(body, &before, &v) +} + +/// 请求体 `body`（解出来是 `before`）改成了 `after`：只动改了的那几个字段，别的字节一个不动 +/// （见 [`crate::splice`]）。一样的原样交回 +pub(crate) fn rewritten( + body: &Bytes, + before: &serde_json::Value, + after: &serde_json::Value, +) -> Bytes { + match crate::splice::rewrite(body, before, after) { crate::splice::Rewrite::Changed(b) => return Bytes::from(b), crate::splice::Rewrite::Same => return body.clone(), crate::splice::Rewrite::Unusual => {} } - match serde_json::to_vec(&v) { + match serde_json::to_vec(after) { Ok(b) => Bytes::from(b), // 序列化不该失败，但真失败了宁可发原文也不要发半个 body Err(e) => { @@ -119,6 +133,7 @@ fn set_params( v: &mut serde_json::Value, set: &tw_engine::SetAction, client: Option, + model: &str, ) { use tw_dialect::params; let client = client.unwrap_or(tw_dialect::ir::Dialect::Anthropic); @@ -128,14 +143,16 @@ fn set_params( if let Some(t) = set.max_tokens { params::set_max_output_tokens(client, v, t); } - match set.thinking { - // 开启思考需要一个 budget，而我们没有一个合理的值可以编。 - // **只做「关掉」这一个方向** —— 那是降级场景真正需要的。 - Some(true) => tracing::warn!( - "set.thinking: true is not supported yet (it needs budget_tokens); ignoring it" - ), - Some(false) => params::disable_thinking(client, v), - None => {} + // 推理只改开着推理的请求：没开的不替它打开（开推理要一个预算，也要多花钱），关掉的也 + // 不动 —— 去掉 `reasoning_effort: "none"` 反倒让推理按模型的默认打开了 + match set.reasoning { + Some(tw_engine::SetReasoning::Off) if params::reasoning_on(client, v) => { + params::disable_thinking(client, v) + } + Some(tw_engine::SetReasoning::Level(e)) => { + params::set_reasoning_effort(client, v, e, model); + } + _ => {} } } @@ -239,7 +256,10 @@ mod tests { // 出站直通。cc-switch 那次把缓存命中率从 99% 打到 20%， // 就是因为一个「看起来无害」的重写跑在了每个请求上。 let b = Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#); - assert_eq!(apply_set(&b, &tw_engine::SetAction::default(), None), b); + assert_eq!( + apply_set(&b, &tw_engine::SetAction::default(), None, "m"), + b + ); } #[test] @@ -252,6 +272,7 @@ mod tests { ..Default::default() }, None, + "m", ); let v: serde_json::Value = serde_json::from_slice(&out).unwrap(); assert_eq!(v["model"], "haiku"); @@ -266,15 +287,36 @@ mod tests { let out = apply_set( &b, &tw_engine::SetAction { - thinking: Some(false), + reasoning: Some(tw_engine::SetReasoning::Off), ..Default::default() }, None, + "m", ); let v: serde_json::Value = serde_json::from_slice(&out).unwrap(); assert!(v.get("thinking").is_none()); } + /// Gemini 的预算按发给它的模型压在上限以内：规则写 max 不能变成一个必然 400 的 64000 + #[test] + fn a_gemini_budget_from_a_rule_stays_within_the_model_limit() { + let b = Bytes::from_static( + br#"{"contents":[],"generationConfig":{"thinkingConfig":{"thinkingBudget":2048}}}"#, + ); + let s = tw_engine::SetAction { + reasoning: Some(tw_engine::SetReasoning::Level(tw_dialect::ir::Effort::Max)), + ..Default::default() + }; + for (model, want) in [("gemini-2.5-flash", 24576), ("gemini-2.5-pro", 32768)] { + let out = apply_set(&b, &s, Some(tw_dialect::ir::Dialect::Gemini), model); + let v: serde_json::Value = serde_json::from_slice(&out).unwrap(); + assert_eq!( + v["generationConfig"]["thinkingConfig"]["thinkingBudget"], want, + "{model}" + ); + } + } + #[test] fn a_body_we_cannot_parse_is_forwarded_untouched() { // 我们的解析器不认识的东西，上游可能完全认识。 @@ -286,6 +328,7 @@ mod tests { ..Default::default() }, None, + "m", ); assert_eq!(out, b); } @@ -296,7 +339,7 @@ mod tests { let set = tw_engine::SetAction { model: Some("small".into()), max_tokens: Some(512), - thinking: Some(false), + reasoning: Some(tw_engine::SetReasoning::Off), ..Default::default() }; let run = |body: &'static str, d| { @@ -304,6 +347,7 @@ mod tests { &Bytes::from_static(body.as_bytes()), &set, Some(d), + "m", )) .unwrap() }; @@ -315,13 +359,15 @@ mod tests { (v["model"].as_str(), v["max_completion_tokens"].as_u64()), (Some("small"), Some(512)) ); - assert!(v.get("reasoning_effort").is_none() && v.get("max_tokens").is_none()); + // OpenAI 的格式里 `off` 写 `none`：去掉的话按模型默认的那一档想 + assert_eq!(v["reasoning_effort"], "none"); + assert!(v.get("max_tokens").is_none()); let v = run( r#"{"model":"big","reasoning":{"effort":"high"}}"#, Dialect::Responses, ); assert_eq!(v["max_output_tokens"], 512); - assert!(v.get("reasoning").is_none()); + assert_eq!(v["reasoning"], serde_json::json!({"effort": "none"})); let v = run( r#"{"contents":[],"generationConfig":{"thinkingConfig":{"thinkingBudget":9}}}"#, Dialect::Gemini, @@ -340,12 +386,12 @@ mod tests { fn set( model: Option<&str>, max_tokens: Option, - thinking: Option, + reasoning: Option, ) -> tw_engine::SetAction { tw_engine::SetAction { model: model.map(str::to_string), max_tokens, - thinking, + reasoning, ..Default::default() } } @@ -353,7 +399,7 @@ mod tests { /// 以前的写法：解析、改、整份写回（键按字母重排） fn whole(body: &[u8], s: &tw_engine::SetAction, d: Option) -> Bytes { let mut v: serde_json::Value = serde_json::from_slice(body).unwrap(); - set_params(&mut v, s, d); + set_params(&mut v, s, d, "m"); Bytes::from(serde_json::to_vec(&v).unwrap()) } @@ -369,8 +415,13 @@ mod tests { ); let out = apply_set( &Bytes::from(body.clone()), - &set(Some("claude-haiku-5"), Some(4096), Some(false)), + &set( + Some("claude-haiku-5"), + Some(4096), + Some(tw_engine::SetReasoning::Off), + ), Some(Dialect::Anthropic), + "m", ); assert_eq!( std::str::from_utf8(&out).unwrap(), @@ -383,6 +434,7 @@ mod tests { &Bytes::from(format!(r#"{{"model":"m",{tools}}}"#)), &set(None, Some(10), None), Some(Dialect::Anthropic), + "m", ); assert_eq!( std::str::from_utf8(&out).unwrap(), @@ -394,8 +446,13 @@ mod tests { ); let out = apply_set( &Bytes::from(gemini), - &set(Some("ignored"), Some(512), Some(false)), + &set( + Some("ignored"), + Some(512), + Some(tw_engine::SetReasoning::Off), + ), Some(Dialect::Gemini), + "m", ); assert_eq!( std::str::from_utf8(&out).unwrap(), @@ -409,7 +466,7 @@ mod tests { #[test] fn a_set_that_changes_nothing_sends_the_same_bytes() { let b = Bytes::from_static(br#"{ "max_tokens" : 100, "b":{"z":1,"a":2} }"#); - let out = apply_set(&b, &set(None, Some(100), None), None); + let out = apply_set(&b, &set(None, Some(100), None), None, "m"); assert_eq!(out.as_ptr(), b.as_ptr()); } @@ -420,7 +477,7 @@ mod tests { let b = br#"{"model":"a","x":1,"model":"b"}"#; let s = set(Some("c"), None, None); assert_eq!( - apply_set(&Bytes::from_static(b), &s, None), + apply_set(&Bytes::from_static(b), &s, None, "m"), whole(b, &s, None) ); } @@ -461,6 +518,7 @@ mod tests { "\"thinking\":{\"type\":\"enabled\",\"budget_tokens\":9}", "\"reasoning_effort\":\"high\"", "\"reasoning\":{\"effort\":\"low\"}", + "\"output_config\":{\"effort\":\"xhigh\"}", "\"inferenceConfig\":{\"temperature\":1,\"maxTokens\":3}", "\"additionalModelRequestFields\":{\"thinking\":{\"type\":\"enabled\"},\"k\":1}", "\"tools\":[{\"input_schema\":{\"type\":\"object\",\"properties\":{\"b\":{},\"a\":{}}}}]", @@ -491,13 +549,17 @@ mod tests { let s = set( [None, Some("small"), Some("big")][r.below(3)], [None, Some(9), Some(512)][r.below(3)], - [None, Some(false), Some(true)][r.below(3)], + [ + None, + Some(tw_engine::SetReasoning::Off), + Some(tw_engine::SetReasoning::Level(tw_dialect::ir::Effort::Max)), + ][r.below(3)], ); if s.is_empty() { continue; } let d = Some(dialects[r.below(dialects.len())]); - let out = apply_set(&Bytes::from(body.clone()), &s, d); + let out = apply_set(&Bytes::from(body.clone()), &s, d, "m"); let value = |b: &[u8]| serde_json::from_slice::(b).unwrap(); assert_eq!( value(&out), @@ -506,7 +568,7 @@ mod tests { ); let before = value(body.as_bytes()); let mut after = before.clone(); - set_params(&mut after, &s, d); + set_params(&mut after, &s, d, "m"); if crate::splice::rewrite(body.as_bytes(), &before, &after) == crate::splice::Rewrite::Unusual { diff --git a/crates/tw-gateway/src/lib.rs b/crates/tw-gateway/src/lib.rs index 3ddbb9d4..2a766094 100644 --- a/crates/tw-gateway/src/lib.rs +++ b/crates/tw-gateway/src/lib.rs @@ -20,6 +20,7 @@ pub mod client_api; pub mod clientprobe; pub mod content; pub mod count; +pub mod effort_memory; pub mod egress; pub mod ending; pub mod error; diff --git a/crates/tw-gateway/src/server.rs b/crates/tw-gateway/src/server.rs index b376472a..d9b605ad 100644 --- a/crates/tw-gateway/src/server.rs +++ b/crates/tw-gateway/src/server.rs @@ -389,6 +389,7 @@ pub(crate) fn hop( pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } @@ -419,6 +420,7 @@ pub(crate) fn hop_busy( pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } @@ -471,6 +473,7 @@ pub(crate) fn hop_waiting( pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } @@ -502,6 +505,7 @@ pub(crate) fn hop_used_up( pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } @@ -526,6 +530,7 @@ pub(crate) fn hop_failed( pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } diff --git a/crates/tw-gateway/src/server/pipeline/hop.rs b/crates/tw-gateway/src/server/pipeline/hop.rs index 2334e34c..cce2b940 100644 --- a/crates/tw-gateway/src/server/pipeline/hop.rs +++ b/crates/tw-gateway/src/server/pipeline/hop.rs @@ -24,6 +24,8 @@ use crate::failure::{Cause, Verdict}; use crate::forward; use crate::server::{hop, hop_failed, note_health}; use crate::state::{AppState, Runtime}; +use tw_dialect::params::WrittenEffort; +use tw_dialect::think::EffortFix; use tw_types::msg; /// 接下了这个请求的那一家，和回程要用的东西。 @@ -199,9 +201,21 @@ pub(super) async fn try_upstreams<'a>( // 插件写进来的、这个请求已经报过的命中：每一跳再查一遍，同样的只报一次（见 // `crate::guard::Reported`） let mut reported = crate::guard::Reported::default(); + // 推理强度被上游拒了（见 `crate::effort_memory`）：同一家改一档再发的那一跳要用的 —— 发给 + // 哪一家、被拒那一跳本来要发的那一档、改成什么。**是尝试链上新的一跳**，不算退避重试 + let mut effort_resend: Option<(&String, WrittenEffort, EffortFix)> = None; + // 重发的那一跳接着用被拒那一跳在这家占着的位置（和探路的名额）：**马上发**，不排到别的 + // 候选后面、不重新等空位 + let mut effort_hold: Option<(&String, crate::slots::Slot, Option)> = + None; + // 为推理强度重发过的那几家：一家只重发一次 + let mut effort_resent: Vec<&String> = Vec::new(); + // 这一跳发出去的推理强度和客户端要的不一样：那一跳在尝试链上的位置和那一项 + let mut effort_note: Option<(usize, tw_api::EffortChange)> = None; loop { stamp_queued(&mut chain, queued.take()); + stamp_effort(&mut chain, effort_note.take()); // 候选都试过了，最后一跳在内容之前暂时出错：等待期限之内退避一阵，把暂时出错的那几家再 // 试一轮。不重试的，交出去的是最后那个错误 if queue.is_empty() && waiting.is_empty() { @@ -425,13 +439,18 @@ pub(super) async fn try_upstreams<'a>( // 数 token 不排队、不占位置：它不跑模型，一眨眼就回来；为它排在几个长回答后面等上半分钟、 // 最后回一个 429，客户端连上下文还剩多少都看不到。这一家暂时发不出去的，由网关估 let limits = crate::throttle::usage_limits(provider); - let (slot, mut probe) = match granted.take() { - Some((ready, waited)) => { + let held = match effort_hold.take() { + Some((n, slot, probe)) if n == name => Some((slot, probe)), + _ => None, + }; + let (slot, mut probe) = match (held, granted.take()) { + (Some(held), _) => held, + (None, Some((ready, waited))) => { queued = Some((chain.len(), waited.as_millis() as u64)); hop_started = std::time::Instant::now(); (ready.slot, ready.probe) } - None if counting => { + (None, None) if counting => { if state.throttles.peek(&provider.name, limits) != crate::throttle::Peek::Open { chain.push(estimated_hop(provider, None, None, hop_started)); estimated = Some(provider); @@ -439,7 +458,7 @@ pub(super) async fn try_upstreams<'a>( } (crate::slots::Slot::free(), None) } - None => { + (None, None) => { let model = Some(sent.clone()).filter(asked_other); // 这段对话留在这家是为了它的缓存：等它，等不到再换下一家（缓存就丢在这家了）。 // 别的候选发不出去当场跳过，记下来，候选都看过之后再等 @@ -591,8 +610,33 @@ pub(super) async fn try_upstreams<'a>( let model = Some(plugged.model.clone().unwrap_or(sent)).filter(asked_other); let asked = Asked::of(req, reading, &plugged); - let out = match super::heavy(asked.body, || { - prepare(state, req, reading, &asked, provider, &effective_set, id) + // 发给这一家的模型名：推理强度按（上游，它）记（见 `crate::effort_memory`） + let sent_model = effective_set + .model + .clone() + .unwrap_or_else(|| reading.facts.model.clone()); + // 这一跳是推理强度被拒之后的重发：用定好的那一档。**真的发出去了才算用掉**：发之前就失败了 + // 的（取不到凭据、转换不了），下一次轮到这一家时照样用 + let carried = match effort_resend { + Some((n, pre, fix)) if n == name => Some((pre, fix)), + _ => None, + }; + // `hop_effort`：这一跳的推理强度改了什么（见 `adjust_effort`） + let (out, mut hop_effort) = match super::heavy(asked.body, || { + prepare(state, req, reading, &asked, provider, &effective_set, id).map(|mut out| { + let effort = adjust_effort( + state, + req, + reading, + &asked, + provider, + &sent_model, + &effective_set, + &mut out, + carried.map(|(_, fix)| fix), + ); + (out, effort) + }) }) { Ok(out) => out, Err(err) => { @@ -700,16 +744,18 @@ pub(super) async fn try_upstreams<'a>( attempt = attempts.len(), "forwarding" ); - // 这一跳要发出去了：插件改过的话，它收到的就是改过的那一份 + // 这一跳要发出去了：重发定好的那一档用掉了，推理强度改了什么记在这一跳上 + if carried.is_some() { + effort_resend = None; + } + effort_note = hop_effort.change.take().map(|c| (chain.len(), c)); + // 插件改过的话，上游收到的就是改过的那一份 after_plugins = plugged .rewritten .as_ref() .map(|r| (r.body.clone(), ledger.clone())); - // 这一跳接下了的话，回答钩子要的 - let sent_model = effective_set - .model - .clone() - .unwrap_or_else(|| reading.facts.model.clone()); + // 这一跳接下了的话，回答钩子要的：插件改了名的是改过的 + let sent_model = effective_set.model.clone().unwrap_or(sent_model); let (attempt, bridge) = (chain.len(), plugged.bridge); // 这一跳在报文记录里记到哪儿（见 `crate::content`）：尝试链上的第几跳，从 1 数 let tap = crate::content::Tap::new( @@ -862,6 +908,47 @@ pub(super) async fn try_upstreams<'a>( if !matches!(verdict, Verdict::Failed(Cause::RateLimited { .. })) { to_throttle(state, &provider.name, sent_at, verdict); } + // 上游因为推理强度拒了它（见 `crate::effort_memory`）：这一跳是一跳普通的 400，同一家 + // 改一档（或者去掉强度）马上再发一次，是尝试链上新的一跳，接着用这一跳占着的位置。 + // **一家只重发一次**，再被拒照常交给客户端。上游好好的，不算失败。记着的改法自己 + // 被拒了：那一条删掉 + if status == 400 + && reading.generates + && !counting + && !effort_resent.contains(&name) + && let Some((wire, pre, fix)) = effort_refused(req, &out, &hop_effort, &head) + { + if hop_effort.remembered { + state + .effort_memory + .forget(&provider.name, &sent_model, wire, pre); + } + tap.said(&head); + state.note_proxy_ok(&provider.proxy); + note_health( + &state.bus, + &state.health, + &provider.name, + state.health.record_success(&provider.name), + ); + tracing::info!( + provider = %provider.name, + model = %sent_model, + "the upstream refused the reasoning effort; sending again with another one" + ); + chain.push(hop( + &provider.name, + model.clone(), + tw_api::AttemptOutcome::Status, + status, + hop_started, + )); + effort_resent.push(name); + effort_resend = Some((name, pre, fix)); + effort_hold = Some((name, slot, probe.take())); + queue.push_front(name); + continue; + } // **最后一家的 4xx 交给客户端**：没有下一家可换了，上游自己的原话 // （「invalid x-api-key」）比我们转述的一句「回了 401」有用得多。 // 5xx 和 429 照旧报我们的话，带着尝试链 @@ -1023,11 +1110,15 @@ pub(super) async fn try_upstreams<'a>( }); // 不压开头的最后一家：它答没答上（成败的账）要等回程看到第一段内容才知道 let mut unconfirmed = last && hold.is_some() && !held; + // 这一家答上了：响应头是 2xx、不压开头的，或者压着开头等到了内容。推理强度改了一档 + // 重发的，这时才记下这个模型不认原来那一档（见 `crate::effort_memory`） + let mut answered = false; let r = match hold.filter(|_| held) { None => { // 不压开头：响应头就是它的答复，这一家没在限流（探路的过了，连续限流的次数 // 归零） state.throttles.answered(&provider.name, sent_at); + answered = true; r } Some(hold) => { @@ -1064,6 +1155,7 @@ pub(super) async fn try_upstreams<'a>( // 上游，响应头不说明它好了 Some(super::opening::Opening::Go(r)) => { state.throttles.answered(&provider.name, sent_at); + answered = true; r } // 到点了还没有内容：这一跳没有结论 @@ -1225,6 +1317,19 @@ pub(super) async fn try_upstreams<'a>( state.health.record_success(&provider.name), ); } + if answered + && let Some((pre, fix)) = carried + && let Some(wire) = wire(req, &out) + { + state.effort_memory.note( + provider, + &sent_model, + wire, + pre, + fix, + crate::effort_memory::now_secs(), + ); + } chain.push(hop( &provider.name, model.clone(), @@ -1301,6 +1406,7 @@ pub(super) async fn try_upstreams<'a>( } stamp_queued(&mut chain, queued.take()); + stamp_effort(&mut chain, effort_note.take()); stamp_retries(&mut chain, &retries.rounds); // 尝试链走完了，两条路都要发 —— 挂在 RequestFinished 上的话， // 失败那条路就没有尝试链，而那恰恰是最需要看它的时候。 @@ -1455,6 +1561,15 @@ fn stamp_queued(chain: &mut [tw_api::AttemptView], queued: Option<(usize, u64)>) } } +/// 推理强度改过的那一跳：在尝试链上补上改了什么（见 [`tw_api::AttemptView::effort`]） +fn stamp_effort(chain: &mut [tw_api::AttemptView], note: Option<(usize, tw_api::EffortChange)>) { + if let Some((at, change)) = note + && let Some(a) = chain.get_mut(at) + { + a.effort = Some(change); + } +} + /// 发出去了的一跳在这一家用量上限上占着的（见 [`crate::upstream_limits`]）。**丢掉就交给请求号**， /// 这一跳算一个请求、等结算：不管它怎么收场，上游都收到了它。回了 429 的、其实没发出去的调 /// [`Spent::release`] 还回去 @@ -1848,6 +1963,7 @@ fn estimated_hop( pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } @@ -1962,7 +2078,11 @@ fn prepare( // 参数改写。**只在这里动 body，而且只动被点名的那几个字段** —— // 出站直通说过任何 body 改写都可能是缓存杀手，所以这是 // 一个用户显式要求的例外，不是默认行为。 - let out = forward::apply_set(asked.body, effective_set, client_dialect); + let model = effective_set + .model + .as_deref() + .unwrap_or(&reading.facts.model); + let out = forward::apply_set(asked.body, effective_set, client_dialect, model); if let (Some(tw_dialect::ir::Dialect::Gemini), Some(m)) = (client_dialect, &effective_set.model) { @@ -2165,6 +2285,140 @@ fn prepare( }) } +/// 一跳的推理强度改了什么（见 [`adjust_effort`]）。 +#[derive(Default)] +struct HopEffort { + /// 这里改之前（规则改过之后）请求体里写明的那一档（[`tw_dialect::params::written_effort`]）。 + /// 没算过、没写明的（只写了预算的也是）是 None + pre: Option, + /// 用上了记着的改法 + remembered: bool, + /// 尝试链上这一跳要记的那一项 + change: Option, +} + +/// 这一跳的推理强度（见 [`crate::effort_memory`]）：规则改过之后（`out` 是按规则改好、要发出去 +/// 的那一份），这一家的这个模型记着不认这一档的，或者这一跳是被拒之后的重发（`carried`），在 +/// 发出去的这一份上改掉写明的那一档。 +/// +/// 尝试链上记的（[`tw_api::EffortChange`]）：这里改了的，`asked` 是改之前写明的那一档；只有规则 +/// 改了的，`asked` 是客户端要的那一档。和发出去的一样的不记。 +/// +/// **用不着时不解请求体**：没有规则改推理、这一家的这个模型什么都没记着、也不是重发的，什么都 +/// 不做 +#[allow(clippy::too_many_arguments)] +fn adjust_effort( + state: &AppState, + req: &Inbound, + reading: &crate::client_api::Reading, + asked: &Asked<'_>, + provider: &tw_config::Provider, + model: &str, + set: &tw_engine::SetAction, + out: &mut Outbound, + carried: Option, +) -> HopEffort { + use tw_dialect::params; + let rule = set.reasoning.is_some(); + let Some(wire) = wire(req, out).filter(|_| reading.generates) else { + return HopEffort::default(); + }; + if !rule && carried.is_none() && !state.effort_memory.has(&provider.name, model, wire) { + return HopEffort::default(); + } + let Ok(before) = serde_json::from_slice::(&out.body) else { + return HopEffort::default(); + }; + let pre = params::written_effort(wire, &before); + let mut fixed = None; + let mut after = before.clone(); + if let Some(p) = pre { + let fix = match carried { + Some(f) => Some((f, tw_api::EffortReason::Rejected)), + None => state + .effort_memory + .lookup(provider, model, wire, p, crate::effort_memory::now_secs()) + .map(|f| (f, tw_api::EffortReason::Remembered)), + }; + if let Some((f, why)) = fix + && apply_fix(wire, &mut after, f) + { + out.body = forward::rewritten(&out.body, &before, &after); + // 去掉、改掉了上游不认的值：发出去的不再是客户端那一份（报文记录另存它） + out.reshaped = true; + fixed = Some(why); + } + } + let shown = |v: &serde_json::Value| { + params::written_effort(wire, v) + .map(|w| w.slug()) + .or_else(|| params::reasoning_effort(wire, v).map(tw_dialect::think::openai)) + }; + let sent = shown(&after); + let from = match fixed { + Some(_) => pre.map(|p| p.slug()), + None if rule => match asked.decoded { + Some(Ok(d)) => d + .request + .reasoning + .as_ref() + .filter(|r| r.enabled) + .and_then(tw_dialect::think::effort) + .map(tw_dialect::think::openai), + _ => req.api.and_then(|a| { + let v = serde_json::from_slice::(asked.body).ok()?; + params::reasoning_effort(a.dialect(), &v).map(tw_dialect::think::openai) + }), + }, + None => None, + }; + let reason = fixed.unwrap_or(tw_api::EffortReason::Rule); + let change = from + .filter(|f| Some(*f) != sent) + .map(|f| tw_api::EffortChange { + asked: f.to_string(), + sent: sent.map(str::to_string), + reason, + }); + HopEffort { + pre, + remembered: fixed == Some(tw_api::EffortReason::Remembered), + change, + } +} + +/// 照 `fix` 改请求体 `v`（`wire` 格式）里写明的那一档。返回改没改 +fn apply_fix(wire: tw_dialect::ir::Dialect, v: &mut serde_json::Value, fix: EffortFix) -> bool { + match fix { + EffortFix::Use(e) => tw_dialect::params::replace_reasoning_effort(wire, v, e), + EffortFix::Drop => tw_dialect::params::remove_reasoning_effort(wire, v), + } +} + +/// 上游回的这个 400 是不是因为写明的推理强度（[`tw_dialect::think::effort_rejection`]，发出去的 +/// 是 `none` 的看 [`tw_dialect::think::none_rejection`]，那时去掉它）：是的话，交回发出去的格式、 +/// 记的时候按哪一档记（这一跳改强度之前写明的那一档；没算过的就是发出去的那一档）和重发时怎么 +/// 改。只写了预算的不算；照着改了也和发出去的一样的不算：重发一遍还是被拒 +fn effort_refused( + req: &Inbound, + out: &Outbound, + hop: &HopEffort, + said: &[u8], +) -> Option<(tw_dialect::ir::Dialect, WrittenEffort, EffortFix)> { + let wire = wire(req, out)?; + let v = serde_json::from_slice::(&out.body).ok()?; + let sent = tw_dialect::params::written_effort(wire, &v)?; + let said = String::from_utf8_lossy(said); + let fix = match sent { + WrittenEffort::Level(e) => tw_dialect::think::effort_rejection(400, &said, e)?, + WrittenEffort::Off => { + tw_dialect::think::none_rejection(400, &said).then_some(EffortFix::Drop)? + } + }; + let mut tried = v; + apply_fix(wire, &mut tried, fix).then_some((wire, hop.pre.unwrap_or(sent), fix)) +} + /// 这一跳发出去的那一份是什么格式：转换过的是目标格式，直通的是客户端的 fn wire(req: &Inbound, out: &Outbound) -> Option { out.target.or(req.api.map(|a| a.dialect())) diff --git a/crates/tw-gateway/src/server/pipeline/idle.rs b/crates/tw-gateway/src/server/pipeline/idle.rs index 71cf82b1..544fd3e3 100644 --- a/crates/tw-gateway/src/server/pipeline/idle.rs +++ b/crates/tw-gateway/src/server/pipeline/idle.rs @@ -101,6 +101,7 @@ pub(super) fn abandoned( pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } @@ -126,6 +127,7 @@ pub(super) fn aborted_hop( pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } } diff --git a/crates/tw-gateway/src/state.rs b/crates/tw-gateway/src/state.rs index 4e2009b7..669ee5f9 100644 --- a/crates/tw-gateway/src/state.rs +++ b/crates/tw-gateway/src/state.rs @@ -262,6 +262,9 @@ pub struct AppState { /// 哪些（上游，模型）拒过转换时自动标的提示缓存断点（见 [`crate::cache_marks`]）。 /// **跨重载存活** pub cache_marks: Arc, + /// 哪些（上游，模型）不认哪一档推理强度、改用什么（见 [`crate::effort_memory`]）。 + /// **跨重载存活**，存在数据目录里 + pub effort_memory: Arc, /// 脚本插件里跨重载存活的那一半：运行时、插件文件在哪儿、计数和日志、编译缓存 /// （见 [`crate::plugin::Plugins`]）。跟着配置换的那一半在 `Runtime::plugins` pub plugins: Arc, @@ -367,6 +370,7 @@ impl AppState { balance: Default::default(), seals: Default::default(), cache_marks: Default::default(), + effort_memory: Default::default(), plugins, swap: Default::default(), ping_every: crate::PING_EVERY, @@ -608,6 +612,8 @@ impl AppState { self.key_limits.configure(&next.config); self.upstream_limits.configure(&next.config); self.throttles.configure(&next.config); + // 删掉的、地址密钥协议变了的上游，记着的推理强度作废 + self.effort_memory.reconcile(&next.config); self.rt.store(Arc::new(next)); self.announce_broken(broken); // 模型汇总马上按新配置重算：删掉、停用的上游的模型必须立刻消失（列表 @@ -644,6 +650,8 @@ impl AppState { /// 告诉网关配置文件在哪个目录：**插件文件的路径相对它**。控制面拿到配置文件的 /// 路径时调；目录变了就把插件重读一遍 pub fn set_config_dir(&self, dir: std::path::PathBuf) { + // 记着的推理强度也存在这个目录里（见 `crate::effort_memory`） + self.effort_memory.set_dir(dir.clone()); if self.plugins.set_dir(dir) { self.reload_plugins(); } diff --git a/crates/tw-gateway/src/translate.rs b/crates/tw-gateway/src/translate.rs index 8a74ee0f..70afc575 100644 --- a/crates/tw-gateway/src/translate.rs +++ b/crates/tw-gateway/src/translate.rs @@ -57,9 +57,15 @@ pub fn apply_set(r: &mut Request, set: &tw_engine::SetAction) { if let Some(t) = set.max_tokens { r.max_tokens = Some(t); } - // 和直通时一样只做「关掉」：去掉推理配置，按模型默认走 - if set.thinking == Some(false) { - r.reasoning = None; + // 和直通时一样只改开着推理的请求：关掉是去掉推理配置；改一档是改强度，预算跟着强度 + // 由编码器折算（客户端写的预算不再作数） + match (set.reasoning, r.reasoning.as_mut()) { + (Some(tw_engine::SetReasoning::Off), Some(re)) if re.enabled => r.reasoning = None, + (Some(tw_engine::SetReasoning::Level(e)), Some(re)) if re.enabled => { + re.effort = Some(e); + re.budget = None; + } + _ => {} } } diff --git a/crates/tw-gateway/src/ws.rs b/crates/tw-gateway/src/ws.rs index d86f00e3..07f32d75 100644 --- a/crates/tw-gateway/src/ws.rs +++ b/crates/tw-gateway/src/ws.rs @@ -2212,7 +2212,8 @@ fn rewrite(text: String, model: &str, set: &tw_engine::SetAction) -> String { return text; } let body = bytes::Bytes::from(text); - let out = crate::forward::apply_set(&body, &set, Some(tw_dialect::ir::Dialect::Responses)); + let out = + crate::forward::apply_set(&body, &set, Some(tw_dialect::ir::Dialect::Responses), model); String::from_utf8(Vec::from(out)) .unwrap_or_else(|_| String::from_utf8_lossy(&body).into_owned()) } diff --git a/crates/tw-gateway/tests/alias_ws.rs b/crates/tw-gateway/tests/alias_ws.rs index 7a476802..82fa47fb 100644 --- a/crates/tw-gateway/tests/alias_ws.rs +++ b/crates/tw-gateway/tests/alias_ws.rs @@ -664,14 +664,14 @@ fn create_reasoning(model: &str) -> WsMsg { } /// 规则的参数改写每一帧照 HTTP 那条路改（Responses 的字段名）：`set.max_tokens` 写成 -/// `max_output_tokens`，`set.thinking: false` 去掉 `reasoning`。条件按这一帧求 —— 按模型附加的 +/// `max_output_tokens`，`set.reasoning: off` 把强度写成 `none`。条件按这一帧求 —— 按模型附加的 /// 改写、按模型拒绝的规则，升级时还不知道模型，每一帧照样生效 #[tokio::test] async fn rule_parameter_rewrites_and_model_conditions_apply_to_each_frame() { let (up, seen) = upstream().await; let rules = " - { name: Codex 限长, when: { client: codex }, set: { max_tokens: 1234 } } -- { name: Max 不思考, when: { model: gpt-5.1-codex-max }, set: { thinking: false, max_tokens: 99 } } +- { name: Max 不思考, when: { model: gpt-5.1-codex-max }, set: { reasoning: off, max_tokens: 99 } } - { name: 不给, when: { model: banned }, deny: 这个模型不走这里 } - { name: 兜底, to: __all__ } "; @@ -695,7 +695,12 @@ async fn rule_parameter_rewrites_and_model_conditions_apply_to_each_frame() { { let seen = seen.lock().unwrap(); assert_eq!(seen[1]["max_output_tokens"], 99, "{:?}", seen[1]); - assert!(seen[1].get("reasoning").is_none(), "{:?}", seen[1]); + assert_eq!( + seen[1]["reasoning"], + json!({"effort": "none"}), + "{:?}", + seen[1] + ); assert_eq!(seen[1]["model"], "gpt-5.1-codex-max"); } diff --git a/crates/tw-gateway/tests/effort.rs b/crates/tw-gateway/tests/effort.rs new file mode 100644 index 00000000..90985cff --- /dev/null +++ b/crates/tw-gateway/tests/effort.rs @@ -0,0 +1,651 @@ +//! 推理强度，端到端：规则改写（`set.reasoning`），上游因为推理强度回 400 时同一家改一档再发 +//! 一次、记下来（见 `tw_gateway::effort_memory`）。 + +use std::net::SocketAddr; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use axum::Router; +use axum::extract::State; +use serde_json::{Value, json}; + +/// 一个假上游收到的请求体 +type Seen = Arc>>; + +/// 一个假上游：按到达的顺序依次用 `replies` 回（用完了一直用最后一个），记下收到的请求体 +async fn upstream(replies: Vec<(u16, Value)>) -> (SocketAddr, Seen) { + let seen: Seen = Default::default(); + let app = Router::new() + .fallback(move |State(s): State, body: bytes::Bytes| { + let replies = replies.clone(); + async move { + let mut s = s.lock().unwrap(); + let n = s.len(); + s.push(serde_json::from_slice(&body).unwrap_or(Value::Null)); + let (status, reply) = replies[n.min(replies.len() - 1)].clone(); + axum::response::Response::builder() + .status(status) + .header("content-type", "application/json") + .body(axum::body::Body::from(reply.to_string())) + .unwrap() + } + }) + .with_state(seen.clone()); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(l, app).await.unwrap() }); + (addr, seen) +} + +fn config(up: SocketAddr, protocol: &str, rules: &str) -> tw_config::Config { + let routes = if rules.is_empty() { + String::new() + } else { + format!("routes:\n - name: default\n rules:\n{rules}") + }; + serde_yaml_ng::from_str(&format!( + "version: 1 +clients: + - name: c + key: tw-k +providers: + - name: relay + base_url: http://{up} + key: sk-upstream + protocol: {protocol} +{routes}" + )) + .unwrap() +} + +struct Gw { + addr: SocketAddr, + state: tw_gateway::AppState, + rx: tokio::sync::broadcast::Receiver, +} + +async fn gateway(cfg: tw_config::Config, dir: Option<&std::path::Path>) -> Gw { + let state = tw_gateway::AppState::new(cfg).unwrap(); + if let Some(d) = dir { + state.set_config_dir(d.to_path_buf()); + } + let rx = state.bus.subscribe(); + let addr = tw_gateway::serve(state.clone(), ([127, 0, 0, 1], 0).into()) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + Gw { addr, state, rx } +} + +async fn post(gw: &Gw, path: &str, body: Value) -> (u16, String) { + let resp = reqwest::Client::new() + .post(format!("http://{}{path}", gw.addr)) + .header("content-type", "application/json") + .header("x-api-key", "tw-k") + .header("authorization", "Bearer tw-k") + .body(body.to_string()) + .send() + .await + .unwrap(); + let status = resp.status().as_u16(); + (status, resp.text().await.unwrap()) +} + +/// 这个请求的尝试链 +async fn attempts( + rx: &mut tokio::sync::broadcast::Receiver, +) -> Vec { + loop { + let e = tokio::time::timeout(Duration::from_secs(5), rx.recv()) + .await + .expect("no routing event") + .unwrap(); + if let tw_api::Event::RequestRouted { attempts, .. } = e { + return attempts; + } + } +} + +fn anthropic_ok() -> Value { + json!({"id": "msg_1", "type": "message", "role": "assistant", "model": "claude-opus-4-6", + "content": [{"type": "text", "text": "好"}], "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 1}}) +} + +fn chat_ok() -> Value { + json!({"id": "c1", "object": "chat.completion", "model": "glm-5", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "好"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}) +} + +fn anthropic_error(message: &str) -> Value { + json!({"type": "error", "error": {"type": "invalid_request_error", "message": message}}) +} + +/// 自适应思考、写着 `effort` 的 Claude 请求 +fn claude(effort: &str) -> Value { + json!({ + "model": "claude-opus-4-6", "max_tokens": 2000, + "thinking": {"type": "adaptive"}, "output_config": {"effort": effort}, + "messages": [{"role": "user", "content": "hi"}], + }) +} + +fn effort_of(v: &Value) -> Option<&str> { + v.pointer("/output_config/effort").and_then(Value::as_str) +} + +fn change( + asked: &str, + sent: Option<&str>, + reason: tw_api::EffortReason, +) -> Option { + Some(tw_api::EffortChange { + asked: asked.into(), + sent: sent.map(str::to_string), + reason, + }) +} + +const NOT_VALID: &str = "The parameter `output_config.effort` specified in the request are not valid: expected `low`, `medium`, `high` or `max`, but got `xhigh` instead."; + +/// 上游不认 `xhigh`、列出了认的那几档：同一家改用最近的 `max` 再发，是尝试链上新的一跳；记下来， +/// 下一个请求直接发 `max`。存在数据目录里 +#[tokio::test] +async fn a_refused_effort_is_sent_again_once_with_the_nearest_level_and_remembered() { + let dir = tempfile::tempdir().unwrap(); + let (up, seen) = upstream(vec![ + (400, anthropic_error(NOT_VALID)), + (200, anthropic_ok()), + ]) + .await; + let mut gw = gateway(config(up, "anthropic", ""), Some(dir.path())).await; + let (status, body) = post(&gw, "/v1/messages", claude("xhigh")).await; + assert_eq!(status, 200, "{body}"); + { + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), 2); + assert_eq!(effort_of(&seen[0]), Some("xhigh")); + assert_eq!(effort_of(&seen[1]), Some("max")); + // 别的照旧 + assert_eq!(seen[1]["thinking"], json!({"type": "adaptive"})); + assert_eq!(seen[1]["messages"], seen[0]["messages"]); + } + let chain = attempts(&mut gw.rx).await; + assert_eq!(chain.len(), 2, "{chain:?}"); + assert_eq!(chain[0].outcome, tw_api::AttemptOutcome::Status); + assert_eq!(chain[0].status, Some(400)); + assert_eq!(chain[0].effort, None, "被拒的那一跳发的就是客户端要的"); + assert_eq!(chain[1].outcome, tw_api::AttemptOutcome::Served); + assert_eq!(chain[1].provider, "relay"); + assert_eq!(chain[1].retry, None, "不是退避重试"); + assert_eq!( + chain[1].effort, + change("xhigh", Some("max"), tw_api::EffortReason::Rejected) + ); + + // 记住了：直接发 max，上游只收到一次 + let (status, _) = post(&gw, "/v1/messages", claude("xhigh")).await; + assert_eq!(status, 200); + assert_eq!(seen.lock().unwrap().len(), 3); + assert_eq!(effort_of(&seen.lock().unwrap()[2]), Some("max")); + let chain = attempts(&mut gw.rx).await; + assert_eq!(chain.len(), 1); + assert_eq!( + chain[0].effort, + change("xhigh", Some("max"), tw_api::EffortReason::Remembered) + ); + // 别的档位不受影响 + post(&gw, "/v1/messages", claude("high")).await; + assert_eq!(effort_of(&seen.lock().unwrap()[3]), Some("high")); + assert_eq!(attempts(&mut gw.rx).await[0].effort, None); + + // 存在数据目录里（写文件挪到了阻塞线程上，等它一下） + let file = dir.path().join(tw_gateway::effort_memory::FILE); + let mut text = String::new(); + for _ in 0..50 { + text = std::fs::read_to_string(&file).unwrap_or_default(); + if !text.is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert!( + text.contains("claude-opus-4-6") && text.contains("\"use\": \"max\""), + "{text}" + ); + assert!(!text.contains("sk-upstream"), "{text}"); + + // 换了密钥：已经是另一个上游了，记着的作废，再试原来那一档 + let mut cfg = gw.state.config().as_ref().clone(); + cfg.providers[0].key = Some("sk-other".into()); + gw.state.reload(cfg).unwrap(); + post(&gw, "/v1/messages", claude("xhigh")).await; + assert_eq!(effort_of(&seen.lock().unwrap()[4]), Some("xhigh")); +} + +/// 「high 或以下」是上限：改用 high +#[tokio::test] +async fn a_ceiling_in_the_refusal_is_the_level_sent_again() { + let (up, seen) = upstream(vec![ + ( + 400, + anthropic_error( + "output_config.effort 'xhigh' is not supported when thinking is disabled on this model. Use effort 'high' or below, or enable thinking.", + ), + ), + (200, anthropic_ok()), + ]) + .await; + let mut gw = gateway(config(up, "anthropic", ""), None).await; + let (status, body) = post(&gw, "/v1/messages", claude("xhigh")).await; + assert_eq!(status, 200, "{body}"); + assert_eq!(effort_of(&seen.lock().unwrap()[1]), Some("high")); + let chain = attempts(&mut gw.rx).await; + assert_eq!( + chain[1].effort, + change("xhigh", Some("high"), tw_api::EffortReason::Rejected) + ); +} + +/// 不提 `effort`、只列了几档的（GLM 始终思考的模型，码 1210）：离 medium 最近的 low 和 high +/// 一样近，取 high +#[tokio::test] +async fn levels_listed_without_the_word_effort_are_understood_too() { + let refusal = json!({"error": {"code": "1210", + "message": "该模型始终思考，不支持关闭思考；请使用 low、high 或 max。"}}); + let (up, seen) = upstream(vec![(400, refusal), (200, chat_ok())]).await; + let mut gw = gateway(config(up, "openai-chat", ""), None).await; + let (status, body) = post( + &gw, + "/v1/chat/completions", + json!({"model": "glm-5", "reasoning_effort": "medium", + "messages": [{"role": "user", "content": "hi"}]}), + ) + .await; + assert_eq!(status, 200, "{body}"); + { + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), 2); + assert_eq!(seen[1]["reasoning_effort"], "high"); + } + let chain = attempts(&mut gw.rx).await; + assert_eq!( + chain[1].effort, + change("medium", Some("high"), tw_api::EffortReason::Rejected) + ); +} + +/// 与推理强度无关的 400：原样交给客户端，不重发 +#[tokio::test] +async fn another_400_is_not_sent_again() { + let reply = anthropic_error("messages.0.content: field required"); + let (up, seen) = upstream(vec![(400, reply.clone()), (200, anthropic_ok())]).await; + let mut gw = gateway(config(up, "anthropic", ""), None).await; + let (status, body) = post(&gw, "/v1/messages", claude("xhigh")).await; + assert_eq!(status, 400); + assert_eq!(serde_json::from_str::(&body).unwrap(), reply); + assert_eq!(seen.lock().unwrap().len(), 1); + let chain = attempts(&mut gw.rx).await; + assert_eq!(chain.len(), 1); + assert_eq!(chain[0].outcome, tw_api::AttemptOutcome::Served); +} + +/// 原话里碰巧有发出去的那一档的名字（「max context length」），又说到了思考：照规矩重发一次，再被拒就把那个 +/// 错误交给客户端，**而且不记**：下一个请求照样发 max +#[tokio::test] +async fn a_resend_refused_again_is_handed_on_and_not_remembered() { + let reply = + anthropic_error("prompt is too long for extended thinking: max context length exceeded"); + let (up, seen) = upstream(vec![(400, reply.clone())]).await; + let mut gw = gateway(config(up, "anthropic", ""), None).await; + let (status, body) = post(&gw, "/v1/messages", claude("max")).await; + assert_eq!(status, 400); + assert_eq!(serde_json::from_str::(&body).unwrap(), reply); + { + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), 2, "只重发一次"); + assert_eq!(effort_of(&seen[0]), Some("max")); + assert_eq!(effort_of(&seen[1]), None, "说不出用哪一档：去掉强度"); + } + let chain = attempts(&mut gw.rx).await; + assert_eq!(chain.len(), 2); + assert_eq!(chain[1].status, Some(400)); + assert_eq!( + chain[1].effort, + change("max", None, tw_api::EffortReason::Rejected) + ); + assert!(!gw.state.effort_memory.has( + "relay", + "claude-opus-4-6", + tw_dialect::ir::Dialect::Anthropic + )); + post(&gw, "/v1/messages", claude("max")).await; + let seen = seen.lock().unwrap(); + assert_eq!(effort_of(&seen[2]), Some("max"), "没有记住"); +} + +/// 规则改写推理：改一档只改开着推理的请求，`off` 连强度一起去掉，没开推理的原样不动 +#[tokio::test] +async fn a_rule_rewrites_reasoning_only_where_it_is_on() { + let ok = anthropic_ok(); + let (up, seen) = upstream(vec![(200, ok)]).await; + let rules = + " - { name: 想多点, when: { model: claude-opus-4-6 }, set: { reasoning: xhigh } } + - { name: 别想, when: { model: claude-haiku-4-5 }, set: { reasoning: off } } + - { name: 兜底, to: relay } +"; + let mut gw = gateway(config(up, "anthropic", rules), None).await; + post(&gw, "/v1/messages", claude("high")).await; + assert_eq!(effort_of(&seen.lock().unwrap()[0]), Some("xhigh")); + assert_eq!( + attempts(&mut gw.rx).await[0].effort, + change("high", Some("xhigh"), tw_api::EffortReason::Rule) + ); + // 没开推理的（生成标题这类）：一个字节都不动 + let plain = json!({"model": "claude-opus-4-6", "max_tokens": 50, + "messages": [{"role": "user", "content": "起个标题"}]}); + post(&gw, "/v1/messages", plain.clone()).await; + assert_eq!(seen.lock().unwrap()[1], plain); + assert_eq!(attempts(&mut gw.rx).await[0].effort, None); + // off：开关和强度一起去掉 + let mut haiku = claude("high"); + haiku["model"] = json!("claude-haiku-4-5"); + post(&gw, "/v1/messages", haiku).await; + { + let seen = seen.lock().unwrap(); + assert!(seen[2].get("thinking").is_none(), "{}", seen[2]); + assert!(seen[2].get("output_config").is_none(), "{}", seen[2]); + } + assert_eq!( + attempts(&mut gw.rx).await[0].effort, + change("high", None, tw_api::EffortReason::Rule) + ); +} + +/// 规则定的那一档上游不认：照样改一档重发（自动兼容在规则之后） +#[tokio::test] +async fn a_level_set_by_a_rule_is_fixed_like_any_other() { + let (up, seen) = upstream(vec![ + (400, anthropic_error(NOT_VALID)), + (200, anthropic_ok()), + ]) + .await; + let rules = " - { name: 想多点, set: { reasoning: xhigh } } + - { name: 兜底, to: relay } +"; + let mut gw = gateway(config(up, "anthropic", rules), None).await; + let (status, _) = post(&gw, "/v1/messages", claude("low")).await; + assert_eq!(status, 200); + { + let seen = seen.lock().unwrap(); + assert_eq!(effort_of(&seen[0]), Some("xhigh")); + assert_eq!(effort_of(&seen[1]), Some("max")); + } + let chain = attempts(&mut gw.rx).await; + assert_eq!( + chain[0].effort, + change("low", Some("xhigh"), tw_api::EffortReason::Rule) + ); + assert_eq!( + chain[1].effort, + change("xhigh", Some("max"), tw_api::EffortReason::Rejected), + "被拒的是规则改出来的 xhigh" + ); +} + +/// 转换过格式的请求一样：改的是转成上游格式之后的那一份 +#[tokio::test] +async fn a_converted_request_is_fixed_in_the_upstream_format() { + let (up, seen) = upstream(vec![ + (400, anthropic_error(NOT_VALID)), + (200, anthropic_ok()), + ]) + .await; + let mut gw = gateway(config(up, "anthropic", ""), None).await; + let (status, body) = post( + &gw, + "/v1/chat/completions", + json!({"model": "claude-opus-4-6", "reasoning_effort": "xhigh", + "messages": [{"role": "user", "content": "hi"}]}), + ) + .await; + assert_eq!(status, 200, "{body}"); + { + let seen = seen.lock().unwrap(); + assert_eq!(effort_of(&seen[0]), Some("xhigh")); + assert_eq!(effort_of(&seen[1]), Some("max")); + } + let chain = attempts(&mut gw.rx).await; + assert_eq!( + chain[1].effort, + change("xhigh", Some("max"), tw_api::EffortReason::Rejected) + ); +} + +/// 关着思考、写着强度的请求（Anthropic 说「关着思考时不认 xhigh，用 high 或以下」）：按写着的 +/// 字段认，照样改一档重发 +#[tokio::test] +async fn an_effort_written_with_thinking_off_is_fixed_too() { + let (up, seen) = upstream(vec![ + ( + 400, + anthropic_error( + "output_config.effort 'xhigh' is not supported when thinking is disabled on this model. Use effort 'high' or below, or enable thinking.", + ), + ), + (200, anthropic_ok()), + ]) + .await; + let mut gw = gateway(config(up, "anthropic", ""), None).await; + let mut body = claude("xhigh"); + body["thinking"] = json!({"type": "disabled"}); + let (status, text) = post(&gw, "/v1/messages", body).await; + assert_eq!(status, 200, "{text}"); + { + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), 2); + assert_eq!(effort_of(&seen[1]), Some("high")); + assert_eq!(seen[1]["thinking"], json!({"type": "disabled"})); + } + let chain = attempts(&mut gw.rx).await; + assert_eq!( + chain[1].effort, + change("xhigh", Some("high"), tw_api::EffortReason::Rejected) + ); +} + +/// 记着的改法自己被拒了：那一条删掉，照新的原话再改一次，成了记新的；重发成了用的正是原来那一档 +/// 的（上游又认了）：删掉，往后照原样发 +#[tokio::test] +async fn a_remembered_fix_that_is_refused_is_replaced_or_forgotten() { + let ceiling = anthropic_error( + "effort 'max' is not supported when thinking is disabled. Use effort 'high' or below.", + ); + let back = anthropic_error("effort 'max' is not available for this model: use xhigh"); + let (up, seen) = upstream(vec![ + (400, anthropic_error(NOT_VALID)), + (200, anthropic_ok()), + (400, ceiling), + (200, anthropic_ok()), + (200, anthropic_ok()), + ]) + .await; + let mut gw = gateway(config(up, "anthropic", ""), None).await; + let e = |i: usize| effort_of(&seen.lock().unwrap()[i]).map(str::to_string); + // 记下 xhigh → max + post(&gw, "/v1/messages", claude("xhigh")).await; + attempts(&mut gw.rx).await; + // max 自己被拒，上限 high：改记 xhigh → high + let (status, _) = post(&gw, "/v1/messages", claude("xhigh")).await; + assert_eq!(status, 200); + assert_eq!((e(2), e(3)), (Some("max".into()), Some("high".into()))); + let chain = attempts(&mut gw.rx).await; + assert_eq!( + chain[0].effort, + change("xhigh", Some("max"), tw_api::EffortReason::Remembered) + ); + assert_eq!( + chain[1].effort, + change("xhigh", Some("high"), tw_api::EffortReason::Rejected), + "asked 是改之前写明的那一档" + ); + post(&gw, "/v1/messages", claude("xhigh")).await; + assert_eq!(e(4), Some("high".into())); + attempts(&mut gw.rx).await; + + // 另一家：记着 xhigh → max，max 被拒、原话让用 xhigh，重发 xhigh 成了 —— 删掉 + let (up, seen) = upstream(vec![ + (400, anthropic_error(NOT_VALID)), + (200, anthropic_ok()), + (400, back), + (200, anthropic_ok()), + ]) + .await; + let mut gw = gateway(config(up, "anthropic", ""), None).await; + let e = |i: usize| effort_of(&seen.lock().unwrap()[i]).map(str::to_string); + post(&gw, "/v1/messages", claude("xhigh")).await; + attempts(&mut gw.rx).await; + post(&gw, "/v1/messages", claude("xhigh")).await; + assert_eq!((e(2), e(3)), (Some("max".into()), Some("xhigh".into()))); + attempts(&mut gw.rx).await; + assert!(!gw.state.effort_memory.has( + "relay", + "claude-opus-4-6", + tw_dialect::ir::Dialect::Anthropic + )); + post(&gw, "/v1/messages", claude("xhigh")).await; + assert_eq!(e(4), Some("xhigh".into())); +} + +/// 只写了预算的请求没有写明的强度：上游的 400 说到了思考和档位名也不重发、不记 +#[tokio::test] +async fn a_budget_only_request_is_not_an_effort_rejection() { + let reply = anthropic_error( + "thinking.budget_tokens: the max is lower for this model; use low, medium or high effort instead", + ); + let (up, seen) = upstream(vec![(400, reply.clone()), (200, anthropic_ok())]).await; + let gw = gateway(config(up, "anthropic", ""), None).await; + let (status, _) = post( + &gw, + "/v1/messages", + json!({"model": "claude-sonnet-4-5", "max_tokens": 64000, + "thinking": {"type": "enabled", "budget_tokens": 50000}, + "messages": [{"role": "user", "content": "hi"}]}), + ) + .await; + assert_eq!(status, 400); + assert_eq!(seen.lock().unwrap().len(), 1); + assert!(!gw.state.effort_memory.has( + "relay", + "claude-sonnet-4-5", + tw_dialect::ir::Dialect::Anthropic + )); + // 记着这个模型不认 xhigh，预算折算出来正是 xhigh 的请求也不改：预算不是写明的强度 + let p = gw.state.config().providers[0].clone(); + gw.state.effort_memory.note( + &p, + "claude-sonnet-4-5", + tw_dialect::ir::Dialect::Anthropic, + tw_dialect::params::WrittenEffort::Level(tw_dialect::ir::Effort::XHigh), + tw_dialect::think::EffortFix::Use(tw_dialect::ir::Effort::Low), + tw_gateway::effort_memory::now_secs(), + ); + post( + &gw, + "/v1/messages", + json!({"model": "claude-sonnet-4-5", "max_tokens": 64000, + "thinking": {"type": "enabled", "budget_tokens": 50000}, + "messages": [{"role": "user", "content": "hi"}]}), + ) + .await; + assert_eq!(seen.lock().unwrap()[1]["thinking"]["budget_tokens"], 50000); +} + +/// 重发马上发给同一家：排在它后面的候选一个都不碰，这家只许一个并发也不用等 +#[tokio::test] +async fn the_resend_goes_to_the_same_upstream_at_once() { + let (up, seen) = upstream(vec![ + (400, anthropic_error(NOT_VALID)), + (200, anthropic_ok()), + ]) + .await; + let (backup, backup_seen) = upstream(vec![(200, anthropic_ok())]).await; + let cfg: tw_config::Config = serde_yaml_ng::from_str(&format!( + "version: 1 +clients: + - name: c + key: tw-k +providers: + - name: relay + base_url: http://{up} + key: sk-upstream + protocol: anthropic + max_concurrent: 1 + - name: backup + base_url: http://{backup} + key: sk-backup + protocol: anthropic +groups: + - name: both + type: fallback + providers: [relay, backup] +routes: + - name: default + rules: + - {{ name: 兜底, to: both }} +" + )) + .unwrap(); + let mut gw = gateway(cfg, None).await; + let (status, body) = post(&gw, "/v1/messages", claude("xhigh")).await; + assert_eq!(status, 200, "{body}"); + assert_eq!(seen.lock().unwrap().len(), 2); + assert!(backup_seen.lock().unwrap().is_empty()); + let chain = attempts(&mut gw.rx).await; + assert_eq!( + chain + .iter() + .map(|a| a.provider.as_str()) + .collect::>(), + ["relay", "relay"] + ); + assert_eq!(chain[1].queued_ms, None, "没有排队"); +} + +/// `off` 在 OpenAI 的格式里写 `none`（去掉的话 GPT-5 这类按默认的那一档想）；上游不认 `none` +/// 的去掉再发，记下来，往后直接去掉 +#[tokio::test] +async fn off_writes_none_and_an_upstream_that_refuses_it_gets_the_field_removed() { + let refusal = json!({"error": {"message": "Unsupported value: 'none' is not supported with this model for reasoning_effort. Supported values are: 'low', 'medium', and 'high'.", + "type": "invalid_request_error", "param": "reasoning_effort", "code": "unsupported_value"}}); + let (up, seen) = upstream(vec![(400, refusal), (200, chat_ok())]).await; + let rules = " - { name: 别想, set: { reasoning: off } } + - { name: 兜底, to: relay } +"; + let mut gw = gateway(config(up, "openai-chat", rules), None).await; + let ask = json!({"model": "o3-mini", "reasoning_effort": "high", + "messages": [{"role": "user", "content": "hi"}]}); + let (status, body) = post(&gw, "/v1/chat/completions", ask.clone()).await; + assert_eq!(status, 200, "{body}"); + { + let seen = seen.lock().unwrap(); + assert_eq!(seen[0]["reasoning_effort"], "none"); + assert!(seen[1].get("reasoning_effort").is_none(), "{}", seen[1]); + } + let chain = attempts(&mut gw.rx).await; + assert_eq!( + chain[0].effort, + change("high", Some("none"), tw_api::EffortReason::Rule) + ); + assert_eq!( + chain[1].effort, + change("none", None, tw_api::EffortReason::Rejected) + ); + post(&gw, "/v1/chat/completions", ask).await; + assert!(seen.lock().unwrap()[2].get("reasoning_effort").is_none()); + assert_eq!( + attempts(&mut gw.rx).await[0].effort, + change("none", None, tw_api::EffortReason::Remembered) + ); +} diff --git a/crates/tw-gateway/tests/passthrough.rs b/crates/tw-gateway/tests/passthrough.rs index 405b984d..44d2c114 100644 --- a/crates/tw-gateway/tests/passthrough.rs +++ b/crates/tw-gateway/tests/passthrough.rs @@ -1191,7 +1191,7 @@ async fn a_phase_two_rule_is_recomputed_after_failover() { when: serde_yaml_ng::from_str("{ provider_would_be: relay }").unwrap(), to: None, set: Some(tw_engine::SetAction { - thinking: Some(false), + reasoning: Some(tw_engine::SetReasoning::Off), ..Default::default() }), deny: None, diff --git a/crates/tw-gateway/tests/routing_facts.rs b/crates/tw-gateway/tests/routing_facts.rs index 475671ca..5d4fc997 100644 --- a/crates/tw-gateway/tests/routing_facts.rs +++ b/crates/tw-gateway/tests/routing_facts.rs @@ -60,7 +60,7 @@ fn rule( fn thinking_off() -> Option { Some(SetAction { - thinking: Some(false), + reasoning: Some(tw_engine::SetReasoning::Off), ..Default::default() }) } diff --git a/crates/tw-observe/src/bus.rs b/crates/tw-observe/src/bus.rs index 1955b049..3d98229e 100644 --- a/crates/tw-observe/src/bus.rs +++ b/crates/tw-observe/src/bus.rs @@ -512,6 +512,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }, tw_api::AttemptView { @@ -527,6 +528,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }, ], diff --git a/crates/tw-store/src/db.rs b/crates/tw-store/src/db.rs index 683fa0eb..1493a575 100644 --- a/crates/tw-store/src/db.rs +++ b/crates/tw-store/src/db.rs @@ -3639,6 +3639,7 @@ mod cost_state_tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], ..Default::default() @@ -4151,6 +4152,7 @@ mod cost { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, } }) diff --git a/crates/tw-store/src/recorder.rs b/crates/tw-store/src/recorder.rs index 8dbbdfb6..00e15b15 100644 --- a/crates/tw-store/src/recorder.rs +++ b/crates/tw-store/src/recorder.rs @@ -995,6 +995,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }, tw_api::AttemptView { @@ -1010,6 +1011,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: Some("机场".into()), }, ], @@ -1078,6 +1080,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing: tw_api::Billing::PerToken, @@ -1115,6 +1118,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }, tw_api::AttemptView { @@ -1130,6 +1134,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }, ], @@ -1186,6 +1191,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing: tw_api::Billing::PerToken, @@ -1317,6 +1323,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing: tw_api::Billing::PerToken, @@ -1646,6 +1653,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing: tw_api::Billing::Free, @@ -1691,6 +1699,7 @@ mod tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing: tw_api::Billing::Free, @@ -1775,6 +1784,7 @@ mod billing_tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing, @@ -1870,6 +1880,7 @@ mod billing_tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing, @@ -2319,6 +2330,7 @@ mod security_detail_tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing: tw_api::Billing::PerToken, @@ -2617,6 +2629,7 @@ mod translation_tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }) .collect(), @@ -3312,6 +3325,7 @@ mod settle_hook_tests { pause_ms: None, retry: None, backoff_ms: None, + effort: None, proxy: None, }], billing: tw_api::Billing::PerToken, diff --git a/docs/config.md b/docs/config.md index aba312de..267f2ba7 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1500,6 +1500,23 @@ read, naming the rule and the value. `set.model` may name an alias; each upstream then receives its own name for it. Answers carry the name the client asked for, as with aliases. +`set.reasoning` writes the effort where the client's format keeps it +(`output_config.effort`, `reasoning_effort`, `reasoning.effort`, Gemini's +`thinkingLevel`, or the thinking budget when the client gave a budget). + +When an upstream answers 400 because of the reasoning effort, whether the +client or a rule chose it, the request is sent once more to the same upstream +with a level the upstream named (the nearest one, or the ceiling it gave), or +without an effort when it named none. The resend is a new attempt in the +request details and does not count as a retry. If it succeeds, the level is +remembered for that upstream and model for 7 days in `effort-memory.json` in +the data directory, so later requests send the working level directly. The +entries of an upstream are forgotten when its address, key or protocol +changes, or when it is removed. Requests that only give a thinking budget are +not resent. On WebSocket connections (Codex), a turn refused for its effort is +not resent and nothing is remembered: the refusal reaches the client as it +is. + @@ -1507,7 +1524,7 @@ it. Answers carry the name the client asked for, as with aliases. |---|---|---|---| | `model` | string | — | Send a different model. The prompt cache of the session is lost. | | `max_tokens` | integer | — | Replace `max_tokens`. | -| `thinking` | bool | — | Turn extended thinking on or off. | +| `reasoning` | `off` \| `low` \| `medium` \| `high` \| `xhigh` \| `max` | — | Reasoning. `off` turns it off: the reasoning settings and the effort are removed, and in the OpenAI formats the effort is set to `none`. A level changes the effort. Only requests that already have reasoning on are changed; requests without reasoning (such as title generation) are left as they are. | | `only_at_session_start` | bool | `false` | Apply only when a session starts. Recorded and shown; not in effect yet. | diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index 7f8d133c..87bc681c 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -1089,6 +1089,10 @@ routes: `set.model` 可以写别名，每家上游收到各自对它的叫法。和别名一样，回答里的模型名写成客户端请求的名称。 +`set.reasoning` 按客户端的格式写推理强度：`output_config.effort`、`reasoning_effort`、`reasoning.effort`、Gemini 的 `thinkingLevel`；客户端给的是思考预算时改预算。 + +上游因为推理强度回 400 时（无论这一档是客户端写的还是规则改的），请求在同一家上游再发一次：改用上游原话里列出的、离原来那一档最近的一档，原话给了上限的用上限，一档都没列的去掉推理强度。重发在请求详情里是新的一跳，不算重试。重发成功后，这一档按上游和模型记 7 天，存在数据目录的 `effort-memory.json` 里，之后的请求直接发可用的那一档。上游的地址、密钥或协议变了，或者上游被删除，它的记录随之作废。只给了思考预算的请求不重发。WebSocket 连接（Codex）上，一轮因推理强度被拒时不重发、也不记录，拒绝原样交给客户端。 + @@ -1096,7 +1100,7 @@ routes: |---|---|---|---| | `model` | 字符串 | — | 换成另一个模型发送。该会话的 prompt cache 随之失效。 | | `max_tokens` | 整数 | — | 替换 `max_tokens`。 | -| `thinking` | 布尔 | — | 开启或关闭扩展思考。 | +| `reasoning` | `off` \| `low` \| `medium` \| `high` \| `xhigh` \| `max` | — | 推理。`off` 关闭推理：去掉推理设置和推理强度，OpenAI 的格式改为写 `none`。写一档则改为该档强度。只改写已开启推理的请求，未开启推理的请求（如生成标题）保持不变。 | | `only_at_session_start` | 布尔 | `false` | 只在会话开始时应用。目前只记录和显示，尚未生效。 |