diff --git a/CHANGELOG.md b/CHANGELOG.md index f0e3060ee..8e9f7082e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## Unreleased - Share chat history across git worktrees of the same repo, and merge workspace cache writes from concurrent servers instead of overwriting. #558 +- Fix ChatGPT OAuth routing and Responses Lite payloads for GPT-5.6 models. +- Retry overloaded OpenAI post-tool requests without rerunning completed tools. ## 0.151.1 diff --git a/integration-test/integration/chat/subagent_test.clj b/integration-test/integration/chat/subagent_test.clj index f6e069a89..b301e78e6 100644 --- a/integration-test/integration/chat/subagent_test.clj +++ b/integration-test/integration/chat/subagent_test.clj @@ -126,3 +126,70 @@ (= "assistant" (:role e)) (matches? {:type "text" :text "Final answer"} (:content e)))) events)))))) + +(deftest spawn-subagent-retries-overloaded-post-tool-request-test + (eca/start-process!) + (eca/request! (fixture/initialize-request)) + (eca/notify! (fixture/initialized-notification)) + + (llm.mocks/set-case! :subagent-retry-0) + (let [resp (eca/request! (fixture/chat-prompt-request + {:model "openai/gpt-5.2" + :message "What can you find?"})) + parent-chat-id (:chatId resp) + events (drain-content-events-until + (fn [e] + (and (= parent-chat-id (:chatId e)) + (= "assistant" (:role e)) + (= "text" (-> e :content :type)) + (= "Final answer" (-> e :content :text))))) + subagent-chat-id (->> events + (keep :chatId) + (filter #(string/starts-with? % "subagent-")) + first)] + (is (string? subagent-chat-id)) + + (testing "subagent reports retrying the overloaded post-tool request" + (is (some (fn [e] + (and (= subagent-chat-id (:chatId e)) + (= "system" (:role e)) + (matches? {:type "progress" + :state "running" + :text #"Provider overloaded.*Retrying"} + (:content e)))) + events))) + + (testing "subagent tool executes exactly once" + (is (= 1 + (count + (filter (fn [e] + (and (= subagent-chat-id (:chatId e)) + (= "assistant" (:role e)) + (matches? {:type "toolCalled" + :name "directory_tree" + :error false} + (:content e)))) + events))))) + + (testing "subagent finishes after the retried request succeeds" + (is (some (fn [e] + (and (= subagent-chat-id (:chatId e)) + (= parent-chat-id (:parentChatId e)) + (= "assistant" (:role e)) + (matches? {:type "text" :text "Subagent done"} (:content e)))) + events))) + + (testing "parent receives a successful subagent result and finishes" + (is (some (fn [e] + (and (= parent-chat-id (:chatId e)) + (= "assistant" (:role e)) + (matches? {:type "toolCalled" + :name "spawn_agent" + :error false} + (:content e)))) + events)) + (is (some (fn [e] + (and (= parent-chat-id (:chatId e)) + (= "assistant" (:role e)) + (matches? {:type "text" :text "Final answer"} (:content e)))) + events))))) diff --git a/integration-test/llm_mock/openai.clj b/integration-test/llm_mock/openai.clj index 6ed718bde..2309e5e6a 100644 --- a/integration-test/llm_mock/openai.clj +++ b/integration-test/llm_mock/openai.clj @@ -190,73 +190,94 @@ :status "completed"}}) (hk/close ch)) +(defn ^:private send-text-response! [ch text] + (sse-send! ch "response.output_text.delta" + {:type "response.output_text.delta" :delta text}) + (sse-send! ch "response.completed" + {:type "response.completed" + :response {:output [] + :usage {:input_tokens 10 + :output_tokens 5} + :status "completed"}}) + (hk/close ch)) + +(defn ^:private send-function-call-response! [ch item-id call-id tool-name arguments] + (let [args-json (json/generate-string arguments)] + (sse-send! ch "response.output_item.added" + {:type "response.output_item.added" + :item {:type "function_call" + :id item-id + :call_id call-id + :name tool-name + :arguments ""}}) + (sse-send! ch "response.function_call_arguments.delta" + {:type "response.function_call_arguments.delta" + :item_id item-id + :delta args-json}) + (sse-send! ch "response.completed" + {:type "response.completed" + :response {:output [{:type "function_call" + :id item-id + :call_id call-id + :name tool-name + :arguments args-json}] + :usage {:input_tokens 10 + :output_tokens 5} + :status "completed"}}) + (hk/close ch))) + +(defonce ^:private subagent-follow-up-attempt* (atom 0)) + (defn ^:private subagent-spawn-0 - "Three-stage scenario for parent <-> subagent communication: - - Parent's first call: returns a tool_use for `eca__spawn_agent`. - - Subagent's call (recognized by user message \"find files\"): returns text and finishes. - - Parent's follow-up call (recognized by `function_call_output` in input): returns final text." - [ch body] - (let [input (:input body) - has-tool-output? (some #(= "function_call_output" (:type %)) input) - first-user-text (some->> input - (filter #(= "user" (:role %))) - first - :content - first - :text) - subagent-call? (and (not has-tool-output?) - (= "find files" first-user-text))] - (cond - has-tool-output? - (do - (sse-send! ch "response.output_text.delta" - {:type "response.output_text.delta" :delta "Final answer"}) - (sse-send! ch "response.completed" - {:type "response.completed" - :response {:output [] - :usage {:input_tokens 10 - :output_tokens 5} - :status "completed"}}) - (hk/close ch)) + "Parent/subagent scenario, optionally overloading the child's first post-tool request." + ([ch body] (subagent-spawn-0 ch body false)) + ([ch body overload-subagent-follow-up?] + (let [input (:input body) + tool-output-call-ids (->> input + (filter #(= "function_call_output" (:type %))) + (keep :call_id) + set) + parent-follow-up? (contains? tool-output-call-ids "tool-1") + subagent-follow-up? (contains? tool-output-call-ids "sub-tool-1") + first-user-text (some->> input + (filter #(= "user" (:role %))) + first + :content + first + :text) + subagent-call? (and (empty? tool-output-call-ids) + (= "find files" first-user-text))] + (cond + subagent-follow-up? + (if (and overload-subagent-follow-up? + (= 1 (swap! subagent-follow-up-attempt* inc))) + (do + (sse-send! ch "error" + {:type "service_unavailable_error" + :code "server_is_overloaded" + :message "Our servers are currently overloaded. Please try again later."}) + (hk/close ch)) + (send-text-response! ch "Subagent done")) - subagent-call? - (do - (sse-send! ch "response.output_text.delta" - {:type "response.output_text.delta" :delta "Subagent done"}) - (sse-send! ch "response.completed" - {:type "response.completed" - :response {:output [] - :usage {:input_tokens 5 - :output_tokens 3} - :status "completed"}}) - (hk/close ch)) + parent-follow-up? + (send-text-response! ch "Final answer") - :else - (let [args-json (json/generate-string {:agent "explorer" - :task "find files" - :activity "exploring"})] - (sse-send! ch "response.output_item.added" - {:type "response.output_item.added" - :item {:type "function_call" - :id "item-1" - :call_id "tool-1" - :name "eca__spawn_agent" - :arguments ""}}) - (sse-send! ch "response.function_call_arguments.delta" - {:type "response.function_call_arguments.delta" - :item_id "item-1" - :delta args-json}) - (sse-send! ch "response.completed" - {:type "response.completed" - :response {:output [{:type "function_call" - :id "item-1" - :call_id "tool-1" - :name "eca__spawn_agent" - :arguments args-json}] - :usage {:input_tokens 10 - :output_tokens 5} - :status "completed"}}) - (hk/close ch))))) + subagent-call? + (if overload-subagent-follow-up? + (send-function-call-response! ch "sub-item-1" "sub-tool-1" + "eca__directory_tree" + {:path h/default-root-project-path + :max_depth 1}) + (send-text-response! ch "Subagent done")) + + :else + (do + (when overload-subagent-follow-up? + (reset! subagent-follow-up-attempt* 0)) + (send-function-call-response! ch "item-1" "tool-1" "eca__spawn_agent" + {:agent "explorer" + :task "find files" + :activity "exploring"})))))) (defn handle-openai-responses [req] (let [body (some-> (slurp (:body req)) @@ -281,4 +302,5 @@ :reasoning-0 (reasoning-0 ch) :reasoning-1 (reasoning-1 ch) :tool-calling-0 (tool-calling-0 ch body) - :subagent-spawn-0 (subagent-spawn-0 ch body)))))}))) + :subagent-spawn-0 (subagent-spawn-0 ch body) + :subagent-retry-0 (subagent-spawn-0 ch body true)))))}))) diff --git a/src/eca/features/chat.clj b/src/eca/features/chat.clj index 152d49e37..82e40ab77 100644 --- a/src/eca/features/chat.clj +++ b/src/eca/features/chat.clj @@ -1055,6 +1055,7 @@ (lifecycle/maybe-renew-auth-token chat-ctx) (get-in @db* [:auth provider])) :variant (:variant chat-ctx) + :chat-id chat-id :prompt-cache-key (prompt-cache-key agent) :subagent? (some? (get-in @db* [:chats chat-id :subagent])) :cancelled? (fn [] diff --git a/src/eca/llm_api.clj b/src/eca/llm_api.clj index 6df8a5e28..5b069c7a5 100644 --- a/src/eca/llm_api.clj +++ b/src/eca/llm_api.clj @@ -18,6 +18,7 @@ [eca.llm-providers.ollama :as llm-providers.ollama] [eca.llm-providers.openai :as llm-providers.openai] [eca.llm-providers.openai-chat :as llm-providers.openai-chat] + [eca.llm-providers.openai-codex :as llm-providers.openai-codex] [eca.llm-providers.openrouter] [eca.llm-providers.z-ai] [eca.llm-util :as llm-util] @@ -243,8 +244,8 @@ (defn ^:private prompt! [{:keys [provider model model-capabilities instructions user-messages config variant on-message-received on-error on-prepare-tool-call on-tools-called on-reason on-usage-updated - on-server-web-search on-server-image-generation on-history-sanitized - past-messages tools provider-auth sync? subagent? cancelled? prompt-cache-key] + on-server-web-search on-server-image-generation on-history-sanitized retry-request + past-messages tools provider-auth sync? subagent? cancelled? prompt-cache-key turn-context] :or {on-error identity}}] (let [real-model (real-model-name model model-capabilities) tools (when (:tools model-capabilities) tools) @@ -274,6 +275,17 @@ (:extraHeaders model-config)) reasoning-history (or (:reasoningHistory model-config) :all) [auth-type api-key] (llm-util/provider-api-key provider provider-auth config) + openai-codex? (and (= "openai" provider) + (= :auth/oauth auth-type)) + codex-model-fallback (when openai-codex? + (llm-providers.openai-codex/responses-lite-fallback + real-model)) + responses-lite? (if (contains? model-capabilities :codex-responses-lite?) + (:codex-responses-lite? model-capabilities) + (:discovered-codex-responses-lite? codex-model-fallback)) + default-reasoning-effort (or (:default-reasoning-effort model-capabilities) + (:discovered-default-reasoning-effort + codex-model-fallback)) api-url (llm-util/provider-api-url provider config) ;; Flatten {:static :dynamic} instructions map into a single string for non-Anthropic providers flat-instructions (if (map? instructions) (f.prompt/instructions->str instructions) instructions) @@ -303,7 +315,8 @@ :on-reason on-reason :on-usage-updated on-usage-updated :on-server-web-search on-server-web-search - :on-server-image-generation on-server-image-generation})] + :on-server-image-generation on-server-image-generation + :retry-request retry-request})] (try (when-not api-url (throw (ex-info (format "API url not found.\nMake sure you have provider '%s' configured properly." provider) {}))) (cond @@ -319,14 +332,20 @@ :tools tools :web-search web-search :image-generation image-generation - :extra-payload (merge {:parallel_tool_calls true} + :extra-payload (merge {:parallel_tool_calls + (not= false (:supports-parallel-tool-calls? model-capabilities))} extra-payload) :extra-headers extra-headers :reasoning-history reasoning-history :api-url api-url :api-key api-key - :auth-type auth-type + :request-profile (if openai-codex? + llm-providers.openai-codex/request-profile + :openai-responses) :account-id (:account-id provider-auth) + :turn-context turn-context + :responses-lite? responses-lite? + :default-reasoning-effort default-reasoning-effort :prompt-cache-key prompt-cache-key :cancelled? cancelled? :stream-idle-timeout-seconds (:streamIdleTimeoutSeconds config)} @@ -475,7 +494,8 @@ [{:keys [provider model model-capabilities instructions user-messages config on-first-response-received on-message-received on-error on-prepare-tool-call on-tools-called on-reason on-usage-updated on-server-web-search on-server-image-generation on-history-sanitized - past-messages tools provider-auth refresh-provider-auth-fn variant cancelled? on-retry subagent? prompt-cache-key] + past-messages tools provider-auth refresh-provider-auth-fn variant cancelled? on-retry subagent? + prompt-cache-key chat-id] :or {on-first-response-received identity on-message-received identity on-error identity @@ -488,6 +508,11 @@ on-history-sanitized identity cancelled? (constantly false)}}] (let [first-response-received* (atom false) + openai-oauth? (and (= "openai" provider) + (= :auth/oauth + (first (llm-util/provider-api-key provider provider-auth config)))) + turn-context (when openai-oauth? + (llm-providers.openai-codex/new-turn-context chat-id)) ;; Fire :on-history-sanitized at most once per sync-or-async-prompt! call: ;; prompt! is invoked multiple times on retries and we don't want to ;; spam the chat with repeated "history sanitized" notices. @@ -541,7 +566,7 @@ {:exception (ex-message e)}) provider-auth)) provider-auth)) - maybe-retry (fn [error-data attempt on-give-up retry-prompt-fn] + maybe-retry (fn [error-data attempt replay-safe? on-give-up retry-prompt-fn] (let [{error-type :error/type :as classified} (llm-providers.errors/classify-error error-data retry-rules) policy (retry-policy provider-config error-type) @@ -559,10 +584,10 @@ default-rate-limit-max-wait-seconds))) wait-too-long? (boolean (and rate-limit-delay-ms (> rate-limit-delay-ms max-wait-ms)))] - (if (and (contains? #{:rate-limited :overloaded :retryable-custom :premature-stop} error-type) + (if (and replay-safe? + (contains? #{:rate-limited :overloaded :retryable-custom :premature-stop} error-type) (< attempt max-retries) (not wait-too-long?) - (not @first-response-received*) (not (cancelled?))) (let [delay-ms (or rate-limit-delay-ms (retry-delay-ms attempt policy))] @@ -611,12 +636,14 @@ :variant variant :subagent? subagent? :prompt-cache-key prompt-cache-key + :turn-context turn-context :on-error on-error-wrapper :on-history-sanitized on-history-sanitized-wrapper :config config})] (let [{:keys [error output-text reason-text reasoning-content tools-to-call call-tools-fn reason-id usage]} result] (if error - (maybe-retry error attempt on-error-wrapper sync-prompt-with-retry) + (maybe-retry error attempt (not @first-response-received*) + on-error-wrapper sync-prompt-with-retry) (do (when reason-text (on-reason-wrapper {:status :started :id reason-id}) @@ -648,6 +675,7 @@ :variant variant :subagent? subagent? :prompt-cache-key prompt-cache-key + :turn-context turn-context :cancelled? cancelled? :on-message-received on-message-received-wrapper :on-prepare-tool-call on-prepare-tool-call-wrapper @@ -657,10 +685,14 @@ :on-server-image-generation on-server-image-generation-wrapper :on-reason on-reason-wrapper :on-history-sanitized on-history-sanitized-wrapper + :retry-request (fn [{:keys [error-data attempt replay-safe? retry-fn on-give-up]}] + (maybe-retry error-data (or attempt 0) (true? replay-safe?) + (or on-give-up on-error-wrapper) retry-fn)) :on-error (fn [error-data] (if (:silent? (ex-data (:exception error-data))) (on-error-wrapper error-data) - (maybe-retry error-data attempt on-error-wrapper async-prompt-with-retry))) + (maybe-retry error-data attempt (not @first-response-received*) + on-error-wrapper async-prompt-with-retry))) :config config}))] (async-prompt-with-retry* 0))))) diff --git a/src/eca/llm_providers/openai.clj b/src/eca/llm_providers/openai.clj index a36456c46..cdbaa36d4 100644 --- a/src/eca/llm_providers/openai.clj +++ b/src/eca/llm_providers/openai.clj @@ -7,6 +7,7 @@ [eca.config :as config] [eca.features.login :as f.login] [eca.features.providers :as f.providers] + [eca.llm-providers.openai-codex :as openai-codex] [eca.llm-util :as llm-util] [eca.logger :as logger] [eca.message-sanitize :as message-sanitize] @@ -20,8 +21,6 @@ (def ^:private logger-tag "[OPENAI]") (def ^:private responses-path "/v1/responses") -(def ^:private codex-url "https://chatgpt.com/backend-api/codex/responses") -(def ^:private codex-models-url "https://chatgpt.com/backend-api/codex/models?client_version=1.0.0") (def ^:private codex-models-timeout-ms 10000) ;; OpenAI OAuth requests go through the ChatGPT Codex backend, whose context @@ -38,6 +37,9 @@ "gpt-5.5" 272000 "gpt-5.4" 272000 "gpt-5.2" 272000 + "gpt-5.6-sol" 272000 + "gpt-5.6-terra" 272000 + "gpt-5.6-luna" 272000 "gpt-5" 272000}) (defn ^:private pos-num [n] @@ -48,6 +50,11 @@ (merge-with deep-merge-config a b) b)) +(defn ^:private merge-codex-model-config [fallback live] + (cond-> (deep-merge-config fallback live) + (contains? live :discovered-variants) + (assoc :discovered-variants (:discovered-variants live)))) + (defn ^:private codex-context-fallback-for [model] (let [model-name (string/lower-case (str model))] (some (fn [[slug context-limit]] @@ -69,13 +76,17 @@ context-limit (:context_window model) output-limit (or (:max_output_tokens model) (:max_completion_tokens model))] (when (and (string? slug) (not (string/blank? slug))) - [slug (codex-model-config context-limit output-limit)]))) + [slug (deep-merge-config + (codex-model-config context-limit output-limit) + (openai-codex/live-model-discovery model))]))) (defn ^:private codex-fallback-models [static-models] (merge (into {} (map (fn [[model context-limit]] - [model (codex-model-config context-limit nil)])) + [model (deep-merge-config + (codex-model-config context-limit nil) + (or (openai-codex/responses-lite-fallback model) {}))])) codex-context-fallback) (into {} (keep (fn [[model model-config]] @@ -88,33 +99,38 @@ "Resolves OpenAI OAuth (ChatGPT/Codex) model limits from the Codex /models endpoint, falling back to known Codex caps on any failure. Returns a map of model-id -> model-config in user-override shape." - [api-key static-models] - (let [fallback-models (codex-fallback-models static-models)] - (try - (if-not api-key - fallback-models - (let [{:keys [status body]} (http/get codex-models-url - {:headers {"Authorization" (str "Bearer " api-key)} - :throw-exceptions? false - :as :json - :http-client (client/merge-with-global-http-client {}) - :timeout codex-models-timeout-ms})] - (if (not= 200 status) - (do - (logger/warn logger-tag (format "Codex /models endpoint returned status %s" status)) - fallback-models) - (let [live-models (not-empty (into {} - (keep codex-live-model-entry) - (:models body)))] - (or (not-empty (merge-with deep-merge-config fallback-models live-models)) - fallback-models))))) - (catch Exception e - (logger/warn logger-tag (format "Failed to fetch Codex /models endpoint: %s" e)) - fallback-models)))) + ([api-key static-models] + (fetch-oauth-models api-key nil static-models)) + ([api-key account-id static-models] + (let [fallback-models (codex-fallback-models static-models)] + (try + (if-not api-key + fallback-models + (let [{:keys [status body]} (http/get openai-codex/models-url + {:headers (merge + {"Authorization" (str "Bearer " api-key)} + (openai-codex/request-headers + {:account-id account-id})) + :throw-exceptions? false + :as :json + :http-client (client/merge-with-global-http-client {}) + :timeout codex-models-timeout-ms})] + (if (not= 200 status) + (do + (logger/warn logger-tag (format "Codex /models endpoint returned status %s" status)) + fallback-models) + (let [live-models (not-empty (into {} + (keep codex-live-model-entry) + (:models body)))] + (or (not-empty (merge-with merge-codex-model-config fallback-models live-models)) + fallback-models))))) + (catch Exception e + (logger/warn logger-tag (format "Failed to fetch Codex /models endpoint: %s" e)) + fallback-models))))) (defmethod llm-util/provider-models-override ["openai" :auth/oauth] - [{:keys [api-key static-models]}] - (fetch-oauth-models api-key static-models)) + [{:keys [api-key account-id static-models]}] + (fetch-oauth-models api-key account-id static-models)) (defn ^:private jwt-payload->account-id "Extract account ID from JWT payload, checking multiple locations like opencode does." @@ -203,11 +219,14 @@ :request-id request-id :headers response-headers))) -(defn ^:private base-responses-request! [{:keys [rid body api-url auth-type url-relative-path api-key account-id on-error on-stream http-client extra-headers cancelled? stream-idle-timeout-seconds]}] - (let [oauth? (= :auth/oauth auth-type) +(defn ^:private base-responses-request! + [{:keys [rid body api-url request-profile url-relative-path api-key account-id + turn-context responses-lite? on-error on-stream http-client extra-headers + cancelled? stream-idle-timeout-seconds]}] + (let [codex? (openai-codex/responses-profile? request-profile) stream? (and on-stream (not= false (:stream body))) - url (if oauth? - codex-url + url (if codex? + openai-codex/responses-url (join-api-url api-url (or url-relative-path responses-path))) ;; Use persisted account-id first, fall back to extracting from JWT resolved-account-id (or account-id (jwt-token->account-id api-key)) @@ -216,13 +235,13 @@ extra-headers) headers (client/merge-llm-headers (merge - (assoc-some - {"Authorization" (str "Bearer " api-key) - "Content-Type" "application/json"} - "ChatGPT-Account-Id" resolved-account-id - "OpenAI-Beta" (when oauth? "responses=experimental"), - "Originator" (when oauth? "codex_cli_rs") - "Session-ID" (when oauth? (str (random-uuid)))) + {"Authorization" (str "Bearer " api-key) + "Content-Type" "application/json"} + (when codex? + (openai-codex/request-headers + {:account-id resolved-account-id + :turn-context turn-context + :responses-lite? responses-lite?})) extra-headers)) on-error (or on-error (fn [error-data] @@ -237,6 +256,10 @@ :throw-exceptions? false :http-client (client/merge-with-global-http-client http-client) :as (if stream? :stream :json)})] + (when (and codex? (= 200 status)) + (openai-codex/capture-turn-state! + turn-context + (response-header resp-headers "x-codex-turn-state"))) (if (not= 200 status) (let [body-str (if stream? (slurp body) body)] (logger/warn logger-tag "Unexpected response status: %s body: %s" status body-str) @@ -320,6 +343,38 @@ (format "Internal error: %s" (or (ex-message e) (.getName (class e)))) (llm-util/connection-error-message e))}))))) +(def ^:private responses-replay-blocking-events + #{"response.output_text.delta" + "response.output_text.annotation.added" + "response.reasoning_summary_text.delta" + "response.reasoning_summary_text.done" + "response.output_item.added" + "response.output_item.done" + "response.completed"}) + +(defn ^:private request-with-retry! + "Runs one exact Responses API request with request-scoped retries. + The shared retry controller owns policy and backoff; this wrapper only tracks + whether the current HTTP attempt emitted output that makes replay unsafe." + [{:keys [on-error on-stream retry-request] :as request-opts}] + (letfn [(request! [attempt] + (let [replay-safe?* (atom true)] + (base-responses-request! + (assoc request-opts + :on-error (fn [error-data] + (if retry-request + (retry-request {:error-data error-data + :attempt attempt + :replay-safe? @replay-safe?* + :on-give-up on-error + :retry-fn request!}) + (on-error error-data))) + :on-stream (fn [event data & args] + (when (contains? responses-replay-blocking-events event) + (reset! replay-safe?* false)) + (apply on-stream event data args))))))] + (request! 0))) + (defn ^:private normalize-messages [messages supports-image?] ;; Each history entry maps to one or more provider messages. Switched from ;; `keep` to `mapcat` so a single history role can emit multiple messages @@ -419,33 +474,41 @@ (defn create-response! [{:keys [model user-messages instructions reason? supports-image? api-key api-url url-relative-path max-output-tokens past-messages tools web-search image-generation extra-payload extra-headers - auth-type account-id http-client prompt-cache-key cancelled? stream-idle-timeout-seconds]} + account-id http-client prompt-cache-key cancelled? stream-idle-timeout-seconds + request-profile turn-context responses-lite? default-reasoning-effort]} {:keys [on-message-received on-error on-prepare-tool-call on-tools-called on-reason on-usage-updated - on-server-web-search on-server-image-generation] :as callbacks}] - (let [oauth? (= :auth/oauth auth-type) + on-server-web-search on-server-image-generation retry-request] :as callbacks}] + (let [codex? (openai-codex/responses-profile? request-profile) + responses-lite? (and codex? responses-lite?) + turn-context (when codex? + (or turn-context (openai-codex/new-turn-context nil))) input (concat (normalize-messages past-messages supports-image?) (normalize-messages user-messages supports-image?)) tools (->tools tools web-search image-generation) - body (merge - (assoc-some - {:model model - :input (if oauth? - (concat [{:role "system" :content instructions}] input) - input) - :prompt_cache_key (or prompt-cache-key - (str (System/getProperty "user.name") "@ECA")) - :instructions instructions - :tools tools - :include (when reason? - ["reasoning.encrypted_content"]) - :store false - :reasoning (when reason? - {:effort "medium" - :summary "auto"}) - :stream true} - :max_output_tokens (when-not oauth? max-output-tokens) - :parallel_tool_calls (:parallel_tool_calls extra-payload)) - extra-payload) + base-body (merge + (assoc-some + {:model model + :input input + :prompt_cache_key (or prompt-cache-key + (str (System/getProperty "user.name") "@ECA")) + :instructions instructions + :tools tools + :include (when reason? + ["reasoning.encrypted_content"]) + :store false + :reasoning (when reason? + {:effort (or default-reasoning-effort "medium") + :summary "auto"}) + :stream true} + :max_output_tokens (when-not codex? max-output-tokens) + :tool_choice (when codex? "auto") + :parallel_tool_calls (:parallel_tool_calls extra-payload)) + extra-payload) + prepare-body (fn [body] + (if responses-lite? + (openai-codex/responses-lite-body body) + body)) + body (prepare-body base-body) tool-call-by-item-id* (atom {}) reasoning-item-id* (atom nil) sync-result* (when-not callbacks (atom nil)) @@ -573,22 +636,26 @@ (when-let [{:keys [new-messages tools fresh-api-key provider-auth]} (on-tools-called tool-calls)] (let [new-messages (message-sanitize/sanitize-outbound-messages new-messages)] (reset! tool-call-by-item-id* {}) - (base-responses-request! + (request-with-retry! {:rid (llm-util/gen-rid) - :body (assoc body - :input (normalize-messages new-messages supports-image?) - :tools (->tools tools web-search image-generation)) + :body (prepare-body + (assoc base-body + :input (normalize-messages new-messages supports-image?) + :tools (->tools tools web-search image-generation))) :api-url api-url :url-relative-path url-relative-path :api-key (or fresh-api-key api-key) :account-id (or (:account-id provider-auth) account-id) :http-client http-client :extra-headers extra-headers - :auth-type auth-type + :request-profile request-profile + :turn-context turn-context + :responses-lite? responses-lite? :cancelled? cancelled? :stream-idle-timeout-seconds stream-idle-timeout-seconds :on-error on-error - :on-stream handle-stream}))) + :on-stream handle-stream + :retry-request retry-request}))) (on-message-received {:type :finish :finish-reason (-> data :response :status)}))) nil)) @@ -610,7 +677,9 @@ :account-id account-id :http-client http-client :extra-headers extra-headers - :auth-type auth-type + :request-profile request-profile + :turn-context turn-context + :responses-lite? responses-lite? :cancelled? cancelled? :stream-idle-timeout-seconds stream-idle-timeout-seconds :on-error on-error diff --git a/src/eca/llm_providers/openai_codex.clj b/src/eca/llm_providers/openai_codex.clj new file mode 100644 index 000000000..6feb8fae6 --- /dev/null +++ b/src/eca/llm_providers/openai_codex.clj @@ -0,0 +1,121 @@ +(ns eca.llm-providers.openai-codex + (:require + [clojure.string :as string] + + [eca.shared :refer [assoc-some]])) + +(set! *warn-on-reflection* true) + +(def ^:private codex-compatibility-version "0.146.0") + +(def responses-url "https://chatgpt.com/backend-api/codex/responses") +(def models-url + (str "https://chatgpt.com/backend-api/codex/models?client_version=" + codex-compatibility-version)) + +(def request-profile :chatgpt-codex) + +(def ^:private responses-lite-fallbacks + {"gpt-5.6-sol" {:default-reasoning-effort "low" + :supported-reasoning-efforts ["low" "medium" "high" "xhigh" "max"]} + "gpt-5.6-terra" {:default-reasoning-effort "medium" + :supported-reasoning-efforts ["low" "medium" "high" "xhigh" "max"]} + "gpt-5.6-luna" {:default-reasoning-effort "medium" + :supported-reasoning-efforts ["low" "medium" "high" "xhigh" "max"]}}) + +(def ^:private reasoning-efforts + #{"none" "low" "medium" "high" "xhigh" "max"}) + +(defn responses-profile? [profile] + (= request-profile profile)) + +(defn new-turn-context + "Creates state shared by every request and retry in one user turn. + A supplied conversation ID keeps routing stable across turns in the same chat." + [conversation-id] + (let [conversation-id (or (some-> conversation-id str not-empty) + (str (random-uuid)))] + {:session-id conversation-id + :thread-id conversation-id + :turn-state* (atom nil)})) + +(defn request-headers + [{:keys [account-id turn-context responses-lite?]}] + (let [{:keys [session-id thread-id turn-state*]} turn-context] + (assoc-some + ;; The backend reports a mismatched Originator/User-Agent pair as + ;; server_is_overloaded, so keep Codex's default identity paired. + {"Originator" "codex_cli_rs" + "User-Agent" (str "codex_cli_rs/" codex-compatibility-version)} + "ChatGPT-Account-ID" account-id + "Session-ID" session-id + "Thread-ID" thread-id + "x-client-request-id" thread-id + "x-codex-turn-state" (some-> turn-state* deref) + "x-openai-internal-codex-responses-lite" (when responses-lite? "true")))) + +(defn capture-turn-state! + "Stores the first routing state returned for a turn, matching Codex's + first-write-wins behavior." + [turn-context turn-state] + (when-let [turn-state* (:turn-state* turn-context)] + (when-not (string/blank? turn-state) + (compare-and-set! turn-state* nil turn-state)))) + +(defn ^:private normalize-reasoning-efforts [levels] + (->> levels + (keep #(if (map? %) (:effort %) %)) + (map #(if (= "ultra" %) "max" %)) + (filter reasoning-efforts) + distinct + vec + not-empty)) + +(defn ^:private reasoning-variants [efforts] + (when efforts + (into {} + (map (fn [effort] + [effort {:reasoning {:effort effort :summary "auto"}}])) + efforts))) + +(defn live-model-discovery + [{:keys [use_responses_lite default_reasoning_level + supported_reasoning_levels supports_parallel_tool_calls] :as model}] + (let [efforts (normalize-reasoning-efforts supported_reasoning_levels)] + (assoc-some (cond-> {} + (contains? model :use_responses_lite) + (assoc :discovered-codex-responses-lite? (true? use_responses_lite))) + :discovered-default-reasoning-effort default_reasoning_level + :discovered-supports-parallel-tool-calls? supports_parallel_tool_calls + :discovered-variants (reasoning-variants efforts)))) + +(defn responses-lite-fallback [model] + (when-let [{:keys [default-reasoning-effort supported-reasoning-efforts]} + (get responses-lite-fallbacks (string/lower-case (str model)))] + {:discovered-codex-responses-lite? true + :discovered-default-reasoning-effort default-reasoning-effort + :discovered-variants (reasoning-variants supported-reasoning-efforts)})) + +(defn responses-lite-body + "Projects a regular Responses request into the Codex Responses Lite shape." + [body] + (let [instructions (:instructions body) + tools (->> (:tools body) + (filterv #(= "function" (:type %)))) + input (cond-> [{:type "additional_tools" + :role "developer" + :tools tools}] + (not (string/blank? instructions)) + (conj {:type "message" + :role "developer" + :content [{:type "input_text" + :text instructions}]}) + + true + (into (:input body)))] + (-> body + (dissoc :instructions :tools) + (assoc :input input + :parallel_tool_calls false + :reasoning (assoc (or (:reasoning body) {}) + :context "all_turns"))))) diff --git a/src/eca/models.clj b/src/eca/models.clj index d3b5ee233..4b8604a01 100644 --- a/src/eca/models.clj +++ b/src/eca/models.clj @@ -441,8 +441,9 @@ (when (contains? native-models-endpoint-providers (:api provider-config)) (let [api-url (or (get-in db [:auth provider :api-url]) (llm-util/provider-api-url provider config)) + provider-auth (get-in db [:auth provider]) [auth-type api-key] (llm-util/provider-api-key provider - (get-in db [:auth provider]) + provider-auth config) api-type (:api provider-config)] ;; Provider+auth specific source first (e.g. OpenAI OAuth -> ChatGPT Codex @@ -452,6 +453,7 @@ {:provider provider :auth-type auth-type :api-key api-key + :account-id (:account-id provider-auth) :static-models (:models provider-config)}) (when api-url (fetch-provider-native-models @@ -537,6 +539,9 @@ :reason? (:discovered-reason? model-config) :api (:discovered-api model-config) :variants (:discovered-variants model-config) + :codex-responses-lite? (:discovered-codex-responses-lite? model-config) + :default-reasoning-effort (:discovered-default-reasoning-effort model-config) + :supports-parallel-tool-calls? (:discovered-supports-parallel-tool-calls? model-config) :image-input? (:imageInput model-config) :limit (not-empty limit-overrides) :max-output-tokens output-override diff --git a/test/eca/llm_api_test.clj b/test/eca/llm_api_test.clj index 3533320aa..5185062f0 100644 --- a/test/eca/llm_api_test.clj +++ b/test/eca/llm_api_test.clj @@ -1,6 +1,8 @@ (ns eca.llm-api-test (:require [clojure.test :refer [deftest is testing]] + [cheshire.core :as json] + [hato.client :as http] [eca.client-test-helpers :refer [with-client-proxied *http-client-captures*]] [eca.config :as config] [eca.llm-api :as llm-api] @@ -8,6 +10,7 @@ [eca.llm-providers.ollama :as llm-providers.ollama] [eca.llm-providers.openai :as llm-providers.openai] [eca.llm-providers.openai-chat :as llm-providers.openai-chat] + [eca.llm-providers.openai-codex :as llm-providers.openai-codex] [eca.secrets :as secrets] [eca.test-helper :as h])) @@ -518,6 +521,181 @@ (is (not (true? (:image-generation @captured*))) "openai handler should NOT receive :image-generation true when capability is off")))) +(deftest prompt-selects-codex-profile-only-for-openai-oauth-test + (let [base-opts {:model "gpt-5.6-sol" + :model-capabilities {:api :openai-responses + :tools true + :reason? true + :web-search false + :model-name "gpt-5.6-sol" + :codex-responses-lite? true + :default-reasoning-effort "low"} + :instructions "test" + :user-messages [{:role "user" :content [{:type :text :text "hi"}]}] + :past-messages [] + :tools [] + :sync? false}] + (testing "OpenAI OAuth selects the Codex profile and forwards model metadata" + (let [captured* (atom nil) + turn-context (llm-providers.openai-codex/new-turn-context "chat-1")] + (with-redefs [llm-providers.openai/create-response! + (fn [opts _callbacks] (reset! captured* opts) :ok)] + (#'eca.llm-api/prompt! + (merge base-opts + {:provider "openai" + :provider-auth {:api-key "oauth-token" + :type :auth/oauth + :account-id "account-1"} + :turn-context turn-context + :config {:providers {"openai" {:api "openai-responses" + :url "https://api.openai.com" + :models {}}}}}))) + (is (= llm-providers.openai-codex/request-profile + (:request-profile @captured*))) + (is (identical? turn-context (:turn-context @captured*))) + (is (true? (:responses-lite? @captured*))) + (is (= "low" (:default-reasoning-effort @captured*))))) + + (testing "known GPT-5.6 models use Lite before OAuth model discovery completes" + (let [captured* (atom nil)] + (with-redefs [llm-providers.openai/create-response! + (fn [opts _callbacks] (reset! captured* opts) :ok)] + (#'eca.llm-api/prompt! + (merge base-opts + {:provider "openai" + :model-capabilities (dissoc (:model-capabilities base-opts) + :codex-responses-lite? + :default-reasoning-effort) + :provider-auth {:api-key "oauth-token" :type :auth/oauth} + :config {:providers {"openai" {:api "openai-responses" + :url "https://api.openai.com" + :models {}}}}}))) + (is (true? (:responses-lite? @captured*))) + (is (= "low" (:default-reasoning-effort @captured*))))) + + (testing "OpenAI API keys keep the standard Responses profile" + (let [captured* (atom nil)] + (with-redefs [llm-providers.openai/create-response! + (fn [opts _callbacks] (reset! captured* opts) :ok)] + (#'eca.llm-api/prompt! + (merge base-opts + {:provider "openai" + :provider-auth {:api-key "sk-test" :type :auth/api-key} + :config {:providers {"openai" {:api "openai-responses" + :url "https://api.openai.com" + :models {}}}}}))) + (is (= :openai-responses (:request-profile @captured*))))) + + (testing "Copilot and custom Responses providers receive no Codex profile" + (doseq [[provider provider-auth provider-config] + [["github-copilot" + {:api-key "copilot-token" + :api-url "https://api.githubcopilot.com" + :type :auth/oauth} + {:api "openai-chat" + :url "https://api.githubcopilot.com" + :models {}}] + ["gateway" + {:api-key "gateway-token" :type :auth/api-key} + {:api "openai-responses" + :url "https://gateway.example.com" + :models {"gpt-5.6-sol" {}}}]]] + (let [captured* (atom nil)] + (with-redefs [llm-providers.openai/create-response! + (fn [opts _callbacks] (reset! captured* opts) :ok)] + (#'eca.llm-api/prompt! + (merge base-opts + {:provider provider + :provider-auth provider-auth + :config {:providers {provider provider-config}}}))) + (is (nil? (:request-profile @captured*)) + (str provider " must not inherit ChatGPT Codex transport"))))))) + +(deftest oauth-gpt-5-6-lite-retries-post-tool-request-test + (testing "GPT-5.6 Lite keeps routing state and executes the tool once across overload retry" + (let [requests* (atom []) + tools-called* (atom 0) + messages* (atom []) + terminal-errors* (atom []) + stream (fn [text] + (java.io.ByteArrayInputStream. + (.getBytes ^String text java.nio.charset.StandardCharsets/UTF_8))) + tool-stream (str + "event: response.completed\n" + "data: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"output\":[{\"type\":\"function_call\",\"id\":\"item-1\",\"call_id\":\"call-1\",\"name\":\"eca__directory_tree\",\"arguments\":\"{}\"}],\"usage\":{\"input_tokens\":1,\"output_tokens\":1}}}\n\n") + overloaded-stream (str + "event: error\n" + "data: {\"type\":\"service_unavailable_error\",\"code\":\"server_is_overloaded\",\"message\":\"Our servers are currently overloaded.\"}\n\n") + final-stream (str + "event: response.output_text.delta\n" + "data: {\"type\":\"response.output_text.delta\",\"delta\":\"done\"}\n\n" + "event: response.completed\n" + "data: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"output\":[],\"usage\":{\"input_tokens\":1,\"output_tokens\":1}}}\n\n")] + (with-redefs [http/post + (fn [url opts] + (let [request-number (inc (count @requests*)) + body (json/parse-string (:body opts) true)] + (swap! requests* conj {:url url + :headers (:headers opts) + :body body}) + {:status 200 + :headers {"x-codex-turn-state" + (if (= 1 request-number) "state-1" "state-2")} + :body (stream (case request-number + 1 tool-stream + 2 overloaded-stream + final-stream))})) + eca.llm-api/sleep-with-cancel (fn [_ _] true)] + (llm-api/sync-or-async-prompt! + {:provider "openai" + :model "gpt-5.6-sol" + :model-capabilities {:api :openai-responses + :model-name "gpt-5.6-sol" + :tools true + :reason? true + :web-search true + :image-generation? true + :codex-responses-lite? true + :default-reasoning-effort "low"} + :instructions "test" + :user-messages [{:role "user" :content [{:type :text :text "hello"}]}] + :past-messages [] + :tools [{:full-name "eca__directory_tree" + :description "list" + :parameters {:type "object"}}] + :chat-id "chat-123" + :config {:providers {"openai" {:api "openai-responses" + :url "https://api.openai.com" + :models {"gpt-5.6-sol" {}}}}} + :provider-auth {:api-key "oauth-token" + :account-id "account-1" + :type :auth/oauth} + :on-message-received #(swap! messages* conj %) + :on-error #(swap! terminal-errors* conj %) + :on-tools-called (fn [_] + (swap! tools-called* inc) + {:new-messages [{:role "tool_call_output" + :content {:id "call-1" + :output {:contents [{:type :text + :text "result"}]}}}] + :tools []})}) + + (is (= 3 (count @requests*))) + (is (= 1 @tools-called*)) + (is (empty? @terminal-errors*)) + (is (some #(= {:type :text :text "done"} %) @messages*)) + (doseq [{:keys [body]} @requests*] + (is (= "additional_tools" (get-in body [:input 0 :type]))) + (is (nil? (:instructions body))) + (is (nil? (:tools body))) + (is (false? (:parallel_tool_calls body)))) + (is (nil? (get-in (first @requests*) [:headers "x-codex-turn-state"]))) + (is (= ["state-1" "state-1"] + (mapv #(get-in % [:headers "x-codex-turn-state"]) + (rest @requests*)))) + (is (every? #(= "chat-123" (get-in % [:headers "Session-ID"])) + @requests*)))))) + (deftest prompt-forwards-stream-idle-timeout-and-cache-retention-to-anthropic-handler-test (testing "custom provider with :api anthropic forwards :stream-idle-timeout-seconds and :cache-retention to chat!" (let [captured* (atom nil)] @@ -666,6 +844,42 @@ :provider-auth {:api-key "test-key"}} (dissoc overrides :stream)))) +(deftest openai-retry-reuses-turn-context-test + (testing "outer retries keep one chat-scoped Codex turn context" + (let [contexts* (atom []) + attempts* (atom 0)] + (with-redefs [eca.llm-api/prompt! + (fn [{:keys [turn-context]}] + (swap! contexts* conj turn-context) + (if (= 1 (swap! attempts* inc)) + {:error {:status 503 + :body "server_is_overloaded" + :message "OpenAI response status: 503"}} + {:output-text "ok" + :usage {:input-tokens 1 :output-tokens 1}})) + eca.llm-api/sleep-with-cancel (fn [_ _] true)] + (llm-api/sync-or-async-prompt! + (make-prompt-opts + {:stream false + :provider "openai" + :model "gpt-5.6-sol" + :model-capabilities {:tools true + :reason? true + :web-search false + :codex-responses-lite? true} + :chat-id "chat-123" + :provider-auth {:api-key "oauth-token" :type :auth/oauth} + :config {:providers {"openai" {:api "openai-responses" + :url "https://api.openai.com" + :models {"gpt-5.6-sol" + {:extraPayload {:stream false}}}}}} + :on-error identity + :on-message-received identity}))) + (is (= 2 (count @contexts*))) + (is (identical? (first @contexts*) (second @contexts*))) + (is (= "chat-123" (:session-id (first @contexts*)))) + (is (= "chat-123" (:thread-id (first @contexts*))))))) + (deftest refresh-provider-auth-fn-used-for-initial-and-retry-test (testing "refresh-provider-auth-fn supplies a fresh provider-auth on each prompt! call" ;; Previously provider-auth was captured once and reused across retries, @@ -1168,6 +1382,116 @@ :error/source :openai-responses}] @errors*)))))) +(deftest async-request-scoped-retry-test + (testing "retries a replay-safe request after earlier visible output" + (let [prompt-calls* (atom 0) + request-retries* (atom 0) + retry-events* (atom []) + errors* (atom [])] + (with-redefs [eca.llm-api/prompt! (fn [{:keys [on-prepare-tool-call on-message-received on-error retry-request]}] + (swap! prompt-calls* inc) + (on-prepare-tool-call {:id "call_1" + :full-name "tool" + :arguments-text "{}"}) + (retry-request + {:error-data {:code "server_is_overloaded" + :message "Request failed" + :error/source :openai-responses} + :attempt 0 + :replay-safe? true + :on-give-up on-error + :retry-fn (fn [next-attempt] + (is (= 1 next-attempt)) + (swap! request-retries* inc) + (on-message-received {:type :text :text "recovered"}) + (on-message-received {:type :finish :finish-reason "stop"}))})) + eca.llm-api/sleep-with-cancel (fn [_ cancelled?] (not (cancelled?)))] + (llm-api/sync-or-async-prompt! + (make-prompt-opts + {:on-retry (fn [event] (swap! retry-events* conj event)) + :on-error (fn [error] (swap! errors* conj error)) + :on-prepare-tool-call identity + :on-message-received identity}))) + (is (= 1 @prompt-calls*) "the outer prompt must not be replayed") + (is (= 1 @request-retries*)) + (is (= 1 (count @retry-events*))) + (is (= :overloaded (get-in (first @retry-events*) [:classified :error/type]))) + (is (empty? @errors*)))) + + (testing "does not retry a request that is not replay-safe" + (let [request-retries* (atom 0) + sleep-calls* (atom 0) + errors* (atom [])] + (with-redefs [eca.llm-api/prompt! (fn [{:keys [on-message-received on-error retry-request]}] + (on-message-received {:type :text :text "partial"}) + (retry-request + {:error-data {:code "server_error" + :message "Request failed" + :error/source :openai-responses} + :attempt 0 + :replay-safe? false + :on-give-up on-error + :retry-fn (fn [_] + (swap! request-retries* inc))})) + eca.llm-api/sleep-with-cancel (fn [_ _] + (swap! sleep-calls* inc) + true)] + (llm-api/sync-or-async-prompt! + (make-prompt-opts + {:on-error (fn [error] (swap! errors* conj error)) + :on-message-received identity}))) + (is (zero? @request-retries*)) + (is (zero? @sleep-calls*)) + (is (= 1 (count @errors*))))) + + (testing "gives up when the request retry budget is exhausted" + (let [request-retries* (atom 0) + errors* (atom [])] + (with-redefs [eca.llm-api/prompt! (fn [{:keys [on-message-received on-error retry-request]}] + (on-message-received {:type :text :text "earlier output"}) + (retry-request + {:error-data {:code "server_error" + :message "Request failed" + :error/source :openai-responses} + :attempt 1 + :replay-safe? true + :on-give-up on-error + :retry-fn (fn [_] + (swap! request-retries* inc))})) + eca.llm-api/sleep-with-cancel (fn [_ _] true)] + (llm-api/sync-or-async-prompt! + (make-prompt-opts + {:config {:providers {"anthropic" {:key "test-key" + :url "http://test" + :retry {:maxRetries 1} + :models {"claude-sonnet-4-6" {}}}}} + :on-error (fn [error] (swap! errors* conj error)) + :on-message-received identity}))) + (is (zero? @request-retries*)) + (is (= 1 (count @errors*))))) + + (testing "gives up when cancellation interrupts request retry backoff" + (let [request-retries* (atom 0) + errors* (atom [])] + (with-redefs [eca.llm-api/prompt! (fn [{:keys [on-message-received on-error retry-request]}] + (on-message-received {:type :text :text "earlier output"}) + (retry-request + {:error-data {:code "server_error" + :message "Request failed" + :error/source :openai-responses} + :attempt 0 + :replay-safe? true + :on-give-up on-error + :retry-fn (fn [_] + (swap! request-retries* inc))})) + eca.llm-api/sleep-with-cancel (fn [_ _] false)] + (llm-api/sync-or-async-prompt! + (make-prompt-opts + {:on-error (fn [error] (swap! errors* conj error)) + :on-message-received identity}))) + (is (zero? @request-retries*)) + (is (= 1 (count @errors*)))))) + (deftest async-duplicate-errors-delivered-once-test (testing "only the first terminal error reaches on-error when stacked requests fail together (#547)" (let [errors* (atom [])] diff --git a/test/eca/llm_providers/openai_codex_test.clj b/test/eca/llm_providers/openai_codex_test.clj new file mode 100644 index 000000000..d2dfc905b --- /dev/null +++ b/test/eca/llm_providers/openai_codex_test.clj @@ -0,0 +1,91 @@ +(ns eca.llm-providers.openai-codex-test + (:require + [clojure.test :refer [deftest is testing]] + [eca.llm-providers.openai-codex :as openai-codex])) + +(set! *warn-on-reflection* true) + +(deftest turn-context-headers-test + (let [context (openai-codex/new-turn-context "chat-123") + client-version (second (re-find #"client_version=(\d+\.\d+\.\d+)" + openai-codex/models-url))] + (testing "session and thread routing stay stable for the chat" + (let [headers (openai-codex/request-headers + {:account-id "account-1" + :turn-context context})] + (is (= {"ChatGPT-Account-ID" "account-1" + "Originator" "codex_cli_rs" + "Session-ID" "chat-123" + "Thread-ID" "chat-123" + "x-client-request-id" "chat-123"} + (dissoc headers "User-Agent"))) + (is (= (str "codex_cli_rs/" client-version) + (get headers "User-Agent"))))) + + (testing "the first turn state is replayed and later values cannot replace it" + (openai-codex/capture-turn-state! context "state-1") + (openai-codex/capture-turn-state! context "state-2") + (is (= "state-1" + (get (openai-codex/request-headers {:turn-context context}) + "x-codex-turn-state")))))) + +(deftest responses-lite-body-test + (let [body (openai-codex/responses-lite-body + {:model "gpt-5.6-sol" + :instructions "Use the repository rules." + :input [{:role "user" :content "Fix it"}] + :tools [{:type "function" :name "eca__read_file"} + {:type "web_search"} + {:type "image_generation"}] + :parallel_tool_calls true + :reasoning {:effort "low" :summary "auto"} + :stream true})] + (is (nil? (:instructions body))) + (is (nil? (:tools body))) + (is (false? (:parallel_tool_calls body))) + (is (= {:effort "low" :summary "auto" :context "all_turns"} + (:reasoning body))) + (is (= [{:type "additional_tools" + :role "developer" + :tools [{:type "function" :name "eca__read_file"}]} + {:type "message" + :role "developer" + :content [{:type "input_text" + :text "Use the repository rules."}]} + {:role "user" :content "Fix it"}] + (:input body))))) + +(deftest responses-lite-body-without-reasoning-test + (testing "Lite always declares all-turn reasoning context" + (is (= {:context "all_turns"} + (:reasoning + (openai-codex/responses-lite-body + {:input [{:role "user" :content "Generate a title"}]})))))) + +(deftest live-model-discovery-test + (is (= {:discovered-codex-responses-lite? true + :discovered-default-reasoning-effort "low" + :discovered-supports-parallel-tool-calls? true + :discovered-variants + {"low" {:reasoning {:effort "low" :summary "auto"}} + "max" {:reasoning {:effort "max" :summary "auto"}}}} + (openai-codex/live-model-discovery + {:use_responses_lite true + :default_reasoning_level "low" + :supported_reasoning_levels [{:effort "low"} + {:effort "max"} + {:effort "ultra"}] + :supports_parallel_tool_calls true}))) + (testing "an explicit live false value overrides static Lite fallbacks" + (is (false? (:discovered-codex-responses-lite? + (openai-codex/live-model-discovery + {:use_responses_lite false})))))) + +(deftest responses-lite-fallback-test + (testing "all current GPT-5.6 Codex models retain Lite metadata without /models" + (doseq [model ["gpt-5.6-sol" "gpt-5.6-terra" "gpt-5.6-luna"]] + (is (true? (:discovered-codex-responses-lite? + (openai-codex/responses-lite-fallback model)))))) + (testing "model matching is case-insensitive" + (is (true? (:discovered-codex-responses-lite? + (openai-codex/responses-lite-fallback "GPT-5.6-SOL")))))) diff --git a/test/eca/llm_providers/openai_test.clj b/test/eca/llm_providers/openai_test.clj index c851ac875..eb2cda26f 100644 --- a/test/eca/llm_providers/openai_test.clj +++ b/test/eca/llm_providers/openai_test.clj @@ -4,6 +4,7 @@ [clojure.test :refer [deftest is testing]] [eca.client-test-helpers :refer [blocking-input-stream with-client-proxied]] [eca.llm-providers.openai :as llm-providers.openai] + [eca.llm-providers.openai-codex :as openai-codex] [hato.client :as http] [matcher-combinators.test :refer [match?]])) @@ -57,6 +58,89 @@ (is (= error-body (get-in result [:error :body]))) (is (= "7" (get-in result [:error :headers "retry-after"])))))))) +(deftest base-responses-codex-routing-test + (testing "Codex profile sends stable routing headers and replays returned turn state" + (let [requests* (atom []) + turn-context (openai-codex/new-turn-context "chat-123")] + (with-redefs [http/post (fn [url opts] + (swap! requests* conj [url opts]) + {:status 200 + :headers {"x-codex-turn-state" "turn-state-1"} + :body {:output [{:content [{:text "ok"}]}]}})] + (dotimes [_ 2] + (#'llm-providers.openai/base-responses-request! + {:rid "r1" + :api-key "oauth-token" + :account-id "account-1" + :api-url "https://api.openai.com" + :request-profile openai-codex/request-profile + :turn-context turn-context + :responses-lite? true + :body {:model "gpt-5.6-sol" :input "hi" :stream false}})) + + (let [[first-url first-request] (first @requests*) + [_ second-request] (second @requests*)] + (is (= openai-codex/responses-url first-url)) + (is (= "chat-123" (get-in first-request [:headers "Session-ID"]))) + (is (= "chat-123" (get-in first-request [:headers "Thread-ID"]))) + (is (= "chat-123" (get-in first-request [:headers "x-client-request-id"]))) + (is (= "account-1" (get-in first-request [:headers "ChatGPT-Account-ID"]))) + (is (= "true" + (get-in first-request + [:headers "x-openai-internal-codex-responses-lite"]))) + (is (nil? (get-in first-request [:headers "OpenAI-Beta"]))) + (is (nil? (get-in first-request [:headers "x-codex-turn-state"]))) + (is (= "turn-state-1" + (get-in second-request [:headers "x-codex-turn-state"])))))))) + +(deftest create-response-codex-tool-continuation-replays-turn-state-test + (testing "the post-tool request stays on the first response's Codex turn state" + (let [requests* (atom []) + first-stream (str + "event: response.completed\n" + "data: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"output\":[{\"type\":\"function_call\",\"id\":\"item-1\",\"call_id\":\"call-1\",\"name\":\"eca__shell_command\",\"arguments\":\"{}\"}],\"usage\":{\"input_tokens\":1,\"output_tokens\":1}}}\n\n") + final-stream (str + "event: response.completed\n" + "data: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"output\":[],\"usage\":{\"input_tokens\":1,\"output_tokens\":1}}}\n\n")] + (with-redefs [http/post + (fn [_url opts] + (swap! requests* conj opts) + {:status 200 + :headers {"x-codex-turn-state" + (if (= 1 (count @requests*)) "state-1" "state-2")} + :body (java.io.ByteArrayInputStream. + (.getBytes ^String (if (= 1 (count @requests*)) + first-stream + final-stream) + java.nio.charset.StandardCharsets/UTF_8))})] + (llm-providers.openai/create-response! + {:model "gpt-test" + :user-messages [{:role "user" :content [{:type :text :text "hi"}]}] + :instructions "test" + :reason? false + :supports-image? false + :api-key "oauth-token" + :api-url "https://api.openai.com" + :past-messages [] + :tools [{:full-name "eca__shell_command" + :description "run" + :parameters {:type "object"}}] + :web-search false + :request-profile openai-codex/request-profile + :turn-context (openai-codex/new-turn-context "chat-1")} + {:on-message-received (fn [_]) + :on-error (fn [error] (throw (ex-info "unexpected error" error))) + :on-prepare-tool-call (fn [_]) + :on-tools-called (fn [_] {:new-messages [] :tools []}) + :on-reason (fn [_]) + :on-usage-updated (fn [_]) + :on-server-web-search (fn [_]) + :on-server-image-generation (fn [_])}) + (is (= 2 (count @requests*))) + (is (nil? (get-in (first @requests*) [:headers "x-codex-turn-state"]))) + (is (= "state-1" + (get-in (second @requests*) [:headers "x-codex-turn-state"]))))))) + (deftest oauth-authorize-test (testing "that OAuth token exchange is routed through the http proxy" (let [req* (atom nil) @@ -185,6 +269,87 @@ :account-id "new-account"}] @requests*)))))) +(deftest create-response-retries-post-tool-request-test + (testing "retries the exact post-tool request without executing the tool again" + (let [requests* (atom []) + retry-events* (atom []) + tools-called* (atom 0) + messages* (atom []) + errors* (atom [])] + (with-redefs [llm-providers.openai/base-responses-request! + (fn [{:keys [body on-error on-stream] :as _opts}] + (let [request-number (count (swap! requests* conj body))] + (case request-number + 1 (on-stream "response.completed" + {:response {:status "completed" + :output [{:type "function_call" + :id "item-1" + :call_id "call-1" + :name "eca__read_file" + :arguments "{\"path\":\"/tmp/a\"}"}] + :usage {:input_tokens 1 + :output_tokens 1}}}) + 2 (on-error {:type "service_unavailable_error" + :code "server_is_overloaded" + :message "Our servers are currently overloaded. Please try again later." + :error/source :openai-responses}) + 3 (do + (on-stream "response.output_text.delta" {:delta "Recovered"}) + (on-stream "response.completed" + {:response {:status "completed" + :output [] + :usage {:input_tokens 2 + :output_tokens 1}}})))) + :ok)] + (llm-providers.openai/create-response! + {:model "gpt-test" + :user-messages [{:role "user" :content [{:type :text :text "inspect"}]}] + :instructions "ins" + :reason? false + :supports-image? false + :api-key "token" + :api-url "http://localhost:1" + :past-messages [] + :tools [{:full-name "eca__read_file" :description "read" :parameters {:type "object"}}] + :web-search false + :extra-payload {} + :extra-headers nil + :auth-type :auth/api-key} + {:on-message-received (fn [message] (swap! messages* conj message)) + :on-error (fn [error] (swap! errors* conj error)) + :on-prepare-tool-call (fn [_]) + :on-tools-called (fn [_] + (swap! tools-called* inc) + {:new-messages [{:role "tool_call" + :content {:id "call-1" + :full-name "eca__read_file" + :arguments {:path "/tmp/a"}}} + {:role "tool_call_output" + :content {:id "call-1" + :full-name "eca__read_file" + :output {:error false + :contents [{:type :text :text "contents"}]}}}] + :tools []}) + :on-reason (fn [_]) + :on-usage-updated (fn [_]) + :on-server-web-search (fn [_]) + :on-server-image-generation (fn [_]) + :retry-request (fn [{:keys [error-data attempt replay-safe? retry-fn]}] + (swap! retry-events* conj {:error-data error-data + :attempt attempt + :replay-safe? replay-safe?}) + (retry-fn (inc attempt)))}) + (is (= 3 (count @requests*))) + (is (= 1 @tools-called*) "the completed tool call must not be replayed") + (is (= (second @requests*) (nth @requests* 2)) + "the retry must replay the exact post-tool request body") + (is (true? (:replay-safe? (first @retry-events*)))) + (is (= "server_is_overloaded" (get-in (first @retry-events*) [:error-data :code]))) + (is (= [{:type :text :text "Recovered"} + {:type :finish :finish-reason "completed"}] + @messages*)) + (is (empty? @errors*)))))) + (deftest ->normalize-messages-test (testing "no previous history" (is (match? @@ -663,8 +828,8 @@ :properties {"limit" {:type "number"}}}}] false false))))) -(deftest create-response-oauth-preserves-built-in-tools-test - (testing "OAuth requests keep web_search and image_generation when capabilities are enabled" +(deftest create-response-standard-preserves-built-in-tools-test + (testing "standard Responses requests keep web_search and image_generation when enabled" (let [requests* (atom [])] (with-redefs [llm-providers.openai/base-responses-request! (fn [{:keys [on-stream] :as opts}] @@ -675,7 +840,6 @@ :status "completed"}}))] (llm-providers.openai/create-response! (assoc (base-provider-params) - :auth-type :auth/oauth :web-search true :image-generation true) (base-callbacks {})) @@ -684,6 +848,77 @@ {:type "image_generation" :output_format "png"}] (get-in (first @requests*) [:body :tools]))))))) +(deftest create-response-codex-request-shapes-test + (testing "public Responses requests ignore Codex-only Lite metadata" + (let [request* (atom nil)] + (with-redefs [llm-providers.openai/base-responses-request! + (fn [{:keys [on-stream] :as opts}] + (reset! request* opts) + (on-stream "response.completed" + {:response {:output [] + :usage {:input_tokens 0 :output_tokens 0} + :status "completed"}}))] + (llm-providers.openai/create-response! + (assoc (base-provider-params) + :model "gpt-5.6-sol" + :request-profile :openai-responses + :responses-lite? true) + (base-callbacks {})) + (is (= "test" (get-in @request* [:body :instructions]))) + (is (= ["function"] (mapv :type (get-in @request* [:body :tools])))) + (is (nil? (get-in @request* [:body :tool_choice]))) + (is (= "user" (-> @request* :body :input first :role)))))) + + (testing "regular Codex requests use top-level instructions without duplicating a system input" + (let [request* (atom nil)] + (with-redefs [llm-providers.openai/base-responses-request! + (fn [{:keys [on-stream] :as opts}] + (reset! request* opts) + (on-stream "response.completed" + {:response {:output [] + :usage {:input_tokens 0 :output_tokens 0} + :status "completed"}}))] + (llm-providers.openai/create-response! + (assoc (base-provider-params) + :request-profile openai-codex/request-profile + :turn-context (openai-codex/new-turn-context "chat-1")) + (base-callbacks {})) + (is (= "test" (get-in @request* [:body :instructions]))) + (is (= "auto" (get-in @request* [:body :tool_choice]))) + (is (= ["user"] (mapv :role (get-in @request* [:body :input]))))))) + + (testing "Responses Lite moves instructions and function tools into developer input items" + (let [request* (atom nil)] + (with-redefs [llm-providers.openai/base-responses-request! + (fn [{:keys [on-stream] :as opts}] + (reset! request* opts) + (on-stream "response.completed" + {:response {:output [] + :usage {:input_tokens 0 :output_tokens 0} + :status "completed"}}))] + (llm-providers.openai/create-response! + (assoc (base-provider-params) + :model "gpt-5.6-sol" + :request-profile openai-codex/request-profile + :turn-context (openai-codex/new-turn-context "chat-1") + :responses-lite? true + :reason? true + :default-reasoning-effort "low" + :web-search true + :image-generation true) + (base-callbacks {})) + (let [body (:body @request*)] + (is (nil? (:instructions body))) + (is (nil? (:tools body))) + (is (= "additional_tools" (get-in body [:input 0 :type]))) + (is (= ["function"] (mapv :type (get-in body [:input 0 :tools])))) + (is (= "developer" (get-in body [:input 1 :role]))) + (is (= {:effort "low" :summary "auto" :context "all_turns"} + (:reasoning body))) + (is (= "auto" (:tool_choice body))) + (is (false? (:parallel_tool_calls body))) + (is (true? (:responses-lite? @request*)))))))) + (deftest create-response-image-generation-tool-on-request-test (testing "request body includes image_generation tool when :image-generation is true" (let [requests* (atom [])] @@ -943,13 +1178,34 @@ {:status 200 :body {:models [{:slug "gpt-5.5" :context_window 272000 - :max_output_tokens 128000}]}})] - (let [result (#'llm-providers.openai/fetch-oauth-models "oauth-token" {"gpt-5.5" {}})] - (is (= "https://chatgpt.com/backend-api/codex/models?client_version=1.0.0" - (first @request*))) + :max_output_tokens 128000} + {:slug "gpt-5.6-sol" + :context_window 272000 + :use_responses_lite true + :default_reasoning_level "low" + :supported_reasoning_levels [{:effort "low"} + {:effort "max"}]} + {:slug "gpt-5.6-terra" + :use_responses_lite false}]}})] + (let [result (#'llm-providers.openai/fetch-oauth-models + "oauth-token" "account-1" {"gpt-5.5" {}}) + client-version (second (re-find #"client_version=(\d+\.\d+\.\d+)" + (first @request*)))] + (is (re-matches + #"https://chatgpt\.com/backend-api/codex/models\?client_version=\d+\.\d+\.\d+" + (first @request*))) (is (= "Bearer oauth-token" (get-in @request* [1 :headers "Authorization"]))) + (is (= "codex_cli_rs" (get-in @request* [1 :headers "Originator"]))) + (is (= (str "codex_cli_rs/" client-version) + (get-in @request* [1 :headers "User-Agent"]))) + (is (= "account-1" (get-in @request* [1 :headers "ChatGPT-Account-ID"]))) (is (= 272000 (get-in result ["gpt-5.5" :limit :context]))) - (is (= 128000 (get-in result ["gpt-5.5" :limit :output])))))))) + (is (= 128000 (get-in result ["gpt-5.5" :limit :output]))) + (is (true? (get-in result ["gpt-5.6-sol" :discovered-codex-responses-lite?]))) + (is (false? (get-in result ["gpt-5.6-terra" :discovered-codex-responses-lite?]))) + (is (= "low" (get-in result ["gpt-5.6-sol" :discovered-default-reasoning-effort]))) + (is (= #{"low" "max"} + (set (keys (get-in result ["gpt-5.6-sol" :discovered-variants])))))))))) (deftest fetch-oauth-models-fallback-on-error-test (testing "falls back to known Codex caps when /models is unauthorized" diff --git a/test/eca/models_test.clj b/test/eca/models_test.clj index d5d7ec886..79172f05c 100644 --- a/test/eca/models_test.clj +++ b/test/eca/models_test.clj @@ -812,6 +812,18 @@ (is (= 272000 (get-in supported ["openai/gpt-5.5" :limit :context]))) (is (zero? @native-calls*) "native /models must not be called when an override wins")))))) +(deftest codex-discovered-metadata-builds-model-capabilities-test + (is (= {:variants {"low" {:reasoning {:effort "low" :summary "auto"}}} + :codex-responses-lite? true + :default-reasoning-effort "low" + :supports-parallel-tool-calls? false} + (#'models/config-overrides->capabilities + {:discovered-codex-responses-lite? true + :discovered-default-reasoning-effort "low" + :discovered-supports-parallel-tool-calls? false + :discovered-variants {"low" {:reasoning {:effort "low" + :summary "auto"}}}})))) + (deftest openai-token-keeps-direct-api-context-window-test (testing "API-key (non-OAuth) OpenAI keeps the direct API context window (no override)" (let [request* (atom nil)