From 477cadfda53956a134a4bffeb5ed3a7f41de0116 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Thu, 16 Jul 2026 14:39:57 +0800 Subject: [PATCH 01/12] fix(ai-transport): stop internal LLM calls leaking client headers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The HTTP transport's construct_forward_headers pulled the downstream request's headers straight from ctx and always forwarded them. That is correct for the transparent proxy path (ai-proxy / ai-proxy-multi) but wrong for plugins that make a self-contained internal LLM call: ai-request-rewrite forwarded the client's Authorization / Cookie to the third-party LLM endpoint used to rewrite the body — a header leak. Make the transport ctx-free: construct_forward_headers now takes an explicit client_headers table (nil for internal requests) instead of reading core.request.headers(ctx). build_request forwards opts.client_headers, and only the proxy path (ai-proxy/base.lua) sets it. Internal callers (ai-request-rewrite, and any future embeddings/RAG/moderation call) pass none, so their own credentials — not the client's — reach the LLM. Add a test asserting the internal rewrite call receives no client Cookie while the transparent proxy path still forwards it. --- apisix/plugins/ai-providers/base.lua | 7 +- apisix/plugins/ai-proxy/base.lua | 4 + apisix/plugins/ai-transport/http.lua | 11 +- t/plugin/ai-transport-header-forwarding.t | 176 ++++++++++++++++++++++ 4 files changed, 194 insertions(+), 4 deletions(-) create mode 100644 t/plugin/ai-transport-header-forwarding.t diff --git a/apisix/plugins/ai-providers/base.lua b/apisix/plugins/ai-providers/base.lua index a992defbaabc..4525090492da 100644 --- a/apisix/plugins/ai-providers/base.lua +++ b/apisix/plugins/ai-providers/base.lua @@ -188,7 +188,12 @@ function _M.build_request(self, ctx, conf, request_body, opts) .. "includes a path", 400 end - local headers = transport_http.construct_forward_headers(auth.header or {}, ctx) + -- Only the proxy path (ai-proxy / ai-proxy-multi) forwards the downstream + -- request's headers, and it passes them in via opts.client_headers. A + -- self-contained internal request (ai-request-rewrite, embeddings, ...) + -- passes none, so its own credentials — not the client's — reach the LLM. + local headers = transport_http.construct_forward_headers(auth.header or {}, + opts.client_headers) if opts.host_header then headers["Host"] = opts.host_header end diff --git a/apisix/plugins/ai-proxy/base.lua b/apisix/plugins/ai-proxy/base.lua index 97c980dc5028..345923d8248e 100644 --- a/apisix/plugins/ai-proxy/base.lua +++ b/apisix/plugins/ai-proxy/base.lua @@ -178,6 +178,10 @@ function _M.before_proxy(conf, ctx, on_error) auth = ai_instance.auth, host_header = ai_instance._resolved_host_header, ssl_server_name = ai_instance._resolved_ssl_server_name, + -- ai-proxy is a transparent proxy, so it forwards the client's + -- request headers to the LLM. Internal callers (ai-request-rewrite, + -- embeddings, ...) omit this and never forward client headers. + client_headers = core.request.headers(ctx), override_llm_options = core.table.try_read_attr(ai_instance, "override", "llm_options"), request_body_override_map = diff --git a/apisix/plugins/ai-transport/http.lua b/apisix/plugins/ai-transport/http.lua index 5ea9d2194545..a2075aa3f4c3 100644 --- a/apisix/plugins/ai-transport/http.lua +++ b/apisix/plugins/ai-transport/http.lua @@ -43,9 +43,14 @@ end --- Build forwarded headers from client request + extra headers. --- Copies client headers, merges ext_opts_headers (lowercased), +-- Copies `client_headers`, merges ext_opts_headers (lowercased), -- forces Content-Type to application/json, removes host/content-length. -function _M.construct_forward_headers(ext_opts_headers, ctx) +-- `client_headers` is the downstream request's headers to forward (proxy path), +-- or nil for a self-contained internal request (e.g. ai-request-rewrite calling +-- an LLM to rewrite the body), which must not leak the client's Authorization, +-- Cookie or other headers to a third-party endpoint. The caller passes them in +-- explicitly, so the transport carries no `ctx` / downstream-request coupling. +function _M.construct_forward_headers(ext_opts_headers, client_headers) local blacklist = { "host", "content-length", @@ -53,7 +58,7 @@ function _M.construct_forward_headers(ext_opts_headers, ctx) } local headers = {} - for k, v in pairs(core.request.headers(ctx) or {}) do + for k, v in pairs(client_headers or {}) do headers[str_lower(k)] = v end for k, v in pairs(ext_opts_headers or {}) do diff --git a/t/plugin/ai-transport-header-forwarding.t b/t/plugin/ai-transport-header-forwarding.t new file mode 100644 index 000000000000..b5c882bb36aa --- /dev/null +++ b/t/plugin/ai-transport-header-forwarding.t @@ -0,0 +1,176 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +log_level("info"); +repeat_each(1); +no_long_string(); +no_root_location(); +no_shuffle(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } + + my $http_config = $block->http_config // <<_EOC_; + server { + listen 6820; + default_type 'application/json'; + # Mock LLM endpoint. It logs whether the client's Cookie reached it, + # so tests can distinguish the transparent proxy path (ai-proxy, which + # SHOULD forward client headers) from a self-contained internal request + # (ai-request-rewrite, which must NOT leak them to a third party). + location /v1/chat/completions { + content_by_lua_block { + ngx.log(ngx.WARN, "llm-recv-cookie:", + ngx.var.http_cookie or "none") + ngx.req.read_body() + ngx.status = 200 + ngx.say('{"choices":[{"message":{"content":"rewritten body"}}]}') + } + } + # Upstream target for the proxied (post-rewrite) request. + location /anything { + content_by_lua_block { + ngx.status = 200 + ngx.say("upstream-ok") + } + } + } +_EOC_ + $block->set_value("http_config", $http_config); + + my $user_yaml_config = <<_EOC_; +plugins: + - ai-request-rewrite + - ai-proxy-multi +_EOC_ + $block->set_value("extra_yaml_config", $user_yaml_config); +}); + +run_tests(); + +__DATA__ + +=== TEST 1: configure ai-request-rewrite pointing at the mock LLM +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{ + "uri": "/anything", + "plugins": { + "ai-request-rewrite": { + "prompt": "rewrite this", + "auth": { "header": { "Authorization": "Bearer llm-key" } }, + "provider": "openai", + "override": { + "endpoint": "http://127.0.0.1:6820/v1/chat/completions" + }, + "ssl_verify": false + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { "127.0.0.1:6820": 1 } + } + }]]) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + ngx.say("passed") + } + } +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 2: ai-request-rewrite's internal LLM call must not receive the client Cookie +--- request +POST /anything +some content to rewrite +--- more_headers +Content-Type: text/plain +Cookie: session=super-secret +--- response_body +upstream-ok +--- error_log +llm-recv-cookie:none +--- no_error_log +llm-recv-cookie:session=super-secret + + + +=== TEST 3: configure ai-proxy-multi routing to the mock LLM +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{ + "uri": "/chat", + "plugins": { + "ai-proxy-multi": { + "instances": [ + { + "name": "openai", + "provider": "openai", + "weight": 1, + "auth": { "header": { "Authorization": "Bearer llm-key" } }, + "options": { "model": "gpt-4" }, + "override": { + "endpoint": "http://127.0.0.1:6820/v1/chat/completions" + } + } + ], + "ssl_verify": false + } + } + }]]) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + ngx.say("passed") + } + } +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 4: ai-proxy is a transparent proxy and DOES forward the client Cookie +--- request +POST /chat +{"messages":[{"role":"user","content":"hello"}]} +--- more_headers +Content-Type: application/json +Cookie: session=proxy-secret +--- error_log +llm-recv-cookie:session=proxy-secret +--- no_error_log +[error] From 61fc5b12734711ca67731b67fc9de5aa4052b415 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Thu, 16 Jul 2026 15:30:18 +0800 Subject: [PATCH 02/12] fix(ai-transport): redact forwarded client headers from logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The proxy path now carries the client's request headers in extra_opts (client_headers) so ai-proxy can forward them. But build_request logs extra_opts at info level via redact_extra_opts(), which only stripped auth — so the client's Cookie/Authorization would be written to the log. Strip client_headers in redact_extra_opts too, closing the log-side leak that mirrors the network-side one. Extend the test: the mock LLM now also records Authorization; assert the internal rewrite call reaches the LLM with its own credentials (not the client's) and that the client's secret never appears in any log, including the extra_opts info line, on the proxy path. --- apisix/utils/log-sanitize.lua | 3 +++ t/plugin/ai-transport-header-forwarding.t | 19 +++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/apisix/utils/log-sanitize.lua b/apisix/utils/log-sanitize.lua index 03fe02acf1fb..bf2948a9aeb7 100644 --- a/apisix/utils/log-sanitize.lua +++ b/apisix/utils/log-sanitize.lua @@ -56,6 +56,9 @@ end function _M.redact_extra_opts(extra_opts) local redacted = core.table.deepcopy(extra_opts) redacted.auth = nil + -- client_headers are the raw downstream request headers forwarded on the + -- proxy path; they can carry Authorization/Cookie and must not be logged. + redacted.client_headers = nil return redacted end diff --git a/t/plugin/ai-transport-header-forwarding.t b/t/plugin/ai-transport-header-forwarding.t index b5c882bb36aa..cc5b84fb9a70 100644 --- a/t/plugin/ai-transport-header-forwarding.t +++ b/t/plugin/ai-transport-header-forwarding.t @@ -33,14 +33,16 @@ add_block_preprocessor(sub { server { listen 6820; default_type 'application/json'; - # Mock LLM endpoint. It logs whether the client's Cookie reached it, - # so tests can distinguish the transparent proxy path (ai-proxy, which - # SHOULD forward client headers) from a self-contained internal request - # (ai-request-rewrite, which must NOT leak them to a third party). + # Mock LLM endpoint. It logs which of the client's Cookie / Authorization + # reached it, so tests can distinguish the transparent proxy path + # (ai-proxy, which SHOULD forward client headers) from a self-contained + # internal request (ai-request-rewrite, which must NOT leak them). location /v1/chat/completions { content_by_lua_block { ngx.log(ngx.WARN, "llm-recv-cookie:", ngx.var.http_cookie or "none") + ngx.log(ngx.WARN, "llm-recv-auth:", + ngx.var.http_authorization or "none") ngx.req.read_body() ngx.status = 200 ngx.say('{"choices":[{"message":{"content":"rewritten body"}}]}') @@ -107,19 +109,22 @@ passed -=== TEST 2: ai-request-rewrite's internal LLM call must not receive the client Cookie +=== TEST 2: ai-request-rewrite's internal LLM call uses its own creds, not the client's --- request POST /anything some content to rewrite --- more_headers Content-Type: text/plain Cookie: session=super-secret +Authorization: Bearer client-secret-abc --- response_body upstream-ok --- error_log llm-recv-cookie:none +llm-recv-auth:Bearer llm-key --- no_error_log llm-recv-cookie:session=super-secret +client-secret-abc @@ -163,14 +168,16 @@ passed -=== TEST 4: ai-proxy is a transparent proxy and DOES forward the client Cookie +=== TEST 4: ai-proxy-multi is a transparent proxy and DOES forward the client Cookie --- request POST /chat {"messages":[{"role":"user","content":"hello"}]} --- more_headers Content-Type: application/json Cookie: session=proxy-secret +Authorization: Bearer client-secret-xyz --- error_log llm-recv-cookie:session=proxy-secret --- no_error_log [error] +client-secret-xyz From 72a8a4a26d8d9f5b00ecb09eedf82b5d35b81d25 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Fri, 17 Jul 2026 13:57:33 +0800 Subject: [PATCH 03/12] refactor(ai-providers): make build_request a ctx-free pure client build_request/request took the request ctx and reached into the downstream request through it, so an internal caller had to fake a ctx to use them. None of what ctx carried is meaningful for a self-contained LLM call. Drop the ctx parameter. Everything derived from the downstream request or from request state is now resolved by the caller and passed via opts: target_protocol was ctx.ai_target_protocol header_transform was ctx.ai_converter.convert_headers access_token was fetch_gcp_access_token(ctx, ...) (ctx keyed its cache) method/client_args was core.request.get_method() / ctx.var.args (passthrough) raw_request_body was ctx.ai_raw_request_body / core.request.get_body() client_headers (already) Protocol conversion moves to the caller too: converters stash state on ctx for the response side to read back, so ai-proxy converts and passes the converted body. ai-proxy also keeps reading ctx.ai_request_body_changed -- the cross-plugin 'an earlier plugin rewrote the body' signal -- and simply withholds raw_request_body then, which preserves the raw-body reuse optimisation without the transport knowing about ctx. ai-request-rewrite now calls the provider as a plain client and resolves its own GCP token; it sets ai_request_body_changed where it actually rewrites the body, for later plugins rather than for its own call. _M.request only ever forwarded ctx to build_request, so it is now ctx-free too: a genuine pure client for internal callers. --- apisix/plugins/ai-providers/base.lua | 107 +++++++++------------- apisix/plugins/ai-proxy/base.lua | 47 +++++++++- apisix/plugins/ai-request-rewrite.lua | 19 +++- t/plugin/ai-proxy-request-body-override.t | 46 ++++++++-- t/plugin/ai-proxy.t | 8 +- 5 files changed, 148 insertions(+), 79 deletions(-) diff --git a/apisix/plugins/ai-providers/base.lua b/apisix/plugins/ai-providers/base.lua index 4525090492da..24dd0d90d2ae 100644 --- a/apisix/plugins/ai-providers/base.lua +++ b/apisix/plugins/ai-providers/base.lua @@ -31,7 +31,6 @@ local url = require("socket.url") local sse = require("apisix.plugins.ai-transport.sse") local aws_eventstream = require("apisix.plugins.ai-transport.aws-eventstream") local transport_http = require("apisix.plugins.ai-transport.http") -local transport_auth = require("apisix.plugins.ai-transport.auth") local log_sanitize = require("apisix.utils.log-sanitize") local protocols = require("apisix.plugins.ai-protocols") local deep_merge = require("apisix.plugins.ai-proxy.merge").deep_merge @@ -100,25 +99,24 @@ end --- Build HTTP request parameters from driver config and extra_opts. +-- +-- This is a pure client: it never reads the downstream request. Everything +-- derived from it is passed in through `opts` by the caller — client_headers, +-- client_args and method (proxy path only), raw_request_body, a resolved +-- access_token, the target_protocol and an optional header_transform. +-- +-- Protocol conversion is deliberately NOT done here: converters carry state on +-- the request ctx for the response side to read back, so the caller converts and +-- passes the already-converted body. -- @return table params HTTP parameters ready for transport_http.request() -- @return string|nil err Error message -function _M.build_request(self, ctx, conf, request_body, opts) +function _M.build_request(self, conf, request_body, opts) local body_changed = false - -- Protocol conversion (when a converter bridges client→target protocol) - local converter = ctx.ai_converter - if converter and converter.convert_request then - local converted, err = converter.convert_request(request_body, ctx) - if not converted then - return nil, err or "invalid protocol", 400 - end - request_body = converted - body_changed = true - end - -- Inject target-protocol-specific parameters (e.g. stream_options for OpenAI). - -- This runs after conversion so it covers both passthrough and convert scenarios. - local target_protocol = ctx.ai_target_protocol + -- This runs after the caller's conversion so it covers both passthrough and + -- convert scenarios. + local target_protocol = opts.target_protocol if target_protocol then local target_proto = protocols.get(target_protocol) if target_proto and target_proto.prepare_outgoing_request then @@ -131,17 +129,7 @@ function _M.build_request(self, ctx, conf, request_body, opts) core.log.info("request extra_opts to LLM server: ", core.json.delay_encode(log_sanitize.redact_extra_opts(opts), true)) - -- Auth: GCP token local auth = opts.auth or {} - local token - if auth.gcp then - local access_token, err = transport_auth.fetch_gcp_access_token(ctx, opts.name, - auth.gcp) - if not access_token then - return nil, "failed to get gcp access token: " .. (err or "unknown") - end - token = access_token - end -- Parse endpoint URL local endpoint = opts.endpoint @@ -197,29 +185,27 @@ function _M.build_request(self, ctx, conf, request_body, opts) if opts.host_header then headers["Host"] = opts.host_header end - if token then - headers["authorization"] = "Bearer " .. token + -- The caller resolves credentials that need request state (e.g. a GCP + -- access token) and hands the finished token over. + if opts.access_token then + headers["authorization"] = "Bearer " .. opts.access_token end - -- Protocol converter header transformation (e.g. Anthropic → OpenAI headers) - if ctx.ai_converter and ctx.ai_converter.convert_headers then - ctx.ai_converter.convert_headers(headers) + -- Optional header transformation (e.g. the Anthropic → OpenAI converter's). + if opts.header_transform then + opts.header_transform(headers) end - -- For the passthrough protocol the gateway acts as a catch-all proxy, so - -- forward the client's HTTP method and original query string unchanged. - -- Other protocols always issue a POST with provider-specific query args. - local method = "POST" - if ctx.ai_target_protocol == "passthrough" then - method = core.request.get_method() - local client_args = ctx.var.args and core.string.decode_args(ctx.var.args) - if type(client_args) == "table" then - -- client query overrides the endpoint query, but configured - -- auth.query credentials must stay non-overridable by the caller - for k, v in pairs(client_args) do - if not (auth.query and auth.query[k] ~= nil) then - query_params[k] = v - end + -- The caller forwards the downstream method and query string only when it + -- wants them proxied verbatim (the passthrough protocol). Otherwise this is + -- a plain POST with provider-specific query args. + local method = opts.method or "POST" + if type(opts.client_args) == "table" then + -- client query overrides the endpoint query, but configured + -- auth.query credentials must stay non-overridable by the caller + for k, v in pairs(opts.client_args) do + if not (auth.query and auth.query[k] ~= nil) then + query_params[k] = v end end end @@ -251,7 +237,7 @@ function _M.build_request(self, ctx, conf, request_body, opts) -- Apply llm_options via provider capability hook (always force-overwrites) if opts.override_llm_options then - local cap = self.capabilities and self.capabilities[ctx.ai_target_protocol] + local cap = self.capabilities and self.capabilities[target_protocol] if cap and cap.rewrite_request_body then cap.rewrite_request_body(request_body, opts.override_llm_options, true) if next(opts.override_llm_options) ~= nil then @@ -262,10 +248,10 @@ function _M.build_request(self, ctx, conf, request_body, opts) -- Apply per-target-protocol request body override (deep merge) if opts.request_body_override_map then - local patch = opts.request_body_override_map[ctx.ai_target_protocol] + local patch = opts.request_body_override_map[target_protocol] if patch then core.log.info("applying request_body override for target protocol '", - ctx.ai_target_protocol, "'") + target_protocol, "'") request_body = deep_merge(request_body, patch, opts.request_body_force_override) body_changed = true end @@ -276,19 +262,13 @@ function _M.build_request(self, ctx, conf, request_body, opts) body_changed = true end - if body_changed then - ctx.ai_request_body_changed = true - end - - if not ctx.ai_request_body_changed then - if ctx.ai_raw_request_body == nil then - ctx.ai_raw_request_body = core.request.get_body() - end - if type(ctx.ai_raw_request_body) == "string" then - params.body = ctx.ai_raw_request_body - else - params.body = request_body - end + -- Send the downstream body verbatim when nothing above modified it, so a + -- pure passthrough stays byte-identical. The caller supplies + -- opts.raw_request_body only when it has not transformed the body itself + -- (i.e. no protocol conversion ran); an internal caller omits it and always + -- sends the body it built. + if not body_changed and type(opts.raw_request_body) == "string" then + params.body = opts.raw_request_body else params.body = request_body end @@ -806,8 +786,11 @@ end -- @return number|nil HTTP status code -- @return string|nil Raw response body (JSON string) -- @return string|nil Error message -function _M.request(self, ctx, conf, request_table, extra_opts) - local params, err = self:build_request(ctx, conf, request_table, extra_opts) +--- Send a self-contained request to an LLM and return its raw response. +-- Fully decoupled from the downstream request: internal callers (ai-request-rewrite, +-- embeddings, ...) use this as a plain client and parse the body themselves. +function _M.request(self, conf, request_table, extra_opts) + local params, err = self:build_request(conf, request_table, extra_opts) if not params then core.log.error("failed to build request: ", err) return 500, nil, err diff --git a/apisix/plugins/ai-proxy/base.lua b/apisix/plugins/ai-proxy/base.lua index 345923d8248e..e306b61beb60 100644 --- a/apisix/plugins/ai-proxy/base.lua +++ b/apisix/plugins/ai-proxy/base.lua @@ -26,6 +26,7 @@ local table = table local exporter = require("apisix.plugins.prometheus.exporter") local protocols = require("apisix.plugins.ai-protocols") local transport_http = require("apisix.plugins.ai-transport.http") +local transport_auth = require("apisix.plugins.ai-transport.auth") local log_sanitize = require("apisix.utils.log-sanitize") local apisix_upstream = require("resty.apisix.upstream") @@ -244,6 +245,50 @@ function _M.before_proxy(conf, ctx, on_error) extra_opts.target_path = target_path extra_opts.target_host = target_host + extra_opts.target_protocol = target_proto + -- The transport is a pure client, so everything below that depends on the + -- downstream request or on ctx is resolved here and handed over via + -- extra_opts. + extra_opts.header_transform = converter and converter.convert_headers + + -- Credentials that need request state (GCP token cache is keyed on ctx). + local instance_auth = ai_instance.auth + if instance_auth and instance_auth.gcp then + local token, token_err = transport_auth.fetch_gcp_access_token( + ctx, ai_instance.name, instance_auth.gcp) + if not token then + core.log.error("failed to get gcp access token: ", token_err) + return 500, {error_msg = "failed to get gcp access token: " + .. (token_err or "unknown")} + end + extra_opts.access_token = token + end + + -- passthrough proxies the client's method and query string verbatim. + if target_proto == "passthrough" then + extra_opts.method = core.request.get_method() + local client_args = ctx.var.args and core.string.decode_args(ctx.var.args) + if type(client_args) == "table" then + extra_opts.client_args = client_args + end + end + + -- Step 2.5: protocol conversion. It lives here rather than in the + -- transport because converters stash state on ctx for the response side. + local body_for_llm = request_body + if converter and converter.convert_request then + local converted, conv_err = converter.convert_request(request_body, ctx) + if not converted then + return 400, {error_msg = conv_err or "invalid protocol"} + end + body_for_llm = converted + elseif not ctx.ai_request_body_changed then + -- Nothing has rewritten the body -- neither a converter here nor an + -- earlier plugin (ai-request-rewrite marks that on ctx) -- so the + -- client's verbatim bytes can be reused, keeping a pure passthrough + -- byte-identical and skipping a re-encode. + extra_opts.raw_request_body = core.request.get_body() + end local do_request = function() ctx.llm_request_start_time = ngx.now() @@ -251,7 +296,7 @@ function _M.before_proxy(conf, ctx, on_error) -- Step 3: Build HTTP request params local params, build_err, code = ai_provider:build_request( - ctx, conf, request_body, extra_opts) + conf, body_for_llm, extra_opts) if not params then local body = {error_msg = build_err} if code then diff --git a/apisix/plugins/ai-request-rewrite.lua b/apisix/plugins/ai-request-rewrite.lua index 5bef20771582..930143aed353 100644 --- a/apisix/plugins/ai-request-rewrite.lua +++ b/apisix/plugins/ai-request-rewrite.lua @@ -17,6 +17,7 @@ local core = require("apisix.core") local ai_providers_schema = require("apisix.plugins.ai-providers.schema") local protocols = require("apisix.plugins.ai-protocols") +local transport_auth = require("apisix.plugins.ai-transport.auth") local require = require local pcall = pcall local next = next @@ -125,10 +126,21 @@ local function request_to_llm(conf, request_table, ctx, target_path) model_options = conf.options, target_path = target_path, } + -- The provider is a pure client, so resolve here whatever needs the request + -- ctx. Nothing downstream-derived is handed over: no client headers, no + -- verbatim client body -- this call carries its own credentials and body. + if conf.auth and conf.auth.gcp then + local token, token_err = transport_auth.fetch_gcp_access_token(ctx, plugin_name, + conf.auth.gcp) + if not token then + return nil, nil, "failed to get gcp access token: " .. (token_err or "unknown") + end + extra_opts.access_token = token + end + ctx.llm_request_start_time = ngx.now() ctx.var.llm_request_body = request_table - ctx.ai_request_body_changed = true - return ai_provider:request(ctx, conf, request_table, extra_opts) + return ai_provider:request(conf, request_table, extra_opts) end @@ -222,6 +234,9 @@ function _M.access(conf, ctx) -- Replace the original request body with the rewritten content ngx.req.set_body_data(content) + -- Tell later AI plugins (ai-proxy) that the downstream body is no longer the + -- client's original, so they must not reuse the raw bytes as-is. + ctx.ai_request_body_changed = true end return _M diff --git a/t/plugin/ai-proxy-request-body-override.t b/t/plugin/ai-proxy-request-body-override.t index 1bac6c31b72b..f705d41d5903 100644 --- a/t/plugin/ai-proxy-request-body-override.t +++ b/t/plugin/ai-proxy-request-body-override.t @@ -258,10 +258,14 @@ same body -=== TEST 5c: build_request does not reuse raw body after an earlier rewrite +=== TEST 5c: build_request sends the parsed body when the caller offers no raw body --- config location /t { content_by_lua_block { + -- The caller withholds raw_request_body whenever the body is no longer + -- the client's original -- a converter ran, or an earlier plugin + -- rewrote it (ai-request-rewrite marks that on ctx). build_request + -- then sends the parsed table. local base = require("apisix.plugins.ai-providers.base") local provider = base.new({ capabilities = { @@ -271,20 +275,15 @@ same body }, }, }) - local ctx = { - ai_target_protocol = "openai-chat", - ai_request_body_changed = true, - var = {}, - } local opts = { auth = {}, conf = {}, - raw_request_body = '{"messages":[]}', + target_protocol = "openai-chat", target_path = "/v1/chat/completions", } local request_body = {messages = {{role = "user", content = "changed"}}} - local params = assert(provider:build_request(ctx, {ssl_verify = false}, + local params = assert(provider:build_request({ssl_verify = false}, request_body, opts)) ngx.say(type(params.body)) ngx.say(params.body == request_body and "table body" or "raw body") @@ -296,6 +295,37 @@ table body +=== TEST 5d: build_request reuses the caller's raw body when nothing changes it +--- config + location /t { + content_by_lua_block { + local base = require("apisix.plugins.ai-providers.base") + local provider = base.new({ + capabilities = { + ["openai-chat"] = { + path = "/v1/chat/completions", + host = "localhost", + }, + }, + }) + local opts = { + auth = {}, + conf = {}, + target_protocol = "openai-chat", + target_path = "/v1/chat/completions", + raw_request_body = '{"messages":[]}', + } + + local params = assert(provider:build_request({ssl_verify = false}, + {messages = {}}, opts)) + ngx.say(params.body) + } + } +--- response_body +{"messages":[]} + + + === TEST 6: llm_options: openai provider maps max_tokens to max_completion_tokens --- config location /t { diff --git a/t/plugin/ai-proxy.t b/t/plugin/ai-proxy.t index 24658a5b792c..e38decf15eb6 100644 --- a/t/plugin/ai-proxy.t +++ b/t/plugin/ai-proxy.t @@ -1240,10 +1240,6 @@ got token usage from ai service: capabilities = {}, }) - local ctx = { - var = {}, - ai_client_protocol = "openai-chat", - } local conf = { ssl_verify = false } local opts = { endpoint = "http://127.0.0.1:1980/v1/chat/completions?extra=value", @@ -1251,8 +1247,8 @@ got token usage from ai service: conf = {}, } - provider:build_request(ctx, conf, {messages = {{role="user", content="hi"}}}, opts) - provider:build_request(ctx, conf, {messages = {{role="user", content="hi"}}}, opts) + provider:build_request(conf, {messages = {{role="user", content="hi"}}}, opts) + provider:build_request(conf, {messages = {{role="user", content="hi"}}}, opts) if auth_query["extra"] then ngx.say("FAIL: auth.query was mutated, extra=" .. auth_query["extra"]) From ae691d7c63c24c924201d550c3f6c34830862260 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Fri, 17 Jul 2026 14:49:40 +0800 Subject: [PATCH 04/12] style(test): reindex ai-proxy-request-body-override.t The new build_request test block broke the blank-line spacing utils/reindex enforces; run it and take its renumbering (the file's existing 3a/4b/5c convention continues as 6d). --- t/plugin/ai-proxy-request-body-override.t | 30 +++++++++++------------ 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/t/plugin/ai-proxy-request-body-override.t b/t/plugin/ai-proxy-request-body-override.t index f705d41d5903..3d8d0c945d70 100644 --- a/t/plugin/ai-proxy-request-body-override.t +++ b/t/plugin/ai-proxy-request-body-override.t @@ -295,7 +295,7 @@ table body -=== TEST 5d: build_request reuses the caller's raw body when nothing changes it +=== TEST 6d: build_request reuses the caller's raw body when nothing changes it --- config location /t { content_by_lua_block { @@ -326,7 +326,7 @@ table body -=== TEST 6: llm_options: openai provider maps max_tokens to max_completion_tokens +=== TEST 7: llm_options: openai provider maps max_tokens to max_completion_tokens --- config location /t { content_by_lua_block { @@ -369,7 +369,7 @@ max_completion_tokens=555 -=== TEST 7: llm_options: openai-compatible provider maps max_tokens to max_tokens +=== TEST 8: llm_options: openai-compatible provider maps max_tokens to max_tokens --- config location /t { content_by_lua_block { @@ -412,7 +412,7 @@ max_tokens=444 -=== TEST 8: llm_options: openai responses API maps max_tokens to max_output_tokens +=== TEST 9: llm_options: openai responses API maps max_tokens to max_output_tokens --- config location /t { content_by_lua_block { @@ -455,7 +455,7 @@ max_output_tokens=333 -=== TEST 9: llm_options: ai-proxy-multi per-instance override +=== TEST 10: llm_options: ai-proxy-multi per-instance override --- config location /t { content_by_lua_block { @@ -502,7 +502,7 @@ max_completion_tokens=222 -=== TEST 10: llm_options always force-overwrites client value +=== TEST 11: llm_options always force-overwrites client value --- config location /t { content_by_lua_block { @@ -547,7 +547,7 @@ max_tokens=555 -=== TEST 11: request_body: openai-chat override writes fields on outgoing body +=== TEST 12: request_body: openai-chat override writes fields on outgoing body --- config location /t { content_by_lua_block { @@ -595,7 +595,7 @@ max_tokens=555 temperature=0.1 -=== TEST 12: request_body: non-force deep merge fills missing nested keys without overwriting existing +=== TEST 13: request_body: non-force deep merge fills missing nested keys without overwriting existing --- config location /t { content_by_lua_block { @@ -644,7 +644,7 @@ include_usage=true extra=1 -=== TEST 13: request_body: array values are replaced wholesale (stop sequences) +=== TEST 14: request_body: array values are replaced wholesale (stop sequences) --- config location /t { content_by_lua_block { @@ -690,7 +690,7 @@ stop=["a","b"] -=== TEST 14: request_body: override keyed by non-matching target protocol is ignored +=== TEST 15: request_body: override keyed by non-matching target protocol is ignored --- config location /t { content_by_lua_block { @@ -733,7 +733,7 @@ max_tokens=nil -=== TEST 15: request_body: default mode - client value takes priority +=== TEST 16: request_body: default mode - client value takes priority --- config location /t { content_by_lua_block { @@ -778,7 +778,7 @@ max_tokens=999 -=== TEST 16: request_body: force_override mode - override overwrites client fields +=== TEST 17: request_body: force_override mode - override overwrites client fields --- config location /t { content_by_lua_block { @@ -824,7 +824,7 @@ max_tokens=555 -=== TEST 17: request_body: override applies to target protocol after converter +=== TEST 18: request_body: override applies to target protocol after converter --- config location /t { content_by_lua_block { @@ -871,7 +871,7 @@ max_tokens=77 has_messages=true -=== TEST 18: ai-proxy-multi per-instance request_body override +=== TEST 19: ai-proxy-multi per-instance request_body override --- config location /t { content_by_lua_block { @@ -919,7 +919,7 @@ max_tokens=321 -=== TEST 19: both llm_options and request_body coexist, request_body wins +=== TEST 20: both llm_options and request_body coexist, request_body wins --- config location /t { content_by_lua_block { From 667793dfbf52118a76528dbd61c153c9d230f7d5 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Fri, 17 Jul 2026 15:09:30 +0800 Subject: [PATCH 05/12] refactor(ai-providers): group downstream-derived inputs under opts.client The ctx removal replaced one opaque parameter with four flat fields (client_headers, client_args, method, raw_request_body) scattered through an already-large opts table. That named no concept, let a caller half-set the group, and made redaction a matter of remembering each field -- which had already gone wrong: opts.raw_request_body put the client's verbatim body (the user's prompt) into the info-level 'request extra_opts' log line, since redact_extra_opts only stripped auth and client_headers. Group them under a single opts.client: client = { headers, args, method, raw_body } Its presence is now what marks a call as proxying an inbound request; an internal call omits it entirely, so nothing of the client's can reach the LLM or the logs. redact_extra_opts drops the one key and with it every piece of downstream data, instead of enumerating fields. Extend the proxy test to assert the client's prompt never appears in any log; it fails without the redaction. --- apisix/plugins/ai-providers/base.lua | 40 ++++++++++++----------- apisix/plugins/ai-proxy/base.lua | 17 ++++++---- apisix/utils/log-sanitize.lua | 8 +++-- t/plugin/ai-proxy-request-body-override.t | 2 +- t/plugin/ai-transport-header-forwarding.t | 3 +- 5 files changed, 39 insertions(+), 31 deletions(-) diff --git a/apisix/plugins/ai-providers/base.lua b/apisix/plugins/ai-providers/base.lua index 24dd0d90d2ae..3c50f411385c 100644 --- a/apisix/plugins/ai-providers/base.lua +++ b/apisix/plugins/ai-providers/base.lua @@ -100,10 +100,13 @@ end --- Build HTTP request parameters from driver config and extra_opts. -- --- This is a pure client: it never reads the downstream request. Everything --- derived from it is passed in through `opts` by the caller — client_headers, --- client_args and method (proxy path only), raw_request_body, a resolved --- access_token, the target_protocol and an optional header_transform. +-- This is a pure client: it never reads the downstream request. Everything taken +-- from that request is grouped under `opts.client` and supplied by the caller — +-- its presence is what marks this as proxying an inbound request. A +-- self-contained internal call (ai-request-rewrite, embeddings, ...) omits +-- `opts.client` entirely, so nothing of the client's can reach the LLM or the +-- logs. Request state the caller resolves for us arrives as opts.access_token / +-- opts.target_protocol / opts.header_transform. -- -- Protocol conversion is deliberately NOT done here: converters carry state on -- the request ctx for the response side to read back, so the caller converts and @@ -176,12 +179,12 @@ function _M.build_request(self, conf, request_body, opts) .. "includes a path", 400 end - -- Only the proxy path (ai-proxy / ai-proxy-multi) forwards the downstream - -- request's headers, and it passes them in via opts.client_headers. A - -- self-contained internal request (ai-request-rewrite, embeddings, ...) - -- passes none, so its own credentials — not the client's — reach the LLM. + -- Everything below that comes from the downstream request lives on + -- opts.client; an internal call leaves it nil and therefore forwards none of + -- the client's headers, query or body. + local client = opts.client local headers = transport_http.construct_forward_headers(auth.header or {}, - opts.client_headers) + client and client.headers) if opts.host_header then headers["Host"] = opts.host_header end @@ -196,14 +199,14 @@ function _M.build_request(self, conf, request_body, opts) opts.header_transform(headers) end - -- The caller forwards the downstream method and query string only when it + -- The caller passes the downstream method and query string only when it -- wants them proxied verbatim (the passthrough protocol). Otherwise this is -- a plain POST with provider-specific query args. - local method = opts.method or "POST" - if type(opts.client_args) == "table" then + local method = client and client.method or "POST" + if client and type(client.args) == "table" then -- client query overrides the endpoint query, but configured -- auth.query credentials must stay non-overridable by the caller - for k, v in pairs(opts.client_args) do + for k, v in pairs(client.args) do if not (auth.query and auth.query[k] ~= nil) then query_params[k] = v end @@ -263,12 +266,11 @@ function _M.build_request(self, conf, request_body, opts) end -- Send the downstream body verbatim when nothing above modified it, so a - -- pure passthrough stays byte-identical. The caller supplies - -- opts.raw_request_body only when it has not transformed the body itself - -- (i.e. no protocol conversion ran); an internal caller omits it and always - -- sends the body it built. - if not body_changed and type(opts.raw_request_body) == "string" then - params.body = opts.raw_request_body + -- pure passthrough stays byte-identical. The caller sets client.raw_body only + -- when it has not transformed the body itself (i.e. no protocol conversion + -- ran); an internal call has no client at all and always sends what it built. + if not body_changed and client and type(client.raw_body) == "string" then + params.body = client.raw_body else params.body = request_body end diff --git a/apisix/plugins/ai-proxy/base.lua b/apisix/plugins/ai-proxy/base.lua index e306b61beb60..420cbc209123 100644 --- a/apisix/plugins/ai-proxy/base.lua +++ b/apisix/plugins/ai-proxy/base.lua @@ -179,10 +179,6 @@ function _M.before_proxy(conf, ctx, on_error) auth = ai_instance.auth, host_header = ai_instance._resolved_host_header, ssl_server_name = ai_instance._resolved_ssl_server_name, - -- ai-proxy is a transparent proxy, so it forwards the client's - -- request headers to the LLM. Internal callers (ai-request-rewrite, - -- embeddings, ...) omit this and never forward client headers. - client_headers = core.request.headers(ctx), override_llm_options = core.table.try_read_attr(ai_instance, "override", "llm_options"), request_body_override_map = @@ -264,12 +260,19 @@ function _M.before_proxy(conf, ctx, on_error) extra_opts.access_token = token end + -- ai-proxy is a transparent proxy of an inbound request, so it hands the + -- transport everything taken from that request under one `client` key. + -- Internal callers omit it entirely and thus forward nothing of the + -- client's -- see ai-providers/base.lua build_request. + local client = { headers = core.request.headers(ctx) } + extra_opts.client = client + -- passthrough proxies the client's method and query string verbatim. if target_proto == "passthrough" then - extra_opts.method = core.request.get_method() + client.method = core.request.get_method() local client_args = ctx.var.args and core.string.decode_args(ctx.var.args) if type(client_args) == "table" then - extra_opts.client_args = client_args + client.args = client_args end end @@ -287,7 +290,7 @@ function _M.before_proxy(conf, ctx, on_error) -- earlier plugin (ai-request-rewrite marks that on ctx) -- so the -- client's verbatim bytes can be reused, keeping a pure passthrough -- byte-identical and skipping a re-encode. - extra_opts.raw_request_body = core.request.get_body() + client.raw_body = core.request.get_body() end local do_request = function() diff --git a/apisix/utils/log-sanitize.lua b/apisix/utils/log-sanitize.lua index bf2948a9aeb7..10ebb48f3a8d 100644 --- a/apisix/utils/log-sanitize.lua +++ b/apisix/utils/log-sanitize.lua @@ -56,9 +56,11 @@ end function _M.redact_extra_opts(extra_opts) local redacted = core.table.deepcopy(extra_opts) redacted.auth = nil - -- client_headers are the raw downstream request headers forwarded on the - -- proxy path; they can carry Authorization/Cookie and must not be logged. - redacted.client_headers = nil + -- Everything taken from the downstream request is grouped under `client`: + -- its headers (Authorization/Cookie), its query args and its verbatim body + -- (the user's prompt). None of it belongs in a log line, and dropping the + -- one key drops all of it. + redacted.client = nil return redacted end diff --git a/t/plugin/ai-proxy-request-body-override.t b/t/plugin/ai-proxy-request-body-override.t index 3d8d0c945d70..ce9137f2ebb5 100644 --- a/t/plugin/ai-proxy-request-body-override.t +++ b/t/plugin/ai-proxy-request-body-override.t @@ -313,7 +313,7 @@ table body conf = {}, target_protocol = "openai-chat", target_path = "/v1/chat/completions", - raw_request_body = '{"messages":[]}', + client = { raw_body = '{"messages":[]}' }, } local params = assert(provider:build_request({ssl_verify = false}, diff --git a/t/plugin/ai-transport-header-forwarding.t b/t/plugin/ai-transport-header-forwarding.t index cc5b84fb9a70..5790dcaadb18 100644 --- a/t/plugin/ai-transport-header-forwarding.t +++ b/t/plugin/ai-transport-header-forwarding.t @@ -171,7 +171,7 @@ passed === TEST 4: ai-proxy-multi is a transparent proxy and DOES forward the client Cookie --- request POST /chat -{"messages":[{"role":"user","content":"hello"}]} +{"messages":[{"role":"user","content":"super-secret-prompt"}]} --- more_headers Content-Type: application/json Cookie: session=proxy-secret @@ -181,3 +181,4 @@ llm-recv-cookie:session=proxy-secret --- no_error_log [error] client-secret-xyz +super-secret-prompt From df39a4219d8b215c3c5c6cf5c967022c643d7292 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Fri, 17 Jul 2026 16:45:47 +0800 Subject: [PATCH 06/12] refactor(ai-providers): split the client into build_body and build_request build_request did three jobs at once: shape the body for the target protocol, assemble the HTTP request, and sign it. The seam between them followed an implementation detail rather than a concept -- it knew about target_protocol and mutated the body in four places, yet protocol *conversion* sat outside, purely because that one function takes ctx. A reader could not state what the function's job was. Every protocol-driven decision in there turned out to be a body decision: nothing protocol-shaped touched the URL, headers, method or signing. So split by what each layer produces: build_body(request_body, opts) -> body, changed prepare_outgoing_request / model_options / llm_options(capability) / request_body_override_map / remove_model. This is where the target protocol matters. build_request(conf, body, opts) -> params endpoint, auth, headers, method, query, SigV4 last. Protocol-agnostic; the body is passed through as given. request(conf, request_table, opts) = build_body + build_request + send The raw-body reuse rule goes with it. The transport already sends a string body verbatim and encodes a table, so 'reuse the client's bytes' is just the caller passing the string -- an explicit line at the call site instead of an implicit contract (client.raw_body plus an internal body_changed) that the caller had to know build_request's internals to use correctly. opts.client now carries only downstream facts: headers, args, method. Retarget the build_request unit tests at the new contract (pass-through) and add one for build_body's changed flag, which is what the reuse decision hangs on. --- apisix/plugins/ai-providers/base.lua | 144 +++++++++++----------- apisix/plugins/ai-proxy/base.lua | 35 ++++-- t/plugin/ai-proxy-request-body-override.t | 111 ++++++++++------- 3 files changed, 165 insertions(+), 125 deletions(-) diff --git a/apisix/plugins/ai-providers/base.lua b/apisix/plugins/ai-providers/base.lua index 3c50f411385c..9087958bdf01 100644 --- a/apisix/plugins/ai-providers/base.lua +++ b/apisix/plugins/ai-providers/base.lua @@ -98,37 +98,84 @@ local function merge_usage(ctx, parsed) end ---- Build HTTP request parameters from driver config and extra_opts. +--- Shape the request body for the target protocol and this provider. -- --- This is a pure client: it never reads the downstream request. Everything taken --- from that request is grouped under `opts.client` and supplied by the caller — --- its presence is what marks this as proxying an inbound request. A --- self-contained internal call (ai-request-rewrite, embeddings, ...) omits --- `opts.client` entirely, so nothing of the client's can reach the LLM or the --- logs. Request state the caller resolves for us arrives as opts.access_token / --- opts.target_protocol / opts.header_transform. --- --- Protocol conversion is deliberately NOT done here: converters carry state on +-- Pure: no ctx, no downstream request, no network -- body in, body out. This is +-- where the target protocol matters; build_request below does not know protocols +-- exist. Protocol *conversion* is not done here either: converters carry state on -- the request ctx for the response side to read back, so the caller converts and --- passes the already-converted body. --- @return table params HTTP parameters ready for transport_http.request() --- @return string|nil err Error message -function _M.build_request(self, conf, request_body, opts) - local body_changed = false +-- hands us the converted body. +-- @return table body The body to send +-- @return boolean changed Whether anything here modified it. The caller uses this +-- to decide whether the client's verbatim bytes may still be reused. +function _M.build_body(self, request_body, opts) + local changed = false + local target_protocol = opts.target_protocol -- Inject target-protocol-specific parameters (e.g. stream_options for OpenAI). - -- This runs after the caller's conversion so it covers both passthrough and - -- convert scenarios. - local target_protocol = opts.target_protocol if target_protocol then local target_proto = protocols.get(target_protocol) if target_proto and target_proto.prepare_outgoing_request then if target_proto.prepare_outgoing_request(request_body) then - body_changed = true + changed = true end end end + -- Inject model options (flat overwrite) + if opts.model_options then + for opt, val in pairs(opts.model_options) do + if request_body[opt] ~= nil then + core.log.info("model_options overwriting request field '", opt, "'") + end + request_body[opt] = val + changed = true + end + end + + -- Apply llm_options via provider capability hook (always force-overwrites) + if opts.override_llm_options then + local cap = self.capabilities and self.capabilities[target_protocol] + if cap and cap.rewrite_request_body then + cap.rewrite_request_body(request_body, opts.override_llm_options, true) + if next(opts.override_llm_options) ~= nil then + changed = true + end + end + end + + -- Apply per-target-protocol request body override (deep merge) + if opts.request_body_override_map then + local patch = opts.request_body_override_map[target_protocol] + if patch then + core.log.info("applying request_body override for target protocol '", + target_protocol, "'") + request_body = deep_merge(request_body, patch, opts.request_body_force_override) + changed = true + end + end + + if self.remove_model and request_body.model ~= nil then + request_body.model = nil + changed = true + end + + return request_body, changed +end + + +--- Assemble the outbound HTTP request: endpoint, auth, headers, query, and -- +-- last -- signing. +-- +-- Pure and protocol-agnostic: `body` arrives already shaped (see build_body) and +-- is passed through untouched, so a string goes out verbatim and a table is +-- encoded by the transport. Everything taken from the downstream request lives on +-- `opts.client`; its presence is what marks this as proxying an inbound request. +-- A self-contained internal call omits it, so nothing of the client's can reach +-- the LLM or the logs. +-- @return table params HTTP parameters ready for transport_http.request() +-- @return string|nil err Error message +function _M.build_request(self, conf, body, opts) core.log.info("request extra_opts to LLM server: ", core.json.delay_encode(log_sanitize.redact_extra_opts(opts), true)) @@ -225,56 +272,11 @@ function _M.build_request(self, conf, request_body, opts) ssl_server_name = opts.ssl_server_name or parsed_url and parsed_url.host or opts.target_host or self.host, + -- Passed through as given: a string goes out verbatim (the client's own + -- bytes), a table is encoded by the transport. + body = body, } - -- Inject model options (flat overwrite) - if opts.model_options then - for opt, val in pairs(opts.model_options) do - if request_body[opt] ~= nil then - core.log.info("model_options overwriting request field '", opt, "'") - end - request_body[opt] = val - body_changed = true - end - end - - -- Apply llm_options via provider capability hook (always force-overwrites) - if opts.override_llm_options then - local cap = self.capabilities and self.capabilities[target_protocol] - if cap and cap.rewrite_request_body then - cap.rewrite_request_body(request_body, opts.override_llm_options, true) - if next(opts.override_llm_options) ~= nil then - body_changed = true - end - end - end - - -- Apply per-target-protocol request body override (deep merge) - if opts.request_body_override_map then - local patch = opts.request_body_override_map[target_protocol] - if patch then - core.log.info("applying request_body override for target protocol '", - target_protocol, "'") - request_body = deep_merge(request_body, patch, opts.request_body_force_override) - body_changed = true - end - end - - if self.remove_model and request_body.model ~= nil then - request_body.model = nil - body_changed = true - end - - -- Send the downstream body verbatim when nothing above modified it, so a - -- pure passthrough stays byte-identical. The caller sets client.raw_body only - -- when it has not transformed the body itself (i.e. no protocol conversion - -- ran); an internal call has no client at all and always sends what it built. - if not body_changed and client and type(client.raw_body) == "string" then - params.body = client.raw_body - else - params.body = request_body - end - -- AWS SigV4 signing (must be last — signs the finalized body) if self.aws_sigv4 and auth.aws then local auth_aws = require("apisix.plugins.ai-transport.auth-aws") @@ -789,10 +791,12 @@ end -- @return string|nil Raw response body (JSON string) -- @return string|nil Error message --- Send a self-contained request to an LLM and return its raw response. --- Fully decoupled from the downstream request: internal callers (ai-request-rewrite, --- embeddings, ...) use this as a plain client and parse the body themselves. +-- Shapes the body, assembles the HTTP request, sends it. Fully decoupled from the +-- downstream request: internal callers (ai-request-rewrite, embeddings, ...) use +-- this as a plain client and parse the body themselves. function _M.request(self, conf, request_table, extra_opts) - local params, err = self:build_request(conf, request_table, extra_opts) + local body = self:build_body(request_table, extra_opts) + local params, err = self:build_request(conf, body, extra_opts) if not params then core.log.error("failed to build request: ", err) return 500, nil, err diff --git a/apisix/plugins/ai-proxy/base.lua b/apisix/plugins/ai-proxy/base.lua index 420cbc209123..539d2f6ba141 100644 --- a/apisix/plugins/ai-proxy/base.lua +++ b/apisix/plugins/ai-proxy/base.lua @@ -276,30 +276,39 @@ function _M.before_proxy(conf, ctx, on_error) end end - -- Step 2.5: protocol conversion. It lives here rather than in the - -- transport because converters stash state on ctx for the response side. + -- Step 2.5: protocol conversion. It lives here rather than in the provider + -- because converters stash state on ctx for the response side to read back. local body_for_llm = request_body + local converted = false if converter and converter.convert_request then - local converted, conv_err = converter.convert_request(request_body, ctx) - if not converted then + local new_body, conv_err = converter.convert_request(request_body, ctx) + if not new_body then return 400, {error_msg = conv_err or "invalid protocol"} end - body_for_llm = converted - elseif not ctx.ai_request_body_changed then - -- Nothing has rewritten the body -- neither a converter here nor an - -- earlier plugin (ai-request-rewrite marks that on ctx) -- so the - -- client's verbatim bytes can be reused, keeping a pure passthrough - -- byte-identical and skipping a re-encode. - client.raw_body = core.request.get_body() + body_for_llm = new_body + converted = true end local do_request = function() ctx.llm_request_start_time = ngx.now() ctx.var.llm_request_body = request_body - -- Step 3: Build HTTP request params + -- Step 3: shape the body for the target protocol, then decide which + -- bytes actually go out. When nothing has touched the body -- no + -- conversion above, no shaping just now, and no earlier plugin rewrite + -- (ai-request-rewrite marks that on ctx) -- the client's verbatim bytes + -- are reused, keeping a pure passthrough byte-identical. + local body, shaped = ai_provider:build_body(body_for_llm, extra_opts) + if not shaped and not converted and not ctx.ai_request_body_changed then + local raw = core.request.get_body() + if type(raw) == "string" then + body = raw + end + end + + -- Step 4: assemble the HTTP request local params, build_err, code = ai_provider:build_request( - conf, body_for_llm, extra_opts) + conf, body, extra_opts) if not params then local body = {error_msg = build_err} if code then diff --git a/t/plugin/ai-proxy-request-body-override.t b/t/plugin/ai-proxy-request-body-override.t index ce9137f2ebb5..58a5bbfd11a3 100644 --- a/t/plugin/ai-proxy-request-body-override.t +++ b/t/plugin/ai-proxy-request-body-override.t @@ -258,66 +258,51 @@ same body -=== TEST 5c: build_request sends the parsed body when the caller offers no raw body +=== TEST 5c: build_request passes the body through untouched --- config location /t { content_by_lua_block { - -- The caller withholds raw_request_body whenever the body is no longer - -- the client's original -- a converter ran, or an earlier plugin - -- rewrote it (ai-request-rewrite marks that on ctx). build_request - -- then sends the parsed table. + -- build_request assembles the HTTP request only: whatever body the + -- caller hands it goes out as-is. A table is encoded by the transport. local base = require("apisix.plugins.ai-providers.base") - local provider = base.new({ - capabilities = { - ["openai-chat"] = { - path = "/v1/chat/completions", - host = "localhost", - }, - }, - }) + local provider = base.new({}) local opts = { auth = {}, conf = {}, - target_protocol = "openai-chat", target_path = "/v1/chat/completions", + target_host = "localhost", } local request_body = {messages = {{role = "user", content = "changed"}}} local params = assert(provider:build_request({ssl_verify = false}, request_body, opts)) ngx.say(type(params.body)) - ngx.say(params.body == request_body and "table body" or "raw body") + ngx.say(params.body == request_body and "same table" or "other body") } } --- response_body table -table body +same table -=== TEST 6d: build_request reuses the caller's raw body when nothing changes it +=== TEST 6d: build_request sends a string body verbatim --- config location /t { content_by_lua_block { + -- The caller reuses the client's raw bytes simply by passing the + -- string; the transport sends a string body as-is. local base = require("apisix.plugins.ai-providers.base") - local provider = base.new({ - capabilities = { - ["openai-chat"] = { - path = "/v1/chat/completions", - host = "localhost", - }, - }, - }) + local provider = base.new({}) local opts = { auth = {}, conf = {}, - target_protocol = "openai-chat", target_path = "/v1/chat/completions", - client = { raw_body = '{"messages":[]}' }, + target_host = "localhost", } local params = assert(provider:build_request({ssl_verify = false}, - {messages = {}}, opts)) + '{"messages":[]}', opts)) ngx.say(params.body) } } @@ -326,7 +311,49 @@ table body -=== TEST 7: llm_options: openai provider maps max_tokens to max_completion_tokens +=== TEST 7e: build_body reports whether it changed the body +--- config + location /t { + content_by_lua_block { + -- `changed` is what tells the caller the client's verbatim bytes are + -- no longer safe to reuse. + local base = require("apisix.plugins.ai-providers.base") + local provider = base.new({ + capabilities = { + ["openai-chat"] = { path = "/v1/chat/completions" }, + }, + }) + + -- nothing to apply -> untouched + local _, changed = provider:build_body({messages = {}}, + {target_protocol = "openai-chat"}) + ngx.say("untouched: ", tostring(changed)) + + -- a request_body override -> changed + local body, changed2 = provider:build_body({messages = {}}, { + target_protocol = "openai-chat", + request_body_override_map = { + ["openai-chat"] = {temperature = 0.5}, + }, + }) + ngx.say("overridden: ", tostring(changed2), " temperature=", body.temperature) + + -- model_options -> changed + local body3, changed3 = provider:build_body({messages = {}}, { + target_protocol = "openai-chat", + model_options = {model = "gpt-4o"}, + }) + ngx.say("model_options: ", tostring(changed3), " model=", body3.model) + } + } +--- response_body +untouched: false +overridden: true temperature=0.5 +model_options: true model=gpt-4o + + + +=== TEST 8: llm_options: openai provider maps max_tokens to max_completion_tokens --- config location /t { content_by_lua_block { @@ -369,7 +396,7 @@ max_completion_tokens=555 -=== TEST 8: llm_options: openai-compatible provider maps max_tokens to max_tokens +=== TEST 9: llm_options: openai-compatible provider maps max_tokens to max_tokens --- config location /t { content_by_lua_block { @@ -412,7 +439,7 @@ max_tokens=444 -=== TEST 9: llm_options: openai responses API maps max_tokens to max_output_tokens +=== TEST 10: llm_options: openai responses API maps max_tokens to max_output_tokens --- config location /t { content_by_lua_block { @@ -455,7 +482,7 @@ max_output_tokens=333 -=== TEST 10: llm_options: ai-proxy-multi per-instance override +=== TEST 11: llm_options: ai-proxy-multi per-instance override --- config location /t { content_by_lua_block { @@ -502,7 +529,7 @@ max_completion_tokens=222 -=== TEST 11: llm_options always force-overwrites client value +=== TEST 12: llm_options always force-overwrites client value --- config location /t { content_by_lua_block { @@ -547,7 +574,7 @@ max_tokens=555 -=== TEST 12: request_body: openai-chat override writes fields on outgoing body +=== TEST 13: request_body: openai-chat override writes fields on outgoing body --- config location /t { content_by_lua_block { @@ -595,7 +622,7 @@ max_tokens=555 temperature=0.1 -=== TEST 13: request_body: non-force deep merge fills missing nested keys without overwriting existing +=== TEST 14: request_body: non-force deep merge fills missing nested keys without overwriting existing --- config location /t { content_by_lua_block { @@ -644,7 +671,7 @@ include_usage=true extra=1 -=== TEST 14: request_body: array values are replaced wholesale (stop sequences) +=== TEST 15: request_body: array values are replaced wholesale (stop sequences) --- config location /t { content_by_lua_block { @@ -690,7 +717,7 @@ stop=["a","b"] -=== TEST 15: request_body: override keyed by non-matching target protocol is ignored +=== TEST 16: request_body: override keyed by non-matching target protocol is ignored --- config location /t { content_by_lua_block { @@ -733,7 +760,7 @@ max_tokens=nil -=== TEST 16: request_body: default mode - client value takes priority +=== TEST 17: request_body: default mode - client value takes priority --- config location /t { content_by_lua_block { @@ -778,7 +805,7 @@ max_tokens=999 -=== TEST 17: request_body: force_override mode - override overwrites client fields +=== TEST 18: request_body: force_override mode - override overwrites client fields --- config location /t { content_by_lua_block { @@ -824,7 +851,7 @@ max_tokens=555 -=== TEST 18: request_body: override applies to target protocol after converter +=== TEST 19: request_body: override applies to target protocol after converter --- config location /t { content_by_lua_block { @@ -871,7 +898,7 @@ max_tokens=77 has_messages=true -=== TEST 19: ai-proxy-multi per-instance request_body override +=== TEST 20: ai-proxy-multi per-instance request_body override --- config location /t { content_by_lua_block { @@ -919,7 +946,7 @@ max_tokens=321 -=== TEST 20: both llm_options and request_body coexist, request_body wins +=== TEST 21: both llm_options and request_body coexist, request_body wins --- config location /t { content_by_lua_block { From 5198aa3699c45071e71772509a6ecdef74780072 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Mon, 20 Jul 2026 14:40:25 +0800 Subject: [PATCH 07/12] refactor(ai-transport): key the GCP token on credentials, rename opts.client Review follow-ups. fetch_gcp_access_token took a ctx only to build its cache key via plugin_ctx_id(ctx, name) -- i.e. route id + conf version, which says nothing about the token. Moving the call out to each caller (so the provider could be ctx-free) therefore bought nothing and duplicated the same block in every plugin that supports a gcp provider. Key it on the credentials instead: the token depends only on the service account, so identical credentials now share one token across routes and the lookup needs no ctx. The call moves back inside build_request and no caller repeats it. The key keeps its '<...>#' shape, which the cache-hit test asserts on. Rename opts.client to opts.downstream: it is not a client object, it is the data taken from the inbound request (headers, args, method). The grouping stays -- its presence is what marks a call as proxying, and redact_extra_opts drops the one key to keep all of it out of the logs -- but the name no longer suggests an HTTP client. Keep protocol conversion inside the pcall boundary. Moving it out of do_request took it outside the pcall that bounds a request attempt, which is reachable with input-controlled but valid JSON: an Anthropic image block whose "source" is a boolean makes convert_media_block index a boolean, and the exception then escapes before_proxy instead of becoming a controlled 500. Run it inside do_request again, keeping its returned-400 path, and add a regression test driving exactly that payload. Assert both build_request calls in the auth.query mutation test; a build failure would otherwise leave auth_query untouched and still report OK. --- apisix/plugins/ai-providers/base.lua | 37 +++++++++-------- apisix/plugins/ai-proxy/base.lua | 53 ++++++++++--------------- apisix/plugins/ai-request-rewrite.lua | 15 +------ apisix/plugins/ai-transport/auth.lua | 14 +++++-- apisix/utils/log-sanitize.lua | 9 ++--- t/plugin/ai-proxy-protocol-conversion.t | 49 +++++++++++++++++++++++ t/plugin/ai-proxy.t | 6 ++- 7 files changed, 112 insertions(+), 71 deletions(-) diff --git a/apisix/plugins/ai-providers/base.lua b/apisix/plugins/ai-providers/base.lua index 9087958bdf01..754aa56f4fe3 100644 --- a/apisix/plugins/ai-providers/base.lua +++ b/apisix/plugins/ai-providers/base.lua @@ -31,6 +31,7 @@ local url = require("socket.url") local sse = require("apisix.plugins.ai-transport.sse") local aws_eventstream = require("apisix.plugins.ai-transport.aws-eventstream") local transport_http = require("apisix.plugins.ai-transport.http") +local transport_auth = require("apisix.plugins.ai-transport.auth") local log_sanitize = require("apisix.utils.log-sanitize") local protocols = require("apisix.plugins.ai-protocols") local deep_merge = require("apisix.plugins.ai-proxy.merge").deep_merge @@ -169,10 +170,10 @@ end -- -- Pure and protocol-agnostic: `body` arrives already shaped (see build_body) and -- is passed through untouched, so a string goes out verbatim and a table is --- encoded by the transport. Everything taken from the downstream request lives on --- `opts.client`; its presence is what marks this as proxying an inbound request. --- A self-contained internal call omits it, so nothing of the client's can reach --- the LLM or the logs. +-- encoded by the transport. Everything taken from the inbound request lives on +-- `opts.downstream`; its presence is what marks this as proxying one. A +-- self-contained internal call omits it, so nothing of the client's can reach the +-- LLM or the logs. -- @return table params HTTP parameters ready for transport_http.request() -- @return string|nil err Error message function _M.build_request(self, conf, body, opts) @@ -226,19 +227,23 @@ function _M.build_request(self, conf, body, opts) .. "includes a path", 400 end - -- Everything below that comes from the downstream request lives on - -- opts.client; an internal call leaves it nil and therefore forwards none of - -- the client's headers, query or body. - local client = opts.client + -- Everything below that comes from the inbound request lives on + -- opts.downstream; an internal call leaves it nil and therefore forwards none + -- of the client's headers or query. + local downstream = opts.downstream local headers = transport_http.construct_forward_headers(auth.header or {}, - client and client.headers) + downstream and downstream.headers) if opts.host_header then headers["Host"] = opts.host_header end - -- The caller resolves credentials that need request state (e.g. a GCP - -- access token) and hands the finished token over. - if opts.access_token then - headers["authorization"] = "Bearer " .. opts.access_token + -- GCP tokens are fetched here: the lookup is keyed on the credentials, not on + -- the request, so it needs no ctx and no caller has to repeat it. + if auth.gcp then + local token, token_err = transport_auth.fetch_gcp_access_token(opts.name, auth.gcp) + if not token then + return nil, "failed to get gcp access token: " .. (token_err or "unknown") + end + headers["authorization"] = "Bearer " .. token end -- Optional header transformation (e.g. the Anthropic → OpenAI converter's). @@ -249,11 +254,11 @@ function _M.build_request(self, conf, body, opts) -- The caller passes the downstream method and query string only when it -- wants them proxied verbatim (the passthrough protocol). Otherwise this is -- a plain POST with provider-specific query args. - local method = client and client.method or "POST" - if client and type(client.args) == "table" then + local method = downstream and downstream.method or "POST" + if downstream and type(downstream.args) == "table" then -- client query overrides the endpoint query, but configured -- auth.query credentials must stay non-overridable by the caller - for k, v in pairs(client.args) do + for k, v in pairs(downstream.args) do if not (auth.query and auth.query[k] ~= nil) then query_params[k] = v end diff --git a/apisix/plugins/ai-proxy/base.lua b/apisix/plugins/ai-proxy/base.lua index 539d2f6ba141..8920d107f7de 100644 --- a/apisix/plugins/ai-proxy/base.lua +++ b/apisix/plugins/ai-proxy/base.lua @@ -26,7 +26,6 @@ local table = table local exporter = require("apisix.plugins.prometheus.exporter") local protocols = require("apisix.plugins.ai-protocols") local transport_http = require("apisix.plugins.ai-transport.http") -local transport_auth = require("apisix.plugins.ai-transport.auth") local log_sanitize = require("apisix.utils.log-sanitize") local apisix_upstream = require("resty.apisix.upstream") @@ -247,52 +246,42 @@ function _M.before_proxy(conf, ctx, on_error) -- extra_opts. extra_opts.header_transform = converter and converter.convert_headers - -- Credentials that need request state (GCP token cache is keyed on ctx). - local instance_auth = ai_instance.auth - if instance_auth and instance_auth.gcp then - local token, token_err = transport_auth.fetch_gcp_access_token( - ctx, ai_instance.name, instance_auth.gcp) - if not token then - core.log.error("failed to get gcp access token: ", token_err) - return 500, {error_msg = "failed to get gcp access token: " - .. (token_err or "unknown")} - end - extra_opts.access_token = token - end - -- ai-proxy is a transparent proxy of an inbound request, so it hands the - -- transport everything taken from that request under one `client` key. + -- provider everything taken from that request under one `downstream` key. -- Internal callers omit it entirely and thus forward nothing of the -- client's -- see ai-providers/base.lua build_request. - local client = { headers = core.request.headers(ctx) } - extra_opts.client = client + local downstream = { headers = core.request.headers(ctx) } + extra_opts.downstream = downstream -- passthrough proxies the client's method and query string verbatim. if target_proto == "passthrough" then - client.method = core.request.get_method() + downstream.method = core.request.get_method() local client_args = ctx.var.args and core.string.decode_args(ctx.var.args) if type(client_args) == "table" then - client.args = client_args + downstream.args = client_args end end - -- Step 2.5: protocol conversion. It lives here rather than in the provider - -- because converters stash state on ctx for the response side to read back. - local body_for_llm = request_body - local converted = false - if converter and converter.convert_request then - local new_body, conv_err = converter.convert_request(request_body, ctx) - if not new_body then - return 400, {error_msg = conv_err or "invalid protocol"} - end - body_for_llm = new_body - converted = true - end - local do_request = function() ctx.llm_request_start_time = ngx.now() ctx.var.llm_request_body = request_body + -- Step 2.5: protocol conversion. It runs in the provider's stead -- + -- converters stash state on ctx for the response side to read back -- + -- but stays inside do_request so the pcall below still bounds it: a + -- converter fed hostile-but-valid JSON can raise (e.g. an Anthropic + -- image block whose "source" is not an object). + local body_for_llm = request_body + local converted = false + if converter and converter.convert_request then + local new_body, conv_err = converter.convert_request(request_body, ctx) + if not new_body then + return 400, {error_msg = conv_err or "invalid protocol"} + end + body_for_llm = new_body + converted = true + end + -- Step 3: shape the body for the target protocol, then decide which -- bytes actually go out. When nothing has touched the body -- no -- conversion above, no shaping just now, and no earlier plugin rewrite diff --git a/apisix/plugins/ai-request-rewrite.lua b/apisix/plugins/ai-request-rewrite.lua index 930143aed353..29e1b1d58ebf 100644 --- a/apisix/plugins/ai-request-rewrite.lua +++ b/apisix/plugins/ai-request-rewrite.lua @@ -17,7 +17,6 @@ local core = require("apisix.core") local ai_providers_schema = require("apisix.plugins.ai-providers.schema") local protocols = require("apisix.plugins.ai-protocols") -local transport_auth = require("apisix.plugins.ai-transport.auth") local require = require local pcall = pcall local next = next @@ -126,18 +125,8 @@ local function request_to_llm(conf, request_table, ctx, target_path) model_options = conf.options, target_path = target_path, } - -- The provider is a pure client, so resolve here whatever needs the request - -- ctx. Nothing downstream-derived is handed over: no client headers, no - -- verbatim client body -- this call carries its own credentials and body. - if conf.auth and conf.auth.gcp then - local token, token_err = transport_auth.fetch_gcp_access_token(ctx, plugin_name, - conf.auth.gcp) - if not token then - return nil, nil, "failed to get gcp access token: " .. (token_err or "unknown") - end - extra_opts.access_token = token - end - + -- Nothing downstream-derived is handed over: no client headers, no verbatim + -- client body -- this call carries its own credentials and its own body. ctx.llm_request_start_time = ngx.now() ctx.var.llm_request_body = request_table return ai_provider:request(conf, request_table, extra_opts) diff --git a/apisix/plugins/ai-transport/auth.lua b/apisix/plugins/ai-transport/auth.lua index 147067e8f236..18452575d072 100644 --- a/apisix/plugins/ai-transport/auth.lua +++ b/apisix/plugins/ai-transport/auth.lua @@ -23,6 +23,7 @@ local google_oauth = require("apisix.utils.google-cloud-oauth") local lrucache = require("resty.lrucache") local type = type local os = os +local ngx_crc32_long = ngx.crc32_long local _M = {} @@ -30,13 +31,20 @@ local gcp_access_token_cache = lrucache.new(1024 * 4) --- Fetch (or retrieve from cache) a GCP OAuth2 access token. --- @param ctx table Request context +-- +-- Keyed on the credentials themselves rather than on the request ctx: the token +-- depends only on the service account, so identical credentials share one token +-- across routes, and callers need no ctx to ask for one. -- @param name string Cache key name (driver instance name) -- @param gcp_conf table GCP configuration (service_account_json, expire_early_secs, max_ttl) -- @return string|nil Access token -- @return string|nil Error message -function _M.fetch_gcp_access_token(ctx, name, gcp_conf) - local key = core.lrucache.plugin_ctx_id(ctx, name) +function _M.fetch_gcp_access_token(name, gcp_conf) + -- The credentials may arrive as a secret ref resolved per request, so key on + -- the resolved value; hash it so the cache never holds the raw JSON. + local sa = type(gcp_conf) == "table" and gcp_conf.service_account_json + or os.getenv("GCP_SERVICE_ACCOUNT") + local key = ngx_crc32_long(sa or "") .. "#" .. (name or "") local access_token = gcp_access_token_cache:get(key) if not access_token then local auth_conf = {} diff --git a/apisix/utils/log-sanitize.lua b/apisix/utils/log-sanitize.lua index 10ebb48f3a8d..a75f896a65c2 100644 --- a/apisix/utils/log-sanitize.lua +++ b/apisix/utils/log-sanitize.lua @@ -56,11 +56,10 @@ end function _M.redact_extra_opts(extra_opts) local redacted = core.table.deepcopy(extra_opts) redacted.auth = nil - -- Everything taken from the downstream request is grouped under `client`: - -- its headers (Authorization/Cookie), its query args and its verbatim body - -- (the user's prompt). None of it belongs in a log line, and dropping the - -- one key drops all of it. - redacted.client = nil + -- Everything taken from the inbound request is grouped under `downstream`: + -- its headers (Authorization/Cookie) and query args. None of it belongs in a + -- log line, and dropping the one key drops all of it. + redacted.downstream = nil return redacted end diff --git a/t/plugin/ai-proxy-protocol-conversion.t b/t/plugin/ai-proxy-protocol-conversion.t index b28d1c9eba51..a737e8e7b993 100644 --- a/t/plugin/ai-proxy-protocol-conversion.t +++ b/t/plugin/ai-proxy-protocol-conversion.t @@ -1108,3 +1108,52 @@ cannot parse any events and the gateway should return 502 instead of crashing. status: 502 --- error_log streaming response completed without producing any output + + + +=== TEST 29: Set up route for converter fault isolation +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/v1/messages", + "plugins": { + "ai-proxy": { + "provider": "openai", + "auth": { + "header": { + "Authorization": "Bearer token" + } + }, + "override": { + "endpoint": "http://127.0.0.1:1980/v1/chat/completions" + } + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 30: A converter fault on valid JSON stays a controlled 500 +--- request +POST /v1/messages +{"model":"claude-3","messages":[{"role":"user","content":[{"type":"image","source":true}]}]} +--- more_headers +Content-Type: application/json +--- error_code: 500 +--- error_log +failed to send request to AI service +attempt to index diff --git a/t/plugin/ai-proxy.t b/t/plugin/ai-proxy.t index e38decf15eb6..8f57acae6d9f 100644 --- a/t/plugin/ai-proxy.t +++ b/t/plugin/ai-proxy.t @@ -1247,8 +1247,10 @@ got token usage from ai service: conf = {}, } - provider:build_request(conf, {messages = {{role="user", content="hi"}}}, opts) - provider:build_request(conf, {messages = {{role="user", content="hi"}}}, opts) + -- assert both builds: a build failure would otherwise leave auth_query + -- untouched and the test would still report OK + assert(provider:build_request(conf, {messages = {{role="user", content="hi"}}}, opts)) + assert(provider:build_request(conf, {messages = {{role="user", content="hi"}}}, opts)) if auth_query["extra"] then ngx.say("FAIL: auth.query was mutated, extra=" .. auth_query["extra"]) From 4669eaa3990a5527d25b47123c63469bce940e6d Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Mon, 20 Jul 2026 15:11:59 +0800 Subject: [PATCH 08/12] refactor(ai-providers): flatten opts.downstream into opts.client_* The nesting was justified on two grounds; only one held up. It did not make the group atomic: build_request reads each field with its own nil check (`downstream and downstream.headers`, `... and downstream.method`), so a caller could always set some and not others. The nesting made them look like one unit without making them behave as one. What it did buy was redaction: one `redacted.downstream = nil` kept every client-derived field out of the logs, and a field added later was covered for free. Keep that property without the nesting by naming the fields in log-sanitize instead -- CLIENT_DERIVED_FIELDS is an explicit list that says out loud which options are sensitive, which reads better than an implicit 'anything under this key'. The header-forwarding test covers it: drop a field from the list and it fails. opts.client_headers / client_args / client_method now sit directly on opts, which is what they are -- static options taken from the inbound request, not a client object. --- apisix/plugins/ai-providers/base.lua | 22 ++++++++++------------ apisix/plugins/ai-proxy/base.lua | 16 ++++++++-------- apisix/utils/log-sanitize.lua | 17 +++++++++++++---- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/apisix/plugins/ai-providers/base.lua b/apisix/plugins/ai-providers/base.lua index 754aa56f4fe3..e04b57c8b876 100644 --- a/apisix/plugins/ai-providers/base.lua +++ b/apisix/plugins/ai-providers/base.lua @@ -170,10 +170,9 @@ end -- -- Pure and protocol-agnostic: `body` arrives already shaped (see build_body) and -- is passed through untouched, so a string goes out verbatim and a table is --- encoded by the transport. Everything taken from the inbound request lives on --- `opts.downstream`; its presence is what marks this as proxying one. A --- self-contained internal call omits it, so nothing of the client's can reach the --- LLM or the logs. +-- encoded by the transport. Anything taken from the inbound request reaches us +-- as an explicit opts.client_* field; a self-contained internal call sets none of +-- them, so nothing of the client's can reach the LLM or the logs. -- @return table params HTTP parameters ready for transport_http.request() -- @return string|nil err Error message function _M.build_request(self, conf, body, opts) @@ -227,12 +226,11 @@ function _M.build_request(self, conf, body, opts) .. "includes a path", 400 end - -- Everything below that comes from the inbound request lives on - -- opts.downstream; an internal call leaves it nil and therefore forwards none - -- of the client's headers or query. - local downstream = opts.downstream + -- opts.client_* is whatever the caller took from the inbound request; an + -- internal call sets none of it and therefore forwards none of the client's + -- headers or query. local headers = transport_http.construct_forward_headers(auth.header or {}, - downstream and downstream.headers) + opts.client_headers) if opts.host_header then headers["Host"] = opts.host_header end @@ -254,11 +252,11 @@ function _M.build_request(self, conf, body, opts) -- The caller passes the downstream method and query string only when it -- wants them proxied verbatim (the passthrough protocol). Otherwise this is -- a plain POST with provider-specific query args. - local method = downstream and downstream.method or "POST" - if downstream and type(downstream.args) == "table" then + local method = opts.client_method or "POST" + if type(opts.client_args) == "table" then -- client query overrides the endpoint query, but configured -- auth.query credentials must stay non-overridable by the caller - for k, v in pairs(downstream.args) do + for k, v in pairs(opts.client_args) do if not (auth.query and auth.query[k] ~= nil) then query_params[k] = v end diff --git a/apisix/plugins/ai-proxy/base.lua b/apisix/plugins/ai-proxy/base.lua index 8920d107f7de..b98ae43e37da 100644 --- a/apisix/plugins/ai-proxy/base.lua +++ b/apisix/plugins/ai-proxy/base.lua @@ -246,19 +246,19 @@ function _M.before_proxy(conf, ctx, on_error) -- extra_opts. extra_opts.header_transform = converter and converter.convert_headers - -- ai-proxy is a transparent proxy of an inbound request, so it hands the - -- provider everything taken from that request under one `downstream` key. - -- Internal callers omit it entirely and thus forward nothing of the - -- client's -- see ai-providers/base.lua build_request. - local downstream = { headers = core.request.headers(ctx) } - extra_opts.downstream = downstream + -- ai-proxy is a transparent proxy of an inbound request, so it passes what + -- it takes from that request as client_* options. Internal callers set + -- none of them and thus forward nothing of the client's -- see + -- ai-providers/base.lua build_request. Keep the names in sync with + -- log-sanitize's CLIENT_DERIVED_FIELDS so they stay out of the logs. + extra_opts.client_headers = core.request.headers(ctx) -- passthrough proxies the client's method and query string verbatim. if target_proto == "passthrough" then - downstream.method = core.request.get_method() + extra_opts.client_method = core.request.get_method() local client_args = ctx.var.args and core.string.decode_args(ctx.var.args) if type(client_args) == "table" then - downstream.args = client_args + extra_opts.client_args = client_args end end diff --git a/apisix/utils/log-sanitize.lua b/apisix/utils/log-sanitize.lua index a75f896a65c2..816a56514791 100644 --- a/apisix/utils/log-sanitize.lua +++ b/apisix/utils/log-sanitize.lua @@ -18,6 +18,7 @@ local core = require("apisix.core") local pairs = pairs +local ipairs = ipairs local _M = {} @@ -53,13 +54,21 @@ function _M.redact_params(params) end +-- extra_opts fields carrying data taken from the inbound request: its headers +-- (Authorization/Cookie) and query args. None of it belongs in a log line. Add +-- any new client-derived option here. +local CLIENT_DERIVED_FIELDS = { + "client_headers", + "client_args", +} + + function _M.redact_extra_opts(extra_opts) local redacted = core.table.deepcopy(extra_opts) redacted.auth = nil - -- Everything taken from the inbound request is grouped under `downstream`: - -- its headers (Authorization/Cookie) and query args. None of it belongs in a - -- log line, and dropping the one key drops all of it. - redacted.downstream = nil + for _, field in ipairs(CLIENT_DERIVED_FIELDS) do + redacted[field] = nil + end return redacted end From 60e58c3df0c1936d4d3c329753c80027b8c38b13 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Mon, 20 Jul 2026 15:38:23 +0800 Subject: [PATCH 09/12] fix(ai-transport): make extra_opts redaction an allowlist The denylist only protected the fields someone remembered to name. That is the failure mode that already happened once in this branch: raw_request_body was added to extra_opts and the user's prompt went into the info log, because the sanitizer stripped only auth. Flattening opts.client_* back onto opts made that worse -- the nested form at least covered every field under one key. Invert it: name the fields that are safe to log (connection/routing shape and non-secret provider config) and drop everything else. A field added to extra_opts later is now redacted by default rather than leaked by default. redact_params right below already works this way; this makes the two consistent. Also drops the per-request deepcopy -- the allowlist copies a dozen scalars instead. --- apisix/utils/log-sanitize.lua | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/apisix/utils/log-sanitize.lua b/apisix/utils/log-sanitize.lua index 816a56514791..a9c0ce6e0a38 100644 --- a/apisix/utils/log-sanitize.lua +++ b/apisix/utils/log-sanitize.lua @@ -15,7 +15,6 @@ -- limitations under the License. -- -local core = require("apisix.core") local pairs = pairs local ipairs = ipairs @@ -54,22 +53,32 @@ function _M.redact_params(params) end --- extra_opts fields carrying data taken from the inbound request: its headers --- (Authorization/Cookie) and query args. None of it belongs in a log line. Add --- any new client-derived option here. -local CLIENT_DERIVED_FIELDS = { - "client_headers", - "client_args", +-- extra_opts fields that are safe to log: connection/routing shape and +-- non-secret provider config. Deliberately an allowlist, like redact_params +-- below -- anything added to extra_opts later (credentials, client headers, a +-- verbatim request body) stays out of the logs unless it is named here. +local LOGGABLE_EXTRA_OPTS = { + "name", + "endpoint", + "target_host", + "target_path", + "target_protocol", + "host_header", + "ssl_server_name", + "model_options", + "override_llm_options", + "request_body_override_map", + "request_body_force_override", + "conf", } function _M.redact_extra_opts(extra_opts) - local redacted = core.table.deepcopy(extra_opts) - redacted.auth = nil - for _, field in ipairs(CLIENT_DERIVED_FIELDS) do - redacted[field] = nil + local safe = {} + for _, field in ipairs(LOGGABLE_EXTRA_OPTS) do + safe[field] = extra_opts[field] end - return redacted + return safe end From 6e06f6a659aec822aa78f8921749fe9f955de02c Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Mon, 20 Jul 2026 15:57:29 +0800 Subject: [PATCH 10/12] fix(ai-transport): log named options instead of dumping extra_opts Neither list was the right answer. A denylist leaked whatever nobody remembered to name -- it missed client_headers and then raw_request_body inside this branch alone. An allowlist fixed the default but made log-sanitize track the caller's whole field vocabulary: which options exist is the caller's business, only what counts as a secret is the sanitizer's. The real problem was serialising an open-ended options bag into a log line at all. Most of what it printed is already logged from the resolved `params` further down (endpoint, path, host, headers -- the last with per-header redaction), so the dump was largely duplicate. Log the three things it uniquely carried -- instance name, target protocol, model -- by name. redact_extra_opts has no callers left and goes with it; log-sanitize is back to one job, redacting the params it is handed. A field added to extra_opts later is now private by default with no list to maintain. --- apisix/plugins/ai-providers/base.lua | 9 +++++++-- apisix/utils/log-sanitize.lua | 30 ---------------------------- 2 files changed, 7 insertions(+), 32 deletions(-) diff --git a/apisix/plugins/ai-providers/base.lua b/apisix/plugins/ai-providers/base.lua index e04b57c8b876..6ed57b84e2ec 100644 --- a/apisix/plugins/ai-providers/base.lua +++ b/apisix/plugins/ai-providers/base.lua @@ -176,8 +176,13 @@ end -- @return table params HTTP parameters ready for transport_http.request() -- @return string|nil err Error message function _M.build_request(self, conf, body, opts) - core.log.info("request extra_opts to LLM server: ", - core.json.delay_encode(log_sanitize.redact_extra_opts(opts), true)) + -- Name the few options worth tracing rather than dumping opts wholesale: the + -- resolved endpoint, path and headers are already logged from `params` below, + -- and serialising an open-ended options bag is how credentials and client data + -- end up in logs. + core.log.info("building LLM request: instance=", opts.name, + ", target_protocol=", opts.target_protocol, + ", model=", opts.model_options and opts.model_options.model) local auth = opts.auth or {} diff --git a/apisix/utils/log-sanitize.lua b/apisix/utils/log-sanitize.lua index a9c0ce6e0a38..59a3068cc9e1 100644 --- a/apisix/utils/log-sanitize.lua +++ b/apisix/utils/log-sanitize.lua @@ -17,7 +17,6 @@ local pairs = pairs -local ipairs = ipairs local _M = {} @@ -53,33 +52,4 @@ function _M.redact_params(params) end --- extra_opts fields that are safe to log: connection/routing shape and --- non-secret provider config. Deliberately an allowlist, like redact_params --- below -- anything added to extra_opts later (credentials, client headers, a --- verbatim request body) stays out of the logs unless it is named here. -local LOGGABLE_EXTRA_OPTS = { - "name", - "endpoint", - "target_host", - "target_path", - "target_protocol", - "host_header", - "ssl_server_name", - "model_options", - "override_llm_options", - "request_body_override_map", - "request_body_force_override", - "conf", -} - - -function _M.redact_extra_opts(extra_opts) - local safe = {} - for _, field in ipairs(LOGGABLE_EXTRA_OPTS) do - safe[field] = extra_opts[field] - end - return safe -end - - return _M From bbe83574bdad27d805625f6c8618928489c2d7ba Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Tue, 21 Jul 2026 07:49:32 +0800 Subject: [PATCH 11/12] revert(log-sanitize): keep redact_extra_opts; the removal was churn The security fix is the build_request log-site change -- it stopped dumping the whole options bag and logs three named values (instance, target_protocol, model) instead, which is what closed the leak. Deleting redact_extra_opts on top of that bought nothing: it is a small, tested, exported util in a shared module, its removal only enlarged a security diff, dropped a used require, and forced deleting a passing test. It also left HEAD internally inconsistent -- the function gone but t/utils/log-sanitize.t still calling it -- which would break CI. Restore log-sanitize.lua and t/utils/log-sanitize.t to their master state. The log-site change stands on its own. --- apisix/utils/log-sanitize.lua | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apisix/utils/log-sanitize.lua b/apisix/utils/log-sanitize.lua index 59a3068cc9e1..03fe02acf1fb 100644 --- a/apisix/utils/log-sanitize.lua +++ b/apisix/utils/log-sanitize.lua @@ -15,6 +15,7 @@ -- limitations under the License. -- +local core = require("apisix.core") local pairs = pairs @@ -52,4 +53,11 @@ function _M.redact_params(params) end +function _M.redact_extra_opts(extra_opts) + local redacted = core.table.deepcopy(extra_opts) + redacted.auth = nil + return redacted +end + + return _M From 9c993791b1190e33f4c6e7f19c46852ef4839db8 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Tue, 21 Jul 2026 13:10:21 +0800 Subject: [PATCH 12/12] fix(ai-transport): make the GCP token cache key collision-resistant The ctx-free refactor keyed the GCP access-token cache on ngx.crc32_long(service_account_json) .. name. A 32-bit CRC can collide, so distinct credentials could reuse the wrong cached token; and because the key omitted expire_early_secs and max_ttl, identical credentials under different TTL policies shared one entry and the second policy was bypassed. Key on a 128-bit digest of the resolved credentials plus those TTL fields. The raw JSON still never enters the key or the log. --- apisix/plugins/ai-transport/auth.lua | 36 +++++++++++++++------------- t/plugin/ai-proxy-vertex-ai.t | 36 ++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 16 deletions(-) diff --git a/apisix/plugins/ai-transport/auth.lua b/apisix/plugins/ai-transport/auth.lua index 18452575d072..2670caa44cbc 100644 --- a/apisix/plugins/ai-transport/auth.lua +++ b/apisix/plugins/ai-transport/auth.lua @@ -23,7 +23,8 @@ local google_oauth = require("apisix.utils.google-cloud-oauth") local lrucache = require("resty.lrucache") local type = type local os = os -local ngx_crc32_long = ngx.crc32_long +local tostring = tostring +local ngx_md5 = ngx.md5 local _M = {} @@ -41,22 +42,25 @@ local gcp_access_token_cache = lrucache.new(1024 * 4) -- @return string|nil Error message function _M.fetch_gcp_access_token(name, gcp_conf) -- The credentials may arrive as a secret ref resolved per request, so key on - -- the resolved value; hash it so the cache never holds the raw JSON. - local sa = type(gcp_conf) == "table" and gcp_conf.service_account_json - or os.getenv("GCP_SERVICE_ACCOUNT") - local key = ngx_crc32_long(sa or "") .. "#" .. (name or "") + -- the resolved value; hash it so the cache never holds the raw JSON. Include + -- the fields that change how long the token is cached (expire_early_secs, + -- max_ttl) so that identical credentials under different TTL policies do not + -- share an entry, and use a 128-bit digest instead of a 32-bit CRC so that + -- distinct credentials cannot collide onto the wrong cached token. + local conf = type(gcp_conf) == "table" and gcp_conf or {} + local sa = conf.service_account_json or os.getenv("GCP_SERVICE_ACCOUNT") + local key = ngx_md5((sa or "") .. "\0" + .. tostring(conf.expire_early_secs or "") .. "\0" + .. tostring(conf.max_ttl or "")) .. "#" .. (name or "") local access_token = gcp_access_token_cache:get(key) if not access_token then local auth_conf = {} - gcp_conf = type(gcp_conf) == "table" and gcp_conf or {} - local service_account_json = gcp_conf.service_account_json or - os.getenv("GCP_SERVICE_ACCOUNT") - if type(service_account_json) == "string" and service_account_json ~= "" then - local conf, err = core.json.decode(service_account_json) - if not conf then + if type(sa) == "string" and sa ~= "" then + local decoded, err = core.json.decode(sa) + if not decoded then return nil, "invalid gcp service account json: " .. (err or "unknown error") end - auth_conf = conf + auth_conf = decoded end local oauth = google_oauth.new(auth_conf) access_token = oauth:generate_access_token() @@ -64,11 +68,11 @@ function _M.fetch_gcp_access_token(name, gcp_conf) return nil, "failed to get google oauth token" end local ttl = oauth.access_token_ttl or 3600 - if gcp_conf.expire_early_secs and ttl > gcp_conf.expire_early_secs then - ttl = ttl - gcp_conf.expire_early_secs + if conf.expire_early_secs and ttl > conf.expire_early_secs then + ttl = ttl - conf.expire_early_secs end - if gcp_conf.max_ttl and ttl > gcp_conf.max_ttl then - ttl = gcp_conf.max_ttl + if conf.max_ttl and ttl > conf.max_ttl then + ttl = conf.max_ttl end gcp_access_token_cache:set(key, access_token, ttl) core.log.debug("set gcp access token in cache with ttl: ", ttl, ", key: ", key) diff --git a/t/plugin/ai-proxy-vertex-ai.t b/t/plugin/ai-proxy-vertex-ai.t index e3710f3f1b1f..88a6c2aa7268 100644 --- a/t/plugin/ai-proxy-vertex-ai.t +++ b/t/plugin/ai-proxy-vertex-ai.t @@ -486,3 +486,39 @@ request head: GET /status/gpt4 /v1/projects/my-project/locations/us-central1/publishers/google/models/text-embedding-004:predict /v1/projects/my-project/locations/us-central1/publishers/google/models/textembedding-gecko:predict nil + + + +=== TEST 13: same credentials under different TTL policies do not share a token +--- config + location /t { + content_by_lua_block { + local auth = require("apisix.plugins.ai-transport.auth") + local google_oauth = require("apisix.utils.google-cloud-oauth") + local calls = 0 + local orig_new = google_oauth.new + google_oauth.new = function(conf) + return { + access_token_ttl = 3600, + generate_access_token = function() + calls = calls + 1 + return "token-" .. calls + end, + } + end + + local sa = '{"type":"service_account","project_id":"p"}' + -- same instance name and credentials, different max_ttl policy: + -- must not alias onto one cache entry + local t1 = auth.fetch_gcp_access_token("inst", {service_account_json = sa, max_ttl = 100}) + local t2 = auth.fetch_gcp_access_token("inst", {service_account_json = sa, max_ttl = 200}) + -- repeating the first policy must hit the cache, no new token + local t3 = auth.fetch_gcp_access_token("inst", {service_account_json = sa, max_ttl = 100}) + + google_oauth.new = orig_new + ngx.say("calls=", calls, " distinct=", tostring(t1 ~= t2), + " cachehit=", tostring(t1 == t3)) + } + } +--- response_body +calls=2 distinct=true cachehit=true