diff --git a/.env.example b/.env.example index 61dbdc64..c00d2dbc 100644 --- a/.env.example +++ b/.env.example @@ -209,11 +209,11 @@ QWEN_CHAT_PROXY_URL= QWEN_CLI_PROXY_URL= # HTTP/HTTPS 代理配置 -# 支持 HTTP, HTTPS, SOCKS5 代理 +# 支持 HTTP, HTTPS, SOCKS5, SOCKS5H 代理(socks5h:// 由代理端解析 DNS) # 全局回退代理:当账号未配置专属代理时使用(账号级代理可在 dashboard 或 ACCOUNTS 中设置) # 优先级:account.proxy > PROXY_URL > 不使用代理 # HTTP/HTTPS proxy configuration -# Supports HTTP, HTTPS, SOCKS5 proxies +# Supports HTTP, HTTPS, SOCKS5, SOCKS5H proxies (socks5h:// resolves DNS at the proxy) # Global fallback proxy — used only when an account has no dedicated proxy # (per-account proxy can be set via the dashboard or ACCOUNTS env var) # Priority: account.proxy > PROXY_URL > no proxy diff --git a/README-en.md b/README-en.md index 83e0dbd8..b6b84069 100644 --- a/README-en.md +++ b/README-en.md @@ -37,7 +37,7 @@ Each account can be configured with its own outbound proxy, allowing multiple ac **Priority:** `account.proxy` > Global `PROXY_URL` > No proxy -**Supported Proxy Protocols:** HTTP / HTTPS / SOCKS5 (consistent with `PROXY_URL`) +**Supported Proxy Protocols:** HTTP / HTTPS / SOCKS5 / SOCKS5H (consistent with `PROXY_URL`) **Frontend Configuration (Recommended):** Open the management panel → Fill in the "Proxy Address" field when adding accounts, or click the "Modify Proxy" button on existing account cards. @@ -103,7 +103,7 @@ AGENT_TURN_MAX_TOOL_CALLS=24 # Anthropic path: text-channel tool_use cap per ag # 🌐 Proxy and Reverse Proxy Configuration QWEN_CHAT_PROXY_URL= # Custom Chat API reverse proxy URL (default: https://chat.qwen.ai) QWEN_CLI_PROXY_URL= # Custom CLI API reverse proxy URL (default: https://portal.qwen.ai) -PROXY_URL= # HTTP/HTTPS/SOCKS5 proxy address (example: http://127.0.0.1:7890) +PROXY_URL= # HTTP/HTTPS/SOCKS5/SOCKS5H proxy address (example: http://127.0.0.1:7890) # 🗄️ Data Storage DATA_SAVE_MODE=none # Data save mode (none/file/redis) @@ -132,7 +132,7 @@ CACHE_MODE=default # Image cache mode (default/file) | `AGENT_CONTEXT_LIVE_PROMPT_BYTES` | Maximum size of the tool protocol and current turn kept in the live request after context externalization | `49152` (48 KiB) | | `QWEN_CHAT_PROXY_URL` | Custom Chat API reverse proxy address | `https://your-proxy.com` | | `QWEN_CLI_PROXY_URL` | Custom CLI API reverse proxy address | `https://your-cli-proxy.com` | -| `PROXY_URL` | Outbound request proxy address, supports HTTP/HTTPS/SOCKS5 | `http://127.0.0.1:7890` | +| `PROXY_URL` | Outbound request proxy address, supports HTTP/HTTPS/SOCKS5/SOCKS5H | `http://127.0.0.1:7890` | | `DATA_SAVE_MODE` | Data persistence method | `none`/`file`/[redis](file://d:\Code\Qwen2API\src\utils\logger.js#L294-L296) | | `REDIS_URL` | Redis database connection address, use `rediss://` protocol when using TLS encryption | `redis://localhost:6379` or `rediss://xxx.upstash.io` | | `BATCH_LOGIN_CONCURRENCY` | Login concurrency during batch account addition, can be adjusted dynamically in frontend system settings | `5` | diff --git a/README-ru.md b/README-ru.md index 7d4e572b..03007ec8 100644 --- a/README-ru.md +++ b/README-ru.md @@ -57,7 +57,7 @@ QWEN_CHAT_PROXY_URL=http://127.0.0.1:8000/qwen # Адрес обратного **Приоритет:** `account.proxy` > глобальный `PROXY_URL` > без прокси -**Поддерживаемые протоколы:** HTTP / HTTPS / SOCKS5 (как у `PROXY_URL`) +**Поддерживаемые протоколы:** HTTP / HTTPS / SOCKS5 / SOCKS5H (как у `PROXY_URL`) **Через панель (рекомендуется):** Откройте dashboard → при добавлении аккаунта заполните поле «Proxy URL», либо нажмите кнопку «Изменить proxy» на карточке существующего аккаунта. @@ -123,7 +123,7 @@ AGENT_TURN_MAX_TOOL_CALLS=24 # Anthropic: лимит tool_use по тексто # 🌐 Прокси и обратный прокси QWEN_CHAT_PROXY_URL= # Пользовательский URL обратного прокси Chat API (по умолчанию: https://chat.qwen.ai) QWEN_CLI_PROXY_URL= # Пользовательский URL обратного прокси CLI API (по умолчанию: https://portal.qwen.ai) -PROXY_URL= # Адрес HTTP/HTTPS/SOCKS5 прокси (например: http://127.0.0.1:7890) +PROXY_URL= # Адрес HTTP/HTTPS/SOCKS5/SOCKS5H прокси (например: http://127.0.0.1:7890) # 🗄️ Хранение данных DATA_SAVE_MODE=none # Режим сохранения данных (none/file/redis) @@ -150,7 +150,7 @@ CACHE_MODE=default # Режим кэширования изображ | `AGENT_TURN_MAX_TOOL_CALLS` | Путь Anthropic: лимит блоков `tool_use` по текстовому каналу за один ход агента (4–256). Если модель «идёт вразнос» после нарративного `[TOOL CALL]` (сотни повторов одного вызова, выдуманная сессия целиком), upstream обрывается сразу после N-го принятого вызова, а принятые вызовы отдаются со `stop_reason=tool_use`; после уже принятого в более раннем delta вызова дубликат, отклонённый вызов или проза/размышление также обрывают ход | `24` | | `QWEN_CHAT_PROXY_URL` | Пользовательский адрес обратного прокси Chat API | `https://your-proxy.com` | | `QWEN_CLI_PROXY_URL` | Пользовательский адрес обратного прокси CLI API | `https://your-cli-proxy.com` | -| `PROXY_URL` | Адрес прокси для исходящих запросов, поддержка HTTP/HTTPS/SOCKS5 | `http://127.0.0.1:7890` | +| `PROXY_URL` | Адрес прокси для исходящих запросов, поддержка HTTP/HTTPS/SOCKS5/SOCKS5H | `http://127.0.0.1:7890` | | `DATA_SAVE_MODE` | Способ персистентного хранения данных | `none`/`file`/`redis` | | `REDIS_URL` | Адрес подключения к Redis, при использовании TLS-шифрования необходим протокол `rediss://` | `redis://localhost:6379` или `rediss://xxx.upstash.io` | | `BATCH_LOGIN_CONCURRENCY` | Параллельность входа при массовом добавлении аккаунтов, можно динамически менять в системных настройках фронтенда | `5` | diff --git a/README.md b/README.md index b85d07b1..1ed570c2 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Qwen-Proxy 是一个将 `https://chat.qwen.ai` 和 `Qwen Code / Qwen Cli` 转换 **优先级:** `account.proxy` > 全局 `PROXY_URL` > 不使用代理 -**支持的代理协议:** HTTP / HTTPS / SOCKS5(与 `PROXY_URL` 一致) +**支持的代理协议:** HTTP / HTTPS / SOCKS5 / SOCKS5H(与 `PROXY_URL` 一致) **前端配置(推荐):** 打开管理面板 → 添加账号时填写 "代理地址" 字段,或在已有账号卡片上点击 "修改代理" 按钮。 @@ -108,7 +108,7 @@ AGENT_CONTEXT_LIVE_PROMPT_BYTES=49152 # 外置后仍内联保留的关键任 # 🌐 代理与反代配置 QWEN_CHAT_PROXY_URL= # 自定义 Chat API 反代URL (默认: https://chat.qwen.ai) QWEN_CLI_PROXY_URL= # 自定义 CLI API 反代URL (默认: https://portal.qwen.ai) -PROXY_URL= # HTTP/HTTPS/SOCKS5 代理地址 (例如: http://127.0.0.1:7890) +PROXY_URL= # HTTP/HTTPS/SOCKS5/SOCKS5H 代理地址 (例如: http://127.0.0.1:7890) # 🗄️ 数据存储 DATA_SAVE_MODE=none # 数据保存模式 (none/file/redis) @@ -140,7 +140,7 @@ CACHE_MODE=default # 图片缓存模式 (default/file) | `AGENT_CONTEXT_LIVE_PROMPT_BYTES` | 上下文外置后,实时请求中保留的工具协议、system/developer 指令、原始任务、最近工具进度和当前结果的最大大小 | `49152`(48 KiB) | | `QWEN_CHAT_PROXY_URL` | 自定义 Chat API 反代地址 | `https://your-proxy.com` | | `QWEN_CLI_PROXY_URL` | 自定义 CLI API 反代地址 | `https://your-cli-proxy.com` | -| `PROXY_URL` | 出站请求代理地址,支持 HTTP/HTTPS/SOCKS5 | `http://127.0.0.1:7890` | +| `PROXY_URL` | 出站请求代理地址,支持 HTTP/HTTPS/SOCKS5/SOCKS5H | `http://127.0.0.1:7890` | | `DATA_SAVE_MODE` | 数据持久化方式 | `none`/`file`/`redis` | | `REDIS_URL` | Redis 数据库连接地址,使用TLS加密时需使用 `rediss://` 协议 | `redis://localhost:6379` 或 `rediss://xxx.upstash.io` | | `BATCH_LOGIN_CONCURRENCY` | 批量添加账号时的登录并发数,可在前端系统设置中动态调整 | `5` | diff --git a/public/src/locales/en.json b/public/src/locales/en.json index c3df5599..a7eadbda 100644 --- a/public/src/locales/en.json +++ b/public/src/locales/en.json @@ -118,7 +118,7 @@ "forceRefreshAllConfirm": "Are you sure you want to force refresh the tokens of all accounts? \nThis will refresh all accounts, regardless of whether they are about to expire, and may take a long time.", "forceRefreshAllFailed": "Forced refresh failed:", "progressRetry": "Progress query failed, retrying...", - "proxyInvalid": "Format error: must start with http://, https:// or socks5://", + "proxyInvalid": "Format error: must start with http://, https://, socks5:// or socks5h://", "proxyUpdateFailed": "Agent update failed:", "proxyUpdateSuccess": "Agent updated successfully", "refreshAllComplete": "Batch refresh completed, {n} accounts were successfully refreshed", diff --git a/public/src/locales/ru.json b/public/src/locales/ru.json index ec2899f0..ec302a9e 100644 --- a/public/src/locales/ru.json +++ b/public/src/locales/ru.json @@ -120,7 +120,7 @@ "forceRefreshAllFailed": "Ошибка принудительного обновления: ", "proxyUpdateSuccess": "Proxy обновлён", "proxyUpdateFailed": "Ошибка обновления proxy: ", - "proxyInvalid": "Неверный формат: ожидается http://, https:// или socks5://" + "proxyInvalid": "Неверный формат: ожидается http://, https://, socks5:// или socks5h://" }, "settings": { "title": "Настройки", diff --git a/public/src/locales/zh.json b/public/src/locales/zh.json index 8f968c5d..d917f7a0 100644 --- a/public/src/locales/zh.json +++ b/public/src/locales/zh.json @@ -120,7 +120,7 @@ "forceRefreshAllFailed": "强制刷新失败: ", "proxyUpdateSuccess": "代理更新成功", "proxyUpdateFailed": "代理更新失败: ", - "proxyInvalid": "格式错误:必须以 http://、https:// 或 socks5:// 开头" + "proxyInvalid": "格式错误:必须以 http://、https://、socks5:// 或 socks5h:// 开头" }, "settings": { "title": "系统设置", diff --git a/public/src/views/dashboard.vue b/public/src/views/dashboard.vue index 96bb1c02..9d08eaa2 100644 --- a/public/src/views/dashboard.vue +++ b/public/src/views/dashboard.vue @@ -576,7 +576,7 @@ const editProxy = ref({ email: '', proxy: '' }) const isSavingProxy = ref(false) // 与后端 src/utils/proxy-helper.js#PROXY_URL_REGEX 保持一致 -const PROXY_URL_REGEX = /^(https?|socks5):\/\/[^\s]+$/i +const PROXY_URL_REGEX = /^(https?|socks5h?):\/\/[^\s]+$/i const isValidProxy = (value) => { if (!value) return true const trimmed = String(value).trim() diff --git a/src/config/index.js b/src/config/index.js index bc51cb0c..5c3bf4c6 100644 --- a/src/config/index.js +++ b/src/config/index.js @@ -5,16 +5,26 @@ dotenv.config() * 解析API_KEY环境变量,支持逗号分隔的多个key * @returns {Object} 包含apiKeys数组和adminKey的对象 */ -const parseApiKeys = () => { - const apiKeyEnv = process.env.API_KEY +const parseApiKeys = (apiKeyEnv = process.env.API_KEY) => { if (!apiKeyEnv) { return { apiKeys: [], adminKey: null } } - const keys = apiKeyEnv.split(',').map(key => key.trim()).filter(key => key.length > 0) + const slots = apiKeyEnv.split(',').map(key => key.trim()) + // 空槽位 = 变量没展开(`API_KEY=${A},${B}` 里 A 未定义就变成 `,B`)。旧代码把空槽 + // 过滤掉照常启动:admin key 静默变成了后面那个客户端 key,dashboard 用真正的 admin + // key 反而报 "invalid"。2026-09-16 qwen-next 就是这样丢掉后台访问的。没人会故意写 + // 出空槽,所以这里直接拒绝启动,而不是带着残缺的密钥表跑下去。 + const emptyAt = slots.map((key, index) => (key.length === 0 ? index + 1 : 0)).filter(Boolean) + if (emptyAt.length > 0) { + throw new Error( + `API_KEY 有 ${emptyAt.length} 个空槽位(第 ${emptyAt.join('、')} 位,共 ${slots.length} 位)——` + + '多半是环境变量没有展开。请检查部署时是否注入了全部密钥(例如 compose 的 ${...} 是否有值)。' + ) + } return { - apiKeys: keys, - adminKey: keys.length > 0 ? keys[0] : null + apiKeys: slots, + adminKey: slots.length > 0 ? slots[0] : null } } @@ -137,4 +147,7 @@ const config = { ) } +// 暴露解析器本身以便测试空槽位的拒绝逻辑(config 对象是模块级单例,无法重复解析)。 +config.parseApiKeys = parseApiKeys + module.exports = config diff --git a/src/controllers/anthropic.js b/src/controllers/anthropic.js index d1f6344c..8ce770ac 100644 --- a/src/controllers/anthropic.js +++ b/src/controllers/anthropic.js @@ -57,10 +57,14 @@ const { logger } = require('../utils/logger'); const { assertNoUpstreamFailure, describeUpstreamFailure, + isRateLimitError, + isTransportInterruption, noteRateLimitedAccount, RATE_LIMIT_ANTHROPIC_TYPE, UpstreamResponseError } = require('../utils/upstream-error.js'); +const { describeEgress } = require('../utils/proxy-helper.js'); +const { recordFailedAccount } = require('../utils/agent-account-failover.js'); const { analyzeAnthropicCompatibility, buildAnthropicCompatibilityHeaders @@ -1209,6 +1213,44 @@ const runWithAnthropicPing = async (res, work, intervalMs) => { } }; +// Reintento por cuenta cuando el upstream se cae ANTES de que el cliente haya visto un solo +// bloque de contenido. Dos causas, una regla: +// - transporte: el socket se cerro a mitad del SSE (`UND_ERR_SOCKET: other side closed`, +// 3 de 70 peticiones en 8 h el 2026-09-16 en qwen-next; sing-box sin un solo error, asi +// que no se sabe si corto WARP o Qwen); +// - cuota: el primer frame util es `RateLimited` — la cuenta se pausa (recordFailedAccount) +// y otra sirve el turno en vez de devolverle al cliente un 429 a medio stream. +// El guard es "cero content_block emitidos": con uno ya en el cable, reenviar duplicaria +// texto en la pantalla del cliente, asi que ahi el error sigue saliendo como hasta ahora y +// el cliente (Claude Code) reintenta el. Una sola vuelta por peticion: la segunda caida +// consecutiva es senal, no ruido. +const MID_STREAM_FAILOVER_MAX_RETRIES = 1; + +const classifyMidStreamFailure = (error) => { + if (isRateLimitError(error)) return 'quota'; + if (isTransportInterruption(error)) return 'transport'; + return null; +}; + +/** + * Una linea por interrupcion, se reintente o no. Es la medida que faltaba: sin tasa de + * cierres por egress no hay forma de saber si WARP los empeora respecto a salir directo. + * `bytes`/`frames` los anota consumeSSEStream en el error; `proxy` nunca lleva credenciales + * (describeEgress). + */ +const logEgressInterruption = (kind, error, account, { emittedBlocks, action }) => { + logger.warn( + `mid_stream_${kind} code=${error?.code || 'unknown'}` + + ` bytes=${Number(error?.upstreamBytesRead) || 0}` + + ` frames=${Number(error?.upstreamEventCount) || 0}` + + ` emitted_blocks=${emittedBlocks}` + + ` account=${account?.email || 'none'}` + + ` proxy=${describeEgress(account)}` + + ` action=${action}`, + 'EGRESS' + ); +}; + /** * 处理流式 Anthropic 响应 * @param {object} res - Express 响应 @@ -1262,8 +1304,6 @@ const handleAnthropicStream = async (res, ctx, upstream) => { let textBlockOpen = false; let thinkingBlockOpen = false; let thinkingSignature = null; - let promptTokens = 0; - let completionTokens = 0; let upstreamUsage = null; // 上游逐帧累计的 usage(DashScope 命名已归一化;null = 还没报) let upstreamFinishReason = null; let upstreamCompleted; @@ -1703,12 +1743,14 @@ const handleAnthropicStream = async (res, ctx, upstream) => { const maxAttempts = Math.max(1, Number(config.agentTurnMaxAttempts) || 1); let currentUpstream = upstream; + // Vueltas en las que el modelo llego a responder. Un failover no cuenta: el modelo aun no + // hablo, y el cupo de correccion de protocolo (maxAttempts) es para lo que SI dijo. let attemptsMade = 0; let retriedAfterVisibleText = false; let protocolRecoveryRetried = false; + let failoverRetries = 0; for (;;) { - attemptsMade += 1; startAttempt(); try { @@ -1719,9 +1761,53 @@ const handleAnthropicStream = async (res, ctx, upstream) => { upstreamCompleted = result.completed; upstreamEventCount = result.eventCount; } catch (e) { - logger.error('Anthropic 流式心跳包装失败', 'ANTHROPIC', '', e); - throw e; + const kind = classifyMidStreamFailure(e); + const emittedBlocks = blockIndex + 1; + const canFailover = kind !== null + && emittedBlocks === 0 + && failoverRetries < MID_STREAM_FAILOVER_MAX_RETRIES + && !res.writableEnded && !res.destroyed; + if (kind) { + logEgressInterruption(kind, e, ctx.currentAccount, { + emittedBlocks, + action: canFailover ? 'failover' : 'deliver_error' + }); + } + if (!canFailover) { + // Quien servia cuando se cayo, para que el catch del handler marque ESA cuenta si el + // error es de cuota (tras un failover ya no es la del sorteo inicial). Solo el email: + // el error se loguea y no debe arrastrar el token. Mismo campo que recordFailedAccount. + if (e && typeof e === 'object' && !e.failedAccountEmail) { + e.failedAccountEmail = ctx.currentAccount?.email || null; + } + logger.error('Anthropic 流式心跳包装失败', 'ANTHROPIC', '', e); + throw e; + } + + failoverRetries += 1; + const failedEmail = ctx.currentAccount?.email || null; + // Cuota: pausa la cuenta antes de sortear otra. Transporte: solo anota el email; un + // cierre a mitad de stream no es culpa de la cuenta y no debe acercarla al cooldown. + recordFailedAccount(e, ctx.currentAccount); + let retryResp = null; + try { + await runWithAnthropicPing(res, async () => { + retryResp = await sendRequest(requestBody, { + ...upstreamOptions, + excludeEmails: failedEmail ? [failedEmail] : [] + }); + }); + } catch (retryError) { + logger.error('Anthropic 流式 failover 重试失败', 'ANTHROPIC', '', retryError); + throw retryError.publicMessage ? retryError : e; + } + if (!retryResp?.status || !retryResp.response) throw e; + currentUpstream = retryResp.response; + // Stats y un eventual 429 posterior se atribuyen a quien sirvio de verdad. + if (retryResp.currentAccount) ctx.currentAccount = retryResp.currentAccount; + continue; } + attemptsMade += 1; // 本轮收尾。解析器的尾巴属于这一轮,必须在判定之前放出来。文本通道截断之后例外: // 根本不 flush —— 解析器里压着的只是失控那一 push 的残余(半个触发器 / 半截负载), @@ -2001,8 +2087,8 @@ const handleAnthropicStream = async (res, ctx, upstream) => { // 只对上游没报的字段补本地估算(早停的回合收不到尾部 usage 帧) const usage = reportUsage(upstreamUsage, () => createUsageObject(requestBody?.messages || '', completionContent), 'ANTHROPIC'); - promptTokens = usage.prompt_tokens; - completionTokens = usage.completion_tokens; + const promptTokens = usage.prompt_tokens; + const completionTokens = usage.completion_tokens; // Daily stats 累计——一次性归属主账户(见模块顶部 attributeChatUsage 注释) attributeChatUsage(ctx.currentAccount, promptTokens, completionTokens); @@ -2041,8 +2127,6 @@ const handleAnthropicNonStream = async (res, ctx, upstream) => { // 与流式分支同一条纪律,上一轮的泄漏已经重试过了。 let attemptThinkingContent = ''; let answerContent = ''; - let promptTokens = 0; - let completionTokens = 0; let upstreamUsage = null; // 上游逐帧累计的 usage(DashScope 命名已归一化;null = 还没报) let webSearchInfo = null; let upstreamFinishReason = null; @@ -2577,8 +2661,8 @@ const handleAnthropicNonStream = async (res, ctx, upstream) => { const nativeArgsText = nativeToolCalls.map(call => call.function.arguments || '').join(''); return createUsageObject(requestBody?.messages || '', thinkingContent + answerContent + nativeArgsText); }, 'ANTHROPIC'); - promptTokens = usage.prompt_tokens; - completionTokens = usage.completion_tokens; + const promptTokens = usage.prompt_tokens; + const completionTokens = usage.completion_tokens; const contentBlocks = []; if (thinkingContent && thinkingContent.trim()) { @@ -2706,7 +2790,16 @@ const handleAnthropicMessages = async (req, res) => { : (failure.overloaded ? 'overloaded_error' : 'api_error'); // La otra mitad: sin esto el cliente deja de reintentar pero el servidor sigue // devolviendo la misma cuenta agotada al sorteo, y la quema en cada vuelta. - noteRateLimitedAccount(error, currentAccount); + // Si el failover a mitad de stream ya paso la cuenta a cooldown (recordFailedAccount) + // y luego no hubo otra cuenta a la que saltar, el error que llega aqui es el mismo: + // no se marca dos veces. Tras un failover la cuenta que fallo no es la del sorteo + // inicial: el stream la deja en error.failedAccountEmail. Gemelo: chat.js. + if (!error?.accountFailureRecorded) { + noteRateLimitedAccount( + error, + error?.failedAccountEmail ? { email: error.failedAccountEmail } : currentAccount + ); + } // Un prefijo de historial reutilizado pudo ser la causa (file_id que Qwen ya no // reconoce): se olvida y el reintento del cliente hornea uno nuevo. Un 529 por // ContextExternalizationError nunca llega aqui con contextPrefixReused. diff --git a/src/routes/accounts.js b/src/routes/accounts.js index 19ce0c93..3d2ab03e 100644 --- a/src/routes/accounts.js +++ b/src/routes/accounts.js @@ -11,7 +11,7 @@ const { isValidProxyUrl } = require('../utils/proxy-helper') const { DEFAULT_CLI_QUOTA_LIMIT, getAccountCliState } = require('../utils/cli-support') // 仅在 proxy 字段存在时触发;空字符串/null 一律视为"清除代理",无需校验 -const PROXY_FORMAT_ERROR = '代理 URL 格式无效,应以 http://、https:// 或 socks5:// 开头' +const PROXY_FORMAT_ERROR = '代理 URL 格式无效,应以 http://、https://、socks5:// 或 socks5h:// 开头' const batchAccountTasks = new Map() const BATCH_TASK_RETENTION_MS = 1000 * 60 * 30 diff --git a/src/utils/proxy-helper.js b/src/utils/proxy-helper.js index 23f59d6e..5ec0fc92 100644 --- a/src/utils/proxy-helper.js +++ b/src/utils/proxy-helper.js @@ -1,7 +1,7 @@ const { once } = require('node:events'); const { STATUS_CODES } = require('node:http'); const { isIP } = require('node:net'); -const { PassThrough, Readable } = require('node:stream'); +const { PassThrough } = require('node:stream'); const { checkServerIdentity } = require('node:tls'); const axios = require('axios'); const { HttpsProxyAgent } = require('https-proxy-agent'); @@ -19,6 +19,14 @@ const MAX_AGENT_CACHE_SIZE = 50 const PROXY_CONNECT_TIMEOUT_MS = 10_000; const proxyUrls = new WeakMap(); const proxyTransports = new WeakMap(); +// undici transport failures mapped onto the codes request.js already retries / cools down on. +const UNDICI_ERROR_CODES = { + UND_ERR_SOCKET: 'ECONNRESET', + UND_ERR_CONNECT_TIMEOUT: 'ETIMEDOUT', + UND_ERR_HEADERS_TIMEOUT: 'ECONNABORTED', + UND_ERR_BODY_TIMEOUT: 'ECONNABORTED', + UND_ERR_ABORTED: 'ERR_CANCELED' +}; /** * Reuse the account's SOCKS/CONNECT implementation with an HTTP client that honors sockets. @@ -66,6 +74,11 @@ const getProxyTransport = (proxyAgent) => { } }); const requestDispatcher = dispatcher.compose(interceptors.redirect({ maxRedirections: 20 }), interceptors.decompress()); + // Never hand a Readable.toWeb/fromWeb-bridged body to a consumer: under Bun the bridge drops + // undici's body error on a mid-body close, so the consumer hangs and the rejection escapes as a + // process crash (qwen-next died 6x in 2.5 h on 2026-09-16; reproduced with a proxy that closes + // the socket mid-body). + // Streams are returned as undici's own Node Readable; everything else is buffered right here. const fetch = async (url, options) => { const request = new globalThis.Request(url, options); // Keep buffered upstream payloads replayable across 307/308 redirects. @@ -80,34 +93,52 @@ const getProxyTransport = (proxyAgent) => { }); const noBody = request.method === 'HEAD' || [204, 205, 304].includes(response.statusCode); if (noBody) await response.body.dump(); - // Bun stalls on Undici fetch's WebStream bridge; its native Response streams correctly. - try { - return new globalThis.Response(noBody ? null : Readable.toWeb(response.body), { - status: response.statusCode, - statusText: STATUS_CODES[response.statusCode] || '', - headers: response.headers - }); - } catch (error) { - response.body.destroy(); - throw error; - } + const payload = noBody ? null : Buffer.from(await response.body.arrayBuffer()); + return new globalThis.Response(payload, { + status: response.statusCode, + statusText: STATUS_CODES[response.statusCode] || '', + headers: response.headers + }); + }; + const readResponseData = (response, responseType) => { + if (responseType === 'stream') return response.body; + if (responseType === 'arraybuffer') return response.body.arrayBuffer().then((buffer) => Buffer.from(buffer)); + // json/text/undefined: axios' transformResponse parses the text, exactly as with the Node adapter. + return response.body.text(); }; const adapter = async (requestConfig) => { - const adaptedConfig = { - ...requestConfig, - env: { ...requestConfig.env, fetch, Request: globalThis.Request, Response: globalThis.Response } - }; + let response; try { - const response = await axios.getAdapter('fetch', adaptedConfig)(adaptedConfig); - // Existing SSE consumers rely on Node streams for both success and error bodies. - if (requestConfig.responseType === 'stream') response.data = Readable.fromWeb(response.data); - return response; + response = await proxyRequest(axios.getUri(requestConfig), { + dispatcher: requestDispatcher, + method: String(requestConfig.method || 'get').toUpperCase(), + headers: axios.AxiosHeaders.from(requestConfig.headers).toJSON(), + body: requestConfig.data, + signal: requestConfig.signal, + headersTimeout: requestConfig.timeout || undefined, + bodyTimeout: requestConfig.timeout || undefined, + maxRedirections: 20 + }); } catch (error) { - if (requestConfig.responseType === 'stream' && error.response?.data?.getReader) { - error.response.data = Readable.fromWeb(error.response.data); - } - throw error; + throw axios.AxiosError.from(error, UNDICI_ERROR_CODES[error.code] || error.code || axios.AxiosError.ERR_NETWORK, requestConfig); } + const axiosResponse = { + data: await readResponseData(response, requestConfig.responseType), + status: response.statusCode, + statusText: STATUS_CODES[response.statusCode] || '', + headers: axios.AxiosHeaders.from(response.headers), + config: requestConfig, + request: null + }; + const { validateStatus } = requestConfig; + if (!validateStatus || validateStatus(axiosResponse.status)) return axiosResponse; + throw new axios.AxiosError( + `Request failed with status code ${axiosResponse.status}`, + axiosResponse.status >= 500 ? axios.AxiosError.ERR_BAD_RESPONSE : axios.AxiosError.ERR_BAD_REQUEST, + requestConfig, + null, + axiosResponse + ); }; transport = { dispatcher, fetch, adapter }; proxyTransports.set(proxyAgent, transport); @@ -125,8 +156,13 @@ const destroyProxyAgent = (agent) => { agent.destroy(); }; -// Accept http/https/socks5 protocols; regex intentionally loose to catch common typos only -const PROXY_URL_REGEX = /^(https?|socks5):\/\/[^\s]+$/i +// Accept http/https/socks5/socks5h; regex intentionally loose to catch common typos only. +// socks5:// resolves the target hostname LOCALLY and hands the proxy an IP (socks-proxy-agent +// sets `lookup = true`), so the DNS query leaves through the host's resolver while the TCP +// goes through the proxy. socks5h:// delegates resolution to the proxy (curl semantics): +// DNS and TCP share one egress. Measured on qwen-next 2026-09-16: 76/80 upstream connections +// reached sing-box as bare IPs under socks5://. +const PROXY_URL_REGEX = /^(https?|socks5h?):\/\/[^\s]+$/i /** * Validate proxy URL format. @@ -155,6 +191,24 @@ const resolveProxyUrl = (account) => { return config.proxyUrl || null } +/** + * Egress identity for logs: `protocol//host:port` of the proxy an account's + * requests leave through, or 'direct' when none applies. Qwen's WAF judges by + * egress IP, not by account, so parse failures name this instead of the account. + * Credentials never make it out: the WHATWG parser drops userinfo, and the regex + * fallback (unparseable URL) masks up to the last `@` of the authority. + */ +const describeEgress = (account) => { + const url = resolveProxyUrl(account) + if (!url) return 'direct' + try { + const parsed = new URL(url) + return `${parsed.protocol}//${parsed.host}` + } catch { + return url.replace(/\/\/[^/?#]*@/, '//***@') + } +} + /** * Evict oldest entry from agent cache when over limit. * Map iteration order is insertion order, so first key is oldest. @@ -195,6 +249,8 @@ const getOrCreateAgent = (url, account) => { const proxyUrl = new URL(url); switch (proxyUrl.protocol) { case 'socks5:': + case 'socks5h:': + // The agent reads the scheme itself: socks5h → shouldLookup=false. agent = new SocksProxyAgent(proxyUrl); break; case 'http:': @@ -292,11 +348,13 @@ const fetchWithProxy = (url, fetchOptions = {}, account) => { module.exports = { resolveProxyUrl, + describeEgress, getProxyAgent, invalidateProxyAgent, getChatBaseUrl, getCliBaseUrl, applyProxyToAxiosConfig, + getProxyTransport, fetchWithProxy, isValidProxyUrl } diff --git a/src/utils/request.js b/src/utils/request.js index ac69d75b..ccbec424 100644 --- a/src/utils/request.js +++ b/src/utils/request.js @@ -7,28 +7,22 @@ const { applyProxyToAxiosConfig, getChatBaseUrl } = require('./proxy-helper'); const { generateUUID, jitter } = require('./tools.js') const { uploadAgentContextFile, buildChatFileDescriptor } = require('./upload.js') const { buildRequestHeaders } = require('./header-profile') -const { ContextExternalizationError } = require('./upstream-error.js') +const { ContextExternalizationError, isTransportInterruption } = require('./upstream-error.js') const { contextPrefixCache, prefixMatches, canonicalHistoryHash } = require('./context-prefix-cache.js') const { TOOL_CALL_OPEN, LEDGER_HEADER, LEDGER_CAPTION, truncateToolHistoryLedger, stripRetainedThinking } = require('./agent-turn.js') -// 传输层(非 HTTP)错误码 — 这些重试的, HTTP 响应不重试 -const RETRYABLE_ERROR_CODES = new Set([ - 'ECONNRESET', - 'ECONNREFUSED', - 'ETIMEDOUT', - 'ECONNABORTED', - 'EAI_AGAIN' -]) +// 连接阶段失败(拒绝 / DNS 暂时不可用)。传输中断(socket 关闭、超时)的判定在 +// upstream-error.js#isTransportInterruption,这里只补连接前的两个码,不再各自维护一份。 +const CONNECT_FAILURE_CODES = new Set(['ECONNREFUSED', 'EAI_AGAIN']) const isRetryableNetworkError = (error) => { - if (!error) return false + if (!error || typeof error !== 'object') return false + if (isTransportInterruption(error)) return true // 已收到 HTTP 响应 = 上游回包了, 不是传输问题 if (error.response) return false - if (error.code && RETRYABLE_ERROR_CODES.has(error.code)) return true - if (typeof error.message === 'string' && error.message.includes('socket hang up')) return true - return false + return CONNECT_FAILURE_CODES.has(String(error.code || error.cause?.code || '')) } const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms)) @@ -649,9 +643,12 @@ const externalizeOversizedAgentContext = async ( */ const sendChatRequest = async (body, options = {}) => { // 获取可用的账户(包含 proxy 等完整字段) + // excludeEmails:本次 HTTP 请求里已经烧掉的账户(流中途 failover)——轮换器跳过它们, + // 即使它们对其他请求仍然可用。 + const excludeEmails = Array.isArray(options.excludeEmails) ? options.excludeEmails : [] const currentAccount = options.currentAccount?.token ? options.currentAccount - : accountManager.getAccount() + : accountManager.getAccount(excludeEmails) const currentToken = currentAccount ? currentAccount.token : null if (!currentToken) { @@ -884,5 +881,6 @@ module.exports = { compactAgentContextFallback, externalizeOversizedAgentContext, buildPrefixReusePrompt, - invalidateContextPrefix + invalidateContextPrefix, + isRetryableNetworkError } diff --git a/src/utils/sse.js b/src/utils/sse.js index fcf3955c..4b2ef67e 100644 --- a/src/utils/sse.js +++ b/src/utils/sse.js @@ -154,6 +154,7 @@ const consumeSSEStream = async (stream, onFrame, options = {}) => { const decoder = new SSEDecoder() let sawDone = false let eventCount = 0 + let bytesRead = 0 let stopped = false const consumeFrames = async (frames) => { @@ -168,16 +169,28 @@ const consumeSSEStream = async (stream, onFrame, options = {}) => { } } - for await (const chunk of stream) { - await consumeFrames(decoder.push(chunk)) - if (stopped) { - if (typeof stream.on === 'function') stream.on('error', () => {}) - break + try { + for await (const chunk of stream) { + bytesRead += typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length + await consumeFrames(decoder.push(chunk)) + if (stopped) { + if (typeof stream.on === 'function') stream.on('error', () => {}) + break + } + } + if (!stopped) await consumeFrames(decoder.end()) + } catch (error) { + // 中途失败带上"上游走到了哪":控制器据此判断重试是否安全(已转发多少), + // egress 日志据此区分早断(几 KB)与晚断(几十 KB)。onFrame 抛出的业务错误 + // (RateLimited 帧)同样经过这里,同样标注。 + if (error && typeof error === 'object') { + error.upstreamBytesRead = bytesRead + error.upstreamEventCount = eventCount } + throw error } - if (!stopped) await consumeFrames(decoder.end()) - return { sawDone, eventCount, completed: true, stopped } + return { sawDone, eventCount, bytesRead, completed: true, stopped } } /** diff --git a/src/utils/upload.js b/src/utils/upload.js index 15827ce1..995a709a 100644 --- a/src/utils/upload.js +++ b/src/utils/upload.js @@ -3,7 +3,7 @@ const OSS = require('ali-oss') const mimetypes = require('mime-types') const { logger } = require('./logger') const { generateUUID } = require('./tools.js') -const { getProxyAgent, getChatBaseUrl, applyProxyToAxiosConfig } = require('./proxy-helper') +const { getProxyAgent, getChatBaseUrl, applyProxyToAxiosConfig, describeEgress } = require('./proxy-helper') const { buildRequestHeaders } = require('./header-profile') const config = require('../config/index.js') @@ -391,17 +391,22 @@ const parseServiceFailureCode = (response) => { return null } -const throwIfParseServiceFailed = (response, fileId) => { +// `egress` names the proxy (or 'direct') the parse left through. The WAF +// challenge is per egress IP, so this is the field that tells one burnt proxy +// apart from a Qwen-side outage. +const throwIfParseServiceFailed = (response, fileId, egress = 'unknown') => { const code = parseServiceFailureCode(response) if (code === null) return - const error = new Error(`Qwen 文档解析服务失败: ${code} (${fileId})`) + const error = new Error(`Qwen 文档解析服务失败: ${code} (${fileId}, via ${egress})`) error.code = code === WAF_CAPTCHA_CODE ? 'qwen_parse_waf_challenge' : 'qwen_parse_unavailable' error.parseCode = code + error.egress = egress throw error } const parseUploadedTextFile = async (fileId, authToken, account, options = {}) => { if (!fileId || !authToken) throw new Error('解析文档缺少 fileId 或认证 Token') + const egress = describeEgress(account) const baseUrl = getChatBaseUrl() const requestConfig = applyProxyToAxiosConfig({ @@ -409,19 +414,29 @@ const parseUploadedTextFile = async (fileId, authToken, account, options = {}) = timeout: Math.max(1000, Number(options.timeoutMs) || 30000) }, account) - const parseResponse = await axios.post(`${baseUrl}/api/v2/files/parse`, { file_id: fileId }, requestConfig) - throwIfParseServiceFailed(parseResponse, fileId) + // Transport failures (dead proxy, reset, timeout) never reach the JSON checks in + // throwIfParseServiceFailed; tag them with the egress too, or a burnt proxy logs as a bare ECONNRESET. + const postViaEgress = async (path, body) => { + try { + return await axios.post(`${baseUrl}${path}`, body, requestConfig) + } catch (error) { + if (error && typeof error === 'object' && !error.egress) { + error.egress = egress + error.message = `${error.message} (${fileId}, via ${egress})` + } + throw error + } + } + + const parseResponse = await postViaEgress('/api/v2/files/parse', { file_id: fileId }) + throwIfParseServiceFailed(parseResponse, fileId, egress) const maxAttempts = Math.max(1, Number(options.maxAttempts) || 30) const intervalMs = Math.max(50, Number(options.intervalMs) || 500) let lastStatus = '' for (let attempt = 1; attempt <= maxAttempts; attempt++) { - const response = await axios.post( - `${baseUrl}/api/v2/files/parse/status`, - { file_id_list: [fileId] }, - requestConfig - ) - throwIfParseServiceFailed(response, fileId) + const response = await postViaEgress('/api/v2/files/parse/status', { file_id_list: [fileId] }) + throwIfParseServiceFailed(response, fileId, egress) const payload = unwrapApiData(response) const records = Array.isArray(payload) ? payload : (payload?.list || payload?.items || []) const record = records.find(item => item?.file_id === fileId) || records[0] @@ -519,16 +534,18 @@ const noteParseOutcome = (error) => { if (cooldownSeconds > 0 && parseBreaker.strikes >= PARSE_BREAKER_STRIKES) { parseBreaker.openUntil = nowMs() + cooldownSeconds * 1000 error.retryAfterSeconds = cooldownSeconds - logger.warn(`Agent 上下文解析被 WAF 连续拦截 ${parseBreaker.strikes} 次,${cooldownSeconds}s 内不再上传`, 'UPLOAD') + logger.warn(`Agent 上下文解析被 WAF 连续拦截 ${parseBreaker.strikes} 次,${cooldownSeconds}s 内不再上传 (egress ${error.egress || 'unknown'})`, 'UPLOAD') } } -const assertParseBreakerClosed = () => { +const assertParseBreakerClosed = (account) => { const remaining = parseBreakerRemainingSeconds() if (remaining <= 0) return - const error = new Error(`Qwen 文档解析服务失败: ${WAF_CAPTCHA_CODE} (breaker open, ${remaining}s left, upload skipped)`) + const egress = describeEgress(account) + const error = new Error(`Qwen 文档解析服务失败: ${WAF_CAPTCHA_CODE} (breaker open, ${remaining}s left, upload skipped, via ${egress})`) error.code = 'qwen_parse_waf_challenge' error.parseCode = WAF_CAPTCHA_CODE + error.egress = egress error.retryAfterSeconds = remaining error.breakerOpen = true throw error @@ -550,7 +567,7 @@ const parseWindow = [] const resetParseRateLimiter = () => { parseWindow.length = 0 } -const takeParseSlot = () => { +const takeParseSlot = (account) => { const max = Math.max(0, parseInt(config.agentParseMaxPerWindow, 10) || 0) if (max <= 0) return const windowSeconds = Math.max(1, parseInt(config.agentParseWindowSeconds, 10) || 120) @@ -563,10 +580,12 @@ const takeParseSlot = () => { } const untilFree = Math.ceil((parseWindow[0] + windowMs - now) / 1000) const retryAfter = Math.min(PARSE_RATE_RETRY_MAX_SECONDS, Math.max(PARSE_RATE_RETRY_MIN_SECONDS, untilFree)) - logger.warn(`Agent 上下文解析已达速率上限 (${max}/${windowSeconds}s),${retryAfter}s 后重试`, 'UPLOAD') - const error = new Error(`Qwen 文档解析服务失败: ${PARSE_RATE_LIMITED_CODE} (${max}/${windowSeconds}s reached, upload skipped)`) + const egress = describeEgress(account) + logger.warn(`Agent 上下文解析已达速率上限 (${max}/${windowSeconds}s),${retryAfter}s 后重试 (egress ${egress})`, 'UPLOAD') + const error = new Error(`Qwen 文档解析服务失败: ${PARSE_RATE_LIMITED_CODE} (${max}/${windowSeconds}s reached, upload skipped, via ${egress})`) error.code = 'qwen_parse_rate_limited' error.parseCode = PARSE_RATE_LIMITED_CODE + error.egress = egress error.retryAfterSeconds = retryAfter error.breakerOpen = false throw error @@ -575,8 +594,8 @@ const takeParseSlot = () => { const uploadAgentContextFile = async (text, authToken, account, options = {}) => { const content = Buffer.from(String(text || ''), 'utf8') if (content.length === 0) throw new Error('Agent 上下文为空') - assertParseBreakerClosed() - takeParseSlot() + assertParseBreakerClosed(account) + takeParseSlot(account) const filename = options.filename || `QWEN2API_AGENT_CONTEXT_${Date.now()}.txt` const uploaded = await uploadFileToQwenOss(content, filename, authToken, account) try { diff --git a/src/utils/upstream-error.js b/src/utils/upstream-error.js index 29c4eec6..553e38a4 100644 --- a/src/utils/upstream-error.js +++ b/src/utils/upstream-error.js @@ -46,6 +46,29 @@ const isRateLimitError = (error) => { return RATE_LIMIT_MESSAGE_RE.test(String(error.publicMessage || error.message || '')); }; +/** + * ¿El transporte se cayo antes o a mitad de la respuesta? Cierres de socket y timeouts tal + * y como los emiten Node (`ECONNRESET`), undici (`UND_ERR_SOCKET: other side closed` — + * 3 de 70 peticiones en 8 h el 2026-09-16 en qwen-next, con 63–90 KiB ya escritos) y los + * streams (`ERR_STREAM_PREMATURE_CLOSE`). NO incluye la cancelacion del propio cliente + * (`ERR_CANCELED` / AbortError): si quien corto fue el cliente no hay a quien reintentarle. + * Tampoco un error con respuesta HTTP: eso lo decidio el upstream, no la red. + */ +const TRANSPORT_INTERRUPTION_CODES = new Set([ + 'ECONNRESET', 'ECONNABORTED', 'ETIMEDOUT', 'EPIPE', + 'UND_ERR_SOCKET', 'UND_ERR_BODY_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT', + 'ERR_STREAM_PREMATURE_CLOSE' +]); +const TRANSPORT_INTERRUPTION_MESSAGE_RE = /other side closed|socket hang up|premature close/i; +const isTransportInterruption = (error) => { + if (!error || typeof error !== 'object') return false; + if (error.response) return false; + if (error.name === 'AbortError' || error.code === 'ERR_CANCELED') return false; + const code = String(error.code || error.cause?.code || ''); + if (TRANSPORT_INTERRUPTION_CODES.has(code)) return true; + return TRANSPORT_INTERRUPTION_MESSAGE_RE.test(String(error.message || error.cause?.message || '')); +}; + /** * El adjunto de contexto largo (upload + parse en Qwen) fallo en una peticion que NO * puede compactarse (lleva tools). Es una averia temporal del upstream —el servicio de @@ -225,6 +248,7 @@ module.exports = { assertNoUpstreamFailure, isRateLimitError, isWafChallengeError, + isTransportInterruption, rateLimitRetryAfterSeconds, describeUpstreamFailure, noteRateLimitedAccount, diff --git a/tests/anthropic-midstream-failover.test.js b/tests/anthropic-midstream-failover.test.js new file mode 100644 index 00000000..f7b238af --- /dev/null +++ b/tests/anthropic-midstream-failover.test.js @@ -0,0 +1,215 @@ +// Failover por cuenta cuando el upstream se cae ANTES del primer bloque de contenido. +// +// Medido en qwen-next el 2026-09-16: 3 de 70 peticiones terminaron en +// `UND_ERR_SOCKET: other side closed` a mitad del SSE y 1 en `RateLimited` en el primer +// frame. Las cuatro salieron al cliente como error a medio stream aunque todavia no habia +// visto nada. Con cero content_block emitidos reenviar es seguro (no duplica texto), asi +// que el handler sortea otra cuenta y sigue; con uno ya en el cable, el error sale como +// siempre y el cliente reintenta el. +// Misma cabecera que tests/agent-account-failover.test.js: sin API_KEY el arranque corta el +// proceso antes de que el runner reporte, y sin DATA_SAVE_MODE=none account.js intentaria +// un login real al importarse. +process.env.API_KEY = 'midstream-failover-test-key' +process.env.DATA_SAVE_MODE = 'none' +process.env.ACCOUNTS = '' +process.env.ENABLE_CLI = 'false' +process.env.ENABLE_FILE_LOG = 'false' +process.env.PROXY_URL = '' + +const { describe, it, after } = require('node:test') +const assert = require('node:assert/strict') +const { Readable } = require('node:stream') + +const accountManager = require('../src/utils/account') +const { handleAnthropicStream } = require('../src/controllers/anthropic.js') +const { isTransportInterruption } = require('../src/utils/upstream-error.js') +const { isRetryableNetworkError } = require('../src/utils/request.js') + +after(() => { accountManager.destroy() }) + +const createMockResponse = () => ({ + output: '', + headers: {}, + writableEnded: false, + destroyed: false, + set(headers) { Object.assign(this.headers, headers); return this }, + status() { return this }, + write(chunk) { this.output += String(chunk); return true }, + end(chunk = '') { this.output += String(chunk); this.writableEnded = true } +}) + +const frame = (content) => + `data: ${JSON.stringify({ choices: [{ delta: { phase: 'answer', content }, finish_reason: null }] })}\n\n` +const DONE = 'data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n' +// El paquete real de cuota (tests/upstream-quota-429.test.js lo fija). +const QUOTA = `data: ${JSON.stringify({ + success: false, + data: { code: 'RateLimited', details: "You've reached the upper limit for today's usage." } +})}\n\n` + +const socketClose = () => Object.assign(new Error('other side closed'), { code: 'UND_ERR_SOCKET' }) +const streamOf = (chunks, failure = null) => Readable.from((async function* () { + for (const chunk of chunks) yield chunk + if (failure) throw failure +})()) + +const emittedText = (output) => + [...output.matchAll(/"type":"text_delta","text":("(?:[^"\\]|\\.)*")/g)].map(m => JSON.parse(m[1])).join('') +const errorEvents = (output) => output.match(/^event: error$/gm) || [] + +const FIRST = { email: 'first@example.com', proxy: 'socks5h://lohari-warp-qwen:9091' } +const SECOND = { email: 'second@example.com', proxy: 'socks5h://lohari-warp-qwen:9091' } + +const recordingSendRequest = (respond) => { + const calls = [] + const fn = async (body, options) => { + calls.push({ body, options }) + return respond(calls.length) + } + fn.calls = calls + return fn +} +const recovered = () => ({ status: true, response: streamOf([frame('Recovered.'), DONE]), currentAccount: { ...SECOND } }) + +const ctxFor = (sendRequest, overrides = {}) => ({ + message_id: 'msg_failover', + model: 'qwen-test', + hasTools: false, + toolChoice: 'auto', + requestBody: { messages: [{ role: 'user', content: 'hola' }] }, + currentAccount: { ...FIRST }, + upstreamOptions: { allowContextCompaction: true, contextPrefixKey: 'session-k1' }, + sendRequest, + ...overrides +}) + +describe('mid-stream failover before the first content block', () => { + it('a socket close with nothing emitted is served by another account, transparently', async () => { + const sendRequest = recordingSendRequest(() => recovered()) + const res = createMockResponse() + const ctx = ctxFor(sendRequest) + + await handleAnthropicStream(res, ctx, streamOf([], socketClose())) + + assert.equal(emittedText(res.output), 'Recovered.') + assert.equal(errorEvents(res.output).length, 0, 'the client never sees the failure') + assert.match(res.output, /event: message_stop/) + assert.equal(sendRequest.calls.length, 1) + const [{ body, options }] = sendRequest.calls + assert.deepEqual(body, ctx.requestBody, 'the same internal body is replayed, no hint appended') + assert.deepEqual(options.excludeEmails, ['first@example.com'], 'the account that failed is skipped') + assert.equal(options.contextPrefixKey, 'session-k1', 'the uploaded history prefix is reused, not re-parsed') + assert.equal(options.allowContextCompaction, true) + assert.equal(ctx.currentAccount.email, 'second@example.com', 'stats and later 429s go to who actually served') + }) + + it('a RateLimited first frame rotates to another account instead of a 429 mid-stream', async () => { + const sendRequest = recordingSendRequest(() => recovered()) + const res = createMockResponse() + const ctx = ctxFor(sendRequest) + + await handleAnthropicStream(res, ctx, streamOf([QUOTA])) + + assert.equal(emittedText(res.output), 'Recovered.') + assert.equal(errorEvents(res.output).length, 0) + assert.equal(sendRequest.calls.length, 1) + assert.deepEqual(sendRequest.calls[0].options.excludeEmails, ['first@example.com']) + assert.equal(ctx.currentAccount.email, 'second@example.com') + }) + + it('a socket close AFTER a content block is delivered as an error: replaying would duplicate text', async () => { + const sendRequest = recordingSendRequest(() => recovered()) + const res = createMockResponse() + + await assert.rejects( + handleAnthropicStream(res, ctxFor(sendRequest), streamOf([frame('Half an ans')], socketClose())), + (err) => err.code === 'UND_ERR_SOCKET' && err.failedAccountEmail === 'first@example.com' + ) + assert.equal(sendRequest.calls.length, 0, 'no failover once the client has seen a block') + assert.equal(emittedText(res.output), 'Half an ans', 'what was already sent stays sent') + }) + + it('only one failover per request: a second consecutive close propagates', async () => { + const sendRequest = recordingSendRequest(() => ({ + status: true, response: streamOf([], socketClose()), currentAccount: { ...SECOND } + })) + const res = createMockResponse() + + await assert.rejects( + handleAnthropicStream(res, ctxFor(sendRequest), streamOf([], socketClose())), + (err) => err.code === 'UND_ERR_SOCKET' + && err.failedAccountEmail === 'second@example.com' + ) + assert.equal(sendRequest.calls.length, 1) + }) + + it('a failed replay surfaces the ORIGINAL failure when the retry carries no public message', async () => { + const sendRequest = recordingSendRequest(() => ({ status: false, message: 'no account' })) + const res = createMockResponse() + + await assert.rejects( + handleAnthropicStream(res, ctxFor(sendRequest), streamOf([], socketClose())), + (err) => err.code === 'UND_ERR_SOCKET' + ) + assert.equal(sendRequest.calls.length, 1) + }) + + it('no failover when the client is already gone', async () => { + const sendRequest = recordingSendRequest(() => recovered()) + const res = createMockResponse() + res.destroyed = true + + await assert.rejects( + handleAnthropicStream(res, ctxFor(sendRequest), streamOf([], socketClose())), + (err) => err.code === 'UND_ERR_SOCKET' + ) + assert.equal(sendRequest.calls.length, 0) + }) + + it('an unrelated error (a bug, not the transport) is not retried', async () => { + const sendRequest = recordingSendRequest(() => recovered()) + const res = createMockResponse() + const bug = new TypeError('cannot read properties of undefined') + + await assert.rejects( + handleAnthropicStream(res, ctxFor(sendRequest), streamOf([], bug)), + (err) => err === bug + ) + assert.equal(sendRequest.calls.length, 0) + }) +}) + +describe('isTransportInterruption', () => { + it('recognises socket closes and transport timeouts from Node, undici and streams', () => { + for (const code of ['ECONNRESET', 'ECONNABORTED', 'ETIMEDOUT', 'EPIPE', 'UND_ERR_SOCKET', + 'UND_ERR_BODY_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT', 'ERR_STREAM_PREMATURE_CLOSE']) { + assert.equal(isTransportInterruption(Object.assign(new Error('x'), { code })), true, code) + } + assert.equal(isTransportInterruption(new Error('other side closed')), true, 'undici message without code') + assert.equal(isTransportInterruption(new Error('socket hang up')), true) + assert.equal(isTransportInterruption(Object.assign(new TypeError('fetch failed'), { cause: { code: 'ECONNRESET' } })), true, 'fetch wraps the code in cause') + }) + + it('never treats the client cancelling, an HTTP response, or a business error as a transport failure', () => { + assert.equal(isTransportInterruption(Object.assign(new Error('canceled'), { code: 'ERR_CANCELED' })), false) + assert.equal(isTransportInterruption(Object.assign(new Error('aborted'), { name: 'AbortError' })), false) + assert.equal(isTransportInterruption(Object.assign(new Error('other side closed'), { response: { status: 502 } })), false) + assert.equal(isTransportInterruption(Object.assign(new Error('quota'), { code: 'RateLimited' })), false) + assert.equal(isTransportInterruption(null), false) + assert.equal(isTransportInterruption('ECONNRESET'), false) + }) +}) + +describe('isRetryableNetworkError (pre-response retry in sendRequest)', () => { + it('is isTransportInterruption plus the two connect-phase codes, with the same exclusions', () => { + for (const code of ['ECONNREFUSED', 'EAI_AGAIN', 'ECONNRESET', 'ETIMEDOUT', 'UND_ERR_SOCKET', 'EPIPE']) { + assert.equal(isRetryableNetworkError(Object.assign(new Error('x'), { code })), true, code) + } + assert.equal(isRetryableNetworkError(Object.assign(new TypeError('fetch failed'), { cause: { code: 'ECONNREFUSED' } })), true, 'connect code wrapped in cause') + assert.equal(isRetryableNetworkError(new Error('socket hang up')), true) + assert.equal(isRetryableNetworkError(Object.assign(new Error('refused'), { code: 'ECONNREFUSED', response: { status: 502 } })), false, 'an HTTP response is never a network error') + assert.equal(isRetryableNetworkError(Object.assign(new Error('canceled'), { code: 'ERR_CANCELED' })), false) + assert.equal(isRetryableNetworkError(Object.assign(new Error('quota'), { code: 'RateLimited' })), false) + assert.equal(isRetryableNetworkError(null), false) + }) +}) diff --git a/tests/config-api-key-slots.test.js b/tests/config-api-key-slots.test.js new file mode 100644 index 00000000..cfba5d98 --- /dev/null +++ b/tests/config-api-key-slots.test.js @@ -0,0 +1,58 @@ +// An unexpanded API_KEY must stop the boot, not silently demote the admin key. +// +// `docker-compose.next.yml` carries `API_KEY=${QWEN2API_ADMIN_KEY_V2},${QWEN2API_API_KEY}`. +// Deployed without the admin variable in scope it expands to `,sk-client…`; the old parser +// filtered the empty slot away and promoted the CLIENT key to admin, so the dashboard +// answered "invalid API key" to the real admin key. That is how qwen-next lost its +// dashboard on 2026-09-16 — a config bug that only surfaced as an auth failure. +process.env.API_KEY = process.env.API_KEY || 'test-only-key' + +const test = require('node:test') +const assert = require('node:assert/strict') + +const { parseApiKeys } = require('../src/config/index.js') + +test('an empty slot is refused: an unexpanded variable never boots', () => { + // The exact shape the broken deploy produced. + assert.throws(() => parseApiKeys(',sk-client'), /API_KEY/) + // Trailing and middle slots too, and whitespace-only counts as empty. + assert.throws(() => parseApiKeys('sk-admin,'), /API_KEY/) + assert.throws(() => parseApiKeys('sk-admin,,sk-client'), /API_KEY/) + assert.throws(() => parseApiKeys('sk-admin, ,sk-client'), /API_KEY/) +}) + +test('the error names how many slots are empty and where, so the fix is obvious', () => { + assert.throws(() => parseApiKeys(',sk-client'), (error) => { + assert.match(error.message, /第 1 位/) + assert.match(error.message, /共 2 位/) + return true + }) + assert.throws(() => parseApiKeys(',sk-client,'), (error) => { + assert.match(error.message, /第 1、3 位/) + return true + }) +}) + +test('an unset API_KEY still yields an empty config, not a throw', () => { + // Boot without any key is a separate, already-handled case (the server logs and refuses + // requests); only a PARTIALLY expanded value is the deploy bug this guards. + // `undefined` is not testable here: it falls through to the parameter default, which + // reads the real process.env.API_KEY this file has to set to import the module at all. + for (const value of ['', null]) { + assert.deepEqual(parseApiKeys(value), { apiKeys: [], adminKey: null }) + } +}) + +test('well-formed values keep parsing exactly as before: slot 1 is the admin key', () => { + assert.deepEqual(parseApiKeys('sk-admin,sk-client'), { + apiKeys: ['sk-admin', 'sk-client'], + adminKey: 'sk-admin' + }) + // Surrounding whitespace is trimmed, as it always was. + assert.deepEqual(parseApiKeys(' sk-admin , sk-client '), { + apiKeys: ['sk-admin', 'sk-client'], + adminKey: 'sk-admin' + }) + // A single key is both the only key and the admin key. + assert.deepEqual(parseApiKeys('sk-only'), { apiKeys: ['sk-only'], adminKey: 'sk-only' }) +}) diff --git a/tests/expected-counts.json b/tests/expected-counts.json index b070841b..095af32e 100644 --- a/tests/expected-counts.json +++ b/tests/expected-counts.json @@ -1,6 +1,6 @@ { - "tests": 1146, - "suites": 134, + "tests": 1181, + "suites": 137, "note": "Authoritative count. Verify with the per-file sum in AGENTS.md (\"The test gate\"). The -a on that grep is load-bearing: tool-prompt.test.js emits bytes that make grep call the stream binary, and without -a its whole summary line — 133 tests — is silently dropped from the sum.", "updated": "2026-09-16" } diff --git a/tests/proxy-egress-describe.test.js b/tests/proxy-egress-describe.test.js new file mode 100644 index 00000000..0333a50e --- /dev/null +++ b/tests/proxy-egress-describe.test.js @@ -0,0 +1,105 @@ +const test = require('node:test') +const assert = require('node:assert/strict') + +const config = require('../src/config/index.js') +const { describeEgress } = require('../src/utils/proxy-helper') +const { assertParseBreakerClosed, noteParseOutcome, resetParseBreaker } = require('../src/utils/upload') + +// The WAF challenge is per egress IP. Every parse failure names its egress so a +// burnt proxy can be told apart from a Qwen-side outage in one log line. + +const withGlobalProxy = (value, fn) => { + const saved = config.proxyUrl + config.proxyUrl = value + try { + return fn() + } finally { + config.proxyUrl = saved + } +} + +test('describeEgress: "direct" when neither the account nor PROXY_URL sets a proxy', () => { + withGlobalProxy(null, () => { + assert.equal(describeEgress({ email: 'a@example.com' }), 'direct') + assert.equal(describeEgress(null), 'direct') + assert.equal(describeEgress(undefined), 'direct') + }) +}) + +test('describeEgress: the account proxy wins over PROXY_URL and is trimmed', () => { + withGlobalProxy('http://global.example:8080', () => { + assert.equal( + describeEgress({ proxy: ' socks5://lohari-warp-qwen:9091 ' }), + 'socks5://lohari-warp-qwen:9091' + ) + }) +}) + +test('describeEgress: falls back to PROXY_URL when the account has no proxy', () => { + withGlobalProxy('socks5://127.0.0.1:1080', () => { + assert.equal(describeEgress({ proxy: '' }), 'socks5://127.0.0.1:1080') + assert.equal(describeEgress({}), 'socks5://127.0.0.1:1080') + }) +}) + +test('describeEgress: credentials are dropped, only protocol//host:port remains', () => { + withGlobalProxy(null, () => { + assert.equal( + describeEgress({ proxy: 'http://user:s3cr3t@proxy.example:3128' }), + 'http://proxy.example:3128' + ) + assert.equal( + describeEgress({ proxy: 'socks5://u:p@10.0.0.2:1080' }), + 'socks5://10.0.0.2:1080' + ) + }) +}) + +test('describeEgress: an "@" in the password, query or path never leaks or eats the host', () => { + withGlobalProxy(null, () => { + // password containing "@": userinfo ends at the LAST "@" of the authority + const leaky = describeEgress({ proxy: 'http://user:p@ss@proxy.example:3128' }) + assert.equal(leaky, 'http://proxy.example:3128') + assert.doesNotMatch(leaky, /ss/) + // "@" in the query with no path segment: no credentials, host must survive + assert.equal( + describeEgress({ proxy: 'http://proxy.example:3128?token=abc@def' }), + 'http://proxy.example:3128' + ) + // "@" in the path + assert.equal( + describeEgress({ proxy: 'socks5://proxy.example:1080/a@b/c' }), + 'socks5://proxy.example:1080' + ) + }) +}) + +test('describeEgress: an unparseable URL still gets its userinfo masked', () => { + withGlobalProxy(null, () => { + assert.throws(() => new URL('http://u:p@ss@[::1'), 'precondition: this shape must not parse') + assert.equal(describeEgress({ proxy: 'http://u:p@ss@[::1' }), 'http://***@[::1') + assert.equal(describeEgress({ proxy: 'http://[::1' }), 'http://[::1') + }) +}) + +test('breaker-open rejection names the egress like every other parse failure', () => { + const savedBreaker = config.agentParseBreakerSeconds + config.agentParseBreakerSeconds = 90 + resetParseBreaker() + try { + const waf = () => Object.assign(new Error('waf'), { code: 'qwen_parse_waf_challenge', parseCode: 'WAF_CAPTCHA' }) + for (let i = 0; i < 5; i++) noteParseOutcome(waf()) + assert.throws( + () => assertParseBreakerClosed({ proxy: 'socks5://u:p@lohari-warp-qwen:9091' }), + (error) => + error.breakerOpen === true && + error.parseCode === 'WAF_CAPTCHA' && + error.egress === 'socks5://lohari-warp-qwen:9091' && + /upload skipped, via socks5:\/\/lohari-warp-qwen:9091\)$/.test(error.message) && + !error.message.includes('u:p@') + ) + } finally { + resetParseBreaker() + config.agentParseBreakerSeconds = savedBreaker + } +}) diff --git a/tests/proxy-socks5h.test.js b/tests/proxy-socks5h.test.js new file mode 100644 index 00000000..771f846c --- /dev/null +++ b/tests/proxy-socks5h.test.js @@ -0,0 +1,46 @@ +const test = require('node:test') +const assert = require('node:assert/strict') + +const { isValidProxyUrl, getProxyAgent, describeEgress, invalidateProxyAgent } = require('../src/utils/proxy-helper') + +// socks5:// resolves the target hostname locally and hands the proxy an IP; socks5h:// +// lets the proxy resolve (curl semantics). On qwen-next (2026-09-16) 76 of 80 upstream +// connections reached sing-box as bare IPs: DNS for Qwen's domains was leaving through +// Hetzner's resolver while the TCP went through WARP. The `h` variant closes that gap, +// and the dashboard/route validators must let it through or the field cannot be saved. + +test('socks5h:// is a valid account proxy URL, alongside socks5/http/https', () => { + assert.equal(isValidProxyUrl('socks5h://lohari-warp-qwen:9091'), true) + assert.equal(isValidProxyUrl('socks5://lohari-warp-qwen:9091'), true) + assert.equal(isValidProxyUrl('SOCKS5H://x:1'), true) + assert.equal(isValidProxyUrl('http://x:1'), true) + assert.equal(isValidProxyUrl('socks4://x:1'), false) + assert.equal(isValidProxyUrl('socks5hh://x:1'), false) + assert.equal(isValidProxyUrl('socks5h:/x:1'), false) +}) + +test('socks5h:// builds a SOCKS agent that leaves DNS to the proxy; socks5:// resolves locally', () => { + const remoteUrl = 'socks5h://127.0.0.1:1080' + const localUrl = 'socks5://127.0.0.1:1080' + const remote = getProxyAgent({ email: 'socks5h@example.com', proxy: remoteUrl }) + const local = getProxyAgent({ email: 'socks5@example.com', proxy: localUrl }) + try { + assert.equal(remote.shouldLookup, false, 'socks5h must hand the hostname to the proxy') + assert.equal(local.shouldLookup, true, 'socks5 keeps the historical local lookup') + assert.equal(remote.proxy.type, 5) + } finally { + invalidateProxyAgent(remoteUrl) + invalidateProxyAgent(localUrl) + } +}) + +test('describeEgress keeps the scheme so a log line tells socks5h apart from socks5, without credentials', () => { + assert.equal( + describeEgress({ email: 'a@example.com', proxy: 'socks5h://user:secret@lohari-warp-qwen:9091' }), + 'socks5h://lohari-warp-qwen:9091' + ) + assert.equal( + describeEgress({ email: 'a@example.com', proxy: 'socks5://lohari-warp-qwen:9091' }), + 'socks5://lohari-warp-qwen:9091' + ) +}) diff --git a/tests/proxy-transport-no-bridge.test.js b/tests/proxy-transport-no-bridge.test.js new file mode 100644 index 00000000..ee9cb0bb --- /dev/null +++ b/tests/proxy-transport-no-bridge.test.js @@ -0,0 +1,171 @@ +const test = require('node:test') +const assert = require('node:assert/strict') +const http = require('node:http') +const net = require('node:net') +const axios = require('axios') + +// The Bun/undici proxy transport must never bridge a body through Readable.toWeb/fromWeb: under +// Bun the bridge drops undici's body error on a mid-body close, the consumer hangs and the +// rejection escapes as a process crash (qwen-next, 2026-09-16). The transport itself is +// runtime-agnostic, so this drives it under Node through a real SOCKS5 hop against an upstream +// that cuts the socket mid-body. + +process.env.DATA_SAVE_MODE = 'none' +process.env.ACCOUNTS = '' +const { getProxyAgent, getProxyTransport } = require('../src/utils/proxy-helper') + +const SSE_FRAME = `data: ${'x'.repeat(120)}\n\n` +const listen = (server) => new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve(server.address().port))) + +// Minimal SOCKS5: no auth, CONNECT, IPv4 or domain targets; propagates the upstream close. +const startSocks5 = () => net.createServer((client) => { + client.on('error', () => {}) + client.once('data', (greeting) => { + if (greeting[0] !== 5) return client.destroy() + client.write(Buffer.from([5, 0])) + client.once('data', (request) => { + if (request[0] !== 5 || request[1] !== 1) return client.destroy() + let host, offset + if (request[3] === 1) { + host = request.subarray(4, 8).join('.') + offset = 8 + } else if (request[3] === 3) { + const length = request[4] + host = request.subarray(5, 5 + length).toString() + offset = 5 + length + } else { + return client.destroy() + } + const upstream = net.connect(request.readUInt16BE(offset), host, () => { + client.write(Buffer.from([5, 0, 0, 1, 0, 0, 0, 0, 0, 0])) + client.pipe(upstream).pipe(client) + }) + upstream.on('error', () => client.destroy()) + upstream.on('close', () => client.destroy()) + }) + }) +}) + +const startUpstream = () => http.createServer((req, res) => { + const chunks = [] + req.on('data', (chunk) => chunks.push(chunk)) + req.on('end', () => { + const [route, query = ''] = req.url.split('?') + if (route === '/sse-cut') { + res.writeHead(200, { 'content-type': 'text/event-stream' }) + for (let sent = 0; sent < 5708; sent += SSE_FRAME.length) res.write(SSE_FRAME) + setTimeout(() => res.socket.destroy(), 30) + } else if (route === '/sse-401') { + res.writeHead(401, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ error: 'unauthorized' })) + } else if (route === '/json') { + res.writeHead(200, { 'content-type': 'application/json', 'x-probe': '1' }) + res.end(JSON.stringify({ ok: true, method: req.method, query, body: Buffer.concat(chunks).toString(), auth: req.headers.authorization || '' })) + } else if (route === '/json-500') { + res.writeHead(500, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ error: 'boom' })) + } else if (route === '/bytes') { + res.writeHead(200, { 'content-type': 'application/octet-stream' }) + res.end(Buffer.from([1, 2, 3, 250])) + } else if (route === '/reset') { + req.socket.destroy() + } else if (route === '/slow-headers') { + setTimeout(() => { res.writeHead(200); res.end('late') }, 1500) + } else { + res.writeHead(404) + res.end() + } + }) +}) + +let socks, upstream, base, transport +const escaped = [] +const onEscape = (error) => escaped.push(error) + +test.before(async () => { + process.on('unhandledRejection', onEscape) + process.on('uncaughtException', onEscape) + socks = startSocks5() + upstream = startUpstream() + const socksPort = await listen(socks) + base = `http://127.0.0.1:${await listen(upstream)}` + transport = getProxyTransport(getProxyAgent({ email: 'transport@example.com', proxy: `socks5://127.0.0.1:${socksPort}` })) +}) + +test.after(() => { + process.off('unhandledRejection', onEscape) + process.off('uncaughtException', onEscape) + socks.close() + upstream.close() +}) + +const request = (config) => axios.request({ adapter: transport.adapter, proxy: false, ...config }) +const settle = () => new Promise((resolve) => setTimeout(resolve, 150)) +const rejection = (promise) => promise.then(() => assert.fail('expected a rejection'), (error) => error) +const readAll = async (stream) => { + let text = '' + for await (const chunk of stream) text += chunk + return text +} + +test('stream: a mid-body close rejects the consumer and escapes nowhere', async () => { + const response = await request({ url: `${base}/sse-cut`, method: 'post', data: '{}', responseType: 'stream', timeout: 5000 }) + assert.equal(response.status, 200) + assert.equal(typeof response.data.on, 'function', 'SSE consumers expect a Node stream') + let received = 0 + await assert.rejects( + (async () => { for await (const chunk of response.data) received += chunk.length })(), + (error) => error.code === 'UND_ERR_SOCKET' + ) + assert.ok(received > 0, 'the cut happens after bytes were delivered') + await settle() + assert.deepEqual(escaped, []) +}) + +test('stream: a non-2xx still rejects with the body as a Node stream', async () => { + const error = await rejection(request({ url: `${base}/sse-401`, responseType: 'stream' })) + assert.equal(error.response?.status, 401) + assert.equal(typeof error.response.data.on, 'function') + assert.equal(await readAll(error.response.data), '{"error":"unauthorized"}') +}) + +test('json: method, query, body and headers reach the upstream; axios parses the text', async () => { + const response = await request({ + url: `${base}/json`, + method: 'post', + params: { chat_id: 'abc' }, + data: { hello: 'world' }, + headers: { Authorization: 'Bearer test' } + }) + assert.equal(response.status, 200) + assert.equal(response.headers['x-probe'], '1') + assert.deepEqual(response.data, { ok: true, method: 'POST', query: 'chat_id=abc', body: '{"hello":"world"}', auth: 'Bearer test' }) + const error = await rejection(request({ url: `${base}/json-500` })) + assert.equal(error.response?.status, 500) + assert.deepEqual(error.response.data, { error: 'boom' }) +}) + +test('arraybuffer: returns a Buffer like the Node adapter', async () => { + const response = await request({ url: `${base}/bytes`, responseType: 'arraybuffer' }) + assert.ok(Buffer.isBuffer(response.data)) + assert.deepEqual([...response.data], [1, 2, 3, 250]) +}) + +test('request-phase transport errors map onto the codes request.js retries', async () => { + const reset = await rejection(request({ url: `${base}/reset`, responseType: 'stream' })) + assert.equal(reset.isAxiosError, true) + assert.equal(reset.code, 'ECONNRESET') + assert.equal(reset.cause?.code, 'UND_ERR_SOCKET') + const timeout = await rejection(request({ url: `${base}/slow-headers`, timeout: 200 })) + assert.equal(timeout.code, 'ECONNABORTED') + assert.equal(timeout.cause?.code, 'UND_ERR_HEADERS_TIMEOUT') +}) + +test('fetch: the Response is buffered, so a mid-body close rejects instead of hanging', async () => { + const ok = await transport.fetch(`${base}/json`) + assert.equal(ok.status, 200) + assert.equal((await ok.json()).ok, true) + await assert.rejects(transport.fetch(`${base}/sse-cut`, { method: 'POST', body: '{}' }), (error) => error.code === 'UND_ERR_SOCKET') + await settle() + assert.deepEqual(escaped, []) +}) diff --git a/tests/sse-interrupt-metrics.test.js b/tests/sse-interrupt-metrics.test.js new file mode 100644 index 00000000..7f05042b --- /dev/null +++ b/tests/sse-interrupt-metrics.test.js @@ -0,0 +1,67 @@ +const test = require('node:test') +const assert = require('node:assert/strict') +const { Readable } = require('node:stream') + +const { consumeSSEStream } = require('../src/utils/sse') + +// A mid-stream failure has to say how far the upstream got. The controller decides from +// it whether a retry is safe, and the [EGRESS] log needs the byte count to tell an early +// close (a few KB) from a late one (tens of KB) — the three live closes on 2026-09-16 were +// all late, 63–90 KiB in. + +const frame = (obj) => `data: ${JSON.stringify(obj)}\n\n` +const socketClose = () => Object.assign(new Error('other side closed'), { code: 'UND_ERR_SOCKET' }) + +test('a transport failure mid-stream carries the bytes and frames consumed so far', async () => { + const first = frame({ a: 1 }) + const failing = Readable.from((async function* () { + yield Buffer.from(first) + throw socketClose() + })()) + const seen = [] + await assert.rejects( + consumeSSEStream(failing, async (f) => { seen.push(f.data) }), + (err) => { + assert.equal(err.code, 'UND_ERR_SOCKET') + assert.equal(err.upstreamBytesRead, Buffer.byteLength(first)) + assert.equal(err.upstreamEventCount, 1) + return true + } + ) + assert.equal(seen.length, 1, 'the frame before the failure was still delivered') +}) + +test('a failure before any byte reports zero, not undefined', async () => { + const failing = new Readable({ read() { this.destroy(socketClose()) } }) + await assert.rejects(consumeSSEStream(failing, async () => {}), (err) => { + assert.equal(err.upstreamBytesRead, 0) + assert.equal(err.upstreamEventCount, 0) + return true + }) +}) + +test('an onFrame rejection (upstream business error such as RateLimited) is annotated too', async () => { + const text = frame({ success: false, data: { code: 'RateLimited' } }) + await assert.rejects( + consumeSSEStream(Readable.from([text]), async () => { + throw Object.assign(new Error('quota'), { code: 'RateLimited' }) + }), + (err) => { + assert.equal(err.code, 'RateLimited') + assert.equal(err.upstreamBytesRead, Buffer.byteLength(text)) + assert.equal(err.upstreamEventCount, 1) + return true + } + ) +}) + +test('a clean stream reports bytesRead alongside eventCount; string and Buffer chunks count the same', async () => { + const text = frame({ a: 1 }) + frame({ b: 2 }) + 'data: [DONE]\n\n' + const asString = await consumeSSEStream(Readable.from([text]), async () => {}) + const asBuffer = await consumeSSEStream(Readable.from([Buffer.from(text)]), async () => {}) + assert.equal(asString.bytesRead, Buffer.byteLength(text)) + assert.equal(asBuffer.bytesRead, Buffer.byteLength(text)) + assert.equal(asString.eventCount, 3) + assert.equal(asString.sawDone, true) + assert.equal(asString.completed, true) +}) diff --git a/tests/upload-parse-status.test.js b/tests/upload-parse-status.test.js index f1695ef2..4ffc94be 100644 --- a/tests/upload-parse-status.test.js +++ b/tests/upload-parse-status.test.js @@ -55,7 +55,10 @@ const axiosStub = { } }; } - if (url.endsWith('/api/v2/files/parse')) return parseResponse + if (url.endsWith('/api/v2/files/parse')) { + if (parseResponse instanceof Error) throw parseResponse + return parseResponse + } if (url.endsWith('/api/v2/files/parse/status')) { return statusQueue.length > 1 ? statusQueue.shift() : statusQueue[0] } @@ -119,6 +122,24 @@ test('parse POST answering with an error code fails before any status poll', asy assert.equal(statusCalls(), 0) }) +test('a transport failure on the parse POST names the egress, credentials dropped', async (context) => { + reset() + const account = { email: 'egress@example.invalid', proxy: 'socks5h://user:s3cr3t@lohari-warp-qwen:9091' } + context.after(() => invalidateProxyAgent(account.proxy)) + parseResponse = Object.assign(new Error('socket hang up'), { code: 'ECONNRESET' }) + await assert.rejects( + parseUploadedTextFile('f1', 'token', account, { intervalMs: 10 }), + (error) => { + assert.equal(error.code, 'ECONNRESET') + assert.equal(error.egress, 'socks5h://lohari-warp-qwen:9091') + assert.match(error.message, /socket hang up \(f1, via socks5h:\/\/lohari-warp-qwen:9091\)/) + assert.doesNotMatch(error.message, /s3cr3t/) + return true + } + ) + assert.equal(statusCalls(), 0) +}) + test('a genuinely pending parse still resolves once the file reports success', async () => { reset() statusQueue = [perFile('pending'), perFile('parsing'), perFile('success')] diff --git a/tests/upstream-quota-429.test.js b/tests/upstream-quota-429.test.js index 6fd96fe1..4b8f2356 100644 --- a/tests/upstream-quota-429.test.js +++ b/tests/upstream-quota-429.test.js @@ -37,9 +37,18 @@ const requestModule = require('../src/utils/request.js'); let upstreamFactory = null; /** La cuenta que el upstream dice haber usado. Sin esto no hay a quien culpar del gasto. */ let upstreamAccount = null; -requestModule.sendChatRequest = async () => (upstreamFactory - ? { status: true, response: upstreamFactory(), currentAccount: upstreamAccount } - : { status: false }); +requestModule.sendChatRequest = async (_body, options = {}) => { + // Como el rotador real: una cuenta excluida (ya quemada en este mismo request por el + // failover a mitad de stream) no vuelve a salir. Con una sola cuenta eso es "no hay + // alternativa", y el controlador debe entregar el fallo original. + const excluded = Array.isArray(options.excludeEmails) ? options.excludeEmails : []; + if (upstreamAccount?.email && excluded.includes(upstreamAccount.email)) { + return { status: false, response: null, message: 'offline test: no alternative account' }; + } + return upstreamFactory + ? { status: true, response: upstreamFactory(), currentAccount: upstreamAccount } + : { status: false }; +}; const { UpstreamResponseError,