diff --git a/open-sse/services/rateLimitManager.ts b/open-sse/services/rateLimitManager.ts index 290678b503..5e92751b00 100644 --- a/open-sse/services/rateLimitManager.ts +++ b/open-sse/services/rateLimitManager.ts @@ -569,11 +569,25 @@ export async function withRateLimit(provider, connectionId, model, fn, signal = const abortPromise = new Promise((_, reject) => { const onAbort = () => { const reason = signal.reason; - const err = + // Build a fresh Error rather than mutating `reason` in place: the + // default abort reason (when `controller.abort()` is called with no + // argument, e.g. modelTestRunner's timeout path) is a native + // DOMException, whose `name` is a read-only getter — assigning + // `err.name = "AbortError"` on it throws `TypeError: Cannot set + // property name of [object DOMException] which has only a getter`, + // which then surfaces as an unhandled rejection instead of the + // intended "slow"/timeout result. + const message = reason instanceof Error - ? reason - : new Error(typeof reason === "string" ? reason : "The operation was aborted"); + ? reason.message + : typeof reason === "string" + ? reason + : "The operation was aborted"; + const err = new Error(message); err.name = "AbortError"; + if (reason !== undefined) { + (err as Error & { cause?: unknown }).cause = reason; + } reject(err); }; if (signal.aborted) { diff --git a/src/app/(dashboard)/dashboard/providers/[id]/components/PassthroughModelsSection.tsx b/src/app/(dashboard)/dashboard/providers/[id]/components/PassthroughModelsSection.tsx index ea36db0671..6b201ce48a 100644 --- a/src/app/(dashboard)/dashboard/providers/[id]/components/PassthroughModelsSection.tsx +++ b/src/app/(dashboard)/dashboard/providers/[id]/components/PassthroughModelsSection.tsx @@ -159,7 +159,7 @@ export default function PassthroughModelsSection({ results?: Record< string, { - status?: "ok" | "error"; + status?: "ok" | "error" | "slow"; rateLimited?: boolean; isTimeout?: boolean; error?: string; diff --git a/src/app/(dashboard)/dashboard/providers/[id]/hooks/useModelVisibilityHandlers.ts b/src/app/(dashboard)/dashboard/providers/[id]/hooks/useModelVisibilityHandlers.ts index 3fd4b67529..66ccee8c2d 100644 --- a/src/app/(dashboard)/dashboard/providers/[id]/hooks/useModelVisibilityHandlers.ts +++ b/src/app/(dashboard)/dashboard/providers/[id]/hooks/useModelVisibilityHandlers.ts @@ -352,7 +352,7 @@ export function useModelVisibilityHandlers({ results?: Record< string, { - status?: "ok" | "error"; + status?: "ok" | "error" | "slow"; rateLimited?: boolean; isTimeout?: boolean; error?: string; diff --git a/src/app/(dashboard)/dashboard/providers/[id]/providerPageHelpers.ts b/src/app/(dashboard)/dashboard/providers/[id]/providerPageHelpers.ts index 1865b819b0..3b85c92db0 100644 --- a/src/app/(dashboard)/dashboard/providers/[id]/providerPageHelpers.ts +++ b/src/app/(dashboard)/dashboard/providers/[id]/providerPageHelpers.ts @@ -130,6 +130,7 @@ export interface TestAllModelOutcome { status: "ok" | "error"; shouldHide: boolean; } +type TestAllEntryStatus = "ok" | "error" | "slow"; /** * Decide a model's per-row test status (the green/red icon) and whether it should @@ -142,27 +143,27 @@ export interface TestAllModelOutcome { * model failed (unlike the single-model ▶ test). * * Auto-hide policy: when `autoHideFailed` is on, only NON-TRANSIENT failures are - * hidden. Transient failures (rate-limited, timeout) are surfaced as 'error' on - * the row icon but NOT hidden, because: - * - The provider may have been temporarily throttled during a parallel batch - * (a single Test All across 10+ models routinely trips per-account rate - * limits on subscription-tier APIs). - * - The model itself is not broken — a retry seconds later would succeed. - * - Hidden state persists across server restarts and silently removes the - * model from `/v1/models`, so a transient blip turns into a permanent - * catalog gap that the user can only recover from by editing the DB or - * hand-toggling each row. + * hidden. Transient failures remain visible because the model may still be healthy + * and hidden state persists across restarts, removing it from `/v1/models`. * * Genuine failures (`status:"error"` without a transient flag — e.g. upstream * 400 "invalid model", schema mismatch, auth failure) ARE still auto-hidden, * which is the intended use of the toggle. */ export function evaluateTestAllEntry( - entry: { status?: "ok" | "error"; rateLimited?: boolean; isTimeout?: boolean } | null | undefined, + entry: + | { + status?: TestAllEntryStatus; + rateLimited?: boolean; + isTimeout?: boolean; + isTransient?: boolean; + } + | null + | undefined, autoHideFailed: boolean ): TestAllModelOutcome { const ok = entry?.status === "ok"; - const transient = Boolean(entry?.rateLimited || entry?.isTimeout); + const transient = [entry?.rateLimited, entry?.isTimeout, entry?.isTransient].some(Boolean); return { status: ok ? "ok" : "error", // Hide only persistent failures. Transient (rate-limited, timeout) are diff --git a/src/app/api/models/test-all/route.ts b/src/app/api/models/test-all/route.ts index 28a418dd48..65044431de 100644 --- a/src/app/api/models/test-all/route.ts +++ b/src/app/api/models/test-all/route.ts @@ -41,12 +41,13 @@ const testAllSchema = z.object({ }); export interface BatchTestResultEntry { - status: "ok" | "error"; + status: "ok" | "error" | "slow"; latencyMs: number; responseText?: string; error?: string; statusCode?: number; rateLimited?: boolean; + isTransient?: boolean; hidden?: boolean; isTimeout?: boolean; } @@ -55,13 +56,14 @@ function toBatchEntry( result: Awaited> ): BatchTestResultEntry { const entry: BatchTestResultEntry = { - status: result.status === "ok" ? "ok" : "error", + status: result.status === "ok" ? "ok" : result.status === "slow" ? "slow" : "error", latencyMs: result.latencyMs, }; if (result.responseText !== undefined) entry.responseText = result.responseText; if (result.error !== undefined) entry.error = result.error; if (result.statusCode !== undefined) entry.statusCode = result.statusCode; if (result.rateLimited === true) entry.rateLimited = true; + if (result.isTransient === true) entry.isTransient = true; if (result.isTimeout === true) entry.isTimeout = true; return entry; } @@ -152,6 +154,7 @@ export async function POST(request: Request) { modelId, ...(effectiveConnectionId ? { connectionId: effectiveConnectionId } : {}), timeoutMs: PER_MODEL_TIMEOUT_MS, + streamChat: true, }); entry = toBatchEntry(result); testedUpstream += 1; @@ -184,7 +187,13 @@ export async function POST(request: Request) { consecutiveBotBlocks = 0; } - if (autoHideFailed && entry.status === "error" && !entry.rateLimited && !entry.isTimeout) { + if ( + autoHideFailed && + entry.status === "error" && + !entry.rateLimited && + !entry.isTimeout && + !entry.isTransient + ) { try { await setModelIsHidden(providerId, modelId, true); entry.hidden = true; diff --git a/src/app/api/models/test/route.ts b/src/app/api/models/test/route.ts index c8974707a1..1ca642908e 100644 --- a/src/app/api/models/test/route.ts +++ b/src/app/api/models/test/route.ts @@ -13,6 +13,7 @@ const testModelSchema = z.object({ }); const SINGLE_TEST_TIMEOUT_MS = 20_000; +const NVIDIA_SINGLE_TEST_TIMEOUT_MS = 180_000; export async function POST(request: Request) { const authError = await requireManagementAuth(request); @@ -67,7 +68,11 @@ export async function POST(request: Request) { providerId, modelId, ...(connectionId ? { connectionId } : {}), - timeoutMs: SINGLE_TEST_TIMEOUT_MS, + timeoutMs: + providerId.trim().toLowerCase() === "nvidia" + ? NVIDIA_SINGLE_TEST_TIMEOUT_MS + : SINGLE_TEST_TIMEOUT_MS, + streamChat: true, }); if (result.status === "ok") { diff --git a/src/lib/api/modelTestRunner.ts b/src/lib/api/modelTestRunner.ts index 87a512fa30..f042b89341 100644 --- a/src/lib/api/modelTestRunner.ts +++ b/src/lib/api/modelTestRunner.ts @@ -2,7 +2,11 @@ import { randomUUID } from "node:crypto"; import { POST as postChatCompletion } from "@/app/api/v1/chat/completions/route"; import { handleValidatedEmbeddingRequestBody } from "@/app/api/v1/embeddings/route"; import { POST as postRerank } from "@/app/api/v1/rerank/route"; -import { buildComboTestRequestBody, extractComboTestResponseText } from "@/lib/combos/testHealth"; +import { + buildComboTestRequestBody, + extractComboTestResponseText, + extractComboTestStreamResult, +} from "@/lib/combos/testHealth"; import { getCustomModels } from "@/lib/localDb"; import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/error"; import { withRateLimit } from "@omniroute/open-sse/services/rateLimitManager"; @@ -12,6 +16,7 @@ const DEFAULT_TEST_TIMEOUT_MS = 10_000; const DOLA_PRO_TEST_TIMEOUT_MS = 90_000; const DOUBAO_WEB_PROVIDER_ID = "doubao-web"; const SLOW_WEB_TEST_MODELS = new Set(["dola-pro"]); +const STREAMING_CHAT_TEST_MAX_TOKENS = 64; function asRecord(value: unknown): Record { return value && typeof value === "object" && !Array.isArray(value) @@ -195,21 +200,47 @@ export interface RunSingleModelTestOptions { modelId: string; connectionId?: string; timeoutMs?: number; + streamChat?: boolean; } export interface SingleModelTestResult { modelId: string; - status: "ok" | "error" | "rate_limited"; + status: "ok" | "error" | "rate_limited" | "slow"; latencyMs: number; responseText?: string; statusCode?: number; httpStatus: number; error?: string; rateLimited?: boolean; + isTransient?: boolean; isTimeout?: boolean; retryAfter?: number; } +export type ModelTestResponseText = { + text: string; + error?: { message: string; statusCode?: number }; +}; + +export async function extractModelTestResponseText( + response: Response, + streamChat: boolean +): Promise { + const contentType = (response.headers.get("content-type") || "").toLowerCase(); + if (streamChat && !contentType.includes("application/json")) { + return extractComboTestStreamResult(await response.text()); + } + return { text: extractComboTestResponseText(await response.json()) }; +} + +function isRateLimitMessage(message: string): boolean { + return /rate[ -]?limit|too many requests|quota exceeded/i.test(message); +} + +function isBotBlockMessage(message: string): boolean { + return /cloudflare|bot management|recaptcha|cf-chl|just a moment/i.test(message); +} + /** * Run a single model test. When `connectionId` is provided, wraps the * upstream call with `withRateLimit` (Bottleneck). Returns a plain @@ -219,7 +250,13 @@ export interface SingleModelTestResult { export async function runSingleModelTest( options: RunSingleModelTestOptions ): Promise { - const { providerId, modelId, connectionId, timeoutMs = DEFAULT_TEST_TIMEOUT_MS } = options; + const { + providerId, + modelId, + connectionId, + timeoutMs = DEFAULT_TEST_TIMEOUT_MS, + streamChat = true, + } = options; let fullModelStr = modelId; if (!fullModelStr.includes("/")) { @@ -242,7 +279,10 @@ export async function runSingleModelTest( top_n: 1, return_documents: false, } - : buildComboTestRequestBody(fullModelStr, isEmbedding); + : buildComboTestRequestBody(fullModelStr, isEmbedding, { + stream: !isEmbedding && streamChat, + maxTokens: !isEmbedding && streamChat ? STREAMING_CHAT_TEST_MAX_TOKENS : undefined, + }); // Per-model AbortController. We track whether the timeout fired so we can // distinguish "rate-limit queue aborted" (withRateLimit threw AbortError @@ -287,10 +327,10 @@ export async function runSingleModelTest( if (timedOut) { return { modelId: fullModelStr, - status: "error", + status: "slow", latencyMs, - httpStatus: 500, - error: `Timeout (${Math.round(effectiveTimeoutMs / 1000)}s)`, + httpStatus: 504, + error: `No model output within ${Math.round(effectiveTimeoutMs / 1000)}s`, isTimeout: true, }; } @@ -313,9 +353,19 @@ export async function runSingleModelTest( error: getErrorMessage(error), }; } - clearTimeout(timeoutHandle); + let latencyMs = Date.now() - startTime; - const latencyMs = Date.now() - startTime; + if (timedOut) { + clearTimeout(timeoutHandle); + return { + modelId: fullModelStr, + status: "slow", + latencyMs, + httpStatus: 504, + error: `No model output within ${Math.round(effectiveTimeoutMs / 1000)}s`, + isTimeout: true, + }; + } if (res.status === 429) { const retryAfter = parseRetryAfterHeader(res.headers.get("retry-after")); @@ -327,7 +377,7 @@ export async function runSingleModelTest( } catch { errorMsg = res.statusText || errorMsg; } - return { + const result: SingleModelTestResult = { modelId: fullModelStr, status: "rate_limited", latencyMs, @@ -337,17 +387,51 @@ export async function runSingleModelTest( rateLimited: true, ...(retryAfter !== undefined ? { retryAfter } : {}), }; + clearTimeout(timeoutHandle); + return result; } if (res.ok) { - let responseBody = null; + let responseText = ""; + let streamError: ModelTestResponseText["error"]; try { - responseBody = await res.json(); + const parsedResponse = await extractModelTestResponseText( + res, + !isEmbedding && !isRerank && streamChat + ); + responseText = parsedResponse.text; + streamError = parsedResponse.error; } catch { - responseBody = null; + responseText = ""; + } finally { + clearTimeout(timeoutHandle); + } + latencyMs = Date.now() - startTime; + if (streamError) { + const error = sanitizeErrorMessage(streamError.message) || "Upstream stream failed"; + const rateLimited = streamError.statusCode === 429 || isRateLimitMessage(error); + const isBotBlock = streamError.statusCode === 403 || isBotBlockMessage(error); + return { + modelId: fullModelStr, + status: rateLimited ? "rate_limited" : "error", + latencyMs, + ...(streamError.statusCode !== undefined ? { statusCode: streamError.statusCode } : {}), + httpStatus: streamError.statusCode ?? 502, + error, + ...(rateLimited ? { rateLimited: true } : {}), + ...(rateLimited || isBotBlock ? { isTransient: true } : {}), + }; + } + if (timedOut && !responseText) { + return { + modelId: fullModelStr, + status: "slow", + latencyMs, + httpStatus: 504, + error: `No model output within ${Math.round(effectiveTimeoutMs / 1000)}s`, + isTimeout: true, + }; } - - const responseText = extractComboTestResponseText(responseBody); if (isRerank) { return { modelId: fullModelStr, @@ -382,6 +466,8 @@ export async function runSingleModelTest( errorMsg = extractProviderErrorMessage(errBody, res.statusText); } catch { errorMsg = res.statusText; + } finally { + clearTimeout(timeoutHandle); } return { modelId: fullModelStr, diff --git a/src/lib/combos/testHealth.ts b/src/lib/combos/testHealth.ts index 34d8f0661b..0fcca804dd 100644 --- a/src/lib/combos/testHealth.ts +++ b/src/lib/combos/testHealth.ts @@ -1,6 +1,7 @@ type JsonRecord = Record; const COMBO_TEST_MAX_TOKENS = 2048; +const STREAMING_MODEL_TEST_MAX_TOKENS = 64; const COMBO_TEST_OPERAND_MIN = 10000; const COMBO_TEST_OPERAND_RANGE = 90000; @@ -118,7 +119,11 @@ function buildComboTestPrompt() { return `Calculate ${left}+${right}, and reply with the result only.`; } -export function buildComboTestRequestBody(modelStr: string, isEmbedding: boolean = false) { +export function buildComboTestRequestBody( + modelStr: string, + isEmbedding: boolean = false, + options: { stream?: boolean; maxTokens?: number } = {} +) { if (isEmbedding) { return { model: modelStr, @@ -133,11 +138,84 @@ export function buildComboTestRequestBody(modelStr: string, isEmbedding: boolean messages: [{ role: "user", content: buildComboTestPrompt() }], // Give reasoning-heavy models enough headroom to finish the request and // still emit a visible answer without immediate truncation. - max_tokens: COMBO_TEST_MAX_TOKENS, - stream: false, + max_tokens: + options.maxTokens ?? + (options.stream ? STREAMING_MODEL_TEST_MAX_TOKENS : COMBO_TEST_MAX_TOKENS), + stream: options.stream ?? false, }; } +export type ComboTestStreamResult = { + text: string; + error?: { message: string; statusCode?: number }; +}; + +function extractStreamError(body: JsonRecord): ComboTestStreamResult["error"] { + const error = asRecord(body.error); + const message = + typeof error.message === "string" + ? error.message.trim() + : typeof body.message === "string" + ? body.message.trim() + : ""; + if (!message) return undefined; + + const status = error.status ?? error.statusCode ?? error.code ?? body.status ?? body.statusCode; + const statusCode = + typeof status === "number" + ? status + : typeof status === "string" && /^\d+$/.test(status) + ? Number(status) + : undefined; + return { + message, + ...(Number.isInteger(statusCode) ? { statusCode } : {}), + }; +} + +function extractStreamPayload(payload: string): ComboTestStreamResult | undefined { + try { + const body = JSON.parse(payload); + const error = extractStreamError(asRecord(body)); + if (error) return { text: "", error }; + + const collected: string[] = []; + const direct = extractComboTestResponseText(body); + if (direct) collected.push(direct); + for (const choice of Array.isArray(body?.choices) ? body.choices : []) { + const delta = asRecord(choice?.delta); + const content = extractTextFromContent(delta.content); + const reasoning = extractReasoningText(delta); + if (content) collected.push(content); + else if (reasoning) collected.push(reasoning); + } + return { text: collected.join("") }; + } catch { + // Ignore malformed/non-JSON SSE events; a later valid event can still + // prove that the model is healthy. + return undefined; + } +} + +export function extractComboTestStreamResult(streamBody: string): ComboTestStreamResult { + const collected: string[] = []; + let streamError: ComboTestStreamResult["error"]; + for (const line of streamBody.split(/\r?\n/)) { + if (!line.startsWith("data:")) continue; + const payload = line.slice(5).trim(); + if (!payload || payload === "[DONE]") continue; + const result = extractStreamPayload(payload); + if (!result) continue; + if (result.error) streamError = result.error; + else if (result.text) collected.push(result.text); + } + return { text: collected.join("").trim(), ...(streamError ? { error: streamError } : {}) }; +} + +export function extractComboTestStreamText(streamBody: string): string { + return extractComboTestStreamResult(streamBody).text; +} + export function extractComboTestResponseText(responseBody: unknown): string { const body = asRecord(responseBody); diff --git a/stryker.conf.json b/stryker.conf.json index bc5a1b1ad3..d8354ffb1f 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -150,6 +150,7 @@ "tests/unit/combo-scoring-inspector.test.ts", "tests/unit/combo-selected-connection-success.test.ts", "tests/unit/combo-sessionless-pin-3825.test.ts", + "tests/unit/combo-speed-telemetry-6875.test.ts", "tests/unit/combo-strategies.test.ts", "tests/unit/combo-strategy-fallbacks.test.ts", "tests/unit/combo-stream-readiness-fallback.test.ts", @@ -207,6 +208,7 @@ "tests/unit/observability-payloads.test.ts", "tests/unit/ollama-cloud-weekly-quota-cooldown-3709.test.ts", "tests/unit/openapi-security-tiers.test.ts", + "tests/unit/openrouter-quota-6842.test.ts", "tests/unit/persist-429-cooldown-account-fallback.test.ts", "tests/unit/plan3-p0.test.ts", "tests/unit/plugin-sandbox-permissions.test.ts", @@ -240,6 +242,7 @@ "tests/unit/serial/combo-quota-share-cooldown-wait-timing.test.ts", "tests/unit/serial/combo-strategy-fallbacks-half-open-timing.test.ts", "tests/unit/serial/provider-health-autopilot.test.ts", + "tests/unit/session-affinity-generic-7274.test.ts", "tests/unit/service-combo-metrics.test.ts", "tests/unit/services-branch-hardening.test.ts", "tests/unit/services/bifrost-route-guard.test.ts", @@ -260,6 +263,7 @@ "tests/unit/trae-publiccred.test.ts", "tests/unit/types-barrel-model-cooldown.test.ts", "tests/unit/upstream-retry-hints-toggle.test.ts", + "tests/unit/upstream-timeout-model-override.test.ts", "tests/unit/usage-service-hardening.test.ts", "tests/unit/validate-response-quality.test.ts" ], diff --git a/tests/unit/combo-test-health.test.ts b/tests/unit/combo-test-health.test.ts index 6b764c979b..dcdc27cccb 100644 --- a/tests/unit/combo-test-health.test.ts +++ b/tests/unit/combo-test-health.test.ts @@ -1,8 +1,12 @@ import test from "node:test"; import assert from "node:assert/strict"; -const { buildComboTestRequestBody, extractComboTestResponseText } = - await import("../../src/lib/combos/testHealth.ts"); +const { + buildComboTestRequestBody, + extractComboTestResponseText, + extractComboTestStreamResult, + extractComboTestStreamText, +} = await import("../../src/lib/combos/testHealth.ts"); test("combo test helper builds a realistic smoke payload", () => { const originalRandom = Math.random; @@ -26,6 +30,30 @@ test("combo test helper builds a realistic smoke payload", () => { assert.equal(body.stream, false); }); +test("combo test helper builds a small streaming model probe", () => { + const body = buildComboTestRequestBody("nvidia/z-ai/glm-5.2", false, { stream: true }); + assert.equal(body.stream, true); + assert.equal(body.max_tokens, 64); +}); + +test("combo test helper ignores keepalives and extracts streamed model content", () => { + const text = extractComboTestStreamText( + ': omniroute-keepalive\n\ndata: {"choices":[{"delta":{"content":"O"}}]}\n\n' + + 'data: {"choices":[{"delta":{"content":"K"}}]}\n\ndata: [DONE]\n\n' + ); + assert.equal(text, "OK"); +}); + +test("combo test helper preserves streamed upstream errors", () => { + const result = extractComboTestStreamResult( + 'data: {"error":{"message":"Rate limit exceeded","code":"429"}}\n\n' + ); + assert.deepEqual(result, { + text: "", + error: { message: "Rate limit exceeded", statusCode: 429 }, + }); +}); + test("combo test helper extracts text from chat-completions responses", () => { const text = extractComboTestResponseText({ choices: [ diff --git a/tests/unit/model-test-runner.test.ts b/tests/unit/model-test-runner.test.ts index f7a8a55c8b..b0e3d8cc47 100644 --- a/tests/unit/model-test-runner.test.ts +++ b/tests/unit/model-test-runner.test.ts @@ -1,9 +1,12 @@ import { test } from "node:test"; import assert from "node:assert/strict"; +import { createProviderConnection } from "@/lib/db/providers"; import { parseRetryAfterHeader, detectTestKind, extractProviderErrorMessage, + extractModelTestResponseText, + runSingleModelTest, resolveModelTestTimeoutMs, } from "@/lib/api/modelTestRunner.ts"; @@ -117,3 +120,108 @@ test("resolveModelTestTimeoutMs leaves ordinary models unchanged", () => { assert.equal(resolveModelTestTimeoutMs("doubao-web", "dola-speed", 10_000), 10_000); assert.equal(resolveModelTestTimeoutMs("openai", "dola-pro", 10_000), 10_000); }); + +test("extractModelTestResponseText accepts JSON when a streaming probe is ignored upstream", async () => { + const response = new Response( + JSON.stringify({ choices: [{ message: { role: "assistant", content: "OK" } }] }), + { headers: { "content-type": "Application/JSON; charset=utf-8" } } + ); + + assert.deepEqual(await extractModelTestResponseText(response, true), { text: "OK" }); +}); + +test("extractModelTestResponseText extracts content from SSE responses", async () => { + const response = new Response( + 'data: {"choices":[{"delta":{"content":"OK"}}]}\n\ndata: [DONE]\n', + { + headers: { "content-type": "text/event-stream" }, + } + ); + + assert.deepEqual(await extractModelTestResponseText(response, true), { text: "OK" }); +}); + +test("extractModelTestResponseText preserves SSE error status for transient classification", async () => { + const response = new Response( + 'data: {"error":{"message":"Rate limit exceeded","status":429}}\n\n', + { headers: { "content-type": "text/event-stream" } } + ); + + assert.deepEqual(await extractModelTestResponseText(response, true), { + text: "", + error: { message: "Rate limit exceeded", statusCode: 429 }, + }); +}); + +test("runSingleModelTest preserves slow timeout after chatCore converts AbortError to a Response", async () => { + const connection = await createProviderConnection({ + provider: "openai", + authType: "apikey", + name: "model-test-timeout-regression", + apiKey: "sk-model-test-timeout-regression", + isActive: true, + testStatus: "active", + }); + const originalFetch = globalThis.fetch; + + // Warm up the chat-completions pipeline (SSE translators, compression + // settings, etc. all lazy-init on the very first real request in a + // process) with a fast, immediately-resolving mock and a generous + // timeout *before* asserting on the 1s abort-timing below. Without this, + // the first-request cold-start cost can eat the entire 1s budget below, + // so the AbortController fires before chatCore ever reaches the + // executor's fetch() call — by the time that in-flight call actually + // dispatches, this test's own `finally` block has already restored + // `globalThis.fetch`, and the assertions below race real upstream I/O + // instead of exercising the mock. + globalThis.fetch = async () => + new Response(JSON.stringify({ choices: [{ message: { role: "assistant", content: "OK" } }] }), { + headers: { "content-type": "application/json" }, + }); + await runSingleModelTest({ + providerId: "openai", + modelId: "gpt-4o", + connectionId: String(connection.id), + timeoutMs: 10_000, + }); + + let upstreamSignal: AbortSignal | null = null; + let upstreamCalled = false; + + globalThis.fetch = async (_input, init = {}) => { + upstreamCalled = true; + upstreamSignal = (init.signal as AbortSignal | null | undefined) ?? null; + return new Promise((_resolve, reject) => { + let fallbackTimer: ReturnType; + const rejectOnAbort = () => { + clearTimeout(fallbackTimer); + reject(new DOMException("The operation was aborted", "AbortError")); + }; + fallbackTimer = setTimeout(rejectOnAbort, 1_500); + + if (upstreamSignal?.aborted) { + rejectOnAbort(); + } else { + upstreamSignal?.addEventListener("abort", rejectOnAbort, { once: true }); + } + }); + }; + + try { + const result = await runSingleModelTest({ + providerId: "openai", + modelId: "gpt-4o", + connectionId: String(connection.id), + timeoutMs: 1_000, + }); + + assert.equal(upstreamCalled, true); + assert.ok(upstreamSignal, "the chat completion request should receive an abort signal"); + assert.equal(result.status, "slow"); + assert.equal(result.httpStatus, 504); + assert.equal(result.isTimeout, true); + assert.equal(result.error, "No model output within 1s"); + } finally { + globalThis.fetch = originalFetch; + } +}); diff --git a/tests/unit/rate-limit-manager.test.ts b/tests/unit/rate-limit-manager.test.ts index 908e752325..861a0a7e75 100644 --- a/tests/unit/rate-limit-manager.test.ts +++ b/tests/unit/rate-limit-manager.test.ts @@ -284,3 +284,46 @@ test("rate limit manager recomputes auto-enabled API key connections when queue assert.equal(rateLimitManager.isRateLimitEnabled(explicitConnection.id), true); assert.ok(rateLimitManager.getAllRateLimitStatus()[`openai:${autoConnection.id}`]); }); + +test("withRateLimit rejects cleanly when the caller aborts with the default DOMException reason", async () => { + // `AbortController.abort()` called with no argument (e.g. modelTestRunner's + // timeout path) produces a native DOMException as `signal.reason`, whose + // `name` is a read-only getter. withRateLimit's abort handling used to + // mutate `reason.name = "AbortError"` in place, which throws + // `TypeError: Cannot set property name of [object DOMException] which has + // only a getter` instead of rejecting with a clean AbortError — surfacing + // as an unhandled rejection rather than the intended timeout/slow result. + const connection = await providersDb.createProviderConnection({ + provider: "openai", + authType: "apikey", + name: "abort-reason-regression", + apiKey: "sk-abort-reason-regression", + isActive: true, + }); + rateLimitManager.enableRateLimitProtection(String(connection.id)); + + const controller = new AbortController(); + // Mirror how a real executor call behaves: it settles once the signal it + // was handed aborts, so this job doesn't dangle forever in Bottleneck once + // withRateLimit's own Promise.race settles via the abort path below. + const settlesOnAbort = (signal) => + new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => reject(signal.reason), { once: true }); + }); + + const pending = rateLimitManager.withRateLimit( + "openai", + String(connection.id), + "gpt-4o", + () => settlesOnAbort(controller.signal), + controller.signal + ); + + controller.abort(); // no reason argument -> default DOMException + + await assert.rejects(pending, (err) => { + assert.ok(err instanceof Error); + assert.equal(err.name, "AbortError"); + return true; + }); +}); diff --git a/tests/unit/test-all-model-status.test.ts b/tests/unit/test-all-model-status.test.ts index 8ec63149bf..c70067bedb 100644 --- a/tests/unit/test-all-model-status.test.ts +++ b/tests/unit/test-all-model-status.test.ts @@ -47,6 +47,10 @@ test("rate-limited / timeout failures show 'error' but are NOT auto-hidden", () status: "error", shouldHide: false, }); + assert.deepEqual(evaluateTestAllEntry({ status: "error", isTransient: true }, true), { + status: "error", + shouldHide: false, + }); // Toggle off → still not hidden, of course. assert.deepEqual(evaluateTestAllEntry({ status: "error", rateLimited: true }, false), { status: "error", @@ -54,6 +58,13 @@ test("rate-limited / timeout failures show 'error' but are NOT auto-hidden", () }); }); +test("slow batch probes remain visible and unconfirmed", () => { + assert.deepEqual(evaluateTestAllEntry({ status: "slow", isTimeout: true }, true), { + status: "error", + shouldHide: false, + }); +}); + test("missing / null / empty entry is treated as a failure", () => { for (const entry of [undefined, null, {}]) { const out = evaluateTestAllEntry(entry, true);