diff --git a/open-sse/services/autoCombo/chaosEngine.ts b/open-sse/services/autoCombo/chaosEngine.ts index 89813fe48f..32f5b09f48 100644 --- a/open-sse/services/autoCombo/chaosEngine.ts +++ b/open-sse/services/autoCombo/chaosEngine.ts @@ -180,7 +180,22 @@ function dispatchOnePanelModel(opts: { log?.info?.( `CHAOS panel ${index} (${model}) ok=${res.ok} status=${res.status} textLen=${text.length}` ); - const part: ChaosPart = { model, index, ok: true, text }; + // G5b: honor the upstream response status — a 4xx/5xx is a panel FAILURE, + // not a success (previously ok:true was hardcoded, so an all-error panel + // never reached the all-failed branch and the error text was streamed as + // if it were a successful answer). + if (res.ok) { + const part: ChaosPart = { model, index, ok: true, text }; + await onResult?.(part); + return part; + } + const part: ChaosPart = { + model, + index, + ok: false, + text: "", + error: `upstream ${res.status}: ${text.slice(0, 200) || res.statusText || "error"}`, + }; await onResult?.(part); return part; } catch (err) { @@ -466,8 +481,17 @@ export async function handleChaosChat(opts: { } if (successes.length === 0) { - const errText = "All chaos panel models failed"; - await safeEnqueue(chatChunk(chunkId, panelToDispatch[0] ?? "", errText)); + // G5 (silent-stop fix): make an all-panel failure visible server-side. + // The status stays 200 (SSE envelope must stay well-formed), but the + // failure is now logged with the per-model errors so operators can see + // why the chaos panel produced nothing. + const modelErrors = allParts.map((p) => `${p.model}: ${p.error ?? "unknown"}`).join(" | "); + log?.warn?.( + "CHAOS", + `All chaos panel models failed for ${comboName ?? "panel"}: ${modelErrors}` + ); + const errText = `All chaos panel models failed — ${modelErrors}`; + await safeEnqueue(chatChunk(chunkId, panelToDispatch[0] ?? panel[0] ?? "", errText)); await safeEnqueue(SSE_DONE); await enqueueChain; closed = true; diff --git a/open-sse/services/autoCombo/pipelineRouter.ts b/open-sse/services/autoCombo/pipelineRouter.ts index 5fbc9eea02..bc2ce3c5d5 100644 --- a/open-sse/services/autoCombo/pipelineRouter.ts +++ b/open-sse/services/autoCombo/pipelineRouter.ts @@ -343,6 +343,17 @@ export async function handlePipelineCombo({ } } + // G6 (silent-stop fix): if the reflection loop burned its retry budget and the + // verdict is still "fail", the fall-through below returns a FAILED result + // indistinguishable from a first-attempt failure. Surface it loudly so the + // caller (and operator logs) can tell "retries exhausted" apart. + if (result.reflectVerdict === "fail" && reflectionCount > 0) { + log.warn( + "PIPELINE", + `Reflection retries exhausted (${reflectionCount}/${maxReflectionLoops}) — pipeline verdict still "fail", returning the original failed result` + ); + } + // ── Return result ───────────────────────────────────────────────────────── // Check if the last stage has a streaming Response const lastStage = result.stages[result.stages.length - 1]; diff --git a/open-sse/services/autoRefreshDaemon.ts b/open-sse/services/autoRefreshDaemon.ts index 120b081545..3a177a87ae 100644 --- a/open-sse/services/autoRefreshDaemon.ts +++ b/open-sse/services/autoRefreshDaemon.ts @@ -125,8 +125,13 @@ class AutoRefreshDaemon { `[AutoRefreshDaemon] Credential expired for "${providerId}" (${config.displayName})` ); } - } catch { - // Network errors are non-fatal — retry next cycle + } catch (err) { + // Network errors are non-fatal — retry next cycle. G8: log which + // provider failed so credential problems are not silently masked. + console.warn( + `[AutoRefreshDaemon] Network error validating credential for "${providerId}" — retry next cycle`, + err instanceof Error ? err.message : err + ); } } @@ -165,8 +170,16 @@ class AutoRefreshDaemon { } return true; - } catch { - // Network errors (timeout, DNS failure) don't mean the credential is bad + } catch (err) { + // Network errors (timeout, DNS failure) don't mean the credential is bad. + // G8 (silent-stop fix): the previous bare `catch { return true; }` swallowed + // the error entirely — operators could never tell a credential was failing + // to validate due to network trouble. Log it (provider + reason) before + // returning the fail-open result. + console.warn( + `[AutoRefreshDaemon] Network error validating credential for "${providerId}" — treated as valid (fail-open), will retry next cycle`, + err instanceof Error ? err.message : err + ); return true; } finally { clearTimeout(timeout); diff --git a/open-sse/services/batchProcessor.ts b/open-sse/services/batchProcessor.ts index 578814427a..e9fe915afd 100644 --- a/open-sse/services/batchProcessor.ts +++ b/open-sse/services/batchProcessor.ts @@ -506,14 +506,46 @@ async function processSingleItemWithRetry(item: BatchRequestItem, apiKey: string } } +// G10 (silent-stop fix): individual batch-item dispatches can hang indefinitely +// if the upstream route stalls (no signal/timeout plumbed through). Bound each +// item with a wall-clock timeout so a stuck item fails fast (recorded as an item +// error) instead of freezing the whole batch loop. The orphaned dispatch keeps +// running in the background but can no longer block the batch. +export const BATCH_ITEM_DISPATCH_TIMEOUT_MS = 120_000; + +/** + * G10: race a promise against a wall-clock deadline. Exported for unit testing + * (batch dispatch is a module-internal import, so the timeout mechanism itself + * is verified directly here). + */ +export function withItemDispatchTimeout( + promise: Promise, + timeoutMs: number, + label: string +): Promise { + let timer: ReturnType | undefined; + const timeoutPromise = new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), + timeoutMs + ); + }); + return Promise.race([promise, timeoutPromise]).finally(() => { + if (timer) clearTimeout(timer); + }); +} + async function processSingleItem(item: BatchRequestItem, apiKey: string) { const body = buildRequestBody(item); - - return await dispatch.dispatchBatchApiRequest({ - endpoint: item.url, - body, - apiKey, - }); + return withItemDispatchTimeout( + dispatch.dispatchBatchApiRequest({ + endpoint: item.url, + body, + apiKey, + }), + BATCH_ITEM_DISPATCH_TIMEOUT_MS, + `Batch item dispatch (${item.url})` + ); } export function buildRequestBody(item: BatchRequestItem) { diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 9b4069bbc0..6a1d7cbbf2 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -179,6 +179,8 @@ import { TRANSIENT_FOR_SEMAPHORE, MAX_FALLBACK_WAIT_MS, MAX_GLOBAL_ATTEMPTS, + COMBO_LOOP_SAFETY_TIMEOUT_MS, + COMBO_SAFETY_DRAIN_MS, isAllAccountsRateLimitedResponse, clampComboDepth, shouldSkipForPredictedTtft, @@ -1046,11 +1048,54 @@ async function handleComboChatInner({ const globalPromise = new Promise((res) => { globalResolve = res; }); + + // G1 (silent-stop fix): the speculative loop's `Promise.race` waits on + // `globalPromise`, which is ONLY resolved from inside a task (success or + // fatal error). If a target hangs — e.g. the operator disabled the per-model + // timeout (`targetTimeoutMs: 0`) and the upstream never settles — the race + // never resolves and the request hangs forever with no response. This safety + // promise force-resolves after the combo budget (comboTimeoutMs when set, + // otherwise a hard ceiling) so the request ALWAYS terminates with an + // actionable 504 instead of dying silently. `comboExpired` is flipped so the + // target loop stops launching new work; the existing comboExpired branch + // returns the aggregated 504. + const loopSafetyMs = + comboTimeoutMs > 0 ? comboTimeoutMs : COMBO_LOOP_SAFETY_TIMEOUT_MS; + let loopSafetyFired = false; + let loopSafetyTimer: ReturnType | null = null; + const loopSafetyPromise = new Promise((resolve) => { + loopSafetyTimer = setTimeout(() => { + loopSafetyFired = true; + log.warn( + "COMBO", + `Combo loop safety timeout (${loopSafetyMs}ms) reached without a terminal response — force-terminating` + ); + resolve( + errorResponseWithComboDiagnostics( + 504, + `Combo global timeout (${loopSafetyMs}ms) without a terminal response`, + buildComboDiag("combo_timeout"), + { code: "COMBO_TIMEOUT", type: "server_error" } + ) + ); + }, loopSafetyMs); + loopSafetyTimer.unref?.(); + }); const runningTasks = new Set>(); let anySuccess = false; // #10681: steps already recorded as dispatched (so per-target retries do not // duplicate the decision). const dispatchedTargets = new Set(); + // G1: flip comboExpired as soon as the safety timer fires so the next loop + // iteration breaks instead of launching more targets after the budget, and + // abort every in-flight target so a hung upstream actually gets cancelled + // (not just "response stops"). + const markLoopExpiredIfSafetyFired = () => { + if (loopSafetyFired) { + comboExpired = true; + for (const [, ac] of abortControllers.entries()) ac.abort(); + } + }; const abortControllers = new Map(); const zeroLatencyOptimizationsEnabled = config.zeroLatencyOptimizationsEnabled === true; const hasProtectedPriorityTarget = @@ -2363,6 +2408,17 @@ async function handleComboChatInner({ })().catch((err) => { const logError = log.error ?? log.warn; logError("COMBO", `Speculative task error for target ${i}`, err); + // G2 (silent-stop fix): never leave the speculative loop waiting on an + // unresolved globalPromise. If a task throws unexpectedly (outside + // executeTarget's error handling) and no other task succeeds, the post-loop + // `Promise.race([globalPromise, ...])` would hang forever. Resolve with a + // 502 so the request terminates with an actionable error. + if (!anySuccess && globalResolve) { + anySuccess = true; + globalResolve( + errorResponse(502, `Combo target ${i} failed with an unexpected error`) + ); + } }); runningTasks.add(task); @@ -2380,10 +2436,11 @@ async function handleComboChatInner({ timeoutResolve = r; setTimeout(r, hedgeDelay); }); - await Promise.race([task, globalPromise, timeoutPromise]); + await Promise.race([task, globalPromise, timeoutPromise, loopSafetyPromise]); } else { - await Promise.race([task, globalPromise]); + await Promise.race([task, globalPromise, loopSafetyPromise]); } + markLoopExpiredIfSafetyFired(); // Global combo timeout check: after each target completes, stop trying // further targets if the total elapsed time exceeds comboTimeoutMs. @@ -2398,13 +2455,51 @@ async function handleComboChatInner({ } if (!anySuccess && runningTasks.size > 0) { - await Promise.race([globalPromise, Promise.all([...runningTasks])]); + // G1: include loopSafetyPromise so a hung last task (per-model timeout + // disabled) cannot freeze this post-loop race forever. + await Promise.race([globalPromise, Promise.all([...runningTasks]), loopSafetyPromise]); + markLoopExpiredIfSafetyFired(); + } + + // G1: if the safety timer won the race (request would otherwise hang), give + // in-flight tasks a short drain window to land their per-model errors into + // comboErrors so the 504 carries the same "tried: a (500)" summary the + // regular comboExpired branch produces — then return the safety 504. + if (loopSafetyFired && !anySuccess) { + if (runningTasks.size > 0) { + await Promise.race([ + Promise.allSettled([...runningTasks]), + new Promise((resolve) => setTimeout(resolve, COMBO_SAFETY_DRAIN_MS)), + ]); + } + const summary = comboErrors + .slice(0, 5) + .map((e) => `${e.model} (${e.status})`) + .join(", "); + const msg = + `Combo global timeout (${loopSafetyMs}ms) after ${recordedAttempts}/${orderedTargets.length} targets` + + (comboErrors.length > 0 + ? ` | tried: ${summary}${comboErrors.length > 5 ? `... (+${comboErrors.length - 5})` : ""}` + : "") + + " without a terminal response"; + return errorResponseWithComboDiagnostics( + 504, + msg, + buildComboDiag("combo_timeout"), + { code: "COMBO_TIMEOUT", type: "server_error" } + ); } // #10681: finalize the decision trace (success). finalizeComboTrace(traceInvocationId, orderedTargets); finishComboTrace(traceInvocationId, { status: 200 }); if (anySuccess) { + // G1: clear the safety timer on the happy path so a successful combo does + // not leave a 10-minute timer alive per request. + if (loopSafetyTimer) { + clearTimeout(loopSafetyTimer); + loopSafetyTimer = null; + } return await globalPromise; } @@ -2923,6 +3018,33 @@ async function handleRoundRobinCombo({ // and the "Done with this model" path below), mirroring handleComboChat. const rrOutcomes: Array = []; + // G4 (silent-stop fix): round-robin has NO global timeout — a hung model + // (per-model timeout disabled via targetTimeoutMs: 0) would freeze the request + // forever with no response. Safety promise + timer bound the whole loop; when + // it fires, rrExpired flips and every subsequent model attempt short-circuits + // to the 504. Cleaned up in the loop's finally. + const rrConfiguredTimeoutMs = + (config as { comboTimeoutMs?: number }).comboTimeoutMs ?? 0; + const rrLoopSafetyMs = + rrConfiguredTimeoutMs > 0 ? rrConfiguredTimeoutMs : COMBO_LOOP_SAFETY_TIMEOUT_MS; + let rrExpired = false; + let rrLoopSafetyTimer: ReturnType | null = null; + let rrResolveSafety: ((res: Response) => void) | null = null; + const rrSafetyPromise = new Promise((resolve) => { + rrResolveSafety = resolve; + }); + rrLoopSafetyTimer = setTimeout(() => { + rrExpired = true; + log.warn( + "COMBO-RR", + `Round-robin loop exceeded ${rrLoopSafetyMs}ms without a terminal response — force-terminating` + ); + rrResolveSafety?.( + errorResponse(504, `Round-robin combo exceeded ${rrLoopSafetyMs}ms without a terminal response`) + ); + }, rrLoopSafetyMs); + rrLoopSafetyTimer.unref?.(); + // #1731: Per-request in-memory set of providers whose quota is fully exhausted. // When a target returns a quota-exhausted 429, remaining targets from the same // provider are skipped to avoid the cascade through N same-provider targets. @@ -2931,8 +3053,11 @@ async function handleRoundRobinCombo({ const transientRateLimitedProviders = new Set(); // Try each model starting from the round-robin target - for (let offset = 0; offset < modelCount; offset++) { - const modelIndex = (rrStartIndex + offset) % modelCount; + try { + for (let offset = 0; offset < modelCount; offset++) { + // G4: stop launching new work once the safety timer fired. + if (rrExpired) break; + const modelIndex = (rrStartIndex + offset) % modelCount; const target = filteredTargets[modelIndex]; const modelStr = target.modelStr; const provider = target.provider; @@ -3077,11 +3202,15 @@ async function handleRoundRobinCombo({ fingerprint: resolveTargetFingerprint(target) ?? "", }); - const result = await handleSingleModel(attemptBody, modelStr, { - ...targetForAttempt, - effectiveComboStrategy: "round-robin", - failoverBeforeRetry: config.failoverBeforeRetry, - }); + const result = await Promise.race([ + handleSingleModel(attemptBody, modelStr, { + ...targetForAttempt, + effectiveComboStrategy: "round-robin", + failoverBeforeRetry: config.failoverBeforeRetry, + }), + rrSafetyPromise, + ]); + if (rrExpired) return result; // G4: safety timer won — stop everything // Quota-aware scheduling: reserve the estimated budget for this // dispatch (opt-in, same env gate as the pre-request check). Best-effort @@ -3519,6 +3648,26 @@ async function handleRoundRobinCombo({ release(); } } + } catch (err) { + // G4: unexpected exception in the round-robin loop must never crash the + // request silently — surface a 500 instead of hanging the client. + log.error?.("COMBO-RR", "Unexpected error in round-robin loop", err); + return errorResponse(500, "Unexpected error in round-robin combo"); + } finally { + if (rrLoopSafetyTimer) { + clearTimeout(rrLoopSafetyTimer); + rrLoopSafetyTimer = null; + } + } + + // G4: if the safety timer fired between iterations (no race captured it), + // terminate with the actionable 504 instead of the generic exhaustion path. + if (rrExpired) { + return errorResponse( + 504, + `Round-robin combo exceeded ${rrLoopSafetyMs}ms without a terminal response` + ); + } // All models exhausted const latencyMs = Date.now() - startTime; diff --git a/open-sse/services/combo/comboPredicates.ts b/open-sse/services/combo/comboPredicates.ts index dc87509236..f7cfa3322d 100644 --- a/open-sse/services/combo/comboPredicates.ts +++ b/open-sse/services/combo/comboPredicates.ts @@ -18,6 +18,15 @@ import type { ResolvedComboTarget } from "./types.ts"; // Status codes that should mark round-robin target semaphores as cooling down. export const TRANSIENT_FOR_SEMAPHORE = [429, 502, 503, 504]; +// G1 (silent-stop fix): hard ceiling for the combo target loop when the operator +// left comboTimeoutMs at 0 ("unlimited"). Without this, a hung upstream (per-model +// timeout disabled) would freeze the request forever with no response. 10 minutes +// is a generous bound for legitimate long-running fallback cascades. +export const COMBO_LOOP_SAFETY_TIMEOUT_MS = 10 * 60 * 1000; +// G1: after the safety timer fires, wait this long for in-flight targets to land +// their per-model errors into comboErrors (so the 504 carries the same "tried:" +// summary as the regular timeout path) before returning the safety response. +export const COMBO_SAFETY_DRAIN_MS = 2000; // Patterns that signal all accounts for a provider are rate-limited / exhausted. // Used to detect 503 responses from handleNoCredentials so combo can fallback. export const ALL_ACCOUNTS_RATE_LIMITED_PATTERNS = [ diff --git a/open-sse/services/combo/targetTimeoutRunner.ts b/open-sse/services/combo/targetTimeoutRunner.ts index 4eb6cb8b68..fb5c2262f0 100644 --- a/open-sse/services/combo/targetTimeoutRunner.ts +++ b/open-sse/services/combo/targetTimeoutRunner.ts @@ -106,6 +106,13 @@ export function buildTargetTimeoutRunner(deps: { target?: SingleModelTarget ): Promise => { if (comboTargetTimeoutMs <= 0) { + // G3 (silent-stop fix): a disabled per-model timeout means a hung upstream + // stalls the target until the combo loop safety timer (COMBO_LOOP_SAFETY_TIMEOUT_MS) + // force-terminates — surface that dependency instead of silently running bare. + log.warn( + "COMBO", + `Per-model combo timeout is DISABLED (comboTargetTimeoutMs=${comboTargetTimeoutMs}) for ${modelStr} — a hung upstream will hang this target until the combo loop safety timeout` + ); return handleSingleModel(b, modelStr, target).catch((err) => errorResponse(502, err?.message ?? "Upstream model error") ); diff --git a/src/lib/evals/evalRunner.ts b/src/lib/evals/evalRunner.ts index d67ea9d74b..d0b9280013 100644 --- a/src/lib/evals/evalRunner.ts +++ b/src/lib/evals/evalRunner.ts @@ -9,6 +9,7 @@ */ import { getCustomEvalSuite, listCustomEvalSuites } from "@/lib/db/evals"; +import safeRegex from "safe-regex"; import { goldenSet, codingSuite, @@ -161,6 +162,16 @@ export function evaluateCase(evalCase: any, actualOutput: string) { details.error = "Regex pattern too large for safe evaluation."; break; } + // G7 (silent-stop fix): a catastrophic regex (nested quantifiers like + // `(a+)+$`) can hang the event loop for minutes on adversarial output — + // the eval loop then "stops doing anything" with no error. safe-regex + // statically rejects such patterns before test() runs. + if (!safeRegex(regex)) { + passed = false; + details.error = + "Regex pattern rejected as potentially unsafe (catastrophic backtracking risk). Simplify the pattern."; + break; + } passed = regex.test(actualOutput); details.pattern = String(expectedValue); break; diff --git a/tests/unit/combo-silent-stop-gaps.test.ts b/tests/unit/combo-silent-stop-gaps.test.ts new file mode 100644 index 0000000000..3b101a20c0 --- /dev/null +++ b/tests/unit/combo-silent-stop-gaps.test.ts @@ -0,0 +1,305 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +// Isolate DATA_DIR before any DB-touching import (combo.ts pulls in the +// SQLite layer) — mirrors tests/unit/combo-routing-engine.test.ts. +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-silent-stop-gaps-")); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +process.env.DATA_DIR = TEST_DATA_DIR; + +// --------------------------------------------------------------------------- +// Silent-stop gap fixes — regression tests +// --------------------------------------------------------------------------- +// These lock the G1–G10 fixes for loops that could terminate silently: +// G1 combo.ts globalPromise safety timer (hung target → 504, never hang) +// G2 combo.ts task wrapper catch resolves globalPromise (unexpected throw → 502) +// G3 targetTimeoutRunner warns when per-model timeout is disabled +// G4 round-robin loop safety timer (hung model → 504) +// G5 chaosEngine logs all-panel failures with per-model errors +// G7 evalRunner rejects catastrophic regex (ReDoS) via safe-regex +// G8 autoRefreshDaemon logs network errors per provider +// G10 batchProcessor item dispatch timeout (hung item fails fast) +// --------------------------------------------------------------------------- + +const { buildTargetTimeoutRunner } = + await import("../../open-sse/services/combo/targetTimeoutRunner.ts"); +const { handleComboChat } = await import("../../open-sse/services/combo.ts"); +const { handleChaosChat } = await import("../../open-sse/services/autoCombo/chaosEngine.ts"); +const { evaluateCase } = await import("../../src/lib/evals/evalRunner.ts"); +const { withItemDispatchTimeout } = await import("../../open-sse/services/batchProcessor.ts"); +const { saveModelsDevCapabilities } = await import("../../src/lib/modelsDevSync.ts"); + +function capabilityEntry(limitContext: unknown, overrides: Record = {}) { + return { + tool_call: true, + reasoning: false, + attachment: false, + structured_output: true, + temperature: true, + modalities_input: JSON.stringify(["text"]), + modalities_output: JSON.stringify(["text"]), + knowledge_cutoff: null, + release_date: null, + last_updated: null, + status: null, + family: null, + open_weights: false, + limit_context: limitContext, + limit_input: limitContext, + limit_output: 4096, + interleaved_field: null, + ...overrides, + }; +} + +function createLog() { + const entries: Array<{ level: string; tag: string; msg: string }> = []; + return { + info: (tag: string, msg: string) => entries.push({ level: "info", tag, msg }), + warn: (tag: string, msg: string) => entries.push({ level: "warn", tag, msg }), + error: (tag: string, msg: string) => entries.push({ level: "error", tag, msg }), + debug: (tag: string, msg: string) => entries.push({ level: "debug", tag, msg }), + entries, + }; +} + +function okResponse(body: Record = { choices: [{ message: { content: "ok" } }] }) { + return new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" }, + }); +} + +function errorResponse(status: number, message: string = `Error ${status}`) { + return new Response(JSON.stringify({ error: { message } }), { + status, + headers: { "content-type": "application/json" }, + }); +} + +// ── G1: combo loop safety timer ───────────────────────────────────────────── +// A hung target (per-model timeout disabled / upstream never settles) must NOT +// freeze the request forever: the safety timer force-resolves with 504. +test.after(async () => { + if (ORIGINAL_DATA_DIR === undefined) { + delete process.env.DATA_DIR; + } else { + process.env.DATA_DIR = ORIGINAL_DATA_DIR; + } + try { + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + } catch { + /* best effort */ + } +}); + +test("G1: hung target with per-model timeout disabled → 504, not a silent hang", async () => { + const log = createLog(); + const combo = { + name: "g1-hang", + models: ["openai/gpt-4o-mini"], + config: { + // Per-model timeout off + tiny global budget so the test runs in ms. + targetTimeoutMs: 0, + comboTimeoutMs: 100, + }, + }; + saveModelsDevCapabilities({ openai: { "gpt-4o-mini": capabilityEntry(128000) } }); + const startedAt = Date.now(); + const result = await handleComboChat({ + body: {}, + combo, + // Never settles — simulates an upstream that accepts the connection and + // then stalls forever. + handleSingleModel: () => new Promise(() => {}), + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + const elapsed = Date.now() - startedAt; + assert.equal(result.status, 504); + assert.ok(elapsed < 10_000, `safety timeout took ${elapsed}ms — too slow`); + assert.ok( + log.entries.some((e) => e.level === "warn" && /safety timeout/i.test(String(e.msg))), + "expected a warn about the combo loop safety timeout" + ); +}); + +// ── G2: task wrapper catch resolves globalPromise ─────────────────────────── +// An unexpected throw inside executeTarget (e.g. isModelAvailable exploding) +// must surface as 502, not leave the request hanging. +test("G2: unexpected throw in a target surfaces 502 instead of hanging", async () => { + const log = createLog(); + const combo = { + name: "g2-throw", + models: ["openai/gpt-4o-mini"], + }; + const result = await handleComboChat({ + body: {}, + combo, + handleSingleModel: async () => okResponse(), + // Throw inside executeTarget before dispatch — the wrapper catch must + // resolve globalPromise so the loop terminates. + isModelAvailable: async () => { + throw new Error("unexpected availability explosion"); + }, + log, + settings: null, + allCombos: null, + }); + assert.equal(result.status, 502); + const body = await result.text(); + assert.match(body, /unexpected error/i); + assert.ok( + log.entries.some((e) => e.level === "error" && /Speculative task error/i.test(String(e.msg))), + "expected the speculative task error to be logged" + ); +}); + +// ── G3: targetTimeoutRunner warns when per-model timeout is disabled ──────── +test("G3: targetTimeoutRunner warns when comboTargetTimeoutMs <= 0", async () => { + const log = createLog(); + const runner = buildTargetTimeoutRunner({ + handleSingleModel: async () => new Response("ok"), + comboTargetTimeoutMs: 0, + log, + }); + const res = await runner({}, "m"); + assert.equal(await res.text(), "ok"); + assert.ok( + log.entries.some((e) => e.level === "warn" && /DISABLED|disabled/i.test(String(e.msg))), + "expected a warn about the disabled per-model timeout" + ); +}); + +// ── G4: round-robin loop safety timer ─────────────────────────────────────── +test("G4: round-robin hung model → 504 via loop safety timer", async () => { + const log = createLog(); + const combo = { + name: "g4-rr-hang", + models: ["openai/gpt-4o-mini", "claude/sonnet"], + strategy: "round-robin", + config: { + targetTimeoutMs: 0, + comboTimeoutMs: 100, + }, + }; + saveModelsDevCapabilities({ + openai: { "gpt-4o-mini": capabilityEntry(128000) }, + claude: { sonnet: capabilityEntry(200000) }, + }); + const startedAt = Date.now(); + const result = await handleComboChat({ + body: {}, + combo, + handleSingleModel: () => new Promise(() => {}), + isModelAvailable: async () => true, + log, + settings: null, + allCombos: null, + }); + const elapsed = Date.now() - startedAt; + assert.equal(result.status, 504); + assert.ok(elapsed < 10_000, `RR safety timeout took ${elapsed}ms — too slow`); + assert.ok( + log.entries.some((e) => e.level === "warn" && /Round-robin/i.test(String(e.msg))), + "expected a warn about the round-robin safety timeout" + ); +}); + +// ── G5: chaosEngine logs all-panel failures ───────────────────────────────── +test("G5: chaos all-panel failure is logged with per-model errors", async () => { + const log = createLog(); + const res = await handleChaosChat({ + body: {}, + models: ["openai/gpt-4o-mini", "claude/sonnet"], + handleSingleModel: async () => errorResponse(503, "upstream down"), + log, + comboName: "g5-chaos", + }); + assert.equal(res.status, 200); // SSE envelope stays well-formed + const body = await res.text(); + assert.match(body, /All chaos panel models failed/); + assert.match(body, /upstream down/); + assert.ok( + log.entries.some( + (e) => e.level === "warn" && /All chaos panel models failed/i.test(String(e.msg)) + ), + "expected the all-failed warn with model errors" + ); +}); + +// ── G7: evalRunner rejects catastrophic regex (ReDoS guard) ───────────────── +test("G7: catastrophic regex is rejected instead of hanging the eval loop", () => { + const startedAt = Date.now(); + const result = evaluateCase( + { id: "redos", name: "redos", expected: { strategy: "regex", value: "(a+)+$" } }, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa!" + ); + const elapsed = Date.now() - startedAt; + assert.equal(result.passed, false); + assert.match(String(result.details?.error ?? ""), /unsafe|backtracking/i); + assert.ok(elapsed < 1000, `regex eval took ${elapsed}ms — ReDoS guard failed`); +}); + +test("G7: benign regex still evaluates normally", () => { + const result = evaluateCase( + { id: "ok-regex", name: "ok-regex", expected: { strategy: "regex", value: "^hello" } }, + "hello world" + ); + assert.equal(result.passed, true); +}); + +// ── G8: autoRefreshDaemon logs network errors per provider ────────────────── +test("G8: autoRefreshDaemon logs network errors instead of swallowing them", async () => { + const { autoRefreshDaemon } = await import("../../open-sse/services/autoRefreshDaemon.ts"); + const originalFetch = globalThis.fetch; + const originalWarn = console.warn; + const warnings: string[] = []; + console.warn = (...args: unknown[]) => warnings.push(args.map(String).join(" ")); + // Simulate a provider whose validation request blows up (network error). + globalThis.fetch = (async () => { + throw new Error("ECONNRESET"); + }) as typeof fetch; + try { + autoRefreshDaemon.registerCredential("claude-web", "cookie-value"); + await autoRefreshDaemon.check(); + assert.ok( + warnings.some((w) => w.includes("claude-web") && w.includes("ECONNRESET")), + "expected a warn naming the provider and the network error, got: " + warnings.join(" | ") + ); + } finally { + globalThis.fetch = originalFetch; + console.warn = originalWarn; + autoRefreshDaemon.unregisterCredential("claude-web"); + } +}); + +// ── G10: batch item dispatch timeout ──────────────────────────────────────── +test("G10: hung batch item dispatch fails fast via wall-clock timeout", async () => { + const startedAt = Date.now(); + await assert.rejects( + withItemDispatchTimeout( + new Promise(() => {}), // never settles + 50, + "Batch item dispatch (/v1/chat/completions)" + ), + /timed out after 50ms/ + ); + const elapsed = Date.now() - startedAt; + assert.ok(elapsed < 5000, `timeout fired after ${elapsed}ms — too slow`); +}); + +test("G10: fast dispatch wins the race untouched", async () => { + const res = await withItemDispatchTimeout( + Promise.resolve(new Response("ok", { status: 200 })), + 1000, + "Batch item dispatch" + ); + assert.equal(res.status, 200); + assert.equal(await res.text(), "ok"); +});