diff --git a/changelog.d/fixes/12956-weighted-combo-cooling-down-503.md b/changelog.d/fixes/12956-weighted-combo-cooling-down-503.md new file mode 100644 index 0000000000..0df9ca8845 --- /dev/null +++ b/changelog.d/fixes/12956-weighted-combo-cooling-down-503.md @@ -0,0 +1 @@ +- **fix(combo):** A weighted combo whose every target was excluded before dispatch by a resilience timer (model lockout, open circuit breaker, provider cooldown) now answers `503` `all_targets_cooling_down` with `Retry-After` set to the earliest exclusion to lapse, the excluded targets and reasons in `diagnostics.excluded`, a `wait` recovery hint, and a `[COMBO]` warning naming the reasons; previously the pool was dropped silently and the host answered `404 "Combo has no executable targets"` (recovery hint "switch combo / reconnect the missing providers") for a pool that was configured, connected and merely cooling down — which clients such as Claude Code render as "this model may not exist". A pool with nothing to run keeps its `404` ([#12954](https://github.com/diegosouzapw/OmniRoute/issues/12954), [#12956](https://github.com/diegosouzapw/OmniRoute/pull/12956) — thanks @insoln) diff --git a/docs/architecture/RESILIENCE_GUIDE.md b/docs/architecture/RESILIENCE_GUIDE.md index b5196fedab..cc49c443b2 100644 --- a/docs/architecture/RESILIENCE_GUIDE.md +++ b/docs/architecture/RESILIENCE_GUIDE.md @@ -634,6 +634,7 @@ rate limit is the same signal as an exhausted quota. Honest limits: ## Debugging +- Weighted combo answers `503 all_targets_cooling_down` (`Retry-After` set, `diagnostics.excluded` lists every target with `model_lockout` / `circuit_open` / `provider_cooldown` / `unavailable`) → the pool is configured and connected, every target is just excluded by a resilience timer; the `[COMBO] Weighted selection: every target excluded before dispatch — …` warning names the reasons and remaining seconds. A `404 no_executable_targets` from the same combo means no resilience timer was involved (nothing to run, or every account failed the availability probe). Built in `open-sse/services/combo/pinRecovery.ts` from the exclusions collected in `targetResolution.ts`. - All keys for a provider skipped → check both circuit breaker state AND each connection's `rateLimitedUntil`/`testStatus`. - Provider permanently excluded after reset window → code reading raw `state` instead of `getStatus()`/`canExecute()`. - One key fails, others should work → prefer connection cooldown over circuit breaker. diff --git a/open-sse/services/combo/pinRecovery.ts b/open-sse/services/combo/pinRecovery.ts index 6923eb5204..528dc00d5f 100644 --- a/open-sse/services/combo/pinRecovery.ts +++ b/open-sse/services/combo/pinRecovery.ts @@ -1,4 +1,8 @@ -import type { ComboDiagnostics, ComboRecoveryHint } from "../../utils/error.ts"; +import { + errorResponseWithComboDiagnostics, + type ComboDiagnostics, + type ComboRecoveryHint, +} from "../../utils/error.ts"; /** * Build the recovery hint that travels with a terminal combo failure. Lives @@ -47,6 +51,15 @@ export function buildRecoveryHint( ? { retry_after_seconds: retryAfterSeconds } : {}), }; + case "all_targets_cooling_down": + return { + action: "wait", + next_step: + "Every target is temporarily excluded by resilience state (model lockout, circuit breaker or provider cooldown); the pool itself is configured and connected. Wait for the cooldown and retry, or switch combo.", + ...(typeof retryAfterSeconds === "number" && retryAfterSeconds > 0 + ? { retry_after_seconds: retryAfterSeconds } + : {}), + }; case "no_executable_targets": return { action: "switch-combo", @@ -133,3 +146,80 @@ export function buildEmptyComboTargetsPayload( }, }; } + +/** Which weighted-selection gate dropped a target before dispatch. */ +export type PreDispatchExclusionReason = + "circuit_open" | "provider_cooldown" | "model_lockout" | "free_tier_drained" | "unavailable"; + +export interface PreDispatchExclusion { + provider: string; + /** Bare model id (no provider prefix) — the diagnostics header joins provider/model. */ + model: string; + reason: PreDispatchExclusionReason; + /** Remaining exclusion time when the gate knows it (resilience gates), else null. */ + retryAfterMs: number | null; +} + +/** + * Gates whose exclusion is a timer on a configured, connected target — the pool + * is intact, it is just cooling down. `unavailable` (the host's account probe) + * and `free_tier_drained` are not timers the combo layer can vouch for. + */ +const TEMPORARY_EXCLUSION_REASONS: ReadonlySet = new Set([ + "circuit_open", + "provider_cooldown", + "model_lockout", +]); + +/** One-line operator summary: `openai/a: model_lockout (57s), claude/b: circuit_open`. */ +export function formatPreDispatchExclusions(exclusions: readonly PreDispatchExclusion[]): string { + return exclusions + .map((e) => { + const wait = + typeof e.retryAfterMs === "number" && e.retryAfterMs > 0 + ? ` (${Math.ceil(e.retryAfterMs / 1000)}s)` + : ""; + return `${e.provider}/${e.model}: ${e.reason}${wait}`; + }) + .join(", "); +} + +/** + * The weighted strategy filters targets before dispatch (breaker, provider + * cooldown, model lockout, availability probe). When that leaves nothing, the + * host used to answer 404 `no_executable_targets` — "switch combo / reconnect + * the missing providers" — for a pool that is configured, connected and merely + * cooling down, and clients such as Claude Code render a 404 as "this model may + * not exist". When at least one target was excluded by a resilience timer this + * builds the 503 the in-loop skip path already uses, with `Retry-After` set to + * the earliest exclusion to lapse and every excluded target in the diagnostics. + * Returns null when no resilience gate was involved (caller keeps its 404). + */ +export function buildAllTargetsCoolingDownResponse( + exclusions: readonly PreDispatchExclusion[] +): Response | null { + const cooling = exclusions.filter((e) => TEMPORARY_EXCLUSION_REASONS.has(e.reason)); + if (cooling.length === 0) return null; + const known = cooling + .map((e) => e.retryAfterMs) + .filter((ms): ms is number => typeof ms === "number" && ms > 0); + const retryAfterSeconds = + known.length > 0 ? Math.max(1, Math.ceil(Math.min(...known) / 1000)) : undefined; + const response = errorResponseWithComboDiagnostics( + 503, + "Service temporarily unavailable: every target in this combo is cooling down (model lockout, circuit breaker or provider cooldown)", + { + poolSize: exclusions.length, + attempted: 0, + excluded: exclusions.map(({ provider, model, reason }) => ({ provider, model, reason })), + attemptOrder: [], + terminalReason: "all_targets_cooling_down", + recovery: buildRecoveryHint("all_targets_cooling_down", retryAfterSeconds), + }, + { code: "all_targets_cooling_down", type: "service_unavailable" } + ); + if (retryAfterSeconds !== undefined) { + response.headers.set("Retry-After", String(retryAfterSeconds)); + } + return response; +} diff --git a/open-sse/services/combo/targetResolution.ts b/open-sse/services/combo/targetResolution.ts index dd86b4827d..8f041e73fb 100644 --- a/open-sse/services/combo/targetResolution.ts +++ b/open-sse/services/combo/targetResolution.ts @@ -24,13 +24,13 @@ * * See _tasks/quality/2026-06-19-DESIGN-godfiles-decomposition.md §4. */ -import { isModelLocked } from "../accountFallback.ts"; +import { getModelLockoutInfo, isModelLocked } from "../accountFallback.ts"; import { parseAutoPrefix } from "../autoCombo/autoPrefix.ts"; import { handlePipelineCombo, buildPipelineResponse } from "../autoCombo/pipelineRouter.ts"; import type { resolveComboSetupConfig } from "../comboConfig.ts"; import { orderTargetsByEvalScores } from "../evalRouting.ts"; import { parseModel } from "../model.ts"; -import { isProviderInCooldown } from "../providerCooldownTracker.ts"; +import { getRemainingCooldownMs, isProviderInCooldown } from "../providerCooldownTracker.ts"; import { classifyTask, getConversationCacheKey, @@ -52,7 +52,13 @@ import { } from "./comboStructure.ts"; import { applyContextRequirements } from "./contextRequirements.ts"; import { recordComboFailure } from "./failureTracker.ts"; -import { buildEmptyComboTargetsPayload, buildRecoveryHint } from "./pinRecovery.ts"; +import { + buildAllTargetsCoolingDownResponse, + buildEmptyComboTargetsPayload, + buildRecoveryHint, + formatPreDispatchExclusions, + type PreDispatchExclusion, +} from "./pinRecovery.ts"; import { applyPromptCacheAffinity, expandPromptCacheAffinityTargets, @@ -146,37 +152,55 @@ type WeightedStepGroups = /** * Weighted-strategy eligibility predicate: a step counts as selectable only when at * least one of its targets clears the provider breaker, the connection cooldown, the - * per-model lockout and the caller's availability probe. + * per-model lockout and the caller's availability probe. Returns `null` for a + * selectable target, otherwise which gate excluded it and — for the resilience + * gates — how long it stays excluded, so an emptied pool can be reported as + * "cooling down" instead of the silent drop that used to end as a 404. */ -async function isTargetSelectableForWeighted( +async function describeWeightedExclusion( target: ResolvedComboTarget, resilienceSettings: ResilienceSettings, isModelAvailable?: IsModelAvailable -): Promise { +): Promise { const rawModel = parseModel(target.modelStr).model || target.modelStr; - if (target.provider && getCircuitBreaker(target.provider).getStatus().state === "OPEN") - return false; + const exclude = ( + reason: PreDispatchExclusion["reason"], + retryAfterMs: number | null = null + ): PreDispatchExclusion => ({ provider: target.provider, model: rawModel, reason, retryAfterMs }); + if (target.provider) { + const breaker = getCircuitBreaker(target.provider).getStatus(); + if (breaker.state === "OPEN") return exclude("circuit_open", breaker.retryAfterMs); + } if ( resilienceSettings.providerCooldown.enabled && Boolean(target.provider && target.provider !== "unknown") && isProviderInCooldown(target.provider, target.connectionId ?? undefined, resilienceSettings) ) { - return false; + return exclude( + "provider_cooldown", + getRemainingCooldownMs(target.provider, target.connectionId ?? undefined, resilienceSettings) + ); } if ( target.provider && rawModel && isModelLocked(target.provider, target.connectionId || "", rawModel) ) { - return false; + return exclude( + "model_lockout", + getModelLockoutInfo(target.provider, target.connectionId || "", rawModel)?.remainingMs ?? null + ); } if (target.provider && rawModel && target.connectionId) { const { isAlibabaFreeTierModelRoutable } = await import("../alibabaFreeTier.ts"); if (!(await isAlibabaFreeTierModelRoutable(target.provider, target.connectionId, rawModel))) { - return false; + return exclude("free_tier_drained"); } } - return isModelAvailable ? await isModelAvailable(target.modelStr, target) : true; + if (isModelAvailable && !(await isModelAvailable(target.modelStr, target))) { + return exclude("unavailable"); + } + return null; } /** @@ -223,22 +247,32 @@ async function collectWeightedEligibility( resilienceSettings: ResilienceSettings, isModelAvailable?: IsModelAvailable, hiddenModelsByProvider?: HiddenModelsByProvider -): Promise<{ stepGroups: WeightedStepGroups; weightedEligibleKeys: Set }> { +): Promise<{ + stepGroups: WeightedStepGroups; + weightedEligibleKeys: Set; + /** Targets of the steps that had no selectable target — why, and for how long. */ + exclusions: PreDispatchExclusion[]; +}> { const weightedEligibleKeys = new Set(); + const exclusions: PreDispatchExclusion[] = []; const stepGroups = resolveWeightedStepGroups( expandedCombo, expandedAllCombos, hiddenModelsByProvider ); for (const group of stepGroups) { - const availability = await Promise.all( + const verdicts = await Promise.all( group.targets.map((target) => - isTargetSelectableForWeighted(target, resilienceSettings, isModelAvailable) + describeWeightedExclusion(target, resilienceSettings, isModelAvailable) ) ); - if (availability.some(Boolean)) weightedEligibleKeys.add(group.step.executionKey); + if (verdicts.some((verdict) => verdict === null)) { + weightedEligibleKeys.add(group.step.executionKey); + } else { + for (const verdict of verdicts) if (verdict) exclusions.push(verdict); + } } - return { stepGroups, weightedEligibleKeys }; + return { stepGroups, weightedEligibleKeys, exclusions }; } /** @@ -271,12 +305,17 @@ async function resolveWeightedSelection( expandedCombo: ComboLike, expandedAllCombos: ComboCollectionLike, stickyWeightedLimit: number -): Promise<{ weightedResolution: WeightedResolution; stickyWeightedKey: string | null }> { +): Promise<{ + weightedResolution: WeightedResolution; + stickyWeightedKey: string | null; + exclusions: PreDispatchExclusion[]; +}> { const { strategy } = deps; const comboName = deps.combo.name; evictOldestWeightedSticky(strategy, comboName); let stepGroups: WeightedStepGroups; let weightedEligibleKeys = new Set(); + let exclusions: PreDispatchExclusion[] = []; if (strategy === "weighted") { const eligibility = await collectWeightedEligibility( expandedCombo, @@ -287,6 +326,7 @@ async function resolveWeightedSelection( ); stepGroups = eligibility.stepGroups; weightedEligibleKeys = eligibility.weightedEligibleKeys; + exclusions = eligibility.exclusions; } const stickyWeightedKey = resolveStickyWeightedKey( strategy, @@ -304,7 +344,7 @@ async function resolveWeightedSelection( stepGroups ) : null; - return { weightedResolution, stickyWeightedKey }; + return { weightedResolution, stickyWeightedKey, exclusions }; } /** Maps an attempted target back to the weighted step it came from (sticky write-back). */ @@ -696,6 +736,31 @@ async function applyPromptCacheStage( return nextTargets; } +function buildWeightedExhaustionResponse( + deps: ResolveComboTargetPipelineDeps, + weightedResolution: WeightedResolution, + exclusions: PreDispatchExclusion[] +): Response | null { + if (deps.strategy !== "weighted" || (weightedResolution?.orderedTargets.length ?? 0) > 0) { + return null; + } + // Every step was excluded before dispatch. When a resilience timer (model + // lockout, open breaker, provider cooldown) did it, the pool is configured and + // connected and merely cooling down: answer 503 + Retry-After with the + // excluded targets, not the host's 404 "no executable targets / switch combo". + const coolingDown = buildAllTargetsCoolingDownResponse(exclusions); + if (!coolingDown) return null; + deps.log.warn( + "COMBO", + `Weighted selection: every target excluded before dispatch — ${formatPreDispatchExclusions(exclusions)}` + ); + recordComboFailure( + deps.combo.context_cache_protection ? (deps.relayOptions?.sessionId ?? null) : null, + deps.combo.name + ); + return coolingDown; +} + export async function resolveComboTargetPipeline( deps: ResolveComboTargetPipelineDeps ): Promise { @@ -705,13 +770,15 @@ export async function resolveComboTargetPipeline( const stickyWeightedLimit = clampStickyWeightedTargetLimit( (config as Record).stickyWeightedLimit ); - const { weightedResolution, stickyWeightedKey } = await resolveWeightedSelection( + const { weightedResolution, stickyWeightedKey, exclusions } = await resolveWeightedSelection( deps, expandedCombo, expandedAllCombos, stickyWeightedLimit ); const getWeightedStepKeyForTarget = buildWeightedStepKeyMapper(weightedResolution); + const weightedExhaustion = buildWeightedExhaustionResponse(deps, weightedResolution, exclusions); + if (weightedExhaustion) return { earlyResponse: weightedExhaustion }; let orderedTargets = strategy === "weighted" ? weightedResolution?.orderedTargets || [] diff --git a/open-sse/utils/error.ts b/open-sse/utils/error.ts index 9a86344eb0..2f3b3ee868 100644 --- a/open-sse/utils/error.ts +++ b/open-sse/utils/error.ts @@ -50,6 +50,7 @@ const SAFE_PUBLIC_ERROR_IDENTIFIERS = new Set([ "admission_shutdown", "admission_unavailable", "all_accounts_inactive", + "all_targets_cooling_down", "all_targets_skipped", "antigravity_pre_response_timeout", "api_error", diff --git a/stryker.conf.json b/stryker.conf.json index 83a6099e74..6a5dbb7ce8 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -230,6 +230,7 @@ "tests/unit/combo-system-prompt-templates-5501.test.ts", "tests/unit/combo-target-defensive-modelstr.test.ts", "tests/unit/combo-vision-aware-routing.test.ts", + "tests/unit/combo-weighted-all-targets-cooling-down.test.ts", "tests/unit/combo/auto-quota-cutoff.test.ts", "tests/unit/combo/auto-status-penalty-4540.test.ts", "tests/unit/combo/combo-exhausted-skip.test.ts", diff --git a/tests/unit/combo-weighted-all-targets-cooling-down.test.ts b/tests/unit/combo-weighted-all-targets-cooling-down.test.ts new file mode 100644 index 0000000000..3561c4b145 --- /dev/null +++ b/tests/unit/combo-weighted-all-targets-cooling-down.test.ts @@ -0,0 +1,204 @@ +/** + * Weighted combos filter targets before dispatch (model lockout, circuit breaker, + * provider cooldown, availability probe). When that leaves nothing, the response + * used to be the host's 404 `no_executable_targets` — "switch combo / reconnect + * providers" — for a pool that is configured, connected and merely cooling down; + * Claude Code renders a 404 as "this model may not exist". A pool emptied by + * resilience timers now answers 503 + Retry-After with every excluded target in + * the diagnostics. A pool with nothing to run keeps its 404. + * + * Harness mirrors tests/unit/combo-strategy-fallbacks.test.ts. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-weighted-cooling-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const { handleComboChat } = await import("../../open-sse/services/combo.ts"); +const { weightedStickyTargets } = await import("../../open-sse/services/combo/rrState.ts"); +const core = await import("../../src/lib/db/core.ts"); +const { resetAllComboMetrics } = await import("../../open-sse/services/comboMetrics.ts"); +const { resetAllCircuitBreakers, getCircuitBreaker } = + await import("../../src/shared/utils/circuitBreaker.ts"); +const { recordModelLockoutFailure, clearAllModelLockouts } = + await import("../../open-sse/services/accountFallback.ts"); + +type LogEntry = { level: string; tag: unknown; msg: unknown }; +function createLog() { + const entries: LogEntry[] = []; + const push = (level: string) => (tag: unknown, msg: unknown) => { + entries.push({ level, tag, msg }); + }; + return { + info: push("info"), + warn: push("warn"), + error: push("error"), + debug: push("debug"), + entries, + }; +} + +function okResponse() { + return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), { + status: 200, + headers: { "content-type": "application/json" }, + }); +} + +const COMBO = { + name: "weighted-cooling", + strategy: "weighted", + models: [ + { model: "openai/a", weight: 50 }, + { model: "claude/b", weight: 50 }, + ], + config: { maxRetries: 0, retryDelayMs: 0, fallbackDelayMs: 0 }, +}; + +function lockModel(provider: string, model: string, cooldownMs: number) { + recordModelLockoutFailure(provider, "", model, "unknown", 502, cooldownMs, null, { + maxCooldownMs: cooldownMs, + }); +} + +async function openBreaker(provider: string) { + const breaker = getCircuitBreaker(provider, { failureThreshold: 1, resetTimeout: 30_000 }); + await breaker + .execute(async () => { + throw new Error("simulated failure"); + }) + .catch(() => {}); + assert.equal(breaker.getStatus().state, "OPEN"); +} + +async function run(opts: { + isModelAvailable?: (modelStr: string) => Promise | boolean; + settings?: Record | null; + log?: ReturnType; + calls?: string[]; +}) { + return handleComboChat({ + body: {}, + combo: COMBO, + handleSingleModel: async (_body: Record, modelStr: string) => { + opts.calls?.push(modelStr); + return okResponse(); + }, + isModelAvailable: opts.isModelAvailable ?? (async () => true), + log: opts.log ?? createLog(), + settings: opts.settings ?? null, + allCombos: null, + }); +} + +test.beforeEach(() => { + resetAllComboMetrics(); + resetAllCircuitBreakers(); + clearAllModelLockouts(); + weightedStickyTargets.clear(); +}); + +test.after(() => { + resetAllCircuitBreakers(); + clearAllModelLockouts(); + try { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } catch {} +}); + +test("every target model-locked → 503 + Retry-After with the excluded targets, nothing dispatched", async () => { + lockModel("openai", "a", 60_000); + lockModel("claude", "b", 120_000); + const calls: string[] = []; + const log = createLog(); + + const res = await run({ calls, log }); + assert.equal(res.status, 503); + assert.deepEqual(calls, [], "no upstream call is made for a cooling pool"); + + const retryAfter = Number(res.headers.get("Retry-After")); + assert.ok( + retryAfter >= 1 && retryAfter <= 60, + `Retry-After follows the earliest lapse, got ${retryAfter}` + ); + assert.equal(res.headers.get("x-omniroute-combo-terminal-reason"), "all_targets_cooling_down"); + assert.equal(res.headers.get("x-omniroute-recovery-action"), "wait"); + assert.equal(res.headers.get("x-omniroute-retry-after-seconds"), String(retryAfter)); + + const body = (await res.json()) as { + error: { code?: string; message: string }; + diagnostics: { + poolSize: number; + attempted: number; + excluded: Array<{ provider: string; model?: string; reason: string }>; + terminalReason: string; + }; + recovery_hint: { action: string; retry_after_seconds?: number }; + }; + assert.equal(body.error.code, "all_targets_cooling_down"); + assert.match(body.error.message, /cooling down/); + assert.equal(body.diagnostics.terminalReason, "all_targets_cooling_down"); + assert.equal(body.diagnostics.poolSize, 2); + assert.equal(body.diagnostics.attempted, 0); + assert.deepEqual( + body.diagnostics.excluded.map((e) => `${e.provider}/${e.model}:${e.reason}`).sort(), + ["claude/b:model_lockout", "openai/a:model_lockout"] + ); + assert.equal(body.recovery_hint.action, "wait"); + assert.equal(body.recovery_hint.retry_after_seconds, retryAfter); + + const warned = log.entries.find( + (e) => e.level === "warn" && String(e.msg).includes("every target excluded before dispatch") + ); + assert.ok(warned, "the silent drop is now logged with the reasons"); + assert.match(String(warned?.msg), /openai\/a: model_lockout \(\d+s\)/); +}); + +test("lockout + open circuit breaker are both reported as cooling down", async () => { + lockModel("openai", "a", 60_000); + await openBreaker("claude"); + const res = await run({}); + assert.equal(res.status, 503); + const body = (await res.json()) as { + diagnostics: { excluded: Array<{ provider: string; reason: string }> }; + }; + assert.deepEqual(body.diagnostics.excluded.map((e) => `${e.provider}:${e.reason}`).sort(), [ + "claude:circuit_open", + "openai:model_lockout", + ]); +}); + +test("one target cooling down, the other healthy → dispatches to the healthy one", async () => { + lockModel("openai", "a", 60_000); + const calls: string[] = []; + const res = await run({ calls }); + assert.equal(res.status, 200); + assert.deepEqual(calls, ["claude/b"]); +}); + +test("pool emptied only by the availability probe keeps the 404 (nothing is cooling down)", async () => { + const calls: string[] = []; + const res = await run({ calls, isModelAvailable: async () => false }); + assert.equal(res.status, 404, "no resilience timer involved — the pool has nothing to run"); + assert.deepEqual(calls, []); + const body = (await res.json()) as { diagnostics?: { terminalReason?: string } }; + assert.equal(body.diagnostics?.terminalReason, "no_executable_targets"); +}); + +test("a cooling target plus an unavailable one is still 503 — the unavailable one is listed too", async () => { + lockModel("openai", "a", 60_000); + const res = await run({ isModelAvailable: async (modelStr) => modelStr !== "claude/b" }); + assert.equal(res.status, 503); + const body = (await res.json()) as { + diagnostics: { excluded: Array<{ provider: string; reason: string }> }; + }; + assert.deepEqual(body.diagnostics.excluded.map((e) => `${e.provider}:${e.reason}`).sort(), [ + "claude:unavailable", + "openai:model_lockout", + ]); +}); diff --git a/tests/unit/combo/pin-recovery.test.ts b/tests/unit/combo/pin-recovery.test.ts index 143b8ce115..bcc2e4cb70 100644 --- a/tests/unit/combo/pin-recovery.test.ts +++ b/tests/unit/combo/pin-recovery.test.ts @@ -6,9 +6,11 @@ import test from "node:test"; import assert from "node:assert/strict"; import { + buildAllTargetsCoolingDownResponse, buildRecoveryHint, buildNoUpstreamResponseDiagnostics, buildEmptyComboTargetsPayload, + formatPreDispatchExclusions, } from "../../../open-sse/services/combo/pinRecovery.ts"; test("buildRecoveryHint: reasoning_budget_exhausted maps to switch-combo", () => { @@ -105,3 +107,81 @@ test("buildEmptyComboTargetsPayload: empty pre-filter pool → generic no_execut assert.equal(diagnostics.poolSize, 0); assert.deepEqual(diagnostics.excluded, []); }); + +// ── all_targets_cooling_down: weighted pre-dispatch exclusions ────────────── + +test("buildRecoveryHint: all_targets_cooling_down maps to wait with retry_after_seconds", () => { + const hint = buildRecoveryHint("all_targets_cooling_down", 42); + assert.equal(hint.action, "wait"); + assert.equal(hint.retry_after_seconds, 42); + assert.match(hint.next_step, /cooling down|cooldown/i); + assert.equal("retry_after_seconds" in buildRecoveryHint("all_targets_cooling_down"), false); +}); + +test("buildAllTargetsCoolingDownResponse: null when no resilience timer excluded anything", () => { + assert.equal(buildAllTargetsCoolingDownResponse([]), null); + assert.equal( + buildAllTargetsCoolingDownResponse([ + { provider: "openai", model: "a", reason: "unavailable", retryAfterMs: null }, + { provider: "zai", model: "glm", reason: "free_tier_drained", retryAfterMs: null }, + ]), + null + ); +}); + +test("buildAllTargetsCoolingDownResponse: 503 with Retry-After = earliest lapse and every exclusion listed", async () => { + const res = buildAllTargetsCoolingDownResponse([ + { provider: "openai", model: "a", reason: "model_lockout", retryAfterMs: 57_400 }, + { provider: "claude", model: "b", reason: "circuit_open", retryAfterMs: 12_000 }, + { provider: "gemini", model: "c", reason: "provider_cooldown", retryAfterMs: null }, + { provider: "zai", model: "d", reason: "unavailable", retryAfterMs: null }, + ]); + assert.ok(res); + assert.equal(res.status, 503); + assert.equal(res.headers.get("Retry-After"), "12"); + assert.equal(res.headers.get("x-omniroute-retry-after-seconds"), "12"); + assert.equal(res.headers.get("x-omniroute-combo-terminal-reason"), "all_targets_cooling_down"); + const body = (await res.json()) as { + error: { code?: string }; + diagnostics: { + poolSize: number; + excluded: Array<{ provider: string; model?: string; reason: string }>; + }; + recovery_hint: { action: string; retry_after_seconds?: number }; + }; + assert.equal(body.error.code, "all_targets_cooling_down"); + assert.equal(body.diagnostics.poolSize, 4); + assert.deepEqual( + body.diagnostics.excluded.map((e) => `${e.provider}/${e.model}:${e.reason}`), + [ + "openai/a:model_lockout", + "claude/b:circuit_open", + "gemini/c:provider_cooldown", + "zai/d:unavailable", + ] + ); + assert.deepEqual(body.recovery_hint, { + action: "wait", + retry_after_seconds: 12, + next_step: body.recovery_hint.next_step, + }); +}); + +test("buildAllTargetsCoolingDownResponse: no Retry-After when no exclusion carries a timer", () => { + const res = buildAllTargetsCoolingDownResponse([ + { provider: "openai", model: "a", reason: "model_lockout", retryAfterMs: null }, + ]); + assert.ok(res); + assert.equal(res.status, 503); + assert.equal(res.headers.get("Retry-After"), null); +}); + +test("formatPreDispatchExclusions: operator one-liner with remaining seconds when known", () => { + assert.equal( + formatPreDispatchExclusions([ + { provider: "openai", model: "a", reason: "model_lockout", retryAfterMs: 57_400 }, + { provider: "claude", model: "b", reason: "circuit_open", retryAfterMs: null }, + ]), + "openai/a: model_lockout (58s), claude/b: circuit_open" + ); +});