diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 3b21d5761c..ecc9c47155 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -140,7 +140,7 @@ "open-sse/services/browserBackedChat.ts": 850, "open-sse/services/claudeCodeCompatible.ts": 1202, "_rebaseline_pr4592_exclude_exhausted_auto": "Reconcile #4592 already-merged growth: combo.ts 2991->3036 (+45, terminal-status quota-cutoff exclusion in buildAutoCandidates + opt-in gate). Fast-gate PR->release does not run check:file-size.", - "open-sse/services/combo.ts": 3069, + "open-sse/services/combo.ts": 3168, "open-sse/services/compression/strategySelector.ts": 848, "open-sse/services/rateLimitManager.ts": 1035, "open-sse/services/tokenRefresh.ts": 2070, diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 46ec4f62cc..6fc911a256 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -582,6 +582,73 @@ export async function buildAutoCandidates( }); } +const TERMINAL_PIN_STATUSES = new Set(["credits_exhausted", "banned", "expired"]); + +/** + * Pure decision: should a context-cache pin be DROPPED because its provider has + * DURABLY fallen? A ccp pin keeps the prompt cache warm by bypassing the combo + * strategy — but if the pinned provider is dead (credits exhausted / banned / + * expired, circuit-open, repeated failures, or a long rate-limit) honoring the + * pin pounds a dead account forever with no failover (laila throttle + credits + * incidents, 2026-06-22). A brief transient cooldown is tolerated (pin kept) so + * an unstable provider does not churn the pin every turn. Connection-level + * `backoffLevel` already resets on success, so `backoffLevel >= K` ≈ K + * consecutive failures — no per-session counter needed. + * + * Returns true ⇒ drop the pin and use the strategy. Pure + unit-testable. + */ +export function pinIsDurablyUnhealthy( + circuitState: string | undefined, + connections: Array<{ + testStatus?: string | null; + backoffLevel?: number | null; + rateLimitedUntil?: string | null; + }>, + now: number, + opts: { backoffLevel?: number; graceMs?: number } = {} +): boolean { + if (circuitState === "OPEN") return true; + if (!Array.isArray(connections) || connections.length === 0) return true; + const backoffThreshold = + opts.backoffLevel ?? Number(process.env.PIN_DROP_BACKOFF_LEVEL || "2"); + const graceMs = opts.graceMs ?? Number(process.env.PIN_DROP_GRACE_MS || "20000"); + // The pin survives as long as AT LEAST ONE connection is healthy or only + // briefly cooling down — failover only when every connection is durably down. + const anyUsable = connections.some((c) => { + const status = typeof c.testStatus === "string" ? c.testStatus : ""; + if (TERMINAL_PIN_STATUSES.has(status)) return false; + if (Number(c.backoffLevel ?? 0) >= backoffThreshold) return false; + const rl = c.rateLimitedUntil ? new Date(String(c.rateLimitedUntil)).getTime() : 0; + if (Number.isFinite(rl) && rl - now > graceMs) return false; + return true; + }); + return !anyUsable; +} + +/** + * Async wrapper: resolve the pinned model's provider, read its circuit state and + * active connections, and decide via {@link pinIsDurablyUnhealthy}. Fail-open + * (return false) on any error so a lookup bug never drops a healthy pin. + */ +async function isPinnedModelDurablyUnhealthy(pinnedModel: string): Promise { + try { + const provider = parseModel(pinnedModel).provider; + if (!provider) return false; + const circuitState = getCircuitBreaker(provider)?.getStatus?.()?.state; + const connections = (await getProviderConnections({ + provider, + isActive: true, + })) as Array<{ + testStatus?: string | null; + backoffLevel?: number | null; + rateLimitedUntil?: string | null; + }>; + return pinIsDurablyUnhealthy(circuitState, connections || [], Date.now()); + } catch { + return false; + } +} + /** * Handle combo chat with fallback. * @param {Object} options @@ -731,9 +798,41 @@ export async function handleComboChat({ // Route to pinned model if context caching specifies one (Fix #679) if (pinnedModel) { - log.info( + // The pin is read from session_model_history (a PRIOR turn) and may name a + // model that has since been removed from this combo, or a provider whose + // credentials are gone. Without this guard a stale pin bypasses the strategy + // and routes to a dead model forever — incident 2026-06-21: cli-claude-heavy + // pinned to a deepseek connection with no active credentials → instant fail, + // never falling through to the live targets; and combos re-pointed Opus→Sonnet + // kept serving the old model. Validate the pin is still reachable in THIS + // combo's resolved targets (refs flattened) before honoring it. Only validate + // when allCombos is authoritative (non-empty) so we can resolve combo-refs; + // the auto-combo redirect path passes an empty list and keeps prior behavior. + const haveFullCombos = Array.isArray(allCombos) ? allCombos.length > 0 : !!allCombos; + const pinInCombo = + !haveFullCombos || + resolveComboTargets(combo, allCombos, clampComboDepth(config.maxComboDepth)).some( + (t) => t.modelStr === pinnedModel + ); + // Honor the pin only if it is still a combo target AND its provider is not + // DURABLY down. Without the health gate a pin keeps routing a session to a + // dead/credits-exhausted/throttled account forever (strategy bypassed, no + // failover) — incident 2026-06-22: laila stuck on a throttled claude account + // and credits_exhausted accounts never failing over. A transient cooldown is + // tolerated (pin kept) so an unstable provider does not churn the pin. + const pinDurablyDown = pinInCombo ? await isPinnedModelDurablyUnhealthy(pinnedModel) : false; + if (pinInCombo && !pinDurablyDown) { + log.info( + "COMBO", + `Bypassing strategy — routing directly to pinned context model: ${pinnedModel}` + ); + return handleSingleModelWithTimeout(body, pinnedModel); + } + log.warn( "COMBO", - `Bypassing strategy — routing directly to pinned context model: ${pinnedModel}` + pinInCombo + ? `Context-cache pin "${pinnedModel}" provider durably unhealthy — dropping pin, using strategy` + : `Stale context-cache pin "${pinnedModel}" not in combo "${combo.name}" targets — dropping pin, using strategy` ); return handleSingleModelWithTimeout(body, pinnedModel); } diff --git a/tests/unit/combo-pin-health-gate.test.ts b/tests/unit/combo-pin-health-gate.test.ts new file mode 100644 index 0000000000..ad02650975 --- /dev/null +++ b/tests/unit/combo-pin-health-gate.test.ts @@ -0,0 +1,77 @@ +/** + * ccp pin health gate — drop a context-cache pin when its provider is DURABLY + * down so the session fails over instead of pounding a dead account, while + * tolerating transient cooldowns so an unstable provider does not churn the pin. + * + * Incident 2026-06-22: a session pinned to a throttled/credits-exhausted account + * stayed pinned forever (strategy bypassed, no failover) because the pin was only + * dropped when the model LEFT the combo, never on connection health. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { pinIsDurablyUnhealthy } from "../../open-sse/services/combo.ts"; + +const NOW = 1_900_000_000_000; +const opts = { backoffLevel: 2, graceMs: 20_000 }; +const healthy = { testStatus: "active", backoffLevel: 0, rateLimitedUntil: null }; + +test("keeps pin when a healthy connection exists", () => { + assert.equal(pinIsDurablyUnhealthy("CLOSED", [healthy], NOW, opts), false); +}); + +test("drops pin when circuit is OPEN", () => { + assert.equal(pinIsDurablyUnhealthy("OPEN", [healthy], NOW, opts), true); +}); + +test("drops pin when there are no active connections", () => { + assert.equal(pinIsDurablyUnhealthy("CLOSED", [], NOW, opts), true); +}); + +test("drops pin when the only connection has credits exhausted (terminal)", () => { + const conn = { testStatus: "credits_exhausted", backoffLevel: 0, rateLimitedUntil: null }; + assert.equal(pinIsDurablyUnhealthy("CLOSED", [conn], NOW, opts), true); +}); + +test("drops pin on banned/expired terminal status", () => { + assert.equal( + pinIsDurablyUnhealthy("CLOSED", [{ testStatus: "banned", backoffLevel: 0 }], NOW, opts), + true + ); + assert.equal( + pinIsDurablyUnhealthy("CLOSED", [{ testStatus: "expired", backoffLevel: 0 }], NOW, opts), + true + ); +}); + +test("drops pin once backoffLevel reaches the threshold (repeated failures)", () => { + assert.equal( + pinIsDurablyUnhealthy("CLOSED", [{ testStatus: "active", backoffLevel: 2 }], NOW, opts), + true + ); +}); + +test("anti-flap: keeps pin on a brief transient cooldown (low backoff, short rate-limit)", () => { + const conn = { + testStatus: "active", + backoffLevel: 1, + rateLimitedUntil: new Date(NOW + 4_000).toISOString(), // 4s out — within grace + }; + assert.equal(pinIsDurablyUnhealthy("CLOSED", [conn], NOW, opts), false); +}); + +test("drops pin on a long rate-limit window (beyond grace)", () => { + const conn = { + testStatus: "active", + backoffLevel: 0, + rateLimitedUntil: new Date(NOW + 60_000).toISOString(), // 60s out — durable + }; + assert.equal(pinIsDurablyUnhealthy("CLOSED", [conn], NOW, opts), true); +}); + +test("keeps pin if ANY connection is usable (terminal + healthy mix)", () => { + const conns = [ + { testStatus: "credits_exhausted", backoffLevel: 0 }, + healthy, + ]; + assert.equal(pinIsDurablyUnhealthy("CLOSED", conns, NOW, opts), false); +});