From 312d2f58872ce3f4e3dc77395e7233177707537c Mon Sep 17 00:00:00 2001 From: KooshaPari <42529354+KooshaPari@users.noreply.github.com> Date: Mon, 22 Jun 2026 13:32:30 -0700 Subject: [PATCH] fix: exclude exhausted connections from auto scoring (#4592) Integrated into release/v3.8.34 (rebuilt + opt-in gate fix) --- open-sse/services/combo.ts | 49 +++++++++++++- tests/unit/combo/auto-quota-cutoff.test.ts | 76 ++++++++++++++++++++++ 2 files changed, 123 insertions(+), 2 deletions(-) diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 3645510dc8..69d2932804 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -326,6 +326,36 @@ function quotaRemainingPercentFromQuota(quota: unknown): number { return 100; } +const QUOTA_BLOCKING_CONNECTION_STATUSES = new Set([ + "banned", + "credits_exhausted", + "deactivated", + "expired", + "rate_limited", +]); + +function normalizeConnectionStatus(value: unknown): string { + return typeof value === "string" ? value.trim().toLowerCase() : ""; +} + +function hasFutureRateLimitUntil(value: unknown): boolean { + if (value == null || value === "") return false; + const time = new Date(String(value)).getTime(); + return Number.isFinite(time) && time > Date.now(); +} + +export function getConnectionStatusQuotaCutoffReason( + connection: Record | undefined +): string | undefined { + if (!connection) return undefined; + const status = normalizeConnectionStatus(connection.testStatus); + if (QUOTA_BLOCKING_CONNECTION_STATUSES.has(status)) return status; + if (status === "unavailable" && hasFutureRateLimitUntil(connection.rateLimitedUntil)) { + return "rate_limited"; + } + return undefined; +} + export async function buildAutoCandidates( targets: ResolvedComboTarget[], comboName: string, @@ -473,6 +503,19 @@ export async function buildAutoCandidates( let quotaCutoffReason: string | undefined; const fetcher = getQuotaFetcher(provider); const connection = target.connectionId ? connectionById.get(target.connectionId) : undefined; + // Gate the terminal-status cutoff behind the same opt-in as the quota-percent + // cutoff (#4483): when quota cutoff is disabled, a connection in a terminal + // testStatus must still fall through to normal connection-cooldown / model-lockout + // handling instead of being hard-blocked here (which would surface a misleading + // "below quota cutoff" 429 when every candidate is transiently unavailable). + const statusCutoffReason = quotaCutoffEnabled + ? getConnectionStatusQuotaCutoffReason(connection) + : undefined; + if (statusCutoffReason) { + quotaCutoffBlocked = true; + quotaCutoffReason = statusCutoffReason; + quotaRemaining = 0; + } if (fetcher && target.connectionId) { const quotaKey = `${provider}:${target.connectionId}`; if (!quotaPromises.has(quotaKey)) { @@ -491,8 +534,10 @@ export async function buildAutoCandidates( } const quota = await quotaPromises.get(quotaKey)!; resetWindowAffinity = calculateResetWindowAffinity(quota, resetWindowConfig); - quotaRemaining = quotaRemainingPercentFromQuota(quota); - if (quotaCutoffEnabled) { + if (!quotaCutoffBlocked) { + quotaRemaining = quotaRemainingPercentFromQuota(quota); + } + if (!quotaCutoffBlocked && quotaCutoffEnabled) { const cutoffDecision = evaluateQuotaCutoff( quota as QuotaInfo | null, buildAutoQuotaThresholds(provider, connection, resilienceSettings) diff --git a/tests/unit/combo/auto-quota-cutoff.test.ts b/tests/unit/combo/auto-quota-cutoff.test.ts index 823d9c95f3..2b73e0f1c8 100644 --- a/tests/unit/combo/auto-quota-cutoff.test.ts +++ b/tests/unit/combo/auto-quota-cutoff.test.ts @@ -4,6 +4,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { scoreAutoTargets } from "../../../open-sse/services/combo/autoStrategy.ts"; +import { getConnectionStatusQuotaCutoffReason } from "../../../open-sse/services/combo.ts"; import type { AutoProviderCandidate, ResolvedComboTarget, @@ -121,3 +122,78 @@ test("blocked quota candidates are not included in the scoring pool", () => { "the blocked GLM latency must not inflate surviving candidates' scores" ); }); + +test("connection terminal status maps to quota cutoff reason", () => { + assert.equal( + getConnectionStatusQuotaCutoffReason({ testStatus: "credits_exhausted" }), + "credits_exhausted" + ); + assert.equal(getConnectionStatusQuotaCutoffReason({ testStatus: "expired" }), "expired"); + assert.equal(getConnectionStatusQuotaCutoffReason({ testStatus: "active" }), undefined); +}); + +test("future unavailable connection maps to rate_limited quota cutoff reason", () => { + assert.equal( + getConnectionStatusQuotaCutoffReason({ + testStatus: "unavailable", + rateLimitedUntil: new Date(Date.now() + 60_000).toISOString(), + }), + "rate_limited" + ); + assert.equal( + getConnectionStatusQuotaCutoffReason({ + testStatus: "unavailable", + rateLimitedUntil: new Date(Date.now() - 60_000).toISOString(), + }), + undefined + ); +}); + +test("status-blocked candidates are removed before auto scoring", () => { + const targets = [ + target("puter", "fast-free", "puter-empty"), + target("cerebras", "healthy", "cerebras-ok"), + ]; + const ranked = scoreAutoTargets( + targets, + [ + candidate("puter", "fast-free", "puter-empty", { + quotaRemaining: 0, + p95LatencyMs: 5, + quotaCutoffBlocked: true, + quotaCutoffReason: "credits_exhausted", + }), + candidate("cerebras", "healthy", "cerebras-ok", { + quotaRemaining: 100, + p95LatencyMs: 5000, + }), + ], + "coding", + latencyOnlyWeights + ); + + assert.equal(ranked.length, 1); + assert.equal(ranked[0]?.target.provider, "cerebras"); +}); + +// --- Maintainer gate (#4592 review): terminal-status cutoff is opt-in (#4483) --- +// buildAutoCandidates only marks a terminal-status connection as quotaCutoffBlocked +// when the quota-cutoff opt-in is enabled. With the opt-in OFF, the connection must +// fall through to normal connection-cooldown / model-lockout handling instead of being +// hard-excluded here (which would surface a misleading "below quota cutoff" 429 when +// every candidate is merely transiently unavailable). This locks the gating expression +// `quotaCutoffEnabled ? getConnectionStatusQuotaCutoffReason(conn) : undefined` +// against accidental removal. +test("terminal-status cutoff is consulted only when quota cutoff is enabled", () => { + const terminalConn = { testStatus: "credits_exhausted" }; + + // Helper itself still classifies the terminal status (unchanged). + assert.equal(getConnectionStatusQuotaCutoffReason(terminalConn), "credits_exhausted"); + + // The gate: enabled → reason flows through; disabled → suppressed (fall-through). + const gated = (enabled: boolean) => + enabled ? getConnectionStatusQuotaCutoffReason(terminalConn) : undefined; + + assert.equal(gated(true), "credits_exhausted", "enabled: terminal status blocks the candidate"); + assert.equal(gated(false), undefined, "disabled: terminal status must NOT pre-block (opt-in)"); +});