mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
fix(auto): enforce quota cutoff before scoring (#4483)
Thanks @megamen32! Rebased onto release/v3.8.32. On review (owner decision) the auto-routing quota cutoff was made OPT-IN behind QuotaPreflightSettings.enabled (default OFF, so current behavior is unchanged) and the ...eligibleTargets last-resort fallback was restored so a quota-blocked target survives as final fallback rather than vanishing. The 429-when-all-blocked guard stays for the enabled path. typecheck clean, 13/13 tests.
This commit is contained in:
committed by
GitHub
parent
5a7c1e1731
commit
1831c6eca8
@@ -1,5 +1,6 @@
|
||||
{
|
||||
"_comment": "Catraca de tamanho (check-file-size.mjs). frozen so pode encolher; arquivos novos <= cap. --update ratcheta.",
|
||||
"_rebaseline_2026_06_21_4483_auto_quota_cutoff": "PR #4483 (megamen32) own growth + review fix: open-sse/services/combo.ts 2611->2623 (+12). The PR adds an auto-routing hard quota cutoff in buildAutoCandidates (evaluateQuotaCutoff + buildAutoQuotaThresholds/clampPercent/asThresholdMap/quotaWindowLookupNames helpers) that drops low-quota candidates before scoring, plus a 429 guard when all candidates are below cutoff. On review (owner decision) the cutoff was made OPT-IN behind a new QuotaPreflightSettings.enabled flag (default OFF via QUOTA_PREFLIGHT_CUTOFF_ENABLED) so default behavior is unchanged, and the `...eligibleTargets` last-resort fallback the PR removed was restored (dedupe makes it identical to pre-cutoff when OFF; when ON a blocked target survives as final fallback rather than vanishing). Cohesive at the existing candidate-build/select chokepoints; not extractable. Covered by tests/unit/combo/auto-quota-cutoff.test.ts + tests/unit/resilience-settings-quota-preflight.test.ts (default-off + opt-in round-trip).",
|
||||
"_rebaseline_2026_06_21_4475_target_format_badge": "PR #4475 (adivekar-utexas) review: src/app/(dashboard)/dashboard/providers/[id]/providerPageHelpers.ts 955->974 (+19). Extracted the pure targetFormatBadgeI18nKey (the 6 targetFormat value->i18n-key mapping) out of the CustomModelsSection.tsx badge so it is unit-testable outside the .tsx (Rule #18 gap — the PR had no UI test). The .tsx now calls the helper instead of an inline if-chain. This leaf is the strangler-fig home for pure provider-page helpers (#3501), so receiving the extraction is on-purpose. Covered by tests/unit/provider-target-format-badge-4475.test.ts.",
|
||||
"_rebaseline_2026_06_21_4427_low_noise_catalog": "PR #4427 (Rahulsharma0810) own growth: src/app/api/v1/models/catalog.ts 1478->1486 (+8). The opt-in MODELS_CATALOG_PREFIX_MODE (dual default | alias | canonical, with ?prefix= per-request override) gates the dual alias+canonical model emission at the three /v1/models push sites (static, synced, custom) behind includeAlias/includeCanonical, suppressing the duplicate cross-prefix entries (net +3 from the feature). On review, 4 incidental explanatory comments removed by the PR were restored (synced-models resolve, skip-static, try-block intent, strip-modelIdPrefix; +5) since their code is unchanged — useful docs on a non-trivial catalog function. Default `dual` keeps byte-identical output; request-side alias resolution unchanged. Structural shrink of this route tracked in #3789. Covered by tests/unit/models-catalog-low-noise-flag.test.ts.",
|
||||
"_rebaseline_2026_06_21_qg9_chatcore_service_tier": "QG v2 Fase 9 T5 (#3501) — chatCore.ts 5137->5110 (shrink -27). The two inline Codex service-tier resolvers (resolveEffectiveServiceTier / resolveReportedServiceTier, ~36 LOC) were extracted byte-identically into the new pure leaf open-sse/handlers/chatCore/serviceTier.ts; the handler now keeps a `let effectiveServiceTier` + two thin binding closures that pass provider/credentials?.providerSpecificData to the extracted functions, so every call site stays unchanged. The orphaned getCodexRequestDefaults/normalizeCodexServiceTier/CodexServiceTier imports moved to the leaf. Ratcheted the frozen value down to lock the freed budget. Covered by tests/unit/chatcore-service-tier.test.ts.",
|
||||
@@ -121,7 +122,7 @@
|
||||
"open-sse/services/batchProcessor.ts": 828,
|
||||
"open-sse/services/browserBackedChat.ts": 850,
|
||||
"open-sse/services/claudeCodeCompatible.ts": 1202,
|
||||
"open-sse/services/combo.ts": 2611,
|
||||
"open-sse/services/combo.ts": 2623,
|
||||
"open-sse/services/rateLimitManager.ts": 1035,
|
||||
"open-sse/services/tokenRefresh.ts": 1997,
|
||||
"open-sse/services/usage.ts": 3408,
|
||||
|
||||
@@ -44,7 +44,12 @@ import {
|
||||
import { extractSessionAffinityKey } from "@/sse/services/auth";
|
||||
import { resolveModelLockoutSettings } from "../../src/lib/resilience/modelLockoutSettings";
|
||||
import { fetchCodexQuota } from "./codexQuotaFetcher.ts";
|
||||
import { getQuotaFetcher } from "./quotaPreflight.ts";
|
||||
import {
|
||||
evaluateQuotaCutoff,
|
||||
getQuotaFetcher,
|
||||
type PreflightQuotaThresholds,
|
||||
type QuotaInfo,
|
||||
} from "./quotaPreflight.ts";
|
||||
import * as semaphore from "./rateLimitSemaphore.ts";
|
||||
import { getCircuitBreaker } from "../../src/shared/utils/circuitBreaker";
|
||||
import { fisherYatesShuffle, getNextFromDeck } from "../../src/shared/utils/shuffleDeck";
|
||||
@@ -224,13 +229,96 @@ function getBootstrapLatencyMs(modelId: string): number {
|
||||
return DEFAULT_MODEL_P95_MS[normalized] ?? 1500;
|
||||
}
|
||||
|
||||
function clampPercent(value: number): number {
|
||||
if (!Number.isFinite(value)) return 100;
|
||||
return Math.max(0, Math.min(100, value));
|
||||
}
|
||||
|
||||
function asThresholdMap(value: unknown): Record<string, number> {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return {};
|
||||
const result: Record<string, number> = {};
|
||||
for (const [key, raw] of Object.entries(value as Record<string, unknown>)) {
|
||||
const numeric = Number(raw);
|
||||
if (key && Number.isFinite(numeric)) result[key] = numeric;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function quotaWindowLookupNames(provider: string, windowName: string): string[] {
|
||||
const names = [windowName];
|
||||
const lower = windowName.toLowerCase();
|
||||
if (lower !== windowName) names.push(lower);
|
||||
if (provider === "codex") {
|
||||
if (lower.includes("session") || lower === "5h" || lower === "five_hour") names.push("session");
|
||||
if (lower.includes("weekly") || lower === "7d" || lower === "seven_day") names.push("weekly");
|
||||
if (lower.includes("monthly") || lower === "30d") names.push("monthly");
|
||||
}
|
||||
return [...new Set(names)];
|
||||
}
|
||||
|
||||
function buildAutoQuotaThresholds(
|
||||
provider: string,
|
||||
connection: Record<string, unknown> | undefined,
|
||||
resilienceSettings: ResilienceSettings | null | undefined
|
||||
): PreflightQuotaThresholds {
|
||||
const quotaPreflight = (resilienceSettings ?? resolveResilienceSettings(null))?.quotaPreflight;
|
||||
const defaultThresholdPercent = quotaPreflight?.defaultThresholdPercent ?? 2;
|
||||
const warnThresholdPercent = quotaPreflight?.warnThresholdPercent ?? 20;
|
||||
const providerWindowMap = asThresholdMap(quotaPreflight?.providerWindowDefaults?.[provider]);
|
||||
const perConnectionWindowOverrides = asThresholdMap(connection?.quotaWindowThresholds);
|
||||
|
||||
return {
|
||||
resolveMinRemainingPercent: (windowName: string | null): number => {
|
||||
if (windowName !== null) {
|
||||
for (const lookupWindowName of quotaWindowLookupNames(provider, windowName)) {
|
||||
const override = perConnectionWindowOverrides[lookupWindowName];
|
||||
if (typeof override === "number") return override;
|
||||
const providerDefault = providerWindowMap[lookupWindowName];
|
||||
if (typeof providerDefault === "number") return providerDefault;
|
||||
}
|
||||
}
|
||||
return defaultThresholdPercent;
|
||||
},
|
||||
resolveWarnRemainingPercent: () => warnThresholdPercent,
|
||||
};
|
||||
}
|
||||
|
||||
function quotaRemainingPercentFromQuota(quota: unknown): number {
|
||||
if (!quota || typeof quota !== "object") return 100;
|
||||
const record = quota as Record<string, unknown>;
|
||||
if (record.limitReached === true) return 0;
|
||||
|
||||
const windows = record.windows;
|
||||
if (windows && typeof windows === "object" && !Array.isArray(windows)) {
|
||||
let minRemaining: number | null = null;
|
||||
for (const windowInfo of Object.values(windows as Record<string, unknown>)) {
|
||||
if (!windowInfo || typeof windowInfo !== "object") continue;
|
||||
const percentUsed = Number((windowInfo as Record<string, unknown>).percentUsed);
|
||||
if (!Number.isFinite(percentUsed)) continue;
|
||||
const remaining = clampPercent((1 - percentUsed) * 100);
|
||||
minRemaining = minRemaining === null ? remaining : Math.min(minRemaining, remaining);
|
||||
}
|
||||
if (minRemaining !== null) return minRemaining;
|
||||
}
|
||||
|
||||
const percentUsed = Number(record.percentUsed);
|
||||
if (Number.isFinite(percentUsed)) return clampPercent((1 - percentUsed) * 100);
|
||||
return 100;
|
||||
}
|
||||
|
||||
export async function buildAutoCandidates(
|
||||
targets: ResolvedComboTarget[],
|
||||
comboName: string,
|
||||
sessionId: string | null | undefined = null,
|
||||
resetWindowConfig: ResetWindowConfig = resolveResetWindowConfig(null)
|
||||
resetWindowConfig: ResetWindowConfig = resolveResetWindowConfig(null),
|
||||
resilienceSettings: ResilienceSettings | null = null
|
||||
): Promise<AutoProviderCandidate[]> {
|
||||
const metrics = getComboMetrics(comboName);
|
||||
// Opt-in hard quota cutoff (default OFF). When disabled, candidates are never
|
||||
// dropped for low quota here — the soft quota penalty + connection cooldown still
|
||||
// apply, so auto-routing behavior is unchanged.
|
||||
const quotaCutoffEnabled =
|
||||
(resilienceSettings ?? resolveResilienceSettings(null))?.quotaPreflight?.enabled === true;
|
||||
const { getPricingForModel } = await import("../../src/lib/localDb");
|
||||
const quotaPromises = new Map<string, Promise<unknown>>();
|
||||
let historicalLatencyStats: Record<string, HistoricalLatencyStatsEntry> = {};
|
||||
@@ -252,6 +340,7 @@ export async function buildAutoCandidates(
|
||||
);
|
||||
const connectionPoolCounts = new Map<string, number>();
|
||||
const connectionsByProvider = new Map<string, Array<Record<string, unknown>>>();
|
||||
const connectionById = new Map<string, Record<string, unknown>>();
|
||||
await Promise.all(
|
||||
uniqueProviders.map(async (provider) => {
|
||||
try {
|
||||
@@ -259,6 +348,11 @@ export async function buildAutoCandidates(
|
||||
const active = Array.isArray(connections) ? connections : [];
|
||||
connectionPoolCounts.set(provider, active.length);
|
||||
connectionsByProvider.set(provider, active);
|
||||
for (const connection of active) {
|
||||
if (connection && typeof connection === "object" && typeof connection.id === "string") {
|
||||
connectionById.set(connection.id, connection as Record<string, unknown>);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
connectionPoolCounts.set(provider, 0);
|
||||
connectionsByProvider.set(provider, []);
|
||||
@@ -353,7 +447,11 @@ export async function buildAutoCandidates(
|
||||
breakerStateRaw === "OPEN" || breakerStateRaw === "HALF_OPEN" ? breakerStateRaw : "CLOSED";
|
||||
const contextAffinity = calculateTargetContextAffinity(target, sessionId);
|
||||
let resetWindowAffinity = 0.5;
|
||||
let quotaRemaining = 100;
|
||||
let quotaCutoffBlocked = false;
|
||||
let quotaCutoffReason: string | undefined;
|
||||
const fetcher = getQuotaFetcher(provider);
|
||||
const connection = target.connectionId ? connectionById.get(target.connectionId) : undefined;
|
||||
if (fetcher && target.connectionId) {
|
||||
const quotaKey = `${provider}:${target.connectionId}`;
|
||||
if (!quotaPromises.has(quotaKey)) {
|
||||
@@ -362,6 +460,7 @@ export async function buildAutoCandidates(
|
||||
fetchResetAwareQuotaWithCache({
|
||||
provider,
|
||||
connectionId: target.connectionId,
|
||||
connection,
|
||||
fetcher,
|
||||
config: resetWindowConfig,
|
||||
log: {},
|
||||
@@ -371,6 +470,17 @@ export async function buildAutoCandidates(
|
||||
}
|
||||
const quota = await quotaPromises.get(quotaKey)!;
|
||||
resetWindowAffinity = calculateResetWindowAffinity(quota, resetWindowConfig);
|
||||
quotaRemaining = quotaRemainingPercentFromQuota(quota);
|
||||
if (quotaCutoffEnabled) {
|
||||
const cutoffDecision = evaluateQuotaCutoff(
|
||||
quota as QuotaInfo | null,
|
||||
buildAutoQuotaThresholds(provider, connection, resilienceSettings)
|
||||
);
|
||||
if (!cutoffDecision.proceed) {
|
||||
quotaCutoffBlocked = true;
|
||||
quotaCutoffReason = cutoffDecision.reason || "quota_exhausted";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -379,7 +489,7 @@ export async function buildAutoCandidates(
|
||||
modelStr,
|
||||
provider,
|
||||
model,
|
||||
quotaRemaining: 100,
|
||||
quotaRemaining,
|
||||
quotaTotal: 100,
|
||||
circuitBreakerState,
|
||||
costPer1MTokens,
|
||||
@@ -390,6 +500,8 @@ export async function buildAutoCandidates(
|
||||
quotaResetIntervalSecs: 86400,
|
||||
contextAffinity,
|
||||
resetWindowAffinity,
|
||||
quotaCutoffBlocked,
|
||||
quotaCutoffReason,
|
||||
connectionPoolSize: connectionPoolCounts.get(provider) ?? 1,
|
||||
connectionId: target.connectionId ?? undefined,
|
||||
};
|
||||
@@ -719,11 +831,28 @@ export async function handleComboChat({
|
||||
eligibleTargets,
|
||||
combo.name,
|
||||
relayOptions?.sessionId,
|
||||
resetWindowConfig
|
||||
resetWindowConfig,
|
||||
resilienceSettings
|
||||
);
|
||||
const routableCandidates = candidates.filter(
|
||||
(candidate) => candidate.quotaCutoffBlocked !== true
|
||||
);
|
||||
const quotaBlockedCount = candidates.length - routableCandidates.length;
|
||||
if (quotaBlockedCount > 0) {
|
||||
log.info(
|
||||
"COMBO",
|
||||
`Auto strategy: quota cutoff skipped ${quotaBlockedCount}/${candidates.length} account candidates`
|
||||
);
|
||||
}
|
||||
// G2: Register candidates so chatCore can mark quotaSoftPenalty via setCandidateQuotaSoftPenalty.
|
||||
_registerExecutionCandidates(candidates);
|
||||
if (candidates.length > 0) {
|
||||
_registerExecutionCandidates(routableCandidates);
|
||||
if (candidates.length > 0 && routableCandidates.length === 0) {
|
||||
return unavailableResponse(
|
||||
429,
|
||||
"All auto strategy candidates are below configured quota cutoffs"
|
||||
);
|
||||
}
|
||||
if (routableCandidates.length > 0) {
|
||||
let selectedProvider: string | null = null;
|
||||
let selectedModel: string | null = null;
|
||||
let selectionReason = "";
|
||||
@@ -731,7 +860,7 @@ export async function handleComboChat({
|
||||
if (routingStrategy !== "rules") {
|
||||
try {
|
||||
const decision = selectWithStrategy(
|
||||
candidates,
|
||||
routableCandidates,
|
||||
{
|
||||
taskType,
|
||||
requestHasTools,
|
||||
@@ -764,7 +893,7 @@ export async function handleComboChat({
|
||||
budgetCap,
|
||||
explorationRate,
|
||||
},
|
||||
candidates,
|
||||
routableCandidates,
|
||||
taskType
|
||||
);
|
||||
selectedProvider = selection.provider;
|
||||
@@ -786,7 +915,7 @@ export async function handleComboChat({
|
||||
|
||||
const scoredTargets = scoreAutoTargets(
|
||||
eligibleTargets,
|
||||
candidates,
|
||||
routableCandidates,
|
||||
taskType,
|
||||
weights,
|
||||
autoManifestHint
|
||||
@@ -800,7 +929,17 @@ export async function handleComboChat({
|
||||
})?.target ||
|
||||
rankedTargets[0] ||
|
||||
eligibleTargets[0];
|
||||
if (!selectedTarget) {
|
||||
return unavailableResponse(
|
||||
429,
|
||||
"No auto strategy targets remained after quota cutoff filtering"
|
||||
);
|
||||
}
|
||||
|
||||
// Keep eligibleTargets as the last-resort fallback tail: dedupe drops the
|
||||
// routable ranked ones (and, when the cutoff is OFF, makes this identical to
|
||||
// the pre-cutoff behavior), but a quota-blocked target still survives as a
|
||||
// final fallback instead of vanishing — the hard cutoff only de-prioritizes.
|
||||
orderedTargets = dedupeTargetsByExecutionKey(
|
||||
[selectedTarget, ...rankedTargets, ...eligibleTargets].filter(
|
||||
(entry): entry is ResolvedComboTarget => entry !== undefined && entry !== null
|
||||
|
||||
@@ -330,19 +330,31 @@ export function scoreAutoTargets(
|
||||
weights: ScoringWeights,
|
||||
manifestHint?: RoutingHint | null
|
||||
) {
|
||||
const candidateByExecutionKey = new Map(
|
||||
candidates.map((candidate: ProviderCandidate & { executionKey: string }) => [
|
||||
candidate.executionKey,
|
||||
candidate,
|
||||
])
|
||||
);
|
||||
return targets
|
||||
.map((target) => {
|
||||
const candidate = candidateByExecutionKey.get(target.executionKey);
|
||||
if (!candidate) return null;
|
||||
const targetByExecutionKey = new Map(targets.map((target) => [target.executionKey, target]));
|
||||
const activeCandidates = candidates.filter((candidate) => candidate.quotaCutoffBlocked !== true);
|
||||
|
||||
return activeCandidates
|
||||
.map((candidate) => {
|
||||
const baseTarget =
|
||||
targetByExecutionKey.get(candidate.executionKey) ||
|
||||
targets.find(
|
||||
(target) =>
|
||||
target.stepId === candidate.stepId ||
|
||||
(target.provider === candidate.provider && target.modelStr === candidate.modelStr)
|
||||
);
|
||||
if (!baseTarget) return null;
|
||||
|
||||
const target: ResolvedComboTarget = {
|
||||
...baseTarget,
|
||||
stepId: candidate.stepId,
|
||||
executionKey: candidate.executionKey,
|
||||
modelStr: candidate.modelStr,
|
||||
provider: candidate.provider,
|
||||
connectionId: candidate.connectionId ?? baseTarget.connectionId,
|
||||
};
|
||||
const factors = calculateFactors(
|
||||
candidate as ProviderCandidate,
|
||||
candidates,
|
||||
activeCandidates,
|
||||
taskType ?? "general",
|
||||
getTaskFitness,
|
||||
manifestHint ?? undefined
|
||||
|
||||
@@ -101,6 +101,10 @@ export type AutoProviderCandidate = ProviderCandidate & {
|
||||
* for the key routed through this target's connectionId.
|
||||
*/
|
||||
quotaSoftPenalty?: boolean;
|
||||
/** True when provider-account quota preflight cutoff says this candidate must not be routed. */
|
||||
quotaCutoffBlocked?: boolean;
|
||||
/** Diagnostic reason for quotaCutoffBlocked. */
|
||||
quotaCutoffReason?: string;
|
||||
};
|
||||
|
||||
export type ResolvedComboTarget = {
|
||||
|
||||
@@ -151,6 +151,7 @@ export function convertUsageToQuotaInfo(usage: unknown): QuotaInfo | null {
|
||||
percentUsed: worstPercent,
|
||||
resetAt: worstResetAt,
|
||||
windows,
|
||||
limitReached: worstPercent >= 1 - 1e-9,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -44,6 +44,8 @@ export interface QuotaInfo {
|
||||
* (e.g. "session", "weekly", "monthly").
|
||||
*/
|
||||
windows?: Record<string, QuotaWindowInfo>;
|
||||
/** True when the upstream usage endpoint explicitly reports exhausted quota. */
|
||||
limitReached?: boolean;
|
||||
}
|
||||
|
||||
export type QuotaFetcher = (
|
||||
@@ -140,6 +142,88 @@ function isRemainingAtOrBelowThreshold(
|
||||
return remainingPercent <= thresholdPercent + REMAINING_PERCENT_EPSILON;
|
||||
}
|
||||
|
||||
function exhaustedResult(quotaPercent: number, resetAt: string | null): PreflightQuotaResult {
|
||||
return {
|
||||
proceed: false,
|
||||
reason: "quota_exhausted",
|
||||
quotaPercent,
|
||||
resetAt,
|
||||
};
|
||||
}
|
||||
|
||||
function quotaWindowCutoffResult(
|
||||
windows: NonNullable<QuotaInfo["windows"]>,
|
||||
thresholds?: PreflightQuotaThresholds
|
||||
): PreflightQuotaResult | null {
|
||||
let worstUsedPercent = 0;
|
||||
let worstWindow: string | null = null;
|
||||
let worstResetAt: string | null = null;
|
||||
|
||||
for (const [windowName, windowInfo] of Object.entries(windows)) {
|
||||
if (!Number.isFinite(windowInfo.percentUsed)) continue;
|
||||
const minRemainingPercent = resolveOrDefault(
|
||||
thresholds?.resolveMinRemainingPercent,
|
||||
windowName,
|
||||
DEFAULT_MIN_REMAINING_PERCENT
|
||||
);
|
||||
if (
|
||||
!isRemainingAtOrBelowThreshold(
|
||||
remainingPercentFrom(windowInfo.percentUsed),
|
||||
minRemainingPercent
|
||||
)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if (windowInfo.percentUsed <= worstUsedPercent && worstWindow !== null) continue;
|
||||
worstUsedPercent = windowInfo.percentUsed;
|
||||
worstWindow = windowName;
|
||||
worstResetAt = windowInfo.resetAt ?? null;
|
||||
}
|
||||
|
||||
return worstWindow === null ? null : exhaustedResult(worstUsedPercent, worstResetAt);
|
||||
}
|
||||
|
||||
function quotaPercentCutoffResult(
|
||||
quota: QuotaInfo,
|
||||
thresholds?: PreflightQuotaThresholds
|
||||
): PreflightQuotaResult {
|
||||
if (!Number.isFinite(quota.percentUsed)) return { proceed: true };
|
||||
|
||||
const minRemainingPercent = resolveOrDefault(
|
||||
thresholds?.resolveMinRemainingPercent,
|
||||
null,
|
||||
DEFAULT_MIN_REMAINING_PERCENT
|
||||
);
|
||||
const remainingPercent = remainingPercentFrom(quota.percentUsed);
|
||||
return isRemainingAtOrBelowThreshold(remainingPercent, minRemainingPercent)
|
||||
? exhaustedResult(quota.percentUsed, quota.resetAt ?? null)
|
||||
: { proceed: true, quotaPercent: quota.percentUsed };
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure cutoff evaluator used by routing paths that already fetched quota.
|
||||
* Mirrors preflightQuota threshold semantics without performing I/O or logging.
|
||||
*/
|
||||
export function evaluateQuotaCutoff(
|
||||
quota: QuotaInfo | null | undefined,
|
||||
thresholds?: PreflightQuotaThresholds
|
||||
): PreflightQuotaResult {
|
||||
if (!quota) return { proceed: true };
|
||||
if (quota.limitReached === true) return exhaustedResult(1, quota.resetAt ?? null);
|
||||
|
||||
const windows = quota.windows;
|
||||
if (windows && Object.keys(windows).length > 0) {
|
||||
return (
|
||||
quotaWindowCutoffResult(windows, thresholds) ?? {
|
||||
proceed: true,
|
||||
quotaPercent: quota.percentUsed,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
return quotaPercentCutoffResult(quota, thresholds);
|
||||
}
|
||||
|
||||
export async function preflightQuota(
|
||||
provider: string,
|
||||
connectionId: string,
|
||||
|
||||
@@ -64,6 +64,14 @@ export interface ProviderCooldownSettings {
|
||||
}
|
||||
|
||||
export interface QuotaPreflightSettings {
|
||||
/**
|
||||
* Master switch for the auto-routing quota cutoff (buildAutoCandidates). When
|
||||
* disabled (default), candidates are NOT dropped for low quota before scoring —
|
||||
* the soft quota penalty + connection cooldown still apply, so behavior is
|
||||
* unchanged. Opt-in because the hard cutoff interacts with the auto-routing
|
||||
* scorer and must be validated per deployment. Default: false.
|
||||
*/
|
||||
enabled: boolean;
|
||||
/**
|
||||
* Global minimum-remaining cutoff (percent, 0-100). A connection is skipped
|
||||
* when its remaining quota drops to this value or below. Matches the
|
||||
@@ -215,6 +223,13 @@ export const DEFAULT_RESILIENCE_SETTINGS: ResilienceSettings = {
|
||||
),
|
||||
},
|
||||
quotaPreflight: {
|
||||
// Opt-in (default OFF): the auto-routing hard cutoff drops low-quota candidates
|
||||
// before scoring, overlapping the existing soft quota penalty + connection
|
||||
// cooldown, so it must be explicitly enabled by the operator until its
|
||||
// interaction with the scorer is validated in production.
|
||||
enabled: ["true", "1", "on"].includes(
|
||||
(process.env.QUOTA_PREFLIGHT_CUTOFF_ENABLED || "").trim().toLowerCase()
|
||||
),
|
||||
// Remaining-% semantics. 2 = "stop when only 2% remaining" (= 98% used).
|
||||
// Uniform across all providers and windows; operators set per-window
|
||||
// overrides per connection via the Cutoff modal in Dashboard › Limits,
|
||||
@@ -428,7 +443,8 @@ function normalizeQuotaPreflightSettings(
|
||||
record.providerWindowDefaults,
|
||||
fallback.providerWindowDefaults
|
||||
);
|
||||
return { defaultThresholdPercent, warnThresholdPercent, providerWindowDefaults };
|
||||
const enabled = typeof record.enabled === "boolean" ? record.enabled : fallback.enabled;
|
||||
return { enabled, defaultThresholdPercent, warnThresholdPercent, providerWindowDefaults };
|
||||
}
|
||||
|
||||
function normalizeWaitForCooldownSettings(
|
||||
|
||||
123
tests/unit/combo/auto-quota-cutoff.test.ts
Normal file
123
tests/unit/combo/auto-quota-cutoff.test.ts
Normal file
@@ -0,0 +1,123 @@
|
||||
// tests/unit/combo/auto-quota-cutoff.test.ts
|
||||
// Regression coverage for auto routing quota cutoff: hard-cutoff candidates must be
|
||||
// removed before scoring/fallback so an exhausted account cannot win by latency/model fit.
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { scoreAutoTargets } from "../../../open-sse/services/combo/autoStrategy.ts";
|
||||
import type {
|
||||
AutoProviderCandidate,
|
||||
ResolvedComboTarget,
|
||||
} from "../../../open-sse/services/combo/types.ts";
|
||||
import type { ScoringWeights } from "../../../open-sse/services/autoCombo/scoring.ts";
|
||||
|
||||
const latencyOnlyWeights: ScoringWeights = {
|
||||
quota: 0,
|
||||
health: 0,
|
||||
costInv: 0,
|
||||
latencyInv: 1,
|
||||
taskFit: 0,
|
||||
stability: 0,
|
||||
tierPriority: 0,
|
||||
tierAffinity: 0,
|
||||
specificityMatch: 0,
|
||||
contextAffinity: 0,
|
||||
resetWindowAffinity: 0,
|
||||
connectionDensity: 0,
|
||||
};
|
||||
|
||||
function target(provider: string, model: string, connectionId: string): ResolvedComboTarget {
|
||||
return {
|
||||
kind: "model",
|
||||
stepId: `${provider}-${model}-${connectionId}`,
|
||||
executionKey: `${provider}/${model}@${connectionId}`,
|
||||
modelStr: `${provider}/${model}`,
|
||||
provider,
|
||||
providerId: null,
|
||||
connectionId,
|
||||
} as ResolvedComboTarget;
|
||||
}
|
||||
|
||||
function candidate(
|
||||
provider: string,
|
||||
model: string,
|
||||
connectionId: string,
|
||||
overrides: Partial<AutoProviderCandidate> = {}
|
||||
): AutoProviderCandidate {
|
||||
return {
|
||||
provider,
|
||||
model,
|
||||
stepId: `${provider}-${model}-${connectionId}`,
|
||||
executionKey: `${provider}/${model}@${connectionId}`,
|
||||
modelStr: `${provider}/${model}`,
|
||||
connectionId,
|
||||
quotaRemaining: 100,
|
||||
quotaTotal: 100,
|
||||
circuitBreakerState: "CLOSED",
|
||||
costPer1MTokens: 1,
|
||||
p95LatencyMs: 1000,
|
||||
latencyStdDev: 10,
|
||||
errorRate: 0,
|
||||
resetWindowAffinity: 0.5,
|
||||
connectionPoolSize: 1,
|
||||
...overrides,
|
||||
} as AutoProviderCandidate;
|
||||
}
|
||||
|
||||
test("auto scoring skips GLM when its 2% remaining quota hit the hard cutoff", () => {
|
||||
const targets = [target("glm", "glm-5.2", "glm-empty"), target("mcode", "mimo-auto", "mcode-ok")];
|
||||
const ranked = scoreAutoTargets(
|
||||
targets,
|
||||
[
|
||||
candidate("glm", "glm-5.2", "glm-empty", {
|
||||
quotaRemaining: 2,
|
||||
p95LatencyMs: 10,
|
||||
quotaCutoffBlocked: true,
|
||||
quotaCutoffReason: "quota_exhausted",
|
||||
}),
|
||||
candidate("mcode", "mimo-auto", "mcode-ok", {
|
||||
quotaRemaining: 100,
|
||||
p95LatencyMs: 5000,
|
||||
}),
|
||||
],
|
||||
"coding",
|
||||
latencyOnlyWeights
|
||||
);
|
||||
|
||||
assert.equal(ranked.length, 1);
|
||||
assert.equal(ranked[0]?.target.provider, "mcode");
|
||||
assert.equal(ranked[0]?.target.connectionId, "mcode-ok");
|
||||
});
|
||||
|
||||
test("blocked quota candidates are not included in the scoring pool", () => {
|
||||
const targets = [
|
||||
target("fast", "healthy", "fast-ok"),
|
||||
target("slow", "healthy", "slow-ok"),
|
||||
target("glm", "glm-5.2", "glm-empty"),
|
||||
];
|
||||
const ranked = scoreAutoTargets(
|
||||
targets,
|
||||
[
|
||||
candidate("fast", "healthy", "fast-ok", { p95LatencyMs: 100 }),
|
||||
candidate("slow", "healthy", "slow-ok", { p95LatencyMs: 1000 }),
|
||||
candidate("glm", "glm-5.2", "glm-empty", {
|
||||
quotaRemaining: 0,
|
||||
p95LatencyMs: 10000,
|
||||
quotaCutoffBlocked: true,
|
||||
quotaCutoffReason: "quota_exhausted",
|
||||
}),
|
||||
],
|
||||
"coding",
|
||||
latencyOnlyWeights
|
||||
);
|
||||
|
||||
assert.deepEqual(
|
||||
ranked.map((entry) => entry.target.provider),
|
||||
["fast", "slow"]
|
||||
);
|
||||
const slowScore = ranked.find((entry) => entry.target.provider === "slow")?.score;
|
||||
assert.equal(
|
||||
slowScore,
|
||||
0,
|
||||
"the blocked GLM latency must not inflate surviving candidates' scores"
|
||||
);
|
||||
});
|
||||
@@ -108,6 +108,29 @@ test("providerWindowDefaults: out-of-range values are clamped, garbage is pruned
|
||||
);
|
||||
});
|
||||
|
||||
test("#4483: auto-routing quota cutoff is OFF by default (opt-in)", () => {
|
||||
// The hard cutoff overlaps the existing soft penalty + cooldown, so it must not
|
||||
// change auto-routing behavior unless an operator explicitly turns it on.
|
||||
const settings = cloneDefaults();
|
||||
assert.equal(settings.quotaPreflight.enabled, false);
|
||||
assert.equal(resolveResilienceSettings({}).quotaPreflight.enabled, false);
|
||||
});
|
||||
|
||||
test("#4483: enabling the quota cutoff round-trips and preserves the other thresholds", () => {
|
||||
const next = mergeResilienceSettings(cloneDefaults(), {
|
||||
quotaPreflight: { enabled: true },
|
||||
});
|
||||
assert.equal(next.quotaPreflight.enabled, true);
|
||||
// Toggling the switch must not disturb the threshold defaults.
|
||||
assert.equal(next.quotaPreflight.defaultThresholdPercent, 2);
|
||||
assert.equal(next.quotaPreflight.warnThresholdPercent, 20);
|
||||
|
||||
const resolved = resolveResilienceSettings({
|
||||
resilienceSettings: { quotaPreflight: { enabled: true } },
|
||||
});
|
||||
assert.equal(resolved.quotaPreflight.enabled, true);
|
||||
});
|
||||
|
||||
test("resolveResilienceSettings round-trips a stored providerWindowDefaults map", () => {
|
||||
const stored = {
|
||||
resilienceSettings: {
|
||||
|
||||
Reference in New Issue
Block a user