mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-13 18:32:12 +03:00
refactor(combo): split executeTarget into gates, attempt, and loop (#12746)
Validado numa worktree combinada com as 16 PRs desta leva sobre `release/v3.8.51`: typecheck:core limpo, check-file-size e check-changelog-integrity OK, complexity 2788/3218 e cognitive 1261/1437, ESLint 0 erros nos 152 arquivos alterados, 771 testes unitários focados, 49 de integração e a suíte vitest:ui completa (2149) verdes. Guardar os contratos com testes ANTES de levantar o bloco (`05059880` no irmão, e o `287658f5` marcando os `executeTargetGates` como lift-as-is) é o que torna um refactor deste tamanho auditável. Sem essa ordem, um extract de 3 mil linhas é indistinguível de uma reescrita. Revalidei depois do merge da base: `combo-attempt-loop`, `execute-target-attempt`, `execute-target-gates` e `combo-loop-safety-timer-leak-11804` — 20/20 — mais typecheck:core limpo e o cap de arquivo OK. O #12811 entra na sequência logo em seguida, com os três commits do round-robin sobre este.
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
"_rebaseline_2026_09_05_claude_extra_usage_preflight": "Own growth: open-sse/services/combo.ts 4080->4084 (+4). buildAutoCandidates now forwards connection.providerSpecificData into evaluateQuotaCutoff so a Claude account with blockExtraUsage=false is not dropped at the 5h bar. Irreducible at the existing cutoff call site; the helper lives in claudeExtraUsage.ts (under cap). Covered by tests/unit/quota-preflight.test.ts.",
|
||||
"_rebaseline_2026_09_04_12697_combo_pin_allowlist": "PR #12697 own growth: src/sse/handlers/chat.ts 2454->2458 (+4). checkModelAvailable preflight and handleSingleModelChat now call comboPinAllowlist so a pin-only combo step cannot scan the provider pool after 502/429. Helper lives in src/lib/combos/steps.ts under cap. Covered by tests/unit/combo-pin-implicit-allowlist.test.ts (11/11).",
|
||||
"_rebaseline_2026_09_05_quota_weighted": "feat/quota-weighted-routing own growth: src/app/(dashboard)/dashboard/combos/page.tsx 5066->5080 (+14 = STRATEGY_GUIDANCE_FALLBACK + STRATEGY_RECOMMENDATIONS_FALLBACK entries for quota-weighted; copy is the spec-mandated when/avoid/example and tips, irreducible at the existing fallback maps). Rebased onto 9d1a896c6 where #12671 already grew the same file 5018->5066. Covered by tests/unit/combo/quota-weighted-strategy.test.ts + autocombo-unification.test.ts.",
|
||||
"_rebaseline_2026_09_03_combo_execute_target_attempt": "Task 3 of handleComboChat split: new leaf open-sse/services/combo/executeTargetAttempt.ts lands at 1205 (check-file-size split-newline; wc -l 1204) above new-file cap 1200. Lift-as-is from combo.ts:1533-2616 retry loop. Pure classify predicates already extracted to executeTargetClassify.ts (54 LOC). Remaining growth is I/O + side effects (handleSingleModel, quality, pin/LKGP, cooldown, lockout) that cannot leave this file without splitting the retry loop mid-request. Frozen at exact LOC so it can only shrink. Covered by tests/unit/combo/execute-target-attempt.test.ts (7/7).",
|
||||
"_rebaseline_2026_09_03_reset_aware_model_family": "Own growth: open-sse/services/combo.ts 4036->4041 (+5). buildAutoCandidates now keys the reset-aware quota cache by getQuotaFetchScope and spreads requestedModel onto the connection so Gemini windows stay off a Claude-empty Antigravity account. Irreducible wiring at the existing fetchResetAwareQuotaWithCache call site; the family helper itself lives in antigravityQuotaFamily.ts. Covered by tests/unit/reset-aware-request-scope-12600.test.ts.",
|
||||
"_rebaseline_2026_09_03_overloaded_not_provider_breaker": "fix/overloaded-not-provider-breaker own growth: open-sse/services/combo.ts 4036->4075 (check-file-size split-newline, +39). Circuit-open pre-skip now records the breaker retryAfter and, when every target was skipped that way, waits the short reset via resolveCircuitOpenWaitDecision (new leaf in comboCooldownRetry.ts) instead of crystallizing ALL_TARGETS_SKIPPED in ~43ms. skippedForCircuitOpen / earliestCircuitOpenRetryMs reset each setTry so a later iteration cannot inherit a stale retryAfter. Irreducible at the existing ALL_TARGETS_SKIPPED chokepoint (same pattern as #7301/#8213 cooldown-wait). Predicate itself lives in circuitBreaker.ts / comboPredicates.ts / chatPredicates.ts, all under cap. Covered by tests/unit/overloaded-not-provider-breaker.test.ts + combo-cooldown-retry.test.ts.",
|
||||
"_rebaseline_2026_09_03_12649_free_tier_reaudit_gateways": "PR #12649 (fix/free-tier-quota-reaudit) own growth: src/shared/constants/providers/apikey/gateways.ts 1459->1462 (+3 = the nara authHint rewritten for the re-audited 7M/day plan now wraps to two lines, plus the Prettier reflow of two pre-existing >100-col authHint lines (oneminai, freebuff) that lint-staged enforces on any touch of the file; additive text at the existing registry chokepoint, same god-file no-split rationale as prior gateways.ts rebaselines: #11786 seekai, #10987 logfare, #10531 freebuff). Covered by tests/unit/free-tier-reaudit-2026-09.test.ts and tests/unit/free-providers-batch-2026-07.test.ts.",
|
||||
@@ -430,6 +431,8 @@
|
||||
"open-sse/services/accountFallback.ts": 2467,
|
||||
"open-sse/services/adobeFireflyBrowserLogin.ts": 1401,
|
||||
"open-sse/services/combo.ts": 4084,
|
||||
"open-sse/services/combo.ts": 4080,
|
||||
"open-sse/services/combo/executeTargetAttempt.ts": 1205,
|
||||
"open-sse/translator/response/openai-responses.ts": 1466,
|
||||
"open-sse/utils/cursorAgentProtobuf.ts": 1547,
|
||||
"open-sse/utils/proxyFetch.ts": 1271,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
117
open-sse/services/combo/attemptLoopTypes.ts
Normal file
117
open-sse/services/combo/attemptLoopTypes.ts
Normal file
@@ -0,0 +1,117 @@
|
||||
/**
|
||||
* Shared types for the handleComboChat attempt-loop split (ROADMAP 3.8.52).
|
||||
*
|
||||
* Mutable loop state lives on AttemptLoopState. Read-only dependencies live on
|
||||
* AttemptLoopDeps. Do not merge the two into ComboContext.
|
||||
*
|
||||
* @internal — not part of the public combo.ts barrel.
|
||||
*/
|
||||
import type { PerTargetAdmissionHook } from "../admission/types.ts";
|
||||
import type { ResilienceSettings } from "../../../src/lib/resilience/settings";
|
||||
import type { ContextRelayConfig, UniversalHandoffConfig } from "../contextHandoff.ts";
|
||||
import type { ComboErrorEntry } from "./comboErrorAggregation.ts";
|
||||
import type { ResetWindowConfig } from "./quotaScoring.ts";
|
||||
import type { ResponseValidationConfig } from "./responseValidation.ts";
|
||||
import type { ApplyStickinessResult } from "./sessionStickiness.ts";
|
||||
import type {
|
||||
ComboLike,
|
||||
ComboLogger,
|
||||
ComboRetryAfter,
|
||||
HandleSingleModel,
|
||||
IsModelAvailable,
|
||||
ResolvedComboTarget,
|
||||
} from "./types.ts";
|
||||
|
||||
export type ExecuteTargetResult = { ok: boolean; response?: Response } | null;
|
||||
|
||||
export type AttemptLoopState = {
|
||||
orderedTargets: ResolvedComboTarget[];
|
||||
fallbackCount: number;
|
||||
recordedAttempts: number;
|
||||
comboErrors: ComboErrorEntry[];
|
||||
lastError: string | null;
|
||||
lastStatus: number | null;
|
||||
earliestRetryAfter: ComboRetryAfter | null;
|
||||
comboExpired: boolean;
|
||||
exhaustedProviders: Set<string>;
|
||||
exhaustedConnections: Set<string>;
|
||||
transientRateLimitedProviders: Set<string>;
|
||||
abortControllers: Map<number, AbortController>;
|
||||
dispatchedTargets: Set<string>;
|
||||
targetFailureTrust: Map<string, { observedFailure: boolean; allObservedFailuresQuota: boolean }>;
|
||||
comboAttemptOrder: Array<{ provider: string; model: string }>;
|
||||
skippedForCircuitOpen: boolean;
|
||||
earliestCircuitOpenRetryMs: number;
|
||||
/** Mutable attempt budget shared with dispatchWithCooldownRetry (Task 4). */
|
||||
globalAttempts: number;
|
||||
/** Quota-trust accumulators; persist across set retries and cooldown re-dispatch. */
|
||||
observedFailure: boolean;
|
||||
allObservedFailuresQuota: boolean;
|
||||
observeFailure(quotaExhausted: boolean, targetExecutionKey?: string): void;
|
||||
};
|
||||
|
||||
export type AttemptLoopDeps = {
|
||||
strategy: string;
|
||||
combo: ComboLike;
|
||||
config: Record<string, unknown> & {
|
||||
zeroLatencyOptimizationsEnabled?: boolean;
|
||||
responseValidation?: ResponseValidationConfig | null;
|
||||
failoverBeforeRetryExplicit?: boolean;
|
||||
failoverBeforeRetry?: boolean;
|
||||
predictiveTtftMs?: number;
|
||||
fallbackCompressionMode?: string;
|
||||
fallbackCompressionThreshold?: number;
|
||||
retryDelayMs?: number;
|
||||
fallbackDelayMs?: number;
|
||||
maxGlobalAttempts?: unknown;
|
||||
hedging?: boolean;
|
||||
hedgeDelayMs?: unknown;
|
||||
};
|
||||
log: ComboLogger;
|
||||
settings: Record<string, unknown> | null;
|
||||
resilienceSettings: ResilienceSettings;
|
||||
sticky: ApplyStickinessResult;
|
||||
effectiveSessionId: string | null;
|
||||
preScreenMap: Map<string, { profile?: unknown }>;
|
||||
quotaCutoffResetWindowConfig: ResetWindowConfig;
|
||||
maxRetries: number;
|
||||
traceInvocationId: string;
|
||||
clientRequestedStream: boolean;
|
||||
handleSingleModelWithTimeout: HandleSingleModel;
|
||||
isModelAvailable?: IsModelAvailable;
|
||||
perTargetAdmission?: PerTargetAdmissionHook | null;
|
||||
signal?: AbortSignal | null;
|
||||
body: Record<string, unknown>;
|
||||
startTime: number;
|
||||
releaseStickyPinOnFailure: (
|
||||
messageHash: string | null | undefined,
|
||||
failedConnectionId: string | null | undefined
|
||||
) => void;
|
||||
clearStaleLKGP: (
|
||||
comboName: string,
|
||||
executionKey: string | undefined,
|
||||
comboId: string | undefined,
|
||||
log: ComboLogger,
|
||||
tag: string
|
||||
) => void;
|
||||
/**
|
||||
* Closed-over setup values from handleComboChatInner. Optional so Task 2
|
||||
* gate tests keep compiling; attempt uses defaults when absent.
|
||||
*/
|
||||
clientManagedResponsesContext?: boolean;
|
||||
reasoningTokenBufferEnabled?: boolean;
|
||||
stickyWeightedLimit?: number;
|
||||
getWeightedStepKeyForTarget?: (target: ResolvedComboTarget) => string | null;
|
||||
universalHandoffConfig?: UniversalHandoffConfig;
|
||||
relayOptions?: { sessionId?: string | null } | null;
|
||||
relayConfig?: ContextRelayConfig | null;
|
||||
};
|
||||
|
||||
export type GateDecision =
|
||||
| { kind: "skip"; result: ExecuteTargetResult }
|
||||
| {
|
||||
kind: "proceed";
|
||||
targetForAttempt: ResolvedComboTarget;
|
||||
profile: unknown;
|
||||
protectedPriorityTarget: boolean;
|
||||
};
|
||||
575
open-sse/services/combo/comboAttemptLoop.ts
Normal file
575
open-sse/services/combo/comboAttemptLoop.ts
Normal file
@@ -0,0 +1,575 @@
|
||||
/**
|
||||
* Set-try + speculative dispatch loop for handleComboChatInner.
|
||||
* Extracted from combo.ts dispatchWithCooldownRetry (#11804 finally lives here).
|
||||
*
|
||||
* @internal — not part of the public combo.ts barrel.
|
||||
*/
|
||||
import { formatRetryAfter, getModelLockoutInfo } from "../accountFallback.ts";
|
||||
import {
|
||||
errorResponse,
|
||||
errorResponseWithComboDiagnostics,
|
||||
unavailableResponse,
|
||||
} from "../../utils/error.ts";
|
||||
import type { ComboDiagnostics } from "../../utils/error.ts";
|
||||
import { COMBO_FAILURE_THRESHOLD, recordComboFailure } from "./failureTracker.ts";
|
||||
import { buildNoUpstreamResponseDiagnostics, buildRecoveryHint } from "./pinRecovery.ts";
|
||||
import { formatExhaustedConnectionKey } from "./comboDiagFormat.ts";
|
||||
import { recordComboRequest } from "../comboMetrics.ts";
|
||||
import { notifyWebhookEvent } from "../../../src/lib/webhookDispatcher.ts";
|
||||
import { parseModel } from "../model.ts";
|
||||
import {
|
||||
formatComboOutcomes,
|
||||
buildRedactedSummary,
|
||||
resolveComboTerminalStatus,
|
||||
} from "./comboErrorAggregation.ts";
|
||||
import {
|
||||
resolveComboCooldownWaitDecision,
|
||||
resolveCircuitOpenWaitDecision,
|
||||
type ResolveComboCooldownDecisionResult,
|
||||
} from "./comboCooldownRetry.ts";
|
||||
import {
|
||||
computeClosestRetryAfter,
|
||||
waitForCooldownAwareRetry,
|
||||
} from "../../../src/sse/services/cooldownAwareRetry.ts";
|
||||
import { toRetryAfterDisplayValue } from "./validateQuality.ts";
|
||||
import { finalizeComboTrace, finishComboTrace } from "./decisionTrace.ts";
|
||||
import { isRetryAfterEligibleStatus } from "./unavailableRetryGate.ts";
|
||||
import { withQuotaExhaustionClassification } from "./quotaExhaustion.ts";
|
||||
import {
|
||||
COMBO_LOOP_SAFETY_TIMEOUT_MS,
|
||||
COMBO_SAFETY_DRAIN_MS,
|
||||
resolveDelayMs,
|
||||
} from "./comboPredicates.ts";
|
||||
import { evaluateExecuteTargetGates } from "./executeTargetGates.ts";
|
||||
import { executeTargetAttempt } from "./executeTargetAttempt.ts";
|
||||
import type { AttemptLoopDeps, AttemptLoopState, ExecuteTargetResult } from "./attemptLoopTypes.ts";
|
||||
|
||||
export type DispatchWithCooldownRetryExtra = {
|
||||
maxSetRetries: number;
|
||||
setRetryDelayMs: number;
|
||||
comboTimeoutMs: number;
|
||||
comboStartTime: number;
|
||||
comboCooldownWaitEnabled: boolean;
|
||||
comboCooldownAttempt: { current: number };
|
||||
comboCooldownBudgetLeftMs: { current: number };
|
||||
evaluateGates: typeof evaluateExecuteTargetGates;
|
||||
executeAttempt: typeof executeTargetAttempt;
|
||||
};
|
||||
|
||||
export async function dispatchWithCooldownRetry(opts: {
|
||||
state: AttemptLoopState;
|
||||
deps: AttemptLoopDeps;
|
||||
extra: DispatchWithCooldownRetryExtra;
|
||||
}): Promise<Response> {
|
||||
const { state, deps, extra } = opts;
|
||||
// #7360: persist lastStatus/earliestRetryAfter across set retries; reset
|
||||
// only on a fresh dispatch (including cooldown-aware re-dispatch).
|
||||
state.lastError = null;
|
||||
state.earliestRetryAfter = null;
|
||||
state.lastStatus = null;
|
||||
state.skippedForCircuitOpen = false;
|
||||
state.earliestCircuitOpenRetryMs = 0;
|
||||
// #11804: the loop-safety timer is armed per setTry iteration but must be
|
||||
// cleared on EVERY exit path, not just the happy one. Hoisted to function
|
||||
// scope so the `finally` at the end of this function always reaches it.
|
||||
let activeLoopSafetyTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
try {
|
||||
for (let setTry = 0; setTry <= extra.maxSetRetries; setTry++) {
|
||||
// #1731: Per-set-iteration set of providers whose quota is fully exhausted.
|
||||
// Reset each retry so providers excluded in a previous attempt get another chance.
|
||||
state.exhaustedProviders = new Set<string>();
|
||||
state.exhaustedConnections = new Set<string>();
|
||||
state.transientRateLimitedProviders = new Set<string>();
|
||||
state.skippedForCircuitOpen = false;
|
||||
state.earliestCircuitOpenRetryMs = 0;
|
||||
if (setTry > 0) {
|
||||
deps.log.info(
|
||||
"COMBO",
|
||||
`All targets failed — retrying set (${setTry}/${extra.maxSetRetries})`
|
||||
);
|
||||
await new Promise((resolve) => {
|
||||
const timer = setTimeout(resolve, extra.setRetryDelayMs);
|
||||
deps.signal?.addEventListener(
|
||||
"abort",
|
||||
() => {
|
||||
clearTimeout(timer);
|
||||
resolve(undefined);
|
||||
},
|
||||
{ once: true }
|
||||
);
|
||||
});
|
||||
if (deps.signal?.aborted) {
|
||||
deps.log.info("COMBO", "Client disconnected during set retry delay — aborting");
|
||||
return errorResponse(499, "Client disconnected");
|
||||
}
|
||||
}
|
||||
|
||||
deps.startTime = Date.now();
|
||||
state.fallbackCount = 0;
|
||||
state.recordedAttempts = 0;
|
||||
state.comboErrors = [];
|
||||
|
||||
// QA P0: assemble a sanitized diagnostic trace from the state already in scope
|
||||
// (pool size + this set-try's exhausted providers/connections + attempt order +
|
||||
// a terminal-reason code). Never touches keys/tokens — provider/model ids only.
|
||||
// Silent-stop fix: include a `recovery` hint (action verb + human next-step) so the
|
||||
// OC plugin + non-header-aware clients can render an actionable error instead of an
|
||||
// opaque 5xx. The optional `retryAfterSeconds` carries the upstream Retry-After hint.
|
||||
const buildComboDiag = (
|
||||
terminalReason: string,
|
||||
retryAfterSeconds?: number
|
||||
): ComboDiagnostics => ({
|
||||
poolSize: state.orderedTargets.length,
|
||||
attempted: state.recordedAttempts,
|
||||
excluded: [
|
||||
...[...state.exhaustedProviders].map((p) => ({ provider: p, reason: "exhausted" })),
|
||||
...[...state.exhaustedConnections].map((c) => formatExhaustedConnectionKey(String(c))),
|
||||
],
|
||||
attemptOrder: state.comboAttemptOrder,
|
||||
terminalReason,
|
||||
recovery: buildRecoveryHint(terminalReason, retryAfterSeconds),
|
||||
});
|
||||
|
||||
let globalResolve: ((res: Response) => void) | null = null;
|
||||
const globalPromise = new Promise<Response>((res) => {
|
||||
globalResolve = res;
|
||||
});
|
||||
|
||||
// G1 (silent-stop fix): the speculative loop's `Promise.race` waits on
|
||||
// `globalPromise`, which is ONLY resolved from inside a task (success or
|
||||
// fatal error). If a target hangs — e.g. the operator disabled the per-model
|
||||
// timeout (`targetTimeoutMs: 0`) and the upstream never settles — the race
|
||||
// never resolves and the request hangs forever with no response. This safety
|
||||
// promise force-resolves after the combo budget (extra.comboTimeoutMs when set,
|
||||
// otherwise a hard ceiling) so the request ALWAYS terminates with an
|
||||
// actionable 504 instead of dying silently. `state.comboExpired` is flipped so the
|
||||
// target loop stops launching new work; the existing state.comboExpired branch
|
||||
// returns the aggregated 504.
|
||||
const loopSafetyMs =
|
||||
extra.comboTimeoutMs > 0 ? extra.comboTimeoutMs : COMBO_LOOP_SAFETY_TIMEOUT_MS;
|
||||
let loopSafetyFired = false;
|
||||
let loopSafetyTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
const loopSafetyPromise = new Promise<Response>((resolve) => {
|
||||
loopSafetyTimer = setTimeout(() => {
|
||||
loopSafetyFired = true;
|
||||
deps.log.warn(
|
||||
"COMBO",
|
||||
`Combo loop safety timeout (${loopSafetyMs}ms) reached without a terminal response — force-terminating`
|
||||
);
|
||||
resolve(
|
||||
errorResponseWithComboDiagnostics(
|
||||
504,
|
||||
`Combo global timeout (${loopSafetyMs}ms) without a terminal response`,
|
||||
buildComboDiag("combo_timeout"),
|
||||
{ code: "COMBO_TIMEOUT", type: "server_error" }
|
||||
)
|
||||
);
|
||||
}, loopSafetyMs);
|
||||
loopSafetyTimer.unref?.();
|
||||
activeLoopSafetyTimer = loopSafetyTimer;
|
||||
});
|
||||
const runningTasks = new Set<Promise<void>>();
|
||||
let anySuccess = false;
|
||||
// #10681: steps already recorded as dispatched (so per-target retries do not
|
||||
// duplicate the decision).
|
||||
state.dispatchedTargets = new Set<string>();
|
||||
// G1: flip state.comboExpired as soon as the safety timer fires so the next loop
|
||||
// iteration breaks instead of launching more targets after the budget, and
|
||||
// abort every in-flight target so a hung upstream actually gets cancelled
|
||||
// (not just "response stops").
|
||||
const markLoopExpiredIfSafetyFired = () => {
|
||||
if (loopSafetyFired) {
|
||||
state.comboExpired = true;
|
||||
for (const [, ac] of state.abortControllers.entries()) ac.abort();
|
||||
}
|
||||
};
|
||||
state.abortControllers = new Map<number, AbortController>();
|
||||
const zeroLatencyOptimizationsEnabled = deps.config.zeroLatencyOptimizationsEnabled === true;
|
||||
const hasProtectedPriorityTarget =
|
||||
deps.strategy === "priority" &&
|
||||
state.orderedTargets.some((target) => target.fallbackOnlyOnQuotaExhaustion === true);
|
||||
|
||||
const executeTarget = async (i: number): Promise<ExecuteTargetResult> => {
|
||||
const gate = await extra.evaluateGates({ index: i, state, deps });
|
||||
if (gate.kind === "skip") return gate.result;
|
||||
return extra.executeAttempt({
|
||||
index: i,
|
||||
state,
|
||||
deps,
|
||||
targetForAttempt: gate.targetForAttempt,
|
||||
profile: gate.profile,
|
||||
protectedPriorityTarget: gate.protectedPriorityTarget,
|
||||
});
|
||||
};
|
||||
|
||||
for (let i = 0; i < state.orderedTargets.length; i++) {
|
||||
if (anySuccess || state.comboExpired) break;
|
||||
|
||||
const abortController = new AbortController();
|
||||
state.abortControllers.set(i, abortController);
|
||||
const onClientAbort = () => abortController.abort();
|
||||
deps.signal?.addEventListener("abort", onClientAbort);
|
||||
|
||||
const task = (async () => {
|
||||
try {
|
||||
const res = await executeTarget(i);
|
||||
if (res && !anySuccess) {
|
||||
if (res.ok) {
|
||||
anySuccess = true;
|
||||
globalResolve!(res.response!);
|
||||
for (const [idx, ac] of state.abortControllers.entries()) {
|
||||
if (idx !== i) ac.abort();
|
||||
}
|
||||
} else if (res.response) {
|
||||
// Fatal error, abort combo
|
||||
anySuccess = true;
|
||||
globalResolve!(res.response);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
deps.signal?.removeEventListener("abort", onClientAbort);
|
||||
}
|
||||
})().catch((err) => {
|
||||
const logError = deps.log.error ?? deps.log.warn;
|
||||
logError("COMBO", `Speculative task error for target ${i}`, err);
|
||||
// G2 (silent-stop fix): never leave the speculative loop waiting on an
|
||||
// unresolved globalPromise. If a task throws unexpectedly (outside
|
||||
// executeTarget's error handling) and no other task succeeds, the post-loop
|
||||
// `Promise.race([globalPromise, ...])` would hang forever. Resolve with a
|
||||
// 502 so the request terminates with an actionable error.
|
||||
if (!anySuccess && globalResolve) {
|
||||
anySuccess = true;
|
||||
globalResolve(errorResponse(502, `Combo target ${i} failed with an unexpected error`));
|
||||
}
|
||||
});
|
||||
|
||||
runningTasks.add(task);
|
||||
task.finally(() => runningTasks.delete(task));
|
||||
|
||||
if (
|
||||
zeroLatencyOptimizationsEnabled &&
|
||||
deps.config.hedging &&
|
||||
!hasProtectedPriorityTarget &&
|
||||
i + 1 < state.orderedTargets.length
|
||||
) {
|
||||
const hedgeDelay = resolveDelayMs(deps.config.hedgeDelayMs, 500);
|
||||
const timeoutPromise = new Promise<void>((r) => {
|
||||
setTimeout(r, hedgeDelay);
|
||||
});
|
||||
await Promise.race([task, globalPromise, timeoutPromise, loopSafetyPromise]);
|
||||
} else {
|
||||
await Promise.race([task, globalPromise, loopSafetyPromise]);
|
||||
}
|
||||
markLoopExpiredIfSafetyFired();
|
||||
|
||||
// Global combo timeout check: after each target completes, stop trying
|
||||
// further targets if the total elapsed time exceeds extra.comboTimeoutMs.
|
||||
if (
|
||||
!anySuccess &&
|
||||
extra.comboTimeoutMs > 0 &&
|
||||
Date.now() - extra.comboStartTime >= extra.comboTimeoutMs
|
||||
) {
|
||||
state.comboExpired = true;
|
||||
deps.log.info(
|
||||
"COMBO",
|
||||
`Combo global timeout (${extra.comboTimeoutMs}ms) reached after ` +
|
||||
`${i + 1}/${state.orderedTargets.length} targets (${state.recordedAttempts} attempted) — stopping`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (!anySuccess && runningTasks.size > 0) {
|
||||
// G1: include loopSafetyPromise so a hung last task (per-model timeout
|
||||
// disabled) cannot freeze this post-loop race forever.
|
||||
await Promise.race([globalPromise, Promise.all([...runningTasks]), loopSafetyPromise]);
|
||||
markLoopExpiredIfSafetyFired();
|
||||
}
|
||||
|
||||
// G1: if the safety timer won the race (request would otherwise hang), give
|
||||
// in-flight tasks a short drain window to land their per-model errors into
|
||||
// state.comboErrors so the 504 carries the same "tried: a (500)" summary the
|
||||
// regular state.comboExpired branch produces — then return the safety 504.
|
||||
if (loopSafetyFired && !anySuccess) {
|
||||
if (runningTasks.size > 0) {
|
||||
await Promise.race([
|
||||
Promise.allSettled([...runningTasks]),
|
||||
new Promise((resolve) => setTimeout(resolve, COMBO_SAFETY_DRAIN_MS)),
|
||||
]);
|
||||
}
|
||||
const summary = state.comboErrors
|
||||
.slice(0, 5)
|
||||
.map((e) => `${e.model} (${e.status})`)
|
||||
.join(", ");
|
||||
const msg =
|
||||
`Combo global timeout (${loopSafetyMs}ms) after ${state.recordedAttempts}/${state.orderedTargets.length} targets` +
|
||||
(state.comboErrors.length > 0
|
||||
? ` | tried: ${summary}${state.comboErrors.length > 5 ? `... (+${state.comboErrors.length - 5})` : ""}`
|
||||
: "") +
|
||||
" without a terminal response";
|
||||
return errorResponseWithComboDiagnostics(504, msg, buildComboDiag("combo_timeout"), {
|
||||
code: "COMBO_TIMEOUT",
|
||||
type: "server_error",
|
||||
});
|
||||
}
|
||||
|
||||
// #10681: finalize the decision trace (success).
|
||||
finalizeComboTrace(deps.traceInvocationId, state.orderedTargets);
|
||||
finishComboTrace(deps.traceInvocationId, { status: 200 });
|
||||
if (anySuccess) {
|
||||
// G1: clear the safety timer on the happy path so a successful combo does
|
||||
// not leave a 10-minute timer alive per request.
|
||||
if (loopSafetyTimer) {
|
||||
clearTimeout(loopSafetyTimer);
|
||||
loopSafetyTimer = null;
|
||||
}
|
||||
return await globalPromise;
|
||||
}
|
||||
|
||||
// #10681: finalize the decision trace (global timeout).
|
||||
finalizeComboTrace(deps.traceInvocationId, state.orderedTargets);
|
||||
finishComboTrace(deps.traceInvocationId, { status: 504 });
|
||||
// Global combo timeout: return aggregated error immediately, skipping set retries.
|
||||
if (state.comboExpired) {
|
||||
const summary = buildRedactedSummary(state.comboErrors);
|
||||
const msg =
|
||||
`Combo global timeout (${extra.comboTimeoutMs}ms) after ${state.recordedAttempts}/${state.orderedTargets.length} targets` +
|
||||
(state.comboErrors.length > 0 ? ` | tried: ${summary}` : "");
|
||||
const latencyMs = Date.now() - deps.startTime;
|
||||
if (state.recordedAttempts === 0) {
|
||||
recordComboRequest(deps.combo.name, null, {
|
||||
success: false,
|
||||
latencyMs,
|
||||
fallbackCount: state.fallbackCount,
|
||||
strategy: deps.strategy,
|
||||
});
|
||||
}
|
||||
notifyWebhookEvent("request.failed", {
|
||||
combo: deps.combo.name,
|
||||
reason: "COMBO_TIMEOUT",
|
||||
latencyMs,
|
||||
fallbackCount: state.fallbackCount,
|
||||
});
|
||||
return errorResponseWithComboDiagnostics(504, msg, buildComboDiag("combo_timeout"), {
|
||||
code: "COMBO_TIMEOUT",
|
||||
type: "server_error",
|
||||
});
|
||||
}
|
||||
|
||||
// All models failed in this set try
|
||||
const latencyMs = Date.now() - deps.startTime;
|
||||
if (state.recordedAttempts === 0) {
|
||||
recordComboRequest(deps.combo.name, null, {
|
||||
success: false,
|
||||
latencyMs,
|
||||
fallbackCount: state.fallbackCount,
|
||||
strategy: deps.strategy,
|
||||
});
|
||||
}
|
||||
|
||||
// Retry the entire set if more attempts remain
|
||||
if (setTry < extra.maxSetRetries) continue;
|
||||
|
||||
if (!state.lastStatus && state.recordedAttempts === 0 && extra.comboCooldownWaitEnabled) {
|
||||
const circuitOpenWait = resolveCircuitOpenWaitDecision({
|
||||
skippedForCircuitOpen: state.skippedForCircuitOpen,
|
||||
retryAfterMs: state.earliestCircuitOpenRetryMs,
|
||||
attempt: extra.comboCooldownAttempt.current,
|
||||
budgetLeftMs: extra.comboCooldownBudgetLeftMs.current,
|
||||
settings: deps.resilienceSettings.comboCooldownWait,
|
||||
});
|
||||
if (circuitOpenWait.wait) {
|
||||
deps.log.info(
|
||||
"COMBO",
|
||||
`${deps.strategy} circuit-open wait: waiting ${Math.ceil(circuitOpenWait.waitMs / 1000)}s (reason=${circuitOpenWait.reason ?? "circuit_open"}) then retrying (attempt ${extra.comboCooldownAttempt.current + 1}/${deps.resilienceSettings.comboCooldownWait.maxAttempts})`
|
||||
);
|
||||
const completed = await waitForCooldownAwareRetry(circuitOpenWait.waitMs, deps.signal);
|
||||
if (!completed) {
|
||||
return errorResponse(499, "Request aborted");
|
||||
}
|
||||
extra.comboCooldownAttempt.current += 1;
|
||||
extra.comboCooldownBudgetLeftMs.current = Math.max(
|
||||
0,
|
||||
extra.comboCooldownBudgetLeftMs.current - circuitOpenWait.waitMs
|
||||
);
|
||||
return dispatchWithCooldownRetry({ state, deps, extra });
|
||||
}
|
||||
}
|
||||
|
||||
// All set retries exhausted — return the final error
|
||||
// #10681: finalize the decision trace (all targets failed or skipped).
|
||||
finalizeComboTrace(deps.traceInvocationId, state.orderedTargets);
|
||||
finishComboTrace(deps.traceInvocationId, { status: 503 });
|
||||
if (!state.lastStatus) {
|
||||
if (state.recordedAttempts === 0) {
|
||||
notifyWebhookEvent("request.failed", {
|
||||
combo: deps.combo.name,
|
||||
reason: "ALL_TARGETS_SKIPPED",
|
||||
latencyMs,
|
||||
fallbackCount: state.fallbackCount,
|
||||
});
|
||||
return withQuotaExhaustionClassification(
|
||||
errorResponseWithComboDiagnostics(
|
||||
503,
|
||||
"Service temporarily unavailable: all targets were skipped by pre-dispatch filters",
|
||||
buildComboDiag("all_targets_skipped"),
|
||||
{ code: "ALL_TARGETS_SKIPPED", type: "service_unavailable" }
|
||||
),
|
||||
state.observedFailure ? state.allObservedFailuresQuota : null
|
||||
);
|
||||
}
|
||||
notifyWebhookEvent("request.failed", {
|
||||
combo: deps.combo.name,
|
||||
reason: "ALL_ACCOUNTS_INACTIVE",
|
||||
latencyMs,
|
||||
fallbackCount: state.fallbackCount,
|
||||
});
|
||||
recordComboFailure(deps.effectiveSessionId, deps.combo.name);
|
||||
return errorResponseWithComboDiagnostics(
|
||||
503,
|
||||
"Service temporarily unavailable: all upstream accounts are inactive",
|
||||
buildComboDiag("all_accounts_inactive"),
|
||||
{ code: "ALL_ACCOUNTS_INACTIVE", type: "service_unavailable" }
|
||||
);
|
||||
}
|
||||
|
||||
// #10501: derive the terminal HTTP status from the structured per-target
|
||||
// outcomes instead of `state.lastStatus` (whichever target happened to fail
|
||||
// LAST). A 4xx is preserved only when the request itself is genuinely
|
||||
// invalid across every eligible target; a heterogeneous mix of failure
|
||||
// classes (e.g. a quality failure + a sibling's 401) normalizes to a
|
||||
// 5xx-class status reflecting an infra/provider problem, not a client
|
||||
// error. See comboErrorAggregation.ts::resolveComboTerminalStatus.
|
||||
const status = resolveComboTerminalStatus(state.comboErrors, state.lastStatus);
|
||||
// #10314: build the terminal message from the structured per-target
|
||||
// outcomes (each distinct class+reason listed separately) instead of
|
||||
// mashing a single state.lastError with raw `[model (status)]` markers. Connection
|
||||
// identifiers are redacted. Falls back to state.lastError when no target recorded
|
||||
// a structured outcome.
|
||||
const msg =
|
||||
formatComboOutcomes(state.comboErrors) || state.lastError || "All combo models unavailable";
|
||||
|
||||
// Cooldown-aware retry: instead of crystallizing a transient failure, wait
|
||||
// out a SHORT cooldown and re-run the whole set loop. Guarded by the helper
|
||||
// (quota_exhausted/auth/not-found excluded, ceiling, attempts, budget).
|
||||
// MAX_GLOBAL_ATTEMPTS still bounds total dispatches. Available to ALL combo
|
||||
// strategies when enabled — entry is driven by earliestRetryAfter + the
|
||||
// real model-lockout reason, NOT by whichever target last overwrote
|
||||
// `status` (a later 403 must not skip the allow-list check for an earlier
|
||||
// 429's retry-after hint). SECURITY (see comboCooldownRetry.ts header): the
|
||||
// allow-list is the PRIMARY barrier and `maxWaitMs` only the SECOND one.
|
||||
// Hardcoding reason:"rate_limit" would drop the primary barrier and leave
|
||||
// only the ceiling — which does NOT cover a quota_exhausted lock carrying a
|
||||
// SHORT upstream retry-after. Model lockouts are recorded for all strategies,
|
||||
// so the real reason is always available.
|
||||
if (extra.comboCooldownWaitEnabled && state.earliestRetryAfter) {
|
||||
const decision: ResolveComboCooldownDecisionResult = resolveComboCooldownWaitDecision({
|
||||
targets: state.orderedTargets,
|
||||
earliestRetryAfter: state.earliestRetryAfter,
|
||||
attempt: extra.comboCooldownAttempt.current,
|
||||
budgetLeftMs: extra.comboCooldownBudgetLeftMs.current,
|
||||
settings: deps.resilienceSettings.comboCooldownWait,
|
||||
// Key each lookup on the TARGET's own model: quota-share combos are
|
||||
// single-model/multi-account (so this is identical to the previous
|
||||
// state.orderedTargets[0] behavior), but heterogeneous combos carry a
|
||||
// different model per target.
|
||||
lookupLock: (provider, connectionId, target) => {
|
||||
const rawModel = parseModel(target?.modelStr ?? "").model || "";
|
||||
if (!rawModel) return null;
|
||||
return getModelLockoutInfo(provider, connectionId, rawModel);
|
||||
},
|
||||
computeWaitMs: (retryAfter) => computeClosestRetryAfter(retryAfter).waitMs,
|
||||
});
|
||||
|
||||
if (decision.wait) {
|
||||
deps.log.info(
|
||||
"COMBO",
|
||||
`${deps.strategy} cooldown wait: ${msg} — waiting ${Math.ceil(
|
||||
decision.waitMs / 1000
|
||||
)}s (reason=${decision.reason ?? "?"}) then retrying (attempt ${
|
||||
extra.comboCooldownAttempt.current + 1
|
||||
}/${deps.resilienceSettings.comboCooldownWait.maxAttempts})`
|
||||
);
|
||||
const completed = await waitForCooldownAwareRetry(decision.waitMs, deps.signal);
|
||||
if (!completed) {
|
||||
deps.log.info("COMBO", `${deps.strategy} cooldown wait aborted by client disconnect`);
|
||||
return errorResponse(499, "Request aborted");
|
||||
}
|
||||
extra.comboCooldownAttempt.current += 1;
|
||||
extra.comboCooldownBudgetLeftMs.current = Math.max(
|
||||
0,
|
||||
extra.comboCooldownBudgetLeftMs.current - decision.waitMs
|
||||
);
|
||||
return dispatchWithCooldownRetry({ state, deps, extra });
|
||||
}
|
||||
}
|
||||
|
||||
// #10681: finalize the decision trace with the aggregated terminal status.
|
||||
finalizeComboTrace(deps.traceInvocationId, state.orderedTargets);
|
||||
finishComboTrace(deps.traceInvocationId, { status });
|
||||
// Retry-after decoration is separate from the wait decision above: only
|
||||
// rate-limit-class final statuses may carry a `(reset after ...)` suffix
|
||||
// (see unavailableRetryGate.ts — do not stitch a peer target's window onto
|
||||
// a config-class status like 403/422).
|
||||
if (state.earliestRetryAfter && isRetryAfterEligibleStatus(status)) {
|
||||
const retryHuman = formatRetryAfter(toRetryAfterDisplayValue(state.earliestRetryAfter));
|
||||
deps.log.warn("COMBO", `All models failed | ${msg} (${retryHuman})`);
|
||||
return withQuotaExhaustionClassification(
|
||||
unavailableResponse(status, msg, state.earliestRetryAfter, retryHuman),
|
||||
state.observedFailure ? state.allObservedFailuresQuota : null
|
||||
);
|
||||
}
|
||||
|
||||
// Silent-stop fix: bump the failure counter (pin clears on 3rd consecutive) and emit
|
||||
// `try-auto` recovery action via buildRecoveryHint so the OC plugin can show "→ Try
|
||||
// model: auto" instead of an opaque 5xx. We pass the upstream retry-after seconds to
|
||||
// the hint so the client can render a precise "wait Ns and retry" message.
|
||||
deps.log.warn("COMBO", `All models failed | ${msg}`);
|
||||
const { pinClearedNow } = recordComboFailure(deps.effectiveSessionId, deps.combo.name);
|
||||
if (pinClearedNow) {
|
||||
deps.log.info(
|
||||
"COMBO",
|
||||
`Auto-cleared session_model_history pin for combo "${deps.combo.name}" after ${COMBO_FAILURE_THRESHOLD} consecutive failures to break the silent-stop loop`
|
||||
);
|
||||
}
|
||||
const retryAfterSeconds = undefined;
|
||||
// #10966: when every observed failure was independently classified as quota/
|
||||
// balance exhaustion (isQuotaExhaustionResponse, tracked via state.observeFailure's
|
||||
// state.allObservedFailuresQuota accumulator), stamp a stable `quota_exhausted`
|
||||
// terminalReason instead of forwarding the raw upstream error string. The raw
|
||||
// string falls through buildRecoveryHint's default branch ("retry" / "failed
|
||||
// transiently"), which is actively misleading for a durable wallet/quota
|
||||
// exhaustion — retrying the same combo will never refill it.
|
||||
const terminalReason =
|
||||
state.observedFailure && state.allObservedFailuresQuota
|
||||
? "quota_exhausted"
|
||||
: (state.lastError ?? "all_models_failed");
|
||||
return withQuotaExhaustionClassification(
|
||||
errorResponseWithComboDiagnostics(
|
||||
status,
|
||||
msg,
|
||||
buildComboDiag(terminalReason, retryAfterSeconds)
|
||||
),
|
||||
state.observedFailure ? state.allObservedFailuresQuota : null
|
||||
);
|
||||
}
|
||||
|
||||
// Final fallback — when the dispatch returned without crystallizing a status (rare).
|
||||
// Surface the recovery hint with a generic retry recommendation so the client at least
|
||||
// gets a non-opaque message instead of "Combo routing completed without an upstream response".
|
||||
recordComboFailure(deps.effectiveSessionId, deps.combo.name);
|
||||
return errorResponseWithComboDiagnostics(
|
||||
503,
|
||||
"Combo routing completed without an upstream response",
|
||||
buildNoUpstreamResponseDiagnostics(state.orderedTargets.length)
|
||||
);
|
||||
} finally {
|
||||
// #11804: always release the loop-safety timer. Covering every exit path by
|
||||
// construction here means a future `return` added to this function cannot
|
||||
// silently reintroduce the leak.
|
||||
if (activeLoopSafetyTimer) {
|
||||
clearTimeout(activeLoopSafetyTimer);
|
||||
activeLoopSafetyTimer = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
1204
open-sse/services/combo/executeTargetAttempt.ts
Normal file
1204
open-sse/services/combo/executeTargetAttempt.ts
Normal file
File diff suppressed because it is too large
Load Diff
54
open-sse/services/combo/executeTargetClassify.ts
Normal file
54
open-sse/services/combo/executeTargetClassify.ts
Normal file
@@ -0,0 +1,54 @@
|
||||
/**
|
||||
* Pure classify helpers for executeTarget's retry loop.
|
||||
* Lift-as-is from combo.ts #8375 / #2101 / #4279. No I/O.
|
||||
*
|
||||
* @internal — not part of the public combo.ts barrel.
|
||||
*/
|
||||
import {
|
||||
isContextOverflow400,
|
||||
isInputBoundRequestFailure,
|
||||
isModelScoped400,
|
||||
isParamValidation400,
|
||||
} from "./comboPredicates.ts";
|
||||
|
||||
export function remainderIsHomogeneous(
|
||||
orderedTargets: { modelStr: string }[],
|
||||
index: number,
|
||||
modelStr: string
|
||||
): boolean {
|
||||
return orderedTargets.slice(index + 1).every((nextInPool) => nextInPool.modelStr === modelStr);
|
||||
}
|
||||
|
||||
export function shouldAbortOnInputBoundFailure(opts: {
|
||||
structuredError: unknown;
|
||||
remainderIsHomogeneous: boolean;
|
||||
}): boolean {
|
||||
const structured = opts.structuredError as
|
||||
{ code?: string | null; type?: string | null } | undefined;
|
||||
return isInputBoundRequestFailure(structured) && opts.remainderIsHomogeneous;
|
||||
}
|
||||
|
||||
/**
|
||||
* #2101 / #4279: body-specific 400 must surface via {ok,response}, not null.
|
||||
* Same predicate chain as combo.ts (overflow / param / model-scoped excluded).
|
||||
*/
|
||||
export function shouldSurfaceBodySpecific400(opts: {
|
||||
status: number;
|
||||
errorText: string;
|
||||
shouldFallback: boolean;
|
||||
}): boolean {
|
||||
const errorText = opts.errorText;
|
||||
return (
|
||||
opts.status === 400 &&
|
||||
opts.shouldFallback &&
|
||||
!isContextOverflow400(errorText) &&
|
||||
!isParamValidation400(errorText) &&
|
||||
!isModelScoped400(errorText) &&
|
||||
(errorText.toLowerCase().includes("context") ||
|
||||
errorText.toLowerCase().includes("prompt") ||
|
||||
errorText.toLowerCase().includes("token") ||
|
||||
errorText.toLowerCase().includes("malformed") ||
|
||||
errorText.toLowerCase().includes("invalid") ||
|
||||
errorText.toLowerCase().includes("bad request"))
|
||||
);
|
||||
}
|
||||
329
open-sse/services/combo/executeTargetGates.ts
Normal file
329
open-sse/services/combo/executeTargetGates.ts
Normal file
@@ -0,0 +1,329 @@
|
||||
/**
|
||||
* Pre-dispatch skip gates for handleComboChat's executeTarget.
|
||||
* Order is locked (spec §4.1). Do not reorder.
|
||||
*
|
||||
* Extracted from combo.ts executeTarget entry through the retry loop.
|
||||
*
|
||||
* @internal — not part of the public combo.ts barrel.
|
||||
*/
|
||||
import {
|
||||
getRuntimeProviderProfile,
|
||||
isAccountSemaphoreFull,
|
||||
isModelLocked,
|
||||
} from "../accountFallback.ts";
|
||||
import { isProviderInCooldown } from "../providerCooldownTracker.ts";
|
||||
import { checkCredentialGate, logCredentialSkip } from "../credentialGate.ts";
|
||||
import { errorResponse } from "../../utils/error.ts";
|
||||
import { getCircuitBreaker } from "../../../src/shared/utils/circuitBreaker";
|
||||
import { parseModel } from "../model.ts";
|
||||
import { canAffordRequest } from "../../../src/lib/quota/quotaScheduler.ts";
|
||||
import { getCachedProviderConnectionById } from "../../../src/lib/db/readCache.ts";
|
||||
import { lookupPositiveCap } from "./concurrencyCaps.ts";
|
||||
import { recordComboDecision } from "./decisionTrace.ts";
|
||||
import {
|
||||
getExhaustedTargetSkipReason,
|
||||
resolvePersistedConnectionCooldownSkipReason,
|
||||
} from "./comboPredicates.ts";
|
||||
import { resolveQuotaExhaustionCutoffForTarget } from "./quotaExhaustionCutoff.ts";
|
||||
import type { AttemptLoopDeps, AttemptLoopState, GateDecision } from "./attemptLoopTypes.ts";
|
||||
import type { ResolvedComboTarget } from "./types.ts";
|
||||
|
||||
/**
|
||||
* Cached vs fresh connection read for the persisted-cooldown gate.
|
||||
* `fresh: false` (first attempt) uses the 5s readCache. `fresh: true`
|
||||
* (every retry) goes straight to SQLite.
|
||||
*
|
||||
* Task 2 call sites pass `false` — same as combo.ts executeTarget today.
|
||||
* Retry-path `fresh: true` is Task 4 wiring, not this extract.
|
||||
*/
|
||||
export async function readConnectionForCooldownGate(
|
||||
connectionId: string,
|
||||
fresh: boolean
|
||||
): Promise<Record<string, unknown> | null | undefined> {
|
||||
if (!fresh) return getCachedProviderConnectionById(connectionId);
|
||||
const { getProviderConnectionById } = await import("@/lib/db/providers");
|
||||
return (await getProviderConnectionById(connectionId)) as Record<string, unknown> | null;
|
||||
}
|
||||
|
||||
export async function evaluateExecuteTargetGates(opts: {
|
||||
index: number;
|
||||
state: AttemptLoopState;
|
||||
deps: AttemptLoopDeps;
|
||||
}): Promise<GateDecision> {
|
||||
const { index: i, state, deps } = opts;
|
||||
const target = state.orderedTargets[i];
|
||||
const modelStr = target.modelStr;
|
||||
const rawModel = parseModel(modelStr).model || modelStr;
|
||||
const provider = target.provider;
|
||||
const protectedPriorityTarget =
|
||||
deps.strategy === "priority" && target.fallbackOnlyOnQuotaExhaustion === true;
|
||||
|
||||
const stopProtectedPriorityTarget = (message: string) => {
|
||||
state.observeFailure(false, target.executionKey);
|
||||
deps.clearStaleLKGP(deps.combo.name, target.executionKey, deps.combo.id, deps.log, "COMBO");
|
||||
return protectedPriorityTarget
|
||||
? { ok: false as const, response: errorResponse(503, message) }
|
||||
: null;
|
||||
};
|
||||
|
||||
// Lift-as-is from combo.ts executeTarget: only count a fallback when
|
||||
// this is not the first ordered target. Do not change the condition.
|
||||
const bumpFallback = () => {
|
||||
if (i > 0) state.fallbackCount++;
|
||||
};
|
||||
|
||||
const cb = getCircuitBreaker(provider);
|
||||
const cbStatus = cb.getStatus();
|
||||
if (cbStatus.state === "OPEN") {
|
||||
state.skippedForCircuitOpen = true;
|
||||
if (
|
||||
cbStatus.retryAfterMs > 0 &&
|
||||
(state.earliestCircuitOpenRetryMs === 0 ||
|
||||
cbStatus.retryAfterMs < state.earliestCircuitOpenRetryMs)
|
||||
) {
|
||||
state.earliestCircuitOpenRetryMs = cbStatus.retryAfterMs;
|
||||
}
|
||||
deps.log.info("COMBO", `Skipping ${modelStr} — circuit breaker OPEN for ${provider}`);
|
||||
recordComboDecision(deps.traceInvocationId, {
|
||||
step: target.executionKey,
|
||||
target: modelStr,
|
||||
decision: "skipped_before_dispatch",
|
||||
reason: "circuit_open",
|
||||
});
|
||||
bumpFallback();
|
||||
return {
|
||||
kind: "skip",
|
||||
result: stopProtectedPriorityTarget(`Provider ${provider} circuit breaker is open`),
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
deps.resilienceSettings.providerCooldown.enabled &&
|
||||
Boolean(provider && provider !== "unknown") &&
|
||||
(isProviderInCooldown(provider, target.connectionId ?? undefined, deps.resilienceSettings) ||
|
||||
isProviderInCooldown(provider, undefined, deps.resilienceSettings))
|
||||
) {
|
||||
deps.log.info("COMBO", `Skipping ${modelStr} — provider ${provider} in global cooldown`);
|
||||
recordComboDecision(deps.traceInvocationId, {
|
||||
step: target.executionKey,
|
||||
target: modelStr,
|
||||
decision: "skipped_before_dispatch",
|
||||
reason: "provider_cooldown",
|
||||
});
|
||||
bumpFallback();
|
||||
return {
|
||||
kind: "skip",
|
||||
result: stopProtectedPriorityTarget(`Provider ${provider} is in cooldown`),
|
||||
};
|
||||
}
|
||||
|
||||
const preScreenEntry = deps.preScreenMap.get(target.executionKey);
|
||||
const profile = preScreenEntry?.profile ?? (await getRuntimeProviderProfile(provider));
|
||||
|
||||
const allowRateLimitedConnection =
|
||||
Boolean(provider && provider !== "unknown") &&
|
||||
state.transientRateLimitedProviders.has(provider);
|
||||
const abortSignal = state.abortControllers.get(i)?.signal;
|
||||
const targetForAttempt = allowRateLimitedConnection
|
||||
? {
|
||||
...target,
|
||||
allowRateLimitedConnection: true,
|
||||
modelAbortSignal: abortSignal,
|
||||
}
|
||||
: { ...target, modelAbortSignal: abortSignal };
|
||||
|
||||
if (target.connectionId && !allowRateLimitedConnection) {
|
||||
const persistedSkip = await resolvePersistedConnectionCooldownSkipReason(
|
||||
target,
|
||||
(id) => readConnectionForCooldownGate(id, false),
|
||||
allowRateLimitedConnection
|
||||
);
|
||||
if (persistedSkip) {
|
||||
// Lift-as-is: combo.ts skips without observeFailure / stopProtectedPriorityTarget.
|
||||
deps.log.info("COMBO", persistedSkip);
|
||||
deps.clearStaleLKGP(deps.combo.name, target.executionKey, deps.combo.id, deps.log, "COMBO");
|
||||
bumpFallback();
|
||||
return { kind: "skip", result: null };
|
||||
}
|
||||
}
|
||||
|
||||
const exhaustedSkip = getExhaustedTargetSkipReason(
|
||||
target,
|
||||
state.exhaustedProviders,
|
||||
state.exhaustedConnections
|
||||
);
|
||||
if (exhaustedSkip) {
|
||||
deps.log.info("COMBO", exhaustedSkip);
|
||||
recordComboDecision(deps.traceInvocationId, {
|
||||
step: target.executionKey,
|
||||
target: modelStr,
|
||||
decision: "skipped_before_dispatch",
|
||||
reason: "request_exhaustion",
|
||||
});
|
||||
bumpFallback();
|
||||
return {
|
||||
kind: "skip",
|
||||
result: stopProtectedPriorityTarget(`Target ${modelStr} is unavailable`),
|
||||
};
|
||||
}
|
||||
|
||||
if (provider && rawModel && isModelLocked(provider, target.connectionId || "", rawModel)) {
|
||||
deps.log.info("COMBO", `Skipping ${modelStr} — model locked by resilience (cooldown active)`);
|
||||
recordComboDecision(deps.traceInvocationId, {
|
||||
step: target.executionKey,
|
||||
target: modelStr,
|
||||
decision: "skipped_before_dispatch",
|
||||
reason: "model_lockout",
|
||||
});
|
||||
bumpFallback();
|
||||
return {
|
||||
kind: "skip",
|
||||
result: stopProtectedPriorityTarget(`Model ${modelStr} is locked`),
|
||||
};
|
||||
}
|
||||
|
||||
if (deps.strategy !== "auto" && provider && target.connectionId) {
|
||||
const quotaCutoff = await resolveQuotaExhaustionCutoffForTarget(
|
||||
provider,
|
||||
target.connectionId,
|
||||
deps.resilienceSettings,
|
||||
deps.quotaCutoffResetWindowConfig,
|
||||
deps.combo.name,
|
||||
deps.log,
|
||||
modelStr
|
||||
);
|
||||
if (quotaCutoff.blocked) {
|
||||
deps.log.info(
|
||||
"COMBO",
|
||||
`Skipping ${modelStr} — quota exhaustion cutoff (${quotaCutoff.reason || "quota_exhausted"})`
|
||||
);
|
||||
deps.clearStaleLKGP(deps.combo.name, target.executionKey, deps.combo.id, deps.log, "COMBO");
|
||||
recordComboDecision(deps.traceInvocationId, {
|
||||
step: target.executionKey,
|
||||
target: modelStr,
|
||||
decision: "skipped_before_dispatch",
|
||||
reason: "quota_cutoff",
|
||||
});
|
||||
bumpFallback();
|
||||
state.observeFailure(true, target.executionKey);
|
||||
if (protectedPriorityTarget) {
|
||||
const protectedTargetTrust = state.targetFailureTrust.get(target.executionKey);
|
||||
if (!protectedTargetTrust?.allObservedFailuresQuota) {
|
||||
return {
|
||||
kind: "skip",
|
||||
result: {
|
||||
ok: false,
|
||||
response: errorResponse(503, `Target ${modelStr} is unavailable`),
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
return { kind: "skip", result: null };
|
||||
}
|
||||
}
|
||||
|
||||
// Lift-as-is: combo.ts reads the env flag inline, not via AttemptLoopDeps.
|
||||
if (process.env.OMNIROUTE_QUOTA_AWARE_ROUTING === "1" && provider && target.connectionId) {
|
||||
const quotaDecision = canAffordRequest(
|
||||
target.connectionId,
|
||||
modelStr,
|
||||
deps.body as Record<string, unknown> | null | undefined
|
||||
);
|
||||
if (!quotaDecision.affordable) {
|
||||
deps.log.info(
|
||||
"COMBO",
|
||||
`Skipping ${modelStr} — quota budget ${quotaDecision.reason} (remaining ${quotaDecision.tokensRemaining ?? 0}, cost ${quotaDecision.estimatedCost ?? 0})`
|
||||
);
|
||||
deps.clearStaleLKGP(deps.combo.name, target.executionKey, deps.combo.id, deps.log, "COMBO");
|
||||
bumpFallback();
|
||||
return { kind: "skip", result: null };
|
||||
}
|
||||
}
|
||||
|
||||
if (deps.isModelAvailable) {
|
||||
const available = await deps.isModelAvailable(modelStr, targetForAttempt);
|
||||
if (!available) {
|
||||
deps.log.debug?.(
|
||||
"COMBO",
|
||||
`Skipping ${modelStr} — no credentials available or model excluded`
|
||||
);
|
||||
deps.clearStaleLKGP(deps.combo.name, target.executionKey, deps.combo.id, deps.log, "COMBO");
|
||||
recordComboDecision(deps.traceInvocationId, {
|
||||
step: target.executionKey,
|
||||
target: modelStr,
|
||||
decision: "skipped_before_dispatch",
|
||||
reason: "availability",
|
||||
});
|
||||
bumpFallback();
|
||||
return {
|
||||
kind: "skip",
|
||||
result: stopProtectedPriorityTarget(`Model ${modelStr} is unavailable`),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Lift-as-is: combo.ts uses the same `as string | undefined` cast.
|
||||
const connectionId = target.connectionId as string | undefined;
|
||||
if (connectionId) {
|
||||
const gateResult = checkCredentialGate(connectionId, provider, modelStr);
|
||||
if (gateResult.allowed === false) {
|
||||
logCredentialSkip(deps.log, modelStr, gateResult.reason || "Credential gate blocked");
|
||||
recordComboDecision(deps.traceInvocationId, {
|
||||
step: target.executionKey,
|
||||
target: modelStr,
|
||||
decision: "skipped_before_dispatch",
|
||||
reason: "credential_gate",
|
||||
});
|
||||
bumpFallback();
|
||||
return {
|
||||
kind: "skip",
|
||||
result: stopProtectedPriorityTarget(`Credential gate blocked ${modelStr}`),
|
||||
};
|
||||
}
|
||||
|
||||
const maxConcurrentCap = await lookupPositiveCap(connectionId);
|
||||
if (maxConcurrentCap && isAccountSemaphoreFull(provider, connectionId, maxConcurrentCap)) {
|
||||
deps.log.info(
|
||||
"COMBO",
|
||||
`Skipping ${modelStr} — connection ${connectionId} is at max concurrency cap (${maxConcurrentCap})`
|
||||
);
|
||||
recordComboDecision(deps.traceInvocationId, {
|
||||
step: target.executionKey,
|
||||
target: modelStr,
|
||||
decision: "skipped_before_dispatch",
|
||||
reason: "concurrency_cap",
|
||||
});
|
||||
bumpFallback();
|
||||
return {
|
||||
kind: "skip",
|
||||
result: stopProtectedPriorityTarget(`Connection capacity reached for ${modelStr}`),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
deps.perTargetAdmission &&
|
||||
!(await deps.perTargetAdmission({
|
||||
modelStr,
|
||||
executionKey: target.executionKey,
|
||||
body: deps.body,
|
||||
}))
|
||||
) {
|
||||
deps.log.info("COMBO", `Skipping ${modelStr} — admission lane full (#9654)`);
|
||||
recordComboDecision(deps.traceInvocationId, {
|
||||
step: target.executionKey,
|
||||
target: modelStr,
|
||||
decision: "skipped_before_dispatch",
|
||||
reason: "admission_lane",
|
||||
});
|
||||
bumpFallback();
|
||||
return { kind: "skip", result: null };
|
||||
}
|
||||
|
||||
return {
|
||||
kind: "proceed",
|
||||
targetForAttempt: targetForAttempt as ResolvedComboTarget,
|
||||
profile,
|
||||
protectedPriorityTarget,
|
||||
};
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
/**
|
||||
* #11804 — the combo loop-safety timer must be cleared on EVERY exit path.
|
||||
*
|
||||
* `dispatchWithCooldownRetry` (open-sse/services/combo.ts) arms a
|
||||
* `dispatchWithCooldownRetry` (open-sse/services/combo/comboAttemptLoop.ts) arms a
|
||||
* `setTimeout(..., loopSafetyMs)` — 10 minutes by default — once per `setTry`
|
||||
* iteration, so a combo that never produces a terminal response still answers
|
||||
* the client with a 504 instead of hanging forever.
|
||||
@@ -29,7 +29,10 @@ import { fileURLToPath } from "node:url";
|
||||
import { dirname, resolve } from "node:path";
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const comboSrc = readFileSync(resolve(here, "../../open-sse/services/combo.ts"), "utf8");
|
||||
const comboSrc = readFileSync(
|
||||
resolve(here, "../../open-sse/services/combo/comboAttemptLoop.ts"),
|
||||
"utf8"
|
||||
);
|
||||
|
||||
test("#11804: the loop-safety timer is released in a finally, not only on success", () => {
|
||||
assert.match(
|
||||
|
||||
66
tests/unit/combo/combo-attempt-loop.test.ts
Normal file
66
tests/unit/combo/combo-attempt-loop.test.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
/**
|
||||
* Characterization for comboAttemptLoop.ts (#11804 finally + gates/attempt wiring).
|
||||
* Plan Task 4. RED until that module exists.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { dirname, resolve } from "node:path";
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
test("dispatchWithCooldownRetry clears activeLoopSafetyTimer in finally", async () => {
|
||||
const src = readFileSync(
|
||||
resolve(here, "../../../open-sse/services/combo/comboAttemptLoop.ts"),
|
||||
"utf8"
|
||||
);
|
||||
assert.match(src, /finally\s*\{[^}]*clearTimeout\(activeLoopSafetyTimer\)/s);
|
||||
assert.match(src, /activeLoopSafetyTimer = loopSafetyTimer/);
|
||||
});
|
||||
|
||||
test("dispatchWithCooldownRetry calls evaluateGates then executeAttempt, not inline executeTarget", async () => {
|
||||
const src = readFileSync(
|
||||
resolve(here, "../../../open-sse/services/combo/comboAttemptLoop.ts"),
|
||||
"utf8"
|
||||
);
|
||||
assert.match(src, /extra\.evaluateGates/);
|
||||
assert.match(src, /extra\.executeAttempt/);
|
||||
// Thin wrapper may keep the local name; the old inline retry/gate body must not.
|
||||
assert.doesNotMatch(src, /getCircuitBreaker\(provider\)/);
|
||||
assert.doesNotMatch(src, /for \(let retry = 0; retry <= deps\.maxRetries/);
|
||||
});
|
||||
|
||||
test("attempt budget lives on state.globalAttempts, not extra.globalAttempts box", async () => {
|
||||
const loopSrc = readFileSync(
|
||||
resolve(here, "../../../open-sse/services/combo/comboAttemptLoop.ts"),
|
||||
"utf8"
|
||||
);
|
||||
const comboSrc = readFileSync(resolve(here, "../../../open-sse/services/combo.ts"), "utf8");
|
||||
const attemptSrc = readFileSync(
|
||||
resolve(here, "../../../open-sse/services/combo/executeTargetAttempt.ts"),
|
||||
"utf8"
|
||||
);
|
||||
assert.match(attemptSrc, /state\.globalAttempts\+\+/);
|
||||
assert.doesNotMatch(loopSrc, /globalAttempts:\s*\{\s*current:\s*number\s*\}/);
|
||||
assert.doesNotMatch(comboSrc, /globalAttempts:\s*\{\s*current:\s*0\s*\}/);
|
||||
});
|
||||
|
||||
test("handleComboChatInner does not leave unused delay locals or unused failureTracker import", async () => {
|
||||
const comboSrc = readFileSync(resolve(here, "../../../open-sse/services/combo.ts"), "utf8");
|
||||
const inner = comboSrc.slice(
|
||||
comboSrc.indexOf("async function handleComboChatInner"),
|
||||
comboSrc.indexOf("async function handleRoundRobinCombo")
|
||||
);
|
||||
assert.doesNotMatch(inner, /const retryDelayMs = resolveDelayMs/);
|
||||
assert.doesNotMatch(inner, /const fallbackDelayMs = resolveDelayMs/);
|
||||
assert.doesNotMatch(comboSrc, /clearComboFailureTracking/);
|
||||
});
|
||||
|
||||
test("hedge delay does not declare unused timeoutResolve", async () => {
|
||||
const loopSrc = readFileSync(
|
||||
resolve(here, "../../../open-sse/services/combo/comboAttemptLoop.ts"),
|
||||
"utf8"
|
||||
);
|
||||
assert.doesNotMatch(loopSrc, /let timeoutResolve/);
|
||||
});
|
||||
288
tests/unit/combo/execute-target-attempt.test.ts
Normal file
288
tests/unit/combo/execute-target-attempt.test.ts
Normal file
@@ -0,0 +1,288 @@
|
||||
/**
|
||||
* Characterization for executeTarget retry loop + classify
|
||||
* (open-sse/services/combo/executeTargetAttempt.ts,
|
||||
* open-sse/services/combo/executeTargetClassify.ts).
|
||||
*
|
||||
* Plan Task 3. RED until those modules exist.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import type {
|
||||
AttemptLoopDeps,
|
||||
AttemptLoopState,
|
||||
} from "../../../open-sse/services/combo/attemptLoopTypes.ts";
|
||||
import type { ResolvedComboTarget } from "../../../open-sse/services/combo/types.ts";
|
||||
|
||||
function emptyState(overrides: Partial<AttemptLoopState> = {}): AttemptLoopState {
|
||||
return {
|
||||
orderedTargets: [],
|
||||
fallbackCount: 0,
|
||||
recordedAttempts: 0,
|
||||
comboErrors: [],
|
||||
lastError: null,
|
||||
lastStatus: null,
|
||||
earliestRetryAfter: null,
|
||||
comboExpired: false,
|
||||
exhaustedProviders: new Set(),
|
||||
exhaustedConnections: new Set(),
|
||||
transientRateLimitedProviders: new Set(),
|
||||
abortControllers: new Map([[0, new AbortController()]]),
|
||||
dispatchedTargets: new Set(),
|
||||
targetFailureTrust: new Map(),
|
||||
comboAttemptOrder: [],
|
||||
skippedForCircuitOpen: false,
|
||||
earliestCircuitOpenRetryMs: 0,
|
||||
globalAttempts: 0,
|
||||
observedFailure: false,
|
||||
allObservedFailuresQuota: true,
|
||||
observeFailure() {},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function baseDeps(overrides: Partial<AttemptLoopDeps> = {}): AttemptLoopDeps {
|
||||
const handleSingleModelWithTimeout = async () => {
|
||||
throw new Error("handleSingleModel must be stubbed");
|
||||
};
|
||||
return {
|
||||
strategy: "priority",
|
||||
combo: { name: "t", models: [] },
|
||||
config: {},
|
||||
log: { info() {}, warn() {}, debug() {}, error() {} },
|
||||
settings: null,
|
||||
resilienceSettings: {
|
||||
providerCooldown: { enabled: false },
|
||||
} as AttemptLoopDeps["resilienceSettings"],
|
||||
sticky: { targets: [], messageHash: null, stuck: false },
|
||||
effectiveSessionId: null,
|
||||
preScreenMap: new Map(),
|
||||
quotaCutoffResetWindowConfig: {} as AttemptLoopDeps["quotaCutoffResetWindowConfig"],
|
||||
maxRetries: 0,
|
||||
traceInvocationId: "inv-attempt",
|
||||
clientRequestedStream: false,
|
||||
handleSingleModelWithTimeout,
|
||||
body: { messages: [{ role: "user", content: "hi" }] },
|
||||
startTime: Date.now(),
|
||||
releaseStickyPinOnFailure() {},
|
||||
clearStaleLKGP() {},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function modelTarget(overrides: Partial<ResolvedComboTarget> = {}): ResolvedComboTarget {
|
||||
return {
|
||||
kind: "model",
|
||||
stepId: "s1",
|
||||
executionKey: "ek-1",
|
||||
modelStr: "openai/gpt-4o",
|
||||
provider: "openai",
|
||||
providerId: null,
|
||||
connectionId: "c-fail",
|
||||
weight: 1,
|
||||
label: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function emptyContent200(connectionId = "c-fail"): Response {
|
||||
return new Response(
|
||||
JSON.stringify({ choices: [{ message: { role: "assistant", content: "" } }] }),
|
||||
{
|
||||
status: 200,
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"x-omniroute-selected-connection-id": connectionId,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
// "invalid message format" — same fixture as combo-body-specific-400-stop-4279.test.ts
|
||||
function bodySpecific400(): Response {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
detail: "Invalid message format: the request body is malformed.",
|
||||
}),
|
||||
{ status: 400, headers: { "Content-Type": "application/json" } }
|
||||
);
|
||||
}
|
||||
|
||||
test("remainderIsHomogeneous is true only when remaining targets share modelStr", async () => {
|
||||
const { remainderIsHomogeneous } =
|
||||
await import("../../../open-sse/services/combo/executeTargetClassify.ts");
|
||||
const same = [
|
||||
{ modelStr: "openai/gpt-4o" },
|
||||
{ modelStr: "openai/gpt-4o" },
|
||||
{ modelStr: "openai/gpt-4o" },
|
||||
];
|
||||
assert.equal(remainderIsHomogeneous(same, 0, "openai/gpt-4o"), true);
|
||||
const mixed = [{ modelStr: "openai/gpt-4o" }, { modelStr: "anthropic/claude" }];
|
||||
assert.equal(remainderIsHomogeneous(mixed, 0, "openai/gpt-4o"), false);
|
||||
assert.equal(remainderIsHomogeneous(same, 2, "openai/gpt-4o"), true);
|
||||
});
|
||||
|
||||
test("shouldAbortOnInputBoundFailure requires homogeneous remainder", async () => {
|
||||
const { shouldAbortOnInputBoundFailure } =
|
||||
await import("../../../open-sse/services/combo/executeTargetClassify.ts");
|
||||
const structured = { code: "context_length_exceeded" };
|
||||
assert.equal(
|
||||
shouldAbortOnInputBoundFailure({ structuredError: structured, remainderIsHomogeneous: true }),
|
||||
true
|
||||
);
|
||||
assert.equal(
|
||||
shouldAbortOnInputBoundFailure({ structuredError: structured, remainderIsHomogeneous: false }),
|
||||
false
|
||||
);
|
||||
assert.equal(
|
||||
shouldAbortOnInputBoundFailure({
|
||||
structuredError: { code: "rate_limit" },
|
||||
remainderIsHomogeneous: true,
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("shouldSurfaceBodySpecific400 matches #4279 invalid-format 400, not model-scoped", async () => {
|
||||
const { shouldSurfaceBodySpecific400 } =
|
||||
await import("../../../open-sse/services/combo/executeTargetClassify.ts");
|
||||
assert.equal(
|
||||
shouldSurfaceBodySpecific400({
|
||||
status: 400,
|
||||
errorText: "Invalid message format: the request body is malformed.",
|
||||
shouldFallback: true,
|
||||
}),
|
||||
true
|
||||
);
|
||||
assert.equal(
|
||||
shouldSurfaceBodySpecific400({
|
||||
status: 400,
|
||||
errorText: "The requested model is not supported",
|
||||
shouldFallback: true,
|
||||
}),
|
||||
false
|
||||
);
|
||||
assert.equal(
|
||||
shouldSurfaceBodySpecific400({
|
||||
status: 429,
|
||||
errorText: "Invalid message format: the request body is malformed.",
|
||||
shouldFallback: true,
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("quality-rejected 200 calls releaseStickyPinOnFailure and records kind quality", async () => {
|
||||
const { executeTargetAttempt } =
|
||||
await import("../../../open-sse/services/combo/executeTargetAttempt.ts");
|
||||
let released: string | null = null;
|
||||
const target = modelTarget({ connectionId: "c-fail" });
|
||||
const deps = baseDeps({
|
||||
maxRetries: 0,
|
||||
clientRequestedStream: false,
|
||||
releaseStickyPinOnFailure(_hash, id) {
|
||||
released = String(id);
|
||||
},
|
||||
handleSingleModelWithTimeout: async () => emptyContent200("c-fail"),
|
||||
sticky: { targets: [], messageHash: "h", stuck: true },
|
||||
});
|
||||
const state = emptyState({
|
||||
orderedTargets: [target],
|
||||
abortControllers: new Map([[0, new AbortController()]]),
|
||||
});
|
||||
const result = await executeTargetAttempt({
|
||||
index: 0,
|
||||
state,
|
||||
deps,
|
||||
targetForAttempt: target,
|
||||
profile: {},
|
||||
protectedPriorityTarget: false,
|
||||
});
|
||||
assert.equal(released, "c-fail");
|
||||
assert.equal(
|
||||
state.comboErrors.some((e) => e.kind === "quality"),
|
||||
true
|
||||
);
|
||||
assert.equal(result, null);
|
||||
});
|
||||
|
||||
test("injection: missing releaseStickyPinOnFailure forwarding goes red on quality fail", async () => {
|
||||
const { executeTargetAttempt } =
|
||||
await import("../../../open-sse/services/combo/executeTargetAttempt.ts");
|
||||
let callCount = 0;
|
||||
const target = modelTarget({ connectionId: "c-fail" });
|
||||
const deps = baseDeps({
|
||||
maxRetries: 0,
|
||||
clientRequestedStream: false,
|
||||
releaseStickyPinOnFailure() {
|
||||
callCount += 1;
|
||||
},
|
||||
handleSingleModelWithTimeout: async () => emptyContent200("c-fail"),
|
||||
sticky: { targets: [], messageHash: "h", stuck: true },
|
||||
});
|
||||
const state = emptyState({
|
||||
orderedTargets: [target],
|
||||
abortControllers: new Map([[0, new AbortController()]]),
|
||||
});
|
||||
await executeTargetAttempt({
|
||||
index: 0,
|
||||
state,
|
||||
deps,
|
||||
targetForAttempt: target,
|
||||
profile: {},
|
||||
protectedPriorityTarget: false,
|
||||
});
|
||||
assert.equal(callCount, 1);
|
||||
});
|
||||
|
||||
test("499 surfaces {ok:false,response} and does not continue retries", async () => {
|
||||
const { executeTargetAttempt } =
|
||||
await import("../../../open-sse/services/combo/executeTargetAttempt.ts");
|
||||
let calls = 0;
|
||||
const target = modelTarget({ connectionId: "c1" });
|
||||
const deps = baseDeps({
|
||||
maxRetries: 3,
|
||||
handleSingleModelWithTimeout: async () => {
|
||||
calls += 1;
|
||||
return new Response("disconnected", { status: 499 });
|
||||
},
|
||||
});
|
||||
const state = emptyState({
|
||||
orderedTargets: [target],
|
||||
abortControllers: new Map([[0, new AbortController()]]),
|
||||
});
|
||||
const result = await executeTargetAttempt({
|
||||
index: 0,
|
||||
state,
|
||||
deps,
|
||||
targetForAttempt: target,
|
||||
profile: {},
|
||||
protectedPriorityTarget: false,
|
||||
});
|
||||
assert.equal(result?.ok, false);
|
||||
assert.equal(result?.response?.status, 499);
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
|
||||
test("body-specific 400 surfaces via {ok,response} not null", async () => {
|
||||
const { executeTargetAttempt } =
|
||||
await import("../../../open-sse/services/combo/executeTargetAttempt.ts");
|
||||
const target = modelTarget({ connectionId: "c1", modelStr: "codex/gpt-5.2" });
|
||||
const deps = baseDeps({
|
||||
maxRetries: 0,
|
||||
handleSingleModelWithTimeout: async () => bodySpecific400(),
|
||||
});
|
||||
const state = emptyState({
|
||||
orderedTargets: [target],
|
||||
abortControllers: new Map([[0, new AbortController()]]),
|
||||
});
|
||||
const result = await executeTargetAttempt({
|
||||
index: 0,
|
||||
state,
|
||||
deps,
|
||||
targetForAttempt: target,
|
||||
profile: {},
|
||||
protectedPriorityTarget: false,
|
||||
});
|
||||
assert.equal(result?.ok, false);
|
||||
assert.equal(result?.response?.status, 400);
|
||||
});
|
||||
158
tests/unit/combo/execute-target-gates.test.ts
Normal file
158
tests/unit/combo/execute-target-gates.test.ts
Normal file
@@ -0,0 +1,158 @@
|
||||
/**
|
||||
* Characterization for executeTarget pre-dispatch gates
|
||||
* (open-sse/services/combo/executeTargetGates.ts).
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { getCircuitBreaker, STATE } from "../../../src/shared/utils/circuitBreaker.ts";
|
||||
import type {
|
||||
AttemptLoopDeps,
|
||||
AttemptLoopState,
|
||||
GateDecision,
|
||||
} from "../../../open-sse/services/combo/attemptLoopTypes.ts";
|
||||
import type { ResolvedComboTarget } from "../../../open-sse/services/combo/types.ts";
|
||||
|
||||
test("attemptLoopTypes exports GateDecision discriminant", async () => {
|
||||
const mod = await import("../../../open-sse/services/combo/attemptLoopTypes.ts");
|
||||
assert.equal(typeof mod, "object");
|
||||
});
|
||||
|
||||
function emptyState(overrides: Partial<AttemptLoopState> = {}): AttemptLoopState {
|
||||
return {
|
||||
orderedTargets: [],
|
||||
fallbackCount: 0,
|
||||
recordedAttempts: 0,
|
||||
comboErrors: [],
|
||||
lastError: null,
|
||||
lastStatus: null,
|
||||
earliestRetryAfter: null,
|
||||
comboExpired: false,
|
||||
exhaustedProviders: new Set(),
|
||||
exhaustedConnections: new Set(),
|
||||
transientRateLimitedProviders: new Set(),
|
||||
abortControllers: new Map([[0, new AbortController()]]),
|
||||
dispatchedTargets: new Set(),
|
||||
targetFailureTrust: new Map(),
|
||||
comboAttemptOrder: [],
|
||||
skippedForCircuitOpen: false,
|
||||
earliestCircuitOpenRetryMs: 0,
|
||||
globalAttempts: 0,
|
||||
observedFailure: false,
|
||||
allObservedFailuresQuota: true,
|
||||
observeFailure() {},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function baseDeps(overrides: Partial<AttemptLoopDeps> = {}): AttemptLoopDeps {
|
||||
const handleSingleModelWithTimeout = async () => {
|
||||
throw new Error("handleSingleModel must not be called from gates");
|
||||
};
|
||||
return {
|
||||
strategy: "priority",
|
||||
combo: { name: "t", models: [] },
|
||||
config: {},
|
||||
log: { info() {}, warn() {}, debug() {}, error() {} },
|
||||
settings: null,
|
||||
resilienceSettings: {
|
||||
providerCooldown: { enabled: false },
|
||||
} as AttemptLoopDeps["resilienceSettings"],
|
||||
sticky: { targets: [], messageHash: null, stuck: false },
|
||||
effectiveSessionId: null,
|
||||
preScreenMap: new Map(),
|
||||
quotaCutoffResetWindowConfig: {} as AttemptLoopDeps["quotaCutoffResetWindowConfig"],
|
||||
maxRetries: 0,
|
||||
traceInvocationId: "inv-test",
|
||||
clientRequestedStream: false,
|
||||
handleSingleModelWithTimeout,
|
||||
body: { messages: [{ role: "user", content: "hi" }] },
|
||||
startTime: Date.now(),
|
||||
releaseStickyPinOnFailure() {},
|
||||
clearStaleLKGP() {},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function modelTarget(overrides: Partial<ResolvedComboTarget> = {}): ResolvedComboTarget {
|
||||
return {
|
||||
kind: "model",
|
||||
stepId: "s1",
|
||||
executionKey: "ek-1",
|
||||
modelStr: "openai/gpt-4o",
|
||||
provider: "openai",
|
||||
providerId: null,
|
||||
connectionId: "c1",
|
||||
weight: 1,
|
||||
label: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test("breaker OPEN skips and does not call handleSingleModel", async () => {
|
||||
const { evaluateExecuteTargetGates } =
|
||||
await import("../../../open-sse/services/combo/executeTargetGates.ts");
|
||||
const provider = `openai-gates-test-${Date.now()}`;
|
||||
const cb = getCircuitBreaker(provider, { failureThreshold: 1, resetTimeout: 60_000 });
|
||||
cb._onFailure("transient");
|
||||
assert.equal(cb.getStatus().state, STATE.OPEN);
|
||||
const target = modelTarget({ provider, modelStr: `${provider}/gpt-4o-mini` });
|
||||
const state = emptyState({ orderedTargets: [target] });
|
||||
const decision: GateDecision = await evaluateExecuteTargetGates({
|
||||
index: 0,
|
||||
state,
|
||||
deps: baseDeps(),
|
||||
});
|
||||
assert.equal(decision.kind, "skip");
|
||||
assert.equal(state.skippedForCircuitOpen, true);
|
||||
});
|
||||
|
||||
test("exhausted connection skip uses getExhaustedTargetSkipReason", async () => {
|
||||
const { evaluateExecuteTargetGates } =
|
||||
await import("../../../open-sse/services/combo/executeTargetGates.ts");
|
||||
const target = modelTarget({ connectionId: "conn-1", provider: "openai" });
|
||||
const state = emptyState({
|
||||
orderedTargets: [target],
|
||||
exhaustedConnections: new Set(["openai:conn-1"]),
|
||||
});
|
||||
const decision = await evaluateExecuteTargetGates({ index: 0, state, deps: baseDeps() });
|
||||
assert.equal(decision.kind, "skip");
|
||||
if (decision.kind === "skip") {
|
||||
assert.equal(decision.result, null);
|
||||
}
|
||||
});
|
||||
|
||||
test("quota cutoff skipped for strategy auto", async () => {
|
||||
const { evaluateExecuteTargetGates } =
|
||||
await import("../../../open-sse/services/combo/executeTargetGates.ts");
|
||||
const target = modelTarget();
|
||||
const state = emptyState({ orderedTargets: [target] });
|
||||
const decision = await evaluateExecuteTargetGates({
|
||||
index: 0,
|
||||
state,
|
||||
deps: baseDeps({ strategy: "auto" }),
|
||||
});
|
||||
assert.equal(decision.kind, "proceed");
|
||||
});
|
||||
|
||||
test("protected priority non-quota skip returns 503 response not null", async () => {
|
||||
const { evaluateExecuteTargetGates } =
|
||||
await import("../../../open-sse/services/combo/executeTargetGates.ts");
|
||||
const target = modelTarget({
|
||||
connectionId: "c1",
|
||||
fallbackOnlyOnQuotaExhaustion: true,
|
||||
});
|
||||
const state = emptyState({
|
||||
orderedTargets: [target],
|
||||
exhaustedConnections: new Set(["openai:c1"]),
|
||||
});
|
||||
const decision = await evaluateExecuteTargetGates({
|
||||
index: 0,
|
||||
state,
|
||||
deps: baseDeps({ strategy: "priority" }),
|
||||
});
|
||||
assert.equal(decision.kind, "skip");
|
||||
if (decision.kind === "skip") {
|
||||
assert.equal(decision.result?.ok, false);
|
||||
assert.equal(decision.result?.response?.status, 503);
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user