mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-17 04:32:31 +03:00
Compare commits
1 Commits
fix/10314-
...
fix/10225-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cad308942b |
@@ -0,0 +1 @@
|
||||
- **fix(combo):** defer the known-context-overflow hard rejection for compressible requests so compression runs before the final context gate, instead of a raw-body estimate 400'ing generic Responses clients targeting a large model before OmniRoute can shrink it ([#10225](https://github.com/diegosouzapw/OmniRoute/issues/10225))
|
||||
@@ -1 +0,0 @@
|
||||
- fix(resilience): keep combo quality and auth failure reasons separate and redact connection labels in terminal errors (#10314)
|
||||
@@ -93,13 +93,6 @@ import {
|
||||
expandPromptCacheAffinityTargetsFromConnections,
|
||||
resolvePromptCacheAffinityKey,
|
||||
} from "./combo/promptCacheAffinity.ts";
|
||||
import {
|
||||
classifyComboOutcome,
|
||||
formatComboOutcomes,
|
||||
redactConnectionLabel,
|
||||
buildRedactedSummary,
|
||||
} from "./combo/comboErrorAggregation.ts";
|
||||
import type { ComboErrorEntry } from "./combo/comboErrorAggregation.ts";
|
||||
import type { CompressionMode } from "./compression/types.ts";
|
||||
import { getCachedProviderConnections } from "../../src/lib/db/readCache";
|
||||
import { isProviderInCooldown, recordProviderCooldown } from "./providerCooldownTracker.ts";
|
||||
@@ -598,6 +591,8 @@ export async function handleComboChat({
|
||||
nesting = null,
|
||||
hiddenModelsByProvider = getHiddenModelsByProvider(),
|
||||
clientManagedResponsesContext = false,
|
||||
deferContextOverflowWhenCompressible = false,
|
||||
compressionExclusions,
|
||||
}: HandleComboChatOptions): Promise<Response> {
|
||||
const comboCtx = createComboContext({ body, combo, settings, relayOptions, log });
|
||||
const {
|
||||
@@ -658,6 +653,8 @@ export async function handleComboChat({
|
||||
signal,
|
||||
apiKeyAllowedConnections,
|
||||
hiddenModelsByProvider,
|
||||
deferContextOverflowWhenCompressible,
|
||||
compressionExclusions,
|
||||
runCombo: handleComboChat,
|
||||
});
|
||||
if (fusionDispatch) return fusionDispatch;
|
||||
@@ -707,6 +704,8 @@ export async function handleComboChat({
|
||||
signal,
|
||||
apiKeyAllowedConnections,
|
||||
hiddenModelsByProvider,
|
||||
deferContextOverflowWhenCompressible,
|
||||
compressionExclusions,
|
||||
runCombo: handleComboChat,
|
||||
});
|
||||
if (runtimeUnitDispatch) return runtimeUnitDispatch;
|
||||
@@ -730,6 +729,8 @@ export async function handleComboChat({
|
||||
signal,
|
||||
hiddenModelsByProvider,
|
||||
clientManagedResponsesContext,
|
||||
deferContextOverflowWhenCompressible,
|
||||
compressionExclusions,
|
||||
relayOptions,
|
||||
});
|
||||
}
|
||||
@@ -757,6 +758,8 @@ export async function handleComboChat({
|
||||
buildAutoCandidates,
|
||||
hiddenModelsByProvider,
|
||||
clientManagedResponsesContext,
|
||||
deferContextOverflowWhenCompressible,
|
||||
compressionExclusions,
|
||||
});
|
||||
if ("earlyResponse" in targetResolution) return targetResolution.earlyResponse;
|
||||
const { stickyWeightedLimit, getWeightedStepKeyForTarget, preScreenMap } = targetResolution;
|
||||
@@ -860,7 +863,7 @@ export async function handleComboChat({
|
||||
let comboExpired = false;
|
||||
// Accumulator for per-model error details across targets in the current set try.
|
||||
// Reset at the start of each set retry (same lifecycle as lastError/recordedAttempts).
|
||||
let comboErrors: Array<ComboErrorEntry> = [];
|
||||
let comboErrors: Array<{ model: string; status: number; error: string }> = [];
|
||||
// Quota trust spans set retries and recursive cooldown re-dispatches. Once any
|
||||
// failure is non-quota, a nested caller must never treat this dispatch as quota-only.
|
||||
let observedFailure = false;
|
||||
@@ -1350,15 +1353,6 @@ export async function handleComboChat({
|
||||
// misleading ALL_ACCOUNTS_INACTIVE when the real issue is quality.
|
||||
lastError = `Upstream response failed quality validation: ${quality.reason}`;
|
||||
lastStatus = 502;
|
||||
// #10314: record quality failures as a FIRST-CLASS per-target outcome
|
||||
// so a quality reason is never silently dropped from the aggregated
|
||||
// terminal message when a later sibling overwrites lastError.
|
||||
comboErrors.push({
|
||||
model: modelStr,
|
||||
status: 502,
|
||||
error: quality.reason || "upstream response failed quality validation",
|
||||
kind: "quality",
|
||||
});
|
||||
if (i > 0) fallbackCount++;
|
||||
if (provider && rawModel) {
|
||||
const mlSettings = resolveModelLockoutSettings(settings);
|
||||
@@ -1866,7 +1860,6 @@ export async function handleComboChat({
|
||||
model: modelStr,
|
||||
status: result.status,
|
||||
error: errorText || String(result.status),
|
||||
kind: classifyComboOutcome(result.status, errorText),
|
||||
});
|
||||
lastStatus = result.status;
|
||||
if (i > 0) fallbackCount++;
|
||||
@@ -2060,7 +2053,6 @@ export async function handleComboChat({
|
||||
model: modelStr,
|
||||
status: result.status,
|
||||
error: errorText || String(result.status),
|
||||
kind: classifyComboOutcome(result.status, errorText),
|
||||
});
|
||||
lastStatus = result.status;
|
||||
if (i > 0) fallbackCount++;
|
||||
@@ -2215,10 +2207,15 @@ export async function handleComboChat({
|
||||
|
||||
// Global combo timeout: return aggregated error immediately, skipping set retries.
|
||||
if (comboExpired) {
|
||||
const summary = buildRedactedSummary(comboErrors);
|
||||
const summary = comboErrors
|
||||
.slice(0, 5)
|
||||
.map((e) => `${e.model} (${e.status})`)
|
||||
.join(", ");
|
||||
const msg =
|
||||
`Combo global timeout (${comboTimeoutMs}ms) after ${recordedAttempts}/${orderedTargets.length} targets` +
|
||||
(comboErrors.length > 0 ? ` | tried: ${summary}` : "");
|
||||
(comboErrors.length > 0
|
||||
? ` | tried: ${summary}${comboErrors.length > 5 ? `... (+${comboErrors.length - 5})` : ""}`
|
||||
: "");
|
||||
const latencyMs = Date.now() - startTime;
|
||||
if (recordedAttempts === 0) {
|
||||
recordComboRequest(combo.name, null, {
|
||||
@@ -2289,12 +2286,18 @@ export async function handleComboChat({
|
||||
}
|
||||
|
||||
const status = lastStatus;
|
||||
// #10314: build the terminal message from the structured per-target
|
||||
// outcomes (each distinct class+reason listed separately) instead of
|
||||
// mashing a single lastError with raw `[model (status)]` markers. Connection
|
||||
// identifiers are redacted. Falls back to lastError when no target recorded
|
||||
// a structured outcome.
|
||||
const msg = formatComboOutcomes(comboErrors) || lastError || "All combo models unavailable";
|
||||
// Build aggregated error message with per-model failure details for diagnostics.
|
||||
const comboErrorSummary =
|
||||
comboErrors.length > 0
|
||||
? " [" +
|
||||
comboErrors
|
||||
.slice(0, 5)
|
||||
.map((e) => `${e.model} (${e.status})`)
|
||||
.join(", ") +
|
||||
(comboErrors.length > 5 ? `... (+${comboErrors.length - 5})` : "") +
|
||||
"]"
|
||||
: "";
|
||||
const msg = (lastError || "All combo models unavailable") + comboErrorSummary;
|
||||
|
||||
// Cooldown-aware retry: instead of crystallizing a transient failure, wait
|
||||
// out a SHORT cooldown and re-run the whole set loop. Guarded by the helper
|
||||
@@ -2448,6 +2451,8 @@ async function handleRoundRobinCombo({
|
||||
nesting = null,
|
||||
hiddenModelsByProvider = getHiddenModelsByProvider(),
|
||||
clientManagedResponsesContext,
|
||||
deferContextOverflowWhenCompressible = false,
|
||||
compressionExclusions,
|
||||
relayOptions,
|
||||
}: HandleRoundRobinOptions): Promise<Response> {
|
||||
const config = settings
|
||||
@@ -2505,6 +2510,8 @@ async function handleRoundRobinCombo({
|
||||
const evalRankedTargets = orderTargetsByEvalScores(tagFilteredTargets, config.evalRouting, log);
|
||||
const knownContextOverflow = getKnownContextOverflow(evalRankedTargets, body, {
|
||||
clientManagedResponsesContext,
|
||||
deferContextOverflowWhenCompressible,
|
||||
compressionExclusions,
|
||||
});
|
||||
if (knownContextOverflow) {
|
||||
return errorResponseWithComboDiagnostics(
|
||||
@@ -2722,10 +2729,6 @@ async function handleRoundRobinCombo({
|
||||
let globalAttempts = 0;
|
||||
let fallbackCount = 0;
|
||||
let recordedAttempts = 0;
|
||||
// #10314: per-target outcome accumulator for the round-robin twin so the
|
||||
// terminal message lists each distinct reason separately (see the quality path
|
||||
// and the "Done with this model" path below), mirroring handleComboChat.
|
||||
const rrOutcomes: Array<ComboErrorEntry> = [];
|
||||
|
||||
// #1731: Per-request in-memory set of providers whose quota is fully exhausted.
|
||||
// When a target returns a quota-exhausted 429, remaining targets from the same
|
||||
@@ -2922,12 +2925,6 @@ async function handleRoundRobinCombo({
|
||||
// misleading ALL_ACCOUNTS_INACTIVE when the real issue is quality.
|
||||
lastError = `Upstream response failed quality validation: ${quality.reason}`;
|
||||
lastStatus = 502;
|
||||
rrOutcomes.push({
|
||||
model: modelStr,
|
||||
status: 502,
|
||||
error: quality.reason || "upstream response failed quality validation",
|
||||
kind: "quality",
|
||||
});
|
||||
if (offset > 0) fallbackCount++;
|
||||
break; // move to next model
|
||||
}
|
||||
@@ -3234,12 +3231,6 @@ async function handleRoundRobinCombo({
|
||||
recordedAttempts++;
|
||||
lastError = errorText || String(result.status);
|
||||
lastStatus = result.status;
|
||||
rrOutcomes.push({
|
||||
model: modelStr,
|
||||
status: result.status,
|
||||
error: errorText || String(result.status),
|
||||
kind: classifyComboOutcome(result.status, errorText),
|
||||
});
|
||||
if (offset > 0) fallbackCount++;
|
||||
log.warn("COMBO-RR", `${modelStr} failed, trying next model`, { status: result.status });
|
||||
|
||||
@@ -3360,10 +3351,7 @@ async function handleRoundRobinCombo({
|
||||
}
|
||||
|
||||
const status = lastStatus;
|
||||
// #10314: same structured per-target aggregation as handleComboChat — list each
|
||||
// distinct reason separately (redacted), fall back to lastError when no outcome.
|
||||
const msg =
|
||||
formatComboOutcomes(rrOutcomes) || lastError || "All round-robin combo models unavailable";
|
||||
const msg = lastError || "All round-robin combo models unavailable";
|
||||
|
||||
if (earliestRetryAfter && isRetryAfterEligibleStatus(status)) {
|
||||
const retryHuman = formatRetryAfter(toRetryAfterDisplayValue(earliestRetryAfter));
|
||||
|
||||
@@ -1,115 +0,0 @@
|
||||
/**
|
||||
* Shared combo terminal-error aggregation.
|
||||
*
|
||||
* #10314 — combo error aggregation mixes quality and auth. Prior to this module
|
||||
* the combo terminal message was built as a single `lastError` string (last
|
||||
* writer wins — it can only ever represent ONE target's reason) concatenated
|
||||
* with a raw `[model (status)]` suffix. A quality-failure reason from one
|
||||
* target and a sibling's 401 were collapsed into one client-facing sentence
|
||||
* (`invalid_api_key [openai/proxy-account-b (401)]`) and a quality reason that
|
||||
* was not the final failing target was dropped entirely.
|
||||
*
|
||||
* This module gives each per-target failure a structured {model, status, error,
|
||||
* kind} entry, so the terminal message can list every distinct reason
|
||||
* separately (and classification-labelled) instead of mashing them, and it
|
||||
* redacts connection/account identifiers that, on openai-compatible proxy
|
||||
* connections, used to surface verbatim in client-visible and shared-warn
|
||||
* strings (ops/PII leak).
|
||||
*/
|
||||
|
||||
export type ComboOutcomeKind =
|
||||
| "quality"
|
||||
| "auth"
|
||||
| "model"
|
||||
| "provider"
|
||||
| "timeout"
|
||||
| "skipped"
|
||||
| "upstream";
|
||||
|
||||
export interface ComboErrorEntry {
|
||||
model: string;
|
||||
status: number;
|
||||
error: string;
|
||||
kind: ComboOutcomeKind;
|
||||
}
|
||||
|
||||
const KIND_LABELS: Record<ComboOutcomeKind, string> = {
|
||||
quality: "quality validation",
|
||||
auth: "auth",
|
||||
model: "model",
|
||||
provider: "provider",
|
||||
timeout: "timeout",
|
||||
skipped: "skipped",
|
||||
upstream: "upstream",
|
||||
};
|
||||
|
||||
/**
|
||||
* Classify a single target's terminal outcome for the client-facing message.
|
||||
* Auth-class errors (401/403 or auth-sounding text) are kept distinct from
|
||||
* model-class (400/422) and provider-class (5xx) so a sibling's 401 is never
|
||||
* presented as "quality failed". Fall through to `model` for everything else.
|
||||
*/
|
||||
export function classifyComboOutcome(status: number, errorText: string): ComboOutcomeKind {
|
||||
const text = typeof errorText === "string" ? errorText : "";
|
||||
if (
|
||||
status === 401 ||
|
||||
status === 403 ||
|
||||
/(invalid.?api.?key|unauthorized|not.?authorized|auth(entication|orization)?)/i.test(text)
|
||||
) {
|
||||
return "auth";
|
||||
}
|
||||
if (status === 408 || status >= 499) return "timeout";
|
||||
if (status >= 500) return "provider";
|
||||
return "model";
|
||||
}
|
||||
|
||||
/**
|
||||
* Redact connection/account identifiers that can ride inside a proxy target's
|
||||
* model string (openai-compatible proxy model names often carry a connection
|
||||
* label). UUIDs and long hex hashes are truncated to a short `conn:` prefix.
|
||||
* Provider/model names operators need for debugging are left intact.
|
||||
*/
|
||||
export function redactConnectionLabel(modelStr: string | null | undefined): string {
|
||||
const label = typeof modelStr === "string" && modelStr ? modelStr : "unknown";
|
||||
return label
|
||||
.replace(
|
||||
/\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b/g,
|
||||
(m) => `conn:${m.slice(0, 8)}`
|
||||
)
|
||||
.replace(/\b[0-9a-fA-F]{16,}\b/g, (m) => `conn:${m.slice(0, 8)}`);
|
||||
}
|
||||
|
||||
/** Build the redacted, collision-free `model (status)` summary used by the
|
||||
* global-combo-timeout diagnostics path. */
|
||||
export function buildRedactedSummary(
|
||||
entries: Array<{ model: string; status: number }> | ReadonlyArray<{ model: string; status: number }>
|
||||
): string {
|
||||
const slice = entries.slice(0, 5);
|
||||
const parts = slice.map((e) => `${redactConnectionLabel(e.model)} (${e.status})`).join(", ");
|
||||
return entries.length > 5 ? `${parts}... (+${entries.length - 5})` : parts;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format per-target terminal outcomes into one client-facing sentence that keeps
|
||||
* every distinct reason separate (and classification-labelled) instead of
|
||||
* mashing a single `lastError` with raw status markers. Always redacts
|
||||
* connection identifiers unless `{ redact: false }` is explicitly passed.
|
||||
*/
|
||||
export function formatComboOutcomes(
|
||||
entries: ReadonlyArray<{ model: string; status: number; error: string; kind?: ComboOutcomeKind }>,
|
||||
opts?: { redact?: boolean }
|
||||
): string {
|
||||
if (!entries.length) return "";
|
||||
const redact = opts?.redact !== false;
|
||||
const slice = entries.slice(0, 5);
|
||||
const parts = slice.map((e) => {
|
||||
const label = redact ? redactConnectionLabel(e.model) : e.model;
|
||||
const kind = e.kind ? KIND_LABELS[e.kind] ?? e.kind : null;
|
||||
const reason = e.error || `HTTP ${e.status}`;
|
||||
const statusTxt = ` (HTTP ${e.status})`;
|
||||
return kind ? `${label}: ${kind} — ${reason}${statusTxt}` : `${label}: ${reason}${statusTxt}`;
|
||||
});
|
||||
return entries.length > 5
|
||||
? `${parts.join("; ")}... (+${entries.length - 5} more)`
|
||||
: parts.join("; ");
|
||||
}
|
||||
@@ -76,6 +76,10 @@ type PreludeBaseOptionArgs = {
|
||||
apiKeyAllowedConnections?: string[] | null;
|
||||
hiddenModelsByProvider?: HiddenModelsByProvider;
|
||||
clientManagedResponsesContext?: boolean;
|
||||
/** #10225 — defer the hard context-overflow preflight when compression is enabled. */
|
||||
deferContextOverflowWhenCompressible?: boolean;
|
||||
/** Server-side compression exclusions (#8034). */
|
||||
compressionExclusions?: import("../compression/exclusions.ts").CompressionExclusions;
|
||||
};
|
||||
|
||||
/** Rebuild handleComboChat's option bag verbatim for a recursive dispatch. */
|
||||
@@ -93,6 +97,8 @@ function buildBaseOptions(a: PreludeBaseOptionArgs): HandleComboChatOptions {
|
||||
apiKeyAllowedConnections: a.apiKeyAllowedConnections,
|
||||
hiddenModelsByProvider: a.hiddenModelsByProvider,
|
||||
clientManagedResponsesContext: a.clientManagedResponsesContext,
|
||||
deferContextOverflowWhenCompressible: a.deferContextOverflowWhenCompressible,
|
||||
compressionExclusions: a.compressionExclusions,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -366,6 +372,8 @@ export async function tryFusionDispatch(args: {
|
||||
signal?: AbortSignal | null;
|
||||
apiKeyAllowedConnections?: string[] | null;
|
||||
hiddenModelsByProvider?: HiddenModelsByProvider;
|
||||
deferContextOverflowWhenCompressible?: boolean;
|
||||
compressionExclusions?: import("../compression/exclusions.ts").CompressionExclusions;
|
||||
runCombo: RunCombo;
|
||||
}): Promise<Response | null> {
|
||||
const { cfg, combo, config, strategy, log } = args;
|
||||
@@ -589,6 +597,8 @@ export async function tryRuntimeUnitDispatch(args: {
|
||||
signal?: AbortSignal | null;
|
||||
apiKeyAllowedConnections?: string[] | null;
|
||||
hiddenModelsByProvider?: HiddenModelsByProvider;
|
||||
deferContextOverflowWhenCompressible?: boolean;
|
||||
compressionExclusions?: import("../compression/exclusions.ts").CompressionExclusions;
|
||||
runCombo: RunCombo;
|
||||
}): Promise<Response | null> {
|
||||
const { body, combo, config, strategy, allCombos, log, settings } = args;
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
*/
|
||||
|
||||
import { getResolvedModelCapabilities } from "../modelCapabilities.ts";
|
||||
import { isCompressionExcluded, type CompressionExclusions } from "../compression/exclusions.ts";
|
||||
import { deriveRequestCompatibilityRequirements } from "./comboStructure.ts";
|
||||
import type { ResolvedComboTarget } from "./types.ts";
|
||||
|
||||
@@ -28,6 +29,19 @@ export type KnownContextOverflow = {
|
||||
targetCount: number;
|
||||
};
|
||||
|
||||
export type KnownContextOverflowOptions = {
|
||||
clientManagedResponsesContext?: boolean;
|
||||
/**
|
||||
* When prompt compression is enabled for this request (global compression switch
|
||||
* AND not API-key opted-out), defer the hard preflight so chatCore's compression
|
||||
* pipeline runs before the final context gate — instead of a raw-body estimate
|
||||
* rejecting a compressible request up front. (#10225)
|
||||
*/
|
||||
deferContextOverflowWhenCompressible?: boolean;
|
||||
/** Server-side compression exclusions (#8034) — targets matching one cannot run compression. */
|
||||
compressionExclusions?: CompressionExclusions;
|
||||
};
|
||||
|
||||
// #7177: an empty array/object (e.g. a default `messages: []` some combo entrypoints inject
|
||||
// when the caller sent none) has no real content — counting it would charge a few phantom
|
||||
// "structural" tokens (JSON.stringify braces/brackets) toward the estimate, which is enough
|
||||
@@ -69,7 +83,7 @@ export function getKnownContextLimit(
|
||||
export function getKnownContextOverflow(
|
||||
targets: ResolvedComboTarget[],
|
||||
body: Record<string, unknown>,
|
||||
options: { clientManagedResponsesContext?: boolean } = {}
|
||||
options: KnownContextOverflowOptions = {}
|
||||
): KnownContextOverflow | null {
|
||||
if (targets.length === 0) return null;
|
||||
// Native Codex Responses clients compact their own item history. Let the concrete
|
||||
@@ -85,6 +99,31 @@ export function getKnownContextOverflow(
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
// #10225: a conservative raw-body context estimate must not be treated as proof
|
||||
// that a compression-enabled request cannot fit. When compression is available
|
||||
// for this request AND at least one target can actually run it, defer the hard
|
||||
// rejection so handleChatCore runs proactive compression (chatCore.ts) and its
|
||||
// post-compression enforceOutputTokenBudget becomes the final context gate —
|
||||
// returning a local `context_length_exceeded` only if the compressed body still
|
||||
// cannot fit (no upstream dispatch). Each excluded/native-codex-passthrough
|
||||
// target is skipped; if no target can compress, the fast preflight is kept.
|
||||
if (
|
||||
options.deferContextOverflowWhenCompressible === true &&
|
||||
targets.some(
|
||||
(target) =>
|
||||
!isCompressionExcluded(
|
||||
{
|
||||
provider: target.provider,
|
||||
model: target.modelStr.includes("/")
|
||||
? target.modelStr.split("/").slice(1).join("/")
|
||||
: target.modelStr,
|
||||
},
|
||||
options.compressionExclusions
|
||||
)
|
||||
)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const requirements = deriveRequestCompatibilityRequirements(body);
|
||||
if (requirements.requiredContextTokens <= 0) return null;
|
||||
|
||||
|
||||
@@ -115,6 +115,10 @@ export interface ResolveComboTargetPipelineDeps {
|
||||
hiddenModelsByProvider?: HiddenModelsByProvider;
|
||||
/** Native Responses clients (for example Codex CLI/Desktop) manage compaction themselves. */
|
||||
clientManagedResponsesContext?: boolean;
|
||||
/** #10225 — defer the hard context-overflow preflight when compression is enabled for this request. */
|
||||
deferContextOverflowWhenCompressible?: boolean;
|
||||
/** Server-side compression exclusions (#8034) — which targets can run compression. */
|
||||
compressionExclusions?: import("../compression/exclusions.ts").CompressionExclusions;
|
||||
}
|
||||
|
||||
export interface ResolvedComboTargetPipeline {
|
||||
@@ -730,6 +734,8 @@ export async function resolveComboTargetPipeline(
|
||||
|
||||
const overflow = getKnownContextOverflow(orderedTargets, body, {
|
||||
clientManagedResponsesContext: deps.clientManagedResponsesContext,
|
||||
deferContextOverflowWhenCompressible: deps.deferContextOverflowWhenCompressible,
|
||||
compressionExclusions: deps.compressionExclusions,
|
||||
});
|
||||
if (overflow) {
|
||||
return { earlyResponse: buildContextOverflowResponse(overflow, orderedTargets, log) };
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
* — logic unchanged, re-exported from combo.ts for backward compatibility.
|
||||
*/
|
||||
|
||||
import type { CompressionExclusions } from "../compression/exclusions.ts";
|
||||
import type { ProviderCandidate } from "../autoCombo/scoring.ts";
|
||||
|
||||
export const RESET_WINDOW_NAMES = ["weekly", "session", "monthly"] as const;
|
||||
@@ -112,6 +113,15 @@ export type HandleComboChatOptions = {
|
||||
hiddenModelsByProvider?: HiddenModelsByProvider;
|
||||
/** Native Responses clients (for example Codex CLI/Desktop) manage compaction themselves. */
|
||||
clientManagedResponsesContext?: boolean;
|
||||
/**
|
||||
* #10225: request-scoped flag — prompt compression is enabled for this request
|
||||
* (global compression switch ON and not opted-out by the API key). When set, the
|
||||
* combo preflight defers its hard context-overflow rejection so chatCore's
|
||||
* compression runs before the final context gate.
|
||||
*/
|
||||
deferContextOverflowWhenCompressible?: boolean;
|
||||
/** Server-side compression exclusions (#8034) — used to check which targets can run compression. */
|
||||
compressionExclusions?: CompressionExclusions;
|
||||
};
|
||||
|
||||
export type HandleRoundRobinOptions = Omit<HandleComboChatOptions, "apiKeyAllowedConnections">;
|
||||
|
||||
@@ -33,6 +33,8 @@ import type { SingleModelTarget } from "@omniroute/open-sse/services/combo/types
|
||||
import { mergeAbortSignals } from "@omniroute/open-sse/executors/base.ts";
|
||||
import { resolveRequestAutoControls } from "@omniroute/open-sse/services/autoCombo/requestControls.ts";
|
||||
import { isVerifiedNativeCodexRequest } from "@omniroute/open-sse/config/codexIdentity.ts";
|
||||
import { resolveCompressionSettings } from "@omniroute/open-sse/handlers/chatCore/compressionSettings.ts";
|
||||
import type { CompressionExclusions } from "@omniroute/open-sse/services/compression/exclusions.ts";
|
||||
import { resolveComboConfig } from "@omniroute/open-sse/services/comboConfig.ts";
|
||||
import { injectHandoffIntoBody } from "@omniroute/open-sse/services/contextHandoff.ts";
|
||||
import {
|
||||
@@ -209,6 +211,31 @@ let combosCacheTs = 0;
|
||||
let combosCacheVersionSnapshot = -1;
|
||||
const COMBOS_CACHE_TTL_MS = 10_000;
|
||||
|
||||
/**
|
||||
* #10225 — resolve whether this request's combo preflight should DEFER its hard
|
||||
* context-overflow rejection so chatCore's compression runs first.
|
||||
*
|
||||
* Mirrors handleChatCore's own enablement determination (chatCore.ts): defer only
|
||||
* when the global compression switch is ON and the API key has not opted out
|
||||
* (`apiKeyInfo.compressionEnabled !== false`). Per-target applicability (server-side
|
||||
* exclusions) is checked inside getKnownContextOverflow via the returned exclusions.
|
||||
* Fail closed (defer=false) on any lookup error — the existing hard preflight stays.
|
||||
*/
|
||||
async function resolveComboContextOverflowDeferral(
|
||||
logger: { warn?: (...args: unknown[]) => void } | null | undefined,
|
||||
apiKeyInfo: { compressionEnabled?: boolean } | null | undefined
|
||||
): Promise<{ defer: boolean; exclusions: CompressionExclusions | undefined }> {
|
||||
try {
|
||||
const compression = await resolveCompressionSettings(logger);
|
||||
return {
|
||||
defer: compression.enabled && apiKeyInfo?.compressionEnabled !== false,
|
||||
exclusions: compression.settings?.exclusions,
|
||||
};
|
||||
} catch {
|
||||
return { defer: false, exclusions: undefined };
|
||||
}
|
||||
}
|
||||
|
||||
async function getCombosCachedForChat(): Promise<unknown[]> {
|
||||
const now = Date.now();
|
||||
// Explicit non-null check: we intentionally cache and return the Promise
|
||||
@@ -824,9 +851,13 @@ async function handleChatImplementation(
|
||||
|
||||
// Context-relay keeps generation in combo.ts, but handoff injection lives here
|
||||
// because only this layer knows which connectionId was actually selected.
|
||||
const { defer: deferContextOverflowWhenCompressible, exclusions: compressionExclusions } =
|
||||
await resolveComboContextOverflowDeferral(log, apiKeyInfo);
|
||||
const response = await (handleComboChat as any)({
|
||||
body,
|
||||
combo,
|
||||
deferContextOverflowWhenCompressible,
|
||||
compressionExclusions,
|
||||
clientManagedResponsesContext:
|
||||
sourceFormat === "openai-responses" &&
|
||||
new URL(request.url).pathname.split("/").includes("responses") &&
|
||||
@@ -1103,9 +1134,13 @@ async function handleSingleModelChat(
|
||||
);
|
||||
log.info("ROUTING", `Auto-combo redirect from handleSingleModelChat for "${modelStr}"`);
|
||||
log.info("ROUTING", `Auto-combo redirect to combo flow for "${modelStr}"`);
|
||||
const { defer: sNetDefer, exclusions: sNetExclusions } =
|
||||
await resolveComboContextOverflowDeferral(log, apiKeyInfo);
|
||||
return handleComboChat({
|
||||
body,
|
||||
combo: redirectCombo,
|
||||
deferContextOverflowWhenCompressible: sNetDefer,
|
||||
compressionExclusions: sNetExclusions,
|
||||
clientManagedResponsesContext:
|
||||
detectFormatFromEndpoint(body, clientRawRequest?.endpoint || "") === "openai-responses" &&
|
||||
String(clientRawRequest?.endpoint || "")
|
||||
|
||||
173
tests/unit/combo-context-overflow-compression-probe.test.ts
Normal file
173
tests/unit/combo-context-overflow-compression-probe.test.ts
Normal file
@@ -0,0 +1,173 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
/**
|
||||
* #10225 — combo known-context-overflow must NOT hard-reject a compressible
|
||||
* request before OmniRoute's compression pipeline can run.
|
||||
*
|
||||
* Root cause: getKnownContextOverflow() estimates the RAW body (ceil(serializedChars/4)
|
||||
* over the whole Responses input[]) during combo target resolution, before any
|
||||
* compression. When every known target limit is below that raw estimate, both call
|
||||
* sites (round-robin + target-resolution) convert it into an immediate local 400
|
||||
* `context_length_exceeded` with attempted:0 — so chatCore's proactive compression
|
||||
* (which can shrink 294133→111529, 62% in the reporter's case) never runs. The only
|
||||
* existing bypass (clientManagedResponsesContext) is gated to VERIFIED native Codex
|
||||
* clients, so a generic Responses client (e.g. OpenCode) pointed at a codex model
|
||||
* still hits the hard gate.
|
||||
*
|
||||
* Fix: thread a request-scoped `deferContextOverflowWhenCompressible` flag (set when
|
||||
* the global compression switch is ON and not API-key opted-out). When set AND at
|
||||
* least one target can run compression, getKnownContextOverflow returns null so the
|
||||
* request reaches chatCore, whose post-compression enforceOutputTokenBudget becomes
|
||||
* the final context gate — a local 400 only if the compressed body still cannot fit.
|
||||
*/
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-overflow-compress-"));
|
||||
const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const { saveModelsDevCapabilities, clearModelsDevCapabilities } =
|
||||
await import("../../src/lib/modelsDevSync.ts");
|
||||
const { getKnownContextOverflow, handleComboChat } = await import(
|
||||
"../../open-sse/services/combo.ts"
|
||||
);
|
||||
|
||||
test.after(() => {
|
||||
core.resetDbInstance();
|
||||
if (ORIGINAL_DATA_DIR === undefined) {
|
||||
delete process.env.DATA_DIR;
|
||||
} else {
|
||||
process.env.DATA_DIR = ORIGINAL_DATA_DIR;
|
||||
}
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test.beforeEach(() => {
|
||||
clearModelsDevCapabilities();
|
||||
});
|
||||
|
||||
function capabilityEntry(limitContext: number | null) {
|
||||
return {
|
||||
tool_call: true,
|
||||
reasoning: false,
|
||||
attachment: false,
|
||||
structured_output: true,
|
||||
temperature: true,
|
||||
modalities_input: JSON.stringify(["text"]),
|
||||
modalities_output: JSON.stringify(["text"]),
|
||||
knowledge_cutoff: null,
|
||||
release_date: null,
|
||||
last_updated: null,
|
||||
status: null,
|
||||
family: null,
|
||||
open_weights: false,
|
||||
limit_context: limitContext,
|
||||
limit_input: limitContext,
|
||||
limit_output: 4096,
|
||||
interleaved_field: null,
|
||||
};
|
||||
}
|
||||
|
||||
function target(modelStr: string) {
|
||||
return {
|
||||
kind: "model" as const,
|
||||
stepId: modelStr,
|
||||
executionKey: modelStr,
|
||||
modelStr,
|
||||
provider: modelStr.includes("/") ? modelStr.split("/")[0] : modelStr,
|
||||
providerId: null,
|
||||
connectionId: null,
|
||||
weight: 1,
|
||||
label: null,
|
||||
};
|
||||
}
|
||||
|
||||
// A generic Responses-API body whose estimate lands near `tokens` tokens (4 chars/token).
|
||||
// Uses `input:` (not `messages:`) to mirror the OpenCode/Codex Responses surface.
|
||||
function bigResponsesBody(tokens: number) {
|
||||
return { input: [["user", "x".repeat(tokens * 4)]] };
|
||||
}
|
||||
|
||||
const noopLog = { info() {}, warn() {}, error() {}, debug() {} };
|
||||
|
||||
test("#10225 getKnownContextOverflow defers the hard overflow when compression is available", () => {
|
||||
saveModelsDevCapabilities({ codex: { "gpt-5.6-terra": capabilityEntry(272_000) } });
|
||||
const body = bigResponsesBody(275_000);
|
||||
|
||||
// Compression enabled + target can compress -> defer (null).
|
||||
assert.equal(
|
||||
getKnownContextOverflow([target("codex/gpt-5.6-terra")], body, {
|
||||
deferContextOverflowWhenCompressible: true,
|
||||
}),
|
||||
null,
|
||||
"compressible request must defer so chatCore compression can run (#10225)"
|
||||
);
|
||||
|
||||
// Compression disabled -> the existing hard overflow is preserved (never lose #7177).
|
||||
const hard = getKnownContextOverflow([target("codex/gpt-5.6-terra")], body);
|
||||
assert.ok(hard);
|
||||
assert.ok(hard.requiredContextTokens > hard.maxKnownContextTokens);
|
||||
|
||||
// Compression enabled but EVERY target is excluded from compression -> keep the hard gate.
|
||||
const excluded = getKnownContextOverflow([target("codex/gpt-5.6-terra")], body, {
|
||||
deferContextOverflowWhenCompressible: true,
|
||||
compressionExclusions: ["gpt-5.6-terra"],
|
||||
});
|
||||
assert.ok(excluded, "fully-excluded targets must retain the hard preflight");
|
||||
});
|
||||
|
||||
test("#10225 combo does not early-400 a compressible over-limit request when deferral is on", async () => {
|
||||
saveModelsDevCapabilities({ codex: { "gpt-5.6-terra": capabilityEntry(272_000) } });
|
||||
let dispatches = 0;
|
||||
|
||||
const response = await handleComboChat({
|
||||
body: bigResponsesBody(275_000),
|
||||
combo: {
|
||||
name: "codex-compress-overflow",
|
||||
strategy: "priority",
|
||||
models: ["codex/gpt-5.6-terra"],
|
||||
},
|
||||
deferContextOverflowWhenCompressible: true,
|
||||
clientManagedResponsesContext: false,
|
||||
isModelAvailable: async () => true,
|
||||
handleSingleModel: async () => {
|
||||
dispatches += 1;
|
||||
return new Response("ok", { status: 200 });
|
||||
},
|
||||
log: noopLog,
|
||||
});
|
||||
|
||||
assert.notEqual(response.status, 400, "compression-enabled request must reach chatCore");
|
||||
assert.equal(dispatches, 1, "must dispatch so chatCore compaction runs first");
|
||||
});
|
||||
|
||||
test("#10225 combo keeps the fast 400 when compression is disabled", async () => {
|
||||
saveModelsDevCapabilities({ codex: { "gpt-5.6-terra": capabilityEntry(272_000) } });
|
||||
let dispatches = 0;
|
||||
|
||||
const response = await handleComboChat({
|
||||
body: bigResponsesBody(275_000),
|
||||
combo: {
|
||||
name: "codex-compress-disabled",
|
||||
strategy: "priority",
|
||||
models: ["codex/gpt-5.6-terra"],
|
||||
},
|
||||
deferContextOverflowWhenCompressible: false,
|
||||
clientManagedResponsesContext: false,
|
||||
isModelAvailable: async () => true,
|
||||
handleSingleModel: async () => {
|
||||
dispatches += 1;
|
||||
return new Response("ok", { status: 200 });
|
||||
},
|
||||
log: noopLog,
|
||||
});
|
||||
|
||||
assert.equal(response.status, 400);
|
||||
assert.equal(dispatches, 0, "#7177 anti-exhaustion guard must survive when compression is off");
|
||||
const body = await response.json();
|
||||
assert.equal(body.error.code, "context_length_exceeded");
|
||||
});
|
||||
@@ -1,54 +0,0 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
classifyComboOutcome,
|
||||
formatComboOutcomes,
|
||||
redactConnectionLabel,
|
||||
buildRedactedSummary,
|
||||
} from "../../open-sse/services/combo/comboErrorAggregation.ts";
|
||||
|
||||
// #10314 — combo error aggregation mixes quality and auth.
|
||||
// Regression guard for the pure aggregation helpers: a quality-failure reason from one
|
||||
// target and a sibling's 401 must be presented as SEPARATE classified outcomes (never
|
||||
// mashed into a single lastError), and account/connection identifiers must be redacted
|
||||
// from client-visible and shared-warn strings.
|
||||
|
||||
test("#10314: classifyComboOutcome keeps auth distinct from quality/model", () => {
|
||||
assert.equal(classifyComboOutcome(401, "invalid_api_key"), "auth");
|
||||
assert.equal(classifyComboOutcome(403, "not authorized"), "auth");
|
||||
// 5xx sleep to the "timeout" class (>=499 is checked before >=500).
|
||||
assert.equal(classifyComboOutcome(503, "upstream unavailable"), "timeout");
|
||||
assert.equal(classifyComboOutcome(408, "timeout"), "timeout");
|
||||
assert.equal(classifyComboOutcome(400, "bad request"), "model");
|
||||
});
|
||||
|
||||
test("#10314: formatComboOutcomes lists quality and auth reasons SEPARATELY (both visible)", () => {
|
||||
const msg = formatComboOutcomes([
|
||||
{ model: "openai/model-quality", status: 502, error: "response failed quality validation", kind: "quality" },
|
||||
{ model: "openai/proxy-account-b", status: 401, error: "invalid_api_key", kind: "auth" },
|
||||
]);
|
||||
assert.match(msg, /quality validation/);
|
||||
assert.match(msg, /invalid_api_key/);
|
||||
assert.match(msg, /auth/);
|
||||
assert.ok(msg.indexOf("quality validation") < msg.indexOf("invalid_api_key"));
|
||||
});
|
||||
|
||||
test("#10314: redactConnectionLabel masks connection/account identifiers", () => {
|
||||
assert.equal(
|
||||
redactConnectionLabel("openai/proxy-account-b"),
|
||||
"openai/proxy-account-b"
|
||||
);
|
||||
const withUuid = redactConnectionLabel("openai/8a4f0c6e-3b27-4c51-9d88-1f2a3b4c5d6e");
|
||||
assert.equal(withUuid, "openai/conn:8a4f0c6e");
|
||||
const withHex = redactConnectionLabel("openai/0f1e2d3c4b5a69788796170a1b2c3d4e5f607182");
|
||||
assert.equal(withHex, "openai/conn:0f1e2d3c");
|
||||
});
|
||||
|
||||
test("#10314: buildRedactedSummary is redacted and truncates past 5 entries", () => {
|
||||
const s = buildRedactedSummary(
|
||||
Array.from({ length: 6 }, (_, i) => ({ model: `openai/8a4f0c6e-3b27-4c51-9d88-1f2a3b4c5d6e-${i}`, status: 401 + i }))
|
||||
);
|
||||
assert.ok(!s.includes("8a4f0c6e-3b27"), "summary must not leak a full UUID");
|
||||
assert.match(s, /conn:8a4f0c6e/);
|
||||
assert.match(s, /\(\+1\)/);
|
||||
});
|
||||
Reference in New Issue
Block a user