mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-17 20:52:15 +03:00
* fix(antigravity): classify geo-blocked egress, exclude account, real connection probe Google refuses the Cloud Code model API from unsupported egress locations with 400 FAILED_PRECONDITION "User location is not supported for the API use." Previously this surfaced as a cryptic "Antigravity upstream error (400)", never excluded the account, and the dashboard connection test stayed green because it only probed the (non-geo-restricted) OAuth userinfo endpoint. - errorClassifier: new GEO_BLOCKED type + isGeoBlockedError detection (400/403 + location-not-supported wording); non-terminal classification. - chatCore fallback: GEO_BLOCKED marks the connection and caches a 24h rate-limit-until exclusion so routing moves to other accounts instead of re-selecting the same one; never bans/expires the account. - auth: GEO_BLOCKED joins the non-terminal group (no banned/expired state). - antigravityUpstreamError: geo refusals carry an actionable message (egress location vs account problem, proxy-in-supported-region guidance). - connection test: antigravity/agy now probe the REAL streamGenerateContent surface (buildProbe), so a green tick means the model path actually works and a geo-blocked egress shows red with a clear diagnosis. * chore(changelog): fragment for #10420 antigravity geo-block resilience * chore(pr): drop prettier-version drift noise, keep only real hunks The earlier format pass (local prettier differs from the repo's pinned version) rewrapped unrelated lines in chatCore.ts and the provider test route. Restore the base formatting and re-apply only the GEO_BLOCKED fallback branch and the buildProbe connection-test changes. * fix(antigravity): strip competing-agent system prompts (429 RESOURCE_EXHAUSTED) Port decolua/9router b566b20, generalized: Antigravity flags system prompts advertising competing agents ('You are a Claude agent, built on Anthropic's Claude Agent SDK.' — Zed, Claude Code, etc.) and answers with a 429 quota error. sanitizeAntigravityGeminiRequest now strips known competitor identity sentences from systemInstruction.parts before dispatch; surrounding instruction text is untouched and non-matching prompts pass through without allocation. * chore(changelog): cover competitive prompt strip in #10420 fragment * fix(antigravity): scope GEO_BLOCKED classification to Google AI surfaces Address reviewer feedback: classifyProviderError is shared across every provider, so a lookalike 'not available in your region' body from an unrelated upstream must not receive the egress-fixable 24h exclusion treatment. Gate GEO_BLOCKED behind isGeoBlockEligibleProvider, which matches the surfaces that actually emit Google's regional-availability refusal: Cloud Code / Gemini Code Assist (antigravity, agy, cloudcode*), the Gemini Developer API (gemini, gemini-cli, vertex), plus a registry-driven fallback on executor/format. Non-Google providers fall through to their existing 400/403 classification (typically null for an unclassified 400), so a permanent block still follows its own path. * ci: re-run quality gates Trigger a fresh CI run for the PR: the previous run's 'Vitest (fast-path)' job failed in 'npm ci' because the onnxruntime-node postinstall could not download its binary from the Microsoft CDN (connect ETIMEDOUT 150.171.109.118:443). No tests ran; no code changed in this commit. * fix(antigravity): guard provider before registry lookup in geo-block gate isGeoBlockEligibleProvider passes the raw provider (string | null | undefined) to getRegistryEntry(provider: string), failing typecheck:core and the ts7-diagnostics ratchet (TS2345 at errorClassifier.ts:166). Add an explicit null guard; runtime behavior is unchanged — a falsy provider already resolved to !entry -> false. * ci: re-run quality gates (vitest npm ci onnxruntime CDN flake) --------- Co-authored-by: Rouzbeh <rqzbeh@users.noreply.github.com>
273 lines
8.8 KiB
TypeScript
273 lines
8.8 KiB
TypeScript
export type ErrorInfo = {
|
|
type: string;
|
|
code: string;
|
|
};
|
|
|
|
export type ConfiguredErrorReason =
|
|
| "auth_error"
|
|
| "quota_exhausted"
|
|
| "rate_limit_exceeded"
|
|
| "model_capacity"
|
|
| "server_error"
|
|
| "unknown";
|
|
|
|
export type ErrorRule = {
|
|
id: string;
|
|
text?: string;
|
|
status?: number;
|
|
reason?: ConfiguredErrorReason;
|
|
cooldownMs?: number;
|
|
backoff?: boolean;
|
|
};
|
|
|
|
// OpenAI-compatible error types mapping (client-facing)
|
|
export const ERROR_TYPES: Record<number, ErrorInfo> = {
|
|
400: { type: "invalid_request_error", code: "bad_request" },
|
|
401: { type: "authentication_error", code: "invalid_api_key" },
|
|
402: { type: "billing_error", code: "payment_required" },
|
|
403: { type: "permission_error", code: "insufficient_quota" },
|
|
404: { type: "invalid_request_error", code: "model_not_found" },
|
|
406: { type: "invalid_request_error", code: "model_not_supported" },
|
|
410: { type: "invalid_request_error", code: "model_shutdown" },
|
|
429: { type: "rate_limit_error", code: "rate_limit_exceeded" },
|
|
499: { type: "client_disconnected", code: "client_disconnected" },
|
|
500: { type: "server_error", code: "internal_server_error" },
|
|
502: { type: "server_error", code: "bad_gateway" },
|
|
503: { type: "server_error", code: "service_unavailable" },
|
|
504: { type: "server_error", code: "gateway_timeout" },
|
|
};
|
|
|
|
// Default error messages per status code (client-facing)
|
|
export const DEFAULT_ERROR_MESSAGES: Record<number, string> = {
|
|
400: "Bad request",
|
|
401: "Invalid API key provided",
|
|
402: "Payment required",
|
|
403: "You exceeded your current quota",
|
|
404: "Model not found",
|
|
406: "Model not supported",
|
|
410: "Model has been shut down",
|
|
429: "Rate limit exceeded",
|
|
499: "Client disconnected",
|
|
500: "Internal server error",
|
|
502: "Bad gateway - upstream provider error",
|
|
503: "Service temporarily unavailable",
|
|
504: "Gateway timeout",
|
|
};
|
|
|
|
// Exponential backoff config for rate limits.
|
|
// Preserve OmniRoute's existing 2-minute cap to avoid changing runtime behavior.
|
|
export const BACKOFF_CONFIG = {
|
|
base: 1000,
|
|
max: 2 * 60 * 1000,
|
|
maxLevel: 15,
|
|
};
|
|
|
|
export const TRANSIENT_COOLDOWN_MS = 5 * 1000;
|
|
|
|
// Cooldown durations (ms)
|
|
export const COOLDOWN_MS = {
|
|
unauthorized: 2 * 60 * 1000,
|
|
paymentRequired: 2 * 60 * 1000,
|
|
notFound: 2 * 60 * 1000,
|
|
notFoundLocal: 5 * 1000,
|
|
transientInitial: TRANSIENT_COOLDOWN_MS,
|
|
transientMax: 60 * 1000,
|
|
transient: TRANSIENT_COOLDOWN_MS,
|
|
requestNotAllowed: 5 * 1000,
|
|
rateLimit: 2 * 60 * 1000,
|
|
serviceUnavailable: 2 * 1000,
|
|
authExpired: 2 * 60 * 1000,
|
|
// Google regional-availability refusal: nothing changes region-wise on the
|
|
// account, so re-probe only after a long window (or when the operator routes
|
|
// egress through a supported-region proxy).
|
|
geoBlocked: 24 * 60 * 60 * 1000,
|
|
};
|
|
|
|
/**
|
|
* Shared rules for account fallback classification.
|
|
* Checked top-to-bottom: text rules first, then status rules.
|
|
*/
|
|
export const ERROR_RULES: ErrorRule[] = [
|
|
{
|
|
id: "no_credentials",
|
|
text: "no credentials",
|
|
cooldownMs: COOLDOWN_MS.notFound,
|
|
reason: "auth_error",
|
|
},
|
|
{
|
|
id: "request_not_allowed",
|
|
text: "request not allowed",
|
|
cooldownMs: COOLDOWN_MS.requestNotAllowed,
|
|
reason: "rate_limit_exceeded",
|
|
},
|
|
{
|
|
id: "improperly_formed_request",
|
|
text: "improperly formed request",
|
|
cooldownMs: 0,
|
|
reason: "model_capacity",
|
|
},
|
|
{ id: "rate_limit", text: "rate limit", backoff: true, reason: "rate_limit_exceeded" },
|
|
{
|
|
id: "too_many_requests",
|
|
text: "too many requests",
|
|
backoff: true,
|
|
reason: "rate_limit_exceeded",
|
|
},
|
|
{
|
|
id: "hour_quota_exceeded",
|
|
text: "hour quota",
|
|
backoff: true,
|
|
reason: "quota_exhausted",
|
|
},
|
|
{
|
|
id: "quota_has_been_exceeded",
|
|
text: "quota has been exceeded",
|
|
backoff: true,
|
|
reason: "quota_exhausted",
|
|
},
|
|
{
|
|
id: "quota_exceeded",
|
|
text: "quota exceeded",
|
|
backoff: true,
|
|
reason: "quota_exhausted",
|
|
},
|
|
{
|
|
id: "quota_will_reset",
|
|
text: "quota will reset",
|
|
backoff: true,
|
|
reason: "quota_exhausted",
|
|
},
|
|
{
|
|
id: "capacity_exhausted",
|
|
text: "exhausted your capacity",
|
|
backoff: true,
|
|
reason: "quota_exhausted",
|
|
},
|
|
{
|
|
id: "quota_exhausted",
|
|
text: "quota exhausted",
|
|
backoff: true,
|
|
reason: "quota_exhausted",
|
|
},
|
|
{
|
|
id: "free_tier_exhausted",
|
|
text: "free tier of the model has been exhausted",
|
|
backoff: true,
|
|
reason: "quota_exhausted",
|
|
},
|
|
{
|
|
id: "out_of_extra_usage",
|
|
text: "out of extra usage",
|
|
backoff: true,
|
|
reason: "quota_exhausted",
|
|
},
|
|
{
|
|
id: "extra_usage_required",
|
|
text: "extra usage required",
|
|
backoff: true,
|
|
reason: "quota_exhausted",
|
|
},
|
|
{ id: "capacity", text: "capacity", backoff: true, reason: "model_capacity" },
|
|
{ id: "overloaded", text: "overloaded", backoff: true, reason: "model_capacity" },
|
|
{ id: "high_demand", text: "high demand", backoff: true, reason: "model_capacity" },
|
|
{ id: "status_401", status: 401, cooldownMs: 0, reason: "auth_error" },
|
|
{ id: "status_402", status: 402, cooldownMs: 0, reason: "quota_exhausted" },
|
|
{ id: "status_403", status: 403, cooldownMs: 0, reason: "quota_exhausted" },
|
|
{ id: "status_404", status: 404, cooldownMs: COOLDOWN_MS.notFound, reason: "unknown" },
|
|
{ id: "status_406", status: 406, backoff: true, reason: "server_error" },
|
|
{ id: "status_408", status: 408, backoff: true, reason: "server_error" },
|
|
{ id: "status_429", status: 429, backoff: true, reason: "rate_limit_exceeded" },
|
|
{ id: "status_500", status: 500, backoff: true, reason: "server_error" },
|
|
{ id: "status_502", status: 502, backoff: true, reason: "server_error" },
|
|
{ id: "status_503", status: 503, backoff: true, reason: "server_error" },
|
|
{ id: "status_504", status: 504, backoff: true, reason: "server_error" },
|
|
];
|
|
|
|
function normalizeErrorMessage(message: unknown): string {
|
|
return String(message || "").toLowerCase();
|
|
}
|
|
|
|
export function getErrorInfo(statusCode: number): ErrorInfo {
|
|
return (
|
|
ERROR_TYPES[statusCode] ||
|
|
(statusCode >= 500
|
|
? { type: "server_error", code: "internal_server_error" }
|
|
: { type: "invalid_request_error", code: "" })
|
|
);
|
|
}
|
|
|
|
export function getDefaultErrorMessage(statusCode: number): string {
|
|
return DEFAULT_ERROR_MESSAGES[statusCode] || "An error occurred";
|
|
}
|
|
|
|
export function calculateBackoffCooldown(level = 0): number {
|
|
const safeLevel = Math.max(0, Math.floor(level));
|
|
const cooldown = BACKOFF_CONFIG.base * Math.pow(2, safeLevel);
|
|
return Math.min(cooldown, BACKOFF_CONFIG.max);
|
|
}
|
|
|
|
export function matchErrorRuleByText(message: unknown): ErrorRule | null {
|
|
const lower = normalizeErrorMessage(message);
|
|
if (!lower) return null;
|
|
return ERROR_RULES.find((rule) => rule.text && lower.includes(rule.text)) || null;
|
|
}
|
|
|
|
export function matchErrorRuleByStatus(statusCode: number): ErrorRule | null {
|
|
return ERROR_RULES.find((rule) => rule.status === statusCode) || null;
|
|
}
|
|
|
|
export function findMatchingErrorRule(statusCode: number, message: unknown): ErrorRule | null {
|
|
return matchErrorRuleByText(message) || matchErrorRuleByStatus(statusCode);
|
|
}
|
|
|
|
// #8248: NVIDIA NIM function-state DEGRADED — some NIM deployments signal a non-standard
|
|
// HTTP 400 whose body reports the backing "function" is DEGRADED (e.g. `Function id "<uuid>"
|
|
// submitted for inference is DEGRADED`) instead of a clean model-not-found/5xx. Bounded
|
|
// lookahead ({0,80}) — ReDoS-safe, no nested quantifiers.
|
|
const NIM_FUNCTION_DEGRADED_PATTERNS = [
|
|
/\bfunction\b[\s\S]{0,80}?\bDEGRADED\b/i,
|
|
/\bDEGRADED\b[\s\S]{0,80}?\bfunction\b/i,
|
|
];
|
|
|
|
export function isNimFunctionDegraded(errorText: string): boolean {
|
|
return NIM_FUNCTION_DEGRADED_PATTERNS.some((p) => p.test(errorText));
|
|
}
|
|
|
|
export interface ServiceSupervisorCooldown {
|
|
shouldFallback: true;
|
|
cooldownMs: number;
|
|
baseCooldownMs: number;
|
|
newBackoffLevel: 0;
|
|
reason: string;
|
|
skipProviderBreaker: true;
|
|
}
|
|
|
|
/**
|
|
* G-02: detect embedded service supervisor failures (X-Omni-Fallback-Hint: connection_cooldown).
|
|
* These are NOT upstream AI provider failures — they are local supervisor state changes. Returns
|
|
* a short 5s connection-cooldown decision (no provider circuit-breaker trip), or null when the
|
|
* status/header don't match.
|
|
*/
|
|
export function serviceSupervisorCooldown(
|
|
status: number,
|
|
headers: Headers | Record<string, string> | null
|
|
): ServiceSupervisorCooldown | null {
|
|
if (status !== 503 || !headers) return null;
|
|
const hintValue =
|
|
typeof (headers as Headers).get === "function"
|
|
? (headers as Headers).get("x-omni-fallback-hint")
|
|
: (headers as Record<string, string>)["x-omni-fallback-hint"] ||
|
|
(headers as Record<string, string>)["X-Omni-Fallback-Hint"];
|
|
if (typeof hintValue !== "string" || hintValue.toLowerCase() !== "connection_cooldown") {
|
|
return null;
|
|
}
|
|
return {
|
|
shouldFallback: true,
|
|
cooldownMs: 5_000,
|
|
baseCooldownMs: 5_000,
|
|
newBackoffLevel: 0,
|
|
reason: "service_not_running",
|
|
skipProviderBreaker: true,
|
|
};
|
|
}
|