mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-31 04:12:10 +03:00
Merge branch 'pr-2019' into release/v3.8.0
# Conflicts: # open-sse/handlers/chatCore.ts # open-sse/services/comboConfig.ts # open-sse/services/usage.ts # src/app/(dashboard)/dashboard/providers/[id]/page.tsx # src/app/(dashboard)/dashboard/usage/components/ProviderLimits/index.tsx # src/app/(dashboard)/dashboard/usage/components/ProviderLimits/utils.tsx # src/app/api/usage/analytics/route.ts # src/lib/db/migrationRunner.ts # src/lib/usage/providerLimits.ts # src/shared/constants/providers.ts # src/sse/handlers/chat.ts # tests/unit/provider-limits-ui.test.ts # tests/unit/usage-analytics.test.ts # tests/unit/usage-service-hardening.test.ts
This commit is contained in:
@@ -602,14 +602,39 @@ function toFiniteNumberOrNull(value: unknown): number | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
function isSemaphoreTimeoutError(error: unknown): error is Error & { code: string } {
|
||||
function isSemaphoreCapacityError(error: unknown): error is Error & { code: string } {
|
||||
return (
|
||||
!!error &&
|
||||
typeof error === "object" &&
|
||||
(error as { code?: unknown }).code === "SEMAPHORE_TIMEOUT"
|
||||
((error as { code?: unknown }).code === "SEMAPHORE_TIMEOUT" ||
|
||||
(error as { code?: unknown }).code === "SEMAPHORE_QUEUE_FULL")
|
||||
);
|
||||
}
|
||||
|
||||
function createStreamingErrorResult(statusCode: number, message: string, code?: string) {
|
||||
const errorBody = buildErrorBody(statusCode, message);
|
||||
if (code) {
|
||||
errorBody.error.code = code;
|
||||
}
|
||||
|
||||
const body = `data: ${JSON.stringify(errorBody)}\n\ndata: [DONE]\n\n`;
|
||||
|
||||
return {
|
||||
success: false as const,
|
||||
status: statusCode,
|
||||
error: message,
|
||||
response: new Response(body, {
|
||||
status: statusCode,
|
||||
headers: {
|
||||
"Content-Type": "text/event-stream",
|
||||
"Cache-Control": "no-cache, no-transform",
|
||||
Connection: "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function wrapReadableStreamWithFinalize<T>(
|
||||
readable: ReadableStream<T>,
|
||||
finalize: () => void
|
||||
@@ -3051,16 +3076,13 @@ export async function handleChatCore({
|
||||
);
|
||||
} catch (error) {
|
||||
trackPendingRequest(model, provider, connectionId, false);
|
||||
if (isSemaphoreTimeoutError(error)) {
|
||||
if (isSemaphoreCapacityError(error)) {
|
||||
appendRequestLog({
|
||||
model,
|
||||
provider,
|
||||
connectionId,
|
||||
status: `FAILED ${error.code}`,
|
||||
}).catch(() => {});
|
||||
if (isCombo) {
|
||||
throw error;
|
||||
}
|
||||
const failureMessage = error.message || "Semaphore timeout";
|
||||
persistAttemptLogs({
|
||||
status: HTTP_STATUS.RATE_LIMITED,
|
||||
@@ -3071,7 +3093,14 @@ export async function handleChatCore({
|
||||
cacheSource: "upstream",
|
||||
});
|
||||
persistFailureUsage(HTTP_STATUS.RATE_LIMITED, error.code);
|
||||
return createErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage);
|
||||
const result = stream
|
||||
? createStreamingErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage, error.code)
|
||||
: createErrorResult(HTTP_STATUS.RATE_LIMITED, failureMessage);
|
||||
return {
|
||||
...result,
|
||||
errorType: "account_semaphore_capacity",
|
||||
errorCode: error.code,
|
||||
};
|
||||
}
|
||||
const failureStatus =
|
||||
error.name === "AbortError"
|
||||
|
||||
@@ -156,6 +156,12 @@ function getGlmTokenQuotaName(
|
||||
return existingQuotas.session ? "weekly" : "session";
|
||||
}
|
||||
|
||||
function getGlmQuotaDisplayName(quotaName: string): string {
|
||||
if (quotaName === "session") return "5 Hours Quota";
|
||||
if (quotaName === "weekly") return "Weekly Quota";
|
||||
return quotaName;
|
||||
}
|
||||
|
||||
function getFieldValue(source: unknown, snakeKey: string, camelKey: string): unknown {
|
||||
const obj = toRecord(source);
|
||||
return obj[snakeKey] ?? obj[camelKey] ?? null;
|
||||
@@ -635,6 +641,13 @@ async function getGlmUsage(apiKey: string, providerSpecificData?: Record<string,
|
||||
remaining,
|
||||
remainingPercentage: remaining,
|
||||
resetAt,
|
||||
displayName: getGlmQuotaDisplayName(quotaName),
|
||||
details: Array.isArray(src.models)
|
||||
? src.models.map((m: any) => ({
|
||||
name: String(m.model || ""),
|
||||
used: toNumber(m.percentage, 0),
|
||||
}))
|
||||
: [],
|
||||
unlimited: false,
|
||||
};
|
||||
continue;
|
||||
@@ -876,6 +889,7 @@ export async function getUsageForProvider(connection, options: { forceRefresh?:
|
||||
return await getQoderUsage(accessToken);
|
||||
case "glm":
|
||||
case "glm-cn":
|
||||
case "zai":
|
||||
case "glmt":
|
||||
return await getGlmUsage(apiKey, {
|
||||
...(providerSpecificData || {}),
|
||||
|
||||
@@ -26,6 +26,11 @@ const QUOTA_LABEL_MAP: Record<string, string> = {
|
||||
"search-prime": "Web Search",
|
||||
"web-reader": "Web Reader",
|
||||
zread: "Zread",
|
||||
"5 Hours Quota": "5 Hours",
|
||||
"Weekly Quota": "Weekly",
|
||||
"Monthly Tools": "Monthly Tools",
|
||||
tokens: "Tokens",
|
||||
time_limit: "Time Limit",
|
||||
};
|
||||
|
||||
const GLM_QUOTA_ORDER: Record<string, number> = {
|
||||
|
||||
@@ -555,6 +555,7 @@ export async function GET(request: Request) {
|
||||
COALESCE(NULLIF(service_tier, ''), 'standard') as serviceTier,
|
||||
LOWER(provider) as provider,
|
||||
LOWER(model) as model,
|
||||
COALESCE(NULLIF(service_tier, ''), 'standard') as serviceTier,
|
||||
COUNT(*) as requests,
|
||||
COALESCE(SUM(tokens_input), 0) as promptTokens,
|
||||
COALESCE(SUM(tokens_output), 0) as completionTokens,
|
||||
@@ -563,6 +564,7 @@ export async function GET(request: Request) {
|
||||
COALESCE(SUM(tokens_reasoning), 0) as reasoningTokens,
|
||||
COALESCE(SUM(tokens_input + tokens_output), 0) as totalTokens
|
||||
FROM usage_history
|
||||
|
||||
${whereClause}
|
||||
GROUP BY serviceTier, LOWER(provider), LOWER(model)
|
||||
`
|
||||
|
||||
@@ -22,7 +22,22 @@ function isOpenAiCompatiblePath(pathname: string): boolean {
|
||||
return OPENAI_COMPAT_PATHS.some((pattern) => pattern.test(pathname));
|
||||
}
|
||||
|
||||
function requestWantsStreaming(req: IncomingMessage): boolean {
|
||||
const accept = String(req.headers.accept || "").toLowerCase();
|
||||
if (accept.includes("text/event-stream")) return true;
|
||||
|
||||
const pathname = (req.url || "/").split("?")[0] || "/";
|
||||
return /^\/(?:v1\/)?(?:responses|chat\/completions)(?:\/|$)/.test(pathname);
|
||||
}
|
||||
|
||||
function getProxyTimeoutMs(req: IncomingMessage): number {
|
||||
if (!requestWantsStreaming(req)) return API_BRIDGE_TIMEOUTS.proxyTimeoutMs;
|
||||
|
||||
return Math.max(API_BRIDGE_TIMEOUTS.proxyTimeoutMs, API_BRIDGE_TIMEOUTS.serverRequestTimeoutMs);
|
||||
}
|
||||
|
||||
function proxyRequest(req: IncomingMessage, res: ServerResponse, dashboardPort: number): void {
|
||||
const proxyTimeoutMs = getProxyTimeoutMs(req);
|
||||
const targetReq = http.request(
|
||||
{
|
||||
hostname: "127.0.0.1",
|
||||
@@ -33,9 +48,14 @@ function proxyRequest(req: IncomingMessage, res: ServerResponse, dashboardPort:
|
||||
...req.headers,
|
||||
host: `127.0.0.1:${dashboardPort}`,
|
||||
},
|
||||
timeout: API_BRIDGE_TIMEOUTS.proxyTimeoutMs,
|
||||
timeout: proxyTimeoutMs,
|
||||
},
|
||||
(targetRes) => {
|
||||
const contentType = String(targetRes.headers["content-type"] || "").toLowerCase();
|
||||
if (contentType.includes("text/event-stream")) {
|
||||
targetReq.setTimeout(0);
|
||||
}
|
||||
|
||||
res.writeHead(targetRes.statusCode || 502, targetRes.headers);
|
||||
targetRes.pipe(res);
|
||||
}
|
||||
@@ -48,7 +68,7 @@ function proxyRequest(req: IncomingMessage, res: ServerResponse, dashboardPort:
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error: "api_bridge_timeout",
|
||||
detail: `Proxy request timed out after ${API_BRIDGE_TIMEOUTS.proxyTimeoutMs}ms`,
|
||||
detail: `Proxy request timed out after ${proxyTimeoutMs}ms`,
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
@@ -47,6 +47,7 @@ interface ProviderConnectionLike {
|
||||
const PROVIDER_LIMITS_APIKEY_PROVIDERS = new Set([
|
||||
"glm",
|
||||
"glm-cn",
|
||||
"zai",
|
||||
"glmt",
|
||||
"minimax",
|
||||
"minimax-cn",
|
||||
|
||||
@@ -1982,6 +1982,7 @@ export const USAGE_SUPPORTED_PROVIDERS = [
|
||||
"kimi-coding",
|
||||
"glm",
|
||||
"glm-cn",
|
||||
"zai",
|
||||
"glmt",
|
||||
"minimax",
|
||||
"minimax-cn",
|
||||
|
||||
@@ -11,7 +11,7 @@ export const DEFAULT_STREAM_IDLE_TIMEOUT_MS = 600_000;
|
||||
export const DEFAULT_STREAM_READINESS_TIMEOUT_MS = 30_000;
|
||||
export const DEFAULT_FETCH_CONNECT_TIMEOUT_MS = 30_000;
|
||||
export const DEFAULT_FETCH_KEEPALIVE_TIMEOUT_MS = 4_000;
|
||||
export const DEFAULT_API_BRIDGE_PROXY_TIMEOUT_MS = 30_000;
|
||||
export const DEFAULT_API_BRIDGE_PROXY_TIMEOUT_MS = 600_000;
|
||||
export const DEFAULT_API_BRIDGE_SERVER_REQUEST_TIMEOUT_MS = 300_000;
|
||||
export const DEFAULT_API_BRIDGE_SERVER_HEADERS_TIMEOUT_MS = 60_000;
|
||||
export const DEFAULT_API_BRIDGE_SERVER_KEEPALIVE_TIMEOUT_MS = 5_000;
|
||||
|
||||
@@ -874,6 +874,25 @@ async function handleSingleModelChat(
|
||||
return result.response;
|
||||
}
|
||||
|
||||
if (result.errorType === "account_semaphore_capacity") {
|
||||
// Local concurrency pressure is not an upstream quota failure. Prefer another
|
||||
// account when possible; pinned combo steps fall through to combo orchestration.
|
||||
if (hasForcedConnection) {
|
||||
return result.response;
|
||||
}
|
||||
|
||||
log.warn(
|
||||
"AUTH",
|
||||
`Account ${accountId}... at local concurrency cap, trying fallback account`
|
||||
);
|
||||
excludedConnectionIds.add(credentials.connectionId);
|
||||
lastError = result.error;
|
||||
lastStatus = result.status;
|
||||
requestRetryLastError = result.error;
|
||||
requestRetryLastStatus = result.status;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Emergency fallback for budget exhaustion (402 / billing / quota keywords):
|
||||
// reroute to a free model (default provider/model: nvidia + openai/gpt-oss-120b) exactly once.
|
||||
if (!runtimeOptions.emergencyFallbackTried) {
|
||||
|
||||
@@ -94,3 +94,10 @@ test("API bridge timeouts align request timeout with long proxy timeout by defau
|
||||
serverSocketTimeoutMs: 0,
|
||||
});
|
||||
});
|
||||
|
||||
test("API bridge proxy timeout defaults to the long upstream request window", () => {
|
||||
const config = runtimeTimeouts.getApiBridgeTimeoutConfig({});
|
||||
|
||||
assert.equal(config.proxyTimeoutMs, 600000);
|
||||
assert.equal(config.serverRequestTimeoutMs, 600000);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user