diff --git a/open-sse/config/providerRegistry.ts b/open-sse/config/providerRegistry.ts index a8cfac03b0..efb4174f06 100644 --- a/open-sse/config/providerRegistry.ts +++ b/open-sse/config/providerRegistry.ts @@ -480,6 +480,7 @@ export const REGISTRY: Record = { tokenUrl: "https://auth.openai.com/oauth/token", }, models: [ + { id: "codex-auto-review", name: "Codex Auto Review", targetFormat: "openai-responses" }, { id: "gpt-5.5", name: "GPT 5.5", targetFormat: "openai-responses" }, { id: "gpt-5.4", name: "GPT 5.4", targetFormat: "openai-responses" }, { id: "gpt-5.4-mini", name: "GPT 5.4 Mini", targetFormat: "openai-responses" }, diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index e7fbeba6fd..e52cd50c05 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -24,7 +24,7 @@ import { parseUpstreamError, formatProviderError, } from "../utils/error.ts"; -import { HTTP_STATUS, PROVIDER_MAX_TOKENS } from "../config/constants.ts"; +import { COOLDOWN_MS, HTTP_STATUS, PROVIDER_MAX_TOKENS } from "../config/constants.ts"; import { classifyProviderError, PROVIDER_ERROR_TYPES, @@ -96,6 +96,7 @@ import { buildAccountSemaphoreKey, markBlocked as markAccountSemaphoreBlocked, } from "../services/accountSemaphore.ts"; +import { lockModelIfPerModelQuota } from "../services/accountFallback.ts"; import { generateSignature, getCachedResponse, @@ -490,6 +491,45 @@ function isSemaphoreTimeoutError(error: unknown): error is Error & { code: strin ); } +function wrapReadableStreamWithFinalize( + readable: ReadableStream, + finalize: () => void +): ReadableStream { + const reader = readable.getReader(); + let finalized = false; + + const runFinalize = () => { + if (finalized) return; + finalized = true; + finalize(); + }; + + return new ReadableStream({ + async pull(controller) { + try { + const { done, value } = await reader.read(); + if (done) { + runFinalize(); + controller.close(); + return; + } + controller.enqueue(value); + } catch (error) { + runFinalize(); + controller.error(error); + } + }, + + async cancel(reason) { + try { + await reader.cancel(reason); + } finally { + runFinalize(); + } + }, + }); +} + function resolveAccountSemaphoreAccountKey( connectionId: string | null | undefined, credentials: Record | null | undefined @@ -1968,82 +2008,90 @@ export async function handleChatCore({ }) : () => {}; - const rawResult = await withRateLimit(provider, connectionId, modelToCall, async () => { - let attempts = 0; - const maxAttempts = provider === "qwen" ? 3 : 1; + try { + const rawResult = await withRateLimit(provider, connectionId, modelToCall, async () => { + let attempts = 0; + const maxAttempts = provider === "qwen" ? 3 : 1; - while (attempts < maxAttempts) { - const res = await executor.execute({ - model: modelToCall, - body: bodyToSend, - stream: upstreamStream, - credentials: executionCredentials, - signal: streamController.signal, - log, - extendedContext, - upstreamExtraHeaders: buildUpstreamHeadersForExecute(modelToCall), - clientHeaders: buildExecutorClientHeaders(clientRawRequest?.headers, userAgent), - onCredentialsRefreshed, - }); + while (attempts < maxAttempts) { + const res = await executor.execute({ + model: modelToCall, + body: bodyToSend, + stream: upstreamStream, + credentials: executionCredentials, + signal: streamController.signal, + log, + extendedContext, + upstreamExtraHeaders: buildUpstreamHeadersForExecute(modelToCall), + clientHeaders: buildExecutorClientHeaders(clientRawRequest?.headers, userAgent), + onCredentialsRefreshed, + }); - // Qwen 429 strict quota backoff (wait 1.5s, 3s and retry) - if (provider === "qwen" && res.response.status === 429 && attempts < maxAttempts - 1) { - const bodyPeek = await res.response - .clone() - .text() - .catch(() => ""); - if (bodyPeek.toLowerCase().includes("exceeded your current quota")) { - const delay = 1500 * (attempts + 1); - log?.warn?.("QWEN_RETRY", `Quota 429 hit. Retrying in ${delay}ms...`); - await new Promise((r) => setTimeout(r, delay)); - attempts++; - continue; + // Qwen 429 strict quota backoff (wait 1.5s, 3s and retry) + if (provider === "qwen" && res.response.status === 429 && attempts < maxAttempts - 1) { + const bodyPeek = await res.response + .clone() + .text() + .catch(() => ""); + if (bodyPeek.toLowerCase().includes("exceeded your current quota")) { + const delay = 1500 * (attempts + 1); + log?.warn?.("QWEN_RETRY", `Quota 429 hit. Retrying in ${delay}ms...`); + await new Promise((r) => setTimeout(r, delay)); + attempts++; + continue; + } } - } - // For streaming: wrap response body to release semaphore only when stream is fully consumed - if (stream) { - const originalBody = res.response.body; - const wrappedBody = originalBody - ? originalBody.pipeThrough( - new TransformStream({ - flush: () => { - acquireAccountSemaphoreRelease(); - }, - }) - ) - : null; - return { - ...res, - response: new Response(wrappedBody, { - status: res.response.status, - statusText: res.response.statusText, - headers: res.response.headers, - }), - }; - } + // For streaming: release the semaphore when the client drains or cancels the stream. + if (stream) { + const originalBody = res.response.body; + if (!originalBody) { + acquireAccountSemaphoreRelease(); + return res; + } - return res; + return { + ...res, + response: new Response( + wrapReadableStreamWithFinalize(originalBody, acquireAccountSemaphoreRelease), + { + status: res.response.status, + statusText: res.response.statusText, + headers: res.response.headers, + } + ), + }; + } + + return res; + } + }); + + if (stream) { + return rawResult; } - }); - // Non-stream: release semaphore immediately after reading full response body - const status = rawResult.response.status; - const statusText = rawResult.response.statusText; - const headers = Array.from(rawResult.response.headers.entries()) as [string, string][]; - const payload = await rawResult.response.text(); - acquireAccountSemaphoreRelease(); + // Non-stream: release semaphore immediately after reading full response body. + const status = rawResult.response.status; + const statusText = rawResult.response.statusText; + const headers = Array.from(rawResult.response.headers.entries()) as [string, string][]; + const payload = await rawResult.response.text(); + acquireAccountSemaphoreRelease(); - return { - ...rawResult, - response: new Response(payload, { status, statusText, headers }), - _dedupSnapshot: { - status, - statusText, - headers, - payload, - }, - }; + return { + ...rawResult, + response: new Response(payload, { status, statusText, headers }), + _dedupSnapshot: { + status, + statusText, + headers, + payload, + }, + }; + } catch (error) { + acquireAccountSemaphoreRelease(); + throw error; + } }; if (allowDedup && dedupEnabled && dedupHash) { diff --git a/open-sse/services/model.ts b/open-sse/services/model.ts index 5c5e7bb470..6719d1431c 100644 --- a/open-sse/services/model.ts +++ b/open-sse/services/model.ts @@ -73,6 +73,7 @@ for (const [aliasOrId, models] of Object.entries(PROVIDER_MODELS)) { } } const KNOWN_MODEL_IDS = new Set(MODEL_TO_PROVIDERS.keys()); +const CODEX_PREFERRED_UNPREFIXED_MODELS = new Set(["codex-auto-review", "gpt-5.5"]); /** * Resolve provider alias to provider ID @@ -286,6 +287,14 @@ function resolveModelByProviderInference(modelId, extendedContext) { } const nonOpenAIProviders = providers.filter((p) => p !== "openai"); + if (providers.includes("codex") && CODEX_PREFERRED_UNPREFIXED_MODELS.has(modelId)) { + return { + provider: "codex", + model: modelId, + extendedContext, + }; + } + if (nonOpenAIProviders.length === 1) { const provider = nonOpenAIProviders[0]; const canonicalModel = resolveProviderModelAlias(provider, modelId); diff --git a/public/providers/aws-polly.png b/public/providers/aws-polly.png new file mode 100644 index 0000000000..7bc66fa485 Binary files /dev/null and b/public/providers/aws-polly.png differ diff --git a/public/providers/blackbox-web.png b/public/providers/blackbox-web.png new file mode 100644 index 0000000000..c11589826b Binary files /dev/null and b/public/providers/blackbox-web.png differ diff --git a/public/providers/cliproxyapi.png b/public/providers/cliproxyapi.png new file mode 100644 index 0000000000..4aaacd3118 Binary files /dev/null and b/public/providers/cliproxyapi.png differ diff --git a/public/providers/databricks.png b/public/providers/databricks.png new file mode 100644 index 0000000000..2ae8754538 Binary files /dev/null and b/public/providers/databricks.png differ diff --git a/public/providers/empower.png b/public/providers/empower.png new file mode 100644 index 0000000000..1540ddb37d Binary files /dev/null and b/public/providers/empower.png differ diff --git a/public/providers/gigachat.png b/public/providers/gigachat.png new file mode 100644 index 0000000000..f175bf6db5 Binary files /dev/null and b/public/providers/gigachat.png differ diff --git a/public/providers/gitlab-duo.png b/public/providers/gitlab-duo.png new file mode 100644 index 0000000000..8dfcb51780 Binary files /dev/null and b/public/providers/gitlab-duo.png differ diff --git a/public/providers/gitlab.png b/public/providers/gitlab.png new file mode 100644 index 0000000000..8dfcb51780 Binary files /dev/null and b/public/providers/gitlab.png differ diff --git a/public/providers/heroku.png b/public/providers/heroku.png new file mode 100644 index 0000000000..b96cb3f6a7 Binary files /dev/null and b/public/providers/heroku.png differ diff --git a/public/providers/linkup-search.png b/public/providers/linkup-search.png new file mode 100644 index 0000000000..43452df575 Binary files /dev/null and b/public/providers/linkup-search.png differ diff --git a/public/providers/llamagate.png b/public/providers/llamagate.png new file mode 100644 index 0000000000..2fe71a5a22 Binary files /dev/null and b/public/providers/llamagate.png differ diff --git a/public/providers/maritalk.png b/public/providers/maritalk.png new file mode 100644 index 0000000000..c80425a0ef Binary files /dev/null and b/public/providers/maritalk.png differ diff --git a/public/providers/modal.png b/public/providers/modal.png new file mode 100644 index 0000000000..05f51a49ae Binary files /dev/null and b/public/providers/modal.png differ diff --git a/public/providers/nanogpt.png b/public/providers/nanogpt.png new file mode 100644 index 0000000000..c4558c4670 Binary files /dev/null and b/public/providers/nanogpt.png differ diff --git a/public/providers/nscale.png b/public/providers/nscale.png new file mode 100644 index 0000000000..cb6cdf41ea Binary files /dev/null and b/public/providers/nscale.png differ diff --git a/public/providers/oci.png b/public/providers/oci.png new file mode 100644 index 0000000000..96672d7bd4 Binary files /dev/null and b/public/providers/oci.png differ diff --git a/public/providers/ovhcloud.png b/public/providers/ovhcloud.png new file mode 100644 index 0000000000..a88d1b448d Binary files /dev/null and b/public/providers/ovhcloud.png differ diff --git a/public/providers/piapi.png b/public/providers/piapi.png new file mode 100644 index 0000000000..ac4b72706b Binary files /dev/null and b/public/providers/piapi.png differ diff --git a/public/providers/poe.png b/public/providers/poe.png new file mode 100644 index 0000000000..b2a77049e5 Binary files /dev/null and b/public/providers/poe.png differ diff --git a/public/providers/predibase.png b/public/providers/predibase.png new file mode 100644 index 0000000000..55ecaed763 Binary files /dev/null and b/public/providers/predibase.png differ diff --git a/public/providers/qoder.png b/public/providers/qoder.png index 1bddeae625..fd03f28dd2 100644 Binary files a/public/providers/qoder.png and b/public/providers/qoder.png differ diff --git a/public/providers/recraft.png b/public/providers/recraft.png new file mode 100644 index 0000000000..dff533eb8c Binary files /dev/null and b/public/providers/recraft.png differ diff --git a/public/providers/reka.png b/public/providers/reka.png new file mode 100644 index 0000000000..000162576a Binary files /dev/null and b/public/providers/reka.png differ diff --git a/public/providers/runwayml.png b/public/providers/runwayml.png new file mode 100644 index 0000000000..bbb54dd99c Binary files /dev/null and b/public/providers/runwayml.png differ diff --git a/public/providers/triton.png b/public/providers/triton.png new file mode 100644 index 0000000000..9670b2ca22 Binary files /dev/null and b/public/providers/triton.png differ diff --git a/public/providers/venice.png b/public/providers/venice.png new file mode 100644 index 0000000000..a72e8ba1df Binary files /dev/null and b/public/providers/venice.png differ diff --git a/public/providers/voyage-ai.png b/public/providers/voyage-ai.png new file mode 100644 index 0000000000..72c41963e5 Binary files /dev/null and b/public/providers/voyage-ai.png differ diff --git a/public/providers/wandb.png b/public/providers/wandb.png new file mode 100644 index 0000000000..3594e78071 Binary files /dev/null and b/public/providers/wandb.png differ diff --git a/public/providers/youcom-search.png b/public/providers/youcom-search.png new file mode 100644 index 0000000000..6220753cc0 Binary files /dev/null and b/public/providers/youcom-search.png differ diff --git a/scripts/i18n_autotranslate.py b/scripts/i18n_autotranslate.py index 26f90a35d4..9321ff57f6 100755 --- a/scripts/i18n_autotranslate.py +++ b/scripts/i18n_autotranslate.py @@ -6,7 +6,7 @@ API (like OmniRoute itself) to translate any English paragraphs into the target language. Usage: - python3 scripts/i18n_autotranslate.py --api-url http://192.168.0.15:20128/v1 --api-key sk-14e76c286e84ff2d-agn73z-5a1fd283 --model cx/gpt-5.4 + python3 scripts/i18n_autotranslate.py --api-url http://localhost:20128/v1 --api-key sk-your-omniroute-key --model cx/gpt-5.4 """ import os diff --git a/src/shared/components/ProviderIcon.tsx b/src/shared/components/ProviderIcon.tsx index 0ccd797532..3ed7a4b0b7 100644 --- a/src/shared/components/ProviderIcon.tsx +++ b/src/shared/components/ProviderIcon.tsx @@ -79,6 +79,34 @@ const LOBEHUB_PROVIDER_MAP: Record = { "github-copilot": "githubcopilot", github: "github", mistralai: "mistral", + "azure-ai": "azureai", + baseten: "baseten", + "black-forest-labs": "bfl", + deepinfra: "deepinfra", + "fal-ai": "fal", + "featherless-ai": "featherless", + friendliai: "friendli", + "glm-cn": "zhipu", + "inference-net": "inference", + "jina-ai": "jina", + "lambda-ai": "lambda", + "lm-studio": "lmstudio", + "meta-llama": "meta", + "muse-spark-web": "meta", + "nous-research": "nousresearch", + novita: "novita", + sambanova: "sambanova", + "searchapi-search": "searchapi", + snowflake: "snowflake", + upstage: "upstage", + "v0-vercel": "v0", + "vercel-ai-gateway": "vercelaigateway", + "vertex-partner": "vertexai", + vllm: "vllm", + volcengine: "volcengine", + watsonx: "ibm", + "xiaomi-mimo": "xiaomimimo", + xinference: "xinference", }; interface ProviderIconProps { @@ -186,6 +214,35 @@ const KNOWN_PNGS = new Set([ "together", "xai", "zeroclaw", + "aws-polly", + "blackbox-web", + "cliproxyapi", + "databricks", + "empower", + "gigachat", + "gitlab-duo", + "gitlab", + "heroku", + "linkup-search", + "llamagate", + "maritalk", + "modal", + "nanogpt", + "nscale", + "oci", + "ovhcloud", + "piapi", + "poe", + "predibase", + "qoder", + "recraft", + "reka", + "runwayml", + "triton", + "venice", + "voyage-ai", + "wandb", + "youcom-search", ]); const KNOWN_SVGS = new Set([ "apikey", diff --git a/tests/unit/chatcore-translation-paths.test.ts b/tests/unit/chatcore-translation-paths.test.ts index 2ccbb96e7f..326de77cd0 100644 --- a/tests/unit/chatcore-translation-paths.test.ts +++ b/tests/unit/chatcore-translation-paths.test.ts @@ -18,6 +18,13 @@ const { clearCache, getCachedResponse, generateSignature } = await import("../../src/lib/semanticCache.ts"); const { clearIdempotency } = await import("../../src/lib/idempotencyLayer.ts"); const { clearInflight } = await import("../../open-sse/services/requestDedup.ts"); +const { + buildAccountSemaphoreKey, + getStats: getAccountSemaphoreStats, + resetAll: resetAccountSemaphores, +} = await import("../../open-sse/services/accountSemaphore.ts"); +const { clearModelLock, isModelLocked } = + await import("../../open-sse/services/accountFallback.ts"); const { saveModelsDevCapabilities, clearModelsDevCapabilities } = await import("../../src/lib/modelsDevSync.ts"); const { @@ -346,12 +353,14 @@ async function invokeChatCore({ test.afterEach(async () => { globalThis.fetch = originalFetch; + resetAccountSemaphores(); await waitForAsyncSideEffects(); await resetStorage(); }); test.after(async () => { globalThis.fetch = originalFetch; + resetAccountSemaphores(); await waitForAsyncSideEffects(); await resetStorage(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); @@ -2007,6 +2016,146 @@ test("chatCore maps upstream aborts to request-aborted errors", async () => { assert.equal(result.error, "Request aborted"); }); +test("chatCore returns streaming responses without waiting for upstream completion", async () => { + const encoder = new TextEncoder(); + let closeUpstream: (() => void) | null = null; + + const invocation = invokeChatCore({ + provider: "openai", + model: "gpt-4o-mini", + accept: "text/event-stream", + body: { + model: "gpt-4o-mini", + stream: true, + messages: [{ role: "user", content: "do not buffer streaming" }], + }, + responseFactory() { + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + encoder.encode( + `data: ${JSON.stringify({ + id: "chatcmpl-stream", + object: "chat.completion.chunk", + choices: [ + { + index: 0, + delta: { role: "assistant", content: "streamed-without-buffering" }, + }, + ], + })}\n\n` + ) + ); + closeUpstream = () => { + controller.enqueue(encoder.encode("data: [DONE]\n\n")); + controller.close(); + }; + }, + }), + { + status: 200, + headers: { "Content-Type": "text/event-stream" }, + } + ); + }, + }); + + const raceResult = await Promise.race([ + invocation.then(() => "returned"), + new Promise((resolve) => setTimeout(() => resolve("blocked"), 1000)), + ]); + + if (raceResult !== "returned") { + closeUpstream?.(); + } + const { result } = await invocation; + + assert.equal(raceResult, "returned"); + closeUpstream?.(); + + const streamText = await result.response.text(); + assert.equal(result.success, true); + assert.match(streamText, /streamed-without-buffering/); +}); + +test("chatCore releases account semaphore slots when upstream execution throws", async () => { + const connectionId = "sem-exception"; + const semaphoreKey = buildAccountSemaphoreKey({ + provider: "openai", + accountKey: connectionId, + }); + + const { result } = await invokeChatCore({ + provider: "openai", + model: "gpt-4o-mini", + connectionId, + credentials: { + apiKey: "sk-test", + maxConcurrent: 1, + providerSpecificData: {}, + }, + body: { + model: "gpt-4o-mini", + stream: false, + messages: [{ role: "user", content: "executor throws" }], + }, + responseFactory() { + throw new Error("simulated upstream network failure"); + }, + }); + + await waitForAsyncSideEffects(); + + assert.equal(result.success, false); + assert.equal(result.status, 502); + assert.equal(getAccountSemaphoreStats()[semaphoreKey], undefined); +}); + +test("chatCore locks per-model quota failures without dropping quota helper references", async () => { + const model = "gemini-1.5-pro"; + const connection = await providersDb.createProviderConnection({ + provider: "gemini", + authType: "apikey", + name: "gemini-quota-lock", + apiKey: "gemini-key", + isActive: true, + providerSpecificData: {}, + }); + + try { + const { result } = await invokeChatCore({ + provider: "gemini", + model, + connectionId: connection.id, + credentials: { + apiKey: "gemini-key", + providerSpecificData: {}, + }, + body: { + model, + stream: false, + messages: [{ role: "user", content: "quota lock" }], + }, + responseFactory() { + return new Response( + JSON.stringify({ error: { message: "insufficient_quota: quota exhausted" } }), + { + status: 402, + headers: { "Content-Type": "application/json" }, + } + ); + }, + }); + + assert.equal(result.success, false); + assert.equal(result.status, 402); + assert.equal(isModelLocked("gemini", connection.id, model), true); + } finally { + clearModelLock("gemini", connection.id, model); + } +}); + // ── Streaming semantic cache tests ────────────────────────────────────────── test("chatCore caches streaming response and serves cache HIT on repeat", async () => { diff --git a/tests/unit/pack-artifact-policy.test.ts b/tests/unit/pack-artifact-policy.test.ts index a8e5ef7655..64811a50a9 100644 --- a/tests/unit/pack-artifact-policy.test.ts +++ b/tests/unit/pack-artifact-policy.test.ts @@ -56,6 +56,8 @@ test("findMissingArtifactPaths flags missing root runtime files in the tarball", ); assert.deepEqual(missingPaths, [ + "app/responses-ws-proxy.mjs", + "app/server-ws.mjs", "bin/mcp-server.mjs", "bin/nodeRuntimeSupport.mjs", "scripts/native-binary-compat.mjs",