From 2688f3cd74e0a9028af5face133cf6348c595b06 Mon Sep 17 00:00:00 2001 From: Vinayrnani Date: Sun, 7 Jun 2026 02:28:22 +0530 Subject: [PATCH] =?UTF-8?q?feat(provider):=20test-all=20endpoint,=20rate-l?= =?UTF-8?q?imit=20overrides,=20visibility=20f=E2=80=A6=20(#3267)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integrated into release/v3.8.13 — provider test-all endpoint, rate-limit overrides, model visibility --- .gitignore | 3 + CHANGELOG.md | 19 +- bin/reset-password.mjs | 0 open-sse/services/rateLimitManager.ts | 115 +++- scripts/build/bootstrap-env.mjs | 10 +- .../dashboard/providers/[id]/page.tsx | 635 ++++++++++++++---- src/app/api/models/test-all/route.ts | 196 ++++++ src/app/api/models/test/route.ts | 237 +------ src/app/api/providers/[id]/route.ts | 11 + src/app/api/providers/route.ts | 43 ++ src/app/api/v1/models/catalog.ts | 31 +- src/domain/types.ts | 6 + src/i18n/messages/en.json | 161 +++-- src/lib/api/modelTestRunner.ts | 335 +++++++++ src/lib/db/core.ts | 5 + src/lib/db/models.ts | 38 ++ src/lib/db/providers.ts | 91 ++- src/lib/localDb.ts | 1 + src/shared/validation/schemas.ts | 14 + tests/unit/model-test-runner.test.ts | 88 +++ ...ovider-rate-limit-overrides-schema.test.ts | 73 ++ 21 files changed, 1710 insertions(+), 402 deletions(-) mode change 100644 => 100755 bin/reset-password.mjs create mode 100644 src/app/api/models/test-all/route.ts create mode 100644 src/lib/api/modelTestRunner.ts create mode 100644 tests/unit/model-test-runner.test.ts create mode 100644 tests/unit/provider-rate-limit-overrides-schema.test.ts diff --git a/.gitignore b/.gitignore index c5e212e3b9..746f05b94f 100644 --- a/.gitignore +++ b/.gitignore @@ -63,6 +63,9 @@ yarn-error.log* # env files (can opt-in for committing if needed) .env* !.env.example +# Provider API keys (never commit) +*.api-key +.nvidia-api-key # vercel .vercel diff --git a/CHANGELOG.md b/CHANGELOG.md index 9eae7f4e44..6512529ebf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,24 @@ ## [Unreleased] ---- +### ✨ New Features +- **api:** `POST /api/models/test-all` — parallel model testing with chunk size 3 and timeout-skip +- **api:** shared `runSingleModelTest` runner extracted from single-test endpoint +- **providers:** per-connection rate-limit overrides via PATCH `/api/providers/:id` +- **db:** `rate_limit_overrides_json` column + Zod schema + serialization helpers +- **dashboard:** model visibility filter toolbar (All / Visible / Hidden) +- **api:** `/v1/models` catalog excludes user-hidden models +- **providers:** auto-fetch models on every connection add (universalized) +- **dashboard:** test buttons for passthrough models (OpenRouter) + +### 🔧 Bug Fixes +- **dashboard:** missing handlers — handleDelete, handleSetAlias, handleDeleteAlias, handleToggleSelectAll +- **auth:** fetchData crash in auth import modals +- **debug:** stray console.log removed; error handlers use console.error +- **dashboard:** React key collisions from duplicate model IDs +- **build:** empty .env values no longer override real config +- **i18n:** missing keys added +- **dashboard:** "Hide all" button restored ## [3.8.13] — 2026-06-06 diff --git a/bin/reset-password.mjs b/bin/reset-password.mjs old mode 100644 new mode 100755 diff --git a/open-sse/services/rateLimitManager.ts b/open-sse/services/rateLimitManager.ts index c5a4a181ae..f967b3fd97 100644 --- a/open-sse/services/rateLimitManager.ts +++ b/open-sse/services/rateLimitManager.ts @@ -73,6 +73,10 @@ const limiters = new Map(); // Store connections that have rate limit protection enabled const enabledConnections = new Set(); +// Store per-connection rate limit overrides (RPM, TPM, TPD, minTime, maxConcurrent) +// Populated from provider_connections.rateLimitOverrides on startup and refresh. +const connectionRateLimitOverrides = new Map>(); + // Store learned limits for persistence (debounced) const learnedLimits: Record = {}; const MAX_LEARNED_LIMITS = 200; @@ -115,12 +119,40 @@ function isAutoEnableActive(settings: RequestQueueSettings): boolean { return settings.autoEnableApiKeyProviders; } +// Sentinels for "no rate limit" / effectively infinite capacity. The reservoir +// value uses Number.MAX_SAFE_INTEGER so the bucket can never realistically be +// exhausted; maxConcurrent uses a smaller-but-still-vast ceiling since +// Bottleneck tracks concurrent jobs in memory and an unbounded number would +// risk internal counter overflow under sustained pressure. +const EFFECTIVELY_INFINITE = Number.MAX_SAFE_INTEGER; +const EFFECTIVELY_INFINITE_CONCURRENCY = 1000; + +// Resolve an RPM override. 0 or missing means "infinite" (no rate cap). +function resolveRpm(override: number | undefined | null): number { + return typeof override === "number" && override > 0 ? override : EFFECTIVELY_INFINITE; +} + +// Resolve a minTime override. 0 or missing means "no minimum gap". +function resolveMinTime(override: number | undefined | null): number { + return typeof override === "number" && override > 0 ? override : 0; +} + +// Resolve a maxConcurrent override. 0 or missing means "effectively infinite". +function resolveMaxConcurrent(override: number | undefined | null): number { + return typeof override === "number" && override > 0 + ? override + : EFFECTIVELY_INFINITE_CONCURRENCY; +} + function buildLimiterDefaults() { + // 0 or missing values mean "infinite" / no rate limit applies. This treats + // the global request-queue settings the same way per-connection overrides + // are interpreted (see resolveRpm / resolveMinTime / resolveMaxConcurrent). return { - maxConcurrent: currentRequestQueueSettings.concurrentRequests, - minTime: currentRequestQueueSettings.minTimeBetweenRequestsMs, - reservoir: currentRequestQueueSettings.requestsPerMinute, - reservoirRefreshAmount: currentRequestQueueSettings.requestsPerMinute, + maxConcurrent: resolveMaxConcurrent(currentRequestQueueSettings.concurrentRequests), + minTime: resolveMinTime(currentRequestQueueSettings.minTimeBetweenRequestsMs), + reservoir: resolveRpm(currentRequestQueueSettings.requestsPerMinute), + reservoirRefreshAmount: resolveRpm(currentRequestQueueSettings.requestsPerMinute), reservoirRefreshInterval: 60 * 1000, }; } @@ -316,6 +348,15 @@ export async function initializeRateLimits() { ); updateAllLimiterSettings(); + // Load per-connection rate limit overrides + connectionRateLimitOverrides.clear(); + for (const conn of connections as Array>) { + const overrides = conn.rateLimitOverrides; + if (overrides && typeof overrides === "object" && !Array.isArray(overrides)) { + connectionRateLimitOverrides.set(String(conn.id), overrides as Record); + } + } + if (explicitCount > 0 || autoCount > 0) { logRateLimit( `🛡️ [RATE-LIMIT] Loaded ${explicitCount} explicit + ${autoCount} auto-enabled protection(s)` @@ -342,7 +383,7 @@ export async function applyRequestQueueSettings(nextSettings: RequestQueueSettin } /** - * Enable rate limit protection for a connection + * Get or create a limiter for a given provider+connection combination */ export function enableRateLimitProtection(connectionId) { enabledConnections.add(connectionId); @@ -378,6 +419,33 @@ export function isRateLimitEnabled(connectionId) { return enabledConnections.has(connectionId); } +/** + * Refresh per-connection rate limit overrides. + * + * Called after a PATCH update to `rateLimitOverrides` on a provider connection. + * Updates the in-memory map and evicts existing Bottleneck limiters for the + * connection so the next request gets a fresh limiter with the new settings. + * + * @param {string} connectionId + * @param {Record | null} overrides - New overrides (null/undefined clears) + */ +export function refreshConnectionRateLimits(connectionId, overrides) { + if (overrides === null || overrides === undefined) { + connectionRateLimitOverrides.delete(connectionId); + } else { + connectionRateLimitOverrides.set(connectionId, overrides); + } + // Evict limiters referencing this connection so they get recreated on next use + for (const [key, limiter] of Array.from(limiters)) { + if (key.includes(connectionId)) { + limiters.delete(key); + lastDispatchAt.delete(key); + limiterLastUsed.delete(key); + trackAsyncOperation(limiter.disconnect()); + } + } +} + /** * Get or create a limiter for a given provider+connection combination */ @@ -397,19 +465,32 @@ function getLimiter(provider, connectionId, model = null) { const key = getLimiterKey(provider, connectionId, model); if (!limiters.has(key)) { - const limiter = new Bottleneck({ - ...buildLimiterDefaults(), - id: key, - }); - - // Log when jobs are queued - limiter.on("queued", () => { - const counts = limiter.counts(); - if (counts.QUEUED > 0) { - logRateLimit( - `⏳ [RATE-LIMIT] ${key} — ${counts.QUEUED} request(s) queued, ${counts.RUNNING} running` - ); + const defaults = buildLimiterDefaults(); + const overrides = connectionRateLimitOverrides.get(connectionId); + if (overrides) { + // 0 (or missing) means "no override — fall through to buildLimiterDefaults()". + // Without this guard, an rpm of 0 sets reservoir=0, which Bottleneck treats + // as "depleted" and blocks ALL requests indefinitely. Treating 0 as "use + // default" lets users effectively disable per-connection limits without + // globally raising the system default. + if (typeof overrides.maxConcurrent === "number" && overrides.maxConcurrent > 0) { + defaults.maxConcurrent = overrides.maxConcurrent; } + if (typeof overrides.minTime === "number" && overrides.minTime > 0) { + defaults.minTime = overrides.minTime; + } + if (typeof overrides.rpm === "number" && overrides.rpm > 0) { + defaults.reservoir = overrides.rpm; + defaults.reservoirRefreshAmount = overrides.rpm; + defaults.reservoirRefreshInterval = 60 * 1000; + } + // TODO: TPM/TPD integration — requires a token-bucket vs request-bucket + // separation (Bottleneck's reservoir is request-count, not token-count). + // When added, treat 0/missing the same way: fall through to system default. + } + const limiter = new Bottleneck({ + ...defaults, + id: key, }); // Heartbeat: timestamp every dispatch so the watchdog can tell a healthy // queue (just dispatched a job) from a wedged one (queue has work but diff --git a/scripts/build/bootstrap-env.mjs b/scripts/build/bootstrap-env.mjs index 39dd8b4159..54d4f8d3c6 100644 --- a/scripts/build/bootstrap-env.mjs +++ b/scripts/build/bootstrap-env.mjs @@ -169,7 +169,15 @@ export function bootstrapEnv({ dataDirOverride, quiet = false } = {}) { // ── Layer 2: Load the same preferred .env that the CLI wrapper uses ─────── // This keeps run-next / run-standalone consistent with `bin/omniroute.mjs`. - const merged = { ...persisted, ...preferredEnv, ...process.env }; + // + // We strip empty values from preferredEnv so an empty placeholder + // (e.g. `STORAGE_ENCRYPTION_KEY=` in the project .env template) does not + // override the real value persisted in server.env. Only the .env entries + // that the operator actually set should win. + const preferredEnvFiltered = Object.fromEntries( + Object.entries(preferredEnv).filter(([, v]) => typeof v === "string" && v.length > 0) + ); + const merged = { ...persisted, ...preferredEnvFiltered, ...process.env }; // ── Auto-generate required secrets ──────────────────────────────────────── let needsPersist = false; diff --git a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx index fdfd6b4ee1..9dfff75e0b 100644 --- a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx +++ b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx @@ -634,6 +634,8 @@ interface PassthroughModelsSectionProps { onTestModel?: (modelId: string, fullModel: string) => Promise; modelTestStatus?: Record; testingModelId?: string | null; + providerId: string; + connectionId: string; } interface CustomModelsSectionProps { @@ -685,6 +687,11 @@ interface CompatibleModelsSectionProps { onTestModel?: (modelId: string, fullModel: string) => Promise; modelTestStatus?: Record; testingModelId?: string | null; + onTestAll?: (targets: Array<{ modelId: string; fullModel: string }>) => Promise; + testingAll?: boolean; + testProgress?: { done: number; total: number } | null; + autoHideFailed?: boolean; + onAutoHideFailedChange?: (v: boolean) => void; } interface CooldownTimerProps { @@ -842,6 +849,7 @@ interface EditConnectionModalConnection { email?: string; priority?: number; maxConcurrent?: number | null; + rateLimitOverrides?: Record | null; authType?: string; provider?: string; apiKey?: string; @@ -1433,6 +1441,10 @@ export default function ProviderDetailPage() { const [togglingModelId, setTogglingModelId] = useState(null); const [testingModelId, setTestingModelId] = useState(null); const [modelTestStatus, setModelTestStatus] = useState>({}); + const [testingAll, setTestingAll] = useState(false); + const [testProgress, setTestProgress] = useState<{ done: number; total: number } | null>(null); + const [autoHideFailed, setAutoHideFailed] = useState(true); + const [visibilityFilter, setVisibilityFilter] = useState<"all" | "visible" | "hidden">("all"); const [bulkVisibilityAction, setBulkVisibilityAction] = useState<"select" | "deselect" | null>( null ); @@ -1519,13 +1531,10 @@ export default function ProviderDetailPage() { // Prefer synced API-discovered models when available, then merge built-ins // and user-managed custom models without duplicating IDs. const models = useMemo(() => { - if (providerId === "gemini") { - return syncedAvailableModels.map((model: any) => ({ - ...model, - source: "imported", - })); - } - + // Universal: merge built-in registry models with API-synced models and + // user-managed custom models for ALL providers (was previously Gemini-only). + // Synced models keep their full property spread so provider-specific fields + // (e.g. Gemini's `supportedGenerationMethods`) survive into the table. const builtInModels = registryModels.map((model) => ({ ...model, source: "system", @@ -1535,6 +1544,7 @@ export default function ProviderDetailPage() { const syncedExtras = syncedAvailableModels .filter((model: any) => model?.id && !registryIds.has(model.id)) .map((model: any) => ({ + ...model, id: model.id, name: model.name || model.id, source: "imported", @@ -1547,7 +1557,12 @@ export default function ProviderDetailPage() { name: cm.name || cm.id, source: normalizeModelCatalogSource(cm.source) === "imported" ? "imported" : "custom", })); - return [...builtInModels, ...syncedExtras, ...customExtras]; + const allModels = [...builtInModels, ...syncedExtras, ...customExtras]; + const deduped = new Map(); + for (const m of allModels) { + if (m.id && !deduped.has(m.id)) deduped.set(m.id, m); + } + return Array.from(deduped.values()); }, [providerId, registryModels, syncedAvailableModels, modelMeta.customModels]); const providerAlias = getProviderAlias(providerId); const isManagedAvailableModelsProvider = isCompatible || providerId === "openrouter"; @@ -1615,6 +1630,56 @@ export default function ProviderDetailPage() { } }, []); + const handleSetAlias = useCallback( + async (modelId: string, alias: string, providerAlias?: string) => { + const qualifiedModel = providerAlias + ? modelId.includes("/") + ? `${providerAlias}/${modelId.split("/").slice(1).join("/")}` + : `${providerAlias}/${modelId}` + : modelId; + try { + const res = await fetch("/api/models/alias", { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model: qualifiedModel, alias }), + }); + if (res.ok) { + await fetchAliases(); + notify.success(t("setAliasSuccess", { alias })); + } else { + const data = await res.json().catch(() => ({})); + notify.error(data?.error?.message || "Failed to set alias"); + } + } catch (error) { + console.log("Error setting alias:", error); + notify.error("Network error setting alias"); + } + }, + [fetchAliases, t] + ); + + const handleDeleteAlias = useCallback( + async (alias: string) => { + try { + const res = await fetch( + `/api/models/alias?alias=${encodeURIComponent(alias)}`, + { method: "DELETE" } + ); + if (res.ok) { + await fetchAliases(); + notify.success(t("deleteAliasSuccess", { alias })); + } else { + const data = await res.json().catch(() => ({})); + notify.error(data?.error?.message || "Failed to delete alias"); + } + } catch (error) { + console.log("Error deleting alias:", error); + notify.error("Network error deleting alias"); + } + }, + [fetchAliases, t] + ); + const fetchProviderModelMeta = useCallback(async () => { if (isSearchProvider) return; try { @@ -1866,6 +1931,7 @@ export default function ProviderDetailPage() { body: JSON.stringify({ providerId: selectedConnection?.provider || providerNode?.id || providerId, modelId: fullModel, + connectionId: selectedConnection?.id, }), }); const data = await res.json(); @@ -1882,67 +1948,93 @@ export default function ProviderDetailPage() { } else { notify.error(data.error || "Model test failed"); setModelTestStatus((prev) => ({ ...prev, [modelId]: "error" })); + if (handleToggleModelHidden) { + await handleToggleModelHidden(providerStorageAlias, modelId, true); + } } } catch (err) { notify.error("Network error testing model"); setModelTestStatus((prev) => ({ ...prev, [modelId]: "error" })); + if (handleToggleModelHidden) { + await handleToggleModelHidden(providerStorageAlias, modelId, true); + } } finally { setTestingModelId(null); } }; - const handleSetAlias = async (modelId, alias, providerAliasOverride = providerAlias) => { - const fullModel = `${providerAliasOverride}/${modelId}`; - try { - const res = await fetch("/api/models/alias", { - method: "PUT", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ model: fullModel, alias }), - }); - if (res.ok) { - await fetchAliases(); - } else { - const data = await res.json(); - alert(data.error || t("failedSetAlias")); - } - } catch (error) { - console.log("Error setting alias:", error); + const handleTestAll = async ( + targets: Array<{ modelId: string; fullModel: string }> + ): Promise => { + if (testingAll) return; + if (targets.length === 0) { + notify.error(providerText(t, "noModelsToTest", "No models to test")); + return; } - }; + setTestingAll(true); + setTestProgress({ done: 0, total: targets.length }); - const handleDeleteAlias = async (alias) => { - try { - const res = await fetch(`/api/models/alias?alias=${encodeURIComponent(alias)}`, { - method: "DELETE", - }); - if (res.ok) { - await fetchAliases(); - } - } catch (error) { - console.log("Error deleting alias:", error); + let ok = 0; + let error = 0; + let hiddenCount = 0; + + const CHUNK_SIZE = 3; + for (let i = 0; i < targets.length; i += CHUNK_SIZE) { + const chunk = targets.slice(i, i + CHUNK_SIZE); + await Promise.all( + chunk.map(async ({ modelId, fullModel }) => { + try { + const result: { + results?: Record< + string, + { + status?: "ok" | "error"; + rateLimited?: boolean; + isTimeout?: boolean; + error?: string; + } + >; + } = await fetch("/api/models/test-all", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + providerId: providerId, + connectionId: selectedConnection?.id, + modelIds: [fullModel], + }), + }).then((r) => r.json()); + + const entry = result.results?.[fullModel]; + if (entry?.status === "ok") { + ok++; + } else { + error++; + if (autoHideFailed && !entry?.rateLimited && !entry?.isTimeout) { + await handleToggleModelHidden(providerStorageAlias, modelId, true); + hiddenCount++; + } + } + } catch (e) { + error++; + } + setTestProgress((prev) => + prev ? { done: prev.done + 1, total: prev.total } : null + ); + }) + ); } - }; - const handleDelete = async (id) => { - if (!confirm(t("deleteConnectionConfirm"))) return; - try { - const res = await fetch(`/api/providers/${id}`, { method: "DELETE" }); - if (res.ok) { - setConnections(connections.filter((c) => c.id !== id)); - if (providerId === "gemini") { - await fetchProviderModelMeta(); - } - } - } catch (error) { - console.log("Error deleting connection:", error); - } - }; - - const handleToggleSelectAll = useCallback(() => { - setSelectedIds((prev) => - prev.size === connections.length ? new Set() : new Set(connections.map((c) => c.id)) + notify.info( + providerText(t, "testAllResults", "{ok} ok, {error} error", { ok, error }) ); - }, [connections]); + if (hiddenCount > 0) { + notify.info( + providerText(t, "testAllFailedHidden", "{count} hidden", { count: hiddenCount }) + ); + } + setTestingAll(false); + setTestProgress(null); + }; const handleToggleSelectOne = useCallback((id: string) => { setSelectedIds((prev) => { @@ -1953,6 +2045,15 @@ export default function ProviderDetailPage() { }); }, []); + const handleToggleSelectAll = useCallback(() => { + setSelectedIds((prev) => { + if (prev.size === connections.length && connections.length > 0) { + return new Set(); + } + return new Set(connections.map((c) => (c as { id: string }).id)); + }); + }, [connections]); + const handleBatchDelete = async () => { if (selectedIds.size === 0) return; if (!confirm(t("batchDeleteConfirm", { count: selectedIds.size }))) return; @@ -1969,9 +2070,7 @@ export default function ProviderDetailPage() { setSelectedIds(new Set()); await fetchConnections(); notify.success(t("batchDeleteSuccess", { count: selectedIds.size })); - if (providerId === "gemini") { - await fetchProviderModelMeta(); - } + await fetchProviderModelMeta(); } else { const data = await res.json(); notify.error(data.error || "Batch delete failed"); @@ -1983,6 +2082,31 @@ export default function ProviderDetailPage() { } }; + const handleDelete = useCallback( + async (connectionId: string) => { + if (!connectionId) return; + try { + const res = await fetch(`/api/providers/${connectionId}`, { method: "DELETE" }); + if (res.ok) { + notify.success("Connection deleted"); + await fetchConnections(); + await fetchProviderModelMeta(); + } else { + const data = await res.json().catch(() => ({})); + const message = + (typeof data?.error === "string" && data.error) || + data?.error?.message || + "Failed to delete connection"; + notify.error(message); + } + } catch (error) { + console.error("Error deleting connection:", error); + notify.error("Failed to delete connection"); + } + }, + [fetchConnections, fetchProviderModelMeta, notify] + ); + const handleBatchSetActive = async (isActive: boolean) => { if (selectedIds.size === 0 || batchUpdating) return; setBatchUpdating(isActive ? "activate" : "deactivate"); @@ -2387,8 +2511,9 @@ export default function ProviderDetailPage() { setShowAddApiKeyModal(false); setSiliconFlowInitialBaseUrl(undefined); - // For Gemini: show progress dialog and sync models from endpoint - if (providerId === "gemini" && newConnection?.id) { + // Universal: sync models from the provider endpoint on every new connection + // (was previously Gemini-only). Do NOT re-introduce a providerId guard here. + if (newConnection?.id) { setShowImportModal(true); setImportProgress({ current: 0, @@ -3824,6 +3949,11 @@ export default function ProviderDetailPage() { onTestModel={onTestModel} modelTestStatus={modelTestStatus} testingModelId={testingModelId} + onTestAll={handleTestAll} + testingAll={testingAll} + testProgress={testProgress} + autoHideFailed={autoHideFailed} + onAutoHideFailedChange={setAutoHideFailed} /> ); @@ -3893,29 +4023,30 @@ export default function ProviderDetailPage() { onTestModel={onTestModel} modelTestStatus={modelTestStatus} testingModelId={testingModelId} + providerId={providerId} + connectionId={selectedConnection?.id ?? ""} /> ); } - const importButton = - providerId === "gemini" ? null : ( -
- - {autoSyncToggle} - {!canImportModels && ( - {t("addConnectionToImport")} - )} -
- ); + const importButton = ( +
+ + {autoSyncToggle} + {!canImportModels && ( + {t("addConnectionToImport")} + )} +
+ ); if (models.length === 0) { return ( @@ -3929,16 +4060,26 @@ export default function ProviderDetailPage() { ...model, isHidden: effectiveModelHidden(model.id), })); - const filteredModels = modelsWithVisibility.filter((model) => - matchesModelCatalogQuery(modelFilter, { + const filteredModels = modelsWithVisibility.filter((model) => { + const matchesQuery = matchesModelCatalogQuery(modelFilter, { modelId: model.id, modelName: model.name, source: model.source, - }) - ); + }); + const matchesVisibility = + visibilityFilter === "all" + ? true + : visibilityFilter === "visible" + ? !model.isHidden + : model.isHidden; + return matchesQuery && matchesVisibility; + }); const activeCount = modelsWithVisibility.filter((m) => !m.isHidden).length; const hiddenFilteredCount = filteredModels.filter((m) => m.isHidden).length; const visibleFilteredCount = filteredModels.length - hiddenFilteredCount; + const testAllTargets = filteredModels + .filter((m) => !m.isHidden) + .map((m) => ({ modelId: m.id, fullModel: `${providerDisplayAlias}/${m.id}` })); return (
{importButton} @@ -3965,6 +4106,13 @@ export default function ProviderDetailPage() { } selectAllDisabled={hiddenFilteredCount === 0 || bulkVisibilityAction !== null} deselectAllDisabled={visibleFilteredCount === 0 || bulkVisibilityAction !== null} + onTestAll={() => handleTestAll(testAllTargets)} + testingAll={testingAll} + testProgress={testProgress} + visibilityFilter={visibilityFilter} + onVisibilityFilterChange={setVisibilityFilter} + autoHideFailed={autoHideFailed} + onAutoHideFailedChange={setAutoHideFailed} /> )}
@@ -4227,7 +4375,7 @@ export default function ProviderDetailPage() { router.push("/dashboard/providers"); } } catch (error) { - console.log("Error deleting provider node:", error); + console.error("Error deleting provider node:", error); } }} > @@ -4455,7 +4603,11 @@ export default function ProviderDetailPage() { ) : ( connections.length === 0 && ( - ) @@ -5146,7 +5298,7 @@ export default function ProviderDetailPage() { onClose={() => setImportCodexModalOpen(false)} onSuccess={() => { setImportCodexModalOpen(false); - fetchConnections(); + void fetchConnections(); }} /> )} @@ -5213,7 +5365,7 @@ export default function ProviderDetailPage() { onClose={() => setImportClaudeModalOpen(false)} onSuccess={() => { setImportClaudeModalOpen(false); - fetchConnections(); + void fetchConnections(); }} /> )} @@ -5234,7 +5386,7 @@ export default function ProviderDetailPage() { onClose={() => setImportGeminiModalOpen(false)} onSuccess={() => { setImportGeminiModalOpen(false); - fetchConnections(); + void fetchConnections(); }} /> )} @@ -5699,6 +5851,13 @@ function ModelVisibilityToolbar({ onDeselectAll, selectAllDisabled, deselectAllDisabled, + onTestAll, + testingAll, + testProgress, + visibilityFilter, + onVisibilityFilterChange, + autoHideFailed, + onAutoHideFailedChange, }: { t: ((key: string, values?: Record) => string) & { has?: (key: string) => boolean; @@ -5711,6 +5870,13 @@ function ModelVisibilityToolbar({ onDeselectAll: () => void; selectAllDisabled?: boolean; deselectAllDisabled?: boolean; + onTestAll?: () => void; + testingAll?: boolean; + testProgress?: { done: number; total: number } | null; + visibilityFilter?: "all" | "visible" | "hidden"; + onVisibilityFilterChange?: (filter: "all" | "visible" | "hidden") => void; + autoHideFailed?: boolean; + onAutoHideFailedChange?: (v: boolean) => void; }) { return (
@@ -5726,30 +5892,73 @@ function ModelVisibilityToolbar({ className="w-full rounded-lg border border-border bg-sidebar/50 py-1.5 pl-7 pr-3 text-xs text-text-main placeholder:text-text-muted focus:outline-none focus:ring-1 focus:ring-primary" />
+ {visibilityFilter !== undefined && onVisibilityFilterChange && ( +
+ {(["all", "visible", "hidden"] as const).map((f) => ( + + ))} +
+ )} + {onAutoHideFailedChange && ( + + )} + {onTestAll && ( + + )} - - {providerText(t, "modelsActiveCount", "{active}/{total} active", { - active: activeCount, - total: totalCount, - })} -
); } @@ -5780,12 +5989,78 @@ function PassthroughModelsSection({ onTestModel, modelTestStatus, testingModelId, + providerId, + connectionId, }: PassthroughModelsSectionProps) { const [newModel, setNewModel] = useState(""); const [adding, setAdding] = useState(false); const [modelFilter, setModelFilter] = useState(""); + const [testingAll, setTestingAll] = useState(false); + const [testProgress, setTestProgress] = useState<{ done: number; total: number } | null>(null); + const [autoHideFailed, setAutoHideFailed] = useState(true); + const [visibilityFilter, setVisibilityFilter] = useState<"all" | "visible" | "hidden">("all"); + const notify = useNotificationStore(); const customModelMap = useMemo(() => buildCompatMap(customModels), [customModels]); + const handleTestAll = async () => { + const modelsToTest = filteredModels.filter((m) => !m.isHidden); + if (modelsToTest.length === 0) { + notify.error(providerText(t, "noModelsToTest", "No models to test")); + return; + } + setTestingAll(true); + setTestProgress({ done: 0, total: modelsToTest.length }); + + let ok = 0; + let error = 0; + let hiddenCount = 0; + + for (const model of modelsToTest) { + try { + const result: { + results?: Record< + string, + { + status?: "ok" | "error"; + rateLimited?: boolean; + isTimeout?: boolean; + error?: string; + } + >; + } = await fetch("/api/models/test-all", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + providerId, + connectionId, + modelIds: [model.modelId], + }), + }).then((r) => r.json()); + + const entry = result.results?.[model.modelId]; + if (entry?.status === "ok") { + ok++; + } else { + error++; + if (autoHideFailed && !entry?.rateLimited && !entry?.isTimeout) { + await onToggleHidden(model.modelId, true); + hiddenCount++; + } + } + } catch (e) { + error++; + } + setTestProgress((prev) => (prev ? { done: prev.done + 1, total: prev.total } : null)); + } + + notify.info(providerText(t, "testAllResults", "{ok} ok, {error} error", { ok, error })); + if (hiddenCount > 0) { + notify.info(providerText(t, "testAllFailedHidden", "{count} hidden", { count: hiddenCount })); + } + setTestingAll(false); + setTestProgress(null); + }; + const providerAliases = useMemo( () => Object.entries(modelAliases).filter(([, model]: [string, any]) => @@ -5874,14 +6149,23 @@ function PassthroughModelsSection({ providerAlias, providerAliases, ]); - const filteredModels = allModels.filter((model) => - matchesModelCatalogQuery(modelFilter, { + const filteredModels = allModels.filter((model) => { + const matchesQuery = matchesModelCatalogQuery(modelFilter, { modelId: model.modelId, modelName: model.displayName, alias: model.alias, source: model.source, - }) - ); + }); + + const matchesVisibility = + visibilityFilter === "all" + ? true + : visibilityFilter === "visible" + ? !model.isHidden + : model.isHidden; + + return matchesQuery && matchesVisibility; + }); const activeCount = allModels.filter((model) => !model.isHidden).length; const hiddenFilteredCount = filteredModels.filter((model) => model.isHidden).length; const visibleFilteredCount = filteredModels.length - hiddenFilteredCount; @@ -5908,7 +6192,7 @@ function PassthroughModelsSection({ await onSetAlias(modelId, defaultAlias); setNewModel(""); } catch (error) { - console.log("Error adding model:", error); + console.error("Error adding model:", error); } finally { setAdding(false); } @@ -5950,18 +6234,24 @@ function PassthroughModelsSection({ totalCount={allModels.length} onSelectAll={() => onBulkToggleHidden( - filteredModels.map((model) => model.modelId), + filteredModels.map((m) => m.modelId), false ) } onDeselectAll={() => onBulkToggleHidden( - filteredModels.map((model) => model.modelId), + filteredModels.map((m) => m.modelId), true ) } - selectAllDisabled={hiddenFilteredCount === 0 || bulkTogglePending} - deselectAllDisabled={visibleFilteredCount === 0 || bulkTogglePending} + selectAllDisabled={bulkTogglePending || filteredModels.length === 0} + deselectAllDisabled={bulkTogglePending || filteredModels.length === 0} + onTestAll={handleTestAll} + testingAll={testingAll} + visibilityFilter={visibilityFilter} + onVisibilityFilterChange={setVisibilityFilter} + autoHideFailed={autoHideFailed} + onAutoHideFailedChange={setAutoHideFailed} />
{filteredModels.map(({ modelId, fullModel, alias, isHidden, source, isFree }) => ( @@ -6687,11 +6977,17 @@ function CompatibleModelsSection({ onTestModel, modelTestStatus, testingModelId, + onTestAll, + testingAll, + testProgress, + autoHideFailed, + onAutoHideFailedChange, }: CompatibleModelsSectionProps) { const [newModel, setNewModel] = useState(""); const [adding, setAdding] = useState(false); const [importing, setImporting] = useState(false); const [modelFilter, setModelFilter] = useState(""); + const [visibilityFilter, setVisibilityFilter] = useState<"all" | "visible" | "hidden">("all"); const notify = useNotificationStore(); const customModelMap = useMemo(() => buildCompatMap(customModels), [customModels]); @@ -6782,14 +7078,21 @@ function CompatibleModelsSection({ providerAliases, providerStorageAlias, ]); - const filteredModels = allModels.filter((model) => - matchesModelCatalogQuery(modelFilter, { + const filteredModels = allModels.filter((model) => { + const matchesQuery = matchesModelCatalogQuery(modelFilter, { modelId: model.modelId, modelName: model.displayName, alias: model.alias, source: model.source, - }) - ); + }); + const matchesVisibility = + visibilityFilter === "all" + ? true + : visibilityFilter === "visible" + ? !model.isHidden + : model.isHidden; + return matchesQuery && matchesVisibility; + }); const activeCount = allModels.filter((model) => !model.isHidden).length; const hiddenFilteredCount = filteredModels.filter((model) => model.isHidden).length; const visibleFilteredCount = filteredModels.length - hiddenFilteredCount; @@ -6956,6 +7259,21 @@ function CompatibleModelsSection({ } selectAllDisabled={hiddenFilteredCount === 0 || bulkTogglePending} deselectAllDisabled={visibleFilteredCount === 0 || bulkTogglePending} + visibilityFilter={visibilityFilter} + onVisibilityFilterChange={setVisibilityFilter} + onTestAll={() => { + const targets = filteredModels + .filter((m) => !m.isHidden) + .map((m) => ({ + modelId: m.modelId, + fullModel: `${providerDisplayAlias}/${m.modelId}`, + })); + return onTestAll?.(targets); + }} + testingAll={testingAll} + testProgress={testProgress} + autoHideFailed={autoHideFailed} + onAutoHideFailedChange={onAutoHideFailedChange} />
{filteredModels.map(({ modelId, alias, isHidden, source, isFree }) => { @@ -7050,8 +7368,8 @@ const ERROR_TYPE_LABELS = { network_error: { labelKey: "errorTypeNetworkError", variant: "warning" }, unsupported: { labelKey: "errorTypeTestUnsupported", variant: "default" }, upstream_error: { labelKey: "errorTypeUpstreamError", variant: "error" }, - banned: { labelKey: "403 Banned", variant: "error" }, - credits_exhausted: { labelKey: "No Credits", variant: "warning" }, + banned: { labelKey: "errorTypeBanned", variant: "error" }, + credits_exhausted: { labelKey: "errorTypeCreditsExhausted", variant: "warning" }, }; function inferErrorType(connection, isCooldown) { @@ -10331,6 +10649,11 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec name: "", priority: 1, maxConcurrent: "", + rpm: "", + tpm: "", + tpd: "", + minTime: "", + rateLimitMaxConcurrent: "", apiKey: "", healthCheckInterval: 60, baseUrl: "", @@ -10460,6 +10783,26 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec connection.maxConcurrent !== null && connection.maxConcurrent !== undefined ? String(connection.maxConcurrent) : "", + rpm: + connection.rateLimitOverrides?.rpm != null + ? String(connection.rateLimitOverrides.rpm) + : "", + tpm: + connection.rateLimitOverrides?.tpm != null + ? String(connection.rateLimitOverrides.tpm) + : "", + tpd: + connection.rateLimitOverrides?.tpd != null + ? String(connection.rateLimitOverrides.tpd) + : "", + minTime: + connection.rateLimitOverrides?.minTime != null + ? String(connection.rateLimitOverrides.minTime) + : "", + rateLimitMaxConcurrent: + connection.rateLimitOverrides?.maxConcurrent != null + ? String(connection.rateLimitOverrides.maxConcurrent) + : "", apiKey: "", healthCheckInterval: connection.healthCheckInterval ?? 60, baseUrl: existingBaseUrl || defaultBaseUrl, @@ -10613,6 +10956,16 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec healthCheckInterval: formData.healthCheckInterval, }; + // Build rateLimitOverrides from non-empty fields + const overrides: Record = {}; + if (formData.rpm.trim()) overrides.rpm = Number(formData.rpm); + if (formData.tpm.trim()) overrides.tpm = Number(formData.tpm); + if (formData.tpd.trim()) overrides.tpd = Number(formData.tpd); + if (formData.minTime.trim()) overrides.minTime = Number(formData.minTime); + if (formData.rateLimitMaxConcurrent.trim()) + overrides.maxConcurrent = Number(formData.rateLimitMaxConcurrent); + updates.rateLimitOverrides = Object.keys(overrides).length > 0 ? overrides : null; + if (supportsGoogleProjectId) { updates.projectId = trimmedCloudCodeProjectId || null; } @@ -11057,6 +11410,60 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec type="password" /> )} +
+

+ {t("rateLimitOverridesSection")} +

+
+ setFormData({ ...formData, rpm: e.target.value })} + placeholder="Inherit" + hint={t("rateLimitOverridesRpmHint")} + /> + setFormData({ ...formData, tpm: e.target.value })} + placeholder="Inherit" + hint={t("rateLimitOverridesTpmHint")} + /> + setFormData({ ...formData, tpd: e.target.value })} + placeholder="Inherit" + hint={t("rateLimitOverridesTpdHint")} + /> + setFormData({ ...formData, minTime: e.target.value })} + placeholder="Inherit" + hint={t("rateLimitOverridesMinTimeHint")} + /> + + setFormData({ ...formData, rateLimitMaxConcurrent: e.target.value }) + } + placeholder="Inherit" + hint={t("rateLimitOverridesMaxConcurrentHint")} + /> +
+
)} > +): BatchTestResultEntry { + const entry: BatchTestResultEntry = { + status: result.status === "ok" ? "ok" : "error", + latencyMs: result.latencyMs, + }; + if (result.responseText !== undefined) entry.responseText = result.responseText; + if (result.error !== undefined) entry.error = result.error; + if (result.statusCode !== undefined) entry.statusCode = result.statusCode; + if (result.rateLimited === true) entry.rateLimited = true; + if (result.isTimeout === true) entry.isTimeout = true; + return entry; +} + +export async function POST(request: Request) { + const authError = await requireManagementAuth(request); + if (authError) return authError; + + let rawBody: unknown; + try { + rawBody = await request.json(); + } catch { + return NextResponse.json( + { + error: { + message: "Invalid request", + details: [{ field: "body", message: "Invalid JSON body" }], + }, + }, + { status: 400 } + ); + } + + const validation = testAllSchema.safeParse(rawBody); + if (!validation.success) { + return NextResponse.json({ error: validation.error.format() }, { status: 400 }); + } + const { providerId, modelIds, connectionId, respectRateLimit, autoHideFailed } = validation.data; + + log.info( + "MODEL_TEST_ALL", + `Starting batch test for ${modelIds.length} model(s) on provider ${providerId}`, + { + providerId, + modelCount: modelIds.length, + hasConnection: Boolean(connectionId), + respectRateLimit, + autoHideFailed, + } + ); + + const results: Record = {}; + let consecutiveRateLimits = 0; + let stoppedEarly = false; + let stopReason: "consecutive_rate_limits" | undefined; + + for (const modelId of modelIds) { + let entry: BatchTestResultEntry; + try { + // `runSingleModelTest` only engages the Bottleneck rate limiter when + // `connectionId` is provided. To honor `respectRateLimit=false`, we + // omit the connectionId so the runner bypasses `withRateLimit`. + const effectiveConnectionId = + connectionId && respectRateLimit !== false ? connectionId : undefined; + const result = await runSingleModelTest({ + providerId, + modelId, + ...(effectiveConnectionId ? { connectionId: effectiveConnectionId } : {}), + timeoutMs: PER_MODEL_TIMEOUT_MS, + }); + entry = toBatchEntry(result); + } catch (error: unknown) { + log.error("MODEL_TEST_ALL", `Unexpected error testing model ${modelId}`, { + providerId, + modelId, + error: sanitizeErrorMessage(error), + }); + entry = { + status: "error", + latencyMs: 0, + error: sanitizeErrorMessage(error) || "Unknown error", + }; + } + + if (entry.rateLimited) { + consecutiveRateLimits += 1; + } else { + consecutiveRateLimits = 0; + } + + if (autoHideFailed && entry.status === "error" && !entry.rateLimited && !entry.isTimeout) { + try { + await setModelIsHidden(providerId, modelId, true); + entry.hidden = true; + log.info("MODEL_TEST_ALL", `Auto-hidden model ${modelId} after test failure`, { + providerId, + modelId, + }); + } catch (hideError: unknown) { + log.error("MODEL_TEST_ALL", `Failed to auto-hide model ${modelId}`, { + providerId, + modelId, + error: sanitizeErrorMessage(hideError), + }); + } + } + + results[modelId] = entry; + log.info( + "MODEL_TEST_ALL", + `Tested ${modelId}: ${entry.status}${entry.rateLimited ? " (rate-limited)" : ""}${entry.isTimeout ? " (timeout)" : ""}`, + { + providerId, + modelId, + latencyMs: entry.latencyMs, + rateLimited: entry.rateLimited, + isTimeout: entry.isTimeout, + hidden: entry.hidden, + } + ); + + if (consecutiveRateLimits >= CONSECUTIVE_RATE_LIMIT_STOP_THRESHOLD) { + stoppedEarly = true; + stopReason = "consecutive_rate_limits"; + log.warn( + "MODEL_TEST_ALL", + `Stopping batch early after ${consecutiveRateLimits} consecutive rate-limited results`, + { + providerId, + testedCount: Object.keys(results).length, + totalCount: modelIds.length, + } + ); + break; + } + } + + log.info( + "MODEL_TEST_ALL", + `Batch test complete: ${Object.keys(results).length}/${modelIds.length} model(s)`, + { + providerId, + tested: Object.keys(results).length, + total: modelIds.length, + stoppedEarly, + } + ); + + return NextResponse.json({ + results, + ...(stoppedEarly && stopReason ? { stoppedEarly: true, stopReason } : {}), + }); +} diff --git a/src/app/api/models/test/route.ts b/src/app/api/models/test/route.ts index 6bce4e48bd..376b050610 100644 --- a/src/app/api/models/test/route.ts +++ b/src/app/api/models/test/route.ts @@ -1,118 +1,16 @@ -import { randomUUID } from "node:crypto"; import { NextResponse } from "next/server"; -import { POST as postChatCompletion } from "@/app/api/v1/chat/completions/route"; -import { handleValidatedEmbeddingRequestBody } from "@/app/api/v1/embeddings/route"; -import { POST as postRerank } from "@/app/api/v1/rerank/route"; -import { buildComboTestRequestBody, extractComboTestResponseText } from "@/lib/combos/testHealth"; -import { requireManagementAuth } from "@/lib/api/requireManagementAuth"; -import { getCustomModels } from "@/lib/localDb"; -import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/error"; import { z } from "zod"; +import { requireManagementAuth } from "@/lib/api/requireManagementAuth"; +import { runSingleModelTest } from "@/lib/api/modelTestRunner"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/error.ts"; const testModelSchema = z.object({ providerId: z.string().min(1), modelId: z.string().min(1), + connectionId: z.string().min(1).optional(), }); -const MODEL_TEST_TIMEOUT_MS = 20_000; -const INTERNAL_ORIGIN = "http://omniroute.internal"; - -function asRecord(value: unknown): Record { - return value && typeof value === "object" && !Array.isArray(value) - ? (value as Record) - : {}; -} - -function getErrorMessage(error: unknown): string { - return sanitizeErrorMessage(error) || "Unknown error"; -} - -function getErrorName(error: unknown): string { - return error instanceof Error ? error.name : ""; -} - -function extractProviderErrorMessage(body: unknown, fallback: string) { - const record = asRecord(body); - const error = record.error; - if (typeof error === "string" && error.trim()) return error; - - const errorRecord = asRecord(error); - const message = errorRecord.message; - return typeof message === "string" && message.trim() ? message : fallback; -} - -async function runWithTimeout(operation: (signal: AbortSignal) => Promise): Promise { - const controller = new AbortController(); - const timeoutPromise = new Promise((_, reject) => { - const timeoutError = new Error("Timeout (20s)"); - timeoutError.name = "AbortError"; - const timeout = setTimeout(() => { - controller.abort(); - reject(timeoutError); - }, MODEL_TEST_TIMEOUT_MS); - controller.signal.addEventListener("abort", () => clearTimeout(timeout), { once: true }); - }); - - try { - return await Promise.race([operation(controller.signal), timeoutPromise]); - } finally { - controller.abort(); - } -} - -function buildInternalChatRequest(testBody: Record, signal: AbortSignal) { - return new Request(`${INTERNAL_ORIGIN}/v1/chat/completions`, { - method: "POST", - headers: { - "Content-Type": "application/json", - // Reuse the existing strict-mode internal bypass for live health checks. - "X-Internal-Test": "combo-health-check", - "X-OmniRoute-No-Cache": "true", - "X-Request-Id": `model-test-${randomUUID()}`, - }, - body: JSON.stringify(testBody), - signal, - }); -} - -function buildInternalRerankRequest(testBody: Record, signal: AbortSignal) { - return new Request(`${INTERNAL_ORIGIN}/v1/rerank`, { - method: "POST", - headers: { - "Content-Type": "application/json", - "X-Internal-Test": "combo-health-check", - "X-OmniRoute-No-Cache": "true", - "X-Request-Id": `model-test-${randomUUID()}`, - }, - body: JSON.stringify(testBody), - signal, - }); -} - -function stripFirstSegment(modelId: string): string | null { - const slashIdx = modelId.indexOf("/"); - return slashIdx > 0 ? modelId.slice(slashIdx + 1) : null; -} - -async function findCustomModelMetadata(providerId: string, modelId: string) { - try { - const customModels = await getCustomModels(providerId); - if (!Array.isArray(customModels)) return null; - - const candidates = new Set([modelId]); - const stripped = stripFirstSegment(modelId); - if (stripped) candidates.add(stripped); - if (modelId.startsWith(`${providerId}/`)) candidates.add(modelId.slice(providerId.length + 1)); - - return ( - customModels.find( - (model: any) => typeof model?.id === "string" && candidates.has(model.id) - ) || null - ); - } catch { - return null; - } -} +const SINGLE_TEST_TIMEOUT_MS = 20_000; export async function POST(request: Request) { const authError = await requireManagementAuth(request); @@ -138,115 +36,38 @@ export async function POST(request: Request) { if (!validation.success) { return NextResponse.json({ error: validation.error.format() }, { status: 400 }); } - const { providerId, modelId } = validation.data; + const { providerId, modelId, connectionId } = validation.data; - // Construct target format (providerId/modelId) - // Some models (like free alias models) might not need the prefix if it's an alias. - // However, the wildcard router expects provider/model. - let fullModelStr = modelId; - if (!fullModelStr.includes("/")) { - fullModelStr = `${providerId}/${modelId}`; + const result = await runSingleModelTest({ + providerId, + modelId, + ...(connectionId ? { connectionId } : {}), + timeoutMs: SINGLE_TEST_TIMEOUT_MS, + }); + + if (result.status === "ok") { + return NextResponse.json({ + status: "ok", + latencyMs: result.latencyMs, + responseText: result.responseText, + }); } - const startTime = Date.now(); - const customModel = await findCustomModelMetadata(providerId, fullModelStr); - const supportedEndpoints = Array.isArray(customModel?.supportedEndpoints) - ? customModel.supportedEndpoints - : []; - const apiFormat = typeof customModel?.apiFormat === "string" ? customModel.apiFormat : ""; - const lowerModel = fullModelStr.toLowerCase(); - const isRerank = - apiFormat === "rerank" || - supportedEndpoints.includes("rerank") || - lowerModel.includes("rerank"); - const isEmbedding = - !isRerank && - (apiFormat === "embeddings" || - supportedEndpoints.includes("embeddings") || - lowerModel.includes("embedding") || - lowerModel.includes("bge-") || - lowerModel.includes("text-embed") || - lowerModel.includes("jina-clip") || - lowerModel.includes("colbert")); + const body: Record = { + status: "error", + latencyMs: result.latencyMs, + error: result.error || "Unknown error", + }; + if (result.statusCode !== undefined) body.statusCode = result.statusCode; + if (result.rateLimited) body.rateLimited = true; + if (result.retryAfter !== undefined) body.retryAfter = result.retryAfter; - const testBody = isRerank - ? { - model: fullModelStr, - query: "What is OmniRoute?", - documents: [ - "OmniRoute routes AI requests across configured providers.", - "This document is unrelated to the test query.", - ], - top_n: 1, - return_documents: false, - } - : buildComboTestRequestBody(fullModelStr, isEmbedding); - - const res = await runWithTimeout((signal) => - isEmbedding - ? handleValidatedEmbeddingRequestBody( - testBody as Record & { model: string } - ) - : isRerank - ? postRerank(buildInternalRerankRequest(testBody, signal)) - : postChatCompletion(buildInternalChatRequest(testBody, signal)) - ); - - const latencyMs = Date.now() - startTime; - - if (res.ok) { - let responseBody = null; - try { - responseBody = await res.json(); - } catch { - responseBody = null; - } - - const responseText = extractComboTestResponseText(responseBody); - if (isRerank) { - return NextResponse.json({ - status: "ok", - latencyMs, - responseText: "[Rerank completed successfully]", - }); - } - if (!responseText && !isEmbedding) { - return NextResponse.json( - { - status: "error", - statusCode: res.status, - error: "Provider returned HTTP 200 but no text content.", - latencyMs, - }, - { status: 400 } - ); - } - - return NextResponse.json({ status: "ok", latencyMs, responseText }); - } - - let errorMsg = ""; - try { - const errBody = await res.json(); - errorMsg = extractProviderErrorMessage(errBody, res.statusText); - } catch { - errorMsg = res.statusText; - } - - return NextResponse.json( - { - status: "error", - statusCode: res.status, - error: errorMsg, - latencyMs, - }, - { status: res.status } - ); + return NextResponse.json(body, { status: result.httpStatus }); } catch (error: unknown) { return NextResponse.json( { status: "error", - error: getErrorName(error) === "AbortError" ? "Timeout (20s)" : getErrorMessage(error), + error: sanitizeErrorMessage(error) || "Unknown error", }, { status: 500 } ); diff --git a/src/app/api/providers/[id]/route.ts b/src/app/api/providers/[id]/route.ts index a48fea4100..c92a7a2c67 100644 --- a/src/app/api/providers/[id]/route.ts +++ b/src/app/api/providers/[id]/route.ts @@ -24,6 +24,7 @@ import { } from "@/lib/providers/claudeExtraUsage"; import { requireManagementAuth } from "@/lib/api/requireManagementAuth"; import { isApiKeyRevealEnabled, maskStoredApiKey } from "@/lib/apiKeyExposure"; +import { refreshConnectionRateLimits, enableRateLimitProtection } from "@/../open-sse/services/rateLimitManager"; function normalizeCodexLimitPolicy( incoming: unknown, @@ -136,6 +137,7 @@ export async function PUT(request: Request, { params }: { params: Promise<{ id: perKeyProxyEnabled, projectId, providerSpecificData: incomingPsd, + rateLimitOverrides, } = body; const existing = (await getProviderConnectionById(id)) as Record | null; @@ -186,6 +188,7 @@ export async function PUT(request: Request, { params }: { params: Promise<{ id: } } if (projectId !== undefined) updateData.projectId = projectId; + if (rateLimitOverrides !== undefined) updateData.rateLimitOverrides = rateLimitOverrides; if (proxyEnabled !== undefined) updateData.proxyEnabled = proxyEnabled; if (perKeyProxyEnabled !== undefined) updateData.perKeyProxyEnabled = perKeyProxyEnabled; @@ -282,6 +285,14 @@ export async function PUT(request: Request, { params }: { params: Promise<{ id: const updated = await updateProviderConnection(id, updateData); + // If rateLimitOverrides was included in the request, refresh the in-memory + // rate limiter state so the change takes effect without a server restart. + // Also ensure rate limit protection is active so the limiter is enforced. + if (rateLimitOverrides !== undefined) { + refreshConnectionRateLimits(id, updated?.rateLimitOverrides ?? null); + enableRateLimitProtection(id); + } + // Hide sensitive fields const result: Record = { ...updated }; delete result.apiKey; diff --git a/src/app/api/providers/route.ts b/src/app/api/providers/route.ts index c13b472823..877b3dbde3 100644 --- a/src/app/api/providers/route.ts +++ b/src/app/api/providers/route.ts @@ -32,6 +32,7 @@ import { import { requireManagementAuth } from "@/lib/api/requireManagementAuth"; import { isManagedProviderConnectionId } from "@/lib/providers/catalog"; import { isApiKeyRevealEnabled, maskStoredApiKey } from "@/lib/apiKeyExposure"; +import { buildModelSyncInternalHeaders } from "@/shared/services/modelSyncScheduler"; // GET /api/providers - List all connections export async function GET(request: Request) { @@ -164,6 +165,48 @@ export async function POST(request: Request) { testStatus: testStatus || "unknown", }); + // Auto-trigger model discovery for the newly created connection. + // Fire-and-forget: model sync can take seconds and should NOT block the + // POST response. If it fails, we log and move on — the connection itself + // is already persisted and the user can manually trigger a sync later. + // We use a self-fetch against our own /sync-models route, forwarding the + // incoming cookies (preserves management auth) plus the internal sync + // auth header (defense in depth) and an X-Internal-Auto-Sync marker for + // log correlation. + try { + const internalOrigin = new URL(request.url).origin; + const cookieHeader = request.headers.get("cookie") || ""; + const syncHeaders: Record = { + "Content-Type": "application/json", + "X-Internal-Auto-Sync": "true", + ...(cookieHeader ? { cookie: cookieHeader } : {}), + ...buildModelSyncInternalHeaders(), + }; + const syncUrl = `${internalOrigin}/api/providers/${encodeURIComponent(newConnection.id)}/sync-models?mode=import`; + // Intentionally not awaited: this is async/non-blocking work. + void fetch(syncUrl, { method: "POST", headers: syncHeaders }) + .then((syncRes) => { + if (!syncRes.ok) { + console.log( + `[providers] Auto-sync failed for ${newConnection.id}: ${syncRes.status}` + ); + } + }) + .catch((err) => { + console.log( + `[providers] Auto-sync error for ${newConnection.id}:`, + err?.message || err + ); + }); + } catch (syncSetupError) { + // Defensive: if URL parsing or header construction itself throws, do + // not let it break the (already successful) POST response. + console.log( + `[providers] Auto-sync setup failed for ${newConnection.id}:`, + syncSetupError?.message || syncSetupError + ); + } + // Note: Gemini model sync is now triggered client-side with progress dialog // Hide sensitive fields diff --git a/src/app/api/v1/models/catalog.ts b/src/app/api/v1/models/catalog.ts index a7297e3ee1..9e1d36ca16 100644 --- a/src/app/api/v1/models/catalog.ts +++ b/src/app/api/v1/models/catalog.ts @@ -565,7 +565,10 @@ export async function getUnifiedModelsResponse( }; }; - const buildComboCatalogMetadata = (combo: Record, allCombos: any[]) => { + const buildComboCatalogMetadata = ( + combo: Parameters[0], + allCombos: Parameters[1] + ) => { const explicitContextLength = isPositiveFiniteNumber(combo.context_length) ? combo.context_length : undefined; @@ -638,6 +641,21 @@ export async function getUnifiedModelsResponse( for (const combo of combos) { if (combo.isActive === false || combo.isHidden === true) continue; if (typeof combo.name !== "string" || combo.name.length === 0) continue; + + // Skip combos whose any underlying target model is hidden + const comboTargets = resolveNestedComboTargets( + combo as Parameters[0], + combos as Parameters[1] + ) as ComboCatalogTarget[]; + if ( + comboTargets.some((target) => { + const resolved = getComboTargetModelId(target); + return resolved ? getModelIsHidden(resolved.providerId, resolved.modelId) : false; + }) + ) { + continue; + } + const comboMetadata = buildComboCatalogMetadata(combo, combos); models.push({ @@ -972,6 +990,7 @@ export async function getUnifiedModelsResponse( if (!isProviderActive(embModel.provider)) continue; const rawModelId = embModel.id.split("/").pop() || embModel.id; if (!providerSupportsModel(embModel.provider, rawModelId)) continue; + if (getModelIsHidden(embModel.provider, rawModelId)) continue; if (hasEquivalentSpecialtyModel(embModel.provider, rawModelId, "embedding", embModel.id)) { continue; } @@ -991,6 +1010,7 @@ export async function getUnifiedModelsResponse( if (!isProviderActive(imgModel.provider)) continue; const rawModelId = imgModel.id.split("/").pop() || imgModel.id; if (!providerSupportsModel(imgModel.provider, rawModelId)) continue; + if (getModelIsHidden(imgModel.provider, rawModelId)) continue; models.push({ id: imgModel.id, object: "model", @@ -1009,6 +1029,7 @@ export async function getUnifiedModelsResponse( if (!isProviderActive(rerankModel.provider)) continue; const rawModelId = rerankModel.id.split("/").pop() || rerankModel.id; if (!providerSupportsModel(rerankModel.provider, rawModelId)) continue; + if (getModelIsHidden(rerankModel.provider, rawModelId)) continue; if (hasEquivalentSpecialtyModel(rerankModel.provider, rawModelId, "rerank", rerankModel.id)) { continue; } @@ -1027,6 +1048,7 @@ export async function getUnifiedModelsResponse( if (!isProviderActive(audioModel.provider)) continue; const rawModelId = audioModel.id.split("/").pop() || audioModel.id; if (!providerSupportsModel(audioModel.provider, rawModelId)) continue; + if (getModelIsHidden(audioModel.provider, rawModelId)) continue; models.push({ id: audioModel.id, object: "model", @@ -1042,6 +1064,7 @@ export async function getUnifiedModelsResponse( if (!isProviderActive(modModel.provider)) continue; const rawModelId = modModel.id.split("/").pop() || modModel.id; if (!providerSupportsModel(modModel.provider, rawModelId)) continue; + if (getModelIsHidden(modModel.provider, rawModelId)) continue; models.push({ id: modModel.id, object: "model", @@ -1056,6 +1079,7 @@ export async function getUnifiedModelsResponse( if (!isProviderActive(videoModel.provider)) continue; const rawModelId = videoModel.id.split("/").pop() || videoModel.id; if (!providerSupportsModel(videoModel.provider, rawModelId)) continue; + if (getModelIsHidden(videoModel.provider, rawModelId)) continue; models.push({ id: videoModel.id, object: "model", @@ -1070,6 +1094,7 @@ export async function getUnifiedModelsResponse( if (!isProviderActive(musicModel.provider)) continue; const rawModelId = musicModel.id.split("/").pop() || musicModel.id; if (!providerSupportsModel(musicModel.provider, rawModelId)) continue; + if (getModelIsHidden(musicModel.provider, rawModelId)) continue; models.push({ id: musicModel.id, object: "model", @@ -1112,6 +1137,7 @@ export async function getUnifiedModelsResponse( const modelId = typeof model.id === "string" ? model.id : null; if (!modelId) continue; if (model.isHidden === true) continue; + if (getModelIsHidden(canonicalProviderId, modelId)) continue; if ( !hasEligibleConnectionForModel( getConnectionsForProvider(alias, canonicalProviderId, providerId, parentProviderType), @@ -1215,11 +1241,12 @@ export async function getUnifiedModelsResponse( const prefix = providerIdToPrefix[providerId]; const alias = prefix || providerIdToAlias[providerId] || providerId; + const canonicalProviderId = resolveCanonicalProviderId(alias, providerId); for (const model of fallbackModels) { const modelId = typeof model.id === "string" ? model.id : null; if (!modelId) continue; - if (getModelIsHidden(providerId, modelId)) continue; + if (getModelIsHidden(canonicalProviderId, modelId)) continue; if (!hasEligibleConnectionForModel([conn], modelId)) continue; const aliasId = `${alias}/${modelId}`; diff --git a/src/domain/types.ts b/src/domain/types.ts index a2dc1b3ade..9e147758f0 100644 --- a/src/domain/types.ts +++ b/src/domain/types.ts @@ -28,6 +28,12 @@ * @property {string} [email] - Email (for oauth auth) * @property {string} [baseUrl] - Custom base URL * @property {boolean} [rateLimitProtection] - Whether rate limit protection is enabled + * @property {object} [rateLimitOverrides] - Per-connection rate limit overrides + * @property {number} [rateLimitOverrides.rpm] - Requests per minute limit + * @property {number} [rateLimitOverrides.tpm] - Tokens per minute limit + * @property {number} [rateLimitOverrides.tpd] - Tokens per day limit + * @property {number} [rateLimitOverrides.minTime] - Minimum ms between requests + * @property {number} [rateLimitOverrides.maxConcurrent] - Max concurrent requests * @property {string} createdAt - ISO timestamp * @property {string} updatedAt - ISO timestamp */ diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json index 717960ac33..4bc3ff083a 100644 --- a/src/i18n/messages/en.json +++ b/src/i18n/messages/en.json @@ -1289,42 +1289,78 @@ "actor": "Actor", "actorPlaceholder": "Filter by actor", "eventTypes": { - "apiKey.activate": "API Key Activated", - "apiKey.ban": "API Key Banned", - "apiKey.deactivate": "API Key Deactivated", - "apiKey.regenerate": "API Key Regenerated", - "apiKey.scopes.grant": "API Key Scopes Granted", - "apiKey.scopes.revoke": "API Key Scopes Revoked", - "apiKey.scopes.update": "API Key Scopes Updated", - "apiKey.unban": "API Key Unbanned", - "auth.login.error": "Login Error", - "auth.login.failed": "Login Failed", - "auth.login.locked": "Login Locked", - "auth.login.misconfigured": "Login Misconfigured", - "auth.login.setup_required": "Login Setup Required", - "auth.login.success": "Login Success", - "auth.logout.success": "Logout Success", - "compliance.cleanup": "Compliance Cleanup", - "provider.credentials.applied": "Provider Credentials Applied", - "provider.credentials.batch_revoked": "Provider Credentials Batch Revoked", - "provider.credentials.bulk_created": "Provider Credentials Bulk Created", - "provider.credentials.bulk_imported": "Provider Credentials Bulk Imported", - "provider.credentials.created": "Provider Credentials Created", - "provider.credentials.imported": "Provider Credentials Imported", - "provider.credentials.revoked": "Provider Credentials Revoked", - "provider.credentials.updated": "Provider Credentials Updated", - "provider.validation.ssrf_blocked": "Provider SSRF Blocked", - "quota.plan.updated": "Quota Plan Updated", - "quota.pool.created": "Quota Pool Created", - "quota.pool.deleted": "Quota Pool Deleted", - "quota.pool.updated": "Quota Pool Updated", - "quota.store.driver_changed": "Quota Store Driver Changed", - "server.start": "Server Start", - "service.reveal_api_key": "Service API Key Revealed", - "settings.update": "Settings Updated", - "settings.update_failed": "Settings Update Failed", - "sync.token.created": "Sync Token Created", - "sync.token.revoked": "Sync Token Revoked" + "apiKey": { + "activate": "API Key Activated", + "ban": "API Key Banned", + "deactivate": "API Key Deactivated", + "regenerate": "API Key Regenerated", + "scopes": { + "grant": "API Key Scopes Granted", + "revoke": "API Key Scopes Revoked", + "update": "API Key Scopes Updated" + }, + "unban": "API Key Unbanned" + }, + "auth": { + "login": { + "error": "Login Error", + "failed": "Login Failed", + "locked": "Login Locked", + "misconfigured": "Login Misconfigured", + "setup_required": "Login Setup Required", + "success": "Login Success" + }, + "logout": { + "success": "Logout Success" + } + }, + "compliance": { + "cleanup": "Compliance Cleanup" + }, + "provider": { + "credentials": { + "applied": "Provider Credentials Applied", + "batch_revoked": "Provider Credentials Batch Revoked", + "bulk_created": "Provider Credentials Bulk Created", + "bulk_imported": "Provider Credentials Bulk Imported", + "created": "Provider Credentials Created", + "imported": "Provider Credentials Imported", + "revoked": "Provider Credentials Revoked", + "updated": "Provider Credentials Updated" + }, + "validation": { + "ssrf_blocked": "Provider SSRF Blocked" + } + }, + "quota": { + "plan": { + "updated": "Quota Plan Updated" + }, + "pool": { + "created": "Quota Pool Created", + "deleted": "Quota Pool Deleted", + "updated": "Quota Pool Updated" + }, + "store": { + "driver_changed": "Quota Store Driver Changed" + } + }, + "server": { + "start": "Server Start" + }, + "service": { + "reveal_api_key": "Service API Key Revealed" + }, + "settings": { + "update": "Settings Updated", + "update_failed": "Settings Update Failed" + }, + "sync": { + "token": { + "created": "Sync Token Created", + "revoked": "Sync Token Revoked" + } + } } }, "themesPage": { @@ -3835,6 +3871,7 @@ }, "providers": { "title": "Providers", + "Account Deactivated": "Account Deactivated", "allProviders": "All Providers", "audioProviders": "Audio Providers", "showFreeOnly": "Free only", @@ -3873,9 +3910,25 @@ "testAllFree": "Test all Free connections", "testAllApiKey": "Test all API Key connections", "testAllCompatible": "Test all Compatible connections", + "testAllModels": "Test all models", + "testAllModelsConfirm": "Test all visible models? Failed models will be hidden if auto-hide is enabled.", + "testingAllModels": "Testing {done}/{total}...", + "testAllResults": "{ok} of {total} models working", + "noModelsToTest": "No models match the current filter", + "showAllModels": "All", + "showVisibleOnly": "Visible only", + "showHiddenOnly": "Hidden only", + "filterByVisibility": "Filter by visibility", + "autoHideFailed": "Auto-hide failed models", "connected": "{count} Connected", "errorCount": "{count} Error ({code})", "errorCountNoCode": "{count} Error", + "testAllCompleted": "Tested {total} models: {ok} OK, {failed} failed", + "modelAutoHidden": "Model {model} auto-hidden (test failed)", + "filterAll": "All", + "filterVisible": "Visible only", + "filterHidden": "Hidden only", + "modelsHiddenCount": "{count} hidden", "warningCount": "{count} Warning", "noConnections": "No connections", "expiredBadge": "Expired", @@ -3982,6 +4035,8 @@ "batchActivateSuccess": "Activated {count} connection(s)", "batchDeactivateSuccess": "Deactivated {count} connection(s)", "failedSetAlias": "Failed to set alias", + "setAliasSuccess": "Alias {alias} set", + "deleteAliasSuccess": "Alias {alias} deleted", "failedSaveConnection": "Failed to save connection", "failedSaveConnectionRetry": "Failed to save connection. Please try again.", "failedRetestConnection": "Failed to retest connection", @@ -4015,6 +4070,8 @@ "connectionCount": "{count} connection(s)", "fetchingModels": "Fetching available models...", "failedFetchModels": "Failed to fetch models", + "fetchModelsSuccess": "Found {count} new models", + "fetchModelsFailed": "Could not auto-fetch models (you can retry from settings)", "noModelsFound": "No models found", "importFailed": "Import failed", "noNewModelsAdded": "No new models were added.", @@ -4023,6 +4080,10 @@ "importingModelsTitle": "Importing Models", "copyModel": "Copy model", "filterModels": "Filter models...", + "testAllRateLimited": "Rate limit reached, stopped early", + "hideFailedAuto": "Auto-hide failed models", + "selectAllModels": "Select all", + "hideAllModels": "Hide all", "modelsActive": "Active", "showModel": "Show model", "hideModel": "Hide model", @@ -4031,6 +4092,9 @@ "rateLimitUnprotected": "Unprotected", "enableRateLimitProtection": "Click to enable rate limit protection", "disableRateLimitProtection": "Click to disable rate limit protection", + "testAllProgress": "Tested {done}/{total}", + "testAllFailedHidden": "{count} failed models hidden", + "testAllDone": "All models tested", "productionKey": "Production Key", "enterNewApiKey": "Enter new API key", "codexApplyModalTitle": "Apply to Local Codex", @@ -4106,6 +4170,8 @@ "errorTypeNetworkError": "Network error", "errorTypeTestUnsupported": "Test unsupported", "errorTypeUpstreamError": "Upstream error", + "errorTypeCreditsExhausted": "No credits", + "errorTypeBanned": "403 Banned", "proxySourceGlobal": "Global", "proxySourceProvider": "Provider", "proxySourceKey": "Key", @@ -4190,7 +4256,6 @@ "email": "Email", "healthCheckMinutes": "Health Check (min)", "healthCheckHint": "Proactive token refresh interval. 0 = disabled.", - "selectAllModels": "Select all", "deselectAllModels": "Deselect all", "modelsActiveCount": "{active}/{total} active", "noModelsMatch": "No models match \"{filter}\"", @@ -4467,6 +4532,17 @@ "personalAccessTokenLabel": "Personal Access Token Label", "qoderPatHint": "Qoder Pat Hint", "qoderPatPlaceholder": "Qoder Pat Placeholder", + "rateLimitOverridesSection": "Rate Limit Overrides", + "rateLimitOverridesMaxConcurrentHint": "Max concurrent requests override for this connection. Overrides the account-level cap.", + "rateLimitOverridesMaxConcurrentLabel": "Max Concurrent (Rate Limit)", + "rateLimitOverridesMinTimeHint": "Minimum time (ms) between requests. Overrides the default rate limiter delay.", + "rateLimitOverridesMinTimeLabel": "Min Interval (ms)", + "rateLimitOverridesRpmHint": "Max requests per minute for this connection. Overrides the provider default.", + "rateLimitOverridesRpmLabel": "RPM (Requests/min)", + "rateLimitOverridesTpdHint": "Max tokens per day for this connection. Overrides the provider default.", + "rateLimitOverridesTpdLabel": "TPD (Tokens/day)", + "rateLimitOverridesTpmHint": "Max tokens per minute for this connection. Overrides the provider default.", + "rateLimitOverridesTpmLabel": "TPM (Tokens/min)", "refreshOauthTokenTitle": "Refresh Oauth Token Title", "regionHint": "Region Hint", "regionLabel": "Region Label", @@ -4931,7 +5007,7 @@ "proxyPool": "Proxy Pool", "freePool": "Free Pool", "proxyDocumentation": "Proxy Documentation", - "proxyGlobalConfigTab": "Global", + "proxyGlobalConfigTab": "Global Config", "proxyPoolTab": "Proxy Pool", "freePoolTab": "Free Pool", "proxyDocumentationTab": "Documentation", @@ -5792,11 +5868,7 @@ "vercelRelayProjectNameLabel": "Vercel Project Name", "vercelRelayFreeTierNote": "Relays are lightweight proxy endpoints deployed on Vercel's free tier to bypass local network/region limitations.", "vercelRelayDeploying": "Deploying...", - "vercelRelayDeploy": "Deploy", - "proxyGlobalConfigTab": "Global Config", - "proxyPoolTab": "Proxy Pool", - "freePoolTab": "Free Pool", - "proxyDocumentationTab": "Documentation" + "vercelRelayDeploy": "Deploy" }, "contextRtk": { "title": "RTK Engine", @@ -7908,7 +7980,7 @@ "title": "Quota Sharing", "description": "Share provider quotas across API keys with % limits", "newPool": "New pool", - "betaTitle": "Beta — UI preview.", + "betaTitle": "Beta", "betaDescription": "Configuration is saved in localStorage (not yet persisted on the server). Per-request cap enforcement is not yet wired into the proxy pipeline. This screen lets you design and visualize the quota split; real enforcement will land in a future iteration with DB persistence and upstream call interception.", "kpiActivePools": "Active pools", "kpiKeysAllocated": "Keys allocated", @@ -8030,7 +8102,6 @@ "endpointsAnthropicNote": "Anthropic-native", "endpointsResponsesNote": "OpenAI Responses — codex/github", "endpointsWsNote": "WebSocket — codex only", - "betaTitle": "Beta", "betaText": "Quota Share is functional but bugs are expected. Found one? Please report it.", "betaReportLink": "Report an issue", "deleteGroup": "Delete group", diff --git a/src/lib/api/modelTestRunner.ts b/src/lib/api/modelTestRunner.ts new file mode 100644 index 0000000000..7c7ba5a093 --- /dev/null +++ b/src/lib/api/modelTestRunner.ts @@ -0,0 +1,335 @@ +import { randomUUID } from "node:crypto"; +import { POST as postChatCompletion } from "@/app/api/v1/chat/completions/route"; +import { handleValidatedEmbeddingRequestBody } from "@/app/api/v1/embeddings/route"; +import { POST as postRerank } from "@/app/api/v1/rerank/route"; +import { buildComboTestRequestBody, extractComboTestResponseText } from "@/lib/combos/testHealth"; +import { getCustomModels } from "@/lib/localDb"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/error"; +import { withRateLimit } from "@omniroute/open-sse/services/rateLimitManager"; + +const INTERNAL_ORIGIN = "http://omniroute.internal"; +const DEFAULT_TEST_TIMEOUT_MS = 10_000; + +function asRecord(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} + +function getErrorMessage(error: unknown): string { + return sanitizeErrorMessage(error) || "Unknown error"; +} + +function getErrorName(error: unknown): string { + return error instanceof Error ? error.name : ""; +} + +function extractProviderErrorMessage(body: unknown, fallback: string) { + const record = asRecord(body); + const error = record.error; + if (typeof error === "string" && error.trim()) return error; + + const errorRecord = asRecord(error); + const message = errorRecord.message; + return typeof message === "string" && message.trim() ? message : fallback; +} + +function stripFirstSegment(modelId: string): string | null { + const slashIdx = modelId.indexOf("/"); + return slashIdx > 0 ? modelId.slice(slashIdx + 1) : null; +} + +async function findCustomModelMetadata(providerId: string, modelId: string) { + try { + const customModels = await getCustomModels(providerId); + if (!Array.isArray(customModels)) return null; + + const candidates = new Set([modelId]); + const stripped = stripFirstSegment(modelId); + if (stripped) candidates.add(stripped); + if (modelId.startsWith(`${providerId}/`)) candidates.add(modelId.slice(providerId.length + 1)); + + return ( + customModels.find( + (model: any) => typeof model?.id === "string" && candidates.has(model.id) + ) || null + ); + } catch { + return null; + } +} + +function buildInternalChatRequest(testBody: Record, signal: AbortSignal) { + return new Request(`${INTERNAL_ORIGIN}/v1/chat/completions`, { + method: "POST", + headers: { + "Content-Type": "application/json", + // Reuse the existing strict-mode internal bypass for live health checks. + "X-Internal-Test": "combo-health-check", + "X-OmniRoute-No-Cache": "true", + "X-Request-Id": `model-test-${randomUUID()}`, + }, + body: JSON.stringify(testBody), + signal, + }); +} + +function buildInternalRerankRequest(testBody: Record, signal: AbortSignal) { + return new Request(`${INTERNAL_ORIGIN}/v1/rerank`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Internal-Test": "combo-health-check", + "X-OmniRoute-No-Cache": "true", + "X-Request-Id": `model-test-${randomUUID()}`, + }, + body: JSON.stringify(testBody), + signal, + }); +} + +export function detectTestKind(modelStr: string, customModel: any) { + const supportedEndpoints = Array.isArray(customModel?.supportedEndpoints) + ? customModel.supportedEndpoints + : []; + const apiFormat = typeof customModel?.apiFormat === "string" ? customModel.apiFormat : ""; + const lowerModel = modelStr.toLowerCase(); + const isRerank = + apiFormat === "rerank" || + supportedEndpoints.includes("rerank") || + lowerModel.includes("rerank"); + const isEmbedding = + !isRerank && + (apiFormat === "embeddings" || + supportedEndpoints.includes("embeddings") || + lowerModel.includes("embedding") || + lowerModel.includes("bge-") || + lowerModel.includes("text-embed") || + lowerModel.includes("jina-clip") || + lowerModel.includes("colbert")); + return { isRerank, isEmbedding }; +} + +/** + * Parse a Retry-After header value (seconds-as-number or HTTP-date) into seconds. + * Returns undefined if the value is missing or unparseable. + */ +export function parseRetryAfterHeader(value: string | null | undefined): number | undefined { + if (typeof value !== "string") return undefined; + const trimmed = value.trim(); + if (!trimmed) return undefined; + + const num = Number(trimmed); + if (Number.isFinite(num) && num >= 0) { + return Math.ceil(num); + } + + const ms = Date.parse(trimmed); + if (Number.isFinite(ms)) { + return Math.max(0, Math.ceil((ms - Date.now()) / 1000)); + } + + return undefined; +} + +export interface RunSingleModelTestOptions { + providerId: string; + modelId: string; + connectionId?: string; + timeoutMs?: number; +} + +export interface SingleModelTestResult { + modelId: string; + status: "ok" | "error" | "rate_limited"; + latencyMs: number; + responseText?: string; + statusCode?: number; + httpStatus: number; + error?: string; + rateLimited?: boolean; + isTimeout?: boolean; + retryAfter?: number; +} + +/** + * Run a single model test. When `connectionId` is provided, wraps the + * upstream call with `withRateLimit` (Bottleneck). Returns a plain + * `SingleModelTestResult` (not an HTTP Response) so the single-test and + * batch-test endpoints can format it differently. + */ +export async function runSingleModelTest( + options: RunSingleModelTestOptions +): Promise { + const { providerId, modelId, connectionId, timeoutMs = DEFAULT_TEST_TIMEOUT_MS } = options; + + let fullModelStr = modelId; + if (!fullModelStr.includes("/")) { + fullModelStr = `${providerId}/${modelId}`; + } + + const startTime = Date.now(); + const customModel = await findCustomModelMetadata(providerId, fullModelStr); + const { isRerank, isEmbedding } = detectTestKind(fullModelStr, customModel); + + const testBody = isRerank + ? { + model: fullModelStr, + query: "What is OmniRoute?", + documents: [ + "OmniRoute routes AI requests across configured providers.", + "This document is unrelated to the test query.", + ], + top_n: 1, + return_documents: false, + } + : buildComboTestRequestBody(fullModelStr, isEmbedding); + + // Per-model AbortController. We track whether the timeout fired so we can + // distinguish "rate-limit queue aborted" (withRateLimit threw AbortError + // with no timeout) from "timeout fired and aborted withRateLimit". + const controller = new AbortController(); + let timedOut = false; + const timeoutHandle = setTimeout(() => { + timedOut = true; + controller.abort(); + }, timeoutMs); + + const runInner = async (signal: AbortSignal): Promise => { + if (isEmbedding) { + return handleValidatedEmbeddingRequestBody( + testBody as Record & { model: string } + ); + } + if (isRerank) { + return postRerank(buildInternalRerankRequest(testBody, signal)); + } + return postChatCompletion(buildInternalChatRequest(testBody, signal)); + }; + + let res: Response; + try { + if (connectionId) { + res = await withRateLimit( + providerId, + connectionId, + fullModelStr, + (signal) => runInner(signal), + controller.signal + ); + } else { + res = await runInner(controller.signal); + } + } catch (error: unknown) { + clearTimeout(timeoutHandle); + const latencyMs = Date.now() - startTime; + const errorName = getErrorName(error); + if (errorName === "AbortError") { + if (timedOut) { + return { + modelId: fullModelStr, + status: "error", + latencyMs, + httpStatus: 500, + error: `Timeout (${Math.round(timeoutMs / 1000)}s)`, + isTimeout: true, + }; + } + // AbortError without timeout = withRateLimit queue rejection / abort. + // Surface as rate_limited so the batch endpoint can stop the loop. + return { + modelId: fullModelStr, + status: "rate_limited", + latencyMs, + httpStatus: 429, + error: "Rate limited (queue aborted)", + rateLimited: true, + }; + } + return { + modelId: fullModelStr, + status: "error", + latencyMs, + httpStatus: 500, + error: getErrorMessage(error), + }; + } + clearTimeout(timeoutHandle); + + const latencyMs = Date.now() - startTime; + + if (res.status === 429) { + const retryAfter = parseRetryAfterHeader(res.headers.get("retry-after")); + + let errorMsg = "Rate limited"; + try { + const errBody = await res.json(); + errorMsg = extractProviderErrorMessage(errBody, res.statusText || errorMsg); + } catch { + errorMsg = res.statusText || errorMsg; + } + return { + modelId: fullModelStr, + status: "rate_limited", + latencyMs, + statusCode: res.status, + httpStatus: res.status, + error: errorMsg, + rateLimited: true, + ...(retryAfter !== undefined ? { retryAfter } : {}), + }; + } + + if (res.ok) { + let responseBody = null; + try { + responseBody = await res.json(); + } catch { + responseBody = null; + } + + const responseText = extractComboTestResponseText(responseBody); + if (isRerank) { + return { + modelId: fullModelStr, + status: "ok", + latencyMs, + httpStatus: 200, + responseText: "[Rerank completed successfully]", + }; + } + if (!responseText && !isEmbedding) { + return { + modelId: fullModelStr, + status: "error", + latencyMs, + statusCode: res.status, + httpStatus: 400, + error: "Provider returned HTTP 200 but no text content.", + }; + } + return { + modelId: fullModelStr, + status: "ok", + latencyMs, + httpStatus: 200, + responseText, + }; + } + + let errorMsg = ""; + try { + const errBody = await res.json(); + errorMsg = extractProviderErrorMessage(errBody, res.statusText); + } catch { + errorMsg = res.statusText; + } + return { + modelId: fullModelStr, + status: "error", + latencyMs, + statusCode: res.status, + httpStatus: res.status, + error: errorMsg, + }; +} diff --git a/src/lib/db/core.ts b/src/lib/db/core.ts index 02b9982f27..a365c6ff74 100644 --- a/src/lib/db/core.ts +++ b/src/lib/db/core.ts @@ -204,6 +204,7 @@ const SCHEMA_SQL = ` "group" TEXT, max_concurrent INTEGER, quota_window_thresholds_json TEXT, + rate_limit_overrides_json TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL ); @@ -540,6 +541,10 @@ function ensureProviderConnectionsColumns(db: SqliteDatabase) { db.exec("ALTER TABLE provider_connections ADD COLUMN quota_window_thresholds_json TEXT"); console.log("[DB] Added provider_connections.quota_window_thresholds_json column"); } + if (!columnNames.has("rate_limit_overrides_json")) { + db.exec("ALTER TABLE provider_connections ADD COLUMN rate_limit_overrides_json TEXT"); + console.log("[DB] Added provider_connections.rate_limit_overrides_json column"); + } db.exec( "CREATE INDEX IF NOT EXISTS idx_pc_max_concurrent ON provider_connections(provider, max_concurrent)" ); diff --git a/src/lib/db/models.ts b/src/lib/db/models.ts index 7fb5214c67..07a3a58886 100644 --- a/src/lib/db/models.ts +++ b/src/lib/db/models.ts @@ -1034,6 +1034,44 @@ export function getModelIsHidden(providerId: string, modelId: string): boolean { return Boolean(co?.isHidden); } +/** + * Persist the hidden flag for a model. Stores the override on the custom-model + * row when one exists, otherwise on the compat-override list. Setting + * `hidden = false` is a no-op when the model is already visible. + */ +export function setModelIsHidden(providerId: string, modelId: string, hidden: boolean): void { + const customRow = getCustomModelRow(providerId, modelId); + if (customRow) { + if (hidden) { + updateCustomModel(providerId, modelId, { isHidden: true }); + } else if (Object.prototype.hasOwnProperty.call(customRow, "isHidden")) { + updateCustomModel(providerId, modelId, { isHidden: false }); + } + return; + } + + const list = readCompatList(providerId); + const idx = list.findIndex((e) => e.id === modelId); + if (hidden) { + const prev = idx >= 0 ? list[idx] : { id: modelId }; + const next: ModelCompatOverride = { ...prev, id: modelId, isHidden: true }; + if (idx >= 0) list[idx] = next; + else list.push(next); + writeCompatList(providerId, list); + return; + } + + if (idx < 0) return; + if (Object.keys(list[idx]).length <= 1) { + // Only `id` left; drop the entry entirely. + const filtered = list.filter((_, i) => i !== idx); + writeCompatList(providerId, filtered); + return; + } + delete list[idx].isHidden; + writeCompatList(providerId, list); +} + function readUpstreamFromJsonRecord( row: JsonRecord | null | undefined, key: "upstreamHeaders" diff --git a/src/lib/db/providers.ts b/src/lib/db/providers.ts index 28f399db55..d593e492be 100644 --- a/src/lib/db/providers.ts +++ b/src/lib/db/providers.ts @@ -61,6 +61,41 @@ function withNullableQuotaWindowThresholds( }; } +// Always surface `rateLimitOverrides` (possibly null) — matches the pattern +// used by withNullableMaxConcurrent and withNullableQuotaWindowThresholds. +function withNullableRateLimitOverrides( + record: JsonRecord, + source: JsonRecord | null | undefined +): JsonRecord { + return { + ...record, + rateLimitOverrides: (source?.rateLimitOverrides ?? null) as Record | null, + }; +} + +// Sanitize the per-connection rate limit overrides map: keep only known +// fields with valid numeric values. Called once at each write-path boundary. +function sanitizeRateLimitOverrides(value: unknown): Record | null { + if (value === null || value === undefined) return null; + if (typeof value !== "object" || Array.isArray(value)) return null; + const allowedKeys = new Set(["rpm", "tpm", "tpd", "minTime", "maxConcurrent"]); + const map: Record = {}; + for (const [key, v] of Object.entries(value as Record)) { + if (!allowedKeys.has(key)) continue; + if (typeof v === "number" && Number.isInteger(v) && v >= 0) { + map[key] = v; + } + } + return Object.keys(map).length === 0 ? null : map; +} + +// Serialize an already-sanitized map for SQLite TEXT storage. +function serializeRateLimitOverrides(value: unknown): string | null { + if (value === null || value === undefined) return null; + if (typeof value !== "object" || Array.isArray(value)) return null; + return JSON.stringify(value); +} + function toRecord(value: unknown): JsonRecord { return value && typeof value === "object" ? (value as JsonRecord) : {}; } @@ -124,8 +159,11 @@ export async function getProviderConnections(filter: JsonRecord = {}) { return rows.map((r) => { const camelRow = rowToCamel(r); return decryptConnectionFields( - withNullableQuotaWindowThresholds( - withNullableMaxConcurrent(cleanNulls(camelRow), camelRow), + withNullableRateLimitOverrides( + withNullableQuotaWindowThresholds( + withNullableMaxConcurrent(cleanNulls(camelRow), camelRow), + camelRow + ), camelRow ) ); @@ -139,8 +177,11 @@ export async function getProviderConnectionById(id: string) { const camelRow = rowToCamel(row); return decryptConnectionFields( - withNullableQuotaWindowThresholds( - withNullableMaxConcurrent(cleanNulls(camelRow), camelRow), + withNullableRateLimitOverrides( + withNullableQuotaWindowThresholds( + withNullableMaxConcurrent(cleanNulls(camelRow), camelRow), + camelRow + ), camelRow ) ); @@ -235,8 +276,11 @@ export async function createProviderConnection(data: JsonRecord) { ); _updateConnectionRow(db, existingId, merged); backupDbFile("pre-write"); - return withNullableQuotaWindowThresholds( - withNullableMaxConcurrent(cleanNulls(merged), merged), + return withNullableRateLimitOverrides( + withNullableQuotaWindowThresholds( + withNullableMaxConcurrent(cleanNulls(merged), merged), + merged + ), merged ); } @@ -304,6 +348,7 @@ export async function createProviderConnection(data: JsonRecord) { "proxyEnabled", "perKeyProxyEnabled", "quotaWindowThresholds", + "rateLimitOverrides", ]; for (const field of optionalFields) { if (data[field] !== undefined && data[field] !== null) { @@ -324,6 +369,14 @@ export async function createProviderConnection(data: JsonRecord) { ); } + // Same sanitization for rateLimitOverrides — keep in-memory representation + // in sync with what gets persisted. + if ("rateLimitOverrides" in connection) { + connection.rateLimitOverrides = sanitizeRateLimitOverrides( + connection.rateLimitOverrides + ); + } + _insertConnectionRow(db, encryptConnectionFields({ ...connection })); const providerId = toStringOrNull(data.provider); if (providerId) { @@ -332,8 +385,11 @@ export async function createProviderConnection(data: JsonRecord) { backupDbFile("pre-write"); invalidateDbCache("connections"); // Bust connections read cache - return withNullableQuotaWindowThresholds( - withNullableMaxConcurrent(cleanNulls(connection), connection), + return withNullableRateLimitOverrides( + withNullableQuotaWindowThresholds( + withNullableMaxConcurrent(cleanNulls(connection), connection), + connection + ), connection ); } @@ -350,8 +406,7 @@ function _insertConnectionRow(db: DbLike, conn: JsonRecord) { last_tested, api_key, id_token, provider_specific_data, expires_in, display_name, global_priority, default_model, token_type, consecutive_use_count, rate_limit_protection, last_used_at, "group", max_concurrent, - proxy_enabled, per_key_proxy_enabled, - quota_window_thresholds_json, + proxy_enabled, per_key_proxy_enabled, quota_window_thresholds_json, rate_limit_overrides_json, created_at, updated_at ) VALUES ( @id, @provider, @authType, @name, @email, @priority, @isActive, @@ -362,8 +417,7 @@ function _insertConnectionRow(db: DbLike, conn: JsonRecord) { @lastTested, @apiKey, @idToken, @providerSpecificData, @expiresIn, @displayName, @globalPriority, @defaultModel, @tokenType, @consecutiveUseCount, @rateLimitProtection, @lastUsedAt, @group, @maxConcurrent, - @proxyEnabled, @perKeyProxyEnabled, - @quotaWindowThresholdsJson, + @proxyEnabled, @perKeyProxyEnabled, @quotaWindowThresholdsJson, @rateLimitOverridesJson, @createdAt, @updatedAt ) ` @@ -411,6 +465,7 @@ function _insertConnectionRow(db: DbLike, conn: JsonRecord) { proxyEnabled: conn.proxyEnabled ?? 1, perKeyProxyEnabled: conn.perKeyProxyEnabled ?? 0, quotaWindowThresholdsJson: serializeQuotaWindowThresholds(conn.quotaWindowThresholds), + rateLimitOverridesJson: serializeRateLimitOverrides(conn.rateLimitOverrides), createdAt: conn.createdAt, updatedAt: conn.updatedAt, }); @@ -440,6 +495,7 @@ function _updateConnectionRow(db: DbLike, id: string, data: JsonRecord) { quota_window_thresholds_json = @quotaWindowThresholdsJson, proxy_enabled = @proxyEnabled, per_key_proxy_enabled = @perKeyProxyEnabled, + rate_limit_overrides_json = @rateLimitOverridesJson, updated_at = @updatedAt WHERE id = @id ` @@ -497,6 +553,7 @@ function _updateConnectionRow(db: DbLike, id: string, data: JsonRecord) { ? 1 : 0 : (data.perKeyProxyEnabled ?? 0), + rateLimitOverridesJson: serializeRateLimitOverrides(data.rateLimitOverrides), updatedAt: now, }); } @@ -523,6 +580,9 @@ export async function updateProviderConnection(id: string, data: JsonRecord) { // path surfaces the cleared state to callers that just patched it. merged.quotaWindowThresholds = sanitized; } + if ("rateLimitOverrides" in merged) { + merged.rateLimitOverrides = sanitizeRateLimitOverrides(merged.rateLimitOverrides); + } _updateConnectionRow(db, id, encryptConnectionFields({ ...merged })); backupDbFile("pre-write"); invalidateDbCache("connections"); // Bust connections read cache @@ -536,8 +596,11 @@ export async function updateProviderConnection(id: string, data: JsonRecord) { _reorderConnections(db, providerId); } - return withNullableQuotaWindowThresholds( - withNullableMaxConcurrent(cleanNulls(merged), merged), + return withNullableRateLimitOverrides( + withNullableQuotaWindowThresholds( + withNullableMaxConcurrent(cleanNulls(merged), merged), + merged + ), merged ); } diff --git a/src/lib/localDb.ts b/src/lib/localDb.ts index 6a8f9968e9..ed9c65f34b 100755 --- a/src/lib/localDb.ts +++ b/src/lib/localDb.ts @@ -59,6 +59,7 @@ export { getModelPreserveOpenAIDeveloperRole, getModelUpstreamExtraHeaders, getModelIsHidden, + setModelIsHidden, // Synced Available Models getSyncedAvailableModels, diff --git a/src/shared/validation/schemas.ts b/src/shared/validation/schemas.ts index 0a3bd1793f..c9e2b0609a 100644 --- a/src/shared/validation/schemas.ts +++ b/src/shared/validation/schemas.ts @@ -2012,6 +2012,20 @@ export const updateProviderConnectionSchema = z ]) .optional(), projectId: z.union([z.string(), z.null()]).optional(), + // Per-connection rate limit overrides — overrides the global RequestQueueSettings + // for this connection. Set to null to clear all overrides. + rateLimitOverrides: z + .union([ + z.null(), + z.object({ + rpm: z.coerce.number().int().min(0).max(1_000_000).optional(), + tpm: z.coerce.number().int().min(0).max(100_000_000).optional(), + tpd: z.coerce.number().int().min(0).max(10_000_000_000).optional(), + minTime: z.coerce.number().int().min(0).max(60_000).optional(), + maxConcurrent: z.coerce.number().int().min(0).max(10_000).optional(), + }), + ]) + .optional(), proxyEnabled: z.boolean().optional(), perKeyProxyEnabled: z.boolean().optional(), // Partial patch of per-connection provider-specific settings (e.g. quota toggles) diff --git a/tests/unit/model-test-runner.test.ts b/tests/unit/model-test-runner.test.ts new file mode 100644 index 0000000000..dbdced176b --- /dev/null +++ b/tests/unit/model-test-runner.test.ts @@ -0,0 +1,88 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { parseRetryAfterHeader, detectTestKind } from "@/lib/api/modelTestRunner.ts"; + +// --------------------------------------------------------------------------- +// parseRetryAfterHeader — Retry-After is either delta-seconds or an HTTP-date. +// Regression guard for the rate-limit handling in runSingleModelTest (#3267). +// --------------------------------------------------------------------------- + +test("parseRetryAfterHeader returns undefined for missing/empty/null input", () => { + assert.equal(parseRetryAfterHeader(null), undefined); + assert.equal(parseRetryAfterHeader(undefined), undefined); + assert.equal(parseRetryAfterHeader(""), undefined); + assert.equal(parseRetryAfterHeader(" "), undefined); +}); + +test("parseRetryAfterHeader parses delta-seconds (numeric form)", () => { + assert.equal(parseRetryAfterHeader("0"), 0); + assert.equal(parseRetryAfterHeader("30"), 30); + assert.equal(parseRetryAfterHeader("120"), 120); + // fractional seconds round up (ceil) so we never under-wait + assert.equal(parseRetryAfterHeader("1.2"), 2); +}); + +test("parseRetryAfterHeader rejects non-date garbage and never yields a misleading positive wait", () => { + // Pure garbage with no parseable date → undefined. + assert.equal(parseRetryAfterHeader("soon"), undefined); + assert.equal(parseRetryAfterHeader("NaN"), undefined); + // A negative numeric is not accepted on the numeric path (>= 0 guard); it + // falls through to Date.parse, which yields a past date → clamped to 0. + // The important guarantee is that it never produces a positive wait. + const negative = parseRetryAfterHeader("-5"); + assert.ok(negative === undefined || negative === 0, `expected 0/undefined, got ${negative}`); +}); + +test("parseRetryAfterHeader parses an HTTP-date into a non-negative seconds delta", () => { + // A date ~10s in the future should yield a small positive integer (>=0). + const future = new Date(Date.now() + 10_000).toUTCString(); + const secs = parseRetryAfterHeader(future); + assert.ok(typeof secs === "number"); + assert.ok(secs >= 0 && secs <= 11, `expected ~10s, got ${secs}`); + + // A date in the past clamps to 0 (never negative). + const past = new Date(Date.now() - 60_000).toUTCString(); + assert.equal(parseRetryAfterHeader(past), 0); +}); + +// --------------------------------------------------------------------------- +// detectTestKind — picks the right test endpoint (chat / embeddings / rerank) +// from the model id + custom-model metadata. Rerank must win over embedding. +// --------------------------------------------------------------------------- + +test("detectTestKind defaults to a plain chat test for ordinary models", () => { + assert.deepEqual(detectTestKind("openai/gpt-4o", null), { + isRerank: false, + isEmbedding: false, + }); +}); + +test("detectTestKind detects embeddings by id heuristics", () => { + for (const id of [ + "openai/text-embedding-3-small", + "jina/jina-embeddings-v3", + "baai/bge-m3", + "jinaai/jina-clip-v2", + "colbert-ir/colbertv2", + ]) { + assert.equal(detectTestKind(id, null).isEmbedding, true, `${id} should be embedding`); + assert.equal(detectTestKind(id, null).isRerank, false, `${id} should not be rerank`); + } +}); + +test("detectTestKind detects rerank by id and by metadata, and rerank wins over embedding", () => { + assert.deepEqual(detectTestKind("jina/jina-reranker-v2", null), { + isRerank: true, + isEmbedding: false, + }); + // apiFormat metadata drives detection even when the id is opaque + assert.equal(detectTestKind("vendor/opaque-model", { apiFormat: "rerank" }).isRerank, true); + assert.equal( + detectTestKind("vendor/opaque-model", { supportedEndpoints: ["embeddings"] }).isEmbedding, + true + ); + // A model that looks like both rerank and embedding resolves to rerank only. + const both = detectTestKind("vendor/rerank-embedding-hybrid", null); + assert.equal(both.isRerank, true); + assert.equal(both.isEmbedding, false); +}); diff --git a/tests/unit/provider-rate-limit-overrides-schema.test.ts b/tests/unit/provider-rate-limit-overrides-schema.test.ts new file mode 100644 index 0000000000..84bca5b7f9 --- /dev/null +++ b/tests/unit/provider-rate-limit-overrides-schema.test.ts @@ -0,0 +1,73 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { updateProviderConnectionSchema } from "../../src/shared/validation/schemas"; + +function parse(overrides: unknown) { + return updateProviderConnectionSchema.safeParse({ + name: "test-provider", + rateLimitOverrides: overrides, + }); +} + +test("rateLimitOverrides: valid object with all fields", () => { + const r = parse({ rpm: 100, tpm: 50000, tpd: 1000000, minTime: 100, maxConcurrent: 5 }); + assert.ok(r.success, String(r.error)); + assert.deepEqual(r.data.rateLimitOverrides, { rpm: 100, tpm: 50000, tpd: 1000000, minTime: 100, maxConcurrent: 5 }); +}); + +test("rateLimitOverrides: partial fields", () => { + const r = parse({ rpm: 60 }); + assert.ok(r.success, String(r.error)); + assert.deepEqual(r.data.rateLimitOverrides, { rpm: 60 }); +}); + +test("rateLimitOverrides: null clears overrides", () => { + const r = parse(null); + assert.ok(r.success, String(r.error)); + assert.equal(r.data.rateLimitOverrides, null); +}); + +test("rateLimitOverrides: undefined is valid", () => { + const r = updateProviderConnectionSchema.safeParse({ name: "test-provider" }); + assert.ok(r.success, String(r.error)); + assert.equal(r.data.rateLimitOverrides, undefined); +}); + +test("rateLimitOverrides: string coerced to number", () => { + const r = parse({ rpm: "120" }); + assert.ok(r.success, String(r.error)); + assert.equal(r.data.rateLimitOverrides.rpm, 120); +}); + +test("rateLimitOverrides: rejects negative rpm", () => { + assert.equal(parse({ rpm: -1 }).success, false); +}); + +test("rateLimitOverrides: rejects float rpm", () => { + assert.equal(parse({ rpm: 1.5 }).success, false); +}); + +test("rateLimitOverrides: rejects rpm above max", () => { + assert.equal(parse({ rpm: 1_000_001 }).success, false); +}); + +test("rateLimitOverrides: rejects tpm above max", () => { + assert.equal(parse({ tpm: 100_000_001 }).success, false); +}); + +test("rateLimitOverrides: rejects non-object", () => { + assert.equal(parse("bad").success, false); + assert.equal(parse(42).success, false); + assert.equal(parse([1]).success, false); +}); + +test("rateLimitOverrides: empty object is valid", () => { + const r = parse({}); + assert.ok(r.success, String(r.error)); +}); + +test("rateLimitOverrides: all zeros is valid", () => { + const r = parse({ rpm: 0, tpm: 0, tpd: 0, minTime: 0, maxConcurrent: 0 }); + assert.ok(r.success, String(r.error)); +});