diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c854cbdfb..824fbd7368 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ _Living section — bullets land here as PRs merge into `release/v3.8.47` (paral ### 🐛 Bug Fixes +- **fix(providers):** custom models a provider actually has are no longer dropped from the Free Provider Rankings when both **"Configured only"** and **"Available only"** filters are applied ([#6368](https://github.com/diegosouzapw/OmniRoute/issues/6368)), follow-up to #6150 — `freeProviderRankings.ts::getProviderModels()` only ever walked the static `open-sse/config/providerRegistry.ts` catalog, so a user-added custom model (e.g. a Puter `claude-fable-5` model saved as "Claude Fable 5") never entered the candidate model list the ranking scores against, and could never survive the #6150 configured/available filters even when actually configured and available. It now additively merges the provider's custom models (`db/models.ts::getCustomModels`) into that candidate list via a new pure, de-duping `mergeProviderModels()` helper, before scoring/filtering runs — catalog free/paid filtering elsewhere is untouched. Regression guard: `tests/unit/free-provider-rankings-custom-models-6368.test.ts`. (thanks @shabeer) - **fix(providers):** `cloudflare-ai` no longer silently drops image/non-text content parts ([#6390](https://github.com/diegosouzapw/OmniRoute/issues/6390)) — `transformRequest()`'s `flattenContent()` (added for #2539 to satisfy the Workers AI `/ai/v1/chat/completions` plain-string `content` requirement) mapped any non-text OpenAI content part (e.g. `image_url`) to `""` and joined the rest, so a request carrying an image quietly went out as text-only with the attachment gone and no error surfaced. It now throws a clear error on the first non-text part instead of dropping it silently, which the existing top-level `chatCore.ts` catch already routes through `buildErrorBody()`/`sanitizeErrorMessage()` (same pattern as `buildUrl()`'s missing-Account-ID error). Regression guard: `tests/unit/cloudflare-ai-image-parts-6390.test.ts`. - **fix(providers):** stop Antigravity connections from falsely reporting **all-accounts quota-exhausted** ([#6295](https://github.com/diegosouzapw/OmniRoute/issues/6295)) — `genericQuotaFetcher.ts::percentUsedForQuota()` ignored the `fractionReported` flag and defaulted an unreported model's `remainingPercentage` to 0, which computed as 100% used; since `convertUsageToQuotaInfo()` takes the worst-case window across a connection, a single model with no reported fraction dragged the whole account into `limitReached` and `quotaPreflight` skipped it. `percentUsedForQuota()` now returns `null` (unknown, window ignored) whenever `fractionReported === false`, before falling back to `remainingPercentage`. Regression guard: `tests/unit/generic-quota-fetcher.test.ts`. - **fix(providers):** grok-cli (Grok Build) now strips `reasoning_effort`/`reasoning` before forwarding the request ([#6288](https://github.com/diegosouzapw/OmniRoute/issues/6288)) — Claude Code sends `reasoning_effort` on every request (routing the Opus slot), which Grok Build's upstream chat-proxy endpoint rejects with a 400; `transformRequest()`'s existing `UNSUPPORTED` sampling-param strip list (#5273) never covered it. Regression guard: `tests/unit/grok-cli-reasoning-strip-6288.test.ts`. diff --git a/src/lib/freeProviderRankings.ts b/src/lib/freeProviderRankings.ts index dfa58fab82..c5055332b3 100644 --- a/src/lib/freeProviderRankings.ts +++ b/src/lib/freeProviderRankings.ts @@ -12,6 +12,7 @@ import { NOAUTH_PROVIDERS, OAUTH_PROVIDERS, APIKEY_PROVIDERS } from "@/shared/co import { REGISTRY } from "@omniroute/open-sse/config/providerRegistry"; import { listModelIntelligence } from "./db/modelIntelligence"; import { getProviderConnections } from "./db/providers"; +import { getCustomModels } from "./db/models"; export interface ProviderModelScore { modelId: string; @@ -90,12 +91,51 @@ function getFreeProviders() { return providers; } +/** Minimal shape shared by registry models and user-added custom models. */ +export interface RankableModel { + id: string; + name: string; +} + /** - * Get models for a provider from the registry. + * Pure merge: combine a provider's static registry models with its + * user-added custom models, de-duplicating by `id` (registry entry wins on + * collision — a custom model overriding a known catalog ID keeps the + * catalog's richer metadata upstream, only the extra IDs are additive here). + * + * Exported so the #6368 fix ("custom models missing from Free Provider + * Rankings under configured/available filters") can be unit-tested without a + * DB: the ranking builder no longer only walks the static registry — it also + * folds in whatever the operator configured as a custom model for that + * provider, mirroring how #6150's connection-based filters already treat + * "configured" as DB/runtime state rather than catalog membership. */ -function getProviderModels(providerId: string) { +export function mergeProviderModels( + registryModels: RankableModel[], + customModels: RankableModel[] +): RankableModel[] { + if (customModels.length === 0) return registryModels; + const seen = new Set(registryModels.map((m) => m.id)); + const merged = registryModels.slice(); + for (const custom of customModels) { + if (!custom?.id || seen.has(custom.id)) continue; + seen.add(custom.id); + merged.push({ id: custom.id, name: custom.name || custom.id }); + } + return merged; +} + +/** + * Get models for a provider: static registry models plus any user-added + * custom models for that provider (#6368 — custom models were previously + * invisible to the ranking builder, so they never survived the + * configured/available filters even when actually configured+available). + */ +async function getProviderModels(providerId: string): Promise { const entry = REGISTRY[providerId]; - return entry?.models ?? []; + const registryModels = entry?.models ?? []; + const customModels = (await getCustomModels(providerId)) as RankableModel[]; + return mergeProviderModels(registryModels, Array.isArray(customModels) ? customModels : []); } /** @@ -282,7 +322,7 @@ export async function computeFreeProviderRankings( const rankings: FreeProviderRanking[] = []; for (const provider of freeProviders) { - const models = getProviderModels(provider.id); + const models = await getProviderModels(provider.id); if (models.length === 0) continue; const modelScores: ProviderModelScore[] = []; diff --git a/tests/unit/free-provider-rankings-custom-models-6368.test.ts b/tests/unit/free-provider-rankings-custom-models-6368.test.ts new file mode 100644 index 0000000000..95b53064cb --- /dev/null +++ b/tests/unit/free-provider-rankings-custom-models-6368.test.ts @@ -0,0 +1,116 @@ +/** + * Regression test for #6368 (follow-up to #6150). + * + * Custom models a user adds to a provider (e.g. "Claude Fable 5" added on + * top of the Puter provider) were invisible in the Free Provider Rankings + * once the "Configured only" / "Available only" filters were applied, + * because `getProviderModels()` only ever walked the static + * `open-sse/config/providerRegistry.ts` catalog — a provider's user-added + * custom models were never folded into the candidate model list that gets + * matched against intelligence scores, so they could never appear in the + * ranking regardless of the filters. + * + * This test proves: + * 1. `mergeProviderModels` (pure) additively includes custom models and + * de-dupes against the registry list. + * 2. `computeFreeProviderRankings()` end-to-end surfaces a provider whose + * *only* matching model is a user-added custom model, under BOTH + * `configuredOnly` and `availableOnly` filters. + * 3. Existing catalog-model based free/paid... i.e. configured/available + * filtering still behaves as before (#6150 regression guard) — a + * provider with no connection is still dropped under `configuredOnly`. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-rankings-6368-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const modelsDb = await import("../../src/lib/db/models.ts"); +const intelligenceDb = await import("../../src/lib/db/modelIntelligence.ts"); +const rankings = await import("../../src/lib/freeProviderRankings.ts"); + +const CUSTOM_MODEL_ID = "claude-fable-5-6368"; + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("mergeProviderModels: additively includes custom models, de-duping by id", () => { + const registryModels = [{ id: "known-model", name: "Known Model" }]; + const customModels = [ + { id: "known-model", name: "Should not duplicate" }, + { id: "claude-fable-5-6368", name: "Claude Fable 5" }, + ]; + const merged = rankings.mergeProviderModels(registryModels, customModels); + assert.deepEqual( + merged.map((m) => m.id).sort(), + ["claude-fable-5-6368", "known-model"] + ); +}); + +test("mergeProviderModels: no custom models returns the registry list unchanged", () => { + const registryModels = [{ id: "known-model", name: "Known Model" }]; + assert.deepEqual(rankings.mergeProviderModels(registryModels, []), registryModels); +}); + +test("#6368: a provider whose only scored model is a user-added custom model appears under configuredOnly+availableOnly", async () => { + // Give the custom model an arena_elo intelligence entry so it survives + // the ranking builder's scoring step, same as any catalog model would. + intelligenceDb.upsertModelIntelligence({ + model: CUSTOM_MODEL_ID, + source: "arena_elo", + category: "default", + score: 0.91, + eloRaw: 1400, + confidence: "high", + expiresAt: null, + }); + + await modelsDb.addCustomModel("puter", CUSTOM_MODEL_ID, "Claude Fable 5"); + + await providersDb.createProviderConnection({ + provider: "puter", + authType: "apikey", + name: "puter-main-6368", + apiKey: "test-token", + isActive: true, + }); + + const unfiltered = await rankings.computeFreeProviderRankings(undefined, 100, {}); + const puterUnfiltered = unfiltered.find((r) => r.id === "puter"); + assert.ok(puterUnfiltered, "puter must appear in the unfiltered ranking"); + assert.ok( + puterUnfiltered!.topModel?.modelId === CUSTOM_MODEL_ID || + unfiltered.some((r) => r.id === "puter" && r.modelCount >= 1), + "puter ranking must reflect the custom model score" + ); + + const filtered = await rankings.computeFreeProviderRankings(undefined, 100, { + configuredOnly: true, + availableOnly: true, + }); + const puterFiltered = filtered.find((r) => r.id === "puter"); + assert.ok( + puterFiltered, + "puter (configured + available, ranked only via its custom model) must survive configuredOnly+availableOnly filters" + ); +}); + +test("#6150 regression guard: a free provider with no connection is still dropped under configuredOnly", async () => { + // "groq" is a no-auth free provider (always eligible, no connection needed + // to exist as a *candidate*), but configuredOnly still requires an actual + // connection row — assert the filter itself hasn't been loosened by the + // #6368 custom-model change. + const filtered = await rankings.computeFreeProviderRankings(undefined, 100, { + configuredOnly: true, + }); + const groq = filtered.find((r) => r.id === "groq"); + assert.equal(groq, undefined, "groq has no configured connection and must stay excluded"); +});