fix(providers): include custom models in Free Provider Rankings filters (#6368) (#6598)

CHANGELOG.md

@@ -10,6 +10,7 @@ _Living section — bullets land here as PRs merge into `release/v3.8.47` (paral
 
 ### 🐛 Bug Fixes
 
+- **fix(providers):** custom models a provider actually has are no longer dropped from the Free Provider Rankings when both **"Configured only"** and **"Available only"** filters are applied ([#6368](https://github.com/diegosouzapw/OmniRoute/issues/6368)), follow-up to #6150 — `freeProviderRankings.ts::getProviderModels()` only ever walked the static `open-sse/config/providerRegistry.ts` catalog, so a user-added custom model (e.g. a Puter `claude-fable-5` model saved as "Claude Fable 5") never entered the candidate model list the ranking scores against, and could never survive the #6150 configured/available filters even when actually configured and available. It now additively merges the provider's custom models (`db/models.ts::getCustomModels`) into that candidate list via a new pure, de-duping `mergeProviderModels()` helper, before scoring/filtering runs — catalog free/paid filtering elsewhere is untouched. Regression guard: `tests/unit/free-provider-rankings-custom-models-6368.test.ts`. (thanks @shabeer)
 - **fix(providers):** `cloudflare-ai` no longer silently drops image/non-text content parts ([#6390](https://github.com/diegosouzapw/OmniRoute/issues/6390)) — `transformRequest()`'s `flattenContent()` (added for #2539 to satisfy the Workers AI `/ai/v1/chat/completions` plain-string `content` requirement) mapped any non-text OpenAI content part (e.g. `image_url`) to `""` and joined the rest, so a request carrying an image quietly went out as text-only with the attachment gone and no error surfaced. It now throws a clear error on the first non-text part instead of dropping it silently, which the existing top-level `chatCore.ts` catch already routes through `buildErrorBody()`/`sanitizeErrorMessage()` (same pattern as `buildUrl()`'s missing-Account-ID error). Regression guard: `tests/unit/cloudflare-ai-image-parts-6390.test.ts`.
 - **fix(providers):** stop Antigravity connections from falsely reporting **all-accounts quota-exhausted** ([#6295](https://github.com/diegosouzapw/OmniRoute/issues/6295)) — `genericQuotaFetcher.ts::percentUsedForQuota()` ignored the `fractionReported` flag and defaulted an unreported model's `remainingPercentage` to 0, which computed as 100% used; since `convertUsageToQuotaInfo()` takes the worst-case window across a connection, a single model with no reported fraction dragged the whole account into `limitReached` and `quotaPreflight` skipped it. `percentUsedForQuota()` now returns `null` (unknown, window ignored) whenever `fractionReported === false`, before falling back to `remainingPercentage`. Regression guard: `tests/unit/generic-quota-fetcher.test.ts`.
 - **fix(providers):** grok-cli (Grok Build) now strips `reasoning_effort`/`reasoning` before forwarding the request ([#6288](https://github.com/diegosouzapw/OmniRoute/issues/6288)) — Claude Code sends `reasoning_effort` on every request (routing the Opus slot), which Grok Build's upstream chat-proxy endpoint rejects with a 400; `transformRequest()`'s existing `UNSUPPORTED` sampling-param strip list (#5273) never covered it. Regression guard: `tests/unit/grok-cli-reasoning-strip-6288.test.ts`.

src/lib/freeProviderRankings.ts

@@ -12,6 +12,7 @@ import { NOAUTH_PROVIDERS, OAUTH_PROVIDERS, APIKEY_PROVIDERS } from "@/shared/co
 import { REGISTRY } from "@omniroute/open-sse/config/providerRegistry";
 import { listModelIntelligence } from "./db/modelIntelligence";
 import { getProviderConnections } from "./db/providers";
+import { getCustomModels } from "./db/models";
 
 export interface ProviderModelScore {
   modelId: string;
@@ -90,12 +91,51 @@ function getFreeProviders() {
   return providers;
 }
 
+/** Minimal shape shared by registry models and user-added custom models. */
+export interface RankableModel {
+  id: string;
+  name: string;
+}
+
 /**
- * Get models for a provider from the registry.
+ * Pure merge: combine a provider's static registry models with its
+ * user-added custom models, de-duplicating by `id` (registry entry wins on
+ * collision — a custom model overriding a known catalog ID keeps the
+ * catalog's richer metadata upstream, only the extra IDs are additive here).
+ *
+ * Exported so the #6368 fix ("custom models missing from Free Provider
+ * Rankings under configured/available filters") can be unit-tested without a
+ * DB: the ranking builder no longer only walks the static registry — it also
+ * folds in whatever the operator configured as a custom model for that
+ * provider, mirroring how #6150's connection-based filters already treat
+ * "configured" as DB/runtime state rather than catalog membership.
  */
-function getProviderModels(providerId: string) {
+export function mergeProviderModels(
+  registryModels: RankableModel[],
+  customModels: RankableModel[]
+): RankableModel[] {
+  if (customModels.length === 0) return registryModels;
+  const seen = new Set(registryModels.map((m) => m.id));
+  const merged = registryModels.slice();
+  for (const custom of customModels) {
+    if (!custom?.id || seen.has(custom.id)) continue;
+    seen.add(custom.id);
+    merged.push({ id: custom.id, name: custom.name || custom.id });
+  }
+  return merged;
+}
+
+/**
+ * Get models for a provider: static registry models plus any user-added
+ * custom models for that provider (#6368 — custom models were previously
+ * invisible to the ranking builder, so they never survived the
+ * configured/available filters even when actually configured+available).
+ */
+async function getProviderModels(providerId: string): Promise<RankableModel[]> {
   const entry = REGISTRY[providerId];
-  return entry?.models ?? [];
+  const registryModels = entry?.models ?? [];
+  const customModels = (await getCustomModels(providerId)) as RankableModel[];
+  return mergeProviderModels(registryModels, Array.isArray(customModels) ? customModels : []);
 }
 
 /**
@@ -282,7 +322,7 @@ export async function computeFreeProviderRankings(
   const rankings: FreeProviderRanking[] = [];
 
   for (const provider of freeProviders) {
-    const models = getProviderModels(provider.id);
+    const models = await getProviderModels(provider.id);
     if (models.length === 0) continue;
 
     const modelScores: ProviderModelScore[] = [];

tests/unit/free-provider-rankings-custom-models-6368.test.ts

@@ -0,0 +1,116 @@
+/**
+ * Regression test for #6368 (follow-up to #6150).
+ *
+ * Custom models a user adds to a provider (e.g. "Claude Fable 5" added on
+ * top of the Puter provider) were invisible in the Free Provider Rankings
+ * once the "Configured only" / "Available only" filters were applied,
+ * because `getProviderModels()` only ever walked the static
+ * `open-sse/config/providerRegistry.ts` catalog — a provider's user-added
+ * custom models were never folded into the candidate model list that gets
+ * matched against intelligence scores, so they could never appear in the
+ * ranking regardless of the filters.
+ *
+ * This test proves:
+ *  1. `mergeProviderModels` (pure) additively includes custom models and
+ *     de-dupes against the registry list.
+ *  2. `computeFreeProviderRankings()` end-to-end surfaces a provider whose
+ *     *only* matching model is a user-added custom model, under BOTH
+ *     `configuredOnly` and `availableOnly` filters.
+ *  3. Existing catalog-model based free/paid... i.e. configured/available
+ *     filtering still behaves as before (#6150 regression guard) — a
+ *     provider with no connection is still dropped under `configuredOnly`.
+ */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-rankings-6368-"));
+process.env.DATA_DIR = TEST_DATA_DIR;
+
+const core = await import("../../src/lib/db/core.ts");
+const providersDb = await import("../../src/lib/db/providers.ts");
+const modelsDb = await import("../../src/lib/db/models.ts");
+const intelligenceDb = await import("../../src/lib/db/modelIntelligence.ts");
+const rankings = await import("../../src/lib/freeProviderRankings.ts");
+
+const CUSTOM_MODEL_ID = "claude-fable-5-6368";
+
+test.after(() => {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+});
+
+test("mergeProviderModels: additively includes custom models, de-duping by id", () => {
+  const registryModels = [{ id: "known-model", name: "Known Model" }];
+  const customModels = [
+    { id: "known-model", name: "Should not duplicate" },
+    { id: "claude-fable-5-6368", name: "Claude Fable 5" },
+  ];
+  const merged = rankings.mergeProviderModels(registryModels, customModels);
+  assert.deepEqual(
+    merged.map((m) => m.id).sort(),
+    ["claude-fable-5-6368", "known-model"]
+  );
+});
+
+test("mergeProviderModels: no custom models returns the registry list unchanged", () => {
+  const registryModels = [{ id: "known-model", name: "Known Model" }];
+  assert.deepEqual(rankings.mergeProviderModels(registryModels, []), registryModels);
+});
+
+test("#6368: a provider whose only scored model is a user-added custom model appears under configuredOnly+availableOnly", async () => {
+  // Give the custom model an arena_elo intelligence entry so it survives
+  // the ranking builder's scoring step, same as any catalog model would.
+  intelligenceDb.upsertModelIntelligence({
+    model: CUSTOM_MODEL_ID,
+    source: "arena_elo",
+    category: "default",
+    score: 0.91,
+    eloRaw: 1400,
+    confidence: "high",
+    expiresAt: null,
+  });
+
+  await modelsDb.addCustomModel("puter", CUSTOM_MODEL_ID, "Claude Fable 5");
+
+  await providersDb.createProviderConnection({
+    provider: "puter",
+    authType: "apikey",
+    name: "puter-main-6368",
+    apiKey: "test-token",
+    isActive: true,
+  });
+
+  const unfiltered = await rankings.computeFreeProviderRankings(undefined, 100, {});
+  const puterUnfiltered = unfiltered.find((r) => r.id === "puter");
+  assert.ok(puterUnfiltered, "puter must appear in the unfiltered ranking");
+  assert.ok(
+    puterUnfiltered!.topModel?.modelId === CUSTOM_MODEL_ID ||
+      unfiltered.some((r) => r.id === "puter" && r.modelCount >= 1),
+    "puter ranking must reflect the custom model score"
+  );
+
+  const filtered = await rankings.computeFreeProviderRankings(undefined, 100, {
+    configuredOnly: true,
+    availableOnly: true,
+  });
+  const puterFiltered = filtered.find((r) => r.id === "puter");
+  assert.ok(
+    puterFiltered,
+    "puter (configured + available, ranked only via its custom model) must survive configuredOnly+availableOnly filters"
+  );
+});
+
+test("#6150 regression guard: a free provider with no connection is still dropped under configuredOnly", async () => {
+  // "groq" is a no-auth free provider (always eligible, no connection needed
+  // to exist as a *candidate*), but configuredOnly still requires an actual
+  // connection row — assert the filter itself hasn't been loosened by the
+  // #6368 custom-model change.
+  const filtered = await rankings.computeFreeProviderRankings(undefined, 100, {
+    configuredOnly: true,
+  });
+  const groq = filtered.find((r) => r.id === "groq");
+  assert.equal(groq, undefined, "groq has no configured connection and must stay excluded");
+});