From be78e05925ba1e1190ebd2d307e05f5abde44a10 Mon Sep 17 00:00:00 2001 From: Demiurge The Single Date: Tue, 23 Jun 2026 13:03:24 +0300 Subject: [PATCH] =?UTF-8?q?fix(tier):=20noAuth=20providers=20count=20as=20?= =?UTF-8?q?free;=20free=20filter=20returns=20empty=20=E2=80=A6=20(#4753)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit noAuth providers now classified free (union of legacy list + NOAUTH_PROVIDERS chat-tier derivation), -free arena_elo alias, and auto/:free returns an empty pool when no free candidate matches (opt-in legacy fallback via OMNIROUTE_AUTO_FREE_FALLBACK_TO_FULL_POOL). New env var documented in .env.example + ENVIRONMENT.md; CHANGELOG bullet added (maintainer co-author). 46/46 node + 56/56 vitest tests pass on release tip; env-doc-sync, docs-sync, typecheck:core, lint, file-size all green. Integrated into release/v3.8.35. --- .env.example | 8 ++ CHANGELOG.md | 1 + docs/reference/ENVIRONMENT.md | 1 + .../services/__tests__/tierResolver.test.ts | 78 +++++++++++++ .../autoCombo/__tests__/autoCombo.test.ts | 60 ++++++++++ open-sse/services/autoCombo/taskFitness.ts | 109 ++++++++---------- open-sse/services/autoCombo/virtualFactory.ts | 22 +++- open-sse/services/tierConfig.ts | 74 ++++++++++-- src/lib/db/tierConfig.ts | 51 +++++++- .../autoCombo/suffixComposition-4517.test.ts | 94 +++++++++++++++ tests/unit/db-tier-config.test.ts | 82 ++++++++++++- 11 files changed, 497 insertions(+), 83 deletions(-) create mode 100644 tests/unit/autoCombo/suffixComposition-4517.test.ts diff --git a/.env.example b/.env.example index ac457f386b..3335784a6c 100644 --- a/.env.example +++ b/.env.example @@ -1640,6 +1640,14 @@ QUOTA_STORE_DRIVER=sqlite # sqlite | redis # QUOTA_CONSUMPTION_RETENTION_DAYS=14 # GC de buckets quota_consumption.updated_at antigos # QUOTA_PREFLIGHT_CUTOFF_ENABLED=false # opt-in (default OFF): hard quota cutoff drops low-quota candidates before auto-routing scoring +# ─── Auto-Combo tier filter (#4517) ─────────────────────────────────────── +# When an `auto/:free` (or any `:`) request matches NO connected +# candidates, OmniRoute returns an EMPTY pool by default — so `:free` really means +# "free tier only" and a paid model is never picked just because no free provider is +# connected. Set this to `true`/`1` to restore the legacy behavior of falling back to +# the full (unfiltered) pool with a warning. Source: open-sse/services/autoCombo/virtualFactory.ts +# OMNIROUTE_AUTO_FREE_FALLBACK_TO_FULL_POOL=false + # ─── OpenCode config regeneration (scripts/ad-hoc/regen-opencode-config.ts) ─── # Base URL of the OmniRoute instance to query for /v1/models when regenerating # an opencode.json with accurate limit.context values. Used by: diff --git a/CHANGELOG.md b/CHANGELOG.md index c257649f04..4c14928cd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ _In development — bullets added per PR; finalized at release._ ### 🐛 Fixed +- **fix(tier): noAuth providers count as free; `auto/:free` returns an empty pool when no free candidate matches** — `freeProviders` is now the union of the legacy explicit list and every chat-tier `noAuth` provider derived from `NOAUTH_PROVIDERS` (so opencode / mimocode / duckduckgo-web are correctly classified free), the task-fitness lookup inherits a base model's `arena_elo` for its `-free` variant, and the `auto/:` filter no longer silently falls back to the full pool — a `:free` request that matches no connected free model returns empty instead of billing a paid model (opt back into the legacy fallback with `OMNIROUTE_AUTO_FREE_FALLBACK_TO_FULL_POOL=true`). Corrupted/invalid `tier_config` rows now log a structured warning and fall back to defaults instead of throwing. ([#4753](https://github.com/diegosouzapw/OmniRoute/pull/4753), [#4517](https://github.com/diegosouzapw/OmniRoute/issues/4517) — thanks @megamen32) - **fix(db): scheduled VACUUM follows Storage settings** — the SQLite VACUUM scheduler now uses the existing Storage page `scheduledVacuum` / `vacuumHour` configuration as its single source of truth, refreshes immediately when those settings are saved, and no longer exposes a separate environment-variable control path. - **fix(db): scheduled cleanup actually runs + queries target the real tables (DB-bloat / OOM)** — `runAutoCleanup` was never scheduled, so retention cleanup never executed and tables (`compression_analytics`, `usage_history`, …) grew unbounded into multi-GB SQLite files driving high RSS. Worse, several cleanup queries referenced wrong table/column names (`call_logs.created_at`→`timestamp`, `compression_analytics.created_at`→`timestamp`, `mcp_audit_log`→`mcp_tool_audit`, `a2a_events`→`a2a_task_events`, `memory_entries`→`memories`), so even a manual run silently no-op'd or errored. Fixed the five queries to match the real schema, added `cleanupProxyLogs`, and wired a `startCleanupScheduler` (startup + every 6h, VACUUM after deletes) into `server-init` alongside the existing budget-reset and reasoning-cache jobs. ([#4691](https://github.com/diegosouzapw/OmniRoute/pull/4691), extracted from [#4428](https://github.com/diegosouzapw/OmniRoute/pull/4428) — thanks @oyi77 / @diegosouzapw) - **fix(routing): include all noAuth models in auto-combos + add reka-flash + best-free template** — noAuth provider models are no longer skipped when building auto-combos, `reka-flash` is registered, and a `best-free` combo template is added. ([#4621](https://github.com/diegosouzapw/OmniRoute/pull/4621) — thanks @oyi77) diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index be3ac0676f..ffebb7a27d 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -968,6 +968,7 @@ Provider quota endpoints, network tunnels (Tailscale, Ngrok, MITM debug proxy), | `QUOTA_SOFT_DEPRIORITIZE_FACTOR` | `0.7` | `open-sse/services/combo.ts` | Score multiplier (0..1) applied to a target when the soft quota policy deprioritizes it. | | `QUOTA_CONSUMPTION_RETENTION_DAYS` | `14` | `src/lib/db/quotaConsumption.ts` | Retention window (days) for `quota_consumption` buckets before GC (`gcQuotaConsumption`). | | `QUOTA_PREFLIGHT_CUTOFF_ENABLED` | `false` | `src/lib/resilience/settings.ts` | Opt-in (default OFF): enables the auto-routing hard quota cutoff that drops low-quota candidates before scoring. | +| `OMNIROUTE_AUTO_FREE_FALLBACK_TO_FULL_POOL` | `false` | `open-sse/services/autoCombo/virtualFactory.ts` | Opt-in (default OFF): when an `auto/:` filter matches no connected candidates, restore the legacy behavior of falling back to the full (unfiltered) pool instead of returning an empty pool. Default OFF makes `:free` mean "free tier only". | | `AGENTBRIDGE_UPSTREAM_CA_CERT` | _(unset)_ | `src/mitm/manager.ts` | Extra CA certificate (PEM) trusted for AgentBridge upstream TLS connections. | | `INSPECTOR_BUFFER_SIZE` | `1000` | `src/mitm/inspector/buffer.ts` | Max captured requests held in the Traffic Inspector ring buffer. | | `INSPECTOR_MAX_BODY_KB` | `1024` | `src/mitm/inspector/buffer.ts` | Max captured request/response body size (KB) before truncation. | diff --git a/open-sse/services/__tests__/tierResolver.test.ts b/open-sse/services/__tests__/tierResolver.test.ts index 23047eb757..fe14e0c646 100644 --- a/open-sse/services/__tests__/tierResolver.test.ts +++ b/open-sse/services/__tests__/tierResolver.test.ts @@ -13,6 +13,12 @@ import { classifyTiers, } from "../tierResolver.ts"; import { PROVIDER_TIER } from "../tierTypes.ts"; +import { + DEFAULT_TIER_CONFIG, + LEGACY_FREE_PROVIDERS, + deriveNoAuthFreeProviders, +} from "../tierConfig.ts"; +import { NOAUTH_PROVIDERS } from "@/shared/constants/providers.ts"; describe("TierResolver", () => { // Reset cache between tests @@ -213,4 +219,76 @@ describe("TierResolver", () => { assert.ok(stats[PROVIDER_TIER.CHEAP] >= 1); }); }); + + describe("freeProviders from NOAUTH_PROVIDERS (#4517)", () => { + beforeEach(() => clearTierCache()); + + it("LEGACY_FREE_PROVIDERS keeps the historical explicit list", () => { + for (const id of [ + "kiro", + "qoder", + "pollinations", + "longcat", + "cloudflare-ai", + "qwen", + "gemini-cli", + "nvidia-nim", + "cerebras", + "groq", + ]) { + assert.ok(LEGACY_FREE_PROVIDERS.includes(id), `expected ${id} in LEGACY_FREE_PROVIDERS`); + } + }); + + it("deriveNoAuthFreeProviders includes all chat-tier noAuth providers", () => { + const derived = deriveNoAuthFreeProviders(); + // opencode + mimocode are the ones the bug report called out + assert.ok(derived.includes("opencode"), "opencode should be in derived noAuth-free list"); + assert.ok(derived.includes("mimocode"), "mimocode should be in derived noAuth-free list"); + assert.ok(derived.includes("duckduckgo-web")); + }); + + it("deriveNoAuthFreeProviders excludes non-LLM noAuth providers", () => { + const derived = deriveNoAuthFreeProviders(); + assert.ok( + !derived.includes("veoaifree-web"), + "veoaifree-web (serviceKinds: video) must not be classified as chat-free" + ); + }); + + it("DEFAULT_TIER_CONFIG.freeProviders contains the union of legacy + noAuth-derived", () => { + const expected = new Set([...LEGACY_FREE_PROVIDERS, ...deriveNoAuthFreeProviders()]); + const actual = new Set(DEFAULT_TIER_CONFIG.freeProviders); + assert.deepEqual(actual, expected, "freeProviders must be the union, deduplicated"); + }); + + it("classifyTier classifies opencode/big-pickle as free via noAuth derivation", () => { + // No provider override, no cost-based match (big-pickle has no KNOWN_MODEL_PRICING row). + // The fix is that 'opencode' is now in freeProviders. + const result = classifyTier("opencode", "big-pickle"); + assert.equal(result.tier, PROVIDER_TIER.FREE); + assert.equal(result.hasFreeTier, true); + }); + + it("classifyTier classifies mimocode/mimo-auto as free via noAuth derivation", () => { + const result = classifyTier("mimocode", "mimo-auto"); + assert.equal(result.tier, PROVIDER_TIER.FREE); + assert.equal(result.hasFreeTier, true); + }); + + it("classifyTier still returns cheap for paid glm-5.1 (no regression)", () => { + // glm-5.1 is not in freeProviders, costs $0.50/M → cheap tier. + // Make sure the new noAuth derivation didn't accidentally pull it into free. + const result = classifyTier("opencode-go", "glm-5.1"); + assert.equal(result.tier, PROVIDER_TIER.CHEAP); + }); + + it("userConfig.freeProviders is merged on top of the noAuth-derived list", () => { + // Re-merge with a new free provider (e.g. local-llama) and confirm it's added. + setTierConfig({ freeProviders: ["local-llama"] }); + const result = classifyTier("local-llama", "anything"); + assert.equal(result.tier, PROVIDER_TIER.FREE); + clearTierCache(); + }); + }); }); diff --git a/open-sse/services/autoCombo/__tests__/autoCombo.test.ts b/open-sse/services/autoCombo/__tests__/autoCombo.test.ts index 0ae1ec01ea..6ee4b5997c 100644 --- a/open-sse/services/autoCombo/__tests__/autoCombo.test.ts +++ b/open-sse/services/autoCombo/__tests__/autoCombo.test.ts @@ -11,6 +11,8 @@ import { getTaskTypes, getModelsDevTierFitness, invalidateFitnessCache, + setUserFitnessOverride, + clearUserFitnessOverride, } from "../taskFitness"; import { SelfHealingManager } from "../selfHealing"; import { MODE_PACKS, getModePack, getModePackNames } from "../modePacks"; @@ -91,6 +93,64 @@ describe("Task Fitness", () => { const normalScore = getTaskFitness("some-random-model", "coding"); expect(coderScore).toBeGreaterThan(normalScore); }); + + describe("-free alias resolution (#4517)", () => { + beforeEach(() => invalidateFitnessCache()); + + it("returns the base model's arena_elo when given a -free variant", async () => { + // The fix: getTaskFitnessWithSource strips a trailing "-free" suffix + // and re-queries arena_elo with the base id. We seed an arena_elo + // row directly via the DB module, look up the free variant, and + // assert the alias path returns the base score with source + // "arena_elo_free_alias". + const baseId = "alias-base-test-4517"; + const freeId = "alias-base-test-4517-free"; + const { upsertModelIntelligence, deleteModelIntelligence } = + await import("../../../../src/lib/db/modelIntelligence.ts"); + // Seed arena_elo on the base id only — no row exists for the free id. + upsertModelIntelligence({ + model: baseId, + source: "arena_elo", + category: "coding", + score: 0.42, + eloRaw: 1500, + confidence: "high", + expiresAt: null, + }); + invalidateFitnessCache(); + try { + const result = getTaskFitnessWithSource(freeId, "coding"); + // Without the fix: result.source would be "wildcard_boost" (0.5 default). + // With the fix: result.source is "arena_elo_free_alias" with score 0.42. + expect(result.score).toBeCloseTo(0.42, 5); + expect(result.source).toBe("arena_elo_free_alias"); + } finally { + deleteModelIntelligence(baseId, "arena_elo", "coding"); + invalidateFitnessCache(); + } + }); + + it("does not strip -free when arena_elo is present on the literal model id", () => { + // If both "foo-free" and "foo" have arena_elo rows, the literal "foo-free" + // wins (we never go through the alias path). This protects future + // benchmark uploads that specifically tag free tiers. + setUserFitnessOverride("foo-free", "coding", 0.91); + const result = getTaskFitnessWithSource("foo-free", "coding"); + expect(result.score).toBe(0.91); + expect(result.source).toBe("user_override"); + clearUserFitnessOverride("foo-free", "coding"); + invalidateFitnessCache(); + }); + + it("ignores -free suffix only at the end of the model id", () => { + // "free-something" must NOT be treated as a free alias of "free-something-free" + // — the suffix must be at the end. "mimo-free-edition" is left alone. + // We just confirm no exception is thrown and the lookup returns a number. + const score = getTaskFitness("mimo-free-edition", "coding"); + expect(typeof score).toBe("number"); + expect(score).toBeGreaterThan(0); + }); + }); }); describe("Self-Healing", () => { diff --git a/open-sse/services/autoCombo/taskFitness.ts b/open-sse/services/autoCombo/taskFitness.ts index 404a476089..35b38fae12 100644 --- a/open-sse/services/autoCombo/taskFitness.ts +++ b/open-sse/services/autoCombo/taskFitness.ts @@ -199,11 +199,7 @@ const TIER_TASK_FITNESS: Record> = { const _intelligenceCache = new Map(); -function queryModelIntelligence( - model: string, - category: string, - source: string, -): number | null { +function queryModelIntelligence(model: string, category: string, source: string): number | null { const cacheKey = `${model}:${category}:${source}`; if (_intelligenceCache.has(cacheKey)) { return _intelligenceCache.get(cacheKey)!; @@ -233,8 +229,7 @@ interface ModelCapRow { function deriveTierFromCapabilities(cap: ModelCapRow): string { if (cap.reasoning === true) return "premium"; - if (cap.tool_call === true && (cap.limit_context ?? 0) >= 128000) - return "standard"; + if (cap.tool_call === true && (cap.limit_context ?? 0) >= 128000) return "standard"; if (cap.tool_call === true) return "fast"; return "budget"; } @@ -244,10 +239,7 @@ function loadModelCapabilities(): Record | null { try { const db = getDbInstance(); - const rows = db.prepare("SELECT * FROM model_capabilities").all() as Record< - string, - unknown - >[]; + const rows = db.prepare("SELECT * FROM model_capabilities").all() as Record[]; const cache: Record = {}; for (const row of rows) { @@ -267,8 +259,7 @@ function loadModelCapabilities(): Record | null { : row.reasoning === false || row.reasoning === 0 ? false : null, - limit_context: - typeof row.limit_context === "number" ? row.limit_context : null, + limit_context: typeof row.limit_context === "number" ? row.limit_context : null, }; } @@ -279,18 +270,11 @@ function loadModelCapabilities(): Record | null { } } -export function getModelsDevTierFitness( - model: string, - taskType: string, -): number | null { +export function getModelsDevTierFitness(model: string, taskType: string): number | null { const normalizedModel = model.toLowerCase(); const normalizedTask = taskType.toLowerCase(); - const dbScore = queryModelIntelligence( - normalizedModel, - normalizedTask, - "models_dev_tier", - ); + const dbScore = queryModelIntelligence(normalizedModel, normalizedTask, "models_dev_tier"); if (dbScore !== null) return dbScore; const caps = loadModelCapabilities(); @@ -308,10 +292,7 @@ export function getModelsDevTierFitness( // ─── Resolution chain ─────────────────────────────────────────────────── -function lookupStaticFitnessTable( - normalizedModel: string, - normalizedTask: string, -): number | null { +function lookupStaticFitnessTable(normalizedModel: string, normalizedTask: string): number | null { const table = FITNESS_TABLE[normalizedTask] || FITNESS_TABLE.default; for (const [pattern, score] of Object.entries(table)) { if (normalizedModel.includes(pattern)) return score; @@ -319,10 +300,7 @@ function lookupStaticFitnessTable( return null; } -function lookupWildcardBoosts( - normalizedModel: string, - normalizedTask: string, -): number { +function lookupWildcardBoosts(normalizedModel: string, normalizedTask: string): number { let baseScore = 0.5; for (const wc of WILDCARD_BOOSTS) { if (normalizedModel.includes(wc.pattern) && normalizedTask === wc.taskType) { @@ -338,38 +316,38 @@ export function getTaskFitness(model: string, taskType: string): number { export function getTaskFitnessWithSource( model: string, - taskType: string, + taskType: string ): { score: number; source: string } { const normalizedModel = model.toLowerCase(); const normalizedTask = taskType.toLowerCase(); - const userOverride = queryModelIntelligence( - normalizedModel, - normalizedTask, - "user_override", - ); + const userOverride = queryModelIntelligence(normalizedModel, normalizedTask, "user_override"); if (userOverride !== null) { return { score: userOverride, source: "user_override" }; } - const arenaElo = queryModelIntelligence( - normalizedModel, - normalizedTask, - "arena_elo", - ); + // Try arena_elo with the literal model id first (e.g. "mimo-v2.5"). If that's + // a miss and the model id carries a "-free" suffix (e.g. "mimo-v2.5-free"), + // try the un-suffixed base id so free-tier variants inherit the arena_elo + // score of their paid counterpart. This is what operators expect: the + // upstream's `mimo-v2.5` is benchmarked once, and `mimo-v2.5-free` should + // pick up the same signal rather than falling through to the wildcard 0.5 + // and losing every free-vs-paid comparison. + const arenaElo = queryModelIntelligence(normalizedModel, normalizedTask, "arena_elo"); if (arenaElo !== null) { return { score: arenaElo, source: "arena_elo" }; } + const arenaEloBase = lookupFreeAliasArenaElo(normalizedModel, normalizedTask); + if (arenaEloBase !== null) { + return { score: arenaEloBase, source: "arena_elo_free_alias" }; + } const tierScore = getModelsDevTierFitness(normalizedModel, normalizedTask); if (tierScore !== null) { return { score: tierScore, source: "models_dev_tier" }; } - const staticScore = lookupStaticFitnessTable( - normalizedModel, - normalizedTask, - ); + const staticScore = lookupStaticFitnessTable(normalizedModel, normalizedTask); if (staticScore !== null) { return { score: staticScore, source: "fitness_table" }; } @@ -377,35 +355,44 @@ export function getTaskFitnessWithSource( return { score: lookupWildcardBoosts(normalizedModel, normalizedTask), source: "wildcard_boost" }; } -export function setUserFitnessOverride( - model: string, - category: string, - score: number, -): void { +/** Suffix used to mark free-tier model variants (e.g. "mimo-v2.5-free"). */ +const FREE_SUFFIX = "-free"; + +/** + * Strip a trailing "-free" suffix from the model id and re-query arena_elo. + * Returns `null` when the original id has no "-free" suffix, when the base id + * is identical to the original, or when no arena_elo row exists for the base. + * + * Examples: + * "mimo-v2.5-free" → look up "mimo-v2.5" + * "deepseek-v4-flash-free" → look up "deepseek-v4-flash" + * "big-pickle" → no "-free" suffix → return null (skip) + */ +function lookupFreeAliasArenaElo(normalizedModel: string, normalizedTask: string): number | null { + if (!normalizedModel.endsWith(FREE_SUFFIX)) return null; + const baseId = normalizedModel.slice(0, -FREE_SUFFIX.length); + if (baseId.length === 0 || baseId === normalizedModel) return null; + return queryModelIntelligence(baseId, normalizedTask, "arena_elo"); +} + +export function setUserFitnessOverride(model: string, category: string, score: number): void { try { - setUserFitnessOverrideEntry( - model.toLowerCase(), - category.toLowerCase(), - score, - ); + setUserFitnessOverrideEntry(model.toLowerCase(), category.toLowerCase(), score); invalidateFitnessCache(); } catch (err) { throw new Error( - `Failed to set user fitness override for ${model}/${category}: ${err instanceof Error ? err.message : String(err)}`, + `Failed to set user fitness override for ${model}/${category}: ${err instanceof Error ? err.message : String(err)}` ); } } -export function clearUserFitnessOverride( - model: string, - category: string, -): void { +export function clearUserFitnessOverride(model: string, category: string): void { try { deleteUserFitnessOverrideEntry(model.toLowerCase(), category.toLowerCase()); invalidateFitnessCache(); } catch (err) { throw new Error( - `Failed to clear user fitness override for ${model}/${category}: ${err instanceof Error ? err.message : String(err)}`, + `Failed to clear user fitness override for ${model}/${category}: ${err instanceof Error ? err.message : String(err)}` ); } } diff --git a/open-sse/services/autoCombo/virtualFactory.ts b/open-sse/services/autoCombo/virtualFactory.ts index 8912148c4d..dd386b7775 100644 --- a/open-sse/services/autoCombo/virtualFactory.ts +++ b/open-sse/services/autoCombo/virtualFactory.ts @@ -294,8 +294,14 @@ export async function createVirtualAutoCombo( } // #4235 Phase B: narrow the pool by the `auto/:` overlay - // (vision/reasoning capability, free/premium model tier). Fall back to the full - // pool if the filter would empty it — never break routing, just lose the bias. + // (vision/reasoning capability, free/premium model tier). + // + // Default behavior: when the filter yields zero candidates, return an EMPTY + // pool — never silently fall back to the full pool. This makes + // `auto/coding:free` actually mean "free tier only" and prevents a paid + // expensive model from being picked just because no free provider is + // connected. Operators who want the old "never break routing, lose the bias" + // behavior can opt back in via the env var below. let effectivePool = candidatePool; const candidateFilter = spec ? buildAutoCandidateFilter(spec.category, spec.tier) : null; if (candidateFilter) { @@ -304,11 +310,21 @@ export async function createVirtualAutoCombo( ); if (narrowed.length > 0) { effectivePool = narrowed; + } else if ( + process.env.OMNIROUTE_AUTO_FREE_FALLBACK_TO_FULL_POOL === "true" || + process.env.OMNIROUTE_AUTO_FREE_FALLBACK_TO_FULL_POOL === "1" + ) { + // Opt-in legacy behavior: warn loudly, then keep the full pool. + log.warn( + "AUTO", + `auto/${spec?.category ?? ""}${spec?.tier ? `:${spec.tier}` : ""} matched no connected models; falling back to the full pool (OMNIROUTE_AUTO_FREE_FALLBACK_TO_FULL_POOL=true)` + ); } else { log.warn( "AUTO", - `auto/${spec?.category ?? ""}${spec?.tier ? `:${spec.tier}` : ""} matched no connected models; using the full pool` + `auto/${spec?.category ?? ""}${spec?.tier ? `:${spec.tier}` : ""} matched no connected models; returning an empty pool. Set OMNIROUTE_AUTO_FREE_FALLBACK_TO_FULL_POOL=true to restore the legacy "use full pool" behavior.` ); + effectivePool = []; } } diff --git a/open-sse/services/tierConfig.ts b/open-sse/services/tierConfig.ts index 52d7aee2c5..7bc5945fe8 100644 --- a/open-sse/services/tierConfig.ts +++ b/open-sse/services/tierConfig.ts @@ -1,10 +1,22 @@ /** * Tier configuration schema with Zod validation and sensible defaults. + * + * `freeProviders` is the union of two sources: + * 1. Legacy explicit list (`LEGACY_FREE_PROVIDERS`) — keeps historical behavior + * for providers that aren't noAuth (e.g., groq, cerebras) so the old test + * suite and existing DB overrides keep working. + * 2. Auto-derived from `NOAUTH_PROVIDERS` where `noAuth === true` AND + * `serviceKinds` is empty or includes "llm" (excludes free audio/video + * gateways like veo-free that would never feed the chat tier resolver). + * See `deriveNoAuthFreeProviders()`. + * + * The two are merged into `DEFAULT_TIER_CONFIG.freeProviders` at module load. */ import { z } from "zod"; import type { TierConfig, ProviderTierOverride, ModelTierOverride } from "./tierTypes"; import { PROVIDER_TIER } from "./tierTypes"; +import { NOAUTH_PROVIDERS } from "@/shared/constants/providers"; export const providerTierOverrideSchema = z.object({ provider: z.string().min(1), @@ -28,6 +40,55 @@ export const tierConfigSchema = z.object({ freeProviders: z.array(z.string()).default([]), }); +/** + * Legacy explicit free providers — kept for back-compat with existing DB rows + * and test fixtures. New free providers should be added by registering them + * in `NOAUTH_PROVIDERS` with `noAuth: true` instead of editing this list. + */ +export const LEGACY_FREE_PROVIDERS: readonly string[] = [ + "kiro", + "qoder", + "pollinations", + "longcat", + "cloudflare-ai", + "qwen", + "gemini-cli", + "nvidia-nim", + "cerebras", + "groq", +]; + +/** + * Derive free provider IDs from `NOAUTH_PROVIDERS`. Only chat-tier noAuth + * providers count (serviceKinds === undefined || includes "llm"), so the + * free video / audio / image noAuth entries (e.g. veo-free, muse-spark) don't + * accidentally mark themselves as chat-free. + * + * Wrapped in try/catch to tolerate the import failing in non-Node runtimes + * (vitest worker setup, MCP server entry) — the legacy list still applies. + */ +export function deriveNoAuthFreeProviders(): string[] { + try { + const ids: string[] = []; + for (const def of Object.values(NOAUTH_PROVIDERS)) { + if (!def || typeof def !== "object") continue; + if (def.noAuth !== true) continue; + const kinds = (def as { serviceKinds?: unknown }).serviceKinds; + const isLlm = + !Array.isArray(kinds) || kinds.length === 0 || (kinds as unknown[]).includes("llm"); + if (!isLlm) continue; + if (typeof def.id === "string" && def.id.length > 0) { + ids.push(def.id); + } + } + return ids; + } catch { + return []; + } +} + +const NOAUTH_FREE_PROVIDERS = deriveNoAuthFreeProviders(); + export const DEFAULT_TIER_CONFIG: TierConfig = { version: "1.0.0", defaults: { @@ -36,18 +97,7 @@ export const DEFAULT_TIER_CONFIG: TierConfig = { }, providerOverrides: [], modelOverrides: [], - freeProviders: [ - "kiro", - "qoder", - "pollinations", - "longcat", - "cloudflare-ai", - "qwen", - "gemini-cli", - "nvidia-nim", - "cerebras", - "groq", - ], + freeProviders: [...new Set([...LEGACY_FREE_PROVIDERS, ...NOAUTH_FREE_PROVIDERS])], }; export function validateTierConfig(raw: unknown): TierConfig { diff --git a/src/lib/db/tierConfig.ts b/src/lib/db/tierConfig.ts index 1a83fea0f7..0c1d0e7b06 100644 --- a/src/lib/db/tierConfig.ts +++ b/src/lib/db/tierConfig.ts @@ -1,8 +1,10 @@ import { getDbInstance } from "./core"; import type { TierConfig } from "../../../open-sse/services/tierTypes"; import { validateTierConfig, DEFAULT_TIER_CONFIG } from "../../../open-sse/services/tierConfig"; +import { defaultLogger as log } from "@omniroute/open-sse/utils/logger"; const TABLE = "tier_config"; +const CORRUPTED_VALUE_PREVIEW_LEN = 200; export function initTierConfigTable(): void { const db = getDbInstance(); @@ -23,15 +25,60 @@ export function saveTierConfig(config: TierConfig): void { ).run(serialized); } +/** + * Truncate an unknown value (string) for safe inclusion in a log payload. + * Returns the string verbatim when shorter than the cap, otherwise a + * 200-char preview with an ellipsis. `String(value)` is used as a final + * fallback so a non-string never throws here. + */ +function previewCorruptedValue(value: unknown): string { + if (typeof value !== "string") return String(value); + if (value.length <= CORRUPTED_VALUE_PREVIEW_LEN) return value; + return `${value.slice(0, CORRUPTED_VALUE_PREVIEW_LEN)}…`; +} + +/** + * Load the persisted tier config from SQLite. Returns `null` when no row exists + * OR when the stored value is unreadable (invalid JSON, fails Zod validation). + * + * The function NEVER throws on parse failure — instead it logs a structured + * warning so operators can spot the corruption in logs and either: + * 1. Manually delete the bad row: + * DELETE FROM tier_config WHERE key = 'tier_config'; + * 2. Re-save a clean config via the dashboard's Tier settings page. + * + * The caller (`loadTierConfig()`) then falls back to `DEFAULT_TIER_CONFIG`, + * so a corrupted row never silently feeds invalid pricing into the router. + */ export function loadTierConfigFromDb(): TierConfig | null { const db = getDbInstance(); const row = db.prepare(`SELECT value FROM ${TABLE} WHERE key = 'tier_config'`).get() as | { value: string } | undefined; if (!row) return null; + + const raw = row.value; + let parsed: unknown; try { - return validateTierConfig(JSON.parse(row.value)); - } catch { + parsed = JSON.parse(raw); + } catch (err) { + log.warn( + { err: err instanceof Error ? err.message : String(err), value: previewCorruptedValue(raw) }, + "tier_config JSON.parse failed; falling back to DEFAULT_TIER_CONFIG" + ); + return null; + } + + try { + return validateTierConfig(parsed); + } catch (err) { + log.warn( + { + err: err instanceof Error ? err.message : String(err), + value: previewCorruptedValue(raw), + }, + "tier_config Zod validation failed; falling back to DEFAULT_TIER_CONFIG" + ); return null; } } diff --git a/tests/unit/autoCombo/suffixComposition-4517.test.ts b/tests/unit/autoCombo/suffixComposition-4517.test.ts new file mode 100644 index 0000000000..9b02462bdc --- /dev/null +++ b/tests/unit/autoCombo/suffixComposition-4517.test.ts @@ -0,0 +1,94 @@ +/** + * Unit tests for the `auto/:` suffix composition filter. + * + * See: `open-sse/services/autoCombo/suffixComposition.ts` + * + * Focus: the `:free` tier filter. Regression test for the bug where + * opencode (noAuth, free) and mimocode (noAuth, free) were NOT being + * included in the free pool because the legacy `freeProviders` list + * only contained paid-API-key providers with free tiers (kiro, qoder, ...). + * + * #4517. + */ +import { describe, it, before, after, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import { + buildAutoCandidateFilter, + parseAutoSuffix, +} from "../../../open-sse/services/autoCombo/suffixComposition"; + +describe("suffixComposition :free tier (#4517)", () => { + const ORIGINAL_ENV = { ...process.env }; + + before(() => { + // Snapshot env so we can restore it after each test. + }); + + beforeEach(() => { + // Reset env to a known state so OMNIROUTE_AUTO_FREE_FALLBACK_TO_FULL_POOL + // doesn't leak between cases. + process.env = { ...ORIGINAL_ENV }; + }); + + after(() => { + process.env = ORIGINAL_ENV; + }); + + it("parseAutoSuffix recognizes coding:free", () => { + assert.deepEqual(parseAutoSuffix("coding:free"), { + valid: true, + category: "coding", + tier: "free", + }); + }); + + it("buildAutoCandidateFilter keeps noAuth free providers", () => { + // Regression: opencode and mimocode are noAuth and free, but the + // pre-fix `freeProviders` list omitted them, so the filter rejected + // their candidates even though they ARE free upstream. + const filter = buildAutoCandidateFilter("coding", "free"); + assert.notEqual(filter, null); + + assert.equal(filter!({ provider: "opencode", model: "big-pickle" }), true); + assert.equal(filter!({ provider: "opencode", model: "minimax-m3-free" }), true); + assert.equal(filter!({ provider: "mimocode", model: "mimo-auto" }), true); + assert.equal(filter!({ provider: "duckduckgo-web", model: "gpt-4o-mini" }), true); + }); + + it("buildAutoCandidateFilter keeps legacy free providers", () => { + // Don't break the existing list — kiro, qoder, groq, etc. must still pass. + const filter = buildAutoCandidateFilter("coding", "free"); + assert.equal(filter!({ provider: "kiro", model: "claude-sonnet-4-5" }), true); + assert.equal(filter!({ provider: "groq", model: "llama-3.3-70b" }), true); + assert.equal(filter!({ provider: "qoder", model: "qwen3-coder-plus" }), true); + }); + + it("buildAutoCandidateFilter rejects paid models under :free", () => { + // The bug: opencode-go/glm-5.1 was being picked because the filter + // fell back to the full pool when no free candidate was found. + // After the fix, glm-5.1 must be rejected by the :free filter. + const filter = buildAutoCandidateFilter("coding", "free"); + assert.equal(filter!({ provider: "opencode-go", model: "glm-5.1" }), false); + assert.equal(filter!({ provider: "openai", model: "gpt-4o" }), false); + assert.equal(filter!({ provider: "anthropic", model: "claude-sonnet-4-6" }), false); + assert.equal(filter!({ provider: "deepseek", model: "deepseek-chat" }), false); + }); + + it("buildAutoCandidateFilter returns null for category-only (no tier)", () => { + // "coding" with no tier must NOT filter by tier — pass-through. + const filter = buildAutoCandidateFilter("coding", undefined); + assert.equal(filter, null); + }); + + it("buildAutoCandidateFilter keeps free candidates alongside capability checks", () => { + // The category and tier checks are AND-combined. For "coding:free" the + // category check is a pass-through (no vision/reasoning filter), so + // any free model should be kept. + const filter = buildAutoCandidateFilter("coding", "free"); + assert.equal(filter!({ provider: "opencode", model: "minimax-m3-free" }), true); + // The "reasoning" category also pairs with ":free" and keeps free models. + const reasoningFilter = buildAutoCandidateFilter("reasoning", "free"); + // big-pickle (model_capabilities: reasoning=1) should pass the reasoning check. + assert.equal(reasoningFilter!({ provider: "opencode", model: "big-pickle" }), true); + }); +}); diff --git a/tests/unit/db-tier-config.test.ts b/tests/unit/db-tier-config.test.ts index 684a619d45..86eb47017d 100644 --- a/tests/unit/db-tier-config.test.ts +++ b/tests/unit/db-tier-config.test.ts @@ -1,4 +1,4 @@ -import { describe, it, beforeEach } from "node:test"; +import { describe, it, beforeEach, mock } from "node:test"; import assert from "node:assert/strict"; import { @@ -43,14 +43,86 @@ describe("tierConfig DB module", () => { assert.ok(loaded, "should return config after overwrite"); }); - it("loadTierConfigFromDb handles corrupted JSON gracefully", async () => { - // Directly insert corrupted data + it("loadTierConfigFromDb handles corrupted JSON gracefully (#4517)", async () => { + // Reproduce the bug report: a hand-edited / partially written row contains + // non-JSON garbage. We expect null + a warning, NOT a thrown error. const { getDbInstance } = await import("../../src/lib/db/core.ts"); const db = getDbInstance(); db.prepare( "INSERT OR REPLACE INTO tier_config (key, value, updated_at) VALUES ('tier_config', ?, datetime('now'))" ).run("not-valid-json{{{"); - const result = loadTierConfigFromDb(); - assert.equal(result, null, "should return null for corrupted JSON"); + + // Spy on the logger to confirm we emit a warning that operators can spot. + const loggerModule = await import("@omniroute/open-sse/utils/logger.ts"); + const warnSpy = mock.method(loggerModule.defaultLogger, "warn", () => {}); + + try { + const result = loadTierConfigFromDb(); + assert.equal(result, null, "should return null for corrupted JSON"); + assert.ok( + warnSpy.mock.calls.length > 0, + "should emit at least one warning so operators can spot the corruption" + ); + // Sanity: loadTierConfig() still returns DEFAULT_TIER_CONFIG. + const fallback = loadTierConfig(); + assert.deepEqual(fallback.freeProviders, DEFAULT_TIER_CONFIG.freeProviders); + } finally { + warnSpy.mock.restore(); + } + }); + + it("loadTierConfigFromDb handles valid JSON that fails Zod (#4517)", async () => { + // JSON parses fine, but the shape doesn't match the schema (freeThreshold = -1 + // violates the min(0) constraint). + const { getDbInstance } = await import("../../src/lib/db/core.ts"); + const db = getDbInstance(); + const badShape = JSON.stringify({ + version: "1.0.0", + defaults: { freeThreshold: -1, cheapThreshold: 1.0 }, + providerOverrides: [], + modelOverrides: [], + freeProviders: [], + }); + db.prepare( + "INSERT OR REPLACE INTO tier_config (key, value, updated_at) VALUES ('tier_config', ?, datetime('now'))" + ).run(badShape); + + const loggerModule = await import("@omniroute/open-sse/utils/logger.ts"); + const warnSpy = mock.method(loggerModule.defaultLogger, "warn", () => {}); + + try { + const result = loadTierConfigFromDb(); + assert.equal(result, null, "should return null for Zod-failing config"); + assert.ok(warnSpy.mock.calls.length > 0, "should log a warning on Zod failure"); + } finally { + warnSpy.mock.restore(); + } + }); + + it("loadTierConfigFromDb truncates very long corrupted values in the warning preview (#4517)", async () => { + const { getDbInstance } = await import("../../src/lib/db/core.ts"); + const db = getDbInstance(); + // 1000 chars of garbage — the warning preview must truncate to avoid log floods. + const long = "{".repeat(1000); + db.prepare( + "INSERT OR REPLACE INTO tier_config (key, value, updated_at) VALUES ('tier_config', ?, datetime('now'))" + ).run(long); + + const loggerModule = await import("@omniroute/open-sse/utils/logger.ts"); + const warnSpy = mock.method(loggerModule.defaultLogger, "warn", () => {}); + + try { + const result = loadTierConfigFromDb(); + assert.equal(result, null); + assert.ok(warnSpy.mock.calls.length > 0); + const payload = warnSpy.mock.calls[0].arguments[0] as Record; + const preview = typeof payload.value === "string" ? payload.value : ""; + assert.ok( + preview.length <= 250, + `warning preview should be truncated, got length=${preview.length}` + ); + } finally { + warnSpy.mock.restore(); + } }); });