diff --git a/changelog.d/fixes/12792-pricing-valuation.md b/changelog.d/fixes/12792-pricing-valuation.md new file mode 100644 index 0000000000..7eea070726 --- /dev/null +++ b/changelog.d/fixes/12792-pricing-valuation.md @@ -0,0 +1 @@ +- **fix(routing):** off-table models check the free-model catalog before inheriting premium prices, latency bootstraps from the observed pool median, async tiers read live database pricing with a 90-day freshness gate, and the tier cache invalidates on every pricing write ([#12792](https://github.com/diegosouzapw/OmniRoute/pull/12792)) — thanks @maxmad64bis diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 49c5f74724..41d80fbff6 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -467,7 +467,8 @@ "src/sse/handlers/chat.ts": 2458, "src/sse/services/auth.ts": 3450, "tests/unit/account-fallback-service.test.ts": 2453, - "tests/unit/provider-validation-specialty.test.ts": 4656 + "tests/unit/provider-validation-specialty.test.ts": 4656, + "open-sse/services/autoCombo/virtualFactory.ts": 1207 }, "_rebaseline_base_2026_08_10_proxyfetch": "Base-red fix (green-prs sweep, issue #9985): open-sse/utils/proxyFetch.ts 1207 > cap 1000 — new proxied-TLS fetch helper introduced by the Fal reference-image work. Owner-authorized quick rebaseline to green; structural slim tracked for v3.9.0.", "_rebaseline_2026_07_27_v3849_train2": "Merge-train 2 (7 PRs) — owner-approved 2026-07-27. Single entry: chatCore.ts 4955->5006 (#8595, Responses multi-turn image compaction before the context hard-reject). Genuine irreducible growth at the existing compaction chokepoint in handleChatCore — the PR adds a last-resort retry against the concrete budget plus the estimateFinalInputTokens helper, both wired at the pre-existing call site rather than a new branch. Covered by tests/unit/8560-responses-image-compaction.test.ts (4 tests).", @@ -652,5 +653,6 @@ "_rebaseline_2026_09_03_houminxi_combo_stacked": "Leva HouMinXi (#12624 #12626 #12632 #12637): open-sse/services/combo.ts 4075->4080 (+5), medido no tip com os quatro mergeados. Cada PR registrou o proprio crescimento contra o tip de onde forkou (o #12637 ja subira o cap para 4075); as 5 linhas restantes so aparecem quando eles empilham, porque mais de um toca o mesmo chokepoint de scoring reset-aware em combo.ts. Fiacao em ponto existente, sem extracao possivel sem partir a funcao de selecao de alvos. Coberto por combo-strategies e reset-aware-request-scope-12600 (119/119 focados na leva).", "_rebaseline_2026_09_04_12641_continuation_effective_input": "PR #12641 crescimento proprio: src/sse/handlers/chat.ts 2450->2454 (+4). A continuacao por previous_response_id encadeava a partir de clientRawRequest.body.input, que e capturado ANTES da reconstrucao do proprio chat.ts; quando o turno anterior ja era uma continuacao, esse campo guarda so o delta do cliente, e o erro se acumulava a cada salto ate a reconstrucao virar itens de tool sem prefixo. Persistir o input EFETIVO exige as linhas no ponto onde a reconstrucao termina, dentro do fluxo de despacho. Coberto por tests/unit/responses-continuation-store.test.ts (22/22 focados na leva).", "_rebaseline_2026_09_05_12671_combos_usage_guide_external_store": "combos/page.tsx 5018 -> 5066: #12671 replaces the effect-based localStorage read with useSyncExternalStore; the +48 lines are the store helpers (subscribe/getSnapshot/getServerSnapshot/emit) hoisted to module scope, which is the sanctioned shape and what let the react-hooks/set-state-in-effect suppression be dropped.", - "_rebaseline_2026_09_07_chatcore_nonstreaming_regression_fixes": "Own growth: open-sse/handlers/chatCore.ts 5984->6021 (+37). Two of my own PRs on top of #12867: #12963 pins the ok variant of the non-streaming leg result in its own binding (the discriminated-union narrowing was lost across the tool-loop reassignment, 13 TS2339 under tsconfig.typecheck-api.json), and #12990 restores four behaviours the same refactor dropped — abort classification through isLocalStreamLifecycleError, the omitted synthetic clientResponse, the claudePromptCacheLogMeta rebuild on the leg path, and the lazy fail-closed fence identity. Irreducible at the existing chokepoints: each edit sits where chatCore already owns the decision, and the helpers themselves (nonStreamingProviderLeg.ts, serverOwnedToolLoopWire.ts) are under cap. Covered by tests/unit/chatcore-translation-paths.test.ts (72/74; the 2 open are issue #13043)." + "_rebaseline_2026_09_07_chatcore_nonstreaming_regression_fixes": "Own growth: open-sse/handlers/chatCore.ts 5984->6021 (+37). Two of my own PRs on top of #12867: #12963 pins the ok variant of the non-streaming leg result in its own binding (the discriminated-union narrowing was lost across the tool-loop reassignment, 13 TS2339 under tsconfig.typecheck-api.json), and #12990 restores four behaviours the same refactor dropped — abort classification through isLocalStreamLifecycleError, the omitted synthetic clientResponse, the claudePromptCacheLogMeta rebuild on the leg path, and the lazy fail-closed fence identity. Irreducible at the existing chokepoints: each edit sits where chatCore already owns the decision, and the helpers themselves (nonStreamingProviderLeg.ts, serverOwnedToolLoopWire.ts) are under cap. Covered by tests/unit/chatcore-translation-paths.test.ts (72/74; the 2 open are issue #13043).", + "_rebaseline_2026_09_07_virtualfactory_crosses_the_new_file_cap": "open-sse/services/autoCombo/virtualFactory.ts crosses the 1200 new-file cap for the first time (1187 on the pre-wave tip, 1207 here). Growth is spread across the routing/free-tier wave, not one extractable block: #12794 feeds observed breaker state and model quality into snapshot scoring instead of neutral constants, #12792 adds the reliability factor the snapshot path was still ignoring and the pooled-latency bootstrap, and #12744 tightens the free-model predicate the factory consumes. FROZEN RATHER THAN SPLIT, deliberately, and this is debt: two cohesive extraction candidates are ready when someone owns the move — computeSnapshotWeights (~85 lines) and the credential-eligibility group hasUsableOAuthToken/hasProviderSpecificSessionData/isKeylessEligibleConnection/hasUsableConnectionCredential (~70 lines). Either alone clears 1200 from here. Splitting three contributors' just-merged work mid-batch was the larger risk." } diff --git a/open-sse/services/__tests__/manifestAdapter.test.ts b/open-sse/services/__tests__/manifestAdapter.test.ts index e9430851a4..5652888741 100644 --- a/open-sse/services/__tests__/manifestAdapter.test.ts +++ b/open-sse/services/__tests__/manifestAdapter.test.ts @@ -22,8 +22,8 @@ function makeTarget(provider: string, model: string): ResolvedComboTarget { describe("ManifestAdapter", () => { describe("generateRoutingHints - trivial query", () => { - it("returns prefer-free modifier for greeting", () => { - const hints = generateRoutingHints([], { + it("returns prefer-free modifier for greeting", async () => { + const hints = await generateRoutingHints([], { messages: [{ content: "Hello" }], }); expect(hints.strategyModifier).toBe("prefer-free"); @@ -32,8 +32,8 @@ describe("ManifestAdapter", () => { }); describe("generateRoutingHints - expert query", () => { - it("returns a valid modifier for complex input", () => { - const hints = generateRoutingHints([], { + it("returns a valid modifier for complex input", async () => { + const hints = await generateRoutingHints([], { messages: [ { content: @@ -47,25 +47,25 @@ describe("ManifestAdapter", () => { }); describe("generateRoutingHints - target classification", () => { - it("marks free provider as eligible for trivial query", () => { + it("marks free provider as eligible for trivial query", async () => { const targets = [makeTarget("kiro", "claude-sonnet-4.5")]; - const hints = generateRoutingHints(targets, { + const hints = await generateRoutingHints(targets, { messages: [{ content: "Hi" }], }); expect(hints.eligibleTargets.length).toBeGreaterThanOrEqual(0); }); - it("handles empty targets array gracefully", () => { - const hints = generateRoutingHints([], { + it("handles empty targets array gracefully", async () => { + const hints = await generateRoutingHints([], { messages: [{ content: "Hello" }], }); expect(hints.eligibleTargets.length).toBe(0); expect(hints.underqualifiedTargets.length).toBe(0); }); - it("classifies mixed targets for simple query", () => { + it("classifies mixed targets for simple query", async () => { const targets = [makeTarget("kiro", "claude-sonnet-4.5"), makeTarget("openai", "gpt-4o")]; - const hints = generateRoutingHints(targets, { + const hints = await generateRoutingHints(targets, { messages: [{ content: "Hello" }], }); expect(hints.eligibleTargets.length).toBeGreaterThanOrEqual(0); @@ -73,20 +73,20 @@ describe("ManifestAdapter", () => { }); describe("compareByCostEffectiveness", () => { - it("takes 3 arguments and returns a number", () => { + it("takes 3 arguments and returns a number", async () => { const a = makeTarget("deepseek", "deepseek-chat"); const b = makeTarget("openai", "gpt-4o"); - const hints = generateRoutingHints([a, b], { + const hints = await generateRoutingHints([a, b], { messages: [{ content: "Test" }], }); const result = compareByCostEffectiveness(a, b, hints); expect(typeof result).toBe("number"); }); - it("returns negative when a is cheaper than b", () => { + it("returns negative when a is cheaper than b", async () => { const a = makeTarget("deepseek", "deepseek-chat"); const b = makeTarget("openai", "gpt-4o"); - const hints = generateRoutingHints([a, b], { + const hints = await generateRoutingHints([a, b], { messages: [{ content: "Test" }], }); const result = compareByCostEffectiveness(a, b, hints); @@ -115,16 +115,16 @@ describe("ManifestAdapter", () => { }); describe("edge cases", () => { - it("handles empty targets array", () => { - const hints = generateRoutingHints([], { + it("handles empty targets array", async () => { + const hints = await generateRoutingHints([], { messages: [{ content: "Hello" }], }); expect(hints.eligibleTargets.length).toBe(0); expect(hints.underqualifiedTargets.length).toBe(0); }); - it("returns valid hints structure with no targets", () => { - const hints = generateRoutingHints([], { + it("returns valid hints structure with no targets", async () => { + const hints = await generateRoutingHints([], { messages: [{ content: "Test" }], }); expect("specificityLevel" in hints).toBe(true); diff --git a/open-sse/services/autoCombo/complexityRouter.ts b/open-sse/services/autoCombo/complexityRouter.ts index 602fb1450b..5b49adf483 100644 --- a/open-sse/services/autoCombo/complexityRouter.ts +++ b/open-sse/services/autoCombo/complexityRouter.ts @@ -75,11 +75,11 @@ export function classifyRequestComplexity(input: RuleInput): ComplexityClassific * failure — fail-open, so scoring stays tier-neutral. Extracted from combo.ts to * keep the complexity-routing logic in one module. */ -export function buildComplexityRoutingHint( +export async function buildComplexityRoutingHint( modelTargets: Parameters[0], body: { messages?: unknown; tools?: unknown; model?: unknown } | null | undefined, log: { info: (tag: string, message: string) => void } -): RoutingHint | null { +): Promise { try { const ruleInput = { messages: Array.isArray(body?.messages) @@ -92,7 +92,7 @@ export function buildComplexityRoutingHint( : undefined, model: typeof body?.model === "string" ? body.model : undefined, }; - const hint = generateRoutingHints(modelTargets, ruleInput); + const hint = await generateRoutingHints(modelTargets, ruleInput); // Tool-use escalation: floor the recommended tier at "cheap" so scoring // favors function-calling-reliable models for agentic requests. const classification = classifyRequestComplexity(ruleInput); diff --git a/open-sse/services/autoCombo/engine.ts b/open-sse/services/autoCombo/engine.ts index 9e91552f36..07a94e2dc4 100644 --- a/open-sse/services/autoCombo/engine.ts +++ b/open-sse/services/autoCombo/engine.ts @@ -39,6 +39,7 @@ export interface AutoComboConfig { * silently overspending. */ budgetFallback?: "cheapest" | "strict"; + estimatedInputTokens?: number; // tokens the budget is computed against (default 1000) explorationRate: number; // 0.05 = 5% exploratory /** If set, RouterStrategy name to use for selection ('rules' | 'cost' | 'latency') */ routerStrategy?: string; @@ -311,9 +312,13 @@ export function selectProvider( for (const c of candidates) { costMap.set(`${c.provider}\0${c.model}`, c.costPer1MTokens); } + const estimatedTokens = + Number.isFinite(config.estimatedInputTokens) && config.estimatedInputTokens! > 0 + ? config.estimatedInputTokens! + : 1000; const estimatedCostFor = (s: ScoredProvider) => { const cost = costMap.get(`${s.provider}\0${s.model}`) ?? 0; - return (cost / 1_000_000) * 1000; + return (cost / 1_000_000) * estimatedTokens; }; if (estimatedCostFor(selected) > config.budgetCap) { const budgetOk = candidates_.filter((s) => estimatedCostFor(s) <= config.budgetCap!); diff --git a/open-sse/services/autoCombo/scoring.ts b/open-sse/services/autoCombo/scoring.ts index 295d5379b3..a9977ccba5 100644 --- a/open-sse/services/autoCombo/scoring.ts +++ b/open-sse/services/autoCombo/scoring.ts @@ -293,6 +293,14 @@ export function computePoolMaxima(pool: ProviderCandidate[]): PoolMaxima { * `speedRanking.ts` so both consumers of the same signal agree, including on * garbage input. */ +/** Reliability factor from an observed failure (else error) rate; absent reads fully reliable. */ +export function reliabilityFactor(candidate: { + failureRate?: number | null; + errorRate?: number | null; +}): number { + return clamp01(1 - boundedRate(candidate.failureRate ?? candidate.errorRate)); +} + function boundedRate(value: number | null | undefined): number { if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return 0; return Math.min(1, value); @@ -341,7 +349,7 @@ export function calculateFactors( // bounded BEFORE the subtraction, exactly as `toBoundedRate` does there -- // `clamp01(1 - NaN)` would be 0, i.e. "fails every call", which is the // opposite of what corrupt telemetry should mean. - reliability: clamp01(1 - boundedRate(candidate.failureRate ?? candidate.errorRate)), + reliability: reliabilityFactor(candidate), }; } diff --git a/open-sse/services/autoCombo/virtualFactory.ts b/open-sse/services/autoCombo/virtualFactory.ts index f14cec5cd4..51f0be6606 100644 --- a/open-sse/services/autoCombo/virtualFactory.ts +++ b/open-sse/services/autoCombo/virtualFactory.ts @@ -1,6 +1,6 @@ import { AutoComboConfig } from "./engine"; import { MODE_PACKS } from "./modePacks"; -import { DEFAULT_WEIGHTS, ScoringWeights } from "./scoring"; +import { DEFAULT_WEIGHTS, reliabilityFactor, ScoringWeights } from "./scoring"; import { getCachedProviderConnections } from "@/lib/db/readCache"; import { getSettings } from "@/lib/db/settings"; import { getProviderRegistry } from "./providerRegistryAccessor"; @@ -28,6 +28,7 @@ import { import { classifyTier } from "../tierResolver"; import { getQualityScore } from "../routing/quality.ts"; import { readBreakerStates, snapshotHealthFactor, type BreakerState } from "./snapshotBreaker.ts"; +import { resolveVirtualCost } from "../providerCostData"; import type { AutoVariant } from "./autoPrefix"; import { buildFamilyCandidateFilter, type ModelFamily } from "./modelFamily"; import { getHiddenModelsByProvider } from "@/models"; @@ -107,6 +108,8 @@ export interface VirtualAutoComboCandidate { model: string; modelStr: string; // e.g., 'openai/gpt-4o' costPer1MTokens: number; // from providerRegistry + /** Observed failure rate 0..1 when known; null/absent reads fully reliable. */ + failureRate?: number | null; /** Build-local capability snapshot. Runtime calls rebuild it; catalog entries reuse it. */ resolvedContextLength?: number | null; resolvedMaxOutputTokens?: number | null; @@ -444,7 +447,7 @@ function getNoAuthCandidates( connectionId: SYNTHETIC_NOAUTH_CONNECTION_ID, model: modelId, modelStr: `${routingPrefix}/${modelId}`, - costPer1MTokens: 0, + costPer1MTokens: resolveVirtualCost(providerId, modelId), }); } } @@ -726,7 +729,7 @@ export async function prepareVirtualAutoComboInputs( allowedConnectionIds, model: modelId, modelStr: `${providerId}/${modelId}`, - costPer1MTokens: 0, // Not used in virtual auto-combo (LKGP uses session stickiness) + costPer1MTokens: resolveVirtualCost(providerId, modelId), }); } } @@ -893,6 +896,10 @@ export function computeSnapshotWeights( // (no runtime data at snapshot time, so equal baseline) if (weights.latencyInv > 0) score += weights.latencyInv * 0.5; + // reliability (#12792): the same failure-rate factor as scoring.ts; absent reads as + // fully reliable. The snapshot path was the only one still ignoring it. + if (weights.reliability > 0) score += weights.reliability * reliabilityFactor(c); + // health: build-time breaker state (OPEN 0, CLOSED 1, else neutral 0.5); quota stays neutral score += weights.health * snapshotHealthFactor(breakerByProvider, c.provider); score += weights.quota * 0.5; diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 61236649fd..acc72de4ae 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -251,6 +251,63 @@ function getBootstrapLatencyMs(modelId: string): number { return DEFAULT_MODEL_P95_MS[normalized] ?? 1500; } +export function poolMedianP95Ms( + stats: Record +): number | undefined { + const vals = Object.values(stats) + .map((st) => Number(st?.p95LatencyMs)) + .filter((v) => Number.isFinite(v) && v > 0) + .sort((a, b) => a - b); + return vals.length ? vals[(vals.length - 1) >> 1] : undefined; +} + +const BOOTSTRAP_WARN_WINDOW_MS = 3600_000; +export let bootstrapLatencyHits = 0; // exported for testability (reset in tests) +export let bootstrapLatencyTotal = 0; +let bootstrapWarnedAt = 0; +export function resetBootstrapCounters(): void { + bootstrapLatencyHits = 0; + bootstrapLatencyTotal = 0; + bootstrapWarnedAt = 0; +} +export function bootstrapMs(model: string, poolMedian: number | undefined): number { + bootstrapLatencyTotal++; + const table = DEFAULT_MODEL_P95_MS[String(model || "").toLowerCase()]; + if (table !== undefined) return table; + bootstrapLatencyHits++; + return poolMedian ?? 1500; +} + +// Pure and testable without timers: the throttled 1h warn + cold-start exemption live here. +export function shouldWarnBootstrap( + hits: number, + total: number, + hasStats: boolean, + now: number, + lastWarn: number +): boolean { + if (!hasStats || total === 0) return false; + if (hits / total <= 0.3) return false; + return now - lastWarn >= BOOTSTRAP_WARN_WINDOW_MS; +} + +function maybeWarnBootstrapDominant(hasStats: boolean): void { + if ( + !shouldWarnBootstrap( + bootstrapLatencyHits, + bootstrapLatencyTotal, + hasStats, + Date.now(), + bootstrapWarnedAt + ) + ) + return; + bootstrapWarnedAt = Date.now(); + console.warn( + `[combo] bootstrap latency dominant (${bootstrapLatencyHits}/${bootstrapLatencyTotal}) — scoring runs on guesses` + ); +} + export async function buildAutoCandidates( targets: ResolvedComboTarget[], comboName: string, @@ -278,6 +335,8 @@ export async function buildAutoCandidates( } catch { // keep empty stats — auto-combo will use runtime + bootstrap signals } + const poolMedian = poolMedianP95Ms(historicalLatencyStats); + const hasStats = Object.keys(historicalLatencyStats).length > 0; const uniqueProviders = Array.from( new Set( @@ -364,10 +423,10 @@ export async function buildAutoCandidates( const p95LatencyMs = hasHistoricalSignal ? Number.isFinite(historicalP95Latency) && historicalP95Latency > 0 ? historicalP95Latency - : getBootstrapLatencyMs(model) + : bootstrapMs(model, poolMedian) : Number.isFinite(avgLatency) && avgLatency > 0 ? avgLatency - : getBootstrapLatencyMs(model); + : bootstrapMs(model, poolMedian); const errorRate = hasHistoricalSignal ? Number.isFinite(historicalSuccessRate) && @@ -516,6 +575,7 @@ export async function buildAutoCandidates( // Filter out candidates whose model is hidden by the user in the dashboard, // then drop vendor-retired ids so auto-combo cannot pick them (#11625). + maybeWarnBootstrapDominant(hasStats); return rejectRetiredAutoComboCandidates( candidates.filter((c) => { const hiddenModels = hiddenModelsMap.get(c.provider); diff --git a/open-sse/services/combo/applyStrategyOrdering.ts b/open-sse/services/combo/applyStrategyOrdering.ts index 405b042863..a0dc11ebc0 100644 --- a/open-sse/services/combo/applyStrategyOrdering.ts +++ b/open-sse/services/combo/applyStrategyOrdering.ts @@ -157,7 +157,7 @@ export async function applyStrategyOrdering( orderedTargets = await sortTargetsByCost(orderedTargets); if (config.manifestRouting === true) { try { - const manifestHint = generateRoutingHints( + const manifestHint = await generateRoutingHints( orderedTargets.filter((t) => t.kind === "model"), { messages: Array.isArray(body?.messages) diff --git a/open-sse/services/combo/resolveAutoStrategy.ts b/open-sse/services/combo/resolveAutoStrategy.ts index 9882f2da4e..134f8100b6 100644 --- a/open-sse/services/combo/resolveAutoStrategy.ts +++ b/open-sse/services/combo/resolveAutoStrategy.ts @@ -315,7 +315,7 @@ export async function resolveAutoStrategyOrder( // specificityMatch favor candidates whose tier matches the request. const autoManifestHint: RoutingHint | null = config.complexityAwareRouting === true - ? buildComplexityRoutingHint( + ? await buildComplexityRoutingHint( eligibleTargets.filter((t) => t.kind === "model"), body, log @@ -408,6 +408,7 @@ export async function resolveAutoStrategyOrder( modePack, budgetCap, budgetFallback, + estimatedInputTokens, explorationRate, }, routableCandidates, diff --git a/open-sse/services/manifestAdapter.ts b/open-sse/services/manifestAdapter.ts index a0528e48be..5ca43aa78a 100644 --- a/open-sse/services/manifestAdapter.ts +++ b/open-sse/services/manifestAdapter.ts @@ -1,7 +1,7 @@ import type { TierAssignment, ProviderTier } from "./tierTypes"; import { PROVIDER_TIER } from "./tierTypes"; import type { SpecificityResult, SpecificityLevel } from "./specificityTypes"; -import { classifyTier } from "./tierResolver"; +import { classifyTier, classifyTierAsync } from "./tierResolver"; import { analyzeSpecificity, getSpecificityLevel, @@ -29,17 +29,24 @@ export interface RoutingHint { strategyModifier: StrategyModifier; } -export function generateRoutingHints( +export async function generateRoutingHints( targets: ResolvedComboTarget[], input: RuleInput -): RoutingHint { +): Promise { const tierAssignments = new Map(); - for (const target of targets) { - if (target.kind !== "model") continue; - const key = `${target.provider}::${target.modelStr}`; - if (!tierAssignments.has(key)) { - tierAssignments.set(key, classifyTier(target.provider, target.modelStr)); - } + const modelTargets = targets.filter((t) => t.kind === "model"); + const settled = await Promise.all( + modelTargets.map(async (target) => { + const key = `${target.provider}::${target.modelStr}`; + try { + return [key, await classifyTierAsync(target.provider, target.modelStr)] as const; + } catch { + return [key, classifyTier(target.provider, target.modelStr)] as const; + } + }) + ); + for (const [key, assignment] of settled) { + if (!tierAssignments.has(key)) tierAssignments.set(key, assignment); } const specificity = analyzeSpecificity(input); diff --git a/open-sse/services/providerCostData.ts b/open-sse/services/providerCostData.ts index 8464635c92..675514d816 100644 --- a/open-sse/services/providerCostData.ts +++ b/open-sse/services/providerCostData.ts @@ -1,4 +1,5 @@ import { getPricingForModel as getDefaultPricingForModel } from "@/shared/constants/pricing"; +import { isFreeModel } from "@/shared/utils/freeModels"; import type { TierConfig } from "./tierTypes"; export interface ModelPricing { @@ -38,14 +39,16 @@ export const KNOWN_MODEL_PRICING: Record = { }; export function getModelPricing(provider: string, model: string): ModelPricing { - const providerKey = `${provider}/${model}`.toLowerCase(); - if (KNOWN_MODEL_PRICING[providerKey]) { - return KNOWN_MODEL_PRICING[providerKey]; - } - const providerPricing = getDefaultPricingForModel(provider, model); - if (providerPricing) { - const inputCostPer1M = Number(providerPricing.input); - const outputCostPer1M = Number(providerPricing.output); + const normalized = String(model || "") + .split("/") + .pop()! + .toLowerCase(); + const providerHit = KNOWN_MODEL_PRICING[`${provider}/${normalized}`.toLowerCase()]; + if (providerHit) return providerHit; + const defaultPricing = getDefaultPricingForModel(provider, model); + if (defaultPricing) { + const inputCostPer1M = Number(defaultPricing.input); + const outputCostPer1M = Number(defaultPricing.output); if (Number.isFinite(inputCostPer1M) && Number.isFinite(outputCostPer1M)) { return { inputCostPer1M, @@ -54,13 +57,18 @@ export function getModelPricing(provider: string, model: string): ModelPricing { }; } } - const directKey = model.toLowerCase(); - if (KNOWN_MODEL_PRICING[directKey]) { - return KNOWN_MODEL_PRICING[directKey]; - } + const genericHit = KNOWN_MODEL_PRICING[normalized]; + if (genericHit) return genericHit; + if (isFreeModel(provider, { id: normalized })) + return { inputCostPer1M: 0, outputCostPer1M: 0, isFree: true }; return { inputCostPer1M: 5.0, outputCostPer1M: 15.0, isFree: false }; } +/** Input cost per 1M tokens a virtual auto-combo candidate is scored at. */ +export function resolveVirtualCost(providerId: string, modelId: string): number { + return getModelPricing(providerId, modelId).inputCostPer1M; +} + export function isExplicitlyFree(provider: string, config: TierConfig): boolean { return config.freeProviders.includes(provider.toLowerCase()); } diff --git a/open-sse/services/tierResolver.ts b/open-sse/services/tierResolver.ts index b1702c5d9c..d83e43bef0 100644 --- a/open-sse/services/tierResolver.ts +++ b/open-sse/services/tierResolver.ts @@ -142,3 +142,40 @@ export function getTierStats(): Record { } return stats; } + +export let tierAsyncFallbackTotal = 0; // exported for testability + +export async function classifyTierAsync(provider: string, model: string): Promise { + try { + const { getPricingForModel } = await import("@/lib/db/settings"); + const db = await getPricingForModel(provider, model); + const input = Number((db as { input?: unknown } | null)?.input); + const output = Number((db as { output?: unknown } | null)?.output); + if (Number.isFinite(input) && input >= 0) { + const out = Number.isFinite(output) && output >= 0 ? output : input; + // Same thresholds as the sync path below; a DB $0 lands FREE by threshold + // (the DB carries no isFree flag of its own). + let tier: ProviderTier; + if (input <= currentConfig.defaults.freeThreshold) tier = PROVIDER_TIER.FREE; + else if (input <= currentConfig.defaults.cheapThreshold) tier = PROVIDER_TIER.CHEAP; + else tier = PROVIDER_TIER.PREMIUM; + const sync = getModelPricing(provider, model); // keeps freeQuotaLimit when DB is mute + const assignment: TierAssignment = { + provider, + model, + tier, + reason: `DB cost-based: $${input}/M input`, + costPer1MInput: input, + costPer1MOutput: out, + hasFreeTier: input === 0, + freeQuotaLimit: sync.freeQuotaLimit, + }; + tierCache.set(cacheKey(provider, model), assignment); + return assignment; + } + } catch { + // fall through to sync + } + tierAsyncFallbackTotal++; + return classifyTier(provider, model); +} diff --git a/package.json b/package.json index bd3367796d..e8f3389354 100644 --- a/package.json +++ b/package.json @@ -265,6 +265,7 @@ "coverage:report": "cross-env NODE_OPTIONS=--max-old-space-size=8192 c8 report --merge-async --output-dir coverage --exclude=tests/** --exclude=**/*.test.* --reporter=text --reporter=text-summary --reporter=html --reporter=json-summary --reporter=lcov", "coverage:summary": "node scripts/check/test-report-summary.mjs --input coverage/coverage-summary.json --output coverage/coverage-report.md", "check:pr-test-policy": "node scripts/check/check-pr-test-policy.mjs", + "check:pricing-freshness": "node scripts/check/check-pricing-freshness.mjs", "coverage:report:legacy": "c8 report --output-dir coverage --exclude=open-sse --reporter=text --reporter=text-summary", "test:all": "npm run test:unit && npm run test:vitest && npm run test:vitest:ui && npm run test:ecosystem && npm run test:e2e", "check": "npm run lint && npm run test", diff --git a/scripts/check/check-pricing-freshness.mjs b/scripts/check/check-pricing-freshness.mjs new file mode 100644 index 0000000000..075077d13d --- /dev/null +++ b/scripts/check/check-pricing-freshness.mjs @@ -0,0 +1,26 @@ +#!/usr/bin/env node +// STRICT gate: KNOWN_MODEL_PRICING must have been touched within PRICING_STALE_AFTER_DAYS, +// or tiers silently drift from real prices. Exits 1 on stale, 0 on fresh. +// Run: node scripts/check/check-pricing-freshness.mjs +import { execFileSync } from "node:child_process"; + +const PRICING_STALE_AFTER_DAYS = 90; +const TARGET = "open-sse/services/providerCostData.ts"; + +function lastTouchDays() { + const out = execFileSync("git", ["log", "--follow", "-1", "--format=%ct", "--", TARGET], { + encoding: "utf8", + }).trim(); + const touched = Number(out); + if (!Number.isFinite(touched) || touched <= 0) return Infinity; + return (Date.now() / 1000 - touched) / 86400; +} + +const days = lastTouchDays(); +if (days > PRICING_STALE_AFTER_DAYS) { + console.error( + `STALE: ${TARGET} untouched for ${Math.floor(days)}d (> ${PRICING_STALE_AFTER_DAYS}d) — refresh prices or bump the gate with justification` + ); + process.exit(1); +} +console.log(`pricing fresh: ${TARGET} touched ${Math.floor(days)}d ago`); diff --git a/scripts/quality/run-all-gates.mjs b/scripts/quality/run-all-gates.mjs index 54102b6abc..aa6b68ea90 100644 --- a/scripts/quality/run-all-gates.mjs +++ b/scripts/quality/run-all-gates.mjs @@ -29,6 +29,7 @@ const GATES = [ // Group B — fast (<5s) { name: "check:provider-consistency", cmd: ["node", "--import", "tsx", "scripts/check/check-provider-consistency.ts"] }, { name: "check:provider-assets", cmd: ["node", "scripts/check/check-provider-assets.mjs"] }, + { name: "check:pricing-freshness", cmd: ["node", "scripts/check/check-pricing-freshness.mjs"] }, { name: "check:provider-order-sync", cmd: ["node", "scripts/check/check-provider-order-sync.mjs"] }, { name: "check:public-creds", cmd: ["node", "scripts/check/check-public-creds.mjs"] }, { name: "check:error-helper", cmd: ["node", "scripts/check/check-error-helper.mjs"] }, diff --git a/src/lib/db/settings/pricing.ts b/src/lib/db/settings/pricing.ts index b0f7dd1b74..e2496550a2 100644 --- a/src/lib/db/settings/pricing.ts +++ b/src/lib/db/settings/pricing.ts @@ -13,6 +13,16 @@ type PricingByProvider = Record; export type PricingSource = "default" | "litellm" | "modelsDev" | "user"; export type PricingSourceMap = Record>; +async function touchPricing(): Promise { + invalidateDbCache("pricing"); + try { + const { clearTierCache } = await import("@omniroute/open-sse/services/tierResolver"); + clearTierCache(); + } catch { + // fail-open: a missed tier invalidation must never break a price write + } +} + function readPricingNamespace( db: ReturnType, namespace: string @@ -195,7 +205,7 @@ export async function updatePricing(pricingData: PricingByProvider) { }); tx(); backupDbFile("pre-write"); - invalidateDbCache("pricing"); // Bust the pricing read cache + await touchPricing(); const updated: PricingByProvider = {}; const allRows = db.prepare("SELECT key, value FROM key_value WHERE namespace = 'pricing'").all(); for (const row of allRows) { @@ -234,6 +244,7 @@ export async function resetPricing(provider: string, model?: string) { } backupDbFile("pre-write"); + await touchPricing(); const allRows = db.prepare("SELECT key, value FROM key_value WHERE namespace = 'pricing'").all(); const result: Record = {}; for (const row of allRows) { @@ -250,5 +261,6 @@ export async function resetAllPricing() { const db = getDbInstance(); db.prepare("DELETE FROM key_value WHERE namespace = 'pricing'").run(); backupDbFile("pre-write"); + await touchPricing(); return {}; } diff --git a/tests/integration/manifest-routing.test.ts b/tests/integration/manifest-routing.test.ts index 458b9af4ad..878eda7efb 100644 --- a/tests/integration/manifest-routing.test.ts +++ b/tests/integration/manifest-routing.test.ts @@ -3,7 +3,7 @@ import assert from "node:assert/strict"; import { generateRoutingHints } from "../../open-sse/services/manifestAdapter.ts"; test("manifest routing generates hints without error", async () => { - const hints = generateRoutingHints([], { + const hints = await generateRoutingHints([], { messages: [{ content: "Test" }], }); assert.equal(hints.specificityLevel, "trivial"); @@ -11,7 +11,7 @@ test("manifest routing generates hints without error", async () => { }); test("manifest routing failure gracefully falls back - empty targets handled", async () => { - const hints = generateRoutingHints([], { + const hints = await generateRoutingHints([], { messages: [{ content: "Hello world" }], }); assert.equal(hints.eligibleTargets.length, 0); @@ -20,7 +20,7 @@ test("manifest routing failure gracefully falls back - empty targets handled", a }); test("specificity score is non-negative and bounded", async () => { - const hints = generateRoutingHints([], { + const hints = await generateRoutingHints([], { messages: [{ content: "Hello" }], }); assert.ok(hints.specificity.score >= 0); @@ -28,7 +28,7 @@ test("specificity score is non-negative and bounded", async () => { }); test("routing hints contain all required fields", async () => { - const hints = generateRoutingHints([], { + const hints = await generateRoutingHints([], { messages: [{ content: "Test message" }], }); assert.ok("specificityLevel" in hints); @@ -40,7 +40,7 @@ test("routing hints contain all required fields", async () => { }); test("trivial query recommends free tier", async () => { - const hints = generateRoutingHints([], { + const hints = await generateRoutingHints([], { messages: [{ content: "Hello" }], }); assert.equal(hints.recommendedMinTier, "free"); diff --git a/tests/unit/auto-combo-budget.test.ts b/tests/unit/auto-combo-budget.test.ts new file mode 100644 index 0000000000..1e9313ba58 --- /dev/null +++ b/tests/unit/auto-combo-budget.test.ts @@ -0,0 +1,57 @@ +// tests/unit/auto-combo-budget.test.ts +import test from "node:test"; +import assert from "node:assert/strict"; +import { selectProvider } from "../../open-sse/services/autoCombo/engine.ts"; +import { getSelfHealingManager } from "../../open-sse/services/autoCombo/selfHealing.ts"; +import { DEFAULT_WEIGHTS } from "../../open-sse/services/autoCombo/scoring.ts"; + +const healer = getSelfHealingManager(); +test.beforeEach(() => { + healer.exclusions.clear(); + healer.incidentMode = false; +}); + +function config(over: Record = {}) { + return { + id: "budget-probe", + name: "Budget Probe", + type: "auto", + candidatePool: [], + weights: DEFAULT_WEIGHTS, + explorationRate: 0, + ...over, + } as never; +} + +function cand(provider: string, model: string, cost: number) { + return { + provider, + model, + quotaRemaining: 95, + quotaTotal: 100, + circuitBreakerState: "CLOSED", + costPer1MTokens: cost, + p95LatencyMs: 120, + latencyStdDev: 8, + } as never; +} + +test("default budget math is unchanged without the new field", () => { + const c = [cand("openai", "gpt-4o", 5)]; + const r = selectProvider(config({ budgetCap: 1 }), c); + assert.equal(r.provider, "openai"); // (5/1M)*1000 = $0.005 < $1 +}); + +test("32k estimated tokens enforce a 32x tighter budget", () => { + const c = [cand("openai", "gpt-4o", 5)]; + // (5/1M)*32000 = $0.16 — exceeds a $0.01 cap only when tokens are honored. + // Message verified: "No candidate fits within the configured budget cap…" (engine.ts:59-62). + assert.throws( + () => + selectProvider( + config({ budgetCap: 0.01, budgetFallback: "strict", estimatedInputTokens: 32000 }), + c + ), + /budget cap/i + ); +}); diff --git a/tests/unit/combo-bootstrap-latency.test.ts b/tests/unit/combo-bootstrap-latency.test.ts new file mode 100644 index 0000000000..0499d7c36a --- /dev/null +++ b/tests/unit/combo-bootstrap-latency.test.ts @@ -0,0 +1,53 @@ +// tests/unit/combo-bootstrap-latency.test.ts +import { test } from "node:test"; +import assert from "node:assert/strict"; + +// Median helper lives in combo.ts; import it directly (export it non-default). +import { + poolMedianP95Ms, + bootstrapMs, + shouldWarnBootstrap, + resetBootstrapCounters, + bootstrapLatencyHits, + bootstrapLatencyTotal, +} from "../../open-sse/services/combo.ts"; + +test("median ignores corrupt entries", () => { + const m = poolMedianP95Ms({ + "a/x": { p95LatencyMs: 1000 }, + "b/y": { p95LatencyMs: NaN }, + "c/z": { p95LatencyMs: -5 }, + "d/w": { p95LatencyMs: 3000 }, + } as never); + assert.equal(m, 1000); +}); + +test("empty stats yield undefined (caller falls back to 1500, no cold-start alarm)", () => { + assert.equal(poolMedianP95Ms({}), undefined); +}); + +test("counters start at zero", () => { + resetBootstrapCounters(); + assert.equal(bootstrapLatencyHits, 0); + assert.equal(bootstrapLatencyTotal, 0); +}); + +test("table hit bumps Total but not Hits; miss bumps both", () => { + resetBootstrapCounters(); + assert.equal(bootstrapMs("gpt-4o-mini", 9999), 2764); // table hit + assert.equal(bootstrapLatencyTotal, 1); + assert.equal(bootstrapLatencyHits, 0); + assert.equal(bootstrapMs("some-new-model-xyz", 1234), 1234); // miss -> median + assert.equal(bootstrapLatencyTotal, 2); + assert.equal(bootstrapLatencyHits, 1); + assert.equal(bootstrapMs("another-new-model", undefined), 1500); // miss, no median + resetBootstrapCounters(); +}); + +test("warn gate is pure: cold start and low ratio stay silent, dominant warns once", () => { + assert.equal(shouldWarnBootstrap(10, 10, false, 9999, 0), false); // cold start + assert.equal(shouldWarnBootstrap(2, 10, true, 9999, 0), false); // ratio <= 0.3 + assert.equal(shouldWarnBootstrap(5, 10, true, 9999, 0), false); // throttled 1h + assert.equal(shouldWarnBootstrap(5, 10, true, 3600_001, 0), true); // window elapsed + assert.equal(shouldWarnBootstrap(5, 10, true, 9999, 9000), false); // throttled 1h +}); diff --git a/tests/unit/complexity-router.test.ts b/tests/unit/complexity-router.test.ts index 7a6e6745c7..de7b565d75 100644 --- a/tests/unit/complexity-router.test.ts +++ b/tests/unit/complexity-router.test.ts @@ -78,8 +78,8 @@ test("classifyRequestComplexity — tool schemas escalate the tier above free", assert.notEqual(c.recommendedTier, "free", "tool-using requests must not route to the free tier"); }); -test("buildComplexityRoutingHint — a tool-using request floors the hint tier above free", () => { - const hint = buildComplexityRoutingHint( +test("buildComplexityRoutingHint — a tool-using request floors the hint tier above free", async () => { + const hint = await buildComplexityRoutingHint( modelTargets(), { messages: [{ role: "user", content: "weather?" }], @@ -96,8 +96,8 @@ test("buildComplexityRoutingHint — a tool-using request floors the hint tier a ); }); -test("buildComplexityRoutingHint — a null body is safe and still builds a tier-neutral hint", () => { - const hint = buildComplexityRoutingHint(modelTargets(), null, NOOP_LOG); +test("buildComplexityRoutingHint — a null body is safe and still builds a tier-neutral hint", async () => { + const hint = await buildComplexityRoutingHint(modelTargets(), null, NOOP_LOG); assert.ok(hint, "a null body must not throw — messages default to [] and a hint is built"); if (!hint) return; assert.ok( diff --git a/tests/unit/pricing-fallback.test.ts b/tests/unit/pricing-fallback.test.ts new file mode 100644 index 0000000000..13e2f4fffc --- /dev/null +++ b/tests/unit/pricing-fallback.test.ts @@ -0,0 +1,33 @@ +// tests/unit/pricing-fallback.test.ts +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { getModelPricing, KNOWN_MODEL_PRICING } from "../../open-sse/services/providerCostData.ts"; + +// `ai21/jamba-large-1.7` is a documented free model (FREE_MODEL_BUDGETS, +// one-time-initial) that is NOT in the table (grep jamba -> 0 hit). +// `openai/gpt-4o` is paid. Fixtures are verbatim catalog ids — never a suffixed +// invention (Set.has is exact: a mutilated id is no longer in the catalog). +test("off-table free model is free, never premium-priced", () => { + const p = getModelPricing("ai21", "jamba-large-1.7"); + assert.equal(p.isFree, true); + assert.equal(p.inputCostPer1M, 0); + assert.equal(p.outputCostPer1M, 0); +}); + +test("prefixed model id still resolves through the catalog", () => { + const p = getModelPricing("ai21", "ai21/jamba-large-1.7"); + assert.equal(p.isFree, true); +}); + +test("off-table paid model keeps the conservative fallback", () => { + const p = getModelPricing("openai", "gpt-9-never-existed"); + assert.equal(p.isFree, false); + assert.equal(p.inputCostPer1M, 5.0); + assert.equal(p.outputCostPer1M, 15.0); +}); + +test("table hits are untouched", () => { + assert.equal(Object.keys(KNOWN_MODEL_PRICING).length, 21); + assert.deepEqual(getModelPricing("openai", "gpt-4o"), KNOWN_MODEL_PRICING["gpt-4o"]); + assert.deepEqual(getModelPricing("longcat", "LongCat-2.0"), KNOWN_MODEL_PRICING["longcat-2.0"]); +}); diff --git a/tests/unit/tier-pricing-cache.test.ts b/tests/unit/tier-pricing-cache.test.ts new file mode 100644 index 0000000000..844cfaafa6 --- /dev/null +++ b/tests/unit/tier-pricing-cache.test.ts @@ -0,0 +1,50 @@ +// tests/unit/tier-pricing-cache.test.ts +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + classifyTier, + classifyTierAsync, + clearTierCache, +} from "../../open-sse/services/tierResolver.ts"; +import { updatePricing, resetPricing, resetAllPricing } from "../../src/lib/db/settings/pricing.ts"; + +test.beforeEach(() => clearTierCache()); + +test("async tier reflects a DB price write without restart", async () => { + try { + await updatePricing({ testprov: { "testmodel-cache-1": { input: 0, output: 0 } } }); + const a = await classifyTierAsync("testprov", "testmodel-cache-1"); + assert.equal(a.tier, "free"); + } finally { + await resetPricing("testprov", "testmodel-cache-1"); + } +}); + +test("resetAllPricing (full wipe, no read-cache bust) also falls back cleanly", async () => { + try { + await updatePricing({ testprov: { "testmodel-cache-2": { input: 0, output: 0 } } }); + assert.equal((await classifyTierAsync("testprov", "testmodel-cache-2")).tier, "free"); + await resetAllPricing(); + const back = await classifyTierAsync("testprov", "testmodel-cache-2"); + assert.equal(back.tier, classifyTier("testprov", "testmodel-cache-2").tier); + } finally { + await resetPricing("testprov"); + } +}); + +test("async tier falls back to sync when DB is down", async () => { + const a = await classifyTierAsync("openai", "gpt-4o"); + assert.deepEqual(a, classifyTier("openai", "gpt-4o")); +}); + +test("sync tier keeps serving the table while DB changes", async () => { + // Fictitious model on a real provider: never pollutes shared state if cleanup fails. + const before = classifyTier("openai", "gpt-9-never-existed"); + try { + await updatePricing({ openai: { "gpt-9-never-existed": { input: 0, output: 0 } } }); + const during = classifyTier("openai", "gpt-9-never-existed"); + assert.equal(during.tier, before.tier); // sync path untouched by DB + } finally { + await resetPricing("openai", "gpt-9-never-existed"); + } +}); diff --git a/tests/unit/virtualFactory-valuation.test.ts b/tests/unit/virtualFactory-valuation.test.ts new file mode 100644 index 0000000000..79f9665c1f --- /dev/null +++ b/tests/unit/virtualFactory-valuation.test.ts @@ -0,0 +1,47 @@ +// tests/unit/virtualFactory-valuation.test.ts (part 1 — reliability) +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { computeSnapshotWeights } from "../../open-sse/services/autoCombo/virtualFactory.ts"; +import { resolveVirtualCost } from "../../open-sse/services/providerCostData.ts"; +import { DEFAULT_WEIGHTS } from "../../open-sse/services/autoCombo/scoring.ts"; + +function candidate(over: Record = {}) { + return { + provider: "openai", + connectionId: null, + model: "gpt-4o", + modelStr: "openai/gpt-4o", + costPer1MTokens: 5, + ...over, + } as never; +} + +test("absent failure rate reads fully reliable (neutral 1)", () => { + const weights = { ...DEFAULT_WEIGHTS, reliability: 0.2 }; + const scores = computeSnapshotWeights([candidate()], weights); + const base = computeSnapshotWeights([candidate()], { ...DEFAULT_WEIGHTS, reliability: 0 }); + const delta0 = scores.get("openai/gpt-4o")! - base.get("openai/gpt-4o")!; + assert.ok(Math.abs(delta0 - 0.2) < 1e-9, `expected 0.2, got ${delta0}`); +}); + +test("failureRate 0.2 scales the reliability weight to 0.8", () => { + const weights = { ...DEFAULT_WEIGHTS, reliability: 0.2 }; + const scores = computeSnapshotWeights([candidate({ failureRate: 0.2 })], weights); + const base = computeSnapshotWeights([candidate()], { ...DEFAULT_WEIGHTS, reliability: 0 }); + const delta = scores.get("openai/gpt-4o")! - base.get("openai/gpt-4o")!; + assert.ok(Math.abs(delta - 0.16) < 1e-9, `expected 0.16, got ${delta}`); +}); + +test("errorRate backs up a missing failureRate", () => { + const weights = { ...DEFAULT_WEIGHTS, reliability: 0.2 }; + const scores = computeSnapshotWeights([candidate({ errorRate: 0.5 })], weights); + const base = computeSnapshotWeights([candidate()], { ...DEFAULT_WEIGHTS, reliability: 0 }); + const delta = scores.get("openai/gpt-4o")! - base.get("openai/gpt-4o")!; + assert.ok(Math.abs(delta - 0.1) < 1e-9, `expected 0.1, got ${delta}`); +}); + +test("virtual cost helper returns real table costs, not zero", () => { + assert.equal(resolveVirtualCost("openai", "gpt-4o"), 2.5); + assert.equal(resolveVirtualCost("ai21", "jamba-large-1.7"), 0); // catalog free + assert.equal(resolveVirtualCost("openai", "gpt-9-never-existed"), 5.0); // conservative fallback +});