diff --git a/changelog.d/fixes/12731-scoring-packs-quality-reliability.md b/changelog.d/fixes/12731-scoring-packs-quality-reliability.md new file mode 100644 index 0000000000..32033d4456 --- /dev/null +++ b/changelog.d/fixes/12731-scoring-packs-quality-reliability.md @@ -0,0 +1 @@ +- **fix(auto-combo):** every mode pack now carries `quality` and `reliability` weights (`quality` 0.02, 0.03 in `quality-first`; `reliability` 0.03, 0.04 in `reliability-first`), so selecting a pack no longer silences either signal; `DEFAULT_WEIGHTS` is unchanged ([#12731](https://github.com/diegosouzapw/OmniRoute/pull/12731)) — thanks @maxmad64bis diff --git a/config/quality/eslint-suppressions.json b/config/quality/eslint-suppressions.json index 2074040010..849047598f 100644 --- a/config/quality/eslint-suppressions.json +++ b/config/quality/eslint-suppressions.json @@ -411,11 +411,6 @@ "count": 1 } }, - "open-sse/services/autoCombo/__tests__/autoCombo.test.ts": { - "@typescript-eslint/no-unused-vars": { - "count": 2 - } - }, "open-sse/services/autoCombo/chaosEngine.ts": { "@typescript-eslint/no-unused-vars": { "count": 2 diff --git a/docs/diagrams/auto-combo-scoring.mmd b/docs/diagrams/auto-combo-scoring.mmd index be1599df77..55d69cd910 100644 --- a/docs/diagrams/auto-combo-scoring.mmd +++ b/docs/diagrams/auto-combo-scoring.mmd @@ -2,7 +2,7 @@ %% Reflects: open-sse/services/autoCombo/scoring.ts (DEFAULT_WEIGHTS, sum = 1.0) %% v3.8.50 %% svg-title: OmniRoute Auto-Combo 16-factor scoring -%% svg-description: Flow from an incoming request through eligible candidates, the 15 weighted scoring factors, descending score sort, top-N selection, and sequential dispatch. +%% svg-description: Flow from an incoming request through eligible candidates, the 16 weighted scoring factors, descending score sort, top-N selection, and sequential dispatch. flowchart TB Request["Incoming request"] --> Candidates["Eligible candidates
(provider × model × account)"] Candidates --> Score["Compute composite score
per candidate"] @@ -23,6 +23,7 @@ flowchart TB f13["resetWindowAffinity (0.0000)"] f14["connectionDensity (0.0476)"] f15["quality (0.0300)"] + f16["reliability (0.0000 DEFAULT, 0.03 packs, 0.04 reliable)"] end Score --> Factors diff --git a/docs/routing/AUTO-COMBO.md b/docs/routing/AUTO-COMBO.md index e3eaf7e1c5..2b406a7b2c 100644 --- a/docs/routing/AUTO-COMBO.md +++ b/docs/routing/AUTO-COMBO.md @@ -184,7 +184,7 @@ See [#7992](https://github.com/diegosouzapw/OmniRoute/issues/7992) and [#7111](h ## How It Works (Persisted Auto-Combos) -The Auto-Combo Engine dynamically selects the best provider/model for each request using a **16-factor scoring function** (defined in `open-sse/services/autoCombo/scoring.ts` → `DEFAULT_WEIGHTS`). The default weights sum to `1.0`; custom weights are renormalized by `normalizeScoringWeights()`. Three of the sixteen — `cacheAffinity`, `resetWindowAffinity` and `reliability` — carry a default weight of `0`: they are still computed for every candidate, and `cacheAffinity` gates prompt-cache deduplication outside the score, so they are declared factors that simply do not vote by default. +The Auto-Combo Engine dynamically selects the best provider/model for each request using a **16-factor scoring function** (defined in `open-sse/services/autoCombo/scoring.ts` → `DEFAULT_WEIGHTS`). The default weights sum to `1.0`; custom weights are renormalized by `normalizeScoringWeights()`. Two of the sixteen — `cacheAffinity` and `resetWindowAffinity` — carry a default weight of `0`; `reliability` carries `0` in `DEFAULT_WEIGHTS` but `0.03` in generic packs and `0.04` in `reliability-first`, and `quality` carries `0.02` in packs (`0.03` in `quality-first`): they are still computed for every candidate, and `cacheAffinity` gates prompt-cache deduplication outside the score, so the zero-default factors simply do not vote by default while packs do. ![Auto-Combo 16-factor scoring](../diagrams/exported/auto-combo-scoring.svg) @@ -217,30 +217,32 @@ The Auto-Combo Engine dynamically selects the best provider/model for each reque | Factor | ship-fast | cost-saver | quality-first | offline-friendly | reliability-first | chaos-mode | | :-------------------- | :--------- | :--------- | :------------ | :--------------- | :---------------- | :--------- | -| `quota` | 0.1333 | 0.1333 | 0.0952 | **0.3524** | 0.1333 | 0.0476 | +| `quota` | 0.1133 | 0.1133 | 0.0752 | **0.3324** | 0.1133 | 0.0376 | | `health` | 0.2667 | 0.1810 | 0.1714 | 0.2667 | **0.3524** | **0.4000** | -| `costInv` | 0.0476 | **0.3524** | 0.0476 | 0.0952 | 0.0381 | 0.0190 | -| `latencyInv` | **0.3048** | 0.0476 | 0.0476 | 0.0476 | 0.0476 | 0.0286 | +| `costInv` | 0.0276 | **0.3324** | 0.0276 | 0.0752 | 0.0181 | 0.0140 | +| `latencyInv` | **0.3048** | 0.0476 | 0.0476 | 0.0476 | 0.0476 | 0.0186 | | `taskFit` | 0.0952 | 0.0952 | **0.3524** | 0.0000 | 0.0952 | 0.1905 | | `stability` | 0.0000 | 0.0476 | 0.1429 | 0.0952 | 0.1905 | 0.1714 | -| `tierPriority` | 0.0476 | 0.0476 | 0.0476 | 0.0476 | 0.0476 | 0.0190 | +| `tierPriority` | 0.0376 | 0.0376 | 0.0276 | 0.0376 | 0.0276 | 0.0040 | | `tierAffinity` | 0.0000 | 0.0000 | 0.0000 | 0.0000 | 0.0000 | 0.0000 | | `specificityMatch` | 0.0000 | 0.0000 | 0.0000 | 0.0000 | 0.0000 | 0.0000 | -| `contextAffinity` | 0.0095 | 0.0000 | 0.0000 | 0.0000 | 0.0000 | 0.0286 | +| `contextAffinity` | 0.0095 | 0.0000 | 0.0000 | 0.0000 | 0.0000 | 0.0186 | | `sessionAvailability` | 0.0476 | 0.0476 | 0.0476 | 0.0476 | 0.0476 | 0.0476 | | `resetWindowAffinity` | 0.0000 | 0.0000 | 0.0000 | 0.0000 | 0.0000 | 0.0000 | | `connectionDensity` | 0.0476 | 0.0476 | 0.0476 | 0.0476 | 0.0476 | 0.0476 | +| `quality` | 0.02 | 0.02 | **0.03** | 0.02 | 0.02 | 0.02 | +| `reliability` | 0.03 | 0.03 | 0.03 | 0.03 | **0.04** | 0.03 | Notes: -- **No pack sets `quality`, and a pack replaces the weight map wholesale** (`weights = pack`, not a merge). `quality` carries `0.03` in `DEFAULT_WEIGHTS`, but under any mode pack it normalizes to `0` — selecting a pack silences the observed-quality signal completely. If you want quality feedback to influence routing, leave `modePack` unset and tune the weights directly. (`cacheAffinity` is also unset by every pack, but it defaults to `0` anyway, so nothing changes there.) +- **Packs carry `quality` and `reliability`** (`quality 0.02`, `quality-first 0.03`; `reliability 0.03`, `reliability-first 0.04`) and replace the weight map wholesale (`weights = pack`, not a merge). `DEFAULT_WEIGHTS` carries `quality 0.03 / reliability 0`; selecting `balanced`/`default` keeps those defaults, selecting a pack uses the pack's values above. On a cold pool (no observations yet, so `quality 0.5` and `reliability 1`) these two factors add `+0.04` under a generic pack (`0.03 + 0.01`), `+0.045` under `quality-first` and `+0.05` under `reliability-first`. - `tierAffinity`, `specificityMatch` and `resetWindowAffinity` are explicitly `0` in every pack. - Each pack's emphasis at a glance: - **ship-fast** → latencyInv 0.3048 + health 0.2667 (low-latency, healthy connections) - - **cost-saver** → costInv 0.3524 (cheapest tokens win) - - **quality-first** → taskFit 0.3524 + stability 0.1429 (best model for the task, consistent) - - **offline-friendly** → quota 0.3524 + health 0.2667 (max headroom regardless of speed/cost) - - **reliability-first** → health 0.3524 + stability 0.1905 (fewest surprises) + - **cost-saver** → costInv 0.3324 (cheapest tokens win) + - **quality-first** → taskFit 0.3524 + stability 0.1429 + quality 0.03, the highest of any pack (best model for the task, consistent) + - **offline-friendly** → quota 0.3324 + health 0.2667 (max headroom regardless of speed/cost) + - **reliability-first** → health 0.3524 + stability 0.1905 + reliability 0.04, the highest of any pack (fewest surprises) - **chaos-mode** → health 0.4000 + taskFit 0.1905 (fault-injection profile) ### Per-Request Controls (headers) — #6023 / #6024 / #6025 / #3470 diff --git a/open-sse/services/autoCombo/__tests__/autoCombo.test.ts b/open-sse/services/autoCombo/__tests__/autoCombo.test.ts index ba4cce5b83..e1821c002f 100644 --- a/open-sse/services/autoCombo/__tests__/autoCombo.test.ts +++ b/open-sse/services/autoCombo/__tests__/autoCombo.test.ts @@ -2,7 +2,7 @@ * Unit tests for Auto-Combo Engine (Phase 5) */ -import { describe, it, expect, beforeEach, vi } from "vitest"; +import { describe, it, expect, beforeEach } from "vitest"; import { calculateFactors, calculateScore, DEFAULT_WEIGHTS, validateWeights } from "../scoring"; import type { ProviderCandidate, ScoringWeights } from "../scoring"; import { @@ -242,6 +242,115 @@ describe("Mode Packs", () => { it("undefined pack should return undefined", () => { expect(getModePack("nonexistent")).toBeUndefined(); }); + + it("every pack carries quality>0 and reliability>0 and sums to 0.9999", () => { + for (const name of getModePackNames()) { + const w = getModePack(name)!; + expect(Number(w.quality)).toBeGreaterThan(0); + expect(Number(w.reliability)).toBeGreaterThan(0); + const sum = Object.values(w).reduce((a, b) => a + Number(b), 0); + expect(sum).toBeCloseTo(0.9999, 3); + } + }); +}); + +describe("Mode pack ranking gates (cold/warm/health)", () => { + function scoreOne(candidate: ProviderCandidate, pack: ScoringWeights): number { + const factors = calculateFactors(candidate, [candidate], "coding", () => 0.5); + return calculateScore(factors, pack); + } + it("cold pool ranking unchanged (reliability 1, quality 0.5 neutrals)", () => { + const a: ProviderCandidate = { + circuitBreakerState: "CLOSED", + failureRate: undefined, + quality: undefined, + quotaRemaining: 50, + quotaTotal: 100, + costPer1MTokens: 1, + p95LatencyMs: 100, + accountTier: "pro", + latencyStdDev: 10, + }; + const b: ProviderCandidate = { + circuitBreakerState: "CLOSED", + failureRate: undefined, + quality: undefined, + quotaRemaining: 50, + quotaTotal: 100, + costPer1MTokens: 1, + p95LatencyMs: 100, + accountTier: "pro", + latencyStdDev: 10, + }; + const pack = MODE_PACKS["reliability-first"]; + expect(scoreOne(a, pack)).toBeCloseTo(scoreOne(b, pack), 5); + }); + it("warm reliability 0.01 vs 0.4 flips winner at health tie", () => { + const highFail: ProviderCandidate = { + circuitBreakerState: "CLOSED", + failureRate: 0.4, + quality: 0.5, + quotaRemaining: 50, + quotaTotal: 100, + costPer1MTokens: 1, + p95LatencyMs: 100, + accountTier: "pro", + latencyStdDev: 10, + }; + const lowFail: ProviderCandidate = { + circuitBreakerState: "CLOSED", + failureRate: 0.01, + quality: 0.5, + quotaRemaining: 50, + quotaTotal: 100, + costPer1MTokens: 1, + p95LatencyMs: 100, + accountTier: "pro", + latencyStdDev: 10, + }; + const pack = MODE_PACKS["reliability-first"]; + expect(scoreOne(lowFail, pack)).toBeGreaterThan(scoreOne(highFail, pack)); + }); + it("boundedRate NaN yields reliability 1", () => { + const c: ProviderCandidate = { + circuitBreakerState: "CLOSED", + failureRate: NaN, + quotaRemaining: 50, + quotaTotal: 100, + costPer1MTokens: 1, + p95LatencyMs: 100, + accountTier: "pro", + latencyStdDev: 10, + }; + const factors = calculateFactors(c, [c], "coding", () => 0.5); + expect(factors.reliability).toBe(1); + }); + it("health CLOSED vs HALF_OPEN still outweighs reliability gap", () => { + const healthyHighFail: ProviderCandidate = { + circuitBreakerState: "CLOSED", + failureRate: 0.4, + quality: 0.5, + quotaRemaining: 50, + quotaTotal: 100, + costPer1MTokens: 1, + p95LatencyMs: 100, + accountTier: "pro", + latencyStdDev: 10, + }; + const halfOpenLowFail: ProviderCandidate = { + circuitBreakerState: "HALF_OPEN", + failureRate: 0.01, + quality: 0.5, + quotaRemaining: 50, + quotaTotal: 100, + costPer1MTokens: 1, + p95LatencyMs: 100, + accountTier: "pro", + latencyStdDev: 10, + }; + const pack = MODE_PACKS["reliability-first"]; + expect(scoreOne(healthyHighFail, pack)).toBeGreaterThan(scoreOne(halfOpenLowFail, pack)); + }); }); describe("SLA-aware Strategy", () => { diff --git a/open-sse/services/autoCombo/modePacks.ts b/open-sse/services/autoCombo/modePacks.ts index 267dfceeec..4c725a0b8c 100644 --- a/open-sse/services/autoCombo/modePacks.ts +++ b/open-sse/services/autoCombo/modePacks.ts @@ -4,8 +4,10 @@ * Each pack optimizes for a different priority: * - ship-fast: Prioritize latency and health * - cost-saver: Prioritize cost efficiency - * - quality-first: Prioritize task fitness and stability + * - quality-first: Prioritize task fitness and stability (highest quality weight, 0.03) * - offline-friendly: Prioritize quota availability + * - reliability-first: Prioritize health+stability (highest reliability weight, 0.04) + * - chaos-mode: Fault-injection — health > stability > taskFit */ import type { ScoringWeights } from "./scoring"; @@ -14,85 +16,95 @@ export const MODE_PACKS: Record = { // Prioritize latency → health. tierPriority replaces 0.05 from stability. // tierAffinity/specificityMatch stay at 0 (manifest-routing-only weights). "ship-fast": { - quota: 0.1333, + quota: 0.1133, health: 0.2667, - costInv: 0.0476, + costInv: 0.0276, latencyInv: 0.3048, taskFit: 0.0952, stability: 0, - tierPriority: 0.0476, + tierPriority: 0.0376, tierAffinity: 0, specificityMatch: 0, contextAffinity: 0.0095, sessionAvailability: 0.0476, resetWindowAffinity: 0, connectionDensity: 0.0476, + quality: 0.02, + reliability: 0.03, }, // Prioritize cost. tierPriority replaces 0.05 from stability. "cost-saver": { - quota: 0.1333, + quota: 0.1133, health: 0.181, - costInv: 0.3524, + costInv: 0.3324, latencyInv: 0.0476, taskFit: 0.0952, stability: 0.0476, - tierPriority: 0.0476, + tierPriority: 0.0376, tierAffinity: 0, specificityMatch: 0, contextAffinity: 0, sessionAvailability: 0.0476, resetWindowAffinity: 0, connectionDensity: 0.0476, + quality: 0.02, + reliability: 0.03, }, // Prioritize task fitness. tierPriority replaces 0.05 from latencyInv. "quality-first": { - quota: 0.0952, + quota: 0.0752, health: 0.1714, - costInv: 0.0476, + costInv: 0.0276, latencyInv: 0.0476, taskFit: 0.3524, stability: 0.1429, - tierPriority: 0.0476, + tierPriority: 0.0276, tierAffinity: 0, specificityMatch: 0, contextAffinity: 0, sessionAvailability: 0.0476, resetWindowAffinity: 0, connectionDensity: 0.0476, + quality: 0.03, + reliability: 0.03, }, // Prioritize quota availability. tierPriority replaces 0.05 from taskFit. "offline-friendly": { - quota: 0.3524, + quota: 0.3324, health: 0.2667, - costInv: 0.0952, + costInv: 0.0752, latencyInv: 0.0476, taskFit: 0, stability: 0.0952, - tierPriority: 0.0476, + tierPriority: 0.0376, tierAffinity: 0, specificityMatch: 0, contextAffinity: 0, sessionAvailability: 0.0476, resetWindowAffinity: 0, connectionDensity: 0.0476, + quality: 0.02, + reliability: 0.03, }, // #4235 `:reliable` — prioritize healthy, low-variance providers (high availability). // health (circuit-breaker) + stability (latency std-dev) dominate; weights sum to ~1.0 // (re-normalized after #8940 added sessionAvailability without rebalancing — #9985). "reliability-first": { - quota: 0.1333, + quota: 0.1133, health: 0.3524, - costInv: 0.0381, + costInv: 0.0181, latencyInv: 0.0476, taskFit: 0.0952, stability: 0.1905, - tierPriority: 0.0476, + tierPriority: 0.0276, tierAffinity: 0, specificityMatch: 0, contextAffinity: 0, sessionAvailability: 0.0476, resetWindowAffinity: 0, connectionDensity: 0.0476, + quality: 0.02, + reliability: 0.04, }, // Chaos mode — priority: health > stability > taskFit > latency. // Selects top-N healthy providers for parallel dispatch. Favors providers with @@ -101,19 +113,21 @@ export const MODE_PACKS: Record = { // to picking the most stable providers); connectionDensity boosted slightly to // prefer providers with multiple accounts (more resilient to per-account rate limits). "chaos-mode": { - quota: 0.0476, + quota: 0.0376, health: 0.4, - costInv: 0.019, - latencyInv: 0.0286, + costInv: 0.014, + latencyInv: 0.0186, taskFit: 0.1905, stability: 0.1714, - tierPriority: 0.019, + tierPriority: 0.004, tierAffinity: 0, specificityMatch: 0, - contextAffinity: 0.0286, + contextAffinity: 0.0186, sessionAvailability: 0.0476, resetWindowAffinity: 0, connectionDensity: 0.0476, + quality: 0.02, + reliability: 0.03, }, }; diff --git a/open-sse/services/autoCombo/scoring.ts b/open-sse/services/autoCombo/scoring.ts index 92c4844993..295d5379b3 100644 --- a/open-sse/services/autoCombo/scoring.ts +++ b/open-sse/services/autoCombo/scoring.ts @@ -78,11 +78,12 @@ export const DEFAULT_WEIGHTS: ScoringWeights = { // the new quality signal (observed output quality over time) gets a real, // if smaller, vote. Sum remains exactly 1.0. quality: 0.03, - // Declared but silent, like `cacheAffinity` and `resetWindowAffinity`: every - // candidate already carries a measured failure rate (24h of usage history - // behind a ten-sample floor, real-time metrics otherwise) and the scorer had - // no way to read it. Which weight it deserves is a product call backed by - // measurement, so this ships at 0 and leaves the ranking exactly as it was. + // Declared but silent in DEFAULT (like `cacheAffinity`/`resetWindowAffinity`): + // every candidate already carries a measured failure rate (24h of usage + // history behind a ten-sample floor, real-time metrics otherwise) and the + // scorer had no way to read it. Which weight it deserves is a product call + // backed by measurement, so DEFAULT ships at 0 and leaves the ranking as + // it was; `reliability-first` ships at 0.04 and generic packs at 0.03. reliability: 0, }; diff --git a/scripts/check/check-docs-counts-sync.mjs b/scripts/check/check-docs-counts-sync.mjs index 20631ae10f..116d33bcc6 100644 --- a/scripts/check/check-docs-counts-sync.mjs +++ b/scripts/check/check-docs-counts-sync.mjs @@ -614,6 +614,22 @@ export function buildChecks() { files: ["docs/routing/AUTO-COMBO.md"], validate: makeModePackNamesValidator(packs), }, + { + // Every pack must pin `quality` explicitly (6 pins, 0.02/0.03) so + // no pack silently inherits a future DEFAULT. Validates live code, + // not docs — the gate loops `files` content through `validate`. + label: "mode packs pin quality explicitly (live code)", + actual: 6, + docKey: "packs quality pins", + strict: true, + files: ["open-sse/services/autoCombo/modePacks.ts"], + validate: (content) => { + const pins = (content.match(/^\s*quality:\s*0\.0\d,?\s*$/gm) ?? []).length; + return pins >= 6 + ? { ok: true, detail: `${pins} quality pins` } + : { ok: false, detail: `only ${pins} quality pins — every pack must pin quality` }; + }, + }, { label: "Provider reference total (doc vs live modules)", actual: f.providers, diff --git a/tests/unit/auto-combos-suffixes-4235.test.ts b/tests/unit/auto-combos-suffixes-4235.test.ts index e09e7e9a46..c67ebf383f 100644 --- a/tests/unit/auto-combos-suffixes-4235.test.ts +++ b/tests/unit/auto-combos-suffixes-4235.test.ts @@ -88,6 +88,16 @@ test("#4235 reliability-first mode pack exists and is normalized", () => { assert.ok(pack.health >= 0.3, "reliability-first leans on circuit-breaker health"); }); +test("every pack carries quality>0 and reliability>0", () => { + for (const [name, w] of Object.entries(modePacks.MODE_PACKS)) { + const weights = w as { quality?: unknown; reliability?: unknown } & Record; + assert.ok(Number(weights.quality) > 0, `${name} quality>0`); + assert.ok(Number(weights.reliability) > 0, `${name} reliability>0`); + const sum = Object.values(weights).reduce((a: number, b: unknown) => a + Number(b), 0); + assert.ok(Math.abs(sum - 0.9999) < 0.002, `${name} sum ~0.9999 got ${sum}`); + } +}); + test("#4235 createBuiltinAutoCombo composes tier weights for auto/coding:fast", async () => { const combo = await builtinCatalog.createBuiltinAutoCombo("auto/coding:fast", "coding:fast"); assert.equal(combo.id, "auto/coding:fast"); diff --git a/tests/unit/check-docs-counts-quality-pinning.test.ts b/tests/unit/check-docs-counts-quality-pinning.test.ts new file mode 100644 index 0000000000..d759f544ad --- /dev/null +++ b/tests/unit/check-docs-counts-quality-pinning.test.ts @@ -0,0 +1,22 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { MODE_PACKS } from "../../open-sse/services/autoCombo/modePacks.ts"; + +// Every pack pins quality explicitly so none silently inherits a future DEFAULT. +// Locks, not TDD: green on existing behavior, break on silent regression. + +test("every pack pins quality explicitly (no silent DEFAULT inheritance)", () => { + for (const [name, pack] of Object.entries(MODE_PACKS)) { + assert.ok( + typeof (pack as { quality?: unknown }).quality === "number", + `${name} must pin quality` + ); + } +}); + +test("scoring fallback stays neutral (no DEFAULT inheritance)", () => { + const src = readFileSync("open-sse/services/autoCombo/scoring.ts", "utf8"); + assert.match(src, /weights\.quality \?\? 0\)/); + assert.match(src, /factors\.quality \?\? 0\.5\)/); +});