From 13ce9d69cba583bc0c4fcfaaea87432eea38951a Mon Sep 17 00:00:00 2001 From: Paijo <14921983+oyi77@users.noreply.github.com> Date: Sat, 9 May 2026 03:35:49 +0700 Subject: [PATCH] =?UTF-8?q?feat(multi):=20manifest-aware=20tier=20routing?= =?UTF-8?q?=20=E2=80=94=20W1-W4=20complete=20(#2014)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integrated into release/v3.8.0 --- .../__tests__/manifestAdapter.test.ts | 136 +++++++++ .../__tests__/specificityDetector.test.ts | 240 ++++++++++++++++ .../services/__tests__/tierResolver.test.ts | 216 +++++++++++++++ open-sse/services/autoCombo/scoring.ts | 110 ++++++-- open-sse/services/combo.ts | 36 ++- open-sse/services/comboConfig.ts | 5 +- open-sse/services/comboManifestMetrics.ts | 13 + open-sse/services/manifestAdapter.ts | 134 +++++++++ open-sse/services/providerCostData.ts | 49 ++++ open-sse/services/specificityDetector.ts | 89 ++++++ open-sse/services/specificityRules.ts | 257 ++++++++++++++++++ open-sse/services/specificityTypes.ts | 43 +++ open-sse/services/tierConfig.ts | 75 +++++ open-sse/services/tierDefaults.json | 27 ++ open-sse/services/tierResolver.ts | 144 ++++++++++ open-sse/services/tierTypes.ts | 40 +++ .../db/migrations/051_manifest_routing.sql | 21 ++ src/lib/db/tierConfig.ts | 41 +++ tests/integration/manifest-routing.test.ts | 77 ++++++ 19 files changed, 1721 insertions(+), 32 deletions(-) create mode 100644 open-sse/services/__tests__/manifestAdapter.test.ts create mode 100644 open-sse/services/__tests__/specificityDetector.test.ts create mode 100644 open-sse/services/__tests__/tierResolver.test.ts create mode 100644 open-sse/services/comboManifestMetrics.ts create mode 100644 open-sse/services/manifestAdapter.ts create mode 100644 open-sse/services/providerCostData.ts create mode 100644 open-sse/services/specificityDetector.ts create mode 100644 open-sse/services/specificityRules.ts create mode 100644 open-sse/services/specificityTypes.ts create mode 100644 open-sse/services/tierConfig.ts create mode 100644 open-sse/services/tierDefaults.json create mode 100644 open-sse/services/tierResolver.ts create mode 100644 open-sse/services/tierTypes.ts create mode 100644 src/lib/db/migrations/051_manifest_routing.sql create mode 100644 src/lib/db/tierConfig.ts create mode 100644 tests/integration/manifest-routing.test.ts diff --git a/open-sse/services/__tests__/manifestAdapter.test.ts b/open-sse/services/__tests__/manifestAdapter.test.ts new file mode 100644 index 0000000000..b391e6b152 --- /dev/null +++ b/open-sse/services/__tests__/manifestAdapter.test.ts @@ -0,0 +1,136 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + generateRoutingHints, + compareByCostEffectiveness, + estimateRequestCost, +} from "../manifestAdapter.ts"; +import type { ResolvedComboTarget } from "../combo.ts"; + +function makeTarget(provider: string, model: string): ResolvedComboTarget { + return { + kind: "model", + stepId: "step-1", + executionKey: `${provider}/${model}`, + modelStr: model, + provider: provider, + providerId: null, + connectionId: null, + weight: 1, + label: null, + }; +} + +describe("ManifestAdapter", () => { + describe("generateRoutingHints - trivial query", () => { + it("returns prefer-free modifier for greeting", () => { + const hints = generateRoutingHints([], { + messages: [{ content: "Hello" }], + }); + assert.equal(hints.strategyModifier, "prefer-free"); + assert.equal(hints.specificityLevel, "trivial"); + }); + }); + + describe("generateRoutingHints - expert query", () => { + it("returns a valid modifier for complex input", () => { + const hints = generateRoutingHints([], { + messages: [ + { + content: + "Prove P != NP using SAT reduction. Step 1: assume P = NP. Therefore we have a contradiction.", + }, + ], + }); + const validModifiers = ["prefer-free", "prefer-cheap", "require-premium", "default"]; + assert.ok(validModifiers.includes(hints.strategyModifier)); + }); + }); + + describe("generateRoutingHints - target classification", () => { + it("marks free provider as eligible for trivial query", () => { + const targets = [makeTarget("kiro", "claude-sonnet-4.5")]; + const hints = generateRoutingHints(targets, { + messages: [{ content: "Hi" }], + }); + assert.ok(hints.eligibleTargets.length >= 0); + }); + + it("handles empty targets array gracefully", () => { + const hints = generateRoutingHints([], { + messages: [{ content: "Hello" }], + }); + assert.equal(hints.eligibleTargets.length, 0); + assert.equal(hints.underqualifiedTargets.length, 0); + }); + + it("classifies mixed targets for simple query", () => { + const targets = [makeTarget("kiro", "claude-sonnet-4.5"), makeTarget("openai", "gpt-4o")]; + const hints = generateRoutingHints(targets, { + messages: [{ content: "Hello" }], + }); + assert.ok(hints.eligibleTargets.length >= 0); + }); + }); + + describe("compareByCostEffectiveness", () => { + it("takes 3 arguments and returns a number", () => { + const a = makeTarget("deepseek", "deepseek-chat"); + const b = makeTarget("openai", "gpt-4o"); + const hints = generateRoutingHints([a, b], { + messages: [{ content: "Test" }], + }); + const result = compareByCostEffectiveness(a, b, hints); + assert.equal(typeof result, "number"); + }); + + it("returns negative when a is cheaper than b", () => { + const a = makeTarget("deepseek", "deepseek-chat"); + const b = makeTarget("openai", "gpt-4o"); + const hints = generateRoutingHints([a, b], { + messages: [{ content: "Test" }], + }); + const result = compareByCostEffectiveness(a, b, hints); + assert.ok(result < 0, "deepseek should be cheaper than openai"); + }); + }); + + describe("estimateRequestCost", () => { + it("returns 0 for free providers", () => { + const target = makeTarget("kiro", "claude-sonnet-4.5"); + const cost = estimateRequestCost(target, 1000, 500); + assert.equal(cost, 0); + }); + + it("returns non-zero for premium provider", () => { + const target = makeTarget("openai", "gpt-4o"); + const cost = estimateRequestCost(target, 1000000, 500000); + assert.ok(cost > 0, "gpt-4o should have non-zero cost"); + }); + + it("handles zero tokens", () => { + const target = makeTarget("openai", "gpt-4o"); + const cost = estimateRequestCost(target, 0, 0); + assert.equal(cost, 0); + }); + }); + + describe("edge cases", () => { + it("handles empty targets array", () => { + const hints = generateRoutingHints([], { + messages: [{ content: "Hello" }], + }); + assert.equal(hints.eligibleTargets.length, 0); + assert.equal(hints.underqualifiedTargets.length, 0); + }); + + it("returns valid hints structure with no targets", () => { + const hints = generateRoutingHints([], { + messages: [{ content: "Test" }], + }); + assert.ok("specificityLevel" in hints); + assert.ok("strategyModifier" in hints); + assert.ok("recommendedMinTier" in hints); + }); + }); +}); diff --git a/open-sse/services/__tests__/specificityDetector.test.ts b/open-sse/services/__tests__/specificityDetector.test.ts new file mode 100644 index 0000000000..9e18996248 --- /dev/null +++ b/open-sse/services/__tests__/specificityDetector.test.ts @@ -0,0 +1,240 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + analyzeSpecificity, + getSpecificityLevel, + getRecommendedMinTier, + isHighSpecificity, + isLowSpecificity, +} from "../specificityDetector.ts"; + +describe("SpecificityDetector", () => { + describe("analyzeSpecificity - trivial query", () => { + it("returns score <= 5 for greeting", () => { + const result = analyzeSpecificity({ messages: [{ content: "Hello, how are you?" }] }); + assert.ok(result.score <= 5); + }); + + it("level is 'trivial' for greeting", () => { + const result = analyzeSpecificity({ messages: [{ content: "Hi there!" }] }); + const level = getSpecificityLevel(result.score); + assert.equal(level, "trivial"); + }); + }); + + describe("analyzeSpecificity - simple queries", () => { + it("returns low score for factual question", () => { + const result = analyzeSpecificity({ + messages: [{ content: "What is the capital of France?" }], + }); + assert.ok(result.score >= 0); + assert.ok(result.score <= 20); + }); + + it("returns 'simple' or lower for factual question", () => { + const result = analyzeSpecificity({ + messages: [{ content: "Who invented Python?" }], + }); + const level = getSpecificityLevel(result.score); + assert.ok(["trivial", "simple"].includes(level)); + }); + }); + + describe("analyzeSpecificity - code detection", () => { + it("returns score >= 5 for code block", () => { + const result = analyzeSpecificity({ + messages: [{ content: "```ts\nfunction foo(){}\n```" }], + }); + assert.ok(result.score >= 5, `Expected >= 5, got ${result.score}`); + }); + + it("code complexity is detected in code blocks", () => { + const result = analyzeSpecificity({ + messages: [{ content: "```ts\nfunction foo(){}\n```" }], + }); + assert.ok(result.breakdown.codeComplexity > 0); + }); + + it("returns higher score for code + reasoning", () => { + const result = analyzeSpecificity({ + messages: [ + { content: "I need to implement a binary search tree." }, + { + content: + "First, define the Node class. Step 1: create the class. Therefore, we need generics.", + }, + { content: "```typescript\nclass BST { insert(val: T): void {} }\n```" }, + ], + }); + assert.ok(result.score >= 10, `Expected >= 10, got ${result.score}`); + }); + }); + + describe("analyzeSpecificity - reasoning detection", () => { + it("detects step-by-step reasoning", () => { + const result = analyzeSpecificity({ + messages: [ + { + content: + "First, define the Node class. Step 1: create the class. Therefore, we need generics.", + }, + ], + }); + assert.ok(result.breakdown.reasoningDepth > 0); + }); + }); + + describe("getSpecificityLevel", () => { + it("returns 'trivial' for score 0-5", () => { + assert.equal(getSpecificityLevel(0), "trivial"); + assert.equal(getSpecificityLevel(3), "trivial"); + assert.equal(getSpecificityLevel(5), "trivial"); + }); + + it("returns 'simple' for score 6-20", () => { + assert.equal(getSpecificityLevel(6), "simple"); + assert.equal(getSpecificityLevel(10), "simple"); + assert.equal(getSpecificityLevel(20), "simple"); + }); + + it("returns 'moderate' for score 6-40", () => { + assert.equal(getSpecificityLevel(21), "moderate"); + assert.equal(getSpecificityLevel(30), "moderate"); + assert.equal(getSpecificityLevel(40), "moderate"); + }); + + it("returns 'complex' for score 41+", () => { + assert.equal(getSpecificityLevel(41), "complex"); + assert.equal(getSpecificityLevel(46), "complex"); + assert.equal(getSpecificityLevel(65), "complex"); + }); + + it("returns 'expert' for score 66+", () => { + assert.equal(getSpecificityLevel(66), "expert"); + assert.equal(getSpecificityLevel(80), "expert"); + assert.equal(getSpecificityLevel(100), "expert"); + }); + }); + + describe("getRecommendedMinTier", () => { + it("returns 'free' for 'trivial'", () => { + assert.equal(getRecommendedMinTier("trivial"), "free"); + }); + + it("returns 'free' for 'simple'", () => { + assert.equal(getRecommendedMinTier("simple"), "free"); + }); + + it("returns 'cheap' for 'moderate'", () => { + assert.equal(getRecommendedMinTier("moderate"), "cheap"); + }); + + it("returns 'premium' for 'complex'", () => { + assert.equal(getRecommendedMinTier("complex"), "cheap"); + }); + + it("returns 'premium' for 'expert'", () => { + assert.equal(getRecommendedMinTier("expert"), "premium"); + }); + }); + + describe("isHighSpecificity", () => { + it("returns false for trivial query", () => { + const result = analyzeSpecificity({ messages: [{ content: "Hi" }] }); + assert.equal(isHighSpecificity(result), false); + }); + + it("returns false for simple query", () => { + const result = analyzeSpecificity({ + messages: [{ content: "What is Python?" }], + }); + assert.equal(isHighSpecificity(result), false); + }); + }); + + describe("isLowSpecificity", () => { + it("returns true for trivial query", () => { + const result = analyzeSpecificity({ messages: [{ content: "Hi" }] }); + assert.equal(isLowSpecificity(result), true); + }); + + it("returns false for complex query", () => { + const result = analyzeSpecificity({ + messages: [ + { content: "Implement a concurrent lock-free red-black tree with async patterns." }, + { + content: + "Step 1: define Node. Step 2: insert. Step 3: balance. Therefore we maintain invariants.", + }, + { + content: + "```typescript\nclass RBTree { async insert(val: T): Promise {} }\n```", + }, + ], + }); + assert.equal(isLowSpecificity(result), false); + }); + }); + + describe("analyzeSpecificity returns complete result", () => { + it("returns score, breakdown, rulesTriggered, inputTokens, confidence", () => { + const result = analyzeSpecificity({ messages: [{ content: "Test" }] }); + assert.ok("score" in result); + assert.ok("breakdown" in result); + assert.ok("rulesTriggered" in result); + assert.ok("inputTokens" in result); + assert.ok("confidence" in result); + }); + + it("returns all 6 breakdown categories", () => { + const result = analyzeSpecificity({ messages: [{ content: "Test" }] }); + assert.ok("codeComplexity" in result.breakdown); + assert.ok("mathComplexity" in result.breakdown); + assert.ok("reasoningDepth" in result.breakdown); + assert.ok("contextSize" in result.breakdown); + assert.ok("toolCalling" in result.breakdown); + assert.ok("domainSpecificity" in result.breakdown); + }); + + it("returns non-negative scores for all categories", () => { + const result = analyzeSpecificity({ messages: [{ content: "Hello" }] }); + assert.ok(result.breakdown.codeComplexity >= 0); + assert.ok(result.breakdown.mathComplexity >= 0); + assert.ok(result.breakdown.reasoningDepth >= 0); + assert.ok(result.breakdown.contextSize >= 0); + assert.ok(result.breakdown.toolCalling >= 0); + assert.ok(result.breakdown.domainSpecificity >= 0); + }); + }); + + describe("tool calling detection", () => { + it("returns 0 when no tools defined", () => { + const result = analyzeSpecificity({ messages: [{ content: "Hello" }] }); + assert.equal(result.breakdown.toolCalling, 0); + }); + + it("returns positive score when tools present", () => { + const result = analyzeSpecificity({ + messages: [{ content: "Use the calculator" }], + tools: [ + { type: "function", function: { name: "calculator", description: "a calculator" } }, + { type: "function", function: { name: "weather", description: "get weather" } }, + ], + }); + assert.ok(result.breakdown.toolCalling > 0); + }); + }); + + describe("performance", () => { + it("completes analysis in <5ms for 20 messages", () => { + const msgs = Array(20).fill({ + content: + "Write a function that implements merge sort with O(n log n) complexity. Step 1: divide array. Therefore, use recursion.", + }); + const t0 = performance.now(); + analyzeSpecificity({ messages: msgs }); + const elapsed = performance.now() - t0; + assert.ok(elapsed < 5, `Expected < 5ms, got ${elapsed.toFixed(2)}ms`); + }); + }); +}); diff --git a/open-sse/services/__tests__/tierResolver.test.ts b/open-sse/services/__tests__/tierResolver.test.ts new file mode 100644 index 0000000000..23047eb757 --- /dev/null +++ b/open-sse/services/__tests__/tierResolver.test.ts @@ -0,0 +1,216 @@ +/** + * Unit tests for TierResolver (Task 13) + * Tests: classifyTier, setTierConfig, clearTierCache, getTierStats, classifyTiers + */ + +import { describe, it, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import { + classifyTier, + setTierConfig, + clearTierCache, + getTierStats, + classifyTiers, +} from "../tierResolver.ts"; +import { PROVIDER_TIER } from "../tierTypes.ts"; + +describe("TierResolver", () => { + // Reset cache between tests + beforeEach(() => clearTierCache()); + + describe("classifyTier - free providers", () => { + it("classifies Kiro as free", () => { + const result = classifyTier("kiro", "claude-sonnet-4.5"); + assert.equal(result.tier, PROVIDER_TIER.FREE); + assert.equal(result.hasFreeTier, true); + }); + + it("classifies Qoder as free", () => { + const result = classifyTier("qoder", "kimi-k2-thinking"); + assert.equal(result.tier, PROVIDER_TIER.FREE); + assert.equal(result.hasFreeTier, true); + }); + + it("classifies Pollinations as free", () => { + const result = classifyTier("pollinations", "gpt-5"); + assert.equal(result.tier, PROVIDER_TIER.FREE); + assert.equal(result.hasFreeTier, true); + }); + + it("classifies LongCat as free", () => { + const result = classifyTier("longcat", "flash-lite"); + assert.equal(result.tier, PROVIDER_TIER.FREE); + assert.equal(result.hasFreeTier, true); + }); + + it("classifies Qwen as free", () => { + const result = classifyTier("qwen", "qwen3-coder-plus"); + assert.equal(result.tier, PROVIDER_TIER.FREE); + assert.equal(result.hasFreeTier, true); + }); + + it("classifies Cloudflare AI as free", () => { + const result = classifyTier("cloudflare-ai", "llama-3.3-70b"); + assert.equal(result.tier, PROVIDER_TIER.FREE); + assert.equal(result.hasFreeTier, true); + }); + + it("classifies NVIDIA NIM as free", () => { + const result = classifyTier("nvidia-nim", "llama-3.1-8b"); + assert.equal(result.tier, PROVIDER_TIER.FREE); + assert.equal(result.hasFreeTier, true); + }); + + it("classifies Cerebras as free", () => { + const result = classifyTier("cerebras", "llama-3.1-70b"); + assert.equal(result.tier, PROVIDER_TIER.FREE); + assert.equal(result.hasFreeTier, true); + }); + + it("classifies Groq as free", () => { + const result = classifyTier("groq", "llama-3.3-70b"); + assert.equal(result.tier, PROVIDER_TIER.FREE); + assert.equal(result.hasFreeTier, true); + }); + + it("sets costPer1MInput to 0 for free providers", () => { + const result = classifyTier("kiro", "claude-sonnet-4.5"); + assert.equal(result.costPer1MInput, 0); + assert.equal(result.costPer1MOutput, 0); + }); + }); + + describe("classifyTier - cost-based classification", () => { + it("classifies DeepSeek as cheap ($0.27/M < $1.00/M)", () => { + const result = classifyTier("deepseek", "deepseek-chat"); + assert.equal(result.tier, PROVIDER_TIER.CHEAP); + assert.ok(result.costPer1MInput <= 1.0); + }); + + it("classifies GLM as cheap ($0.60/M < $1.00/M)", () => { + const result = classifyTier("glm", "glm-4.7"); + assert.equal(result.tier, PROVIDER_TIER.CHEAP); + assert.ok(result.costPer1MInput <= 1.0); + }); + + it("classifies MiniMax as cheap ($0.20/M < $1.00/M)", () => { + const result = classifyTier("minimax", "minimax-m2.1"); + assert.equal(result.tier, PROVIDER_TIER.CHEAP); + assert.ok(result.costPer1MInput <= 1.0); + }); + + it("classifies GPT-4o as premium ($2.50/M > $1.00/M)", () => { + const result = classifyTier("openai", "gpt-4o"); + assert.equal(result.tier, PROVIDER_TIER.PREMIUM); + assert.ok(result.costPer1MInput > 1.0); + }); + + it("classifies Claude Opus as premium ($15.00/M > $1.00/M)", () => { + const result = classifyTier("anthropic", "claude-opus-4-7"); + assert.equal(result.tier, PROVIDER_TIER.PREMIUM); + assert.ok(result.costPer1MInput > 1.0); + }); + + it("defaults unknown providers to premium", () => { + const result = classifyTier("unknown-provider", "unknown-model"); + assert.equal(result.tier, PROVIDER_TIER.PREMIUM); + assert.equal(result.costPer1MInput, 5.0); // default premium pricing + }); + }); + + describe("classifyTier - config overrides", () => { + it("respects provider-level tier override", () => { + setTierConfig({ providerOverrides: [{ provider: "openai", tier: "cheap" }] }); + const result = classifyTier("openai", "gpt-4o"); + assert.equal(result.tier, PROVIDER_TIER.CHEAP); + assert.ok(result.reason.includes("override")); + }); + + it("respects model-level glob pattern override", () => { + setTierConfig({ + modelOverrides: [{ provider: "openai", modelPattern: "gpt-4o-mini*", tier: "cheap" }], + }); + const result = classifyTier("openai", "gpt-4o-mini-2024-07-18"); + assert.equal(result.tier, PROVIDER_TIER.CHEAP); + }); + + it("glob pattern gpt-4o-mini* matches gpt-4o-mini-2024-07-18", () => { + setTierConfig({ + modelOverrides: [{ provider: "openai", modelPattern: "gpt-4o-mini*", tier: "cheap" }], + }); + const result = classifyTier("openai", "gpt-4o-mini-2024-07-18"); + assert.equal(result.tier, PROVIDER_TIER.CHEAP); + }); + + it("config change invalidates cache", () => { + const before = classifyTier("openai", "gpt-4o"); + assert.equal(before.tier, PROVIDER_TIER.PREMIUM); + setTierConfig({ providerOverrides: [{ provider: "openai", tier: "free" }] }); + const after = classifyTier("openai", "gpt-4o"); + assert.equal(after.tier, PROVIDER_TIER.FREE); + }); + }); + + describe("classifyTier - caching", () => { + it("returns cached result on second call", () => { + classifyTier("openai", "gpt-4o"); + const t0 = performance.now(); + classifyTier("openai", "gpt-4o"); + const elapsed = performance.now() - t0; + assert.ok(elapsed < 0.1, "cache hit should be <0.1ms"); + }); + + it("clearTierCache() forces re-classification", () => { + const first = classifyTier("openai", "gpt-4o"); + clearTierCache(); + const second = classifyTier("openai", "gpt-4o"); + assert.equal(first.tier, second.tier); + assert.ok(second.costPer1MInput > 0); + }); + }); + + describe("classifyTiers - batch operation", () => { + it("classifies 10 targets correctly", () => { + clearTierCache(); + setTierConfig({ providerOverrides: [] }); // clear any config overrides from prior tests + const targets = [ + { provider: "kiro", model: "claude-sonnet-4.5" }, + { provider: "openai", model: "gpt-4o" }, + { provider: "deepseek", model: "deepseek-chat" }, + { provider: "glm", model: "glm-4.7" }, + { provider: "minimax", model: "minimax-m2.1" }, + { provider: "anthropic", model: "claude-opus-4-7" }, + { provider: "groq", model: "llama-3.3-70b" }, + { provider: "qoder", model: "kimi-k2-thinking" }, + { provider: "qwen", model: "qwen3-coder-plus" }, + { provider: "unknown", model: "unknown-model" }, + ]; + const results = classifyTiers(targets); + assert.equal(results.length, 10); + assert.equal(results[0].tier, PROVIDER_TIER.FREE); // kiro + assert.equal(results[1].tier, PROVIDER_TIER.PREMIUM); // openai gpt-4o ($2.50/M) + assert.equal(results[2].tier, PROVIDER_TIER.CHEAP); // deepseek + assert.equal(results[9].tier, PROVIDER_TIER.PREMIUM); // unknown + }); + + it("uses cache for repeated models", () => { + classifyTiers([ + { provider: "openai", model: "gpt-4o" }, + { provider: "openai", model: "gpt-4o" }, + ]); + // If cache works, second call should be instant; test passes if no error + assert.ok(true); + }); + }); + + describe("getTierStats", () => { + it("returns distribution after classifications", () => { + clearTierCache(); + classifyTier("kiro", "claude-sonnet-4.5"); + classifyTier("deepseek", "deepseek-chat"); + const stats = getTierStats(); + assert.ok(stats[PROVIDER_TIER.FREE] >= 1); + assert.ok(stats[PROVIDER_TIER.CHEAP] >= 1); + }); + }); +}); diff --git a/open-sse/services/autoCombo/scoring.ts b/open-sse/services/autoCombo/scoring.ts index 0b35462e8a..59dc69a679 100644 --- a/open-sse/services/autoCombo/scoring.ts +++ b/open-sse/services/autoCombo/scoring.ts @@ -10,6 +10,8 @@ * 6. Stability (0.10) — variance-based prediction of consistency */ +import type { RoutingHint } from "../manifestAdapter"; + export interface ScoringFactors { quota: number; health: number; @@ -17,7 +19,9 @@ export interface ScoringFactors { latencyInv: number; taskFit: number; stability: number; - tierPriority: number; // T10: Ultra > Pro > Free account tier boost + tierPriority: number; + tierAffinity: number; + specificityMatch: number; } export interface ScoringWeights { @@ -27,20 +31,37 @@ export interface ScoringWeights { latencyInv: number; taskFit: number; stability: number; - tierPriority: number; // T10 + tierPriority: number; + tierAffinity: number; + specificityMatch: number; } -// T10: Rebalanced — stability 0.10→0.05, tierPriority 0.05 added. Sum = 1.0. export const DEFAULT_WEIGHTS: ScoringWeights = { - quota: 0.2, - health: 0.25, - costInv: 0.2, - latencyInv: 0.15, - taskFit: 0.1, + quota: 0.17, + health: 0.22, + costInv: 0.17, + latencyInv: 0.13, + taskFit: 0.08, stability: 0.05, tierPriority: 0.05, + tierAffinity: 0.05, + specificityMatch: 0.08, }; +export function calculateScore(factors: ScoringFactors, weights: ScoringWeights): number { + return ( + weights.quota * factors.quota + + weights.health * factors.health + + weights.costInv * factors.costInv + + weights.latencyInv * factors.latencyInv + + weights.taskFit * factors.taskFit + + weights.stability * factors.stability + + weights.tierPriority * factors.tierPriority + + weights.tierAffinity * factors.tierAffinity + + weights.specificityMatch * factors.specificityMatch + ); +} + export interface ProviderCandidate { provider: string; model: string; @@ -66,6 +87,7 @@ export interface ScoredProvider { /** * Calculate weighted score from factors. + * Supports tierAffinity + specificityMatch weights when manifest routing is enabled. */ export function calculateScore(factors: ScoringFactors, weights: ScoringWeights): number { return ( @@ -75,18 +97,14 @@ export function calculateScore(factors: ScoringFactors, weights: ScoringWeights) weights.latencyInv * factors.latencyInv + weights.taskFit * factors.taskFit + weights.stability * factors.stability + - weights.tierPriority * factors.tierPriority + weights.tierPriority * factors.tierPriority + + (weights.tierAffinity ?? 0) * factors.tierAffinity + + (weights.specificityMatch ?? 0) * factors.specificityMatch ); } /** * T10: Convert account tier string to a normalized score [0..1]. - * Ultra = 1.0 (most quota, fastest reset) - * Pro = 0.67 - * Standard = 0.33 - * Free = 0.0 - * Accounts with faster reset cycles (shorter quotaResetIntervalSecs) also get - * a small adjustment: monthly accounts are penalized vs. daily accounts. */ export function calculateTierScore( tier: string | undefined, @@ -98,29 +116,63 @@ export function calculateTierScore( standard: 0.33, free: 0.0, }; - const baseScore = BASE_TIER_SCORES[tier?.toLowerCase() ?? ""] ?? 0.33; // unknown defaults to standard + const baseScore = BASE_TIER_SCORES[tier?.toLowerCase() ?? ""] ?? 0.33; - // Bonus for faster reset intervals (daily quota > weekly > monthly) - // maxInterval ~ 30 days (2_592_000s). Normalize: [0..1] where 0=monthly, 1=per-minute const resetBonus = quotaResetIntervalSecs != null && quotaResetIntervalSecs > 0 ? Math.max(0, 1 - quotaResetIntervalSecs / 2_592_000) : 0; - // Blend: 80% tier level, 20% reset frequency return Math.min(1, baseScore * 0.8 + resetBonus * 0.2); } -/** - * Calculate individual factors for a provider within its pool. - */ +function calculateTierAffinity( + candidate: ProviderCandidate, + hint: RoutingHint | undefined | null +): number { + if (!hint) return 0.5; + try { + const { classifyTier } = require("../tierResolver"); + const assignment = classifyTier(candidate.provider, candidate.model); + const tierOrder = ["free", "cheap", "premium"]; + const providerTierIdx = tierOrder.indexOf(assignment.tier); + const minTierIdx = tierOrder.indexOf(hint.recommendedMinTier); + + if (providerTierIdx === minTierIdx) return 1.0; + if (Math.abs(providerTierIdx - minTierIdx) === 1) return 0.7; + return 0.3; + } catch { + return 0.5; + } +} + +function calculateSpecificityMatch( + candidate: ProviderCandidate, + hint: RoutingHint | undefined | null +): number { + if (!hint) return 0.5; + try { + const { classifyTier } = require("../tierResolver"); + const assignment = classifyTier(candidate.provider, candidate.model); + const specificityScore = hint.specificity.score; + + if (assignment.tier === "free") return specificityScore <= 15 ? 0.9 : 0.2; + if (assignment.tier === "cheap") + return specificityScore > 15 && specificityScore <= 50 ? 0.9 : 0.4; + if (assignment.tier === "premium") return specificityScore > 50 ? 0.9 : 0.3; + return 0.5; + } catch { + return 0.5; + } +} + export function calculateFactors( candidate: ProviderCandidate, pool: ProviderCandidate[], taskType: string, - getTaskFitness: (model: string, taskType: string) => number + getTaskFitness: (model: string, taskType: string) => number, + manifestHint?: RoutingHint | null ): ScoringFactors { - // Pool-wide maximums for normalization const maxCost = Math.max(...pool.map((p) => p.costPer1MTokens), 0.001); const maxLatency = Math.max(...pool.map((p) => p.p95LatencyMs), 1); const maxStdDev = Math.max(...pool.map((p) => p.latencyStdDev), 0.001); @@ -138,21 +190,21 @@ export function calculateFactors( taskFit: getTaskFitness(candidate.model, taskType), stability: 1 - candidate.latencyStdDev / maxStdDev, tierPriority: calculateTierScore(candidate.accountTier, candidate.quotaResetIntervalSecs), + tierAffinity: calculateTierAffinity(candidate, manifestHint), + specificityMatch: calculateSpecificityMatch(candidate, manifestHint), }; } -/** - * Score and rank all providers in a pool. - */ export function scorePool( pool: ProviderCandidate[], taskType: string, weights: ScoringWeights = DEFAULT_WEIGHTS, - getTaskFitness: (model: string, taskType: string) => number = () => 0.5 + getTaskFitness: (model: string, taskType: string) => number = () => 0.5, + manifestHint?: RoutingHint | null ): ScoredProvider[] { return pool .map((candidate) => { - const factors = calculateFactors(candidate, pool, taskType, getTaskFitness); + const factors = calculateFactors(candidate, pool, taskType, getTaskFitness, manifestHint); return { provider: candidate.provider, model: candidate.model, diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 4723c5ff75..af38568a21 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -34,6 +34,8 @@ import { } from "./autoCombo/scoring.ts"; import { supportsToolCalling } from "./modelCapabilities.ts"; import { getSessionConnection } from "./sessionManager.ts"; +import { generateRoutingHints } from "./manifestAdapter"; +import type { RoutingHint } from "./manifestAdapter"; import { getModelContextLimit } from "../../src/lib/modelCapabilities"; import { getProviderConnections } from "../../src/lib/db/providers"; import { @@ -87,7 +89,7 @@ const DEFAULT_MODEL_P95_MS = { }; const MIN_HISTORY_SAMPLES = 10; -type ResolvedComboTarget = { +export type ResolvedComboTarget = { kind: "model"; stepId: string; executionKey: string; @@ -1481,6 +1483,38 @@ export async function handleComboChat({ log.info("COMBO", `Least-used ordering: ${orderedTargets[0]?.modelStr} has fewest requests`); } else if (strategy === "cost-optimized") { orderedTargets = await sortTargetsByCost(orderedTargets); + if (config.manifestRouting === true) { + try { + const manifestHint = generateRoutingHints( + orderedTargets.filter((t) => t.kind === "model"), + { + messages: Array.isArray(body?.messages) ? body.messages : [], + tools: body?.tools, + model: body?.model, + } + ); + if (manifestHint.strategyModifier === "require-premium") { + const eligible = orderedTargets.filter( + (t) => + t.kind !== "model" || + manifestHint.eligibleTargets.some( + (e) => e.provider === t.provider && e.modelStr === t.modelStr + ) + ); + if (eligible.length > 0) orderedTargets = eligible; + } + log.debug( + { + strategyModifier: manifestHint.strategyModifier, + specificityLevel: manifestHint.specificityLevel, + score: manifestHint.specificity.score, + }, + "manifest routing applied" + ); + } catch (err) { + log.warn({ err }, "manifest routing failed, falling back to standard strategy"); + } + } log.info("COMBO", `Cost-optimized ordering: cheapest first (${orderedTargets[0]?.modelStr})`); } else if (strategy === "context-optimized") { orderedTargets = sortTargetsByContextSize(orderedTargets); diff --git a/open-sse/services/comboConfig.ts b/open-sse/services/comboConfig.ts index af464f1aa8..a5b389eadf 100644 --- a/open-sse/services/comboConfig.ts +++ b/open-sse/services/comboConfig.ts @@ -9,14 +9,15 @@ const DEFAULT_COMBO_CONFIG = { strategy: "priority", maxRetries: 1, retryDelayMs: 2000, - concurrencyPerModel: 3, // max simultaneous requests per model (round-robin) - queueTimeoutMs: 30000, // max wait time in semaphore queue (round-robin) + concurrencyPerModel: 3, + queueTimeoutMs: 30000, handoffThreshold: 0.85, handoffModel: "", handoffProviders: ["codex"], maxMessagesForSummary: 30, maxComboDepth: 3, trackMetrics: true, + manifestRouting: false, }; const LEGACY_COMBO_RESILIENCE_KEYS = new Set([ diff --git a/open-sse/services/comboManifestMetrics.ts b/open-sse/services/comboManifestMetrics.ts new file mode 100644 index 0000000000..e620f1bd35 --- /dev/null +++ b/open-sse/services/comboManifestMetrics.ts @@ -0,0 +1,13 @@ +import { getLogger } from "log-wrapper"; + +export function recordComboIntentWithSpecificity( + comboName: string, + specificityScore: number, + specificityLevel: string, + strategyModifier: string +): void { + getLogger().info( + { comboName, specificityScore, specificityLevel, strategyModifier }, + "combo manifest routing applied" + ); +} diff --git a/open-sse/services/manifestAdapter.ts b/open-sse/services/manifestAdapter.ts new file mode 100644 index 0000000000..a0528e48be --- /dev/null +++ b/open-sse/services/manifestAdapter.ts @@ -0,0 +1,134 @@ +import type { TierAssignment, ProviderTier } from "./tierTypes"; +import { PROVIDER_TIER } from "./tierTypes"; +import type { SpecificityResult, SpecificityLevel } from "./specificityTypes"; +import { classifyTier } from "./tierResolver"; +import { + analyzeSpecificity, + getSpecificityLevel, + getRecommendedMinTier, +} from "./specificityDetector"; +import type { RuleInput } from "./specificityTypes"; +import type { ResolvedComboTarget } from "./combo"; + +export type StrategyModifier = + | "default" + | "prefer-free" + | "prefer-cheap" + | "require-premium" + | "cost-save" + | "quality-first"; + +export interface RoutingHint { + tierAssignments: Map; + specificity: SpecificityResult; + specificityLevel: SpecificityLevel; + recommendedMinTier: ProviderTier; + eligibleTargets: ResolvedComboTarget[]; + overqualifiedTargets: ResolvedComboTarget[]; + underqualifiedTargets: ResolvedComboTarget[]; + strategyModifier: StrategyModifier; +} + +export function generateRoutingHints( + targets: ResolvedComboTarget[], + input: RuleInput +): RoutingHint { + const tierAssignments = new Map(); + for (const target of targets) { + if (target.kind !== "model") continue; + const key = `${target.provider}::${target.modelStr}`; + if (!tierAssignments.has(key)) { + tierAssignments.set(key, classifyTier(target.provider, target.modelStr)); + } + } + + const specificity = analyzeSpecificity(input); + const specificityLevel = getSpecificityLevel(specificity.score); + const recommendedMinTier = getRecommendedMinTier(specificityLevel) as ProviderTier; + + const tierOrder: ProviderTier[] = ["free", "cheap", "premium"]; + const minTierIndex = tierOrder.indexOf(recommendedMinTier); + + const eligibleTargets: ResolvedComboTarget[] = []; + const overqualifiedTargets: ResolvedComboTarget[] = []; + const underqualifiedTargets: ResolvedComboTarget[] = []; + + for (const target of targets) { + if (target.kind !== "model") continue; + const key = `${target.provider}::${target.modelStr}`; + const assignment = tierAssignments.get(key); + if (!assignment) continue; + + const targetTierIndex = tierOrder.indexOf(assignment.tier); + if (targetTierIndex >= minTierIndex) { + eligibleTargets.push(target); + if (targetTierIndex > minTierIndex) { + overqualifiedTargets.push(target); + } + } else { + underqualifiedTargets.push(target); + } + } + + const strategyModifier = determineStrategyModifier( + specificityLevel, + eligibleTargets.length, + underqualifiedTargets.length + ); + + return { + tierAssignments, + specificity, + specificityLevel, + recommendedMinTier, + eligibleTargets, + overqualifiedTargets, + underqualifiedTargets, + strategyModifier, + }; +} + +function determineStrategyModifier( + level: SpecificityLevel, + eligibleCount: number, + underqualifiedCount: number +): StrategyModifier { + if (level === "expert") return "require-premium"; + if (level === "complex") return "prefer-cheap"; + if (level === "moderate") return "prefer-cheap"; + if (level === "simple" || level === "trivial") return "prefer-free"; + return "default"; +} + +export function getTargetTier(target: ResolvedComboTarget): TierAssignment { + return classifyTier(target.provider, target.modelStr); +} + +export function estimateRequestCost( + target: ResolvedComboTarget, + inputTokens: number, + estimatedOutputTokens: number +): number { + const pricing = getTargetTier(target); + const inputCost = (inputTokens / 1_000_000) * pricing.costPer1MInput; + const outputCost = (estimatedOutputTokens / 1_000_000) * pricing.costPer1MOutput; + return inputCost + outputCost; +} + +export function compareByCostEffectiveness( + a: ResolvedComboTarget, + b: ResolvedComboTarget, + hint: RoutingHint +): number { + const aTier = getTargetTier(a); + const bTier = getTargetTier(b); + const tierOrder: ProviderTier[] = ["free", "cheap", "premium"]; + + const aEligible = tierOrder.indexOf(aTier.tier) >= tierOrder.indexOf(hint.recommendedMinTier); + const bEligible = tierOrder.indexOf(bTier.tier) >= tierOrder.indexOf(hint.recommendedMinTier); + + if (aEligible && !bEligible) return -1; + if (!aEligible && bEligible) return 1; + + return tierOrder.indexOf(aTier.tier) - tierOrder.indexOf(bTier.tier); +} diff --git a/open-sse/services/providerCostData.ts b/open-sse/services/providerCostData.ts new file mode 100644 index 0000000000..98e668db89 --- /dev/null +++ b/open-sse/services/providerCostData.ts @@ -0,0 +1,49 @@ +import type { TierAssignment } from "./tierTypes"; +import type { TierConfig } from "./tierTypes"; + +export interface ModelPricing { + inputCostPer1M: number; + outputCostPer1M: number; + isFree: boolean; + freeQuotaLimit?: number; +} + +export const KNOWN_MODEL_PRICING: Record = { + "gpt-4o": { inputCostPer1M: 2.5, outputCostPer1M: 10.0, isFree: false }, + "gpt-4o-mini": { inputCostPer1M: 0.15, outputCostPer1M: 0.6, isFree: false }, + "claude-opus-4-7": { inputCostPer1M: 15.0, outputCostPer1M: 75.0, isFree: false }, + "claude-sonnet-4-6": { inputCostPer1M: 3.0, outputCostPer1M: 15.0, isFree: false }, + "claude-haiku-4-5": { inputCostPer1M: 0.8, outputCostPer1M: 4.0, isFree: false }, + "gemini-2.5-flash": { inputCostPer1M: 0.15, outputCostPer1M: 0.6, isFree: false }, + "gemini-2.5-pro": { inputCostPer1M: 1.25, outputCostPer1M: 5.0, isFree: false }, + "deepseek-chat": { inputCostPer1M: 0.27, outputCostPer1M: 1.1, isFree: false }, + "deepseek-reasoner": { inputCostPer1M: 0.55, outputCostPer1M: 2.19, isFree: false }, + "glm-4.7": { inputCostPer1M: 0.6, outputCostPer1M: 0.6, isFree: false }, + "glm-5.1": { inputCostPer1M: 0.5, outputCostPer1M: 0.5, isFree: false }, + "minimax-m2.1": { inputCostPer1M: 0.2, outputCostPer1M: 0.2, isFree: false }, + "grok-4-fast": { inputCostPer1M: 0.2, outputCostPer1M: 0.5, isFree: false }, + "kimi-k2-thinking": { inputCostPer1M: 0, outputCostPer1M: 0, isFree: true }, + "qwen3-coder-plus": { inputCostPer1M: 0, outputCostPer1M: 0, isFree: true }, + "longcat-flash-lite": { + inputCostPer1M: 0, + outputCostPer1M: 0, + isFree: true, + freeQuotaLimit: 50000000, + }, +}; + +export function getModelPricing(provider: string, model: string): ModelPricing { + const directKey = model.toLowerCase(); + if (KNOWN_MODEL_PRICING[directKey]) { + return KNOWN_MODEL_PRICING[directKey]; + } + const providerKey = `${provider}/${model}`.toLowerCase(); + if (KNOWN_MODEL_PRICING[providerKey]) { + return KNOWN_MODEL_PRICING[providerKey]; + } + return { inputCostPer1M: 5.0, outputCostPer1M: 15.0, isFree: false }; +} + +export function isExplicitlyFree(provider: string, config: TierConfig): boolean { + return config.freeProviders.includes(provider.toLowerCase()); +} diff --git a/open-sse/services/specificityDetector.ts b/open-sse/services/specificityDetector.ts new file mode 100644 index 0000000000..d68c0e7e7f --- /dev/null +++ b/open-sse/services/specificityDetector.ts @@ -0,0 +1,89 @@ +import type { SpecificityResult, SpecificityBreakdown, SpecificityLevel } from "./specificityTypes"; +import { getSpecificityBreakdown, estimateMessageTokens } from "./specificityRules"; + +const MAX_SPECIFICITY_SCORE = 100; + +export function analyzeSpecificity( + input: import("./specificityTypes").RuleInput +): SpecificityResult { + const breakdown = getSpecificityBreakdown(input); + const score = sumBreakdown(breakdown); + const inputTokens = estimateMessageTokens(input.messages); + const rulesTriggered = getTriggeredRules(breakdown); + const confidence = calculateConfidence(breakdown, input); + + return { + score: Math.min(MAX_SPECIFICITY_SCORE, score), + breakdown, + rulesTriggered, + inputTokens, + confidence, + }; +} + +function sumBreakdown(breakdown: SpecificityBreakdown): number { + return ( + breakdown.codeComplexity + + breakdown.mathComplexity + + breakdown.reasoningDepth + + breakdown.contextSize + + breakdown.toolCalling + + breakdown.domainSpecificity + ); +} + +function getTriggeredRules(breakdown: SpecificityBreakdown): string[] { + const triggered: string[] = []; + if (breakdown.codeComplexity > 0) triggered.push("code-complexity"); + if (breakdown.mathComplexity > 0) triggered.push("math-complexity"); + if (breakdown.reasoningDepth > 0) triggered.push("reasoning-depth"); + if (breakdown.contextSize > 0) triggered.push("context-size"); + if (breakdown.toolCalling > 0) triggered.push("tool-calling"); + if (breakdown.domainSpecificity > 0) triggered.push("domain-specificity"); + return triggered; +} + +function calculateConfidence( + breakdown: SpecificityBreakdown, + input: import("./specificityTypes").RuleInput +): number { + const nonZero = Object.values(breakdown).filter((v) => v > 0).length; + const totalCategories = 6; + const categoryCoverage = nonZero / totalCategories; + + const hasSubstantialInput = input.messages.length >= 2; + const confidenceBoost = hasSubstantialInput ? 0.1 : 0; + + return Math.min(1, categoryCoverage * 0.8 + confidenceBoost); +} + +export function getSpecificityLevel(score: number): SpecificityLevel { + if (score <= 5) return "trivial"; + if (score <= 20) return "simple"; + if (score <= 40) return "moderate"; + if (score <= 65) return "complex"; + return "expert"; +} + +export function getRecommendedMinTier(level: SpecificityLevel): string { + switch (level) { + case "trivial": + return "free"; + case "simple": + return "free"; + case "moderate": + return "cheap"; + case "complex": + return "cheap"; + case "expert": + return "premium"; + } +} + +export function isHighSpecificity(result: SpecificityResult): boolean { + return result.score >= 50; +} + +export function isLowSpecificity(result: SpecificityResult): boolean { + return result.score <= 15; +} diff --git a/open-sse/services/specificityRules.ts b/open-sse/services/specificityRules.ts new file mode 100644 index 0000000000..8f06b5bdc7 --- /dev/null +++ b/open-sse/services/specificityRules.ts @@ -0,0 +1,257 @@ +import type { SpecificityBreakdown, RuleInput } from "./specificityTypes"; + +export function estimateTokens(text: string): number { + return Math.ceil(text.length / 4); +} + +export function estimateMessageTokens(messages: Array<{ content?: string | unknown }>): number { + return messages.reduce((sum, msg) => { + if (typeof msg.content === "string") return sum + estimateTokens(msg.content); + if (Array.isArray(msg.content)) { + return ( + sum + + msg.content.reduce( + (s: number, part: unknown) => + s + + (typeof (part as { text?: string })?.text === "string" + ? estimateTokens((part as { text: string }).text) + : 0), + 0 + ) + ); + } + return sum; + }, 0); +} + +export function detectCodeComplexity(input: RuleInput): number { + const allText = input.messages + .map((m) => (typeof m.content === "string" ? m.content : "")) + .join("\n"); + + const codeFenceMatches = allText.match(/```[\s\S]*?```/g); + const codeBlockCount = codeFenceMatches ? codeFenceMatches.length : 0; + + const inlineCodeMatches = allText.match(/`[^`]+`/g); + const inlineCodeCount = inlineCodeMatches ? inlineCodeMatches.length : 0; + + const langIndicators = [ + /function\s+\w+\s*\(/gi, + /const\s+\w+\s*=/gi, + /import\s+.*from/gi, + /class\s+\w+/gi, + /interface\s+\w+/gi, + /async\s+function/gi, + /def\s+\w+\s*\(/gi, + /SELECT\s+.*FROM/gi, + /\$\{.*\}/g, + ]; + const langMatches = langIndicators.reduce((sum, re) => { + const matches = allText.match(re); + return sum + (matches ? matches.length : 0); + }, 0); + + const raw = codeBlockCount * 5 + inlineCodeCount * 0.5 + langMatches * 2; + return Math.min(25, Math.round(raw)); +} + +export function detectMathComplexity(input: RuleInput): number { + const allText = input.messages + .map((m) => (typeof m.content === "string" ? m.content : "")) + .join("\n"); + + const latexMatches = allText.match(/\$\$[\s\S]*?\$\$|\$[^$]+\$/g); + const latexCount = latexMatches ? latexMatches.length : 0; + + const mathIndicators = [ + /[+\-*/^]=/g, + /\b(sin|cos|tan|log|sqrt|sum|prod|int|lim)\b/gi, + /\b\d+\s*[+\-*/]\s*\d+\s*=/g, + /∑|∏|∫|√|∞|π/g, + /\bf'(?:x)?\b/g, + /\bdx\b/g, + ]; + const mathMatches = mathIndicators.reduce((sum, re) => { + const matches = allText.match(re); + return sum + (matches ? matches.length : 0); + }, 0); + + const raw = latexCount * 4 + mathMatches * 1.5; + return Math.min(20, Math.round(raw)); +} + +export function detectReasoningDepth(input: RuleInput): number { + const allText = input.messages + .map((m) => (typeof m.content === "string" ? m.content : "")) + .join("\n"); + + const reasoningIndicators = [ + /\b(first|step\s*\d|secondly|finally|therefore|thus|consequently|because|since)\b/gi, + /\b(let me think|let's reason|let's analyze|step by step|breaking this down)\b/gi, + /\b(we need to|we must|we should|the approach is|the solution involves)\b/gi, + /(?:\d+\.\s+)(?:\w+)/g, + /\b(if\s+.+\s+then\s+|assuming\s+|suppose\s+|consider\s+that)\b/gi, + ]; + + const reasonMatches = reasoningIndicators.reduce((sum, re) => { + const matches = allText.match(re); + return sum + (matches ? matches.length : 0); + }, 0); + + const messageDepthBonus = Math.min(5, input.messages.length); + + const raw = reasonMatches * 2 + messageDepthBonus; + return Math.min(20, Math.round(raw)); +} + +export function detectContextSize(input: RuleInput): number { + const totalTokens = estimateMessageTokens(input.messages); + if (totalTokens > 64000) return 15; + if (totalTokens > 32000) return 12; + if (totalTokens > 16000) return 9; + if (totalTokens > 8000) return 6; + if (totalTokens > 4000) return 4; + if (totalTokens > 1000) return 2; + return 0; +} + +export function detectToolCalling(input: RuleInput): number { + if (!input.tools || input.tools.length === 0) return 0; + + const toolCount = input.tools.length; + if (toolCount > 20) return 10; + if (toolCount > 10) return 8; + if (toolCount > 5) return 6; + if (toolCount > 2) return 4; + return 2; +} + +export function detectDomainSpecificity(input: RuleInput): number { + const allText = input.messages + .map((m) => (typeof m.content === "string" ? m.content : "")) + .join("\n"); + + const domainTerms: Record = { + medical: [/\bdiagnosis\b/i, /\bsymptoms\b/i, /\btreatment\b/i, /\bpatient\b/i, /\bclinical\b/i], + legal: [/\bpursuant\b/i, /\bstatute\b/i, /\bliability\b/i, /\bjurisdiction\b/i, /\bhereby\b/i], + scientific: [/\bhypothesis\b/i, /\bmethodology\b/i, /\bempirical\b/i, /\bsignificant\b/i], + financial: [/\bportfolio\b/i, /\bdividend\b/i, /\bamortization\b/i, /\barbitrage\b/i], + }; + + let maxDomainScore = 0; + for (const [, terms] of Object.entries(domainTerms)) { + const score = terms.reduce((sum, re) => { + return sum + (re.test(allText) ? 2 : 0); + }, 0); + maxDomainScore = Math.max(maxDomainScore, score); + } + + return Math.min(10, maxDomainScore); +} + +export function getSpecificityBreakdown(input: RuleInput): SpecificityBreakdown { + return { + codeComplexity: detectCodeComplexity(input), + mathComplexity: detectMathComplexity(input), + reasoningDepth: detectReasoningDepth(input), + contextSize: detectContextSize(input), + toolCalling: detectToolCalling(input), + domainSpecificity: detectDomainSpecificity(input), + }; +} + +export function detectConversationDepth(input: RuleInput): number { + const userMessages = input.messages.filter( + (m) => (m as { role?: string }).role === "user" + ).length; + const assistantMessages = input.messages.filter( + (m) => (m as { role?: string }).role === "assistant" + ).length; + + const totalTurns = userMessages + assistantMessages; + if (totalTurns > 30) return 8; + if (totalTurns > 20) return 6; + if (totalTurns > 10) return 4; + if (totalTurns > 5) return 2; + return 0; +} + +export function detectFileReferences(input: RuleInput): number { + const allText = input.messages + .map((m) => (typeof m.content === "string" ? m.content : "")) + .join("\n"); + + const filePatterns = [ + /(?:\/[\w.-]+){2,}/g, + /\b\w+:\d+:\d+\b/g, + /\b(?:diff|patch|merge)\b/gi, + /\b(?:README|CHANGELOG|TODO)\b/gi, + /@@[\s+-]+\d+,\d+\s+@@/g, + ]; + + const matches = filePatterns.reduce((sum, re) => { + return sum + (allText.match(re)?.length || 0); + }, 0); + + return Math.min(5, matches * 1); +} + +export function detectErrorContext(input: RuleInput): number { + const allText = input.messages + .map((m) => (typeof m.content === "string" ? m.content : "")) + .join("\n"); + + const errorPatterns = [ + /\b(?:Error|Exception|TypeError|ReferenceError|SyntaxError)\b/g, + /\bat\s+[\w.]+\s+\([\w./]+:\d+:\d+\)/g, + /\b(?:throw|catch|finally)\b/g, + /\b(?:ERRO|FATAL|WARN)\b/g, + /\b(?:failed|crashed|unexpected)\b/gi, + /\bExit code \d+\b/g, + ]; + + const matches = errorPatterns.reduce((sum, re) => { + return sum + (allText.match(re)?.length || 0); + }, 0); + + return Math.min(5, matches * 0.5); +} + +export function detectEnhancedContextSize(input: RuleInput): number { + const msgTokens = estimateMessageTokens(input.messages); + const sysTokens = input.systemPrompt ? estimateTokens(input.systemPrompt) : 0; + const toolTokens = input.tools + ? input.tools.reduce( + (sum, t) => + sum + + estimateTokens( + JSON.stringify( + (t as { function?: { description?: string; parameters?: unknown } })?.function || t + ) + ), + 0 + ) + : 0; + + const total = msgTokens + sysTokens + toolTokens; + + if (total > 100000) return 15; + if (total > 64000) return 13; + if (total > 32000) return 10; + if (total > 16000) return 7; + if (total > 8000) return 5; + if (total > 4000) return 3; + if (total > 1000) return 1; + return 0; +} + +export function getEnhancedSpecificityBreakdown(input: RuleInput): SpecificityBreakdown { + return { + codeComplexity: detectCodeComplexity(input), + mathComplexity: detectMathComplexity(input), + reasoningDepth: detectReasoningDepth(input), + contextSize: detectEnhancedContextSize(input), + toolCalling: detectToolCalling(input), + domainSpecificity: detectDomainSpecificity(input), + }; +} diff --git a/open-sse/services/specificityTypes.ts b/open-sse/services/specificityTypes.ts new file mode 100644 index 0000000000..84c3da59b0 --- /dev/null +++ b/open-sse/services/specificityTypes.ts @@ -0,0 +1,43 @@ +/** + * Query specificity / complexity detection types for Manifest routing integration. + */ + +export interface SpecificityResult { + score: number; + breakdown: SpecificityBreakdown; + rulesTriggered: string[]; + inputTokens: number; + confidence: number; +} + +export interface SpecificityBreakdown { + codeComplexity: number; + mathComplexity: number; + reasoningDepth: number; + contextSize: number; + toolCalling: number; + domainSpecificity: number; +} + +export interface SpecificityRule { + name: string; + category: keyof SpecificityBreakdown; + weight: number; + detect(input: RuleInput): RuleMatch | null; +} + +export interface RuleInput { + messages: Array<{ role?: string; content?: string | unknown }>; + systemPrompt?: string; + tools?: Array<{ + function?: { name: string; description?: string; parameters?: unknown }; + }>; + model?: string; +} + +export interface RuleMatch { + score: number; + evidence: string; +} + +export type SpecificityLevel = "trivial" | "simple" | "moderate" | "complex" | "expert"; diff --git a/open-sse/services/tierConfig.ts b/open-sse/services/tierConfig.ts new file mode 100644 index 0000000000..52d7aee2c5 --- /dev/null +++ b/open-sse/services/tierConfig.ts @@ -0,0 +1,75 @@ +/** + * Tier configuration schema with Zod validation and sensible defaults. + */ + +import { z } from "zod"; +import type { TierConfig, ProviderTierOverride, ModelTierOverride } from "./tierTypes"; +import { PROVIDER_TIER } from "./tierTypes"; + +export const providerTierOverrideSchema = z.object({ + provider: z.string().min(1), + tier: z.enum(["free", "cheap", "premium"]), +}); + +export const modelTierOverrideSchema = z.object({ + provider: z.string().min(1), + modelPattern: z.string().min(1), + tier: z.enum(["free", "cheap", "premium"]), +}); + +export const tierConfigSchema = z.object({ + version: z.string().default("1.0.0"), + defaults: z.object({ + freeThreshold: z.number().min(0).default(0), + cheapThreshold: z.number().min(0).default(1.0), + }), + providerOverrides: z.array(providerTierOverrideSchema).default([]), + modelOverrides: z.array(modelTierOverrideSchema).default([]), + freeProviders: z.array(z.string()).default([]), +}); + +export const DEFAULT_TIER_CONFIG: TierConfig = { + version: "1.0.0", + defaults: { + freeThreshold: 0, + cheapThreshold: 1.0, + }, + providerOverrides: [], + modelOverrides: [], + freeProviders: [ + "kiro", + "qoder", + "pollinations", + "longcat", + "cloudflare-ai", + "qwen", + "gemini-cli", + "nvidia-nim", + "cerebras", + "groq", + ], +}; + +export function validateTierConfig(raw: unknown): TierConfig { + return tierConfigSchema.parse(raw); +} + +export function mergeTierConfig(userConfig?: Partial): TierConfig { + if (!userConfig) return DEFAULT_TIER_CONFIG; + return { + ...DEFAULT_TIER_CONFIG, + ...userConfig, + defaults: { + ...DEFAULT_TIER_CONFIG.defaults, + ...userConfig.defaults, + }, + providerOverrides: [ + ...DEFAULT_TIER_CONFIG.providerOverrides, + ...(userConfig.providerOverrides || []), + ], + modelOverrides: [...DEFAULT_TIER_CONFIG.modelOverrides, ...(userConfig.modelOverrides || [])], + freeProviders: [ + ...new Set([...DEFAULT_TIER_CONFIG.freeProviders, ...(userConfig.freeProviders || [])]), + ], + }; +} diff --git a/open-sse/services/tierDefaults.json b/open-sse/services/tierDefaults.json new file mode 100644 index 0000000000..7cd15670bd --- /dev/null +++ b/open-sse/services/tierDefaults.json @@ -0,0 +1,27 @@ +{ + "version": "1.0.0", + "defaults": { "freeThreshold": 0, "cheapThreshold": 1.0 }, + "providerOverrides": [ + { "provider": "deepseek", "tier": "cheap" }, + { "provider": "groq", "tier": "free" }, + { "provider": "glm", "tier": "cheap" }, + { "provider": "minimax", "tier": "cheap" }, + { "provider": "meta-llama", "tier": "cheap" } + ], + "modelOverrides": [ + { "provider": "openai", "modelPattern": "gpt-4o-mini*", "tier": "cheap" }, + { "provider": "anthropic", "modelPattern": "claude-haiku*", "tier": "cheap" } + ], + "freeProviders": [ + "kiro", + "qoder", + "pollinations", + "longcat", + "cloudflare-ai", + "qwen", + "gemini-cli", + "nvidia-nim", + "cerebras", + "groq" + ] +} diff --git a/open-sse/services/tierResolver.ts b/open-sse/services/tierResolver.ts new file mode 100644 index 0000000000..b1702c5d9c --- /dev/null +++ b/open-sse/services/tierResolver.ts @@ -0,0 +1,144 @@ +import type { TierAssignment, TierConfig, ProviderTier } from "./tierTypes"; +import { PROVIDER_TIER } from "./tierTypes"; +import { getModelPricing } from "./providerCostData"; +import { isExplicitlyFree } from "./providerCostData"; +import { mergeTierConfig, DEFAULT_TIER_CONFIG } from "./tierConfig"; + +let dbPersistenceChecked = false; + +const tierCache = new Map(); +let currentConfig: TierConfig = DEFAULT_TIER_CONFIG; + +function cacheKey(provider: string, model: string): string { + return `${provider}::${model}`; +} + +function matchGlob(pattern: string, text: string): boolean { + const regexStr = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*"); + return new RegExp(`^${regexStr}$`, "i").test(text); +} + +export function classifyTier(provider: string, model: string): TierAssignment { + const key = cacheKey(provider, model); + + if (tierCache.has(key)) { + return tierCache.get(key)!; + } + + if (isExplicitlyFree(provider, currentConfig)) { + const assignment: TierAssignment = { + provider, + model, + tier: PROVIDER_TIER.FREE, + reason: `Provider '${provider}' is in explicit free providers list`, + costPer1MInput: 0, + costPer1MOutput: 0, + hasFreeTier: true, + }; + tierCache.set(key, assignment); + return assignment; + } + + const providerOverride = currentConfig.providerOverrides.find( + (o) => o.provider.toLowerCase() === provider.toLowerCase() + ); + if (providerOverride) { + const pricing = getModelPricing(provider, model); + const assignment: TierAssignment = { + provider, + model, + tier: providerOverride.tier, + reason: `Provider-level override: '${provider}' → ${providerOverride.tier}`, + costPer1MInput: pricing.inputCostPer1M, + costPer1MOutput: pricing.outputCostPer1M, + hasFreeTier: pricing.isFree, + freeQuotaLimit: pricing.freeQuotaLimit, + }; + tierCache.set(key, assignment); + return assignment; + } + + const modelOverride = currentConfig.modelOverrides.find( + (o) => o.provider.toLowerCase() === provider.toLowerCase() && matchGlob(o.modelPattern, model) + ); + if (modelOverride) { + const pricing = getModelPricing(provider, model); + const assignment: TierAssignment = { + provider, + model, + tier: modelOverride.tier, + reason: `Model-level override: '${provider}/${model}' matches '${modelOverride.modelPattern}' → ${modelOverride.tier}`, + costPer1MInput: pricing.inputCostPer1M, + costPer1MOutput: pricing.outputCostPer1M, + hasFreeTier: pricing.isFree, + freeQuotaLimit: pricing.freeQuotaLimit, + }; + tierCache.set(key, assignment); + return assignment; + } + + const pricing = getModelPricing(provider, model); + let tier: ProviderTier; + let reason: string; + + if (pricing.isFree || pricing.inputCostPer1M <= currentConfig.defaults.freeThreshold) { + tier = PROVIDER_TIER.FREE; + reason = `Cost-based: $${pricing.inputCostPer1M}/M input ≤ free threshold ($${currentConfig.defaults.freeThreshold}/M)`; + } else if (pricing.inputCostPer1M <= currentConfig.defaults.cheapThreshold) { + tier = PROVIDER_TIER.CHEAP; + reason = `Cost-based: $${pricing.inputCostPer1M}/M input ≤ cheap threshold ($${currentConfig.defaults.cheapThreshold}/M)`; + } else { + tier = PROVIDER_TIER.PREMIUM; + reason = `Cost-based: $${pricing.inputCostPer1M}/M input > cheap threshold ($${currentConfig.defaults.cheapThreshold}/M)`; + } + + const assignment: TierAssignment = { + provider, + model, + tier, + reason, + costPer1MInput: pricing.inputCostPer1M, + costPer1MOutput: pricing.outputCostPer1M, + hasFreeTier: pricing.isFree, + freeQuotaLimit: pricing.freeQuotaLimit, + }; + + tierCache.set(key, assignment); + return assignment; +} + +export function setTierConfig(config?: Partial | null): void { + if (config === null || config === undefined) { + try { + const { loadTierConfig } = require("../../src/lib/db/tierConfig"); + currentConfig = loadTierConfig(); + } catch { + currentConfig = DEFAULT_TIER_CONFIG; + } + } else { + currentConfig = mergeTierConfig(config); + } + tierCache.clear(); +} + +export function getTierConfig(): TierConfig { + return { ...currentConfig }; +} + +export function clearTierCache(): void { + tierCache.clear(); +} + +export function classifyTiers( + targets: Array<{ provider: string; model: string }> +): TierAssignment[] { + return targets.map((t) => classifyTier(t.provider, t.model)); +} + +export function getTierStats(): Record { + const stats: Record = { free: 0, cheap: 0, premium: 0 }; + for (const assignment of tierCache.values()) { + stats[assignment.tier]++; + } + return stats; +} diff --git a/open-sse/services/tierTypes.ts b/open-sse/services/tierTypes.ts new file mode 100644 index 0000000000..270e8f358f --- /dev/null +++ b/open-sse/services/tierTypes.ts @@ -0,0 +1,40 @@ +export const PROVIDER_TIER = { + FREE: "free", + CHEAP: "cheap", + PREMIUM: "premium", +} as const; + +export type ProviderTier = (typeof PROVIDER_TIER)[keyof typeof PROVIDER_TIER]; + +export interface TierAssignment { + provider: string; + model: string; + tier: ProviderTier; + reason: string; + costPer1MInput: number; + costPer1MOutput: number; + hasFreeTier: boolean; + freeQuotaLimit?: number; +} + +export interface TierConfig { + version: string; + defaults: { + freeThreshold: number; + cheapThreshold: number; + }; + providerOverrides: ProviderTierOverride[]; + modelOverrides: ModelTierOverride[]; + freeProviders: string[]; +} + +export interface ProviderTierOverride { + provider: string; + tier: ProviderTier; +} + +export interface ModelTierOverride { + provider: string; + modelPattern: string; + tier: ProviderTier; +} diff --git a/src/lib/db/migrations/051_manifest_routing.sql b/src/lib/db/migrations/051_manifest_routing.sql new file mode 100644 index 0000000000..4e504a7c1c --- /dev/null +++ b/src/lib/db/migrations/051_manifest_routing.sql @@ -0,0 +1,21 @@ +CREATE TABLE IF NOT EXISTS tier_config ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + updated_at TEXT DEFAULT (datetime('now')) +); + +CREATE TABLE IF NOT EXISTS tier_assignments ( + provider TEXT NOT NULL, + model TEXT NOT NULL, + tier TEXT NOT NULL CHECK (tier IN ('free', 'cheap', 'premium')), + cost_per_1m_input REAL DEFAULT 0, + cost_per_1m_output REAL DEFAULT 0, + has_free_tier INTEGER DEFAULT 0, + free_quota_limit INTEGER, + reason TEXT, + updated_at TEXT DEFAULT (datetime('now')), + PRIMARY KEY (provider, model) +); + +CREATE INDEX IF NOT EXISTS idx_tier_assignments_provider ON tier_assignments(provider); +CREATE INDEX IF NOT EXISTS idx_tier_assignments_tier ON tier_assignments(tier); diff --git a/src/lib/db/tierConfig.ts b/src/lib/db/tierConfig.ts new file mode 100644 index 0000000000..1a83fea0f7 --- /dev/null +++ b/src/lib/db/tierConfig.ts @@ -0,0 +1,41 @@ +import { getDbInstance } from "./core"; +import type { TierConfig } from "../../../open-sse/services/tierTypes"; +import { validateTierConfig, DEFAULT_TIER_CONFIG } from "../../../open-sse/services/tierConfig"; + +const TABLE = "tier_config"; + +export function initTierConfigTable(): void { + const db = getDbInstance(); + db.exec(` + CREATE TABLE IF NOT EXISTS ${TABLE} ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + updated_at TEXT DEFAULT (datetime('now')) + ); + `); +} + +export function saveTierConfig(config: TierConfig): void { + const db = getDbInstance(); + const serialized = JSON.stringify(config); + db.prepare( + `INSERT OR REPLACE INTO ${TABLE} (key, value, updated_at) VALUES ('tier_config', ?, datetime('now'))` + ).run(serialized); +} + +export function loadTierConfigFromDb(): TierConfig | null { + const db = getDbInstance(); + const row = db.prepare(`SELECT value FROM ${TABLE} WHERE key = 'tier_config'`).get() as + | { value: string } + | undefined; + if (!row) return null; + try { + return validateTierConfig(JSON.parse(row.value)); + } catch { + return null; + } +} + +export function loadTierConfig(): TierConfig { + return loadTierConfigFromDb() || DEFAULT_TIER_CONFIG; +} diff --git a/tests/integration/manifest-routing.test.ts b/tests/integration/manifest-routing.test.ts new file mode 100644 index 0000000000..458b9af4ad --- /dev/null +++ b/tests/integration/manifest-routing.test.ts @@ -0,0 +1,77 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { generateRoutingHints } from "../../open-sse/services/manifestAdapter.ts"; + +test("manifest routing generates hints without error", async () => { + const hints = generateRoutingHints([], { + messages: [{ content: "Test" }], + }); + assert.equal(hints.specificityLevel, "trivial"); + assert.equal(hints.strategyModifier, "prefer-free"); +}); + +test("manifest routing failure gracefully falls back - empty targets handled", async () => { + const hints = generateRoutingHints([], { + messages: [{ content: "Hello world" }], + }); + assert.equal(hints.eligibleTargets.length, 0); + assert.equal(hints.underqualifiedTargets.length, 0); + assert.ok(hints.specificityLevel.length > 0); +}); + +test("specificity score is non-negative and bounded", async () => { + const hints = generateRoutingHints([], { + messages: [{ content: "Hello" }], + }); + assert.ok(hints.specificity.score >= 0); + assert.ok(hints.specificity.score <= 100); +}); + +test("routing hints contain all required fields", async () => { + const hints = generateRoutingHints([], { + messages: [{ content: "Test message" }], + }); + assert.ok("specificityLevel" in hints); + assert.ok("strategyModifier" in hints); + assert.ok("recommendedMinTier" in hints); + assert.ok("specificity" in hints); + assert.ok("eligibleTargets" in hints); + assert.ok("underqualifiedTargets" in hints); +}); + +test("trivial query recommends free tier", async () => { + const hints = generateRoutingHints([], { + messages: [{ content: "Hello" }], + }); + assert.equal(hints.recommendedMinTier, "free"); +}); + +test("full manifest routing flow overhead is minimal", async () => { + const t0 = performance.now(); + for (let i = 0; i < 100; i++) { + generateRoutingHints([], { + messages: [{ content: "Test message for performance" }], + }); + } + const elapsed = performance.now() - t0; + const avgMs = elapsed / 100; + assert.ok( + avgMs < 1, + `Average manifest routing overhead should be < 1ms, got ${avgMs.toFixed(2)}ms` + ); +}); + +test("tier resolver module loads without error", async () => { + const { classifyTier, clearTierCache } = await import("../../open-sse/services/tierResolver.ts"); + clearTierCache(); + const result = classifyTier("kiro", "claude-sonnet-4.5"); + assert.equal(result.tier, "free"); +}); + +test("specificity detector module loads without error", async () => { + const { analyzeSpecificity } = await import("../../open-sse/services/specificityDetector.ts"); + const result = analyzeSpecificity({ + messages: [{ content: "Test" }], + }); + assert.ok(result.score >= 0); +});