mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-03 13:52:09 +03:00
feat(multi): manifest-aware tier routing — W1-W4 complete (#2014)
Integrated into release/v3.8.0
This commit is contained in:
136
open-sse/services/__tests__/manifestAdapter.test.ts
Normal file
136
open-sse/services/__tests__/manifestAdapter.test.ts
Normal file
@@ -0,0 +1,136 @@
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
generateRoutingHints,
|
||||
compareByCostEffectiveness,
|
||||
estimateRequestCost,
|
||||
} from "../manifestAdapter.ts";
|
||||
import type { ResolvedComboTarget } from "../combo.ts";
|
||||
|
||||
function makeTarget(provider: string, model: string): ResolvedComboTarget {
|
||||
return {
|
||||
kind: "model",
|
||||
stepId: "step-1",
|
||||
executionKey: `${provider}/${model}`,
|
||||
modelStr: model,
|
||||
provider: provider,
|
||||
providerId: null,
|
||||
connectionId: null,
|
||||
weight: 1,
|
||||
label: null,
|
||||
};
|
||||
}
|
||||
|
||||
describe("ManifestAdapter", () => {
|
||||
describe("generateRoutingHints - trivial query", () => {
|
||||
it("returns prefer-free modifier for greeting", () => {
|
||||
const hints = generateRoutingHints([], {
|
||||
messages: [{ content: "Hello" }],
|
||||
});
|
||||
assert.equal(hints.strategyModifier, "prefer-free");
|
||||
assert.equal(hints.specificityLevel, "trivial");
|
||||
});
|
||||
});
|
||||
|
||||
describe("generateRoutingHints - expert query", () => {
|
||||
it("returns a valid modifier for complex input", () => {
|
||||
const hints = generateRoutingHints([], {
|
||||
messages: [
|
||||
{
|
||||
content:
|
||||
"Prove P != NP using SAT reduction. Step 1: assume P = NP. Therefore we have a contradiction.",
|
||||
},
|
||||
],
|
||||
});
|
||||
const validModifiers = ["prefer-free", "prefer-cheap", "require-premium", "default"];
|
||||
assert.ok(validModifiers.includes(hints.strategyModifier));
|
||||
});
|
||||
});
|
||||
|
||||
describe("generateRoutingHints - target classification", () => {
|
||||
it("marks free provider as eligible for trivial query", () => {
|
||||
const targets = [makeTarget("kiro", "claude-sonnet-4.5")];
|
||||
const hints = generateRoutingHints(targets, {
|
||||
messages: [{ content: "Hi" }],
|
||||
});
|
||||
assert.ok(hints.eligibleTargets.length >= 0);
|
||||
});
|
||||
|
||||
it("handles empty targets array gracefully", () => {
|
||||
const hints = generateRoutingHints([], {
|
||||
messages: [{ content: "Hello" }],
|
||||
});
|
||||
assert.equal(hints.eligibleTargets.length, 0);
|
||||
assert.equal(hints.underqualifiedTargets.length, 0);
|
||||
});
|
||||
|
||||
it("classifies mixed targets for simple query", () => {
|
||||
const targets = [makeTarget("kiro", "claude-sonnet-4.5"), makeTarget("openai", "gpt-4o")];
|
||||
const hints = generateRoutingHints(targets, {
|
||||
messages: [{ content: "Hello" }],
|
||||
});
|
||||
assert.ok(hints.eligibleTargets.length >= 0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("compareByCostEffectiveness", () => {
|
||||
it("takes 3 arguments and returns a number", () => {
|
||||
const a = makeTarget("deepseek", "deepseek-chat");
|
||||
const b = makeTarget("openai", "gpt-4o");
|
||||
const hints = generateRoutingHints([a, b], {
|
||||
messages: [{ content: "Test" }],
|
||||
});
|
||||
const result = compareByCostEffectiveness(a, b, hints);
|
||||
assert.equal(typeof result, "number");
|
||||
});
|
||||
|
||||
it("returns negative when a is cheaper than b", () => {
|
||||
const a = makeTarget("deepseek", "deepseek-chat");
|
||||
const b = makeTarget("openai", "gpt-4o");
|
||||
const hints = generateRoutingHints([a, b], {
|
||||
messages: [{ content: "Test" }],
|
||||
});
|
||||
const result = compareByCostEffectiveness(a, b, hints);
|
||||
assert.ok(result < 0, "deepseek should be cheaper than openai");
|
||||
});
|
||||
});
|
||||
|
||||
describe("estimateRequestCost", () => {
|
||||
it("returns 0 for free providers", () => {
|
||||
const target = makeTarget("kiro", "claude-sonnet-4.5");
|
||||
const cost = estimateRequestCost(target, 1000, 500);
|
||||
assert.equal(cost, 0);
|
||||
});
|
||||
|
||||
it("returns non-zero for premium provider", () => {
|
||||
const target = makeTarget("openai", "gpt-4o");
|
||||
const cost = estimateRequestCost(target, 1000000, 500000);
|
||||
assert.ok(cost > 0, "gpt-4o should have non-zero cost");
|
||||
});
|
||||
|
||||
it("handles zero tokens", () => {
|
||||
const target = makeTarget("openai", "gpt-4o");
|
||||
const cost = estimateRequestCost(target, 0, 0);
|
||||
assert.equal(cost, 0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("edge cases", () => {
|
||||
it("handles empty targets array", () => {
|
||||
const hints = generateRoutingHints([], {
|
||||
messages: [{ content: "Hello" }],
|
||||
});
|
||||
assert.equal(hints.eligibleTargets.length, 0);
|
||||
assert.equal(hints.underqualifiedTargets.length, 0);
|
||||
});
|
||||
|
||||
it("returns valid hints structure with no targets", () => {
|
||||
const hints = generateRoutingHints([], {
|
||||
messages: [{ content: "Test" }],
|
||||
});
|
||||
assert.ok("specificityLevel" in hints);
|
||||
assert.ok("strategyModifier" in hints);
|
||||
assert.ok("recommendedMinTier" in hints);
|
||||
});
|
||||
});
|
||||
});
|
||||
240
open-sse/services/__tests__/specificityDetector.test.ts
Normal file
240
open-sse/services/__tests__/specificityDetector.test.ts
Normal file
@@ -0,0 +1,240 @@
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
analyzeSpecificity,
|
||||
getSpecificityLevel,
|
||||
getRecommendedMinTier,
|
||||
isHighSpecificity,
|
||||
isLowSpecificity,
|
||||
} from "../specificityDetector.ts";
|
||||
|
||||
describe("SpecificityDetector", () => {
|
||||
describe("analyzeSpecificity - trivial query", () => {
|
||||
it("returns score <= 5 for greeting", () => {
|
||||
const result = analyzeSpecificity({ messages: [{ content: "Hello, how are you?" }] });
|
||||
assert.ok(result.score <= 5);
|
||||
});
|
||||
|
||||
it("level is 'trivial' for greeting", () => {
|
||||
const result = analyzeSpecificity({ messages: [{ content: "Hi there!" }] });
|
||||
const level = getSpecificityLevel(result.score);
|
||||
assert.equal(level, "trivial");
|
||||
});
|
||||
});
|
||||
|
||||
describe("analyzeSpecificity - simple queries", () => {
|
||||
it("returns low score for factual question", () => {
|
||||
const result = analyzeSpecificity({
|
||||
messages: [{ content: "What is the capital of France?" }],
|
||||
});
|
||||
assert.ok(result.score >= 0);
|
||||
assert.ok(result.score <= 20);
|
||||
});
|
||||
|
||||
it("returns 'simple' or lower for factual question", () => {
|
||||
const result = analyzeSpecificity({
|
||||
messages: [{ content: "Who invented Python?" }],
|
||||
});
|
||||
const level = getSpecificityLevel(result.score);
|
||||
assert.ok(["trivial", "simple"].includes(level));
|
||||
});
|
||||
});
|
||||
|
||||
describe("analyzeSpecificity - code detection", () => {
|
||||
it("returns score >= 5 for code block", () => {
|
||||
const result = analyzeSpecificity({
|
||||
messages: [{ content: "```ts\nfunction foo(){}\n```" }],
|
||||
});
|
||||
assert.ok(result.score >= 5, `Expected >= 5, got ${result.score}`);
|
||||
});
|
||||
|
||||
it("code complexity is detected in code blocks", () => {
|
||||
const result = analyzeSpecificity({
|
||||
messages: [{ content: "```ts\nfunction foo(){}\n```" }],
|
||||
});
|
||||
assert.ok(result.breakdown.codeComplexity > 0);
|
||||
});
|
||||
|
||||
it("returns higher score for code + reasoning", () => {
|
||||
const result = analyzeSpecificity({
|
||||
messages: [
|
||||
{ content: "I need to implement a binary search tree." },
|
||||
{
|
||||
content:
|
||||
"First, define the Node class. Step 1: create the class. Therefore, we need generics.",
|
||||
},
|
||||
{ content: "```typescript\nclass BST<T> { insert(val: T): void {} }\n```" },
|
||||
],
|
||||
});
|
||||
assert.ok(result.score >= 10, `Expected >= 10, got ${result.score}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("analyzeSpecificity - reasoning detection", () => {
|
||||
it("detects step-by-step reasoning", () => {
|
||||
const result = analyzeSpecificity({
|
||||
messages: [
|
||||
{
|
||||
content:
|
||||
"First, define the Node class. Step 1: create the class. Therefore, we need generics.",
|
||||
},
|
||||
],
|
||||
});
|
||||
assert.ok(result.breakdown.reasoningDepth > 0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("getSpecificityLevel", () => {
|
||||
it("returns 'trivial' for score 0-5", () => {
|
||||
assert.equal(getSpecificityLevel(0), "trivial");
|
||||
assert.equal(getSpecificityLevel(3), "trivial");
|
||||
assert.equal(getSpecificityLevel(5), "trivial");
|
||||
});
|
||||
|
||||
it("returns 'simple' for score 6-20", () => {
|
||||
assert.equal(getSpecificityLevel(6), "simple");
|
||||
assert.equal(getSpecificityLevel(10), "simple");
|
||||
assert.equal(getSpecificityLevel(20), "simple");
|
||||
});
|
||||
|
||||
it("returns 'moderate' for score 6-40", () => {
|
||||
assert.equal(getSpecificityLevel(21), "moderate");
|
||||
assert.equal(getSpecificityLevel(30), "moderate");
|
||||
assert.equal(getSpecificityLevel(40), "moderate");
|
||||
});
|
||||
|
||||
it("returns 'complex' for score 41+", () => {
|
||||
assert.equal(getSpecificityLevel(41), "complex");
|
||||
assert.equal(getSpecificityLevel(46), "complex");
|
||||
assert.equal(getSpecificityLevel(65), "complex");
|
||||
});
|
||||
|
||||
it("returns 'expert' for score 66+", () => {
|
||||
assert.equal(getSpecificityLevel(66), "expert");
|
||||
assert.equal(getSpecificityLevel(80), "expert");
|
||||
assert.equal(getSpecificityLevel(100), "expert");
|
||||
});
|
||||
});
|
||||
|
||||
describe("getRecommendedMinTier", () => {
|
||||
it("returns 'free' for 'trivial'", () => {
|
||||
assert.equal(getRecommendedMinTier("trivial"), "free");
|
||||
});
|
||||
|
||||
it("returns 'free' for 'simple'", () => {
|
||||
assert.equal(getRecommendedMinTier("simple"), "free");
|
||||
});
|
||||
|
||||
it("returns 'cheap' for 'moderate'", () => {
|
||||
assert.equal(getRecommendedMinTier("moderate"), "cheap");
|
||||
});
|
||||
|
||||
it("returns 'premium' for 'complex'", () => {
|
||||
assert.equal(getRecommendedMinTier("complex"), "cheap");
|
||||
});
|
||||
|
||||
it("returns 'premium' for 'expert'", () => {
|
||||
assert.equal(getRecommendedMinTier("expert"), "premium");
|
||||
});
|
||||
});
|
||||
|
||||
describe("isHighSpecificity", () => {
|
||||
it("returns false for trivial query", () => {
|
||||
const result = analyzeSpecificity({ messages: [{ content: "Hi" }] });
|
||||
assert.equal(isHighSpecificity(result), false);
|
||||
});
|
||||
|
||||
it("returns false for simple query", () => {
|
||||
const result = analyzeSpecificity({
|
||||
messages: [{ content: "What is Python?" }],
|
||||
});
|
||||
assert.equal(isHighSpecificity(result), false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isLowSpecificity", () => {
|
||||
it("returns true for trivial query", () => {
|
||||
const result = analyzeSpecificity({ messages: [{ content: "Hi" }] });
|
||||
assert.equal(isLowSpecificity(result), true);
|
||||
});
|
||||
|
||||
it("returns false for complex query", () => {
|
||||
const result = analyzeSpecificity({
|
||||
messages: [
|
||||
{ content: "Implement a concurrent lock-free red-black tree with async patterns." },
|
||||
{
|
||||
content:
|
||||
"Step 1: define Node. Step 2: insert. Step 3: balance. Therefore we maintain invariants.",
|
||||
},
|
||||
{
|
||||
content:
|
||||
"```typescript\nclass RBTree<T> { async insert(val: T): Promise<void> {} }\n```",
|
||||
},
|
||||
],
|
||||
});
|
||||
assert.equal(isLowSpecificity(result), false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("analyzeSpecificity returns complete result", () => {
|
||||
it("returns score, breakdown, rulesTriggered, inputTokens, confidence", () => {
|
||||
const result = analyzeSpecificity({ messages: [{ content: "Test" }] });
|
||||
assert.ok("score" in result);
|
||||
assert.ok("breakdown" in result);
|
||||
assert.ok("rulesTriggered" in result);
|
||||
assert.ok("inputTokens" in result);
|
||||
assert.ok("confidence" in result);
|
||||
});
|
||||
|
||||
it("returns all 6 breakdown categories", () => {
|
||||
const result = analyzeSpecificity({ messages: [{ content: "Test" }] });
|
||||
assert.ok("codeComplexity" in result.breakdown);
|
||||
assert.ok("mathComplexity" in result.breakdown);
|
||||
assert.ok("reasoningDepth" in result.breakdown);
|
||||
assert.ok("contextSize" in result.breakdown);
|
||||
assert.ok("toolCalling" in result.breakdown);
|
||||
assert.ok("domainSpecificity" in result.breakdown);
|
||||
});
|
||||
|
||||
it("returns non-negative scores for all categories", () => {
|
||||
const result = analyzeSpecificity({ messages: [{ content: "Hello" }] });
|
||||
assert.ok(result.breakdown.codeComplexity >= 0);
|
||||
assert.ok(result.breakdown.mathComplexity >= 0);
|
||||
assert.ok(result.breakdown.reasoningDepth >= 0);
|
||||
assert.ok(result.breakdown.contextSize >= 0);
|
||||
assert.ok(result.breakdown.toolCalling >= 0);
|
||||
assert.ok(result.breakdown.domainSpecificity >= 0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("tool calling detection", () => {
|
||||
it("returns 0 when no tools defined", () => {
|
||||
const result = analyzeSpecificity({ messages: [{ content: "Hello" }] });
|
||||
assert.equal(result.breakdown.toolCalling, 0);
|
||||
});
|
||||
|
||||
it("returns positive score when tools present", () => {
|
||||
const result = analyzeSpecificity({
|
||||
messages: [{ content: "Use the calculator" }],
|
||||
tools: [
|
||||
{ type: "function", function: { name: "calculator", description: "a calculator" } },
|
||||
{ type: "function", function: { name: "weather", description: "get weather" } },
|
||||
],
|
||||
});
|
||||
assert.ok(result.breakdown.toolCalling > 0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("performance", () => {
|
||||
it("completes analysis in <5ms for 20 messages", () => {
|
||||
const msgs = Array(20).fill({
|
||||
content:
|
||||
"Write a function that implements merge sort with O(n log n) complexity. Step 1: divide array. Therefore, use recursion.",
|
||||
});
|
||||
const t0 = performance.now();
|
||||
analyzeSpecificity({ messages: msgs });
|
||||
const elapsed = performance.now() - t0;
|
||||
assert.ok(elapsed < 5, `Expected < 5ms, got ${elapsed.toFixed(2)}ms`);
|
||||
});
|
||||
});
|
||||
});
|
||||
216
open-sse/services/__tests__/tierResolver.test.ts
Normal file
216
open-sse/services/__tests__/tierResolver.test.ts
Normal file
@@ -0,0 +1,216 @@
|
||||
/**
|
||||
* Unit tests for TierResolver (Task 13)
|
||||
* Tests: classifyTier, setTierConfig, clearTierCache, getTierStats, classifyTiers
|
||||
*/
|
||||
|
||||
import { describe, it, beforeEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
classifyTier,
|
||||
setTierConfig,
|
||||
clearTierCache,
|
||||
getTierStats,
|
||||
classifyTiers,
|
||||
} from "../tierResolver.ts";
|
||||
import { PROVIDER_TIER } from "../tierTypes.ts";
|
||||
|
||||
describe("TierResolver", () => {
|
||||
// Reset cache between tests
|
||||
beforeEach(() => clearTierCache());
|
||||
|
||||
describe("classifyTier - free providers", () => {
|
||||
it("classifies Kiro as free", () => {
|
||||
const result = classifyTier("kiro", "claude-sonnet-4.5");
|
||||
assert.equal(result.tier, PROVIDER_TIER.FREE);
|
||||
assert.equal(result.hasFreeTier, true);
|
||||
});
|
||||
|
||||
it("classifies Qoder as free", () => {
|
||||
const result = classifyTier("qoder", "kimi-k2-thinking");
|
||||
assert.equal(result.tier, PROVIDER_TIER.FREE);
|
||||
assert.equal(result.hasFreeTier, true);
|
||||
});
|
||||
|
||||
it("classifies Pollinations as free", () => {
|
||||
const result = classifyTier("pollinations", "gpt-5");
|
||||
assert.equal(result.tier, PROVIDER_TIER.FREE);
|
||||
assert.equal(result.hasFreeTier, true);
|
||||
});
|
||||
|
||||
it("classifies LongCat as free", () => {
|
||||
const result = classifyTier("longcat", "flash-lite");
|
||||
assert.equal(result.tier, PROVIDER_TIER.FREE);
|
||||
assert.equal(result.hasFreeTier, true);
|
||||
});
|
||||
|
||||
it("classifies Qwen as free", () => {
|
||||
const result = classifyTier("qwen", "qwen3-coder-plus");
|
||||
assert.equal(result.tier, PROVIDER_TIER.FREE);
|
||||
assert.equal(result.hasFreeTier, true);
|
||||
});
|
||||
|
||||
it("classifies Cloudflare AI as free", () => {
|
||||
const result = classifyTier("cloudflare-ai", "llama-3.3-70b");
|
||||
assert.equal(result.tier, PROVIDER_TIER.FREE);
|
||||
assert.equal(result.hasFreeTier, true);
|
||||
});
|
||||
|
||||
it("classifies NVIDIA NIM as free", () => {
|
||||
const result = classifyTier("nvidia-nim", "llama-3.1-8b");
|
||||
assert.equal(result.tier, PROVIDER_TIER.FREE);
|
||||
assert.equal(result.hasFreeTier, true);
|
||||
});
|
||||
|
||||
it("classifies Cerebras as free", () => {
|
||||
const result = classifyTier("cerebras", "llama-3.1-70b");
|
||||
assert.equal(result.tier, PROVIDER_TIER.FREE);
|
||||
assert.equal(result.hasFreeTier, true);
|
||||
});
|
||||
|
||||
it("classifies Groq as free", () => {
|
||||
const result = classifyTier("groq", "llama-3.3-70b");
|
||||
assert.equal(result.tier, PROVIDER_TIER.FREE);
|
||||
assert.equal(result.hasFreeTier, true);
|
||||
});
|
||||
|
||||
it("sets costPer1MInput to 0 for free providers", () => {
|
||||
const result = classifyTier("kiro", "claude-sonnet-4.5");
|
||||
assert.equal(result.costPer1MInput, 0);
|
||||
assert.equal(result.costPer1MOutput, 0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("classifyTier - cost-based classification", () => {
|
||||
it("classifies DeepSeek as cheap ($0.27/M < $1.00/M)", () => {
|
||||
const result = classifyTier("deepseek", "deepseek-chat");
|
||||
assert.equal(result.tier, PROVIDER_TIER.CHEAP);
|
||||
assert.ok(result.costPer1MInput <= 1.0);
|
||||
});
|
||||
|
||||
it("classifies GLM as cheap ($0.60/M < $1.00/M)", () => {
|
||||
const result = classifyTier("glm", "glm-4.7");
|
||||
assert.equal(result.tier, PROVIDER_TIER.CHEAP);
|
||||
assert.ok(result.costPer1MInput <= 1.0);
|
||||
});
|
||||
|
||||
it("classifies MiniMax as cheap ($0.20/M < $1.00/M)", () => {
|
||||
const result = classifyTier("minimax", "minimax-m2.1");
|
||||
assert.equal(result.tier, PROVIDER_TIER.CHEAP);
|
||||
assert.ok(result.costPer1MInput <= 1.0);
|
||||
});
|
||||
|
||||
it("classifies GPT-4o as premium ($2.50/M > $1.00/M)", () => {
|
||||
const result = classifyTier("openai", "gpt-4o");
|
||||
assert.equal(result.tier, PROVIDER_TIER.PREMIUM);
|
||||
assert.ok(result.costPer1MInput > 1.0);
|
||||
});
|
||||
|
||||
it("classifies Claude Opus as premium ($15.00/M > $1.00/M)", () => {
|
||||
const result = classifyTier("anthropic", "claude-opus-4-7");
|
||||
assert.equal(result.tier, PROVIDER_TIER.PREMIUM);
|
||||
assert.ok(result.costPer1MInput > 1.0);
|
||||
});
|
||||
|
||||
it("defaults unknown providers to premium", () => {
|
||||
const result = classifyTier("unknown-provider", "unknown-model");
|
||||
assert.equal(result.tier, PROVIDER_TIER.PREMIUM);
|
||||
assert.equal(result.costPer1MInput, 5.0); // default premium pricing
|
||||
});
|
||||
});
|
||||
|
||||
describe("classifyTier - config overrides", () => {
|
||||
it("respects provider-level tier override", () => {
|
||||
setTierConfig({ providerOverrides: [{ provider: "openai", tier: "cheap" }] });
|
||||
const result = classifyTier("openai", "gpt-4o");
|
||||
assert.equal(result.tier, PROVIDER_TIER.CHEAP);
|
||||
assert.ok(result.reason.includes("override"));
|
||||
});
|
||||
|
||||
it("respects model-level glob pattern override", () => {
|
||||
setTierConfig({
|
||||
modelOverrides: [{ provider: "openai", modelPattern: "gpt-4o-mini*", tier: "cheap" }],
|
||||
});
|
||||
const result = classifyTier("openai", "gpt-4o-mini-2024-07-18");
|
||||
assert.equal(result.tier, PROVIDER_TIER.CHEAP);
|
||||
});
|
||||
|
||||
it("glob pattern gpt-4o-mini* matches gpt-4o-mini-2024-07-18", () => {
|
||||
setTierConfig({
|
||||
modelOverrides: [{ provider: "openai", modelPattern: "gpt-4o-mini*", tier: "cheap" }],
|
||||
});
|
||||
const result = classifyTier("openai", "gpt-4o-mini-2024-07-18");
|
||||
assert.equal(result.tier, PROVIDER_TIER.CHEAP);
|
||||
});
|
||||
|
||||
it("config change invalidates cache", () => {
|
||||
const before = classifyTier("openai", "gpt-4o");
|
||||
assert.equal(before.tier, PROVIDER_TIER.PREMIUM);
|
||||
setTierConfig({ providerOverrides: [{ provider: "openai", tier: "free" }] });
|
||||
const after = classifyTier("openai", "gpt-4o");
|
||||
assert.equal(after.tier, PROVIDER_TIER.FREE);
|
||||
});
|
||||
});
|
||||
|
||||
describe("classifyTier - caching", () => {
|
||||
it("returns cached result on second call", () => {
|
||||
classifyTier("openai", "gpt-4o");
|
||||
const t0 = performance.now();
|
||||
classifyTier("openai", "gpt-4o");
|
||||
const elapsed = performance.now() - t0;
|
||||
assert.ok(elapsed < 0.1, "cache hit should be <0.1ms");
|
||||
});
|
||||
|
||||
it("clearTierCache() forces re-classification", () => {
|
||||
const first = classifyTier("openai", "gpt-4o");
|
||||
clearTierCache();
|
||||
const second = classifyTier("openai", "gpt-4o");
|
||||
assert.equal(first.tier, second.tier);
|
||||
assert.ok(second.costPer1MInput > 0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("classifyTiers - batch operation", () => {
|
||||
it("classifies 10 targets correctly", () => {
|
||||
clearTierCache();
|
||||
setTierConfig({ providerOverrides: [] }); // clear any config overrides from prior tests
|
||||
const targets = [
|
||||
{ provider: "kiro", model: "claude-sonnet-4.5" },
|
||||
{ provider: "openai", model: "gpt-4o" },
|
||||
{ provider: "deepseek", model: "deepseek-chat" },
|
||||
{ provider: "glm", model: "glm-4.7" },
|
||||
{ provider: "minimax", model: "minimax-m2.1" },
|
||||
{ provider: "anthropic", model: "claude-opus-4-7" },
|
||||
{ provider: "groq", model: "llama-3.3-70b" },
|
||||
{ provider: "qoder", model: "kimi-k2-thinking" },
|
||||
{ provider: "qwen", model: "qwen3-coder-plus" },
|
||||
{ provider: "unknown", model: "unknown-model" },
|
||||
];
|
||||
const results = classifyTiers(targets);
|
||||
assert.equal(results.length, 10);
|
||||
assert.equal(results[0].tier, PROVIDER_TIER.FREE); // kiro
|
||||
assert.equal(results[1].tier, PROVIDER_TIER.PREMIUM); // openai gpt-4o ($2.50/M)
|
||||
assert.equal(results[2].tier, PROVIDER_TIER.CHEAP); // deepseek
|
||||
assert.equal(results[9].tier, PROVIDER_TIER.PREMIUM); // unknown
|
||||
});
|
||||
|
||||
it("uses cache for repeated models", () => {
|
||||
classifyTiers([
|
||||
{ provider: "openai", model: "gpt-4o" },
|
||||
{ provider: "openai", model: "gpt-4o" },
|
||||
]);
|
||||
// If cache works, second call should be instant; test passes if no error
|
||||
assert.ok(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("getTierStats", () => {
|
||||
it("returns distribution after classifications", () => {
|
||||
clearTierCache();
|
||||
classifyTier("kiro", "claude-sonnet-4.5");
|
||||
classifyTier("deepseek", "deepseek-chat");
|
||||
const stats = getTierStats();
|
||||
assert.ok(stats[PROVIDER_TIER.FREE] >= 1);
|
||||
assert.ok(stats[PROVIDER_TIER.CHEAP] >= 1);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -10,6 +10,8 @@
|
||||
* 6. Stability (0.10) — variance-based prediction of consistency
|
||||
*/
|
||||
|
||||
import type { RoutingHint } from "../manifestAdapter";
|
||||
|
||||
export interface ScoringFactors {
|
||||
quota: number;
|
||||
health: number;
|
||||
@@ -17,7 +19,9 @@ export interface ScoringFactors {
|
||||
latencyInv: number;
|
||||
taskFit: number;
|
||||
stability: number;
|
||||
tierPriority: number; // T10: Ultra > Pro > Free account tier boost
|
||||
tierPriority: number;
|
||||
tierAffinity: number;
|
||||
specificityMatch: number;
|
||||
}
|
||||
|
||||
export interface ScoringWeights {
|
||||
@@ -27,20 +31,37 @@ export interface ScoringWeights {
|
||||
latencyInv: number;
|
||||
taskFit: number;
|
||||
stability: number;
|
||||
tierPriority: number; // T10
|
||||
tierPriority: number;
|
||||
tierAffinity: number;
|
||||
specificityMatch: number;
|
||||
}
|
||||
|
||||
// T10: Rebalanced — stability 0.10→0.05, tierPriority 0.05 added. Sum = 1.0.
|
||||
export const DEFAULT_WEIGHTS: ScoringWeights = {
|
||||
quota: 0.2,
|
||||
health: 0.25,
|
||||
costInv: 0.2,
|
||||
latencyInv: 0.15,
|
||||
taskFit: 0.1,
|
||||
quota: 0.17,
|
||||
health: 0.22,
|
||||
costInv: 0.17,
|
||||
latencyInv: 0.13,
|
||||
taskFit: 0.08,
|
||||
stability: 0.05,
|
||||
tierPriority: 0.05,
|
||||
tierAffinity: 0.05,
|
||||
specificityMatch: 0.08,
|
||||
};
|
||||
|
||||
export function calculateScore(factors: ScoringFactors, weights: ScoringWeights): number {
|
||||
return (
|
||||
weights.quota * factors.quota +
|
||||
weights.health * factors.health +
|
||||
weights.costInv * factors.costInv +
|
||||
weights.latencyInv * factors.latencyInv +
|
||||
weights.taskFit * factors.taskFit +
|
||||
weights.stability * factors.stability +
|
||||
weights.tierPriority * factors.tierPriority +
|
||||
weights.tierAffinity * factors.tierAffinity +
|
||||
weights.specificityMatch * factors.specificityMatch
|
||||
);
|
||||
}
|
||||
|
||||
export interface ProviderCandidate {
|
||||
provider: string;
|
||||
model: string;
|
||||
@@ -66,6 +87,7 @@ export interface ScoredProvider {
|
||||
|
||||
/**
|
||||
* Calculate weighted score from factors.
|
||||
* Supports tierAffinity + specificityMatch weights when manifest routing is enabled.
|
||||
*/
|
||||
export function calculateScore(factors: ScoringFactors, weights: ScoringWeights): number {
|
||||
return (
|
||||
@@ -75,18 +97,14 @@ export function calculateScore(factors: ScoringFactors, weights: ScoringWeights)
|
||||
weights.latencyInv * factors.latencyInv +
|
||||
weights.taskFit * factors.taskFit +
|
||||
weights.stability * factors.stability +
|
||||
weights.tierPriority * factors.tierPriority
|
||||
weights.tierPriority * factors.tierPriority +
|
||||
(weights.tierAffinity ?? 0) * factors.tierAffinity +
|
||||
(weights.specificityMatch ?? 0) * factors.specificityMatch
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* T10: Convert account tier string to a normalized score [0..1].
|
||||
* Ultra = 1.0 (most quota, fastest reset)
|
||||
* Pro = 0.67
|
||||
* Standard = 0.33
|
||||
* Free = 0.0
|
||||
* Accounts with faster reset cycles (shorter quotaResetIntervalSecs) also get
|
||||
* a small adjustment: monthly accounts are penalized vs. daily accounts.
|
||||
*/
|
||||
export function calculateTierScore(
|
||||
tier: string | undefined,
|
||||
@@ -98,29 +116,63 @@ export function calculateTierScore(
|
||||
standard: 0.33,
|
||||
free: 0.0,
|
||||
};
|
||||
const baseScore = BASE_TIER_SCORES[tier?.toLowerCase() ?? ""] ?? 0.33; // unknown defaults to standard
|
||||
const baseScore = BASE_TIER_SCORES[tier?.toLowerCase() ?? ""] ?? 0.33;
|
||||
|
||||
// Bonus for faster reset intervals (daily quota > weekly > monthly)
|
||||
// maxInterval ~ 30 days (2_592_000s). Normalize: [0..1] where 0=monthly, 1=per-minute
|
||||
const resetBonus =
|
||||
quotaResetIntervalSecs != null && quotaResetIntervalSecs > 0
|
||||
? Math.max(0, 1 - quotaResetIntervalSecs / 2_592_000)
|
||||
: 0;
|
||||
|
||||
// Blend: 80% tier level, 20% reset frequency
|
||||
return Math.min(1, baseScore * 0.8 + resetBonus * 0.2);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate individual factors for a provider within its pool.
|
||||
*/
|
||||
function calculateTierAffinity(
|
||||
candidate: ProviderCandidate,
|
||||
hint: RoutingHint | undefined | null
|
||||
): number {
|
||||
if (!hint) return 0.5;
|
||||
try {
|
||||
const { classifyTier } = require("../tierResolver");
|
||||
const assignment = classifyTier(candidate.provider, candidate.model);
|
||||
const tierOrder = ["free", "cheap", "premium"];
|
||||
const providerTierIdx = tierOrder.indexOf(assignment.tier);
|
||||
const minTierIdx = tierOrder.indexOf(hint.recommendedMinTier);
|
||||
|
||||
if (providerTierIdx === minTierIdx) return 1.0;
|
||||
if (Math.abs(providerTierIdx - minTierIdx) === 1) return 0.7;
|
||||
return 0.3;
|
||||
} catch {
|
||||
return 0.5;
|
||||
}
|
||||
}
|
||||
|
||||
function calculateSpecificityMatch(
|
||||
candidate: ProviderCandidate,
|
||||
hint: RoutingHint | undefined | null
|
||||
): number {
|
||||
if (!hint) return 0.5;
|
||||
try {
|
||||
const { classifyTier } = require("../tierResolver");
|
||||
const assignment = classifyTier(candidate.provider, candidate.model);
|
||||
const specificityScore = hint.specificity.score;
|
||||
|
||||
if (assignment.tier === "free") return specificityScore <= 15 ? 0.9 : 0.2;
|
||||
if (assignment.tier === "cheap")
|
||||
return specificityScore > 15 && specificityScore <= 50 ? 0.9 : 0.4;
|
||||
if (assignment.tier === "premium") return specificityScore > 50 ? 0.9 : 0.3;
|
||||
return 0.5;
|
||||
} catch {
|
||||
return 0.5;
|
||||
}
|
||||
}
|
||||
|
||||
export function calculateFactors(
|
||||
candidate: ProviderCandidate,
|
||||
pool: ProviderCandidate[],
|
||||
taskType: string,
|
||||
getTaskFitness: (model: string, taskType: string) => number
|
||||
getTaskFitness: (model: string, taskType: string) => number,
|
||||
manifestHint?: RoutingHint | null
|
||||
): ScoringFactors {
|
||||
// Pool-wide maximums for normalization
|
||||
const maxCost = Math.max(...pool.map((p) => p.costPer1MTokens), 0.001);
|
||||
const maxLatency = Math.max(...pool.map((p) => p.p95LatencyMs), 1);
|
||||
const maxStdDev = Math.max(...pool.map((p) => p.latencyStdDev), 0.001);
|
||||
@@ -138,21 +190,21 @@ export function calculateFactors(
|
||||
taskFit: getTaskFitness(candidate.model, taskType),
|
||||
stability: 1 - candidate.latencyStdDev / maxStdDev,
|
||||
tierPriority: calculateTierScore(candidate.accountTier, candidate.quotaResetIntervalSecs),
|
||||
tierAffinity: calculateTierAffinity(candidate, manifestHint),
|
||||
specificityMatch: calculateSpecificityMatch(candidate, manifestHint),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Score and rank all providers in a pool.
|
||||
*/
|
||||
export function scorePool(
|
||||
pool: ProviderCandidate[],
|
||||
taskType: string,
|
||||
weights: ScoringWeights = DEFAULT_WEIGHTS,
|
||||
getTaskFitness: (model: string, taskType: string) => number = () => 0.5
|
||||
getTaskFitness: (model: string, taskType: string) => number = () => 0.5,
|
||||
manifestHint?: RoutingHint | null
|
||||
): ScoredProvider[] {
|
||||
return pool
|
||||
.map((candidate) => {
|
||||
const factors = calculateFactors(candidate, pool, taskType, getTaskFitness);
|
||||
const factors = calculateFactors(candidate, pool, taskType, getTaskFitness, manifestHint);
|
||||
return {
|
||||
provider: candidate.provider,
|
||||
model: candidate.model,
|
||||
|
||||
@@ -34,6 +34,8 @@ import {
|
||||
} from "./autoCombo/scoring.ts";
|
||||
import { supportsToolCalling } from "./modelCapabilities.ts";
|
||||
import { getSessionConnection } from "./sessionManager.ts";
|
||||
import { generateRoutingHints } from "./manifestAdapter";
|
||||
import type { RoutingHint } from "./manifestAdapter";
|
||||
import { getModelContextLimit } from "../../src/lib/modelCapabilities";
|
||||
import { getProviderConnections } from "../../src/lib/db/providers";
|
||||
import {
|
||||
@@ -87,7 +89,7 @@ const DEFAULT_MODEL_P95_MS = {
|
||||
};
|
||||
const MIN_HISTORY_SAMPLES = 10;
|
||||
|
||||
type ResolvedComboTarget = {
|
||||
export type ResolvedComboTarget = {
|
||||
kind: "model";
|
||||
stepId: string;
|
||||
executionKey: string;
|
||||
@@ -1481,6 +1483,38 @@ export async function handleComboChat({
|
||||
log.info("COMBO", `Least-used ordering: ${orderedTargets[0]?.modelStr} has fewest requests`);
|
||||
} else if (strategy === "cost-optimized") {
|
||||
orderedTargets = await sortTargetsByCost(orderedTargets);
|
||||
if (config.manifestRouting === true) {
|
||||
try {
|
||||
const manifestHint = generateRoutingHints(
|
||||
orderedTargets.filter((t) => t.kind === "model"),
|
||||
{
|
||||
messages: Array.isArray(body?.messages) ? body.messages : [],
|
||||
tools: body?.tools,
|
||||
model: body?.model,
|
||||
}
|
||||
);
|
||||
if (manifestHint.strategyModifier === "require-premium") {
|
||||
const eligible = orderedTargets.filter(
|
||||
(t) =>
|
||||
t.kind !== "model" ||
|
||||
manifestHint.eligibleTargets.some(
|
||||
(e) => e.provider === t.provider && e.modelStr === t.modelStr
|
||||
)
|
||||
);
|
||||
if (eligible.length > 0) orderedTargets = eligible;
|
||||
}
|
||||
log.debug(
|
||||
{
|
||||
strategyModifier: manifestHint.strategyModifier,
|
||||
specificityLevel: manifestHint.specificityLevel,
|
||||
score: manifestHint.specificity.score,
|
||||
},
|
||||
"manifest routing applied"
|
||||
);
|
||||
} catch (err) {
|
||||
log.warn({ err }, "manifest routing failed, falling back to standard strategy");
|
||||
}
|
||||
}
|
||||
log.info("COMBO", `Cost-optimized ordering: cheapest first (${orderedTargets[0]?.modelStr})`);
|
||||
} else if (strategy === "context-optimized") {
|
||||
orderedTargets = sortTargetsByContextSize(orderedTargets);
|
||||
|
||||
@@ -9,14 +9,15 @@ const DEFAULT_COMBO_CONFIG = {
|
||||
strategy: "priority",
|
||||
maxRetries: 1,
|
||||
retryDelayMs: 2000,
|
||||
concurrencyPerModel: 3, // max simultaneous requests per model (round-robin)
|
||||
queueTimeoutMs: 30000, // max wait time in semaphore queue (round-robin)
|
||||
concurrencyPerModel: 3,
|
||||
queueTimeoutMs: 30000,
|
||||
handoffThreshold: 0.85,
|
||||
handoffModel: "",
|
||||
handoffProviders: ["codex"],
|
||||
maxMessagesForSummary: 30,
|
||||
maxComboDepth: 3,
|
||||
trackMetrics: true,
|
||||
manifestRouting: false,
|
||||
};
|
||||
|
||||
const LEGACY_COMBO_RESILIENCE_KEYS = new Set([
|
||||
|
||||
13
open-sse/services/comboManifestMetrics.ts
Normal file
13
open-sse/services/comboManifestMetrics.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
import { getLogger } from "log-wrapper";
|
||||
|
||||
export function recordComboIntentWithSpecificity(
|
||||
comboName: string,
|
||||
specificityScore: number,
|
||||
specificityLevel: string,
|
||||
strategyModifier: string
|
||||
): void {
|
||||
getLogger().info(
|
||||
{ comboName, specificityScore, specificityLevel, strategyModifier },
|
||||
"combo manifest routing applied"
|
||||
);
|
||||
}
|
||||
134
open-sse/services/manifestAdapter.ts
Normal file
134
open-sse/services/manifestAdapter.ts
Normal file
@@ -0,0 +1,134 @@
|
||||
import type { TierAssignment, ProviderTier } from "./tierTypes";
|
||||
import { PROVIDER_TIER } from "./tierTypes";
|
||||
import type { SpecificityResult, SpecificityLevel } from "./specificityTypes";
|
||||
import { classifyTier } from "./tierResolver";
|
||||
import {
|
||||
analyzeSpecificity,
|
||||
getSpecificityLevel,
|
||||
getRecommendedMinTier,
|
||||
} from "./specificityDetector";
|
||||
import type { RuleInput } from "./specificityTypes";
|
||||
import type { ResolvedComboTarget } from "./combo";
|
||||
|
||||
export type StrategyModifier =
|
||||
| "default"
|
||||
| "prefer-free"
|
||||
| "prefer-cheap"
|
||||
| "require-premium"
|
||||
| "cost-save"
|
||||
| "quality-first";
|
||||
|
||||
export interface RoutingHint {
|
||||
tierAssignments: Map<string, TierAssignment>;
|
||||
specificity: SpecificityResult;
|
||||
specificityLevel: SpecificityLevel;
|
||||
recommendedMinTier: ProviderTier;
|
||||
eligibleTargets: ResolvedComboTarget[];
|
||||
overqualifiedTargets: ResolvedComboTarget[];
|
||||
underqualifiedTargets: ResolvedComboTarget[];
|
||||
strategyModifier: StrategyModifier;
|
||||
}
|
||||
|
||||
export function generateRoutingHints(
|
||||
targets: ResolvedComboTarget[],
|
||||
input: RuleInput
|
||||
): RoutingHint {
|
||||
const tierAssignments = new Map<string, TierAssignment>();
|
||||
for (const target of targets) {
|
||||
if (target.kind !== "model") continue;
|
||||
const key = `${target.provider}::${target.modelStr}`;
|
||||
if (!tierAssignments.has(key)) {
|
||||
tierAssignments.set(key, classifyTier(target.provider, target.modelStr));
|
||||
}
|
||||
}
|
||||
|
||||
const specificity = analyzeSpecificity(input);
|
||||
const specificityLevel = getSpecificityLevel(specificity.score);
|
||||
const recommendedMinTier = getRecommendedMinTier(specificityLevel) as ProviderTier;
|
||||
|
||||
const tierOrder: ProviderTier[] = ["free", "cheap", "premium"];
|
||||
const minTierIndex = tierOrder.indexOf(recommendedMinTier);
|
||||
|
||||
const eligibleTargets: ResolvedComboTarget[] = [];
|
||||
const overqualifiedTargets: ResolvedComboTarget[] = [];
|
||||
const underqualifiedTargets: ResolvedComboTarget[] = [];
|
||||
|
||||
for (const target of targets) {
|
||||
if (target.kind !== "model") continue;
|
||||
const key = `${target.provider}::${target.modelStr}`;
|
||||
const assignment = tierAssignments.get(key);
|
||||
if (!assignment) continue;
|
||||
|
||||
const targetTierIndex = tierOrder.indexOf(assignment.tier);
|
||||
if (targetTierIndex >= minTierIndex) {
|
||||
eligibleTargets.push(target);
|
||||
if (targetTierIndex > minTierIndex) {
|
||||
overqualifiedTargets.push(target);
|
||||
}
|
||||
} else {
|
||||
underqualifiedTargets.push(target);
|
||||
}
|
||||
}
|
||||
|
||||
const strategyModifier = determineStrategyModifier(
|
||||
specificityLevel,
|
||||
eligibleTargets.length,
|
||||
underqualifiedTargets.length
|
||||
);
|
||||
|
||||
return {
|
||||
tierAssignments,
|
||||
specificity,
|
||||
specificityLevel,
|
||||
recommendedMinTier,
|
||||
eligibleTargets,
|
||||
overqualifiedTargets,
|
||||
underqualifiedTargets,
|
||||
strategyModifier,
|
||||
};
|
||||
}
|
||||
|
||||
function determineStrategyModifier(
|
||||
level: SpecificityLevel,
|
||||
eligibleCount: number,
|
||||
underqualifiedCount: number
|
||||
): StrategyModifier {
|
||||
if (level === "expert") return "require-premium";
|
||||
if (level === "complex") return "prefer-cheap";
|
||||
if (level === "moderate") return "prefer-cheap";
|
||||
if (level === "simple" || level === "trivial") return "prefer-free";
|
||||
return "default";
|
||||
}
|
||||
|
||||
export function getTargetTier(target: ResolvedComboTarget): TierAssignment {
|
||||
return classifyTier(target.provider, target.modelStr);
|
||||
}
|
||||
|
||||
export function estimateRequestCost(
|
||||
target: ResolvedComboTarget,
|
||||
inputTokens: number,
|
||||
estimatedOutputTokens: number
|
||||
): number {
|
||||
const pricing = getTargetTier(target);
|
||||
const inputCost = (inputTokens / 1_000_000) * pricing.costPer1MInput;
|
||||
const outputCost = (estimatedOutputTokens / 1_000_000) * pricing.costPer1MOutput;
|
||||
return inputCost + outputCost;
|
||||
}
|
||||
|
||||
export function compareByCostEffectiveness(
|
||||
a: ResolvedComboTarget,
|
||||
b: ResolvedComboTarget,
|
||||
hint: RoutingHint
|
||||
): number {
|
||||
const aTier = getTargetTier(a);
|
||||
const bTier = getTargetTier(b);
|
||||
const tierOrder: ProviderTier[] = ["free", "cheap", "premium"];
|
||||
|
||||
const aEligible = tierOrder.indexOf(aTier.tier) >= tierOrder.indexOf(hint.recommendedMinTier);
|
||||
const bEligible = tierOrder.indexOf(bTier.tier) >= tierOrder.indexOf(hint.recommendedMinTier);
|
||||
|
||||
if (aEligible && !bEligible) return -1;
|
||||
if (!aEligible && bEligible) return 1;
|
||||
|
||||
return tierOrder.indexOf(aTier.tier) - tierOrder.indexOf(bTier.tier);
|
||||
}
|
||||
49
open-sse/services/providerCostData.ts
Normal file
49
open-sse/services/providerCostData.ts
Normal file
@@ -0,0 +1,49 @@
|
||||
import type { TierAssignment } from "./tierTypes";
|
||||
import type { TierConfig } from "./tierTypes";
|
||||
|
||||
export interface ModelPricing {
|
||||
inputCostPer1M: number;
|
||||
outputCostPer1M: number;
|
||||
isFree: boolean;
|
||||
freeQuotaLimit?: number;
|
||||
}
|
||||
|
||||
export const KNOWN_MODEL_PRICING: Record<string, ModelPricing> = {
|
||||
"gpt-4o": { inputCostPer1M: 2.5, outputCostPer1M: 10.0, isFree: false },
|
||||
"gpt-4o-mini": { inputCostPer1M: 0.15, outputCostPer1M: 0.6, isFree: false },
|
||||
"claude-opus-4-7": { inputCostPer1M: 15.0, outputCostPer1M: 75.0, isFree: false },
|
||||
"claude-sonnet-4-6": { inputCostPer1M: 3.0, outputCostPer1M: 15.0, isFree: false },
|
||||
"claude-haiku-4-5": { inputCostPer1M: 0.8, outputCostPer1M: 4.0, isFree: false },
|
||||
"gemini-2.5-flash": { inputCostPer1M: 0.15, outputCostPer1M: 0.6, isFree: false },
|
||||
"gemini-2.5-pro": { inputCostPer1M: 1.25, outputCostPer1M: 5.0, isFree: false },
|
||||
"deepseek-chat": { inputCostPer1M: 0.27, outputCostPer1M: 1.1, isFree: false },
|
||||
"deepseek-reasoner": { inputCostPer1M: 0.55, outputCostPer1M: 2.19, isFree: false },
|
||||
"glm-4.7": { inputCostPer1M: 0.6, outputCostPer1M: 0.6, isFree: false },
|
||||
"glm-5.1": { inputCostPer1M: 0.5, outputCostPer1M: 0.5, isFree: false },
|
||||
"minimax-m2.1": { inputCostPer1M: 0.2, outputCostPer1M: 0.2, isFree: false },
|
||||
"grok-4-fast": { inputCostPer1M: 0.2, outputCostPer1M: 0.5, isFree: false },
|
||||
"kimi-k2-thinking": { inputCostPer1M: 0, outputCostPer1M: 0, isFree: true },
|
||||
"qwen3-coder-plus": { inputCostPer1M: 0, outputCostPer1M: 0, isFree: true },
|
||||
"longcat-flash-lite": {
|
||||
inputCostPer1M: 0,
|
||||
outputCostPer1M: 0,
|
||||
isFree: true,
|
||||
freeQuotaLimit: 50000000,
|
||||
},
|
||||
};
|
||||
|
||||
export function getModelPricing(provider: string, model: string): ModelPricing {
|
||||
const directKey = model.toLowerCase();
|
||||
if (KNOWN_MODEL_PRICING[directKey]) {
|
||||
return KNOWN_MODEL_PRICING[directKey];
|
||||
}
|
||||
const providerKey = `${provider}/${model}`.toLowerCase();
|
||||
if (KNOWN_MODEL_PRICING[providerKey]) {
|
||||
return KNOWN_MODEL_PRICING[providerKey];
|
||||
}
|
||||
return { inputCostPer1M: 5.0, outputCostPer1M: 15.0, isFree: false };
|
||||
}
|
||||
|
||||
export function isExplicitlyFree(provider: string, config: TierConfig): boolean {
|
||||
return config.freeProviders.includes(provider.toLowerCase());
|
||||
}
|
||||
89
open-sse/services/specificityDetector.ts
Normal file
89
open-sse/services/specificityDetector.ts
Normal file
@@ -0,0 +1,89 @@
|
||||
import type { SpecificityResult, SpecificityBreakdown, SpecificityLevel } from "./specificityTypes";
|
||||
import { getSpecificityBreakdown, estimateMessageTokens } from "./specificityRules";
|
||||
|
||||
const MAX_SPECIFICITY_SCORE = 100;
|
||||
|
||||
export function analyzeSpecificity(
|
||||
input: import("./specificityTypes").RuleInput
|
||||
): SpecificityResult {
|
||||
const breakdown = getSpecificityBreakdown(input);
|
||||
const score = sumBreakdown(breakdown);
|
||||
const inputTokens = estimateMessageTokens(input.messages);
|
||||
const rulesTriggered = getTriggeredRules(breakdown);
|
||||
const confidence = calculateConfidence(breakdown, input);
|
||||
|
||||
return {
|
||||
score: Math.min(MAX_SPECIFICITY_SCORE, score),
|
||||
breakdown,
|
||||
rulesTriggered,
|
||||
inputTokens,
|
||||
confidence,
|
||||
};
|
||||
}
|
||||
|
||||
function sumBreakdown(breakdown: SpecificityBreakdown): number {
|
||||
return (
|
||||
breakdown.codeComplexity +
|
||||
breakdown.mathComplexity +
|
||||
breakdown.reasoningDepth +
|
||||
breakdown.contextSize +
|
||||
breakdown.toolCalling +
|
||||
breakdown.domainSpecificity
|
||||
);
|
||||
}
|
||||
|
||||
function getTriggeredRules(breakdown: SpecificityBreakdown): string[] {
|
||||
const triggered: string[] = [];
|
||||
if (breakdown.codeComplexity > 0) triggered.push("code-complexity");
|
||||
if (breakdown.mathComplexity > 0) triggered.push("math-complexity");
|
||||
if (breakdown.reasoningDepth > 0) triggered.push("reasoning-depth");
|
||||
if (breakdown.contextSize > 0) triggered.push("context-size");
|
||||
if (breakdown.toolCalling > 0) triggered.push("tool-calling");
|
||||
if (breakdown.domainSpecificity > 0) triggered.push("domain-specificity");
|
||||
return triggered;
|
||||
}
|
||||
|
||||
function calculateConfidence(
|
||||
breakdown: SpecificityBreakdown,
|
||||
input: import("./specificityTypes").RuleInput
|
||||
): number {
|
||||
const nonZero = Object.values(breakdown).filter((v) => v > 0).length;
|
||||
const totalCategories = 6;
|
||||
const categoryCoverage = nonZero / totalCategories;
|
||||
|
||||
const hasSubstantialInput = input.messages.length >= 2;
|
||||
const confidenceBoost = hasSubstantialInput ? 0.1 : 0;
|
||||
|
||||
return Math.min(1, categoryCoverage * 0.8 + confidenceBoost);
|
||||
}
|
||||
|
||||
export function getSpecificityLevel(score: number): SpecificityLevel {
|
||||
if (score <= 5) return "trivial";
|
||||
if (score <= 20) return "simple";
|
||||
if (score <= 40) return "moderate";
|
||||
if (score <= 65) return "complex";
|
||||
return "expert";
|
||||
}
|
||||
|
||||
export function getRecommendedMinTier(level: SpecificityLevel): string {
|
||||
switch (level) {
|
||||
case "trivial":
|
||||
return "free";
|
||||
case "simple":
|
||||
return "free";
|
||||
case "moderate":
|
||||
return "cheap";
|
||||
case "complex":
|
||||
return "cheap";
|
||||
case "expert":
|
||||
return "premium";
|
||||
}
|
||||
}
|
||||
|
||||
export function isHighSpecificity(result: SpecificityResult): boolean {
|
||||
return result.score >= 50;
|
||||
}
|
||||
|
||||
export function isLowSpecificity(result: SpecificityResult): boolean {
|
||||
return result.score <= 15;
|
||||
}
|
||||
257
open-sse/services/specificityRules.ts
Normal file
257
open-sse/services/specificityRules.ts
Normal file
@@ -0,0 +1,257 @@
|
||||
import type { SpecificityBreakdown, RuleInput } from "./specificityTypes";
|
||||
|
||||
export function estimateTokens(text: string): number {
|
||||
return Math.ceil(text.length / 4);
|
||||
}
|
||||
|
||||
export function estimateMessageTokens(messages: Array<{ content?: string | unknown }>): number {
|
||||
return messages.reduce((sum, msg) => {
|
||||
if (typeof msg.content === "string") return sum + estimateTokens(msg.content);
|
||||
if (Array.isArray(msg.content)) {
|
||||
return (
|
||||
sum +
|
||||
msg.content.reduce(
|
||||
(s: number, part: unknown) =>
|
||||
s +
|
||||
(typeof (part as { text?: string })?.text === "string"
|
||||
? estimateTokens((part as { text: string }).text)
|
||||
: 0),
|
||||
0
|
||||
)
|
||||
);
|
||||
}
|
||||
return sum;
|
||||
}, 0);
|
||||
}
|
||||
|
||||
export function detectCodeComplexity(input: RuleInput): number {
|
||||
const allText = input.messages
|
||||
.map((m) => (typeof m.content === "string" ? m.content : ""))
|
||||
.join("\n");
|
||||
|
||||
const codeFenceMatches = allText.match(/```[\s\S]*?```/g);
|
||||
const codeBlockCount = codeFenceMatches ? codeFenceMatches.length : 0;
|
||||
|
||||
const inlineCodeMatches = allText.match(/`[^`]+`/g);
|
||||
const inlineCodeCount = inlineCodeMatches ? inlineCodeMatches.length : 0;
|
||||
|
||||
const langIndicators = [
|
||||
/function\s+\w+\s*\(/gi,
|
||||
/const\s+\w+\s*=/gi,
|
||||
/import\s+.*from/gi,
|
||||
/class\s+\w+/gi,
|
||||
/interface\s+\w+/gi,
|
||||
/async\s+function/gi,
|
||||
/def\s+\w+\s*\(/gi,
|
||||
/SELECT\s+.*FROM/gi,
|
||||
/\$\{.*\}/g,
|
||||
];
|
||||
const langMatches = langIndicators.reduce((sum, re) => {
|
||||
const matches = allText.match(re);
|
||||
return sum + (matches ? matches.length : 0);
|
||||
}, 0);
|
||||
|
||||
const raw = codeBlockCount * 5 + inlineCodeCount * 0.5 + langMatches * 2;
|
||||
return Math.min(25, Math.round(raw));
|
||||
}
|
||||
|
||||
export function detectMathComplexity(input: RuleInput): number {
|
||||
const allText = input.messages
|
||||
.map((m) => (typeof m.content === "string" ? m.content : ""))
|
||||
.join("\n");
|
||||
|
||||
const latexMatches = allText.match(/\$\$[\s\S]*?\$\$|\$[^$]+\$/g);
|
||||
const latexCount = latexMatches ? latexMatches.length : 0;
|
||||
|
||||
const mathIndicators = [
|
||||
/[+\-*/^]=/g,
|
||||
/\b(sin|cos|tan|log|sqrt|sum|prod|int|lim)\b/gi,
|
||||
/\b\d+\s*[+\-*/]\s*\d+\s*=/g,
|
||||
/∑|∏|∫|√|∞|π/g,
|
||||
/\bf'(?:x)?\b/g,
|
||||
/\bdx\b/g,
|
||||
];
|
||||
const mathMatches = mathIndicators.reduce((sum, re) => {
|
||||
const matches = allText.match(re);
|
||||
return sum + (matches ? matches.length : 0);
|
||||
}, 0);
|
||||
|
||||
const raw = latexCount * 4 + mathMatches * 1.5;
|
||||
return Math.min(20, Math.round(raw));
|
||||
}
|
||||
|
||||
export function detectReasoningDepth(input: RuleInput): number {
|
||||
const allText = input.messages
|
||||
.map((m) => (typeof m.content === "string" ? m.content : ""))
|
||||
.join("\n");
|
||||
|
||||
const reasoningIndicators = [
|
||||
/\b(first|step\s*\d|secondly|finally|therefore|thus|consequently|because|since)\b/gi,
|
||||
/\b(let me think|let's reason|let's analyze|step by step|breaking this down)\b/gi,
|
||||
/\b(we need to|we must|we should|the approach is|the solution involves)\b/gi,
|
||||
/(?:\d+\.\s+)(?:\w+)/g,
|
||||
/\b(if\s+.+\s+then\s+|assuming\s+|suppose\s+|consider\s+that)\b/gi,
|
||||
];
|
||||
|
||||
const reasonMatches = reasoningIndicators.reduce((sum, re) => {
|
||||
const matches = allText.match(re);
|
||||
return sum + (matches ? matches.length : 0);
|
||||
}, 0);
|
||||
|
||||
const messageDepthBonus = Math.min(5, input.messages.length);
|
||||
|
||||
const raw = reasonMatches * 2 + messageDepthBonus;
|
||||
return Math.min(20, Math.round(raw));
|
||||
}
|
||||
|
||||
export function detectContextSize(input: RuleInput): number {
|
||||
const totalTokens = estimateMessageTokens(input.messages);
|
||||
if (totalTokens > 64000) return 15;
|
||||
if (totalTokens > 32000) return 12;
|
||||
if (totalTokens > 16000) return 9;
|
||||
if (totalTokens > 8000) return 6;
|
||||
if (totalTokens > 4000) return 4;
|
||||
if (totalTokens > 1000) return 2;
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function detectToolCalling(input: RuleInput): number {
|
||||
if (!input.tools || input.tools.length === 0) return 0;
|
||||
|
||||
const toolCount = input.tools.length;
|
||||
if (toolCount > 20) return 10;
|
||||
if (toolCount > 10) return 8;
|
||||
if (toolCount > 5) return 6;
|
||||
if (toolCount > 2) return 4;
|
||||
return 2;
|
||||
}
|
||||
|
||||
export function detectDomainSpecificity(input: RuleInput): number {
|
||||
const allText = input.messages
|
||||
.map((m) => (typeof m.content === "string" ? m.content : ""))
|
||||
.join("\n");
|
||||
|
||||
const domainTerms: Record<string, RegExp[]> = {
|
||||
medical: [/\bdiagnosis\b/i, /\bsymptoms\b/i, /\btreatment\b/i, /\bpatient\b/i, /\bclinical\b/i],
|
||||
legal: [/\bpursuant\b/i, /\bstatute\b/i, /\bliability\b/i, /\bjurisdiction\b/i, /\bhereby\b/i],
|
||||
scientific: [/\bhypothesis\b/i, /\bmethodology\b/i, /\bempirical\b/i, /\bsignificant\b/i],
|
||||
financial: [/\bportfolio\b/i, /\bdividend\b/i, /\bamortization\b/i, /\barbitrage\b/i],
|
||||
};
|
||||
|
||||
let maxDomainScore = 0;
|
||||
for (const [, terms] of Object.entries(domainTerms)) {
|
||||
const score = terms.reduce((sum, re) => {
|
||||
return sum + (re.test(allText) ? 2 : 0);
|
||||
}, 0);
|
||||
maxDomainScore = Math.max(maxDomainScore, score);
|
||||
}
|
||||
|
||||
return Math.min(10, maxDomainScore);
|
||||
}
|
||||
|
||||
export function getSpecificityBreakdown(input: RuleInput): SpecificityBreakdown {
|
||||
return {
|
||||
codeComplexity: detectCodeComplexity(input),
|
||||
mathComplexity: detectMathComplexity(input),
|
||||
reasoningDepth: detectReasoningDepth(input),
|
||||
contextSize: detectContextSize(input),
|
||||
toolCalling: detectToolCalling(input),
|
||||
domainSpecificity: detectDomainSpecificity(input),
|
||||
};
|
||||
}
|
||||
|
||||
export function detectConversationDepth(input: RuleInput): number {
|
||||
const userMessages = input.messages.filter(
|
||||
(m) => (m as { role?: string }).role === "user"
|
||||
).length;
|
||||
const assistantMessages = input.messages.filter(
|
||||
(m) => (m as { role?: string }).role === "assistant"
|
||||
).length;
|
||||
|
||||
const totalTurns = userMessages + assistantMessages;
|
||||
if (totalTurns > 30) return 8;
|
||||
if (totalTurns > 20) return 6;
|
||||
if (totalTurns > 10) return 4;
|
||||
if (totalTurns > 5) return 2;
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function detectFileReferences(input: RuleInput): number {
|
||||
const allText = input.messages
|
||||
.map((m) => (typeof m.content === "string" ? m.content : ""))
|
||||
.join("\n");
|
||||
|
||||
const filePatterns = [
|
||||
/(?:\/[\w.-]+){2,}/g,
|
||||
/\b\w+:\d+:\d+\b/g,
|
||||
/\b(?:diff|patch|merge)\b/gi,
|
||||
/\b(?:README|CHANGELOG|TODO)\b/gi,
|
||||
/@@[\s+-]+\d+,\d+\s+@@/g,
|
||||
];
|
||||
|
||||
const matches = filePatterns.reduce((sum, re) => {
|
||||
return sum + (allText.match(re)?.length || 0);
|
||||
}, 0);
|
||||
|
||||
return Math.min(5, matches * 1);
|
||||
}
|
||||
|
||||
export function detectErrorContext(input: RuleInput): number {
|
||||
const allText = input.messages
|
||||
.map((m) => (typeof m.content === "string" ? m.content : ""))
|
||||
.join("\n");
|
||||
|
||||
const errorPatterns = [
|
||||
/\b(?:Error|Exception|TypeError|ReferenceError|SyntaxError)\b/g,
|
||||
/\bat\s+[\w.]+\s+\([\w./]+:\d+:\d+\)/g,
|
||||
/\b(?:throw|catch|finally)\b/g,
|
||||
/\b(?:ERRO|FATAL|WARN)\b/g,
|
||||
/\b(?:failed|crashed|unexpected)\b/gi,
|
||||
/\bExit code \d+\b/g,
|
||||
];
|
||||
|
||||
const matches = errorPatterns.reduce((sum, re) => {
|
||||
return sum + (allText.match(re)?.length || 0);
|
||||
}, 0);
|
||||
|
||||
return Math.min(5, matches * 0.5);
|
||||
}
|
||||
|
||||
export function detectEnhancedContextSize(input: RuleInput): number {
|
||||
const msgTokens = estimateMessageTokens(input.messages);
|
||||
const sysTokens = input.systemPrompt ? estimateTokens(input.systemPrompt) : 0;
|
||||
const toolTokens = input.tools
|
||||
? input.tools.reduce(
|
||||
(sum, t) =>
|
||||
sum +
|
||||
estimateTokens(
|
||||
JSON.stringify(
|
||||
(t as { function?: { description?: string; parameters?: unknown } })?.function || t
|
||||
)
|
||||
),
|
||||
0
|
||||
)
|
||||
: 0;
|
||||
|
||||
const total = msgTokens + sysTokens + toolTokens;
|
||||
|
||||
if (total > 100000) return 15;
|
||||
if (total > 64000) return 13;
|
||||
if (total > 32000) return 10;
|
||||
if (total > 16000) return 7;
|
||||
if (total > 8000) return 5;
|
||||
if (total > 4000) return 3;
|
||||
if (total > 1000) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function getEnhancedSpecificityBreakdown(input: RuleInput): SpecificityBreakdown {
|
||||
return {
|
||||
codeComplexity: detectCodeComplexity(input),
|
||||
mathComplexity: detectMathComplexity(input),
|
||||
reasoningDepth: detectReasoningDepth(input),
|
||||
contextSize: detectEnhancedContextSize(input),
|
||||
toolCalling: detectToolCalling(input),
|
||||
domainSpecificity: detectDomainSpecificity(input),
|
||||
};
|
||||
}
|
||||
43
open-sse/services/specificityTypes.ts
Normal file
43
open-sse/services/specificityTypes.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
/**
|
||||
* Query specificity / complexity detection types for Manifest routing integration.
|
||||
*/
|
||||
|
||||
export interface SpecificityResult {
|
||||
score: number;
|
||||
breakdown: SpecificityBreakdown;
|
||||
rulesTriggered: string[];
|
||||
inputTokens: number;
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
export interface SpecificityBreakdown {
|
||||
codeComplexity: number;
|
||||
mathComplexity: number;
|
||||
reasoningDepth: number;
|
||||
contextSize: number;
|
||||
toolCalling: number;
|
||||
domainSpecificity: number;
|
||||
}
|
||||
|
||||
export interface SpecificityRule {
|
||||
name: string;
|
||||
category: keyof SpecificityBreakdown;
|
||||
weight: number;
|
||||
detect(input: RuleInput): RuleMatch | null;
|
||||
}
|
||||
|
||||
export interface RuleInput {
|
||||
messages: Array<{ role?: string; content?: string | unknown }>;
|
||||
systemPrompt?: string;
|
||||
tools?: Array<{
|
||||
function?: { name: string; description?: string; parameters?: unknown };
|
||||
}>;
|
||||
model?: string;
|
||||
}
|
||||
|
||||
export interface RuleMatch {
|
||||
score: number;
|
||||
evidence: string;
|
||||
}
|
||||
|
||||
export type SpecificityLevel = "trivial" | "simple" | "moderate" | "complex" | "expert";
|
||||
75
open-sse/services/tierConfig.ts
Normal file
75
open-sse/services/tierConfig.ts
Normal file
@@ -0,0 +1,75 @@
|
||||
/**
|
||||
* Tier configuration schema with Zod validation and sensible defaults.
|
||||
*/
|
||||
|
||||
import { z } from "zod";
|
||||
import type { TierConfig, ProviderTierOverride, ModelTierOverride } from "./tierTypes";
|
||||
import { PROVIDER_TIER } from "./tierTypes";
|
||||
|
||||
export const providerTierOverrideSchema = z.object({
|
||||
provider: z.string().min(1),
|
||||
tier: z.enum(["free", "cheap", "premium"]),
|
||||
});
|
||||
|
||||
export const modelTierOverrideSchema = z.object({
|
||||
provider: z.string().min(1),
|
||||
modelPattern: z.string().min(1),
|
||||
tier: z.enum(["free", "cheap", "premium"]),
|
||||
});
|
||||
|
||||
export const tierConfigSchema = z.object({
|
||||
version: z.string().default("1.0.0"),
|
||||
defaults: z.object({
|
||||
freeThreshold: z.number().min(0).default(0),
|
||||
cheapThreshold: z.number().min(0).default(1.0),
|
||||
}),
|
||||
providerOverrides: z.array(providerTierOverrideSchema).default([]),
|
||||
modelOverrides: z.array(modelTierOverrideSchema).default([]),
|
||||
freeProviders: z.array(z.string()).default([]),
|
||||
});
|
||||
|
||||
export const DEFAULT_TIER_CONFIG: TierConfig = {
|
||||
version: "1.0.0",
|
||||
defaults: {
|
||||
freeThreshold: 0,
|
||||
cheapThreshold: 1.0,
|
||||
},
|
||||
providerOverrides: [],
|
||||
modelOverrides: [],
|
||||
freeProviders: [
|
||||
"kiro",
|
||||
"qoder",
|
||||
"pollinations",
|
||||
"longcat",
|
||||
"cloudflare-ai",
|
||||
"qwen",
|
||||
"gemini-cli",
|
||||
"nvidia-nim",
|
||||
"cerebras",
|
||||
"groq",
|
||||
],
|
||||
};
|
||||
|
||||
export function validateTierConfig(raw: unknown): TierConfig {
|
||||
return tierConfigSchema.parse(raw);
|
||||
}
|
||||
|
||||
export function mergeTierConfig(userConfig?: Partial<TierConfig>): TierConfig {
|
||||
if (!userConfig) return DEFAULT_TIER_CONFIG;
|
||||
return {
|
||||
...DEFAULT_TIER_CONFIG,
|
||||
...userConfig,
|
||||
defaults: {
|
||||
...DEFAULT_TIER_CONFIG.defaults,
|
||||
...userConfig.defaults,
|
||||
},
|
||||
providerOverrides: [
|
||||
...DEFAULT_TIER_CONFIG.providerOverrides,
|
||||
...(userConfig.providerOverrides || []),
|
||||
],
|
||||
modelOverrides: [...DEFAULT_TIER_CONFIG.modelOverrides, ...(userConfig.modelOverrides || [])],
|
||||
freeProviders: [
|
||||
...new Set([...DEFAULT_TIER_CONFIG.freeProviders, ...(userConfig.freeProviders || [])]),
|
||||
],
|
||||
};
|
||||
}
|
||||
27
open-sse/services/tierDefaults.json
Normal file
27
open-sse/services/tierDefaults.json
Normal file
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"version": "1.0.0",
|
||||
"defaults": { "freeThreshold": 0, "cheapThreshold": 1.0 },
|
||||
"providerOverrides": [
|
||||
{ "provider": "deepseek", "tier": "cheap" },
|
||||
{ "provider": "groq", "tier": "free" },
|
||||
{ "provider": "glm", "tier": "cheap" },
|
||||
{ "provider": "minimax", "tier": "cheap" },
|
||||
{ "provider": "meta-llama", "tier": "cheap" }
|
||||
],
|
||||
"modelOverrides": [
|
||||
{ "provider": "openai", "modelPattern": "gpt-4o-mini*", "tier": "cheap" },
|
||||
{ "provider": "anthropic", "modelPattern": "claude-haiku*", "tier": "cheap" }
|
||||
],
|
||||
"freeProviders": [
|
||||
"kiro",
|
||||
"qoder",
|
||||
"pollinations",
|
||||
"longcat",
|
||||
"cloudflare-ai",
|
||||
"qwen",
|
||||
"gemini-cli",
|
||||
"nvidia-nim",
|
||||
"cerebras",
|
||||
"groq"
|
||||
]
|
||||
}
|
||||
144
open-sse/services/tierResolver.ts
Normal file
144
open-sse/services/tierResolver.ts
Normal file
@@ -0,0 +1,144 @@
|
||||
import type { TierAssignment, TierConfig, ProviderTier } from "./tierTypes";
|
||||
import { PROVIDER_TIER } from "./tierTypes";
|
||||
import { getModelPricing } from "./providerCostData";
|
||||
import { isExplicitlyFree } from "./providerCostData";
|
||||
import { mergeTierConfig, DEFAULT_TIER_CONFIG } from "./tierConfig";
|
||||
|
||||
let dbPersistenceChecked = false;
|
||||
|
||||
const tierCache = new Map<string, TierAssignment>();
|
||||
let currentConfig: TierConfig = DEFAULT_TIER_CONFIG;
|
||||
|
||||
function cacheKey(provider: string, model: string): string {
|
||||
return `${provider}::${model}`;
|
||||
}
|
||||
|
||||
function matchGlob(pattern: string, text: string): boolean {
|
||||
const regexStr = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
|
||||
return new RegExp(`^${regexStr}$`, "i").test(text);
|
||||
}
|
||||
|
||||
export function classifyTier(provider: string, model: string): TierAssignment {
|
||||
const key = cacheKey(provider, model);
|
||||
|
||||
if (tierCache.has(key)) {
|
||||
return tierCache.get(key)!;
|
||||
}
|
||||
|
||||
if (isExplicitlyFree(provider, currentConfig)) {
|
||||
const assignment: TierAssignment = {
|
||||
provider,
|
||||
model,
|
||||
tier: PROVIDER_TIER.FREE,
|
||||
reason: `Provider '${provider}' is in explicit free providers list`,
|
||||
costPer1MInput: 0,
|
||||
costPer1MOutput: 0,
|
||||
hasFreeTier: true,
|
||||
};
|
||||
tierCache.set(key, assignment);
|
||||
return assignment;
|
||||
}
|
||||
|
||||
const providerOverride = currentConfig.providerOverrides.find(
|
||||
(o) => o.provider.toLowerCase() === provider.toLowerCase()
|
||||
);
|
||||
if (providerOverride) {
|
||||
const pricing = getModelPricing(provider, model);
|
||||
const assignment: TierAssignment = {
|
||||
provider,
|
||||
model,
|
||||
tier: providerOverride.tier,
|
||||
reason: `Provider-level override: '${provider}' → ${providerOverride.tier}`,
|
||||
costPer1MInput: pricing.inputCostPer1M,
|
||||
costPer1MOutput: pricing.outputCostPer1M,
|
||||
hasFreeTier: pricing.isFree,
|
||||
freeQuotaLimit: pricing.freeQuotaLimit,
|
||||
};
|
||||
tierCache.set(key, assignment);
|
||||
return assignment;
|
||||
}
|
||||
|
||||
const modelOverride = currentConfig.modelOverrides.find(
|
||||
(o) => o.provider.toLowerCase() === provider.toLowerCase() && matchGlob(o.modelPattern, model)
|
||||
);
|
||||
if (modelOverride) {
|
||||
const pricing = getModelPricing(provider, model);
|
||||
const assignment: TierAssignment = {
|
||||
provider,
|
||||
model,
|
||||
tier: modelOverride.tier,
|
||||
reason: `Model-level override: '${provider}/${model}' matches '${modelOverride.modelPattern}' → ${modelOverride.tier}`,
|
||||
costPer1MInput: pricing.inputCostPer1M,
|
||||
costPer1MOutput: pricing.outputCostPer1M,
|
||||
hasFreeTier: pricing.isFree,
|
||||
freeQuotaLimit: pricing.freeQuotaLimit,
|
||||
};
|
||||
tierCache.set(key, assignment);
|
||||
return assignment;
|
||||
}
|
||||
|
||||
const pricing = getModelPricing(provider, model);
|
||||
let tier: ProviderTier;
|
||||
let reason: string;
|
||||
|
||||
if (pricing.isFree || pricing.inputCostPer1M <= currentConfig.defaults.freeThreshold) {
|
||||
tier = PROVIDER_TIER.FREE;
|
||||
reason = `Cost-based: $${pricing.inputCostPer1M}/M input ≤ free threshold ($${currentConfig.defaults.freeThreshold}/M)`;
|
||||
} else if (pricing.inputCostPer1M <= currentConfig.defaults.cheapThreshold) {
|
||||
tier = PROVIDER_TIER.CHEAP;
|
||||
reason = `Cost-based: $${pricing.inputCostPer1M}/M input ≤ cheap threshold ($${currentConfig.defaults.cheapThreshold}/M)`;
|
||||
} else {
|
||||
tier = PROVIDER_TIER.PREMIUM;
|
||||
reason = `Cost-based: $${pricing.inputCostPer1M}/M input > cheap threshold ($${currentConfig.defaults.cheapThreshold}/M)`;
|
||||
}
|
||||
|
||||
const assignment: TierAssignment = {
|
||||
provider,
|
||||
model,
|
||||
tier,
|
||||
reason,
|
||||
costPer1MInput: pricing.inputCostPer1M,
|
||||
costPer1MOutput: pricing.outputCostPer1M,
|
||||
hasFreeTier: pricing.isFree,
|
||||
freeQuotaLimit: pricing.freeQuotaLimit,
|
||||
};
|
||||
|
||||
tierCache.set(key, assignment);
|
||||
return assignment;
|
||||
}
|
||||
|
||||
export function setTierConfig(config?: Partial<TierConfig> | null): void {
|
||||
if (config === null || config === undefined) {
|
||||
try {
|
||||
const { loadTierConfig } = require("../../src/lib/db/tierConfig");
|
||||
currentConfig = loadTierConfig();
|
||||
} catch {
|
||||
currentConfig = DEFAULT_TIER_CONFIG;
|
||||
}
|
||||
} else {
|
||||
currentConfig = mergeTierConfig(config);
|
||||
}
|
||||
tierCache.clear();
|
||||
}
|
||||
|
||||
export function getTierConfig(): TierConfig {
|
||||
return { ...currentConfig };
|
||||
}
|
||||
|
||||
export function clearTierCache(): void {
|
||||
tierCache.clear();
|
||||
}
|
||||
|
||||
export function classifyTiers(
|
||||
targets: Array<{ provider: string; model: string }>
|
||||
): TierAssignment[] {
|
||||
return targets.map((t) => classifyTier(t.provider, t.model));
|
||||
}
|
||||
|
||||
export function getTierStats(): Record<ProviderTier, number> {
|
||||
const stats: Record<ProviderTier, number> = { free: 0, cheap: 0, premium: 0 };
|
||||
for (const assignment of tierCache.values()) {
|
||||
stats[assignment.tier]++;
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
40
open-sse/services/tierTypes.ts
Normal file
40
open-sse/services/tierTypes.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
export const PROVIDER_TIER = {
|
||||
FREE: "free",
|
||||
CHEAP: "cheap",
|
||||
PREMIUM: "premium",
|
||||
} as const;
|
||||
|
||||
export type ProviderTier = (typeof PROVIDER_TIER)[keyof typeof PROVIDER_TIER];
|
||||
|
||||
export interface TierAssignment {
|
||||
provider: string;
|
||||
model: string;
|
||||
tier: ProviderTier;
|
||||
reason: string;
|
||||
costPer1MInput: number;
|
||||
costPer1MOutput: number;
|
||||
hasFreeTier: boolean;
|
||||
freeQuotaLimit?: number;
|
||||
}
|
||||
|
||||
export interface TierConfig {
|
||||
version: string;
|
||||
defaults: {
|
||||
freeThreshold: number;
|
||||
cheapThreshold: number;
|
||||
};
|
||||
providerOverrides: ProviderTierOverride[];
|
||||
modelOverrides: ModelTierOverride[];
|
||||
freeProviders: string[];
|
||||
}
|
||||
|
||||
export interface ProviderTierOverride {
|
||||
provider: string;
|
||||
tier: ProviderTier;
|
||||
}
|
||||
|
||||
export interface ModelTierOverride {
|
||||
provider: string;
|
||||
modelPattern: string;
|
||||
tier: ProviderTier;
|
||||
}
|
||||
21
src/lib/db/migrations/051_manifest_routing.sql
Normal file
21
src/lib/db/migrations/051_manifest_routing.sql
Normal file
@@ -0,0 +1,21 @@
|
||||
CREATE TABLE IF NOT EXISTS tier_config (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
updated_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tier_assignments (
|
||||
provider TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
tier TEXT NOT NULL CHECK (tier IN ('free', 'cheap', 'premium')),
|
||||
cost_per_1m_input REAL DEFAULT 0,
|
||||
cost_per_1m_output REAL DEFAULT 0,
|
||||
has_free_tier INTEGER DEFAULT 0,
|
||||
free_quota_limit INTEGER,
|
||||
reason TEXT,
|
||||
updated_at TEXT DEFAULT (datetime('now')),
|
||||
PRIMARY KEY (provider, model)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_tier_assignments_provider ON tier_assignments(provider);
|
||||
CREATE INDEX IF NOT EXISTS idx_tier_assignments_tier ON tier_assignments(tier);
|
||||
41
src/lib/db/tierConfig.ts
Normal file
41
src/lib/db/tierConfig.ts
Normal file
@@ -0,0 +1,41 @@
|
||||
import { getDbInstance } from "./core";
|
||||
import type { TierConfig } from "../../../open-sse/services/tierTypes";
|
||||
import { validateTierConfig, DEFAULT_TIER_CONFIG } from "../../../open-sse/services/tierConfig";
|
||||
|
||||
const TABLE = "tier_config";
|
||||
|
||||
export function initTierConfigTable(): void {
|
||||
const db = getDbInstance();
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS ${TABLE} (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
updated_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
`);
|
||||
}
|
||||
|
||||
export function saveTierConfig(config: TierConfig): void {
|
||||
const db = getDbInstance();
|
||||
const serialized = JSON.stringify(config);
|
||||
db.prepare(
|
||||
`INSERT OR REPLACE INTO ${TABLE} (key, value, updated_at) VALUES ('tier_config', ?, datetime('now'))`
|
||||
).run(serialized);
|
||||
}
|
||||
|
||||
export function loadTierConfigFromDb(): TierConfig | null {
|
||||
const db = getDbInstance();
|
||||
const row = db.prepare(`SELECT value FROM ${TABLE} WHERE key = 'tier_config'`).get() as
|
||||
| { value: string }
|
||||
| undefined;
|
||||
if (!row) return null;
|
||||
try {
|
||||
return validateTierConfig(JSON.parse(row.value));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function loadTierConfig(): TierConfig {
|
||||
return loadTierConfigFromDb() || DEFAULT_TIER_CONFIG;
|
||||
}
|
||||
77
tests/integration/manifest-routing.test.ts
Normal file
77
tests/integration/manifest-routing.test.ts
Normal file
@@ -0,0 +1,77 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { generateRoutingHints } from "../../open-sse/services/manifestAdapter.ts";
|
||||
|
||||
test("manifest routing generates hints without error", async () => {
|
||||
const hints = generateRoutingHints([], {
|
||||
messages: [{ content: "Test" }],
|
||||
});
|
||||
assert.equal(hints.specificityLevel, "trivial");
|
||||
assert.equal(hints.strategyModifier, "prefer-free");
|
||||
});
|
||||
|
||||
test("manifest routing failure gracefully falls back - empty targets handled", async () => {
|
||||
const hints = generateRoutingHints([], {
|
||||
messages: [{ content: "Hello world" }],
|
||||
});
|
||||
assert.equal(hints.eligibleTargets.length, 0);
|
||||
assert.equal(hints.underqualifiedTargets.length, 0);
|
||||
assert.ok(hints.specificityLevel.length > 0);
|
||||
});
|
||||
|
||||
test("specificity score is non-negative and bounded", async () => {
|
||||
const hints = generateRoutingHints([], {
|
||||
messages: [{ content: "Hello" }],
|
||||
});
|
||||
assert.ok(hints.specificity.score >= 0);
|
||||
assert.ok(hints.specificity.score <= 100);
|
||||
});
|
||||
|
||||
test("routing hints contain all required fields", async () => {
|
||||
const hints = generateRoutingHints([], {
|
||||
messages: [{ content: "Test message" }],
|
||||
});
|
||||
assert.ok("specificityLevel" in hints);
|
||||
assert.ok("strategyModifier" in hints);
|
||||
assert.ok("recommendedMinTier" in hints);
|
||||
assert.ok("specificity" in hints);
|
||||
assert.ok("eligibleTargets" in hints);
|
||||
assert.ok("underqualifiedTargets" in hints);
|
||||
});
|
||||
|
||||
test("trivial query recommends free tier", async () => {
|
||||
const hints = generateRoutingHints([], {
|
||||
messages: [{ content: "Hello" }],
|
||||
});
|
||||
assert.equal(hints.recommendedMinTier, "free");
|
||||
});
|
||||
|
||||
test("full manifest routing flow overhead is minimal", async () => {
|
||||
const t0 = performance.now();
|
||||
for (let i = 0; i < 100; i++) {
|
||||
generateRoutingHints([], {
|
||||
messages: [{ content: "Test message for performance" }],
|
||||
});
|
||||
}
|
||||
const elapsed = performance.now() - t0;
|
||||
const avgMs = elapsed / 100;
|
||||
assert.ok(
|
||||
avgMs < 1,
|
||||
`Average manifest routing overhead should be < 1ms, got ${avgMs.toFixed(2)}ms`
|
||||
);
|
||||
});
|
||||
|
||||
test("tier resolver module loads without error", async () => {
|
||||
const { classifyTier, clearTierCache } = await import("../../open-sse/services/tierResolver.ts");
|
||||
clearTierCache();
|
||||
const result = classifyTier("kiro", "claude-sonnet-4.5");
|
||||
assert.equal(result.tier, "free");
|
||||
});
|
||||
|
||||
test("specificity detector module loads without error", async () => {
|
||||
const { analyzeSpecificity } = await import("../../open-sse/services/specificityDetector.ts");
|
||||
const result = analyzeSpecificity({
|
||||
messages: [{ content: "Test" }],
|
||||
});
|
||||
assert.ok(result.score >= 0);
|
||||
});
|
||||
Reference in New Issue
Block a user