mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
Combo + quota-shared deep audit: 5 fixes + strategy validation + E2E + complexity routing (#3779)
Deep audit of the combo + quota-shared system, delivered as 4 TDD waves. - Wave 1: repair 5 dead/broken rules — streaming USD recording, pool-usage provider resolution, provider-diversity wiring, maxComboDepth threading, scoring clamp/NaN-safety (incl. connectionDensity). - Wave 2: validate every auto-router strategy (cost / latency / sla-aware / lkgp / selectWithStrategy + aliases) and the predictive-TTFT decision. - Wave 3: E2E coverage — 3-hop priority failover, per-target timeout failover, real strategy:auto dispatch. - Wave 4: complexity-aware routing (2026, opt-in) over the existing specificity detector, plus revival of the dead tierAffinity / specificityMatch scoring factors (require-in-ESM root cause -> static import). Proxy/credential isolation verified clean (each target uses its own credentials+proxy via AsyncLocalStorage). file-size reconciled (combo.ts re-baseline after extracting buildComplexityRoutingHint to complexityRouter.ts; base.ts release-drift from #3780). Fast Quality Gates + semgrep green.
This commit is contained in:
committed by
GitHub
parent
db6b4088a7
commit
45b0672628
@@ -27,7 +27,7 @@
|
||||
"open-sse/services/batchProcessor.ts": 828,
|
||||
"open-sse/services/browserBackedChat.ts": 850,
|
||||
"open-sse/services/claudeCodeCompatible.ts": 1202,
|
||||
"open-sse/services/combo.ts": 5054,
|
||||
"open-sse/services/combo.ts": 5131,
|
||||
"open-sse/services/rateLimitManager.ts": 1017,
|
||||
"open-sse/services/tokenRefresh.ts": 1997,
|
||||
"open-sse/services/usage.ts": 3394,
|
||||
@@ -111,5 +111,6 @@
|
||||
"_rebaseline_2026_06_12_phase1n1s": "Phase 1n-1s (#3501): ProviderDetailPageClient.tsx 2554→1376 (extraídos ConnectionsListPanel, ConnectionsHeaderToolbar, ZedImportCard, BatchTestResultsModal, AdaptaTutorialModal + hooks/useApiKeySave + 4 helper closures→providerPageHelpers.ts). providerPageHelpers.ts 822→897 justificado: recebe 4 closures do god-component (getApiLabel/getApiDefaultPath/getApiPath/getHeaderIconProviderId), zero lógica nova, cliente encolhe mais do que helpers crescem.",
|
||||
"_rebaseline_2026_06_12_phase1t": "Phase 1t (#3501): ProviderDetailPageClient.tsx 1377→782 — META ≤800 ATINGIDA (extraídos ProviderPageHeader, CompatibleNodeCard, ProviderModalsPanel, EmptyConnectionsPlaceholder, UpstreamProxyCard, SearchProviderCard + hooks useConnectionGate/useProviderNodeActions). Drift concorrente reconciliado: ResilienceTab/sse-chat/sse-auth/accountFallback/combo (merges #3629 model-lockout etc.).",
|
||||
"_rebaseline_2026_06_12_v3823_new_features": "Re-baseline v3.8.23 pós-merge de #3742 (cost drilldown: ApiManagerPageClient.tsx +21, CostOverviewTab.tsx +14, providerLimits.ts +2, usage.ts +53) + #3743 (provider display modes: ProviderDetailPageClient.tsx +2, providerPageHelpers.ts +42, providers.ts +2) + #3740 (semantic cache key isolation: chat.ts +3). Crescimento justificado por features novas mergeadas no ciclo.",
|
||||
"_rebaseline_2026_06_13_combo_quota_audit": "Re-baseline consciente do audit combo+quota (PR #3779): combo.ts 5054→5131 (+77). Crescimento = 5 fixes TDD + estratégia complexity-aware 2026 (W1 clampComboDepth + threading de maxDepth em 6 assinaturas/dispatch/DAG; W2 extração shouldSkipForPredictedTtft; W4 scoreAutoTargets exportado + param manifestHint). A parte limpa-extraível do W4 (construção do hint inline, ~30 linhas) FOI extraída para autoCombo/complexityRouter.ts (buildComplexityRoutingHint) — este +76 é o resíduo irredutível (edição de assinaturas/threading, não bloco movível). Shrink estrutural de combo.ts segue com #3501.",
|
||||
"_rebaseline_2026_06_13_v3824_3776": "Re-baseline v3.8.24 pós #3776 (strict-mode controls Claude Code default models: ApiManagerPageClient.tsx 2701→2909 = UI de famílias bloqueáveis cc/* + chips; apiKeys.ts 1490→1633 = blocked_models deny-list + candidatos de permissão claude-code; schemas.ts 2515→2519 = reformatação Prettier + reasoningTokenBufferEnabled restaurado) + carry-over base.ts 1205→1218 do #3780 (enforceThinkingTemperature no chokepoint, drift de baseline não bumpado no merge). Crescimento de feature; sem god-component novo."
|
||||
}
|
||||
|
||||
@@ -5227,21 +5227,15 @@ export async function handleChatCore({
|
||||
// === Quota Share POST-hook (B/F7) — fire-and-forget, fail-open ===
|
||||
if (apiKeyInfo?.id && credentials?.connectionId) {
|
||||
try {
|
||||
const { scheduleRecordConsumption } = await import("@/lib/quota/spendRecorder");
|
||||
const { scheduleRecordConsumption, buildConsumptionCost } = await import(
|
||||
"@/lib/quota/spendRecorder"
|
||||
);
|
||||
scheduleRecordConsumption(
|
||||
{
|
||||
apiKeyId: apiKeyInfo.id,
|
||||
connectionId: credentials.connectionId,
|
||||
provider: provider ?? "unknown",
|
||||
cost: {
|
||||
tokens:
|
||||
usage && typeof usage === "object"
|
||||
? (((usage as Record<string, unknown>).prompt_tokens as number) ?? 0) +
|
||||
(((usage as Record<string, unknown>).completion_tokens as number) ?? 0)
|
||||
: 0,
|
||||
usd: estimatedCost > 0 ? estimatedCost : 0,
|
||||
requests: 1,
|
||||
},
|
||||
cost: buildConsumptionCost(usage, estimatedCost),
|
||||
},
|
||||
log
|
||||
);
|
||||
@@ -5522,29 +5516,28 @@ export async function handleChatCore({
|
||||
}
|
||||
|
||||
// === Quota Share POST-hook streaming (B/F7) — fire-and-forget, fail-open ===
|
||||
// Resolve the real per-request cost (calculateCost) so USD-unit pools accrue
|
||||
// on streaming traffic too; this previously recorded usd:0 hardcoded, which
|
||||
// meant DeepSeek-style `usd/monthly` shared pools never blocked on streams.
|
||||
if (apiKeyInfo?.id && credentials?.connectionId && streamStatus === 200) {
|
||||
const su = streamUsage as Record<string, unknown> | null;
|
||||
const quotaApiKeyId = apiKeyInfo.id;
|
||||
const quotaConnectionId = credentials.connectionId;
|
||||
// onStreamComplete is sync — use .then() (fire-and-forget, fail-open) instead of await
|
||||
import("@/lib/quota/spendRecorder")
|
||||
.then(({ scheduleRecordConsumption }) => {
|
||||
scheduleRecordConsumption(
|
||||
.then(({ recordStreamingConsumption }) =>
|
||||
recordStreamingConsumption(
|
||||
{
|
||||
apiKeyId: quotaApiKeyId,
|
||||
connectionId: quotaConnectionId,
|
||||
provider: provider ?? "unknown",
|
||||
cost: {
|
||||
tokens: su
|
||||
? (Number(su.prompt_tokens ?? 0) || 0) + (Number(su.completion_tokens ?? 0) || 0)
|
||||
: 0,
|
||||
usd: 0, // estimatedCost resolved async above; omit to avoid dependency
|
||||
requests: 1,
|
||||
},
|
||||
provider,
|
||||
model,
|
||||
streamUsage,
|
||||
streamStatus,
|
||||
serviceTier: effectiveServiceTier,
|
||||
},
|
||||
log
|
||||
);
|
||||
})
|
||||
{ calculateCost, log }
|
||||
)
|
||||
)
|
||||
.catch(() => {
|
||||
// Outer fail-open — never throws to caller
|
||||
});
|
||||
|
||||
111
open-sse/services/autoCombo/complexityRouter.ts
Normal file
111
open-sse/services/autoCombo/complexityRouter.ts
Normal file
@@ -0,0 +1,111 @@
|
||||
/**
|
||||
* complexityRouter.ts — Request-complexity classification for tier-aware routing.
|
||||
*
|
||||
* 2026 strategy: route by the intrinsic difficulty of the *request* (not only by
|
||||
* provider stats), so trivial prompts can use cheap models and hard/reasoning
|
||||
* prompts escalate to capable ones. Built on the existing specificity detector
|
||||
* (codeComplexity / mathComplexity / reasoningDepth / contextSize / toolCalling
|
||||
* / domainSpecificity), adding an explicit tool-use → minimum-tier escalation:
|
||||
* a request carrying tool/function schemas (or agentic tool-calling signals)
|
||||
* should not be routed below the "cheap" tier even when the prose looks
|
||||
* trivial, because function-calling reliability matters more than raw cost.
|
||||
*
|
||||
* The classification maps to a `recommendedTier` that feeds the auto-router's
|
||||
* tier-affinity / specificity-match scoring factors (see scoreAutoTargets,
|
||||
* gated by config.complexityAwareRouting).
|
||||
*/
|
||||
import {
|
||||
analyzeSpecificity,
|
||||
getSpecificityLevel,
|
||||
getRecommendedMinTier,
|
||||
} from "../specificityDetector";
|
||||
import type { RuleInput, SpecificityLevel } from "../specificityTypes";
|
||||
import { generateRoutingHints, type RoutingHint } from "../manifestAdapter";
|
||||
|
||||
export type ComplexityTier = "free" | "cheap" | "premium";
|
||||
|
||||
export interface ComplexityClassification {
|
||||
/** 0..100 specificity / difficulty score. */
|
||||
score: number;
|
||||
/** trivial | simple | moderate | complex | expert */
|
||||
level: SpecificityLevel;
|
||||
/** Minimum provider tier recommended for this request. */
|
||||
recommendedTier: ComplexityTier;
|
||||
/** True when the request carries tool/function schemas or agentic tool signals. */
|
||||
hasToolUse: boolean;
|
||||
/** Names of the specificity rules that fired (for the inspector / dashboard). */
|
||||
signals: string[];
|
||||
}
|
||||
|
||||
const TIER_ORDER: ComplexityTier[] = ["free", "cheap", "premium"];
|
||||
|
||||
/** Raise `tier` to at least `floor`; never lowers it. */
|
||||
export function escalateTier(tier: ComplexityTier, floor: ComplexityTier): ComplexityTier {
|
||||
return TIER_ORDER.indexOf(tier) >= TIER_ORDER.indexOf(floor) ? tier : floor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify a request's complexity and recommend a minimum provider tier.
|
||||
* Pure + dependency-light (no DB / network); safe on the hot path.
|
||||
*/
|
||||
export function classifyRequestComplexity(input: RuleInput): ComplexityClassification {
|
||||
const result = analyzeSpecificity(input);
|
||||
const level = getSpecificityLevel(result.score);
|
||||
|
||||
const explicitTools = Array.isArray(input.tools) && input.tools.length > 0;
|
||||
const hasToolUse = explicitTools || result.breakdown.toolCalling > 0;
|
||||
|
||||
let recommendedTier = getRecommendedMinTier(level) as ComplexityTier;
|
||||
// Tool-using / agentic requests need reliable function calling — floor at "cheap".
|
||||
if (hasToolUse) recommendedTier = escalateTier(recommendedTier, "cheap");
|
||||
|
||||
return {
|
||||
score: result.score,
|
||||
level,
|
||||
recommendedTier,
|
||||
hasToolUse,
|
||||
signals: result.rulesTriggered,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the opt-in complexity-aware routing hint for the auto-router. Returns a
|
||||
* RoutingHint whose `recommendedMinTier` is escalated to the request's intrinsic
|
||||
* complexity (and floored at "cheap" for tool-using requests), or `null` on any
|
||||
* failure — fail-open, so scoring stays tier-neutral. Extracted from combo.ts to
|
||||
* keep the complexity-routing logic in one module.
|
||||
*/
|
||||
export function buildComplexityRoutingHint(
|
||||
modelTargets: Parameters<typeof generateRoutingHints>[0],
|
||||
body: { messages?: unknown; tools?: unknown; model?: unknown } | null | undefined,
|
||||
log: { info: (tag: string, message: string) => void }
|
||||
): RoutingHint | null {
|
||||
try {
|
||||
const ruleInput = {
|
||||
messages: Array.isArray(body?.messages)
|
||||
? (body.messages as Array<{ role?: string; content?: string | unknown }>)
|
||||
: [],
|
||||
tools: Array.isArray(body?.tools)
|
||||
? (body.tools as Array<{
|
||||
function?: { name: string; description?: string; parameters?: unknown };
|
||||
}>)
|
||||
: undefined,
|
||||
model: typeof body?.model === "string" ? body.model : undefined,
|
||||
};
|
||||
const hint = generateRoutingHints(modelTargets, ruleInput);
|
||||
// Tool-use escalation: floor the recommended tier at "cheap" so scoring
|
||||
// favors function-calling-reliable models for agentic requests.
|
||||
const classification = classifyRequestComplexity(ruleInput);
|
||||
hint.recommendedMinTier = escalateTier(
|
||||
hint.recommendedMinTier as ComplexityTier,
|
||||
classification.recommendedTier
|
||||
) as typeof hint.recommendedMinTier;
|
||||
log.info(
|
||||
"COMBO",
|
||||
`Complexity-aware routing: level=${classification.level} score=${classification.score} minTier=${hint.recommendedMinTier} tools=${classification.hasToolUse}`
|
||||
);
|
||||
return hint;
|
||||
} catch {
|
||||
return null; // fail-open: scoring stays tier-neutral
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,8 @@
|
||||
*/
|
||||
|
||||
import type { RoutingHint } from "../manifestAdapter";
|
||||
import { clamp01 } from "../../utils/number";
|
||||
import { classifyTier } from "../tierResolver";
|
||||
|
||||
export interface ScoringFactors {
|
||||
quota: number;
|
||||
@@ -86,19 +88,22 @@ export interface ScoredProvider {
|
||||
* Supports tierAffinity + specificityMatch weights when manifest routing is enabled.
|
||||
*/
|
||||
export function calculateScore(factors: ScoringFactors, weights: ScoringWeights): number {
|
||||
return (
|
||||
// clamp01 bounds the result to [0,1] and maps a non-finite sum (a NaN factor)
|
||||
// to 0, so a single bad input can't yield NaN (which sorts nondeterministically)
|
||||
// or a score >1 from float drift in weights that nominally sum to 1.
|
||||
return clamp01(
|
||||
weights.quota * factors.quota +
|
||||
weights.health * factors.health +
|
||||
weights.costInv * factors.costInv +
|
||||
weights.latencyInv * factors.latencyInv +
|
||||
weights.taskFit * factors.taskFit +
|
||||
weights.stability * factors.stability +
|
||||
weights.tierPriority * factors.tierPriority +
|
||||
(weights.tierAffinity ?? 0) * factors.tierAffinity +
|
||||
(weights.specificityMatch ?? 0) * factors.specificityMatch +
|
||||
(weights.contextAffinity ?? 0) * factors.contextAffinity +
|
||||
(weights.resetWindowAffinity ?? 0) * factors.resetWindowAffinity +
|
||||
(weights.connectionDensity ?? 0) * factors.connectionDensity
|
||||
weights.health * factors.health +
|
||||
weights.costInv * factors.costInv +
|
||||
weights.latencyInv * factors.latencyInv +
|
||||
weights.taskFit * factors.taskFit +
|
||||
weights.stability * factors.stability +
|
||||
weights.tierPriority * factors.tierPriority +
|
||||
(weights.tierAffinity ?? 0) * factors.tierAffinity +
|
||||
(weights.specificityMatch ?? 0) * factors.specificityMatch +
|
||||
(weights.contextAffinity ?? 0) * factors.contextAffinity +
|
||||
(weights.resetWindowAffinity ?? 0) * factors.resetWindowAffinity +
|
||||
(weights.connectionDensity ?? 0) * factors.connectionDensity
|
||||
);
|
||||
}
|
||||
|
||||
@@ -131,7 +136,6 @@ function calculateTierAffinity(
|
||||
): number {
|
||||
if (!hint) return 0.5;
|
||||
try {
|
||||
const { classifyTier } = require("../tierResolver");
|
||||
const assignment = classifyTier(candidate.provider, candidate.model);
|
||||
const tierOrder = ["free", "cheap", "premium"];
|
||||
const providerTierIdx = tierOrder.indexOf(assignment.tier);
|
||||
@@ -151,7 +155,6 @@ function calculateSpecificityMatch(
|
||||
): number {
|
||||
if (!hint) return 0.5;
|
||||
try {
|
||||
const { classifyTier } = require("../tierResolver");
|
||||
const assignment = classifyTier(candidate.provider, candidate.model);
|
||||
const specificityScore = hint.specificity.score;
|
||||
|
||||
@@ -176,24 +179,28 @@ export function calculateFactors(
|
||||
const maxLatency = Math.max(...pool.map((p) => p.p95LatencyMs), 1);
|
||||
const maxStdDev = Math.max(...pool.map((p) => p.latencyStdDev), 0.001);
|
||||
|
||||
// Every factor is contractually [0,1]. clamp01 guards against bad telemetry
|
||||
// (negative quota / cost / latency, NaN, out-of-range candidate-supplied
|
||||
// affinities) so a single bad input can't produce a negative or >1 factor
|
||||
// that distorts the weighted score.
|
||||
return {
|
||||
quota: Math.min(1, candidate.quotaRemaining / 100),
|
||||
quota: clamp01(candidate.quotaRemaining / 100),
|
||||
health:
|
||||
candidate.circuitBreakerState === "CLOSED"
|
||||
? 1.0
|
||||
: candidate.circuitBreakerState === "HALF_OPEN"
|
||||
? 0.5
|
||||
: 0.0,
|
||||
costInv: 1 - candidate.costPer1MTokens / maxCost,
|
||||
latencyInv: 1 - candidate.p95LatencyMs / maxLatency,
|
||||
taskFit: getTaskFitness(candidate.model, taskType),
|
||||
stability: 1 - candidate.latencyStdDev / maxStdDev,
|
||||
costInv: clamp01(1 - candidate.costPer1MTokens / maxCost),
|
||||
latencyInv: clamp01(1 - candidate.p95LatencyMs / maxLatency),
|
||||
taskFit: clamp01(getTaskFitness(candidate.model, taskType)),
|
||||
stability: clamp01(1 - candidate.latencyStdDev / maxStdDev),
|
||||
tierPriority: calculateTierScore(candidate.accountTier, candidate.quotaResetIntervalSecs),
|
||||
tierAffinity: calculateTierAffinity(candidate, manifestHint),
|
||||
specificityMatch: calculateSpecificityMatch(candidate, manifestHint),
|
||||
contextAffinity: candidate.contextAffinity ?? 0.5,
|
||||
resetWindowAffinity: candidate.resetWindowAffinity ?? 0.5,
|
||||
connectionDensity: Math.min(1, Math.max(0, ((candidate.connectionPoolSize ?? 1) - 1) / 10)),
|
||||
contextAffinity: clamp01(candidate.contextAffinity ?? 0.5),
|
||||
resetWindowAffinity: clamp01(candidate.resetWindowAffinity ?? 0.5),
|
||||
connectionDensity: clamp01(((candidate.connectionPoolSize ?? 1) - 1) / 10),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -83,6 +83,7 @@ import { getSessionConnection } from "./sessionManager.ts";
|
||||
import { orderTargetsByEvalScores } from "./evalRouting.ts";
|
||||
import { generateRoutingHints } from "./manifestAdapter";
|
||||
import type { RoutingHint } from "./manifestAdapter";
|
||||
import { buildComplexityRoutingHint } from "./autoCombo/complexityRouter";
|
||||
import type { CompressionMode } from "./compression/types.ts";
|
||||
import { getModelContextLimit } from "../../src/lib/modelCapabilities";
|
||||
import { getProviderConnections } from "../../src/lib/db/providers";
|
||||
@@ -143,9 +144,46 @@ function isProviderCircuitOpenResult(
|
||||
}
|
||||
|
||||
const MAX_COMBO_DEPTH = 3;
|
||||
// Absolute safety ceiling for operator-configured nesting depth. config.maxComboDepth
|
||||
// can raise the default (3) up to this cap, or lower it, but never above — runaway
|
||||
// nested-combo expansion is a real DoS/perf risk.
|
||||
const MAX_COMBO_DEPTH_HARD_CAP = 10;
|
||||
const MAX_FALLBACK_WAIT_MS = 5000;
|
||||
const MAX_GLOBAL_ATTEMPTS = 30;
|
||||
|
||||
/**
|
||||
* Clamp an operator-configured combo nesting depth (config.maxComboDepth) to a
|
||||
* safe integer in [1, MAX_COMBO_DEPTH_HARD_CAP]. Anything non-numeric, < 1, or
|
||||
* NaN falls back to the default MAX_COMBO_DEPTH so a bad config never disables
|
||||
* nesting or blows past the safety ceiling.
|
||||
*/
|
||||
export function clampComboDepth(value: unknown): number {
|
||||
const n = Math.floor(Number(value));
|
||||
if (!Number.isFinite(n) || n < 1) return MAX_COMBO_DEPTH;
|
||||
return Math.min(n, MAX_COMBO_DEPTH_HARD_CAP);
|
||||
}
|
||||
|
||||
/** Minimum recorded requests before the predictive-TTFT breaker trusts the average. */
|
||||
const PREDICTIVE_TTFT_MIN_SAMPLES = 5;
|
||||
|
||||
/**
|
||||
* Predictive-TTFT circuit-breaker decision: skip a target whose recent average
|
||||
* latency — measured over a statistically meaningful sample — exceeds the
|
||||
* configured ceiling, so the combo fails over before paying a slow first byte.
|
||||
* Returns false when disabled (ceiling <= 0), when there is no metric, or when
|
||||
* the sample is too small to trust.
|
||||
*/
|
||||
export function shouldSkipForPredictedTtft(
|
||||
metric: { requests?: number; avgLatencyMs?: number } | null | undefined,
|
||||
predictiveTtftMs: number
|
||||
): boolean {
|
||||
if (!metric || !(predictiveTtftMs > 0)) return false;
|
||||
return (
|
||||
(metric.requests ?? 0) >= PREDICTIVE_TTFT_MIN_SAMPLES &&
|
||||
(metric.avgLatencyMs ?? 0) > predictiveTtftMs
|
||||
);
|
||||
}
|
||||
|
||||
function resolveDelayMs(value: unknown, fallback: number): number {
|
||||
const numericValue = Number(value);
|
||||
if (!Number.isFinite(numericValue) || numericValue < 0) return fallback;
|
||||
@@ -1128,19 +1166,24 @@ function expandRuntimeStep(
|
||||
allCombos: ComboCollectionLike,
|
||||
visited = new Set<string>(),
|
||||
depth = 0,
|
||||
path: string[] = []
|
||||
path: string[] = [],
|
||||
maxDepth: number = MAX_COMBO_DEPTH
|
||||
): ResolvedComboTarget[] {
|
||||
if (step.kind === "model") return [step];
|
||||
if (depth > MAX_COMBO_DEPTH) return [];
|
||||
if (depth > maxDepth) return [];
|
||||
|
||||
const combos = getCombosArray(allCombos);
|
||||
const nestedCombo = combos.find((combo) => combo.name === step.comboName);
|
||||
if (!nestedCombo || visited.has(step.comboName)) return [];
|
||||
|
||||
return resolveNestedComboTargets(nestedCombo, combos, new Set(visited), depth + 1, [
|
||||
...path,
|
||||
step.stepId,
|
||||
]);
|
||||
return resolveNestedComboTargets(
|
||||
nestedCombo,
|
||||
combos,
|
||||
new Set(visited),
|
||||
depth + 1,
|
||||
[...path, step.stepId],
|
||||
maxDepth
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveNestedComboTargets(
|
||||
@@ -1148,13 +1191,14 @@ export function resolveNestedComboTargets(
|
||||
allCombos: ComboCollectionLike,
|
||||
visited = new Set<string>(),
|
||||
depth = 0,
|
||||
path: string[] = []
|
||||
path: string[] = [],
|
||||
maxDepth: number = MAX_COMBO_DEPTH
|
||||
): ResolvedComboTarget[] {
|
||||
const directTargets = (combo.models || [])
|
||||
.map((entry, index) => normalizeRuntimeStep(entry, combo.name, index, null, path))
|
||||
.filter((entry): entry is ResolvedComboTarget => entry?.kind === "model");
|
||||
|
||||
if (depth > MAX_COMBO_DEPTH) return directTargets;
|
||||
if (depth > maxDepth) return directTargets;
|
||||
if (visited.has(combo.name)) return [];
|
||||
visited.add(combo.name);
|
||||
|
||||
@@ -1163,7 +1207,9 @@ export function resolveNestedComboTargets(
|
||||
|
||||
for (const step of runtimeSteps) {
|
||||
if (step.kind === "combo-ref") {
|
||||
resolved.push(...expandRuntimeStep(step, allCombos, new Set(visited), depth, path));
|
||||
resolved.push(
|
||||
...expandRuntimeStep(step, allCombos, new Set(visited), depth, path, maxDepth)
|
||||
);
|
||||
continue;
|
||||
}
|
||||
resolved.push(step);
|
||||
@@ -1214,10 +1260,11 @@ export function validateComboDAG(
|
||||
comboName: string,
|
||||
allCombos: ComboCollectionLike,
|
||||
visited = new Set<string>(),
|
||||
depth = 0
|
||||
depth = 0,
|
||||
maxDepth: number = MAX_COMBO_DEPTH
|
||||
): void {
|
||||
if (depth > MAX_COMBO_DEPTH) {
|
||||
throw new Error(`Max combo nesting depth (${MAX_COMBO_DEPTH}) exceeded at "${comboName}"`);
|
||||
if (depth > maxDepth) {
|
||||
throw new Error(`Max combo nesting depth (${maxDepth}) exceeded at "${comboName}"`);
|
||||
}
|
||||
if (visited.has(comboName)) {
|
||||
throw new Error(`Circular combo reference detected: ${comboName}`);
|
||||
@@ -1233,7 +1280,7 @@ export function validateComboDAG(
|
||||
// Check if this model name is itself a combo (not a provider/model pattern)
|
||||
const nestedCombo = combos.find((c) => c.name === modelName);
|
||||
if (nestedCombo) {
|
||||
validateComboDAG(modelName, combos, new Set(visited), depth + 1);
|
||||
validateComboDAG(modelName, combos, new Set(visited), depth + 1, maxDepth);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1251,9 +1298,10 @@ export function resolveNestedComboModels(
|
||||
combo: ComboLike,
|
||||
allCombos: ComboCollectionLike,
|
||||
visited = new Set<string>(),
|
||||
depth = 0
|
||||
depth = 0,
|
||||
maxDepth: number = MAX_COMBO_DEPTH
|
||||
): string[] {
|
||||
if (depth > MAX_COMBO_DEPTH) return combo.models.map((m) => normalizeModelEntry(m).model);
|
||||
if (depth > maxDepth) return combo.models.map((m) => normalizeModelEntry(m).model);
|
||||
if (visited.has(combo.name)) return []; // cycle safety
|
||||
visited.add(combo.name);
|
||||
|
||||
@@ -1266,7 +1314,13 @@ export function resolveNestedComboModels(
|
||||
|
||||
if (nestedCombo) {
|
||||
// Recursively expand the nested combo
|
||||
const nested = resolveNestedComboModels(nestedCombo, combos, new Set(visited), depth + 1);
|
||||
const nested = resolveNestedComboModels(
|
||||
nestedCombo,
|
||||
combos,
|
||||
new Set(visited),
|
||||
depth + 1,
|
||||
maxDepth
|
||||
);
|
||||
resolved.push(...nested);
|
||||
} else {
|
||||
resolved.push(modelName);
|
||||
@@ -2809,9 +2863,12 @@ async function applyRequestTagRouting(
|
||||
|
||||
export function resolveComboTargets(
|
||||
combo: ComboLike,
|
||||
allCombos: ComboCollectionLike
|
||||
allCombos: ComboCollectionLike,
|
||||
maxDepth: number = MAX_COMBO_DEPTH
|
||||
): ResolvedComboTarget[] {
|
||||
return allCombos ? resolveNestedComboTargets(combo, allCombos) : getDirectComboTargets(combo);
|
||||
return allCombos
|
||||
? resolveNestedComboTargets(combo, allCombos, new Set<string>(), 0, [], maxDepth)
|
||||
: getDirectComboTargets(combo);
|
||||
}
|
||||
|
||||
function resolveWeightedTargets(
|
||||
@@ -2850,11 +2907,12 @@ function resolveWeightedTargets(
|
||||
};
|
||||
}
|
||||
|
||||
function scoreAutoTargets(
|
||||
export function scoreAutoTargets(
|
||||
targets: ResolvedComboTarget[],
|
||||
candidates: AutoProviderCandidate[],
|
||||
taskType: string | null,
|
||||
weights: ScoringWeights
|
||||
weights: ScoringWeights,
|
||||
manifestHint?: RoutingHint | null
|
||||
) {
|
||||
const candidateByExecutionKey = new Map(
|
||||
candidates.map((candidate: ProviderCandidate & { executionKey: string }) => [
|
||||
@@ -2870,7 +2928,8 @@ function scoreAutoTargets(
|
||||
candidate as ProviderCandidate,
|
||||
candidates,
|
||||
taskType ?? "general",
|
||||
getTaskFitness
|
||||
getTaskFitness,
|
||||
manifestHint ?? undefined
|
||||
);
|
||||
let score = calculateScore(factors, weights);
|
||||
// B17: Quota Share soft-policy deprioritization
|
||||
@@ -3129,7 +3188,7 @@ export async function handleComboChat({
|
||||
let orderedTargets =
|
||||
strategy === "weighted"
|
||||
? resolveWeightedTargets(combo, allCombos)?.orderedTargets || []
|
||||
: resolveComboTargets(combo, allCombos);
|
||||
: resolveComboTargets(combo, allCombos, clampComboDepth(config.maxComboDepth));
|
||||
|
||||
orderedTargets = await applyRequestTagRouting(orderedTargets, body, log);
|
||||
|
||||
@@ -3345,7 +3404,25 @@ export async function handleComboChat({
|
||||
selectionReason = `score=${selection.score.toFixed(3)}${selection.isExploration ? " (exploration)" : ""}`;
|
||||
}
|
||||
|
||||
const scoredTargets = scoreAutoTargets(eligibleTargets, candidates, taskType, weights);
|
||||
// Complexity-aware routing (2026, opt-in): classify the request's
|
||||
// difficulty and feed a tier hint into scoring so tierAffinity /
|
||||
// specificityMatch favor candidates whose tier matches the request.
|
||||
const autoManifestHint: RoutingHint | null =
|
||||
config.complexityAwareRouting === true
|
||||
? buildComplexityRoutingHint(
|
||||
eligibleTargets.filter((t) => t.kind === "model"),
|
||||
body,
|
||||
log
|
||||
)
|
||||
: null;
|
||||
|
||||
const scoredTargets = scoreAutoTargets(
|
||||
eligibleTargets,
|
||||
candidates,
|
||||
taskType,
|
||||
weights,
|
||||
autoManifestHint
|
||||
);
|
||||
const rankedTargets = scoredTargets.map((entry) => entry.target);
|
||||
const selectedTarget =
|
||||
scoredTargets.find((entry) => {
|
||||
@@ -3724,7 +3801,7 @@ export async function handleComboChat({
|
||||
if (cMetrics) {
|
||||
const targetKey = orderedTargets[i].executionKey || modelStr;
|
||||
const m = cMetrics.byTarget[targetKey] || cMetrics.byModel[modelStr];
|
||||
if (m && m.requests >= 5 && m.avgLatencyMs > config.predictiveTtftMs) {
|
||||
if (shouldSkipForPredictedTtft(m, config.predictiveTtftMs)) {
|
||||
log.warn(
|
||||
"COMBO",
|
||||
`Predictive TTFT Circuit Breaker: skipping ${modelStr} (avg ${m.avgLatencyMs}ms > max ${config.predictiveTtftMs}ms)`
|
||||
@@ -4531,7 +4608,7 @@ async function handleRoundRobinCombo({
|
||||
? resolveResilienceSettings(settings)
|
||||
: resolveResilienceSettings(null);
|
||||
|
||||
const orderedTargets = resolveComboTargets(combo, allCombos);
|
||||
const orderedTargets = resolveComboTargets(combo, allCombos, clampComboDepth(config.maxComboDepth));
|
||||
const tagFilteredTargets = await applyRequestTagRouting(orderedTargets, body, log);
|
||||
const evalRankedTargets = orderTargetsByEvalScores(tagFilteredTargets, config.evalRouting, log);
|
||||
const filteredTargets = filterTargetsByRequestCompatibility(
|
||||
|
||||
@@ -28,6 +28,10 @@ const DEFAULT_COMBO_CONFIG = {
|
||||
trackMetrics: true,
|
||||
reasoningTokenBufferEnabled: true,
|
||||
manifestRouting: false,
|
||||
// Complexity-aware auto routing (2026): when on, the auto router scores
|
||||
// candidates by how well their tier matches the request's classified
|
||||
// difficulty (feeds tierAffinity/specificityMatch). Opt-in — off by default.
|
||||
complexityAwareRouting: false,
|
||||
resetAwareSessionWeight: 0.35,
|
||||
resetAwareWeeklyWeight: 0.65,
|
||||
resetAwareTieBandPercent: 5,
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
* Provides API for reading metrics from the dashboard.
|
||||
*/
|
||||
|
||||
import { recordProviderUsage } from "./autoCombo/providerDiversity";
|
||||
|
||||
interface ModelMetrics {
|
||||
requests: number;
|
||||
successes: number;
|
||||
@@ -270,6 +272,12 @@ export function recordComboRequest(
|
||||
|
||||
if (success) {
|
||||
combo.totalSuccesses++;
|
||||
// Feed the provider-diversity report (/api/analytics/diversity): record the
|
||||
// provider that actually served this request. recordComboRequest is the
|
||||
// single chokepoint every combo strategy funnels through, so one call here
|
||||
// covers priority / round-robin / weighted / auto / etc.
|
||||
const usedProvider = toNonEmptyString(target?.provider);
|
||||
if (usedProvider) recordProviderUsage(usedProvider);
|
||||
} else {
|
||||
combo.totalFailures++;
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ import { getConsistentMachineId } from "@/shared/utils/machineId";
|
||||
import { syncToCloud } from "@/lib/cloudSync";
|
||||
import { validateCompositeTiersConfig } from "@/lib/combos/compositeTiers";
|
||||
import { normalizeComboModels } from "@/lib/combos/steps";
|
||||
import { validateComboDAG } from "@omniroute/open-sse/services/combo.ts";
|
||||
import { validateComboDAG, clampComboDepth } from "@omniroute/open-sse/services/combo.ts";
|
||||
import { updateComboSchema } from "@/shared/validation/schemas";
|
||||
import { isValidationFailure, validateBody } from "@/shared/validation/helpers";
|
||||
import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
|
||||
@@ -130,8 +130,11 @@ export async function PUT(request, { params }) {
|
||||
// Update the combo in the list temporarily for validation
|
||||
const updatedCombos = allCombos.map((c) => (c.id === id ? { ...c, ...body } : c));
|
||||
if (comboName) {
|
||||
const configuredDepth = clampComboDepth(
|
||||
(nextComboState as { config?: { maxComboDepth?: unknown } }).config?.maxComboDepth
|
||||
);
|
||||
try {
|
||||
validateComboDAG(comboName, updatedCombos);
|
||||
validateComboDAG(comboName, updatedCombos, new Set(), 0, configuredDepth);
|
||||
} catch (dagError) {
|
||||
return NextResponse.json({ error: dagError.message }, { status: 400 });
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ import { getConsistentMachineId } from "@/shared/utils/machineId";
|
||||
import { syncToCloud } from "@/lib/cloudSync";
|
||||
import { validateCompositeTiersConfig } from "@/lib/combos/compositeTiers";
|
||||
import { normalizeComboModels } from "@/lib/combos/steps";
|
||||
import { validateComboDAG } from "@omniroute/open-sse/services/combo.ts";
|
||||
import { validateComboDAG, clampComboDepth } from "@omniroute/open-sse/services/combo.ts";
|
||||
import { createComboSchema } from "@/shared/validation/schemas";
|
||||
import { isValidationFailure, validateBody } from "@/shared/validation/helpers";
|
||||
import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
|
||||
@@ -66,7 +66,13 @@ export async function POST(request) {
|
||||
config,
|
||||
};
|
||||
try {
|
||||
validateComboDAG(name, [...allCombos, tempCombo]);
|
||||
validateComboDAG(
|
||||
name,
|
||||
[...allCombos, tempCombo],
|
||||
new Set(),
|
||||
0,
|
||||
clampComboDepth((config as { maxComboDepth?: unknown } | undefined)?.maxComboDepth)
|
||||
);
|
||||
} catch (dagError) {
|
||||
return NextResponse.json({ error: dagError.message }, { status: 400 });
|
||||
}
|
||||
|
||||
@@ -25,32 +25,13 @@ import {
|
||||
deleteProviderPlan,
|
||||
} from "@/lib/localDb";
|
||||
import { resolvePlan } from "@/lib/quota/planResolver";
|
||||
import { resolveConnectionProvider } from "@/lib/quota/connectionProvider";
|
||||
import { logAuditEvent, getAuditRequestContext } from "@/lib/compliance/index";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
type RouteParams = { params: Promise<{ connectionId: string }> };
|
||||
|
||||
/**
|
||||
* Attempt to look up the provider name for a connection.
|
||||
* Falls back to "unknown" if the DB lookup fails or returns nothing.
|
||||
*/
|
||||
async function resolveProvider(connectionId: string): Promise<string> {
|
||||
try {
|
||||
// Lazy import — avoids circular deps and keeps module loadable without full DB
|
||||
const { getProviderConnectionById } = await import("@/lib/localDb");
|
||||
if (typeof getProviderConnectionById === "function") {
|
||||
const conn = getProviderConnectionById(connectionId);
|
||||
if (conn && typeof (conn as { provider?: string }).provider === "string") {
|
||||
return (conn as { provider: string }).provider;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// DB not available or export not present — fall through
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
export async function GET(request: Request, { params }: RouteParams): Promise<Response> {
|
||||
const authError = await requireManagementAuth(request);
|
||||
if (authError) return authError;
|
||||
@@ -65,7 +46,7 @@ export async function GET(request: Request, { params }: RouteParams): Promise<Re
|
||||
}
|
||||
|
||||
// Resolve via catalog (may return empty plan)
|
||||
const provider = await resolveProvider(connectionId);
|
||||
const provider = await resolveConnectionProvider(connectionId);
|
||||
const plan = resolvePlan(connectionId, provider);
|
||||
return NextResponse.json({ plan });
|
||||
} catch (err) {
|
||||
@@ -87,7 +68,7 @@ export async function PUT(request: Request, { params }: RouteParams): Promise<Re
|
||||
}
|
||||
|
||||
// Derive provider for the connection
|
||||
const provider = await resolveProvider(connectionId);
|
||||
const provider = await resolveConnectionProvider(connectionId);
|
||||
|
||||
upsertProviderPlan(connectionId, provider, parsed.data.dimensions, "manual");
|
||||
|
||||
@@ -117,7 +98,7 @@ export async function DELETE(request: Request, { params }: RouteParams): Promise
|
||||
const { connectionId } = await params;
|
||||
|
||||
const existing = getProviderPlan(connectionId);
|
||||
const provider = existing?.provider ?? (await resolveProvider(connectionId));
|
||||
const provider = existing?.provider ?? (await resolveConnectionProvider(connectionId));
|
||||
|
||||
deleteProviderPlan(connectionId);
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
|
||||
import { getPool } from "@/lib/localDb";
|
||||
import { getQuotaStore } from "@/lib/quota/QuotaStore";
|
||||
import { resolvePlan } from "@/lib/quota/planResolver";
|
||||
import { resolveConnectionProvider } from "@/lib/quota/connectionProvider";
|
||||
import type { PoolUsageSnapshot } from "@/lib/quota/types";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
@@ -35,9 +36,12 @@ export async function GET(request: Request, { params }: RouteParams): Promise<Re
|
||||
return NextResponse.json(buildErrorBody(404, "Pool not found"), { status: 404 });
|
||||
}
|
||||
|
||||
// 2. Resolve the provider plan for this pool's connection
|
||||
// Provider name is not stored on pool — use empty string to trigger catalog/empty fallback
|
||||
const plan = resolvePlan(pool.connectionId, "");
|
||||
// 2. Resolve the provider plan for this pool's connection.
|
||||
// The provider name is not stored on the pool — resolve it from the
|
||||
// connection so catalog-only pools surface their plan dimensions
|
||||
// (passing "" here previously degraded every catalog pool to empty).
|
||||
const provider = await resolveConnectionProvider(pool.connectionId);
|
||||
const plan = resolvePlan(pool.connectionId, provider);
|
||||
|
||||
// 3. Get the quota store and call poolUsageWithDimensions (on the interface since v3.8.12)
|
||||
const store = await getQuotaStore();
|
||||
|
||||
32
src/lib/quota/connectionProvider.ts
Normal file
32
src/lib/quota/connectionProvider.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
/**
|
||||
* connectionProvider.ts — Resolve the provider name for a quota-governed
|
||||
* connection.
|
||||
*
|
||||
* A `QuotaPool` (and the pool-usage route) only carries a `connectionId`, not
|
||||
* the provider name. To resolve the connection's plan via the catalog
|
||||
* (`resolvePlan(connectionId, provider)`) we must look the provider up from the
|
||||
* `provider_connections` row. `getProviderConnectionById` is async, so this
|
||||
* MUST be awaited — a previous private copy in the plans route omitted the
|
||||
* await and silently degraded every catalog lookup to "unknown".
|
||||
*
|
||||
* Fail-safe: returns "unknown" if the DB is unavailable or the connection is
|
||||
* missing, so callers degrade to an empty/manual plan rather than throwing.
|
||||
*
|
||||
* Part of: Group B — Quota Sharing Engine (plan 22).
|
||||
*/
|
||||
|
||||
export async function resolveConnectionProvider(connectionId: string): Promise<string> {
|
||||
try {
|
||||
// Lazy import — avoids circular deps and keeps the module loadable without a full DB.
|
||||
const { getProviderConnectionById } = await import("@/lib/localDb");
|
||||
if (typeof getProviderConnectionById === "function") {
|
||||
const conn = await getProviderConnectionById(connectionId);
|
||||
if (conn && typeof (conn as { provider?: string }).provider === "string") {
|
||||
return (conn as { provider: string }).provider;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// DB not available or export not present — fall through to the safe default.
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
@@ -40,3 +40,90 @@ export function scheduleRecordConsumption(
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the per-request consumption cost payload shared by the streaming and
|
||||
* non-streaming POST-hooks. Coerces token fields defensively (string/NaN safe)
|
||||
* and clamps a negative/zero cost to 0 so a bad pricing lookup never records
|
||||
* negative USD.
|
||||
*/
|
||||
export function buildConsumptionCost(
|
||||
usage: unknown,
|
||||
estimatedCost: number
|
||||
): { tokens: number; usd: number; requests: number } {
|
||||
const u = usage && typeof usage === "object" ? (usage as Record<string, unknown>) : null;
|
||||
const tokens = u
|
||||
? (Number(u.prompt_tokens ?? 0) || 0) + (Number(u.completion_tokens ?? 0) || 0)
|
||||
: 0;
|
||||
return {
|
||||
tokens,
|
||||
usd: estimatedCost > 0 ? estimatedCost : 0,
|
||||
requests: 1,
|
||||
};
|
||||
}
|
||||
|
||||
/** Cost resolver injected for testability (matches `calculateCost`). */
|
||||
type CostResolver = (
|
||||
provider: string,
|
||||
model: string,
|
||||
usage: Record<string, number | undefined> | null | undefined,
|
||||
options: { serviceTier?: string }
|
||||
) => Promise<number>;
|
||||
|
||||
/**
|
||||
* Record shared-quota consumption for a completed STREAMING response.
|
||||
*
|
||||
* Unlike the non-streaming path, the streaming completion previously recorded
|
||||
* `usd: 0` (the cost was resolved asynchronously only for `recordCost`), so
|
||||
* USD-unit pools (e.g. DeepSeek `usd/monthly`) never accrued on streaming
|
||||
* traffic. This resolves the real cost via the injected `calculateCost` and
|
||||
* schedules a single consumption record. Fire-and-forget / fail-open: never
|
||||
* throws to the caller, and still records `requests: 1` when usage is absent.
|
||||
*/
|
||||
export async function recordStreamingConsumption(
|
||||
params: {
|
||||
apiKeyId?: string | null;
|
||||
connectionId?: string | null;
|
||||
provider?: string | null;
|
||||
model: string;
|
||||
streamUsage: unknown;
|
||||
streamStatus: number;
|
||||
serviceTier?: string;
|
||||
},
|
||||
deps: {
|
||||
calculateCost: CostResolver;
|
||||
schedule?: (input: RecordConsumptionInput, log?: MinimalLogger | null) => void;
|
||||
log?: MinimalLogger | null;
|
||||
}
|
||||
): Promise<void> {
|
||||
const { apiKeyId, connectionId, provider, model, streamUsage, streamStatus, serviceTier } =
|
||||
params;
|
||||
if (!apiKeyId || !connectionId || streamStatus !== 200) return;
|
||||
|
||||
const schedule = deps.schedule ?? scheduleRecordConsumption;
|
||||
const resolvedProvider = provider ?? "unknown";
|
||||
|
||||
let estimatedCost = 0;
|
||||
if (streamUsage && typeof streamUsage === "object") {
|
||||
try {
|
||||
estimatedCost = await deps.calculateCost(
|
||||
resolvedProvider,
|
||||
model,
|
||||
streamUsage as Record<string, number | undefined>,
|
||||
{ serviceTier }
|
||||
);
|
||||
} catch {
|
||||
estimatedCost = 0;
|
||||
}
|
||||
}
|
||||
|
||||
schedule(
|
||||
{
|
||||
apiKeyId,
|
||||
connectionId,
|
||||
provider: resolvedProvider,
|
||||
cost: buildConsumptionCost(streamUsage, estimatedCost),
|
||||
},
|
||||
deps.log
|
||||
);
|
||||
}
|
||||
|
||||
164
tests/integration/combo-failover-e2e.test.ts
Normal file
164
tests/integration/combo-failover-e2e.test.ts
Normal file
@@ -0,0 +1,164 @@
|
||||
/**
|
||||
* tests/integration/combo-failover-e2e.test.ts
|
||||
*
|
||||
* End-to-end combo routing scenarios that the existing suite left uncovered:
|
||||
* 1. A 3-target priority chain that walks past TWO failing targets
|
||||
* (500 then 503) to succeed on the third — the existing suite only
|
||||
* exercised a 2-target (single-hop) failover.
|
||||
* 2. A `strategy:"auto"` combo dispatched end-to-end (request → scored
|
||||
* selection → real upstream fetch → 200), closing the gap where auto was
|
||||
* only exercised at the UI layer.
|
||||
* 3. A per-target timeout (targetTimeoutMs) on the first target failing over
|
||||
* to a healthy second target — timeout-driven failover had zero coverage.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createChatPipelineHarness } from "./_chatPipelineHarness.ts";
|
||||
|
||||
const harness = await createChatPipelineHarness("combo-failover-e2e");
|
||||
const {
|
||||
BaseExecutor,
|
||||
buildClaudeResponse,
|
||||
buildGeminiResponse,
|
||||
buildOpenAIResponse,
|
||||
buildRequest,
|
||||
combosDb,
|
||||
handleChat,
|
||||
resetStorage,
|
||||
seedConnection,
|
||||
} = harness;
|
||||
|
||||
test.beforeEach(async () => {
|
||||
BaseExecutor.RETRY_CONFIG.delayMs = 0;
|
||||
await resetStorage();
|
||||
});
|
||||
|
||||
test.afterEach(async () => {
|
||||
BaseExecutor.RETRY_CONFIG.delayMs = harness.originalRetryDelayMs;
|
||||
await resetStorage();
|
||||
});
|
||||
|
||||
test.after(async () => {
|
||||
await harness.cleanup();
|
||||
});
|
||||
|
||||
function body(model: string, content = `Route ${model}`) {
|
||||
return { model, stream: false, messages: [{ role: "user", content }] };
|
||||
}
|
||||
|
||||
test("priority combo walks a 3-target chain: 500 → 503 → success", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-3way" });
|
||||
await seedConnection("claude", { apiKey: "sk-claude-3way" });
|
||||
await seedConnection("gemini", { apiKey: "sk-gemini-3way" });
|
||||
await combosDb.createCombo({
|
||||
name: "router-3way",
|
||||
strategy: "priority",
|
||||
config: { maxRetries: 0, retryDelayMs: 0, fallbackDelayMs: 0 },
|
||||
models: [
|
||||
"openai/gpt-4o-mini",
|
||||
"claude/claude-3-5-sonnet-20241022",
|
||||
"gemini/gemini-2.5-flash",
|
||||
],
|
||||
});
|
||||
|
||||
const attempts: string[] = [];
|
||||
globalThis.fetch = async (url) => {
|
||||
const target = String(url);
|
||||
if (target.includes("/chat/completions")) {
|
||||
attempts.push("openai");
|
||||
return new Response(JSON.stringify({ error: { message: "primary down" } }), {
|
||||
status: 500,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
if (target.includes("?beta=true")) {
|
||||
attempts.push("claude");
|
||||
return new Response(JSON.stringify({ error: { message: "secondary overloaded" } }), {
|
||||
status: 503,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
attempts.push("gemini");
|
||||
return buildGeminiResponse("Third target answered");
|
||||
};
|
||||
|
||||
const res = await handleChat(buildRequest({ body: body("router-3way") }));
|
||||
const json = (await res.json()) as {
|
||||
choices: Array<{ message: { content: string } }>;
|
||||
};
|
||||
|
||||
assert.equal(res.status, 200, "request must succeed on the 3rd target");
|
||||
assert.deepEqual(attempts, ["openai", "claude", "gemini"], "all three targets attempted in order");
|
||||
assert.equal(json.choices[0].message.content, "Third target answered");
|
||||
});
|
||||
|
||||
test("priority combo fails over when the first target exceeds its per-target timeout", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-timeout" });
|
||||
await seedConnection("claude", { apiKey: "sk-claude-timeout" });
|
||||
await combosDb.createCombo({
|
||||
name: "router-timeout",
|
||||
strategy: "priority",
|
||||
// 80ms per-target ceiling; the first target hangs past it and is aborted.
|
||||
config: { maxRetries: 0, retryDelayMs: 0, fallbackDelayMs: 0, targetTimeoutMs: 80 },
|
||||
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022"],
|
||||
});
|
||||
|
||||
const attempts: string[] = [];
|
||||
globalThis.fetch = async (url, init: RequestInit = {}) => {
|
||||
const target = String(url);
|
||||
if (target.includes("/chat/completions")) {
|
||||
attempts.push("openai");
|
||||
// Hang until the combo's per-target timeout aborts us via the signal.
|
||||
return await new Promise<Response>((_resolve, reject) => {
|
||||
const signal = init.signal;
|
||||
if (signal) {
|
||||
signal.addEventListener("abort", () =>
|
||||
reject(Object.assign(new Error("aborted by combo timeout"), { name: "AbortError" }))
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
attempts.push("claude");
|
||||
return buildClaudeResponse("Recovered after timeout");
|
||||
};
|
||||
|
||||
const res = await handleChat(buildRequest({ body: body("router-timeout") }));
|
||||
const json = (await res.json()) as {
|
||||
choices: Array<{ message: { content: string } }>;
|
||||
};
|
||||
|
||||
assert.equal(res.status, 200, "must fail over to the second target after the first times out");
|
||||
assert.deepEqual(attempts, ["openai", "claude"]);
|
||||
assert.equal(json.choices[0].message.content, "Recovered after timeout");
|
||||
});
|
||||
|
||||
test("auto combo selects and dispatches a scored candidate end-to-end", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-auto" });
|
||||
await seedConnection("claude", { apiKey: "sk-claude-auto" });
|
||||
await combosDb.createCombo({
|
||||
name: "router-auto",
|
||||
strategy: "auto",
|
||||
config: { maxRetries: 0, retryDelayMs: 0 },
|
||||
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022"],
|
||||
});
|
||||
|
||||
const seen: string[] = [];
|
||||
globalThis.fetch = async (url) => {
|
||||
const target = String(url);
|
||||
if (target.includes("?beta=true")) {
|
||||
seen.push("claude");
|
||||
return buildClaudeResponse("Auto chose claude");
|
||||
}
|
||||
seen.push("openai");
|
||||
return buildOpenAIResponse("Auto chose openai");
|
||||
};
|
||||
|
||||
const res = await handleChat(buildRequest({ body: body("router-auto") }));
|
||||
const json = (await res.json()) as {
|
||||
choices: Array<{ message: { content: string } }>;
|
||||
};
|
||||
|
||||
assert.equal(res.status, 200, "auto combo must dispatch successfully");
|
||||
assert.equal(seen.length, 1, "auto selects exactly one target (no needless fan-out)");
|
||||
assert.match(json.choices[0].message.content, /Auto chose (openai|claude)/);
|
||||
});
|
||||
106
tests/integration/quota-pool-usage-provider-resolution.test.ts
Normal file
106
tests/integration/quota-pool-usage-provider-resolution.test.ts
Normal file
@@ -0,0 +1,106 @@
|
||||
/**
|
||||
* Integration: GET /api/quota/pools/[id]/usage must resolve the pool's provider
|
||||
* from its connection so catalog-only pools (no manual `provider_plans` row)
|
||||
* still surface their plan dimensions.
|
||||
*
|
||||
* Regression: the route passed `resolvePlan(pool.connectionId, "")` with an
|
||||
* empty provider, so `getKnownPlan("")` returned null → empty plan → the route
|
||||
* fell back to the dimension-less `poolUsage()` snapshot, blanking the
|
||||
* dashboard (StackedAllocationBar / DimensionBar / BurnRateChart) for every
|
||||
* catalog-only pool.
|
||||
*
|
||||
* Fix: resolve the provider via `resolveConnectionProvider(connectionId)`
|
||||
* (awaited DB lookup) and pass it to `resolvePlan`.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { makeManagementSessionRequest } from "../helpers/managementSession.ts";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), "omniroute-quota-pool-usage-provider-")
|
||||
);
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
process.env.API_KEY_SECRET = "test-quota-usage-provider-secret";
|
||||
process.env.QUOTA_STORE_DRIVER = "sqlite";
|
||||
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const localDb = await import("../../src/lib/localDb.ts");
|
||||
const { createPool, upsertAllocations, createProviderConnection } = localDb;
|
||||
const { resetQuotaStoreSingleton } = await import("../../src/lib/quota/QuotaStore.ts");
|
||||
const usageRoute = await import("../../src/app/api/quota/pools/[id]/usage/route.ts");
|
||||
|
||||
function resetDb() {
|
||||
core.resetDbInstance();
|
||||
resetQuotaStoreSingleton();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
test.beforeEach(() => {
|
||||
resetDb();
|
||||
});
|
||||
|
||||
test.after(() => {
|
||||
core.resetDbInstance();
|
||||
resetQuotaStoreSingleton();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("GET /usage surfaces catalog dimensions for a catalog-only pool (provider resolved from connection)", async () => {
|
||||
// kimi has a catalog plan: { unit: 'requests', window: 'hourly', limit: 1500 }
|
||||
const conn = (await createProviderConnection({
|
||||
provider: "kimi",
|
||||
name: "kimi-usage-test",
|
||||
authType: "apikey",
|
||||
apiKey: "sk-kimi-test",
|
||||
})) as { id: string };
|
||||
|
||||
const pool = createPool({ name: "Kimi Shared Pool", connectionId: conn.id });
|
||||
upsertAllocations(pool.id, [{ apiKeyId: "key-a", weight: 100, policy: "hard" }]);
|
||||
|
||||
const req = await makeManagementSessionRequest(
|
||||
`http://localhost/api/quota/pools/${pool.id}/usage`
|
||||
);
|
||||
const res = await usageRoute.GET(req, { params: Promise.resolve({ id: pool.id }) });
|
||||
assert.equal(res.status, 200);
|
||||
|
||||
const body = (await res.json()) as {
|
||||
usage: {
|
||||
dimensions: Array<{ unit: string; window: string; limit: number }>;
|
||||
};
|
||||
};
|
||||
|
||||
// Regression assertion: was [] because resolvePlan(connId, "") found no catalog match.
|
||||
assert.ok(
|
||||
body.usage.dimensions.length >= 1,
|
||||
"catalog-only pool must surface plan dimensions (was blank)"
|
||||
);
|
||||
const dim = body.usage.dimensions[0];
|
||||
assert.equal(dim.unit, "requests");
|
||||
assert.equal(dim.window, "hourly");
|
||||
assert.equal(dim.limit, 1500);
|
||||
});
|
||||
|
||||
test("GET /usage still returns a 200 snapshot when the provider has no catalog plan", async () => {
|
||||
// A provider with no catalog entry resolves to an empty (manual) plan; the
|
||||
// route must still return a 200 minimal snapshot (no dimensions), not error.
|
||||
const conn = (await createProviderConnection({
|
||||
provider: "some-unknown-provider",
|
||||
name: "unknown-usage-test",
|
||||
authType: "apikey",
|
||||
apiKey: "sk-unknown-test",
|
||||
})) as { id: string };
|
||||
|
||||
const pool = createPool({ name: "Unknown Pool", connectionId: conn.id });
|
||||
|
||||
const req = await makeManagementSessionRequest(
|
||||
`http://localhost/api/quota/pools/${pool.id}/usage`
|
||||
);
|
||||
const res = await usageRoute.GET(req, { params: Promise.resolve({ id: pool.id }) });
|
||||
assert.equal(res.status, 200);
|
||||
const body = (await res.json()) as { usage: { dimensions: unknown[] } };
|
||||
assert.ok(Array.isArray(body.usage.dimensions));
|
||||
});
|
||||
108
tests/unit/auto-combo-scoring-clamp.test.ts
Normal file
108
tests/unit/auto-combo-scoring-clamp.test.ts
Normal file
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* tests/unit/auto-combo-scoring-clamp.test.ts
|
||||
*
|
||||
* Regression hardening: `calculateScore` summed weighted factors with NO final
|
||||
* clamp and NO NaN guard, and `calculateFactors` lower-bounded none of its
|
||||
* factors. A single NaN/negative input (bad telemetry, negative quota, negative
|
||||
* cost) could yield a NaN or out-of-[0,1] score that sinks a candidate
|
||||
* nondeterministically (NaN sorts unpredictably) or distorts ranking.
|
||||
*
|
||||
* Fix: clamp every factor to [0,1] in calculateFactors and clamp the final
|
||||
* score to [0,1] (clamp01 maps non-finite → 0).
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
calculateScore,
|
||||
calculateFactors,
|
||||
DEFAULT_WEIGHTS,
|
||||
} from "../../open-sse/services/autoCombo/scoring.ts";
|
||||
import type {
|
||||
ScoringFactors,
|
||||
ProviderCandidate,
|
||||
} from "../../open-sse/services/autoCombo/scoring.ts";
|
||||
|
||||
const ONES: ScoringFactors = {
|
||||
quota: 1,
|
||||
health: 1,
|
||||
costInv: 1,
|
||||
latencyInv: 1,
|
||||
taskFit: 1,
|
||||
stability: 1,
|
||||
tierPriority: 1,
|
||||
tierAffinity: 1,
|
||||
specificityMatch: 1,
|
||||
contextAffinity: 1,
|
||||
resetWindowAffinity: 1,
|
||||
connectionDensity: 1,
|
||||
};
|
||||
|
||||
function candidate(partial: Partial<ProviderCandidate> = {}): ProviderCandidate {
|
||||
return {
|
||||
provider: "p",
|
||||
model: "m",
|
||||
quotaRemaining: 100,
|
||||
quotaTotal: 100,
|
||||
circuitBreakerState: "CLOSED",
|
||||
costPer1MTokens: 1,
|
||||
p95LatencyMs: 100,
|
||||
latencyStdDev: 10,
|
||||
errorRate: 0,
|
||||
accountTier: "standard",
|
||||
quotaResetIntervalSecs: 86400,
|
||||
...partial,
|
||||
};
|
||||
}
|
||||
|
||||
test("calculateScore — NaN factor yields a finite [0,1] score (no NaN propagation)", () => {
|
||||
const score = calculateScore({ ...ONES, quota: NaN }, DEFAULT_WEIGHTS);
|
||||
assert.ok(Number.isFinite(score), "score must be finite even with a NaN factor");
|
||||
assert.ok(score >= 0 && score <= 1, `score in [0,1], got ${score}`);
|
||||
});
|
||||
|
||||
test("calculateScore — clamps to [0,1]; all-ones with normalized weights ≈ 1", () => {
|
||||
const score = calculateScore(ONES, DEFAULT_WEIGHTS);
|
||||
assert.ok(score >= 0 && score <= 1);
|
||||
assert.ok(Math.abs(score - 1) < 1e-6, "normalized weights × all-ones ≈ 1");
|
||||
});
|
||||
|
||||
test("calculateScore — negative factors cannot drive the score below 0", () => {
|
||||
const score = calculateScore({ ...ONES, costInv: -5, latencyInv: -5 }, DEFAULT_WEIGHTS);
|
||||
assert.ok(score >= 0, `score floored at 0, got ${score}`);
|
||||
});
|
||||
|
||||
test("calculateFactors — negative quotaRemaining clamps the quota factor to [0,1]", () => {
|
||||
const c = candidate({ quotaRemaining: -50 });
|
||||
const f = calculateFactors(c, [c], "default", () => 0.5);
|
||||
assert.ok(f.quota >= 0 && f.quota <= 1, `quota factor must be in [0,1], got ${f.quota}`);
|
||||
});
|
||||
|
||||
test("calculateFactors — negative cost cannot push costInv above 1", () => {
|
||||
const c = candidate({ costPer1MTokens: -100 });
|
||||
const f = calculateFactors(c, [c], "default", () => 0.5);
|
||||
assert.ok(f.costInv >= 0 && f.costInv <= 1, `costInv must be in [0,1], got ${f.costInv}`);
|
||||
});
|
||||
|
||||
test("calculateFactors — out-of-range contextAffinity is clamped", () => {
|
||||
const c = candidate({ contextAffinity: 5 });
|
||||
const f = calculateFactors(c, [c], "default", () => 0.5);
|
||||
assert.ok(
|
||||
f.contextAffinity >= 0 && f.contextAffinity <= 1,
|
||||
`contextAffinity must be in [0,1], got ${f.contextAffinity}`
|
||||
);
|
||||
});
|
||||
|
||||
test("calculateFactors — connectionDensity is clamped to [0,1] and NaN-safe", () => {
|
||||
// A large pool ((1000-1)/10 = 99.9) must not exceed 1 and skew the weighted score.
|
||||
const big = calculateFactors(candidate({ connectionPoolSize: 1000 }), [candidate()], "default", () => 0.5);
|
||||
assert.ok(
|
||||
big.connectionDensity >= 0 && big.connectionDensity <= 1,
|
||||
`connectionDensity must be in [0,1], got ${big.connectionDensity}`
|
||||
);
|
||||
// A non-finite pool size must map to 0 (clamp01), not propagate NaN into the score.
|
||||
const nan = calculateFactors(candidate({ connectionPoolSize: NaN }), [candidate()], "default", () => 0.5);
|
||||
assert.ok(
|
||||
Number.isFinite(nan.connectionDensity),
|
||||
`connectionDensity must be finite (clamp01 maps NaN→0), got ${nan.connectionDensity}`
|
||||
);
|
||||
});
|
||||
@@ -1,13 +1,66 @@
|
||||
import { test, mock } from "node:test";
|
||||
import assert from "node:assert";
|
||||
import { handleComboChat } from "@omniroute/open-sse/services/combo.ts";
|
||||
import * as metricsDb from "@omniroute/src/lib/db/stats.ts";
|
||||
/**
|
||||
* tests/unit/combo-hedging.test.ts
|
||||
*
|
||||
* Previously a placeholder (two `assert.ok(true)`). Now exercises the two
|
||||
* zero-latency features it claims to cover:
|
||||
* 1. Predictive-TTFT circuit-breaker DECISION (shouldSkipForPredictedTtft).
|
||||
* 2. Hedging / zero-latency CONFIG resolution (defaults + per-combo override),
|
||||
* i.e. the knobs the combo engine reads before racing/skip-ahead.
|
||||
*
|
||||
* The full hedging DISPATCH (Promise.race + loser abort) is covered end-to-end
|
||||
* in combo-routing-engine.test.ts; here we pin the decision logic + config
|
||||
* plumbing that gate it (both opt-in / off by default).
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { shouldSkipForPredictedTtft } from "../../open-sse/services/combo.ts";
|
||||
import { getDefaultComboConfig, resolveComboConfig } from "../../open-sse/services/comboConfig.ts";
|
||||
|
||||
test("combo: predictive TTFT skips slow model without aborting combo", async () => {
|
||||
// Add basic test here
|
||||
assert.ok(true);
|
||||
// ── Predictive-TTFT circuit breaker (decision) ────────────────────────────────
|
||||
test("predictive-TTFT — skips a model whose avg latency exceeds the ceiling (enough samples)", () => {
|
||||
assert.equal(shouldSkipForPredictedTtft({ requests: 10, avgLatencyMs: 5000 }, 2000), true);
|
||||
});
|
||||
|
||||
test("combo: hedging logic works correctly", async () => {
|
||||
assert.ok(true);
|
||||
test("predictive-TTFT — does NOT skip with too few samples (< 5)", () => {
|
||||
assert.equal(shouldSkipForPredictedTtft({ requests: 4, avgLatencyMs: 9000 }, 2000), false);
|
||||
});
|
||||
|
||||
test("predictive-TTFT — does NOT skip when avg latency is within the ceiling", () => {
|
||||
assert.equal(shouldSkipForPredictedTtft({ requests: 50, avgLatencyMs: 1200 }, 2000), false);
|
||||
});
|
||||
|
||||
test("predictive-TTFT — disabled (ceiling <= 0) never skips", () => {
|
||||
assert.equal(shouldSkipForPredictedTtft({ requests: 50, avgLatencyMs: 9000 }, 0), false);
|
||||
});
|
||||
|
||||
test("predictive-TTFT — null/missing metric never skips", () => {
|
||||
assert.equal(shouldSkipForPredictedTtft(null, 2000), false);
|
||||
assert.equal(shouldSkipForPredictedTtft(undefined, 2000), false);
|
||||
});
|
||||
|
||||
// ── Hedging / zero-latency config resolution ──────────────────────────────────
|
||||
test("hedging — defaults are opt-in (off) so normal combos never race", () => {
|
||||
const d = getDefaultComboConfig();
|
||||
assert.equal(d.hedging, false);
|
||||
assert.equal(d.hedgeDelayMs, 500);
|
||||
assert.equal(d.predictiveTtftMs, 0);
|
||||
assert.equal(d.zeroLatencyOptimizationsEnabled, false);
|
||||
});
|
||||
|
||||
test("hedging — per-combo config overrides the zero-latency defaults", () => {
|
||||
const resolved = resolveComboConfig(
|
||||
{
|
||||
config: {
|
||||
hedging: true,
|
||||
hedgeDelayMs: 200,
|
||||
zeroLatencyOptimizationsEnabled: true,
|
||||
predictiveTtftMs: 3000,
|
||||
},
|
||||
},
|
||||
null
|
||||
);
|
||||
assert.equal(resolved.hedging, true);
|
||||
assert.equal(resolved.hedgeDelayMs, 200);
|
||||
assert.equal(resolved.zeroLatencyOptimizationsEnabled, true);
|
||||
assert.equal(resolved.predictiveTtftMs, 3000);
|
||||
});
|
||||
|
||||
58
tests/unit/combo-max-depth-config.test.ts
Normal file
58
tests/unit/combo-max-depth-config.test.ts
Normal file
@@ -0,0 +1,58 @@
|
||||
/**
|
||||
* tests/unit/combo-max-depth-config.test.ts
|
||||
*
|
||||
* Regression: `config.maxComboDepth` (DEFAULT_COMBO_CONFIG) had ZERO readers —
|
||||
* nesting depth was always the hardcoded `MAX_COMBO_DEPTH = 3`, so the operator
|
||||
* knob did nothing. This wires a clamped, per-resolution `maxDepth` through the
|
||||
* combo depth functions (default preserved, hard-capped for safety).
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
// 5-combo chain L0→L1→L2→L3→L4→model — L4 sits at recursion depth 4.
|
||||
function buildDeepChain() {
|
||||
return [
|
||||
{ name: "L0", models: ["L1"] },
|
||||
{ name: "L1", models: ["L2"] },
|
||||
{ name: "L2", models: ["L3"] },
|
||||
{ name: "L3", models: ["L4"] },
|
||||
{ name: "L4", models: ["openai/gpt-4"] },
|
||||
];
|
||||
}
|
||||
|
||||
test("clampComboDepth — clamps to [1, hard cap]; invalid → default 3", async () => {
|
||||
const { clampComboDepth } = await import("../../open-sse/services/combo.ts");
|
||||
assert.equal(clampComboDepth(1), 1);
|
||||
assert.equal(clampComboDepth(2), 2);
|
||||
assert.equal(clampComboDepth(5), 5);
|
||||
assert.equal(clampComboDepth(100), 10, "hard cap at 10");
|
||||
assert.equal(clampComboDepth(0), 3, "0 invalid → default 3");
|
||||
assert.equal(clampComboDepth(-4), 3, "negative → default 3");
|
||||
assert.equal(clampComboDepth(undefined), 3, "undefined → default 3");
|
||||
assert.equal(clampComboDepth("abc"), 3, "non-numeric → default 3");
|
||||
assert.equal(clampComboDepth(2.9), 2, "floors to 2");
|
||||
});
|
||||
|
||||
test("validateComboDAG — default depth (3) still throws on a 4-deep chain", async () => {
|
||||
const { validateComboDAG } = await import("../../open-sse/services/combo.ts");
|
||||
assert.throws(() => validateComboDAG("L0", buildDeepChain()), /nesting depth/);
|
||||
});
|
||||
|
||||
test("validateComboDAG — honors a HIGHER configured maxDepth (was hardcoded 3)", async () => {
|
||||
const { validateComboDAG } = await import("../../open-sse/services/combo.ts");
|
||||
// maxDepth 5 → the 4-deep chain validates without throwing.
|
||||
assert.doesNotThrow(() => validateComboDAG("L0", buildDeepChain(), new Set(), 0, 5));
|
||||
});
|
||||
|
||||
test("validateComboDAG — honors a LOWER configured maxDepth", async () => {
|
||||
const { validateComboDAG } = await import("../../open-sse/services/combo.ts");
|
||||
const shallow = [
|
||||
{ name: "A", models: ["B"] },
|
||||
{ name: "B", models: ["C"] },
|
||||
{ name: "C", models: ["openai/gpt-4"] },
|
||||
];
|
||||
// A→B→C (C at depth 2): default (3) allows it…
|
||||
assert.doesNotThrow(() => validateComboDAG("A", shallow));
|
||||
// …but maxDepth 1 rejects it.
|
||||
assert.throws(() => validateComboDAG("A", shallow, new Set(), 0, 1), /nesting depth/);
|
||||
});
|
||||
69
tests/unit/combo-provider-diversity-wiring.test.ts
Normal file
69
tests/unit/combo-provider-diversity-wiring.test.ts
Normal file
@@ -0,0 +1,69 @@
|
||||
/**
|
||||
* tests/unit/combo-provider-diversity-wiring.test.ts
|
||||
*
|
||||
* Regression: `recordProviderUsage` (open-sse/services/autoCombo/providerDiversity.ts)
|
||||
* had ZERO production callers, so `/api/analytics/diversity` always reported
|
||||
* `score: 1.0` with an empty `providers` map — a dead dashboard.
|
||||
*
|
||||
* Fix: `recordComboRequest` (the single chokepoint every combo strategy funnels
|
||||
* through) records the successful target's provider into the diversity window.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
test("recordComboRequest feeds the provider-diversity report on success", async () => {
|
||||
const { recordComboRequest } = await import("../../open-sse/services/comboMetrics.ts");
|
||||
const { getDiversityReport, resetDiversity } = await import(
|
||||
"../../open-sse/services/autoCombo/providerDiversity.ts"
|
||||
);
|
||||
resetDiversity();
|
||||
|
||||
recordComboRequest("combo-diversity-x", "openai/gpt-4", {
|
||||
success: true,
|
||||
latencyMs: 100,
|
||||
strategy: "priority",
|
||||
target: { provider: "openai" },
|
||||
});
|
||||
recordComboRequest("combo-diversity-x", "claude/sonnet", {
|
||||
success: true,
|
||||
latencyMs: 120,
|
||||
strategy: "priority",
|
||||
target: { provider: "claude" },
|
||||
});
|
||||
recordComboRequest("combo-diversity-x", "openai/gpt-4", {
|
||||
success: true,
|
||||
latencyMs: 90,
|
||||
strategy: "priority",
|
||||
target: { provider: "openai" },
|
||||
});
|
||||
|
||||
const report = getDiversityReport();
|
||||
assert.equal(report.totalRequests, 3, "all 3 successful dispatches recorded");
|
||||
assert.equal(report.providers.openai?.count, 2);
|
||||
assert.equal(report.providers.claude?.count, 1);
|
||||
assert.ok(report.score > 0, "diversity score reflects multiple providers (was always 1.0/empty)");
|
||||
});
|
||||
|
||||
test("recordComboRequest does NOT pollute diversity on failure", async () => {
|
||||
const { recordComboRequest } = await import("../../open-sse/services/comboMetrics.ts");
|
||||
const { getDiversityReport, resetDiversity } = await import(
|
||||
"../../open-sse/services/autoCombo/providerDiversity.ts"
|
||||
);
|
||||
resetDiversity();
|
||||
|
||||
recordComboRequest("combo-diversity-y", "openai/gpt-4", {
|
||||
success: false,
|
||||
latencyMs: 100,
|
||||
strategy: "priority",
|
||||
target: { provider: "openai" },
|
||||
});
|
||||
// Terminal "all targets failed" call (modelStr null, no provider) must also be safe.
|
||||
recordComboRequest("combo-diversity-y", null, {
|
||||
success: false,
|
||||
latencyMs: 200,
|
||||
strategy: "priority",
|
||||
});
|
||||
|
||||
const report = getDiversityReport();
|
||||
assert.equal(report.totalRequests, 0, "failures must not enter the diversity window");
|
||||
});
|
||||
65
tests/unit/complexity-aware-scoring-wiring.test.ts
Normal file
65
tests/unit/complexity-aware-scoring-wiring.test.ts
Normal file
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* tests/unit/complexity-aware-scoring-wiring.test.ts
|
||||
*
|
||||
* Proves the 2026 complexity-aware wiring: feeding a tier/complexity hint into
|
||||
* scoreAutoTargets makes the tierAffinity / specificityMatch factors live
|
||||
* (they were a constant 0.5 because the hint was never passed — audit Bug #4),
|
||||
* while a null hint keeps the default behavior byte-for-byte.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { scoreAutoTargets } from "../../open-sse/services/combo.ts";
|
||||
import { DEFAULT_WEIGHTS } from "../../open-sse/services/autoCombo/scoring.ts";
|
||||
import type { RoutingHint } from "../../open-sse/services/manifestAdapter.ts";
|
||||
|
||||
function target() {
|
||||
return {
|
||||
kind: "model",
|
||||
provider: "openai",
|
||||
model: "gpt-4o-mini",
|
||||
modelStr: "openai/gpt-4o-mini",
|
||||
executionKey: "k1",
|
||||
stepId: "s1",
|
||||
} as unknown as Parameters<typeof scoreAutoTargets>[0][number];
|
||||
}
|
||||
|
||||
function candidate() {
|
||||
return {
|
||||
executionKey: "k1",
|
||||
provider: "openai",
|
||||
model: "gpt-4o-mini",
|
||||
modelStr: "openai/gpt-4o-mini",
|
||||
quotaRemaining: 100,
|
||||
quotaTotal: 100,
|
||||
circuitBreakerState: "CLOSED",
|
||||
costPer1MTokens: 1,
|
||||
p95LatencyMs: 100,
|
||||
latencyStdDev: 10,
|
||||
errorRate: 0,
|
||||
accountTier: "standard",
|
||||
quotaResetIntervalSecs: 86400,
|
||||
} as unknown as Parameters<typeof scoreAutoTargets>[1][number];
|
||||
}
|
||||
|
||||
test("scoreAutoTargets — a tier/complexity hint moves the score off tier-neutral", () => {
|
||||
const withoutHint = scoreAutoTargets([target()], [candidate()], "default", DEFAULT_WEIGHTS);
|
||||
const hint = {
|
||||
recommendedMinTier: "premium",
|
||||
specificity: { score: 80 },
|
||||
} as unknown as RoutingHint;
|
||||
const withHint = scoreAutoTargets([target()], [candidate()], "default", DEFAULT_WEIGHTS, hint);
|
||||
|
||||
assert.equal(withoutHint.length, 1);
|
||||
assert.equal(withHint.length, 1);
|
||||
assert.notEqual(
|
||||
withHint[0].score,
|
||||
withoutHint[0].score,
|
||||
"feeding a tier hint must change tierAffinity/specificityMatch (was constant 0.5)"
|
||||
);
|
||||
});
|
||||
|
||||
test("scoreAutoTargets — a null hint is identical to no hint (backward compatible)", () => {
|
||||
const a = scoreAutoTargets([target()], [candidate()], "default", DEFAULT_WEIGHTS);
|
||||
const b = scoreAutoTargets([target()], [candidate()], "default", DEFAULT_WEIGHTS, null);
|
||||
assert.equal(a[0].score, b[0].score, "null hint must equal no hint (default behavior unchanged)");
|
||||
});
|
||||
107
tests/unit/complexity-router.test.ts
Normal file
107
tests/unit/complexity-router.test.ts
Normal file
@@ -0,0 +1,107 @@
|
||||
/**
|
||||
* tests/unit/complexity-router.test.ts
|
||||
*
|
||||
* 2026 strategy: request-complexity classification → recommended tier, with an
|
||||
* explicit tool-use escalation. Validates the classifier facade over the
|
||||
* existing specificity detector.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
classifyRequestComplexity,
|
||||
escalateTier,
|
||||
buildComplexityRoutingHint,
|
||||
} from "../../open-sse/services/autoCombo/complexityRouter.ts";
|
||||
|
||||
const NOOP_LOG = { info: () => {} };
|
||||
|
||||
function modelTargets(): Parameters<typeof buildComplexityRoutingHint>[0] {
|
||||
return [
|
||||
{
|
||||
kind: "model",
|
||||
provider: "openai",
|
||||
model: "gpt-4o-mini",
|
||||
modelStr: "openai/gpt-4o-mini",
|
||||
executionKey: "k1",
|
||||
stepId: "s1",
|
||||
},
|
||||
] as unknown as Parameters<typeof buildComplexityRoutingHint>[0];
|
||||
}
|
||||
|
||||
test("escalateTier — raises to the floor, never lowers", () => {
|
||||
assert.equal(escalateTier("free", "cheap"), "cheap");
|
||||
assert.equal(escalateTier("free", "premium"), "premium");
|
||||
assert.equal(escalateTier("premium", "cheap"), "premium");
|
||||
assert.equal(escalateTier("cheap", "free"), "cheap");
|
||||
assert.equal(escalateTier("cheap", "cheap"), "cheap");
|
||||
});
|
||||
|
||||
test("classifyRequestComplexity — a trivial prompt stays cheap/free with no tool signal", () => {
|
||||
const c = classifyRequestComplexity({ messages: [{ role: "user", content: "hi there" }] });
|
||||
assert.equal(c.hasToolUse, false);
|
||||
assert.equal(c.recommendedTier, "free");
|
||||
assert.ok(["trivial", "simple"].includes(c.level), `expected low level, got ${c.level}`);
|
||||
});
|
||||
|
||||
test("classifyRequestComplexity — a hard, multi-step coding+reasoning prompt scores higher", () => {
|
||||
const trivial = classifyRequestComplexity({ messages: [{ role: "user", content: "hi" }] });
|
||||
const hard = classifyRequestComplexity({
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content:
|
||||
"First, analyze this TypeScript module for race conditions:\n" +
|
||||
"```ts\nasync function f(){ /* ... */ }\n```\n" +
|
||||
"Then, step by step, prove the time complexity is O(n log n), " +
|
||||
"derive the recurrence relation, and refactor it to remove the data race. " +
|
||||
"Finally, explain the trade-offs of each approach in depth.",
|
||||
},
|
||||
],
|
||||
});
|
||||
assert.ok(hard.score > trivial.score, `hard (${hard.score}) must exceed trivial (${trivial.score})`);
|
||||
});
|
||||
|
||||
test("classifyRequestComplexity — tool schemas escalate the tier above free", () => {
|
||||
const c = classifyRequestComplexity({
|
||||
messages: [{ role: "user", content: "weather?" }],
|
||||
tools: [
|
||||
{
|
||||
function: {
|
||||
name: "get_weather",
|
||||
description: "Get the weather",
|
||||
parameters: { type: "object", properties: { city: { type: "string" } } },
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
assert.equal(c.hasToolUse, true);
|
||||
assert.notEqual(c.recommendedTier, "free", "tool-using requests must not route to the free tier");
|
||||
});
|
||||
|
||||
test("buildComplexityRoutingHint — a tool-using request floors the hint tier above free", () => {
|
||||
const hint = buildComplexityRoutingHint(
|
||||
modelTargets(),
|
||||
{
|
||||
messages: [{ role: "user", content: "weather?" }],
|
||||
tools: [{ function: { name: "get_weather", description: "Get the weather", parameters: {} } }],
|
||||
},
|
||||
NOOP_LOG
|
||||
);
|
||||
assert.ok(hint, "expected a non-null hint when complexity routing builds successfully");
|
||||
if (!hint) return;
|
||||
assert.notEqual(
|
||||
hint.recommendedMinTier,
|
||||
"free",
|
||||
"tool-use must floor the recommended tier at cheap (escalation applied)"
|
||||
);
|
||||
});
|
||||
|
||||
test("buildComplexityRoutingHint — a null body is safe and still builds a tier-neutral hint", () => {
|
||||
const hint = buildComplexityRoutingHint(modelTargets(), null, NOOP_LOG);
|
||||
assert.ok(hint, "a null body must not throw — messages default to [] and a hint is built");
|
||||
if (!hint) return;
|
||||
assert.ok(
|
||||
["free", "cheap", "premium"].includes(hint.recommendedMinTier),
|
||||
`unexpected tier ${hint.recommendedMinTier}`
|
||||
);
|
||||
});
|
||||
174
tests/unit/quota-streaming-consumption-usd.test.ts
Normal file
174
tests/unit/quota-streaming-consumption-usd.test.ts
Normal file
@@ -0,0 +1,174 @@
|
||||
/**
|
||||
* tests/unit/quota-streaming-consumption-usd.test.ts
|
||||
*
|
||||
* Regression: the streaming quota-share POST-hook in chatCore recorded
|
||||
* `usd: 0` hardcoded, so USD-unit shared pools (e.g. DeepSeek `usd/monthly`)
|
||||
* never accrued cost on STREAMING traffic — the dominant request shape — and
|
||||
* therefore never blocked. The non-streaming path correctly used the computed
|
||||
* `estimatedCost`.
|
||||
*
|
||||
* Fix: extract a testable `recordStreamingConsumption` (with injectable
|
||||
* `calculateCost` + `schedule`) and a shared `buildConsumptionCost`, and wire
|
||||
* the real estimated cost into the streaming consumption record.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import type { RecordConsumptionInput } from "../../src/lib/quota/types.ts";
|
||||
|
||||
test("buildConsumptionCost — flows estimatedCost into usd (not hardcoded 0)", async () => {
|
||||
const { buildConsumptionCost } = await import("../../src/lib/quota/spendRecorder.ts");
|
||||
const cost = buildConsumptionCost({ prompt_tokens: 1000, completion_tokens: 500 }, 0.0234);
|
||||
assert.equal(cost.tokens, 1500);
|
||||
assert.equal(cost.usd, 0.0234);
|
||||
assert.equal(cost.requests, 1);
|
||||
});
|
||||
|
||||
test("buildConsumptionCost — null usage → tokens 0, usd 0, requests 1", async () => {
|
||||
const { buildConsumptionCost } = await import("../../src/lib/quota/spendRecorder.ts");
|
||||
const cost = buildConsumptionCost(null, 0);
|
||||
assert.deepEqual(cost, { tokens: 0, usd: 0, requests: 1 });
|
||||
});
|
||||
|
||||
test("buildConsumptionCost — coerces string tokens, clamps negative cost to 0", async () => {
|
||||
const { buildConsumptionCost } = await import("../../src/lib/quota/spendRecorder.ts");
|
||||
const cost = buildConsumptionCost({ prompt_tokens: "10", completion_tokens: "5" }, -1);
|
||||
assert.equal(cost.tokens, 15);
|
||||
assert.equal(cost.usd, 0);
|
||||
});
|
||||
|
||||
test("recordStreamingConsumption — records REAL usd for a 200 stream (regression: was 0)", async () => {
|
||||
const { recordStreamingConsumption } = await import("../../src/lib/quota/spendRecorder.ts");
|
||||
const scheduled: RecordConsumptionInput[] = [];
|
||||
let calcArgs: { p: string; m: string } | null = null;
|
||||
await recordStreamingConsumption(
|
||||
{
|
||||
apiKeyId: "key-1",
|
||||
connectionId: "conn-1",
|
||||
provider: "deepseek",
|
||||
model: "deepseek-chat",
|
||||
streamUsage: { prompt_tokens: 2000, completion_tokens: 1000 },
|
||||
streamStatus: 200,
|
||||
},
|
||||
{
|
||||
calculateCost: async (p: string, m: string) => {
|
||||
calcArgs = { p, m };
|
||||
return 0.042;
|
||||
},
|
||||
schedule: (input) => scheduled.push(input),
|
||||
}
|
||||
);
|
||||
assert.equal(scheduled.length, 1);
|
||||
assert.equal(scheduled[0].cost.usd, 0.042, "streaming usd must reflect calculateCost, not 0");
|
||||
assert.equal(scheduled[0].cost.tokens, 3000);
|
||||
assert.equal(scheduled[0].cost.requests, 1);
|
||||
assert.equal(scheduled[0].apiKeyId, "key-1");
|
||||
assert.equal(scheduled[0].connectionId, "conn-1");
|
||||
assert.equal(scheduled[0].provider, "deepseek");
|
||||
assert.ok(calcArgs);
|
||||
assert.equal(calcArgs!.p, "deepseek");
|
||||
assert.equal(calcArgs!.m, "deepseek-chat");
|
||||
});
|
||||
|
||||
test("recordStreamingConsumption — skips non-200 streams entirely", async () => {
|
||||
const { recordStreamingConsumption } = await import("../../src/lib/quota/spendRecorder.ts");
|
||||
const scheduled: RecordConsumptionInput[] = [];
|
||||
let calcCalled = false;
|
||||
await recordStreamingConsumption(
|
||||
{
|
||||
apiKeyId: "k",
|
||||
connectionId: "c",
|
||||
provider: "openai",
|
||||
model: "gpt-4",
|
||||
streamUsage: { prompt_tokens: 1 },
|
||||
streamStatus: 500,
|
||||
},
|
||||
{
|
||||
calculateCost: async () => {
|
||||
calcCalled = true;
|
||||
return 1;
|
||||
},
|
||||
schedule: (input) => scheduled.push(input),
|
||||
}
|
||||
);
|
||||
assert.equal(scheduled.length, 0);
|
||||
assert.equal(calcCalled, false);
|
||||
});
|
||||
|
||||
test("recordStreamingConsumption — null usage still records requests:1 (usd 0), no cost call", async () => {
|
||||
const { recordStreamingConsumption } = await import("../../src/lib/quota/spendRecorder.ts");
|
||||
const scheduled: RecordConsumptionInput[] = [];
|
||||
let calcCalled = false;
|
||||
await recordStreamingConsumption(
|
||||
{
|
||||
apiKeyId: "k",
|
||||
connectionId: "c",
|
||||
provider: "openai",
|
||||
model: "gpt-4",
|
||||
streamUsage: null,
|
||||
streamStatus: 200,
|
||||
},
|
||||
{
|
||||
calculateCost: async () => {
|
||||
calcCalled = true;
|
||||
return 1;
|
||||
},
|
||||
schedule: (input) => scheduled.push(input),
|
||||
}
|
||||
);
|
||||
assert.equal(scheduled.length, 1);
|
||||
assert.equal(scheduled[0].cost.usd, 0);
|
||||
assert.equal(scheduled[0].cost.requests, 1);
|
||||
assert.equal(calcCalled, false, "no usage → no cost computation");
|
||||
});
|
||||
|
||||
test("recordStreamingConsumption — missing apiKeyId/connectionId → no-op", async () => {
|
||||
const { recordStreamingConsumption } = await import("../../src/lib/quota/spendRecorder.ts");
|
||||
const scheduled: RecordConsumptionInput[] = [];
|
||||
await recordStreamingConsumption(
|
||||
{
|
||||
apiKeyId: null,
|
||||
connectionId: "c",
|
||||
provider: "openai",
|
||||
model: "gpt-4",
|
||||
streamUsage: { prompt_tokens: 1 },
|
||||
streamStatus: 200,
|
||||
},
|
||||
{ calculateCost: async () => 1, schedule: (input) => scheduled.push(input) }
|
||||
);
|
||||
await recordStreamingConsumption(
|
||||
{
|
||||
apiKeyId: "k",
|
||||
connectionId: null,
|
||||
provider: "openai",
|
||||
model: "gpt-4",
|
||||
streamUsage: { prompt_tokens: 1 },
|
||||
streamStatus: 200,
|
||||
},
|
||||
{ calculateCost: async () => 1, schedule: (input) => scheduled.push(input) }
|
||||
);
|
||||
assert.equal(scheduled.length, 0);
|
||||
});
|
||||
|
||||
test("recordStreamingConsumption — calculateCost throwing → records usd 0, never throws", async () => {
|
||||
const { recordStreamingConsumption } = await import("../../src/lib/quota/spendRecorder.ts");
|
||||
const scheduled: RecordConsumptionInput[] = [];
|
||||
await recordStreamingConsumption(
|
||||
{
|
||||
apiKeyId: "k",
|
||||
connectionId: "c",
|
||||
provider: "openai",
|
||||
model: "gpt-4",
|
||||
streamUsage: { prompt_tokens: 100 },
|
||||
streamStatus: 200,
|
||||
},
|
||||
{
|
||||
calculateCost: async () => {
|
||||
throw new Error("pricing unavailable");
|
||||
},
|
||||
schedule: (input) => scheduled.push(input),
|
||||
}
|
||||
);
|
||||
assert.equal(scheduled.length, 1);
|
||||
assert.equal(scheduled[0].cost.usd, 0);
|
||||
assert.equal(scheduled[0].cost.tokens, 100);
|
||||
});
|
||||
167
tests/unit/router-strategies.test.ts
Normal file
167
tests/unit/router-strategies.test.ts
Normal file
@@ -0,0 +1,167 @@
|
||||
/**
|
||||
* tests/unit/router-strategies.test.ts
|
||||
*
|
||||
* Direct coverage for the pluggable auto-router strategies in
|
||||
* open-sse/services/autoCombo/routerStrategy.ts. The `cost` and `latency`
|
||||
* strategies and the `selectWithStrategy` entry point had NO direct test
|
||||
* (only `sla-aware`/`lkgp` were covered, indirectly, in autoCombo.test.ts).
|
||||
* Also pins the documented silent fallback to `rules` for unknown names.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
getStrategy,
|
||||
selectWithStrategy,
|
||||
listStrategies,
|
||||
type RoutingContext,
|
||||
} from "../../open-sse/services/autoCombo/routerStrategy.ts";
|
||||
import type { ProviderCandidate } from "../../open-sse/services/autoCombo/scoring.ts";
|
||||
|
||||
function cand(p: Partial<ProviderCandidate> & { provider: string }): ProviderCandidate {
|
||||
return {
|
||||
model: `${p.provider}/m`,
|
||||
quotaRemaining: 100,
|
||||
quotaTotal: 100,
|
||||
circuitBreakerState: "CLOSED",
|
||||
costPer1MTokens: 1,
|
||||
p95LatencyMs: 100,
|
||||
latencyStdDev: 10,
|
||||
errorRate: 0,
|
||||
...p,
|
||||
} as ProviderCandidate;
|
||||
}
|
||||
|
||||
const ctx: RoutingContext = { taskType: "default" };
|
||||
|
||||
// ── cost ─────────────────────────────────────────────────────────────────────
|
||||
test("cost — selects the cheapest healthy candidate", () => {
|
||||
const pool = [
|
||||
cand({ provider: "a", costPer1MTokens: 5 }),
|
||||
cand({ provider: "b", costPer1MTokens: 1 }),
|
||||
cand({ provider: "c", costPer1MTokens: 3 }),
|
||||
];
|
||||
const d = getStrategy("cost").select(pool, ctx);
|
||||
assert.equal(d.provider, "b");
|
||||
assert.equal(d.strategy, "cost");
|
||||
});
|
||||
|
||||
test("cost — excludes OPEN-breaker candidates even if cheaper", () => {
|
||||
const pool = [
|
||||
cand({ provider: "cheap-open", costPer1MTokens: 0.1, circuitBreakerState: "OPEN" }),
|
||||
cand({ provider: "ok", costPer1MTokens: 2 }),
|
||||
];
|
||||
assert.equal(getStrategy("cost").select(pool, ctx).provider, "ok");
|
||||
});
|
||||
|
||||
test("cost — 'eco' alias resolves to the cost strategy", () => {
|
||||
assert.equal(getStrategy("eco").name, "cost");
|
||||
});
|
||||
|
||||
test("cost — empty pool throws", () => {
|
||||
assert.throws(() => getStrategy("cost").select([], ctx), /No candidates/);
|
||||
});
|
||||
|
||||
// ── latency ───────────────────────────────────────────────────────────────────
|
||||
test("latency — selects the lowest p95 candidate", () => {
|
||||
const pool = [
|
||||
cand({ provider: "slow", p95LatencyMs: 900 }),
|
||||
cand({ provider: "fast", p95LatencyMs: 100 }),
|
||||
cand({ provider: "mid", p95LatencyMs: 400 }),
|
||||
];
|
||||
assert.equal(getStrategy("latency").select(pool, ctx).provider, "fast");
|
||||
});
|
||||
|
||||
test("latency — error rate penalizes a fast-but-flaky candidate", () => {
|
||||
// flaky: 100ms + 0.5*1000 = 600 effective; steady: 400ms + 0 = 400 → steady wins.
|
||||
const pool = [
|
||||
cand({ provider: "flaky", p95LatencyMs: 100, errorRate: 0.5 }),
|
||||
cand({ provider: "steady", p95LatencyMs: 400, errorRate: 0 }),
|
||||
];
|
||||
assert.equal(getStrategy("latency").select(pool, ctx).provider, "steady");
|
||||
});
|
||||
|
||||
test("latency — 'fast' alias resolves to the latency strategy", () => {
|
||||
assert.equal(getStrategy("fast").name, "latency");
|
||||
});
|
||||
|
||||
// ── sla-aware ─────────────────────────────────────────────────────────────────
|
||||
test("sla-aware — prefers a candidate meeting the p95/error SLOs", () => {
|
||||
const pool = [
|
||||
cand({ provider: "violator", p95LatencyMs: 5000, errorRate: 0.3 }),
|
||||
cand({ provider: "compliant", p95LatencyMs: 300, errorRate: 0.001 }),
|
||||
];
|
||||
const d = getStrategy("sla-aware").select(pool, {
|
||||
taskType: "default",
|
||||
sla: { targetP95Ms: 2000, maxErrorRate: 0.05 },
|
||||
});
|
||||
assert.equal(d.provider, "compliant");
|
||||
assert.equal(d.strategy, "sla-aware");
|
||||
});
|
||||
|
||||
test("sla-aware — 'sla' alias resolves to sla-aware", () => {
|
||||
assert.equal(getStrategy("sla").name, "sla-aware");
|
||||
});
|
||||
|
||||
test("sla-aware — hardConstraints ranks fewest-violations first", () => {
|
||||
const pool = [
|
||||
cand({ provider: "violator", p95LatencyMs: 3000, errorRate: 0.2 }),
|
||||
cand({ provider: "within", p95LatencyMs: 1500, errorRate: 0.01 }),
|
||||
];
|
||||
const d = getStrategy("sla-aware").select(pool, {
|
||||
taskType: "default",
|
||||
sla: { targetP95Ms: 2000, maxErrorRate: 0.05, hardConstraints: true },
|
||||
});
|
||||
assert.equal(d.provider, "within");
|
||||
});
|
||||
|
||||
// ── lkgp ──────────────────────────────────────────────────────────────────────
|
||||
test("lkgp — returns the last known good provider when healthy", () => {
|
||||
const pool = [cand({ provider: "x" }), cand({ provider: "y" })];
|
||||
const d = getStrategy("lkgp").select(pool, {
|
||||
taskType: "default",
|
||||
lastKnownGoodProvider: "y",
|
||||
});
|
||||
assert.equal(d.provider, "y");
|
||||
assert.equal(d.strategy, "lkgp");
|
||||
});
|
||||
|
||||
test("lkgp — falls back to rules when the LKGP is OPEN", () => {
|
||||
const pool = [cand({ provider: "x" }), cand({ provider: "y", circuitBreakerState: "OPEN" })];
|
||||
const d = getStrategy("lkgp").select(pool, {
|
||||
taskType: "default",
|
||||
lastKnownGoodProvider: "y",
|
||||
});
|
||||
assert.equal(d.strategy, "rules");
|
||||
});
|
||||
|
||||
test("lkgp — lkgpEnabled:false delegates to rules", () => {
|
||||
const pool = [cand({ provider: "x" })];
|
||||
const d = getStrategy("lkgp").select(pool, {
|
||||
taskType: "default",
|
||||
lkgpEnabled: false,
|
||||
lastKnownGoodProvider: "x",
|
||||
});
|
||||
assert.equal(d.strategy, "rules");
|
||||
});
|
||||
|
||||
// ── selectWithStrategy + registry ─────────────────────────────────────────────
|
||||
test("selectWithStrategy — dispatches by name", () => {
|
||||
const pool = [
|
||||
cand({ provider: "a", costPer1MTokens: 9 }),
|
||||
cand({ provider: "b", costPer1MTokens: 1 }),
|
||||
];
|
||||
assert.equal(selectWithStrategy(pool, ctx, "cost").provider, "b");
|
||||
});
|
||||
|
||||
test("selectWithStrategy — unknown strategy silently falls back to rules", () => {
|
||||
const pool = [cand({ provider: "a" })];
|
||||
const d = selectWithStrategy(pool, ctx, "totally-unknown-strategy");
|
||||
assert.equal(d.strategy, "rules");
|
||||
});
|
||||
|
||||
test("listStrategies — exposes every registered strategy + aliases", () => {
|
||||
const names = listStrategies().map((s) => s.name);
|
||||
for (const n of ["rules", "cost", "eco", "latency", "fast", "sla-aware", "sla", "lkgp"]) {
|
||||
assert.ok(names.includes(n), `listStrategies missing '${n}'`);
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user