mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-04 06:12:10 +03:00
refactor(combo): god-file split part 2 — shadow + sorters + structure (QG v2 Fase 9 T5 D4-D6) (#4175)
Integrated into release/v3.8.29 — combo.ts god-file split part 2 (QG v2 Fase 9 T5 D4-D6): shadow routing, target sorters, and combo structure resolution extracted byte-identically to combo/{shadowRouting,targetSorters,comboStructure,comboData}.ts; combo.ts 4740->3819, public surface re-exported (callers unchanged). Validated on the rebased+merged tree: typecheck:core, file-size, test-discovery green; 356/356 full combo suite + 51/51 smoke pass. Resynced with current release (baseline union: combo-split entries + #4176 free-models entry; combo.ts=3819, rateLimitManager=1035).
This commit is contained in:
committed by
GitHub
parent
5ef6a46977
commit
798fce727c
@@ -3,6 +3,9 @@
|
||||
"_rebaseline_2026_06_18_4176_free_models": "PR #4176 own growth: AddApiKeyModal.tsx 845->866 (+21) and EditConnectionModal.tsx 1174->1204 (+30) = the 'import only free models' connection option — a free-models Toggle gated by providerHasFreeModels() plus its form-state wiring (importFreeModelsOnly field + providerSpecificData persistence, explicit-false on edit so the PUT merge doesn't keep a stale true) added to both connection modals, mirroring the prior per-modal toggle bumps #3879 (redact-thinking) and #2997 (disable-cooling). Detection lives in the new shared src/shared/utils/freeModels.ts (112 LOC, <cap); the duplicated Toggle is ~6 lines and the per-modal state wiring is intrinsic, so the remaining growth is cohesive UI, not an extractable block.",
|
||||
"_rebaseline_2026_06_18_3931_qwen_web_models": "Issue #3931 (Problem #3) own growth: providers/[id]/models/route.ts 2512->2531 (+19 = one PROVIDER_MODELS_CONFIG entry for `qwen-web` + a 4-line comment). qwen-web was missing from the config map so its model-discovery page returned nothing (the OAuth fallback only fires for provider===qwen). Pure additive config entry pointing at the public chat.qwen.ai/api/v2/models endpoint; standard per-provider addition, not extractable.",
|
||||
"_rebaseline_2026_06_18_4165_queue_timeout_msg": "Issue #4165 own growth: rateLimitManager.ts 1022->1035 (+13 at the existing withRateLimit catch chokepoint). Bottleneck's raw `This job timed out after <maxWaitMs> ms.` is rewritten into a clear OmniRoute-owned error (names resilienceSettings.requestQueue.maxWaitMs, disclaims upstream, keeps the original as `cause`, tags code=RATE_LIMIT_QUEUE_TIMEOUT) so queue-saturation 502s stop masquerading as provider outages. The branch already existed (it only logged); this adds the error construction at the same point. Not extractable — closes over provider/model/maxWaitMs locals of the single catch.",
|
||||
"_rebaseline_2026_06_18_qg9_combo_split_d4": "QG v2 Fase 9 T5 D4: combo.ts 4740->4589 — shadow routing extracted byte-identically to the new open-sse/services/combo/shadowRouting.ts (<cap, normalizeShadowRoutingConfig/resolveShadowTargets/drainShadowResponse/withTimeout/cloneRequestBodyForShadowRouting/scheduleShadowRouting). The shared isRecord guard moved to the new combo/comboData.ts leaf (<cap) so submodules can use it without importing the barrel. resolveShadowTargets/scheduleShadowRouting re-exported from combo.ts for compatibility (handleComboChat/handleRoundRobinCombo callers unchanged). Pure move, no logic change.",
|
||||
"_rebaseline_2026_06_18_qg9_combo_split_d5": "QG v2 Fase 9 T5 D5: combo.ts 4589->4430 — target sorters extracted byte-identically to the new open-sse/services/combo/targetSorters.ts (<cap, normalizeModelEntry/selectWeightedTarget/orderTargetsForWeightedFallback/sortModelsByCost/sortTargetsByCost/sortModelsByUsage/sortTargetsByUsage/getP2CTargetScore/orderTargetsByPowerOfTwoChoices). None were ever public, so combo.ts imports back only the six still called by code that stays (no re-export). getComboStepTarget/getComboStepWeight imports dropped from combo.ts (only normalizeModelEntry used them). Pure move, no logic change.",
|
||||
"_rebaseline_2026_06_18_qg9_combo_split_d6": "QG v2 Fase 9 T5 D6: combo.ts 4430->3819 — combo structure resolution extracted byte-identically to the new open-sse/services/combo/comboStructure.ts (638, <cap): runtime-step normalization, nested-combo/DAG expansion (resolveNestedComboTargets/expandRuntimeStep/getComboFromData/getComboModelsFromData/validateComboDAG/resolveNestedComboModels), weighted/direct resolution (resolveComboTargets/resolveWeightedTargets/getDirectComboTargets + composite-tier ordering), request-compatibility filtering (filterTargetsByRequestCompatibility + deriveRequestCompatibilityRequirements/getTargetCompatibilityFailures and their private helpers) and context-size sorting. Shared dedupeTargetsByExecutionKey + the toTrimmedString/toComboLike/getCombosArray normalizers also moved (dedupe to comboData.ts since both combo.ts and comboStructure use it). The 7 previously-public symbols are re-exported from combo.ts; the ~20 external consumers (chatCore.ts, /api/combos routes, embeddings, usage, catalog) are unchanged. shadowRouting.ts's resolveNestedComboTargets import redirected from the barrel to ./comboStructure.ts (removing the D4 temporary edge). Orphaned combo.ts imports dropped (getComboModelString/normalizeComboStep, getModelContextLimit, getResolvedModelCapabilities, getTargetProvider, MAX_COMBO_DEPTH, and 6 now-unused type imports). Pure move, no logic change.",
|
||||
"_rebaseline_2026_06_18_8_2_sliding_window": "Fase 8.2 own growth: rateLimitManager.ts 1017->1022 (+5 = one import + one `await awaitProviderDefaultSlot(...)` call + a 2-line comment at the existing withRateLimit chokepoint). All sliding-window logic was extracted to the new open-sse/services/providerDefaultRateLimit.ts + open-sse/services/slidingWindowLimiter.ts (both <cap), NOT inlined. Thin wiring only; not further shrinkable.",
|
||||
"_rebaseline_2026_06_18_8_1_no_thinking_alias": "Fase 8.1 own growth: catalog.ts 1435->1440 (+5 = appendNoThinkingVariants(finalModels) call + comment at the existing finalModels chokepoint) and chat.ts 1458->1471 (+13 = applyNoThinkingAlias(body) call + comment right after body.model is read, before model resolution). All real logic lives in the new open-sse/utils/noThinkingAlias.ts (<cap); both edits are thin wiring of tested helpers at the single correct integration point in each file. Not extractable.",
|
||||
"_rebaseline_2026_06_18_4_4_midstream": "Fase 4.4 own growth: chatCore.ts 5980->6009 (+29 at the existing streaming-return chokepoint = capture streamRecovery.continueMidStream alongside .enabled; refactor the early-retry reopen thunk into a shared runUpstreamStream(body) helper — net DRY — and add the gated continueStream(assistantSoFar) thunk that re-runs the upstream with makeContinuationBody(bodyToSend, …), plus the onContinue log). All continuation logic (scanOpenAiSseText, makeContinuationBody, trimContinuationOverlap, the createRecoverableStream continuation path) lives in open-sse/services/streamRecovery.ts (<cap). Closes over the same per-attempt executor locals as the existing reopen; not extractable without hiding the dispatch boundary. Opt-in (default OFF).",
|
||||
@@ -68,7 +71,7 @@
|
||||
"open-sse/services/batchProcessor.ts": 828,
|
||||
"open-sse/services/browserBackedChat.ts": 850,
|
||||
"open-sse/services/claudeCodeCompatible.ts": 1202,
|
||||
"open-sse/services/combo.ts": 4740,
|
||||
"open-sse/services/combo.ts": 3819,
|
||||
"open-sse/services/rateLimitManager.ts": 1035,
|
||||
"open-sse/services/tokenRefresh.ts": 1997,
|
||||
"open-sse/services/usage.ts": 3408,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
24
open-sse/services/combo/comboData.ts
Normal file
24
open-sse/services/combo/comboData.ts
Normal file
@@ -0,0 +1,24 @@
|
||||
/**
|
||||
* Shared combo data-normalization helpers extracted from combo.ts.
|
||||
*
|
||||
* Tiny side-effect-free guards/normalizers that several combo submodules depend
|
||||
* on (shadow routing, combo structure resolution) plus combo.ts itself. Moving
|
||||
* them to this leaf module out of the combo.ts god-file (Quality Gate v2 /
|
||||
* Fase 9) lets the submodules import them without reaching back into the barrel
|
||||
* (no cycles). Logic unchanged; combo.ts imports them back for compatibility.
|
||||
*/
|
||||
|
||||
import type { ResolvedComboTarget } from "./types.ts";
|
||||
|
||||
export function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return !!value && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
export function dedupeTargetsByExecutionKey(targets: ResolvedComboTarget[]) {
|
||||
const seen = new Set<string>();
|
||||
return targets.filter((target) => {
|
||||
if (seen.has(target.executionKey)) return false;
|
||||
seen.add(target.executionKey);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
637
open-sse/services/combo/comboStructure.ts
Normal file
637
open-sse/services/combo/comboStructure.ts
Normal file
@@ -0,0 +1,637 @@
|
||||
/**
|
||||
* Combo structure resolution extracted from combo.ts.
|
||||
*
|
||||
* Runtime-step normalization, nested-combo / DAG expansion, weighted/direct
|
||||
* target resolution, and request-compatibility filtering moved out of the
|
||||
* combo.ts god-file (Quality Gate v2 / Fase 9). Logic unchanged; the public
|
||||
* entry points (resolveComboTargets, resolveNestedComboTargets, getComboFromData,
|
||||
* getComboModelsFromData, validateComboDAG, resolveNestedComboModels,
|
||||
* filterTargetsByRequestCompatibility) are re-exported from combo.ts for the
|
||||
* ~20 external consumers (chatCore.ts, the /api/combos routes, embeddings, etc.).
|
||||
* No barrel import — depends only on sibling leaves.
|
||||
*/
|
||||
|
||||
import { getModelContextLimit } from "../../../src/lib/modelCapabilities";
|
||||
import { getComboModelString, normalizeComboStep } from "../../../src/lib/combos/steps.ts";
|
||||
import { estimateTokens } from "../contextManager.ts";
|
||||
import { getResolvedModelCapabilities } from "../modelCapabilities.ts";
|
||||
import { parseModel } from "../model.ts";
|
||||
import { dedupeTargetsByExecutionKey, isRecord } from "./comboData.ts";
|
||||
import { getTargetProvider, MAX_COMBO_DEPTH } from "./comboPredicates.ts";
|
||||
import {
|
||||
normalizeModelEntry,
|
||||
orderTargetsForWeightedFallback,
|
||||
selectWeightedTarget,
|
||||
} from "./targetSorters.ts";
|
||||
import type {
|
||||
ComboCollectionLike,
|
||||
ComboInput,
|
||||
ComboLike,
|
||||
ComboLogger,
|
||||
ComboRuntimeStep,
|
||||
ResolvedComboTarget,
|
||||
} from "./types.ts";
|
||||
|
||||
function toTrimmedString(value: unknown): string | null {
|
||||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
||||
}
|
||||
|
||||
function toComboLike(combo: ComboInput): ComboLike {
|
||||
return {
|
||||
...combo,
|
||||
id: toTrimmedString(combo.id) || undefined,
|
||||
name: toTrimmedString(combo.name) || "",
|
||||
models: Array.isArray(combo.models) ? combo.models : [],
|
||||
config: isRecord(combo.config) ? combo.config : null,
|
||||
autoConfig: isRecord(combo.autoConfig) ? combo.autoConfig : null,
|
||||
context_cache_protection:
|
||||
typeof combo.context_cache_protection === "boolean" ||
|
||||
typeof combo.context_cache_protection === "number"
|
||||
? combo.context_cache_protection
|
||||
: undefined,
|
||||
system_message: typeof combo.system_message === "string" ? combo.system_message : null,
|
||||
};
|
||||
}
|
||||
|
||||
function getCombosArray(allCombos: ComboCollectionLike): ComboLike[] {
|
||||
const combos = Array.isArray(allCombos) ? allCombos : allCombos?.combos || [];
|
||||
return combos.map((combo) => toComboLike(combo));
|
||||
}
|
||||
|
||||
function buildExecutionKey(path: string[], stepId: string): string {
|
||||
return [...path, stepId].join(">");
|
||||
}
|
||||
|
||||
function normalizeRuntimeStep(
|
||||
entry: unknown,
|
||||
comboName: string,
|
||||
index: number,
|
||||
allCombos: ComboCollectionLike,
|
||||
path: string[] = []
|
||||
): ComboRuntimeStep | null {
|
||||
const step = normalizeComboStep(entry, {
|
||||
comboName,
|
||||
index,
|
||||
allCombos,
|
||||
});
|
||||
if (!step) return null;
|
||||
|
||||
const executionKey = buildExecutionKey(path, step.id);
|
||||
const label = typeof step.label === "string" ? step.label : null;
|
||||
const weight = step.weight || 0;
|
||||
|
||||
if (step.kind === "combo-ref") {
|
||||
return {
|
||||
kind: "combo-ref",
|
||||
stepId: step.id,
|
||||
executionKey,
|
||||
comboName: step.comboName,
|
||||
weight,
|
||||
label,
|
||||
};
|
||||
}
|
||||
|
||||
const modelStr = getComboModelString(step);
|
||||
if (!modelStr) return null;
|
||||
|
||||
return {
|
||||
kind: "model",
|
||||
stepId: step.id,
|
||||
executionKey,
|
||||
modelStr,
|
||||
provider: getTargetProvider(modelStr, step.providerId),
|
||||
providerId: step.providerId || null,
|
||||
connectionId: step.connectionId || null,
|
||||
weight,
|
||||
label,
|
||||
} satisfies ResolvedComboTarget;
|
||||
}
|
||||
|
||||
function getDirectComboTargets(combo: ComboLike): ResolvedComboTarget[] {
|
||||
return getOrderedTopLevelRuntimeSteps(combo, null).filter(
|
||||
(entry): entry is ResolvedComboTarget => entry?.kind === "model"
|
||||
);
|
||||
}
|
||||
|
||||
function getTopLevelRuntimeSteps(
|
||||
combo: ComboLike,
|
||||
allCombos: ComboCollectionLike,
|
||||
path: string[] = []
|
||||
): ComboRuntimeStep[] {
|
||||
return (combo.models || [])
|
||||
.map((entry, index) => normalizeRuntimeStep(entry, combo.name, index, allCombos, path))
|
||||
.filter((entry): entry is ComboRuntimeStep => entry !== null);
|
||||
}
|
||||
|
||||
function getCompositeTierStepOrder(combo: ComboLike): string[] {
|
||||
const compositeTiers = isRecord(combo?.config) ? combo.config.compositeTiers : null;
|
||||
if (!isRecord(compositeTiers)) return [];
|
||||
|
||||
const defaultTier = toTrimmedString(compositeTiers.defaultTier);
|
||||
const tiers = isRecord(compositeTiers.tiers) ? compositeTiers.tiers : null;
|
||||
if (!defaultTier || !tiers) return [];
|
||||
|
||||
const orderedStepIds: string[] = [];
|
||||
const visitedTiers = new Set<string>();
|
||||
const seenStepIds = new Set<string>();
|
||||
type CompositeTierEntry = readonly [
|
||||
string,
|
||||
{ readonly stepId: string; readonly fallbackTier: string | null },
|
||||
];
|
||||
const tierEntries = new Map(
|
||||
Object.entries(tiers)
|
||||
.map(([tierName, rawTier]) => {
|
||||
if (!isRecord(rawTier)) return null;
|
||||
const normalizedTierName = toTrimmedString(tierName);
|
||||
const stepId = toTrimmedString(rawTier.stepId);
|
||||
const fallbackTier = toTrimmedString(rawTier.fallbackTier);
|
||||
if (!normalizedTierName || !stepId) return null;
|
||||
return [normalizedTierName, { stepId, fallbackTier }] as const;
|
||||
})
|
||||
.filter((entry): entry is CompositeTierEntry => entry !== null)
|
||||
);
|
||||
|
||||
let currentTier: string | null = defaultTier;
|
||||
while (currentTier && tierEntries.has(currentTier) && !visitedTiers.has(currentTier)) {
|
||||
visitedTiers.add(currentTier);
|
||||
const entry = tierEntries.get(currentTier);
|
||||
if (!entry) break;
|
||||
if (!seenStepIds.has(entry.stepId)) {
|
||||
orderedStepIds.push(entry.stepId);
|
||||
seenStepIds.add(entry.stepId);
|
||||
}
|
||||
currentTier = entry.fallbackTier;
|
||||
}
|
||||
|
||||
for (const entry of tierEntries.values()) {
|
||||
if (!seenStepIds.has(entry.stepId)) {
|
||||
orderedStepIds.push(entry.stepId);
|
||||
seenStepIds.add(entry.stepId);
|
||||
}
|
||||
}
|
||||
|
||||
return orderedStepIds;
|
||||
}
|
||||
|
||||
function hasCompositeTierRuntimeOrder(combo: ComboLike): boolean {
|
||||
return getCompositeTierStepOrder(combo).length > 0;
|
||||
}
|
||||
|
||||
function orderRuntimeStepsByCompositeTiers(
|
||||
steps: ComboRuntimeStep[],
|
||||
combo: ComboLike
|
||||
): ComboRuntimeStep[] {
|
||||
const orderedStepIds = getCompositeTierStepOrder(combo);
|
||||
if (orderedStepIds.length === 0) return steps;
|
||||
|
||||
const byStepId = new Map(steps.map((step) => [step.stepId, step]));
|
||||
const seen = new Set<string>();
|
||||
const ordered: ComboRuntimeStep[] = [];
|
||||
|
||||
for (const stepId of orderedStepIds) {
|
||||
const step = byStepId.get(stepId);
|
||||
if (!step || seen.has(step.stepId)) continue;
|
||||
ordered.push(step);
|
||||
seen.add(step.stepId);
|
||||
}
|
||||
|
||||
for (const step of steps) {
|
||||
if (seen.has(step.stepId)) continue;
|
||||
ordered.push(step);
|
||||
seen.add(step.stepId);
|
||||
}
|
||||
|
||||
return ordered;
|
||||
}
|
||||
|
||||
function getOrderedTopLevelRuntimeSteps(
|
||||
combo: ComboLike,
|
||||
allCombos: ComboCollectionLike,
|
||||
path: string[] = []
|
||||
): ComboRuntimeStep[] {
|
||||
return orderRuntimeStepsByCompositeTiers(getTopLevelRuntimeSteps(combo, allCombos, path), combo);
|
||||
}
|
||||
|
||||
function expandRuntimeStep(
|
||||
step: ComboRuntimeStep,
|
||||
allCombos: ComboCollectionLike,
|
||||
visited = new Set<string>(),
|
||||
depth = 0,
|
||||
path: string[] = [],
|
||||
maxDepth: number = MAX_COMBO_DEPTH
|
||||
): ResolvedComboTarget[] {
|
||||
if (step.kind === "model") return [step];
|
||||
if (depth > maxDepth) return [];
|
||||
|
||||
const combos = getCombosArray(allCombos);
|
||||
const nestedCombo = combos.find((combo) => combo.name === step.comboName);
|
||||
if (!nestedCombo || visited.has(step.comboName)) return [];
|
||||
|
||||
return resolveNestedComboTargets(
|
||||
nestedCombo,
|
||||
combos,
|
||||
new Set(visited),
|
||||
depth + 1,
|
||||
[...path, step.stepId],
|
||||
maxDepth
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveNestedComboTargets(
|
||||
combo: ComboLike,
|
||||
allCombos: ComboCollectionLike,
|
||||
visited = new Set<string>(),
|
||||
depth = 0,
|
||||
path: string[] = [],
|
||||
maxDepth: number = MAX_COMBO_DEPTH
|
||||
): ResolvedComboTarget[] {
|
||||
const directTargets = (combo.models || [])
|
||||
.map((entry, index) => normalizeRuntimeStep(entry, combo.name, index, null, path))
|
||||
.filter((entry): entry is ResolvedComboTarget => entry?.kind === "model");
|
||||
|
||||
if (depth > maxDepth) return directTargets;
|
||||
if (visited.has(combo.name)) return [];
|
||||
visited.add(combo.name);
|
||||
|
||||
const runtimeSteps = getOrderedTopLevelRuntimeSteps(combo, allCombos, path);
|
||||
const resolved: ResolvedComboTarget[] = [];
|
||||
|
||||
for (const step of runtimeSteps) {
|
||||
if (step.kind === "combo-ref") {
|
||||
resolved.push(...expandRuntimeStep(step, allCombos, new Set(visited), depth, path, maxDepth));
|
||||
continue;
|
||||
}
|
||||
resolved.push(step);
|
||||
}
|
||||
|
||||
return resolved;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get combo models from combos data (for open-sse standalone use)
|
||||
* @param {string} modelStr - Model string to check
|
||||
* @param {Array|Object} combosData - Array of combos or object with combos
|
||||
* @returns {Object|null} Full combo object or null if not a combo
|
||||
*/
|
||||
export function getComboFromData(
|
||||
modelStr: string,
|
||||
combosData: ComboCollectionLike
|
||||
): ComboLike | null {
|
||||
const combos = getCombosArray(combosData);
|
||||
const combo = combos.find((c) => c.name === modelStr);
|
||||
if (combo?.models && combo.models.length > 0) {
|
||||
return combo;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Legacy: Get combo models as string array (backward compat)
|
||||
*/
|
||||
export function getComboModelsFromData(
|
||||
modelStr: string,
|
||||
combosData: ComboCollectionLike
|
||||
): string[] | null {
|
||||
const combo = getComboFromData(modelStr, combosData);
|
||||
if (!combo) return null;
|
||||
return combo.models.map((m) => normalizeModelEntry(m).model);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate combo DAG — detect circular references and enforce max depth
|
||||
* @param {string} comboName - Name of the combo to validate
|
||||
* @param {Array} allCombos - All combos in the system
|
||||
* @param {Set} [visited] - Set of already visited combo names (for cycle detection)
|
||||
* @param {number} [depth] - Current depth level
|
||||
* @throws {Error} If circular reference or max depth exceeded
|
||||
*/
|
||||
export function validateComboDAG(
|
||||
comboName: string,
|
||||
allCombos: ComboCollectionLike,
|
||||
visited = new Set<string>(),
|
||||
depth = 0,
|
||||
maxDepth: number = MAX_COMBO_DEPTH
|
||||
): void {
|
||||
if (depth > maxDepth) {
|
||||
throw new Error(`Max combo nesting depth (${maxDepth}) exceeded at "${comboName}"`);
|
||||
}
|
||||
if (visited.has(comboName)) {
|
||||
throw new Error(`Circular combo reference detected: ${comboName}`);
|
||||
}
|
||||
visited.add(comboName);
|
||||
|
||||
const combos = getCombosArray(allCombos);
|
||||
const combo = combos.find((c) => c.name === comboName);
|
||||
if (!combo?.models) return;
|
||||
|
||||
for (const entry of combo.models) {
|
||||
const modelName = normalizeModelEntry(entry).model;
|
||||
// Check if this model name is itself a combo (not a provider/model pattern)
|
||||
const nestedCombo = combos.find((c) => c.name === modelName);
|
||||
if (nestedCombo) {
|
||||
validateComboDAG(modelName, combos, new Set(visited), depth + 1, maxDepth);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve nested combos by expanding inline to a flat model list
|
||||
* Respects max depth and detects cycles
|
||||
* @param {Object} combo - The combo object
|
||||
* @param {Array} allCombos - All combos in the system
|
||||
* @param {Set} [visited] - For cycle detection
|
||||
* @param {number} [depth] - Current depth
|
||||
* @returns {Array} Flat array of model strings
|
||||
*/
|
||||
export function resolveNestedComboModels(
|
||||
combo: ComboLike,
|
||||
allCombos: ComboCollectionLike,
|
||||
visited = new Set<string>(),
|
||||
depth = 0,
|
||||
maxDepth: number = MAX_COMBO_DEPTH
|
||||
): string[] {
|
||||
if (depth > maxDepth) return combo.models.map((m) => normalizeModelEntry(m).model);
|
||||
if (visited.has(combo.name)) return []; // cycle safety
|
||||
visited.add(combo.name);
|
||||
|
||||
const combos = getCombosArray(allCombos);
|
||||
const resolved: string[] = [];
|
||||
|
||||
for (const entry of combo.models || []) {
|
||||
const modelName = normalizeModelEntry(entry).model;
|
||||
const nestedCombo = combos.find((c) => c.name === modelName);
|
||||
|
||||
if (nestedCombo) {
|
||||
// Recursively expand the nested combo
|
||||
const nested = resolveNestedComboModels(
|
||||
nestedCombo,
|
||||
combos,
|
||||
new Set(visited),
|
||||
depth + 1,
|
||||
maxDepth
|
||||
);
|
||||
resolved.push(...nested);
|
||||
} else {
|
||||
resolved.push(modelName);
|
||||
}
|
||||
}
|
||||
|
||||
return resolved;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sort models by context window size (largest first) for context-optimized strategy.
|
||||
* Uses models.dev synced capabilities to get context limits.
|
||||
* @param {Array<string>} models - Model strings in "provider/model" format
|
||||
* @returns {Array<string>} Sorted model strings (largest context first)
|
||||
*/
|
||||
function sortModelsByContextSize(models: string[]): string[] {
|
||||
const withContext = models.map((modelStr) => {
|
||||
return { modelStr, context: getModelContextLimitForModelString(modelStr) ?? 0 };
|
||||
});
|
||||
withContext.sort((a, b) => b.context - a.context);
|
||||
return withContext.map((e) => e.modelStr);
|
||||
}
|
||||
|
||||
export function getModelContextLimitForModelString(modelStr: string) {
|
||||
const parsed = parseModel(modelStr);
|
||||
const provider = parsed.provider || parsed.providerAlias || "unknown";
|
||||
const model = parsed.model || modelStr;
|
||||
return getModelContextLimit(provider, model);
|
||||
}
|
||||
|
||||
type RequestCompatibilityRequirements = {
|
||||
requiresTools: boolean;
|
||||
requiresVision: boolean;
|
||||
requiresStructuredOutput: boolean;
|
||||
estimatedInputTokens: number;
|
||||
requestedOutputTokens: number;
|
||||
requiredContextTokens: number;
|
||||
};
|
||||
|
||||
function getPositiveTokenCount(value: unknown): number {
|
||||
const count = Number(value);
|
||||
return Number.isFinite(count) && count > 0 ? Math.ceil(count) : 0;
|
||||
}
|
||||
|
||||
function requestRequiresTools(body: Record<string, unknown>): boolean {
|
||||
if (Array.isArray(body.tools) && body.tools.length > 0) return true;
|
||||
if (Array.isArray(body.functions) && body.functions.length > 0) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function requestRequiresStructuredOutput(body: Record<string, unknown>): boolean {
|
||||
const responseFormat = isRecord(body.response_format) ? body.response_format : null;
|
||||
const type = typeof responseFormat?.type === "string" ? responseFormat.type : null;
|
||||
return type === "json_object" || type === "json_schema";
|
||||
}
|
||||
|
||||
function estimateRequestInputTokens(body: Record<string, unknown>): number {
|
||||
const estimatePayload: Record<string, unknown> = {};
|
||||
for (const key of ["messages", "input", "tools", "functions", "response_format"]) {
|
||||
if (body[key] !== undefined) estimatePayload[key] = body[key];
|
||||
}
|
||||
return Object.keys(estimatePayload).length > 0 ? estimateTokens(estimatePayload) : 0;
|
||||
}
|
||||
|
||||
function valueContainsImagePart(value: unknown, depth = 0): boolean {
|
||||
if (depth > 8 || value === null || value === undefined) return false;
|
||||
if (typeof value === "string") return value.startsWith("data:image/");
|
||||
if (Array.isArray(value)) return value.some((entry) => valueContainsImagePart(entry, depth + 1));
|
||||
if (!isRecord(value)) return false;
|
||||
|
||||
const type = typeof value.type === "string" ? value.type.toLowerCase() : null;
|
||||
if (type === "image" || type === "image_url" || type === "input_image") return true;
|
||||
if ("image_url" in value || "input_image" in value) return true;
|
||||
|
||||
const source = isRecord(value.source) ? value.source : null;
|
||||
const mediaType = typeof source?.media_type === "string" ? source.media_type.toLowerCase() : "";
|
||||
if (mediaType.startsWith("image/")) return true;
|
||||
|
||||
return Object.values(value).some((entry) => valueContainsImagePart(entry, depth + 1));
|
||||
}
|
||||
|
||||
function deriveRequestCompatibilityRequirements(
|
||||
body: Record<string, unknown>
|
||||
): RequestCompatibilityRequirements {
|
||||
const estimatedInputTokens = estimateRequestInputTokens(body);
|
||||
const requestedOutputTokens = Math.max(
|
||||
getPositiveTokenCount(body.max_tokens),
|
||||
getPositiveTokenCount(body.max_completion_tokens)
|
||||
);
|
||||
return {
|
||||
requiresTools: requestRequiresTools(body),
|
||||
requiresVision: valueContainsImagePart(body.messages) || valueContainsImagePart(body.input),
|
||||
requiresStructuredOutput: requestRequiresStructuredOutput(body),
|
||||
estimatedInputTokens,
|
||||
requestedOutputTokens,
|
||||
requiredContextTokens: estimatedInputTokens + requestedOutputTokens,
|
||||
};
|
||||
}
|
||||
|
||||
function getTargetCompatibilityFailures(
|
||||
target: ResolvedComboTarget,
|
||||
requirements: RequestCompatibilityRequirements
|
||||
): string[] {
|
||||
const capabilities = getResolvedModelCapabilities(target.modelStr);
|
||||
const failures: string[] = [];
|
||||
|
||||
if (
|
||||
requirements.requiresTools &&
|
||||
(capabilities.supportsTools === false || !capabilities.toolCalling)
|
||||
) {
|
||||
failures.push("tools");
|
||||
}
|
||||
|
||||
// For a request that carries an image, only route to a target whose vision
|
||||
// support is *confirmed* (`=== true`). Treat `false` AND `null` (unknown) as
|
||||
// incompatible: an unknown-capability model receiving the image is exactly how
|
||||
// a text-only model (e.g. ministral) ended up answering "image not provided".
|
||||
// The caller keeps all targets when none qualify, so combos with no
|
||||
// confirmed-vision member still behave as before.
|
||||
if (requirements.requiresVision && capabilities.supportsVision !== true) {
|
||||
failures.push("vision");
|
||||
}
|
||||
|
||||
if (requirements.requiresStructuredOutput && capabilities.structuredOutput === false) {
|
||||
failures.push("structured_output");
|
||||
}
|
||||
|
||||
if (
|
||||
requirements.requestedOutputTokens > 0 &&
|
||||
Number.isFinite(capabilities.maxOutputTokens) &&
|
||||
capabilities.maxOutputTokens < requirements.requestedOutputTokens
|
||||
) {
|
||||
failures.push("output_tokens");
|
||||
}
|
||||
|
||||
const contextLimit = capabilities.maxInputTokens ?? capabilities.contextWindow ?? null;
|
||||
if (
|
||||
requirements.requiredContextTokens > 0 &&
|
||||
contextLimit !== null &&
|
||||
contextLimit !== undefined &&
|
||||
contextLimit < requirements.requiredContextTokens
|
||||
) {
|
||||
failures.push("context_window");
|
||||
}
|
||||
|
||||
return failures;
|
||||
}
|
||||
|
||||
export function filterTargetsByRequestCompatibility(
|
||||
targets: ResolvedComboTarget[],
|
||||
body: Record<string, unknown>,
|
||||
log: ComboLogger,
|
||||
label = "Context-aware fallback"
|
||||
): ResolvedComboTarget[] {
|
||||
if (targets.length === 0) return targets;
|
||||
const requirements = deriveRequestCompatibilityRequirements(body);
|
||||
const needsFiltering =
|
||||
requirements.requiresTools ||
|
||||
requirements.requiresVision ||
|
||||
requirements.requiresStructuredOutput ||
|
||||
requirements.requiredContextTokens > 0;
|
||||
if (!needsFiltering) return targets;
|
||||
|
||||
const rejected: Array<{ target: ResolvedComboTarget; reasons: string[] }> = [];
|
||||
const compatible = targets.filter((target) => {
|
||||
const reasons = getTargetCompatibilityFailures(target, requirements);
|
||||
if (reasons.length === 0) return true;
|
||||
rejected.push({ target, reasons });
|
||||
return false;
|
||||
});
|
||||
|
||||
if (compatible.length === targets.length) return targets;
|
||||
if (compatible.length === 0) {
|
||||
log.warn(
|
||||
"COMBO",
|
||||
`${label}: all ${targets.length} targets were filtered by request requirements; preserving strategy order`
|
||||
);
|
||||
log.debug?.(
|
||||
"COMBO",
|
||||
`${label}: rejected targets ${rejected
|
||||
.map((entry) => `${entry.target.modelStr}(${entry.reasons.join("+")})`)
|
||||
.join(", ")}`
|
||||
);
|
||||
return targets;
|
||||
}
|
||||
|
||||
log.info(
|
||||
"COMBO",
|
||||
`${label}: kept ${compatible.length}/${targets.length} targets for request requirements`
|
||||
);
|
||||
log.debug?.(
|
||||
"COMBO",
|
||||
`${label}: rejected targets ${rejected
|
||||
.map((entry) => `${entry.target.modelStr}(${entry.reasons.join("+")})`)
|
||||
.join(", ")}`
|
||||
);
|
||||
return compatible;
|
||||
}
|
||||
|
||||
export function sortTargetsByContextSize(targets: ResolvedComboTarget[]) {
|
||||
const hasKnownContext = targets.some(
|
||||
(target) => getModelContextLimitForModelString(target.modelStr) != null
|
||||
);
|
||||
if (!hasKnownContext) return targets;
|
||||
|
||||
const orderedModels = sortModelsByContextSize(targets.map((target) => target.modelStr));
|
||||
const byModel = new Map<string, ResolvedComboTarget[]>();
|
||||
for (const target of targets) {
|
||||
const queue = byModel.get(target.modelStr) || [];
|
||||
queue.push(target);
|
||||
byModel.set(target.modelStr, queue);
|
||||
}
|
||||
return orderedModels
|
||||
.map((modelStr) => {
|
||||
const queue = byModel.get(modelStr);
|
||||
return queue?.shift() || null;
|
||||
})
|
||||
.filter((target): target is ResolvedComboTarget => target !== null);
|
||||
}
|
||||
|
||||
export function resolveComboTargets(
|
||||
combo: ComboLike,
|
||||
allCombos: ComboCollectionLike,
|
||||
maxDepth: number = MAX_COMBO_DEPTH
|
||||
): ResolvedComboTarget[] {
|
||||
return allCombos
|
||||
? resolveNestedComboTargets(combo, allCombos, new Set<string>(), 0, [], maxDepth)
|
||||
: getDirectComboTargets(combo);
|
||||
}
|
||||
|
||||
export function resolveWeightedTargets(
|
||||
combo: ComboLike,
|
||||
allCombos: ComboCollectionLike
|
||||
): {
|
||||
orderedTargets: ResolvedComboTarget[];
|
||||
selectedStep: ComboRuntimeStep | null;
|
||||
} {
|
||||
const topLevelSteps = getOrderedTopLevelRuntimeSteps(combo, allCombos);
|
||||
if (topLevelSteps.length === 0) {
|
||||
return { orderedTargets: [], selectedStep: null };
|
||||
}
|
||||
|
||||
const selectedStep = selectWeightedTarget(topLevelSteps);
|
||||
if (!selectedStep) {
|
||||
return { orderedTargets: [], selectedStep: null };
|
||||
}
|
||||
|
||||
const orderedSteps = orderTargetsForWeightedFallback(
|
||||
topLevelSteps,
|
||||
selectedStep.executionKey,
|
||||
hasCompositeTierRuntimeOrder(combo)
|
||||
);
|
||||
const expandedTargets = orderedSteps.flatMap((step) => {
|
||||
if (!step) return [];
|
||||
if (!allCombos) {
|
||||
return step.kind === "model" ? [step] : [];
|
||||
}
|
||||
return expandRuntimeStep(step, allCombos, new Set([combo.name]));
|
||||
});
|
||||
|
||||
return {
|
||||
orderedTargets: dedupeTargetsByExecutionKey(expandedTargets),
|
||||
selectedStep,
|
||||
};
|
||||
}
|
||||
178
open-sse/services/combo/shadowRouting.ts
Normal file
178
open-sse/services/combo/shadowRouting.ts
Normal file
@@ -0,0 +1,178 @@
|
||||
/**
|
||||
* Combo shadow-routing helpers extracted from combo.ts.
|
||||
*
|
||||
* Shadow routing mirrors a sampled fraction of production traffic to extra
|
||||
* targets (fire-and-forget) to compare behavior without affecting the real
|
||||
* response. Moved out of the combo.ts god-file (Quality Gate v2 / Fase 9) —
|
||||
* logic unchanged; `resolveShadowTargets` and `scheduleShadowRouting` are
|
||||
* re-exported from combo.ts for backward compatibility and used by
|
||||
* handleComboChat / handleRoundRobinCombo (which stay in combo.ts).
|
||||
*
|
||||
* NOTE: `resolveNestedComboTargets` lives in combo/comboStructure.ts (extracted
|
||||
* in the same Fase 9 split). It is only invoked at request time inside
|
||||
* `resolveShadowTargets`, never during module init.
|
||||
*/
|
||||
|
||||
import { recordComboShadowRequest } from "../comboMetrics.ts";
|
||||
import { isRecord } from "./comboData.ts";
|
||||
import { resolveNestedComboTargets } from "./comboStructure.ts";
|
||||
import { toRecordedTarget } from "./comboPredicates.ts";
|
||||
import type {
|
||||
ComboLike,
|
||||
ComboCollectionLike,
|
||||
ComboLogger,
|
||||
HandleSingleModel,
|
||||
IsModelAvailable,
|
||||
ResolvedComboTarget,
|
||||
ShadowRoutingConfig,
|
||||
} from "./types.ts";
|
||||
|
||||
function normalizeShadowRoutingConfig(config: Record<string, unknown>): ShadowRoutingConfig {
|
||||
const raw = isRecord(config.shadowRouting) ? config.shadowRouting : {};
|
||||
const sampleRate = Number(raw.sampleRate ?? 1);
|
||||
const maxTargets = Number(raw.maxTargets ?? 2);
|
||||
const timeoutMs = Number(raw.timeoutMs ?? 30000);
|
||||
return {
|
||||
enabled: raw.enabled === true,
|
||||
targets: Array.isArray(raw.targets) ? raw.targets : [],
|
||||
sampleRate: Number.isFinite(sampleRate) ? Math.max(0, Math.min(1, sampleRate)) : 1,
|
||||
maxTargets: Number.isFinite(maxTargets) ? Math.max(1, Math.min(10, Math.floor(maxTargets))) : 2,
|
||||
timeoutMs: Number.isFinite(timeoutMs)
|
||||
? Math.max(1000, Math.min(120000, Math.floor(timeoutMs)))
|
||||
: 30000,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveShadowTargets(
|
||||
combo: ComboLike,
|
||||
config: Record<string, unknown>,
|
||||
allCombos: ComboCollectionLike
|
||||
): ResolvedComboTarget[] {
|
||||
const shadowConfig = normalizeShadowRoutingConfig(config);
|
||||
if (!shadowConfig.enabled || shadowConfig.targets.length === 0) return [];
|
||||
if (shadowConfig.sampleRate <= 0 || Math.random() > shadowConfig.sampleRate) return [];
|
||||
|
||||
const shadowCombo: ComboLike = {
|
||||
...combo,
|
||||
name: `${combo.name}:shadow`,
|
||||
models: shadowConfig.targets,
|
||||
};
|
||||
return resolveNestedComboTargets(shadowCombo, allCombos, new Set([combo.name]), 0, ["shadow"])
|
||||
.slice(0, shadowConfig.maxTargets)
|
||||
.map((target) => ({
|
||||
...target,
|
||||
trafficType: "shadow" as const,
|
||||
}));
|
||||
}
|
||||
|
||||
async function drainShadowResponse(response: Response): Promise<void> {
|
||||
try {
|
||||
if (!response.body) return;
|
||||
await response.arrayBuffer();
|
||||
} catch {
|
||||
// Shadow draining is best-effort and must never affect the production response.
|
||||
}
|
||||
}
|
||||
|
||||
function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error("Shadow route timed out")), timeoutMs);
|
||||
promise.then(
|
||||
(value) => {
|
||||
clearTimeout(timer);
|
||||
resolve(value);
|
||||
},
|
||||
(error) => {
|
||||
clearTimeout(timer);
|
||||
reject(error);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function cloneRequestBodyForShadowRouting(body: Record<string, unknown>): Record<string, unknown> {
|
||||
if (typeof structuredClone === "function") {
|
||||
return structuredClone(body) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
return JSON.parse(JSON.stringify(body)) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
export function scheduleShadowRouting(
|
||||
combo: ComboLike,
|
||||
config: Record<string, unknown>,
|
||||
body: Record<string, unknown>,
|
||||
targets: ResolvedComboTarget[],
|
||||
handleSingleModel: HandleSingleModel,
|
||||
isModelAvailable: IsModelAvailable | undefined,
|
||||
strategy: string,
|
||||
log: ComboLogger
|
||||
): void {
|
||||
if (targets.length === 0) return;
|
||||
const shadowConfig = normalizeShadowRoutingConfig(config);
|
||||
let shadowBaseBody: Record<string, unknown>;
|
||||
try {
|
||||
shadowBaseBody = cloneRequestBodyForShadowRouting(body);
|
||||
} catch (error) {
|
||||
log.warn("COMBO", "Shadow routing skipped: failed to clone request body", {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return;
|
||||
}
|
||||
const run = async () => {
|
||||
await Promise.all(
|
||||
targets.map(async (target) => {
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
const shadowBody = {
|
||||
...cloneRequestBodyForShadowRouting(shadowBaseBody),
|
||||
model: target.modelStr,
|
||||
stream: false,
|
||||
};
|
||||
if (isModelAvailable) {
|
||||
const available = await isModelAvailable(target.modelStr, target);
|
||||
if (!available) {
|
||||
recordComboShadowRequest(combo.name, target.modelStr, {
|
||||
success: false,
|
||||
latencyMs: Date.now() - startedAt,
|
||||
target: toRecordedTarget(target),
|
||||
});
|
||||
log.info("COMBO", `Shadow target skipped (unavailable): ${target.modelStr}`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const response = await withTimeout(
|
||||
handleSingleModel(shadowBody, target.modelStr, {
|
||||
...target,
|
||||
failoverBeforeRetry: true,
|
||||
trafficType: "shadow",
|
||||
}),
|
||||
shadowConfig.timeoutMs
|
||||
);
|
||||
await drainShadowResponse(response.clone());
|
||||
recordComboShadowRequest(combo.name, target.modelStr, {
|
||||
success: response.ok,
|
||||
latencyMs: Date.now() - startedAt,
|
||||
target: toRecordedTarget(target),
|
||||
});
|
||||
log.info(
|
||||
"COMBO",
|
||||
`Shadow target ${target.modelStr} completed with status ${response.status} (${strategy})`
|
||||
);
|
||||
} catch (error) {
|
||||
recordComboShadowRequest(combo.name, target.modelStr, {
|
||||
success: false,
|
||||
latencyMs: Date.now() - startedAt,
|
||||
target: toRecordedTarget(target),
|
||||
});
|
||||
log.warn("COMBO", `Shadow target ${target.modelStr} failed`, {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
setTimeout(() => void run(), 0);
|
||||
}
|
||||
173
open-sse/services/combo/targetSorters.ts
Normal file
173
open-sse/services/combo/targetSorters.ts
Normal file
@@ -0,0 +1,173 @@
|
||||
/**
|
||||
* Combo target-ordering / sorting helpers extracted from combo.ts.
|
||||
*
|
||||
* Weighted selection, cost / usage / power-of-two-choices ordering, and the
|
||||
* model-entry normalizer moved out of the combo.ts god-file (Quality Gate v2 /
|
||||
* Fase 9). Logic unchanged; the helpers used by the combo handlers (which stay
|
||||
* in combo.ts) are imported back from this module. No barrel import — pure leaf.
|
||||
*/
|
||||
|
||||
import { getCircuitBreaker } from "../../../src/shared/utils/circuitBreaker";
|
||||
import { getComboStepTarget, getComboStepWeight } from "../../../src/lib/combos/steps.ts";
|
||||
import { getComboMetrics } from "../comboMetrics.ts";
|
||||
import { parseModel } from "../model.ts";
|
||||
import type { ResolvedComboTarget } from "./types.ts";
|
||||
|
||||
/**
|
||||
* Normalize a model entry to { model, weight }
|
||||
* Supports both legacy string format and new object format
|
||||
*/
|
||||
export function normalizeModelEntry(entry: unknown): { model: string; weight: number } {
|
||||
return {
|
||||
model: getComboStepTarget(entry) || "",
|
||||
weight: getComboStepWeight(entry),
|
||||
};
|
||||
}
|
||||
|
||||
export function selectWeightedTarget<T extends { weight?: number }>(targets: T[]) {
|
||||
if (targets.length === 0) return null;
|
||||
|
||||
const totalWeight = targets.reduce((sum, target) => sum + (target.weight || 0), 0);
|
||||
if (totalWeight <= 0) {
|
||||
return targets[Math.floor(Math.random() * targets.length)];
|
||||
}
|
||||
|
||||
let random = Math.random() * totalWeight;
|
||||
for (const target of targets) {
|
||||
random -= target.weight || 0;
|
||||
if (random <= 0) return target;
|
||||
}
|
||||
|
||||
return targets.at(-1);
|
||||
}
|
||||
|
||||
export function orderTargetsForWeightedFallback<T extends { executionKey: string; weight: number }>(
|
||||
targets: T[],
|
||||
selectedExecutionKey: string,
|
||||
preserveExistingOrder = false
|
||||
): T[] {
|
||||
const selected = targets.find((target) => target.executionKey === selectedExecutionKey);
|
||||
const rest = targets.filter((target) => target.executionKey !== selectedExecutionKey);
|
||||
if (!preserveExistingOrder) {
|
||||
rest.sort((a, b) => b.weight - a.weight);
|
||||
}
|
||||
return selected ? [selected, ...rest] : rest;
|
||||
}
|
||||
|
||||
// shuffleArray and getNextModelFromDeck moved to src/shared/utils/shuffleDeck.ts
|
||||
// combo.ts now uses the shared, mutex-protected getNextFromDeck with "combo:" namespace.
|
||||
|
||||
/**
|
||||
* Sort models by pricing (cheapest first) for cost-optimized strategy
|
||||
* @param {Array<string>} models - Model strings in "provider/model" format
|
||||
* @returns {Promise<Array<string>>} Sorted model strings
|
||||
*/
|
||||
export async function sortModelsByCost(models: string[]): Promise<string[]> {
|
||||
try {
|
||||
const { getPricingForModel } = await import("../../../src/lib/localDb");
|
||||
const withCost = await Promise.all(
|
||||
models.map(async (modelStr) => {
|
||||
const parsed = parseModel(modelStr);
|
||||
const provider = parsed.provider || parsed.providerAlias || "unknown";
|
||||
const model = parsed.model || modelStr;
|
||||
try {
|
||||
const pricing = await getPricingForModel(provider, model);
|
||||
const cost = Number(pricing?.input);
|
||||
return { modelStr, cost: Number.isFinite(cost) ? cost : Infinity };
|
||||
} catch {
|
||||
return { modelStr, cost: Infinity };
|
||||
}
|
||||
})
|
||||
);
|
||||
withCost.sort((a, b) => a.cost - b.cost);
|
||||
return withCost.map((e) => e.modelStr);
|
||||
} catch {
|
||||
// If pricing lookup fails entirely, return original order
|
||||
return models;
|
||||
}
|
||||
}
|
||||
|
||||
export async function sortTargetsByCost(targets: ResolvedComboTarget[]) {
|
||||
const orderedModels = await sortModelsByCost(targets.map((target) => target.modelStr));
|
||||
const byModel = new Map<string, ResolvedComboTarget[]>();
|
||||
for (const target of targets) {
|
||||
const queue = byModel.get(target.modelStr) || [];
|
||||
queue.push(target);
|
||||
byModel.set(target.modelStr, queue);
|
||||
}
|
||||
return orderedModels
|
||||
.map((modelStr) => {
|
||||
const queue = byModel.get(modelStr);
|
||||
return queue?.shift() || null;
|
||||
})
|
||||
.filter((target): target is ResolvedComboTarget => target !== null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sort models by usage count (least-used first) for least-used strategy
|
||||
* @param {Array<string>} models - Model strings
|
||||
* @param {string} comboName - Combo name for metrics lookup
|
||||
* @returns {Array<string>} Sorted model strings
|
||||
*/
|
||||
export function sortModelsByUsage(models: string[], comboName: string): string[] {
|
||||
const metrics = getComboMetrics(comboName);
|
||||
if (!metrics?.byModel) return models;
|
||||
|
||||
const withUsage = models.map((modelStr) => ({
|
||||
modelStr,
|
||||
requests: metrics.byModel[modelStr]?.requests ?? 0,
|
||||
}));
|
||||
withUsage.sort((a, b) => a.requests - b.requests);
|
||||
return withUsage.map((e) => e.modelStr);
|
||||
}
|
||||
|
||||
export function sortTargetsByUsage(targets: ResolvedComboTarget[], comboName: string) {
|
||||
const orderedModels = sortModelsByUsage(
|
||||
targets.map((target) => target.modelStr),
|
||||
comboName
|
||||
);
|
||||
const byModel = new Map<string, ResolvedComboTarget[]>();
|
||||
for (const target of targets) {
|
||||
const queue = byModel.get(target.modelStr) || [];
|
||||
queue.push(target);
|
||||
byModel.set(target.modelStr, queue);
|
||||
}
|
||||
return orderedModels
|
||||
.map((modelStr) => {
|
||||
const queue = byModel.get(modelStr);
|
||||
return queue?.shift() || null;
|
||||
})
|
||||
.filter((target): target is ResolvedComboTarget => target !== null);
|
||||
}
|
||||
|
||||
function getP2CTargetScore(
|
||||
target: ResolvedComboTarget,
|
||||
metrics: ReturnType<typeof getComboMetrics>
|
||||
): number {
|
||||
const breakerState = getCircuitBreaker(target.provider)?.getStatus?.()?.state;
|
||||
if (breakerState === "OPEN") return -Infinity;
|
||||
const modelMetric = metrics?.byModel?.[target.modelStr] || null;
|
||||
const successRate = Number(modelMetric?.successRate);
|
||||
const avgLatency = Number(modelMetric?.avgLatencyMs);
|
||||
const successScore = Number.isFinite(successRate) ? successRate / 100 : 0.5;
|
||||
const latencyScore =
|
||||
Number.isFinite(avgLatency) && avgLatency > 0 ? 1 / Math.log10(avgLatency + 10) : 0.25;
|
||||
const breakerPenalty = breakerState === "HALF_OPEN" ? 0.25 : 0;
|
||||
return successScore + latencyScore - breakerPenalty;
|
||||
}
|
||||
|
||||
export function orderTargetsByPowerOfTwoChoices(targets: ResolvedComboTarget[], comboName: string) {
|
||||
if (targets.length <= 1) return targets;
|
||||
const metrics = getComboMetrics(comboName);
|
||||
const firstIndex = Math.floor(Math.random() * targets.length);
|
||||
let secondIndex = Math.floor(Math.random() * (targets.length - 1));
|
||||
if (secondIndex >= firstIndex) secondIndex++;
|
||||
|
||||
const first = targets[firstIndex];
|
||||
const second = targets[secondIndex];
|
||||
const selectedIndex =
|
||||
getP2CTargetScore(second, metrics) > getP2CTargetScore(first, metrics)
|
||||
? secondIndex
|
||||
: firstIndex;
|
||||
return [targets[selectedIndex], ...targets.filter((_, index) => index !== selectedIndex)];
|
||||
}
|
||||
Reference in New Issue
Block a user