mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-31 12:22:14 +03:00
feat(combo): add sticky weighted selection limit with exhaustion-aware renormalization (#4489)
Integrated into release/v3.8.33
This commit is contained in:
@@ -9,6 +9,7 @@
|
||||
"_rebaseline_2026_06_21_4427_low_noise_catalog": "PR #4427 (Rahulsharma0810) own growth: src/app/api/v1/models/catalog.ts 1478->1486 (+8). The opt-in MODELS_CATALOG_PREFIX_MODE (dual default | alias | canonical, with ?prefix= per-request override) gates the dual alias+canonical model emission at the three /v1/models push sites (static, synced, custom) behind includeAlias/includeCanonical, suppressing the duplicate cross-prefix entries (net +3 from the feature). On review, 4 incidental explanatory comments removed by the PR were restored (synced-models resolve, skip-static, try-block intent, strip-modelIdPrefix; +5) since their code is unchanged — useful docs on a non-trivial catalog function. Default `dual` keeps byte-identical output; request-side alias resolution unchanged. Structural shrink of this route tracked in #3789. Covered by tests/unit/models-catalog-low-noise-flag.test.ts.",
|
||||
"_rebaseline_2026_06_21_phase2_active_selector": "Compression Phase 2 (named profiles + active selector) own growth: chatCore.ts 5110->5125 (+15 at the existing compression-dispatch chokepoint, just before the selectCompressionStrategy call). chatCore now loads the operator's named compression combos once into a `namedCombos` map (a best-effort try/catch dynamic-import of listCompressionCombos, debug-logged on failure) and passes it as the new `combos` arg to selectCompressionStrategy + selectCompressionPlan, plus an `&& !activeComboResolves(config, namedCombos)` guard term on the legacy default-combo block so the seeded default cannot shadow the operator's active profile. The resolver itself stays pure (the `combos` threading + activeComboResolves live in open-sse/services/compression/strategySelector.ts, <cap). The two UI files net SHRINK or stay flat: CompressionHub.tsx 521->277 (master toggle/mode/reorder removed, replaced by the thin active-profile selector + preview) and CompressionCombosPageClient.tsx grows a few lines under its frozen for the active badge. Cohesive wiring at the existing compression chokepoint, mirroring the prior compression rebaselines (#4217/#4210/#3890); not extractable without hiding the dispatch boundary. Structural shrink of chatCore.ts tracked in #3501. Covered by tests/unit/compression/active-combo-dispatch.test.ts + active-combo-integration.test.ts + tests/unit/ui/compressionHub-active-selector.test.tsx + namedCombos-active-badge.test.tsx.",
|
||||
"_rebaseline_2026_06_21_qg9_chatcore_service_tier": "QG v2 Fase 9 T5 (#3501) — chatCore.ts 5137->5110 (shrink -27). The two inline Codex service-tier resolvers (resolveEffectiveServiceTier / resolveReportedServiceTier, ~36 LOC) were extracted byte-identically into the new pure leaf open-sse/handlers/chatCore/serviceTier.ts; the handler now keeps a `let effectiveServiceTier` + two thin binding closures that pass provider/credentials?.providerSpecificData to the extracted functions, so every call site stays unchanged. The orphaned getCodexRequestDefaults/normalizeCodexServiceTier/CodexServiceTier imports moved to the leaf. Ratcheted the frozen value down to lock the freed budget. Covered by tests/unit/chatcore-service-tier.test.ts.",
|
||||
"_rebaseline_2026_06_21_4489_sticky_weighted_limit": "PR #4489 (adivekar-utexas) own growth: src/app/(dashboard)/dashboard/combos/page.tsx 4385->4386 (+1 = the new Sticky Weighted Limit input in the weighted advanced section, a single FieldLabelWithHelp + number <input> block gated to strategy===weighted, analogous to the existing stickyRoundRobinLimit field for round-robin) and open-sse/services/combo.ts ->2761 (sticky-weighted feature: weightedStickyTargets state wiring + the isTargetSelectableForWeighted availability pre-filter + exhaustion-aware sticky clearing/migration in handleComboChat + the round-robin sticky pre-filter in handleRoundRobinCombo + the 4 gemini-review fixes — provider guard, startsWith-separator key match, stepGroups dedup, stale-sticky cleanup). Frozen to the merge-tree measurement (release/v3.8.33 base 2649 + the feature) since CI measures the merged tree, not the v3.8.32-based branch tip. Cohesive routing logic at the existing combo dispatch chokepoints; not a movable block. Structural shrink of combo.ts tracked in #3501. Covered by tests/unit/combo-strategy-fallbacks.test.ts (sticky-weighted batching/fallback-migration/stale-clear/nested-availability + RR sticky-clear) and tests/unit/combo-config.test.ts (stickyWeightedLimit schema).",
|
||||
"_rebaseline_2026_06_20_reviewprs_mine_r2_filesize": "Reconciliacao file-size pos-lote /review-prs 'apenas minhas' r2: dois frozen cresceram cumulativamente sem bump (cada PR media OK na sua base, mas o crescimento empilhou acima do frozen no tip de merge; o fast-path do release nao roda check:file-size, so release->main). (1) src/shared/constants/pricing.ts 1620->1623 (+3 = linhas de pricing Claude Code (cc) do #4440, sobre o 1620 que o #4447 ja setara para gpt-4.1-mini/nano + o3/o4-mini). (2) open-sse/executors/base.ts 1399->1407 (+8 = handling granular de reasoning_effort para Claude no Copilot do #4443). Ambos dados/wiring coesos nos chokepoints existentes; nao extraiveis. Cobertos por tests/unit (claude-code pricing / base-executor-sanitize-effort + github-claude-reasoning-effort-granular).",
|
||||
"_rebaseline_2026_06_20_4023_web_cookie_noauth_validation": "PR #4023 (oyi77) own growth: src/lib/providers/validation.ts 4450->4518 (+68 = a new validateWebCookieProvider that probes the provider's /models endpoint — 401/403 => AUTH_007 SESSION_EXPIRED, any other status => valid session, empty cookie => invalid, provider-not-in-registry => unsupported — plus a local STANDARD_USER_AGENT const for the probe). Cohesive validator at the validateProviderApiKey dispatch; not extractable. Covered by tests/unit/provider-validation-web-cookie-auth007.test.ts. Heavily curated on merge — the PR's branch was badly stale-based (squash-base-stale), so its tree was DESTRUCTIVE: providers/index.ts deleted live providers openadapter/dit/tokenrouter (added by #4313) and the executor/base.ts edits reverted release fixes (#4037 duckduckgo host, theoldllm gpt5 models, base.ts fetch-start-timeout). Only the purely-additive validation feature was kept (validation.ts validateWebCookieProvider + errorCodes AUTH_007 + the test). Dropped: 5 malformed new registry entries (used non-RegistryEntry fields defaultModel/auth + referenced non-existent executors -> tsc TS2353), the destructive providers/index.ts + executor reverts, the unrelated pr-*.sh automation scripts, and evals/types.ts (belongs to the deferred evals modularization #4422). Also removed the PR's fragile 'Phase 2' executor probe (ran a live upstream chat during validation + classified any 'auth'-containing error as SESSION_EXPIRED) and rewrote the test to install its fetch mock before module load (the original mocked too late and silently hit live chatgpt.com).",
|
||||
"_rebaseline_2026_06_20_1308_model_lockout_honors_reset": "port from 9router#1308 own growth: open-sse/services/accountFallback.ts 1731->1752 (+21 = the new exported pure helper selectLockoutCooldownMs + its doc comment — picks the parsed upstream reset as the model-lockout exactCooldownMs when it exceeds the base cooldown, e.g. Antigravity \"Resets in 160h\", else preserves the existing 0/base behavior) and open-sse/executors/antigravity.ts 1680->1686 (this PR +1 = parseRetryFromErrorMessage regex `reset` -> `resets?` so plural \"Resets in 160h27m24s\" matches, plus a comment line; frozen set to the SUM 1686 with the concurrent #1944 which adds +5 at the disjoint passthroughFields region of the same file, so either merge order passes — pair-file rule). The combo lockout call sites in combo.ts now pass selectLockoutCooldownMs(cooldownMs, mlSettings) instead of always base/exponential, so an exhausted model honors the real upstream reset instead of being retried within minutes. Both edits are cohesive at the existing lockout/parse chokepoints; the helper is its own pure function (not extractable further). Covered by tests/unit/combo-model-lockout-honors-reset-1308.test.ts.",
|
||||
@@ -129,7 +130,7 @@
|
||||
"open-sse/services/batchProcessor.ts": 828,
|
||||
"open-sse/services/browserBackedChat.ts": 850,
|
||||
"open-sse/services/claudeCodeCompatible.ts": 1202,
|
||||
"open-sse/services/combo.ts": 2701,
|
||||
"open-sse/services/combo.ts": 2818,
|
||||
"open-sse/services/rateLimitManager.ts": 1035,
|
||||
"open-sse/services/tokenRefresh.ts": 1997,
|
||||
"open-sse/services/usage.ts": 3450,
|
||||
@@ -144,7 +145,7 @@
|
||||
"src/app/(dashboard)/dashboard/cache/page.tsx": 841,
|
||||
"src/app/(dashboard)/dashboard/cli-code/components/CodexToolCard.tsx": 900,
|
||||
"src/app/(dashboard)/dashboard/cloud-agents/page.tsx": 922,
|
||||
"src/app/(dashboard)/dashboard/combos/page.tsx": 4385,
|
||||
"src/app/(dashboard)/dashboard/combos/page.tsx": 4406,
|
||||
"src/app/(dashboard)/dashboard/costs/CostOverviewTab.tsx": 1495,
|
||||
"src/app/(dashboard)/dashboard/costs/quota-share/components/PoolWizard.tsx": 1007,
|
||||
"src/app/(dashboard)/dashboard/endpoint/EndpointPageClient.tsx": 2570,
|
||||
|
||||
@@ -99,6 +99,7 @@ import type {
|
||||
HandleRoundRobinOptions,
|
||||
ResolvedComboTarget,
|
||||
AutoProviderCandidate,
|
||||
ComboRuntimeStep,
|
||||
HistoricalLatencyStatsEntry,
|
||||
} from "./combo/types.ts";
|
||||
|
||||
@@ -106,9 +107,13 @@ import {
|
||||
MAX_RR_COUNTERS,
|
||||
rrCounters,
|
||||
rrStickyTargets,
|
||||
weightedStickyTargets,
|
||||
clampStickyRoundRobinTargetLimit,
|
||||
clampStickyWeightedTargetLimit,
|
||||
getStickyRoundRobinStartIndex,
|
||||
recordStickyRoundRobinSuccess,
|
||||
getStickyWeightedExecutionKey,
|
||||
recordStickyWeightedSuccess,
|
||||
} from "./combo/rrState.ts";
|
||||
import { validateResponseQuality, toRetryAfterDisplayValue } from "./combo/validateQuality.ts";
|
||||
import {
|
||||
@@ -143,6 +148,7 @@ import {
|
||||
getModelContextLimitForModelString,
|
||||
resolveComboTargets,
|
||||
resolveWeightedTargets,
|
||||
resolveWeightedStepGroups,
|
||||
sortTargetsByContextSize,
|
||||
} from "./combo/comboStructure.ts";
|
||||
import {
|
||||
@@ -697,10 +703,31 @@ export async function handleComboChat({
|
||||
const maxSetRetries = config.maxSetRetries ?? 0;
|
||||
const setRetryDelayMs = resolveDelayMs(config.setRetryDelayMs, 2000);
|
||||
|
||||
const isTargetSelectableForWeighted = async (target: ResolvedComboTarget): Promise<boolean> => {
|
||||
const rawModel = parseModel(target.modelStr).model || target.modelStr;
|
||||
if (target.provider && getCircuitBreaker(target.provider).getStatus().state === "OPEN") return false;
|
||||
if (
|
||||
resilienceSettings.providerCooldown.enabled &&
|
||||
Boolean(target.provider && target.provider !== "unknown") &&
|
||||
isProviderInCooldown(target.provider, target.connectionId ?? undefined, resilienceSettings)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
target.provider &&
|
||||
rawModel &&
|
||||
isModelLocked(target.provider, target.connectionId || "", rawModel)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return isModelAvailable ? await isModelAvailable(target.modelStr, target) : true;
|
||||
};
|
||||
|
||||
// #2562: Expand provider-wildcard steps (e.g. `fta/*`, `openai/gpt-4*`) into
|
||||
// concrete model entries sourced from the live synced-models catalog + registry.
|
||||
// Must run before resolveComboTargets so that wildcard-originated steps are
|
||||
// treated identically to hand-authored entries by all downstream logic.
|
||||
// Must run before any step-group / target resolution so that wildcard-originated
|
||||
// steps are treated identically to hand-authored entries by all downstream logic
|
||||
// (including the sticky-weighted eligibility pass below).
|
||||
const expandedCombo = await expandProviderWildcardsInCombo(combo);
|
||||
const expandedAllCombos = allCombos
|
||||
? Array.isArray(allCombos)
|
||||
@@ -713,21 +740,66 @@ export async function handleComboChat({
|
||||
}
|
||||
: allCombos;
|
||||
|
||||
let orderedTargets =
|
||||
const stickyWeightedLimit = clampStickyWeightedTargetLimit(
|
||||
(config as Record<string, unknown>).stickyWeightedLimit
|
||||
);
|
||||
if (
|
||||
strategy === "weighted" &&
|
||||
!weightedStickyTargets.has(combo.name) &&
|
||||
weightedStickyTargets.size >= MAX_RR_COUNTERS
|
||||
) {
|
||||
const oldest = weightedStickyTargets.keys().next().value;
|
||||
if (oldest !== undefined) weightedStickyTargets.delete(oldest);
|
||||
}
|
||||
let stepGroups: Array<{ step: ComboRuntimeStep; targets: ResolvedComboTarget[] }> | undefined;
|
||||
const weightedEligibleKeys = new Set<string>();
|
||||
if (strategy === "weighted") {
|
||||
stepGroups = resolveWeightedStepGroups(expandedCombo, expandedAllCombos);
|
||||
for (const group of stepGroups) {
|
||||
const availability = await Promise.all(group.targets.map(isTargetSelectableForWeighted));
|
||||
if (availability.some(Boolean)) weightedEligibleKeys.add(group.step.executionKey);
|
||||
}
|
||||
}
|
||||
const rawStickyWeightedKey =
|
||||
strategy === "weighted" ? getStickyWeightedExecutionKey(combo.name, stickyWeightedLimit) : null;
|
||||
const stickyWeightedKey =
|
||||
rawStickyWeightedKey && weightedEligibleKeys.has(rawStickyWeightedKey)
|
||||
? rawStickyWeightedKey
|
||||
: null;
|
||||
if (strategy !== "weighted" || stickyWeightedLimit <= 1) {
|
||||
weightedStickyTargets.delete(combo.name);
|
||||
} else if (rawStickyWeightedKey && !stickyWeightedKey) {
|
||||
weightedStickyTargets.delete(combo.name);
|
||||
}
|
||||
const weightedResolution =
|
||||
strategy === "weighted"
|
||||
? resolveWeightedTargets(expandedCombo, expandedAllCombos)?.orderedTargets || []
|
||||
: resolveComboTargets(
|
||||
? resolveWeightedTargets(
|
||||
expandedCombo,
|
||||
expandedAllCombos,
|
||||
clampComboDepth(config.maxComboDepth)
|
||||
);
|
||||
stickyWeightedKey,
|
||||
weightedEligibleKeys,
|
||||
stepGroups
|
||||
)
|
||||
: null;
|
||||
const getWeightedStepKeyForTarget = (target: ResolvedComboTarget): string | null => {
|
||||
if (!weightedResolution?.orderedSteps) return null;
|
||||
const step = weightedResolution.orderedSteps.find((entry) =>
|
||||
target.executionKey === entry.executionKey ||
|
||||
target.executionKey.startsWith(entry.executionKey + ">")
|
||||
);
|
||||
return step?.executionKey || null;
|
||||
};
|
||||
let orderedTargets =
|
||||
strategy === "weighted"
|
||||
? weightedResolution?.orderedTargets || []
|
||||
: resolveComboTargets(expandedCombo, expandedAllCombos, clampComboDepth(config.maxComboDepth));
|
||||
|
||||
orderedTargets = await applyRequestTagRouting(orderedTargets, body, log);
|
||||
|
||||
if (strategy === "weighted") {
|
||||
log.info(
|
||||
"COMBO",
|
||||
`Weighted selection${allCombos ? " with nested resolution" : ""}: ${orderedTargets.length} total targets`
|
||||
`Weighted selection${stickyWeightedKey ? " (sticky)" : ""}${allCombos ? " with nested resolution" : ""}: ${orderedTargets.length} total targets`
|
||||
);
|
||||
} else if (allCombos) {
|
||||
log.info("COMBO", `${strategy} with nested resolution: ${orderedTargets.length} total targets`);
|
||||
@@ -1575,6 +1647,12 @@ export async function handleComboChat({
|
||||
if (provider && provider !== "unknown") {
|
||||
recordProviderSuccess(provider, target.connectionId ?? undefined);
|
||||
}
|
||||
if (strategy === "weighted" && stickyWeightedLimit > 1) {
|
||||
const stickySuccessKey = getWeightedStepKeyForTarget(target);
|
||||
if (stickySuccessKey) {
|
||||
recordStickyWeightedSuccess(combo.name, stickySuccessKey, stickyWeightedLimit);
|
||||
}
|
||||
}
|
||||
// Webhook fan-out: best-effort, never blocks the response stream.
|
||||
notifyWebhookEvent("request.completed", {
|
||||
combo: combo.name,
|
||||
@@ -2216,6 +2294,45 @@ async function handleRoundRobinCombo({
|
||||
: (settings as Record<string, unknown> | null)?.stickyRoundRobinLimit
|
||||
);
|
||||
const stickyRoundRobinEnabled = stickyLimit > 1;
|
||||
// Exhaustion-aware sticky: if the currently sticky target is no longer
|
||||
// available (circuit breaker OPEN, provider cooldown, model lockout, or
|
||||
// isModelAvailable returns false), clear the sticky record so the rotation
|
||||
// starts at the counter position instead of probing a dead target.
|
||||
if (stickyRoundRobinEnabled) {
|
||||
const sticky = rrStickyTargets.get(combo.name);
|
||||
if (sticky) {
|
||||
const stickyTarget = filteredTargets.find(
|
||||
(target) => target.executionKey === sticky.executionKey
|
||||
);
|
||||
if (stickyTarget) {
|
||||
const rawModel = parseModel(stickyTarget.modelStr).model || stickyTarget.modelStr;
|
||||
const stickyAvailable =
|
||||
(!stickyTarget.provider || getCircuitBreaker(stickyTarget.provider).getStatus().state !== "OPEN") &&
|
||||
!(
|
||||
resilienceSettings.providerCooldown.enabled &&
|
||||
Boolean(stickyTarget.provider && stickyTarget.provider !== "unknown") &&
|
||||
isProviderInCooldown(
|
||||
stickyTarget.provider,
|
||||
stickyTarget.connectionId ?? undefined,
|
||||
resilienceSettings
|
||||
)
|
||||
) &&
|
||||
!(
|
||||
stickyTarget.provider &&
|
||||
rawModel &&
|
||||
isModelLocked(stickyTarget.provider, stickyTarget.connectionId || "", rawModel)
|
||||
) &&
|
||||
(isModelAvailable ? await isModelAvailable(stickyTarget.modelStr, stickyTarget) : true);
|
||||
if (!stickyAvailable) {
|
||||
log.info(
|
||||
"COMBO-RR",
|
||||
`Clearing stale sticky target ${stickyTarget.modelStr} — unavailable`
|
||||
);
|
||||
rrStickyTargets.delete(combo.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (
|
||||
!rrCounters.has(combo.name) &&
|
||||
!rrStickyTargets.has(combo.name) &&
|
||||
|
||||
@@ -607,21 +607,46 @@ export function resolveComboTargets(
|
||||
: getDirectComboTargets(combo);
|
||||
}
|
||||
|
||||
export function resolveWeightedTargets(
|
||||
export function resolveWeightedStepGroups(
|
||||
combo: ComboLike,
|
||||
allCombos: ComboCollectionLike
|
||||
): Array<{ step: ComboRuntimeStep; targets: ResolvedComboTarget[] }> {
|
||||
return getOrderedTopLevelRuntimeSteps(combo, allCombos)
|
||||
.map((step) => ({
|
||||
step,
|
||||
targets: !allCombos
|
||||
? step.kind === "model"
|
||||
? [step]
|
||||
: []
|
||||
: expandRuntimeStep(step, allCombos, new Set([combo.name])),
|
||||
}))
|
||||
.filter((group) => group.targets.length > 0);
|
||||
}
|
||||
|
||||
export function resolveWeightedTargets(
|
||||
combo: ComboLike,
|
||||
allCombos: ComboCollectionLike,
|
||||
preferredExecutionKey: string | null = null,
|
||||
eligibleExecutionKeys: ReadonlySet<string> | null = null,
|
||||
stepGroups?: Array<{ step: ComboRuntimeStep; targets: ResolvedComboTarget[] }>
|
||||
): {
|
||||
orderedTargets: ResolvedComboTarget[];
|
||||
selectedStep: ComboRuntimeStep | null;
|
||||
orderedSteps: ComboRuntimeStep[];
|
||||
} {
|
||||
const topLevelSteps = getOrderedTopLevelRuntimeSteps(combo, allCombos);
|
||||
const topLevelSteps = getOrderedTopLevelRuntimeSteps(combo, allCombos).filter((step) =>
|
||||
eligibleExecutionKeys ? eligibleExecutionKeys.has(step.executionKey) : true
|
||||
);
|
||||
if (topLevelSteps.length === 0) {
|
||||
return { orderedTargets: [], selectedStep: null };
|
||||
return { orderedTargets: [], selectedStep: null, orderedSteps: [] };
|
||||
}
|
||||
|
||||
const selectedStep = selectWeightedTarget(topLevelSteps);
|
||||
const preferredStep = preferredExecutionKey
|
||||
? topLevelSteps.find((step) => step.executionKey === preferredExecutionKey) || null
|
||||
: null;
|
||||
const selectedStep = preferredStep || selectWeightedTarget(topLevelSteps);
|
||||
if (!selectedStep) {
|
||||
return { orderedTargets: [], selectedStep: null };
|
||||
return { orderedTargets: [], selectedStep: null, orderedSteps: [] };
|
||||
}
|
||||
|
||||
const orderedSteps = orderTargetsForWeightedFallback(
|
||||
@@ -629,16 +654,19 @@ export function resolveWeightedTargets(
|
||||
selectedStep.executionKey,
|
||||
hasCompositeTierRuntimeOrder(combo)
|
||||
);
|
||||
const expandedTargets = orderedSteps.flatMap((step) => {
|
||||
if (!step) return [];
|
||||
if (!allCombos) {
|
||||
return step.kind === "model" ? [step] : [];
|
||||
}
|
||||
return expandRuntimeStep(step, allCombos, new Set([combo.name]));
|
||||
});
|
||||
const targetsByStep = new Map(
|
||||
(stepGroups || resolveWeightedStepGroups(combo, allCombos)).map((group) => [
|
||||
group.step.executionKey,
|
||||
group.targets,
|
||||
])
|
||||
);
|
||||
const expandedTargets = orderedSteps.flatMap(
|
||||
(step) => targetsByStep.get(step.executionKey) || []
|
||||
);
|
||||
|
||||
return {
|
||||
orderedTargets: dedupeTargetsByExecutionKey(expandedTargets),
|
||||
selectedStep,
|
||||
orderedSteps,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -21,7 +21,8 @@ import type { ResolvedComboTarget } from "./types.ts";
|
||||
export const MAX_RR_COUNTERS = 500;
|
||||
|
||||
export const rrCounters = new Map<string, number>();
|
||||
export const rrStickyTargets = new Map<
|
||||
export const rrStickyTargets = new Map<string, { executionKey: string; successCount: number }>();
|
||||
export const weightedStickyTargets = new Map<
|
||||
string,
|
||||
{ executionKey: string; successCount: number }
|
||||
>();
|
||||
@@ -32,6 +33,8 @@ export function clampStickyRoundRobinTargetLimit(value: unknown): number {
|
||||
return Math.min(Math.max(Math.floor(numericValue), 1), 1000);
|
||||
}
|
||||
|
||||
export const clampStickyWeightedTargetLimit = clampStickyRoundRobinTargetLimit;
|
||||
|
||||
export function getStickyRoundRobinStartIndex(
|
||||
comboName: string,
|
||||
targets: ResolvedComboTarget[],
|
||||
@@ -69,3 +72,27 @@ export function recordStickyRoundRobinSuccess(
|
||||
|
||||
rrStickyTargets.set(comboName, { executionKey: target.executionKey, successCount });
|
||||
}
|
||||
|
||||
export function getStickyWeightedExecutionKey(
|
||||
comboName: string,
|
||||
stickyLimit: number
|
||||
): string | null {
|
||||
const sticky = weightedStickyTargets.get(comboName);
|
||||
if (!sticky || stickyLimit <= 1 || sticky.successCount >= stickyLimit) return null;
|
||||
return sticky.executionKey;
|
||||
}
|
||||
|
||||
export function recordStickyWeightedSuccess(
|
||||
comboName: string,
|
||||
executionKey: string,
|
||||
stickyLimit: number
|
||||
): void {
|
||||
const sticky = weightedStickyTargets.get(comboName);
|
||||
const successCount = sticky?.executionKey === executionKey ? sticky.successCount + 1 : 1;
|
||||
if (successCount >= stickyLimit) {
|
||||
weightedStickyTargets.delete(comboName);
|
||||
return;
|
||||
}
|
||||
|
||||
weightedStickyTargets.set(comboName, { executionKey, successCount });
|
||||
}
|
||||
|
||||
@@ -159,6 +159,8 @@ const ADVANCED_FIELD_HELP_FALLBACK = {
|
||||
"How long a request can wait for a round-robin model slot before timing out. This queue is separate from any account-only concurrency cap.",
|
||||
stickyLimit:
|
||||
"Round-robin sticky batch size: consecutive successful requests sent to one target before rotating to the next. Empty inherits the global Sticky Limit setting; 1 disables batching (pure one-request rotation).",
|
||||
stickyWeightedLimit:
|
||||
"Weighted sticky batch size: consecutive successful requests sent to the selected weighted target before drawing again. Empty or 1 keeps the current per-request weighted draw.",
|
||||
failoverBeforeRetry:
|
||||
"When enabled, a 429 from the upstream triggers immediate target failover instead of retrying the same URL first.",
|
||||
targetTimeoutMs:
|
||||
@@ -2600,7 +2602,7 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders, combo
|
||||
handleAutoBalance();
|
||||
}
|
||||
|
||||
if (strategy === "round-robin") {
|
||||
if (strategy === "round-robin" || strategy === "weighted") {
|
||||
setShowAdvanced(true);
|
||||
}
|
||||
|
||||
@@ -2722,6 +2724,9 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders, combo
|
||||
if (config.stickyRoundRobinLimit !== undefined)
|
||||
configToSave.stickyRoundRobinLimit = config.stickyRoundRobinLimit;
|
||||
}
|
||||
if (strategy === "weighted" && config.stickyWeightedLimit !== undefined) {
|
||||
configToSave.stickyWeightedLimit = config.stickyWeightedLimit;
|
||||
}
|
||||
const hasConfigToSave = Object.keys(configToSave).length > 0;
|
||||
const hadExistingConfig = Object.keys(sanitizeComboRuntimeConfig(combo?.config)).length > 0;
|
||||
if (hasConfigToSave || (isEdit && hadExistingConfig)) {
|
||||
@@ -3840,6 +3845,41 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders, combo
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{strategy === "weighted" && (
|
||||
<div className="grid grid-cols-1 gap-2 pt-2 border-t border-black/5 dark:border-white/5">
|
||||
<div>
|
||||
<FieldLabelWithHelp
|
||||
label={getI18nOrFallback(
|
||||
t,
|
||||
"stickyWeightedLimit",
|
||||
"Sticky Weighted Limit"
|
||||
)}
|
||||
help={getI18nOrFallback(
|
||||
t,
|
||||
"advancedHelp.stickyWeightedLimit",
|
||||
ADVANCED_FIELD_HELP_FALLBACK.stickyWeightedLimit
|
||||
)}
|
||||
showHelp={!isExpertMode}
|
||||
/>
|
||||
<input
|
||||
type="number"
|
||||
min="0"
|
||||
max="1000"
|
||||
value={config.stickyWeightedLimit ?? ""}
|
||||
placeholder="1"
|
||||
onChange={(e) =>
|
||||
setConfig({
|
||||
...config,
|
||||
stickyWeightedLimit: e.target.value
|
||||
? Number(e.target.value)
|
||||
: undefined,
|
||||
})
|
||||
}
|
||||
className="w-full text-xs py-1.5 px-2 rounded border border-black/10 dark:border-white/10 bg-transparent focus:border-primary focus:outline-none"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{strategy === "context-relay" && (
|
||||
<div className="grid grid-cols-1 md:grid-cols-3 gap-2 pt-2 border-t border-black/5 dark:border-white/5">
|
||||
<div>
|
||||
|
||||
@@ -138,6 +138,7 @@ export const comboRuntimeConfigSchema = z
|
||||
// falls back to the global `settings.stickyRoundRobinLimit` so the existing
|
||||
// knob still controls the default. 0 clamps to 1 (no batching) upstream.
|
||||
stickyRoundRobinLimit: z.coerce.number().int().min(0).max(1000).optional(),
|
||||
stickyWeightedLimit: z.coerce.number().int().min(0).max(1000).optional(),
|
||||
healthCheckEnabled: z.boolean().optional(),
|
||||
healthCheckTimeoutMs: z.coerce.number().int().min(100).max(30000).optional(),
|
||||
handoffThreshold: z.coerce.number().min(0.5).max(0.94).optional(),
|
||||
|
||||
@@ -549,6 +549,24 @@ test("createComboSchema accepts per-combo stickyRoundRobinLimit and rejects out-
|
||||
assert.equal(tooHigh.success, false);
|
||||
});
|
||||
|
||||
test("createComboSchema accepts per-combo stickyWeightedLimit and rejects out-of-range", () => {
|
||||
const parsed = createComboSchema.parse({
|
||||
name: "sticky-weighted",
|
||||
models: [{ model: "openai/gpt-4o-mini", weight: 100 }],
|
||||
strategy: "weighted",
|
||||
config: { stickyWeightedLimit: 2 },
|
||||
});
|
||||
assert.equal(parsed.config.stickyWeightedLimit, 2);
|
||||
|
||||
const tooHigh = createComboSchema.safeParse({
|
||||
name: "sticky-weighted-too-high",
|
||||
models: [{ model: "openai/gpt-4o-mini", weight: 100 }],
|
||||
strategy: "weighted",
|
||||
config: { stickyWeightedLimit: 1001 },
|
||||
});
|
||||
assert.equal(tooHigh.success, false);
|
||||
});
|
||||
|
||||
test("createComboSchema coerces string numbers for maxSetRetries and setRetryDelayMs", () => {
|
||||
const parsed = createComboSchema.parse({
|
||||
name: "coerce-test",
|
||||
|
||||
@@ -13,6 +13,8 @@ const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
|
||||
const { handleComboChat, preScreenTargets } = await import("../../open-sse/services/combo.ts");
|
||||
const { weightedStickyTargets, rrStickyTargets } =
|
||||
await import("../../open-sse/services/combo/rrState.ts");
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const settingsDb = await import("../../src/lib/db/settings.ts");
|
||||
const { resetAllComboMetrics } = await import("../../open-sse/services/comboMetrics.ts");
|
||||
@@ -69,6 +71,8 @@ test.beforeEach(async () => {
|
||||
resetAllSemaphores();
|
||||
_resetAllDecks();
|
||||
clearSessions();
|
||||
weightedStickyTargets.clear();
|
||||
rrStickyTargets.clear();
|
||||
await cleanupTestDataDir();
|
||||
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
||||
await settingsDb.resetAllPricing();
|
||||
@@ -80,6 +84,8 @@ test.after(async () => {
|
||||
resetAllCircuitBreakers();
|
||||
resetAllSemaphores();
|
||||
_resetAllDecks();
|
||||
weightedStickyTargets.clear();
|
||||
rrStickyTargets.clear();
|
||||
settingsDb.clearAllLKGP();
|
||||
if (ORIGINAL_DATA_DIR === undefined) {
|
||||
delete process.env.DATA_DIR;
|
||||
@@ -385,6 +391,172 @@ test("round-robin sticky batching fallback success becomes sticky target", async
|
||||
assert.deepEqual(calls, ["openai/a", "claude/b", "claude/b", "gemini/c", "gemini/c"]);
|
||||
});
|
||||
|
||||
test("weighted sticky limit batches successful weighted selections", async () => {
|
||||
const calls: string[] = [];
|
||||
const secureRandom = await import("../../src/shared/utils/secureRandom.ts");
|
||||
// First draw targets openai/a (cumulative weight < 0.5), second targets claude/b
|
||||
// (cumulative weight >= 0.5). With sticky=2, openai/a runs for 2 calls then claude/b for 2.
|
||||
secureRandom._setSecureRandomFloatSource(() => (calls.length < 2 ? 0.1 : 0.9));
|
||||
try {
|
||||
const combo = {
|
||||
name: "weighted-sticky-batches",
|
||||
strategy: "weighted",
|
||||
models: [
|
||||
{ model: "openai/a", weight: 50 },
|
||||
{ model: "claude/b", weight: 50 },
|
||||
],
|
||||
config: { maxRetries: 0, retryDelayMs: 0, fallbackDelayMs: 0, stickyWeightedLimit: 2 },
|
||||
};
|
||||
for (let i = 0; i < 4; i += 1) {
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo,
|
||||
handleSingleModel: async (_body: any, modelStr: string) => {
|
||||
calls.push(modelStr);
|
||||
return okResponse();
|
||||
},
|
||||
isModelAvailable: async () => true,
|
||||
log: createLog(),
|
||||
settings: null,
|
||||
allCombos: null,
|
||||
});
|
||||
assert.equal(result.ok, true);
|
||||
}
|
||||
} finally {
|
||||
secureRandom._setSecureRandomFloatSource(null);
|
||||
}
|
||||
assert.deepEqual(calls, ["openai/a", "openai/a", "claude/b", "claude/b"]);
|
||||
});
|
||||
|
||||
test("weighted sticky limit clears unavailable sticky steps before sampling", async () => {
|
||||
const calls: string[] = [];
|
||||
const step0Key = "weighted-sticky-clears-model-1-openai-a";
|
||||
const step1Key = "weighted-sticky-clears-model-2-claude-b";
|
||||
weightedStickyTargets.set("weighted-sticky-clears", { executionKey: step0Key, successCount: 1 });
|
||||
const combo = {
|
||||
name: "weighted-sticky-clears",
|
||||
strategy: "weighted",
|
||||
models: [
|
||||
{ model: "openai/a", weight: 50 },
|
||||
{ model: "claude/b", weight: 50 },
|
||||
],
|
||||
config: { maxRetries: 0, retryDelayMs: 0, fallbackDelayMs: 0, stickyWeightedLimit: 10 },
|
||||
};
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo,
|
||||
handleSingleModel: async (_body: any, modelStr: string) => {
|
||||
calls.push(modelStr);
|
||||
return okResponse();
|
||||
},
|
||||
isModelAvailable: async (modelStr: string) => modelStr !== "openai/a",
|
||||
log: createLog(),
|
||||
settings: null,
|
||||
allCombos: null,
|
||||
});
|
||||
assert.equal(result.ok, true);
|
||||
assert.deepEqual(calls, ["claude/b"]);
|
||||
assert.equal(weightedStickyTargets.get("weighted-sticky-clears")?.executionKey, step1Key);
|
||||
});
|
||||
|
||||
test("weighted sticky limit follows fallback success", async () => {
|
||||
const calls: string[] = [];
|
||||
const secureRandom = await import("../../src/shared/utils/secureRandom.ts");
|
||||
// Always draw claude/b (the failing target). The combo retries within claude/b,
|
||||
// exhausts its leaves, falls through to openai/a, succeeds, and sticky migrates.
|
||||
secureRandom._setSecureRandomFloatSource(() => 0.9);
|
||||
try {
|
||||
const combo = {
|
||||
name: "weighted-sticky-fallback-success",
|
||||
strategy: "weighted",
|
||||
models: [
|
||||
{ model: "openai/a", weight: 50 },
|
||||
{ model: "claude/b", weight: 50 },
|
||||
],
|
||||
config: { maxRetries: 0, retryDelayMs: 0, fallbackDelayMs: 0, stickyWeightedLimit: 2 },
|
||||
};
|
||||
for (let i = 0; i < 4; i += 1) {
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo,
|
||||
handleSingleModel: async (_body: any, modelStr: string) => {
|
||||
calls.push(modelStr);
|
||||
return modelStr === "claude/b" ? errorResponse(503, "b is down") : okResponse();
|
||||
},
|
||||
isModelAvailable: async () => true,
|
||||
log: createLog(),
|
||||
settings: null,
|
||||
allCombos: null,
|
||||
});
|
||||
assert.equal(result.ok, true);
|
||||
}
|
||||
} finally {
|
||||
secureRandom._setSecureRandomFloatSource(null);
|
||||
}
|
||||
assert.deepEqual(calls, ["claude/b", "openai/a", "openai/a", "claude/b", "openai/a", "openai/a"]);
|
||||
});
|
||||
|
||||
test("weighted sticky keeps a top-level step if one nested leaf remains available", async () => {
|
||||
const calls: string[] = [];
|
||||
const step0Key = "weighted-nested-sticky-ref-1-minimax";
|
||||
weightedStickyTargets.set("weighted-nested-sticky", { executionKey: step0Key, successCount: 1 });
|
||||
const combo = {
|
||||
name: "weighted-nested-sticky",
|
||||
strategy: "weighted",
|
||||
models: [{ kind: "combo-ref", comboName: "minimax", weight: 100 }],
|
||||
config: { maxRetries: 0, retryDelayMs: 0, fallbackDelayMs: 0, stickyWeightedLimit: 10 },
|
||||
};
|
||||
const allCombos = [
|
||||
combo,
|
||||
{ name: "minimax", strategy: "priority", models: ["ollama/m3", "oc/m3"] },
|
||||
];
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo,
|
||||
handleSingleModel: async (_body: any, modelStr: string) => {
|
||||
calls.push(modelStr);
|
||||
return okResponse();
|
||||
},
|
||||
isModelAvailable: async (modelStr: string) => modelStr !== "ollama/m3",
|
||||
log: createLog(),
|
||||
settings: null,
|
||||
allCombos,
|
||||
});
|
||||
assert.equal(result.ok, true);
|
||||
assert.deepEqual(calls, ["oc/m3"]);
|
||||
assert.equal(weightedStickyTargets.get("weighted-nested-sticky")?.executionKey, step0Key);
|
||||
});
|
||||
|
||||
test("round-robin sticky clears unavailable sticky target before rotation", async () => {
|
||||
const calls: string[] = [];
|
||||
const step0Key = "rr-sticky-clears-unavailable-model-1-openai-a";
|
||||
rrStickyTargets.set("rr-sticky-clears-unavailable", { executionKey: step0Key, successCount: 1 });
|
||||
const combo = {
|
||||
name: "rr-sticky-clears-unavailable",
|
||||
strategy: "round-robin",
|
||||
models: ["openai/a", "claude/b", "gemini/c"],
|
||||
config: { maxRetries: 0, retryDelayMs: 0, fallbackDelayMs: 0, stickyRoundRobinLimit: 10 },
|
||||
};
|
||||
const result = await handleComboChat({
|
||||
body: {},
|
||||
combo,
|
||||
handleSingleModel: async (_body: any, modelStr: string) => {
|
||||
calls.push(modelStr);
|
||||
return okResponse();
|
||||
},
|
||||
isModelAvailable: async (modelStr: string) => modelStr !== "openai/a",
|
||||
log: createLog(),
|
||||
settings: null,
|
||||
allCombos: null,
|
||||
});
|
||||
assert.equal(result.ok, true);
|
||||
assert.deepEqual(calls, ["claude/b"]);
|
||||
assert.equal(
|
||||
rrStickyTargets.get("rr-sticky-clears-unavailable")?.executionKey,
|
||||
"rr-sticky-clears-unavailable-model-2-claude-b"
|
||||
);
|
||||
});
|
||||
|
||||
test("strict-random survives a stale deck entry after a target is removed", async () => {
|
||||
const comboTwoTargets = {
|
||||
name: "strict-random-stale",
|
||||
|
||||
Reference in New Issue
Block a user