pjandro/OmniRoute
1
0
Fork 0
mirror of https://github.com/diegosouzapw/OmniRoute.git synced 2026-08-01 21:02:12 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(auto): use p95 fallback in speed factors (#7128)

Browse Source 
Co-authored-by: Diego Rodrigues de Sa e Souza <diegosouza.pw@gmail.com>
Co-authored-by: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com>
This commit is contained in:
KooshaPari
2026-07-16 10:14:20 -07:00
committed by GitHub
parent dedf680231
commit 8e9cff3145
2 changed files with 32 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
open-sse/services/autoCombo/__tests__/speedRanking.test.ts
View File

@@ -165,15 +165,34 @@ describe("rankBySpeed — factor breakdown", () => {
});
it("falls back to 0.5 per missing metric so new providers are not crushed", () => {
const ranked = rankBySpeed([candidate({ provider: "fresh", model: "m" })]);
const ranked = rankBySpeed([
candidate({
provider: "fresh",
model: "m",
p95LatencyMs: undefined,
latencyStdDev: undefined,
}),
]);
expect(ranked).toHaveLength(1);
// No telemetry at all → weighted sum lands near 0.5 with reliability multiplier 1
expect(ranked[0].factors.reliability).toBe(1);
expect(ranked[0].factors.health).toBe(1);
expect(ranked[0].factors.ttft).toBe(0.5);
expect(ranked[0].factors.tps).toBe(0.5);
});
});
expect(ranked[0].factors.tps).toBe(0.5);
});
it("uses p95 latency when TTFT and E2E telemetry are unavailable", () => {
const ranked = rankBySpeed([
candidate({ provider: "slow-tail", model: "m", p95LatencyMs: 4000 }),
candidate({ provider: "fast-tail", model: "m", p95LatencyMs: 1000 }),
]);
const fast = ranked.find((entry) => entry.provider === "fast-tail");
const slow = ranked.find((entry) => entry.provider === "slow-tail");
expect(fast?.factors.ttft).toBeGreaterThan(slow?.factors.ttft ?? 1);
expect(fast?.factors.e2e).toBeGreaterThan(slow?.factors.e2e ?? 1);
});
});
describe("rankBySpeed — weight overrides", () => {
it("respects caller weight overrides (e.g. heavy TTFT bias)", () => {
@@ -223,4 +242,4 @@ describe("pickFastest", () => {
const winner = pickFastest([slow, fast]);
expect(winner?.provider).toBe("fast");
});
});
});

10
open-sse/services/autoCombo/speedRanking.ts
View File

@@ -211,9 +211,15 @@ function speedFactorsFor(
failureRate: number
): SpeedFactors {
return {
ttft: lowerIsBetter(positiveFinite(candidate.avgTtftMs), maxima.ttft),
ttft: lowerIsBetter(
positiveFinite(candidate.avgTtftMs) ?? positiveFinite(candidate.p95LatencyMs),
maxima.ttft
),
tps: higherIsBetter(positiveFinite(candidate.avgTokensPerSecond), maxima.tps),
e2e: lowerIsBetter(positiveFinite(candidate.avgE2ELatencyMs), maxima.e2e),
e2e: lowerIsBetter(
positiveFinite(candidate.avgE2ELatencyMs) ?? positiveFinite(candidate.p95LatencyMs),
maxima.e2e
),
p95: lowerIsBetter(positiveFinite(candidate.p95LatencyMs), maxima.p95),
health: healthScoreFor(candidate.circuitBreakerState),
reliability: clamp01(1 - failureRate),