routing: optimize latency strategy with perf metrics (#5629)

Integrated into release/v3.8.43. Thanks @KooshaPari!

open-sse/services/autoCombo/routerStrategy.ts

@@ -100,32 +100,101 @@ class CostStrategyImpl implements RouterStrategy {
 
 // ── LatencyStrategy: prioritize low latency + reliability ───────────────────
 
+function positiveMetric(value: unknown): number | null {
+  const numericValue = Number(value);
+  return Number.isFinite(numericValue) && numericValue > 0 ? numericValue : null;
+}
+
+function boundedRate(value: unknown): number {
+  const numericValue = Number(value);
+  return Number.isFinite(numericValue) && numericValue >= 0 ? Math.min(1, numericValue) : 0;
+}
+
+function maxPositiveMetric(
+  candidates: ProviderCandidate[],
+  readMetric: (candidate: ProviderCandidate) => unknown,
+  fallback = 1
+): number {
+  return Math.max(
+    ...candidates.map((candidate) => positiveMetric(readMetric(candidate)) ?? 0),
+    fallback
+  );
+}
+
+function latencyMetricScore(value: number | null, maxValue: number): number {
+  if (value == null) return 0.5;
+  return inverseNormalized(value, maxValue);
+}
+
+function throughputMetricScore(value: number | null, maxValue: number): number {
+  if (value == null) return 0.5;
+  return clamp01(value / Math.max(maxValue, 0.000_001));
+}
+
 class LatencyStrategyImpl implements RouterStrategy {
   readonly name = "latency";
-  readonly description = "Prioritizes lowest p95 latency with reliability weighting";
+  readonly description =
+    "Prioritizes the fastest reliable provider-model pair using TTFT, TPS, E2E latency, health, fail rate, and stability";
 
   select(pool: ProviderCandidate[], context: RoutingContext): RoutingDecision {
     const healthy = pool.filter((c) => c.circuitBreakerState !== "OPEN");
     const candidates = healthy.length > 0 ? healthy : pool;
-    const sorted = [...candidates].sort((a, b) => {
-      const aPenalty = a.errorRate * 1000;
-      const bPenalty = b.errorRate * 1000;
-      return a.p95LatencyMs + aPenalty - (b.p95LatencyMs + bPenalty);
-    });
-    const best = sorted[0];
+    if (candidates.length === 0) throw new Error("[LatencyStrategy] No candidates available");
+
+    const maxP95 = maxPositiveMetric(candidates, (candidate) => candidate.p95LatencyMs);
+    const maxTtft = maxPositiveMetric(
+      candidates,
+      (candidate) => candidate.avgTtftMs ?? candidate.p95LatencyMs
+    );
+    const maxE2E = maxPositiveMetric(
+      candidates,
+      (candidate) => candidate.avgE2ELatencyMs ?? candidate.p95LatencyMs
+    );
+    const maxTps = maxPositiveMetric(candidates, (candidate) => candidate.avgTokensPerSecond);
+    const maxStdDev = maxPositiveMetric(candidates, (candidate) => candidate.latencyStdDev, 0.001);
+
+    const scored = candidates
+      .map((candidate) => {
+        const p95 = positiveMetric(candidate.p95LatencyMs);
+        const ttft = positiveMetric(candidate.avgTtftMs) ?? p95;
+        const e2e = positiveMetric(candidate.avgE2ELatencyMs) ?? p95;
+        const tps = positiveMetric(candidate.avgTokensPerSecond);
+        const failureRate = boundedRate(candidate.failureRate ?? candidate.errorRate);
+        const healthScore = getHealthScore(candidate);
+        const p95Score = latencyMetricScore(p95, maxP95);
+        const ttftScore = latencyMetricScore(ttft, maxTtft);
+        const e2eScore = latencyMetricScore(e2e, maxE2E);
+        const throughputScore = throughputMetricScore(tps, maxTps);
+        const reliabilityScore = 1 - failureRate;
+        const stabilityScore = latencyMetricScore(
+          positiveMetric(candidate.latencyStdDev),
+          maxStdDev
+        );
+        const rawScore =
+          ttftScore * 0.25 +
+          throughputScore * 0.2 +
+          e2eScore * 0.18 +
+          p95Score * 0.12 +
+          reliabilityScore * 0.15 +
+          healthScore * 0.05 +
+          stabilityScore * 0.05;
+        const reliabilityMultiplier = Math.max(0.05, reliabilityScore * reliabilityScore);
+        const score = rawScore * reliabilityMultiplier * Math.max(0.25, healthScore);
+
+        return { candidate, score, ttft, e2e, tps, failureRate };
+      })
+      .sort((a, b) => b.score - a.score);
+
+    const best = scored[0];
     if (!best) throw new Error("[LatencyStrategy] No candidates available");
 
-    const latencyScore = best.p95LatencyMs > 0 ? Math.max(0.001, 10_000 / best.p95LatencyMs) : 1;
-    const reliability = Math.max(0, 1 - best.errorRate);
-    const finalScore = latencyScore * 0.7 + reliability * 0.3;
-
     return {
-      provider: best.provider,
-      model: best.model,
+      provider: best.candidate.provider,
+      model: best.candidate.model,
       strategy: this.name,
-      reason: `LatencyStrategy: p95=${best.p95LatencyMs}ms, errorRate=${(best.errorRate * 100).toFixed(2)}%`,
+      reason: `LatencyStrategy: ttft=${best.ttft ?? "n/a"}ms, tps=${best.tps ?? "n/a"}, e2e=${best.e2e ?? "n/a"}ms, p95=${best.candidate.p95LatencyMs}ms, failRate=${(best.failureRate * 100).toFixed(2)}%`,
       candidatesConsidered: candidates.length,
-      finalScore,
+      finalScore: best.score,
     };
   }
 }

open-sse/services/autoCombo/scoring.ts

@@ -61,8 +61,16 @@ export interface ProviderCandidate {
   circuitBreakerState: "CLOSED" | "HALF_OPEN" | "OPEN";
   costPer1MTokens: number;
   p95LatencyMs: number;
+  /** Average time-to-first-token in ms, when stream telemetry is available. */
+  avgTtftMs?: number;
+  /** Average end-to-end request latency in ms, when usage telemetry is available. */
+  avgE2ELatencyMs?: number;
+  /** Average generation throughput in output tokens/sec, when token telemetry is available. */
+  avgTokensPerSecond?: number;
   latencyStdDev: number;
   errorRate: number;
+  /** Optional provider/model observed failure rate. Falls back to errorRate. */
+  failureRate?: number;
   /** T10: Optional account tier for priority boosting (Ultra > Pro > Free) */
   accountTier?: "ultra" | "pro" | "standard" | "free";
   /** T10: Optional quota reset interval in seconds (shorter = higher priority when same quota) */

tests/unit/router-strategies.test.ts

@@ -80,6 +80,56 @@ test("latency — error rate penalizes a fast-but-flaky candidate", () => {
   assert.equal(getStrategy("latency").select(pool, ctx).provider, "steady");
 });
 
+test("latency — uses TTFT, TPS, and E2E metrics to pick the fastest provider-model pair", () => {
+  const pool = [
+    cand({
+      provider: "low-p95-slow-stream",
+      p95LatencyMs: 120,
+      avgTtftMs: 160,
+      avgE2ELatencyMs: 1_600,
+      avgTokensPerSecond: 18,
+      latencyStdDev: 120,
+      failureRate: 0.01,
+    }),
+    cand({
+      provider: "fast-streaming",
+      p95LatencyMs: 180,
+      avgTtftMs: 40,
+      avgE2ELatencyMs: 480,
+      avgTokensPerSecond: 120,
+      latencyStdDev: 20,
+      failureRate: 0.01,
+    }),
+  ];
+  const decision = getStrategy("latency").select(pool, ctx);
+  assert.equal(decision.provider, "fast-streaming");
+  assert.match(decision.reason, /ttft=40ms/);
+  assert.match(decision.reason, /tps=120/);
+});
+
+test("latency — failure rate can outweigh excellent raw speed", () => {
+  const pool = [
+    cand({
+      provider: "fast-flaky",
+      p95LatencyMs: 80,
+      avgTtftMs: 20,
+      avgE2ELatencyMs: 260,
+      avgTokensPerSecond: 160,
+      failureRate: 0.9,
+    }),
+    cand({
+      provider: "reliable",
+      p95LatencyMs: 120,
+      avgTtftMs: 80,
+      avgE2ELatencyMs: 400,
+      avgTokensPerSecond: 120,
+      failureRate: 0,
+      latencyStdDev: 10,
+    }),
+  ];
+  assert.equal(getStrategy("latency").select(pool, ctx).provider, "reliable");
+});
+
 test("latency — 'fast' alias resolves to the latency strategy", () => {
   assert.equal(getStrategy("fast").name, "latency");
 });