refactor(auto-combo): fix divergent scoring in combo health reporting (#11854)

Corrige divergência de scoring no relatório de saúde do auto-combo, com testes atualizados em `combo-resolve-auto-strategy-split.test.ts` e `combo-scoring-inspector.test.ts`. Validado no worktree combinado. Obrigado!
2026-09-13 18:32:12 +03:00 · 2026-08-30 08:09:57 -04:00
parent 4c8074ba7a
commit 9903a6d2eb
5 changed files with 377 additions and 65 deletions
--- a/open-sse/services/combo/autoStrategy.ts
+++ b/open-sse/services/combo/autoStrategy.ts
@@ -39,6 +39,7 @@ import {
  calculateScore,
  computePoolMaxima,
  type ProviderCandidate,
+  type ScoringFactors,
  type ScoringWeights,
 } from "../autoCombo/scoring.ts";
 import type { RoutingHint } from "../manifestAdapter";
@@ -407,10 +408,19 @@ export function scoreAutoTargets(
      }
      return {
        target,
+        factors,
        score,
      };
    })
-    .filter((entry): entry is { target: ResolvedComboTarget; score: number } => entry !== null)
+    .filter(
+      (
+        entry
+      ): entry is {
+        target: ResolvedComboTarget;
+        factors: ScoringFactors;
+        score: number;
+      } => entry !== null
+    )
    .sort((a, b) => b.score - a.score);
 }

--- a/open-sse/services/combo/resolveAutoStrategy.ts
+++ b/open-sse/services/combo/resolveAutoStrategy.ts
@@ -4,6 +4,7 @@ import {
  errorResponseWithComboDiagnostics,
 } from "../../utils/error.ts";
 import { BudgetExceededError, selectProvider as selectAutoProvider } from "../autoCombo/engine.ts";
+import type { ScoringWeights } from "../autoCombo/scoring.ts";
 import {
  resolveRequestModePack,
  parseRequestBudgetCap,
@@ -83,6 +84,53 @@ export type ResolveAutoStrategyResult =
  | { earlyResponse: Response }
  | { orderedTargets: ResolvedComboTarget[]; autoUsedExplicitRouter: boolean };

+export interface EvaluateAutoCandidatesOptions {
+  targets: ResolvedComboTarget[];
+  comboName: string;
+  body: Record<string, unknown>;
+  taskType: string;
+  weights: ScoringWeights;
+  sessionId?: string | null;
+  resetWindowConfig?: ResetWindowConfig;
+  resilienceSettings?: ResilienceSettings | null;
+  manifestHint?: RoutingHint | null;
+  buildAutoCandidates: BuildAutoCandidates;
+}
+
+export async function evaluateAutoCandidates(options: EvaluateAutoCandidatesOptions) {
+  const builtCandidates = await options.buildAutoCandidates(
+    options.targets,
+    options.comboName,
+    options.sessionId,
+    options.resetWindowConfig,
+    options.resilienceSettings
+  );
+  const cacheAffinityScores = calculatePromptCacheAffinityScores(
+    builtCandidates,
+    options.body,
+    options.sessionId
+  );
+  const candidates = builtCandidates.map((candidate) => ({
+    ...candidate,
+    cacheAffinity: cacheAffinityScores.get(promptCacheTargetIdentity(candidate)) ?? 0,
+  }));
+  const routableCandidates = candidates.filter(
+    (candidate) => candidate.quotaCutoffBlocked !== true
+  );
+  return {
+    sourceCandidates: builtCandidates,
+    candidates,
+    routableCandidates,
+    scoredTargets: scoreAutoTargets(
+      options.targets,
+      routableCandidates,
+      options.taskType,
+      options.weights,
+      options.manifestHint
+    ),
+  };
+}
+
 /**
 * Resolve target ordering for the `auto` combo strategy.
 *
@@ -262,24 +310,34 @@ export async function resolveAutoStrategyOrder(
          },
        }
      : resilienceSettings;
-  const candidates = await buildAutoCandidates(
-    eligibleTargets,
-    combo.name,
-    relayOptions?.sessionId,
-    resetWindowConfig,
-    autoCandidateResilienceSettings
-  );
-  const cacheAffinityScores = calculatePromptCacheAffinityScores(
-    candidates,
-    body,
-    relayOptions?.sessionId
-  );
-  for (const candidate of candidates) {
-    candidate.cacheAffinity = cacheAffinityScores.get(promptCacheTargetIdentity(candidate)) ?? 0;
+  // Complexity-aware routing (2026, opt-in): classify the request's
+  // difficulty and feed a tier hint into scoring so tierAffinity /
+  // specificityMatch favor candidates whose tier matches the request.
+  const autoManifestHint: RoutingHint | null =
+    config.complexityAwareRouting === true
+      ? buildComplexityRoutingHint(
+          eligibleTargets.filter((t) => t.kind === "model"),
+          body,
+          log
+        )
+      : null;
+
+  const { sourceCandidates, candidates, routableCandidates, scoredTargets } =
+    await evaluateAutoCandidates({
+      targets: eligibleTargets,
+      comboName: combo.name,
+      body,
+      taskType,
+      weights,
+      sessionId: relayOptions?.sessionId,
+      resetWindowConfig,
+      resilienceSettings: autoCandidateResilienceSettings,
+      manifestHint: autoManifestHint,
+      buildAutoCandidates,
+    });
+  for (let index = 0; index < sourceCandidates.length; index += 1) {
+    sourceCandidates[index].cacheAffinity = candidates[index]?.cacheAffinity;
  }
-  const routableCandidates = candidates.filter(
-    (candidate) => candidate.quotaCutoffBlocked !== true
-  );
  const quotaBlockedCount = candidates.length - routableCandidates.length;
  if (quotaBlockedCount > 0) {
    log.info(
@@ -368,25 +426,6 @@ export async function resolveAutoStrategyOrder(
      selectionReason = `score=${selection.score.toFixed(3)}${selection.isExploration ? " (exploration)" : ""}`;
    }

-    // Complexity-aware routing (2026, opt-in): classify the request's
-    // difficulty and feed a tier hint into scoring so tierAffinity /
-    // specificityMatch favor candidates whose tier matches the request.
-    const autoManifestHint: RoutingHint | null =
-      config.complexityAwareRouting === true
-        ? buildComplexityRoutingHint(
-            eligibleTargets.filter((t) => t.kind === "model"),
-            body,
-            log
-          )
-        : null;
-
-    const scoredTargets = scoreAutoTargets(
-      eligibleTargets,
-      routableCandidates,
-      taskType,
-      weights,
-      autoManifestHint
-    );
    const rankedTargets = scoredTargets.map((entry) => entry.target);
    const selectedTarget =
      scoredTargets.find((entry) => {
--- a/src/lib/usage/comboScoringInspector.ts
+++ b/src/lib/usage/comboScoringInspector.ts
@@ -8,6 +8,11 @@ import {
  type ProviderConnectionView,
 } from "@/lib/usage/resilienceExplain";
 import { getCircuitBreaker } from "@/shared/utils/circuitBreaker";
+import { buildAutoCandidates } from "@omniroute/open-sse/services/combo.ts";
+import { parseAutoConfig } from "@omniroute/open-sse/services/combo/autoConfig.ts";
+import { resolveComboTargets } from "@omniroute/open-sse/services/combo/comboStructure.ts";
+import { evaluateAutoCandidates } from "@omniroute/open-sse/services/combo/resolveAutoStrategy.ts";
+import type { AutoProviderCandidate, ComboLike } from "@omniroute/open-sse/services/combo/types.ts";
 import {
  calculateFactors,
  calculateScore,
@@ -315,6 +320,56 @@ function buildCandidate(
  };
 }

+function buildLiveCandidateContext(
+  candidate: AutoProviderCandidate,
+  target: TargetHealth,
+  forecastTarget: ComboForecastTarget | undefined
+): CandidateContext {
+  const context = buildCandidate(target, forecastTarget).context;
+  for (const key of FACTOR_KEYS) context.sources[key] = "runtime";
+  context.notes.quota = "Current credential-level quota from live candidate evaluation.";
+  context.notes.costInv = "Current model pricing from live candidate evaluation.";
+  context.notes.latencyInv = "Current credential-level latency from live candidate evaluation.";
+  context.notes.stability = "Current credential-level stability from live candidate evaluation.";
+  context.notes.contextAffinity = "Live candidate affinity evaluated without a preview session.";
+  if (candidate.cacheAffinity !== undefined) {
+    context.notes.cacheAffinity = "Prompt-cache affinity from live candidate evaluation.";
+  }
+  return context;
+}
+
+function liveTargetHealth(
+  candidate: AutoProviderCandidate,
+  target: {
+    executionKey: string;
+    stepId: string;
+    modelStr: string;
+    provider: string;
+    connectionId?: string | null;
+    label?: string | null;
+  },
+  historicalTarget: TargetHealth | undefined
+): TargetHealth {
+  return {
+    executionKey: target.executionKey,
+    stepId: target.stepId,
+    model: target.modelStr,
+    provider: target.provider,
+    connectionId: target.connectionId ?? null,
+    label: target.label ?? null,
+    requests: historicalTarget?.requests ?? 0,
+    successRate: historicalTarget?.successRate ?? 0,
+    avgLatencyMs: historicalTarget?.avgLatencyMs ?? candidate.p95LatencyMs,
+    lastStatus: historicalTarget?.lastStatus ?? null,
+    lastUsedAt: historicalTarget?.lastUsedAt ?? null,
+    quotaRemainingPct:
+      candidate.quotaTotal > 0 ? (candidate.quotaRemaining / candidate.quotaTotal) * 100 : null,
+    quotaIsExhausted: candidate.quotaCutoffBlocked === true,
+    quotaTrend: historicalTarget?.quotaTrend ?? null,
+    quotaScope: candidate.connectionId ? "connection" : "provider",
+  };
+}
+
 function factorBreakdown(
  factors: ScoringFactors,
  weights: ScoringWeights,
@@ -362,7 +417,9 @@ async function buildInspectorCombo(
  forecastTargets: Map<string, ComboForecastTarget>,
  autopilotCombo: ComboAutopilotCombo | undefined,
  taskType: string,
-  inspectorWeights: InspectorWeights
+  inspectorWeights: InspectorWeights,
+  configuredCombo?: ComboRecord,
+  configuredCombos: ComboRecord[] = []
 ): Promise<ComboScoringInspectorCombo> {
  const warnings: string[] = [];
  const { weights } = inspectorWeights;
@@ -378,23 +435,67 @@ async function buildInspectorCombo(
  }

  const issueCounts = autopilotIssueCounts(autopilotCombo);
-  const contexts = targets.map((target) =>
-    buildCandidate(
-      target,
-      forecastTargets.get(target.executionKey) || forecastTargets.get(target.stepId)
-    )
-  );
-  const pool = contexts.map((entry) => entry.candidate);
+  let scored: Array<{
+    entry: { candidate: ProviderCandidate; context: CandidateContext };
+    factors: ScoringFactors;
+    score: number;
+  }>;

-  const scored = contexts
-    .map((entry) => {
-      const factors = calculateFactors(entry.candidate, pool, taskType, getTaskFitness);
-      const score = calculateScore(factors, weights);
-      const issueCount = issueCounts.get(entry.context.target.executionKey) ?? 0;
-      entry.context.autopilotIssueCount = issueCount;
-      return { entry, factors, score };
-    })
-    .sort((left, right) => right.score - left.score);
+  if (
+    combo.strategy === "auto" &&
+    configuredCombo?.name &&
+    Array.isArray(configuredCombo.models) &&
+    configuredCombo.models.length > 0
+  ) {
+    const comboLike = configuredCombo as ComboLike;
+    const resolvedTargets = resolveComboTargets(comboLike, configuredCombos as ComboLike[]);
+    const { resetWindowConfig } = parseAutoConfig(comboLike, resolvedTargets);
+    const evaluation = await evaluateAutoCandidates({
+      targets: resolvedTargets,
+      comboName: comboLike.name,
+      body: {},
+      taskType,
+      weights,
+      resetWindowConfig,
+      buildAutoCandidates,
+    });
+    const candidatesByExecutionKey = new Map(
+      evaluation.routableCandidates.map((candidate) => [candidate.executionKey, candidate])
+    );
+    scored = evaluation.scoredTargets.map((item) => {
+      const candidate = candidatesByExecutionKey.get(item.target.executionKey);
+      if (!candidate)
+        throw new Error(`Missing evaluated candidate for ${item.target.executionKey}`);
+      const historicalTarget = targets.find(
+        (target) =>
+          target.executionKey === item.target.executionKey || target.stepId === item.target.stepId
+      );
+      const target = liveTargetHealth(candidate, item.target, historicalTarget);
+      const context = buildLiveCandidateContext(
+        candidate,
+        target,
+        forecastTargets.get(target.executionKey) ?? forecastTargets.get(target.stepId)
+      );
+      context.autopilotIssueCount = issueCounts.get(target.executionKey) ?? 0;
+      return { entry: { candidate, context }, factors: item.factors, score: item.score };
+    });
+  } else {
+    const contexts = targets.map((target) =>
+      buildCandidate(
+        target,
+        forecastTargets.get(target.executionKey) || forecastTargets.get(target.stepId)
+      )
+    );
+    const pool = contexts.map((entry) => entry.candidate);
+    scored = contexts
+      .map((entry) => {
+        const factors = calculateFactors(entry.candidate, pool, taskType, getTaskFitness);
+        const score = calculateScore(factors, weights);
+        entry.context.autopilotIssueCount = issueCounts.get(entry.context.target.executionKey) ?? 0;
+        return { entry, factors, score };
+      })
+      .sort((left, right) => right.score - left.score);
+  }

  const connectionsByProvider = new Map<string, ProviderConnectionView[]>();
  await Promise.allSettled(
@@ -514,17 +615,18 @@ export async function buildComboScoringInspectorResponse(
    horizon: options.horizon,
    method: "read_only_recompute",
    combos: await Promise.all(
-      health.combos.map((combo) =>
-        buildInspectorCombo(
+      health.combos.map((combo) => {
+        const configuredCombo = combosById.get(combo.comboId) ?? combosByName.get(combo.comboName);
+        return buildInspectorCombo(
          combo,
          targetForecastMap(forecastByComboId.get(combo.comboId)?.targets ?? []),
          autopilotByComboId.get(combo.comboId),
          taskType,
-          resolveInspectorWeights(
-            combosById.get(combo.comboId) ?? combosByName.get(combo.comboName)
-          )
-        )
-      )
+          resolveInspectorWeights(configuredCombo),
+          configuredCombo,
+          configuredCombos
+        );
+      })
    ),
  };
 }
--- a/tests/unit/combo-resolve-auto-strategy-split.test.ts
+++ b/tests/unit/combo-resolve-auto-strategy-split.test.ts
@@ -1,7 +1,11 @@
 import { test, after } from "node:test";
 import assert from "node:assert/strict";

-import { resolveAutoStrategyOrder } from "@omniroute/open-sse/services/combo/resolveAutoStrategy.ts";
+import {
+  evaluateAutoCandidates,
+  resolveAutoStrategyOrder,
+} from "@omniroute/open-sse/services/combo/resolveAutoStrategy.ts";
+import { DEFAULT_WEIGHTS } from "@omniroute/open-sse/services/autoCombo/scoring.ts";
 import { resetDbInstance } from "@/lib/db/core.ts";

 // resolveAutoStrategyOrder loads the LKGP via the DB singleton (dynamic import);
@@ -87,6 +91,44 @@ test("all candidates quota-cutoff-blocked -> early 429 Response", async () => {
  }
 });

+test("candidate evaluation is deterministic and does not mutate builder-owned candidates", async () => {
+  const candidates = [
+    {
+      kind: "model",
+      stepId: "s1",
+      executionKey: "openai>gpt-4o@account-a",
+      modelStr: "gpt-4o",
+      provider: "openai",
+      model: "gpt-4o",
+      connectionId: "account-a",
+      quotaRemaining: 80,
+      quotaTotal: 100,
+      circuitBreakerState: "CLOSED",
+      costPer1MTokens: 1,
+      p95LatencyMs: 100,
+      latencyStdDev: 10,
+      errorRate: 0,
+    },
+  ];
+  const before = structuredClone(candidates);
+  const options = {
+    targets: [target("openai", "gpt-4o")],
+    comboName: "autoc",
+    body: { prompt_cache_key: "pure-evaluation", messages: [] },
+    taskType: "default",
+    weights: DEFAULT_WEIGHTS,
+    buildAutoCandidates: (async () => candidates) as never,
+  };
+
+  const first = await evaluateAutoCandidates(options);
+  const second = await evaluateAutoCandidates(options);
+
+  assert.deepEqual(first.scoredTargets, second.scoredTargets);
+  assert.deepEqual(candidates, before);
+  assert.notEqual(first.candidates[0], candidates[0]);
+  assert.equal(first.scoredTargets[0].factors.quota, 0.8);
+});
+
 test("cache affinity scores expanded auto account candidates directly", async () => {
  const candidates = [
    {
--- a/tests/unit/combo-scoring-inspector.test.ts
+++ b/tests/unit/combo-scoring-inspector.test.ts
@@ -30,6 +30,11 @@ const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuit
 const { DEFAULT_WEIGHTS, normalizeScoringWeights } =
  await import("../../open-sse/services/autoCombo/scoring.ts");
 const { MODE_PACKS } = await import("../../open-sse/services/autoCombo/modePacks.ts");
+const { buildAutoCandidates } = await import("../../open-sse/services/combo.ts");
+const { parseAutoConfig } = await import("../../open-sse/services/combo/autoConfig.ts");
+const { resolveComboTargets } = await import("../../open-sse/services/combo/comboStructure.ts");
+const { evaluateAutoCandidates } =
+  await import("../../open-sse/services/combo/resolveAutoStrategy.ts");

 async function resetStorage() {
  comboMetrics.resetAllComboMetrics();
@@ -183,9 +188,7 @@ test("scoring inspector ranks targets and explains score contributions", async (
  );
  assert.ok(Math.abs(contributionSum - response.combos[0].targets[0].score) < 0.02);
  assert.ok(response.combos[0].targets[0].factors.some((factor) => factor.key === "quota"));
-  assert.ok(
-    response.combos[0].targets[0].factors.some((factor) => factor.source === "combo_health")
-  );
+  assert.ok(response.combos[0].targets[0].factors.some((factor) => factor.source === "runtime"));
 });

 test("scoring inspector reports mode packs over explicit auto weights", async () => {
@@ -244,6 +247,122 @@ test("scoring inspector reports valid explicit auto weights", async () => {
  assert.equal(response.combos[0].modePack, null);
  assert.deepEqual(response.combos[0].weights, explicitWeights);
 });
+test("configured auto preview uses live evaluation scores and zero quota contribution", async () => {
+  const weights = normalizeScoringWeights({
+    ...DEFAULT_WEIGHTS,
+    quota: 0,
+    latencyInv: DEFAULT_WEIGHTS.latencyInv + DEFAULT_WEIGHTS.quota,
+  });
+  const combo = await combosDb.createCombo({
+    name: "combo-scoring-live-parity",
+    strategy: "auto",
+    models: ["openai/gpt-4o-mini", "anthropic/claude-3-haiku"],
+    autoConfig: { weights },
+  });
+
+  const response = await inspector.buildComboScoringInspectorResponse({
+    range: "24h",
+    horizon: "7d",
+    comboId: String(combo.id),
+    combos: [combo],
+    skipAutopilot: true,
+  });
+  const resolvedTargets = resolveComboTargets(combo as never, [combo] as never);
+  const evaluation = await evaluateAutoCandidates({
+    targets: resolvedTargets,
+    comboName: combo.name,
+    body: {},
+    taskType: "default",
+    weights,
+    resetWindowConfig: parseAutoConfig(combo as never, resolvedTargets).resetWindowConfig,
+    buildAutoCandidates,
+  });
+
+  assert.deepEqual(
+    response.combos[0].targets.map((target) => ({
+      executionKey: target.executionKey,
+      score: target.score,
+      factors: Object.fromEntries(target.factors.map((factor) => [factor.key, factor.value])),
+    })),
+    evaluation.scoredTargets.map((target) => ({
+      executionKey: target.target.executionKey,
+      score: Number(target.score.toFixed(4)),
+      factors: Object.fromEntries(
+        Object.entries(target.factors).map(([key, value]) => [key, Number(value.toFixed(4))])
+      ),
+    }))
+  );
+  for (const target of response.combos[0].targets) {
+    const quota = target.factors.find((factor) => factor.key === "quota");
+    assert.equal(quota?.source, "runtime");
+    assert.equal(quota?.weight, 0);
+    assert.equal(quota?.contribution, 0);
+    assert.equal(
+      target.score,
+      Number(target.factors.reduce((sum, factor) => sum + factor.contribution, 0).toFixed(4))
+    );
+  }
+});
+
+test("configured auto preview with no model steps preserves health-derived targets", async () => {
+  const combo = await combosDb.createCombo({
+    name: "combo-scoring-empty-auto",
+    strategy: "auto",
+    models: [],
+  });
+  const target = {
+    executionKey: "health-only-target",
+    stepId: "health-only-target",
+    model: "openai/gpt-4o-mini",
+    provider: "openai",
+    connectionId: null,
+    label: "Health-only target",
+    requests: 1,
+    successRate: 100,
+    avgLatencyMs: 100,
+    lastStatus: 200,
+    lastUsedAt: new Date().toISOString(),
+    quotaRemainingPct: 100,
+    quotaIsExhausted: false,
+    quotaTrend: null,
+    quotaScope: "provider" as const,
+  };
+
+  const response = await inspector.buildComboScoringInspectorResponse({
+    range: "24h",
+    horizon: "7d",
+    comboId: String(combo.id),
+    combos: [combo],
+    skipAutopilot: true,
+    healthResponse: {
+      timeRange: "24h",
+      combos: [
+        {
+          comboId: String(combo.id),
+          comboName: combo.name,
+          strategy: "auto",
+          models: [],
+          cost: { totalUsd: 0, avgPerRequestUsd: 0, byModel: [] },
+          quotaHealth: { providers: [], worstRemainingPct: 100 },
+          usageSkew: { modelDistribution: [], giniCoefficient: 0 },
+          performance: { avgLatencyMs: 100, successRate: 100, totalRequests: 1 },
+          targetHealth: [target],
+        },
+      ],
+    },
+    forecastResponse: {
+      asOf: new Date().toISOString(),
+      timeRange: "24h",
+      horizon: "7d",
+      method: "linear_history",
+      combos: [],
+    },
+  });
+
+  assert.equal(response.combos[0].targets.length, 1);
+  assert.equal(response.combos[0].targets[0].executionKey, target.executionKey);
+});
+
 test("scoring inspector normalizes partial explicit auto weights like runtime", async () => {
  const explicitWeights = {
    quota: 0.3,