Merge remote-tracking branch 'origin/release/v3.8.47' into HEAD

tests/unit/auto-combo-budget-fallback-3470.test.ts

@@ -0,0 +1,161 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import {
+  BudgetExceededError,
+  selectProvider,
+} from "../../open-sse/services/autoCombo/engine.ts";
+import {
+  parseRequestBudgetFallback,
+  resolveRequestAutoControls,
+} from "../../open-sse/services/autoCombo/requestControls.ts";
+import { getSelfHealingManager } from "../../open-sse/services/autoCombo/selfHealing.ts";
+import { DEFAULT_WEIGHTS } from "../../open-sse/services/autoCombo/scoring.ts";
+
+// #3470 — Auto-combo transparency + budget controls: `budgetFallback: "strict"`
+// must refuse to select (instead of silently overspending) when EVERY candidate
+// exceeds the configured `budgetCap`.
+
+const healer = getSelfHealingManager();
+const originalRandom = Math.random;
+
+function resetHealer() {
+  healer.exclusions.clear();
+  healer.incidentMode = false;
+}
+
+const baseConfig = {
+  id: "auto-main",
+  name: "Auto Main Budget Strict",
+  type: "auto" as const,
+  candidatePool: [],
+  weights: DEFAULT_WEIGHTS,
+  explorationRate: 0,
+};
+
+const overBudgetCandidates = [
+  {
+    provider: "premium",
+    model: "gpt-4o",
+    quotaRemaining: 99,
+    quotaTotal: 100,
+    circuitBreakerState: "CLOSED",
+    costPer1MTokens: 12000,
+    p95LatencyMs: 100,
+    latencyStdDev: 10,
+    errorRate: 0.01,
+    accountTier: "ultra",
+    quotaResetIntervalSecs: 60,
+  },
+  {
+    provider: "cheap",
+    model: "gpt-4o-mini",
+    quotaRemaining: 60,
+    quotaTotal: 100,
+    circuitBreakerState: "CLOSED",
+    costPer1MTokens: 100,
+    p95LatencyMs: 900,
+    latencyStdDev: 50,
+    errorRate: 0.02,
+    accountTier: "free",
+    quotaResetIntervalSecs: 86400,
+  },
+];
+
+test.beforeEach(() => {
+  resetHealer();
+  Math.random = originalRandom;
+});
+
+test.afterEach(() => {
+  resetHealer();
+  Math.random = originalRandom;
+});
+
+test("selectProvider throws BudgetExceededError when budgetFallback='strict' and every candidate exceeds budgetCap", () => {
+  assert.throws(
+    () =>
+      selectProvider(
+        { ...baseConfig, budgetCap: 0.001, budgetFallback: "strict" },
+        overBudgetCandidates,
+        "default"
+      ),
+    BudgetExceededError
+  );
+});
+
+test("BudgetExceededError message reports the cap and the cheapest candidate's cost, no stack leak", () => {
+  try {
+    selectProvider(
+      { ...baseConfig, budgetCap: 0.001, budgetFallback: "strict" },
+      overBudgetCandidates,
+      "default"
+    );
+    assert.fail("expected selectProvider to throw");
+  } catch (err) {
+    assert.ok(err instanceof BudgetExceededError);
+    assert.match(err.message, /budget cap of \$0\.0010/);
+    assert.ok(!err.message.includes("at /"));
+  }
+});
+
+test("selectProvider still falls back to cheapest when budgetFallback is 'cheapest' (default/legacy)", () => {
+  const result = selectProvider(
+    { ...baseConfig, budgetCap: 0.001, budgetFallback: "cheapest" },
+    overBudgetCandidates,
+    "default"
+  );
+  assert.equal(result.provider, "cheap");
+});
+
+test("selectProvider defaults to cheapest fallback when budgetFallback is unset (backward compatible)", () => {
+  const result = selectProvider({ ...baseConfig, budgetCap: 0.001 }, overBudgetCandidates, "default");
+  assert.equal(result.provider, "cheap");
+});
+
+test("selectProvider with strict fallback still picks a within-budget candidate normally", () => {
+  const result = selectProvider(
+    { ...baseConfig, budgetCap: 1, budgetFallback: "strict" },
+    overBudgetCandidates,
+    "default"
+  );
+  assert.equal(result.provider, "cheap");
+});
+
+test("parseRequestBudgetFallback: accepts 'strict' and its aliases", () => {
+  assert.equal(parseRequestBudgetFallback("strict"), "strict");
+  assert.equal(parseRequestBudgetFallback("BLOCK"), "strict");
+  assert.equal(parseRequestBudgetFallback(" hard "), "strict");
+});
+
+test("parseRequestBudgetFallback: accepts 'cheapest' and its aliases", () => {
+  assert.equal(parseRequestBudgetFallback("cheapest"), "cheapest");
+  assert.equal(parseRequestBudgetFallback("Cheapest-Viable"), "cheapest");
+  assert.equal(parseRequestBudgetFallback("soft"), "cheapest");
+});
+
+test("parseRequestBudgetFallback: ignores unknown/empty/non-string values", () => {
+  assert.equal(parseRequestBudgetFallback("garbage"), undefined);
+  assert.equal(parseRequestBudgetFallback(""), undefined);
+  assert.equal(parseRequestBudgetFallback(null), undefined);
+  assert.equal(parseRequestBudgetFallback(42), undefined);
+});
+
+test("resolveRequestAutoControls: aggregates mode/budget/budgetFallback headers, omitting unset ones", () => {
+  const headers = new Headers({
+    "x-omniroute-mode": "fast",
+    "x-omniroute-budget": "0.05",
+    "x-omniroute-budget-fallback": "strict",
+  });
+  const controls = resolveRequestAutoControls(headers);
+  assert.deepEqual(controls, {
+    mode: "fast",
+    budgetCap: 0.05,
+    budgetFallback: "strict",
+  });
+});
+
+test("resolveRequestAutoControls: returns an empty object when no auto-combo headers are present", () => {
+  const controls = resolveRequestAutoControls(new Headers());
+  assert.deepEqual(controls, {});
+});

tests/unit/provider-models-context-window-override-4125.test.ts

@@ -0,0 +1,138 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+// #4125: manual per-model context-window override.
+//
+// Custom-model rows already carried an inputTokenLimit/outputTokenLimit set at
+// creation time, but there was no way to *edit* the real context window afterwards
+// when a provider's own /models endpoint misreports it (e.g. reports 1M when the
+// real limit is 128K), and combo routing would drop the model once a too-small
+// value made it into the catalog / models.dev sync.
+//
+// This reuses the Feature-5004 `model_context_overrides` table (source="manual"),
+// which already wins over the catalog in `getModelContextLimit()` — the same
+// resolver combo routing consults — so no new priority-0 source is needed in
+// modelCapabilities.ts. This test proves the API round trip end-to-end: PUT sets
+// the override, GET surfaces it back on the model row, and getModelContextLimit()
+// (the function combo.ts calls) picks it up.
+
+const TEST_DATA_DIR = fs.mkdtempSync(
+  path.join(os.tmpdir(), "omniroute-provider-model-context-override-4125-")
+);
+process.env.DATA_DIR = TEST_DATA_DIR;
+
+const core = await import("../../src/lib/db/core.ts");
+const modelsDb = await import("../../src/lib/db/models.ts");
+const contextOverrides = await import("../../src/lib/db/modelContextOverrides.ts");
+const modelCapabilities = await import("../../src/lib/modelCapabilities.ts");
+const providerModelsRoute = await import("../../src/app/api/provider-models/route.ts");
+
+async function resetStorage() {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
+}
+
+test.beforeEach(async () => {
+  await resetStorage();
+});
+
+test.after(async () => {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+});
+
+function buildRequest(method: string, body: unknown) {
+  return new Request("http://localhost/api/provider-models", {
+    method,
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(body),
+  });
+}
+
+test("PUT with contextWindowOverride persists a manual override that wins over the catalog", async () => {
+  await modelsDb.addCustomModel(
+    "openai-compatible-demo",
+    "misreported-model",
+    "Misreported Model",
+    "manual",
+    "chat-completions",
+    ["chat"],
+    undefined,
+    { inputTokenLimit: 1_000_000 } // provider misreports 1M
+  );
+
+  const putRes = await providerModelsRoute.PUT(
+    buildRequest("PUT", {
+      provider: "openai-compatible-demo",
+      modelId: "misreported-model",
+      contextWindowOverride: 131072, // real window per the operator
+    })
+  );
+  const putBody = (await putRes.json()) as { contextWindowOverride?: number | null };
+
+  assert.equal(putRes.status, 200);
+  assert.equal(putBody.contextWindowOverride, 131072);
+
+  // Persisted as a "manual" source in the Feature-5004 table.
+  const record = contextOverrides.getModelContextOverrideRecord(
+    "openai-compatible-demo",
+    "misreported-model"
+  );
+  assert.ok(record, "override record should exist");
+  assert.equal(record!.realContext, 131072);
+  assert.equal(record!.source, "manual");
+
+  // getModelContextLimit is what combo.ts's context-window filter reads — the
+  // manual override must win over the (wrong) 1M value on the custom-model row.
+  const limit = modelCapabilities.getModelContextLimit(
+    "openai-compatible-demo",
+    "misreported-model"
+  );
+  assert.equal(limit, 131072);
+});
+
+test("GET surfaces contextWindowOverride on the custom model row", async () => {
+  await modelsDb.addCustomModel("openai-compatible-demo", "m1", "M1");
+  contextOverrides.setModelContextOverride("openai-compatible-demo", "m1", 200000, "manual");
+
+  const getRes = await providerModelsRoute.GET(
+    new Request("http://localhost/api/provider-models?provider=openai-compatible-demo")
+  );
+  const body = (await getRes.json()) as {
+    models: Array<{ id?: string; contextWindowOverride?: number; contextWindowOverrideSource?: string }>;
+  };
+
+  const row = body.models.find((m) => m.id === "m1");
+  assert.ok(row, "model row should be present");
+  assert.equal(row!.contextWindowOverride, 200000);
+  assert.equal(row!.contextWindowOverrideSource, "manual");
+});
+
+test("PUT with contextWindowOverride: null clears a previously set override", async () => {
+  await modelsDb.addCustomModel("openai-compatible-demo", "m2", "M2");
+  contextOverrides.setModelContextOverride("openai-compatible-demo", "m2", 50000, "manual");
+
+  const putRes = await providerModelsRoute.PUT(
+    buildRequest("PUT", {
+      provider: "openai-compatible-demo",
+      modelId: "m2",
+      contextWindowOverride: null,
+    })
+  );
+  assert.equal(putRes.status, 200);
+
+  const record = contextOverrides.getModelContextOverrideRecord("openai-compatible-demo", "m2");
+  assert.equal(record, null);
+});
+
+test("default behavior unchanged: no override means getModelContextLimit falls back to the catalog", async () => {
+  await modelsDb.addCustomModel("openai-compatible-demo", "m3", "M3");
+  const limit = modelCapabilities.getModelContextLimit("openai-compatible-demo", "m3");
+  // No override, no catalog entry for this unknown custom model → null (not dropped
+  // by the combo prefilter, which treats unknown context as "include to be safe").
+  assert.equal(limit, null);
+});

tests/unit/quota-card-grid-horizontal-layout.test.ts

@@ -0,0 +1,120 @@
+// #3520 — Provider Quota page should use horizontal whitespace better.
+//
+// QuotaCardGrid previously stacked provider groups vertically via a single
+// `flex flex-col` container and kept cards to a conservative 1/2/3/4-column
+// breakpoint ladder starting at `grid-cols-1`. This regression guard asserts
+// the shipped JSX structure and grouping logic directly:
+//  1. Grouping still produces one header per distinct provider with the
+//     correct account count ("N account(s)").
+//  2. The per-group card grid starts multi-column (`grid-cols-2`), not
+//     single-column, so cards fill horizontal space sooner.
+//  3. Provider groups themselves flow into multiple columns on wide screens
+//     (`columns-*`) instead of an unconditional vertical `flex flex-col`
+//     stack.
+//
+// Note: QuotaCardGrid's sibling QuotaCard pulls in next/image + provider-icon
+// resolution that only works inside the real Next.js runtime, so this file
+// exercises the two testable seams directly instead of full SSR-rendering the
+// tree: (a) the pure grouping function extracted below, mirroring the
+// component's own grouping logic, and (b) the literal className contract of
+// the component's JSX (static string literals, not derived at runtime),
+// parsed from source via the TypeScript compiler API so the assertions track
+// the real shipped markup rather than a hand-copied string.
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import path from "node:path";
+import ts from "typescript";
+
+const COMPONENT_PATH = path.resolve(
+  import.meta.dirname,
+  "../../src/app/(dashboard)/dashboard/usage/components/ProviderLimits/QuotaCardGrid.tsx"
+);
+
+function groupByProvider<T extends { provider: string }>(connections: T[]): Map<string, T[]> {
+  const groups = new Map<string, T[]>();
+  for (const conn of connections) {
+    const list = groups.get(conn.provider) ?? [];
+    list.push(conn);
+    groups.set(conn.provider, list);
+  }
+  return groups;
+}
+
+test("QuotaCardGrid (#3520) — groups connections by provider with correct counts", () => {
+  const groups = groupByProvider([
+    { id: "conn-a1", provider: "openai" },
+    { id: "conn-a2", provider: "openai" },
+    { id: "conn-b1", provider: "anthropic" },
+  ]);
+  assert.deepEqual([...groups.keys()], ["openai", "anthropic"]);
+  assert.equal(groups.get("openai")!.length, 2);
+  assert.equal(groups.get("anthropic")!.length, 1);
+});
+
+/**
+ * Extract the string literal passed to `className={...}` (or `className="..."`)
+ * for every JSX `<div>` opening element in the component's `return (...)` JSX,
+ * in source order, via the TypeScript compiler API (not a hand-rolled regex —
+ * tracks the real AST so it can't be fooled by comments/whitespace).
+ */
+function extractDivClassNames(sourcePath: string): string[] {
+  const sourceText = fs.readFileSync(sourcePath, "utf8");
+  const sourceFile = ts.createSourceFile(
+    sourcePath,
+    sourceText,
+    ts.ScriptTarget.Latest,
+    true,
+    ts.ScriptKind.TSX
+  );
+  const classNames: string[] = [];
+
+  function visit(node: ts.Node) {
+    if (ts.isJsxOpeningElement(node) || ts.isJsxSelfClosingElement(node)) {
+      const tagName = node.tagName.getText(sourceFile);
+      if (tagName === "div") {
+        for (const attr of node.attributes.properties) {
+          if (ts.isJsxAttribute(attr) && attr.name.getText(sourceFile) === "className") {
+            const init = attr.initializer;
+            if (init && ts.isStringLiteral(init)) {
+              classNames.push(init.text);
+            } else if (
+              init &&
+              ts.isJsxExpression(init) &&
+              init.expression &&
+              ts.isStringLiteral(init.expression)
+            ) {
+              classNames.push(init.expression.text);
+            }
+          }
+        }
+      }
+    }
+    ts.forEachChild(node, visit);
+  }
+  visit(sourceFile);
+  return classNames;
+}
+
+test("QuotaCardGrid (#3520) — outer container flows groups into multiple columns, not a single vertical stack", () => {
+  const [outerClassName] = extractDivClassNames(COMPONENT_PATH);
+  assert.ok(outerClassName, "expected the component to render an outer <div className=...>");
+  assert.match(outerClassName, /\bcolumns-/);
+  assert.notEqual(outerClassName, "flex flex-col gap-6");
+});
+
+test("QuotaCardGrid (#3520) — per-group card grid starts multi-column (grid-cols-2), not single-column", () => {
+  const classNames = extractDivClassNames(COMPONENT_PATH);
+  const cardGridClassName = classNames.find(
+    (c) => /\bgrid\b/.test(c) && /grid-cols-/.test(c)
+  );
+  assert.ok(cardGridClassName, "expected to find the per-group card grid's className");
+  assert.match(cardGridClassName!, /\bgrid-cols-2\b/);
+  assert.doesNotMatch(cardGridClassName!, /\bgrid-cols-1\b/);
+});
+
+test("QuotaCardGrid (#3520) — early-returns null when there are no connections", () => {
+  const sourceText = fs.readFileSync(COMPONENT_PATH, "utf8");
+  assert.match(sourceText, /if\s*\(\s*connections\.length\s*===\s*0\s*\)\s*return\s*null;/);
+});

tests/unit/xai-exact-cost-2453.test.ts

@@ -0,0 +1,138 @@
+/**
+ * xAI exact provider-reported cost passthrough (port of decolua/9router#2453,
+ * capability A — @ryanngit).
+ *
+ * xAI's chat-completions `usage` object reports the exact billed cost of a
+ * request via `cost_in_usd_ticks`. Per the official docs
+ * (https://docs.x.ai/developers/cost-tracking and the API reference's usage
+ * schema): "TICKS_IN_USD_CENT: i64 = 100_000_000" ⇒ 10_000_000_000 (1e10)
+ * ticks per USD. Example given in the docs: 37756000 ticks ≈ $0.0038.
+ *
+ * NOTE: the upstream PR used a /1e12 divisor (100x under-report) — this port
+ * uses the doc-verified /1e10 divisor instead.
+ *
+ * OmniRoute previously always estimated cost from token counts × static
+ * pricing, discarding this exact figure. This test proves calculateCost()/
+ * computeCostFromPricing() now trust the exact figure when present, and
+ * still fall back to the token-based estimate when it is absent (control).
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import { calculateCost, computeCostFromPricing } from "../../src/lib/usage/costCalculator.ts";
+import { extractUsageFromResponse } from "../../open-sse/handlers/usageExtractor.ts";
+import { extractUsage, normalizeUsage } from "../../open-sse/utils/usageTracking.ts";
+
+// $1/1M input, $2/1M output → 1M+1M tokens would estimate to $3 at the
+// metered rate. Chosen so the exact-cost value (~$0.0038) is unmistakably
+// NOT the token-based estimate — proves the early return actually fires.
+const PRICING = { input: 1, output: 2 };
+const TOKENS_1M_EACH = { input: 1_000_000, output: 1_000_000 };
+
+// Doc example: 37756000 ticks ≈ $0.0038 (docs.x.ai/developers/cost-tracking).
+const DOC_EXAMPLE_TICKS = 37_756_000;
+const DOC_EXAMPLE_USD = 0.0037756; // 37756000 / 1e10, exact
+
+test("computeCostFromPricing: xAI exact cost_in_usd_ticks overrides the token-based estimate", () => {
+  const cost = computeCostFromPricing(PRICING, {
+    ...TOKENS_1M_EACH,
+    cost_in_usd_ticks: DOC_EXAMPLE_TICKS,
+  });
+  assert.ok(
+    Math.abs(cost - DOC_EXAMPLE_USD) < 1e-9,
+    `expected ${DOC_EXAMPLE_USD}, got ${cost}`
+  );
+  assert.notEqual(cost, 3, "must not fall back to the $3 token-based estimate");
+});
+
+test("computeCostFromPricing: xAI exact cost works even with no pricing record at all", () => {
+  const cost = computeCostFromPricing(null, {
+    ...TOKENS_1M_EACH,
+    cost_in_usd_ticks: DOC_EXAMPLE_TICKS,
+  });
+  assert.ok(Math.abs(cost - DOC_EXAMPLE_USD) < 1e-9);
+});
+
+test("computeCostFromPricing CONTROL: no cost_in_usd_ticks still falls back to the token-based estimate", () => {
+  assert.equal(computeCostFromPricing(PRICING, TOKENS_1M_EACH), 3);
+});
+
+test("calculateCost: xAI exact cost_in_usd_ticks overrides whatever the token-based estimate would be", async () => {
+  // Baseline: the token-based estimate calculateCost would otherwise compute
+  // for this provider/model/token-count (whatever xai/grok-4.3's local
+  // pricing table says — not hardcoded here, so this test doesn't break if
+  // pricing data changes).
+  const baseline = await calculateCost("xai", "grok-4.3", { input: 500, output: 500 });
+
+  const cost = await calculateCost("xai", "grok-4.3", {
+    input: 500,
+    output: 500,
+    cost_in_usd_ticks: DOC_EXAMPLE_TICKS,
+  });
+
+  assert.ok(Math.abs(cost - DOC_EXAMPLE_USD) < 1e-9, `expected ${DOC_EXAMPLE_USD}, got ${cost}`);
+  assert.notEqual(cost, baseline, "exact cost must override the token-based estimate");
+});
+
+test("calculateCost CONTROL: no cost_in_usd_ticks still falls back to the token-based estimate (unchanged)", async () => {
+  const before = await calculateCost("xai", "grok-4.3", { input: 500, output: 500 });
+  const after = await calculateCost("xai", "grok-4.3", { input: 500, output: 500 });
+  assert.equal(after, before, "identical calls without the exact field must stay deterministic");
+  assert.notEqual(after, DOC_EXAMPLE_USD, "must not coincidentally match the exact-cost value");
+});
+
+test("normalizeUsage: passes through a finite cost_in_usd_ticks", () => {
+  const normalized = normalizeUsage({ prompt_tokens: 10, cost_in_usd_ticks: DOC_EXAMPLE_TICKS });
+  assert.equal(normalized.cost_in_usd_ticks, DOC_EXAMPLE_TICKS);
+});
+
+test("normalizeUsage: drops a non-finite cost_in_usd_ticks", () => {
+  const normalized = normalizeUsage({ prompt_tokens: 10, cost_in_usd_ticks: "not-a-number" });
+  assert.equal(normalized.cost_in_usd_ticks, undefined);
+});
+
+test("extractUsageFromResponse: xAI OpenAI-shaped usage carries cost_in_usd_ticks through", () => {
+  const usage = extractUsageFromResponse(
+    {
+      usage: {
+        prompt_tokens: 12,
+        completion_tokens: 8,
+        cost_in_usd_ticks: DOC_EXAMPLE_TICKS,
+      },
+    },
+    "xai"
+  );
+  assert.equal(usage.cost_in_usd_ticks, DOC_EXAMPLE_TICKS);
+});
+
+test("extractUsageFromResponse CONTROL: non-xAI OpenAI usage without the field stays unchanged (no stray key)", () => {
+  const usage = extractUsageFromResponse(
+    {
+      usage: {
+        prompt_tokens: 12,
+        completion_tokens: 8,
+        prompt_tokens_details: { cached_tokens: 3 },
+        completion_tokens_details: { reasoning_tokens: 2 },
+      },
+    },
+    "openai"
+  );
+  assert.deepEqual(usage, {
+    prompt_tokens: 12,
+    completion_tokens: 8,
+    cached_tokens: 3,
+    reasoning_tokens: 2,
+  });
+  assert.ok(!("cost_in_usd_ticks" in usage), "must not add a stray undefined key");
+});
+
+test("extractUsage (streaming): xAI OpenAI-format chunk carries cost_in_usd_ticks through", () => {
+  const usage = extractUsage({
+    usage: {
+      prompt_tokens: 12,
+      completion_tokens: 8,
+      cost_in_usd_ticks: DOC_EXAMPLE_TICKS,
+    },
+  });
+  assert.equal(usage.cost_in_usd_ticks, DOC_EXAMPLE_TICKS);
+});