Merge branch 'fix/v3851-basereds-orphans-g4' into fix/release-v3.8.51-basereds-orphans

open-sse/services/autoCombo/virtualFactory.ts

@@ -685,10 +685,21 @@ export async function prepareVirtualAutoComboInputs(
     // back to the static catalog only when the user has none. This keeps catalog-only
     // models (e.g. openrouter/auto) out of every auto/* pool when the operator only
     // synced a subset (e.g. OpenRouter with importFreeModelsOnly).
-    const [syncedByConnection, customModels] = await Promise.all([
+    const [syncedByConnection, rawCustomModels] = await Promise.all([
       getSyncedAvailableModelsByConnection(providerId),
       getCustomModels(providerId),
     ]);
+    // The `customModels` key_value blob is operator-writable and is stored as raw
+    // parsed JSON, so a row can be `null` or a non-object. The catalog builder
+    // already filters those out (catalog.ts, "Add custom models"); without the same
+    // filter here every read below null-derefs and the whole auto/* pool fails to
+    // materialize ("Could not materialize built-in auto model auto/<id>").
+    const customModels: Array<{ id?: string }> = (
+      Array.isArray(rawCustomModels) ? rawCustomModels : []
+    ).filter(
+      (model: unknown): model is { id?: string } =>
+        !!model && typeof model === "object" && !Array.isArray(model)
+    );
     const userVisibleIds = new Set<string>();
     for (const models of Object.values(syncedByConnection)) {
       for (const m of models) if (m.id && !hiddenModels?.has(m.id)) userVisibleIds.add(m.id);

src/lib/db/core.ts

@@ -408,7 +408,12 @@ const SCHEMA_SQL = `
   CREATE INDEX IF NOT EXISTS idx_cl_timestamp ON call_logs(timestamp);
   CREATE INDEX IF NOT EXISTS idx_cl_status ON call_logs(status);
   CREATE INDEX IF NOT EXISTS idx_cl_provider_timestamp ON call_logs(provider, timestamp);
-  CREATE INDEX IF NOT EXISTS idx_cl_request_provider ON call_logs(request_type, provider);
+  -- idx_cl_request_provider is NOT declared here: SCHEMA_SQL runs before
+  -- ensureCallLogsColumns() heals a legacy call_logs table, and a lineage that
+  -- predates the request_type column has none yet — the CREATE INDEX would abort
+  -- the whole schema exec with "no such column: request_type" and the server would
+  -- never boot. It is created next to the other request_type/combo indexes in
+  -- ensureCallLogsColumns() (db/schemaColumns.ts), after the columns exist.
 
   CREATE TABLE IF NOT EXISTS proxy_logs (
     id TEXT PRIMARY KEY,

src/lib/db/models/activeSyncedCatalog.ts

@@ -184,10 +184,12 @@ async function unionCustomModels(
  */
 async function loadConnectionCatalog(storedProviderId: string): Promise<SyncedAvailableModel[]> {
   const [connections, modelsByConnection] = await Promise.all([
-    getRawProviderConnections({ provider: storedProviderId, isActive: true }, undefined, undefined, [
-      "id",
-      "provider",
-    ]),
+    getRawProviderConnections(
+      { provider: storedProviderId, isActive: true },
+      undefined,
+      undefined,
+      ["id", "provider"]
+    ),
     getSyncedAvailableModelsByConnection(storedProviderId),
   ]);
 
@@ -248,6 +250,18 @@ export async function getActiveSyncedCatalog(providerId: string): Promise<Active
 /**
  * Return non-empty synced catalogs grouped by provider, restricted to active
  * connections. This is the authoritative live source for /v1/models.
+ *
+ * Deliberately does NOT union `customModels` the way the dispatch-time readers
+ * above do (#12597). Its three consumers all read this as "what the provider's
+ * live sync reported": /v1/models emits synced rows through their own loop and
+ * has a separate custom-model pass right after it (with the specialty-registry
+ * dedupe, hidePaid and vision overrides that pass owns); /api/models uses it to
+ * decide whether an exclusive-listing provider's live catalog suppresses a static
+ * row; getSyncedAutoAliases derives tier aliases from it. Blurring operator custom
+ * rows into "synced" made /v1/models emit a custom specialty model under the
+ * provider's ALIAS as the parentless primary row (`jina/<id>`) and demote the
+ * embedding/rerank registry's canonical `jina-ai/<id>` to a child — the inverse of
+ * the identity every other specialty model of that provider carries.
  */
 export async function getAllActiveSyncedModels(): Promise<Record<string, SyncedAvailableModel[]>> {
   try {
@@ -277,10 +291,7 @@ export async function getAllActiveSyncedModels(): Promise<Record<string, SyncedA
 
         const models = enrichCursorCatalog(
           providerId,
-          await unionCustomModels(
-            providerId,
-            collectModelsForConnections(modelsByConnection, connectionIds)
-          )
+          collectModelsForConnections(modelsByConnection, connectionIds)
         );
 
         if (models.length > 0) {

src/lib/db/schemaColumns.ts

@@ -265,6 +265,12 @@ export function ensureCallLogsColumns(db: SqliteDatabase) {
       "CREATE INDEX IF NOT EXISTS idx_call_logs_requested_model ON call_logs(requested_model)"
     );
     db.exec("CREATE INDEX IF NOT EXISTS idx_call_logs_request_type ON call_logs(request_type)");
+    // #12832's provider-stats index. It lives here rather than in SCHEMA_SQL because
+    // SCHEMA_SQL runs before this healing pass: on a legacy call_logs table that
+    // predates `request_type` the CREATE INDEX aborts the whole schema exec.
+    db.exec(
+      "CREATE INDEX IF NOT EXISTS idx_cl_request_provider ON call_logs(request_type, provider)"
+    );
     db.exec(
       "CREATE INDEX IF NOT EXISTS idx_cl_combo_target ON call_logs(combo_name, combo_execution_key, timestamp)"
     );

tests/unit/combo-auto-pool-visible-only.test.ts

@@ -173,3 +173,40 @@ test("virtual auto-combo pool filters EVERY provider with partial sync, not just
     "kilocode pool must contain exactly the two synced models"
   );
 });
+
+test("virtual auto-combo pool survives a malformed customModels row", async () => {
+  // The `customModels` key_value blob is operator-writable and is read back as raw
+  // parsed JSON, so a row can be null / a non-object / carry no id. Before the guard
+  // in prepareVirtualAutoComboInputs those rows threw
+  // "Cannot read properties of null (reading 'id')" and EVERY auto/* combo failed to
+  // materialize ("[catalog] Could not materialize built-in auto model auto/<id>").
+  const conn = await providersDb.createProviderConnection({
+    provider: "openai",
+    authType: "apikey",
+    name: "OpenAI",
+    apiKey: "sk-test-openai",
+  });
+  const connectionId = (conn as { id?: string }).id;
+  await modelsDb.replaceSyncedAvailableModelsForConnection("openai", connectionId, [
+    { id: "gpt-4o-mini", name: "GPT-4o mini", source: "imported" as const },
+  ]);
+  core
+    .getDbInstance()
+    .prepare("INSERT OR REPLACE INTO key_value (namespace, key, value) VALUES (?, ?, ?)")
+    .run(
+      "customModels",
+      "openai",
+      JSON.stringify([null, "not-an-object", { name: "Missing Id" }, { id: "operator-custom" }])
+    );
+
+  const prepared = await virtualFactory.prepareVirtualAutoComboInputs();
+  const openaiCandidates = prepared.regularCandidates.filter((c) => c.provider === "openai");
+  assert.ok(
+    openaiCandidates.some((c) => c.model === "gpt-4o-mini"),
+    "the synced model must still reach the pool despite the malformed custom rows"
+  );
+  assert.ok(
+    openaiCandidates.some((c) => c.model === "operator-custom"),
+    "the one well-formed custom row must still reach the pool"
+  );
+});

tests/unit/combo-provider-cooldown-sibling.test.ts

@@ -129,12 +129,33 @@ test("source guard: auth.ts skips model lockout for per-model-quota providers on
 });
 
 test("source guard: combo.ts skips provider cooldown for per-model-quota on 500", () => {
-  const src = fs.readFileSync(
+  // The combo dispatcher was split out of combo.ts into open-sse/services/combo/*
+  // (#12746 executeTarget → gates/attempt/loop, #12811 round-robin). The invariant
+  // did not move: EVERY call site that records a provider cooldown after a failed
+  // combo target must first exclude a 500 on a per-model-quota provider, or one
+  // model's outage cools down its siblings. Scan the whole combo surface so the
+  // guard follows the code instead of one file name.
+  const comboDir = path.join(process.cwd(), "open-sse", "services", "combo");
+  const comboFiles = [
     path.join(process.cwd(), "open-sse", "services", "combo.ts"),
-    "utf-8"
+    ...fs
+      .readdirSync(comboDir, { withFileTypes: true })
+      .filter((entry) => entry.isFile() && entry.name.endsWith(".ts"))
+      .map((entry) => path.join(comboDir, entry.name)),
+  ].filter((file) => fs.existsSync(file));
+
+  const recordingFiles = comboFiles.filter((file) =>
+    fs.readFileSync(file, "utf-8").includes("recordProviderCooldown(")
   );
   assert.ok(
-    src.includes("hasPerModelQuota(provider, rawModel)") && src.includes("recordProviderCooldown"),
-    "combo.ts must skip provider cooldown recording for per-model-quota providers on 500"
+    recordingFiles.length > 0,
+    "no combo module records a provider cooldown — the guarded call site vanished"
   );
+  for (const file of recordingFiles) {
+    const normalized = fs.readFileSync(file, "utf-8").replace(/\s+/g, " ");
+    assert.ok(
+      /result\.status === 500[^;]{0,160}?hasPerModelQuota\(provider,/.test(normalized),
+      `${path.basename(file)} must skip provider cooldown recording for per-model-quota providers on 500`
+    );
+  }
 });

tests/unit/custom-models-live-catalog-12597.test.ts

@@ -14,10 +14,14 @@ process.env.DATA_DIR = TEST_DATA_DIR;
 process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "custom-live-12597-test-secret";
 
 const core = await import("../../src/lib/db/core.ts");
-const { addCustomModel, replaceSyncedAvailableModelsForConnection, getActiveProvidersWithSyncedModel } =
-  await import("../../src/lib/db/models.ts");
+const {
+  addCustomModel,
+  replaceSyncedAvailableModelsForConnection,
+  getActiveProvidersWithSyncedModel,
+} = await import("../../src/lib/db/models.ts");
 const {
   getActiveSyncedCatalog,
+  getAllActiveSyncedModels,
   catalogContainsModel,
   reconcileProvidersWithActiveSyncedCatalog,
 } = await import("../../src/lib/db/models/activeSyncedCatalog.ts");
@@ -124,6 +128,21 @@ test("#12597 sparse custom overlay does not wipe synced capability fields", asyn
   assert.deepEqual(match?.supportedThinkingEfforts, ["low", "high"]);
 });
 
+test("#12597 the union is dispatch-only — getAllActiveSyncedModels stays live-sync-shaped", async () => {
+  await addCustomModel(PROVIDER, PICKER_MODEL, "DeepSeek R1 via picker");
+
+  const allActive = await getAllActiveSyncedModels();
+  const ids = (allActive[PROVIDER] ?? []).map((model) => model.id);
+
+  assert.deepEqual(
+    ids,
+    [SYNCED_MODEL],
+    "getAllActiveSyncedModels reports what the provider's live sync returned; its consumers (/v1/models' synced loop, /api/models' exclusive-listing suppression, getSyncedAutoAliases) each handle custom rows on their own"
+  );
+  // The dispatch-time readers still see it — that is what #12597 fixed.
+  assert.equal(catalogContainsModel(await getActiveSyncedCatalog(PROVIDER), PICKER_MODEL), true);
+});
+
 test("#12597 without a custom row the picker id is still absent (lock the old contract)", async () => {
   const catalog = await getActiveSyncedCatalog(PROVIDER);
   assert.equal(catalogContainsModel(catalog, PICKER_MODEL), false);

tests/unit/hard-session-lease-bypass-inventory.test.ts

@@ -13,7 +13,12 @@ type BypassClass = "A" | "B" | "C";
 
 const EXPECTED: Record<InventoryKind, Record<string, number>> = {
   credential: {
-    "open-sse/handlers/chatCore.ts": 2,
+    // #12867 extracted chatCore.ts's streaming provider-execution loop into
+    // chatCore/providerExecutionPipeline.ts. Its two getProviderCredentials()
+    // sites (codex 429 rotation, antigravity BYOP rotation) moved with it and are
+    // now reached through the injected `connection.getProviderCredentials` handle,
+    // so countCalls() also inventories property-access calls.
+    "open-sse/handlers/chatCore/providerExecutionPipeline.ts": 2,
     "open-sse/services/imageCombo.ts": 1,
     "open-sse/services/speechCombo.ts": 1,
     "open-sse/services/videoCombo.ts": 2,
@@ -88,8 +93,9 @@ const EXPECTED: Record<InventoryKind, Record<string, number>> = {
     "open-sse/services/antigravityFamilyCooldown.ts": 1,
     // v3.8.50 back-merge additions (f95b03d7): combo routing infra and the
     // volcengine-plan binding/auto-sync services query connections the same
-    // way as their classified siblings.
-    "open-sse/services/combo.ts": 1,
+    // way as their classified siblings. #12746 split executeTarget out of
+    // combo.ts, moving this lookup into combo/executeTargetGates.ts unchanged.
+    "open-sse/services/combo/executeTargetGates.ts": 1,
     "open-sse/services/combo/providerWildcard.ts": 1,
     "open-sse/services/tokenRefresh.ts": 1,
     "src/lib/providers/volcPlanAutoSyncBackfill.ts": 1,
@@ -127,6 +133,11 @@ const EXPECTED: Record<InventoryKind, Record<string, number>> = {
     "src/app/api/translator/send/route.ts": 1,
     "src/app/api/translator/translate/route.ts": 1,
     "src/app/api/usage/call-logs/route.ts": 1,
+    // #12805: the reset-credit route resolves the connection's PROVIDER to pick
+    // the codex or grok-cli library; the exclusive-lease fence itself lives in
+    // those libraries (both listed in auxiliaryIsolationSources below). It never
+    // selects a connection to serve a request, so it stays class C.
+    "src/app/api/usage/codex-reset-credit/route.ts": 1,
     "src/app/api/usage/quota/route.ts": 1,
     "src/app/api/usage/utilization/route.ts": 1,
     "src/app/api/v1/vscode/[token]/api/tags/route.ts": 1,
@@ -174,6 +185,9 @@ const EXPECTED: Record<InventoryKind, Record<string, number>> = {
     "src/lib/usage/callLogs.ts": 1,
     "src/lib/usage/codexResetCredits.ts": 1,
     "src/lib/usage/comboScoringInspector.ts": 1,
+    // #12805: Grok Build sibling of codexResetCredits.ts — same auxiliary-activity
+    // fence in front of the same connection lookup, so same class B.
+    "src/lib/usage/grokResetCredits.ts": 1,
     "src/lib/usage/providerLimits.ts": 4,
     "src/lib/usage/resilienceExplain.ts": 1,
     "src/lib/usage/usageStats.ts": 1,
@@ -212,10 +226,9 @@ const CLASSIFICATION: Record<InventoryKind, Record<string, BypassClass>> = {
       [
         "open-sse/handlers/autoComboCandidates.ts",
         "open-sse/handlers/chatCore.ts",
-        "open-sse/services/combo.ts",
         "open-sse/services/alibabaFreeTier.ts",
         "open-sse/services/alibabaFreeTierQuotaFetcher.ts",
-        "open-sse/services/combo.ts",
+        "open-sse/services/combo/executeTargetGates.ts",
         "open-sse/services/combo/providerWildcard.ts",
         "open-sse/services/tokenRefresh.ts",
         "src/app/api/translator/send/route.ts",
@@ -224,6 +237,7 @@ const CLASSIFICATION: Record<InventoryKind, Record<string, BypassClass>> = {
         "src/lib/providers/volcenginePlanBinding.ts",
         "src/lib/services/quotaAutoPing.ts",
         "src/lib/usage/codexResetCredits.ts",
+        "src/lib/usage/grokResetCredits.ts",
         "src/lib/usage/providerLimits.ts",
         "src/lib/vncSession/service.ts",
         "src/lib/warmupScheduler.ts",
@@ -260,14 +274,13 @@ function countCalls(): Record<InventoryKind, Record<string, number>> {
       const key = file.split(path.sep).join("/");
       actual[kind][key] = (actual[kind][key] ?? 0) + 1;
     };
+    const isCredentialName = (name: string) =>
+      name === "getProviderCredentials" || name === "getProviderCredentialsWithQuotaPreflight";
     const visit = (node: ts.Node): void => {
       if (ts.isCallExpression(node)) {
         const expression = node.expression;
         if (ts.isIdentifier(expression)) {
-          if (
-            expression.text === "getProviderCredentials" ||
-            expression.text === "getProviderCredentialsWithQuotaPreflight"
-          ) {
+          if (isCredentialName(expression.text)) {
             increment("credential");
           }
           if (
@@ -276,15 +289,22 @@ function countCalls(): Record<InventoryKind, Record<string, number>> {
           ) {
             increment("connection");
           }
-        } else if (
-          ts.isPropertyAccessExpression(expression) &&
-          expression.name.text === "execute" &&
-          ts.isIdentifier(expression.expression) &&
-          ["executor", "fallbackExecutor", "providerExecutor", "streamExecutor"].includes(
-            expression.expression.text
-          )
-        ) {
-          increment("executor");
+        } else if (ts.isPropertyAccessExpression(expression)) {
+          // #12867: the extracted execution pipeline reaches the resolver through an
+          // injected handle (`connection.getProviderCredentials(...)`), so a
+          // bare-identifier scan alone would let those sites leave the inventory.
+          if (isCredentialName(expression.name.text)) {
+            increment("credential");
+          }
+          if (
+            expression.name.text === "execute" &&
+            ts.isIdentifier(expression.expression) &&
+            ["executor", "fallbackExecutor", "providerExecutor", "streamExecutor"].includes(
+              expression.expression.text
+            )
+          ) {
+            increment("executor");
+          }
         }
       }
       ts.forEachChild(node, visit);
@@ -312,6 +332,10 @@ test("managed request surfaces are fenced centrally or rejected before independe
     path.join(REPO_ROOT, "src/app/api/internal/codex-responses-ws/route.ts"),
     "utf8"
   );
+  const executionPipeline = fs.readFileSync(
+    path.join(REPO_ROOT, "open-sse/handlers/chatCore/providerExecutionPipeline.ts"),
+    "utf8"
+  );
   const internalKeys = fs.readFileSync(path.join(REPO_ROOT, "src/lib/db/apiKeys.ts"), "utf8");
   const auxiliaryIsolationSources = [
     "src/app/api/providers/[id]/models/route.ts",
@@ -320,6 +344,7 @@ test("managed request surfaces are fenced centrally or rejected before independe
     "src/lib/api/modelTestRunner.ts",
     "src/lib/services/quotaAutoPing.ts",
     "src/lib/usage/codexResetCredits.ts",
+    "src/lib/usage/grokResetCredits.ts",
     "src/lib/vncSession/service.ts",
     "src/lib/warmupScheduler.ts",
     "src/shared/services/modelSyncScheduler.ts",
@@ -336,7 +361,14 @@ test("managed request surfaces are fenced centrally or rejected before independe
     core,
     /assertManagedLeaseFence\(getExecutionConnectionId\(getExecutionCredentials\(\)\)\)/
   );
-  assert.match(core, /provider === "codex" &&\s*!managedLease/);
+  // #12867 moved the codex 429 account-rotation out of chatCore.ts into
+  // chatCore/providerExecutionPipeline.ts. The managed-lease fence moved with it:
+  // the inline `provider === "codex" && !managedLease` became the policy flag
+  // chatCore computes and the pipeline gates every rotation on. Assert both halves
+  // so the fence cannot be dropped on either side of that seam.
+  assert.match(core, /allowAccountRotation:\s*!managedLease\b/);
+  assert.match(executionPipeline, /canRotateAccount\s*=\s*policy\.allowAccountRotation\b/);
+  assert.match(executionPipeline, /canRotateAccount &&\s*target\.provider === "codex"/);
   assert.match(ws, /LEASE_UNSUPPORTED_TRANSPORT/);
   assert.match(internalKeys, /!k\.scopes\?\.includes\(EXCLUSIVE_LEASE_SCOPE\)/);
   for (const source of auxiliaryIsolationSources) {

tests/unit/models-catalog-route.test.ts

@@ -7,6 +7,14 @@ import path from "node:path";
 const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-model-catalog-"));
 process.env.DATA_DIR = TEST_DATA_DIR;
 process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "catalog-test-secret";
+// #12627 bounds a cold catalog build at 8s and, with no last-good response to fall
+// back on, surfaces `catalog_build_timeout` as an error body — no `data` array. That
+// bound is sized for a warm production process; a tsx-transpiled test runner building
+// the full 500+ model catalog from a fresh SQLite file on a loaded CI box crosses it
+// (10-13s observed), which turned the assertions below into a load-dependent flake.
+// Raise it here so these cases test catalog CONTENT; the timeout behavior itself is
+// covered by tests/unit/12627-catalog-inflight-timeout.test.ts.
+process.env.CATALOG_BUILD_TIMEOUT_MS = process.env.CATALOG_BUILD_TIMEOUT_MS || "120000";
 
 const core = await import("../../src/lib/db/core.ts");
 const providersDb = await import("../../src/lib/db/providers.ts");