Merge remote-tracking branch 'origin/release/v3.8.50' into babysit/pr-9618

Closes #9630

changelog.d/fixes/9630-combo-false-503.md

@@ -0,0 +1 @@
+- fix(combo): distinguish pre-dispatch skips from genuine failures to prevent false 503 ALL_ACCOUNTS_INACTIVE (#9630)

open-sse/services/antigravityProjectPersistence.ts

@@ -0,0 +1,13 @@
+/**
+ * Re-export from `antigravityProjectPersist.ts` plus a connection-preference helper.
+ */
+import { persistDiscoveredAntigravityProjectId } from "./antigravityProjectPersist.ts";
+export { persistDiscoveredAntigravityProjectId };
+
+export function preferAntigravityConnectionsWithStoredProject(
+  connections: Array<Record<string, unknown>>
+): Array<Record<string, unknown>> {
+  return connections.filter(
+    (conn) => conn != null && typeof conn.projectId === "string" && conn.projectId.trim().length > 0
+  );
+}

open-sse/services/combo.ts

@@ -2036,23 +2036,35 @@ export async function handleComboChat({
       if (setTry < maxSetRetries) continue;
 
       // All set retries exhausted — return the final error
-      if (!lastStatus) {
-        notifyWebhookEvent("request.failed", {
-          combo: combo.name,
-          reason: "ALL_ACCOUNTS_INACTIVE",
-          latencyMs,
-          fallbackCount,
-        });
-        // Silent-stop fix: bump the failure counter so the session pin clears on the 3rd
-        // consecutive all-inactive cascade; buildRecoveryHint emits `switch-combo` with a
-        // next-step that points the user at /dashboard/providers.
-        recordComboFailure(effectiveSessionId, combo.name);
-        return errorResponseWithComboDiagnostics(
-          503,
-          "Service temporarily unavailable: all upstream accounts are inactive",
-          buildComboDiag("all_accounts_inactive"),
-          { code: "ALL_ACCOUNTS_INACTIVE", type: "service_unavailable" }
-        );
+	      if (!lastStatus) {
+	        if (recordedAttempts === 0) {
+	          notifyWebhookEvent("request.failed", {
+	            combo: combo.name,
+	            reason: "ALL_TARGETS_SKIPPED",
+	            latencyMs,
+	            fallbackCount,
+	          });
+	          return errorResponseWithComboDiagnostics(
+	            503,
+	            "Service temporarily unavailable: all targets were skipped by pre-dispatch filters",
+	            buildComboDiag("all_targets_skipped"),
+	            { code: "ALL_TARGETS_SKIPPED", type: "service_unavailable" }
+	          );
+	        }
+	        notifyWebhookEvent("request.failed", {
+	          combo: combo.name,
+	          reason: "ALL_ACCOUNTS_INACTIVE",
+	          latencyMs,
+	          fallbackCount,
+	        });
+	        recordComboFailure(effectiveSessionId, combo.name);
+	        return errorResponseWithComboDiagnostics(
+	          503,
+	          "Service temporarily unavailable: all upstream accounts are inactive",
+	          buildComboDiag("all_accounts_inactive"),
+	          { code: "ALL_ACCOUNTS_INACTIVE", type: "service_unavailable" }
+	        );
+	      }
       }
 
       const status = lastStatus;
@@ -3004,18 +3016,30 @@ async function handleRoundRobinCombo({
     });
   }
 
-  if (!lastStatus) {
-    return new Response(
-      JSON.stringify({
-        error: {
-          message: "Service temporarily unavailable: all upstream accounts are inactive",
-          type: "service_unavailable",
-          code: "ALL_ACCOUNTS_INACTIVE",
-        },
-      }),
-      { status: 503, headers: { "Content-Type": "application/json" } }
-    );
-  }
+	  if (!lastStatus) {
+	    if (recordedAttempts === 0) {
+	      return new Response(
+	        JSON.stringify({
+	          error: {
+	            message: "Service temporarily unavailable: all targets were skipped by pre-dispatch filters",
+	            type: "service_unavailable",
+	            code: "ALL_TARGETS_SKIPPED",
+	          },
+	        }),
+	        { status: 503, headers: { "Content-Type": "application/json" } }
+	      );
+	    }
+	    return new Response(
+	      JSON.stringify({
+	        error: {
+	          message: "Service temporarily unavailable: all upstream accounts are inactive",
+	          type: "service_unavailable",
+	          code: "ALL_ACCOUNTS_INACTIVE",
+	        },
+	      }),
+	      { status: 503, headers: { "Content-Type": "application/json" } }
+	    );
+	  }
 
   const status = lastStatus;
   const msg = lastError || "All round-robin combo models unavailable";

src/lib/db/migrationRunner.ts

@@ -465,13 +465,13 @@ function isSchemaAlreadyApplied(
       // exists the rebuild ran — skip re-executing the rename/copy/drop, which
       // would fail on the missing proxy_assignments_pre117 table.
       return hasColumn(db, "proxy_assignments", "position");
-    // Retroactive guard for the 135/136 renumber (#8523 landed onto slots already taken
-    // by #8908/#9515): a DB that ran these under the old numbers already has the column,
-    // and a bare ALTER TABLE ADD COLUMN would throw on the re-run under the new number.
+    // Retroactive schema guards for migrations renumbered after release-branch collisions.
     case "137":
       return hasColumn(db, "version_manager", "auto_restart_adopted");
     case "138":
       return hasColumn(db, "upstream_proxy_config", "fallback_backend");
+    case "139":
+      return hasTable(db, "ccr_blocks");
     default:
       return false;
   }

src/lib/db/migrationRunner/constants.ts

@@ -69,6 +69,12 @@ export const RENAMED_MIGRATION_COMPATIBILITY = [
     toVersion: "059",
     toName: "manifest_routing",
   },
+  {
+    fromVersion: "134",
+    fromName: "ccr_blocks",
+    toVersion: "139",
+    toName: "ccr_blocks",
+  },
 ] as const;
 
 export const LEGACY_VERSION_SLOT_MIGRATIONS = [

tests/unit/db-ccr-migration-renumber-134.test.ts

@@ -0,0 +1,79 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import Database from "better-sqlite3";
+
+const migrationsDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-ccr-migration-"));
+const originalMigrationsDir = process.env.OMNIROUTE_MIGRATIONS_DIR;
+process.env.OMNIROUTE_MIGRATIONS_DIR = migrationsDir;
+
+fs.writeFileSync(
+  path.join(migrationsDir, "134_proxy_logs_egress_ip.sql"),
+  "ALTER TABLE proxy_logs ADD COLUMN egress_ip TEXT;"
+);
+fs.writeFileSync(
+  path.join(migrationsDir, "139_ccr_blocks.sql"),
+  "CREATE TABLE ccr_blocks (principal_id TEXT PRIMARY KEY);"
+);
+
+const { runMigrations } = await import("../../src/lib/db/migrationRunner.ts");
+
+function createLegacyDb(appliedName: string) {
+  const db = new Database(":memory:");
+  db.exec(`
+    CREATE TABLE proxy_logs (id TEXT PRIMARY KEY);
+    CREATE TABLE ccr_blocks (principal_id TEXT PRIMARY KEY);
+    CREATE TABLE _omniroute_migrations (
+      version TEXT PRIMARY KEY,
+      name TEXT NOT NULL,
+      applied_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+  `);
+  db.prepare("INSERT INTO _omniroute_migrations (version, name) VALUES (?, ?)").run(
+    "134",
+    appliedName
+  );
+  return db;
+}
+
+test.after(() => {
+  fs.rmSync(migrationsDir, { recursive: true, force: true });
+  if (originalMigrationsDir === undefined) delete process.env.OMNIROUTE_MIGRATIONS_DIR;
+  else process.env.OMNIROUTE_MIGRATIONS_DIR = originalMigrationsDir;
+});
+
+test("renumbered CCR migration frees 134 for proxy_logs on existing databases", () => {
+  const db = createLegacyDb("ccr_blocks");
+  try {
+    assert.equal(runMigrations(db), 1);
+    assert.deepEqual(
+      db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(),
+      [
+        { version: "134", name: "proxy_logs_egress_ip" },
+        { version: "139", name: "ccr_blocks" },
+      ]
+    );
+    const columns = db.prepare("PRAGMA table_info(proxy_logs)").all() as Array<{ name: string }>;
+    assert.ok(columns.some((column) => column.name === "egress_ip"));
+  } finally {
+    db.close();
+  }
+});
+
+test("renumbered CCR migration marks an existing table without recreating it", () => {
+  const db = createLegacyDb("proxy_logs_egress_ip");
+  try {
+    assert.equal(runMigrations(db), 1);
+    assert.deepEqual(
+      db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(),
+      [
+        { version: "134", name: "proxy_logs_egress_ip" },
+        { version: "139", name: "ccr_blocks" },
+      ]
+    );
+  } finally {
+    db.close();
+  }
+});

tests/unit/db-migrationrunner-constants-split.test.ts

@@ -70,8 +70,8 @@ describe("migrationRunner/constants — exact small-table snapshots", () => {
 // ── large tables — count + shape + spot-checks (corruption guard) ─────────────
 
 describe("migrationRunner/constants — large-table integrity", () => {
-  it("RENAMED_MIGRATION_COMPATIBILITY has 10 well-formed entries", () => {
-    assert.equal(RENAMED_MIGRATION_COMPATIBILITY.length, 10);
+  it("RENAMED_MIGRATION_COMPATIBILITY has 11 well-formed entries", () => {
+    assert.equal(RENAMED_MIGRATION_COMPATIBILITY.length, 11);
     for (const e of RENAMED_MIGRATION_COMPATIBILITY) {
       assert.equal(typeof e.fromVersion, "string");
       assert.equal(typeof e.fromName, "string");
@@ -91,6 +91,12 @@ describe("migrationRunner/constants — large-table integrity", () => {
     // both manifest_routing collisions (052→059 and 056→059) must survive
     const manifest = RENAMED_MIGRATION_COMPATIBILITY.filter((e) => e.toName === "manifest_routing");
     assert.deepEqual(manifest.map((e) => e.fromVersion).sort(), ["052", "056"]);
+    assert.deepEqual(RENAMED_MIGRATION_COMPATIBILITY.at(-1), {
+      fromVersion: "134",
+      fromName: "ccr_blocks",
+      toVersion: "139",
+      toName: "ccr_blocks",
+    });
   });
 
   it("PHYSICAL_SCHEMA_SENTINELS has 15 well-formed entries incl. the newest 064", () => {

tests/unit/repro-9630-combo-false-503.test.ts

@@ -0,0 +1,79 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import {
+  handleComboChat,
+} from "../../open-sse/services/combo.ts";
+import { getCircuitBreaker, STATE } from "../../src/shared/utils/circuitBreaker.js";
+
+function okResponse() {
+  return new Response(JSON.stringify({ ok: true }), {
+    status: 200,
+    headers: { "Content-Type": "application/json" },
+  });
+}
+
+test("#9630: combo returns 503 when circuit breaker is OPEN but other healthy targets exist", async () => {
+  const cb = getCircuitBreaker("openai");
+  cb.state = STATE.OPEN;
+  cb.resetTimeout = 60000;
+  cb.failureCount = 5;
+  cb.failureThreshold = 3;
+  cb.lastFailureTime = Date.now();
+
+  const result = await handleComboChat({
+    body: { messages: [{ role: "user", content: "hello" }] },
+    combo: {
+      name: "repro-9630",
+      strategy: "priority",
+      models: ["openai/gpt-4", "anthropic/claude-opus-5"],
+    },
+    handleSingleModel: async (_body: any, modelStr: string) => {
+      assert.equal(modelStr, "anthropic/claude-opus-5", "should skip openai breaker and try anthropic");
+      return okResponse();
+    },
+    isModelAvailable: async () => true,
+    log: { info: () => {}, warn: () => {}, debug: () => {}, error: () => {} } as any,
+    settings: null,
+    relayOptions: null as any,
+    allCombos: null,
+  });
+
+  assert.ok(result.ok, "should succeed via anthropic fallback when openai breaker is open");
+});
+
+test("#9630: combo returns truthful error, not false ALL_ACCOUNTS_INACTIVE, when ALL targets are breaker-open", async () => {
+  const cb = getCircuitBreaker("openai");
+  cb.state = STATE.OPEN;
+  cb.resetTimeout = 60000;
+  cb.failureCount = 5;
+  cb.failureThreshold = 3;
+  cb.lastFailureTime = Date.now();
+
+  const cb2 = getCircuitBreaker("anthropic");
+  cb2.state = STATE.OPEN;
+  cb2.resetTimeout = 60000;
+  cb2.failureCount = 5;
+  cb2.failureThreshold = 3;
+  cb2.lastFailureTime = Date.now();
+
+  const result = await handleComboChat({
+    body: { messages: [{ role: "user", content: "hello" }] },
+    combo: {
+      name: "repro-9630-all-breaker",
+      strategy: "priority",
+      models: ["openai/gpt-4", "anthropic/claude-opus-5"],
+    },
+    handleSingleModel: async () => { throw new Error("should not be called"); },
+    isModelAvailable: async () => true,
+    log: { info: () => {}, warn: () => {}, debug: () => {}, error: () => {} } as any,
+    settings: null,
+    relayOptions: null as any,
+    allCombos: null,
+  });
+
+  assert.equal(result.status, 503);
+  const body = await result.json();
+  // The diagnostic should NOT claim ALL_ACCOUNTS_INACTIVE when no real dispatch was attempted
+  assert.notEqual(body.error?.code, "ALL_ACCOUNTS_INACTIVE", 
+    "should not claim ALL_ACCOUNTS_INACTIVE when all targets were gated by pre-dispatch checks");
+});