Compare commits

..

3 Commits

Author SHA1 Message Date
diegosouzapw
272b3c4eaa Merge remote-tracking branch 'origin/release/v3.8.50' into babysit/pr-9618 2026-08-07 14:11:44 -03:00
Diego Rodrigues de Sa e Souza
976d670ff3 fix(combo): distinguish pre-dispatch skips from genuine failures to prevent false 503 ALL_ACCOUNTS_INACTIVE (#9630)
Closes #9630
2026-08-07 13:45:58 -03:00
fenix007
90948a9b0c fix(db): resolve ccr migration version collision
Renumber the CCR block-store migration from 134 to 139, reconcile databases that already applied the legacy slot, and add regression coverage for both upgrade paths.
2026-08-06 23:46:58 -03:00
13 changed files with 246 additions and 284 deletions

View File

@@ -1,11 +0,0 @@
- **feat(plugins):** add onStreamComplete built-in event exposing streaming usage and timing (#9571)
Adds a new `onStreamComplete` plugin event that fires after an SSE stream is fully
consumed, carrying usage token counts and timing metrics (latency, TTFT). Built-in
events now include `onStreamComplete` as a fire-and-forget lifecycle hook.
Payload: `status`, `usage` (prompt_tokens, completion_tokens, reasoning_tokens,
cache_read_input_tokens, cache_creation_input_tokens), `timing` (latencyMs, ttft),
`model`, `provider`, `errorCode`.
Non-breaking — existing `onResponse` hooks with `{ streamed: true }` remain unchanged.

View File

@@ -0,0 +1 @@
- fix(combo): distinguish pre-dispatch skips from genuine failures to prevent false 503 ALL_ACCOUNTS_INACTIVE (#9630)

View File

@@ -245,10 +245,7 @@ import { recordCompressionCacheStats } from "./chatCore/compressionCacheStats.ts
import { writeCavemanOutputAnalytics } from "./chatCore/cavemanOutputAnalytics.ts";
import { scheduleQuotaShareConsumption } from "./chatCore/quotaShareConsumption.ts";
import { emitRequestGamificationEvent } from "./chatCore/gamificationEvent.ts";
import {
runPluginOnResponseHook,
runPluginOnStreamCompleteHook,
} from "./chatCore/pluginOnResponse.ts";
import { runPluginOnResponseHook } from "./chatCore/pluginOnResponse.ts";
import { scheduleStreamingQuotaShareConsumption } from "./chatCore/streamingQuotaShare.ts";
import { recordStreamingUsageStats } from "./chatCore/streamingUsageStats.ts";
import { recordStreamingCost } from "./chatCore/streamingCost.ts";
@@ -4898,17 +4895,6 @@ export async function handleChatCore({
streamUsage,
log,
});
// Plugin onStreamComplete hook — fire-and-forget, fail-open (#9571)
runPluginOnStreamCompleteHook({
status: normalizedStreamStatus,
usage: streamUsage as Record<string, unknown> | undefined,
ttft,
model,
provider,
errorCode: streamErrorCode,
startTime,
});
};
const streamFailureFinalizers = streamFailure.createStreamFailureFinalizers({

View File

@@ -43,57 +43,3 @@ export async function runPluginOnResponseHook(args: {
/* plugin onResponse optional */
}
}
/**
* Payload passed to plugin onStreamComplete hooks after a streaming response is consumed.
* Carries usage token counts, timing metrics (latency, TTFT), model, provider, and error code.
*/
export type PluginOnStreamCompletePayload = {
status: number;
usage?: {
prompt_tokens?: number;
completion_tokens?: number;
reasoning_tokens?: number;
cache_read_input_tokens?: number;
cache_creation_input_tokens?: number;
};
timing?: {
latencyMs: number;
ttft?: number;
};
model?: string;
provider?: string;
errorCode?: string;
};
/**
* Run plugin onStreamComplete hooks — fire-and-forget and fail-open.
* Called inside the onStreamComplete callback (chatCore.ts) where usage and timing data
* converge after an SSE stream is fully consumed.
*/
export async function runPluginOnStreamCompleteHook(args: {
status: number;
usage?: Record<string, unknown>;
ttft?: number;
model: string | null | undefined;
provider: string | null | undefined;
errorCode?: string | null | undefined;
startTime: number;
}): Promise<void> {
try {
const { runOnStreamComplete } = await import("@/lib/plugins/hooks");
runOnStreamComplete({
status: args.status,
usage: args.usage as PluginOnStreamCompletePayload["usage"],
timing: {
latencyMs: Date.now() - args.startTime,
ttft: args.ttft,
},
model: args.model ?? undefined,
provider: args.provider ?? undefined,
errorCode: args.errorCode ?? undefined,
}).catch(() => {});
} catch (_) {
/* plugin onStreamComplete optional */
}
}

View File

@@ -0,0 +1,13 @@
/**
* Re-export from `antigravityProjectPersist.ts` plus a connection-preference helper.
*/
import { persistDiscoveredAntigravityProjectId } from "./antigravityProjectPersist.ts";
export { persistDiscoveredAntigravityProjectId };
export function preferAntigravityConnectionsWithStoredProject(
connections: Array<Record<string, unknown>>
): Array<Record<string, unknown>> {
return connections.filter(
(conn) => conn != null && typeof conn.projectId === "string" && conn.projectId.trim().length > 0
);
}

View File

@@ -2036,23 +2036,35 @@ export async function handleComboChat({
if (setTry < maxSetRetries) continue;
// All set retries exhausted — return the final error
if (!lastStatus) {
notifyWebhookEvent("request.failed", {
combo: combo.name,
reason: "ALL_ACCOUNTS_INACTIVE",
latencyMs,
fallbackCount,
});
// Silent-stop fix: bump the failure counter so the session pin clears on the 3rd
// consecutive all-inactive cascade; buildRecoveryHint emits `switch-combo` with a
// next-step that points the user at /dashboard/providers.
recordComboFailure(effectiveSessionId, combo.name);
return errorResponseWithComboDiagnostics(
503,
"Service temporarily unavailable: all upstream accounts are inactive",
buildComboDiag("all_accounts_inactive"),
{ code: "ALL_ACCOUNTS_INACTIVE", type: "service_unavailable" }
);
if (!lastStatus) {
if (recordedAttempts === 0) {
notifyWebhookEvent("request.failed", {
combo: combo.name,
reason: "ALL_TARGETS_SKIPPED",
latencyMs,
fallbackCount,
});
return errorResponseWithComboDiagnostics(
503,
"Service temporarily unavailable: all targets were skipped by pre-dispatch filters",
buildComboDiag("all_targets_skipped"),
{ code: "ALL_TARGETS_SKIPPED", type: "service_unavailable" }
);
}
notifyWebhookEvent("request.failed", {
combo: combo.name,
reason: "ALL_ACCOUNTS_INACTIVE",
latencyMs,
fallbackCount,
});
recordComboFailure(effectiveSessionId, combo.name);
return errorResponseWithComboDiagnostics(
503,
"Service temporarily unavailable: all upstream accounts are inactive",
buildComboDiag("all_accounts_inactive"),
{ code: "ALL_ACCOUNTS_INACTIVE", type: "service_unavailable" }
);
}
}
const status = lastStatus;
@@ -3004,18 +3016,30 @@ async function handleRoundRobinCombo({
});
}
if (!lastStatus) {
return new Response(
JSON.stringify({
error: {
message: "Service temporarily unavailable: all upstream accounts are inactive",
type: "service_unavailable",
code: "ALL_ACCOUNTS_INACTIVE",
},
}),
{ status: 503, headers: { "Content-Type": "application/json" } }
);
}
if (!lastStatus) {
if (recordedAttempts === 0) {
return new Response(
JSON.stringify({
error: {
message: "Service temporarily unavailable: all targets were skipped by pre-dispatch filters",
type: "service_unavailable",
code: "ALL_TARGETS_SKIPPED",
},
}),
{ status: 503, headers: { "Content-Type": "application/json" } }
);
}
return new Response(
JSON.stringify({
error: {
message: "Service temporarily unavailable: all upstream accounts are inactive",
type: "service_unavailable",
code: "ALL_ACCOUNTS_INACTIVE",
},
}),
{ status: 503, headers: { "Content-Type": "application/json" } }
);
}
const status = lastStatus;
const msg = lastError || "All round-robin combo models unavailable";

View File

@@ -465,13 +465,13 @@ function isSchemaAlreadyApplied(
// exists the rebuild ran — skip re-executing the rename/copy/drop, which
// would fail on the missing proxy_assignments_pre117 table.
return hasColumn(db, "proxy_assignments", "position");
// Retroactive guard for the 135/136 renumber (#8523 landed onto slots already taken
// by #8908/#9515): a DB that ran these under the old numbers already has the column,
// and a bare ALTER TABLE ADD COLUMN would throw on the re-run under the new number.
// Retroactive schema guards for migrations renumbered after release-branch collisions.
case "137":
return hasColumn(db, "version_manager", "auto_restart_adopted");
case "138":
return hasColumn(db, "upstream_proxy_config", "fallback_backend");
case "139":
return hasTable(db, "ccr_blocks");
default:
return false;
}

View File

@@ -69,6 +69,12 @@ export const RENAMED_MIGRATION_COMPATIBILITY = [
toVersion: "059",
toName: "manifest_routing",
},
{
fromVersion: "134",
fromName: "ccr_blocks",
toVersion: "139",
toName: "ccr_blocks",
},
] as const;
export const LEGACY_VERSION_SLOT_MIGRATIONS = [

View File

@@ -40,7 +40,6 @@ export const BUILTIN_EVENTS = [
"onActivate",
"onDeactivate",
"onUninstall",
"onStreamComplete",
] as const;
export type BuiltinEvent = (typeof BUILTIN_EVENTS)[number];
@@ -252,35 +251,6 @@ export interface Plugin {
onActivate?: (payload: unknown) => Promise<void> | void;
onDeactivate?: (payload: unknown) => Promise<void> | void;
onUninstall?: (payload: unknown) => Promise<void> | void;
onStreamComplete?: (payload: PluginOnStreamCompletePayload) => Promise<void> | void;
}
// ── onStreamComplete event types ──
export type PluginOnStreamCompletePayload = {
status: number;
usage?: {
prompt_tokens?: number;
completion_tokens?: number;
reasoning_tokens?: number;
cache_read_input_tokens?: number;
cache_creation_input_tokens?: number;
};
timing?: {
latencyMs: number;
ttft?: number;
};
model?: string;
provider?: string;
errorCode?: string;
};
/**
* Run onStreamComplete hooks — fire-and-forget notification with usage/timing data.
* Called when an SSE stream is fully consumed and usage/timing data is available.
*/
export async function runOnStreamComplete(payload: PluginOnStreamCompletePayload): Promise<void> {
await emitHook("onStreamComplete", payload);
}
/**

View File

@@ -7,8 +7,9 @@ import { test, after } from "node:test";
import assert from "node:assert/strict";
const { registerHook, unregisterHook } = await import("../../src/lib/plugins/hooks.ts");
const { runPluginOnResponseHook, runPluginOnStreamCompleteHook } =
await import("../../open-sse/handlers/chatCore/pluginOnResponse.ts");
const { runPluginOnResponseHook } = await import(
"../../open-sse/handlers/chatCore/pluginOnResponse.ts"
);
async function waitFor(pred: () => boolean, timeoutMs = 2000): Promise<void> {
const deadline = Date.now() + timeoutMs;
@@ -19,7 +20,6 @@ async function waitFor(pred: () => boolean, timeoutMs = 2000): Promise<void> {
after(() => {
unregisterHook("onResponse", "test-onresponse-plugin");
unregisterHook("onStreamComplete", "test-onstreamcomplete-plugin");
});
test("no registered hooks → resolves without throwing (no-op)", async () => {
@@ -101,140 +101,3 @@ test("a throwing hook never rejects the caller (fail-open)", async () => {
);
await new Promise((r) => setTimeout(r, 30));
});
// ── onStreamComplete hook tests (#9571) ──
test("onStreamComplete: no registered hooks resolves without throwing (no-op)", async () => {
const start = Date.now();
await assert.doesNotReject(
runPluginOnStreamCompleteHook({
status: 200,
usage: { prompt_tokens: 10, completion_tokens: 20 },
ttft: 150,
model: "gpt-4",
provider: "openai",
errorCode: undefined,
startTime: start - 500,
})
);
});
test("onStreamComplete: registered hook receives usage + timing payload", async () => {
let captured: Record<string, unknown> | undefined;
registerHook(
"onStreamComplete",
"test-onstreamcomplete-plugin",
async (payload: Record<string, unknown>) => {
captured = payload;
}
);
const startTime = Date.now() - 500;
await runPluginOnStreamCompleteHook({
status: 200,
usage: { prompt_tokens: 42, completion_tokens: 100, reasoning_tokens: 5 },
ttft: 200,
model: "claude-3-opus",
provider: "anthropic",
errorCode: undefined,
startTime,
});
await waitFor(() => captured !== undefined);
assert.ok(captured, "expected onStreamComplete hook to be invoked");
// payload shape: status, usage, timing, model, provider
assert.equal(captured!.status, 200);
assert.ok(captured!.usage, "usage should be present");
assert.equal((captured!.usage as Record<string, number>).prompt_tokens, 42);
assert.equal((captured!.usage as Record<string, number>).completion_tokens, 100);
assert.equal((captured!.usage as Record<string, number>).reasoning_tokens, 5);
assert.ok(captured!.timing, "timing should be present");
const timing = captured!.timing as Record<string, number>;
assert.equal(timing.ttft, 200);
assert.ok(timing.latencyMs > 450, "latencyMs should be near 500");
assert.equal(captured!.model, "claude-3-opus");
assert.equal(captured!.provider, "anthropic");
assert.equal(captured!.errorCode, undefined);
});
test("onStreamComplete: payload includes cache token fields when present", async () => {
let captured: Record<string, unknown> | undefined;
registerHook(
"onStreamComplete",
"test-onstreamcomplete-plugin",
async (payload: Record<string, unknown>) => {
captured = payload;
}
);
await runPluginOnStreamCompleteHook({
status: 200,
usage: {
prompt_tokens: 50,
completion_tokens: 30,
cache_read_input_tokens: 20,
cache_creation_input_tokens: 10,
},
ttft: 100,
model: "gpt-4",
provider: "openai",
errorCode: undefined,
startTime: Date.now(),
});
await waitFor(() => captured !== undefined);
assert.ok(captured);
const usage = captured!.usage as Record<string, number>;
assert.equal(usage.cache_read_input_tokens, 20);
assert.equal(usage.cache_creation_input_tokens, 10);
});
test("onStreamComplete: throwing hook never rejects the caller (fail-open)", async () => {
registerHook("onStreamComplete", "test-onstreamcomplete-plugin", async () => {
throw new Error("stream-complete-boom");
});
await assert.doesNotReject(
runPluginOnStreamCompleteHook({
status: 500,
usage: undefined,
ttft: undefined,
model: "gpt-4",
provider: "openai",
errorCode: "upstream_error",
startTime: Date.now(),
})
);
await new Promise((r) => setTimeout(r, 30));
});
test("onStreamComplete: errorCode is passed through when provided", async () => {
let captured: Record<string, unknown> | undefined;
registerHook(
"onStreamComplete",
"test-onstreamcomplete-plugin",
async (payload: Record<string, unknown>) => {
captured = payload;
}
);
await runPluginOnStreamCompleteHook({
status: 502,
usage: undefined,
ttft: undefined,
model: "grok-3",
provider: "xai",
errorCode: "upstream_timeout",
startTime: Date.now(),
});
await waitFor(() => captured !== undefined);
assert.ok(captured);
assert.equal(captured!.status, 502);
assert.equal(captured!.errorCode, "upstream_timeout");
assert.equal(captured!.model, "grok-3");
assert.equal(captured!.provider, "xai");
});

View File

@@ -0,0 +1,79 @@
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import Database from "better-sqlite3";
const migrationsDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-ccr-migration-"));
const originalMigrationsDir = process.env.OMNIROUTE_MIGRATIONS_DIR;
process.env.OMNIROUTE_MIGRATIONS_DIR = migrationsDir;
fs.writeFileSync(
path.join(migrationsDir, "134_proxy_logs_egress_ip.sql"),
"ALTER TABLE proxy_logs ADD COLUMN egress_ip TEXT;"
);
fs.writeFileSync(
path.join(migrationsDir, "139_ccr_blocks.sql"),
"CREATE TABLE ccr_blocks (principal_id TEXT PRIMARY KEY);"
);
const { runMigrations } = await import("../../src/lib/db/migrationRunner.ts");
function createLegacyDb(appliedName: string) {
const db = new Database(":memory:");
db.exec(`
CREATE TABLE proxy_logs (id TEXT PRIMARY KEY);
CREATE TABLE ccr_blocks (principal_id TEXT PRIMARY KEY);
CREATE TABLE _omniroute_migrations (
version TEXT PRIMARY KEY,
name TEXT NOT NULL,
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
);
`);
db.prepare("INSERT INTO _omniroute_migrations (version, name) VALUES (?, ?)").run(
"134",
appliedName
);
return db;
}
test.after(() => {
fs.rmSync(migrationsDir, { recursive: true, force: true });
if (originalMigrationsDir === undefined) delete process.env.OMNIROUTE_MIGRATIONS_DIR;
else process.env.OMNIROUTE_MIGRATIONS_DIR = originalMigrationsDir;
});
test("renumbered CCR migration frees 134 for proxy_logs on existing databases", () => {
const db = createLegacyDb("ccr_blocks");
try {
assert.equal(runMigrations(db), 1);
assert.deepEqual(
db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(),
[
{ version: "134", name: "proxy_logs_egress_ip" },
{ version: "139", name: "ccr_blocks" },
]
);
const columns = db.prepare("PRAGMA table_info(proxy_logs)").all() as Array<{ name: string }>;
assert.ok(columns.some((column) => column.name === "egress_ip"));
} finally {
db.close();
}
});
test("renumbered CCR migration marks an existing table without recreating it", () => {
const db = createLegacyDb("proxy_logs_egress_ip");
try {
assert.equal(runMigrations(db), 1);
assert.deepEqual(
db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(),
[
{ version: "134", name: "proxy_logs_egress_ip" },
{ version: "139", name: "ccr_blocks" },
]
);
} finally {
db.close();
}
});

View File

@@ -70,8 +70,8 @@ describe("migrationRunner/constants — exact small-table snapshots", () => {
// ── large tables — count + shape + spot-checks (corruption guard) ─────────────
describe("migrationRunner/constants — large-table integrity", () => {
it("RENAMED_MIGRATION_COMPATIBILITY has 10 well-formed entries", () => {
assert.equal(RENAMED_MIGRATION_COMPATIBILITY.length, 10);
it("RENAMED_MIGRATION_COMPATIBILITY has 11 well-formed entries", () => {
assert.equal(RENAMED_MIGRATION_COMPATIBILITY.length, 11);
for (const e of RENAMED_MIGRATION_COMPATIBILITY) {
assert.equal(typeof e.fromVersion, "string");
assert.equal(typeof e.fromName, "string");
@@ -91,6 +91,12 @@ describe("migrationRunner/constants — large-table integrity", () => {
// both manifest_routing collisions (052→059 and 056→059) must survive
const manifest = RENAMED_MIGRATION_COMPATIBILITY.filter((e) => e.toName === "manifest_routing");
assert.deepEqual(manifest.map((e) => e.fromVersion).sort(), ["052", "056"]);
assert.deepEqual(RENAMED_MIGRATION_COMPATIBILITY.at(-1), {
fromVersion: "134",
fromName: "ccr_blocks",
toVersion: "139",
toName: "ccr_blocks",
});
});
it("PHYSICAL_SCHEMA_SENTINELS has 15 well-formed entries incl. the newest 064", () => {

View File

@@ -0,0 +1,79 @@
import test from "node:test";
import assert from "node:assert/strict";
import {
handleComboChat,
} from "../../open-sse/services/combo.ts";
import { getCircuitBreaker, STATE } from "../../src/shared/utils/circuitBreaker.js";
function okResponse() {
return new Response(JSON.stringify({ ok: true }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
test("#9630: combo returns 503 when circuit breaker is OPEN but other healthy targets exist", async () => {
const cb = getCircuitBreaker("openai");
cb.state = STATE.OPEN;
cb.resetTimeout = 60000;
cb.failureCount = 5;
cb.failureThreshold = 3;
cb.lastFailureTime = Date.now();
const result = await handleComboChat({
body: { messages: [{ role: "user", content: "hello" }] },
combo: {
name: "repro-9630",
strategy: "priority",
models: ["openai/gpt-4", "anthropic/claude-opus-5"],
},
handleSingleModel: async (_body: any, modelStr: string) => {
assert.equal(modelStr, "anthropic/claude-opus-5", "should skip openai breaker and try anthropic");
return okResponse();
},
isModelAvailable: async () => true,
log: { info: () => {}, warn: () => {}, debug: () => {}, error: () => {} } as any,
settings: null,
relayOptions: null as any,
allCombos: null,
});
assert.ok(result.ok, "should succeed via anthropic fallback when openai breaker is open");
});
test("#9630: combo returns truthful error, not false ALL_ACCOUNTS_INACTIVE, when ALL targets are breaker-open", async () => {
const cb = getCircuitBreaker("openai");
cb.state = STATE.OPEN;
cb.resetTimeout = 60000;
cb.failureCount = 5;
cb.failureThreshold = 3;
cb.lastFailureTime = Date.now();
const cb2 = getCircuitBreaker("anthropic");
cb2.state = STATE.OPEN;
cb2.resetTimeout = 60000;
cb2.failureCount = 5;
cb2.failureThreshold = 3;
cb2.lastFailureTime = Date.now();
const result = await handleComboChat({
body: { messages: [{ role: "user", content: "hello" }] },
combo: {
name: "repro-9630-all-breaker",
strategy: "priority",
models: ["openai/gpt-4", "anthropic/claude-opus-5"],
},
handleSingleModel: async () => { throw new Error("should not be called"); },
isModelAvailable: async () => true,
log: { info: () => {}, warn: () => {}, debug: () => {}, error: () => {} } as any,
settings: null,
relayOptions: null as any,
allCombos: null,
});
assert.equal(result.status, 503);
const body = await result.json();
// The diagnostic should NOT claim ALL_ACCOUNTS_INACTIVE when no real dispatch was attempted
assert.notEqual(body.error?.code, "ALL_ACCOUNTS_INACTIVE",
"should not claim ALL_ACCOUNTS_INACTIVE when all targets were gated by pre-dispatch checks");
});