pjandro/OmniRoute
1
0
Fork 0
mirror of https://github.com/diegosouzapw/OmniRoute.git synced 2026-07-26 09:52:11 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(lockout): classify Gemini Antigravity resource exhaustion as quota_exhausted

Browse Source
This commit is contained in:
Dmitry Kuznetsov
2026-05-28 08:41:36 +03:00
parent 1b6deb815a
commit c791af7db3
11 changed files with 519 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
open-sse/handlers/responseSanitizer.ts
View File

@@ -46,6 +46,56 @@ function toNumber(value: unknown): number | undefined {
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
}
function stripZeroWidthText(value: string): string {
return value.replace(/[\u200B-\u200D\uFEFF]/g, "");
}
function stripZeroWidthValue(value: unknown): unknown {
if (typeof value === "string") return stripZeroWidthText(value);
if (Array.isArray(value)) return value.map((item) => stripZeroWidthValue(item));
const record = toRecord(value);
if (record) {
return Object.fromEntries(
Object.entries(record).map(([key, item]) => [key, stripZeroWidthValue(item)])
);
}
return value;
}
function parseTextualToolCallContent(content: unknown): { name: string; args: unknown } | null {
if (typeof content !== "string") return null;
const normalized = stripZeroWidthText(content);
const toolCallIndex = normalized.lastIndexOf("[Tool call:");
if (toolCallIndex < 0) return null;
const candidate = normalized.slice(toolCallIndex);
const headerMatch = candidate.match(/^\[Tool call:\s*([^\]\n]+)\]\s*\nArguments:\s*/);
if (!headerMatch) return null;
const name = headerMatch[1]?.trim();
const rawArgs = candidate.slice(headerMatch[0].length).trim();
if (!name || !rawArgs) return null;
const decoders = [
(value: string) => value,
(value: string) => {
if (value.startsWith('"') && value.endsWith('"')) {
const decoded = JSON.parse(value);
return typeof decoded === "string" ? decoded : value;
}
return value;
},
];
for (const decode of decoders) {
try {
const decoded = decode(rawArgs);
return { name, args: stripZeroWidthValue(JSON.parse(decoded)) };
} catch {}
}
return null;
}
function containsTextualToolCallContent(content: unknown): boolean {
return typeof content === "string" && stripZeroWidthText(content).includes("[Tool call:");
}
function hasVisibleMessageContent(content: unknown): boolean {
if (typeof content === "string") {
return content.trim().length > 0;
@@ -127,9 +177,19 @@ export function sanitizeOpenAIResponse(body: unknown): unknown {
// Sanitize choices
if (Array.isArray(bodyRecord.choices)) {
sanitized.choices = bodyRecord.choices.map((choice, idx) =>
sanitizeChoice(choice, idx, isDeepSeekV4)
);
sanitized.choices = bodyRecord.choices.map((choice, idx) => {
const sanitizedChoice = sanitizeChoice(choice, idx, isDeepSeekV4);
const message = toRecord(sanitizedChoice.message);
if (
message &&
Array.isArray(message.tool_calls) &&
message.tool_calls.length > 0 &&
sanitizedChoice.finish_reason !== "tool_calls"
) {
sanitizedChoice.finish_reason = "tool_calls";
}
return sanitizedChoice;
});
} else {
sanitized.choices = [];
}
@@ -304,6 +364,23 @@ function sanitizeMessage(msg: unknown, isDeepSeekV4 = false): unknown {
delete sanitized.reasoning_content;
}
const textualToolCall = parseTextualToolCallContent(sanitized.content);
if (textualToolCall && !msgRecord.tool_calls) {
sanitized.content = null;
sanitized.tool_calls = [
{
id: `call_${Date.now()}_0`,
type: "function",
function: {
name: textualToolCall.name,
arguments: JSON.stringify(textualToolCall.args || {}),
},
},
];
} else if (containsTextualToolCallContent(sanitized.content) && !msgRecord.tool_calls) {
sanitized.content = null;
}
// Preserve tool_calls
if (msgRecord.tool_calls) {
sanitized.tool_calls = msgRecord.tool_calls;

37
open-sse/handlers/responseTranslator.ts
View File

@@ -34,6 +34,17 @@ function firstPositiveNumber(...values: unknown[]): number {
return 0;
}
function normalizeToolCallArgs(args: unknown): unknown {
if (typeof args !== "string") return args;
const trimmed = args.trim();
if (!trimmed || !(trimmed.startsWith("{") || trimmed.startsWith("["))) return args;
try {
return JSON.parse(trimmed);
} catch {
return args;
}
}
function parseTextualToolCall(text: unknown): { name: string; args: unknown } | null {
if (typeof text !== "string") return null;
@@ -50,10 +61,24 @@ function parseTextualToolCall(text: unknown): { name: string; args: unknown } |
const rawArgs = match[2]?.trim();
if (!name || !rawArgs) return null;
try {
return { name, args: JSON.parse(rawArgs) };
} catch {
return null;
}
let args = JSON.parse(rawArgs);
if (typeof args === "string") {
const trimmed = args.trim();
if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
args = JSON.parse(trimmed);
}
}
if (args && typeof args === "object" && !Array.isArray(args)) {
return { name, args };
}
} catch {}
return null;
}
function containsTextualToolCallMarker(text: unknown): boolean {
return (
typeof text === "string" && text.replace(/[\u200B-\u200D\uFEFF]/g, "").includes("[Tool call:")
);
}
function extractMessageOutputText(item: JsonRecord): string {
@@ -335,7 +360,7 @@ export function translateNonStreamingResponse(
arguments: JSON.stringify(textualToolCall.args || {}),
},
});
} else {
} else if (!containsTextualToolCallMarker(partObj.text)) {
textContent += partObj.text;
contentParts.push({ type: "text", text: partObj.text });
}
@@ -378,7 +403,7 @@ export function translateNonStreamingResponse(
type: "function",
function: {
name: restoredName,
arguments: JSON.stringify(fn.args || {}),
arguments: JSON.stringify(normalizeToolCallArgs(fn.args || {})),
},
});
}

3
open-sse/services/accountFallback.ts
View File

@@ -109,6 +109,9 @@ export const CREDITS_EXHAUSTED_SIGNALS = [
"credits exhausted",
"out of credits",
"payment required",
"resource has been exhausted",
"resource_exhausted",
"check quota",
];
// T11: Signals that indicate OAuth token is invalid/expired (not permanent deactivation)

43
open-sse/translator/response/gemini-to-openai.ts
View File

@@ -23,6 +23,17 @@ type GeminiFunctionCallPart = {
};
};
function normalizeToolCallArgs(args: unknown): unknown {
if (typeof args !== "string") return args;
const trimmed = args.trim();
if (!trimmed || !(trimmed.startsWith("{") || trimmed.startsWith("["))) return args;
try {
return JSON.parse(trimmed);
} catch {
return args;
}
}
function parseTextualToolCall(text: unknown): { name: string; args: unknown } | null {
if (typeof text !== "string") return null;
@@ -39,10 +50,24 @@ function parseTextualToolCall(text: unknown): { name: string; args: unknown } |
const rawArgs = match[2]?.trim();
if (!name || !rawArgs) return null;
try {
return { name, args: JSON.parse(rawArgs) };
} catch {
return null;
}
let args = JSON.parse(rawArgs);
if (typeof args === "string") {
const trimmed = args.trim();
if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
args = JSON.parse(trimmed);
}
}
if (args && typeof args === "object" && !Array.isArray(args)) {
return { name, args };
}
} catch {}
return null;
}
function containsTextualToolCallMarker(text: unknown): boolean {
return (
typeof text === "string" && text.replace(/[\u200B-\u200D\uFEFF]/g, "").includes("[Tool call:")
);
}
function buildToolCallId(
@@ -69,7 +94,7 @@ function emitFunctionCallPart(
) {
const rawToolName = part.functionCall.name;
const fcName = state.toolNameMap?.get(rawToolName) || rawToolName;
const fcArgs = part.functionCall.args || {};
const fcArgs = normalizeToolCallArgs(part.functionCall.args || {});
const toolCallIndex = state.functionIndex++;
const toolCall = {
id: buildToolCallId(part.functionCall, fcName, toolCallIndex),
@@ -243,6 +268,14 @@ export function geminiToOpenAIResponse(chunk, state) {
continue;
}
// Never leak a malformed textual pseudo tool-call to clients. If the
// model emits the marker but the arguments are not parseable yet/at all,
// suppress the text; the final finish reason remains `stop` unless a
// structured tool call was emitted elsewhere.
if (containsTextualToolCallMarker(part.text)) {
continue;
}
results.push({
id: `chatcmpl-${state.messageId}`,
object: "chat.completion.chunk",

111
open-sse/utils/stream.ts
View File

@@ -840,6 +840,64 @@ export function createSSEStream(options: StreamOptions = {}) {
return output;
};
const applyTextualToolCallStreamingGuard = (parsed: Record<string, unknown>) => {
const choice = Array.isArray((parsed as JsonRecord).choices)
? (((parsed as JsonRecord).choices as unknown[])[0] as JsonRecord | undefined)
: undefined;
const delta = asRecord(choice?.delta);
let textualToolCallConverted = false;
if (typeof delta?.content === "string") {
const incomingContent = delta.content;
const bufferedCandidate = passthroughBufferedTextualToolCallContent + incomingContent;
if (
passthroughBufferedTextualToolCallContent ||
containsTextualToolCallCandidate(incomingContent)
) {
const parsedCandidate = parseTextualToolCallCandidate(bufferedCandidate);
if (parsedCandidate?.kind === "complete") {
const collectedToolCall = collectPassthroughTextualToolCall(
bufferedCandidate,
passthroughToolCalls,
allowedToolNames
);
if (collectedToolCall) {
parsed = toChatCompletionChunkWithToolCall(
parsed as JsonRecord,
collectedToolCall
) as typeof parsed;
passthroughHasToolCalls = true;
} else {
delete delta.content;
delete delta.reasoning_content;
}
textualToolCallConverted = true;
passthroughBufferedTextualToolCallContent = "";
} else if (parsedCandidate?.kind === "partial") {
passthroughBufferedTextualToolCallContent = appendBoundedText(
passthroughBufferedTextualToolCallContent,
incomingContent
);
textualToolCallConverted = true;
delta.content = "";
} else {
passthroughAccumulatedContent = appendBoundedText(
passthroughAccumulatedContent,
passthroughBufferedTextualToolCallContent + incomingContent
);
passthroughBufferedTextualToolCallContent = "";
}
} else {
passthroughAccumulatedContent = appendBoundedText(
passthroughAccumulatedContent,
incomingContent
);
}
}
return { parsed, textualToolCallConverted };
};
const emitSyntheticClaudeEmptyResponse = (
controller: TransformStreamDefaultController,
options: {
@@ -1536,53 +1594,12 @@ export function createSSEStream(options: StreamOptions = {}) {
if (content && typeof content === "string") {
totalContentLength += content.length;
}
if (typeof delta?.content === "string") {
const incomingContent = delta.content;
const bufferedCandidate =
passthroughBufferedTextualToolCallContent + incomingContent;
if (
passthroughBufferedTextualToolCallContent ||
containsTextualToolCallCandidate(incomingContent)
) {
const parsedCandidate = parseTextualToolCallCandidate(bufferedCandidate);
if (parsedCandidate?.kind === "complete") {
const collectedToolCall = collectPassthroughTextualToolCall(
bufferedCandidate,
passthroughToolCalls,
allowedToolNames
);
if (collectedToolCall) {
parsed = toChatCompletionChunkWithToolCall(
parsed as JsonRecord,
collectedToolCall
) as typeof parsed;
passthroughHasToolCalls = true;
} else {
delete delta.content;
delete delta.reasoning_content;
}
textualToolCallConverted = true;
passthroughBufferedTextualToolCallContent = "";
} else if (parsedCandidate?.kind === "partial") {
passthroughBufferedTextualToolCallContent = appendBoundedText(
passthroughBufferedTextualToolCallContent,
incomingContent
);
textualToolCallConverted = true;
delta.content = "";
} else {
passthroughAccumulatedContent = appendBoundedText(
passthroughAccumulatedContent,
passthroughBufferedTextualToolCallContent + incomingContent
);
passthroughBufferedTextualToolCallContent = "";
}
} else {
passthroughAccumulatedContent = appendBoundedText(
passthroughAccumulatedContent,
incomingContent
);
}
{
const guarded = applyTextualToolCallStreamingGuard(
parsed as Record<string, unknown>
);
parsed = guarded.parsed as typeof parsed;
textualToolCallConverted = guarded.textualToolCallConverted;
}
if (typeof delta?.reasoning_content === "string")
passthroughAccumulatedReasoning = appendBoundedText(

15
scripts/build/build-next-isolated.mjs
View File

@@ -208,6 +208,21 @@ export async function main() {
console.warn("[build-next-isolated] Non-fatal error copying static assets:", copyErr);
}
try {
const publicDir = path.join(projectRoot, "public");
if (await exists(publicDir)) {
await fs.cp(publicDir, path.join(projectRoot, ".next", "standalone", "public"), {
recursive: true,
});
console.log("[build-next-isolated] Copied public/ to standalone output");
}
} catch (publicCopyErr) {
console.warn(
"[build-next-isolated] Non-fatal error copying public/:",
publicCopyErr?.message
);
}
try {
await fs.cp(
path.join(projectRoot, "docs"),

2
src/shared/utils/classify429.ts
View File

@@ -43,6 +43,8 @@ const QUOTA_PATTERNS: ReadonlyArray<RegExp> = [
/out of credits/i,
/hard.?limit/i,
/plan.*limit/i,
/resource.*exhaust/i,
/check.*quota/i,
];
/**

9
src/sse/services/auth.ts
View File

@@ -41,6 +41,7 @@ import {
classifyProviderError,
PROVIDER_ERROR_TYPES,
} from "@omniroute/open-sse/services/errorClassifier.ts";
import { looksLikeQuotaExhausted } from "@/shared/utils/classify429";
import { getCodexModelScope } from "@omniroute/open-sse/executors/codex.ts";
import { getProviderAlias, resolveProviderId, FREE_PROVIDERS } from "@/shared/constants/providers";
import { isModelExcludedByConnection } from "@/domain/connectionModelRules";
@@ -1604,7 +1605,13 @@ export async function markAccountUnavailable(
(status === 404 || status === 429 || status >= 500)
) {
const reason =
status === 404 ? "not_found" : status === 429 ? "rate_limited" : "server_error";
status === 404
? "not_found"
: status === 429 && looksLikeQuotaExhausted(errorText)
? "quota_exhausted"
: status === 429
? "rate_limited"
: "server_error";
const lockout = recordModelLockoutFailure(
provider,
connectionId,

58
tests/unit/response-sanitizer.test.ts
View File

@@ -453,3 +453,61 @@ test("sanitize functions return non-object inputs unchanged", () => {
assert.equal(sanitizeOpenAIResponse(null), null);
assert.equal(sanitizeStreamingChunk("raw text"), "raw text");
});
test("sanitizeOpenAIResponse converts textual pseudo tool-call content into structured tool_calls", () => {
const sanitized = sanitizeOpenAIResponse({
id: "chatcmpl_textual_tool_call",
object: "chat.completion",
created: 1,
model: "MainAgent",
choices: [
{
index: 0,
finish_reason: "stop",
message: {
role: "assistant",
content:
'Проверю.\n[Tool call: terminal]\nArguments: {"command":"echo hermes_textual_toolcall_guard","timeout":10}',
},
},
],
}) as any;
const choice = sanitized.choices[0];
assert.equal(choice.finish_reason, "tool_calls");
assert.equal(choice.message.content, null);
assert.equal(choice.message.tool_calls[0].type, "function");
assert.equal(choice.message.tool_calls[0].function.name, "terminal");
assert.deepEqual(JSON.parse(choice.message.tool_calls[0].function.arguments), {
command: "echo hermes_textual_toolcall_guard",
timeout: 10,
});
assert.equal(JSON.stringify(sanitized).includes("[Tool call:"), false);
assert.equal(JSON.stringify(sanitized).includes("Arguments:"), false);
});
test("sanitizeOpenAIResponse suppresses malformed textual pseudo tool-call content", () => {
const sanitized = sanitizeOpenAIResponse({
id: "chatcmpl_malformed_textual_tool_call",
object: "chat.completion",
created: 1,
model: "MainAgent",
choices: [
{
index: 0,
finish_reason: "stop",
message: {
role: "assistant",
content: "[Tool call: terminal]\nArguments: {not json",
},
},
],
}) as any;
const choice = sanitized.choices[0];
assert.equal(choice.finish_reason, "stop");
assert.equal(choice.message.content, null);
assert.equal(choice.message.tool_calls, undefined);
assert.equal(JSON.stringify(sanitized).includes("[Tool call:"), false);
assert.equal(JSON.stringify(sanitized).includes("Arguments:"), false);
});

114
tests/unit/stream-utils.test.ts
View File

@@ -232,6 +232,120 @@ test("createSSEStream passthrough converts split textual tool-call content at co
assert.doesNotMatch(text, /\[Tool call: terminal\]/);
});
test("createSSEStream passthrough buffers fragmented textual tool-call JSON before emitting", async () => {
let onCompletePayload = null;
const text = await readTransformed(
[
`data: ${JSON.stringify({
id: "chatcmpl_fragmented_live_shape",
object: "chat.completion.chunk",
created: 1,
model: "MainAgent",
choices: [
{
index: 0,
delta: { role: "assistant", content: '[Tool call: terminal]\nArguments: {"' },
},
],
})}\n\n`,
`data: ${JSON.stringify({
id: "chatcmpl_fragmented_live_shape",
object: "chat.completion.chunk",
created: 1,
model: "MainAgent",
choices: [
{
index: 0,
delta: { content: 'command":"echo live_shape","timeout":10}' },
},
],
})}\n\n`,
`data: ${JSON.stringify({
id: "chatcmpl_fragmented_live_shape",
object: "chat.completion.chunk",
created: 1,
model: "MainAgent",
choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
})}\n\n`,
],
{
mode: "passthrough",
sourceFormat: FORMATS.OPENAI,
provider: "omniroute",
model: "MainAgent",
body: { messages: [{ role: "user", content: "inspect" }] },
onComplete(payload) {
onCompletePayload = payload;
},
}
);
assert.doesNotMatch(text, /\[Tool call:/);
assert.doesNotMatch(text, /Arguments:/);
assert.match(text, /"tool_calls":\[/);
assert.match(text, /"finish_reason":"tool_calls"/);
const choice = onCompletePayload.responseBody.choices[0];
assert.equal(choice.finish_reason, "tool_calls");
assert.equal(choice.message.content, null);
assert.equal(choice.message.tool_calls[0].function.name, "terminal");
assert.deepEqual(JSON.parse(choice.message.tool_calls[0].function.arguments), {
command: "echo live_shape",
timeout: 10,
});
});
test("createSSEStream passthrough suppresses trailing prose plus textual tool call", async () => {
let onCompletePayload = null;
const toolArgs = JSON.stringify({
command: "echo should_not_leak",
timeout: 10,
});
const toolText = `Вот оно! Статические файлы Next.js отдают 404. Чанки не найдены.\n\n[Tool call: terminal]\nArguments: ${toolArgs}`;
const text = await readTransformed(
[
`data: ${JSON.stringify({
id: "chatcmpl_trailing_textual_tool",
object: "chat.completion.chunk",
created: 1,
model: "MainAgent",
choices: [{ index: 0, delta: { role: "assistant", content: toolText } }],
})}\n\n`,
`data: ${JSON.stringify({
id: "chatcmpl_trailing_textual_tool",
object: "chat.completion.chunk",
created: 1,
model: "MainAgent",
choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
})}\n\n`,
],
{
mode: "passthrough",
sourceFormat: FORMATS.OPENAI,
provider: "omniroute",
model: "MainAgent",
body: { messages: [{ role: "user", content: "inspect static files" }] },
onComplete(payload) {
onCompletePayload = payload;
},
}
);
assert.equal(onCompletePayload.status, 200);
const choice = onCompletePayload.responseBody.choices[0];
assert.equal(choice.finish_reason, "tool_calls");
assert.equal(choice.message.content, null);
assert.equal(choice.message.tool_calls[0].function.name, "terminal");
assert.deepEqual(JSON.parse(choice.message.tool_calls[0].function.arguments), {
command: "echo should_not_leak",
timeout: 10,
});
assert.doesNotMatch(text, /\[Tool call: terminal\]/);
assert.doesNotMatch(text, /Arguments:/);
assert.doesNotMatch(JSON.stringify(onCompletePayload.responseBody), /\[Tool call: terminal\]/);
assert.doesNotMatch(JSON.stringify(onCompletePayload.responseBody), /Arguments:/);
});
test("createSSEStream passthrough suppresses textual tool calls for unknown tools", async () => {
let onCompletePayload = null;
const toolText = `[Tool call: search_files_ide]

106
tests/unit/translator-resp-gemini-to-openai.test.ts
View File

@@ -470,3 +470,109 @@ test("Gemini stream: safety block without candidates emits role chunk then conte
test("Gemini stream: null chunk is ignored", () => {
assert.equal(geminiToOpenAIResponse(null, createStreamingState()), null);
});
test("Gemini stream: unwraps native functionCall args when emitted as JSON string", () => {
const state = createStreamingState();
const result = geminiToOpenAIResponse(
{
responseId: "resp-native-tool-json-string",
modelVersion: "gemini-3.5-flash-low",
candidates: [
{
content: {
parts: [
{
functionCall: {
name: "terminal",
args: JSON.stringify({
command: 'ssh test-vps "systemctl cat omniroute.service"',
}),
},
},
],
},
finishReason: "STOP",
},
],
},
state
);
const toolCall = result.find((event: any) => event.choices?.[0]?.delta?.tool_calls)?.choices[0]
.delta.tool_calls[0];
assert.equal(toolCall.function.name, "terminal");
assert.equal(
toolCall.function.arguments,
JSON.stringify({ command: 'ssh test-vps "systemctl cat omniroute.service"' })
);
});
test("Gemini stream: converts JSON-string encoded textual Tool call arguments", () => {
const state = createStreamingState();
const result = geminiToOpenAIResponse(
{
responseId: "resp-textual-tool-json-string",
modelVersion: "gemini-3.5-flash-low",
candidates: [
{
content: {
parts: [
{
text: '[Tool call: terminal]\nArguments: "{\\\"command\\\":\\\"ssh test-vps \\\\\\\"systemctl cat omniroute.service\\\\\\\"\\\"}"',
},
],
},
finishReason: "STOP",
},
],
},
state
);
const toolCall = result.find((event: any) => event.choices?.[0]?.delta?.tool_calls)?.choices[0]
.delta.tool_calls[0];
assert.ok(toolCall.id.startsWith("terminal-"));
assert.equal(toolCall.function.name, "terminal");
assert.equal(
toolCall.function.arguments,
JSON.stringify({ command: 'ssh test-vps "systemctl cat omniroute.service"' })
);
assert.equal(
result.some((event: any) => event.choices?.[0]?.delta?.content?.includes("[Tool call:")),
false
);
assert.equal(result.at(-1).choices[0].finish_reason, "tool_calls");
});
test("Gemini stream: suppresses malformed textual Tool call marker", () => {
const state = createStreamingState();
const result = geminiToOpenAIResponse(
{
responseId: "resp-textual-tool-malformed",
modelVersion: "gemini-3.5-flash-low",
candidates: [
{
content: {
parts: [
{
text: '[Tool call: terminal]\nArguments: {"command":"unterminated}',
},
],
},
finishReason: "STOP",
},
],
},
state
);
assert.equal(
result.some((event: any) => event.choices?.[0]?.delta?.content?.includes("[Tool call:")),
false
);
assert.equal(
result.some((event: any) => event.choices?.[0]?.delta?.tool_calls),
false
);
assert.equal(result.at(-1).choices[0].finish_reason, "stop");
});