diff --git a/changelog.d/fixes/12949-auggie-quota-200-no-failover.md b/changelog.d/fixes/12949-auggie-quota-200-no-failover.md new file mode 100644 index 0000000000..1185931572 --- /dev/null +++ b/changelog.d/fixes/12949-auggie-quota-200-no-failover.md @@ -0,0 +1 @@ +- **fix(routing):** Auggie now fails over to the next combo model instead of returning the quota-exhausted CLI warning as a successful reply (#12949) — thanks @honeypot55 diff --git a/open-sse/executors/auggie.ts b/open-sse/executors/auggie.ts index 076b42f683..a65c2dfb40 100644 --- a/open-sse/executors/auggie.ts +++ b/open-sse/executors/auggie.ts @@ -294,6 +294,52 @@ function isEnoentLike(message: string): boolean { return message.includes("ENOENT") || message.includes("not found"); } +// ─── In-band quota-exhausted detection (#12949) ─────────────────────────────── +// When a user's Augment/Auggie quota is exhausted, the real `auggie` CLI does NOT +// exit non-zero — it prints a human-readable warning to stdout and exits 0 (a +// clean exit). Left unchecked, that text is wrapped verbatim as a normal, 200 +// assistant reply, so combo/fallback routing (which only fails over on a non-2xx +// status, or a top-level `error` SSE envelope for streaming) never sees a failure +// and keeps sending requests to the same exhausted connection. Anchored tightly to +// Auggie's actual fixed wording (not generic words like "quota"/"usage" alone) so +// a legitimate reply that merely discusses usage/quota in passing is not +// misclassified — see the "no false positive" case in +// tests/unit/issue-12949-auggie-quota-exhausted-200.test.ts. +const AUGGIE_QUOTA_EXHAUSTED_PATTERNS = [ + /you have run out of usage/i, + /run out of usage for/i, + /usage limit exceeded/i, +]; + +export function isAuggieQuotaExhaustedText(text: string): boolean { + return AUGGIE_QUOTA_EXHAUSTED_PATTERNS.some((pattern) => pattern.test(text)); +} + +const AUGGIE_QUOTA_EXHAUSTED_CODE = "AUGGIE_QUOTA_EXHAUSTED"; + +/** + * Build the 429 error Response for a detected in-band quota-exhausted message + * (non-streaming path). Mirrors the shape `blackbox-web.ts` uses for its own + * in-band, HTTP-200 error text (upgrade/login-required/rate-limit) — see + * open-sse/executors/blackbox-web.ts:590-647 — a direct JSON error body rather + * than buildErrorBody(), whose `code`/`type` fields are projected onto a bounded + * public-identifier vocabulary that does not (yet) include this provider-specific + * code. + */ +function buildAuggieQuotaErrorResponse(message: string): Response { + const body = { + error: { + message: sanitizeErrorMessage(message), + type: "upstream_error", + code: AUGGIE_QUOTA_EXHAUSTED_CODE, + }, + }; + return new Response(JSON.stringify(body), { + status: 429, + headers: { "Content-Type": "application/json" }, + }); +} + // Windows cmd.exe and POSIX shells never raise a Node `spawn` 'error' event for a // missing binary when `shell: true` is used (see buildAuggieSpawnOptions) — they // report it as a normal non-zero exit with the "not found" text on stderr instead. @@ -522,8 +568,8 @@ export class AuggieExecutor extends BaseExecutor { ); }; - const emitError = (message: string) => { - emit(`data: ${JSON.stringify(buildErrorBody(502, message))}\n\n`); + const emitError = (message: string, statusCode = 502) => { + emit(`data: ${JSON.stringify(buildErrorBody(statusCode, message))}\n\n`); emit("data: [DONE]\n\n"); finish(); }; @@ -585,8 +631,41 @@ export class AuggieExecutor extends BaseExecutor { }); let stderrTail = ""; + + // #12949: a quota-exhausted response is always short and delivered on the + // very first stdout chunk(s), so we buffer only the START of the stream + // (bounded — well over the quota message's length) and run the detector + // against it before forwarding anything as a normal delta. Once the buffer + // window is flushed (budget hit, or the process closes first) every later + // chunk is relayed live as before — no added latency for the overwhelming + // majority of successful, longer responses. + const QUOTA_DETECTION_BUFFER_BYTES = 2048; + let pendingBuffer = ""; + let bufferFlushed = false; + let quotaDetected = false; + + const flushPendingBuffer = () => { + if (bufferFlushed) return; + bufferFlushed = true; + if (isAuggieQuotaExhaustedText(pendingBuffer)) { + quotaDetected = true; + emitError(sanitizeErrorMessage(pendingBuffer.trim()), 429); + return; + } + if (pendingBuffer) emitDelta(pendingBuffer); + pendingBuffer = ""; + }; + child.stdout?.on("data", (chunk: Buffer) => { - emitDelta(chunk.toString("utf8")); + if (quotaDetected || finished) return; + if (bufferFlushed) { + emitDelta(chunk.toString("utf8")); + return; + } + pendingBuffer += chunk.toString("utf8"); + if (pendingBuffer.length >= QUOTA_DETECTION_BUFFER_BYTES) { + flushPendingBuffer(); + } }); child.stderr?.on("data", (chunk: Buffer) => { @@ -606,6 +685,8 @@ export class AuggieExecutor extends BaseExecutor { ); return; } + flushPendingBuffer(); + if (quotaDetected || finished) return; emitStop(); }); }, @@ -693,6 +774,12 @@ export class AuggieExecutor extends BaseExecutor { ); return; } + // #12949: a clean exit (code 0) can still carry an in-band quota-exhausted + // warning on stdout — detect it before wrapping the text as a completion. + if (isAuggieQuotaExhaustedText(stdout)) { + settle(buildAuggieQuotaErrorResponse(stdout.trim())); + return; + } settle(buildChatCompletionResponse(model, promptText, stdout)); }); }); diff --git a/tests/unit/issue-12949-auggie-quota-exhausted-200.test.ts b/tests/unit/issue-12949-auggie-quota-exhausted-200.test.ts new file mode 100644 index 0000000000..39ac15d5a6 --- /dev/null +++ b/tests/unit/issue-12949-auggie-quota-exhausted-200.test.ts @@ -0,0 +1,124 @@ +// Regression test for issue #12949: the local `auggie` CLI prints its quota-exhausted +// warning to stdout and exits 0 (a clean exit). The executor used to treat any clean +// exit as a successful completion and wrap that text as a normal 200 assistant reply, +// so combo/fallback routing never failed over to the next configured model. +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const { AuggieExecutor } = await import("@omniroute/open-sse/executors/auggie"); + +const TMP_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-auggie-quota-test-")); + +function writeFakeBin(name: string, script: string): string { + const p = path.join(TMP_DIR, name); + fs.writeFileSync(p, `#!/bin/sh\n${script}\n`, { mode: 0o755 }); + return p; +} + +test.after(() => { + fs.rmSync(TMP_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("issue #12949: auggie quota-exhausted stdout with exit 0 must NOT surface as HTTP 200 (non-streaming)", async () => { + // Note: the CLI's real quota text is wrapped in the ⚠️ emoji, but this fixture + // uses plain ASCII (single-quoted literal, no printf hex escapes — those are not + // portable across shells, e.g. dash's printf does not interpret `\xHH`) — the + // detector matches on the plain-English phrase, so the emoji is irrelevant to it. + const bin = writeFakeBin( + "fake-auggie-quota.sh", + `printf '%s\\n' 'You have run out of usage for ursoambra@gmail.com. Please visit https://app.augmentcode.com/account to upgrade.'\nprintf '%s\\n' 'Request ID: f8f864e4-4559-4595-8514-678d1ff1aa09'\nexit 0` + ); + const prevBin = process.env.AUGGIE_BIN; + process.env.AUGGIE_BIN = bin; + try { + const executor = new AuggieExecutor(); + const { response } = await executor.execute({ + model: "sonnet4.6", + body: { messages: [{ role: "user", content: "hi" }] }, + stream: false, + credentials: {} as never, + }); + assert.notEqual( + response.status, + 200, + "quota-exhausted Auggie output must not be reported as a successful 200 response " + + "(combo routing would never fail over to the next model)" + ); + assert.equal(response.status, 429, "quota-exhausted output should surface as 429"); + const body = (await response.json()) as { error?: { message?: string; code?: string } }; + assert.match(String(body.error?.message ?? ""), /run out of usage/i); + assert.equal(body.error?.code, "AUGGIE_QUOTA_EXHAUSTED"); + } finally { + if (prevBin === undefined) delete process.env.AUGGIE_BIN; + else process.env.AUGGIE_BIN = prevBin; + } +}); + +test("issue #12949: auggie quota-exhausted stdout with exit 0 must NOT surface as a normal completion (streaming)", async () => { + const bin = writeFakeBin( + "fake-auggie-quota-stream.sh", + `printf '%s\\n' 'You have run out of usage for ursoambra@gmail.com. Please visit https://app.augmentcode.com/account to upgrade.'\nprintf '%s\\n' 'Request ID: f8f864e4-4559-4595-8514-678d1ff1aa09'\nexit 0` + ); + const prevBin = process.env.AUGGIE_BIN; + process.env.AUGGIE_BIN = bin; + try { + const executor = new AuggieExecutor(); + const { response } = await executor.execute({ + model: "sonnet4.6", + body: { messages: [{ role: "user", content: "hi" }] }, + stream: true, + credentials: {} as never, + }); + const text = await response.text(); + const frames = text + .split("\n\n") + .map((chunk) => chunk.trim()) + .filter((chunk) => chunk.startsWith("data: ") && chunk !== "data: [DONE]") + .map((chunk) => JSON.parse(chunk.slice("data: ".length))); + + const hasErrorFrame = frames.some( + (frame) => frame && typeof frame === "object" && "error" in frame + ); + assert.ok(hasErrorFrame, "expected an in-band {error:...} SSE frame for the quota text"); + + const leakedQuotaDelta = frames.some((frame) => { + const content = frame?.choices?.[0]?.delta?.content; + return typeof content === "string" && /run out of usage/i.test(content); + }); + assert.equal( + leakedQuotaDelta, + false, + "the quota-exhausted text must not be forwarded as a normal delta.content chunk" + ); + } finally { + if (prevBin === undefined) delete process.env.AUGGIE_BIN; + else process.env.AUGGIE_BIN = prevBin; + } +}); + +test("issue #12949: a normal reply that merely mentions usage/quota in passing is NOT misclassified as quota-exhausted", async () => { + const bin = writeFakeBin( + "fake-auggie-normal.sh", + `printf 'Sure — here is how you can check your API usage and quota dashboard in the settings page.'\nexit 0` + ); + const prevBin = process.env.AUGGIE_BIN; + process.env.AUGGIE_BIN = bin; + try { + const executor = new AuggieExecutor(); + const { response } = await executor.execute({ + model: "sonnet4.6", + body: { messages: [{ role: "user", content: "how do I check usage?" }] }, + stream: false, + credentials: {} as never, + }); + assert.equal(response.status, 200, "a normal reply discussing usage/quota must stay a 200"); + const body = (await response.json()) as { choices?: Array<{ message?: { content?: string } }> }; + assert.match(String(body.choices?.[0]?.message?.content ?? ""), /usage and quota dashboard/i); + } finally { + if (prevBin === undefined) delete process.env.AUGGIE_BIN; + else process.env.AUGGIE_BIN = prevBin; + } +});