fix(routing): fail over when Auggie's quota-exhausted text exits clean (#12949)

Root cause: when a user's Augment/Auggie quota is exhausted, the local
`auggie` CLI prints its "You have run out of usage for ..." warning to
stdout and exits with code 0. AuggieExecutor treated any clean exit as
a successful completion and wrapped that text as a normal 200 assistant
reply, so combo/fallback routing never saw a failure and kept sending
requests to the same exhausted connection.

Fix: detect the CLI's known quota-exhausted phrasing before wrapping
stdout as a completion. Non-streaming returns a 429 in-band error body
(mirroring blackbox-web.ts's precedent for HTTP-200 in-band errors);
streaming buffers the first ~2KB of stdout, and on a match emits the
existing {error:...} SSE envelope (reusing the #7880 combo quality-gate
detection) instead of forwarding the text as a delta.

Regression test: tests/unit/issue-12949-auggie-quota-exhausted-200.test.ts

changelog.d/fixes/12949-auggie-quota-200-no-failover.md

@@ -0,0 +1 @@
+- **fix(routing):** Auggie now fails over to the next combo model instead of returning the quota-exhausted CLI warning as a successful reply (#12949) — thanks @honeypot55

open-sse/executors/auggie.ts

@@ -294,6 +294,52 @@ function isEnoentLike(message: string): boolean {
   return message.includes("ENOENT") || message.includes("not found");
 }
 
+// ─── In-band quota-exhausted detection (#12949) ───────────────────────────────
+// When a user's Augment/Auggie quota is exhausted, the real `auggie` CLI does NOT
+// exit non-zero — it prints a human-readable warning to stdout and exits 0 (a
+// clean exit). Left unchecked, that text is wrapped verbatim as a normal, 200
+// assistant reply, so combo/fallback routing (which only fails over on a non-2xx
+// status, or a top-level `error` SSE envelope for streaming) never sees a failure
+// and keeps sending requests to the same exhausted connection. Anchored tightly to
+// Auggie's actual fixed wording (not generic words like "quota"/"usage" alone) so
+// a legitimate reply that merely discusses usage/quota in passing is not
+// misclassified — see the "no false positive" case in
+// tests/unit/issue-12949-auggie-quota-exhausted-200.test.ts.
+const AUGGIE_QUOTA_EXHAUSTED_PATTERNS = [
+  /you have run out of usage/i,
+  /run out of usage for/i,
+  /usage limit exceeded/i,
+];
+
+export function isAuggieQuotaExhaustedText(text: string): boolean {
+  return AUGGIE_QUOTA_EXHAUSTED_PATTERNS.some((pattern) => pattern.test(text));
+}
+
+const AUGGIE_QUOTA_EXHAUSTED_CODE = "AUGGIE_QUOTA_EXHAUSTED";
+
+/**
+ * Build the 429 error Response for a detected in-band quota-exhausted message
+ * (non-streaming path). Mirrors the shape `blackbox-web.ts` uses for its own
+ * in-band, HTTP-200 error text (upgrade/login-required/rate-limit) — see
+ * open-sse/executors/blackbox-web.ts:590-647 — a direct JSON error body rather
+ * than buildErrorBody(), whose `code`/`type` fields are projected onto a bounded
+ * public-identifier vocabulary that does not (yet) include this provider-specific
+ * code.
+ */
+function buildAuggieQuotaErrorResponse(message: string): Response {
+  const body = {
+    error: {
+      message: sanitizeErrorMessage(message),
+      type: "upstream_error",
+      code: AUGGIE_QUOTA_EXHAUSTED_CODE,
+    },
+  };
+  return new Response(JSON.stringify(body), {
+    status: 429,
+    headers: { "Content-Type": "application/json" },
+  });
+}
+
 // Windows cmd.exe and POSIX shells never raise a Node `spawn` 'error' event for a
 // missing binary when `shell: true` is used (see buildAuggieSpawnOptions) — they
 // report it as a normal non-zero exit with the "not found" text on stderr instead.
@@ -522,8 +568,8 @@ export class AuggieExecutor extends BaseExecutor {
           );
         };
 
-        const emitError = (message: string) => {
-          emit(`data: ${JSON.stringify(buildErrorBody(502, message))}\n\n`);
+        const emitError = (message: string, statusCode = 502) => {
+          emit(`data: ${JSON.stringify(buildErrorBody(statusCode, message))}\n\n`);
           emit("data: [DONE]\n\n");
           finish();
         };
@@ -585,8 +631,41 @@ export class AuggieExecutor extends BaseExecutor {
         });
 
         let stderrTail = "";
+
+        // #12949: a quota-exhausted response is always short and delivered on the
+        // very first stdout chunk(s), so we buffer only the START of the stream
+        // (bounded — well over the quota message's length) and run the detector
+        // against it before forwarding anything as a normal delta. Once the buffer
+        // window is flushed (budget hit, or the process closes first) every later
+        // chunk is relayed live as before — no added latency for the overwhelming
+        // majority of successful, longer responses.
+        const QUOTA_DETECTION_BUFFER_BYTES = 2048;
+        let pendingBuffer = "";
+        let bufferFlushed = false;
+        let quotaDetected = false;
+
+        const flushPendingBuffer = () => {
+          if (bufferFlushed) return;
+          bufferFlushed = true;
+          if (isAuggieQuotaExhaustedText(pendingBuffer)) {
+            quotaDetected = true;
+            emitError(sanitizeErrorMessage(pendingBuffer.trim()), 429);
+            return;
+          }
+          if (pendingBuffer) emitDelta(pendingBuffer);
+          pendingBuffer = "";
+        };
+
         child.stdout?.on("data", (chunk: Buffer) => {
-          emitDelta(chunk.toString("utf8"));
+          if (quotaDetected || finished) return;
+          if (bufferFlushed) {
+            emitDelta(chunk.toString("utf8"));
+            return;
+          }
+          pendingBuffer += chunk.toString("utf8");
+          if (pendingBuffer.length >= QUOTA_DETECTION_BUFFER_BYTES) {
+            flushPendingBuffer();
+          }
         });
 
         child.stderr?.on("data", (chunk: Buffer) => {
@@ -606,6 +685,8 @@ export class AuggieExecutor extends BaseExecutor {
             );
             return;
           }
+          flushPendingBuffer();
+          if (quotaDetected || finished) return;
           emitStop();
         });
       },
@@ -693,6 +774,12 @@ export class AuggieExecutor extends BaseExecutor {
           );
           return;
         }
+        // #12949: a clean exit (code 0) can still carry an in-band quota-exhausted
+        // warning on stdout — detect it before wrapping the text as a completion.
+        if (isAuggieQuotaExhaustedText(stdout)) {
+          settle(buildAuggieQuotaErrorResponse(stdout.trim()));
+          return;
+        }
         settle(buildChatCompletionResponse(model, promptText, stdout));
       });
     });

tests/unit/issue-12949-auggie-quota-exhausted-200.test.ts

@@ -0,0 +1,124 @@
+// Regression test for issue #12949: the local `auggie` CLI prints its quota-exhausted
+// warning to stdout and exits 0 (a clean exit). The executor used to treat any clean
+// exit as a successful completion and wrap that text as a normal 200 assistant reply,
+// so combo/fallback routing never failed over to the next configured model.
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const { AuggieExecutor } = await import("@omniroute/open-sse/executors/auggie");
+
+const TMP_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-auggie-quota-test-"));
+
+function writeFakeBin(name: string, script: string): string {
+  const p = path.join(TMP_DIR, name);
+  fs.writeFileSync(p, `#!/bin/sh\n${script}\n`, { mode: 0o755 });
+  return p;
+}
+
+test.after(() => {
+  fs.rmSync(TMP_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
+});
+
+test("issue #12949: auggie quota-exhausted stdout with exit 0 must NOT surface as HTTP 200 (non-streaming)", async () => {
+  // Note: the CLI's real quota text is wrapped in the ⚠️ emoji, but this fixture
+  // uses plain ASCII (single-quoted literal, no printf hex escapes — those are not
+  // portable across shells, e.g. dash's printf does not interpret `\xHH`) — the
+  // detector matches on the plain-English phrase, so the emoji is irrelevant to it.
+  const bin = writeFakeBin(
+    "fake-auggie-quota.sh",
+    `printf '%s\\n' 'You have run out of usage for ursoambra@gmail.com. Please visit https://app.augmentcode.com/account to upgrade.'\nprintf '%s\\n' 'Request ID: f8f864e4-4559-4595-8514-678d1ff1aa09'\nexit 0`
+  );
+  const prevBin = process.env.AUGGIE_BIN;
+  process.env.AUGGIE_BIN = bin;
+  try {
+    const executor = new AuggieExecutor();
+    const { response } = await executor.execute({
+      model: "sonnet4.6",
+      body: { messages: [{ role: "user", content: "hi" }] },
+      stream: false,
+      credentials: {} as never,
+    });
+    assert.notEqual(
+      response.status,
+      200,
+      "quota-exhausted Auggie output must not be reported as a successful 200 response " +
+        "(combo routing would never fail over to the next model)"
+    );
+    assert.equal(response.status, 429, "quota-exhausted output should surface as 429");
+    const body = (await response.json()) as { error?: { message?: string; code?: string } };
+    assert.match(String(body.error?.message ?? ""), /run out of usage/i);
+    assert.equal(body.error?.code, "AUGGIE_QUOTA_EXHAUSTED");
+  } finally {
+    if (prevBin === undefined) delete process.env.AUGGIE_BIN;
+    else process.env.AUGGIE_BIN = prevBin;
+  }
+});
+
+test("issue #12949: auggie quota-exhausted stdout with exit 0 must NOT surface as a normal completion (streaming)", async () => {
+  const bin = writeFakeBin(
+    "fake-auggie-quota-stream.sh",
+    `printf '%s\\n' 'You have run out of usage for ursoambra@gmail.com. Please visit https://app.augmentcode.com/account to upgrade.'\nprintf '%s\\n' 'Request ID: f8f864e4-4559-4595-8514-678d1ff1aa09'\nexit 0`
+  );
+  const prevBin = process.env.AUGGIE_BIN;
+  process.env.AUGGIE_BIN = bin;
+  try {
+    const executor = new AuggieExecutor();
+    const { response } = await executor.execute({
+      model: "sonnet4.6",
+      body: { messages: [{ role: "user", content: "hi" }] },
+      stream: true,
+      credentials: {} as never,
+    });
+    const text = await response.text();
+    const frames = text
+      .split("\n\n")
+      .map((chunk) => chunk.trim())
+      .filter((chunk) => chunk.startsWith("data: ") && chunk !== "data: [DONE]")
+      .map((chunk) => JSON.parse(chunk.slice("data: ".length)));
+
+    const hasErrorFrame = frames.some(
+      (frame) => frame && typeof frame === "object" && "error" in frame
+    );
+    assert.ok(hasErrorFrame, "expected an in-band {error:...} SSE frame for the quota text");
+
+    const leakedQuotaDelta = frames.some((frame) => {
+      const content = frame?.choices?.[0]?.delta?.content;
+      return typeof content === "string" && /run out of usage/i.test(content);
+    });
+    assert.equal(
+      leakedQuotaDelta,
+      false,
+      "the quota-exhausted text must not be forwarded as a normal delta.content chunk"
+    );
+  } finally {
+    if (prevBin === undefined) delete process.env.AUGGIE_BIN;
+    else process.env.AUGGIE_BIN = prevBin;
+  }
+});
+
+test("issue #12949: a normal reply that merely mentions usage/quota in passing is NOT misclassified as quota-exhausted", async () => {
+  const bin = writeFakeBin(
+    "fake-auggie-normal.sh",
+    `printf 'Sure — here is how you can check your API usage and quota dashboard in the settings page.'\nexit 0`
+  );
+  const prevBin = process.env.AUGGIE_BIN;
+  process.env.AUGGIE_BIN = bin;
+  try {
+    const executor = new AuggieExecutor();
+    const { response } = await executor.execute({
+      model: "sonnet4.6",
+      body: { messages: [{ role: "user", content: "how do I check usage?" }] },
+      stream: false,
+      credentials: {} as never,
+    });
+    assert.equal(response.status, 200, "a normal reply discussing usage/quota must stay a 200");
+    const body = (await response.json()) as { choices?: Array<{ message?: { content?: string } }> };
+    assert.match(String(body.choices?.[0]?.message?.content ?? ""), /usage and quota dashboard/i);
+  } finally {
+    if (prevBin === undefined) delete process.env.AUGGIE_BIN;
+    else process.env.AUGGIE_BIN = prevBin;
+  }
+});