diff --git a/src/shared/utils/classify429.ts b/src/shared/utils/classify429.ts index 6c6f4eee2e..531a5bae67 100644 --- a/src/shared/utils/classify429.ts +++ b/src/shared/utils/classify429.ts @@ -66,6 +66,23 @@ const QUOTA_PATTERNS: ReadonlyArray = [ // the 429 is misclassified as transient rate_limit and retried every // ~60s against a budget that only resets at UTC midnight. /daily free allocation/i, + + // Modal-hosted OpenAI-compatible endpoints (e.g. self-hosted Kimi K3). + // Body: {"error":"usage limit reached"}, no nested "message"/"quota"/ + // "daily" wording. Without this pattern the 429 falls through to + // "rate_limit" (short cooldown), so combo round-robin's per-conversation + // session stickiness (#3825) keeps re-targeting the same exhausted + // connection every turn instead of a long lockout that lets the sticky + // target fail over to another account. + // + // Matches the "error" JSON key with "usage limit reached" as its value. + // Extra sibling fields (e.g. {"error":"usage limit reached", "code":"..."}) + // still match. A different key like {"detail":"..."} or a qualified value + // like {"error":"Per-minute usage limit reached"} does NOT match. Bare + // string bodies without a JSON wrapper also do NOT match. + // Trailing punctuation/whitespace before the closing quote is tolerated + // because real API responses may include a period or trailing space. + /"error"\s*:\s*"usage limit reached[.\s]*"/i, ]; /** diff --git a/tests/unit/classify429.test.ts b/tests/unit/classify429.test.ts index 234c42ee73..08ba94de12 100644 --- a/tests/unit/classify429.test.ts +++ b/tests/unit/classify429.test.ts @@ -164,6 +164,63 @@ test("looksLikeQuotaExhausted: rejects empty / null / non-quota text", () => { assert.equal(looksLikeQuotaExhausted("server error 500"), false); }); +test("classify429: Modal-hosted endpoint 'usage limit reached' body returns 'quota_exhausted'", () => { + // Real body observed from a self-hosted Modal OpenAI-compatible endpoint: + // {"error":"usage limit reached"} - a bare string value, no "message"/ + // "daily"/"quota" wording, so none of the prior patterns matched and the + // 429 fell through to a 60s rate_limit cooldown. Combo round-robin's + // per-conversation session stickiness (#3825) then kept re-targeting the + // same exhausted connection on every turn of a long-running session. + const body = { error: "usage limit reached" }; + assert.equal(looksLikeQuotaExhausted(body), true); + assert.equal(classify429({ status: 429, body }), "quota_exhausted"); + assert.equal(classify429({ status: 429, body: JSON.stringify(body) }), "quota_exhausted"); + // Case variation must also match. + assert.equal( + classify429({ status: 429, body: { error: "USAGE LIMIT REACHED" } }), + "quota_exhausted" + ); + // Whitespace around JSON object must also match (bodyToText does not trim). + assert.equal( + classify429({ status: 429, body: ' { "error" : "usage limit reached" } ' }), + "quota_exhausted" + ); + // Extra sibling fields must still match. + assert.equal( + classify429({ + status: 429, + body: { error: "usage limit reached", code: "RESOURCE_EXHAUSTED" }, + }), + "quota_exhausted" + ); + // Trailing punctuation/whitespace must still match. + assert.equal(classify429({ status: 429, body: { error: "usage limit reached." } }), "quota_exhausted"); + assert.equal(classify429({ status: 429, body: { error: "usage limit reached " } }), "quota_exhausted"); +}); + +test("classify429: qualified transient 'usage limit reached' messages stay rate_limit", () => { + // The Modal pattern requires the "error" JSON key with exactly "usage + // limit reached" as its value - anything else is a transient rate limit + // and must NOT be locked out long-term. + assert.equal( + classify429({ status: 429, body: "Per-minute usage limit reached, retry in 60s." }), + "rate_limit" + ); + assert.equal( + classify429({ status: 429, body: { error: { message: "RPM usage limit reached" } } }), + "rate_limit" + ); + // Bare string body (no JSON "error" key) must NOT match. + assert.equal(classify429({ status: 429, body: "usage limit reached" }), "rate_limit"); + // Different JSON key (not "error") must NOT match. + assert.equal(classify429({ status: 429, body: { detail: "usage limit reached" } }), "rate_limit"); + // Qualified value under the "error" key must NOT match. + assert.equal( + classify429({ status: 429, body: { error: "Per-minute usage limit reached" } }), + "rate_limit" + ); +}); + test("ambiguous 'daily rate limit' messages classify as quota_exhausted (intentional)", () => { // Codex audit LOW: messages combining 'daily' or 'monthly' with 'limit' // match the quota regex even when paired with 'rate'. This is intentional