diff --git a/changelog.d/fixes/13657-rate-limited-429-single-retry-drain.md b/changelog.d/fixes/13657-rate-limited-429-single-retry-drain.md new file mode 100644 index 0000000000..93f2bd0166 --- /dev/null +++ b/changelog.d/fixes/13657-rate-limited-429-single-retry-drain.md @@ -0,0 +1 @@ +- **fix(sse):** opt-in `OPENCODE_RATE_LIMITED_429_EARLY_STOP` flag (default off): an opencode 429 classified as a real rate limit (parseable `Retry-After`, or a body naming a rate/usage limit) stops the cross-account wave and returns that upstream 429 unchanged — body, `Retry-After` and quota headers intact, so the opencode quota error rules still apply; unclassified 429s keep rotating, and with the flag off every 429 rotates as before (#9611) ([#13657](https://github.com/diegosouzapw/OmniRoute/pull/13657)) — thanks @maxmad64bis diff --git a/docs/reference/FEATURE_FLAGS.md b/docs/reference/FEATURE_FLAGS.md index 8f4f9c72a1..7c203eae7d 100644 --- a/docs/reference/FEATURE_FLAGS.md +++ b/docs/reference/FEATURE_FLAGS.md @@ -46,7 +46,7 @@ A boolean flag is considered **enabled** when its effective value is `"true"`, ## Flag Catalog -68 flags across 6 categories. **Default** is the definition default — the value +69 flags across 6 categories. **Default** is the definition default — the value used when neither a DB override nor an environment variable is present. ### Security (10) @@ -64,7 +64,7 @@ used when neither a DB override nor an environment variable is present. | `AUTH_LOG_INCLUDE_ACCOUNT_ID` | boolean | `false` | Include account prefix in AUTH log lines (e.g. "Using account: abc12345..."). Disabled by default so account identifiers are redacted from shared/multi-tenant process logs. Independent from Debug Mode; flipping Debug Mode does not reveal this. | | `OMNIROUTE_OIDC_DISABLE_PASSWORD_LOGIN` | boolean | `false` | When OIDC is enabled, disable password login so users can only authenticate via OIDC Single Sign-On. When disabled (default), both password login and OIDC are available. | -### Network (14) +### Network (15) | Key | Type | Default | Restart | Description | | ----------------------------------------------- | ------- | ------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -78,6 +78,7 @@ used when neither a DB override nor an environment variable is present. | `OPENCODE_RESPONSES_STALL_ROTATION` | boolean | `false` | | For the OpenCode executor, watch the first body byte of a streamed Responses reply (window: `RESPONSES_FIRST_BYTE_TIMEOUT_MS`, default `15000`). A 2xx Responses stream that stays silent past the window is treated as stalled: the account is cooled down and the request rotates to the next account once; a second stall fails fast. Off by default: stalled streams keep today's wait until the stream readiness timeout. | | `OPENCODE_USER_BLOCKED_ROTATION` | boolean | `false` | | OpenCode executor: on a 403/451 carrying a `user_blocked` refusal (not geo, not a Cloudflare fingerprint rejection), cool the refused account down and rotate to the next account at most once per request; a second refusal is returned as-is, without a success mark. Off by default: routing around an upstream user block can look like evasion and spread the flag across the fleet. | | `OPENCODE_TRANSIENT_FAILOVER_BACKOFF` | boolean | `false` | | OpenCode rotation: after two consecutive transient upstream failures (5xx or an empty 400), pause before the next account — 1.5s doubling per further failure, capped at 6s per pause and 10s per request, skipped on client disconnect; the failed body is released before waiting. Off by default: failover stays immediate. | +| `OPENCODE_RATE_LIMITED_429_EARLY_STOP` | boolean | `false` | | OpenCode rotation: stop the account wave at the first 429 classified as a real rate limit (parseable `Retry-After`, or a body naming a rate/usage limit) and return that upstream 429 unchanged. Unclassified 429s keep rotating. Off by default: the free tier is limited per egress IP (#9611), so every 429 rotates and an exhausted wave returns the last upstream 429. | | `MITM_DISABLE_TLS_VERIFY` | boolean | `false` | ✓ | Disable TLS certificate verification for the MITM proxy. **Danger.** | | `OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS` | boolean | `false` | | Allow provider URLs pointing to private/internal networks. | | `OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS` | boolean | `true` | | Allow adding/validating providers on local/private addresses (127.0.0.1, localhost, LAN). On by default (local-first); disable for strict public-only blocking. Cloud-metadata stays blocked. | @@ -208,7 +209,7 @@ Returns every flag with its effective value, source, and a summary. "requiresRestart": false, "warningLevel": "caution", }, - // ... all 68 flags + // ... all 69 flags ], "summary": { "total": 56, diff --git a/open-sse/executors/opencode.ts b/open-sse/executors/opencode.ts index 754f0f284e..9c18e7945a 100644 --- a/open-sse/executors/opencode.ts +++ b/open-sse/executors/opencode.ts @@ -53,7 +53,9 @@ import { isProxySkipRecentlyFailedEnabled, isOpencodeUserBlockedRotationEnabled, isOpencodeTransientFailoverBackoffEnabled, + isOpencodeRateLimited429EarlyStopEnabled, } from "@/shared/utils/featureFlags"; +import { classifyUpstream429 } from "./opencodeRateLimited.ts"; /** * The main OpenCode Zen host, shared by the `opencode` and `opencode-zen` @@ -791,6 +793,19 @@ export class OpencodeExecutor extends BaseExecutor { const setAsideMs = skipRecentlyFailed ? noteProxyRefusal(proxyEgressKey(account.proxy), "ip_quota_429") : null; + // Opt-in (#13657): a 429 that names a real rate limit stops the wave and + // the real upstream 429 is returned untouched (body, Retry-After, quota + // headers), so provider error rules still apply. Flag off → rotate. + if ( + isOpencodeRateLimited429EarlyStopEnabled() && + (await classifyUpstream429(result.response)) === "rate_limited" + ) { + log?.warn?.( + "OPENCODE", + `${cid}rate-limited 429 on account ${masked}, stopping the account wave` + ); + return result; + } log?.warn?.( "OPENCODE", `${cid}Rate limited (429) on account ${masked}` + diff --git a/open-sse/executors/opencodeRateLimited.ts b/open-sse/executors/opencodeRateLimited.ts new file mode 100644 index 0000000000..0bf8e7dec3 --- /dev/null +++ b/open-sse/executors/opencodeRateLimited.ts @@ -0,0 +1,92 @@ +/** + * opencodeRateLimited.ts — 429 classifier for the opencode executor loop (#13657). + * + * Leaf module: zero imports, no registry, no DB. Headers first: a parseable + * Retry-After alone marks a real rate limit; otherwise a bounded prefix of the + * body is matched against generic English rate-limit phrasings (derived from a + * captured 429 body). Anything else is a "burst" 429 that keeps the normal + * cross-account rotation. Only consulted when OPENCODE_RATE_LIMITED_429_EARLY_STOP + * is on; the classifier never rewrites the response it inspects. + */ + +const RATE_LIMITED_SIGNALS: ReadonlyArray = [ + /rate.?limited/i, + /usage.?limit/i, + /too many requests/i, +]; + +/** Bytes of a 429 body inspected by the classifier. */ +export const RATE_LIMIT_BODY_SNIFF_BYTES = 8192; + +/** Seconds until retry from a Retry-After value (delta-seconds or HTTP date), or null. */ +export function parseRetryAfterSeconds( + retryAfter: string | number | null | undefined, + now = Date.now() +): number | null { + if (typeof retryAfter === "number") { + return Number.isFinite(retryAfter) && retryAfter > 0 ? Math.ceil(retryAfter) : null; + } + if (typeof retryAfter !== "string") return null; + const text = retryAfter.trim(); + if (text === "") return null; + if (/^\d+$/.test(text)) return Math.max(Number(text), 1); + const ms = Date.parse(text); + if (Number.isFinite(ms)) return Math.max(Math.ceil((ms - now) / 1000), 1); + return null; +} + +export type RateLimit429Verdict = "rate_limited" | "burst"; + +export function classify429(input: { + retryAfter?: string | number | null; + bodyText?: string | null; +}): RateLimit429Verdict { + if (parseRetryAfterSeconds(input.retryAfter) !== null) return "rate_limited"; + const body = typeof input.bodyText === "string" ? input.bodyText : ""; + if (body !== "" && RATE_LIMITED_SIGNALS.some((re) => re.test(body))) return "rate_limited"; + return "burst"; +} + +/** + * Read at most `maxBytes` of a response body from a clone, then cancel the + * clone's reader. The original response keeps its full, unread body. Returns + * null when the body cannot be read. + */ +export async function readBodyPrefix( + response: Response, + maxBytes = RATE_LIMIT_BODY_SNIFF_BYTES +): Promise { + if (!response.body) return ""; + let reader: ReadableStreamDefaultReader | undefined; + try { + reader = response.clone().body?.getReader(); + } catch { + return null; + } + if (!reader) return ""; + const decoder = new TextDecoder(); + let text = ""; + let bytes = 0; + try { + while (bytes < maxBytes) { + const { done, value } = await reader.read(); + if (done || !value) break; + const chunk = + value.byteLength > maxBytes - bytes ? value.subarray(0, maxBytes - bytes) : value; + bytes += chunk.byteLength; + text += decoder.decode(chunk, { stream: true }); + } + return text + decoder.decode(); + } catch { + return null; + } finally { + void reader.cancel().catch(() => undefined); + } +} + +/** Classify an upstream 429: Retry-After header first, bounded body prefix only if needed. */ +export async function classifyUpstream429(response: Response): Promise { + const retryAfter = response.headers.get("retry-after"); + if (parseRetryAfterSeconds(retryAfter) !== null) return "rate_limited"; + return classify429({ retryAfter, bodyText: await readBodyPrefix(response) }); +} diff --git a/src/shared/constants/featureFlagDefinitions.ts b/src/shared/constants/featureFlagDefinitions.ts index 6b87df1e11..ee089d74a5 100644 --- a/src/shared/constants/featureFlagDefinitions.ts +++ b/src/shared/constants/featureFlagDefinitions.ts @@ -251,6 +251,18 @@ export const FEATURE_FLAG_DEFINITIONS: FeatureFlagDefinition[] = [ requiresRestart: false, warningLevel: "caution", }, + { + key: "OPENCODE_RATE_LIMITED_429_EARLY_STOP", + label: "OpenCode Rate-Limited 429 Early Stop", + description: + "For the OpenCode multi-account rotation, stop the account wave at the first 429 classified as a real rate limit (a parseable Retry-After header, or a body naming a rate/usage limit) and return that upstream 429 unchanged (status, body, Retry-After and quota headers), instead of trying every remaining account. Unclassified 429s keep rotating. Off by default: the free tier is limited per egress IP (#9611), so every 429 rotates to the next account, and an exhausted wave returns the last upstream 429.", + descriptionI18nKey: "featureFlagOpencodeRateLimited429EarlyStopDescription", + category: "network", + defaultValue: "false", + type: "boolean", + requiresRestart: false, + warningLevel: "caution", + }, { key: "MITM_DISABLE_TLS_VERIFY", label: "Disable TLS Verify (MITM)", diff --git a/src/shared/utils/featureFlags.ts b/src/shared/utils/featureFlags.ts index e479e72a61..4f003e9e8a 100644 --- a/src/shared/utils/featureFlags.ts +++ b/src/shared/utils/featureFlags.ts @@ -289,6 +289,23 @@ export function isMistralAmbiguous401SoftLockoutEnabled(): boolean { } } +/** + * OpenCode classified-429 early stop (#13657). Opt-in: when off, every 429 rotates to the + * next account exactly as before. + * Fail closed: an unreadable flag store keeps the pre-flag behavior (disabled). + */ +export function isOpencodeRateLimited429EarlyStopEnabled(): boolean { + try { + return isFeatureFlagEnabled("OPENCODE_RATE_LIMITED_429_EARLY_STOP"); + } catch (error) { + console.error( + "[featureFlags] Failed to resolve OPENCODE_RATE_LIMITED_429_EARLY_STOP, defaulting to disabled:", + error instanceof Error ? error.message : error + ); + return false; + } +} + export function isServerOwnedToolLoopEnabled( reader: (key: string) => boolean = isFeatureFlagEnabled ): boolean { diff --git a/tests/unit/feature-flags-settings.test.ts b/tests/unit/feature-flags-settings.test.ts index e725b8b546..c521b3f793 100644 --- a/tests/unit/feature-flags-settings.test.ts +++ b/tests/unit/feature-flags-settings.test.ts @@ -40,7 +40,7 @@ const { // the dead ONEPROXY_ENABLED (readerless since the 1proxy purge, #12091) // brought it back to 53. UNIVERSAL_CONTEXT_HANDOFF_ENABLED bumped it to 54. // #13641 added SEARCH_STATS_HIDE_DELETED_CONNECTIONS, bumping the count to 56. -const EXPECTED_FEATURE_FLAG_COUNT = 68; +const EXPECTED_FEATURE_FLAG_COUNT = 69; // ────────────────────────────────────────────────────── // Test group 1 — Flag definitions registry @@ -242,6 +242,17 @@ describe("featureFlagDefinitions", () => { assert.strictEqual(def.requiresRestart, false); }); + it("defines OPENCODE_RATE_LIMITED_429_EARLY_STOP as an opt-in network boolean flag disabled by default", () => { + const def = FEATURE_FLAG_DEFINITIONS.find( + (d) => d.key === "OPENCODE_RATE_LIMITED_429_EARLY_STOP" + ); + assert.ok(def, "OPENCODE_RATE_LIMITED_429_EARLY_STOP should exist"); + assert.strictEqual(def.category, "network"); + assert.strictEqual(def.type, "boolean"); + assert.strictEqual(def.defaultValue, "false"); + assert.strictEqual(def.requiresRestart, false); + }); + it("defines network rotation shared-egress guard as a network boolean flag enabled by default", () => { const def = FEATURE_FLAG_DEFINITIONS.find( (d) => d.key === "NETWORK_ROTATION_SHARED_EGRESS_GUARD" diff --git a/tests/unit/opencode-rate-limited-classify.test.ts b/tests/unit/opencode-rate-limited-classify.test.ts new file mode 100644 index 0000000000..43cbcc3b59 --- /dev/null +++ b/tests/unit/opencode-rate-limited-classify.test.ts @@ -0,0 +1,277 @@ +import { describe, it, beforeEach, afterEach, before, after } from "node:test"; +import assert from "node:assert"; +import net from "node:net"; +import { + classify429, + classifyUpstream429, + parseRetryAfterSeconds, + readBodyPrefix, + RATE_LIMIT_BODY_SNIFF_BYTES, +} from "../../open-sse/executors/opencodeRateLimited.ts"; +import { OpencodeExecutor } from "../../open-sse/executors/opencode.ts"; +import type { ExecutorLog, ProviderCredentials } from "../../open-sse/executors/base.ts"; +import { resolveProxyForRequest } from "../../open-sse/utils/proxyFetch.ts"; +import { getProviderErrorRuleMatch } from "../../open-sse/config/providerErrorRules.ts"; +import { resetDbInstance } from "../../src/lib/db/core.ts"; + +// #13657 rework: the 429 classifier is kept; stopping the cross-account wave at a +// classified 429 is opt-in (OPENCODE_RATE_LIMITED_429_EARLY_STOP, default off — +// the free tier is per egress IP, #9611). Whatever ends the wave, the client gets +// the REAL last upstream 429 (status, body, Retry-After, quota headers), never a +// synthetic drain, so the opencode provider error rules keep matching it. +const FLAG = "OPENCODE_RATE_LIMITED_429_EARLY_STOP"; +const log: ExecutorLog = { debug() {}, info() {}, warn() {}, error() {} }; +const FPS = ["x", "y", "z"].map((c) => c.repeat(32)); +const MONTHLY_BODY = JSON.stringify({ + error: { + message: + "[429] Monthly usage limit reached. Resets in 13 days. To continue using this model now, enable usage from your available balance.", + }, +}); + +describe("classify429 / parseRetryAfterSeconds", () => { + it("a parseable Retry-After alone classifies rate_limited", () => { + assert.strictEqual(classify429({ retryAfter: "30" }), "rate_limited"); + assert.strictEqual(classify429({ retryAfter: 45 }), "rate_limited"); + const future = new Date(Date.now() + 120_000).toUTCString(); + assert.strictEqual(classify429({ retryAfter: future }), "rate_limited"); + }); + + it("a body naming a rate or usage limit classifies rate_limited", () => { + assert.strictEqual(classify429({ bodyText: "Rate limited, slow down" }), "rate_limited"); + assert.strictEqual(classify429({ bodyText: "Too many requests" }), "rate_limited"); + assert.strictEqual(classify429({ bodyText: MONTHLY_BODY }), "rate_limited"); + }); + + it("anything else is a burst", () => { + assert.strictEqual(classify429({}), "burst"); + assert.strictEqual(classify429({ bodyText: '{"error":"boom"}' }), "burst"); + assert.strictEqual(classify429({ retryAfter: "not-a-date" }), "burst"); + assert.strictEqual(classify429({ retryAfter: "" }), "burst"); + }); + + it("parses delta-seconds and HTTP dates against an injected clock", () => { + const now = Date.parse("2026-09-15T00:00:00Z"); + assert.strictEqual(parseRetryAfterSeconds("30", now), 30); + assert.strictEqual(parseRetryAfterSeconds("Tue, 15 Sep 2026 00:02:00 GMT", now), 120); + assert.strictEqual(parseRetryAfterSeconds("not-a-date", now), null); + assert.strictEqual(parseRetryAfterSeconds(-5, now), null); + }); +}); + +describe("readBodyPrefix / classifyUpstream429", () => { + it("reads only a bounded prefix and leaves the original body intact", async () => { + const body = "a".repeat(RATE_LIMIT_BODY_SNIFF_BYTES) + " rate limited"; + const response = new Response(body, { status: 429 }); + const prefix = await readBodyPrefix(response); + assert.strictEqual( + prefix?.length, + RATE_LIMIT_BODY_SNIFF_BYTES, + "signal past the cap is unseen" + ); + assert.strictEqual(await classifyUpstream429(response), "burst"); + assert.strictEqual(response.bodyUsed, false); + assert.strictEqual(await response.text(), body, "the caller still gets the full body"); + }); + + it("checks the header before touching the body", async () => { + const response = new Response("Too many requests", { + status: 429, + headers: { "Retry-After": "7" }, + }); + assert.strictEqual(await classifyUpstream429(response), "rate_limited"); + assert.strictEqual(await response.text(), "Too many requests"); + }); +}); + +describe("OpencodeExecutor 429 wave", () => { + const servers: net.Server[] = []; + const ports: number[] = []; + let originalFetch: typeof globalThis.fetch; + let priorFlag: string | undefined; + let observed: string[]; + + before(async () => { + for (let i = 0; i < FPS.length; i++) { + const server = net.createServer((s) => s.destroy()); + servers.push(server); + ports.push( + await new Promise((resolve) => + server.listen(0, "127.0.0.1", () => resolve((server.address() as net.AddressInfo).port)) + ) + ); + } + }); + + after(() => { + servers.forEach((s) => s.close()); + resetDbInstance(); + }); + + beforeEach(() => { + originalFetch = globalThis.fetch; + priorFlag = process.env[FLAG]; + observed = []; + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + if (priorFlag === undefined) delete process.env[FLAG]; + else process.env[FLAG] = priorFlag; + }); + + type Step = { status: number; body?: string; headers?: Record }; + + function installFetch(plan: Step[]) { + let call = 0; + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = + typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + const resolved = resolveProxyForRequest(url); + observed.push(resolved.proxyUrl ? new URL(resolved.proxyUrl).port : "direct"); + const step = plan[Math.min(call, plan.length - 1)]; + call++; + return new Response(step.body ?? JSON.stringify({ ok: step.status === 200, call }), { + status: step.status, + headers: { "Content-Type": "application/json", ...(step.headers ?? {}) }, + }); + }) as typeof globalThis.fetch; + } + + function credentials(count: number): ProviderCredentials { + const fingerprints = FPS.slice(0, count); + return { + apiKey: null, + accessToken: null, + connectionId: "noauth", + providerSpecificData: { + fingerprints, + accountProxies: fingerprints.map((fp, i) => ({ + fingerprint: fp, + proxy: { type: "http", host: "127.0.0.1", port: ports[i] }, + })), + }, + }; + } + + async function run(exec: OpencodeExecutor, count: number) { + const result = (await exec.execute({ + model: "muse-spark-1.3-contributor-free", + body: { messages: [{ role: "user", content: "hi" }], stream: false }, + stream: false, + signal: null, + credentials: credentials(count), + log, + })) as { response: Response }; + return result.response; + } + + function cooled(exec: OpencodeExecutor): number { + const accounts = (exec as unknown as { accounts: Array<{ cooldownUntil: number }> }).accounts; + return accounts.filter((a) => a.cooldownUntil > Date.now()).length; + } + + const RATE_LIMITED: Step = { + status: 429, + body: MONTHLY_BODY, + headers: { "Retry-After": "30", "x-ratelimit-remaining-requests": "0" }, + }; + + it("flag off: a classified 429 still rotates to the next account (#9611)", async () => { + delete process.env[FLAG]; + const exec = new OpencodeExecutor("opencode-zen"); + installFetch([RATE_LIMITED, { status: 200 }]); + + const response = await run(exec, 2); + + assert.strictEqual(response.status, 200); + assert.strictEqual(observed.length, 2); + await response.body?.cancel(); + }); + + it("flag off: an exhausted wave returns the last real upstream 429 untouched", async () => { + delete process.env[FLAG]; + const exec = new OpencodeExecutor("opencode-zen"); + installFetch([ + { status: 429, body: '{"error":"first"}' }, + { status: 429, body: '{"error":"second"}' }, + RATE_LIMITED, + ]); + + const response = await run(exec, 3); + + assert.strictEqual(observed.length, 3); + assert.strictEqual(response.status, 429); + assert.strictEqual(response.headers.get("retry-after"), "30"); + assert.strictEqual(response.headers.get("x-ratelimit-remaining-requests"), "0"); + assert.strictEqual(response.headers.get("x-opencode-retry-state"), null, "nothing synthetic"); + assert.strictEqual(await response.text(), MONTHLY_BODY); + }); + + describe("flag on", () => { + beforeEach(() => { + process.env[FLAG] = "true"; + }); + + it("stops at the first classified 429 and returns it untouched", async () => { + const exec = new OpencodeExecutor("opencode-zen"); + installFetch([RATE_LIMITED, { status: 200 }]); + + const response = await run(exec, 3); + + assert.strictEqual(observed.length, 1, "no further account is tried"); + assert.strictEqual(response.status, 429); + assert.strictEqual(response.headers.get("retry-after"), "30"); + assert.strictEqual(response.headers.get("x-ratelimit-remaining-requests"), "0"); + assert.strictEqual(cooled(exec), 1, "the rate-limited account is cooled down"); + const text = await response.text(); + assert.strictEqual(text, MONTHLY_BODY, "upstream body preserved"); + const rule = getProviderErrorRuleMatch( + "opencode-zen", + 429, + Object.fromEntries(response.headers.entries()), + JSON.parse(text) + ); + assert.strictEqual(rule?.reason, "quota_exhausted", "provider error rules still match"); + assert.ok((rule?.cooldownMs ?? 0) > 24 * 60 * 60 * 1000, "the 13-day reset still applies"); + }); + + it("a body-only signal stops without inventing a Retry-After", async () => { + const exec = new OpencodeExecutor("opencode-zen"); + installFetch([{ status: 429, body: '{"error":"Rate limited"}' }, { status: 200 }]); + + const response = await run(exec, 2); + + assert.strictEqual(observed.length, 1); + assert.strictEqual(response.status, 429); + assert.strictEqual(response.headers.get("retry-after"), null); + assert.strictEqual(await response.text(), '{"error":"Rate limited"}'); + }); + + it("an unclassified (burst) 429 keeps rotating", async () => { + const exec = new OpencodeExecutor("opencode-zen"); + installFetch([{ status: 429, body: '{"error":"boom"}' }, { status: 200 }]); + + const response = await run(exec, 2); + + assert.strictEqual(response.status, 200); + assert.strictEqual(observed.length, 2); + await response.body?.cancel(); + }); + + it("an all-burst wave still returns the last real upstream 429", async () => { + const exec = new OpencodeExecutor("opencode-zen"); + installFetch([ + { status: 429, body: '{"error":"a"}' }, + { status: 429, body: '{"error":"b"}', headers: { "x-upstream": "last" } }, + ]); + + const response = await run(exec, 2); + + assert.strictEqual(observed.length, 2); + assert.strictEqual(response.status, 429); + assert.strictEqual(response.headers.get("x-upstream"), "last"); + assert.strictEqual(await response.text(), '{"error":"b"}'); + }); + }); +}); diff --git a/tests/unit/server-owned-tool-loop-flag.test.ts b/tests/unit/server-owned-tool-loop-flag.test.ts index 7136447948..efd7817287 100644 --- a/tests/unit/server-owned-tool-loop-flag.test.ts +++ b/tests/unit/server-owned-tool-loop-flag.test.ts @@ -68,7 +68,7 @@ describe("isServerOwnedToolLoopEnabled wrapper", () => { describe("feature-flags-settings count update", () => { it("flag count matches updated expected value", () => { - assert.equal(FEATURE_FLAG_DEFINITIONS.length, 68); + assert.equal(FEATURE_FLAG_DEFINITIONS.length, 69); }); });