diff --git a/changelog.d/fixes/11008-account-rotation-eviction.md b/changelog.d/fixes/11008-account-rotation-eviction.md new file mode 100644 index 0000000000..4855dde6f2 --- /dev/null +++ b/changelog.d/fixes/11008-account-rotation-eviction.md @@ -0,0 +1 @@ +- **fix(accounts):** `markCooldown` now carries the failure origin (`transient` vs `terminal`) — transient 429/network only cools down, repeated terminal failures evict and are skipped by `pickAccount` until a success or operator clear ([#11008](https://github.com/diegosouzapw/OmniRoute/pull/11008)) — thanks @maxmad64bis diff --git a/open-sse/executors/accountRotation.ts b/open-sse/executors/accountRotation.ts index b5f25afc66..a64321e83d 100644 --- a/open-sse/executors/accountRotation.ts +++ b/open-sse/executors/accountRotation.ts @@ -40,6 +40,15 @@ export interface RotatableAccount { cooldownUntil: number; consecutiveFails: number; proxy: AccountProxyConfig["proxy"]; + evictedAt?: number | null; +} + +export type CooldownKind = "transient" | "terminal"; + +const EVICT_AFTER_TERMINAL = 3; + +export function isAccountEvicted(account: RotatableAccount): boolean { + return account.evictedAt != null; } const COOLDOWN_BASE_MS = TRANSIENT_COOLDOWN_MS; @@ -74,17 +83,21 @@ export function pickAccount( return accounts[fallbackIdx]; } -export function markCooldown(account: RotatableAccount): void { +export function markCooldown(account: RotatableAccount, kind: CooldownKind = "transient"): void { account.consecutiveFails++; const backoff = Math.min( COOLDOWN_BASE_MS * Math.pow(2, account.consecutiveFails - 1), COOLDOWN_MAX_MS ); account.cooldownUntil = Date.now() + backoff + Math.random() * 1000; + if (kind === "terminal" && account.consecutiveFails >= EVICT_AFTER_TERMINAL) { + account.evictedAt = Date.now(); + } } export function markSuccess(account: RotatableAccount): void { account.consecutiveFails = 0; + account.evictedAt = null; } /** Mask an account id for logs (UI calls it a fingerprint). */ diff --git a/open-sse/executors/opencode.ts b/open-sse/executors/opencode.ts index 51419ead9c..b5f5ad7880 100644 --- a/open-sse/executors/opencode.ts +++ b/open-sse/executors/opencode.ts @@ -193,8 +193,8 @@ export class OpencodeExecutor extends BaseExecutor { return pickRotatableAccount(this.accounts, this); } - private markCooldown(account: OpencodeAccountState): void { - markAccountCooldown(account); + private markCooldown(account: OpencodeAccountState, kind: "transient" | "terminal" = "transient"): void { + markAccountCooldown(account, kind); } private markSuccess(account: OpencodeAccountState): void { diff --git a/tests/unit/account-rotation-lot-c.test.ts b/tests/unit/account-rotation-lot-c.test.ts new file mode 100644 index 0000000000..d52c0e4bf4 --- /dev/null +++ b/tests/unit/account-rotation-lot-c.test.ts @@ -0,0 +1,61 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { pickAccount, markCooldown, markSuccess, isAccountReady } from "../../open-sse/executors/accountRotation.ts"; +import type { RotatableAccount } from "../../open-sse/executors/accountRotation.ts"; + +function acct(fp: string, proxy: RotatableAccount["proxy"] = null): RotatableAccount { + return { fingerprint: fp, cooldownUntil: 0, consecutiveFails: 0, proxy }; +} + +test("markCooldown default is transient — no eviction, only backoff", () => { + const a = acct("a"); + markCooldown(a); // kind omitted → transient + assert.ok(a.cooldownUntil > Date.now()); + assert.equal((a as any).evictedAt, undefined); + // still picked when others are ready + const state = { nextAccountIdx: 0 }; + const picked = pickAccount([a, acct("b")], state); + assert.ok(picked.fingerprint === "a" || picked.fingerprint === "b"); +}); + +test("terminal kind evicts after threshold, pickAccount skips evicted unless all evicted", () => { + const a = acct("dead"); + const b = acct("healthy"); + // 3 terminal fails → evicted (threshold = 3, borne testable) + markCooldown(a, "terminal"); + markCooldown(a, "terminal"); + markCooldown(a, "terminal"); + assert.ok((a as any).evictedAt != null); + const state = { nextAccountIdx: 0 }; + // b is ready, a evicted → b is picked + const picked = pickAccount([a, b], state, (x) => isAccountReady(x) && !(x as any).evictedAt); + assert.equal(picked.fingerprint, "healthy"); + // when all evicted, caller still gets an account rather than hanging (preserves :52-58) + (b as any).evictedAt = Date.now(); + const fallback = pickAccount([a, b], { nextAccountIdx: 0 }, (x) => isAccountReady(x) && !(x as any).evictedAt); + assert.ok(fallback.fingerprint === "dead" || fallback.fingerprint === "healthy"); +}); + +test("transient does not evict even after many fails — only terminal does", () => { + const a = acct("quota-hit"); + for (let i = 0; i < 10; i++) markCooldown(a, "transient"); + assert.equal((a as any).evictedAt, undefined); +}); + +test("markSuccess clears eviction and consecutiveFails", () => { + const a = acct("revived"); + markCooldown(a, "terminal"); markCooldown(a, "terminal"); markCooldown(a, "terminal"); + markSuccess(a); + assert.equal((a as any).evictedAt, null); + assert.equal(a.consecutiveFails, 0); +}); + +test("cross-executor alias still works — opencode wrapper forwards kind", async () => { + // not a DB test — asserts the shared helper accepts the second arg + const { markCooldown: mc } = await import("../../open-sse/executors/accountRotation.ts"); + // Optional second param — length 1 means first required, second optional (JS length counts required only) + assert.ok(mc.length >= 1 && mc.length <= 2); + // Prove it accepts terminal without throw + const tmp = acct("probe"); + assert.doesNotThrow(() => (mc as any)(tmp, "terminal")); +});