mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-21 22:52:19 +03:00
fix(accounts): carry failure kind on rotation — transient cools down, terminal evicts (#11008)
⭐5 — Rotação de conta não distinguia falha transitória (quota) de terminal (credencial morta) — ambas só esfriavam e eram retentadas para sempre. Agora markCooldown aceita kind transient/terminal; 3 terminais consecutivos evictam a conta (com fallback para não travar se todas evictadas). TDD, 17/17 testes, lint/typecheck/cycles OK.
This commit is contained in:
1
changelog.d/fixes/11008-account-rotation-eviction.md
Normal file
1
changelog.d/fixes/11008-account-rotation-eviction.md
Normal file
@@ -0,0 +1 @@
|
||||
- **fix(accounts):** `markCooldown` now carries the failure origin (`transient` vs `terminal`) — transient 429/network only cools down, repeated terminal failures evict and are skipped by `pickAccount` until a success or operator clear ([#11008](https://github.com/diegosouzapw/OmniRoute/pull/11008)) — thanks @maxmad64bis
|
||||
@@ -40,6 +40,15 @@ export interface RotatableAccount {
|
||||
cooldownUntil: number;
|
||||
consecutiveFails: number;
|
||||
proxy: AccountProxyConfig["proxy"];
|
||||
evictedAt?: number | null;
|
||||
}
|
||||
|
||||
export type CooldownKind = "transient" | "terminal";
|
||||
|
||||
const EVICT_AFTER_TERMINAL = 3;
|
||||
|
||||
export function isAccountEvicted(account: RotatableAccount): boolean {
|
||||
return account.evictedAt != null;
|
||||
}
|
||||
|
||||
const COOLDOWN_BASE_MS = TRANSIENT_COOLDOWN_MS;
|
||||
@@ -74,17 +83,21 @@ export function pickAccount<T extends RotatableAccount>(
|
||||
return accounts[fallbackIdx];
|
||||
}
|
||||
|
||||
export function markCooldown(account: RotatableAccount): void {
|
||||
export function markCooldown(account: RotatableAccount, kind: CooldownKind = "transient"): void {
|
||||
account.consecutiveFails++;
|
||||
const backoff = Math.min(
|
||||
COOLDOWN_BASE_MS * Math.pow(2, account.consecutiveFails - 1),
|
||||
COOLDOWN_MAX_MS
|
||||
);
|
||||
account.cooldownUntil = Date.now() + backoff + Math.random() * 1000;
|
||||
if (kind === "terminal" && account.consecutiveFails >= EVICT_AFTER_TERMINAL) {
|
||||
account.evictedAt = Date.now();
|
||||
}
|
||||
}
|
||||
|
||||
export function markSuccess(account: RotatableAccount): void {
|
||||
account.consecutiveFails = 0;
|
||||
account.evictedAt = null;
|
||||
}
|
||||
|
||||
/** Mask an account id for logs (UI calls it a fingerprint). */
|
||||
|
||||
@@ -193,8 +193,8 @@ export class OpencodeExecutor extends BaseExecutor {
|
||||
return pickRotatableAccount(this.accounts, this);
|
||||
}
|
||||
|
||||
private markCooldown(account: OpencodeAccountState): void {
|
||||
markAccountCooldown(account);
|
||||
private markCooldown(account: OpencodeAccountState, kind: "transient" | "terminal" = "transient"): void {
|
||||
markAccountCooldown(account, kind);
|
||||
}
|
||||
|
||||
private markSuccess(account: OpencodeAccountState): void {
|
||||
|
||||
61
tests/unit/account-rotation-lot-c.test.ts
Normal file
61
tests/unit/account-rotation-lot-c.test.ts
Normal file
@@ -0,0 +1,61 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { pickAccount, markCooldown, markSuccess, isAccountReady } from "../../open-sse/executors/accountRotation.ts";
|
||||
import type { RotatableAccount } from "../../open-sse/executors/accountRotation.ts";
|
||||
|
||||
function acct(fp: string, proxy: RotatableAccount["proxy"] = null): RotatableAccount {
|
||||
return { fingerprint: fp, cooldownUntil: 0, consecutiveFails: 0, proxy };
|
||||
}
|
||||
|
||||
test("markCooldown default is transient — no eviction, only backoff", () => {
|
||||
const a = acct("a");
|
||||
markCooldown(a); // kind omitted → transient
|
||||
assert.ok(a.cooldownUntil > Date.now());
|
||||
assert.equal((a as any).evictedAt, undefined);
|
||||
// still picked when others are ready
|
||||
const state = { nextAccountIdx: 0 };
|
||||
const picked = pickAccount([a, acct("b")], state);
|
||||
assert.ok(picked.fingerprint === "a" || picked.fingerprint === "b");
|
||||
});
|
||||
|
||||
test("terminal kind evicts after threshold, pickAccount skips evicted unless all evicted", () => {
|
||||
const a = acct("dead");
|
||||
const b = acct("healthy");
|
||||
// 3 terminal fails → evicted (threshold = 3, borne testable)
|
||||
markCooldown(a, "terminal");
|
||||
markCooldown(a, "terminal");
|
||||
markCooldown(a, "terminal");
|
||||
assert.ok((a as any).evictedAt != null);
|
||||
const state = { nextAccountIdx: 0 };
|
||||
// b is ready, a evicted → b is picked
|
||||
const picked = pickAccount([a, b], state, (x) => isAccountReady(x) && !(x as any).evictedAt);
|
||||
assert.equal(picked.fingerprint, "healthy");
|
||||
// when all evicted, caller still gets an account rather than hanging (preserves :52-58)
|
||||
(b as any).evictedAt = Date.now();
|
||||
const fallback = pickAccount([a, b], { nextAccountIdx: 0 }, (x) => isAccountReady(x) && !(x as any).evictedAt);
|
||||
assert.ok(fallback.fingerprint === "dead" || fallback.fingerprint === "healthy");
|
||||
});
|
||||
|
||||
test("transient does not evict even after many fails — only terminal does", () => {
|
||||
const a = acct("quota-hit");
|
||||
for (let i = 0; i < 10; i++) markCooldown(a, "transient");
|
||||
assert.equal((a as any).evictedAt, undefined);
|
||||
});
|
||||
|
||||
test("markSuccess clears eviction and consecutiveFails", () => {
|
||||
const a = acct("revived");
|
||||
markCooldown(a, "terminal"); markCooldown(a, "terminal"); markCooldown(a, "terminal");
|
||||
markSuccess(a);
|
||||
assert.equal((a as any).evictedAt, null);
|
||||
assert.equal(a.consecutiveFails, 0);
|
||||
});
|
||||
|
||||
test("cross-executor alias still works — opencode wrapper forwards kind", async () => {
|
||||
// not a DB test — asserts the shared helper accepts the second arg
|
||||
const { markCooldown: mc } = await import("../../open-sse/executors/accountRotation.ts");
|
||||
// Optional second param — length 1 means first required, second optional (JS length counts required only)
|
||||
assert.ok(mc.length >= 1 && mc.length <= 2);
|
||||
// Prove it accepts terminal without throw
|
||||
const tmp = acct("probe");
|
||||
assert.doesNotThrow(() => (mc as any)(tmp, "terminal"));
|
||||
});
|
||||
Reference in New Issue
Block a user