fix(accounts): carry failure kind on rotation — transient cools down, terminal evicts (#11008)

5 — Rotação de conta não distinguia falha transitória (quota) de terminal (credencial morta) — ambas só esfriavam e eram retentadas para sempre. Agora markCooldown aceita kind transient/terminal; 3 terminais consecutivos evictam a conta (com fallback para não travar se todas evictadas). TDD, 17/17 testes, lint/typecheck/cycles OK.
This commit is contained in:
Dizzle
2026-08-21 18:58:51 +02:00
committed by GitHub
parent 8d076327f1
commit bc0a272bfc
4 changed files with 78 additions and 3 deletions

View File

@@ -0,0 +1 @@
- **fix(accounts):** `markCooldown` now carries the failure origin (`transient` vs `terminal`) — transient 429/network only cools down, repeated terminal failures evict and are skipped by `pickAccount` until a success or operator clear ([#11008](https://github.com/diegosouzapw/OmniRoute/pull/11008)) — thanks @maxmad64bis

View File

@@ -40,6 +40,15 @@ export interface RotatableAccount {
cooldownUntil: number;
consecutiveFails: number;
proxy: AccountProxyConfig["proxy"];
evictedAt?: number | null;
}
export type CooldownKind = "transient" | "terminal";
const EVICT_AFTER_TERMINAL = 3;
export function isAccountEvicted(account: RotatableAccount): boolean {
return account.evictedAt != null;
}
const COOLDOWN_BASE_MS = TRANSIENT_COOLDOWN_MS;
@@ -74,17 +83,21 @@ export function pickAccount<T extends RotatableAccount>(
return accounts[fallbackIdx];
}
export function markCooldown(account: RotatableAccount): void {
export function markCooldown(account: RotatableAccount, kind: CooldownKind = "transient"): void {
account.consecutiveFails++;
const backoff = Math.min(
COOLDOWN_BASE_MS * Math.pow(2, account.consecutiveFails - 1),
COOLDOWN_MAX_MS
);
account.cooldownUntil = Date.now() + backoff + Math.random() * 1000;
if (kind === "terminal" && account.consecutiveFails >= EVICT_AFTER_TERMINAL) {
account.evictedAt = Date.now();
}
}
export function markSuccess(account: RotatableAccount): void {
account.consecutiveFails = 0;
account.evictedAt = null;
}
/** Mask an account id for logs (UI calls it a fingerprint). */

View File

@@ -193,8 +193,8 @@ export class OpencodeExecutor extends BaseExecutor {
return pickRotatableAccount(this.accounts, this);
}
private markCooldown(account: OpencodeAccountState): void {
markAccountCooldown(account);
private markCooldown(account: OpencodeAccountState, kind: "transient" | "terminal" = "transient"): void {
markAccountCooldown(account, kind);
}
private markSuccess(account: OpencodeAccountState): void {

View File

@@ -0,0 +1,61 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { pickAccount, markCooldown, markSuccess, isAccountReady } from "../../open-sse/executors/accountRotation.ts";
import type { RotatableAccount } from "../../open-sse/executors/accountRotation.ts";
function acct(fp: string, proxy: RotatableAccount["proxy"] = null): RotatableAccount {
return { fingerprint: fp, cooldownUntil: 0, consecutiveFails: 0, proxy };
}
test("markCooldown default is transient — no eviction, only backoff", () => {
const a = acct("a");
markCooldown(a); // kind omitted → transient
assert.ok(a.cooldownUntil > Date.now());
assert.equal((a as any).evictedAt, undefined);
// still picked when others are ready
const state = { nextAccountIdx: 0 };
const picked = pickAccount([a, acct("b")], state);
assert.ok(picked.fingerprint === "a" || picked.fingerprint === "b");
});
test("terminal kind evicts after threshold, pickAccount skips evicted unless all evicted", () => {
const a = acct("dead");
const b = acct("healthy");
// 3 terminal fails → evicted (threshold = 3, borne testable)
markCooldown(a, "terminal");
markCooldown(a, "terminal");
markCooldown(a, "terminal");
assert.ok((a as any).evictedAt != null);
const state = { nextAccountIdx: 0 };
// b is ready, a evicted → b is picked
const picked = pickAccount([a, b], state, (x) => isAccountReady(x) && !(x as any).evictedAt);
assert.equal(picked.fingerprint, "healthy");
// when all evicted, caller still gets an account rather than hanging (preserves :52-58)
(b as any).evictedAt = Date.now();
const fallback = pickAccount([a, b], { nextAccountIdx: 0 }, (x) => isAccountReady(x) && !(x as any).evictedAt);
assert.ok(fallback.fingerprint === "dead" || fallback.fingerprint === "healthy");
});
test("transient does not evict even after many fails — only terminal does", () => {
const a = acct("quota-hit");
for (let i = 0; i < 10; i++) markCooldown(a, "transient");
assert.equal((a as any).evictedAt, undefined);
});
test("markSuccess clears eviction and consecutiveFails", () => {
const a = acct("revived");
markCooldown(a, "terminal"); markCooldown(a, "terminal"); markCooldown(a, "terminal");
markSuccess(a);
assert.equal((a as any).evictedAt, null);
assert.equal(a.consecutiveFails, 0);
});
test("cross-executor alias still works — opencode wrapper forwards kind", async () => {
// not a DB test — asserts the shared helper accepts the second arg
const { markCooldown: mc } = await import("../../open-sse/executors/accountRotation.ts");
// Optional second param — length 1 means first required, second optional (JS length counts required only)
assert.ok(mc.length >= 1 && mc.length <= 2);
// Prove it accepts terminal without throw
const tmp = acct("probe");
assert.doesNotThrow(() => (mc as any)(tmp, "terminal"));
});