mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-17 20:32:25 +03:00
* fix(resilience): don't cool down accounts or trip the breaker on client aborts When the caller drops the connection mid-stream, the in-flight request surfaces request_signal_aborted, "Client disconnected", or a DOM AbortError with no upstream status code. These shapes were counted as provider failures: the serving connection went into cooldown, the provider circuit breaker accrued failures, and healthy accounts ended up marked unavailable from client-side cancellations alone. Treat client aborts as local stream lifecycle events (#4602 policy): extend isLocalStreamLifecycleError() to recognize abort shapes and skip connection disable and breaker accounting for them. Genuine upstream failures (5xx/429/401) are still counted. Fixes #7907. * fix(resilience): guard the two remaining breaker-trip call sites against client aborts (#7907) PR #7908 correctly wired isLocalStreamLifecycleError() into shouldSkipConnDisable() and chatHelpers.ts's onStreamFailure, but two separate breaker._onFailure()-triggering call sites were purely status-code gated and never checked it, so a client-side abort (no upstream status, defaults to 502, error='request_signal_aborted') still tripped the whole-provider circuit breaker — the highest blast-radius of the 3 resilience mechanisms: - src/sse/handlers/chat.ts: the single-model, non-combo terminal-failure path called breaker._onFailure() directly, bypassing the isFailure option (which only applies inside breaker.execute()). Extracted the predicate into shouldTripProviderBreakerForResult() and added the missing isLocalStreamLifecycleError guard. - open-sse/services/combo/comboPredicates.ts::shouldRecordProviderBreakerFailure(), used by handleComboChat's executeTarget (open-sse/services/combo.ts), gained the same guard via a new optional `error` field. Added tests/unit/circuit-breaker-abort-provider-trip-7907.test.ts exercising both real predicates directly (not just the isolated isLocalStreamLifecycleError() helper) — confirmed red on the unfixed code (missing export) and green after the fix, alongside the existing #4602/#7908/combo-breaker-429 suites (24/24 pass, no regressions). file-size-baseline.json: +1 combo.ts (irreducible call-site wiring for the new `error` field) and +1 chatHelpers.ts (own growth from the PR's already-verified onStreamFailure guard, surfaced only now since fast-gates PR->release skip check:file-size). Co-authored-by: Diego Rodrigues de Sa e Souza <diegosouzapw@users.noreply.github.com> Co-authored-by: insoln <insoln@ya.ru> * test(quality): register circuit-breaker abort tests in stryker tap.testFiles The two new tests (circuit-breaker-abort-provider-trip-7907, circuit-breaker-client-abort) import mutated modules (circuitBreaker.ts, comboPredicates.ts) but were not listed in stryker.conf.json tap.testFiles, tripping check:mutation-test-coverage --strict. Co-authored-by: diegosouzapw <diegosouza.pw@gmail.com> --------- Co-authored-by: Diego Rodrigues de Sa e Souza <diegosouza.pw@gmail.com> Co-authored-by: Diego Rodrigues de Sa e Souza <diegosouzapw@users.noreply.github.com>
76 lines
3.0 KiB
TypeScript
76 lines
3.0 KiB
TypeScript
/**
|
|
* Client-side aborts must not cool down provider accounts or trip the breaker.
|
|
*
|
|
* When the caller drops the connection mid-stream (combo race loser, model
|
|
* switch in the client, tab close), the in-flight leg surfaces
|
|
* `request_signal_aborted` / `Client disconnected: ...` / DOM `AbortError`
|
|
* with no upstream status. Counting those as provider failures cascades one
|
|
* user action into provider cooldowns (`lastErrorCode=null`,
|
|
* `lastError=undefined` in the "all accounts cooling down" log) and can
|
|
* dead-end a combo on its last-resort target. Extends the #4602 local
|
|
* stream-lifecycle policy to client aborts.
|
|
*/
|
|
import { test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import {
|
|
CircuitBreaker,
|
|
isLocalStreamLifecycleError,
|
|
} from "../../src/shared/utils/circuitBreaker.ts";
|
|
|
|
const uniqueName = (suffix: string) =>
|
|
`cb-test-client-abort-${suffix}-${Date.now()}-${Math.floor(Math.random() * 1e6)}`;
|
|
|
|
test("client-abort shapes are local lifecycle errors", () => {
|
|
// streamHandler's default client-abort reason
|
|
assert.equal(isLocalStreamLifecycleError("request_signal_aborted"), true);
|
|
assert.equal(isLocalStreamLifecycleError(new Error("request_signal_aborted")), true);
|
|
// chatCore's client-disconnect failure message
|
|
assert.equal(
|
|
isLocalStreamLifecycleError(new Error("Client disconnected: request_signal_aborted")),
|
|
true
|
|
);
|
|
assert.equal(isLocalStreamLifecycleError({ message: "Client disconnected: model_switch" }), true);
|
|
// DOM AbortError (fetch aborted via AbortSignal)
|
|
const abortError = new Error("This operation was aborted");
|
|
abortError.name = "AbortError";
|
|
assert.equal(isLocalStreamLifecycleError(abortError), true);
|
|
// AbortError recognized by name even with a nonstandard message
|
|
const bareAbort = new Error("aborted");
|
|
bareAbort.name = "AbortError";
|
|
assert.equal(isLocalStreamLifecycleError(bareAbort), true);
|
|
});
|
|
|
|
test("genuine upstream failures still count as failures", () => {
|
|
assert.equal(isLocalStreamLifecycleError(new Error("502 Bad Gateway")), false);
|
|
assert.equal(isLocalStreamLifecycleError(new Error("upstream timed out")), false);
|
|
assert.equal(isLocalStreamLifecycleError(new Error("429 rate limited")), false);
|
|
assert.equal(
|
|
isLocalStreamLifecycleError(new Error("401 authentication_error: invalid x-api-key")),
|
|
false
|
|
);
|
|
assert.equal(isLocalStreamLifecycleError(undefined), false);
|
|
assert.equal(isLocalStreamLifecycleError(null), false);
|
|
assert.equal(isLocalStreamLifecycleError(""), false);
|
|
});
|
|
|
|
test("breaker stays CLOSED across repeated client aborts", async () => {
|
|
const cb = new CircuitBreaker(uniqueName("aborts"), {
|
|
failureThreshold: 3,
|
|
resetTimeout: 30_000,
|
|
isFailure: (e) => !isLocalStreamLifecycleError(e),
|
|
});
|
|
|
|
for (let i = 0; i < 5; i++) {
|
|
await assert.rejects(
|
|
cb.execute(async () => {
|
|
throw new Error("Client disconnected: request_signal_aborted");
|
|
}),
|
|
/request_signal_aborted/
|
|
);
|
|
}
|
|
|
|
assert.equal(cb.state, "CLOSED");
|
|
assert.equal(cb.failureCount, 0);
|
|
cb.reset();
|
|
});
|