diff --git a/open-sse/services/combo/comboPredicates.ts b/open-sse/services/combo/comboPredicates.ts index 606c7c4ca4..3ad18343b3 100644 --- a/open-sse/services/combo/comboPredicates.ts +++ b/open-sse/services/combo/comboPredicates.ts @@ -10,7 +10,7 @@ import { EXECUTOR_CONTRACT_VIOLATION_CODE } from "../../config/constants.ts"; import { errorResponse } from "../../utils/error.ts"; import { parseModel } from "../model.ts"; import { isSelfInflictedUpstreamTimeout } from "../../handlers/chatCore/cooldownClassification.ts"; -import { isLocalStreamLifecycleError } from "@/shared/utils/circuitBreaker"; +import { isLocalStreamLifecycleError, isLocalExecutionError } from "@/shared/utils/circuitBreaker"; import { CONTEXT_OVERFLOW_PATTERNS, MODEL_ACCESS_DENIED_PATTERNS } from "../accountFallback.ts"; import { isResourceNotFoundResponse } from "../errorClassifier.ts"; import { getTrustedLocalRateLimitResponse } from "../rateLimitManager/errors.ts"; @@ -216,7 +216,8 @@ export function shouldRecordProviderBreakerFailure(args: { (!args.sameProviderNext || args.isProxyUnreachable === true) && !args.skipProviderBreaker && !args.requestScopedFailure && - !isLocalStreamLifecycleError(args.error) + !isLocalStreamLifecycleError(args.error) && + !isLocalExecutionError(args.error) ); } @@ -313,6 +314,7 @@ export function shouldSkipConnDisable( // Client abort surfaced as a bare error (no statusCode → defaults to 502): // a local lifecycle event, not a provider failure (#4602 policy). isLocalStreamLifecycleError(result.error) || + isLocalExecutionError(result.error) || (result.response ? getTrustedLocalRateLimitResponse(result.response) !== null : false) || result.errorCode === "plugin_block" || result.errorType === "plugin_block" || diff --git a/src/shared/utils/circuitBreaker.ts b/src/shared/utils/circuitBreaker.ts index 724112f526..773d757aa2 100644 --- a/src/shared/utils/circuitBreaker.ts +++ b/src/shared/utils/circuitBreaker.ts @@ -65,6 +65,43 @@ export function isLocalStreamLifecycleError(error: unknown): boolean { ); } +const LOCAL_EXECUTION_CODES = new Set([ + "ENOENT", + "EACCES", + "EPIPE", + "ERR_CHILD_PROCESS_STDIO_MAXBUFFER", +]); + +const LOCAL_EXECUTION_PATTERNS = [ + /\bspawn\b.*\b(ENOENT|EACCES|EPIPE)\b/i, + /\bcommand not found\b/i, + /\bis not recognized as an internal or external command\b/i, + /\bchild process exited with code\b/i, + /\blocal host execution error\b/i, +]; + +/** + * Detect a LOCAL host execution error (missing binary ENOENT, permission EACCES, + * broken pipe EPIPE, child process exit errors, etc.) that must NOT count as a + * whole-provider failure or trip remote provider circuit breakers. + */ +export function isLocalExecutionError(error: unknown): boolean { + if (!error) return false; + const errObj = typeof error === "object" ? (error as Record) : null; + const code = typeof errObj?.code === "string" ? errObj.code : ""; + if (LOCAL_EXECUTION_CODES.has(code)) return true; + + const message = + typeof error === "string" + ? error + : typeof errObj?.message === "string" + ? (errObj.message as string) + : ""; + if (!message) return false; + + return LOCAL_EXECUTION_PATTERNS.some((p) => p.test(message)); +} + export const STATE = { CLOSED: "CLOSED", DEGRADED: "DEGRADED", diff --git a/src/sse/handlers/chatPredicates.ts b/src/sse/handlers/chatPredicates.ts index 2a4ab74988..db9abd3da2 100644 --- a/src/sse/handlers/chatPredicates.ts +++ b/src/sse/handlers/chatPredicates.ts @@ -1,4 +1,7 @@ -import { isLocalStreamLifecycleError } from "../../shared/utils/circuitBreaker"; +import { + isLocalStreamLifecycleError, + isLocalExecutionError, +} from "../../shared/utils/circuitBreaker"; import { isRequestScopedUpstreamFailure } from "./comboFailureLogging"; import { getTrustedLocalRateLimitResponse } from "@omniroute/open-sse/services/rateLimitManager/errors"; @@ -29,6 +32,7 @@ export function shouldTripProviderBreakerForResult( !isRequestScopedUpstreamFailure({ code: result.errorCode, type: result.errorType }) && !(result.response && getTrustedLocalRateLimitResponse(result.response)) && !isLocalStreamLifecycleError(result.error) && + !isLocalExecutionError(result.error) && // Network-layer errors (ECONNREFUSED, ETIMEDOUT) never reached the provider — // the provider may be healthy, only the network path is broken. OmniRoute's own // rate-limit queue timeouts are backpressure we applied, not a provider failure. diff --git a/stryker.conf.json b/stryker.conf.json index 117a117ac4..1520a105e4 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -144,6 +144,7 @@ "tests/unit/circuit-breaker-abort-provider-trip-7907.test.ts", "tests/unit/circuit-breaker-client-abort.test.ts", "tests/unit/circuit-breaker-failure-kind.test.ts", + "tests/unit/circuit-breaker-local-execution.test.ts", "tests/unit/circuit-breaker-registry-cap.test.ts", "tests/unit/circuit-breaker-stream-controller-4602.test.ts", "tests/unit/claude-code-parity.test.ts", diff --git a/tests/unit/circuit-breaker-local-execution.test.ts b/tests/unit/circuit-breaker-local-execution.test.ts new file mode 100644 index 0000000000..70bc5b3f95 --- /dev/null +++ b/tests/unit/circuit-breaker-local-execution.test.ts @@ -0,0 +1,62 @@ +/** + * tests/unit/circuit-breaker-local-execution.test.ts + * + * Tests for local process execution error isolation: + * Local host execution faults (e.g. spawn ENOENT, binary missing, EPIPE, exit codes) + * must be identified via `isLocalExecutionError` and prevented from tripping provider-wide + * circuit breakers or marking provider connections/accounts as disabled. + */ + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { isLocalExecutionError } from "../../src/shared/utils/circuitBreaker.ts"; +import { shouldTripProviderBreakerForResult } from "../../src/sse/handlers/chatPredicates.ts"; +import { + shouldRecordProviderBreakerFailure, + shouldSkipConnDisable, +} from "../../open-sse/services/combo/comboPredicates.ts"; + +test("isLocalExecutionError: correctly identifies system spawn and process errors", () => { + assert.equal(isLocalExecutionError({ code: "ENOENT" }), true); + assert.equal(isLocalExecutionError({ code: "EACCES" }), true); + assert.equal(isLocalExecutionError({ code: "EPIPE" }), true); + assert.equal(isLocalExecutionError({ code: "ERR_CHILD_PROCESS_STDIO_MAXBUFFER" }), true); + + assert.equal(isLocalExecutionError(new Error("spawn ollama ENOENT")), true); + assert.equal(isLocalExecutionError("command not found: llama-cli"), true); + assert.equal(isLocalExecutionError("child process exited with code 1"), true); + assert.equal(isLocalExecutionError("local host execution error: process killed"), true); + + assert.equal(isLocalExecutionError(new Error("502 Bad Gateway")), false); + assert.equal(isLocalExecutionError({ code: "ECONNREFUSED" }), false); + assert.equal(isLocalExecutionError(null), false); + assert.equal(isLocalExecutionError(undefined), false); +}); + +test("shouldTripProviderBreakerForResult: local execution error does NOT trip single-model breaker", () => { + const result = { + status: 500, + error: new Error("spawn llama-cli ENOENT"), + }; + assert.equal(shouldTripProviderBreakerForResult(result, false, false), false); +}); + +test("shouldRecordProviderBreakerFailure: local execution error does NOT record failure for combo breaker", () => { + assert.equal( + shouldRecordProviderBreakerFailure({ + isStreamReadinessFailure: false, + status: 500, + sameProviderNext: false, + error: new Error("spawn python ENOENT"), + }), + false + ); +}); + +test("shouldSkipConnDisable: local execution error skips disabling provider connection", () => { + const result = { + status: 500, + error: { code: "ENOENT" }, + }; + assert.equal(shouldSkipConnDisable(result, false, false, "local-provider"), true); +});