fix(resilience): isolate local host execution errors from provider circuit breakers (#12233)

Local process execution failures (ENOENT spawn errors, binary missing, EPIPE, exit codes) were incorrectly treated as upstream provider failures, opening provider circuit breakers and cooling down valid connections. Added `isLocalExecutionError` guard to skip circuit breaker trips and connection disables when local host execution fails.

open-sse/services/combo/comboPredicates.ts

@@ -10,7 +10,7 @@ import { EXECUTOR_CONTRACT_VIOLATION_CODE } from "../../config/constants.ts";
 import { errorResponse } from "../../utils/error.ts";
 import { parseModel } from "../model.ts";
 import { isSelfInflictedUpstreamTimeout } from "../../handlers/chatCore/cooldownClassification.ts";
-import { isLocalStreamLifecycleError } from "@/shared/utils/circuitBreaker";
+import { isLocalStreamLifecycleError, isLocalExecutionError } from "@/shared/utils/circuitBreaker";
 import { CONTEXT_OVERFLOW_PATTERNS, MODEL_ACCESS_DENIED_PATTERNS } from "../accountFallback.ts";
 import { isResourceNotFoundResponse } from "../errorClassifier.ts";
 import { getTrustedLocalRateLimitResponse } from "../rateLimitManager/errors.ts";
@@ -216,7 +216,8 @@ export function shouldRecordProviderBreakerFailure(args: {
     (!args.sameProviderNext || args.isProxyUnreachable === true) &&
     !args.skipProviderBreaker &&
     !args.requestScopedFailure &&
-    !isLocalStreamLifecycleError(args.error)
+    !isLocalStreamLifecycleError(args.error) &&
+    !isLocalExecutionError(args.error)
   );
 }
 
@@ -313,6 +314,7 @@ export function shouldSkipConnDisable(
     // Client abort surfaced as a bare error (no statusCode → defaults to 502):
     // a local lifecycle event, not a provider failure (#4602 policy).
     isLocalStreamLifecycleError(result.error) ||
+    isLocalExecutionError(result.error) ||
     (result.response ? getTrustedLocalRateLimitResponse(result.response) !== null : false) ||
     result.errorCode === "plugin_block" ||
     result.errorType === "plugin_block" ||

src/shared/utils/circuitBreaker.ts

@@ -65,6 +65,43 @@ export function isLocalStreamLifecycleError(error: unknown): boolean {
   );
 }
 
+const LOCAL_EXECUTION_CODES = new Set([
+  "ENOENT",
+  "EACCES",
+  "EPIPE",
+  "ERR_CHILD_PROCESS_STDIO_MAXBUFFER",
+]);
+
+const LOCAL_EXECUTION_PATTERNS = [
+  /\bspawn\b.*\b(ENOENT|EACCES|EPIPE)\b/i,
+  /\bcommand not found\b/i,
+  /\bis not recognized as an internal or external command\b/i,
+  /\bchild process exited with code\b/i,
+  /\blocal host execution error\b/i,
+];
+
+/**
+ * Detect a LOCAL host execution error (missing binary ENOENT, permission EACCES,
+ * broken pipe EPIPE, child process exit errors, etc.) that must NOT count as a
+ * whole-provider failure or trip remote provider circuit breakers.
+ */
+export function isLocalExecutionError(error: unknown): boolean {
+  if (!error) return false;
+  const errObj = typeof error === "object" ? (error as Record<string, unknown>) : null;
+  const code = typeof errObj?.code === "string" ? errObj.code : "";
+  if (LOCAL_EXECUTION_CODES.has(code)) return true;
+
+  const message =
+    typeof error === "string"
+      ? error
+      : typeof errObj?.message === "string"
+        ? (errObj.message as string)
+        : "";
+  if (!message) return false;
+
+  return LOCAL_EXECUTION_PATTERNS.some((p) => p.test(message));
+}
+
 export const STATE = {
   CLOSED: "CLOSED",
   DEGRADED: "DEGRADED",

src/sse/handlers/chatPredicates.ts

@@ -1,4 +1,7 @@
-import { isLocalStreamLifecycleError } from "../../shared/utils/circuitBreaker";
+import {
+  isLocalStreamLifecycleError,
+  isLocalExecutionError,
+} from "../../shared/utils/circuitBreaker";
 import { isRequestScopedUpstreamFailure } from "./comboFailureLogging";
 import { getTrustedLocalRateLimitResponse } from "@omniroute/open-sse/services/rateLimitManager/errors";
 
@@ -29,6 +32,7 @@ export function shouldTripProviderBreakerForResult(
     !isRequestScopedUpstreamFailure({ code: result.errorCode, type: result.errorType }) &&
     !(result.response && getTrustedLocalRateLimitResponse(result.response)) &&
     !isLocalStreamLifecycleError(result.error) &&
+    !isLocalExecutionError(result.error) &&
     // Network-layer errors (ECONNREFUSED, ETIMEDOUT) never reached the provider —
     // the provider may be healthy, only the network path is broken. OmniRoute's own
     // rate-limit queue timeouts are backpressure we applied, not a provider failure.

stryker.conf.json

@@ -144,6 +144,7 @@
       "tests/unit/circuit-breaker-abort-provider-trip-7907.test.ts",
       "tests/unit/circuit-breaker-client-abort.test.ts",
       "tests/unit/circuit-breaker-failure-kind.test.ts",
+      "tests/unit/circuit-breaker-local-execution.test.ts",
       "tests/unit/circuit-breaker-registry-cap.test.ts",
       "tests/unit/circuit-breaker-stream-controller-4602.test.ts",
       "tests/unit/claude-code-parity.test.ts",

tests/unit/circuit-breaker-local-execution.test.ts

@@ -0,0 +1,62 @@
+/**
+ * tests/unit/circuit-breaker-local-execution.test.ts
+ *
+ * Tests for local process execution error isolation:
+ * Local host execution faults (e.g. spawn ENOENT, binary missing, EPIPE, exit codes)
+ * must be identified via `isLocalExecutionError` and prevented from tripping provider-wide
+ * circuit breakers or marking provider connections/accounts as disabled.
+ */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { isLocalExecutionError } from "../../src/shared/utils/circuitBreaker.ts";
+import { shouldTripProviderBreakerForResult } from "../../src/sse/handlers/chatPredicates.ts";
+import {
+  shouldRecordProviderBreakerFailure,
+  shouldSkipConnDisable,
+} from "../../open-sse/services/combo/comboPredicates.ts";
+
+test("isLocalExecutionError: correctly identifies system spawn and process errors", () => {
+  assert.equal(isLocalExecutionError({ code: "ENOENT" }), true);
+  assert.equal(isLocalExecutionError({ code: "EACCES" }), true);
+  assert.equal(isLocalExecutionError({ code: "EPIPE" }), true);
+  assert.equal(isLocalExecutionError({ code: "ERR_CHILD_PROCESS_STDIO_MAXBUFFER" }), true);
+
+  assert.equal(isLocalExecutionError(new Error("spawn ollama ENOENT")), true);
+  assert.equal(isLocalExecutionError("command not found: llama-cli"), true);
+  assert.equal(isLocalExecutionError("child process exited with code 1"), true);
+  assert.equal(isLocalExecutionError("local host execution error: process killed"), true);
+
+  assert.equal(isLocalExecutionError(new Error("502 Bad Gateway")), false);
+  assert.equal(isLocalExecutionError({ code: "ECONNREFUSED" }), false);
+  assert.equal(isLocalExecutionError(null), false);
+  assert.equal(isLocalExecutionError(undefined), false);
+});
+
+test("shouldTripProviderBreakerForResult: local execution error does NOT trip single-model breaker", () => {
+  const result = {
+    status: 500,
+    error: new Error("spawn llama-cli ENOENT"),
+  };
+  assert.equal(shouldTripProviderBreakerForResult(result, false, false), false);
+});
+
+test("shouldRecordProviderBreakerFailure: local execution error does NOT record failure for combo breaker", () => {
+  assert.equal(
+    shouldRecordProviderBreakerFailure({
+      isStreamReadinessFailure: false,
+      status: 500,
+      sameProviderNext: false,
+      error: new Error("spawn python ENOENT"),
+    }),
+    false
+  );
+});
+
+test("shouldSkipConnDisable: local execution error skips disabling provider connection", () => {
+  const result = {
+    status: 500,
+    error: { code: "ENOENT" },
+  };
+  assert.equal(shouldSkipConnDisable(result, false, false, "local-provider"), true);
+});