Fase 8 · Bloco C — resiliência runtime (chaos + heap-growth + k6 soak) (#3854)

Integrated into release/v3.8.25 — Fase 8 Bloco C (resilience: chaos + heap-growth + k6 soak).

.github/workflows/nightly-resilience.yml

@@ -0,0 +1,70 @@
+name: Nightly Resilience
+on:
+  schedule:
+    - cron: "41 4 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  heap:
+    name: Heap-growth gate
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-node@v6
+        with:
+          node-version: "24"
+          cache: npm
+      - run: npm ci
+      - run: npm run test:heap
+
+  chaos:
+    name: Resilience chaos (fault injection)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-node@v6
+        with:
+          node-version: "24"
+          cache: npm
+      - run: npm ci
+      - run: npm run test:chaos
+
+  k6-soak:
+    name: k6 load/soak
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-node@v6
+        with:
+          node-version: "24"
+          cache: npm
+      - run: npm ci
+      - name: Build CLI bundle
+        env:
+          JWT_SECRET: ci-build-secret-with-sufficient-length-for-validation
+        run: npm run build:cli
+      - name: Start OmniRoute (background)
+        env:
+          JWT_SECRET: ci-build-secret-with-sufficient-length-for-validation
+          PORT: "20128"
+        run: |
+          node dist/server.js > server.log 2>&1 &
+          echo $! > server.pid
+          for i in $(seq 1 30); do
+            if curl -sf http://localhost:20128/api/monitoring/health >/dev/null; then echo "server up"; break; fi
+            sleep 2
+          done
+      - name: Install k6
+        uses: grafana/setup-k6-action@v1
+      - name: Run k6 soak
+        run: k6 run tests/load/k6-soak.js
+        env:
+          BASE_URL: http://localhost:20128
+          SOAK_DURATION: "3m"
+          SOAK_VUS: "10"
+      - name: Stop server
+        if: always()
+        run: kill "$(cat server.pid)" || true

docs/architecture/RESILIENCE_GUIDE.md

@@ -144,6 +144,22 @@ Provider-specific stealth (JA3/JA4, CCH, obfuscation) is separately documented
 
 ---
 
+## Resilience testing (Fase 8 · Bloco C)
+
+Além dos unit tests da lógica de resiliência, três testes exercitam o runtime sob
+estresse/falha real (todos integração/nightly — nenhum bloqueia PR):
+
+| Teste | O quê | Rodar |
+|---|---|---|
+| Chaos | Fake-upstream node injeta latência/reset/timeout/503 reais; valida que o circuit breaker abre/recupera e `checkFallbackError` classifica 503 como fallback recuperável. | `RUN_CHAOS_INT=1 npm run test:chaos` |
+| Heap-growth | ~500 streams por `createSSEStream` sob `--expose-gc`; falha se o heap crescer além do teto (guarda OOM #3069). | `npm run test:heap` |
+| k6 soak | Carga sustentada contra `/api/monitoring/health`; thresholds p95/erro. | `k6 run tests/load/k6-soak.js` (nightly) |
+
+Orquestrados por `.github/workflows/nightly-resilience.yml` (cron + dispatch). No
+`test:integration` default, chaos e heap se auto-skipam (sem `RUN_CHAOS_INT`/`--expose-gc`).
+
+---
+
 ## See Also
 
 - [Architecture Guide](./ARCHITECTURE.md) — System architecture and internals

package.json

@@ -156,6 +156,8 @@
     "backfill-aggregation": "node --import tsx src/scripts/backfillAggregation.ts",
     "env:sync": "node scripts/dev/sync-env.mjs",
     "test:integration": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=1 tests/integration/*.test.ts",
+    "test:heap": "node --expose-gc --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit tests/integration/heap-growth.test.ts",
+    "test:chaos": "cross-env RUN_CHAOS_INT=1 node --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=1 tests/integration/resilience-chaos.test.ts",
     "test:e2e": "node scripts/dev/run-playwright-tests.mjs test tests/e2e/*.spec.ts",
     "test:protocols:e2e": "node scripts/dev/run-protocol-clients-tests.mjs",
     "test:vitest": "vitest run --config vitest.mcp.config.ts",

tests/helpers/faultyUpstream.ts

@@ -0,0 +1,72 @@
+import http from "node:http";
+import type { AddressInfo } from "node:net";
+
+export type FaultMode =
+  | { kind: "ok"; body?: string }
+  | { kind: "status"; code: number; body?: string }
+  | { kind: "latency"; ms: number; body?: string }
+  | { kind: "reset" }
+  | { kind: "timeout" }
+  | { kind: "slowDrip"; chunkMs: number; body?: string };
+
+export interface FaultyUpstream {
+  readonly url: string;
+  setMode(mode: FaultMode): void;
+  stop(): Promise<void>;
+}
+
+export async function startFaultyUpstream(initial: FaultMode = { kind: "ok" }): Promise<FaultyUpstream> {
+  let mode: FaultMode = initial;
+
+  const server = http.createServer((req, res) => {
+    const m = mode;
+    if (m.kind === "reset") {
+      req.socket.destroy();
+      return;
+    }
+    if (m.kind === "timeout") {
+      return; // never respond; caller relies on AbortSignal/timeout
+    }
+    if (m.kind === "latency") {
+      setTimeout(() => {
+        res.writeHead(200, { "content-type": "text/plain" });
+        res.end(m.body ?? "ok");
+      }, m.ms);
+      return;
+    }
+    if (m.kind === "slowDrip") {
+      const body = m.body ?? "drip-drip-drip";
+      res.writeHead(200, { "content-type": "text/plain" });
+      let i = 0;
+      const timer = setInterval(() => {
+        if (i >= body.length) {
+          clearInterval(timer);
+          res.end();
+          return;
+        }
+        res.write(body[i++]);
+      }, m.chunkMs);
+      req.on("close", () => clearInterval(timer));
+      return;
+    }
+    const code = m.kind === "status" ? m.code : 200;
+    res.writeHead(code, { "content-type": "text/plain" });
+    res.end(m.body ?? (m.kind === "status" ? "error" : "ok"));
+  });
+
+  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+  const { port } = server.address() as AddressInfo;
+
+  return {
+    url: `http://127.0.0.1:${port}/`,
+    setMode(next: FaultMode) {
+      mode = next;
+    },
+    stop() {
+      return new Promise<void>((resolve) => {
+        server.closeAllConnections?.();
+        server.close(() => resolve());
+      });
+    },
+  };
+}

tests/integration/heap-growth.test.ts

@@ -0,0 +1,78 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+// DB setup required by createSSEStream (via trackPendingRequest / appendRequestLog)
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-heap-growth-"));
+process.env.DATA_DIR = TEST_DATA_DIR;
+const core = await import("../../src/lib/db/core.ts");
+
+const { createSSEStream } = await import("../../open-sse/utils/stream.ts");
+
+test.after(() => {
+  core.resetDbInstance();
+  if (fs.existsSync(TEST_DATA_DIR)) {
+    fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  }
+});
+
+const HAS_GC = typeof (globalThis as { gc?: () => void }).gc === "function";
+const enc = new TextEncoder();
+
+function sseChunks(n: number): Uint8Array[] {
+  const parts: Uint8Array[] = [];
+  for (let i = 0; i < n; i++) {
+    parts.push(
+      enc.encode(
+        `data: {"id":"chatcmpl","object":"chat.completion.chunk","created":1,"model":"gpt-4o","choices":[{"delta":{"content":"x"},"index":0}]}\n\n`
+      )
+    );
+  }
+  parts.push(enc.encode(`data: [DONE]\n\n`));
+  return parts;
+}
+
+async function runOneStream(i: number): Promise<void> {
+  const ts = createSSEStream({
+    mode: "passthrough",
+    clientResponseFormat: "openai",
+    connectionId: `heap-${i}`,
+  }) as TransformStream<Uint8Array, Uint8Array>;
+  const writer = ts.writable.getWriter();
+  const reader = ts.readable.getReader();
+  const chunks = sseChunks(10);
+  const pump = (async () => {
+    for (const c of chunks) await writer.write(c);
+    await writer.close();
+  })();
+  for (;;) {
+    const { done } = await reader.read();
+    if (done) break;
+  }
+  await pump;
+}
+
+async function gcSettle(): Promise<void> {
+  const gc = (globalThis as { gc?: () => void }).gc!;
+  gc();
+  await new Promise((r) => setTimeout(r, 50));
+  gc();
+}
+
+test("SSE pipeline does not leak heap across 500 streams", { skip: !HAS_GC }, async () => {
+  for (let i = 0; i < 50; i++) await runOneStream(i); // warmup
+  await gcSettle();
+  const baseline = process.memoryUsage().heapUsed;
+
+  for (let i = 0; i < 500; i++) await runOneStream(i);
+  await gcSettle();
+  const after = process.memoryUsage().heapUsed;
+
+  const growthMB = (after - baseline) / 1024 / 1024;
+  console.log(`[heap] growth=${growthMB.toFixed(2)}MB after 500 streams`);
+  // Ceiling is 20MB — goal is detecting linear growth (leaks), not noise.
+  // If growth is stably near this boundary, raise to 30MB with justification.
+  assert.ok(growthMB < 20, `heap grew ${growthMB.toFixed(1)}MB after 500 streams (ceiling 20MB)`);
+});

tests/integration/resilience-chaos.test.ts

@@ -0,0 +1,63 @@
+import { test, before, after, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import {
+  getCircuitBreaker,
+  resetAllCircuitBreakers,
+  STATE,
+} from "../../src/shared/utils/circuitBreaker.ts";
+import { checkFallbackError } from "../../open-sse/services/accountFallback.ts";
+import { startFaultyUpstream, type FaultyUpstream } from "../helpers/faultyUpstream.ts";
+
+const RUN = process.env.RUN_CHAOS_INT === "1";
+
+let up: FaultyUpstream;
+before(async () => {
+  up = await startFaultyUpstream({ kind: "ok" });
+});
+after(async () => {
+  await up?.stop();
+});
+afterEach(() => {
+  resetAllCircuitBreakers();
+});
+
+test("breaker OPENs after real timeouts, then refreshes to HALF_OPEN", { skip: !RUN }, async () => {
+  up.setMode({ kind: "timeout" });
+  const breaker = getCircuitBreaker("chaos-timeout", {
+    failureThreshold: 3,
+    resetTimeout: 300,
+    isFailure: () => true,
+  });
+  for (let i = 0; i < 3; i++) {
+    await assert.rejects(() =>
+      breaker.execute(() => fetch(up.url, { signal: AbortSignal.timeout(120) }))
+    );
+  }
+  assert.equal(breaker.canExecute(), false);
+  assert.equal(breaker.getStatus().state, STATE.OPEN);
+  await new Promise((r) => setTimeout(r, 330));
+  assert.equal(breaker.getStatus().state, STATE.HALF_OPEN);
+  assert.equal(breaker.canExecute(), true);
+});
+
+test("breaker trips on a real connection reset", { skip: !RUN }, async () => {
+  up.setMode({ kind: "reset" });
+  const breaker = getCircuitBreaker("chaos-reset", {
+    failureThreshold: 2,
+    resetTimeout: 300,
+    isFailure: () => true,
+  });
+  for (let i = 0; i < 2; i++) {
+    await assert.rejects(() => breaker.execute(() => fetch(up.url)));
+  }
+  assert.equal(breaker.getStatus().state, STATE.OPEN);
+});
+
+test("checkFallbackError classifies a real 503 response as recoverable fallback", { skip: !RUN }, async () => {
+  up.setMode({ kind: "status", code: 503, body: "service unavailable" });
+  const res = await fetch(up.url);
+  const text = await res.text();
+  const decision = checkFallbackError(res.status, text, 0, null, "chaos-provider", res.headers);
+  assert.equal(decision.shouldFallback, true);
+  assert.ok(decision.cooldownMs > 0, "503 should yield a positive cooldown");
+});

tests/load/k6-soak.js

@@ -0,0 +1,19 @@
+import http from "k6/http";
+import { check } from "k6";
+
+// Soak contra um endpoint leve (sem custo de LLM) para medir liveness/leak do
+// servidor sob carga sustentada. Configurável por env: BASE_URL, SOAK_VUS, SOAK_DURATION.
+export const options = {
+  stages: [{ duration: __ENV.SOAK_DURATION || "3m", target: Number(__ENV.SOAK_VUS || 10) }],
+  thresholds: {
+    http_req_failed: ["rate<0.01"],
+    http_req_duration: ["p(95)<500"],
+  },
+};
+
+const BASE_URL = __ENV.BASE_URL || "http://localhost:20128";
+
+export default function () {
+  const res = http.get(`${BASE_URL}/api/monitoring/health`);
+  check(res, { "status is 200": (r) => r.status === 200 });
+}

tests/unit/faulty-upstream.test.ts

@@ -0,0 +1,48 @@
+import { test, after } from "node:test";
+import assert from "node:assert/strict";
+import { startFaultyUpstream } from "../helpers/faultyUpstream.ts";
+
+test("ok mode returns 200 with body", async () => {
+  const up = await startFaultyUpstream({ kind: "ok", body: "hello" });
+  after(() => up.stop());
+  const res = await fetch(up.url);
+  assert.equal(res.status, 200);
+  assert.equal(await res.text(), "hello");
+  await up.stop();
+});
+
+test("status mode returns the given HTTP error code", async () => {
+  const up = await startFaultyUpstream({ kind: "status", code: 503, body: "down" });
+  const res = await fetch(up.url);
+  assert.equal(res.status, 503);
+  assert.equal(await res.text(), "down");
+  await up.stop();
+});
+
+test("latency mode delays the response", async () => {
+  const up = await startFaultyUpstream({ kind: "latency", ms: 120, body: "slow" });
+  const t0 = Date.now();
+  await (await fetch(up.url)).text();
+  assert.ok(Date.now() - t0 >= 100, "should be delayed ~120ms");
+  await up.stop();
+});
+
+test("reset mode destroys the socket (fetch rejects)", async () => {
+  const up = await startFaultyUpstream({ kind: "reset" });
+  await assert.rejects(() => fetch(up.url).then((r) => r.text()));
+  await up.stop();
+});
+
+test("timeout mode never responds (AbortSignal aborts)", async () => {
+  const up = await startFaultyUpstream({ kind: "timeout" });
+  await assert.rejects(() => fetch(up.url, { signal: AbortSignal.timeout(120) }));
+  await up.stop();
+});
+
+test("setMode switches behavior on a running server", async () => {
+  const up = await startFaultyUpstream({ kind: "ok", body: "a" });
+  up.setMode({ kind: "status", code: 429, body: "rl" });
+  const res = await fetch(up.url);
+  assert.equal(res.status, 429);
+  await up.stop();
+});