diff --git a/.github/workflows/nightly-resilience.yml b/.github/workflows/nightly-resilience.yml new file mode 100644 index 0000000000..9af222d94d --- /dev/null +++ b/.github/workflows/nightly-resilience.yml @@ -0,0 +1,70 @@ +name: Nightly Resilience +on: + schedule: + - cron: "41 4 * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + heap: + name: Heap-growth gate + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: "24" + cache: npm + - run: npm ci + - run: npm run test:heap + + chaos: + name: Resilience chaos (fault injection) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: "24" + cache: npm + - run: npm ci + - run: npm run test:chaos + + k6-soak: + name: k6 load/soak + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: "24" + cache: npm + - run: npm ci + - name: Build CLI bundle + env: + JWT_SECRET: ci-build-secret-with-sufficient-length-for-validation + run: npm run build:cli + - name: Start OmniRoute (background) + env: + JWT_SECRET: ci-build-secret-with-sufficient-length-for-validation + PORT: "20128" + run: | + node dist/server.js > server.log 2>&1 & + echo $! > server.pid + for i in $(seq 1 30); do + if curl -sf http://localhost:20128/api/monitoring/health >/dev/null; then echo "server up"; break; fi + sleep 2 + done + - name: Install k6 + uses: grafana/setup-k6-action@v1 + - name: Run k6 soak + run: k6 run tests/load/k6-soak.js + env: + BASE_URL: http://localhost:20128 + SOAK_DURATION: "3m" + SOAK_VUS: "10" + - name: Stop server + if: always() + run: kill "$(cat server.pid)" || true diff --git a/docs/architecture/RESILIENCE_GUIDE.md b/docs/architecture/RESILIENCE_GUIDE.md index cd9f2c71df..92e2ba0779 100644 --- a/docs/architecture/RESILIENCE_GUIDE.md +++ b/docs/architecture/RESILIENCE_GUIDE.md @@ -144,6 +144,22 @@ Provider-specific stealth (JA3/JA4, CCH, obfuscation) is separately documented --- +## Resilience testing (Fase 8 · Bloco C) + +Além dos unit tests da lógica de resiliência, três testes exercitam o runtime sob +estresse/falha real (todos integração/nightly — nenhum bloqueia PR): + +| Teste | O quê | Rodar | +|---|---|---| +| Chaos | Fake-upstream node injeta latência/reset/timeout/503 reais; valida que o circuit breaker abre/recupera e `checkFallbackError` classifica 503 como fallback recuperável. | `RUN_CHAOS_INT=1 npm run test:chaos` | +| Heap-growth | ~500 streams por `createSSEStream` sob `--expose-gc`; falha se o heap crescer além do teto (guarda OOM #3069). | `npm run test:heap` | +| k6 soak | Carga sustentada contra `/api/monitoring/health`; thresholds p95/erro. | `k6 run tests/load/k6-soak.js` (nightly) | + +Orquestrados por `.github/workflows/nightly-resilience.yml` (cron + dispatch). No +`test:integration` default, chaos e heap se auto-skipam (sem `RUN_CHAOS_INT`/`--expose-gc`). + +--- + ## See Also - [Architecture Guide](./ARCHITECTURE.md) — System architecture and internals diff --git a/package.json b/package.json index 9df6c9ef93..784f2d00b6 100644 --- a/package.json +++ b/package.json @@ -156,6 +156,8 @@ "backfill-aggregation": "node --import tsx src/scripts/backfillAggregation.ts", "env:sync": "node scripts/dev/sync-env.mjs", "test:integration": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=1 tests/integration/*.test.ts", + "test:heap": "node --expose-gc --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit tests/integration/heap-growth.test.ts", + "test:chaos": "cross-env RUN_CHAOS_INT=1 node --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=1 tests/integration/resilience-chaos.test.ts", "test:e2e": "node scripts/dev/run-playwright-tests.mjs test tests/e2e/*.spec.ts", "test:protocols:e2e": "node scripts/dev/run-protocol-clients-tests.mjs", "test:vitest": "vitest run --config vitest.mcp.config.ts", diff --git a/tests/helpers/faultyUpstream.ts b/tests/helpers/faultyUpstream.ts new file mode 100644 index 0000000000..3eee4f1c19 --- /dev/null +++ b/tests/helpers/faultyUpstream.ts @@ -0,0 +1,72 @@ +import http from "node:http"; +import type { AddressInfo } from "node:net"; + +export type FaultMode = + | { kind: "ok"; body?: string } + | { kind: "status"; code: number; body?: string } + | { kind: "latency"; ms: number; body?: string } + | { kind: "reset" } + | { kind: "timeout" } + | { kind: "slowDrip"; chunkMs: number; body?: string }; + +export interface FaultyUpstream { + readonly url: string; + setMode(mode: FaultMode): void; + stop(): Promise; +} + +export async function startFaultyUpstream(initial: FaultMode = { kind: "ok" }): Promise { + let mode: FaultMode = initial; + + const server = http.createServer((req, res) => { + const m = mode; + if (m.kind === "reset") { + req.socket.destroy(); + return; + } + if (m.kind === "timeout") { + return; // never respond; caller relies on AbortSignal/timeout + } + if (m.kind === "latency") { + setTimeout(() => { + res.writeHead(200, { "content-type": "text/plain" }); + res.end(m.body ?? "ok"); + }, m.ms); + return; + } + if (m.kind === "slowDrip") { + const body = m.body ?? "drip-drip-drip"; + res.writeHead(200, { "content-type": "text/plain" }); + let i = 0; + const timer = setInterval(() => { + if (i >= body.length) { + clearInterval(timer); + res.end(); + return; + } + res.write(body[i++]); + }, m.chunkMs); + req.on("close", () => clearInterval(timer)); + return; + } + const code = m.kind === "status" ? m.code : 200; + res.writeHead(code, { "content-type": "text/plain" }); + res.end(m.body ?? (m.kind === "status" ? "error" : "ok")); + }); + + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + + return { + url: `http://127.0.0.1:${port}/`, + setMode(next: FaultMode) { + mode = next; + }, + stop() { + return new Promise((resolve) => { + server.closeAllConnections?.(); + server.close(() => resolve()); + }); + }, + }; +} diff --git a/tests/integration/heap-growth.test.ts b/tests/integration/heap-growth.test.ts new file mode 100644 index 0000000000..d541a9e9a2 --- /dev/null +++ b/tests/integration/heap-growth.test.ts @@ -0,0 +1,78 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +// DB setup required by createSSEStream (via trackPendingRequest / appendRequestLog) +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-heap-growth-")); +process.env.DATA_DIR = TEST_DATA_DIR; +const core = await import("../../src/lib/db/core.ts"); + +const { createSSEStream } = await import("../../open-sse/utils/stream.ts"); + +test.after(() => { + core.resetDbInstance(); + if (fs.existsSync(TEST_DATA_DIR)) { + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + } +}); + +const HAS_GC = typeof (globalThis as { gc?: () => void }).gc === "function"; +const enc = new TextEncoder(); + +function sseChunks(n: number): Uint8Array[] { + const parts: Uint8Array[] = []; + for (let i = 0; i < n; i++) { + parts.push( + enc.encode( + `data: {"id":"chatcmpl","object":"chat.completion.chunk","created":1,"model":"gpt-4o","choices":[{"delta":{"content":"x"},"index":0}]}\n\n` + ) + ); + } + parts.push(enc.encode(`data: [DONE]\n\n`)); + return parts; +} + +async function runOneStream(i: number): Promise { + const ts = createSSEStream({ + mode: "passthrough", + clientResponseFormat: "openai", + connectionId: `heap-${i}`, + }) as TransformStream; + const writer = ts.writable.getWriter(); + const reader = ts.readable.getReader(); + const chunks = sseChunks(10); + const pump = (async () => { + for (const c of chunks) await writer.write(c); + await writer.close(); + })(); + for (;;) { + const { done } = await reader.read(); + if (done) break; + } + await pump; +} + +async function gcSettle(): Promise { + const gc = (globalThis as { gc?: () => void }).gc!; + gc(); + await new Promise((r) => setTimeout(r, 50)); + gc(); +} + +test("SSE pipeline does not leak heap across 500 streams", { skip: !HAS_GC }, async () => { + for (let i = 0; i < 50; i++) await runOneStream(i); // warmup + await gcSettle(); + const baseline = process.memoryUsage().heapUsed; + + for (let i = 0; i < 500; i++) await runOneStream(i); + await gcSettle(); + const after = process.memoryUsage().heapUsed; + + const growthMB = (after - baseline) / 1024 / 1024; + console.log(`[heap] growth=${growthMB.toFixed(2)}MB after 500 streams`); + // Ceiling is 20MB — goal is detecting linear growth (leaks), not noise. + // If growth is stably near this boundary, raise to 30MB with justification. + assert.ok(growthMB < 20, `heap grew ${growthMB.toFixed(1)}MB after 500 streams (ceiling 20MB)`); +}); diff --git a/tests/integration/resilience-chaos.test.ts b/tests/integration/resilience-chaos.test.ts new file mode 100644 index 0000000000..bd16fdb993 --- /dev/null +++ b/tests/integration/resilience-chaos.test.ts @@ -0,0 +1,63 @@ +import { test, before, after, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { + getCircuitBreaker, + resetAllCircuitBreakers, + STATE, +} from "../../src/shared/utils/circuitBreaker.ts"; +import { checkFallbackError } from "../../open-sse/services/accountFallback.ts"; +import { startFaultyUpstream, type FaultyUpstream } from "../helpers/faultyUpstream.ts"; + +const RUN = process.env.RUN_CHAOS_INT === "1"; + +let up: FaultyUpstream; +before(async () => { + up = await startFaultyUpstream({ kind: "ok" }); +}); +after(async () => { + await up?.stop(); +}); +afterEach(() => { + resetAllCircuitBreakers(); +}); + +test("breaker OPENs after real timeouts, then refreshes to HALF_OPEN", { skip: !RUN }, async () => { + up.setMode({ kind: "timeout" }); + const breaker = getCircuitBreaker("chaos-timeout", { + failureThreshold: 3, + resetTimeout: 300, + isFailure: () => true, + }); + for (let i = 0; i < 3; i++) { + await assert.rejects(() => + breaker.execute(() => fetch(up.url, { signal: AbortSignal.timeout(120) })) + ); + } + assert.equal(breaker.canExecute(), false); + assert.equal(breaker.getStatus().state, STATE.OPEN); + await new Promise((r) => setTimeout(r, 330)); + assert.equal(breaker.getStatus().state, STATE.HALF_OPEN); + assert.equal(breaker.canExecute(), true); +}); + +test("breaker trips on a real connection reset", { skip: !RUN }, async () => { + up.setMode({ kind: "reset" }); + const breaker = getCircuitBreaker("chaos-reset", { + failureThreshold: 2, + resetTimeout: 300, + isFailure: () => true, + }); + for (let i = 0; i < 2; i++) { + await assert.rejects(() => breaker.execute(() => fetch(up.url))); + } + assert.equal(breaker.getStatus().state, STATE.OPEN); +}); + +test("checkFallbackError classifies a real 503 response as recoverable fallback", { skip: !RUN }, async () => { + up.setMode({ kind: "status", code: 503, body: "service unavailable" }); + const res = await fetch(up.url); + const text = await res.text(); + const decision = checkFallbackError(res.status, text, 0, null, "chaos-provider", res.headers); + assert.equal(decision.shouldFallback, true); + assert.ok(decision.cooldownMs > 0, "503 should yield a positive cooldown"); +}); diff --git a/tests/load/k6-soak.js b/tests/load/k6-soak.js new file mode 100644 index 0000000000..3ae72c0d03 --- /dev/null +++ b/tests/load/k6-soak.js @@ -0,0 +1,19 @@ +import http from "k6/http"; +import { check } from "k6"; + +// Soak contra um endpoint leve (sem custo de LLM) para medir liveness/leak do +// servidor sob carga sustentada. Configurável por env: BASE_URL, SOAK_VUS, SOAK_DURATION. +export const options = { + stages: [{ duration: __ENV.SOAK_DURATION || "3m", target: Number(__ENV.SOAK_VUS || 10) }], + thresholds: { + http_req_failed: ["rate<0.01"], + http_req_duration: ["p(95)<500"], + }, +}; + +const BASE_URL = __ENV.BASE_URL || "http://localhost:20128"; + +export default function () { + const res = http.get(`${BASE_URL}/api/monitoring/health`); + check(res, { "status is 200": (r) => r.status === 200 }); +} diff --git a/tests/unit/faulty-upstream.test.ts b/tests/unit/faulty-upstream.test.ts new file mode 100644 index 0000000000..729ee86d43 --- /dev/null +++ b/tests/unit/faulty-upstream.test.ts @@ -0,0 +1,48 @@ +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import { startFaultyUpstream } from "../helpers/faultyUpstream.ts"; + +test("ok mode returns 200 with body", async () => { + const up = await startFaultyUpstream({ kind: "ok", body: "hello" }); + after(() => up.stop()); + const res = await fetch(up.url); + assert.equal(res.status, 200); + assert.equal(await res.text(), "hello"); + await up.stop(); +}); + +test("status mode returns the given HTTP error code", async () => { + const up = await startFaultyUpstream({ kind: "status", code: 503, body: "down" }); + const res = await fetch(up.url); + assert.equal(res.status, 503); + assert.equal(await res.text(), "down"); + await up.stop(); +}); + +test("latency mode delays the response", async () => { + const up = await startFaultyUpstream({ kind: "latency", ms: 120, body: "slow" }); + const t0 = Date.now(); + await (await fetch(up.url)).text(); + assert.ok(Date.now() - t0 >= 100, "should be delayed ~120ms"); + await up.stop(); +}); + +test("reset mode destroys the socket (fetch rejects)", async () => { + const up = await startFaultyUpstream({ kind: "reset" }); + await assert.rejects(() => fetch(up.url).then((r) => r.text())); + await up.stop(); +}); + +test("timeout mode never responds (AbortSignal aborts)", async () => { + const up = await startFaultyUpstream({ kind: "timeout" }); + await assert.rejects(() => fetch(up.url, { signal: AbortSignal.timeout(120) })); + await up.stop(); +}); + +test("setMode switches behavior on a running server", async () => { + const up = await startFaultyUpstream({ kind: "ok", body: "a" }); + up.setMode({ kind: "status", code: 429, body: "rl" }); + const res = await fetch(up.url); + assert.equal(res.status, 429); + await up.stop(); +});