/** * Unit tests for src/lib/chaos/chaosExecutor.ts * * Regression coverage for #6679: the original PR dispatched to models via a real * network fetch() to `${OMNIROUTE_INTERNAL_URL || "http://localhost:30129"}` — * 30129 is the author's personal dev port, not the project's default port 20128, * so Chaos Mode silently failed on any standard install. This now dispatches * in-process via the established synthetic-Request/route-handler pattern (see * src/lib/batches/dispatch.ts, src/lib/evals/runtime.ts) — no network hop, no * port dependency. `chatDispatch.postChatCompletion` is mocked here (same * technique tests/unit/batch-processor.test.ts uses for dispatch.ts) so these * tests never hit a real upstream provider. */ import test, { mock } from "node:test"; import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-chaos-executor-")); const ORIGINAL_DATA_DIR = process.env.DATA_DIR; process.env.DATA_DIR = TEST_DATA_DIR; const core = await import("../../src/lib/db/core.ts"); const providersDb = await import("../../src/lib/db/providers.ts"); const chaosConfig = await import("../../src/lib/chaos/chaosConfig.ts"); const chaosExecutor = await import("../../src/lib/chaos/chaosExecutor.ts"); async function resetStorage() { core.resetDbInstance(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); } test.beforeEach(async () => { await resetStorage(); }); test.afterEach(() => { mock.restoreAll(); }); test.after(() => { core.resetDbInstance(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); if (ORIGINAL_DATA_DIR === undefined) { delete process.env.DATA_DIR; } else { process.env.DATA_DIR = ORIGINAL_DATA_DIR; } }); function jsonResponse(body: unknown, status = 200): Response { return new Response(JSON.stringify(body), { status, headers: { "Content-Type": "application/json" }, }); } test("executeChaosRun throws when no active provider connections exist", async () => { await assert.rejects( () => chaosExecutor.executeChaosRun({ task: "hello" }), /No active provider connections found/ ); }); test("executeChaosRun dispatches in-process (no fetch/network call) and returns per-provider results", async () => { await providersDb.createProviderConnection({ provider: "openai", authType: "apikey", name: "Test OpenAI", apiKey: "sk-test-openai", defaultModel: "gpt-4o-mini", }); let capturedRequest: Request | null = null; mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async (req: Request) => { capturedRequest = req; return jsonResponse({ choices: [{ message: { content: "hello from openai" } }] }); }); const result = await chaosExecutor.executeChaosRun({ task: "Summarize this repo" }); assert.equal(result.mode, "parallel"); assert.equal(result.totalProviders, 1); assert.equal(result.models.length, 1); assert.equal(result.models[0].status, "success"); assert.equal(result.models[0].content, "hello from openai"); assert.ok(capturedRequest, "postChatCompletion should have been invoked in-process"); assert.equal( (capturedRequest as unknown as Request).url, "http://localhost/api/v1/chat/completions" ); }); test("executeChaosRun forwards the caller's apiKey as a Bearer Authorization header", async () => { await providersDb.createProviderConnection({ provider: "anthropic", authType: "apikey", name: "Test Anthropic", apiKey: "sk-test-anthropic", defaultModel: "claude-3-5-sonnet", }); let capturedAuth: string | null = null; mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async (req: Request) => { capturedAuth = req.headers.get("Authorization"); return jsonResponse({ choices: [{ message: { content: "ok" } }] }); }); await chaosExecutor.executeChaosRun({ task: "task", apiKey: "sk-caller-key" }); assert.equal(capturedAuth, "Bearer sk-caller-key"); }); test("executeChaosRun omits Authorization header when no apiKey is provided (dashboard/local mode)", async () => { await providersDb.createProviderConnection({ provider: "openai", authType: "apikey", name: "Test OpenAI", apiKey: "sk-test-openai-2", defaultModel: "gpt-4o-mini", }); let capturedAuth: string | null | undefined; mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async (req: Request) => { capturedAuth = req.headers.get("Authorization"); return jsonResponse({ choices: [{ message: { content: "ok" } }] }); }); await chaosExecutor.executeChaosRun({ task: "task" }); assert.equal(capturedAuth, null); }); test("executeChaosRun surfaces upstream errors per-model instead of throwing", async () => { await providersDb.createProviderConnection({ provider: "openai", authType: "apikey", name: "Failing OpenAI", apiKey: "sk-test-failing", defaultModel: "gpt-4o-mini", }); mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async () => jsonResponse({ error: "upstream exploded" }, 502) ); const result = await chaosExecutor.executeChaosRun({ task: "task" }); assert.equal(result.models[0].status, "error"); assert.match(result.models[0].error ?? "", /API 502/); }); test("executeChaosRun collaborative mode chains context between sequential model calls", async () => { await providersDb.createProviderConnection({ provider: "openai", authType: "apikey", name: "Model A", apiKey: "sk-test-a", defaultModel: "gpt-4o-mini", }); await providersDb.createProviderConnection({ provider: "anthropic", authType: "apikey", name: "Model B", apiKey: "sk-test-b", defaultModel: "claude-3-5-sonnet", }); const seenUserMessages: string[] = []; mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async (req: Request) => { const body = (await req.json()) as { messages: { role: string; content: string }[] }; const userMessage = body.messages.find((m) => m.role === "user"); seenUserMessages.push(userMessage?.content ?? ""); return jsonResponse({ choices: [{ message: { content: `reply-${seenUserMessages.length}` } }], }); }); const result = await chaosExecutor.executeChaosRun({ task: "Original task", mode: "collaborative", }); assert.equal(result.mode, "collaborative"); assert.equal(result.models.length, 2); assert.equal(seenUserMessages[0], "Original task"); assert.match(seenUserMessages[1], /Previous model's output:\nreply-1/); assert.ok(result.summary && result.summary.includes("reply-2")); }); test("executeChaosRun respects an explicit providers filter and errors when none match", async () => { await providersDb.createProviderConnection({ provider: "openai", authType: "apikey", name: "Only OpenAI", apiKey: "sk-test-only-openai", defaultModel: "gpt-4o-mini", }); mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async () => jsonResponse({ choices: [{ message: { content: "ok" } }] }) ); await assert.rejects( () => chaosExecutor.executeChaosRun({ task: "task", providers: ["anthropic"] }), /None of the specified providers are active/ ); const result = await chaosExecutor.executeChaosRun({ task: "task", providers: ["openai"] }); assert.equal(result.models.length, 1); }); test("executeChaosRun applies global config provider overrides for model resolution", async () => { await providersDb.createProviderConnection({ provider: "openai", authType: "apikey", name: "Override target", apiKey: "sk-test-override", defaultModel: "gpt-4o-mini", }); await chaosConfig.setChaosConfig({ enabled: true, defaultMode: "parallel", providerOverrides: [{ providerId: "openai", modelId: "gpt-4-turbo", enabled: true }], timeoutMs: 120_000, maxTokens: 4096, }); let capturedModel: string | undefined; mock.method(chaosExecutor.chatDispatch, "postChatCompletion", async (req: Request) => { const body = (await req.json()) as { model: string }; capturedModel = body.model; return jsonResponse({ choices: [{ message: { content: "ok" } }] }); }); await chaosExecutor.executeChaosRun({ task: "task" }); assert.equal(capturedModel, "gpt-4-turbo"); });