From a448b146bf0360697312ac7d431e3bc7fa0fdb09 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Sun, 9 Aug 2026 09:53:07 -0300 Subject: [PATCH] cherry-pick(pr-9744): test(integration): add general live-test tool for the real "default" combo + rootless wire capture (#9862) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(integration): add general live-test tool for the real "default" combo Temporary WIP commit on this deferred branch — lands in its own separate PR once the bug-fix extraction batch is done (never bundled into a bug-fix PR). Unlike liveGeminiShared.ts (provisions its own narrow 2-model Gemini-only combo), this reads the REAL "default" combo currently configured on the target instance directly from the DB and exercises every provider/model step in it directly, bypassing combo routing, so live-test coverage always matches whatever is actually configured instead of a hardcoded snapshot. Live-verified against omniroute-beta (seeded with the real 18-model, 5-provider default combo): 14/18 models pass consistently across non-streaming + streaming Chat Completions and streaming Responses API. The 4 consistent failures are real external state (cerebras credits_exhausted, one deprecated openrouter free-tier model), not code regressions. (cherry picked from commit c40b13a48fd897259c56f5122e9e57a3dc7654ba) * test(integration): add rootless wire-capture correlation to the live-test tool Temporary WIP commit on this deferred branch — lands in the same final live-test-tool PR as the general default-combo suite, never bundled into a bug-fix PR. liveContainerHarness.ts spins up a dedicated, throwaway podman container (same runner-base image target as the operator's local dev/beta containers) so wire-capture tests are fully self-contained: builds the image if missing, starts the container with a persistent data dir, waits for health, seeds the real "default" combo + provider connections from the operator's local omniroute-dev instance (idempotent — only runs once per data dir), and provisions API keys via the running instance's own auth flow. wireCapture.ts captures the container's actual network traffic via `podman unshare nsenter --net= -- tcpdump` — no root needed, verified working live (this generalizes the root-requiring `sudo nsenter -t $PID` command scripts/sre/tcp-close-analyzer.py already documented for the same rootless-Podman netns problem; that script's docstring now documents both). Capture and analysis needed two real fixes found only by running the pipeline live: `-U` (unbuffered tcpdump writes) plus a `pkill -f ` fallback, since `podman unshare -> nsenter -> tcpdump` is a 3-level subprocess chain and SIGTERM to the top-level process doesn't reach the tcpdump grandchild, leaving an orphaned process and a truncated/unreadable pcap; and filtering on the container's internal listening port (20128) rather than the dynamically-assigned host port, since capture happens inside the container's own network namespace where only the internal port is meaningful. live-default-combo-wire-capture.test.ts (gated on RUN_LIVE_WIRE_CAPTURE=1) ties it together: sends a small representative sample of requests through the real default combo, then cross-checks each one's app-level JSON status against the actual HTTP status line observed on the wire via scripts/sre/tcp-close-analyzer.py's stream reassembly — catching bugs where the app layer claims success but the wire shows a truncated/reset stream, not just what liveDefaultComboShared.ts's existing breadth suite already covers. Live-verified end-to-end: 4/4 sampled requests correlated correctly across 8 captured TCP streams, container + capture process fully torn down afterward (verified no orphaned podman container or tcpdump process left running). sendModelRequest/filterActiveModelTargets (liveDefaultComboShared.ts) gain optional baseUrl/apiKey overrides, defaulting to the existing module-level omniroute-beta target, so the wire-capture suite can point the same request-sending logic at its own dedicated container instead. (cherry picked from commit 914a7e42cbe914f257db9f72eedc902ee1532083) --------- Co-authored-by: Markus Hartung --- scripts/sre/tcp-close-analyzer.py | 22 +- .../live-default-combo-wire-capture.test.ts | 143 ++++++++++ .../live-default-combo-workload.test.ts | 113 ++++++++ tests/integration/liveContainerHarness.ts | 260 +++++++++++++++++ tests/integration/liveDefaultComboShared.ts | 266 ++++++++++++++++++ tests/integration/wireCapture.ts | 154 ++++++++++ 6 files changed, 956 insertions(+), 2 deletions(-) create mode 100644 tests/integration/live-default-combo-wire-capture.test.ts create mode 100644 tests/integration/live-default-combo-workload.test.ts create mode 100644 tests/integration/liveContainerHarness.ts create mode 100644 tests/integration/liveDefaultComboShared.ts create mode 100644 tests/integration/wireCapture.ts diff --git a/scripts/sre/tcp-close-analyzer.py b/scripts/sre/tcp-close-analyzer.py index 77f489799b..6b01ad2034 100755 --- a/scripts/sre/tcp-close-analyzer.py +++ b/scripts/sre/tcp-close-analyzer.py @@ -17,8 +17,8 @@ parsing the libpcap file format and IPv4/TCP headers directly. Good enough for this one question; not a general-purpose pcap toolkit. ──────────────────────────────────────────────────────────────────────────── -CAPTURING (run this yourself — needs root/sudo for CAP_NET_RAW; also see ---show-capture-cmd) +CAPTURING (run this yourself — needs root/sudo for CAP_NET_RAW, UNLESS you +use the rootless method below; also see --show-capture-cmd) ──────────────────────────────────────────────────────────────────────────── Rootless Podman gotcha: there is usually NO `podman3`/`podmanN` bridge @@ -33,6 +33,24 @@ container's OWN namespace via its PID instead: sudo nsenter -t "$PID" -n tcpdump -i any -w /tmp/omniroute-capture.pcap \\ 'host and port 20128' +Rootless alternative (NO sudo needed): a bare `nsenter -t $PID -n` fails +with "Invalid argument" for a rootless container, because its network +namespace lives inside a user namespace you're not in yet. `podman unshare` +puts you in that same user namespace first, so `nsenter --net=` against the +container's netns path succeeds as a plain user — verified working live +(captured a real `POST /v1/chat/completions` request body in cleartext this +way, no root at any point): + + NETNS=$(podman inspect omniroute-dev --format '{{.NetworkSettings.SandboxKey}}') + podman unshare nsenter --net="$NETNS" -- \\ + tcpdump -i any -w /tmp/omniroute-capture.pcap 'port 20128' + +No `sudo chmod` needed afterward either, since the file was never +root-owned. This is also what +tests/integration/wireCapture.ts + liveContainerHarness.ts automate for the +live wire-capture test suite (its own dedicated throwaway container, not +omniroute-dev) — see RUN_LIVE_WIRE_CAPTURE=1 in that test file. + Find the container's IP first with: podman inspect omniroute-dev --format '{{.NetworkSettings.Networks}}' diff --git a/tests/integration/live-default-combo-wire-capture.test.ts b/tests/integration/live-default-combo-wire-capture.test.ts new file mode 100644 index 0000000000..d1454ac012 --- /dev/null +++ b/tests/integration/live-default-combo-wire-capture.test.ts @@ -0,0 +1,143 @@ +/** + * tests/integration/live-default-combo-wire-capture.test.ts + * + * Wire-level correlation test. Spins up a dedicated, throwaway podman + * container (liveContainerHarness.ts), captures its network traffic + * (wireCapture.ts — rootless tcpdump via `podman unshare nsenter`, no root), + * sends a representative sample of requests against the real "default" + * combo, then cross-checks each request's app-level result (JSON status) + * against what actually went out on the wire (HTTP response status line, + * verdict on who closed the connection first). Catches bugs where the app + * layer claims success but the wire shows a truncated/reset stream. + * + * Fully self-contained — does not touch omniroute-beta or omniroute-dev + * (only reads from omniroute-dev's DB once, to seed its own dedicated + * container's data dir). Gated on RUN_LIVE_WIRE_CAPTURE=1: needs podman, + * tcpdump, python3, and a real .env with provider credentials, so it must + * never run in CI. + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + LIVE_CONTAINER_ENABLED, + startLiveContainer, + type LiveContainerHandle, +} from "./liveContainerHarness.ts"; +import { + startWireCapture, + analyzeCapture, + indexByCorrelationId, + responseStatusLine, + type CaptureHandle, +} from "./wireCapture.ts"; +import { + getDefaultComboModelTargets, + filterActiveModelTargets, + sendModelRequest, +} from "./liveDefaultComboShared.ts"; + +const skip = !LIVE_CONTAINER_ENABLED + ? "RUN_LIVE_WIRE_CAPTURE not set — skipping wire-capture live test" + : undefined; + +// Wire-level correlation is the point of this suite, not breadth across +// every provider (already covered by live-default-combo-workload.test.ts) — +// keep the sample small so capture/analysis stays fast. +const SAMPLE_SIZE = 4; + +let container: LiveContainerHandle; +let capture: CaptureHandle; + +test.before(async () => { + if (skip) return; + container = await startLiveContainer(); + process.env.DATA_DIR = container.dataDir; + + // PID-scoped so a concurrent session running this same test never + // collides on the capture file or the pkill-by-path cleanup in + // wireCapture.ts's stop(). + const pcapPath = `/tmp/omniroute-live-wire-capture-${process.pid}.pcap`; + // Capture happens INSIDE the container's own netns (podman unshare + // nsenter --net=), so packets there are addressed to the + // container's internal listening port (20128), not the dynamically + // assigned host port used to reach it from outside — filtering on + // hostPort here would silently match nothing. + capture = await startWireCapture(container.netnsPath, pcapPath, "tcp port 20128"); +}); + +test.after(async () => { + if (skip) return; + await capture?.stop(); + await container?.stop(); +}); + +test( + "wire capture: app-level status matches the HTTP status line actually observed on the wire", + { skip }, + async () => { + const allTargets = await getDefaultComboModelTargets(); + assert.ok(allTargets.length > 0, `"default" combo has no model steps — nothing to test`); + + const { active } = await filterActiveModelTargets(allTargets, { + baseUrl: container.baseUrl, + apiKey: container.managementApiKey, + }); + assert.ok(active.length > 0, "no active provider connections in the seeded container"); + + const sample = active.slice(0, SAMPLE_SIZE); + console.log( + `\n [wire-capture] sampling ${sample.length} model(s): ${sample.map((t) => t.model).join(", ")}` + ); + + const results = await Promise.all( + sample.map((t) => + sendModelRequest(t.model, false, "chat", { + baseUrl: container.baseUrl, + apiKey: container.apiKey, + }) + ) + ); + + // Give the capture a moment to flush the last packets before analyzing. + await new Promise((r) => setTimeout(r, 1000)); + await capture.stop(); + const streams = await analyzeCapture(capture.pcapPath); + const byCorrelationId = indexByCorrelationId(streams); + + console.log(` [wire-capture] captured ${streams.length} TCP stream(s)`); + + const mismatches: string[] = []; + for (const r of results) { + if (r.correlationId === "?") { + mismatches.push(`${r.model}: no correlationId returned in response headers`); + continue; + } + const matched = byCorrelationId.get(r.correlationId); + if (!matched || matched.length === 0) { + mismatches.push( + `${r.model}: correlationId ${r.correlationId} not found in any captured wire stream` + ); + continue; + } + const wireStatusLines = matched.map(responseStatusLine).filter(Boolean); + const wireStatusCodes = wireStatusLines.map((line) => line!.split(" ")[1]); + if (!wireStatusCodes.includes(String(r.status))) { + mismatches.push( + `${r.model}: app-level status ${r.status} but wire shows ${wireStatusCodes.join(",") || "no status line"} (cid ${r.correlationId})` + ); + } + } + + if (mismatches.length > 0) { + console.log(`\n Wire/app-level mismatches (${mismatches.length}):`); + for (const m of mismatches) console.log(` ${m}`); + } + + assert.equal( + mismatches.length, + 0, + `${mismatches.length}/${results.length} requests had app-level results that don't match what was observed on the wire` + ); + } +); diff --git a/tests/integration/live-default-combo-workload.test.ts b/tests/integration/live-default-combo-workload.test.ts new file mode 100644 index 0000000000..1e502f278b --- /dev/null +++ b/tests/integration/live-default-combo-workload.test.ts @@ -0,0 +1,113 @@ +/** + * tests/integration/live-default-combo-workload.test.ts + * + * General breadth test against the REAL, currently-configured "default" + * combo on the target instance — unlike live-gemini-workload.test.ts (which + * provisions its own narrow 2-model Gemini-only combo), this targets every + * provider/model step the operator actually has in "default" directly, + * bypassing combo routing. One request per configured model: non-streaming + * + streaming Chat Completions, and streaming Responses API. Skips (never + * fails) any model whose provider connection isn't currently active, so one + * unrelated provider outage doesn't block the rest of the run. + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + skip, + getDefaultComboModelTargets, + filterActiveModelTargets, + sendModelRequest, +} from "./liveDefaultComboShared.ts"; + +let modelNames: string[] = []; + +test.before(async () => { + if (skip) return; + const targets = await getDefaultComboModelTargets(); + assert.ok(targets.length > 0, `"default" combo has no model steps — nothing to test`); + + const { active, skipped } = await filterActiveModelTargets(targets); + if (skipped.length > 0) { + console.log(`\n [setup] skipping ${skipped.length} model(s) with inactive provider:`); + for (const s of skipped) console.log(` - ${s}`); + } + + modelNames = active.map((t) => t.model); + console.log(`\n [setup] testing ${modelNames.length} model(s) from the live "default" combo`); +}); + +test( + "[32] default combo: non-streaming chat completions across every configured model", + { skip }, + async () => { + const failures: string[] = []; + for (const model of modelNames) { + const r = await sendModelRequest(model, false, "chat"); + if (r.status !== 200 || r.contentLength === 0) { + failures.push( + `${model}: HTTP ${r.status}${r.error ? ` (${r.error})` : ""}, ${r.contentLength} chars` + ); + } + } + if (failures.length > 0) { + console.log(`\n Non-streaming failures (${failures.length}/${modelNames.length}):`); + for (const f of failures) console.log(` ${f}`); + } + assert.equal( + failures.length, + 0, + `${failures.length}/${modelNames.length} models failed non-streaming chat` + ); + } +); + +test( + "[33] default combo: streaming chat completions across every configured model", + { skip }, + async () => { + const failures: string[] = []; + for (const model of modelNames) { + const r = await sendModelRequest(model, true, "chat"); + if (r.status !== 200 || r.contentLength === 0) { + failures.push( + `${model}: HTTP ${r.status}${r.error ? ` (${r.error})` : ""}, ${r.contentLength} chars` + ); + } + } + if (failures.length > 0) { + console.log(`\n Streaming failures (${failures.length}/${modelNames.length}):`); + for (const f of failures) console.log(` ${f}`); + } + assert.equal( + failures.length, + 0, + `${failures.length}/${modelNames.length} models failed streaming chat` + ); + } +); + +test( + "[34] default combo: streaming responses API across every configured model", + { skip }, + async () => { + const failures: string[] = []; + for (const model of modelNames) { + const r = await sendModelRequest(model, true, "responses"); + if (r.status !== 200 || r.contentLength === 0) { + failures.push( + `${model}: HTTP ${r.status}${r.error ? ` (${r.error})` : ""}, ${r.contentLength} chars` + ); + } + } + if (failures.length > 0) { + console.log(`\n Responses API failures (${failures.length}/${modelNames.length}):`); + for (const f of failures) console.log(` ${f}`); + } + assert.equal( + failures.length, + 0, + `${failures.length}/${modelNames.length} models failed streaming Responses API` + ); + } +); diff --git a/tests/integration/liveContainerHarness.ts b/tests/integration/liveContainerHarness.ts new file mode 100644 index 0000000000..168fa2d211 --- /dev/null +++ b/tests/integration/liveContainerHarness.ts @@ -0,0 +1,260 @@ +/** + * tests/integration/liveContainerHarness.ts + * + * Spins up a dedicated, throwaway podman container running this checkout's + * own code (runner-base target, same as the operator's local dev/beta + * containers) so wire-capture live tests are fully self-contained — no + * dependency on a manually-managed systemd quadlet. + * + * The container's DATA_DIR is a persistent host directory (not wiped between + * runs) so the "default" combo + real provider connections only need + * seeding once; seeding is idempotent and copies from the operator's local + * omniroute-dev instance (same source used for the manual omniroute-beta + * seed earlier this session). + */ +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import Database from "better-sqlite3"; + +const REPO_ROOT = fileURLToPath(new URL("../..", import.meta.url)); + +export const LIVE_CONTAINER_ENABLED = process.env.RUN_LIVE_WIRE_CAPTURE === "1"; + +const IMAGE_TAG = process.env.LIVE_CONTAINER_IMAGE || "localhost/omniroute:live-wire-test"; +const CONTAINER_NAME = process.env.LIVE_CONTAINER_NAME || "omniroute-live-wire-test"; +const DATA_DIR_HOST = + process.env.LIVE_CONTAINER_DATA_DIR || "/data/podman-data/omniroute-live-wire-test/data"; +const ENV_FILE = process.env.LIVE_CONTAINER_ENV_FILE || "/data/podman-data/omniroute/omniroute.env"; +// Source DB to seed the "default" combo + provider connections from — the +// operator's local omniroute-dev instance, same source used for the manual +// omniroute-beta seed earlier this session. +const SEED_SOURCE_DB = + process.env.LIVE_CONTAINER_SEED_SOURCE_DB || + "/home/markus/code/podman/OmniRoute/data/storage.sqlite"; +const SEED_PROVIDERS = ["gemini", "openrouter", "mistral", "cerebras"]; + +export interface LiveContainerHandle { + baseUrl: string; + apiKey: string; + managementApiKey: string; + containerName: string; + netnsPath: string; + hostPort: number; + dataDir: string; + stop(): Promise; +} + +function run(cmd: string, args: string[], opts: { input?: string } = {}): string { + const result = spawnSync(cmd, args, { + cwd: REPO_ROOT, + encoding: "utf8", + input: opts.input, + maxBuffer: 64 * 1024 * 1024, + }); + if (result.status !== 0) { + throw new Error( + `${cmd} ${args.join(" ")} failed (exit ${result.status}):\n${result.stderr || result.stdout}` + ); + } + return result.stdout.trim(); +} + +function tryRun(cmd: string, args: string[]): string | null { + const result = spawnSync(cmd, args, { cwd: REPO_ROOT, encoding: "utf8" }); + return result.status === 0 ? result.stdout.trim() : null; +} + +function ensureImageBuilt(): void { + const existing = tryRun("podman", ["images", "-q", IMAGE_TAG]); + if (existing) { + console.log(` [container] image ${IMAGE_TAG} already exists (${existing}) — reusing`); + return; + } + console.log(` [container] building ${IMAGE_TAG} (runner-base target — this takes a while)...`); + run("podman", ["build", "--target", "runner-base", "-t", IMAGE_TAG, "."]); +} + +function stopExistingContainer(): void { + tryRun("podman", ["rm", "-f", CONTAINER_NAME]); +} + +function startContainer(): { hostPort: number; netnsPath: string } { + if (!existsSync(DATA_DIR_HOST)) { + mkdirSync(DATA_DIR_HOST, { recursive: true }); + } + // podman unshare owns the rootless user namespace these directories' + // native uid mappings live in — plain chmod as the host user fails with + // EPERM on files podman previously wrote as a different mapped uid. + tryRun("podman", ["unshare", "chmod", "-R", "a+rwX", DATA_DIR_HOST]); + + run("podman", [ + "run", + "-d", + "--name", + CONTAINER_NAME, + "-p", + "127.0.0.1::20128", + "-v", + `${DATA_DIR_HOST}:/app/data`, + "--env-file", + ENV_FILE, + IMAGE_TAG, + ]); + + const portOutput = run("podman", ["port", CONTAINER_NAME, "20128/tcp"]); + const hostPort = Number(portOutput.split(":").pop()); + if (!Number.isFinite(hostPort)) { + throw new Error(`could not parse assigned host port from: ${portOutput}`); + } + + const netnsPath = run("podman", [ + "inspect", + CONTAINER_NAME, + "--format", + "{{.NetworkSettings.SandboxKey}}", + ]); + + return { hostPort, netnsPath }; +} + +async function waitForHealth(baseUrl: string, timeoutMs = 60_000): Promise { + const deadline = Date.now() + timeoutMs; + let lastError: unknown; + while (Date.now() < deadline) { + try { + const res = await fetch(`${baseUrl}/api/monitoring/health`); + if (res.ok) return; + } catch (err) { + lastError = err; + } + await new Promise((r) => setTimeout(r, 1000)); + } + throw new Error(`container never became healthy within ${timeoutMs}ms: ${lastError}`); +} + +// Idempotent: only copies rows if the target has no "default" combo yet. +// Direct SQLite access (not the src/lib/db/ CRUD functions) is deliberate +// here, same as liveGeminiShared.ts's ensureGeminiProvider() — cloning an +// existing row's already-encrypted apiKey blob byte-for-byte has no CRUD +// equivalent, and both instances share the same API_KEY_SECRET (same +// --env-file), so the encrypted value decrypts correctly on the target too. +async function seedDefaultComboAndConnections(): Promise { + const targetPath = `${DATA_DIR_HOST}/storage.sqlite`; + if (!existsSync(targetPath)) { + console.log(` [container] target DB not created yet, skipping seed this pass`); + return; + } + if (!existsSync(SEED_SOURCE_DB)) { + console.warn(` [container] seed source DB not found at ${SEED_SOURCE_DB} — skipping seed`); + return; + } + + const target = new Database(targetPath); + const existingCombo = target.prepare("SELECT 1 FROM combos WHERE name = 'default'").get(); + if (existingCombo) { + console.log(` [container] "default" combo already seeded — skipping`); + target.close(); + return; + } + + const source = new Database(SEED_SOURCE_DB, { readonly: true }); + const connCols = source.prepare("PRAGMA table_info(provider_connections)").all() as Array<{ + name: string; + }>; + const colList = connCols.map((c) => `"${c.name}"`).join(","); + const placeholders = connCols.map((c) => `@${c.name}`).join(","); + const insertConn = target.prepare( + `INSERT OR REPLACE INTO provider_connections (${colList}) VALUES (${placeholders})` + ); + + let copied = 0; + for (const provider of SEED_PROVIDERS) { + const rows = source + .prepare("SELECT * FROM provider_connections WHERE provider = ? AND is_active = 1") + .all(provider); + for (const row of rows) { + insertConn.run(row); + copied++; + } + } + + const comboRow = source.prepare("SELECT * FROM combos WHERE name = 'default'").get() as + Record | undefined; + if (comboRow) { + const comboCols = Object.keys(comboRow); + const comboColList = comboCols.map((c) => `"${c}"`).join(","); + const comboPlaceholders = comboCols.map((c) => `@${c}`).join(","); + target + .prepare(`INSERT OR REPLACE INTO combos (${comboColList}) VALUES (${comboPlaceholders})`) + .run(comboRow); + } + + console.log(` [container] seeded "default" combo + ${copied} provider connection(s)`); + source.close(); + target.close(); +} + +async function provisionApiKeys( + baseUrl: string +): Promise<{ apiKey: string; managementApiKey: string }> { + const passwordLine = spawnSync("grep", ["INITIAL_PASSWORD", ENV_FILE], { + encoding: "utf8", + }).stdout.trim(); + const password = passwordLine.split("=").slice(1).join("=") || "CHANGEME"; + + const login = await fetch(`${baseUrl}/api/auth/login`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ password }), + }); + const cookie = login.headers.get("set-cookie"); + if (!cookie) throw new Error("login did not return a session cookie"); + + async function createKey(name: string, scopes?: string[]): Promise { + const res = await fetch(`${baseUrl}/api/keys`, { + method: "POST", + headers: { "Content-Type": "application/json", Cookie: cookie! }, + body: JSON.stringify({ name, ...(scopes ? { scopes } : {}) }), + }); + if (!res.ok) throw new Error(`failed to create API key "${name}": ${res.status}`); + const data = (await res.json()) as { key: string }; + return data.key; + } + + const apiKey = await createKey("live-wire-capture-test"); + const managementApiKey = await createKey("live-wire-capture-test-mgmt", ["manage"]); + return { apiKey, managementApiKey }; +} + +export async function startLiveContainer(): Promise { + stopExistingContainer(); + ensureImageBuilt(); + const { hostPort, netnsPath } = startContainer(); + const baseUrl = `http://127.0.0.1:${hostPort}`; + + await waitForHealth(baseUrl); + // The container creates storage.sqlite etc. on first boot under its own + // internal uid mapping — chmod again now that those files exist, since + // the earlier pre-start chmod only reached the (then-empty) directory. + // Without this, seedDefaultComboAndConnections()'s direct host-side + // better-sqlite3 open fails with "attempt to write a readonly database" + // (same root cause hit manually with omniroute-beta earlier this session). + tryRun("podman", ["unshare", "chmod", "-R", "a+rwX", DATA_DIR_HOST]); + await seedDefaultComboAndConnections(); + const { apiKey, managementApiKey } = await provisionApiKeys(baseUrl); + + return { + baseUrl, + apiKey, + managementApiKey, + containerName: CONTAINER_NAME, + netnsPath, + hostPort, + dataDir: DATA_DIR_HOST, + async stop() { + tryRun("podman", ["stop", "-t", "5", CONTAINER_NAME]); + tryRun("podman", ["rm", "-f", CONTAINER_NAME]); + }, + }; +} diff --git a/tests/integration/liveDefaultComboShared.ts b/tests/integration/liveDefaultComboShared.ts new file mode 100644 index 0000000000..d5d9291e63 --- /dev/null +++ b/tests/integration/liveDefaultComboShared.ts @@ -0,0 +1,266 @@ +/** + * tests/integration/liveDefaultComboShared.ts + * + * Shared utilities for the general "default combo" live workload test. + * Unlike liveGeminiShared.ts (which provisions its own narrow 2-model + * Gemini-only combo when "default" doesn't already exist), this reads the + * REAL "default" combo currently configured on the target instance directly + * from its own DB (src/lib/db/combos.ts — never raw SQL, per AGENTS.md) and + * exercises every provider/model step in it directly, bypassing combo + * routing, so live-test coverage always matches whatever the operator + * actually has configured instead of a hardcoded snapshot that goes stale + * the moment the combo changes. + */ +import { + API_KEY, + BASE_URL, + readSSEStream, + readResponsesSSEStream, + genSystemMessage, + genUserMessage, + type Message, +} from "./liveGeminiShared.ts"; + +export { API_KEY, BASE_URL }; + +export const skip = !API_KEY ? "OMNIROUTE_API_KEY not set — skipping live test" : undefined; + +export interface ComboModelTarget { + model: string; + providerId: string | null; +} + +async function apiFetch(path: string, options: RequestInit = {}): Promise { + return fetch(`${BASE_URL}${path}`, { + ...options, + headers: { + Authorization: `Bearer ${API_KEY}`, + "Content-Type": "application/json", + ...options.headers, + }, + }); +} + +// Bootstrap seed used ONLY when the target instance has no "default" combo +// at all — mirrors liveGeminiShared.ts's own DEFAULT_COMBO_CONFIG fallback, +// generalized to the real multi-provider spread confirmed live against this +// operator's own production "default" combo (5 providers, 18 models) rather +// than Gemini alone. This is a creation fallback only: whenever a "default" +// combo already exists on the target instance, its actual live config is +// always what gets read and tested — this list never overrides it. +const FALLBACK_COMBO_MODELS: { model: string; providerId: string }[] = [ + { model: "opencode/big-pickle", providerId: "opencode" }, + { model: "opencode/mimo-v2.5-free", providerId: "opencode" }, + { model: "opencode/laguna-s-2.1-free", providerId: "opencode" }, + { model: "openrouter/cohere/north-mini-code:free", providerId: "openrouter" }, + { model: "openrouter/poolside/laguna-m.1:free", providerId: "openrouter" }, + { model: "openrouter/nvidia/nemotron-3-ultra-550b-a55b:free", providerId: "openrouter" }, + { model: "openrouter/nvidia/nemotron-3-super-120b-a12b:free", providerId: "openrouter" }, + { model: "openrouter/nvidia/nemotron-3-nano-30b-a3b:free", providerId: "openrouter" }, + { model: "openrouter/google/gemma-4-26b-a4b-it:free", providerId: "openrouter" }, + { model: "openrouter/google/gemma-4-31b-it:free", providerId: "openrouter" }, + { model: "openrouter/poolside/laguna-s-2.1:free", providerId: "openrouter" }, + { model: "gemini/gemini-3.1-flash-lite", providerId: "gemini" }, + { model: "gemini/gemma-4-31b-it", providerId: "gemini" }, + { model: "gemini/gemma-4-26b-a4b-it", providerId: "gemini" }, + { model: "mistral/mistral-large-latest", providerId: "mistral" }, + { model: "cerebras/gemma-4-31b", providerId: "cerebras" }, + { model: "cerebras/zai-glm-4.7", providerId: "cerebras" }, + { model: "cerebras/gpt-oss-120b", providerId: "cerebras" }, +]; + +async function ensureDefaultComboExists( + getComboByName: (name: string) => Promise | null> +): Promise { + const existing = await getComboByName("default"); + if (existing) return; + + console.log(` [setup] no "default" combo on this instance — creating fallback seed combo`); + const { createCombo } = await import("../../src/lib/db/combos.ts"); + await createCombo({ + name: "default", + strategy: "priority", + models: FALLBACK_COMBO_MODELS.map((m, i) => ({ + kind: "model" as const, + model: m.model, + providerId: m.providerId, + weight: 1, + id: `fallback-${i}`, + })), + }); +} + +// Read the live "default" combo's model steps straight from the DB module — +// intentionally not hardcoded, so this always reflects whatever the operator +// currently has configured on the target instance. Creates a fallback seed +// combo first if none exists at all (see ensureDefaultComboExists above). +export async function getDefaultComboModelTargets(): Promise { + const { getComboByName } = await import("../../src/lib/db/combos.ts"); + await ensureDefaultComboExists(getComboByName); + const combo = (await getComboByName("default")) as Record | null; + const models = + combo && Array.isArray(combo.models) ? (combo.models as Record[]) : []; + + const targets: ComboModelTarget[] = []; + for (const step of models) { + if (step.kind !== "model" || typeof step.model !== "string") continue; + targets.push({ + model: step.model, + providerId: typeof step.providerId === "string" ? step.providerId : null, + }); + } + return targets; +} + +// Skip (never fail) any model whose provider connection isn't currently +// active — this suite's job is breadth across the real combo, not blocking +// the whole run on one unrelated provider outage. baseUrl/apiKey default to +// the module-level omniroute-beta target but can be overridden (see +// sendModelRequest — same rationale, used by the wire-capture suite's +// dedicated container). +export async function filterActiveModelTargets( + targets: ComboModelTarget[], + options: SendModelRequestOptions = {} +): Promise<{ active: ComboModelTarget[]; skipped: string[] }> { + const baseUrl = options.baseUrl ?? BASE_URL; + const apiKey = options.apiKey ?? API_KEY; + const res = await fetch(`${baseUrl}/api/providers`, { + headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" }, + }); + if (!res.ok) return { active: targets, skipped: [] }; + + const data = await res.json(); + const connections = (data.connections || data) as Record[]; + // Terminal states (never self-heal — see AGENTS.md "Resilience Runtime + // State" → Connection Cooldown) plus "unavailable" (active cooldown) are + // the only statuses worth pre-filtering; everything else (including + // transient/lazily-recovered cooldowns that have already expired) is left + // for the request itself to prove out. + const DEAD_STATUSES = new Set(["expired", "unavailable", "banned", "credits_exhausted"]); + const activeProviders = new Set( + connections + .filter((c) => c.isActive && !DEAD_STATUSES.has(c.testStatus as string)) + .map((c) => c.provider as string) + ); + + const active: ComboModelTarget[] = []; + const skipped: string[] = []; + for (const t of targets) { + if (!t.providerId || activeProviders.has(t.providerId)) { + active.push(t); + } else { + skipped.push(`${t.model} (provider "${t.providerId}" not active)`); + } + } + return { active, skipped }; +} + +function ts(): string { + return new Date().toISOString().slice(11, 23); // HH:MM:SS.mmm +} + +export interface ModelRequestResult { + model: string; + status: number; + duration: number; + tokens: number; + contentLength: number; + correlationId: string; + error?: string; +} + +export interface SendModelRequestOptions { + baseUrl?: string; + apiKey?: string; +} + +// Deliberately lighter than liveGeminiShared's sendAndValidate (no retry +// loop, one fixed prompt pair): this suite's job is breadth across every +// model in the real combo, not depth on any single provider. baseUrl/apiKey +// default to the module-level omniroute-beta target but can be overridden — +// e.g. by the wire-capture suite, which points requests at its own +// dedicated throwaway container instead (see liveContainerHarness.ts). +export async function sendModelRequest( + model: string, + stream: boolean, + apiFormat: "chat" | "responses" = "chat", + options: SendModelRequestOptions = {} +): Promise { + const baseUrl = options.baseUrl ?? BASE_URL; + const apiKey = options.apiKey ?? API_KEY; + const endpoint = apiFormat === "responses" ? "/v1/responses" : "/v1/chat/completions"; + const messages: Message[] = [genSystemMessage(), genUserMessage()]; + const body = + apiFormat === "responses" + ? { model, input: messages, stream, max_output_tokens: 1024, temperature: 0.3 } + : { model, messages, stream, max_tokens: 1024, temperature: 0.3 }; + + const controller = new AbortController(); + const timeoutMs = Number(process.env.TEST_REQUEST_TIMEOUT_MS) || 120_000; + const timeout = setTimeout(() => controller.abort(), timeoutMs); + const start = performance.now(); + + try { + const response = await fetch(`${baseUrl}${endpoint}`, { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` }, + body: JSON.stringify(body), + signal: controller.signal, + }); + const duration = performance.now() - start; + clearTimeout(timeout); + const correlationId = response.headers.get("x-correlation-id") || "?"; + + let content = ""; + let totalTokens = 0; + + if (response.status === 200) { + if (stream) { + const streamResult = + apiFormat === "responses" + ? await readResponsesSSEStream(response) + : await readSSEStream(response); + content = streamResult.fullContent; + totalTokens = streamResult.totalTokens; + } else if (apiFormat === "responses") { + const json = await response.json().catch(() => ({})); + const textItem = json?.output?.find((o: Record) => o.type === "message"); + content = textItem?.content?.[0]?.text || ""; + totalTokens = json?.usage?.total_tokens || 0; + } else { + const json = await response.json().catch(() => ({})); + content = json?.choices?.[0]?.message?.content || ""; + totalTokens = json?.usage?.total_tokens || 0; + } + } + + console.log( + `${ts()} ${model.padEnd(40)} HTTP ${response.status} | ` + + `${Math.round(duration).toString().padStart(6)}ms | ` + + `${String(totalTokens).padStart(5)} tok | ` + + `${content.length} chars | cid: ${correlationId}` + ); + + return { + model, + status: response.status, + duration, + tokens: totalTokens, + contentLength: content.length, + correlationId, + }; + } catch (err) { + clearTimeout(timeout); + const errorMessage = err instanceof Error ? err.message : String(err); + console.log(`${ts()} ${model.padEnd(40)} FAILED: ${errorMessage}`); + return { + model, + status: 0, + duration: performance.now() - start, + tokens: 0, + contentLength: 0, + correlationId: "?", + error: errorMessage, + }; + } +} diff --git a/tests/integration/wireCapture.ts b/tests/integration/wireCapture.ts new file mode 100644 index 0000000000..4485c4ea6c --- /dev/null +++ b/tests/integration/wireCapture.ts @@ -0,0 +1,154 @@ +/** + * tests/integration/wireCapture.ts + * + * Rootless wire capture + analysis for live container tests. Uses + * `podman unshare nsenter --net=` to run tcpdump without + * sudo/root (verified working against a rootless podman container — see + * scripts/sre/tcp-close-analyzer.py's docstring for the equivalent + * root-requiring `nsenter -t $PID` command this generalizes from), then + * shells out to that same script to reassemble TCP streams and extract + * HTTP request/response lines + correlationId per stream. + */ +import { spawn, spawnSync } from "node:child_process"; +import { existsSync, readFileSync, unlinkSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +const REPO_ROOT = fileURLToPath(new URL("../..", import.meta.url)); +const ANALYZER_SCRIPT = `${REPO_ROOT}scripts/sre/tcp-close-analyzer.py`; + +export interface WireStreamRecord { + streamKey: string; + client: string | null; + server: string | null; + firstTs: number; + lastTs: number; + durationSec: number; + packetCount: number; + correlationId: string | null; + requestId: string | null; + firstLineFromA: string | null; + firstLineFromB: string | null; + closes: Array<{ ts: number; side: string; src: string; dst: string; flags: string }>; + verdict: + | "client_closed_first" + | "server_closed_first" + | "simultaneous" + | "no_close_seen" + | "unknown_side_closed_first"; +} + +export interface CaptureHandle { + pcapPath: string; + stop(): Promise; +} + +// Best-effort HTTP status line finder — checks both reassembled directions +// since we don't know a priori which one carried the response. +export function responseStatusLine(record: WireStreamRecord): string | null { + for (const line of [record.firstLineFromA, record.firstLineFromB]) { + if (line && /^HTTP\/\d\.\d \d{3}/.test(line)) return line; + } + return null; +} + +export function requestLine(record: WireStreamRecord): string | null { + for (const line of [record.firstLineFromA, record.firstLineFromB]) { + if (line && /^(GET|POST|PUT|PATCH|DELETE) /.test(line)) return line; + } + return null; +} + +export async function startWireCapture( + netnsPath: string, + pcapPath: string, + bpfFilter: string +): Promise { + if (existsSync(pcapPath)) unlinkSync(pcapPath); + + // `-U`: flush each packet to disk as captured instead of buffering, so a + // non-graceful stop still leaves a readable pcap. + const child = spawn( + "podman", + [ + "unshare", + "nsenter", + `--net=${netnsPath}`, + "--", + "tcpdump", + "-i", + "any", + "-U", + "-w", + pcapPath, + bpfFilter, + ], + { stdio: ["ignore", "ignore", "pipe"] } + ); + + await new Promise((resolve, reject) => { + const timeout = setTimeout( + () => reject(new Error("tcpdump did not start listening in time")), + 10_000 + ); + child.stderr?.on("data", (chunk: Buffer) => { + if (chunk.toString().includes("listening on")) { + clearTimeout(timeout); + resolve(); + } + }); + child.on("exit", (code) => { + clearTimeout(timeout); + reject(new Error(`tcpdump exited early with code ${code}`)); + }); + }); + + return { + pcapPath, + async stop() { + // podman unshare -> nsenter -> tcpdump is a 3-level subprocess chain; + // SIGTERM to the top-level `podman` process (the only PID Node's + // child_process handle actually tracks) does not reliably reach the + // tcpdump grandchild, leaving it running as an orphan with a + // never-flushed pcap. pkill by the (unique, per-run) pcap path + // reliably reaches the real tcpdump process regardless of how deep + // the subprocess chain is. + child.kill("SIGTERM"); + spawnSync("pkill", ["-f", `tcpdump.*${pcapPath}`]); + await new Promise((resolve) => { + if (child.exitCode !== null) return resolve(); + child.on("exit", () => resolve()); + setTimeout(resolve, 3_000); + }); + // Give the now-dead tcpdump's OS write buffers a moment to land on + // disk before anything tries to read the pcap. + await new Promise((r) => setTimeout(r, 250)); + }, + }; +} + +export async function analyzeCapture(pcapPath: string): Promise { + const jsonlPath = pcapPath.replace(/\.pcap$/, "") + ".streams.jsonl"; + const result = spawnSync("python3", [ANALYZER_SCRIPT, pcapPath, "--out", jsonlPath], { + encoding: "utf8", + }); + if (result.status !== 0) { + throw new Error(`tcp-close-analyzer.py failed: ${result.stderr || result.stdout}`); + } + if (!existsSync(jsonlPath)) return []; + + return readFileSync(jsonlPath, "utf8") + .split("\n") + .filter((line) => line.trim()) + .map((line) => JSON.parse(line) as WireStreamRecord); +} + +export function indexByCorrelationId(records: WireStreamRecord[]): Map { + const map = new Map(); + for (const record of records) { + if (!record.correlationId) continue; + const existing = map.get(record.correlationId) || []; + existing.push(record); + map.set(record.correlationId, existing); + } + return map; +}