From 38d50a08d176d8cfbcf54af533034e38001f9c90 Mon Sep 17 00:00:00 2001 From: Markus Hartung Date: Fri, 7 Aug 2026 19:00:45 +0200 Subject: [PATCH] test(integration): add rootless wire-capture correlation to the live-test tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Temporary WIP commit on this deferred branch — lands in the same final live-test-tool PR as the general default-combo suite, never bundled into a bug-fix PR. liveContainerHarness.ts spins up a dedicated, throwaway podman container (same runner-base image target as the operator's local dev/beta containers) so wire-capture tests are fully self-contained: builds the image if missing, starts the container with a persistent data dir, waits for health, seeds the real "default" combo + provider connections from the operator's local omniroute-dev instance (idempotent — only runs once per data dir), and provisions API keys via the running instance's own auth flow. wireCapture.ts captures the container's actual network traffic via `podman unshare nsenter --net= -- tcpdump` — no root needed, verified working live (this generalizes the root-requiring `sudo nsenter -t $PID` command scripts/sre/tcp-close-analyzer.py already documented for the same rootless-Podman netns problem; that script's docstring now documents both). Capture and analysis needed two real fixes found only by running the pipeline live: `-U` (unbuffered tcpdump writes) plus a `pkill -f ` fallback, since `podman unshare -> nsenter -> tcpdump` is a 3-level subprocess chain and SIGTERM to the top-level process doesn't reach the tcpdump grandchild, leaving an orphaned process and a truncated/unreadable pcap; and filtering on the container's internal listening port (20128) rather than the dynamically-assigned host port, since capture happens inside the container's own network namespace where only the internal port is meaningful. live-default-combo-wire-capture.test.ts (gated on RUN_LIVE_WIRE_CAPTURE=1) ties it together: sends a small representative sample of requests through the real default combo, then cross-checks each one's app-level JSON status against the actual HTTP status line observed on the wire via scripts/sre/tcp-close-analyzer.py's stream reassembly — catching bugs where the app layer claims success but the wire shows a truncated/reset stream, not just what liveDefaultComboShared.ts's existing breadth suite already covers. Live-verified end-to-end: 4/4 sampled requests correlated correctly across 8 captured TCP streams, container + capture process fully torn down afterward (verified no orphaned podman container or tcpdump process left running). sendModelRequest/filterActiveModelTargets (liveDefaultComboShared.ts) gain optional baseUrl/apiKey overrides, defaulting to the existing module-level omniroute-beta target, so the wire-capture suite can point the same request-sending logic at its own dedicated container instead. (cherry picked from commit 914a7e42cbe914f257db9f72eedc902ee1532083) --- scripts/sre/tcp-close-analyzer.py | 22 +- .../live-default-combo-wire-capture.test.ts | 143 ++++++++++ tests/integration/liveContainerHarness.ts | 260 ++++++++++++++++++ tests/integration/liveDefaultComboShared.ts | 33 ++- tests/integration/wireCapture.ts | 154 +++++++++++ 5 files changed, 603 insertions(+), 9 deletions(-) create mode 100644 tests/integration/live-default-combo-wire-capture.test.ts create mode 100644 tests/integration/liveContainerHarness.ts create mode 100644 tests/integration/wireCapture.ts diff --git a/scripts/sre/tcp-close-analyzer.py b/scripts/sre/tcp-close-analyzer.py index 77f489799b..6b01ad2034 100755 --- a/scripts/sre/tcp-close-analyzer.py +++ b/scripts/sre/tcp-close-analyzer.py @@ -17,8 +17,8 @@ parsing the libpcap file format and IPv4/TCP headers directly. Good enough for this one question; not a general-purpose pcap toolkit. ──────────────────────────────────────────────────────────────────────────── -CAPTURING (run this yourself — needs root/sudo for CAP_NET_RAW; also see ---show-capture-cmd) +CAPTURING (run this yourself — needs root/sudo for CAP_NET_RAW, UNLESS you +use the rootless method below; also see --show-capture-cmd) ──────────────────────────────────────────────────────────────────────────── Rootless Podman gotcha: there is usually NO `podman3`/`podmanN` bridge @@ -33,6 +33,24 @@ container's OWN namespace via its PID instead: sudo nsenter -t "$PID" -n tcpdump -i any -w /tmp/omniroute-capture.pcap \\ 'host and port 20128' +Rootless alternative (NO sudo needed): a bare `nsenter -t $PID -n` fails +with "Invalid argument" for a rootless container, because its network +namespace lives inside a user namespace you're not in yet. `podman unshare` +puts you in that same user namespace first, so `nsenter --net=` against the +container's netns path succeeds as a plain user — verified working live +(captured a real `POST /v1/chat/completions` request body in cleartext this +way, no root at any point): + + NETNS=$(podman inspect omniroute-dev --format '{{.NetworkSettings.SandboxKey}}') + podman unshare nsenter --net="$NETNS" -- \\ + tcpdump -i any -w /tmp/omniroute-capture.pcap 'port 20128' + +No `sudo chmod` needed afterward either, since the file was never +root-owned. This is also what +tests/integration/wireCapture.ts + liveContainerHarness.ts automate for the +live wire-capture test suite (its own dedicated throwaway container, not +omniroute-dev) — see RUN_LIVE_WIRE_CAPTURE=1 in that test file. + Find the container's IP first with: podman inspect omniroute-dev --format '{{.NetworkSettings.Networks}}' diff --git a/tests/integration/live-default-combo-wire-capture.test.ts b/tests/integration/live-default-combo-wire-capture.test.ts new file mode 100644 index 0000000000..d1454ac012 --- /dev/null +++ b/tests/integration/live-default-combo-wire-capture.test.ts @@ -0,0 +1,143 @@ +/** + * tests/integration/live-default-combo-wire-capture.test.ts + * + * Wire-level correlation test. Spins up a dedicated, throwaway podman + * container (liveContainerHarness.ts), captures its network traffic + * (wireCapture.ts — rootless tcpdump via `podman unshare nsenter`, no root), + * sends a representative sample of requests against the real "default" + * combo, then cross-checks each request's app-level result (JSON status) + * against what actually went out on the wire (HTTP response status line, + * verdict on who closed the connection first). Catches bugs where the app + * layer claims success but the wire shows a truncated/reset stream. + * + * Fully self-contained — does not touch omniroute-beta or omniroute-dev + * (only reads from omniroute-dev's DB once, to seed its own dedicated + * container's data dir). Gated on RUN_LIVE_WIRE_CAPTURE=1: needs podman, + * tcpdump, python3, and a real .env with provider credentials, so it must + * never run in CI. + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + LIVE_CONTAINER_ENABLED, + startLiveContainer, + type LiveContainerHandle, +} from "./liveContainerHarness.ts"; +import { + startWireCapture, + analyzeCapture, + indexByCorrelationId, + responseStatusLine, + type CaptureHandle, +} from "./wireCapture.ts"; +import { + getDefaultComboModelTargets, + filterActiveModelTargets, + sendModelRequest, +} from "./liveDefaultComboShared.ts"; + +const skip = !LIVE_CONTAINER_ENABLED + ? "RUN_LIVE_WIRE_CAPTURE not set — skipping wire-capture live test" + : undefined; + +// Wire-level correlation is the point of this suite, not breadth across +// every provider (already covered by live-default-combo-workload.test.ts) — +// keep the sample small so capture/analysis stays fast. +const SAMPLE_SIZE = 4; + +let container: LiveContainerHandle; +let capture: CaptureHandle; + +test.before(async () => { + if (skip) return; + container = await startLiveContainer(); + process.env.DATA_DIR = container.dataDir; + + // PID-scoped so a concurrent session running this same test never + // collides on the capture file or the pkill-by-path cleanup in + // wireCapture.ts's stop(). + const pcapPath = `/tmp/omniroute-live-wire-capture-${process.pid}.pcap`; + // Capture happens INSIDE the container's own netns (podman unshare + // nsenter --net=), so packets there are addressed to the + // container's internal listening port (20128), not the dynamically + // assigned host port used to reach it from outside — filtering on + // hostPort here would silently match nothing. + capture = await startWireCapture(container.netnsPath, pcapPath, "tcp port 20128"); +}); + +test.after(async () => { + if (skip) return; + await capture?.stop(); + await container?.stop(); +}); + +test( + "wire capture: app-level status matches the HTTP status line actually observed on the wire", + { skip }, + async () => { + const allTargets = await getDefaultComboModelTargets(); + assert.ok(allTargets.length > 0, `"default" combo has no model steps — nothing to test`); + + const { active } = await filterActiveModelTargets(allTargets, { + baseUrl: container.baseUrl, + apiKey: container.managementApiKey, + }); + assert.ok(active.length > 0, "no active provider connections in the seeded container"); + + const sample = active.slice(0, SAMPLE_SIZE); + console.log( + `\n [wire-capture] sampling ${sample.length} model(s): ${sample.map((t) => t.model).join(", ")}` + ); + + const results = await Promise.all( + sample.map((t) => + sendModelRequest(t.model, false, "chat", { + baseUrl: container.baseUrl, + apiKey: container.apiKey, + }) + ) + ); + + // Give the capture a moment to flush the last packets before analyzing. + await new Promise((r) => setTimeout(r, 1000)); + await capture.stop(); + const streams = await analyzeCapture(capture.pcapPath); + const byCorrelationId = indexByCorrelationId(streams); + + console.log(` [wire-capture] captured ${streams.length} TCP stream(s)`); + + const mismatches: string[] = []; + for (const r of results) { + if (r.correlationId === "?") { + mismatches.push(`${r.model}: no correlationId returned in response headers`); + continue; + } + const matched = byCorrelationId.get(r.correlationId); + if (!matched || matched.length === 0) { + mismatches.push( + `${r.model}: correlationId ${r.correlationId} not found in any captured wire stream` + ); + continue; + } + const wireStatusLines = matched.map(responseStatusLine).filter(Boolean); + const wireStatusCodes = wireStatusLines.map((line) => line!.split(" ")[1]); + if (!wireStatusCodes.includes(String(r.status))) { + mismatches.push( + `${r.model}: app-level status ${r.status} but wire shows ${wireStatusCodes.join(",") || "no status line"} (cid ${r.correlationId})` + ); + } + } + + if (mismatches.length > 0) { + console.log(`\n Wire/app-level mismatches (${mismatches.length}):`); + for (const m of mismatches) console.log(` ${m}`); + } + + assert.equal( + mismatches.length, + 0, + `${mismatches.length}/${results.length} requests had app-level results that don't match what was observed on the wire` + ); + } +); diff --git a/tests/integration/liveContainerHarness.ts b/tests/integration/liveContainerHarness.ts new file mode 100644 index 0000000000..168fa2d211 --- /dev/null +++ b/tests/integration/liveContainerHarness.ts @@ -0,0 +1,260 @@ +/** + * tests/integration/liveContainerHarness.ts + * + * Spins up a dedicated, throwaway podman container running this checkout's + * own code (runner-base target, same as the operator's local dev/beta + * containers) so wire-capture live tests are fully self-contained — no + * dependency on a manually-managed systemd quadlet. + * + * The container's DATA_DIR is a persistent host directory (not wiped between + * runs) so the "default" combo + real provider connections only need + * seeding once; seeding is idempotent and copies from the operator's local + * omniroute-dev instance (same source used for the manual omniroute-beta + * seed earlier this session). + */ +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import Database from "better-sqlite3"; + +const REPO_ROOT = fileURLToPath(new URL("../..", import.meta.url)); + +export const LIVE_CONTAINER_ENABLED = process.env.RUN_LIVE_WIRE_CAPTURE === "1"; + +const IMAGE_TAG = process.env.LIVE_CONTAINER_IMAGE || "localhost/omniroute:live-wire-test"; +const CONTAINER_NAME = process.env.LIVE_CONTAINER_NAME || "omniroute-live-wire-test"; +const DATA_DIR_HOST = + process.env.LIVE_CONTAINER_DATA_DIR || "/data/podman-data/omniroute-live-wire-test/data"; +const ENV_FILE = process.env.LIVE_CONTAINER_ENV_FILE || "/data/podman-data/omniroute/omniroute.env"; +// Source DB to seed the "default" combo + provider connections from — the +// operator's local omniroute-dev instance, same source used for the manual +// omniroute-beta seed earlier this session. +const SEED_SOURCE_DB = + process.env.LIVE_CONTAINER_SEED_SOURCE_DB || + "/home/markus/code/podman/OmniRoute/data/storage.sqlite"; +const SEED_PROVIDERS = ["gemini", "openrouter", "mistral", "cerebras"]; + +export interface LiveContainerHandle { + baseUrl: string; + apiKey: string; + managementApiKey: string; + containerName: string; + netnsPath: string; + hostPort: number; + dataDir: string; + stop(): Promise; +} + +function run(cmd: string, args: string[], opts: { input?: string } = {}): string { + const result = spawnSync(cmd, args, { + cwd: REPO_ROOT, + encoding: "utf8", + input: opts.input, + maxBuffer: 64 * 1024 * 1024, + }); + if (result.status !== 0) { + throw new Error( + `${cmd} ${args.join(" ")} failed (exit ${result.status}):\n${result.stderr || result.stdout}` + ); + } + return result.stdout.trim(); +} + +function tryRun(cmd: string, args: string[]): string | null { + const result = spawnSync(cmd, args, { cwd: REPO_ROOT, encoding: "utf8" }); + return result.status === 0 ? result.stdout.trim() : null; +} + +function ensureImageBuilt(): void { + const existing = tryRun("podman", ["images", "-q", IMAGE_TAG]); + if (existing) { + console.log(` [container] image ${IMAGE_TAG} already exists (${existing}) — reusing`); + return; + } + console.log(` [container] building ${IMAGE_TAG} (runner-base target — this takes a while)...`); + run("podman", ["build", "--target", "runner-base", "-t", IMAGE_TAG, "."]); +} + +function stopExistingContainer(): void { + tryRun("podman", ["rm", "-f", CONTAINER_NAME]); +} + +function startContainer(): { hostPort: number; netnsPath: string } { + if (!existsSync(DATA_DIR_HOST)) { + mkdirSync(DATA_DIR_HOST, { recursive: true }); + } + // podman unshare owns the rootless user namespace these directories' + // native uid mappings live in — plain chmod as the host user fails with + // EPERM on files podman previously wrote as a different mapped uid. + tryRun("podman", ["unshare", "chmod", "-R", "a+rwX", DATA_DIR_HOST]); + + run("podman", [ + "run", + "-d", + "--name", + CONTAINER_NAME, + "-p", + "127.0.0.1::20128", + "-v", + `${DATA_DIR_HOST}:/app/data`, + "--env-file", + ENV_FILE, + IMAGE_TAG, + ]); + + const portOutput = run("podman", ["port", CONTAINER_NAME, "20128/tcp"]); + const hostPort = Number(portOutput.split(":").pop()); + if (!Number.isFinite(hostPort)) { + throw new Error(`could not parse assigned host port from: ${portOutput}`); + } + + const netnsPath = run("podman", [ + "inspect", + CONTAINER_NAME, + "--format", + "{{.NetworkSettings.SandboxKey}}", + ]); + + return { hostPort, netnsPath }; +} + +async function waitForHealth(baseUrl: string, timeoutMs = 60_000): Promise { + const deadline = Date.now() + timeoutMs; + let lastError: unknown; + while (Date.now() < deadline) { + try { + const res = await fetch(`${baseUrl}/api/monitoring/health`); + if (res.ok) return; + } catch (err) { + lastError = err; + } + await new Promise((r) => setTimeout(r, 1000)); + } + throw new Error(`container never became healthy within ${timeoutMs}ms: ${lastError}`); +} + +// Idempotent: only copies rows if the target has no "default" combo yet. +// Direct SQLite access (not the src/lib/db/ CRUD functions) is deliberate +// here, same as liveGeminiShared.ts's ensureGeminiProvider() — cloning an +// existing row's already-encrypted apiKey blob byte-for-byte has no CRUD +// equivalent, and both instances share the same API_KEY_SECRET (same +// --env-file), so the encrypted value decrypts correctly on the target too. +async function seedDefaultComboAndConnections(): Promise { + const targetPath = `${DATA_DIR_HOST}/storage.sqlite`; + if (!existsSync(targetPath)) { + console.log(` [container] target DB not created yet, skipping seed this pass`); + return; + } + if (!existsSync(SEED_SOURCE_DB)) { + console.warn(` [container] seed source DB not found at ${SEED_SOURCE_DB} — skipping seed`); + return; + } + + const target = new Database(targetPath); + const existingCombo = target.prepare("SELECT 1 FROM combos WHERE name = 'default'").get(); + if (existingCombo) { + console.log(` [container] "default" combo already seeded — skipping`); + target.close(); + return; + } + + const source = new Database(SEED_SOURCE_DB, { readonly: true }); + const connCols = source.prepare("PRAGMA table_info(provider_connections)").all() as Array<{ + name: string; + }>; + const colList = connCols.map((c) => `"${c.name}"`).join(","); + const placeholders = connCols.map((c) => `@${c.name}`).join(","); + const insertConn = target.prepare( + `INSERT OR REPLACE INTO provider_connections (${colList}) VALUES (${placeholders})` + ); + + let copied = 0; + for (const provider of SEED_PROVIDERS) { + const rows = source + .prepare("SELECT * FROM provider_connections WHERE provider = ? AND is_active = 1") + .all(provider); + for (const row of rows) { + insertConn.run(row); + copied++; + } + } + + const comboRow = source.prepare("SELECT * FROM combos WHERE name = 'default'").get() as + Record | undefined; + if (comboRow) { + const comboCols = Object.keys(comboRow); + const comboColList = comboCols.map((c) => `"${c}"`).join(","); + const comboPlaceholders = comboCols.map((c) => `@${c}`).join(","); + target + .prepare(`INSERT OR REPLACE INTO combos (${comboColList}) VALUES (${comboPlaceholders})`) + .run(comboRow); + } + + console.log(` [container] seeded "default" combo + ${copied} provider connection(s)`); + source.close(); + target.close(); +} + +async function provisionApiKeys( + baseUrl: string +): Promise<{ apiKey: string; managementApiKey: string }> { + const passwordLine = spawnSync("grep", ["INITIAL_PASSWORD", ENV_FILE], { + encoding: "utf8", + }).stdout.trim(); + const password = passwordLine.split("=").slice(1).join("=") || "CHANGEME"; + + const login = await fetch(`${baseUrl}/api/auth/login`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ password }), + }); + const cookie = login.headers.get("set-cookie"); + if (!cookie) throw new Error("login did not return a session cookie"); + + async function createKey(name: string, scopes?: string[]): Promise { + const res = await fetch(`${baseUrl}/api/keys`, { + method: "POST", + headers: { "Content-Type": "application/json", Cookie: cookie! }, + body: JSON.stringify({ name, ...(scopes ? { scopes } : {}) }), + }); + if (!res.ok) throw new Error(`failed to create API key "${name}": ${res.status}`); + const data = (await res.json()) as { key: string }; + return data.key; + } + + const apiKey = await createKey("live-wire-capture-test"); + const managementApiKey = await createKey("live-wire-capture-test-mgmt", ["manage"]); + return { apiKey, managementApiKey }; +} + +export async function startLiveContainer(): Promise { + stopExistingContainer(); + ensureImageBuilt(); + const { hostPort, netnsPath } = startContainer(); + const baseUrl = `http://127.0.0.1:${hostPort}`; + + await waitForHealth(baseUrl); + // The container creates storage.sqlite etc. on first boot under its own + // internal uid mapping — chmod again now that those files exist, since + // the earlier pre-start chmod only reached the (then-empty) directory. + // Without this, seedDefaultComboAndConnections()'s direct host-side + // better-sqlite3 open fails with "attempt to write a readonly database" + // (same root cause hit manually with omniroute-beta earlier this session). + tryRun("podman", ["unshare", "chmod", "-R", "a+rwX", DATA_DIR_HOST]); + await seedDefaultComboAndConnections(); + const { apiKey, managementApiKey } = await provisionApiKeys(baseUrl); + + return { + baseUrl, + apiKey, + managementApiKey, + containerName: CONTAINER_NAME, + netnsPath, + hostPort, + dataDir: DATA_DIR_HOST, + async stop() { + tryRun("podman", ["stop", "-t", "5", CONTAINER_NAME]); + tryRun("podman", ["rm", "-f", CONTAINER_NAME]); + }, + }; +} diff --git a/tests/integration/liveDefaultComboShared.ts b/tests/integration/liveDefaultComboShared.ts index 1e86df14be..d5d9291e63 100644 --- a/tests/integration/liveDefaultComboShared.ts +++ b/tests/integration/liveDefaultComboShared.ts @@ -114,11 +114,19 @@ export async function getDefaultComboModelTargets(): Promise // Skip (never fail) any model whose provider connection isn't currently // active — this suite's job is breadth across the real combo, not blocking -// the whole run on one unrelated provider outage. +// the whole run on one unrelated provider outage. baseUrl/apiKey default to +// the module-level omniroute-beta target but can be overridden (see +// sendModelRequest — same rationale, used by the wire-capture suite's +// dedicated container). export async function filterActiveModelTargets( - targets: ComboModelTarget[] + targets: ComboModelTarget[], + options: SendModelRequestOptions = {} ): Promise<{ active: ComboModelTarget[]; skipped: string[] }> { - const res = await apiFetch("/api/providers"); + const baseUrl = options.baseUrl ?? BASE_URL; + const apiKey = options.apiKey ?? API_KEY; + const res = await fetch(`${baseUrl}/api/providers`, { + headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" }, + }); if (!res.ok) return { active: targets, skipped: [] }; const data = await res.json(); @@ -161,14 +169,25 @@ export interface ModelRequestResult { error?: string; } +export interface SendModelRequestOptions { + baseUrl?: string; + apiKey?: string; +} + // Deliberately lighter than liveGeminiShared's sendAndValidate (no retry // loop, one fixed prompt pair): this suite's job is breadth across every -// model in the real combo, not depth on any single provider. +// model in the real combo, not depth on any single provider. baseUrl/apiKey +// default to the module-level omniroute-beta target but can be overridden — +// e.g. by the wire-capture suite, which points requests at its own +// dedicated throwaway container instead (see liveContainerHarness.ts). export async function sendModelRequest( model: string, stream: boolean, - apiFormat: "chat" | "responses" = "chat" + apiFormat: "chat" | "responses" = "chat", + options: SendModelRequestOptions = {} ): Promise { + const baseUrl = options.baseUrl ?? BASE_URL; + const apiKey = options.apiKey ?? API_KEY; const endpoint = apiFormat === "responses" ? "/v1/responses" : "/v1/chat/completions"; const messages: Message[] = [genSystemMessage(), genUserMessage()]; const body = @@ -182,9 +201,9 @@ export async function sendModelRequest( const start = performance.now(); try { - const response = await fetch(`${BASE_URL}${endpoint}`, { + const response = await fetch(`${baseUrl}${endpoint}`, { method: "POST", - headers: { "Content-Type": "application/json", Authorization: `Bearer ${API_KEY}` }, + headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` }, body: JSON.stringify(body), signal: controller.signal, }); diff --git a/tests/integration/wireCapture.ts b/tests/integration/wireCapture.ts new file mode 100644 index 0000000000..4485c4ea6c --- /dev/null +++ b/tests/integration/wireCapture.ts @@ -0,0 +1,154 @@ +/** + * tests/integration/wireCapture.ts + * + * Rootless wire capture + analysis for live container tests. Uses + * `podman unshare nsenter --net=` to run tcpdump without + * sudo/root (verified working against a rootless podman container — see + * scripts/sre/tcp-close-analyzer.py's docstring for the equivalent + * root-requiring `nsenter -t $PID` command this generalizes from), then + * shells out to that same script to reassemble TCP streams and extract + * HTTP request/response lines + correlationId per stream. + */ +import { spawn, spawnSync } from "node:child_process"; +import { existsSync, readFileSync, unlinkSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +const REPO_ROOT = fileURLToPath(new URL("../..", import.meta.url)); +const ANALYZER_SCRIPT = `${REPO_ROOT}scripts/sre/tcp-close-analyzer.py`; + +export interface WireStreamRecord { + streamKey: string; + client: string | null; + server: string | null; + firstTs: number; + lastTs: number; + durationSec: number; + packetCount: number; + correlationId: string | null; + requestId: string | null; + firstLineFromA: string | null; + firstLineFromB: string | null; + closes: Array<{ ts: number; side: string; src: string; dst: string; flags: string }>; + verdict: + | "client_closed_first" + | "server_closed_first" + | "simultaneous" + | "no_close_seen" + | "unknown_side_closed_first"; +} + +export interface CaptureHandle { + pcapPath: string; + stop(): Promise; +} + +// Best-effort HTTP status line finder — checks both reassembled directions +// since we don't know a priori which one carried the response. +export function responseStatusLine(record: WireStreamRecord): string | null { + for (const line of [record.firstLineFromA, record.firstLineFromB]) { + if (line && /^HTTP\/\d\.\d \d{3}/.test(line)) return line; + } + return null; +} + +export function requestLine(record: WireStreamRecord): string | null { + for (const line of [record.firstLineFromA, record.firstLineFromB]) { + if (line && /^(GET|POST|PUT|PATCH|DELETE) /.test(line)) return line; + } + return null; +} + +export async function startWireCapture( + netnsPath: string, + pcapPath: string, + bpfFilter: string +): Promise { + if (existsSync(pcapPath)) unlinkSync(pcapPath); + + // `-U`: flush each packet to disk as captured instead of buffering, so a + // non-graceful stop still leaves a readable pcap. + const child = spawn( + "podman", + [ + "unshare", + "nsenter", + `--net=${netnsPath}`, + "--", + "tcpdump", + "-i", + "any", + "-U", + "-w", + pcapPath, + bpfFilter, + ], + { stdio: ["ignore", "ignore", "pipe"] } + ); + + await new Promise((resolve, reject) => { + const timeout = setTimeout( + () => reject(new Error("tcpdump did not start listening in time")), + 10_000 + ); + child.stderr?.on("data", (chunk: Buffer) => { + if (chunk.toString().includes("listening on")) { + clearTimeout(timeout); + resolve(); + } + }); + child.on("exit", (code) => { + clearTimeout(timeout); + reject(new Error(`tcpdump exited early with code ${code}`)); + }); + }); + + return { + pcapPath, + async stop() { + // podman unshare -> nsenter -> tcpdump is a 3-level subprocess chain; + // SIGTERM to the top-level `podman` process (the only PID Node's + // child_process handle actually tracks) does not reliably reach the + // tcpdump grandchild, leaving it running as an orphan with a + // never-flushed pcap. pkill by the (unique, per-run) pcap path + // reliably reaches the real tcpdump process regardless of how deep + // the subprocess chain is. + child.kill("SIGTERM"); + spawnSync("pkill", ["-f", `tcpdump.*${pcapPath}`]); + await new Promise((resolve) => { + if (child.exitCode !== null) return resolve(); + child.on("exit", () => resolve()); + setTimeout(resolve, 3_000); + }); + // Give the now-dead tcpdump's OS write buffers a moment to land on + // disk before anything tries to read the pcap. + await new Promise((r) => setTimeout(r, 250)); + }, + }; +} + +export async function analyzeCapture(pcapPath: string): Promise { + const jsonlPath = pcapPath.replace(/\.pcap$/, "") + ".streams.jsonl"; + const result = spawnSync("python3", [ANALYZER_SCRIPT, pcapPath, "--out", jsonlPath], { + encoding: "utf8", + }); + if (result.status !== 0) { + throw new Error(`tcp-close-analyzer.py failed: ${result.stderr || result.stdout}`); + } + if (!existsSync(jsonlPath)) return []; + + return readFileSync(jsonlPath, "utf8") + .split("\n") + .filter((line) => line.trim()) + .map((line) => JSON.parse(line) as WireStreamRecord); +} + +export function indexByCorrelationId(records: WireStreamRecord[]): Map { + const map = new Map(); + for (const record of records) { + if (!record.correlationId) continue; + const existing = map.get(record.correlationId) || []; + existing.push(record); + map.set(record.correlationId, existing); + } + return map; +}