OmniRoute/scripts/test/combo-live-vps.mjs

/**
 * scripts/test/combo-live-vps.mjs
 *
 * Phase-3 VPS HTTP scenario driver for OmniRoute combo routing.
 * Exercises 6 strategies (priority / round-robin / weighted / cost-optimized /
 * fusion / auto) against the live server at 192.168.0.15:20128.
 *
 * Usage:
 *   node scripts/test/combo-live-vps.mjs
 *   node scripts/test/combo-live-vps.mjs --only=round-robin
 *   node scripts/test/combo-live-vps.mjs --failover   # runs all 7 base scenarios + real failover
 *
 * Safety rules:
 *   - Only creates/deletes __live_test__* combos
 *   - Always cleans up in `finally` blocks
 *   - Never stops services or touches other data
 *
 * Exit code: 0 if all scenarios PASS or SKIP; non-zero only on real FAIL.
 * Task 8 (--failover) can be appended after the main() call at the bottom.
 */

import { chat, createCombo, deleteCombo, listHealthyProviders, nonce } from "./_vpsClient.mjs";

// ---------------------------------------------------------------------------
// Candidate list — broad to maximise coverage across volatile provider health.
// listHealthyProviders() probes each with a real chat() call.
// ---------------------------------------------------------------------------
const BROAD_CANDIDATES = [
  "groq/llama-3.1-8b-instant",
  "groq/llama-3.3-70b-versatile",
  "minimax/MiniMax-M3",
  "minimax/minimax-m3",
  "kimi-coding-apikey/moonshot-v1-8k",
  "openrouter/openai/gpt-3.5-turbo",
  "cerebras/llama-3.3-70b",
  "cerebras/llama3.1-8b",
  "deepseek/deepseek-chat",
  "ollama-cloud/glm-5.2",
  "glm/glm-4-flash",
  "gemini/gemini-2.0-flash",
];

// Known approximate input cost ($/M tokens) from OmniRoute's default-pricing constants.
// Used only to identify cheap vs pricey pairs for cost-optimized scenario.
// Models absent from this map are treated as unknown cost (Infinity in the server's
// sortModelsByCost, i.e. sorted last — effectively "most expensive").
const KNOWN_INPUT_COST = {
  "groq/llama-3.1-8b-instant": 0,       // inference-hosts.ts: price=0 (free tier)
  "groq/llama-3.3-70b-versatile": 0,     // inference-hosts.ts: price=0 (free tier)
  "cerebras/llama3.1-8b": 0,             // inference-hosts.ts: price=0
  "cerebras/llama-3.3-70b": 0,           // inference-hosts.ts: price=0
  "deepseek/deepseek-chat": 0,           // inference-hosts.ts: price=0
  "minimax/MiniMax-M3": 0.5,             // regional.ts: $0.5/M input
  "minimax/minimax-m3": 0.5,             // regional.ts: $0.5/M input
  "kimi-coding-apikey/moonshot-v1-8k": 1, // not in pricing table → Infinity on server → treated pricey
};

// ---------------------------------------------------------------------------
// CLI args
// ---------------------------------------------------------------------------
const onlyArg = process.argv.find((a) => a.startsWith("--only="));
const onlyScenario = onlyArg ? onlyArg.slice(7) : null;
// --failover: opt-in flag that appends a real-failover scenario (broken primary → healthy fallback)
const failoverFlag = process.argv.includes("--failover");

// ---------------------------------------------------------------------------
// Result tracking
// ---------------------------------------------------------------------------
let exitCode = 0;
const summary = [];

function pass(name, detail = "") {
  summary.push({ name, result: "PASS" });
  console.log(`PASS  [${name}]${detail ? ": " + detail : ""}`);
}

function skip(name, reason) {
  summary.push({ name, result: "SKIP" });
  console.log(`SKIP  [${name}]: ${reason}`);
}

function fail(name, reason, err = null) {
  summary.push({ name, result: "FAIL" });
  console.error(`FAIL  [${name}]: ${reason}`);
  if (err) console.error("       caused by:", err?.message ?? String(err));
  exitCode = 1;
}

async function runScenario(name, fn) {
  if (onlyScenario && onlyScenario !== name) return;
  try {
    await fn();
  } catch (err) {
    fail(name, `unexpected error: ${err?.message ?? String(err)}`, err);
  }
}

// ---------------------------------------------------------------------------
// Helper: split "provider/model" → { providerId, modelPart }
// For multi-segment paths like "openrouter/openai/gpt-3.5-turbo":
//   providerId = "openrouter", modelPart = "openai/gpt-3.5-turbo"
// ---------------------------------------------------------------------------
function splitProviderModel(full) {
  const idx = full.indexOf("/");
  if (idx < 0) return { providerId: full, modelPart: full };
  return { providerId: full.slice(0, idx), modelPart: full.slice(idx + 1) };
}

// ---------------------------------------------------------------------------
// STEP 0: Cache probe
// Verifies that a sqlite-inserted combo is immediately routable.
// getComboByName() in combos.ts does a direct DB read with no TTL cache,
// so the combo should be visible as soon as the SSH INSERT commits.
// If not (unexpected caching behaviour), we poll up to 12s before blocking.
// ---------------------------------------------------------------------------
async function step0CacheProbe(healthy) {
  console.log("\n=== STEP 0: sqlite → routable cache probe ===");
  const probe = healthy[0];
  const probeName = "__live_test__probe";
  let id;
  let blocked = false;

  try {
    id = createCombo({ name: probeName, strategy: "priority", models: [probe] });
    console.log(`  created combo id=${id} model=${probe}`);

    // Attempt immediate chat — expect instant visibility (getComboByName bypasses cache)
    let r = await chat(probeName, { maxTokens: 4 });
    if (r.status === 200 && r.text) {
      console.log(
        `  chat() → ${r.status} model=${r.model} text="${r.text.slice(0, 40)}"`
      );
      console.log("  PROBE RESULT: immediately routable — getComboByName bypasses in-memory cache");
    } else {
      console.log(
        `  Immediate chat → status=${r.status} text=${r.text ?? "(empty)"}`
      );
      console.log("  Polling up to 12 s (TTL cache unexpectedly active)...");
      let resolved = false;
      for (let i = 0; i < 6; i++) {
        await new Promise((res) => setTimeout(res, 2000));
        r = await chat(probeName, { maxTokens: 4 });
        if (r.status === 200 && r.text) {
          console.log(`  Resolved after ~${(i + 1) * 2}s: model=${r.model}`);
          console.log("  PROBE RESULT: visible after TTL expiry");
          resolved = true;
          break;
        }
      }
      if (!resolved) {
        console.error("  PROBE RESULT: BLOCKER — combo not routable within 12s");
        blocked = true;
      }
    }
  } finally {
    if (id) {
      try {
        deleteCombo(probeName);
        console.log(`  cleanup: ${probeName} deleted`);
      } catch (e) {
        console.error(`  cleanup error: ${e?.message}`);
      }
    }
  }

  if (blocked) {
    throw new Error("BLOCKER: sqlite-inserted combo not routable within 12s — cannot run scenarios");
  }
}

// ---------------------------------------------------------------------------
// Scenario 1: priority
// Two healthy models. Single chat() call → 200 + non-empty text.
// ---------------------------------------------------------------------------
async function scenarioPriority(healthy) {
  const name = "__live_test__priority";
  if (healthy.length < 1) {
    skip("priority", "no healthy providers");
    return;
  }
  const models = healthy.slice(0, 2);
  let id;
  try {
    id = createCombo({ name, strategy: "priority", models });
    const r = await chat(name, { maxTokens: 16 });
    if (r.status !== 200) {
      fail("priority", `status=${r.status} raw=${JSON.stringify(r.raw)?.slice(0, 120)}`);
      return;
    }
    if (!r.text) {
      fail("priority", "response text is empty");
      return;
    }
    pass("priority", `status=200 model=${r.model} text="${r.text.slice(0, 40)}"`);
  } finally {
    if (id) try { deleteCombo(name); } catch { /* best effort */ }
  }
}

// ---------------------------------------------------------------------------
// Scenario 2: round-robin
// ≥2 healthy models. 5 calls with unique nonces → at least 2 distinct
// response.model values (each call must return 200).
// ---------------------------------------------------------------------------
async function scenarioRoundRobin(healthy) {
  const name = "__live_test__round-robin";
  if (healthy.length < 2) {
    skip("round-robin", `need ≥2 healthy providers, found ${healthy.length}`);
    return;
  }
  const models = healthy.slice(0, Math.min(3, healthy.length));
  let id;
  try {
    id = createCombo({ name, strategy: "round-robin", models });
    const served = new Set();
    for (let i = 0; i < 5; i++) {
      const r = await chat(name, { maxTokens: 16 });
      if (r.status !== 200) {
        fail("round-robin", `call ${i + 1} returned status=${r.status}`);
        return;
      }
      served.add(r.model);
    }
    if (served.size < 2) {
      fail(
        "round-robin",
        `only 1 distinct model across 5 calls: [${[...served].join(", ")}] — round-robin not distributing`
      );
      return;
    }
    pass("round-robin", `${served.size} distinct models across 5 calls: [${[...served].join(", ")}]`);
  } finally {
    if (id) try { deleteCombo(name); } catch { /* best effort */ }
  }
}

// ---------------------------------------------------------------------------
// Scenario 3: weighted
// Two healthy models with equal weight (50/50). 8 calls → both appear at
// least once (loose statistical check — P(only 1 model in 8 calls) ≈ 0.8%).
// ---------------------------------------------------------------------------
async function scenarioWeighted(healthy) {
  const name = "__live_test__weighted";
  if (healthy.length < 2) {
    skip("weighted", `need ≥2 healthy providers, found ${healthy.length}`);
    return;
  }
  const [m1, m2] = healthy.slice(0, 2);
  const { providerId: p1, modelPart: mp1 } = splitProviderModel(m1);
  const { providerId: p2, modelPart: mp2 } = splitProviderModel(m2);
  let id;
  try {
    id = createCombo({
      name,
      strategy: "weighted",
      models: [
        { providerId: p1, model: mp1, weight: 50 },
        { providerId: p2, model: mp2, weight: 50 },
      ],
    });
    const tally = {};
    for (let i = 0; i < 8; i++) {
      const r = await chat(name, { maxTokens: 16 });
      if (r.status !== 200) {
        fail("weighted", `call ${i + 1} returned status=${r.status}`);
        return;
      }
      tally[r.model] = (tally[r.model] ?? 0) + 1;
    }
    const distinct = Object.keys(tally);
    if (distinct.length < 2) {
      fail(
        "weighted",
        `only 1 distinct model across 8 calls: ${JSON.stringify(tally)} — weighted routing not distributing`
      );
      return;
    }
    pass("weighted", `8 calls distribution: ${JSON.stringify(tally)}`);
  } finally {
    if (id) try { deleteCombo(name); } catch { /* best effort */ }
  }
}

// ---------------------------------------------------------------------------
// Scenario 4: cost-optimized
// Find a cheap+pricey healthy pair (based on KNOWN_INPUT_COST).
// Insert pricey first (position 0) so cost-optimized must reorder.
// Assert: the served model matches the cheap provider.
//
// OmniRoute's sortModelsByCost uses getPricingForModel which merges
// default-pricing constants. groq models have price=0; minimax M3 has $0.5.
// Cost-optimized sorts ascending → groq (0) before minimax (0.5).
// ---------------------------------------------------------------------------
async function scenarioCostOptimized(healthy) {
  const name = "__live_test__cost-optimized";

  // Partition healthy into cheap (price=0) and pricey (price>0 or unknown>0)
  const cheap = healthy.filter(
    (m) => KNOWN_INPUT_COST[m] !== undefined && KNOWN_INPUT_COST[m] === 0
  );
  const pricey = healthy.filter(
    (m) => KNOWN_INPUT_COST[m] !== undefined && KNOWN_INPUT_COST[m] > 0
  );

  if (cheap.length === 0 || pricey.length === 0) {
    skip(
      "cost-optimized",
      `no distinguishable cheap+pricey pair among healthy=[${healthy.join(", ")}]`
    );
    return;
  }

  const cheapModel = cheap[0];
  const priceyModel = pricey[0];
  let id;
  try {
    // Insert pricey first — cost-optimized should reorder to serve cheapModel first
    id = createCombo({
      name,
      strategy: "cost-optimized",
      models: [priceyModel, cheapModel],
    });
    const r = await chat(name, { maxTokens: 16 });
    if (r.status !== 200) {
      fail("cost-optimized", `status=${r.status}`);
      return;
    }
    if (!r.text) {
      fail("cost-optimized", "empty response text");
      return;
    }
    // Verify the cheap model was served (response.model contains cheap provider's model name)
    const cheapProvider = splitProviderModel(cheapModel).providerId;
    // Both direct match and provider-substring match are accepted since OmniRoute
    // returns the raw upstream model name (e.g. "llama-3.1-8b-instant" not "groq/...")
    const cheapModelPart = splitProviderModel(cheapModel).modelPart.toLowerCase();
    const servedModel = (r.model ?? "").toLowerCase();
    const isChapModel =
      servedModel === cheapModelPart ||
      servedModel.includes(cheapModelPart) ||
      servedModel === cheapModel.toLowerCase() ||
      // fallback: check provider header if available (response.model could be bare name)
      servedModel.includes(cheapProvider.toLowerCase());

    if (!isChapModel) {
      fail(
        "cost-optimized",
        `expected cheaper model (${cheapModel}, price=${KNOWN_INPUT_COST[cheapModel]}) but got ${r.model} — cost-optimized may not have reordered`
      );
      return;
    }
    pass(
      "cost-optimized",
      `cheaper model served: ${r.model} (cheap=${cheapModel}@$${KNOWN_INPUT_COST[cheapModel]}, pricey=${priceyModel}@$${KNOWN_INPUT_COST[priceyModel]})`
    );
  } finally {
    if (id) try { deleteCombo(name); } catch { /* best effort */ }
  }
}

// ---------------------------------------------------------------------------
// Scenario 5: fusion
// Panel of 2-3 healthy models fan out in parallel; judge synthesizes 1 answer.
// Cost guard: panel ≤3, max_tokens=16, one call.
// ---------------------------------------------------------------------------
async function scenarioFusion(healthy) {
  const name = "__live_test__fusion";
  if (healthy.length < 2) {
    skip("fusion", `need ≥2 healthy providers, found ${healthy.length}`);
    return;
  }
  // Panel: up to 3 distinct models
  const panelModels = healthy.slice(0, Math.min(3, healthy.length));
  // Judge: reuse first healthy model (cheap, already warmed up)
  const judgeModel = panelModels[0];
  let id;
  try {
    id = createCombo({
      name,
      strategy: "fusion",
      models: panelModels,
      config: {
        judgeModel,
        fusionTuning: { minPanel: 2 },
      },
    });
    // Use a unique nonce in content to defeat semantic cache
    const r = await chat(name, {
      maxTokens: 16,
      content: `hi ${nonce()} answer in one word`,
    });
    if (r.status !== 200) {
      fail("fusion", `status=${r.status} raw=${JSON.stringify(r.raw)?.slice(0, 120)}`);
      return;
    }
    if (!r.text) {
      fail("fusion", "judge returned empty synthesized text");
      return;
    }
    pass("fusion", `synthesized text="${r.text.slice(0, 60)}" served-model=${r.model}`);
  } finally {
    if (id) try { deleteCombo(name); } catch { /* best effort */ }
  }
}

// ---------------------------------------------------------------------------
// Scenario 6: auto
// model="auto" → virtual auto-combo bypasses DB lookup entirely.
// Assert: status=200, non-empty text, response.model is a real model (not "auto").
// Also test "auto/fast" variant (skip if it 400s as unknown).
// ---------------------------------------------------------------------------
async function scenarioAuto() {
  // 6a: bare "auto"
  const r = await chat("auto", { maxTokens: 16 });
  if (r.status !== 200) {
    fail("auto", `status=${r.status} raw=${JSON.stringify(r.raw)?.slice(0, 120)}`);
    return;
  }
  if (!r.text) {
    fail("auto", "empty response text");
    return;
  }
  const servedModel = r.model ?? "";
  if (servedModel === "auto" || !servedModel) {
    fail("auto", `response.model is still "auto" — pool was not resolved`);
    return;
  }
  pass("auto", `status=200 resolved-model=${servedModel} text="${r.text.slice(0, 40)}"`);

  // 6b: "auto/fast" variant
  const r2 = await chat("auto/fast", { maxTokens: 16 });
  if (r2.status === 400 || r2.status === 404) {
    skip("auto/fast", `variant not recognised (${r2.status})`);
  } else if (r2.status !== 200 || !r2.text) {
    fail("auto/fast", `status=${r2.status} text=${r2.text ?? "(empty)"}`);
  } else {
    pass("auto/fast", `status=200 model=${r2.model} text="${r2.text.slice(0, 40)}"`);
  }
}

// ---------------------------------------------------------------------------
// Scenario 7: failover (opt-in via --failover)
//
// Approach: CROSS-PROVIDER BOGUS-MODEL (deterministic, no SSH crypto required).
//
// Why cross-provider?
//   A same-provider bogus-model fails silently: when target[0] (bogus) returns a
//   404, OmniRoute calls recordProviderCooldown(providerA, undefined) — a provider-
//   wide key. The combo then pre-screens target[1] (same providerA, real model) and
//   finds providerA in cooldown → skips it → combo returns the 404 from target[0].
//   Using DIFFERENT providers avoids this: target[0] puts providerA in cooldown,
//   target[1] on providerB is unaffected, providerB serves the healthy response.
//
// Mechanism:
//   - Target[0]: <providerA>/__nonexistent_model_xyz__ → upstream 404 → providerA cooldown
//   - Target[1]: <providerB>/<realModel>              → not in cooldown → 200 served
//   - config.maxRetries:0, retryDelayMs:0 → immediate fallover, no retry delay
//   - Bogus model can never return 200 → any 200 proves fallover to the real target
//
// If only 1 distinct provider is healthy: SKIP (BROKEN-CONNECTION approach needed; see
// comment below for how to implement it using encrypted wrong API key via SSH sqlite).
//
// BROKEN-CONNECTION approach (for future reference or if cross-provider is unavailable):
//   1. SSH to read STORAGE_ENCRYPTION_KEY from /root/.omniroute/.env
//   2. Encrypt a wrong API key using scryptSync(key,"omniroute-field-encryption-v1",32)+AES-256-GCM
//   3. INSERT broken provider_connection row into provider_connections table via SSH sqlite
//   4. Combo: [{providerId:glm, model:glm/glm-4-flash, connectionId:brokenConnId}, realModel]
//   5. Broken conn → 401 → recordProviderCooldown("glm",brokenConnId) → key "glm:brokenConnId"
//   6. Target[1] on different provider → unaffected → 200
//   7. Finally: DELETE combo AND broken connection
// ---------------------------------------------------------------------------
async function scenarioFailover(healthy) {
  const name = "__live_test__failover";

  if (healthy.length < 1) {
    skip("failover", "no healthy providers — cannot run failover scenario");
    return;
  }

  // Group healthy providers by providerId to find two distinct providers
  const byProvider = new Map();
  for (const m of healthy) {
    const { providerId } = splitProviderModel(m);
    if (!byProvider.has(providerId)) byProvider.set(providerId, []);
    byProvider.get(providerId).push(m);
  }
  const distinctProviders = [...byProvider.keys()];

  if (distinctProviders.length < 2) {
    // Cannot use cross-provider approach with only one provider.
    // Same-provider bogus-model fails: 404 → recordProviderCooldown(provider, undefined) →
    // provider-wide cooldown blocks target[1] on the same provider.
    skip(
      "failover",
      `need ≥2 distinct healthy providers for cross-provider approach; ` +
        `found only 1 [${distinctProviders.join(", ")}]. ` +
        `Implement BROKEN-CONNECTION approach to run failover with a single provider.`
    );
    return;
  }

  // CROSS-PROVIDER BOGUS-MODEL:
  //   target[0] = <providerA>/__nonexistent_model_xyz__  (will get 404, puts providerA in cooldown)
  //   target[1] = <providerB>/<realHealthyModel>         (different provider, not in cooldown)
  const bogusProvider = distinctProviders[0];
  const realModel = byProvider.get(distinctProviders[1])[0];
  const bogusModel = `${bogusProvider}/__nonexistent_model_xyz__`;
  const { modelPart: realModelPart } = splitProviderModel(realModel);

  let id;
  try {
    id = createCombo({
      name,
      strategy: "priority",
      models: [bogusModel, realModel],
      config: { maxRetries: 0, retryDelayMs: 0 },
    });

    console.log(
      `  failover combo (CROSS-PROVIDER BOGUS-MODEL):\n` +
        `    [0] ${bogusModel} ← broken primary (will 404)\n` +
        `    [1] ${realModel} ← healthy fallback (different provider)\n` +
        `    strategy=priority, maxRetries=0`
    );

    const r = await chat(name, { maxTokens: 16 });

    // Assertions:
    // 1. status=200: the combo succeeded — ONLY possible if it fell over to target[1],
    //    because target[0] (bogus model) can NEVER return 200 from the upstream.
    // 2. Non-empty text: real LLM content was returned (not an empty error body).
    // 3. Served model is NOT the bogus one (belt-and-suspenders; 200 already proves it).
    // 4. Served model matches the real healthy target (positive proof of which model served).

    if (r.status !== 200) {
      fail(
        "failover",
        `expected status=200 after fallover (broken ${bogusModel} → ${realModel}), ` +
          `got status=${r.status}. ` +
          `raw=${JSON.stringify(r.raw)?.slice(0, 200)}`
      );
      return;
    }

    if (!r.text) {
      fail("failover", `status=200 but empty text — fallover may have served a no-content response`);
      return;
    }

    const bogusModelPart = "__nonexistent_model_xyz__";
    const servedModel = (r.model ?? "").toLowerCase();

    // Negative proof: bogus model did NOT serve (should never happen, but guard anyway)
    if (servedModel.includes(bogusModelPart.toLowerCase())) {
      fail(
        "failover",
        `bogus model string "${bogusModelPart}" appears in served model field "${r.model}" — ` +
          `impossible 200 from a non-existent model; something is wrong`
      );
      return;
    }

    // Positive proof: served model matches the real healthy target
    const realModelLower = realModelPart.toLowerCase();
    const servedMatchesReal =
      servedModel === realModelLower ||
      servedModel.includes(realModelLower) ||
      servedModel === realModel.toLowerCase();

    const proofNote = servedMatchesReal
      ? ""
      : ` [NOTE: served="${r.model}" ≠ expected="${realModelPart}" ` +
        `— upstream alias likely; 200+text from bogus-primary is the failover proof]`;

    pass(
      "failover",
      `CROSS-PROVIDER BOGUS-MODEL: broken primary (${bogusModel}) → ` +
        `fallover → served ${r.model} (real target: ${realModel}) ` +
        `text="${r.text.slice(0, 40)}"${proofNote}`
    );
  } finally {
    if (id) {
      try {
        deleteCombo(name);
        console.log(`  cleanup: ${name} deleted`);
      } catch (e) {
        console.error(`  cleanup error: ${e?.message}`);
      }
    }
  }
}

// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
  console.log("=== OmniRoute combo-live-vps scenario driver ===");
  if (onlyScenario) console.log(`Filtering to scenario: ${onlyScenario}`);
  console.log();

  // --- Health probe ---
  console.log("Probing healthy providers (broad candidate list)...");
  const healthy = await listHealthyProviders(BROAD_CANDIDATES);
  console.log(`  healthy (${healthy.length}/${BROAD_CANDIDATES.length}): [${healthy.join(", ")}]`);

  if (healthy.length === 0) {
    console.error("FATAL: no healthy providers — cannot run any scenarios");
    process.exit(1);
  }

  // --- STEP 0: cache probe ---
  if (!onlyScenario) {
    // Run STEP 0 unconditionally unless --only is passed (targeted run skips housekeeping)
    try {
      await step0CacheProbe(healthy);
    } catch (err) {
      console.error(`\nBLOCKER: ${err.message}`);
      process.exit(1);
    }
  }

  console.log("\n=== Scenarios ===\n");

  await runScenario("priority", () => scenarioPriority(healthy));
  await runScenario("round-robin", () => scenarioRoundRobin(healthy));
  await runScenario("weighted", () => scenarioWeighted(healthy));
  await runScenario("cost-optimized", () => scenarioCostOptimized(healthy));
  await runScenario("fusion", () => scenarioFusion(healthy));
  await runScenario("auto", () => scenarioAuto());

  // --- Scenario 7: failover (opt-in) ---
  // Only runs when --failover is passed. Uses a bogus-model primary to force a real
  // combo failover to the healthy secondary, proving the priority fallover path works
  // against the live server without stopping any service.
  if (failoverFlag) {
    console.log("\n=== Failover scenario (--failover) ===\n");
    await runScenario("failover", () => scenarioFailover(healthy));
  }

  // --- Summary ---
  console.log("\n=== Summary ===");
  for (const { name, result } of summary) {
    console.log(`  ${result.padEnd(4)} [${name}]`);
  }
  const passed = summary.filter((s) => s.result === "PASS").length;
  const skipped = summary.filter((s) => s.result === "SKIP").length;
  const failed = summary.filter((s) => s.result === "FAIL").length;
  console.log(`\n  ${passed} PASS  ${skipped} SKIP  ${failed} FAIL`);

  process.exit(exitCode);
}

main().catch((err) => {
  console.error("FATAL:", err?.message ?? err);
  process.exit(1);
});