feat(homolog): real-environment E2E homologation suite (npm run homolog) (#7133)

* feat(homolog): scaffolding da suíte de homologação E2E (deps + npm run homolog)

* feat(homolog): L0 avaliador de paridade de deploy (TDD)

* feat(homolog): L1a ciclo de vida de API key efêmera (login admin -> create -> revoke)

* feat(homolog): L1b suite httpYac de API (models, chat, auth de management, health)

* feat(homolog): L1c checker SSE de streaming real (TDD no parser)

* feat(homolog): L2 smoke de providers reais via promptfoo gerado do catálogo

* feat(homolog): L4a Playwright homolog config + login storageState

* feat(homolog): L4b smoke de todas as rotas do dashboard (descoberta via fs)

* feat(homolog): L4c fluxo criar/revogar API key pela UI

* fix(homolog): resiliencia real-environment — stream:false no smoke promptfoo, retry de socket keep-alive, key efemera com sufixo unico

* feat(homolog): L5 orquestrador npm run homolog + relatorio CTRF unificado

* docs(homolog): guia de operacao da suite + fragment de changelog + allowlist env-doc-sync

* fix(homolog): paraleliza o sweep de rotas do dashboard (fullyParallel + 8 workers)

* fix(homolog): isola outputs crus em homolog-report/raw para nao quebrar o ctrf merge

* fix(homolog): outputDir absoluto do reporter CTRF da UI (path relativo escapava do worktree)

* chore(quality): allowlist the 5 homolog-suite devDependencies (ctrf-io trio, httpyac, promptfoo) after registry verification

* chore(quality): register the homolog Playwright suite as a test-discovery collector (run.mjs -> tests/homolog/ui)

scripts/check/check-env-doc-sync.mjs

@@ -101,6 +101,15 @@ const IGNORE_FROM_CODE = new Set([
   // ("http://192.168.0.15:20128" / null), never OmniRoute runtime config (#5151).
   "COMBO_LIVE_BASE_URL",
   "COMBO_LIVE_API_KEY",
+  // Homologation E2E suite (npm run homolog) vars — configured via the dedicated
+  // .env.homolog file (template: .env.homolog.example), never in the runtime .env.
+  // Test/ops-only signals against the homologation VPS, same class as COMBO_LIVE_*.
+  // See docs/ops/HOMOLOGATION.md.
+  "HOMOLOG_BASE_URL",
+  "HOMOLOG_ADMIN_PASSWORD",
+  "HOMOLOG_API_KEY",
+  "HOMOLOG_CRITICAL_PROVIDERS",
+  "HOMOLOG_EXPECT_VERSION",
   // update-notifier opt-out for the CLI binary.
   "OMNIROUTE_NO_UPDATE_NOTIFIER",
   // Headless CLI execution flag for Electron.

scripts/check/check-test-discovery.mjs

@@ -127,6 +127,13 @@ export const COLLECTORS = [
     glob: "tests/e2e/protocol-clients.test.ts",
     sources: ["scripts/dev/run-protocol-clients-tests.mjs"],
   },
+  // Playwright — suíte de homologação real (npm run homolog, L4 UI): run.mjs invoca
+  // `playwright test -c tests/homolog/ui/playwright.config.ts` (testMatch **/*.spec.ts).
+  {
+    glob: "tests/homolog/ui/*.spec.ts",
+    sources: ["scripts/homolog/run.mjs"],
+    anchors: { "scripts/homolog/run.mjs": "tests/homolog/ui/playwright.config.ts" },
+  },
 ];
 
 const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");

scripts/homolog/gen-promptfoo.mjs

@@ -0,0 +1,52 @@
+import fs from "node:fs";
+import path from "node:path";
+import { pickSmokeModels } from "./lib/providerTiers.mjs";
+
+const baseUrl = process.env.HOMOLOG_BASE_URL;
+const critical = (process.env.HOMOLOG_CRITICAL_PROVIDERS || "").split(",").filter(Boolean);
+
+const res = await fetch(`${baseUrl}/v1/models`, {
+  headers: { Authorization: `Bearer ${process.env.HOMOLOG_API_KEY}` },
+});
+if (!res.ok) throw new Error(`/v1/models HTTP ${res.status}`);
+const catalog = (await res.json()).data;
+
+const picks = pickSmokeModels(catalog, critical);
+const missing = picks.filter((p) => !p.model);
+const providers = picks
+  .filter((p) => p.model)
+  .map((p) => ({
+    id: `openai:chat:${p.model}`,
+    label: p.provider,
+    config: {
+      apiBaseUrl: `${baseUrl}/v1`,
+      apiKeyEnvar: "HOMOLOG_API_KEY",
+      max_tokens: 5,
+      temperature: 0,
+      // OmniRoute streama por default quando "stream" é omitido (streamDefaultMode
+      // legacy) — o parser JSON do promptfoo precisa da resposta non-stream.
+      passthrough: { stream: false, max_tokens: 5 },
+    },
+  }));
+
+const config = {
+  description: "OmniRoute homolog — smoke real 1 request/provider crítico",
+  prompts: ["Reply with exactly: OK"],
+  providers,
+  // O smoke valida o WIRING do provider (respondeu sem erro), não o comportamento
+  // do modelo: com max_tokens=5, modelos de reasoning podem gastar o budget antes
+  // de emitir o "OK" literal — icontains seria falso-positivo de quebra.
+  tests: [{ assert: [{ type: "javascript", value: "typeof output === 'string'" }] }],
+};
+fs.mkdirSync("homolog-report/raw", { recursive: true });
+fs.writeFileSync(
+  path.join("homolog-report", "promptfooconfig.yaml"),
+  JSON.stringify(config, null, 2) // promptfoo aceita JSON como config YAML-compatível
+);
+fs.writeFileSync(
+  path.join("homolog-report", "raw", "provider-misses.json"),
+  JSON.stringify(missing, null, 2)
+);
+console.log(
+  `[gen-promptfoo] ${providers.length} providers no smoke, ${missing.length} misses de catálogo`
+);

scripts/homolog/lib/adminClient.mjs

@@ -0,0 +1,65 @@
+// Cookie confirmado em src/app/api/auth/login/route.ts (cookieStore.set("auth_token", ...))
+// e em src/shared/utils/apiAuth.ts (isDashboardSessionAuthenticated lê "auth_token").
+const TOKEN_COOKIE = "auth_token";
+
+export function extractJwtCookie(setCookies) {
+  for (const c of setCookies || []) {
+    const m = c.match(new RegExp(`^(${TOKEN_COOKIE}=[^;]+)`));
+    if (m) return m[1];
+  }
+  return null;
+}
+
+export function extractApiKey(body) {
+  if (!body?.key || !body?.id) throw new Error("POST /api/keys sem key/id no corpo");
+  return { key: body.key, id: body.id };
+}
+
+// fetch com 1 retry para erros de socket (keep-alive reciclado pelo servidor
+// entre requests espaçados derruba o 1º write com EPIPE/other side closed).
+async function fetchRetry(url, init, retries = 1) {
+  try {
+    return await fetch(url, init);
+  } catch (err) {
+    if (retries > 0) {
+      await new Promise((r) => setTimeout(r, 1_000));
+      return fetchRetry(url, init, retries - 1);
+    }
+    throw err;
+  }
+}
+
+/** Login admin → cria API key efêmera. Retorna {key, id, cookie, revoke()}. */
+export async function createEphemeralKey(baseUrl, password) {
+  const login = await fetchRetry(`${baseUrl}/api/auth/login`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ password }),
+  });
+  if (!login.ok) throw new Error(`login falhou: HTTP ${login.status}`);
+  const cookie = extractJwtCookie(login.headers.getSetCookie());
+  if (!cookie) throw new Error("login sem cookie de sessão");
+
+  // sufixo único por run: dois runs paralelos (ou um cleanup por nome) nunca colidem
+  const name = `homolog-${new Date().toISOString().slice(0, 10)}-${Math.random().toString(36).slice(2, 8)}`;
+  const create = await fetchRetry(`${baseUrl}/api/keys`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json", cookie },
+    body: JSON.stringify({ name }),
+  });
+  if (!create.ok) throw new Error(`criação de key falhou: HTTP ${create.status}`);
+  const { key, id } = extractApiKey(await create.json());
+
+  return {
+    key,
+    id,
+    cookie,
+    async revoke() {
+      const del = await fetchRetry(`${baseUrl}/api/keys/${id}`, {
+        method: "DELETE",
+        headers: { cookie },
+      });
+      if (!del.ok) throw new Error(`revogação da key ${id} falhou: HTTP ${del.status}`);
+    },
+  };
+}

scripts/homolog/lib/parity.mjs

@@ -0,0 +1,15 @@
+/**
+ * Avaliação pura de paridade do deploy (testável sem rede).
+ * @param {{status?: string, version?: string}} health corpo de /api/monitoring/health
+ * @param {{expectedVersion: string, httpStatus: number}} ctx
+ * @returns {{ok: boolean, failures: string[]}}
+ */
+export function evaluateParity(health, ctx) {
+  const failures = [];
+  if (ctx.httpStatus !== 200) failures.push(`health HTTP ${ctx.httpStatus} (esperado 200)`);
+  if (health?.status !== "healthy")
+    failures.push(`status "${health?.status}" (esperado "healthy")`);
+  if (health?.version !== ctx.expectedVersion)
+    failures.push(`version "${health?.version}" (esperado "${ctx.expectedVersion}")`);
+  return { ok: failures.length === 0, failures };
+}

scripts/homolog/lib/promptfooToCtrf.mjs

@@ -0,0 +1,24 @@
+export function promptfooToCtrf(output) {
+  const rows = output?.results?.results || [];
+  const tests = rows.map((r) => ({
+    name: `provider-smoke: ${r.provider?.label || r.provider?.id || "?"}`,
+    status: r.success ? "passed" : "failed",
+    duration: Math.round(r.latencyMs || 0),
+    ...(r.error ? { message: String(r.error).slice(0, 300) } : {}),
+  }));
+  const passed = tests.filter((t) => t.status === "passed").length;
+  return {
+    results: {
+      tool: { name: "promptfoo" },
+      summary: {
+        tests: tests.length,
+        passed,
+        failed: tests.length - passed,
+        pending: 0,
+        skipped: 0,
+        other: 0,
+      },
+      tests,
+    },
+  };
+}

scripts/homolog/lib/providerTiers.mjs

@@ -0,0 +1,7 @@
+/** Escolhe 1 modelo por provider crítico a partir do catálogo /v1/models. */
+export function pickSmokeModels(catalog, criticalProviders) {
+  return criticalProviders.map((provider) => {
+    const hit = catalog.find((m) => m.id.startsWith(`${provider}/`));
+    return { provider, model: hit ? hit.id : null };
+  });
+}

scripts/homolog/lib/sseCheck.mjs

@@ -0,0 +1,80 @@
+export function parseSseChunk(text) {
+  // Itera LINHAS dentro de cada bloco: a VPS emite comment-lines SSE
+  // (": x-omniroute-*") no mesmo bloco do "data: [DONE]", então olhar só o
+  // início do bloco perde o terminador.
+  const events = [];
+  for (const block of text.split(/\n\n/)) {
+    for (const line of block.split("\n")) {
+      const t = line.trim();
+      if (t.startsWith("data:")) events.push(t.slice(5).trim());
+    }
+  }
+  return events;
+}
+
+export function summarizeStream(events) {
+  let contentDeltas = 0;
+  let done = false;
+  for (const e of events) {
+    if (e === "[DONE]") {
+      done = true;
+      continue;
+    }
+    try {
+      const j = JSON.parse(e);
+      if (j.choices?.[0]?.delta?.content) contentDeltas++;
+    } catch {
+      /* fragmento parcial — ignorado; o caller acumula buffer */
+    }
+  }
+  const ok = contentDeltas >= 1 && done;
+  return { ok, contentDeltas, done };
+}
+
+/** Faz 1 chat streaming real e valida o protocolo SSE ponta-a-ponta. */
+export async function checkSse(baseUrl, apiKey, model, { retries = 1 } = {}) {
+  try {
+    const res = await fetch(`${baseUrl}/v1/chat/completions`, {
+      method: "POST",
+      headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" },
+      body: JSON.stringify({
+        model,
+        messages: [{ role: "user", content: "Reply with exactly: OK" }],
+        max_tokens: 5,
+        stream: true,
+      }),
+    });
+    if (res.status !== 200) return { ok: false, failures: [`HTTP ${res.status}`] };
+    const ct = res.headers.get("content-type") || "";
+    if (!ct.includes("text/event-stream")) return { ok: false, failures: [`content-type "${ct}"`] };
+
+    const events = [];
+    let buffer = "";
+    const reader = res.body.getReader();
+    const decoder = new TextDecoder();
+    const deadline = Date.now() + 60_000;
+    while (Date.now() < deadline) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      buffer += decoder.decode(value, { stream: true });
+      const lastSep = buffer.lastIndexOf("\n\n");
+      if (lastSep >= 0) {
+        events.push(...parseSseChunk(buffer.slice(0, lastSep + 2)));
+        buffer = buffer.slice(lastSep + 2);
+      }
+    }
+    // flush do resto do buffer (último bloco pode chegar sem "\n\n" no read final)
+    if (buffer.trim()) events.push(...parseSseChunk(buffer));
+    const s = summarizeStream(events);
+    return { ok: s.ok, failures: s.ok ? [] : [`contentDeltas=${s.contentDeltas} done=${s.done}`] };
+  } catch (err) {
+    // Socket keep-alive reciclado pelo servidor entre requests é transitório —
+    // 1 retry antes de reportar falha. Erro persistente é FALHA da camada,
+    // nunca crash do orquestrador.
+    if (retries > 0) {
+      await new Promise((r) => setTimeout(r, 1_000));
+      return checkSse(baseUrl, apiKey, model, { retries: retries - 1 });
+    }
+    return { ok: false, failures: [`fetch/stream error: ${err?.cause?.message || err.message}`] };
+  }
+}

scripts/homolog/run.mjs

@@ -0,0 +1,169 @@
+#!/usr/bin/env node
+import { execSync, spawnSync } from "node:child_process";
+import fs from "node:fs";
+import { evaluateParity } from "./lib/parity.mjs";
+import { createEphemeralKey } from "./lib/adminClient.mjs";
+import { checkSse } from "./lib/sseCheck.mjs";
+import { promptfooToCtrf } from "./lib/promptfooToCtrf.mjs";
+
+// ── env ──────────────────────────────────────────────────────────────────
+if (fs.existsSync(".env.homolog")) {
+  for (const line of fs.readFileSync(".env.homolog", "utf8").split("\n")) {
+    const m = line.match(/^([A-Z_]+)=(.*)$/);
+    if (m && !process.env[m[1]]) process.env[m[1]] = m[2];
+  }
+}
+const BASE = process.env.HOMOLOG_BASE_URL;
+if (!BASE || !process.env.HOMOLOG_ADMIN_PASSWORD) {
+  console.error("Configure .env.homolog (HOMOLOG_BASE_URL, HOMOLOG_ADMIN_PASSWORD)");
+  process.exit(2);
+}
+fs.rmSync("homolog-report", { recursive: true, force: true });
+// raw/ fica FORA do merge CTRF: `ctrf merge` tenta mesclar qualquer *.json com
+// chave "results" e quebra no output cru do promptfoo.
+fs.mkdirSync("homolog-report/raw", { recursive: true });
+const layers = []; // {name, ok, detail}
+const record = (name, ok, detail = "") => {
+  layers.push({ name, ok, detail });
+  console.log(`${ok ? "✅" : "❌"} ${name}${detail ? ` — ${detail}` : ""}`);
+};
+
+// ── L0 saúde/paridade ────────────────────────────────────────────────────
+const expectedVersion =
+  process.env.HOMOLOG_EXPECT_VERSION || JSON.parse(fs.readFileSync("package.json", "utf8")).version;
+const healthRes = await fetch(`${BASE}/api/monitoring/health`);
+const health = await healthRes.json().catch(() => ({}));
+const parity = evaluateParity(health, { expectedVersion, httpStatus: healthRes.status });
+record("L0 saúde/paridade", parity.ok, parity.failures.join("; "));
+if (!parity.ok) {
+  console.error("Deploy divergente — abortando.");
+  writeSummary(layers, BASE, expectedVersion);
+  process.exit(1);
+}
+
+// ── chave efêmera ────────────────────────────────────────────────────────
+const eph = await createEphemeralKey(BASE, process.env.HOMOLOG_ADMIN_PASSWORD);
+process.env.HOMOLOG_API_KEY = eph.key;
+try {
+  // modelo de smoke = 1º do tier crítico presente no catálogo
+  const models = (
+    await (
+      await fetch(`${BASE}/v1/models`, { headers: { Authorization: `Bearer ${eph.key}` } })
+    ).json()
+  ).data;
+  const critical = (process.env.HOMOLOG_CRITICAL_PROVIDERS || "openai").split(",");
+  const smokeModel =
+    models.find((m) => critical.some((p) => m.id.startsWith(`${p}/`)))?.id || models[0].id;
+
+  // ── L1 httpYac + SSE ───────────────────────────────────────────────────
+  const hy = spawnSync(
+    "npx",
+    [
+      "httpyac",
+      "send",
+      "tests/homolog/api/core.http",
+      "--all",
+      "--var",
+      `baseUrl=${BASE}`,
+      "--var",
+      `apiKey=${eph.key}`,
+      "--var",
+      `smokeModel=${smokeModel}`,
+      "--junit",
+      "--output",
+      "none",
+    ],
+    { encoding: "utf8" }
+  );
+  fs.writeFileSync("homolog-report/httpyac-junit.xml", hy.stdout || "");
+  record("L1 API (httpYac)", hy.status === 0);
+  const sse = await checkSse(BASE, eph.key, smokeModel);
+  record("L1 SSE streaming", sse.ok, (sse.failures || []).join("; "));
+
+  // ── L2 providers reais ─────────────────────────────────────────────────
+  try {
+    execSync("node scripts/homolog/gen-promptfoo.mjs", { stdio: "inherit", env: process.env });
+    spawnSync(
+      "npx",
+      [
+        "promptfoo",
+        "eval",
+        "-c",
+        "homolog-report/promptfooconfig.yaml",
+        "-o",
+        "homolog-report/raw/promptfoo.json",
+        "--no-cache",
+      ],
+      { encoding: "utf8", env: process.env }
+    );
+    const pfOut = JSON.parse(fs.readFileSync("homolog-report/raw/promptfoo.json", "utf8"));
+    const pfCtrf = promptfooToCtrf(pfOut);
+    fs.writeFileSync("homolog-report/providers-ctrf.json", JSON.stringify(pfCtrf, null, 2));
+    record(
+      "L2 providers reais",
+      pfCtrf.results.summary.failed === 0,
+      `${pfCtrf.results.summary.passed}/${pfCtrf.results.summary.tests} providers OK`
+    );
+  } catch (err) {
+    // gerador/eval quebrando é falha da camada — o run continua para o L4 e o cleanup
+    record("L2 providers reais", false, err.message);
+  }
+
+  // ── L4 UI ──────────────────────────────────────────────────────────────
+  const pw = spawnSync(
+    "npx",
+    ["playwright", "test", "-c", "tests/homolog/ui/playwright.config.ts"],
+    {
+      stdio: "inherit",
+      env: process.env,
+    }
+  );
+  record("L4 UI (Playwright)", pw.status === 0);
+} finally {
+  await eph
+    .revoke()
+    .then(() => record("cleanup: key efêmera revogada", true))
+    .catch((e) => record("cleanup: key efêmera revogada", false, e.message));
+}
+
+// ── L5 relatório unificado ───────────────────────────────────────────────
+spawnSync(
+  "npx",
+  ["junit-to-ctrf", "homolog-report/httpyac-junit.xml", "-o", "homolog-report/api-ctrf.json"],
+  {
+    stdio: "inherit",
+  }
+);
+spawnSync(
+  "npx",
+  [
+    "ctrf",
+    "merge",
+    "homolog-report",
+    "--output",
+    "homolog-ctrf.json",
+    "--output-dir",
+    "homolog-report",
+  ],
+  {
+    stdio: "inherit",
+  }
+);
+
+writeSummary(layers, BASE, expectedVersion);
+const failed = layers.filter((l) => !l.ok);
+process.exit(failed.length ? 1 : 0);
+
+function writeSummary(rows, base, version) {
+  const md = [
+    "# Homologação — relatório",
+    "",
+    `Alvo: ${base} · versão esperada: ${version}`,
+    "",
+    "| camada | resultado | detalhe |",
+    "|---|---|---|",
+    ...rows.map((l) => `| ${l.name} | ${l.ok ? "✅" : "❌"} | ${l.detail} |`),
+  ].join("\n");
+  fs.writeFileSync("homolog-report/summary.md", md);
+  console.log(`\n${md}\n\nRelatório: homolog-report/ (CTRF unificado: homolog-ctrf.json)`);
+}