feat(cli): crash recovery com backoff exponencial e PID granular (Fase 1.9)

Adiciona ServerSupervisor (bin/cli/runtime/processSupervisor.mjs) que reinicia o
servidor com backoff exponencial (1s, 2s, 4s... cap 10s) em caso de crash.
Após maxRestarts falhas em 30s exibe crash log e encerra. Detecta MITM como
causa do crash via heurística e desabilita automaticamente.

PID management agora é granular por subprocesso (~/.omniroute/{service}/.pid)
suportando server, mitm e tunnel/cloudflared|tailscale. `stop` e
`killAllSubprocesses` encerram todos os serviços registrados.

Novas opções em `serve`: --log (passa stdout/stderr inline), --no-recovery
(comportamento legado sem supervisor), --max-restarts <n> (padrão 2).
This commit is contained in:
diegosouzapw
2026-05-15 00:18:53 -03:00
parent 6b63aa3948
commit a5fa94bdcb
9 changed files with 486 additions and 50 deletions

View File

@@ -785,6 +785,11 @@ APP_LOG_TO_FILE=true
# Falls back to LC_ALL / LC_MESSAGES / LANG / en if unset.
# OMNIROUTE_LANG=en
# Show server logs inline when running in supervised mode (omniroute serve).
# Set to "1" to forward server stdout/stderr to the terminal.
# Equivalent to the --log flag on `omniroute serve`.
# OMNIROUTE_SHOW_LOG=1
# Bearer token injected as x-omniroute-cli-token header for machine-auth (task 8.12).
# Auto-generated on first run if machine-id is available; set manually to override.
# OMNIROUTE_CLI_TOKEN=

View File

@@ -4,7 +4,8 @@ import { join, dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { platform } from "node:os";
import { t } from "../i18n.mjs";
import { writePidFile, cleanupPidFile } from "../utils/pid.mjs";
import { writePidFile, cleanupPidFile, waitForServer } from "../utils/pid.mjs";
import { ServerSupervisor, detectMitmCrash } from "../runtime/processSupervisor.mjs";
const __dirname = dirname(fileURLToPath(import.meta.url));
const ROOT = join(__dirname, "..", "..", "..");
@@ -22,6 +23,9 @@ export function registerServe(program) {
.option("--port <port>", t("serve.port"), "20128")
.option("--no-open", t("serve.no_open"))
.option("--daemon", t("serve.daemon"))
.option("--log", t("serve.log"))
.option("--no-recovery", t("serve.no_recovery"))
.option("--max-restarts <n>", t("serve.max_restarts"), parseInt, 2)
.action(async (opts) => {
await runServe(opts);
});
@@ -125,22 +129,48 @@ export async function runServe(opts = {}) {
const isDaemon = opts.daemon === true;
if (isDaemon) {
return runDaemon(serverJs, env, memoryLimit, dashboardPort, apiPort);
}
if (opts.noRecovery) {
return runWithoutRecovery(serverJs, env, memoryLimit, dashboardPort, apiPort, noOpen);
}
return runWithSupervisor(
serverJs,
env,
memoryLimit,
dashboardPort,
apiPort,
noOpen,
opts.log === true,
opts.maxRestarts ?? 2
);
}
function runDaemon(serverJs, env, memoryLimit, dashboardPort, apiPort) {
const server = spawn("node", [`--max-old-space-size=${memoryLimit}`, serverJs], {
cwd: APP_DIR,
env,
stdio: isDaemon ? "ignore" : "pipe",
detached: isDaemon,
stdio: "ignore",
detached: true,
});
writePidFile("server", server.pid);
server.unref();
console.log(`\x1b[32m✔ OmniRoute started in background (PID: ${server.pid})\x1b[0m`);
console.log(` \x1b[1mDashboard:\x1b[0m http://localhost:${dashboardPort}`);
console.log(` \x1b[1mAPI Base:\x1b[0m http://localhost:${apiPort}/v1`);
}
function runWithoutRecovery(serverJs, env, memoryLimit, dashboardPort, apiPort, noOpen) {
const server = spawn("node", [`--max-old-space-size=${memoryLimit}`, serverJs], {
cwd: APP_DIR,
env,
stdio: "pipe",
});
writePidFile(server.pid);
if (isDaemon) {
server.unref();
console.log(`\x1b[32m✔ OmniRoute started in background (PID: ${server.pid})\x1b[0m`);
console.log(` \x1b[1mDashboard:\x1b[0m http://localhost:${dashboardPort}`);
console.log(` \x1b[1mAPI Base:\x1b[0m http://localhost:${apiPort}/v1`);
return;
}
writePidFile("server", server.pid);
let started = false;
@@ -156,9 +186,7 @@ export async function runServe(opts = {}) {
}
});
server.stderr.on("data", (data) => {
process.stderr.write(data);
});
server.stderr.on("data", (data) => process.stderr.write(data));
server.on("error", (err) => {
console.error("\x1b[31m✖ Failed to start server:\x1b[0m", err.message);
@@ -172,15 +200,15 @@ export async function runServe(opts = {}) {
process.exit(code ?? 0);
});
function shutdown() {
const shutdown = () => {
console.log("\n\x1b[33m⏹ Shutting down OmniRoute...\x1b[0m");
cleanupPidFile();
cleanupPidFile("server");
server.kill("SIGTERM");
setTimeout(() => {
server.kill("SIGKILL");
process.exit(0);
}, 5000);
}
};
process.on("SIGINT", shutdown);
process.on("SIGTERM", shutdown);
@@ -193,6 +221,48 @@ export async function runServe(opts = {}) {
}, 15000);
}
async function runWithSupervisor(
serverJs,
env,
memoryLimit,
dashboardPort,
apiPort,
noOpen,
showLog,
maxRestarts
) {
if (showLog) process.env.OMNIROUTE_SHOW_LOG = "1";
const supervisor = new ServerSupervisor({
serverPath: serverJs,
env,
memoryLimit,
maxRestarts,
onCrashCallback: async (crashLog) => {
if (detectMitmCrash(crashLog)) {
try {
const PROJECT_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..", "..");
const { updateSettings } = await import(`${PROJECT_ROOT}/src/lib/db/settings.ts`);
updateSettings({ mitmEnabled: false });
} catch {}
return "disable-mitm-and-retry";
}
return null;
},
});
supervisor.start();
process.on("SIGINT", () => supervisor.stop());
process.on("SIGTERM", () => supervisor.stop());
if (!showLog) {
waitForServer(dashboardPort, 20000).then((up) => {
if (up) onReady(dashboardPort, apiPort, noOpen);
});
}
}
async function onReady(dashboardPort, apiPort, noOpen) {
const dashboardUrl = `http://localhost:${dashboardPort}`;
const apiUrl = `http://localhost:${apiPort}`;

View File

@@ -1,6 +1,12 @@
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { readPidFile, isPidRunning, cleanupPidFile, sleep } from "../utils/pid.mjs";
import {
readPidFile,
isPidRunning,
cleanupPidFile,
killAllSubprocesses,
sleep,
} from "../utils/pid.mjs";
import { t } from "../i18n.mjs";
const execFileAsync = promisify(execFile);
@@ -16,7 +22,7 @@ export function registerStop(program) {
}
export async function runStopCommand(opts = {}) {
const pid = readPidFile();
const pid = readPidFile("server");
if (pid && isPidRunning(pid)) {
console.log(t("stop.stopping", { pid }));
@@ -34,7 +40,8 @@ export async function runStopCommand(opts = {}) {
await sleep(500);
}
cleanupPidFile();
killAllSubprocesses();
cleanupPidFile("server");
console.log(t("stop.stopped"));
return 0;
} catch (err) {
@@ -49,7 +56,8 @@ export async function runStopCommand(opts = {}) {
if (pid === null) {
console.log(t("stop.portFallback"));
await killByPort(port);
cleanupPidFile();
killAllSubprocesses();
cleanupPidFile("server");
console.log(t("stop.stopped"));
return 0;
}

View File

@@ -71,7 +71,10 @@
"notRunning": "Server is not running.",
"port": "Port to listen on (default: 20128)",
"no_open": "Do not open browser automatically",
"daemon": "Run server as a background daemon"
"daemon": "Run server as a background daemon",
"log": "Show server logs inline",
"no_recovery": "Disable auto-restart on crash (debugging mode)",
"max_restarts": "Max crash restarts within 30s before giving up (default: 2)"
},
"backup": {
"title": "Backup",

View File

@@ -71,7 +71,10 @@
"notRunning": "Servidor não está em execução.",
"port": "Porta de escuta (padrão: 20128)",
"no_open": "Não abrir navegador automaticamente",
"daemon": "Executar servidor em segundo plano"
"daemon": "Executar servidor em segundo plano",
"log": "Exibir logs do servidor inline",
"no_recovery": "Desabilitar reinício automático em crash (modo debug)",
"max_restarts": "Máximo de reinícios em 30s antes de desistir (padrão: 2)"
},
"backup": {
"title": "Backup",

View File

@@ -0,0 +1,113 @@
import { spawn } from "node:child_process";
import { dirname } from "node:path";
import { writePidFile, cleanupPidFile, killAllSubprocesses } from "../utils/pid.mjs";
const CRASH_LOG_LINES = 50;
const RESTART_RESET_MS = 30_000;
export class ServerSupervisor {
constructor({ serverPath, env, maxRestarts = 2, memoryLimit = 512, onCrashCallback }) {
this.serverPath = serverPath;
this.env = env;
this.maxRestarts = maxRestarts;
this.memoryLimit = memoryLimit;
this.onCrashCallback = onCrashCallback;
this.restartCount = 0;
this.startedAt = 0;
this.crashLog = [];
this.child = null;
this.isShuttingDown = false;
}
start() {
this.startedAt = Date.now();
this.crashLog = [];
const showLog = process.env.OMNIROUTE_SHOW_LOG === "1";
this.child = spawn("node", [`--max-old-space-size=${this.memoryLimit}`, this.serverPath], {
cwd: dirname(this.serverPath),
env: this.env,
stdio: showLog ? "inherit" : ["ignore", "ignore", "pipe"],
});
writePidFile("server", this.child.pid);
if (this.child.stderr) {
this.child.stderr.on("data", (data) => {
const lines = data.toString().split("\n").filter(Boolean);
this.crashLog.push(...lines);
if (this.crashLog.length > CRASH_LOG_LINES) {
this.crashLog = this.crashLog.slice(-CRASH_LOG_LINES);
}
});
}
this.child.on("error", (err) => this.handleExit(err.code ?? -1, err));
this.child.on("exit", (code) => this.handleExit(code));
return this.child;
}
handleExit(code) {
cleanupPidFile("server");
if (this.isShuttingDown || code === 0) {
process.exit(code || 0);
return;
}
const aliveMs = Date.now() - this.startedAt;
if (aliveMs >= RESTART_RESET_MS) this.restartCount = 0;
if (this.restartCount >= this.maxRestarts) {
console.error(`\n⚠ Server crashed ${this.maxRestarts} times in <30s.`);
if (this.onCrashCallback) {
const action = this.onCrashCallback(this.crashLog);
if (action === "disable-mitm-and-retry") {
console.error("⚠ Disabling MITM and retrying...\n");
this.restartCount = 0;
this.start();
return;
}
}
this.dumpCrashLog();
process.exit(code ?? 1);
return;
}
this.restartCount++;
const delay = Math.min(1000 * 2 ** (this.restartCount - 1), 10_000);
console.error(
`\n⚠ Server exited (code=${code ?? "?"}). Restarting in ${delay / 1000}s... (${this.restartCount}/${this.maxRestarts})`
);
if (this.crashLog.length) this.dumpCrashLog();
setTimeout(() => this.start(), delay);
}
dumpCrashLog() {
console.error("\n--- Server crash log ---");
this.crashLog.forEach((l) => console.error(l));
console.error("--- End crash log ---\n");
}
stop() {
this.isShuttingDown = true;
if (this.child?.pid) {
try {
process.kill(this.child.pid, "SIGTERM");
} catch {}
setTimeout(() => {
try {
process.kill(this.child.pid, "SIGKILL");
} catch {}
}, 5000);
}
killAllSubprocesses();
}
}
export function detectMitmCrash(crashLog) {
const text = crashLog.join("\n").toLowerCase();
const signals = ["mitm", "tls socket", "certificate", "hosts", "eaccess"];
return signals.filter((s) => text.includes(s)).length >= 2;
}

View File

@@ -1,34 +1,52 @@
import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { join } from "node:path";
import { resolveDataDir } from "../data-dir.mjs";
export function getPidFilePath() {
return join(resolveDataDir(), "server.pid");
const SERVICES = ["server", "mitm", "tunnel/cloudflared", "tunnel/tailscale"];
function getServicePidPath(service) {
return join(resolveDataDir(), service, ".pid");
}
export function writePidFile(pid) {
export function writePidFile(service, pid) {
try {
const pidPath = getPidFilePath();
const dir = dirname(pidPath);
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
writeFileSync(pidPath, String(pid), "utf8");
const dir = join(resolveDataDir(), service);
mkdirSync(dir, { recursive: true });
writeFileSync(getServicePidPath(service), String(pid), "utf8");
return true;
} catch {
return false;
}
}
export function readPidFile() {
export function readPidFile(service) {
try {
const pidPath = getPidFilePath();
if (!existsSync(pidPath)) return null;
const content = readFileSync(pidPath, "utf8").trim();
return content ? parseInt(content, 10) : null;
const file = getServicePidPath(service);
if (!existsSync(file)) return null;
const pid = parseInt(readFileSync(file, "utf8").trim(), 10);
return Number.isFinite(pid) ? pid : null;
} catch {
return null;
}
}
export function cleanupPidFile(service) {
try {
unlinkSync(getServicePidPath(service));
} catch {}
}
export function killAllSubprocesses() {
for (const service of SERVICES) {
const pid = readPidFile(service);
if (!pid) continue;
try {
process.kill(pid, "SIGTERM");
} catch {}
cleanupPidFile(service);
}
}
export function isPidRunning(pid) {
if (!pid) return false;
try {
@@ -39,13 +57,6 @@ export function isPidRunning(pid) {
}
}
export function cleanupPidFile() {
try {
const pidPath = getPidFilePath();
if (existsSync(pidPath)) unlinkSync(pidPath);
} catch {}
}
export function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}

View File

@@ -297,12 +297,13 @@ CLI_CLAUDE_BIN=/host-cli/bin/claude
These variables tune the `omniroute` CLI binary's own behavior (not the sidecar
detection above).
| Variable | Default | Source File | Description |
| --------------------------- | ---------- | ------------------ | ---------------------------------------------------------------------------------------------------------------------- |
| `OMNIROUTE_LANG` | _(system)_ | `bin/cli/i18n.mjs` | Force CLI output language. BCP-47 locale (e.g. `en`, `pt-BR`). Overrides system locale env vars (LC_ALL, LC_MESSAGES). |
| `OMNIROUTE_CLI_TOKEN` | _(unset)_ | `bin/cli/api.mjs` | Machine-auth token injected as `x-omniroute-cli-token` header. Auto-generated in task 8.12. |
| `OMNIROUTE_HTTP_TIMEOUT_MS` | `30000` | `bin/cli/api.mjs` | Per-attempt HTTP timeout (ms) for CLI → server requests. |
| `OMNIROUTE_VERBOSE` | `0` | `bin/cli/api.mjs` | Set to `1` to print retry/backoff diagnostics to stderr during CLI commands. |
| Variable | Default | Source File | Description |
| --------------------------- | ---------- | --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- |
| `OMNIROUTE_LANG` | _(system)_ | `bin/cli/i18n.mjs` | Force CLI output language. BCP-47 locale (e.g. `en`, `pt-BR`). Overrides system locale env vars (LC_ALL, LC_MESSAGES). |
| `OMNIROUTE_SHOW_LOG` | _(unset)_ | `bin/cli/runtime/processSupervisor.mjs` | Set to `1` to forward server stdout/stderr to the terminal in supervised mode. Equivalent to `--log` flag on `omniroute serve`. |
| `OMNIROUTE_CLI_TOKEN` | _(unset)_ | `bin/cli/api.mjs` | Machine-auth token injected as `x-omniroute-cli-token` header. Auto-generated in task 8.12. |
| `OMNIROUTE_HTTP_TIMEOUT_MS` | `30000` | `bin/cli/api.mjs` | Per-attempt HTTP timeout (ms) for CLI → server requests. |
| `OMNIROUTE_VERBOSE` | `0` | `bin/cli/api.mjs` | Set to `1` to print retry/backoff diagnostics to stderr during CLI commands. |
---

View File

@@ -0,0 +1,222 @@
import test from "node:test";
import assert from "node:assert/strict";
import { EventEmitter } from "node:events";
// Stub para o processSupervisor: testa a lógica de restart/backoff/MITM sem processos reais.
class StubChild extends EventEmitter {
pid = 99999;
stderr = new EventEmitter();
killed = false;
kill(_sig?: string) {
this.killed = true;
}
}
function makeChildFactory(exitCodes: (number | null)[]) {
let calls = 0;
return () => {
const child = new StubChild();
const code = exitCodes[calls++] ?? null;
// Emite exit no próximo tick para simular processo assíncrono
setImmediate(() => child.emit("exit", code));
return child;
};
}
// --- detectMitmCrash ---
test("detectMitmCrash retorna true quando >=2 sinais MITM presentes", async () => {
const { detectMitmCrash } = await import("../../bin/cli/runtime/processSupervisor.mjs");
assert.ok(detectMitmCrash(["mitm proxy failed", "certificate error in tls socket"]));
assert.ok(detectMitmCrash(["TLS Socket closed", "certificate invalid"]));
});
test("detectMitmCrash retorna false com menos de 2 sinais", async () => {
const { detectMitmCrash } = await import("../../bin/cli/runtime/processSupervisor.mjs");
assert.ok(!detectMitmCrash(["certificate error"]));
assert.ok(!detectMitmCrash(["generic error"]));
assert.ok(!detectMitmCrash([]));
});
// --- ServerSupervisor: lógica de restart ---
test("ServerSupervisor.handleExit com code=0 chama process.exit(0)", async () => {
const { ServerSupervisor } = await import("../../bin/cli/runtime/processSupervisor.mjs");
const exits: number[] = [];
const origExit = process.exit.bind(process);
// @ts-ignore
process.exit = (code?: number) => {
exits.push(code ?? 0);
};
const supervisor = new ServerSupervisor({
serverPath: "/fake/server.js",
env: {},
maxRestarts: 2,
});
supervisor.handleExit(0);
// @ts-ignore
process.exit = origExit;
assert.equal(exits[0], 0);
});
test("ServerSupervisor.handleExit com isShuttingDown=true chama process.exit imediato", async () => {
const { ServerSupervisor } = await import("../../bin/cli/runtime/processSupervisor.mjs");
const exits: number[] = [];
const origExit = process.exit.bind(process);
// @ts-ignore
process.exit = (code?: number) => exits.push(code ?? 0);
const supervisor = new ServerSupervisor({
serverPath: "/fake/server.js",
env: {},
maxRestarts: 2,
});
supervisor.isShuttingDown = true;
supervisor.handleExit(1);
// @ts-ignore
process.exit = origExit;
assert.equal(exits[0], 1);
});
test("ServerSupervisor.handleExit incrementa restartCount e chama start() após delay", async () => {
const { ServerSupervisor } = await import("../../bin/cli/runtime/processSupervisor.mjs");
let startCalls = 0;
const supervisor = new ServerSupervisor({
serverPath: "/fake/server.js",
env: {},
maxRestarts: 5,
});
supervisor.start = () => {
startCalls++;
return null as any;
};
supervisor.startedAt = Date.now() - 100; // viveu <30s
supervisor.handleExit(1);
assert.equal(supervisor.restartCount, 1);
await new Promise((r) => setTimeout(r, 1100)); // aguarda o delay de 1s
assert.equal(startCalls, 1);
});
test("ServerSupervisor.handleExit exibe crash log ao reiniciar", async () => {
const { ServerSupervisor } = await import("../../bin/cli/runtime/processSupervisor.mjs");
const logs: string[] = [];
const origErr = console.error.bind(console);
console.error = (...args: unknown[]) => logs.push(args.join(" "));
const supervisor = new ServerSupervisor({
serverPath: "/fake/server.js",
env: {},
maxRestarts: 5,
});
supervisor.start = () => null as any;
supervisor.startedAt = Date.now() - 100;
supervisor.crashLog = ["line1", "line2"];
supervisor.handleExit(1);
console.error = origErr;
assert.ok(logs.some((l) => l.includes("line1") || l.includes("crash log")));
});
test("ServerSupervisor chama onCrashCallback após maxRestarts atingido", async () => {
const { ServerSupervisor } = await import("../../bin/cli/runtime/processSupervisor.mjs");
let callbackCalled = false;
const exits: number[] = [];
const origExit = process.exit.bind(process);
// @ts-ignore
process.exit = (code?: number) => exits.push(code ?? 0);
const supervisor = new ServerSupervisor({
serverPath: "/fake/server.js",
env: {},
maxRestarts: 2,
onCrashCallback: (log: string[]) => {
callbackCalled = true;
return null;
},
});
supervisor.restartCount = 2; // já no limite
supervisor.startedAt = Date.now() - 100;
supervisor.handleExit(1);
// @ts-ignore
process.exit = origExit;
assert.ok(callbackCalled);
assert.equal(exits[0], 1);
});
test("ServerSupervisor retorna 'disable-mitm-and-retry' chama start() novamente", async () => {
const { ServerSupervisor } = await import("../../bin/cli/runtime/processSupervisor.mjs");
let startCalls = 0;
const supervisor = new ServerSupervisor({
serverPath: "/fake/server.js",
env: {},
maxRestarts: 2,
onCrashCallback: () => "disable-mitm-and-retry",
});
supervisor.start = () => {
startCalls++;
return null as any;
};
supervisor.restartCount = 2;
supervisor.startedAt = Date.now() - 100;
supervisor.handleExit(1);
assert.equal(startCalls, 1);
assert.equal(supervisor.restartCount, 0); // foi resetado
});
test("ServerSupervisor reseta restartCount após processo viver >=30s", async () => {
const { ServerSupervisor } = await import("../../bin/cli/runtime/processSupervisor.mjs");
const supervisor = new ServerSupervisor({
serverPath: "/fake/server.js",
env: {},
maxRestarts: 2,
});
supervisor.start = () => null as any;
supervisor.restartCount = 2;
supervisor.startedAt = Date.now() - 31_000; // viveu 31s
supervisor.handleExit(1);
assert.equal(supervisor.restartCount, 1); // reset p/ 0, depois incrementado p/ 1
});
// --- pid.mjs multi-service ---
test("writePidFile/readPidFile/cleanupPidFile operam por service", async () => {
const os = await import("node:os");
const tmpDir = os.default.tmpdir() + "/omniroute-pid-test-" + Date.now();
process.env.DATA_DIR = tmpDir;
const { writePidFile, readPidFile, cleanupPidFile } = await import("../../bin/cli/utils/pid.mjs");
writePidFile("server", 12345);
assert.equal(readPidFile("server"), 12345);
writePidFile("mitm", 99999);
assert.equal(readPidFile("mitm"), 99999);
// Services são independentes
assert.equal(readPidFile("server"), 12345);
cleanupPidFile("server");
assert.equal(readPidFile("server"), null);
assert.equal(readPidFile("mitm"), 99999); // mitm não foi afetado
cleanupPidFile("mitm");
delete process.env.DATA_DIR;
});