fix(cloudflared): avoid stale restart state

2026-07-26 09:52:11 +03:00 · 2026-04-02 05:40:26 -04:00
parent e121fec599
commit ed72ddc4d3
4 changed files with 217 additions and 12 deletions
--- a/README.md
+++ b/README.md
@@ -1016,7 +1016,9 @@ Dashboard support for Docker deployments now includes a one-click **Cloudflare Q
 Notes:

 - Quick Tunnel URLs are temporary and change after every restart.
+- Quick Tunnels are not auto-restored after an OmniRoute or container restart. Re-enable them from the dashboard when needed.
 - Managed install currently supports Linux, macOS, and Windows on `x64` / `arm64`.
+- Managed Quick Tunnels default to HTTP/2 transport to avoid noisy QUIC UDP buffer warnings in constrained container environments. Set `CLOUDFLARED_PROTOCOL=quic` or `auto` if you want a different transport.
 - Docker images bundle system CA roots and pass them to managed `cloudflared`, which avoids TLS trust failures when the tunnel bootstraps inside the container.
 - SQLite runs in WAL mode. `docker stop` should be allowed to finish so OmniRoute can checkpoint the latest changes back into `storage.sqlite`.
 - The bundled Compose files already set a 40s stop grace period. If you run the image directly, keep `--stop-timeout 40` (or similar) so manual stops do not cut off shutdown cleanup.
--- a/docs/USER_GUIDE.md
+++ b/docs/USER_GUIDE.md
@@ -525,6 +525,7 @@ post_install() {
 | `ENABLE_REQUEST_LOGS`                   | `false`                              | Enables request/response logs                                                                             |
 | `AUTH_COOKIE_SECURE`                    | `false`                              | Force `Secure` auth cookie (behind HTTPS reverse proxy)                                                   |
 | `CLOUDFLARED_BIN`                       | unset                                | Use an existing `cloudflared` binary instead of managed download                                          |
+| `CLOUDFLARED_PROTOCOL`                  | `http2`                              | Transport for managed Quick Tunnels (`http2`, `quic`, or `auto`)                                          |
 | `OMNIROUTE_MEMORY_MB`                   | `512`                                | Node.js heap limit in MB                                                                                  |
 | `PROMPT_CACHE_MAX_SIZE`                 | `50`                                 | Max prompt cache entries                                                                                  |
 | `SEMANTIC_CACHE_MAX_SIZE`               | `100`                                | Max semantic cache entries                                                                                |
@@ -652,7 +653,10 @@ Returns models grouped by provider with types (`chat`, `embedding`, `image`).
 - Available in **Dashboard → Endpoints** for Docker and other self-hosted deployments
 - Creates a temporary `https://*.trycloudflare.com` URL that forwards to your current OpenAI-compatible `/v1` endpoint
 - First enable installs `cloudflared` only when needed; later restarts reuse the same managed binary
+- Quick Tunnels are not auto-restored after an OmniRoute or container restart; re-enable them from the dashboard when needed
 - Tunnel URLs are ephemeral and change every time you stop/start the tunnel
+- Managed Quick Tunnels default to HTTP/2 transport to avoid noisy QUIC UDP buffer warnings in constrained containers
+- Set `CLOUDFLARED_PROTOCOL=quic` or `auto` if you want to override the managed transport choice
 - Set `CLOUDFLARED_BIN` if you prefer using a preinstalled `cloudflared` binary instead of the managed download

 ### LLM Gateway Intelligence (Phase 9)
--- a/src/lib/cloudflaredTunnel.ts
+++ b/src/lib/cloudflaredTunnel.ts
@@ -57,6 +57,7 @@ type BinaryResolution = {
 type PersistedTunnelState = {
  binaryPath?: string | null;
  installSource?: CloudflaredInstallSource | null;
+  ownerPid?: number | null;
  pid?: number | null;
  publicUrl?: string | null;
  apiUrl?: string | null;
@@ -128,6 +129,9 @@ let tunnelProcess: ReturnType<typeof spawn> | null = null;
 let tunnelPid: number | null = null;
 let installPromise: Promise<string> | null = null;
 let startPromise: Promise<CloudflaredTunnelStatus> | null = null;
+const NON_ACTIONABLE_CLOUDFLARED_WARNING_PATTERNS = [
+  /failed to sufficiently increase receive buffer size/i,
+] as const;

 function getTunnelDir() {
  return path.join(resolveDataDir(), "cloudflared");
@@ -264,6 +268,9 @@ export function extractCloudflaredErrorMessage(text: string) {
    .filter(Boolean);

  for (let i = lines.length - 1; i >= 0; i--) {
+    if (NON_ACTIONABLE_CLOUDFLARED_WARNING_PATTERNS.some((pattern) => pattern.test(lines[i]))) {
+      continue;
+    }
    if (/(?:\berror\b|\bfailed\b|\btls:\b|\bx509\b|\bcertificate\b)/i.test(lines[i])) {
      return lines[i];
    }
@@ -327,6 +334,18 @@ export function buildCloudflaredChildEnv(
    childEnv.SSL_CERT_DIR = defaultCertEnv.SSL_CERT_DIR;
  }

+  const requestedProtocol = String(
+    sourceEnv.CLOUDFLARED_PROTOCOL || sourceEnv.TUNNEL_TRANSPORT_PROTOCOL || "http2"
+  )
+    .trim()
+    .toLowerCase();
+  const protocol =
+    requestedProtocol === "quic" || requestedProtocol === "auto" ? requestedProtocol : "http2";
+
+  if (protocol !== "auto") {
+    childEnv.TUNNEL_TRANSPORT_PROTOCOL = protocol;
+  }
+
  return childEnv;
 }

@@ -334,6 +353,41 @@ export function getCloudflaredStartArgs(targetUrl: string) {
  return ["tunnel", "--url", targetUrl, "--no-autoupdate"];
 }

+function isStateOwnedByCurrentProcess(state: PersistedTunnelState) {
+  return !!state.ownerPid && state.ownerPid === process.pid;
+}
+
+function hasTransientRuntimeState(state: PersistedTunnelState) {
+  return !!(
+    state.ownerPid ||
+    state.pid ||
+    state.publicUrl ||
+    state.apiUrl ||
+    state.startedAt ||
+    state.status === "running" ||
+    state.status === "starting" ||
+    state.status === "error"
+  );
+}
+
+function buildStoppedState(
+  state: PersistedTunnelState,
+  binaryResolved: boolean,
+  targetUrl = getLocalTargetUrl()
+): PersistedTunnelState {
+  return {
+    ...state,
+    ownerPid: null,
+    pid: null,
+    publicUrl: null,
+    apiUrl: null,
+    targetUrl,
+    status: binaryResolved ? "stopped" : "not_installed",
+    lastError: null,
+    startedAt: null,
+  };
+}
+
 export function getCloudflaredAssetSpec(
  platform = process.platform,
  arch = process.arch
@@ -549,6 +603,12 @@ async function stopExistingTunnel() {
    return;
  }

+  const state = await readStateFile();
+  if (!isStateOwnedByCurrentProcess(state)) {
+    await clearPidFile();
+    return;
+  }
+
  const pid = await readPidFile();
  if (pid && isProcessAlive(pid)) {
    await killPid(pid);
@@ -558,9 +618,20 @@ async function stopExistingTunnel() {
 export async function getCloudflaredTunnelStatus(): Promise<CloudflaredTunnelStatus> {
  const state = await readStateFile();
  const resolved = await resolveBinary();
-  const pidFromState = tunnelPid || state.pid || (await readPidFile());
+  const pidFromState =
+    tunnelPid || (isStateOwnedByCurrentProcess(state) ? state.pid || (await readPidFile()) : null);
  const running = isProcessAlive(pidFromState);
-  const publicUrl = running ? state.publicUrl || null : null;
+  const needsColdStartReset =
+    !running && !isStateOwnedByCurrentProcess(state) && hasTransientRuntimeState(state);
+  const effectiveState = needsColdStartReset
+    ? buildStoppedState(state, !!resolved.binaryPath)
+    : state;
+
+  if (needsColdStartReset) {
+    await writeStateFile(effectiveState);
+  }
+
+  const publicUrl = running ? effectiveState.publicUrl || null : null;
  const phase =
    !getCloudflaredAssetSpec() && !resolved.binaryPath
      ? "unsupported"
@@ -569,7 +640,7 @@ export async function getCloudflaredTunnelStatus(): Promise<CloudflaredTunnelSta
          ? "running"
          : "starting"
        : resolved.binaryPath
-          ? state.lastError
+          ? effectiveState.lastError
            ? "error"
            : "stopped"
          : "not_installed";
@@ -588,9 +659,9 @@ export async function getCloudflaredTunnelStatus(): Promise<CloudflaredTunnelSta
    pid: running ? pidFromState : null,
    publicUrl,
    apiUrl: publicUrl ? getTunnelApiUrl(publicUrl) : null,
-    targetUrl: state.targetUrl || getLocalTargetUrl(),
+    targetUrl: effectiveState.targetUrl || getLocalTargetUrl(),
    phase,
-    lastError: running ? null : state.lastError || null,
+    lastError: running ? null : effectiveState.lastError || null,
    logPath: getLogFilePath(),
  };
 }
@@ -619,6 +690,7 @@ export async function startCloudflaredTunnel(): Promise<CloudflaredTunnelStatus>
    await writeStateFile({
      binaryPath: binary.binaryPath,
      installSource: binary.source,
+      ownerPid: process.pid,
      pid: null,
      publicUrl: null,
      apiUrl: null,
@@ -662,6 +734,7 @@ export async function startCloudflaredTunnel(): Promise<CloudflaredTunnelStatus>
        const errorMessage = source === "stderr" ? extractCloudflaredErrorMessage(text) : null;
        if (errorMessage) {
          await updateStateFile({
+            ownerPid: process.pid,
            pid: child.pid,
            status: "error",
            lastError: errorMessage,
@@ -672,6 +745,7 @@ export async function startCloudflaredTunnel(): Promise<CloudflaredTunnelStatus>

        const apiUrl = getTunnelApiUrl(url);
        await updateStateFile({
+          ownerPid: process.pid,
          pid: child.pid,
          publicUrl: url,
          apiUrl,
@@ -721,6 +795,7 @@ export async function startCloudflaredTunnel(): Promise<CloudflaredTunnelStatus>
        : "Failed to start cloudflared tunnel";

    await updateStateFile({
+      ownerPid: process.pid,
      status: "error",
      lastError: message,
    });
@@ -734,12 +809,8 @@ export async function stopCloudflaredTunnel() {
  await stopExistingTunnel();
  const current = await readStateFile();
  await writeStateFile({
-    ...current,
-    pid: null,
-    publicUrl: null,
-    apiUrl: null,
-    status: "stopped",
-    lastError: null,
+    ...buildStoppedState(current, !!(await resolveBinary()).binaryPath),
+    ownerPid: null,
  });
  tunnelProcess = null;
  tunnelPid = null;
--- a/tests/unit/cloudflaredTunnel.test.mjs
+++ b/tests/unit/cloudflaredTunnel.test.mjs
@@ -1,10 +1,14 @@
 import test from "node:test";
 import assert from "node:assert/strict";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";

 import {
  buildCloudflaredChildEnv,
  extractCloudflaredErrorMessage,
  extractTryCloudflareUrl,
+  getCloudflaredTunnelStatus,
  getDefaultCloudflaredCertEnv,
  getCloudflaredStartArgs,
  getCloudflaredAssetSpec,
@@ -33,6 +37,14 @@ test("extractCloudflaredErrorMessage keeps the actionable stderr line", () => {
  );
 });

+test("extractCloudflaredErrorMessage ignores the non-actionable UDP buffer warning", () => {
+  const error = extractCloudflaredErrorMessage(
+    "WRN failed to sufficiently increase receive buffer size (was: 208 kiB, wanted: 7168 kiB, got: 416 kiB). See https://github.com/quic-go/quic-go/wiki/UDP-Buffer-Sizes for details."
+  );
+
+  assert.equal(error, null);
+});
+
 test("getCloudflaredAssetSpec resolves linux amd64 binary", () => {
  const spec = getCloudflaredAssetSpec("linux", "x64");

@@ -96,9 +108,56 @@ test("buildCloudflaredChildEnv keeps runtime essentials, isolates runtime dirs,
    TMPDIR: "/managed/runtime/tmp",
    TMP: "/managed/runtime/tmp",
    TEMP: "/managed/runtime/tmp",
+    TUNNEL_TRANSPORT_PROTOCOL: "http2",
  });
 });

+test("buildCloudflaredChildEnv allows overriding the tunnel transport protocol", () => {
+  const env = buildCloudflaredChildEnv(
+    {
+      PATH: "/usr/bin",
+      CLOUDFLARED_PROTOCOL: "quic",
+    },
+    {
+      runtimeRoot: "/managed/runtime",
+      homeDir: "/managed/runtime/home",
+      configDir: "/managed/runtime/config",
+      cacheDir: "/managed/runtime/cache",
+      dataDir: "/managed/runtime/data",
+      tempDir: "/managed/runtime/tmp",
+      userProfileDir: "/managed/runtime/userprofile",
+      appDataDir: "/managed/runtime/userprofile/AppData/Roaming",
+      localAppDataDir: "/managed/runtime/userprofile/AppData/Local",
+    },
+    {}
+  );
+
+  assert.equal(env.TUNNEL_TRANSPORT_PROTOCOL, "quic");
+});
+
+test("buildCloudflaredChildEnv preserves auto negotiation when explicitly requested", () => {
+  const env = buildCloudflaredChildEnv(
+    {
+      PATH: "/usr/bin",
+      CLOUDFLARED_PROTOCOL: "auto",
+    },
+    {
+      runtimeRoot: "/managed/runtime",
+      homeDir: "/managed/runtime/home",
+      configDir: "/managed/runtime/config",
+      cacheDir: "/managed/runtime/cache",
+      dataDir: "/managed/runtime/data",
+      tempDir: "/managed/runtime/tmp",
+      userProfileDir: "/managed/runtime/userprofile",
+      appDataDir: "/managed/runtime/userprofile/AppData/Roaming",
+      localAppDataDir: "/managed/runtime/userprofile/AppData/Local",
+    },
+    {}
+  );
+
+  assert.equal(env.TUNNEL_TRANSPORT_PROTOCOL, undefined);
+});
+
 test("getDefaultCloudflaredCertEnv detects common CA bundle paths", () => {
  const env = getDefaultCloudflaredCertEnv((candidate) =>
    ["/etc/ssl/certs/ca-certificates.crt", "/etc/ssl/certs"].includes(candidate)
@@ -134,7 +193,7 @@ test("buildCloudflaredChildEnv injects discovered CA paths when the parent env o
  assert.equal(env.SSL_CERT_DIR, "/etc/ssl/certs");
 });

-test("getCloudflaredStartArgs relies on cloudflared protocol auto-negotiation", () => {
+test("getCloudflaredStartArgs keeps protocol selection out of argv", () => {
  assert.deepEqual(getCloudflaredStartArgs("http://127.0.0.1:20128"), [
    "tunnel",
    "--url",
@@ -142,3 +201,72 @@ test("getCloudflaredStartArgs relies on cloudflared protocol auto-negotiation",
    "--no-autoupdate",
  ]);
 });
+
+test("getCloudflaredTunnelStatus resets stale runtime state from a previous server process", async () => {
+  const originalDataDir = process.env.DATA_DIR;
+  const originalBinary = process.env.CLOUDFLARED_BIN;
+  const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "omniroute-cloudflared-"));
+  const binDir = path.join(tempDir, "bin");
+  const binaryPath = path.join(binDir, "cloudflared");
+  const stateDir = path.join(tempDir, "cloudflared");
+  const statePath = path.join(stateDir, "quick-tunnel-state.json");
+
+  process.env.DATA_DIR = tempDir;
+  process.env.CLOUDFLARED_BIN = binaryPath;
+
+  try {
+    await fs.mkdir(binDir, { recursive: true });
+    await fs.mkdir(stateDir, { recursive: true });
+    await fs.writeFile(binaryPath, "#!/bin/sh\nexit 0\n", "utf8");
+    await fs.writeFile(
+      statePath,
+      JSON.stringify(
+        {
+          binaryPath,
+          installSource: "env",
+          ownerPid: process.pid + 100000,
+          pid: process.pid,
+          publicUrl: "https://stale.trycloudflare.com",
+          apiUrl: "https://stale.trycloudflare.com/v1",
+          targetUrl: "http://127.0.0.1:20128",
+          status: "running",
+          lastError:
+            "failed to sufficiently increase receive buffer size (was: 208 kiB, wanted: 7168 kiB, got: 416 kiB)",
+          startedAt: "2026-04-02T00:07:16.000Z",
+        },
+        null,
+        2
+      ) + "\n",
+      "utf8"
+    );
+
+    const status = await getCloudflaredTunnelStatus();
+    const persisted = JSON.parse(await fs.readFile(statePath, "utf8"));
+
+    assert.equal(status.running, false);
+    assert.equal(status.phase, "stopped");
+    assert.equal(status.publicUrl, null);
+    assert.equal(status.apiUrl, null);
+    assert.equal(status.lastError, null);
+    assert.equal(persisted.ownerPid, null);
+    assert.equal(persisted.pid, null);
+    assert.equal(persisted.publicUrl, null);
+    assert.equal(persisted.apiUrl, null);
+    assert.equal(persisted.status, "stopped");
+    assert.equal(persisted.lastError, null);
+  } finally {
+    if (originalDataDir === undefined) {
+      delete process.env.DATA_DIR;
+    } else {
+      process.env.DATA_DIR = originalDataDir;
+    }
+
+    if (originalBinary === undefined) {
+      delete process.env.CLOUDFLARED_BIN;
+    } else {
+      process.env.CLOUDFLARED_BIN = originalBinary;
+    }
+
+    await fs.rm(tempDir, { recursive: true, force: true });
+  }
+});