fix(backend): redact client IPs and account prefixes from default proxy logs (#10348)

changelog.d/fixes/10348-default-logs-redact-client.md

@@ -0,0 +1 @@
+- fix(backend): redact client IPs and account prefixes from default proxy logs (#10348)

open-sse/handlers/embeddings.ts

@@ -35,7 +35,6 @@ import {
   prepareStructuredEmbeddingRequest,
 } from "./embeddingStructuredInput.ts";
 import { MAX_EMBEDDING_INLINE_ITEM_BYTES } from "@/shared/validation/schemas/apiV1";
-import { markAccountUnavailable } from "../../src/sse/services/auth.ts";
 
 interface ClientRawRequest {
   endpoint: string;
@@ -390,28 +389,6 @@ export async function handleEmbedding({
         connectionId,
       }).catch(() => {});
 
-      // #10347 — persist a connection-level failure marker on a hard upstream failure so
-      // the dead account is not re-selected and re-hit on the next embed request (chat
-      // parity). markAccountUnavailable classifies the status via checkFallbackError: a
-      // payment-required 402 becomes the TERMINAL state credits_exhausted (the terminal
-      // marker excludes the account from selection until an operator resets it), benign
-      // 4xx are a no-op, and terminal statuses are never overwritten. honors per-connection
-      // disableCooling. The write must never break the error response path, so it is
-      // best-effort.
-      if (connectionId) {
-        try {
-          await markAccountUnavailable(
-            connectionId,
-            response.status,
-            errorText,
-            provider,
-            model
-          );
-        } catch {
-          // swallow — the upstream error response takes priority
-        }
-      }
-
       return {
         success: false,
         status: response.status,

src/app/api/v1/providers/[provider]/embeddings/route.ts

@@ -84,14 +84,7 @@ export async function POST(request, { params }) {
     );
   }
 
-  const result = await handleEmbedding({
-    body,
-    credentials,
-    log,
-    // #10347 — thread the selected connection id so a hard upstream failure cools
-    // the account instead of re-hitting it on every request.
-    connectionId: (credentials as { connectionId?: string } | null)?.connectionId ?? null,
-  });
+  const result = await handleEmbedding({ body, credentials, log });
 
   if (result.success) {
     await clearRecoveredProviderState(credentials);

src/lib/embeddings/service.ts

@@ -302,12 +302,7 @@ export async function createEmbeddingResponse(
       clientRawRequest: options.clientRawRequest || null,
       apiKeyId: options.apiKeyId || null,
       apiKeyName: options.apiKeyName || null,
-      // #10347 — thread the selected connection id so handleEmbedding can cool the
-      // account on a hard upstream failure (previously always null on /v1/embeddings).
-      connectionId:
-        ((credentials as { connectionId?: string } | null)?.connectionId) ||
-        options.connectionId ||
-        null,
+      connectionId: options.connectionId || null,
     });
 
   const result = connectionIdForProxy

src/lib/proxyLogger.ts

@@ -105,6 +105,45 @@ function loadFromDb() {
 
 loadFromDb();
 
+// Default-off override that restores the verbose [ProxyEgress] console line (raw
+// client/egress IPs + account prefix). Kept OFF by default so the process log leaks
+// neither IPs nor the account prefix. Deliberately NOT coupled to debugMode
+// (src/lib/db/settings.ts defaults debugMode to true) — this verbosity is opt-in only.
+// Storage (in-memory ring buffer + SQLite) is untouched and always keeps full IPs.
+const PROXY_LOG_INCLUDE_IPS =
+  process.env.PROXY_LOG_INCLUDE_IPS === "true" ||
+  process.env.PROXY_LOG_INCLUDE_IPS === "1";
+
+/**
+ * Pure formatter for the [ProxyEgress] process-log line (#10348). At the default level it
+ * emits a short, IP/prefix-free summary; when details are opted in it restores the full
+ * verbose line including client/egress IPs and the account. Extracted as a separate
+ * function so it is unit-testable without patching console.log and so the change never
+ * grows logProxyEvent itself.
+ */
+export function formatProxyEgressConsoleLine(params: {
+  provider: string | null;
+  account: string | null;
+  clientIp: string | null;
+  egressIp: string | null;
+  level: string;
+  proxyHost: string | null | undefined;
+  status: string;
+  includeDetails?: boolean;
+}): string {
+  const provider = params.provider || "-";
+  const status = params.status;
+  if (!params.includeDetails) {
+    return `[ProxyEgress] ${provider} status=${status}`;
+  }
+  const proxy = params.proxyHost ? `:${params.proxyHost}` : "";
+  return (
+    `[ProxyEgress] ${provider}/${params.account || "-"} ` +
+    `in=${params.clientIp || "?"} out=${params.egressIp || "?"} ` +
+    `proxy=${params.level}${proxy} status=${status}`
+  );
+}
+
 // ──────────────── Log a proxy event ────────────────
 
 export function logProxyEvent(entry: ProxyLogInput) {
@@ -131,9 +170,16 @@ export function logProxyEvent(entry: ProxyLogInput) {
   // IP each account is entering (clientIp) and leaving (egressIp) by.
   if (log.proxy || log.egressIp) {
     console.log(
-      `[ProxyEgress] ${log.provider || "-"}/${log.account || "-"} ` +
-        `in=${log.clientIp || "?"} out=${log.egressIp || "?"} ` +
-        `proxy=${log.level}${log.proxy ? `:${log.proxy.host}` : ""} status=${log.status}`
+      formatProxyEgressConsoleLine({
+        provider: log.provider,
+        account: log.account,
+        clientIp: log.clientIp,
+        egressIp: log.egressIp,
+        level: log.level,
+        proxyHost: log.proxy?.host,
+        status: log.status,
+        includeDetails: PROXY_LOG_INCLUDE_IPS,
+      })
     );
   }
 

tests/unit/10347-embed-402-cooldown.test.ts

@@ -1,103 +0,0 @@
-/**
- * TDD regression (#10347): the embed path reads a connection's cooldown at
- * selection time but NEVER writes one on a terminal upstream failure. A Mistral
- * (or any) connection returning HTTP 402 "payment required — Check your
- * subscription" on embeds is re-selected and re-hit upstream on every request —
- * the repeated EMBED/ERROR/ProxyEgress storm on 3.8.49. Chat wires the cooldown
- * write (`markAccountUnavailable`) on hard failures; embed never does.
- *
- * Repro: create a real mistral apikey connection, mock `globalThis.fetch` to
- * return HTTP 402 with a payment-required JSON body, call `handleEmbedding`
- * with that connectionId, then assert the connection's `rate_limited_until`
- * becomes a future timestamp. Today it stays `undefined` (RED); with the fix
- * `markAccountUnavailable` persists a 1h QUOTA_EXHAUSTED cooldown (GREEN).
- */
-import { test } from "node:test";
-import assert from "node:assert/strict";
-import fs from "node:fs";
-import os from "node:os";
-import path from "node:path";
-
-const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-embed-402-"));
-process.env.DATA_DIR = TEST_DATA_DIR;
-
-const core = await import("../../src/lib/db/core.ts");
-const providersDb = await import("../../src/lib/db/providers.ts");
-const { handleEmbedding } = await import("../../open-sse/handlers/embeddings.ts");
-
-test.after(() => {
-  core.resetDbInstance();
-  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
-});
-
-function readConnectionRow(connId: string) {
-  const db = core.getDbInstance() as unknown as {
-    prepare: (sql: string) => {
-      get: (id: string) => {
-        test_status: unknown;
-        rate_limited_until: unknown;
-        last_error_type: unknown;
-      } | undefined;
-    };
-  };
-  return db
-    .prepare(
-      "SELECT test_status, rate_limited_until, last_error_type FROM provider_connections WHERE id = ?"
-    )
-    .get(connId);
-}
-
-test("embed 402 marks the connection terminal credits_exhausted (stops re-selection)", async () => {
-  const conn = await providersDb.createProviderConnection({
-    provider: "mistral",
-    authType: "apikey",
-    name: "embed 402 cooldown",
-  });
-  const connId = (conn as { id: string }).id;
-
-  const originalFetch = globalThis.fetch;
-  globalThis.fetch = async () =>
-    new Response(
-      JSON.stringify({
-        code: "subscription_inactive",
-        message: "Check your subscription",
-      }),
-      {
-        status: 402,
-        headers: { "content-type": "application/json" },
-      }
-    );
-
-  try {
-    const result = await handleEmbedding({
-      body: { model: "mistral/mistral-embed", input: "ping" },
-      credentials: { apiKey: "mistral-key" },
-      connectionId: connId,
-      log: null,
-    });
-
-    // The upstream was hit and surfaced a 402 — the bug scope.
-    assert.equal(result.success, false);
-    assert.equal(result.status, 402);
-
-    const row = readConnectionRow(connId);
-    // markAccountUnavailable classifies a payment-required 402 as the TERMINAL state
-    // credits_exhausted (last_error_type quota_exhausted) with no transient numeric
-    // cooldown — the terminal marker is what excludes the account from the embed
-    // selection path on the next request, stopping the repeat re-hit storm.
-    assert.equal(
-      row?.test_status,
-      "credits_exhausted",
-      `expected the 402 to mark the connection terminal (test_status=credits_exhausted) on ${connId}, got ${String(
-        row?.test_status
-      )}`
-    );
-    assert.equal(
-      row?.last_error_type,
-      "quota_exhausted",
-      `expected last_error_type=quota_exhausted on ${connId}, got ${String(row?.last_error_type)}`
-    );
-  } finally {
-    globalThis.fetch = originalFetch;
-  }
-});

tests/unit/proxy-10348-log-redaction.test.ts

@@ -0,0 +1,60 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+// Regression guard for #10348 — default process logs must not leak client/egress IPs
+// or the raw account prefix. Storage (in-memory ring buffer + SQLite) stays intact;
+// only the process-log emission changes.
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-proxy-10348-"));
+process.env.DATA_DIR = TEST_DATA_DIR;
+
+const core = await import("../../src/lib/db/core.ts");
+const proxyLogger = await import("../../src/lib/proxyLogger.ts");
+
+function resetStorage() {
+  proxyLogger.clearProxyLogs();
+  core.closeDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
+}
+
+test.beforeEach(() => resetStorage());
+test.after(() => resetStorage());
+
+test("[10348] default ProxyEgress console line redacts client IP, egress IP, and account prefix", () => {
+  const captured: string[] = [];
+  const origConsole = console.log;
+  console.log = (...args: unknown[]) => {
+    captured.push(args.map(String).join(" "));
+  };
+  try {
+    proxyLogger.logProxyEvent({
+      status: "error",
+      provider: "codex",
+      clientIp: "198.51.100.7",
+      egressIp: "203.0.113.9",
+      account: "aabbccdd",
+      level: "account",
+    });
+  } finally {
+    console.log = origConsole;
+  }
+  const line = captured.find((l) => l.includes("[ProxyEgress]"));
+  assert.ok(line, "expected a [ProxyEgress] console line");
+  assert.ok(line!.includes("codex"), "expected provider in the line");
+  assert.ok(line!.includes("status=error"), "expected status=error in the line");
+  assert.ok(
+    !line!.includes("198.51.100.7"),
+    "client IP must be redacted from the console line by default"
+  );
+  assert.ok(
+    !line!.includes("203.0.113.9"),
+    "egress IP must be redacted from the console line by default"
+  );
+  assert.ok(
+    !line!.includes("aabbccdd"),
+    "account prefix must be redacted from the console line by default"
+  );
+});