fix(encryption): identify failing credential in decrypt errors (#9927)

changelog.d/fixes/9927-encryption-log-identity.md

@@ -0,0 +1 @@
+- fix(encryption): name failing credential + recovery path in decrypt errors, dedupe per connection (#9927)

src/app/api/providers/[id]/models/staleEncryptionGuard.ts

@@ -14,17 +14,35 @@ import { buildErrorBody } from "@omniroute/open-sse/utils/error";
  * Returns a 424 (Failed Dependency) response with a clear, sanitized message
  * when the connection carries that flag; otherwise null (proceed normally).
  */
-const STALE_ENCRYPTION_MESSAGE =
-  "Stored API key cannot be decrypted (STORAGE_ENCRYPTION_KEY changed or unset). Re-enter the API key.";
-
 export function buildStaleEncryptionKeyResponse(
-  connection: { credentialDecryptFailed?: unknown } | null | undefined
+  connection:
+    | {
+        credentialDecryptFailed?: unknown;
+        id?: unknown;
+        provider?: unknown;
+      }
+    | null
+    | undefined
 ): NextResponse | null {
   if (!connection || connection.credentialDecryptFailed !== true) return null;
 
+  // #9927 — surface WHICH credential failed plus the recovery path so the
+  // dashboard points the operator at the account to re-authenticate instead of
+  // a generic "API key cannot be decrypted".
+  const provider = typeof connection.provider === "string" ? connection.provider : "";
+  const id = typeof connection.id === "string" ? connection.id : "";
+  const identity = [provider && `provider "${provider}"`, id && `connection ${id}`]
+    .filter(Boolean)
+    .join(", ");
+
+  const message =
+    `Stored credential${identity ? ` for ${identity}` : ""} cannot be decrypted ` +
+    `(STORAGE_ENCRYPTION_KEY changed or unset). Re-authenticate this account, or verify ` +
+    `STORAGE_ENCRYPTION_KEY matches the key used to store it.`;
+
   // buildErrorBody sanitizes the message (Rule #12); override the type so the
   // client can key off the specific stale-encryption cause.
-  const body = buildErrorBody(424, STALE_ENCRYPTION_MESSAGE);
+  const body = buildErrorBody(424, message);
   body.error.type = "storage_encryption_stale";
   return NextResponse.json(body, { status: 424 });
 }

src/lib/db/encryption.ts

@@ -51,6 +51,31 @@ export interface ConnectionFields {
   [key: string]: unknown;
 }
 
+/**
+ * #9927 — dedupe tracker for credential-decrypt-failure messages. The health
+ * sweep / refresh / request routing re-decrypt the same corrupt row every
+ * cycle; we log the enriched, actionable message ONCE per
+ * (provider + connection + failing-ciphertext) state so it does not spam
+ * every sweep, while still re-logging if the row state actually changes
+ * (e.g. a different field starts failing) instead of permanently suppressing.
+ */
+const loggedDecryptFailures = new Set<string>();
+
+function decryptFailureSignature(
+  connectionId: string,
+  provider: string,
+  failed: Array<{ field: string; value: unknown }>
+): string {
+  const parts = failed
+    .map((f) => `${f.field}:${typeof f.value === "string" ? f.value : ""}`)
+    .sort()
+    .join("|");
+  return `${provider}::${connectionId}::${parts}`;
+}
+
+const RECOVERY_HINT =
+  "Re-authenticate this account, or verify STORAGE_ENCRYPTION_KEY matches the key used to store it.";
+
 /**
  * Derive the PRIMARY encryption key using the static salt.
  * This is the canonical key derivation that all new encryptions use.
@@ -157,7 +182,10 @@ export function encrypt(plaintext: string | null | undefined): string | null | u
  * auto-migration: the next encrypt() call will re-encrypt it with the
  * static-salt key, gradually migrating the database.
  */
-export function decrypt(ciphertext: string | null | undefined): string | null | undefined {
+export function decrypt(
+  ciphertext: string | null | undefined,
+  opts?: { quiet?: boolean }
+): string | null | undefined {
   if (!ciphertext || typeof ciphertext !== "string") return ciphertext;
 
   // Not encrypted — return as-is (legacy plaintext or passthrough mode)
@@ -204,14 +232,21 @@ export function decrypt(ciphertext: string | null | undefined): string | null |
       return decrypted;
     }
 
-    console.error(
-      `[Encryption] Decryption failed. Ciphertext prefix: ${ciphertext.slice(0, 30)}... ` +
-        `Auth tag validation likely failed.`
-    );
+    // #9927 — the low-level generic log is suppressed when called through the
+    // connection-decryption path (quiet:true); decryptConnectionFields emits a
+    // single enriched message naming the credential + recovery path instead.
+    if (!opts?.quiet) {
+      console.error(
+        `[Encryption] Decryption failed. Ciphertext prefix: ${ciphertext.slice(0, 30)}... ` +
+          `Auth tag validation likely failed.`
+      );
+    }
     return null;
   } catch (err: unknown) {
     const message = err instanceof Error ? err.message : String(err);
-    console.error("[Encryption] Decryption failed:", message);
+    if (!opts?.quiet) {
+      console.error("[Encryption] Decryption failed:", message);
+    }
     return null;
   }
 }
@@ -242,10 +277,13 @@ export function decryptConnectionFields<T extends ConnectionFields | null | unde
   if (!row) return row;
   if (!isEncryptionEnabled()) return row;
 
-  const apiKey = decrypt(row.apiKey);
-  const accessToken = decrypt(row.accessToken);
-  const refreshToken = decrypt(row.refreshToken);
-  const idToken = decrypt(row.idToken);
+  // quiet:true — the low-level generic decrypt() log is suppressed here so a
+  // single failure emits ONE enriched message (below) naming the credential
+  // and recovery path (#9927) instead of one generic line per field per cycle.
+  const apiKey = decrypt(row.apiKey, { quiet: true });
+  const accessToken = decrypt(row.accessToken, { quiet: true });
+  const refreshToken = decrypt(row.refreshToken, { quiet: true });
+  const idToken = decrypt(row.idToken, { quiet: true });
 
   // #6148 — a stored credential that is still encrypted (`enc:v1:…`) but
   // decrypts to null means the STORAGE_ENCRYPTION_KEY changed or was unset.
@@ -257,6 +295,31 @@ export function decryptConnectionFields<T extends ConnectionFields | null | unde
     (looksEncrypted(row.refreshToken) && refreshToken === null) ||
     (looksEncrypted(row.idToken) && idToken === null);
 
+  if (credentialDecryptFailed) {
+    const failed: Array<{ field: string; value: unknown }> = [];
+    if (looksEncrypted(row.apiKey) && apiKey === null) failed.push({ field: "apiKey", value: row.apiKey });
+    if (looksEncrypted(row.accessToken) && accessToken === null)
+      failed.push({ field: "accessToken", value: row.accessToken });
+    if (looksEncrypted(row.refreshToken) && refreshToken === null)
+      failed.push({ field: "refreshToken", value: row.refreshToken });
+    if (looksEncrypted(row.idToken) && idToken === null) failed.push({ field: "idToken", value: row.idToken });
+
+    const connectionId = typeof row.id === "string" ? row.id : "";
+    const provider = typeof row.provider === "string" ? row.provider : "unknown";
+    const fields = failed.map((f) => f.field).join(", ");
+
+    // Dedupe per credential/row state: the sweep re-decrypts the same corrupt
+    // row every cycle — log ONCE unless the failing state actually changes.
+    const signature = decryptFailureSignature(connectionId, provider, failed);
+    if (!loggedDecryptFailures.has(signature)) {
+      loggedDecryptFailures.add(signature);
+      console.error(
+        `[Encryption] Failed to decrypt credential(s) [${fields}] for provider ` +
+          `"${provider}" (connection ${connectionId || "unknown"}). ${RECOVERY_HINT}`
+      );
+    }
+  }
+
   return {
     ...row,
     apiKey,

tests/unit/decrypt-failure-identify-credential-9927.test.ts

@@ -0,0 +1,108 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+
+// #9927 — A credential that no longer decrypts (e.g. STORAGE_ENCRYPTION_KEY
+// changed between restarts) must emit a single, enriched error naming the
+// provider + connection id + failing field(s) and a recovery path, instead of
+// the generic low-level `[Encryption] Decryption failed … Auth tag validation
+// likely failed` line that carries no identity and is re-printed every sweep.
+
+const ORIGINAL_STORAGE_KEY = process.env.STORAGE_ENCRYPTION_KEY;
+
+// Cache-busted fresh import so the encryption module re-derives its key from
+// the current STORAGE_ENCRYPTION_KEY and resets module-level dedupe state.
+async function importFresh(modulePath: string) {
+  const url = pathToFileURL(path.resolve(modulePath)).href;
+  return import(`${url}?test=${Date.now()}-${Math.random().toString(16).slice(2)}`);
+}
+
+test.after(() => {
+  if (ORIGINAL_STORAGE_KEY === undefined) {
+    delete process.env.STORAGE_ENCRYPTION_KEY;
+  } else {
+    process.env.STORAGE_ENCRYPTION_KEY = ORIGINAL_STORAGE_KEY;
+  }
+});
+
+function captureConsoleError(fn: () => void): string[] {
+  const original = console.error;
+  const logs: string[] = [];
+  console.error = (...args: unknown[]) => {
+    logs.push(args.join(" "));
+  };
+  try {
+    fn();
+  } finally {
+    console.error = original;
+  }
+  return logs;
+}
+
+test("decryptConnectionFields logs failed credential identity + recovery path (#9927)", async () => {
+  // 1. Encrypt an apiKey under key A.
+  process.env.STORAGE_ENCRYPTION_KEY = "stale-key-9927-A";
+  const encA = await importFresh("src/lib/db/encryption.ts");
+  const ciphertext = encA.encrypt("sk-real-secret-key");
+  assert.match(ciphertext, /^enc:v1:/, "expected a real enc:v1 ciphertext");
+
+  // 2. Read it back under a DIFFERENT key B (simulating a changed key).
+  process.env.STORAGE_ENCRYPTION_KEY = "stale-key-9927-B";
+  const encB = await importFresh("src/lib/db/encryption.ts");
+
+  const logs = captureConsoleError(() => {
+    encB.decryptConnectionFields({
+      id: "conn-9927",
+      provider: "openai",
+      apiKey: ciphertext,
+    });
+  });
+
+  // Must flag the failure so callers can surface the cause.
+  const decrypted = encB.decryptConnectionFields({
+    id: "conn-9927",
+    provider: "openai",
+    apiKey: ciphertext,
+  });
+  assert.equal(decrypted.credentialDecryptFailed, true);
+
+  // The generic low-level log must NOT fire (quiet:true); instead ONE enriched
+  // message names provider + connection id + recovery path.
+  assert.equal(
+    logs.some((l) => /Auth tag validation likely failed/.test(l)),
+    false,
+    "generic low-level decrypt log must be suppressed on the connection path"
+  );
+
+  const enriched = logs.find((l) => l.includes("Failed to decrypt credential(s)"));
+  assert.ok(enriched, "expected an enriched credential-decrypt-failure log");
+  assert.match(enriched, /provider "openai"/, "log must name the provider");
+  assert.match(enriched, /conn-9927/, "log must name the connection id");
+  assert.match(enriched, /apiKey/, "log must name the failing field");
+  assert.match(
+    enriched,
+    /STORAGE_ENCRYPTION_KEY matches the key used to store it/,
+    "log must include the recovery path"
+  );
+});
+
+test("credential-decrypt failure is logged once per connection (dedupe #9927)", async () => {
+  process.env.STORAGE_ENCRYPTION_KEY = "stale-key-9927-dedupe-A";
+  const encA = await importFresh("src/lib/db/encryption.ts");
+  const ciphertext = encA.encrypt("sk-dedupe-key");
+
+  process.env.STORAGE_ENCRYPTION_KEY = "stale-key-9927-dedupe-B";
+  const encB = await importFresh("src/lib/db/encryption.ts");
+
+  const row = { id: "conn-dedupe", provider: "openai", apiKey: ciphertext };
+  const logs = captureConsoleError(() => {
+    // Simulate the health sweep re-decrypting the same corrupt row repeatedly.
+    for (let i = 0; i < 5; i++) {
+      encB.decryptConnectionFields(row);
+    }
+  });
+
+  const enriched = logs.filter((l) => l.includes("Failed to decrypt credential(s)"));
+  assert.equal(enriched.length, 1, "identical failure must be logged once per connection");
+});