From 239200624618992a7d19a37dd2db033de1dc8853 Mon Sep 17 00:00:00 2001
From: Brendan DeBeasi <hi@neueway.com>
Date: Thu, 26 Mar 2026 11:11:49 -0700
Subject: [PATCH] fix: retry expired connections in token health check instead
 of permanently skipping
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Connections marked as 'expired' were permanently skipped by the health check scheduler (line 176: if testStatus === "expired" return). A single transient refresh failure could permanently disable auto-refresh, requiring manual re-authentication.

Replace the hard skip with a bounded retry mechanism: up to 3 attempts with exponential backoff (5min, 10min, 20min). On success, the connection is fully restored to active. On exhaustion, it remains expired (same as before). The existing circuit breaker (5 failures → 30min pause) provides additional protection.

Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-opencode)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
---
 src/lib/tokenHealthCheck.ts | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/src/lib/tokenHealthCheck.ts b/src/lib/tokenHealthCheck.ts
index 051470cf5e..69165c6ab6 100644
--- a/src/lib/tokenHealthCheck.ts
+++ b/src/lib/tokenHealthCheck.ts
@@ -20,6 +20,8 @@ import {
 // ── Constants ────────────────────────────────────────────────────────────────
 const TICK_MS = 60 * 1000; // sweep interval: every 60 seconds
 const DEFAULT_HEALTH_CHECK_INTERVAL_MIN = 60; // default per-connection interval
+const EXPIRED_RETRY_MAX = 3; // max retry attempts for expired connections before giving up
+const EXPIRED_RETRY_BACKOFF_MIN = 5; // backoff between expired retries (minutes)
 const LOG_PREFIX = "[HealthCheck]";
 const TRUE_ENV_VALUES = new Set(["1", "true", "yes", "on"]);
 
@@ -172,8 +174,19 @@ async function checkConnection(conn) {
   if (!conn.isActive) return;
   if (!conn.refreshToken || typeof conn.refreshToken !== "string") return;
 
-  // Skip connections already marked as expired (need re-auth, not retry)
-  if (conn.testStatus === "expired") return;
+  // Retry expired connections with exponential backoff up to EXPIRED_RETRY_MAX times.
+  if (conn.testStatus === "expired") {
+    const retryCount = conn.expiredRetryCount ?? 0;
+    if (retryCount >= EXPIRED_RETRY_MAX) return;
+
+    const lastRetry = conn.expiredRetryAt ? new Date(conn.expiredRetryAt).getTime() : 0;
+    const backoffMs = EXPIRED_RETRY_BACKOFF_MIN * 60 * 1000 * Math.pow(2, retryCount);
+    if (Date.now() - lastRetry < backoffMs) return;
+
+    log(
+      `${LOG_PREFIX} Retrying expired ${conn.provider}/${conn.name || conn.email || conn.id} (attempt ${retryCount + 1}/${EXPIRED_RETRY_MAX})`
+    );
+  }
 
   if (!supportsTokenRefresh(conn.provider)) {
     const now = new Date().toISOString();
@@ -241,7 +254,6 @@ async function checkConnection(conn) {
   }
 
   if (result && result.accessToken) {
-    // Token refreshed successfully — update DB
     const updateData: any = {
       accessToken: result.accessToken,
       lastHealthCheckAt: now,
@@ -251,6 +263,8 @@ async function checkConnection(conn) {
       lastErrorType: null,
       lastErrorSource: null,
       errorCode: null,
+      expiredRetryCount: null,
+      expiredRetryAt: null,
     };
 
     if (result.refreshToken) {
@@ -264,18 +278,22 @@ async function checkConnection(conn) {
     await updateProviderConnection(conn.id, updateData);
     log(`${LOG_PREFIX} ✓ ${conn.provider}/${conn.name || conn.email || conn.id} refreshed`);
   } else {
-    // Refresh failed — record but don't disable the connection
+    const wasExpired = conn.testStatus === "expired";
+    const retryCount = (conn.expiredRetryCount ?? 0) + (wasExpired ? 1 : 0);
+
     await updateProviderConnection(conn.id, {
       lastHealthCheckAt: now,
-      testStatus: "error",
+      testStatus: wasExpired ? "expired" : "error",
       lastError: "Health check: token refresh failed",
       lastErrorAt: now,
       lastErrorType: "token_refresh_failed",
       lastErrorSource: "oauth",
       errorCode: "refresh_failed",
+      ...(wasExpired ? { expiredRetryCount: retryCount, expiredRetryAt: now } : {}),
     });
     logWarn(
-      `${LOG_PREFIX} ✗ ${conn.provider}/${conn.name || conn.email || conn.id} refresh failed`
+      `${LOG_PREFIX} ✗ ${conn.provider}/${conn.name || conn.email || conn.id} refresh failed` +
+        (wasExpired ? ` (expired retry ${retryCount}/${EXPIRED_RETRY_MAX})` : "")
     );
   }
 }