fix(sse): disambiguate Vertex connection-wide vs per-model 403s

src/sse/services/auth.ts

@@ -1898,6 +1898,27 @@ export async function getProviderCredentialsWithQuotaPreflight(
   }
 }
 
+/**
+ * Google's google.rpc.ErrorInfo proto reliably distinguishes a connection-wide
+ * PERMISSION_DENIED (API not enabled, or a project-level IAM denial) from a
+ * model-specific one (IAM denial scoped to a .../models/<id> resource) — see
+ * https://cloud.google.com/apis/design/errors#error_info. Only returns true on
+ * POSITIVE evidence of a connection-wide cause; any other shape (including a
+ * missing/malformed resource field) falls through to the existing per-model
+ * lockout behavior, since that's the safer default and the actual bug this
+ * plan fixes (avoid defaulting BACK toward the connection-wide cooldown this
+ * plan exists to avoid).
+ */
+function isVertexConnectionWidePermissionDenied(errorText: string | null | undefined): boolean {
+  if (!errorText) return false;
+  if (/"reason"\s*:\s*"SERVICE_DISABLED"/.test(errorText)) return true;
+  if (/"reason"\s*:\s*"IAM_PERMISSION_DENIED"/.test(errorText)) {
+    const resourceMatch = errorText.match(/"resource"\s*:\s*"([^"]*)"/);
+    if (resourceMatch && !resourceMatch[1].includes("/models/")) return true;
+  }
+  return false;
+}
+
 /** Persist exponential-backoff state for an unavailable provider connection. */
 export async function markAccountUnavailable(
   connectionId: string,
@@ -2146,7 +2167,14 @@ export async function markAccountUnavailable(
         : rawCooldownMs;
 
     // ── #3027: per-model subscription/permission 403 → model-only lockout ──
-    if (isPerModelQuotaProvider && status === 403 && provider && model && !terminalStatus) {
+    if (
+      isPerModelQuotaProvider &&
+      status === 403 &&
+      provider &&
+      model &&
+      !terminalStatus &&
+      !(provider === "vertex" && isVertexConnectionWidePermissionDenied(errorText))
+    ) {
       const lockout = recordModelLockoutFailure(
         provider,
         connectionId,

tests/unit/vertex-passthrough-model-lockout.test.ts

@@ -119,3 +119,123 @@ test("403 PERMISSION_DENIED on Vertex locks only that model too (accepted trade-
   const lockout = accountFallback.getModelLockoutInfo("vertex", conn.id, "claude-sonnet-5-high");
   assert.equal(lockout?.reason, "forbidden");
 });
+
+// ── 403 disambiguation via google.rpc.ErrorInfo (Task 7) ────────────────────
+// Google Cloud's documented error format (https://cloud.google.com/apis/design/errors#error_info)
+// lets us tell a genuinely connection-wide PERMISSION_DENIED (API disabled, or a
+// project-level IAM denial) apart from one scoped to a single model — the former must
+// fall through to the existing connection-wide cooldown instead of the #3027 per-model
+// lockout path, since a per-model lockout would leave a broken connection "active".
+
+test("403 with SERVICE_DISABLED ErrorInfo reason cools down the whole Vertex connection", async () => {
+  await resetStorage();
+  const conn = await seedVertex();
+
+  const body = JSON.stringify({
+    error: {
+      status: "PERMISSION_DENIED",
+      details: [
+        {
+          "@type": "type.googleapis.com/google.rpc.ErrorInfo",
+          reason: "SERVICE_DISABLED",
+          domain: "googleapis.com",
+          metadata: { service: "aiplatform.googleapis.com", consumer: "projects/12345" },
+        },
+      ],
+    },
+  });
+
+  const result = await auth.markAccountUnavailable(
+    conn.id,
+    403,
+    body,
+    "vertex",
+    "claude-sonnet-5-high"
+  );
+  assert.equal(result.shouldFallback, true);
+
+  const after = await providersDb.getProviderConnectionById(conn.id);
+  assert.equal(after.testStatus, "unavailable");
+  assert.ok(after.rateLimitedUntil, "connection must be cooled down, not left active");
+
+  // The #3027 per-model lockout path must NOT have run.
+  const lockout = accountFallback.getModelLockoutInfo("vertex", conn.id, "claude-sonnet-5-high");
+  assert.equal(lockout, null);
+});
+
+test("403 with IAM_PERMISSION_DENIED reason and a model-scoped resource still locks only that model", async () => {
+  await resetStorage();
+  const conn = await seedVertex();
+
+  const body = JSON.stringify({
+    error: {
+      status: "PERMISSION_DENIED",
+      details: [
+        {
+          "@type": "type.googleapis.com/google.rpc.ErrorInfo",
+          reason: "IAM_PERMISSION_DENIED",
+          domain: "iam.googleapis.com",
+          metadata: {
+            permission: "aiplatform.endpoints.predict",
+            resource:
+              "projects/12345/locations/us-central1/publishers/google/models/claude-sonnet-5",
+          },
+        },
+      ],
+    },
+  });
+
+  const result = await auth.markAccountUnavailable(
+    conn.id,
+    403,
+    body,
+    "vertex",
+    "claude-sonnet-5-high"
+  );
+  assert.equal(result.shouldFallback, true);
+
+  const after = await providersDb.getProviderConnectionById(conn.id);
+  assert.equal(after.testStatus, "active");
+  assert.ok(!after.rateLimitedUntil, "connection must not be rate-limited");
+
+  const lockout = accountFallback.getModelLockoutInfo("vertex", conn.id, "claude-sonnet-5-high");
+  assert.equal(lockout?.reason, "forbidden");
+});
+
+test("403 with IAM_PERMISSION_DENIED reason and a project-level resource cools down the whole connection", async () => {
+  await resetStorage();
+  const conn = await seedVertex();
+
+  const body = JSON.stringify({
+    error: {
+      status: "PERMISSION_DENIED",
+      details: [
+        {
+          "@type": "type.googleapis.com/google.rpc.ErrorInfo",
+          reason: "IAM_PERMISSION_DENIED",
+          domain: "iam.googleapis.com",
+          metadata: {
+            permission: "aiplatform.googleapis.com/models.predict",
+            resource: "projects/12345",
+          },
+        },
+      ],
+    },
+  });
+
+  const result = await auth.markAccountUnavailable(
+    conn.id,
+    403,
+    body,
+    "vertex",
+    "claude-sonnet-5-high"
+  );
+  assert.equal(result.shouldFallback, true);
+
+  const after = await providersDb.getProviderConnectionById(conn.id);
+  assert.equal(after.testStatus, "unavailable");
+  assert.ok(after.rateLimitedUntil, "connection must be cooled down, not left active");
+
+  const lockout = accountFallback.getModelLockoutInfo("vertex", conn.id, "claude-sonnet-5-high");
+  assert.equal(lockout, null);
+});