From 77eb184f9d0428257efd54944736a55fbbd1670b Mon Sep 17 00:00:00 2001 From: Will Gordon Date: Thu, 30 Jul 2026 17:53:00 -0400 Subject: [PATCH] fix(sse): correlate reason and resource within the same ErrorInfo detail --- src/sse/services/auth.ts | 40 +++++++++++++++ .../vertex-passthrough-model-lockout.test.ts | 51 +++++++++++++++++++ 2 files changed, 91 insertions(+) diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index f0302970ac..3128e7f349 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -1908,9 +1908,49 @@ export async function getProviderCredentialsWithQuotaPreflight( * lockout behavior, since that's the safer default and the actual bug this * plan fixes (avoid defaulting BACK toward the connection-wide cooldown this * plan exists to avoid). + * + * Parses the body as JSON and inspects each ErrorInfo-shaped detail object so + * `reason` and `resource` are correlated within the SAME detail entry — a + * multi-detail error body (unusual but possible) must not let one detail's + * resource leak into another detail's reason check. Falls back to a permissive + * regex scan (pre-JSON-parsing behavior) only when the body isn't parseable + * JSON or doesn't contain a `details` array, since Vertex error bodies aren't + * guaranteed to always be well-formed JSON. */ function isVertexConnectionWidePermissionDenied(errorText: string | null | undefined): boolean { if (!errorText) return false; + + try { + const parsed = JSON.parse(errorText); + const details: unknown[] = + parsed?.error?.details ?? parsed?.details ?? (Array.isArray(parsed) ? parsed : []); + if (Array.isArray(details) && details.length > 0) { + for (const detail of details) { + if (!detail || typeof detail !== "object") continue; + const reason = (detail as Record).reason; + if (reason === "SERVICE_DISABLED") return true; + if (reason === "IAM_PERMISSION_DENIED") { + const metadata = (detail as Record).metadata; + const resource = + metadata && typeof metadata === "object" + ? (metadata as Record).resource + : undefined; + if (typeof resource === "string" && !resource.includes("/models/")) return true; + } + } + // Well-formed details array present but no detail matched a connection-wide + // pattern (e.g. IAM_PERMISSION_DENIED with a /models/ resource, or no + // recognized reason at all) — per-model lockout is correct, don't fall + // through to the regex heuristic (it would just re-derive the same answer + // less precisely, or worse, could false-positive on stray substrings). + return false; + } + } catch { + // Not parseable JSON — fall through to the regex heuristic below. + } + + // Fallback for non-JSON or unexpected-shape error bodies (regex-based, + // pre-JSON-parsing heuristic — kept for robustness against malformed bodies). if (/"reason"\s*:\s*"SERVICE_DISABLED"/.test(errorText)) return true; if (/"reason"\s*:\s*"IAM_PERMISSION_DENIED"/.test(errorText)) { const resourceMatch = errorText.match(/"resource"\s*:\s*"([^"]*)"/); diff --git a/tests/unit/vertex-passthrough-model-lockout.test.ts b/tests/unit/vertex-passthrough-model-lockout.test.ts index 2e04b4db2e..68bb5fa7b2 100644 --- a/tests/unit/vertex-passthrough-model-lockout.test.ts +++ b/tests/unit/vertex-passthrough-model-lockout.test.ts @@ -239,3 +239,54 @@ test("403 with IAM_PERMISSION_DENIED reason and a project-level resource cools d const lockout = accountFallback.getModelLockoutInfo("vertex", conn.id, "claude-sonnet-5-high"); assert.equal(lockout, null); }); + +test("multi-detail error body correlates reason+resource per-detail, not across details", async () => { + // Adversarial case: detail[0] has a model-scoped resource under an unrelated reason, + // detail[1] carries the actual IAM_PERMISSION_DENIED with a project-level resource. A + // naive independent-regex scan would match detail[0]'s resource against detail[1]'s + // reason and wrongly conclude "model-scoped" — this must resolve to connection-wide. + await resetStorage(); + const conn = await seedVertex(); + + const body = JSON.stringify({ + error: { + status: "PERMISSION_DENIED", + details: [ + { + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + reason: "SOME_OTHER_REASON", + domain: "iam.googleapis.com", + metadata: { + resource: + "projects/12345/locations/us-central1/publishers/google/models/claude-sonnet-5", + }, + }, + { + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + reason: "IAM_PERMISSION_DENIED", + domain: "iam.googleapis.com", + metadata: { + permission: "aiplatform.googleapis.com/models.predict", + resource: "projects/12345", + }, + }, + ], + }, + }); + + const result = await auth.markAccountUnavailable( + conn.id, + 403, + body, + "vertex", + "claude-sonnet-5-high" + ); + assert.equal(result.shouldFallback, true); + + const after2 = await providersDb.getProviderConnectionById(conn.id); + assert.equal(after2.testStatus, "unavailable"); + assert.ok(after2.rateLimitedUntil, "connection must be cooled down, not left active"); + + const lockout2 = accountFallback.getModelLockoutInfo("vertex", conn.id, "claude-sonnet-5-high"); + assert.equal(lockout2, null); +});