fix(sse): correlate reason and resource within the same ErrorInfo detail

This commit is contained in:
Will Gordon
2026-07-30 17:53:00 -04:00
parent 3780d45d62
commit 77eb184f9d
2 changed files with 91 additions and 0 deletions

View File

@@ -1908,9 +1908,49 @@ export async function getProviderCredentialsWithQuotaPreflight(
* lockout behavior, since that's the safer default and the actual bug this
* plan fixes (avoid defaulting BACK toward the connection-wide cooldown this
* plan exists to avoid).
*
* Parses the body as JSON and inspects each ErrorInfo-shaped detail object so
* `reason` and `resource` are correlated within the SAME detail entry — a
* multi-detail error body (unusual but possible) must not let one detail's
* resource leak into another detail's reason check. Falls back to a permissive
* regex scan (pre-JSON-parsing behavior) only when the body isn't parseable
* JSON or doesn't contain a `details` array, since Vertex error bodies aren't
* guaranteed to always be well-formed JSON.
*/
function isVertexConnectionWidePermissionDenied(errorText: string | null | undefined): boolean {
if (!errorText) return false;
try {
const parsed = JSON.parse(errorText);
const details: unknown[] =
parsed?.error?.details ?? parsed?.details ?? (Array.isArray(parsed) ? parsed : []);
if (Array.isArray(details) && details.length > 0) {
for (const detail of details) {
if (!detail || typeof detail !== "object") continue;
const reason = (detail as Record<string, unknown>).reason;
if (reason === "SERVICE_DISABLED") return true;
if (reason === "IAM_PERMISSION_DENIED") {
const metadata = (detail as Record<string, unknown>).metadata;
const resource =
metadata && typeof metadata === "object"
? (metadata as Record<string, unknown>).resource
: undefined;
if (typeof resource === "string" && !resource.includes("/models/")) return true;
}
}
// Well-formed details array present but no detail matched a connection-wide
// pattern (e.g. IAM_PERMISSION_DENIED with a /models/ resource, or no
// recognized reason at all) — per-model lockout is correct, don't fall
// through to the regex heuristic (it would just re-derive the same answer
// less precisely, or worse, could false-positive on stray substrings).
return false;
}
} catch {
// Not parseable JSON — fall through to the regex heuristic below.
}
// Fallback for non-JSON or unexpected-shape error bodies (regex-based,
// pre-JSON-parsing heuristic — kept for robustness against malformed bodies).
if (/"reason"\s*:\s*"SERVICE_DISABLED"/.test(errorText)) return true;
if (/"reason"\s*:\s*"IAM_PERMISSION_DENIED"/.test(errorText)) {
const resourceMatch = errorText.match(/"resource"\s*:\s*"([^"]*)"/);

View File

@@ -239,3 +239,54 @@ test("403 with IAM_PERMISSION_DENIED reason and a project-level resource cools d
const lockout = accountFallback.getModelLockoutInfo("vertex", conn.id, "claude-sonnet-5-high");
assert.equal(lockout, null);
});
test("multi-detail error body correlates reason+resource per-detail, not across details", async () => {
// Adversarial case: detail[0] has a model-scoped resource under an unrelated reason,
// detail[1] carries the actual IAM_PERMISSION_DENIED with a project-level resource. A
// naive independent-regex scan would match detail[0]'s resource against detail[1]'s
// reason and wrongly conclude "model-scoped" — this must resolve to connection-wide.
await resetStorage();
const conn = await seedVertex();
const body = JSON.stringify({
error: {
status: "PERMISSION_DENIED",
details: [
{
"@type": "type.googleapis.com/google.rpc.ErrorInfo",
reason: "SOME_OTHER_REASON",
domain: "iam.googleapis.com",
metadata: {
resource:
"projects/12345/locations/us-central1/publishers/google/models/claude-sonnet-5",
},
},
{
"@type": "type.googleapis.com/google.rpc.ErrorInfo",
reason: "IAM_PERMISSION_DENIED",
domain: "iam.googleapis.com",
metadata: {
permission: "aiplatform.googleapis.com/models.predict",
resource: "projects/12345",
},
},
],
},
});
const result = await auth.markAccountUnavailable(
conn.id,
403,
body,
"vertex",
"claude-sonnet-5-high"
);
assert.equal(result.shouldFallback, true);
const after2 = await providersDb.getProviderConnectionById(conn.id);
assert.equal(after2.testStatus, "unavailable");
assert.ok(after2.rateLimitedUntil, "connection must be cooled down, not left active");
const lockout2 = accountFallback.getModelLockoutInfo("vertex", conn.id, "claude-sonnet-5-high");
assert.equal(lockout2, null);
});