mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-11 09:42:15 +03:00
fix(sse): correlate reason and resource within the same ErrorInfo detail
This commit is contained in:
@@ -1908,9 +1908,49 @@ export async function getProviderCredentialsWithQuotaPreflight(
|
||||
* lockout behavior, since that's the safer default and the actual bug this
|
||||
* plan fixes (avoid defaulting BACK toward the connection-wide cooldown this
|
||||
* plan exists to avoid).
|
||||
*
|
||||
* Parses the body as JSON and inspects each ErrorInfo-shaped detail object so
|
||||
* `reason` and `resource` are correlated within the SAME detail entry — a
|
||||
* multi-detail error body (unusual but possible) must not let one detail's
|
||||
* resource leak into another detail's reason check. Falls back to a permissive
|
||||
* regex scan (pre-JSON-parsing behavior) only when the body isn't parseable
|
||||
* JSON or doesn't contain a `details` array, since Vertex error bodies aren't
|
||||
* guaranteed to always be well-formed JSON.
|
||||
*/
|
||||
function isVertexConnectionWidePermissionDenied(errorText: string | null | undefined): boolean {
|
||||
if (!errorText) return false;
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(errorText);
|
||||
const details: unknown[] =
|
||||
parsed?.error?.details ?? parsed?.details ?? (Array.isArray(parsed) ? parsed : []);
|
||||
if (Array.isArray(details) && details.length > 0) {
|
||||
for (const detail of details) {
|
||||
if (!detail || typeof detail !== "object") continue;
|
||||
const reason = (detail as Record<string, unknown>).reason;
|
||||
if (reason === "SERVICE_DISABLED") return true;
|
||||
if (reason === "IAM_PERMISSION_DENIED") {
|
||||
const metadata = (detail as Record<string, unknown>).metadata;
|
||||
const resource =
|
||||
metadata && typeof metadata === "object"
|
||||
? (metadata as Record<string, unknown>).resource
|
||||
: undefined;
|
||||
if (typeof resource === "string" && !resource.includes("/models/")) return true;
|
||||
}
|
||||
}
|
||||
// Well-formed details array present but no detail matched a connection-wide
|
||||
// pattern (e.g. IAM_PERMISSION_DENIED with a /models/ resource, or no
|
||||
// recognized reason at all) — per-model lockout is correct, don't fall
|
||||
// through to the regex heuristic (it would just re-derive the same answer
|
||||
// less precisely, or worse, could false-positive on stray substrings).
|
||||
return false;
|
||||
}
|
||||
} catch {
|
||||
// Not parseable JSON — fall through to the regex heuristic below.
|
||||
}
|
||||
|
||||
// Fallback for non-JSON or unexpected-shape error bodies (regex-based,
|
||||
// pre-JSON-parsing heuristic — kept for robustness against malformed bodies).
|
||||
if (/"reason"\s*:\s*"SERVICE_DISABLED"/.test(errorText)) return true;
|
||||
if (/"reason"\s*:\s*"IAM_PERMISSION_DENIED"/.test(errorText)) {
|
||||
const resourceMatch = errorText.match(/"resource"\s*:\s*"([^"]*)"/);
|
||||
|
||||
@@ -239,3 +239,54 @@ test("403 with IAM_PERMISSION_DENIED reason and a project-level resource cools d
|
||||
const lockout = accountFallback.getModelLockoutInfo("vertex", conn.id, "claude-sonnet-5-high");
|
||||
assert.equal(lockout, null);
|
||||
});
|
||||
|
||||
test("multi-detail error body correlates reason+resource per-detail, not across details", async () => {
|
||||
// Adversarial case: detail[0] has a model-scoped resource under an unrelated reason,
|
||||
// detail[1] carries the actual IAM_PERMISSION_DENIED with a project-level resource. A
|
||||
// naive independent-regex scan would match detail[0]'s resource against detail[1]'s
|
||||
// reason and wrongly conclude "model-scoped" — this must resolve to connection-wide.
|
||||
await resetStorage();
|
||||
const conn = await seedVertex();
|
||||
|
||||
const body = JSON.stringify({
|
||||
error: {
|
||||
status: "PERMISSION_DENIED",
|
||||
details: [
|
||||
{
|
||||
"@type": "type.googleapis.com/google.rpc.ErrorInfo",
|
||||
reason: "SOME_OTHER_REASON",
|
||||
domain: "iam.googleapis.com",
|
||||
metadata: {
|
||||
resource:
|
||||
"projects/12345/locations/us-central1/publishers/google/models/claude-sonnet-5",
|
||||
},
|
||||
},
|
||||
{
|
||||
"@type": "type.googleapis.com/google.rpc.ErrorInfo",
|
||||
reason: "IAM_PERMISSION_DENIED",
|
||||
domain: "iam.googleapis.com",
|
||||
metadata: {
|
||||
permission: "aiplatform.googleapis.com/models.predict",
|
||||
resource: "projects/12345",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
const result = await auth.markAccountUnavailable(
|
||||
conn.id,
|
||||
403,
|
||||
body,
|
||||
"vertex",
|
||||
"claude-sonnet-5-high"
|
||||
);
|
||||
assert.equal(result.shouldFallback, true);
|
||||
|
||||
const after2 = await providersDb.getProviderConnectionById(conn.id);
|
||||
assert.equal(after2.testStatus, "unavailable");
|
||||
assert.ok(after2.rateLimitedUntil, "connection must be cooled down, not left active");
|
||||
|
||||
const lockout2 = accountFallback.getModelLockoutInfo("vertex", conn.id, "claude-sonnet-5-high");
|
||||
assert.equal(lockout2, null);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user