diff --git a/open-sse/executors/antigravity.ts b/open-sse/executors/antigravity.ts index ffbeaff31b..030d25118a 100644 --- a/open-sse/executors/antigravity.ts +++ b/open-sse/executors/antigravity.ts @@ -1,6 +1,15 @@ import crypto, { randomUUID } from "crypto"; import { BaseExecutor, mergeUpstreamExtraHeaders } from "./base.ts"; import { PROVIDERS, OAUTH_ENDPOINTS, HTTP_STATUS } from "../config/constants.ts"; +import { scrubProxyAndFingerprintHeaders } from "../services/antigravityHeaderScrub.ts"; +import { antigravityUserAgent, googApiClientHeader } from "../services/antigravityHeaders.ts"; +import { classify429, decide429, type Decision } from "../services/antigravity429Engine.ts"; +import { + injectCreditsField, + shouldRetryWithCredits, + handleCreditsFailure, +} from "../services/antigravityCredits.ts"; +import { obfuscateSensitiveWords } from "../services/antigravityObfuscation.ts"; const MAX_RETRY_AFTER_MS = 60_000; const LONG_RETRY_THRESHOLD_MS = 60_000; @@ -79,13 +88,15 @@ export class AntigravityExecutor extends BaseExecutor { } buildHeaders(credentials, stream = true) { - return { + const raw = { "Content-Type": "application/json", Authorization: `Bearer ${credentials.accessToken}`, - "User-Agent": this.config.headers?.["User-Agent"] || "antigravity/1.104.0 darwin/arm64", - "X-OmniRoute-Source": "omniroute", + "User-Agent": antigravityUserAgent(), + "X-Goog-Api-Client": googApiClientHeader(), Accept: "text/event-stream", }; + // Scrub proxy/fingerprint headers that reveal non-native traffic + return scrubProxyAndFingerprintHeaders(raw); } transformRequest(model, body, stream, credentials) { @@ -159,6 +170,20 @@ export class AntigravityExecutor extends BaseExecutor { const upstreamModel = cleanModelName(model); + // Obfuscate sensitive client names in user content (e.g. "OpenCode", "Cursor") + const requestContents = transformedRequest.contents; + if (Array.isArray(requestContents)) { + for (const msg of requestContents) { + if (Array.isArray(msg.parts)) { + for (const part of msg.parts) { + if (typeof part.text === "string") { + part.text = obfuscateSensitiveWords(part.text); + } + } + } + } + } + return { ...body, project: projectId, @@ -427,40 +452,40 @@ export class AntigravityExecutor extends BaseExecutor { const errorBody = await response.clone().text(); const errorJson = JSON.parse(errorBody); const errorMessage = errorJson?.error?.message || errorJson?.message || ""; - const lowerMsg = errorMessage.toLowerCase(); - // ── AI Credits Overages fallback ───────────────────────────────── - // MUST run BEFORE parseRetryFromErrorMessage: the API embeds the - // reset time in the same message ("reset after 141h22m11s"), so - // parseRetryFromErrorMessage fills retryMs and the !retryMs guard - // below would silently skip the credit injection. + // 1. Try to parse explicit retry time from message + const parsedRetryMs = this.parseRetryFromErrorMessage(errorMessage); + + // 2. Classify 429 + const category = classify429(errorMessage); + + // 3. For quota_exhausted, attempt Google One AI credits retry FIRST! if ( - lowerMsg.includes("exhausted your capacity") || - lowerMsg.includes("exhausted your") || - lowerMsg.includes("daily limit") || - lowerMsg.includes("quota exceeded") + category === "quota_exhausted" && + shouldRetryWithCredits( + credentials?.accessToken || "", + process.env.ANTIGRAVITY_CREDITS === "1" || + process.env.ANTIGRAVITY_CREDITS === "true" + ) ) { - if (!isCreditsExhausted(accountId) && !transformedBody?.enabledCreditTypes) { - log?.info?.( - "CREDITS", - `Quota exhausted for ${model} — retrying with GOOGLE_ONE_AI credits (account: ${accountId})` - ); - const creditBody = { ...transformedBody, enabledCreditTypes: ["GOOGLE_ONE_AI"] }; - const creditRes = await fetch(url, { + log?.info?.("AG_CREDITS", "Retrying with Google One AI credits"); + const creditsBody = injectCreditsField(transformedBody); + try { + const creditsResp = await fetch(url, { method: "POST", headers, - body: JSON.stringify(creditBody), + body: JSON.stringify(creditsBody), signal, }); - - if (creditRes.ok) { + if (creditsResp.ok || creditsResp.status !== HTTP_STATUS.RATE_LIMITED) { + log?.info?.("AG_CREDITS", `Credits retry succeeded: ${creditsResp.status}`); if (!stream) { const collected = await this.collectStreamToResponse( - creditRes, + creditsResp, model, url, headers, - creditBody, + creditsBody, log, signal ); @@ -481,53 +506,28 @@ export class AntigravityExecutor extends BaseExecutor { } return collected; } - return { response: creditRes, url, headers, transformedBody: creditBody }; + return { response: creditsResp, url, headers, transformedBody: creditsBody }; } - // Credit retry also failed — check if credits are exhausted - try { - const creditErrText = await creditRes.clone().text(); - const creditErrJson = JSON.parse(creditErrText); - const creditErrMsg = (creditErrJson?.error?.message || "").toLowerCase(); - if ( - creditErrMsg.includes("credit") || - creditErrMsg.includes("insufficient") || - creditErrMsg.includes("exhausted") - ) { - log?.warn?.( - "CREDITS", - `GOOGLE_ONE_AI credits exhausted for account ${accountId} — caching for 5h` - ); - markCreditsExhausted(accountId); - } - } catch { - /**/ - } - // Fall through to normal fallback logic below - } else if (!isCreditsExhausted(accountId) && transformedBody?.enabledCreditTypes) { - // Already had credits injected and still failed — mark exhausted + // Credit retry also 429'd + handleCreditsFailure(credentials?.accessToken || ""); + log?.warn?.("AG_CREDITS", "Credits retry also 429'd"); + + // Also mark in our legacy exhaustion map to avoid retrying other routes markCreditsExhausted(accountId); - log?.warn?.("CREDITS", `Credits exhausted for account ${accountId}`); - } - - // Hard quota limit — force fallback to next account regardless - retryMs = 24 * 60 * 60 * 1000; - } else { - // Not a quota-exhaustion error — try to parse a Retry-After from the message - retryMs = this.parseRetryFromErrorMessage(errorMessage); - - if (!retryMs) { - if (lowerMsg.includes("free tier") || lowerMsg.includes("free")) { - retryMs = 24 * 60 * 60 * 1000; - } else if ( - lowerMsg.includes("pro") || - lowerMsg.includes("per minute") || - lowerMsg.includes("rpm") - ) { - retryMs = 60 * 1000; - } + } catch (creditsErr) { + handleCreditsFailure(credentials?.accessToken || ""); + log?.warn?.("AG_CREDITS", `Credits retry failed: ${creditsErr}`); } } + + // 4. Decide final retry time (apply 4-tier engine) + const decision: Decision = decide429(category, parsedRetryMs); + retryMs = decision.retryAfterMs; + log?.debug?.( + "AG_429", + `Category: ${category}, Decision: ${decision.kind} — ${decision.reason}` + ); } catch (e) { // Ignore parse errors, will fall back to exponential backoff } diff --git a/open-sse/executors/gemini-cli.ts b/open-sse/executors/gemini-cli.ts index 3c8298a0b6..228b00b38c 100644 --- a/open-sse/executors/gemini-cli.ts +++ b/open-sse/executors/gemini-cli.ts @@ -1,5 +1,8 @@ import { BaseExecutor } from "./base.ts"; import { PROVIDERS, OAUTH_ENDPOINTS } from "../config/constants.ts"; +import { geminiCLIUserAgent, googApiClientHeader } from "../services/antigravityHeaders.ts"; +import { scrubProxyAndFingerprintHeaders } from "../services/antigravityHeaderScrub.ts"; +import { obfuscateSensitiveWords } from "../services/antigravityObfuscation.ts"; const LOAD_CODE_ASSIST_URL = "https://cloudcode-pa.googleapis.com/v1internal:loadCodeAssist"; const PROJECT_TTL_MS = 30_000; // 30 seconds — matches native Gemini CLI @@ -22,19 +25,20 @@ export class GeminiCLIExecutor extends BaseExecutor { } buildHeaders(credentials, stream = true) { - return { + const raw = { "Content-Type": "application/json", Authorization: `Bearer ${credentials.accessToken}`, - // Fingerprint headers matching native GeminiCLI client (prevents upstream rejection) - "User-Agent": "GeminiCLI/0.31.0/unknown (linux; x64)", - "X-Goog-Api-Client": "google-genai-sdk/1.41.0 gl-node/v22.19.0", + // Dynamic headers matching native GeminiCLI client + "User-Agent": geminiCLIUserAgent(this._currentModel || "unknown"), + "X-Goog-Api-Client": googApiClientHeader(), ...(stream && { Accept: "text/event-stream" }), - // NOTE: x-goog-user-project removed — the stored projectId can become stale for - // free-tier accounts, causing 403 "Cloud Code Private API has not been used in - // project X". The API resolves the correct project from the OAuth token alone. }; + return scrubProxyAndFingerprintHeaders(raw); } + // Track current model for dynamic UA (set by transformRequest) + private _currentModel = "unknown"; + /** * Fetch the current cloudaicompanionProject via loadCodeAssist API. * Native Gemini CLI refreshes this every 30 seconds — OmniRoute stores it once @@ -134,15 +138,29 @@ export class GeminiCLIExecutor extends BaseExecutor { } async transformRequest(model, body, stream, credentials) { + // Track model for dynamic User-Agent + this._currentModel = model || "unknown"; + // Refresh the project ID via loadCodeAssist (cached for 30s). - // The translator builds the envelope with the stale stored projectId — - // we replace it here with the fresh one before sending to the API. if (body && typeof body === "object" && body.request && credentials.accessToken) { const freshProject = await this.refreshProject(credentials.accessToken); if (freshProject) { body.project = freshProject; } - // If refresh failed, keep the stale projectId as a best-effort fallback + + // Obfuscate sensitive client names in user content + const contents = body.request?.contents; + if (Array.isArray(contents)) { + for (const msg of contents) { + if (Array.isArray(msg.parts)) { + for (const part of msg.parts) { + if (typeof part.text === "string") { + part.text = obfuscateSensitiveWords(part.text); + } + } + } + } + } } return body; } diff --git a/open-sse/services/antigravity429Engine.ts b/open-sse/services/antigravity429Engine.ts new file mode 100644 index 0000000000..f1b58d7bc0 --- /dev/null +++ b/open-sse/services/antigravity429Engine.ts @@ -0,0 +1,173 @@ +/** + * Antigravity 429 classification and retry decision engine. + * + * CLIProxyAPI classifies 429 responses into 4 categories and makes nuanced + * retry decisions for each. OmniRoute previously had only 2 categories + * (free tier vs RPM). This module brings full parity. + * + * Categories: + * - unknown: Generic 429, exponential backoff + * - rate_limited: Per-minute rate limit, short backoff + same auth retry + * - quota_exhausted: Daily/plan quota gone, switch auth or long cooldown + * - soft_rate_limit: Temporary burst limit, instant retry + * + * Decisions: + * - soft_retry: Wait briefly, retry same auth + * - instant_retry_same_auth: Retry immediately on same auth + * - short_cooldown_switch_auth: 5min cooldown, try next account + * - full_quota_exhausted: 24h cooldown, skip this account + */ + +export type Category = "unknown" | "rate_limited" | "quota_exhausted" | "soft_rate_limit"; + +export type DecisionKind = + | "soft_retry" + | "instant_retry_same_auth" + | "short_cooldown_switch_auth" + | "full_quota_exhausted"; + +export interface Decision { + kind: DecisionKind; + retryAfterMs: number | null; + reason: string; +} + +const QUOTA_EXHAUSTED_KEYWORDS = ["quota_exhausted", "quota exhausted"]; + +const CREDITS_EXHAUSTED_KEYWORDS = [ + "google_one_ai", + "insufficient credit", + "insufficient credits", + "not enough credit", + "not enough credits", + "credit exhausted", + "credits exhausted", + "credit balance", + "minimumcreditamountforusage", + "minimum credit amount for usage", + "minimum credit", + "resource has been exhausted", +]; + +const SHORT_COOLDOWN_MS = 5 * 60 * 1000; // 5 minutes +const INSTANT_RETRY_THRESHOLD_MS = 3 * 1000; // 3 seconds +const FULL_QUOTA_COOLDOWN_MS = 24 * 60 * 60 * 1000; // 24 hours + +export function classify429(errorMessage: string): Category { + const lower = (errorMessage || "").toLowerCase(); + + // Check for quota exhaustion first (most specific) + for (const kw of QUOTA_EXHAUSTED_KEYWORDS) { + if (lower.includes(kw)) return "quota_exhausted"; + } + + // Check for credits exhaustion (also quota-related) + for (const kw of CREDITS_EXHAUSTED_KEYWORDS) { + if (lower.includes(kw)) return "quota_exhausted"; + } + + // Check for RPM/rate limit indicators + if ( + lower.includes("per minute") || + lower.includes("rpm") || + lower.includes("rate limit") || + lower.includes("rate_limit") || + lower.includes("too many requests") + ) { + return "rate_limited"; + } + + // Check for free tier exhaustion + if ( + lower.includes("free tier") || + lower.includes("daily limit") || + lower.includes("exhausted your capacity") + ) { + return "quota_exhausted"; + } + + // Check for soft/burst limits + if (lower.includes("try again") || lower.includes("temporarily")) { + return "soft_rate_limit"; + } + + return "unknown"; +} + +export function decide429(category: Category, retryAfterMs: number | null): Decision { + switch (category) { + case "soft_rate_limit": + return { + kind: + retryAfterMs && retryAfterMs <= INSTANT_RETRY_THRESHOLD_MS + ? "instant_retry_same_auth" + : "soft_retry", + retryAfterMs: retryAfterMs ?? 2000, + reason: "Soft rate limit — brief backoff", + }; + + case "rate_limited": + return { + kind: + retryAfterMs && retryAfterMs <= SHORT_COOLDOWN_MS + ? "soft_retry" + : "short_cooldown_switch_auth", + retryAfterMs: retryAfterMs ?? 60_000, + reason: "RPM rate limit — switch auth if cooldown is long", + }; + + case "quota_exhausted": + return { + kind: "full_quota_exhausted", + retryAfterMs: retryAfterMs ?? FULL_QUOTA_COOLDOWN_MS, + reason: "Quota exhausted — skip this account", + }; + + default: + return { + kind: "soft_retry", + retryAfterMs: retryAfterMs ?? 5000, + reason: "Unknown 429 — generic backoff", + }; + } +} + +/** + * Track credits failure state per auth key. + * Auto-disables after repeated failures with 5h cooldown. + */ +const creditsFailureMap = new Map< + string, + { + count: number; + disabledUntil: number; + } +>(); + +const CREDITS_DISABLE_THRESHOLD = 3; +const CREDITS_COOLDOWN_MS = 5 * 60 * 60 * 1000; // 5 hours + +export function recordCreditsFailure(authKey: string): boolean { + const state = creditsFailureMap.get(authKey) ?? { count: 0, disabledUntil: 0 }; + state.count++; + + if (state.count >= CREDITS_DISABLE_THRESHOLD) { + state.disabledUntil = Date.now() + CREDITS_COOLDOWN_MS; + creditsFailureMap.set(authKey, state); + return true; // disabled + } + + creditsFailureMap.set(authKey, state); + return false; +} + +export function isCreditsDisabled(authKey: string): boolean { + const state = creditsFailureMap.get(authKey); + if (!state) return false; + if (state.disabledUntil > Date.now()) return true; + // Cooldown expired, reset + creditsFailureMap.delete(authKey); + return false; +} + +export { SHORT_COOLDOWN_MS, FULL_QUOTA_COOLDOWN_MS }; diff --git a/open-sse/services/antigravityCredits.ts b/open-sse/services/antigravityCredits.ts new file mode 100644 index 0000000000..31cd744983 --- /dev/null +++ b/open-sse/services/antigravityCredits.ts @@ -0,0 +1,42 @@ +/** + * Google One AI credits injection for Antigravity. + * + * When Antigravity returns a quota_exhausted 429, CLIProxyAPI retries the + * request with `enabledCreditTypes: ["GOOGLE_ONE_AI"]` injected into the + * body. This uses the user's Google One AI credit balance for the retry, + * which is often available on Pro accounts. + * + * Based on CLIProxyAPI's antigravity_executor.go line 268. + */ + +import { isCreditsDisabled, recordCreditsFailure } from "./antigravity429Engine.ts"; + +/** + * Inject enabledCreditTypes into the request body for a credits retry. + * Returns a new body object with the field added. + */ +export function injectCreditsField(body: Record): Record { + return { + ...body, + enabledCreditTypes: ["GOOGLE_ONE_AI"], + }; +} + +/** + * Determine if a credits retry should be attempted for this auth key. + * Returns false if credits are disabled (too many failures) or if the + * config flag is off. + */ +export function shouldRetryWithCredits(authKey: string, creditsEnabled: boolean): boolean { + if (!creditsEnabled) return false; + if (isCreditsDisabled(authKey)) return false; + return true; +} + +/** + * Handle a credits retry failure. Tracks the failure and returns + * true if credits are now disabled for this auth key. + */ +export function handleCreditsFailure(authKey: string): boolean { + return recordCreditsFailure(authKey); +} diff --git a/open-sse/services/antigravityHeaderScrub.ts b/open-sse/services/antigravityHeaderScrub.ts new file mode 100644 index 0000000000..5ecdd91627 --- /dev/null +++ b/open-sse/services/antigravityHeaderScrub.ts @@ -0,0 +1,62 @@ +/** + * Antigravity header scrubbing. + * + * Real Antigravity is a Node.js app. Its outbound HTTP requests never include + * proxy tracing headers, Stainless SDK headers, or Chromium Sec-Ch-* headers. + * Sending any of these reveals the request came through a third-party proxy. + * + * Based on CLIProxyAPI's ScrubProxyAndFingerprintHeaders (misc/header_utils.go). + */ + +const HEADERS_TO_REMOVE = [ + // Proxy tracing + "x-forwarded-for", + "x-forwarded-host", + "x-forwarded-proto", + "x-forwarded-port", + "x-real-ip", + "forwarded", + "via", + // Client identity (Stainless SDK — Claude Code specific, not Antigravity) + "x-title", + "x-stainless-lang", + "x-stainless-package-version", + "x-stainless-os", + "x-stainless-arch", + "x-stainless-runtime", + "x-stainless-runtime-version", + "x-stainless-timeout", + "x-stainless-retry-count", + "x-stainless-helper-method", + "http-referer", + "referer", + // Browser / Chromium fingerprint (Electron clients, NOT Node.js) + "sec-ch-ua", + "sec-ch-ua-mobile", + "sec-ch-ua-platform", + "sec-fetch-mode", + "sec-fetch-site", + "sec-fetch-dest", + "priority", + // Encoding: Antigravity (Node.js) sends "gzip, deflate, br" by default; + // Electron clients add "zstd" which is a fingerprint mismatch. + "accept-encoding", +]; + +/** + * Remove headers that reveal proxy infrastructure or non-native client identity + * from an outgoing request to Antigravity's upstream API. + */ +export function scrubProxyAndFingerprintHeaders( + headers: Record +): Record { + const cleaned: Record = {}; + for (const [key, value] of Object.entries(headers)) { + if (!HEADERS_TO_REMOVE.includes(key.toLowerCase())) { + cleaned[key] = value; + } + } + // Set the standard Node.js accept-encoding + cleaned["Accept-Encoding"] = "gzip, deflate, br"; + return cleaned; +} diff --git a/open-sse/services/antigravityHeaders.ts b/open-sse/services/antigravityHeaders.ts new file mode 100644 index 0000000000..d7adf5e44d --- /dev/null +++ b/open-sse/services/antigravityHeaders.ts @@ -0,0 +1,71 @@ +import os from "node:os"; + +/** + * Antigravity and Gemini CLI header utilities. + * + * Generates User-Agent strings and API client headers that match + * the real Antigravity and Gemini CLI binaries. + * + * Based on CLIProxyAPI's misc/header_utils.go. + */ + +const ANTIGRAVITY_VERSION = "1.21.9"; +const GEMINI_CLI_VERSION = "0.31.0"; +const GEMINI_SDK_VERSION = "1.41.0"; +const NODE_VERSION = "v22.19.0"; + +function getPlatform(): string { + const p = os.platform(); + switch (p) { + case "win32": + return "win32"; + case "darwin": + return "darwin"; + default: + return p; // "linux", etc. + } +} + +function getArch(): string { + const a = os.arch(); + switch (a) { + case "x64": + return "x64"; + case "ia32": + return "x86"; + case "arm64": + return "arm64"; + default: + return a; + } +} + +/** + * Antigravity User-Agent: "antigravity/VERSION darwin/arm64" + * + * Always claims darwin/arm64 regardless of actual server OS. + * Real Antigravity is a macOS desktop tool — most users are on macOS. + * Claiming linux/amd64 from a datacenter IP is MORE suspicious than + * darwin/arm64. Matches CLIProxyAPI's proven production behavior. + */ +export function antigravityUserAgent(): string { + return `antigravity/${ANTIGRAVITY_VERSION} darwin/arm64`; +} + +/** + * Gemini CLI User-Agent: "GeminiCLI/VERSION/MODEL (OS; ARCH)" + * Example: "GeminiCLI/0.31.0/gemini-3-flash (darwin; arm64)" + */ +export function geminiCLIUserAgent(model: string): string { + return `GeminiCLI/${GEMINI_CLI_VERSION}/${model || "unknown"} (${getPlatform()}; ${getArch()})`; +} + +/** + * X-Goog-Api-Client header value matching the real Gemini SDK. + * Example: "google-genai-sdk/1.41.0 gl-node/v22.19.0" + */ +export function googApiClientHeader(): string { + return `google-genai-sdk/${GEMINI_SDK_VERSION} gl-node/${NODE_VERSION}`; +} + +export { ANTIGRAVITY_VERSION, GEMINI_CLI_VERSION, GEMINI_SDK_VERSION }; diff --git a/open-sse/services/antigravityObfuscation.ts b/open-sse/services/antigravityObfuscation.ts new file mode 100644 index 0000000000..63ebf998c7 --- /dev/null +++ b/open-sse/services/antigravityObfuscation.ts @@ -0,0 +1,50 @@ +/** + * Sensitive word obfuscation for Antigravity requests. + * + * Obfuscates client tool names (OpenCode, Cursor, Claude Code, etc.) using + * zero-width joiners so Google's backend can't grep for them in request logs. + * Matching ZeroGravity's ZEROGRAVITY_SENSITIVE_WORDS and CLIProxyAPI's cloak system. + */ + +const ZWJ = "\u200d"; + +const DEFAULT_WORDS = [ + "opencode", + "open-code", + "cline", + "roo-cline", + "roo_cline", + "cursor", + "windsurf", + "aider", + "continue.dev", + "copilot", + "avante", + "codecompanion", + "claude code", + "claude-code", + "kilo code", + "kilocode", + "omniroute", +]; + +let words = [...DEFAULT_WORDS]; + +export function setAntigravitySensitiveWords(w: string[]): void { + words = w.length > 0 ? w : [...DEFAULT_WORDS]; +} + +function escapeRegex(str: string): string { + return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +export function obfuscateSensitiveWords(text: string): string { + if (!text || words.length === 0) return text; + let result = text; + for (const word of words) { + if (!word) continue; + const regex = new RegExp(escapeRegex(word), "gi"); + result = result.replace(regex, (m) => (m.length <= 1 ? m : m[0] + ZWJ + m.slice(1))); + } + return result; +}