diff --git a/open-sse/handlers/chatCore/providerExecutionPipeline.ts b/open-sse/handlers/chatCore/providerExecutionPipeline.ts index b058d8b54c..6a5a1c58be 100644 --- a/open-sse/handlers/chatCore/providerExecutionPipeline.ts +++ b/open-sse/handlers/chatCore/providerExecutionPipeline.ts @@ -142,6 +142,44 @@ function retryAfterMsFrom(attempt: ChatCoreExecutorResult): number | null { return parsed * 1000; } +/** + * Feed the runtime rate limiter from a non-2xx upstream attempt. + * + * Order matters and mirrors the chatCore error path: headers FIRST (a 429 evicts + * the cached limiter so the body can materialize a fresh one), body SECOND (the + * body-embedded retry-after drains that fresh reservoir). Inverting them throws + * the drain away. + * + * The body is read through `response.clone()` — never the original stream. This + * is a shared streaming path, so consuming `attempt.response` here would silently + * break passthrough and SSE; `toOutcome` below drains the same way. + * + * Both hooks are best-effort: rate-limit learning must never fail the request. + */ +async function recordUpstreamRateLimit( + state: PipelineStateHooks, + provider: string, + connectionId: string, + model: string, + attempt: ChatCoreExecutorResult +): Promise { + if (!connectionId) return; + const status = attempt.response.status; + try { + state.recordRateLimitHeaders(provider, connectionId, attempt.response.headers, status, model); + } catch { + // best-effort + } + try { + const text = await attempt.response.clone().text(); + // parseRetryAfterFromBody JSON.parses a string and falls back to "unknown" + // on non-JSON, so the raw text is the safest thing to hand over. + if (text) state.recordRateLimitBody(provider, connectionId, text, status, model); + } catch { + // Body already consumed/unreadable — the header signal above still applied. + } +} + function leaseMismatch(model: string, connectionId: string): ProviderExecutionOutcome { const result = createErrorResult( LEASE_MISMATCH_STATUS, @@ -189,11 +227,21 @@ async function toOutcome( try { // clone() is the drain. sendProviderAttempt must not cancel() a streaming // non-2xx body before we get here (BYOP 422 / Codex 429 Retry-After). - body = JSON.parse(await attempt.response.clone().text()); - const err = (body as { error?: { message?: unknown } } | null)?.error; - if (err && typeof err.message === "string" && err.message) message = err.message; + const text = await attempt.response.clone().text(); + try { + body = JSON.parse(text); + const err = (body as { error?: { message?: unknown } } | null)?.error; + if (err && typeof err.message === "string" && err.message) message = err.message; + } catch { + // Non-JSON upstream body (plain-text 429, HTML error page). parseUpstreamError + // — the pre-pipeline path this replaced — surfaces the raw text as the message; + // collapsing it to statusText ("upstream error") hides what the provider said. + // buildErrorBody()/sanitizeErrorMessage() still sanitize and truncate it before + // it reaches any response body (Hard Rule #12). + if (text.trim()) message = text; + } } catch { - // keep statusText + // Body unreadable (already consumed) — keep statusText. } const restatement = applyStatusRestatement({ provider, @@ -286,6 +334,19 @@ export async function runProviderExecutionPipeline( ); } + // Teach the runtime limiter BEFORE any recovery branch rotates or retries: + // the 429 belongs to the connection that just took it. chatCore's own + // updateFromHeaders/updateFromResponseBody pair only runs on the streaming + // leg — the non-streaming leg returns this pipeline's error outcome straight + // to the caller, so without this the reservoir was never drained (#12945). + await recordUpstreamRateLimit( + state, + target.provider, + currentConnectionId(connection), + wire.currentModel, + attempt + ); + const isolateProbe = await state.isolateProbeFailures(); const canRotateAccount = policy.allowAccountRotation && !isolateProbe; diff --git a/open-sse/handlers/imageGeneration.ts b/open-sse/handlers/imageGeneration.ts index 62a9488565..8c3596e14d 100644 --- a/open-sse/handlers/imageGeneration.ts +++ b/open-sse/handlers/imageGeneration.ts @@ -2778,6 +2778,40 @@ export function saveImageSuccessResult({ }; } +/** + * Render an arbitrary `error` value as a call-log string. + * + * `saveImageErrorResult` takes `error: unknown`, and the Codex fan-out forwards + * whatever `sanitizeImageProviderError()` produced — i.e. the output of + * `sanitizeUpstreamDetails()`, which builds every object with + * `Object.create(null)` on purpose (#12506) so a hostile upstream key such as + * `__proto__` or `constructor` can never reach a real prototype. That object + * therefore has NO `toString`/`Symbol.toPrimitive`, so a bare `String(value)` + * throws `TypeError: Cannot convert object to primitive value` and turned every + * Codex image failure into an unhandled crash instead of the sanitized error. + * The null prototype is the correct behavior at the source, so the sink is what + * has to be total: serialize objects structurally (the same way the Antigravity + * branch already logs its sanitized payload) and keep `String()` semantics for + * everything else. + */ +function stringifyImageErrorForLog(value: unknown): string { + if (typeof value === "string") return value; + if (value instanceof Error) return `${value.name}: ${value.message}`; + if (value !== null && typeof value === "object") { + try { + const serialized = JSON.stringify(value); + if (typeof serialized === "string") return serialized; + } catch { + // Circular graph or a throwing toJSON — fall through to String(). + } + } + try { + return String(value); + } catch { + return "[unserializable error]"; + } +} + export function saveImageErrorResult({ provider, model, @@ -2810,7 +2844,7 @@ export function saveImageErrorResult({ model: `${provider}/${model}`, provider, duration: Date.now() - startTime, - error: typeof error === "string" ? error.slice(0, 500) : String(error).slice(0, 500), + error: stringifyImageErrorForLog(error).slice(0, 500), requestBody, }).catch(() => {}); diff --git a/open-sse/utils/error.ts b/open-sse/utils/error.ts index 0f986694be..fb72dcdb46 100644 --- a/open-sse/utils/error.ts +++ b/open-sse/utils/error.ts @@ -52,9 +52,9 @@ const SAFE_PUBLIC_ERROR_IDENTIFIERS = new Set([ "all_targets_skipped", "antigravity_pre_response_timeout", "api_error", + "auth_error", "authentication_error", "authentication_required", - "auth_error", "bad_gateway", "bad_request", "bedrock_stream_error", @@ -68,36 +68,44 @@ const SAFE_PUBLIC_ERROR_IDENTIFIERS = new Set([ "cf_mitigated_challenge", "chat_admission_busy", "chat_history_too_large", - "chatgpt_web_codex_error", - "chatgpt_web_codex_turn_failed", "chatgpt_session_expired", "chatgpt_submission_ambiguous", "chatgpt_submitted_turn_failed", "chatgpt_subscription_unavailable", + "chatgpt_web_codex_error", + "chatgpt_web_codex_turn_failed", + "chipotle_error", + "claude_web_protocol_error", + "cli_not_found", "client_cancelled", "client_closed_request", "client_disconnected", - "cli_not_found", "cloudflare_challenge", "cloudflare_or_bot", - "codex_app_server_unconfigured", "codex_app_server_turn_failed", + "codex_app_server_unconfigured", + "codex_scope_cooldown", + "codex_tool_timeout", "combo_target_timeout", "combo_timeout", "compaction_control_unavailable", "compaction_handoff_failed", + "compaction_source_unavailable", + "connection_cooldown", + "connection_error", + "connection_not_allowed", + "connection_terminal_status", + "connection_unavailable", "connector_error", "connector_not_found", - "connection_error", "context_length_exceeded", "context_window", - "chipotle_error", "devin_agentic_error", "devin_cli_error", "devin_desktop_error", "devin_internal_tool_execution", - "duplicate_tool_use_id", "direct_response_start_timeout", + "duplicate_tool_use_id", "eai_again", "econnrefused", "econnreset", @@ -105,25 +113,37 @@ const SAFE_PUBLIC_ERROR_IDENTIFIERS = new Set([ "empty_content", "empty_messages", "empty_response", - "executor_contract_violation", "error", "etimedout", + "executor_contract_violation", "executor_error", + "extract_failed", "feature_disabled", + "file_too_large", "gateway_timeout", - "gemini_tpm_exhausted", "gcp_project_required", + "gemini_tpm_exhausted", "grok_error", - "insufficient_quota", + "heap_pressure", + "huggingchat_generation_error", "incompatible_reasoning_effort", + "inspector_error", + "insufficient_quota", "internal_server_error", "invalid_acp_frame", "invalid_acp_upstream", "invalid_api_key", + "invalid_authentication", + "invalid_connection_id", + "invalid_grant", + "invalid_json", "invalid_kiro_tool_call", - "invalid_request", - "invalid_request_error", + "invalid_output_schema", "invalid_previous_response_binding", + "invalid_provider", + "invalid_request", + "invalid_request_body", + "invalid_request_error", "invalid_tool_arguments", "invalid_tool_choice", "invalid_tool_json", @@ -139,33 +159,36 @@ const SAFE_PUBLIC_ERROR_IDENTIFIERS = new Set([ "lease_content_type_required", "lease_context_invalid", "lease_context_required", + "lease_eligibility_unavailable", "lease_error", "lease_fence_stale", "lease_key_configuration_invalid", "lease_key_policy_invalid", "lease_model_invalid", "lease_no_eligible_connection", - "lmarena_error", "lease_required", "lease_scope_required", "lease_service_unavailable", - "lease_eligibility_unavailable", "lease_unsupported_route", "lease_unsupported_transport", + "lmarena_error", "message_limit", - "missing_credits", "meta_ai_empty_response", "meta_ai_mode_switch_failed", "meta_ai_warmup_failed", "meta_ai_ws_error", + "missing_authorization", + "missing_cookie", + "missing_credentials", + "missing_credits", + "missing_project_id", + "missing_session_id", "missing_tool_name", "missing_tool_use_id", "mixed_tool_narrative", - "missing_authorization", - "missing_cookie", - "missing_project_id", - "missing_credentials", - "missing_session_id", + "model_cooldown", + "model_excluded", + "model_lockout", "model_not_found", "model_not_supported", "model_shutdown", @@ -173,104 +196,129 @@ const SAFE_PUBLIC_ERROR_IDENTIFIERS = new Set([ "multiple_tool_requests", "native_codex_pinned_model_unavailable", "network_error", + "no_active_connection", "no_free_eligible_connection", + "no_local_login", + "no_refresh_token", "not_found", "oauth_missing_project_id", + "origin_rejected", "orphan_tool_result", "payload_too_large", "payment_required", + "peer_hop_limit_exceeded", + "peer_loop_detected", + "permission_denied", "permission_error", + "pplx_error", "premium_model_requires_key", + "previous_response_not_found", "prompt_attachment_integrity", + "provider_circuit_half_open", + "provider_circuit_open", + "provider_deprecated", "provider_error", "provider_retired", "provider_unavailable", - "pplx_error", - "proxy_unavailable", "proxy_family_unavailable", "proxy_request_failed", + "proxy_unavailable", "proxy_unreachable", "quota_exhausted", "quota_not_allocated", "quota_only", "rate_limit_error", - "rate_limit_execution_timeout", "rate_limit_exceeded", + "rate_limit_execution_timeout", + "rate_limit_longer_reached", "rate_limit_queue_full", "rate_limit_queue_timeout", "rate_limit_queue_wedged", - "rate_limit_longer_reached", "rate_limit_reached", "rate_limited", "reached_limit", + "read_failed", "relay_timeout", - "resource_pressure", - "resource_exhausted", "request_failed", + "resource_exhausted", + "resource_pressure", "risk_session_stale", - "server_error", "semaphore_queue_full", "semaphore_timeout", - "service_unavailable", + "server_error", + "server_is_overloaded", "service_not_running", + "service_unavailable", "session_expired", "session_pool_exhausted", "spawn_failed", - "stream_error", + "storage_encryption_stale", "stream_disconnected", "stream_early_eof", + "stream_error", "stream_idle_timeout", "stream_pipeline_error", "stream_readiness_timeout", "stream_terminated", "stream_timeout", - "storage_encryption_stale", "structure_limit", "structured_output", "structured_output_validation_failed", - "timeout_error", + "subscription_required", "timeout", - "token_limit_exceeded", - "token_required", - "tls_client_unavailable", + "timeout_error", "tls_circuit_open", + "tls_client_unavailable", "tls_fingerprint_failed", "tls_session_capacity", + "token_limit_exceeded", + "token_required", "tool_calling_not_supported", "tools", - "undeclared_historical_tool", + "uc_auth_error", + "uc_generation_failed", + "uc_message_limit_exceeded", + "uc_paywall_exceeded", + "uc_rate_limit_exceeded", + "uc_timeout", + "uc_upstream_error", + "unauthorized", + "unavailable", "und_err_body_timeout", "und_err_connect_timeout", "und_err_headers_timeout", "und_err_socket", - "unexpected_acp_response", + "undeclared_historical_tool", "unexecuted_tool_intent", - "unavailable", + "unexpected_acp_response", "unknown_devin_model", + "unknown_route", "unknown_tool", - "unverified_codex_client", "unsafe_devin_home", "unsupported_acp_version", "unsupported_content_block", "unsupported_control_for_provider", "unsupported_endpoint", + "unsupported_feature", "unsupported_image_block", + "unsupported_media_type", "unsupported_role", + "unsupported_runtime", "unsupported_system_block", - "upstream_error", + "unverified_codex_client", + "upgrade_required", "upstream_access_denied", "upstream_auth_error", "upstream_empty_response", - "upstream_response_failed", - "upstream_response_error", - "upstream_server_error", + "upstream_error", "upstream_protocol_error", + "upstream_response_error", + "upstream_response_failed", + "upstream_server_error", "upstream_timeout", "upstream_websocket_connect_failed", "upstream_websocket_error", "usage_limit_reached", - "unsupported_feature", - "unsupported_runtime", "video_artifact_content_type_invalid", "video_artifact_download_failed", "video_artifact_not_ready", @@ -280,8 +328,9 @@ const SAFE_PUBLIC_ERROR_IDENTIFIERS = new Set([ "video_artifact_url_blocked", "video_artifact_url_invalid", "vision", - "claude_web_protocol_error", "wreq_unavailable", + "writes_disabled", + "zai_stream_error", ]); function isSafePublicErrorIdentifier(value: string): boolean { diff --git a/open-sse/utils/errorPathRedaction.ts b/open-sse/utils/errorPathRedaction.ts index 2b372af583..24cb50091a 100644 --- a/open-sse/utils/errorPathRedaction.ts +++ b/open-sse/utils/errorPathRedaction.ts @@ -464,8 +464,8 @@ function findUnquotedPathEnd( let hasFilesystemEvidence = false; let hasUnresolvedFragments = false; - const resolveEndpoint = (): number => { - if (hasUnresolvedFragments) { + const resolveEndpoint = (ignoreAmbiguity = false): number => { + if (hasUnresolvedFragments && !ignoreAmbiguity) { return failClosedAmbiguity || hasFilesystemEvidence ? value.length : -1; } if (resolvedExtensionEnd >= 0) return resolvedExtensionEnd; @@ -529,8 +529,16 @@ function findUnquotedPathEnd( let nextTokenStart = tokenEnd; while (nextTokenStart < value.length && isWhitespace(value[nextTokenStart])) nextTokenStart++; if (nextTokenStart >= value.length) return resolveEndpoint(); + // A redaction marker ends the span: whatever follows was already made safe + // by the credential pass, and swallowing it would erase that evidence. + if (startsRedactedToken(value, nextTokenStart)) return resolveEndpoint(true); if (isSyntacticallyAbsolutePathAt(value, nextTokenStart)) { - const endpoint = resolveEndpoint(); + // A route-shielded upcoming span (e.g. "POST /v1/foo") is never + // filesystem-sensitive by design — see hasRouteContextBefore. Its mere + // presence must not force ambiguous prose in between (like "Use POST") + // to fail closed and swallow past it into the shielded route and + // beyond; resolve with whatever evidence was already gathered instead. + const endpoint = resolveEndpoint(hasRouteContextBefore(value, nextTokenStart)); if (endpoint >= 0) return endpoint; return acceptEndpointBeforeAnotherAbsolute ? lastPathTokenEnd : -1; } @@ -890,6 +898,22 @@ export function stripErrorStackTail(value: string): string { * API routes, and punctuation around determinable endpoints. Unequivocal * filesystem prefixes fail closed when an unquoted endpoint is ambiguous. */ +/** + * `[REDACTED]` is the marker an earlier sanitizer pass already wrote over a + * credential. It is never part of a filesystem path, and a path span that grows + * across it costs the operator the one piece of evidence that pass left behind: + * "TLS request failed at /srv/…/client.ts:44:9 access_token=[REDACTED]" + * collapsed to a bare "", hiding *which* credential leaked. + */ +const REDACTION_MARKER = "[REDACTED]"; + +/** True when the token starting at `index` carries a redaction marker. */ +function startsRedactedToken(value: string, index: number): boolean { + let end = index; + while (end < value.length && !isWhitespace(value[end])) end++; + return value.slice(index, end).includes(REDACTION_MARKER); +} + export function redactErrorPaths(value: string): string { const quotedPathsRedacted = redactQuotedAbsolutePaths(value); const pathSpansRedacted = redactUnquotedAbsolutePathSpans(quotedPathsRedacted); diff --git a/open-sse/utils/errorSanitization.ts b/open-sse/utils/errorSanitization.ts index 1b7601d4ee..a8f1c18517 100644 --- a/open-sse/utils/errorSanitization.ts +++ b/open-sse/utils/errorSanitization.ts @@ -689,7 +689,13 @@ function sanitizeErrorMessageWithStackPolicy( // Raw URI credentials must be projected before the path tokenizer consumes // the URI tail; Windows path evidence still stays intact until after this // credential-only pass and is redacted before escape normalization. - str = redactKnownCredentialPatterns(redactSensitiveUrlCredentials(stripStackTail(str))); + // Labeled assignments (access_token=…, api_key=…) are projected here too, for + // the same reason as raw URI credentials: the path tokenizer would otherwise + // absorb "…/client.ts:44:9 access_token=secret" whole and the public message + // would lose the credential marker along with the path. + str = redactLabeledCredentialAssignments( + redactKnownCredentialPatterns(redactSensitiveUrlCredentials(stripStackTail(str))) + ); str = redactErrorPaths(str); str = redactSensitiveErrorText(str); str = truncateSanitizedErrorText(str); diff --git a/src/lib/db/core.ts b/src/lib/db/core.ts index a69db56e64..b0febbef2d 100644 --- a/src/lib/db/core.ts +++ b/src/lib/db/core.ts @@ -519,6 +519,29 @@ const SCHEMA_SQL = ` CREATE INDEX IF NOT EXISTS idx_quota_snapshots_created_at ON quota_snapshots(created_at); `; +// `CREATE TABLE IF NOT EXISTS` is a no-op against a legacy database that already owns the +// table with an older column set — but the `CREATE INDEX` statements that follow it are +// not: they still reference columns the ensure*Columns() healers have yet to backfill, so +// running the whole schema in one exec aborts startup with "no such column". That is how +// the composite idx_cl_request_provider index (#12832) broke booting on a pre-007 +// `call_logs` lineage. Split the inline schema so the boot order can be: create tables → +// heal legacy columns → create indexes. +function splitSchemaStatements(schemaSql: string): { tables: string; indexes: string } { + const tables: string[] = []; + const indexes: string[] = []; + for (const rawStatement of schemaSql.split(";")) { + const statement = rawStatement.trim(); + if (!statement) continue; + // Classify on the first SQL keyword, ignoring any leading `--` comment lines. + const sql = statement.replace(/^(?:[ \t]*--[^\n]*\n)+/, "").trimStart(); + (/^CREATE\s+(?:UNIQUE\s+)?INDEX\b/i.test(sql) ? indexes : tables).push(`${statement};`); + } + return { tables: tables.join("\n"), indexes: indexes.join("\n") }; +} + +const { tables: SCHEMA_TABLES_SQL, indexes: SCHEMA_INDEXES_SQL } = + splitSchemaStatements(SCHEMA_SQL); + // ──────────────── Singleton DB Instance ──────────────── // Use globalThis to survive Next.js dev HMR module re-evaluation. // Module-level `let` resets on every webpack recompile, causing connection leaks. @@ -1254,10 +1277,15 @@ export function getDbInstance(): SqliteDatabase { db.pragma("synchronous = NORMAL"); db.pragma(`cache_size = -${DEFAULT_DATABASE_SETTINGS.optimization.cacheSize}`); db.pragma("temp_store = MEMORY"); - db.exec(SCHEMA_SQL); + // Tables first, then the legacy-column healers, and only then the indexes: an upgraded + // database can already own call_logs/usage_history/provider_connections with an older + // column set, where the CREATE TABLE is a no-op but the indexes still reference columns + // the healers below are the ones adding. + db.exec(SCHEMA_TABLES_SQL); ensureProviderConnectionsColumns(db); ensureUsageHistoryColumns(db); ensureCallLogsColumns(db); + db.exec(SCHEMA_INDEXES_SQL); // ── Versioned Migrations ── // Auto-seed 001 as applied (the inline SCHEMA_SQL already created these tables) diff --git a/tests/unit/error-message-sanitization.test.ts b/tests/unit/error-message-sanitization.test.ts index 8813e7ac71..21f989ef9e 100644 --- a/tests/unit/error-message-sanitization.test.ts +++ b/tests/unit/error-message-sanitization.test.ts @@ -239,6 +239,21 @@ test("sanitizeErrorMessage replaces absolute paths with ", async () => { assert.ok(out2.includes("")); }); +test("sanitizeErrorMessage does not swallow a shielded route hint that follows an earlier redacted path (#6457)", async () => { + // Regression: an unshielded route-looking span ("on /v1/chat/completions") + // followed by ambiguous prose ("Use POST") used to make the unquoted-path + // scanner fail closed all the way to the end of the string, deleting a + // second, legitimately-shielded route reference ("POST /v1/images/...") + // and everything after it instead of just redacting the first span. + const { sanitizeErrorMessage } = await import("../../open-sse/utils/error.ts"); + const input = + "Model 'x' is an image-generation model and cannot be used on /v1/chat/completions. Use POST /v1/images/generations instead."; + const out = sanitizeErrorMessage(input); + assert.match(out, /\/v1\/images\/generations/, "shielded route hint must survive"); + assert.match(out, /instead\.$/, "text after the shielded route hint must not be dropped"); + assert.ok(out.includes(""), "the earlier unshielded route span is still redacted"); +}); + test("sanitizeErrorMessage handles non-string inputs safely", async () => { const { sanitizeErrorMessage } = await import("../../open-sse/utils/error.ts"); assert.equal(sanitizeErrorMessage(undefined), ""); diff --git a/tests/unit/hard-session-lease-bypass-inventory.test.ts b/tests/unit/hard-session-lease-bypass-inventory.test.ts index 97e713a0c7..024972c2da 100644 --- a/tests/unit/hard-session-lease-bypass-inventory.test.ts +++ b/tests/unit/hard-session-lease-bypass-inventory.test.ts @@ -13,7 +13,13 @@ type BypassClass = "A" | "B" | "C"; const EXPECTED: Record> = { credential: { - "open-sse/handlers/chatCore.ts": 2, + // v3.8.51 #12867 (d6f315018): the two credential-resolution sites that used to + // live in chatCore.ts (codex 429 and antigravity 422 account rotation) were + // extracted into the provider execution pipeline. chatCore.ts now only hands + // `getProviderCredentials` across the seam as a dependency (a reference, not a + // call), so the two sites are inventoried at their new home — see the + // property-access branch in countCalls(). + "open-sse/handlers/chatCore/providerExecutionPipeline.ts": 2, "open-sse/services/imageCombo.ts": 1, "open-sse/services/speechCombo.ts": 1, "open-sse/services/videoCombo.ts": 2, @@ -89,7 +95,10 @@ const EXPECTED: Record> = { // v3.8.50 back-merge additions (f95b03d7): combo routing infra and the // volcengine-plan binding/auto-sync services query connections the same // way as their classified siblings. - "open-sse/services/combo.ts": 1, + // v3.8.51 #12746 (6b587d004) split executeTarget out of combo.ts; the + // persisted-cooldown gate's connection read moved here byte-identically + // (readConnectionForCooldownGate), so this is the same site, renamed. + "open-sse/services/combo/executeTargetGates.ts": 1, "open-sse/services/combo/providerWildcard.ts": 1, "open-sse/services/tokenRefresh.ts": 1, "src/lib/providers/volcPlanAutoSyncBackfill.ts": 1, @@ -127,6 +136,12 @@ const EXPECTED: Record> = { "src/app/api/translator/send/route.ts": 1, "src/app/api/translator/translate/route.ts": 1, "src/app/api/usage/call-logs/route.ts": 1, + // v3.8.51 #12805 (c042a5188): the reset-credit endpoint now serves codex and + // grok-cli, so it reads the connection once only to decide which handler runs + // (resolveResetCreditProvider). Read-only lookup behind requireManagementAuth; + // the handlers it delegates to carry the auxiliary-lease fence themselves. It + // never selects a connection to serve a request, so it stays class C. + "src/app/api/usage/codex-reset-credit/route.ts": 1, "src/app/api/usage/quota/route.ts": 1, "src/app/api/usage/utilization/route.ts": 1, "src/app/api/v1/vscode/[token]/api/tags/route.ts": 1, @@ -174,6 +189,10 @@ const EXPECTED: Record> = { "src/lib/usage/callLogs.ts": 1, "src/lib/usage/codexResetCredits.ts": 1, "src/lib/usage/comboScoringInspector.ts": 1, + // v3.8.51 #12805 (c042a5188): grok-cli sibling of codexResetCredits.ts, same + // shape — isConnectionUnavailableToAuxiliaryActivity() gates the lookup, so an + // ACTIVE exclusive lease defers redemption (409 exclusive_lease_active). + "src/lib/usage/grokResetCredits.ts": 1, "src/lib/usage/providerLimits.ts": 4, "src/lib/usage/resilienceExplain.ts": 1, "src/lib/usage/usageStats.ts": 1, @@ -212,10 +231,9 @@ const CLASSIFICATION: Record> = { [ "open-sse/handlers/autoComboCandidates.ts", "open-sse/handlers/chatCore.ts", - "open-sse/services/combo.ts", "open-sse/services/alibabaFreeTier.ts", "open-sse/services/alibabaFreeTierQuotaFetcher.ts", - "open-sse/services/combo.ts", + "open-sse/services/combo/executeTargetGates.ts", "open-sse/services/combo/providerWildcard.ts", "open-sse/services/tokenRefresh.ts", "src/app/api/translator/send/route.ts", @@ -224,6 +242,7 @@ const CLASSIFICATION: Record> = { "src/lib/providers/volcenginePlanBinding.ts", "src/lib/services/quotaAutoPing.ts", "src/lib/usage/codexResetCredits.ts", + "src/lib/usage/grokResetCredits.ts", "src/lib/usage/providerLimits.ts", "src/lib/vncSession/service.ts", "src/lib/warmupScheduler.ts", @@ -276,6 +295,18 @@ function countCalls(): Record> { ) { increment("connection"); } + } else if ( + ts.isPropertyAccessExpression(expression) && + (expression.name.text === "getProviderCredentials" || + expression.name.text === "getProviderCredentialsWithQuotaPreflight") + ) { + // Injected-dependency shape. #12867 moved codex/antigravity account + // rotation behind a seam: chatCore passes `getProviderCredentials` in and + // the provider execution pipeline calls it off its injected `connection` + // context. Counting bare identifier calls only would let an + // extract-to-a-seam refactor silently drop a credential-resolution site + // out of this inventory, which is exactly what this guard exists to catch. + increment("credential"); } else if ( ts.isPropertyAccessExpression(expression) && expression.name.text === "execute" && @@ -320,6 +351,7 @@ test("managed request surfaces are fenced centrally or rejected before independe "src/lib/api/modelTestRunner.ts", "src/lib/services/quotaAutoPing.ts", "src/lib/usage/codexResetCredits.ts", + "src/lib/usage/grokResetCredits.ts", "src/lib/vncSession/service.ts", "src/lib/warmupScheduler.ts", "src/shared/services/modelSyncScheduler.ts", @@ -336,7 +368,28 @@ test("managed request surfaces are fenced centrally or rejected before independe core, /assertManagedLeaseFence\(getExecutionConnectionId\(getExecutionCredentials\(\)\)\)/ ); - assert.match(core, /provider === "codex" &&\s*!managedLease/); + // #12867 (d6f315018) extracted codex 429 / antigravity 422 account rotation out + // of chatCore.ts into the provider execution pipeline. The managed-lease fence was + // NOT dropped — it now crosses the seam as `policy.allowAccountRotation`. Pin both + // ends so neither half can be weakened alone: chatCore must keep deriving the + // policy from `!managedLease` on both legs, and the pipeline must keep gating the + // codex rotation branch on it. (The antigravity 422 branch, which had no lease + // fence at all before the extract, is now gated by the same flag.) + const pipeline = fs.readFileSync( + path.join(REPO_ROOT, "open-sse/handlers/chatCore/providerExecutionPipeline.ts"), + "utf8" + ); + const rotationPolicySites = core.match( + /allowAccountRotation: !managedLease && comboStrategy !== "context-relay"/g + ); + assert.equal( + rotationPolicySites?.length, + 2, + "both the streaming and the non-streaming leg must derive account rotation from !managedLease" + ); + assert.match(pipeline, /const canRotateAccount = policy\.allowAccountRotation && !isolateProbe;/); + assert.match(pipeline, /canRotateAccount &&\s*target\.provider === "codex"/); + assert.match(pipeline, /canRotateAccount &&\s*target\.provider === "antigravity"/); assert.match(ws, /LEASE_UNSUPPORTED_TRANSPORT/); assert.match(internalKeys, /!k\.scopes\?\.includes\(EXCLUSIVE_LEASE_SCOPE\)/); for (const source of auxiliaryIsolationSources) { diff --git a/tests/unit/kiro-tool-call-validation.test.ts b/tests/unit/kiro-tool-call-validation.test.ts index f0e7c47f7f..392b2975b5 100644 --- a/tests/unit/kiro-tool-call-validation.test.ts +++ b/tests/unit/kiro-tool-call-validation.test.ts @@ -236,23 +236,45 @@ test("Kiro stream errors become Responses response.failed events", async () => { null, "kiro-model" ); - const writer = transform.writable.getWriter(); - const responseText = new Response(transform.readable).text(); + // Drive the transform the way production does — `response.body.pipeThrough(transform)` + // read chunk by chunk — instead of `new Response(transform.readable).text()`. + // createStreamFailureAborter forwards the translated failure event and then errors the + // controller on purpose, so a translated upstream error can never end as a clean, + // successful-looking stream (open-sse/utils/streamFailureBoundary.ts). `.text()` cannot + // observe that: it discards the forwarded bytes and rejects, and the abandoned + // rejection lands as an unhandledRejection after the test ends. A reader keeps the + // event that was already delivered and still sees the termination. + const upstream = new ReadableStream({ + start(controller) { + controller.enqueue( + textEncoder.encode( + `data: ${JSON.stringify({ + error: { + message: "Invalid Kiro tool_call payload: missing nested MCP tool name at input.name", + type: "invalid_request_error", + code: "invalid_kiro_tool_call", + }, + })}\n\n` + ) + ); + controller.close(); + }, + }); - await writer.write( - textEncoder.encode( - `data: ${JSON.stringify({ - error: { - message: "Invalid Kiro tool_call payload: missing nested MCP tool name at input.name", - type: "invalid_request_error", - code: "invalid_kiro_tool_call", - }, - })}\n\n` - ) - ); - await writer.close(); - const text = await responseText; + const reader = upstream.pipeThrough(transform).getReader(); + let text = ""; + let streamError: unknown = null; + try { + for (;;) { + const chunk = await reader.read(); + if (chunk.done) break; + text += new TextDecoder().decode(chunk.value); + } + } catch (caught) { + streamError = caught; + } + assert.ok(streamError, "a translated upstream error must terminate the stream"); assert.match(text, /event: response\.failed/); assert.match(text, /invalid_kiro_tool_call/); assert.match(text, /missing nested MCP tool name/); diff --git a/tests/unit/tunnel-routes-error-sanitization.test.ts b/tests/unit/tunnel-routes-error-sanitization.test.ts index 2643af5b01..439a700079 100644 --- a/tests/unit/tunnel-routes-error-sanitization.test.ts +++ b/tests/unit/tunnel-routes-error-sanitization.test.ts @@ -62,28 +62,33 @@ const LEAKS = [ message: "ENOENT: no such file or directory, open '/home/operator/.omniroute/data/tunnels.json'", secrets: ["/home/operator", "tunnels.json"], + sharedSanitizerCovers: true, }, { label: "binary path (no extension)", message: "spawn /usr/local/bin/cloudflared ENOENT", secrets: ["/usr/local/bin/cloudflared"], + sharedSanitizerCovers: true, }, { label: "tailscale auth key", message: "tailscale up failed: invalid key tskey-auth-kMn3Qz7RtY-9fVbXsPq2LdWc", secrets: ["tskey-auth-kMn3Qz7RtY-9fVbXsPq2LdWc"], + sharedSanitizerCovers: false, }, { label: "daemon state path", message: "Command failed: /opt/omniroute/bin/tailscaled --state=/var/lib/tailscale/tailscaled.state", secrets: ["/opt/omniroute/bin/tailscaled", "/var/lib/tailscale"], + sharedSanitizerCovers: true, }, { label: "windows config path", message: "listen EADDRINUSE: address already in use 0.0.0.0:41641 (config C:\\Users\\operator\\AppData\\omniroute\\ngrok.yml)", secrets: ["C:\\Users\\operator", "ngrok.yml"], + sharedSanitizerCovers: true, }, ] as const; @@ -101,18 +106,30 @@ async function withSilencedConsoleError(fn: () => T | Promise): Promise<[T } } -// ── Why a dedicated module: sanitizeErrorMessage does not cover these ─────── +// ── Why a dedicated module: sanitizeErrorMessage does not cover all of these ─ -test("sanitizeErrorMessage alone leaves every tunnel leak shape intact", () => { +test("sanitizeErrorMessage covers the path shapes and still misses the auth key", () => { + // #12506 taught the shared sanitizer to redact filesystem paths, so the four + // path-shaped leaks below are handled upstream now — a real improvement, and + // the reason this test no longer claims "every shape survives". The tailscale + // auth key is not path-shaped and is still echoed verbatim, which is why the + // routes must keep going through publicSafeTunnelError rather than trusting + // the shared sanitizer. Flip an entry's `sharedSanitizerCovers` the day that + // changes; never relax the public-body assertions below it. for (const leak of LEAKS) { const out = sanitizeErrorMessage(leak.message); const stillLeaks = leak.secrets.some((s) => out.includes(s)); - assert.ok( + assert.equal( stillLeaks, - `${leak.label}: sanitizeErrorMessage unexpectedly covers this now — if the ` + - `shared sanitizer grew to handle it, simplify publicSafeTunnelError accordingly. Got: ${out}` + !leak.sharedSanitizerCovers, + `${leak.label}: shared-sanitizer coverage changed — expected ` + + `${leak.sharedSanitizerCovers ? "covered" : "still leaking"}, got: ${out}` ); } + assert.ok( + LEAKS.some((leak) => !leak.sharedSanitizerCovers), + "publicSafeTunnelError would be redundant if the shared sanitizer covered every shape" + ); }); // ── The public-safe contract ───────────────────────────────────────────────