From c4eafaa26d2d211a73d4778a20348584c9c8e72f Mon Sep 17 00:00:00 2001 From: "Bob.Hou" Date: Fri, 11 Sep 2026 19:45:21 -0400 Subject: [PATCH] fix(auth): probe a pinned inactive connection after quota top-up (#13017) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The one-shot framing is what makes this safe: a pin is an explicit operator act, so taking the inactive row only for that request, with siblings out of the pool and a 60s per-connection storm gate, keeps the blast radius at one request. Dashboard deactivate staying off is the right carve-out. Reconciled against the tip after the batch landed: the `chatHelpers.ts` import block conflicted with `buildExhaustionOptions` (#12975) and both imports were kept. 41/41 across the probe suites afterwards. --- Validated in one consolidated worktree cut from `release/v3.8.51`, boarded with the other 19 PRs of this batch. Two in-batch conflicts, both additive and resolved by keeping each side: the `ENVIRONMENT.md` table (#13035 + #13011) and the `chatHelpers.ts` import block (#12975 on the tip + #13017). - `typecheck:core` clean; `check:dashboard-typecheck` OK (206 pre-existing, within baseline); `check:changelog-integrity` OK; `check:docs-counts` migrations ✓ - complexity 2816 / baseline 3218 and cognitive-complexity 1271 / baseline 1437 — both under baseline - 531 of 532 focused assertions green across the batch's 46 test files - `check-file-size` rebaselined for the batch's real growth (annotation `_rebaseline_2026_09_11_mergebatch_v3851_houminxi`, landed on #13038), attributed per PR The single red is **not this batch**: `tests/unit/combo/quota-weighted-strategy.test.ts` → "A/B isolation: 7 hard-empty + 2 at 0.5% + 1 at 40%, floor=1" asserts an order between two connections of identical weight and flakes on the pure tip too — 2 failures in 4 runs at `origin/release/v3.8.51` with nothing from this batch applied. ⚠️ base-red inherited: #12732 — `Docs Gates`, `Merge integrity`, `No new ESLint warnings`, `Unit Tests fast-path` and `Fast Quality Gates` reproduce on the pure tip (provider count 356 vs the 358 the modules define, SKILL.md drift, and `open-sse/utils/stream.ts` at 3115 > frozen 3098, untouched here). Thanks @HouMinXi — the live evidence on these (X500 logs, `storage.sqlite` state, real `/v1/models` probes, the 36-minute outage write-up) is what let a 20-PR batch be reviewed as a unit. --- .../fixes/13017-explicit-inactive-probe.md | 1 + src/lib/quota/connectionRecovery.ts | 2 +- src/sse/handlers/chatHelpers.ts | 8 + src/sse/services/auth.ts | 50 +++- src/sse/services/explicitInactiveProbe.ts | 126 +++++++++ stryker.conf.json | 3 +- .../explicit-inactive-probe-12874.test.ts | 249 ++++++++++++++++++ tests/unit/explicit-inactive-probe-w2.test.ts | 85 ++++++ tests/unit/forced-connection-fallback.test.ts | 4 + 9 files changed, 522 insertions(+), 6 deletions(-) create mode 100644 changelog.d/fixes/13017-explicit-inactive-probe.md create mode 100644 src/sse/services/explicitInactiveProbe.ts create mode 100644 tests/unit/explicit-inactive-probe-12874.test.ts create mode 100644 tests/unit/explicit-inactive-probe-w2.test.ts diff --git a/changelog.d/fixes/13017-explicit-inactive-probe.md b/changelog.d/fixes/13017-explicit-inactive-probe.md new file mode 100644 index 0000000000..f3179991bd --- /dev/null +++ b/changelog.d/fixes/13017-explicit-inactive-probe.md @@ -0,0 +1 @@ +- **fix(auth):** an explicit connection pin may probe a quota-disabled row once and re-enable it on success ([#12874](https://github.com/diegosouzapw/OmniRoute/issues/12874)) ([#13017](https://github.com/diegosouzapw/OmniRoute/pull/13017)) diff --git a/src/lib/quota/connectionRecovery.ts b/src/lib/quota/connectionRecovery.ts index 26952a262d..ec23d8175d 100644 --- a/src/lib/quota/connectionRecovery.ts +++ b/src/lib/quota/connectionRecovery.ts @@ -160,7 +160,7 @@ export function isRecoverableCooldownConnection( * Pure — `nowMs` and `reprobeMs` are injected so callers/tests control the clock. */ -const EXPIRED_REPROBE_BLOCKLIST = new Set([ +export const EXPIRED_REPROBE_BLOCKLIST = new Set([ "account_deactivated", "invalid_grant", "unrecoverable_refresh_error", diff --git a/src/sse/handlers/chatHelpers.ts b/src/sse/handlers/chatHelpers.ts index 1a6b293ddf..e2277ae1ff 100644 --- a/src/sse/handlers/chatHelpers.ts +++ b/src/sse/handlers/chatHelpers.ts @@ -8,6 +8,7 @@ import { markAccountUnavailable, buildExhaustionOptions, } from "../services/auth"; +import { maybeReactivateAfterExplicitProbe } from "../services/explicitInactiveProbe"; import { connectionHasExtraKeys } from "@omniroute/open-sse/services/apiKeyRotator.ts"; import { createBuiltinAutoCombo } from "@omniroute/open-sse/services/autoCombo/builtinCatalog.ts"; import * as log from "../utils/logger"; @@ -535,6 +536,13 @@ export async function executeChatWithBreaker({ onRequestSuccess: async () => { if (isShadowTraffic) return; await clearAccountError(credentials.connectionId, credentials); + await maybeReactivateAfterExplicitProbe({ + connectionId: credentials.connectionId, + reactivatedFromInactive: credentials.reactivatedFromInactive, + isShadowTraffic, + requestedModel: model, + provider, + }); }, onStreamFailure: async (failure: any) => { if (isShadowTraffic) return; diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index 999674480b..65729efc73 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -7,6 +7,7 @@ import { getCachedRawProviderConnections, getCachedProviderNodes, getCachedSettings, + getCachedProviderConnectionById, } from "@/lib/db/readCache"; import { getProviderConnections, @@ -150,6 +151,12 @@ import { planSessionAffinityConnection, syncSessionAffinityRuntimeFields, } from "./sessionAffinityPin"; +import { + EXPLICIT_INACTIVE_PROBE_INTERVAL_MS, + lastExplicitProbeTime, + noteExplicitProbe, + selectExplicitInactiveProbe, +} from "./explicitInactiveProbe"; import { isAnonymousFallbackDisabledBySettings, isNoAuthProviderBlockedBySettings, @@ -1062,7 +1069,10 @@ async function hydrateAccountProxyReferences( async function materializeConnection( connection: ProviderConnectionView, options: CredentialSelectionOptions, - extra: DeferredLeaseSelection & { exclusiveLease?: ExclusiveConnectionLease } = {} + extra: DeferredLeaseSelection & { + exclusiveLease?: ExclusiveConnectionLease; + reactivatedFromInactive?: boolean; + } = {} ) { const providerSpecificData = await hydrateAccountProxyReferences(connection.providerSpecificData); const apiKeyHealth = providerSpecificData.apiKeyHealth as Record | undefined; @@ -1261,6 +1271,29 @@ export async function getProviderCredentials( if (allowedConnections && allowedConnections.length > 0) { connections = connections.filter((conn) => allowedConnections.includes(conn.id)); } + let explicitProbeKind: "probe" | "suppressed" | "skip" = "skip"; + if (forcedConnectionId && !connections.some((c) => c.id === forcedConnectionId)) { + const pinnedRaw = await getCachedProviderConnectionById(forcedConnectionId); + const pinnedRow = pinnedRaw ? toProviderConnection(pinnedRaw) : null; + const nowMs = Date.now(); + const decision = selectExplicitInactiveProbe({ + forcedConnectionId, + activeConnections: connections, + pinnedRow, + providersToSearch, + allowedConnectionIds: allowedConnections ?? null, + nowMs, + lastProbeAtMs: lastExplicitProbeTime(forcedConnectionId), + intervalMs: EXPLICIT_INACTIVE_PROBE_INTERVAL_MS, + }); + explicitProbeKind = decision.kind; + if (decision.kind === "probe" && pinnedRow) { + noteExplicitProbe(forcedConnectionId, nowMs); + connections = [pinnedRow]; + } + } + const probeStamp = + explicitProbeKind === "probe" ? { reactivatedFromInactive: true as const } : {}; const forcedConnectionEligible = connections.some((conn) => conn.id === forcedConnectionId); if (options.lease && forcedConnectionId && !forcedConnectionEligible) return null; if (options.lease?.mode === "request" && forcedConnectionId) { @@ -1487,7 +1520,7 @@ export async function getProviderCredentials( connectionFilterStatus.set(c.id, "modelNotAdvertised"); return false; } - if (!allowSuppressedConnections) { + if (!allowSuppressedConnections && explicitProbeKind !== "probe") { if (!allowRateLimitedConnections && isAccountUnavailable(c.rateLimitedUntil)) { connectionFilterStatus.set(c.id, "rateLimited"); return false; @@ -2120,6 +2153,7 @@ export async function getProviderCredentials( return materializeConnection(connection, options, { commitSelectionSideEffects, selectNextLeaseCandidate, + ...probeStamp, }); } let claim = mutateExclusiveConnectionLease( @@ -2139,7 +2173,12 @@ export async function getProviderCredentials( exclusiveLease = claim.lease; await commitSelectionSideEffects?.(); if (options.materializeCredentials === false) { - return { exclusiveLease, connectionId: connection.id, provider: connection.provider }; + return { + exclusiveLease, + connectionId: connection.id, + provider: connection.provider, + ...probeStamp, + }; } } @@ -2150,7 +2189,10 @@ export async function getProviderCredentials( ); } - return materializeConnection(connection, options, { exclusiveLease }); + return materializeConnection(connection, options, { + exclusiveLease, + ...probeStamp, + }); } finally { selectionLock?.release(); } diff --git a/src/sse/services/explicitInactiveProbe.ts b/src/sse/services/explicitInactiveProbe.ts new file mode 100644 index 0000000000..c6b787df8e --- /dev/null +++ b/src/sse/services/explicitInactiveProbe.ts @@ -0,0 +1,126 @@ +import { updateProviderConnection } from "@/lib/db/providers"; +import { EXPIRED_REPROBE_BLOCKLIST } from "@/lib/quota/connectionRecovery"; + +export const EXPLICIT_PROBE_BLOCKLIST = EXPIRED_REPROBE_BLOCKLIST; + +export const RECOVERABLE_INACTIVE_TEST_STATUSES = new Set([ + "active", + "success", + "credits_exhausted", + "unavailable", + "error", + "", +]); + +export function isRecoverableInactiveConnection( + conn: { + isActive?: boolean; + testStatus?: string | null; + lastErrorType?: string | null; + rateLimitedUntil?: string | null; + }, + nowMs: number = Date.now() +): boolean { + if (conn.isActive !== false) return false; + const status = (conn.testStatus || "").trim().toLowerCase(); + if (status === "banned") return false; + const err = (conn.lastErrorType || "").trim().toLowerCase(); + if (EXPLICIT_PROBE_BLOCKLIST.has(err)) return false; + if (status === "expired") return true; + if (status === "unavailable") { + const until = conn.rateLimitedUntil; + if (until) { + const ms = Date.parse(until); + if (Number.isFinite(ms) && ms > nowMs) return false; + } + } + return RECOVERABLE_INACTIVE_TEST_STATUSES.has(status); +} + +export function selectExplicitInactiveProbe(params: { + forcedConnectionId: string | null; + activeConnections: { id: string }[]; + pinnedRow: { + id: string; + provider?: string | null; + isActive?: boolean; + testStatus?: string | null; + lastErrorType?: string | null; + rateLimitedUntil?: string | null; + } | null; + providersToSearch: string[]; + allowedConnectionIds: string[] | null; + nowMs: number; + lastProbeAtMs: number | null; + intervalMs: number; +}): { kind: "probe" } | { kind: "suppressed" } | { kind: "skip" } { + const id = params.forcedConnectionId; + if (!id) return { kind: "skip" }; + if (params.activeConnections.some((c) => c.id === id)) return { kind: "skip" }; + const row = params.pinnedRow; + if (!row || row.id !== id) return { kind: "skip" }; + if ( + params.allowedConnectionIds && + params.allowedConnectionIds.length > 0 && + !params.allowedConnectionIds.includes(id) + ) { + return { kind: "skip" }; + } + const prov = (row.provider || "").trim(); + if (prov && !params.providersToSearch.includes(prov)) return { kind: "skip" }; + if (!isRecoverableInactiveConnection(row, params.nowMs)) return { kind: "skip" }; + if (params.lastProbeAtMs != null && params.nowMs - params.lastProbeAtMs < params.intervalMs) { + return { kind: "suppressed" }; + } + return { kind: "probe" }; +} + +export const EXPLICIT_INACTIVE_PROBE_INTERVAL_MS = 60_000; +const MAX_PROBE_MAP = 4096; +const lastExplicitProbeAtMs = new Map(); + +export function noteExplicitProbe(id: string, nowMs: number): void { + lastExplicitProbeAtMs.set(id, nowMs); + if (lastExplicitProbeAtMs.size > MAX_PROBE_MAP) { + const oldest = lastExplicitProbeAtMs.keys().next().value; + if (oldest !== undefined) lastExplicitProbeAtMs.delete(oldest); + } +} + +export function lastExplicitProbeTime(id: string): number | null { + return lastExplicitProbeAtMs.get(id) ?? null; +} + +export function resetExplicitProbeMapForTests(): void { + lastExplicitProbeAtMs.clear(); +} + +export async function reactivateRecoveredConnection(connectionId: string): Promise { + await updateProviderConnection(connectionId, { isActive: true }); +} + +export async function maybeReactivateAfterExplicitProbe( + input: { + connectionId: string; + reactivatedFromInactive?: boolean; + explicitProbeSuppressed?: boolean; + isShadowTraffic?: boolean; + allowSuppressedConnections?: boolean; + requestedModel?: string | null; + provider?: string | null; + }, + reactivate: (connectionId: string) => Promise = reactivateRecoveredConnection +): Promise { + if (!input.reactivatedFromInactive) return; + if (input.explicitProbeSuppressed) return; + if (input.isShadowTraffic) return; + if (input.allowSuppressedConnections) return; + if ( + input.provider === "openrouter" && + typeof input.requestedModel === "string" && + input.requestedModel.includes(":free") + ) { + return; + } + await reactivate(input.connectionId); +} diff --git a/stryker.conf.json b/stryker.conf.json index 5f0215a434..767cec38ae 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -71,7 +71,7 @@ "tests/unit/alibaba-free-tier-exhaustion.test.ts", "tests/unit/anthropic-thinking-signature-recovery.test.ts", "tests/unit/agy-family-not-connection-cooldown.test.ts", - "tests/unit/agy-quota-exhaustion-threshold.test.ts", + "tests/unit/agy-quota-exhaustion-threshold.test.ts", "tests/unit/antigravity-429-quota-cooldown.test.ts", "tests/unit/antigravity-429-quota-tdd.test.ts", "tests/unit/antigravity-prefer-stored-project.test.ts", @@ -261,6 +261,7 @@ "tests/unit/executor-contract-violation-terminal.test.ts", "tests/unit/executor-devin-cli-agentic-acp.test.ts", "tests/unit/executor-web-cookie-sweep.test.ts", + "tests/unit/explicit-inactive-probe-w2.test.ts", "tests/unit/false-terminal-401-quota.test.ts", "tests/unit/follow-up-transcript.test.ts", "tests/unit/format-provider-error-cause.test.ts", diff --git a/tests/unit/explicit-inactive-probe-12874.test.ts b/tests/unit/explicit-inactive-probe-12874.test.ts new file mode 100644 index 0000000000..f4926258d5 --- /dev/null +++ b/tests/unit/explicit-inactive-probe-12874.test.ts @@ -0,0 +1,249 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import { EXPIRED_REPROBE_BLOCKLIST } from "../../src/lib/quota/connectionRecovery.ts"; +import { + EXPLICIT_INACTIVE_PROBE_INTERVAL_MS, + EXPLICIT_PROBE_BLOCKLIST, + lastExplicitProbeTime, + maybeReactivateAfterExplicitProbe, + noteExplicitProbe, + resetExplicitProbeMapForTests, + selectExplicitInactiveProbe, +} from "../../src/sse/services/explicitInactiveProbe.ts"; + +const pin = { + id: "c1", + provider: "siliconflow", + isActive: false, + testStatus: "active", + lastErrorType: null, + rateLimitedUntil: null, +}; + +function select(overrides: Partial[0]> = {}) { + return selectExplicitInactiveProbe({ + forcedConnectionId: "c1", + activeConnections: [], + pinnedRow: pin, + providersToSearch: ["siliconflow"], + allowedConnectionIds: null, + nowMs: 1_000_000, + lastProbeAtMs: null, + intervalMs: 60_000, + ...overrides, + }); +} + +test("P-14 EXPLICIT_PROBE_BLOCKLIST is the same object as EXPIRED_REPROBE_BLOCKLIST", () => { + assert.equal(EXPLICIT_PROBE_BLOCKLIST, EXPIRED_REPROBE_BLOCKLIST); +}); + +test("P-1 pin + inactive active -> probe", () => { + assert.equal(select().kind, "probe"); +}); +test("P-2 credits_exhausted -> probe", () => { + assert.equal(select({ pinnedRow: { ...pin, testStatus: "credits_exhausted" } }).kind, "probe"); +}); +test("P-3 no pin -> skip", () => { + assert.equal(select({ forcedConnectionId: null }).kind, "skip"); +}); +test("P-4 live pool already has id -> skip", () => { + assert.equal(select({ activeConnections: [{ id: "c1" }] }).kind, "skip"); +}); +test("P-5 banned -> skip", () => { + assert.equal(select({ pinnedRow: { ...pin, testStatus: "banned" } }).kind, "skip"); +}); +test("P-6 expired + no_refresh_token -> skip", () => { + assert.equal( + select({ pinnedRow: { ...pin, testStatus: "expired", lastErrorType: "no_refresh_token" } }).kind, + "skip" + ); +}); +test("P-7 expired + empty lastErrorType -> probe", () => { + assert.equal(select({ pinnedRow: { ...pin, testStatus: "expired", lastErrorType: "" } }).kind, "probe"); +}); +test("P-8 error + unrecoverable_refresh_error -> skip", () => { + assert.equal( + select({ pinnedRow: { ...pin, testStatus: "error", lastErrorType: "unrecoverable_refresh_error" } }).kind, + "skip" + ); +}); +test("P-9 last probe within 60s -> suppressed", () => { + assert.equal(select({ lastProbeAtMs: 1_000_000 - 10_000 }).kind, "suppressed"); +}); +test("P-10 pin not in allowedConnectionIds -> skip", () => { + assert.equal(select({ allowedConnectionIds: ["other"] }).kind, "skip"); +}); +test("P-11 provider not in providersToSearch -> skip", () => { + assert.equal(select({ providersToSearch: ["openai"] }).kind, "skip"); +}); +test("P-12 unavailable + future cooldown -> skip", () => { + assert.equal( + select({ + pinnedRow: { ...pin, testStatus: "unavailable", rateLimitedUntil: new Date(2_000_000_000).toISOString() }, + nowMs: 1_000_000, + }).kind, + "skip" + ); +}); +test("P-13 unavailable + elapsed cooldown -> probe", () => { + assert.equal( + select({ + pinnedRow: { ...pin, testStatus: "unavailable", rateLimitedUntil: new Date(500_000).toISOString() }, + nowMs: 1_000_000, + }).kind, + "probe" + ); +}); + +test("storm Map note then select within interval is suppressed", () => { + resetExplicitProbeMapForTests(); + noteExplicitProbe("c1", 1e6); + assert.equal(lastExplicitProbeTime("c1"), 1e6); + assert.equal( + select({ lastProbeAtMs: lastExplicitProbeTime("c1"), nowMs: 1e6 + 10_000 }).kind, + "suppressed" + ); +}); + +test("storm Map interval constant is 60s", () => { + assert.equal(EXPLICIT_INACTIVE_PROBE_INTERVAL_MS, 60_000); +}); + +test("storm Map reset clears last probe time", () => { + resetExplicitProbeMapForTests(); + noteExplicitProbe("c1", 1e6); + resetExplicitProbeMapForTests(); + assert.equal(lastExplicitProbeTime("c1"), null); +}); + +test("storm Map evicts oldest when over 4096 entries", () => { + resetExplicitProbeMapForTests(); + for (let i = 0; i < 4096; i++) { + noteExplicitProbe(`id-${i}`, i); + } + noteExplicitProbe("overflow", 4096); + assert.equal(lastExplicitProbeTime("id-0"), null); + assert.equal(lastExplicitProbeTime("id-1"), 1); + assert.equal(lastExplicitProbeTime("overflow"), 4096); +}); + +test("W-6 openrouter :free does not reactivate", async () => { + let called = 0; + await maybeReactivateAfterExplicitProbe( + { + connectionId: "c1", + reactivatedFromInactive: true, + provider: "openrouter", + requestedModel: "openrouter/foo:free", + }, + async () => { + called += 1; + } + ); + assert.equal(called, 0); +}); + +test("maybeReactivateAfterExplicitProbe calls reactivate on recovered pin", async () => { + let called = 0; + let seenId = ""; + await maybeReactivateAfterExplicitProbe( + { + connectionId: "c1", + reactivatedFromInactive: true, + }, + async (id) => { + called += 1; + seenId = id; + } + ); + assert.equal(called, 1); + assert.equal(seenId, "c1"); +}); + +test("maybeReactivateAfterExplicitProbe no-ops without reactivatedFromInactive", async () => { + let called = 0; + await maybeReactivateAfterExplicitProbe({ connectionId: "c1" }, async () => { + called += 1; + }); + assert.equal(called, 0); +}); + +test("maybeReactivateAfterExplicitProbe no-ops when explicitProbeSuppressed", async () => { + let called = 0; + await maybeReactivateAfterExplicitProbe( + { + connectionId: "c1", + reactivatedFromInactive: true, + explicitProbeSuppressed: true, + }, + async () => { + called += 1; + } + ); + assert.equal(called, 0); +}); + +test("maybeReactivateAfterExplicitProbe no-ops on shadow traffic", async () => { + let called = 0; + await maybeReactivateAfterExplicitProbe( + { + connectionId: "c1", + reactivatedFromInactive: true, + isShadowTraffic: true, + }, + async () => { + called += 1; + } + ); + assert.equal(called, 0); +}); + +test("maybeReactivateAfterExplicitProbe no-ops when allowSuppressedConnections", async () => { + let called = 0; + await maybeReactivateAfterExplicitProbe( + { + connectionId: "c1", + reactivatedFromInactive: true, + allowSuppressedConnections: true, + }, + async () => { + called += 1; + } + ); + assert.equal(called, 0); +}); + +test("W-7 chatHelpers onRequestSuccess calls maybeReactivateAfterExplicitProbe", async () => { + const src = fs.readFileSync(new URL("../../src/sse/handlers/chatHelpers.ts", import.meta.url), "utf8"); + assert.match(src, /await maybeReactivateAfterExplicitProbe\(/); + assert.match(src, /clearAccountError/); +}); + +test("W-3 clearAccountError update payload has no isActive key", () => { + const src = fs.readFileSync(new URL("../../src/sse/services/auth.ts", import.meta.url), "utf8"); + const start = src.indexOf("export async function clearAccountError"); + assert.ok(start >= 0, "clearAccountError must exist"); + const next = src.indexOf("\nexport ", start + 1); + const body = next >= 0 ? src.slice(start, next) : src.slice(start); + const updateStart = body.indexOf("await updateProviderConnection("); + assert.ok(updateStart >= 0, "clearAccountError must call updateProviderConnection"); + const brace = body.indexOf("{", updateStart); + assert.ok(brace >= 0); + let depth = 0; + let end = -1; + for (let i = brace; i < body.length; i++) { + if (body[i] === "{") depth += 1; + else if (body[i] === "}") { + depth -= 1; + if (depth === 0) { + end = i; + break; + } + } + } + assert.ok(end > brace); + const literal = body.slice(brace, end + 1); + assert.equal(/\bisActive\b/.test(literal), false); +}); diff --git a/tests/unit/explicit-inactive-probe-w2.test.ts b/tests/unit/explicit-inactive-probe-w2.test.ts new file mode 100644 index 0000000000..eec2fe9c3e --- /dev/null +++ b/tests/unit/explicit-inactive-probe-w2.test.ts @@ -0,0 +1,85 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-explicit-inactive-w2-")); + +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "explicit-inactive-w2-test-secret"; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const auth = await import("../../src/sse/services/auth.ts"); +const { maybeReactivateAfterExplicitProbe, resetExplicitProbeMapForTests } = await import( + "../../src/sse/services/explicitInactiveProbe.ts" +); + +async function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +async function seedInactiveSiliconflow(testStatus: "active" | "credits_exhausted") { + const row = await providersDb.createProviderConnection({ + provider: "siliconflow", + authType: "apikey", + name: "sf-inactive", + apiKey: "sf-inactive-test-key", + isActive: false, + testStatus, + }); + assert.ok(typeof row?.id === "string" && row.id.length > 0); + return { id: row.id }; +} + +function asPinnedCreds(creds: unknown) { + assert.ok(creds); + return creds as { connectionId?: string; reactivatedFromInactive?: boolean }; +} + +test.beforeEach(async () => { + resetExplicitProbeMapForTests(); + await resetStorage(); +}); + +test.after(async () => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("W-2 pin + inactive returns credentials after wiring (was null)", async () => { + const row = await seedInactiveSiliconflow("active"); + const creds = asPinnedCreds( + await auth.getProviderCredentials("siliconflow", null, null, "siliconflow/m", { + forcedConnectionId: row.id, + }) + ); + assert.equal(creds.connectionId, row.id); + assert.equal(creds.reactivatedFromInactive, true); +}); + +test("W-2 pin + credits_exhausted returns credentials after wiring", async () => { + const row = await seedInactiveSiliconflow("credits_exhausted"); + const creds = asPinnedCreds( + await auth.getProviderCredentials("siliconflow", null, null, "siliconflow/m", { + forcedConnectionId: row.id, + }) + ); + assert.equal(creds.connectionId, row.id); + assert.equal(creds.reactivatedFromInactive, true); +}); + +test("W-1 successful probe re-enables inactive pin in SQLite", async () => { + const row = await seedInactiveSiliconflow("active"); + const before = await providersDb.getProviderConnectionById(row.id); + assert.equal(before?.isActive, false); + await maybeReactivateAfterExplicitProbe({ + reactivatedFromInactive: true, + connectionId: row.id, + }); + const after = await providersDb.getProviderConnectionById(row.id); + assert.equal(after?.isActive, true); +}); diff --git a/tests/unit/forced-connection-fallback.test.ts b/tests/unit/forced-connection-fallback.test.ts index 33add57917..935d8b52e2 100644 --- a/tests/unit/forced-connection-fallback.test.ts +++ b/tests/unit/forced-connection-fallback.test.ts @@ -112,6 +112,10 @@ test("BUG CASE: forced connection deactivated (missing from active pool) is dete ); }); +test("recoverable inactive pin is not missing-from-pool once in connections", () => { + assert.equal(isForcedConnectionMissingFromPool("c1", new Set(), [{ id: "c1" }]), false); +}); + test("EXISTING BEHAVIOR: forced connection already excluded after a failed attempt is NOT missing-from-pool", () => { // The account is still present in the (active) pool — it 429'd and the retry loop // added it to excludedConnectionIds. This must keep going through