mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-14 02:42:24 +03:00
fix(auth): probe a pinned inactive connection after quota top-up (#13017)
The one-shot framing is what makes this safe: a pin is an explicit operator act, so taking the inactive row only for that request, with siblings out of the pool and a 60s per-connection storm gate, keeps the blast radius at one request. Dashboard deactivate staying off is the right carve-out. Reconciled against the tip after the batch landed: the `chatHelpers.ts` import block conflicted with `buildExhaustionOptions` (#12975) and both imports were kept. 41/41 across the probe suites afterwards. --- Validated in one consolidated worktree cut from `release/v3.8.51`, boarded with the other 19 PRs of this batch. Two in-batch conflicts, both additive and resolved by keeping each side: the `ENVIRONMENT.md` table (#13035 + #13011) and the `chatHelpers.ts` import block (#12975 on the tip + #13017). - `typecheck:core` clean; `check:dashboard-typecheck` OK (206 pre-existing, within baseline); `check:changelog-integrity` OK; `check:docs-counts` migrations ✓ - complexity 2816 / baseline 3218 and cognitive-complexity 1271 / baseline 1437 — both under baseline - 531 of 532 focused assertions green across the batch's 46 test files - `check-file-size` rebaselined for the batch's real growth (annotation `_rebaseline_2026_09_11_mergebatch_v3851_houminxi`, landed on #13038), attributed per PR The single red is **not this batch**: `tests/unit/combo/quota-weighted-strategy.test.ts` → "A/B isolation: 7 hard-empty + 2 at 0.5% + 1 at 40%, floor=1" asserts an order between two connections of identical weight and flakes on the pure tip too — 2 failures in 4 runs at `origin/release/v3.8.51` with nothing from this batch applied. ⚠️ base-red inherited: #12732 — `Docs Gates`, `Merge integrity`, `No new ESLint warnings`, `Unit Tests fast-path` and `Fast Quality Gates` reproduce on the pure tip (provider count 356 vs the 358 the modules define, SKILL.md drift, and `open-sse/utils/stream.ts` at 3115 > frozen 3098, untouched here). Thanks @HouMinXi — the live evidence on these (X500 logs, `storage.sqlite` state, real `/v1/models` probes, the 36-minute outage write-up) is what let a 20-PR batch be reviewed as a unit.
This commit is contained in:
1
changelog.d/fixes/13017-explicit-inactive-probe.md
Normal file
1
changelog.d/fixes/13017-explicit-inactive-probe.md
Normal file
@@ -0,0 +1 @@
|
||||
- **fix(auth):** an explicit connection pin may probe a quota-disabled row once and re-enable it on success ([#12874](https://github.com/diegosouzapw/OmniRoute/issues/12874)) ([#13017](https://github.com/diegosouzapw/OmniRoute/pull/13017))
|
||||
@@ -160,7 +160,7 @@ export function isRecoverableCooldownConnection(
|
||||
* Pure — `nowMs` and `reprobeMs` are injected so callers/tests control the clock.
|
||||
*/
|
||||
|
||||
const EXPIRED_REPROBE_BLOCKLIST = new Set([
|
||||
export const EXPIRED_REPROBE_BLOCKLIST = new Set([
|
||||
"account_deactivated",
|
||||
"invalid_grant",
|
||||
"unrecoverable_refresh_error",
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
markAccountUnavailable,
|
||||
buildExhaustionOptions,
|
||||
} from "../services/auth";
|
||||
import { maybeReactivateAfterExplicitProbe } from "../services/explicitInactiveProbe";
|
||||
import { connectionHasExtraKeys } from "@omniroute/open-sse/services/apiKeyRotator.ts";
|
||||
import { createBuiltinAutoCombo } from "@omniroute/open-sse/services/autoCombo/builtinCatalog.ts";
|
||||
import * as log from "../utils/logger";
|
||||
@@ -535,6 +536,13 @@ export async function executeChatWithBreaker({
|
||||
onRequestSuccess: async () => {
|
||||
if (isShadowTraffic) return;
|
||||
await clearAccountError(credentials.connectionId, credentials);
|
||||
await maybeReactivateAfterExplicitProbe({
|
||||
connectionId: credentials.connectionId,
|
||||
reactivatedFromInactive: credentials.reactivatedFromInactive,
|
||||
isShadowTraffic,
|
||||
requestedModel: model,
|
||||
provider,
|
||||
});
|
||||
},
|
||||
onStreamFailure: async (failure: any) => {
|
||||
if (isShadowTraffic) return;
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
getCachedRawProviderConnections,
|
||||
getCachedProviderNodes,
|
||||
getCachedSettings,
|
||||
getCachedProviderConnectionById,
|
||||
} from "@/lib/db/readCache";
|
||||
import {
|
||||
getProviderConnections,
|
||||
@@ -150,6 +151,12 @@ import {
|
||||
planSessionAffinityConnection,
|
||||
syncSessionAffinityRuntimeFields,
|
||||
} from "./sessionAffinityPin";
|
||||
import {
|
||||
EXPLICIT_INACTIVE_PROBE_INTERVAL_MS,
|
||||
lastExplicitProbeTime,
|
||||
noteExplicitProbe,
|
||||
selectExplicitInactiveProbe,
|
||||
} from "./explicitInactiveProbe";
|
||||
import {
|
||||
isAnonymousFallbackDisabledBySettings,
|
||||
isNoAuthProviderBlockedBySettings,
|
||||
@@ -1062,7 +1069,10 @@ async function hydrateAccountProxyReferences(
|
||||
async function materializeConnection(
|
||||
connection: ProviderConnectionView,
|
||||
options: CredentialSelectionOptions,
|
||||
extra: DeferredLeaseSelection & { exclusiveLease?: ExclusiveConnectionLease } = {}
|
||||
extra: DeferredLeaseSelection & {
|
||||
exclusiveLease?: ExclusiveConnectionLease;
|
||||
reactivatedFromInactive?: boolean;
|
||||
} = {}
|
||||
) {
|
||||
const providerSpecificData = await hydrateAccountProxyReferences(connection.providerSpecificData);
|
||||
const apiKeyHealth = providerSpecificData.apiKeyHealth as Record<string, KeyHealth> | undefined;
|
||||
@@ -1261,6 +1271,29 @@ export async function getProviderCredentials(
|
||||
if (allowedConnections && allowedConnections.length > 0) {
|
||||
connections = connections.filter((conn) => allowedConnections.includes(conn.id));
|
||||
}
|
||||
let explicitProbeKind: "probe" | "suppressed" | "skip" = "skip";
|
||||
if (forcedConnectionId && !connections.some((c) => c.id === forcedConnectionId)) {
|
||||
const pinnedRaw = await getCachedProviderConnectionById(forcedConnectionId);
|
||||
const pinnedRow = pinnedRaw ? toProviderConnection(pinnedRaw) : null;
|
||||
const nowMs = Date.now();
|
||||
const decision = selectExplicitInactiveProbe({
|
||||
forcedConnectionId,
|
||||
activeConnections: connections,
|
||||
pinnedRow,
|
||||
providersToSearch,
|
||||
allowedConnectionIds: allowedConnections ?? null,
|
||||
nowMs,
|
||||
lastProbeAtMs: lastExplicitProbeTime(forcedConnectionId),
|
||||
intervalMs: EXPLICIT_INACTIVE_PROBE_INTERVAL_MS,
|
||||
});
|
||||
explicitProbeKind = decision.kind;
|
||||
if (decision.kind === "probe" && pinnedRow) {
|
||||
noteExplicitProbe(forcedConnectionId, nowMs);
|
||||
connections = [pinnedRow];
|
||||
}
|
||||
}
|
||||
const probeStamp =
|
||||
explicitProbeKind === "probe" ? { reactivatedFromInactive: true as const } : {};
|
||||
const forcedConnectionEligible = connections.some((conn) => conn.id === forcedConnectionId);
|
||||
if (options.lease && forcedConnectionId && !forcedConnectionEligible) return null;
|
||||
if (options.lease?.mode === "request" && forcedConnectionId) {
|
||||
@@ -1487,7 +1520,7 @@ export async function getProviderCredentials(
|
||||
connectionFilterStatus.set(c.id, "modelNotAdvertised");
|
||||
return false;
|
||||
}
|
||||
if (!allowSuppressedConnections) {
|
||||
if (!allowSuppressedConnections && explicitProbeKind !== "probe") {
|
||||
if (!allowRateLimitedConnections && isAccountUnavailable(c.rateLimitedUntil)) {
|
||||
connectionFilterStatus.set(c.id, "rateLimited");
|
||||
return false;
|
||||
@@ -2120,6 +2153,7 @@ export async function getProviderCredentials(
|
||||
return materializeConnection(connection, options, {
|
||||
commitSelectionSideEffects,
|
||||
selectNextLeaseCandidate,
|
||||
...probeStamp,
|
||||
});
|
||||
}
|
||||
let claim = mutateExclusiveConnectionLease(
|
||||
@@ -2139,7 +2173,12 @@ export async function getProviderCredentials(
|
||||
exclusiveLease = claim.lease;
|
||||
await commitSelectionSideEffects?.();
|
||||
if (options.materializeCredentials === false) {
|
||||
return { exclusiveLease, connectionId: connection.id, provider: connection.provider };
|
||||
return {
|
||||
exclusiveLease,
|
||||
connectionId: connection.id,
|
||||
provider: connection.provider,
|
||||
...probeStamp,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2150,7 +2189,10 @@ export async function getProviderCredentials(
|
||||
);
|
||||
}
|
||||
|
||||
return materializeConnection(connection, options, { exclusiveLease });
|
||||
return materializeConnection(connection, options, {
|
||||
exclusiveLease,
|
||||
...probeStamp,
|
||||
});
|
||||
} finally {
|
||||
selectionLock?.release();
|
||||
}
|
||||
|
||||
126
src/sse/services/explicitInactiveProbe.ts
Normal file
126
src/sse/services/explicitInactiveProbe.ts
Normal file
@@ -0,0 +1,126 @@
|
||||
import { updateProviderConnection } from "@/lib/db/providers";
|
||||
import { EXPIRED_REPROBE_BLOCKLIST } from "@/lib/quota/connectionRecovery";
|
||||
|
||||
export const EXPLICIT_PROBE_BLOCKLIST = EXPIRED_REPROBE_BLOCKLIST;
|
||||
|
||||
export const RECOVERABLE_INACTIVE_TEST_STATUSES = new Set([
|
||||
"active",
|
||||
"success",
|
||||
"credits_exhausted",
|
||||
"unavailable",
|
||||
"error",
|
||||
"",
|
||||
]);
|
||||
|
||||
export function isRecoverableInactiveConnection(
|
||||
conn: {
|
||||
isActive?: boolean;
|
||||
testStatus?: string | null;
|
||||
lastErrorType?: string | null;
|
||||
rateLimitedUntil?: string | null;
|
||||
},
|
||||
nowMs: number = Date.now()
|
||||
): boolean {
|
||||
if (conn.isActive !== false) return false;
|
||||
const status = (conn.testStatus || "").trim().toLowerCase();
|
||||
if (status === "banned") return false;
|
||||
const err = (conn.lastErrorType || "").trim().toLowerCase();
|
||||
if (EXPLICIT_PROBE_BLOCKLIST.has(err)) return false;
|
||||
if (status === "expired") return true;
|
||||
if (status === "unavailable") {
|
||||
const until = conn.rateLimitedUntil;
|
||||
if (until) {
|
||||
const ms = Date.parse(until);
|
||||
if (Number.isFinite(ms) && ms > nowMs) return false;
|
||||
}
|
||||
}
|
||||
return RECOVERABLE_INACTIVE_TEST_STATUSES.has(status);
|
||||
}
|
||||
|
||||
export function selectExplicitInactiveProbe(params: {
|
||||
forcedConnectionId: string | null;
|
||||
activeConnections: { id: string }[];
|
||||
pinnedRow: {
|
||||
id: string;
|
||||
provider?: string | null;
|
||||
isActive?: boolean;
|
||||
testStatus?: string | null;
|
||||
lastErrorType?: string | null;
|
||||
rateLimitedUntil?: string | null;
|
||||
} | null;
|
||||
providersToSearch: string[];
|
||||
allowedConnectionIds: string[] | null;
|
||||
nowMs: number;
|
||||
lastProbeAtMs: number | null;
|
||||
intervalMs: number;
|
||||
}): { kind: "probe" } | { kind: "suppressed" } | { kind: "skip" } {
|
||||
const id = params.forcedConnectionId;
|
||||
if (!id) return { kind: "skip" };
|
||||
if (params.activeConnections.some((c) => c.id === id)) return { kind: "skip" };
|
||||
const row = params.pinnedRow;
|
||||
if (!row || row.id !== id) return { kind: "skip" };
|
||||
if (
|
||||
params.allowedConnectionIds &&
|
||||
params.allowedConnectionIds.length > 0 &&
|
||||
!params.allowedConnectionIds.includes(id)
|
||||
) {
|
||||
return { kind: "skip" };
|
||||
}
|
||||
const prov = (row.provider || "").trim();
|
||||
if (prov && !params.providersToSearch.includes(prov)) return { kind: "skip" };
|
||||
if (!isRecoverableInactiveConnection(row, params.nowMs)) return { kind: "skip" };
|
||||
if (params.lastProbeAtMs != null && params.nowMs - params.lastProbeAtMs < params.intervalMs) {
|
||||
return { kind: "suppressed" };
|
||||
}
|
||||
return { kind: "probe" };
|
||||
}
|
||||
|
||||
export const EXPLICIT_INACTIVE_PROBE_INTERVAL_MS = 60_000;
|
||||
const MAX_PROBE_MAP = 4096;
|
||||
const lastExplicitProbeAtMs = new Map<string, number>();
|
||||
|
||||
export function noteExplicitProbe(id: string, nowMs: number): void {
|
||||
lastExplicitProbeAtMs.set(id, nowMs);
|
||||
if (lastExplicitProbeAtMs.size > MAX_PROBE_MAP) {
|
||||
const oldest = lastExplicitProbeAtMs.keys().next().value;
|
||||
if (oldest !== undefined) lastExplicitProbeAtMs.delete(oldest);
|
||||
}
|
||||
}
|
||||
|
||||
export function lastExplicitProbeTime(id: string): number | null {
|
||||
return lastExplicitProbeAtMs.get(id) ?? null;
|
||||
}
|
||||
|
||||
export function resetExplicitProbeMapForTests(): void {
|
||||
lastExplicitProbeAtMs.clear();
|
||||
}
|
||||
|
||||
export async function reactivateRecoveredConnection(connectionId: string): Promise<void> {
|
||||
await updateProviderConnection(connectionId, { isActive: true });
|
||||
}
|
||||
|
||||
export async function maybeReactivateAfterExplicitProbe(
|
||||
input: {
|
||||
connectionId: string;
|
||||
reactivatedFromInactive?: boolean;
|
||||
explicitProbeSuppressed?: boolean;
|
||||
isShadowTraffic?: boolean;
|
||||
allowSuppressedConnections?: boolean;
|
||||
requestedModel?: string | null;
|
||||
provider?: string | null;
|
||||
},
|
||||
reactivate: (connectionId: string) => Promise<void> = reactivateRecoveredConnection
|
||||
): Promise<void> {
|
||||
if (!input.reactivatedFromInactive) return;
|
||||
if (input.explicitProbeSuppressed) return;
|
||||
if (input.isShadowTraffic) return;
|
||||
if (input.allowSuppressedConnections) return;
|
||||
if (
|
||||
input.provider === "openrouter" &&
|
||||
typeof input.requestedModel === "string" &&
|
||||
input.requestedModel.includes(":free")
|
||||
) {
|
||||
return;
|
||||
}
|
||||
await reactivate(input.connectionId);
|
||||
}
|
||||
@@ -71,7 +71,7 @@
|
||||
"tests/unit/alibaba-free-tier-exhaustion.test.ts",
|
||||
"tests/unit/anthropic-thinking-signature-recovery.test.ts",
|
||||
"tests/unit/agy-family-not-connection-cooldown.test.ts",
|
||||
"tests/unit/agy-quota-exhaustion-threshold.test.ts",
|
||||
"tests/unit/agy-quota-exhaustion-threshold.test.ts",
|
||||
"tests/unit/antigravity-429-quota-cooldown.test.ts",
|
||||
"tests/unit/antigravity-429-quota-tdd.test.ts",
|
||||
"tests/unit/antigravity-prefer-stored-project.test.ts",
|
||||
@@ -261,6 +261,7 @@
|
||||
"tests/unit/executor-contract-violation-terminal.test.ts",
|
||||
"tests/unit/executor-devin-cli-agentic-acp.test.ts",
|
||||
"tests/unit/executor-web-cookie-sweep.test.ts",
|
||||
"tests/unit/explicit-inactive-probe-w2.test.ts",
|
||||
"tests/unit/false-terminal-401-quota.test.ts",
|
||||
"tests/unit/follow-up-transcript.test.ts",
|
||||
"tests/unit/format-provider-error-cause.test.ts",
|
||||
|
||||
249
tests/unit/explicit-inactive-probe-12874.test.ts
Normal file
249
tests/unit/explicit-inactive-probe-12874.test.ts
Normal file
@@ -0,0 +1,249 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import { EXPIRED_REPROBE_BLOCKLIST } from "../../src/lib/quota/connectionRecovery.ts";
|
||||
import {
|
||||
EXPLICIT_INACTIVE_PROBE_INTERVAL_MS,
|
||||
EXPLICIT_PROBE_BLOCKLIST,
|
||||
lastExplicitProbeTime,
|
||||
maybeReactivateAfterExplicitProbe,
|
||||
noteExplicitProbe,
|
||||
resetExplicitProbeMapForTests,
|
||||
selectExplicitInactiveProbe,
|
||||
} from "../../src/sse/services/explicitInactiveProbe.ts";
|
||||
|
||||
const pin = {
|
||||
id: "c1",
|
||||
provider: "siliconflow",
|
||||
isActive: false,
|
||||
testStatus: "active",
|
||||
lastErrorType: null,
|
||||
rateLimitedUntil: null,
|
||||
};
|
||||
|
||||
function select(overrides: Partial<Parameters<typeof selectExplicitInactiveProbe>[0]> = {}) {
|
||||
return selectExplicitInactiveProbe({
|
||||
forcedConnectionId: "c1",
|
||||
activeConnections: [],
|
||||
pinnedRow: pin,
|
||||
providersToSearch: ["siliconflow"],
|
||||
allowedConnectionIds: null,
|
||||
nowMs: 1_000_000,
|
||||
lastProbeAtMs: null,
|
||||
intervalMs: 60_000,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
test("P-14 EXPLICIT_PROBE_BLOCKLIST is the same object as EXPIRED_REPROBE_BLOCKLIST", () => {
|
||||
assert.equal(EXPLICIT_PROBE_BLOCKLIST, EXPIRED_REPROBE_BLOCKLIST);
|
||||
});
|
||||
|
||||
test("P-1 pin + inactive active -> probe", () => {
|
||||
assert.equal(select().kind, "probe");
|
||||
});
|
||||
test("P-2 credits_exhausted -> probe", () => {
|
||||
assert.equal(select({ pinnedRow: { ...pin, testStatus: "credits_exhausted" } }).kind, "probe");
|
||||
});
|
||||
test("P-3 no pin -> skip", () => {
|
||||
assert.equal(select({ forcedConnectionId: null }).kind, "skip");
|
||||
});
|
||||
test("P-4 live pool already has id -> skip", () => {
|
||||
assert.equal(select({ activeConnections: [{ id: "c1" }] }).kind, "skip");
|
||||
});
|
||||
test("P-5 banned -> skip", () => {
|
||||
assert.equal(select({ pinnedRow: { ...pin, testStatus: "banned" } }).kind, "skip");
|
||||
});
|
||||
test("P-6 expired + no_refresh_token -> skip", () => {
|
||||
assert.equal(
|
||||
select({ pinnedRow: { ...pin, testStatus: "expired", lastErrorType: "no_refresh_token" } }).kind,
|
||||
"skip"
|
||||
);
|
||||
});
|
||||
test("P-7 expired + empty lastErrorType -> probe", () => {
|
||||
assert.equal(select({ pinnedRow: { ...pin, testStatus: "expired", lastErrorType: "" } }).kind, "probe");
|
||||
});
|
||||
test("P-8 error + unrecoverable_refresh_error -> skip", () => {
|
||||
assert.equal(
|
||||
select({ pinnedRow: { ...pin, testStatus: "error", lastErrorType: "unrecoverable_refresh_error" } }).kind,
|
||||
"skip"
|
||||
);
|
||||
});
|
||||
test("P-9 last probe within 60s -> suppressed", () => {
|
||||
assert.equal(select({ lastProbeAtMs: 1_000_000 - 10_000 }).kind, "suppressed");
|
||||
});
|
||||
test("P-10 pin not in allowedConnectionIds -> skip", () => {
|
||||
assert.equal(select({ allowedConnectionIds: ["other"] }).kind, "skip");
|
||||
});
|
||||
test("P-11 provider not in providersToSearch -> skip", () => {
|
||||
assert.equal(select({ providersToSearch: ["openai"] }).kind, "skip");
|
||||
});
|
||||
test("P-12 unavailable + future cooldown -> skip", () => {
|
||||
assert.equal(
|
||||
select({
|
||||
pinnedRow: { ...pin, testStatus: "unavailable", rateLimitedUntil: new Date(2_000_000_000).toISOString() },
|
||||
nowMs: 1_000_000,
|
||||
}).kind,
|
||||
"skip"
|
||||
);
|
||||
});
|
||||
test("P-13 unavailable + elapsed cooldown -> probe", () => {
|
||||
assert.equal(
|
||||
select({
|
||||
pinnedRow: { ...pin, testStatus: "unavailable", rateLimitedUntil: new Date(500_000).toISOString() },
|
||||
nowMs: 1_000_000,
|
||||
}).kind,
|
||||
"probe"
|
||||
);
|
||||
});
|
||||
|
||||
test("storm Map note then select within interval is suppressed", () => {
|
||||
resetExplicitProbeMapForTests();
|
||||
noteExplicitProbe("c1", 1e6);
|
||||
assert.equal(lastExplicitProbeTime("c1"), 1e6);
|
||||
assert.equal(
|
||||
select({ lastProbeAtMs: lastExplicitProbeTime("c1"), nowMs: 1e6 + 10_000 }).kind,
|
||||
"suppressed"
|
||||
);
|
||||
});
|
||||
|
||||
test("storm Map interval constant is 60s", () => {
|
||||
assert.equal(EXPLICIT_INACTIVE_PROBE_INTERVAL_MS, 60_000);
|
||||
});
|
||||
|
||||
test("storm Map reset clears last probe time", () => {
|
||||
resetExplicitProbeMapForTests();
|
||||
noteExplicitProbe("c1", 1e6);
|
||||
resetExplicitProbeMapForTests();
|
||||
assert.equal(lastExplicitProbeTime("c1"), null);
|
||||
});
|
||||
|
||||
test("storm Map evicts oldest when over 4096 entries", () => {
|
||||
resetExplicitProbeMapForTests();
|
||||
for (let i = 0; i < 4096; i++) {
|
||||
noteExplicitProbe(`id-${i}`, i);
|
||||
}
|
||||
noteExplicitProbe("overflow", 4096);
|
||||
assert.equal(lastExplicitProbeTime("id-0"), null);
|
||||
assert.equal(lastExplicitProbeTime("id-1"), 1);
|
||||
assert.equal(lastExplicitProbeTime("overflow"), 4096);
|
||||
});
|
||||
|
||||
test("W-6 openrouter :free does not reactivate", async () => {
|
||||
let called = 0;
|
||||
await maybeReactivateAfterExplicitProbe(
|
||||
{
|
||||
connectionId: "c1",
|
||||
reactivatedFromInactive: true,
|
||||
provider: "openrouter",
|
||||
requestedModel: "openrouter/foo:free",
|
||||
},
|
||||
async () => {
|
||||
called += 1;
|
||||
}
|
||||
);
|
||||
assert.equal(called, 0);
|
||||
});
|
||||
|
||||
test("maybeReactivateAfterExplicitProbe calls reactivate on recovered pin", async () => {
|
||||
let called = 0;
|
||||
let seenId = "";
|
||||
await maybeReactivateAfterExplicitProbe(
|
||||
{
|
||||
connectionId: "c1",
|
||||
reactivatedFromInactive: true,
|
||||
},
|
||||
async (id) => {
|
||||
called += 1;
|
||||
seenId = id;
|
||||
}
|
||||
);
|
||||
assert.equal(called, 1);
|
||||
assert.equal(seenId, "c1");
|
||||
});
|
||||
|
||||
test("maybeReactivateAfterExplicitProbe no-ops without reactivatedFromInactive", async () => {
|
||||
let called = 0;
|
||||
await maybeReactivateAfterExplicitProbe({ connectionId: "c1" }, async () => {
|
||||
called += 1;
|
||||
});
|
||||
assert.equal(called, 0);
|
||||
});
|
||||
|
||||
test("maybeReactivateAfterExplicitProbe no-ops when explicitProbeSuppressed", async () => {
|
||||
let called = 0;
|
||||
await maybeReactivateAfterExplicitProbe(
|
||||
{
|
||||
connectionId: "c1",
|
||||
reactivatedFromInactive: true,
|
||||
explicitProbeSuppressed: true,
|
||||
},
|
||||
async () => {
|
||||
called += 1;
|
||||
}
|
||||
);
|
||||
assert.equal(called, 0);
|
||||
});
|
||||
|
||||
test("maybeReactivateAfterExplicitProbe no-ops on shadow traffic", async () => {
|
||||
let called = 0;
|
||||
await maybeReactivateAfterExplicitProbe(
|
||||
{
|
||||
connectionId: "c1",
|
||||
reactivatedFromInactive: true,
|
||||
isShadowTraffic: true,
|
||||
},
|
||||
async () => {
|
||||
called += 1;
|
||||
}
|
||||
);
|
||||
assert.equal(called, 0);
|
||||
});
|
||||
|
||||
test("maybeReactivateAfterExplicitProbe no-ops when allowSuppressedConnections", async () => {
|
||||
let called = 0;
|
||||
await maybeReactivateAfterExplicitProbe(
|
||||
{
|
||||
connectionId: "c1",
|
||||
reactivatedFromInactive: true,
|
||||
allowSuppressedConnections: true,
|
||||
},
|
||||
async () => {
|
||||
called += 1;
|
||||
}
|
||||
);
|
||||
assert.equal(called, 0);
|
||||
});
|
||||
|
||||
test("W-7 chatHelpers onRequestSuccess calls maybeReactivateAfterExplicitProbe", async () => {
|
||||
const src = fs.readFileSync(new URL("../../src/sse/handlers/chatHelpers.ts", import.meta.url), "utf8");
|
||||
assert.match(src, /await maybeReactivateAfterExplicitProbe\(/);
|
||||
assert.match(src, /clearAccountError/);
|
||||
});
|
||||
|
||||
test("W-3 clearAccountError update payload has no isActive key", () => {
|
||||
const src = fs.readFileSync(new URL("../../src/sse/services/auth.ts", import.meta.url), "utf8");
|
||||
const start = src.indexOf("export async function clearAccountError");
|
||||
assert.ok(start >= 0, "clearAccountError must exist");
|
||||
const next = src.indexOf("\nexport ", start + 1);
|
||||
const body = next >= 0 ? src.slice(start, next) : src.slice(start);
|
||||
const updateStart = body.indexOf("await updateProviderConnection(");
|
||||
assert.ok(updateStart >= 0, "clearAccountError must call updateProviderConnection");
|
||||
const brace = body.indexOf("{", updateStart);
|
||||
assert.ok(brace >= 0);
|
||||
let depth = 0;
|
||||
let end = -1;
|
||||
for (let i = brace; i < body.length; i++) {
|
||||
if (body[i] === "{") depth += 1;
|
||||
else if (body[i] === "}") {
|
||||
depth -= 1;
|
||||
if (depth === 0) {
|
||||
end = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.ok(end > brace);
|
||||
const literal = body.slice(brace, end + 1);
|
||||
assert.equal(/\bisActive\b/.test(literal), false);
|
||||
});
|
||||
85
tests/unit/explicit-inactive-probe-w2.test.ts
Normal file
85
tests/unit/explicit-inactive-probe-w2.test.ts
Normal file
@@ -0,0 +1,85 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-explicit-inactive-w2-"));
|
||||
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "explicit-inactive-w2-test-secret";
|
||||
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const providersDb = await import("../../src/lib/db/providers.ts");
|
||||
const auth = await import("../../src/sse/services/auth.ts");
|
||||
const { maybeReactivateAfterExplicitProbe, resetExplicitProbeMapForTests } = await import(
|
||||
"../../src/sse/services/explicitInactiveProbe.ts"
|
||||
);
|
||||
|
||||
async function resetStorage() {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
||||
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
async function seedInactiveSiliconflow(testStatus: "active" | "credits_exhausted") {
|
||||
const row = await providersDb.createProviderConnection({
|
||||
provider: "siliconflow",
|
||||
authType: "apikey",
|
||||
name: "sf-inactive",
|
||||
apiKey: "sf-inactive-test-key",
|
||||
isActive: false,
|
||||
testStatus,
|
||||
});
|
||||
assert.ok(typeof row?.id === "string" && row.id.length > 0);
|
||||
return { id: row.id };
|
||||
}
|
||||
|
||||
function asPinnedCreds(creds: unknown) {
|
||||
assert.ok(creds);
|
||||
return creds as { connectionId?: string; reactivatedFromInactive?: boolean };
|
||||
}
|
||||
|
||||
test.beforeEach(async () => {
|
||||
resetExplicitProbeMapForTests();
|
||||
await resetStorage();
|
||||
});
|
||||
|
||||
test.after(async () => {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
||||
});
|
||||
|
||||
test("W-2 pin + inactive returns credentials after wiring (was null)", async () => {
|
||||
const row = await seedInactiveSiliconflow("active");
|
||||
const creds = asPinnedCreds(
|
||||
await auth.getProviderCredentials("siliconflow", null, null, "siliconflow/m", {
|
||||
forcedConnectionId: row.id,
|
||||
})
|
||||
);
|
||||
assert.equal(creds.connectionId, row.id);
|
||||
assert.equal(creds.reactivatedFromInactive, true);
|
||||
});
|
||||
|
||||
test("W-2 pin + credits_exhausted returns credentials after wiring", async () => {
|
||||
const row = await seedInactiveSiliconflow("credits_exhausted");
|
||||
const creds = asPinnedCreds(
|
||||
await auth.getProviderCredentials("siliconflow", null, null, "siliconflow/m", {
|
||||
forcedConnectionId: row.id,
|
||||
})
|
||||
);
|
||||
assert.equal(creds.connectionId, row.id);
|
||||
assert.equal(creds.reactivatedFromInactive, true);
|
||||
});
|
||||
|
||||
test("W-1 successful probe re-enables inactive pin in SQLite", async () => {
|
||||
const row = await seedInactiveSiliconflow("active");
|
||||
const before = await providersDb.getProviderConnectionById(row.id);
|
||||
assert.equal(before?.isActive, false);
|
||||
await maybeReactivateAfterExplicitProbe({
|
||||
reactivatedFromInactive: true,
|
||||
connectionId: row.id,
|
||||
});
|
||||
const after = await providersDb.getProviderConnectionById(row.id);
|
||||
assert.equal(after?.isActive, true);
|
||||
});
|
||||
@@ -112,6 +112,10 @@ test("BUG CASE: forced connection deactivated (missing from active pool) is dete
|
||||
);
|
||||
});
|
||||
|
||||
test("recoverable inactive pin is not missing-from-pool once in connections", () => {
|
||||
assert.equal(isForcedConnectionMissingFromPool("c1", new Set(), [{ id: "c1" }]), false);
|
||||
});
|
||||
|
||||
test("EXISTING BEHAVIOR: forced connection already excluded after a failed attempt is NOT missing-from-pool", () => {
|
||||
// The account is still present in the (active) pool — it 429'd and the retry loop
|
||||
// added it to excludedConnectionIds. This must keep going through
|
||||
|
||||
Reference in New Issue
Block a user