mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-12 02:02:13 +03:00
test(integration): add general live-test tool for the real "default" combo
Temporary WIP commit on this deferred branch — lands in its own separate PR once the bug-fix extraction batch is done (never bundled into a bug-fix PR). Unlike liveGeminiShared.ts (provisions its own narrow 2-model Gemini-only combo), this reads the REAL "default" combo currently configured on the target instance directly from the DB and exercises every provider/model step in it directly, bypassing combo routing, so live-test coverage always matches whatever is actually configured instead of a hardcoded snapshot. Live-verified against omniroute-beta (seeded with the real 18-model, 5-provider default combo): 14/18 models pass consistently across non-streaming + streaming Chat Completions and streaming Responses API. The 4 consistent failures are real external state (cerebras credits_exhausted, one deprecated openrouter free-tier model), not code regressions. (cherry picked from commit c40b13a48fd897259c56f5122e9e57a3dc7654ba)
This commit is contained in:
Markus Hartung
committed by
diegosouzapw
diegosouzapw
parent
aae408f585
commit
f133704267
113
tests/integration/live-default-combo-workload.test.ts
Normal file
113
tests/integration/live-default-combo-workload.test.ts
Normal file
@@ -0,0 +1,113 @@
|
||||
/**
|
||||
* tests/integration/live-default-combo-workload.test.ts
|
||||
*
|
||||
* General breadth test against the REAL, currently-configured "default"
|
||||
* combo on the target instance — unlike live-gemini-workload.test.ts (which
|
||||
* provisions its own narrow 2-model Gemini-only combo), this targets every
|
||||
* provider/model step the operator actually has in "default" directly,
|
||||
* bypassing combo routing. One request per configured model: non-streaming
|
||||
* + streaming Chat Completions, and streaming Responses API. Skips (never
|
||||
* fails) any model whose provider connection isn't currently active, so one
|
||||
* unrelated provider outage doesn't block the rest of the run.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
skip,
|
||||
getDefaultComboModelTargets,
|
||||
filterActiveModelTargets,
|
||||
sendModelRequest,
|
||||
} from "./liveDefaultComboShared.ts";
|
||||
|
||||
let modelNames: string[] = [];
|
||||
|
||||
test.before(async () => {
|
||||
if (skip) return;
|
||||
const targets = await getDefaultComboModelTargets();
|
||||
assert.ok(targets.length > 0, `"default" combo has no model steps — nothing to test`);
|
||||
|
||||
const { active, skipped } = await filterActiveModelTargets(targets);
|
||||
if (skipped.length > 0) {
|
||||
console.log(`\n [setup] skipping ${skipped.length} model(s) with inactive provider:`);
|
||||
for (const s of skipped) console.log(` - ${s}`);
|
||||
}
|
||||
|
||||
modelNames = active.map((t) => t.model);
|
||||
console.log(`\n [setup] testing ${modelNames.length} model(s) from the live "default" combo`);
|
||||
});
|
||||
|
||||
test(
|
||||
"[32] default combo: non-streaming chat completions across every configured model",
|
||||
{ skip },
|
||||
async () => {
|
||||
const failures: string[] = [];
|
||||
for (const model of modelNames) {
|
||||
const r = await sendModelRequest(model, false, "chat");
|
||||
if (r.status !== 200 || r.contentLength === 0) {
|
||||
failures.push(
|
||||
`${model}: HTTP ${r.status}${r.error ? ` (${r.error})` : ""}, ${r.contentLength} chars`
|
||||
);
|
||||
}
|
||||
}
|
||||
if (failures.length > 0) {
|
||||
console.log(`\n Non-streaming failures (${failures.length}/${modelNames.length}):`);
|
||||
for (const f of failures) console.log(` ${f}`);
|
||||
}
|
||||
assert.equal(
|
||||
failures.length,
|
||||
0,
|
||||
`${failures.length}/${modelNames.length} models failed non-streaming chat`
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
test(
|
||||
"[33] default combo: streaming chat completions across every configured model",
|
||||
{ skip },
|
||||
async () => {
|
||||
const failures: string[] = [];
|
||||
for (const model of modelNames) {
|
||||
const r = await sendModelRequest(model, true, "chat");
|
||||
if (r.status !== 200 || r.contentLength === 0) {
|
||||
failures.push(
|
||||
`${model}: HTTP ${r.status}${r.error ? ` (${r.error})` : ""}, ${r.contentLength} chars`
|
||||
);
|
||||
}
|
||||
}
|
||||
if (failures.length > 0) {
|
||||
console.log(`\n Streaming failures (${failures.length}/${modelNames.length}):`);
|
||||
for (const f of failures) console.log(` ${f}`);
|
||||
}
|
||||
assert.equal(
|
||||
failures.length,
|
||||
0,
|
||||
`${failures.length}/${modelNames.length} models failed streaming chat`
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
test(
|
||||
"[34] default combo: streaming responses API across every configured model",
|
||||
{ skip },
|
||||
async () => {
|
||||
const failures: string[] = [];
|
||||
for (const model of modelNames) {
|
||||
const r = await sendModelRequest(model, true, "responses");
|
||||
if (r.status !== 200 || r.contentLength === 0) {
|
||||
failures.push(
|
||||
`${model}: HTTP ${r.status}${r.error ? ` (${r.error})` : ""}, ${r.contentLength} chars`
|
||||
);
|
||||
}
|
||||
}
|
||||
if (failures.length > 0) {
|
||||
console.log(`\n Responses API failures (${failures.length}/${modelNames.length}):`);
|
||||
for (const f of failures) console.log(` ${f}`);
|
||||
}
|
||||
assert.equal(
|
||||
failures.length,
|
||||
0,
|
||||
`${failures.length}/${modelNames.length} models failed streaming Responses API`
|
||||
);
|
||||
}
|
||||
);
|
||||
247
tests/integration/liveDefaultComboShared.ts
Normal file
247
tests/integration/liveDefaultComboShared.ts
Normal file
@@ -0,0 +1,247 @@
|
||||
/**
|
||||
* tests/integration/liveDefaultComboShared.ts
|
||||
*
|
||||
* Shared utilities for the general "default combo" live workload test.
|
||||
* Unlike liveGeminiShared.ts (which provisions its own narrow 2-model
|
||||
* Gemini-only combo when "default" doesn't already exist), this reads the
|
||||
* REAL "default" combo currently configured on the target instance directly
|
||||
* from its own DB (src/lib/db/combos.ts — never raw SQL, per AGENTS.md) and
|
||||
* exercises every provider/model step in it directly, bypassing combo
|
||||
* routing, so live-test coverage always matches whatever the operator
|
||||
* actually has configured instead of a hardcoded snapshot that goes stale
|
||||
* the moment the combo changes.
|
||||
*/
|
||||
import {
|
||||
API_KEY,
|
||||
BASE_URL,
|
||||
readSSEStream,
|
||||
readResponsesSSEStream,
|
||||
genSystemMessage,
|
||||
genUserMessage,
|
||||
type Message,
|
||||
} from "./liveGeminiShared.ts";
|
||||
|
||||
export { API_KEY, BASE_URL };
|
||||
|
||||
export const skip = !API_KEY ? "OMNIROUTE_API_KEY not set — skipping live test" : undefined;
|
||||
|
||||
export interface ComboModelTarget {
|
||||
model: string;
|
||||
providerId: string | null;
|
||||
}
|
||||
|
||||
async function apiFetch(path: string, options: RequestInit = {}): Promise<Response> {
|
||||
return fetch(`${BASE_URL}${path}`, {
|
||||
...options,
|
||||
headers: {
|
||||
Authorization: `Bearer ${API_KEY}`,
|
||||
"Content-Type": "application/json",
|
||||
...options.headers,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Bootstrap seed used ONLY when the target instance has no "default" combo
|
||||
// at all — mirrors liveGeminiShared.ts's own DEFAULT_COMBO_CONFIG fallback,
|
||||
// generalized to the real multi-provider spread confirmed live against this
|
||||
// operator's own production "default" combo (5 providers, 18 models) rather
|
||||
// than Gemini alone. This is a creation fallback only: whenever a "default"
|
||||
// combo already exists on the target instance, its actual live config is
|
||||
// always what gets read and tested — this list never overrides it.
|
||||
const FALLBACK_COMBO_MODELS: { model: string; providerId: string }[] = [
|
||||
{ model: "opencode/big-pickle", providerId: "opencode" },
|
||||
{ model: "opencode/mimo-v2.5-free", providerId: "opencode" },
|
||||
{ model: "opencode/laguna-s-2.1-free", providerId: "opencode" },
|
||||
{ model: "openrouter/cohere/north-mini-code:free", providerId: "openrouter" },
|
||||
{ model: "openrouter/poolside/laguna-m.1:free", providerId: "openrouter" },
|
||||
{ model: "openrouter/nvidia/nemotron-3-ultra-550b-a55b:free", providerId: "openrouter" },
|
||||
{ model: "openrouter/nvidia/nemotron-3-super-120b-a12b:free", providerId: "openrouter" },
|
||||
{ model: "openrouter/nvidia/nemotron-3-nano-30b-a3b:free", providerId: "openrouter" },
|
||||
{ model: "openrouter/google/gemma-4-26b-a4b-it:free", providerId: "openrouter" },
|
||||
{ model: "openrouter/google/gemma-4-31b-it:free", providerId: "openrouter" },
|
||||
{ model: "openrouter/poolside/laguna-s-2.1:free", providerId: "openrouter" },
|
||||
{ model: "gemini/gemini-3.1-flash-lite", providerId: "gemini" },
|
||||
{ model: "gemini/gemma-4-31b-it", providerId: "gemini" },
|
||||
{ model: "gemini/gemma-4-26b-a4b-it", providerId: "gemini" },
|
||||
{ model: "mistral/mistral-large-latest", providerId: "mistral" },
|
||||
{ model: "cerebras/gemma-4-31b", providerId: "cerebras" },
|
||||
{ model: "cerebras/zai-glm-4.7", providerId: "cerebras" },
|
||||
{ model: "cerebras/gpt-oss-120b", providerId: "cerebras" },
|
||||
];
|
||||
|
||||
async function ensureDefaultComboExists(
|
||||
getComboByName: (name: string) => Promise<Record<string, unknown> | null>
|
||||
): Promise<void> {
|
||||
const existing = await getComboByName("default");
|
||||
if (existing) return;
|
||||
|
||||
console.log(` [setup] no "default" combo on this instance — creating fallback seed combo`);
|
||||
const { createCombo } = await import("../../src/lib/db/combos.ts");
|
||||
await createCombo({
|
||||
name: "default",
|
||||
strategy: "priority",
|
||||
models: FALLBACK_COMBO_MODELS.map((m, i) => ({
|
||||
kind: "model" as const,
|
||||
model: m.model,
|
||||
providerId: m.providerId,
|
||||
weight: 1,
|
||||
id: `fallback-${i}`,
|
||||
})),
|
||||
});
|
||||
}
|
||||
|
||||
// Read the live "default" combo's model steps straight from the DB module —
|
||||
// intentionally not hardcoded, so this always reflects whatever the operator
|
||||
// currently has configured on the target instance. Creates a fallback seed
|
||||
// combo first if none exists at all (see ensureDefaultComboExists above).
|
||||
export async function getDefaultComboModelTargets(): Promise<ComboModelTarget[]> {
|
||||
const { getComboByName } = await import("../../src/lib/db/combos.ts");
|
||||
await ensureDefaultComboExists(getComboByName);
|
||||
const combo = (await getComboByName("default")) as Record<string, unknown> | null;
|
||||
const models =
|
||||
combo && Array.isArray(combo.models) ? (combo.models as Record<string, unknown>[]) : [];
|
||||
|
||||
const targets: ComboModelTarget[] = [];
|
||||
for (const step of models) {
|
||||
if (step.kind !== "model" || typeof step.model !== "string") continue;
|
||||
targets.push({
|
||||
model: step.model,
|
||||
providerId: typeof step.providerId === "string" ? step.providerId : null,
|
||||
});
|
||||
}
|
||||
return targets;
|
||||
}
|
||||
|
||||
// Skip (never fail) any model whose provider connection isn't currently
|
||||
// active — this suite's job is breadth across the real combo, not blocking
|
||||
// the whole run on one unrelated provider outage.
|
||||
export async function filterActiveModelTargets(
|
||||
targets: ComboModelTarget[]
|
||||
): Promise<{ active: ComboModelTarget[]; skipped: string[] }> {
|
||||
const res = await apiFetch("/api/providers");
|
||||
if (!res.ok) return { active: targets, skipped: [] };
|
||||
|
||||
const data = await res.json();
|
||||
const connections = (data.connections || data) as Record<string, unknown>[];
|
||||
// Terminal states (never self-heal — see AGENTS.md "Resilience Runtime
|
||||
// State" → Connection Cooldown) plus "unavailable" (active cooldown) are
|
||||
// the only statuses worth pre-filtering; everything else (including
|
||||
// transient/lazily-recovered cooldowns that have already expired) is left
|
||||
// for the request itself to prove out.
|
||||
const DEAD_STATUSES = new Set(["expired", "unavailable", "banned", "credits_exhausted"]);
|
||||
const activeProviders = new Set(
|
||||
connections
|
||||
.filter((c) => c.isActive && !DEAD_STATUSES.has(c.testStatus as string))
|
||||
.map((c) => c.provider as string)
|
||||
);
|
||||
|
||||
const active: ComboModelTarget[] = [];
|
||||
const skipped: string[] = [];
|
||||
for (const t of targets) {
|
||||
if (!t.providerId || activeProviders.has(t.providerId)) {
|
||||
active.push(t);
|
||||
} else {
|
||||
skipped.push(`${t.model} (provider "${t.providerId}" not active)`);
|
||||
}
|
||||
}
|
||||
return { active, skipped };
|
||||
}
|
||||
|
||||
function ts(): string {
|
||||
return new Date().toISOString().slice(11, 23); // HH:MM:SS.mmm
|
||||
}
|
||||
|
||||
export interface ModelRequestResult {
|
||||
model: string;
|
||||
status: number;
|
||||
duration: number;
|
||||
tokens: number;
|
||||
contentLength: number;
|
||||
correlationId: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
// Deliberately lighter than liveGeminiShared's sendAndValidate (no retry
|
||||
// loop, one fixed prompt pair): this suite's job is breadth across every
|
||||
// model in the real combo, not depth on any single provider.
|
||||
export async function sendModelRequest(
|
||||
model: string,
|
||||
stream: boolean,
|
||||
apiFormat: "chat" | "responses" = "chat"
|
||||
): Promise<ModelRequestResult> {
|
||||
const endpoint = apiFormat === "responses" ? "/v1/responses" : "/v1/chat/completions";
|
||||
const messages: Message[] = [genSystemMessage(), genUserMessage()];
|
||||
const body =
|
||||
apiFormat === "responses"
|
||||
? { model, input: messages, stream, max_output_tokens: 1024, temperature: 0.3 }
|
||||
: { model, messages, stream, max_tokens: 1024, temperature: 0.3 };
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeoutMs = Number(process.env.TEST_REQUEST_TIMEOUT_MS) || 120_000;
|
||||
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
||||
const start = performance.now();
|
||||
|
||||
try {
|
||||
const response = await fetch(`${BASE_URL}${endpoint}`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Authorization: `Bearer ${API_KEY}` },
|
||||
body: JSON.stringify(body),
|
||||
signal: controller.signal,
|
||||
});
|
||||
const duration = performance.now() - start;
|
||||
clearTimeout(timeout);
|
||||
const correlationId = response.headers.get("x-correlation-id") || "?";
|
||||
|
||||
let content = "";
|
||||
let totalTokens = 0;
|
||||
|
||||
if (response.status === 200) {
|
||||
if (stream) {
|
||||
const streamResult =
|
||||
apiFormat === "responses"
|
||||
? await readResponsesSSEStream(response)
|
||||
: await readSSEStream(response);
|
||||
content = streamResult.fullContent;
|
||||
totalTokens = streamResult.totalTokens;
|
||||
} else if (apiFormat === "responses") {
|
||||
const json = await response.json().catch(() => ({}));
|
||||
const textItem = json?.output?.find((o: Record<string, unknown>) => o.type === "message");
|
||||
content = textItem?.content?.[0]?.text || "";
|
||||
totalTokens = json?.usage?.total_tokens || 0;
|
||||
} else {
|
||||
const json = await response.json().catch(() => ({}));
|
||||
content = json?.choices?.[0]?.message?.content || "";
|
||||
totalTokens = json?.usage?.total_tokens || 0;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`${ts()} ${model.padEnd(40)} HTTP ${response.status} | ` +
|
||||
`${Math.round(duration).toString().padStart(6)}ms | ` +
|
||||
`${String(totalTokens).padStart(5)} tok | ` +
|
||||
`${content.length} chars | cid: ${correlationId}`
|
||||
);
|
||||
|
||||
return {
|
||||
model,
|
||||
status: response.status,
|
||||
duration,
|
||||
tokens: totalTokens,
|
||||
contentLength: content.length,
|
||||
correlationId,
|
||||
};
|
||||
} catch (err) {
|
||||
clearTimeout(timeout);
|
||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||
console.log(`${ts()} ${model.padEnd(40)} FAILED: ${errorMessage}`);
|
||||
return {
|
||||
model,
|
||||
status: 0,
|
||||
duration: performance.now() - start,
|
||||
tokens: 0,
|
||||
contentLength: 0,
|
||||
correlationId: "?",
|
||||
error: errorMessage,
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user