/** * tests/integration/liveDefaultComboShared.ts * * Shared utilities for the general "default combo" live workload test. * Unlike liveGeminiShared.ts (which provisions its own narrow 2-model * Gemini-only combo when "default" doesn't already exist), this reads the * REAL "default" combo currently configured on the target instance directly * from its own DB (src/lib/db/combos.ts — never raw SQL, per AGENTS.md) and * exercises every provider/model step in it directly, bypassing combo * routing, so live-test coverage always matches whatever the operator * actually has configured instead of a hardcoded snapshot that goes stale * the moment the combo changes. */ import { API_KEY, BASE_URL, readSSEStream, readResponsesSSEStream, genSystemMessage, genUserMessage, type Message, } from "./liveGeminiShared.ts"; export { API_KEY, BASE_URL }; export const skip = !API_KEY ? "OMNIROUTE_API_KEY not set — skipping live test" : undefined; export interface ComboModelTarget { model: string; providerId: string | null; } async function apiFetch(path: string, options: RequestInit = {}): Promise { return fetch(`${BASE_URL}${path}`, { ...options, headers: { Authorization: `Bearer ${API_KEY}`, "Content-Type": "application/json", ...options.headers, }, }); } // Bootstrap seed used ONLY when the target instance has no "default" combo // at all — mirrors liveGeminiShared.ts's own DEFAULT_COMBO_CONFIG fallback, // generalized to the real multi-provider spread confirmed live against this // operator's own production "default" combo (5 providers, 18 models) rather // than Gemini alone. This is a creation fallback only: whenever a "default" // combo already exists on the target instance, its actual live config is // always what gets read and tested — this list never overrides it. const FALLBACK_COMBO_MODELS: { model: string; providerId: string }[] = [ { model: "opencode/big-pickle", providerId: "opencode" }, { model: "opencode/mimo-v2.5-free", providerId: "opencode" }, { model: "opencode/laguna-s-2.1-free", providerId: "opencode" }, { model: "openrouter/cohere/north-mini-code:free", providerId: "openrouter" }, { model: "openrouter/poolside/laguna-m.1:free", providerId: "openrouter" }, { model: "openrouter/nvidia/nemotron-3-ultra-550b-a55b:free", providerId: "openrouter" }, { model: "openrouter/nvidia/nemotron-3-super-120b-a12b:free", providerId: "openrouter" }, { model: "openrouter/nvidia/nemotron-3-nano-30b-a3b:free", providerId: "openrouter" }, { model: "openrouter/google/gemma-4-26b-a4b-it:free", providerId: "openrouter" }, { model: "openrouter/google/gemma-4-31b-it:free", providerId: "openrouter" }, { model: "openrouter/poolside/laguna-s-2.1:free", providerId: "openrouter" }, { model: "gemini/gemini-3.1-flash-lite", providerId: "gemini" }, { model: "gemini/gemma-4-31b-it", providerId: "gemini" }, { model: "gemini/gemma-4-26b-a4b-it", providerId: "gemini" }, { model: "mistral/mistral-large-latest", providerId: "mistral" }, { model: "cerebras/gemma-4-31b", providerId: "cerebras" }, { model: "cerebras/zai-glm-4.7", providerId: "cerebras" }, { model: "cerebras/gpt-oss-120b", providerId: "cerebras" }, ]; async function ensureDefaultComboExists( getComboByName: (name: string) => Promise | null> ): Promise { const existing = await getComboByName("default"); if (existing) return; console.log(` [setup] no "default" combo on this instance — creating fallback seed combo`); const { createCombo } = await import("../../src/lib/db/combos.ts"); await createCombo({ name: "default", strategy: "priority", models: FALLBACK_COMBO_MODELS.map((m, i) => ({ kind: "model" as const, model: m.model, providerId: m.providerId, weight: 1, id: `fallback-${i}`, })), }); } // Read the live "default" combo's model steps straight from the DB module — // intentionally not hardcoded, so this always reflects whatever the operator // currently has configured on the target instance. Creates a fallback seed // combo first if none exists at all (see ensureDefaultComboExists above). export async function getDefaultComboModelTargets(): Promise { const { getComboByName } = await import("../../src/lib/db/combos.ts"); await ensureDefaultComboExists(getComboByName); const combo = (await getComboByName("default")) as Record | null; const models = combo && Array.isArray(combo.models) ? (combo.models as Record[]) : []; const targets: ComboModelTarget[] = []; for (const step of models) { if (step.kind !== "model" || typeof step.model !== "string") continue; targets.push({ model: step.model, providerId: typeof step.providerId === "string" ? step.providerId : null, }); } return targets; } // Skip (never fail) any model whose provider connection isn't currently // active — this suite's job is breadth across the real combo, not blocking // the whole run on one unrelated provider outage. baseUrl/apiKey default to // the module-level omniroute-beta target but can be overridden (see // sendModelRequest — same rationale, used by the wire-capture suite's // dedicated container). export async function filterActiveModelTargets( targets: ComboModelTarget[], options: SendModelRequestOptions = {} ): Promise<{ active: ComboModelTarget[]; skipped: string[] }> { const baseUrl = options.baseUrl ?? BASE_URL; const apiKey = options.apiKey ?? API_KEY; const res = await fetch(`${baseUrl}/api/providers`, { headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" }, }); if (!res.ok) return { active: targets, skipped: [] }; const data = await res.json(); const connections = (data.connections || data) as Record[]; // Terminal states (never self-heal — see AGENTS.md "Resilience Runtime // State" → Connection Cooldown) plus "unavailable" (active cooldown) are // the only statuses worth pre-filtering; everything else (including // transient/lazily-recovered cooldowns that have already expired) is left // for the request itself to prove out. const DEAD_STATUSES = new Set(["expired", "unavailable", "banned", "credits_exhausted"]); const activeProviders = new Set( connections .filter((c) => c.isActive && !DEAD_STATUSES.has(c.testStatus as string)) .map((c) => c.provider as string) ); const active: ComboModelTarget[] = []; const skipped: string[] = []; for (const t of targets) { if (!t.providerId || activeProviders.has(t.providerId)) { active.push(t); } else { skipped.push(`${t.model} (provider "${t.providerId}" not active)`); } } return { active, skipped }; } function ts(): string { return new Date().toISOString().slice(11, 23); // HH:MM:SS.mmm } export interface ModelRequestResult { model: string; status: number; duration: number; tokens: number; contentLength: number; correlationId: string; error?: string; } export interface SendModelRequestOptions { baseUrl?: string; apiKey?: string; } // Deliberately lighter than liveGeminiShared's sendAndValidate (no retry // loop, one fixed prompt pair): this suite's job is breadth across every // model in the real combo, not depth on any single provider. baseUrl/apiKey // default to the module-level omniroute-beta target but can be overridden — // e.g. by the wire-capture suite, which points requests at its own // dedicated throwaway container instead (see liveContainerHarness.ts). export async function sendModelRequest( model: string, stream: boolean, apiFormat: "chat" | "responses" = "chat", options: SendModelRequestOptions = {} ): Promise { const baseUrl = options.baseUrl ?? BASE_URL; const apiKey = options.apiKey ?? API_KEY; const endpoint = apiFormat === "responses" ? "/v1/responses" : "/v1/chat/completions"; const messages: Message[] = [genSystemMessage(), genUserMessage()]; const body = apiFormat === "responses" ? { model, input: messages, stream, max_output_tokens: 1024, temperature: 0.3 } : { model, messages, stream, max_tokens: 1024, temperature: 0.3 }; const controller = new AbortController(); const timeoutMs = Number(process.env.TEST_REQUEST_TIMEOUT_MS) || 120_000; const timeout = setTimeout(() => controller.abort(), timeoutMs); const start = performance.now(); try { const response = await fetch(`${baseUrl}${endpoint}`, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` }, body: JSON.stringify(body), signal: controller.signal, }); const duration = performance.now() - start; clearTimeout(timeout); const correlationId = response.headers.get("x-correlation-id") || "?"; let content = ""; let totalTokens = 0; if (response.status === 200) { if (stream) { const streamResult = apiFormat === "responses" ? await readResponsesSSEStream(response) : await readSSEStream(response); content = streamResult.fullContent; totalTokens = streamResult.totalTokens; } else if (apiFormat === "responses") { const json = await response.json().catch(() => ({})); const textItem = json?.output?.find((o: Record) => o.type === "message"); content = textItem?.content?.[0]?.text || ""; totalTokens = json?.usage?.total_tokens || 0; } else { const json = await response.json().catch(() => ({})); content = json?.choices?.[0]?.message?.content || ""; totalTokens = json?.usage?.total_tokens || 0; } } console.log( `${ts()} ${model.padEnd(40)} HTTP ${response.status} | ` + `${Math.round(duration).toString().padStart(6)}ms | ` + `${String(totalTokens).padStart(5)} tok | ` + `${content.length} chars | cid: ${correlationId}` ); return { model, status: response.status, duration, tokens: totalTokens, contentLength: content.length, correlationId, }; } catch (err) { clearTimeout(timeout); const errorMessage = err instanceof Error ? err.message : String(err); console.log(`${ts()} ${model.padEnd(40)} FAILED: ${errorMessage}`); return { model, status: 0, duration: performance.now() - start, tokens: 0, contentLength: 0, correlationId: "?", error: errorMessage, }; } }