mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
Release v3.8.38 (#5078)
* chore(release): open v3.8.38 development cycle
* fix(executors): strip client_metadata for cerebras and mistral (#4727)
Integrated into release/v3.8.38 (leva 5)
* fix(codebuddy): only send reasoning params when client requests reasoning (#5019)
Integrated into release/v3.8.38 (leva 5)
* fix(sse): keep streaming for forceStream providers when client requests JSON (#5021)
Integrated into release/v3.8.38 (leva 5)
* fix(sse): guard non-JSON SSE lines and duplicate [DONE] (#4937)
Integrated into release/v3.8.38 (leva 5)
* feat(blackbox): refresh provider model catalog (#4935)
Integrated into release/v3.8.38 (leva 5)
* fix(sse): dedupe case-variant Anthropic version/beta headers (#4846)
Integrated into release/v3.8.38 (leva 5)
* feat(sse): Kiro inline <thinking> stream splitter (#4911)
Integrated into release/v3.8.38 (leva 5)
* feat(cursor): parse Composer DeepSeek-style inline tool calls (#4912)
Integrated into release/v3.8.38 (leva 5)
* feat(proxy): auth-less host:port batch import (#4938)
Integrated into release/v3.8.38 (leva 5)
* fix(oauth): support Kiro IDC (organization) token import (#4944)
Integrated into release/v3.8.38 (leva 5)
* fix(translator): preserve cache_control for DashScope OpenAI-compat providers (port from 9router#2069) (#5013)
Integrated into release/v3.8.38 (leva 5)
* fix(tts): resolve Gemini TTS models from catalog (#4934)
Integrated into release/v3.8.38 (leva 5)
* fix(sse): don't cool down the connection on a self-inflicted upstream timeout (504) (#5064)
Integrated into release/v3.8.38 (leva 5)
* fix(sse): robust Anthropic /v1/messages streaming — real ping keepalive + client-disconnect guard (#5063)
Integrated into release/v3.8.38 (leva 5)
* feat(video): add Alibaba DashScope (wan2.7-t2v) provider (#5051)
Integrated into release/v3.8.38 (leva 5)
* fix: preserve model hidden flags (isHidden) across model sync (#5086)
Integrated into release/v3.8.38 (leva 5)
* fix(models): derive model discovery config from registry modelsUrl (#5087)
Integrated into release/v3.8.38 (leva 5)
* fix(compression): replace fileURLToPath(import.meta.url) with runtime anchors for standalone bundle (#5089)
Integrated into release/v3.8.38 (leva 5)
* feat(cc): add summarized thinking display toggle (#5055)
Integrated into release/v3.8.38 (leva 5)
* Harden selected API error responses (#5032)
Integrated into release/v3.8.38 (leva 5)
* chore(quality): rebaseline file-size for leva 5 PR batch drift
6 frozen files grew from merged leva-5 PRs (cursor #4912, kiro #4911,
videoGeneration #5051, default #4727, base #4846, chat #5064); all covered
by per-PR tests. See _rebaseline_2026_06_26_leva5 in the baseline.
* feat(compression): compression playground (Play + Compare tabs) in the studio (#5080)
Integrated into release/v3.8.38
* fix(combo): fail over on empty-content 502 instead of exhausting the provider (#5085) (#5104)
* fix(dashboard): surface detailed credential-validation error in add-connection modal (#5088) (#5106)
* feat(providers): allow local/private provider URLs by default with scoped metadata-safe guard (#5066) (#5107)
* fix(diagnostics): treat non-streaming Claude messages shape as valid output (#5108) (#5116)
* fix(db): translate pt-BR SQLite driver-fallback log lines to English (#5103) (#5115)
* fix(sse): repair release base-reds — malformed-response false positives + header casing + stale tests (#5117)
Repairs the release/v3.8.38 base-reds; unblocks #5078.
* chore(quality): rebaseline file-size for responseSanitizer (#5117) + AddApiKeyModal drift
* fix(translator): forward image tool_result blocks as image_url (#5100)
Base-reds fixed (#5117); image tool_result→image_url. Integrated into release/v3.8.38.
* fix(responses): default text.format for openai-compatible responses providers (#5101)
Base-reds fixed (#5117); default text.format + file-size rebaseline. Integrated into release/v3.8.38.
* feat(dashboard): expose Fusion judgeModel + fusionTuning in the combo editor (#5074)
Base-reds fixed (#5117); Fusion editor + file-size rebaseline. Integrated into release/v3.8.38.
* feat(quota): add opt-in Codex/Claude auto-ping keepalive (#5102)
Base-reds fixed (#5117); auto-ping keepalive + file-size rebaseline. Integrated into release/v3.8.38.
* test(release): relocate 2 orphan test files into the collected flat tests/unit dir (#5120)
Unblocks Lint (test-discovery) on #5078. Integrated into release/v3.8.38.
* fix(translator): preserve reasoning-replay reasoning_content + repair 3 release-green test reds (#5122)
Repairs 3 release-green test reds + test-masking; unblocks #5078.
* test(golden): redact live Node version from provider translate-path snapshot (#5125)
Final golden unblock for #5078.
* test(golden): redact OmniRoute app version from translate-path snapshot (#5126)
Coverage shard golden unblock for #5078.
* Ignore disconnect races during in-band stream error handling (#5007)
Integrated into release/v3.8.38
* Track final connection IDs in failover logs (#5016)
Integrated into release/v3.8.38
* fix(sse): convert Gemini body to OpenAI format in antigravity MITM handler (#4845)
Integrated into release/v3.8.38 (rebased on tip, CHANGELOG re-injected)
* feat(providers): add ZenMux Free session-cookie provider (#5105)
Integrated into release/v3.8.38 (rebased on tip, CHANGELOG re-injected)
* feat(dashboard): click-to-edit model alias in provider page (#5119)
Integrated into release/v3.8.38 (rebased on tip, i18n scope verified, CHANGELOG re-injected)
* feat(mcp): web-session robustness — cookie dedup (PR6) + browser-pool observability (PR7) (#3368) (#5121)
Integrated into release/v3.8.38 (rebased on tip; cookie-dedup branch extracted to findExistingCookieConnection helper → complexity-neutral; CHANGELOG added)
* fix(usage): dedupe request-usage logging and debounce stats (#4940)
Integrated into release/v3.8.38 (rebased on tip; DB-handle hang was stale-base artifact — resetDbInstance already closes the handle, test green 5/5; file-size drift consolidated at release; CHANGELOG re-injected)
* fix(dashboard): key model visibility toggle on canonical providerId (#5091)
Integrated into release/v3.8.38 (retargeted main→release; .tsx visibility-key test green 2/2)
* chore(deps): bump actions/cache from 5.0.5 to 6.0.0 (#5112)
Integrated into release/v3.8.38 (retargeted main→release; workflow-only actions/cache bump — unit failures were stale main base-reds)
* fix(streaming): harden long OpenAI-compatible SSE streams (#5124)
Integrated into release/v3.8.38 (rebased on tip; streamHandler conflict with #5007 disconnect-guard resolved — both coexist, stream-handler 22/22 green)
* feat: Add Grok Build (xAI) provider with OAuth import-token flow (#5020)
Integrated into release/v3.8.38 (rebased on tip; Hard Rule #11 fix — Grok public client_id now via resolvePublicCred(grok_id), 3 literals removed; grok-oauth 7/7 + check:public-creds green)
* feat(providers): add Factory (factory.ai) as a subscription gateway provider (#5065)
Integrated into release/v3.8.38 (rebased on tip; added factory registry test for PR Test Policy + fixed check:env-doc-sync phantom FACTORY_API_KEY; factory loads in PROVIDERS, no Zod issue — that flag was a false positive)
* chore(test): reconcile golden snapshot + apikey count for new providers
#5020 (grok-cli), #5065 (factory), #5105 (zenmux-free) added providers but did
not regenerate tests/snapshots/provider/translate-path.json (now +3 entries) nor
bump the APIKEY_PROVIDERS count (159->160 for the factory gateway). Test-only
reconciliation; no production change.
* fix(resilience): harden quota and model lockout edge cases (#5093)
Integrated into release/v3.8.38 (rebased on tip). TRUST-BUT-VERIFY: dropped the PR's 0dd7df641 'fix unit gates' commit which reverted #5122 reasoning-replay (preserveReasoningContent) + re-introduced #4849 O(n^2) growth, and restored 5 tests it had realigned. Kept only the 3 declared resilience fixes (quota cutoff guard, gemini MIME, model-lockout maxCooldownMs); 23/23 green.
* Hydrate quota cache and scope auto combo candidates (#5015)
Integrated into release/v3.8.38 (rebased on tip). Kept core quota-cache hydration + auto-combo candidate scoping + combos UI; dropped out-of-scope toolCloaking refactor (conflicted with #4813 stripEnumDescriptions — took tip) and the unrelated sse-auth test split. Added quota-cache-hydrate-5015 regression test (Rule #18); combo-account-allowlist 8/8 + hydration 2/2 green.
* chore(quality): reconcile complexity + file-size baselines for v3.8.38 owner-PR batch
complexity 1972->1978 (+6) and file-size providers.ts 1093->1107 / usageHistory.ts
934->983 — drift from the /review-prs merge batch (#4845/#5105/#5020/#4940/#5093/
#5015 + #5121 cookie-dedup helper extraction). check:complexity/check:file-size do
not run on the PR->release fast-path, so the branch accrued unmeasured; all legit
feature/fix growth, not regression. See per-key justifications in each baseline.
* fix(security): exact-host Anthropic baseUrl check (CodeQL js/incomplete-url-substring-sanitization #674) (#5130)
The anthropic-compatible Bearer-fallback gate decided whether a configured baseUrl
targeted the official api.anthropic.com host via a substring `.includes("api.anthropic.com")`.
A look-alike upstream such as `https://api.anthropic.com.evil.test` or
`https://evil.test/?x=api.anthropic.com` matched the substring and was wrongly treated as
official, suppressing the Bearer fallback meant for third-party gateways
(CodeQL #674, js/incomplete-url-substring-sanitization, high).
Replace the substring test with an exported `isOfficialAnthropicBaseUrl()` helper that
parses the URL and compares the hostname for exact equality. Empty baseUrl stays official;
scheme-less hosts are parsed with an assumed https://; an unparseable baseUrl falls back to
third-party (Bearer emitted) as the safer default. Behavior for legitimate official/third-party
baseUrls is unchanged.
Adds tests/unit/anthropic-official-baseurl-host.test.ts covering official, look-alike,
scheme-less, and unparseable inputs plus a static guard that the substring pattern is gone.
* fix(proxy): repair one-click Deno & Cloudflare relay deployments (#5128) (#5132)
* fix(services): embed WS proxy honours LIVE_WS_HOST; reject empty messages early (#5110) (#5133)
* fix(api): resolve /v1/models/{id} case-insensitively (#5082) (#5135)
* fix(providers): add MiniMax M3 & Nemotron 3 Ultra to Cline catalog (#3321) (#5136)
* fix(proxy): make SOCKS5 handshake timeout tunable via SOCKS_HANDSHAKE_TIMEOUT_MS (#5109) (#5137)
* feat(sidebar): add support for colored menu icons (#3812)
Integrated into release/v3.8.38 (recreated on tip — fork had unrelated history; added getSidebarIconAccent regression test, Rule #18). Clean 2-file UI feature.
* fix(providers): complete grok-cli OAuth wiring + zenmux-free web-session metadata
Base-red repair for #5020 (grok-cli) and #5105 (zenmux-free), surfaced by the
full CI on the release PR (#5078) — the PR->release fast-path does not run the
oauth-providers-config / web-session-credentials / provider-consistency gates.
- grok-cli: register in OAUTH_PROVIDERS (providers.ts canonical list, fixes
check:provider-consistency), add OAUTH_PROVIDER_IDS.GROK_CLI + GROK_CLI_CONFIG
in oauth constants (provider config now sourced there, not a local literal),
align oauth-providers-config.test.ts (EXPECTED_PROVIDER_KEYS + config map).
- zenmux-free: declare its web-session credential requirement (full Cookie header)
in WEB_SESSION_CREDENTIAL_REQUIREMENTS.
Local: oauth-providers-config 27/27, web-session-credentials 4/4, grok-cli-oauth
7/7, check:provider-consistency OK, +115 OAUTH_PROVIDERS tests green.
* Fix resilience settings page response mapping (#5139)
Integrated into release/v3.8.38. Thanks @rdself for the fix and the regression test.
* fix(kiro): retire claude-sonnet-4.5 from catalog + pin 400 model-unavailable test (#5140)
Extracted the real change from #5140 (the bot PR regenerated the entire
freeModelCatalog.data.ts + touched package-lock.json; only the targeted
edits are kept here):
- remove claude-sonnet-4.5 from the Kiro registry entry
- remove the matching kiro free-model catalog row
- pin Kiro's verbatim 400 "Invalid model..." to isModelUnavailableError
Closes #4484
* fix(sidebar): drop orphan `settings` accent color (typecheck:core red) (#5142)
SIDEBAR_ICON_ACCENTS is typed Partial<Record<HideableSidebarItemId, string>>,
but `settings` is not a hideable item id (only `settings-general`,
`settings-appearance`, … and `context-settings` exist; there is no item with
`id: "settings"`), so the accent was unreachable. It broke `typecheck:core`
on the release tip ("'settings' does not exist in type …", introduced by
#3812 colored menu icons). Removing the orphan key restores a clean
typecheck:core (rc=0).
* feat: salvage batch 2 — diagnostics null-guard (#5096) + observed quota reset windows (#5025) (#5141)
* fix(diagnostics): null-guard content blocks in detectMalformedNonStream
A null (or non-object) entry in a Claude-native `content` array made the
non-stream classifier throw `TypeError: Cannot read properties of null
(reading 'type')`, crashing the malformed-response detection path. Guard
before type-asserting each block: a null/non-object block is simply skipped.
Two regression tests added (null block among valid blocks → null; only-null
blocks → empty_choices).
Salvaged from closed PR #5096 (base-stale; only the defensive guard — the
Claude-shape recognition it also carried already landed via #5108).
Co-authored-by: herjarsa <herjarsa@users.noreply.github.com>
* feat(quota): persist observed provider quota reset windows
Adds `provider_quota_reset_events` (migration 108) + `db/quotaResetEvents.ts`
to record real upstream weekly-quota window transitions whenever a quota
refresh shows the reset rolling to a new cycle (different day, later resetAt).
`apiKeyUsageLimits` now prefers the observed window start over the inferred
`resetAt − 7d`, falling back to snapshot inference when no event is recorded
yet. `quotaCache.setQuotaCache` records the transition opportunistically.
`recordProviderQuotaResetEventIfChanged` only fires for the primary weekly
window (not daily/sonnet), is idempotent (INSERT OR IGNORE on the unique
window key), and no-ops when the reset didn't actually roll. 4 unit tests
(tests/unit/lib/quota-reset-events.test.ts).
Salvaged from closed PR #5025 (which bundled this with two unrelated
features + a colliding migration 104). Renumbered to 108; module re-exported
from localDb (Rule #2).
Co-authored-by: Witroch4 <175152067+Witroch4@users.noreply.github.com>
---------
Co-authored-by: herjarsa <herjarsa@users.noreply.github.com>
Co-authored-by: Witroch4 <175152067+Witroch4@users.noreply.github.com>
* docs(i18n): sync 3.8.38 CHANGELOG section to 41 mirrors (unblock docs-accuracy) (#5144)
The root CHANGELOG [3.8.38] section grew with this cycle's merged PRs, but the
docs/i18n/<lang>/CHANGELOG.md mirrors were not re-synced — drifting >25% in body
size and failing check:docs-sync (the "Docs accuracy" fast-gate step) for every
open PR against the release.
Ran scripts/release/sync-changelog-i18n.mjs 3.8.38 3.8.37 to copy the root
[3.8.38] section into all 41 mirrors. check:docs-all now passes (exit 0).
Sections are copied verbatim; the per-language translation pass runs at release
time via i18n:run — this only restores the size-sync the gate enforces.
* feat(compression): pure per-step fidelity checker (4 invariants, fail-open)
* feat(compression): fidelityGate config + rejected breakdown fields
* feat(compression): wire per-step fidelity gate into stacked pipeline (opt-in)
* feat(compression): preview route accepts fidelityGate flag (playground)
* feat(compression): playground fidelity-gate toggle + lane rejection display
* docs(compression): note fidelityGate advanced thresholds are intentionally API-omitted
* refactor(compression): extract fidelity-gate step helpers to shrink strategySelector (file-size gate)
bodyToText and gateAdvance moved to fidelityGateStep.ts; StackAccumulator exported.
strategySelector: 889->854 (-35). Residual +6 vs pre-Milestone-B frozen 848 is the
irreducible StackOptions.fidelityGate field + two stacked-loop dispatch reads + import.
Baseline updated to 854 with justification. No cycle introduced (import type only).
940 compression tests pass; typecheck clean.
* test(usage): wire usageHistoryDedup under unit runner brace-list (#5145)
Integrated into release/v3.8.38.
* feat: salvage batch from closed stale PRs (#5038, #5057, #5076) (#5138)
Integrated into release/v3.8.38.
* test(combo): deterministic routing-decision matrix for all 17 strategies (#5146)
Integrated into release/v3.8.38.
* feat(compression): fuzzy near-duplicate dedup (session-dedup 2nd pass + playground toggle) (#5143)
Integrated into release/v3.8.38.
* chore(quality): rebaseline file-size for sidebarVisibility.ts + chat.ts drift (#5147)
Mid-cycle drift on release/v3.8.38 from already-merged PRs that the fast-path
(PR->release skips check:file-size) let accumulate without a bump:
- src/shared/constants/sidebarVisibility.ts 1100->1198 (#3812 colored menu
icons, per-item accent map; #5142 dropped one orphan, net still above frozen)
- src/sse/handlers/chat.ts 1560->1575 (#5064 self-inflicted-timeout cooldown
skip + #5124 long OpenAI-compatible SSE hardening + #5110 embed-WS
LIVE_WS_HOST honour / early empty-message reject)
Each covered by its own PR tests; structural shrink of chat.ts tracked in #3501.
Unblocks the Fast Quality Gates for PRs targeting release/v3.8.38.
* chore(release): finalize v3.8.38 CHANGELOG + cycle reconciliation
- Reconcile [3.8.38]: +18 bullets (compression fidelity-gate/fuzzy-dedup #5143,
quota keepalive #5102, web-session robustness #5121, MiniMax/Nemotron #5136,
model-visibility #5091, failover logs #5016, disconnect races #5007, sidebar
orphan #5142, SRE playbooks salvage #5138, new Security #5130 + Maintenance roll-up)
- Credit salvaged-PR authors (@JxnLexn / @KooshaPari / @herjarsa / @Witroch4)
- Remove phantom bullet for CLOSED-not-merged #5092 (setup aggregator never landed)
- Fix isHidden bullet PR citation #4389 -> #5086 (@herjarsa)
- Back-fill forgotten v3.8.36 bullet: #5026 crypto.randomUUID ID-gen (@hamsa0x7)
- Sync 41 i18n CHANGELOG mirrors; README What's New -> v3.8.38
- Rebaseline cycle drift: eslint 3987->4002, cognitive 833->841, dead-exports
345->346, cyclomatic 1978->1980 (file-size handled by #5147)
* fix(i18n): add missing English UI labels (#5153)
Integrated into release/v3.8.38
* Preserve non-stream reasoning fields for compatible clients (#5155)
Integrated into release/v3.8.38
* feat(compression): ionizer engine — lossy JSON-array sampling reversible via CCR (#5148)
Integrated into release/v3.8.38
* test(combo): gated live smoke for combo strategies (in-process + VPS HTTP) (#5151)
Integrated into release/v3.8.38
* test: refresh release expectations to match current code (#5150)
Integrated into release/v3.8.38 (test-only base-red alignment extracted from #5150)
---------
Co-authored-by: Éder Costa <eder.almeida.costa@gmail.com>
Co-authored-by: José Victor Ferreira <root@josevictor.me>
Co-authored-by: Hernan Javier Ardila Sanchez <hjasgr@gmail.com>
Co-authored-by: fulorgnas <46461624+fulorgnas@users.noreply.github.com>
Co-authored-by: Randi <55005611+rdself@users.noreply.github.com>
Co-authored-by: Jan Leon <Jan.gaschler@gmail.com>
Co-authored-by: R. Beltran <rbeltran8000@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: KooshaPari <42529354+KooshaPari@users.noreply.github.com>
Co-authored-by: Ramel Tecnologia - Rafa Martins <146174365+rafacpti23@users.noreply.github.com>
Co-authored-by: herjarsa <herjarsa@users.noreply.github.com>
Co-authored-by: Witroch4 <175152067+Witroch4@users.noreply.github.com>
This commit is contained in:
Diego Rodrigues de Sa e Souza
committed by
GitHub
GitHub
parent
b3207ab010
commit
7b139fdb5e
@@ -70,6 +70,7 @@ export const INTENTIONALLY_INTERNAL = new Set([
|
||||
"stateReset", // db-internal: 3 callers dentro de src/lib/db/ (core, backup, apiKeys) para coordenação de reset
|
||||
"stats", // intentionally-internal: src/app/api/settings/database/refresh-stats/route.ts
|
||||
"tierConfig", // intentionally-internal: open-sse/services/tierResolver.ts (require() dinâmico)
|
||||
"webSessionDedup", // db-internal: importado só por db/providers.ts (webSessionCredentialKey/parseProviderSpecificData — helpers puros de dedup de credencial web-session split do providers.ts, #3368 PR6)
|
||||
]);
|
||||
|
||||
// Alias para retrocompatibilidade com os testes existentes que importam KNOWN_UNEXPORTED.
|
||||
|
||||
@@ -93,6 +93,11 @@ const IGNORE_FROM_CODE = new Set([
|
||||
"PR_BODY",
|
||||
// CLI machine-id token opt-out (server-side flag; not user-configurable via .env).
|
||||
"OMNIROUTE_DISABLE_CLI_TOKEN",
|
||||
// Gated combo live-smoke harness (scripts/test/_vpsClient.mjs) — override the VPS HTTP
|
||||
// smoke target host/key. Test/CI-only signals with safe defaults
|
||||
// ("http://192.168.0.15:20128" / null), never OmniRoute runtime config (#5151).
|
||||
"COMBO_LIVE_BASE_URL",
|
||||
"COMBO_LIVE_API_KEY",
|
||||
// update-notifier opt-out for the CLI binary.
|
||||
"OMNIROUTE_NO_UPDATE_NOTIFIER",
|
||||
// Headless CLI execution flag for Electron.
|
||||
|
||||
@@ -43,13 +43,6 @@ export const KNOWN_MISSING_ERROR_HELPER = new Set([
|
||||
// --- original open-sse/executors + handlers scope (pre-6A.8) ---
|
||||
// --- 6A.8 expanded scope: src/app/api/**/route.ts pre-existing violations ---
|
||||
// TODO(6A.8): pre-existing, triage — route through buildErrorBody()/sanitizeErrorMessage()
|
||||
"src/app/api/cli-tools/backups/route.ts",
|
||||
"src/app/api/cli-tools/guide-settings/[toolId]/route.ts",
|
||||
"src/app/api/logs/export/route.ts",
|
||||
"src/app/api/models/catalog/route.ts",
|
||||
"src/app/api/providers/test-batch/route.ts",
|
||||
"src/app/api/settings/import-json/route.ts",
|
||||
"src/app/api/usage/proxy-logs/route.ts",
|
||||
]);
|
||||
|
||||
// Import specifiers that count as "uses the error helper" (path ends in utils/error).
|
||||
@@ -136,9 +129,7 @@ function forwardsRawError(source) {
|
||||
if (m && !/sanitize/i.test(line)) tainted.add(m[1]);
|
||||
}
|
||||
const taintedUse =
|
||||
tainted.size > 0
|
||||
? new RegExp(String.raw`\b(?:${[...tainted].join("|")})\b`)
|
||||
: null;
|
||||
tainted.size > 0 ? new RegExp(String.raw`\b(?:${[...tainted].join("|")})\b`) : null;
|
||||
|
||||
// Pass 2: scan for leak lines.
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
|
||||
@@ -53,11 +53,15 @@ export const COLLECTORS = [
|
||||
// "vitest" e explodem no node runner). Subdir novo: adicione aqui E nos scripts
|
||||
// (o drift-check + o gate de órfãos forçam a manutenção em sincronia).
|
||||
{
|
||||
glob: "tests/unit/{api,auth,authz,build,cli,cli-helper,combo,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts",
|
||||
glob: "tests/unit/{api,auth,authz,build,cli,cli-helper,combo,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui,usage}/**/*.test.ts",
|
||||
sources: ["package.json", ".github/workflows/ci.yml"],
|
||||
},
|
||||
// Node native runner — test:integration (top-level only; tests/integration/services/ NÃO roda)
|
||||
{ glob: "tests/integration/*.test.ts", sources: ["package.json"] },
|
||||
// Node native runner — test:combo:matrix / test:integration (combo strategy decision matrix, 17 strategies)
|
||||
{ glob: "tests/integration/combo-matrix/*.test.ts", sources: ["package.json"] },
|
||||
// Node native runner — test:combo:live (gated real-upstream smoke; RUN_COMBO_LIVE=1 + VPS creds)
|
||||
{ glob: "tests/integration/combo-live/*.live.test.ts", sources: ["package.json"] },
|
||||
// Node native runner — test:system
|
||||
{ glob: "tests/e2e/system-failover.test.ts", sources: ["package.json"] },
|
||||
// vitest.mcp.config.ts — test:vitest
|
||||
|
||||
356
scripts/sre/oncall-rotation.mjs
Normal file
356
scripts/sre/oncall-rotation.mjs
Normal file
@@ -0,0 +1,356 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* On-call rotation calculator.
|
||||
*
|
||||
* PagerDuty-style rotation over a list of engineers with a configurable
|
||||
* shift length (hours). Pure Node stdlib — no `npm install`. File-based
|
||||
* state: the rotation definition lives in a JSON file, and the output
|
||||
* (handoffs, current on-call) is also a JSON file.
|
||||
*
|
||||
* Why a custom calculator instead of just `npm install pd-cli`:
|
||||
* - We want the rotation to work offline (on the operator's laptop,
|
||||
* before they have VPN).
|
||||
* - The PagerDuty schedule XML format has corner cases (DST, week
|
||||
* boundaries, mixed-timezone engineers) that we control here.
|
||||
* - We need to be able to ask "who was on-call at 2026-06-12T07:00:00-07:00?"
|
||||
* without depending on PagerDuty being reachable.
|
||||
*
|
||||
* CLI:
|
||||
* node scripts/sre/oncall-rotation.mjs rotation.json current
|
||||
* node scripts/sre/oncall-rotation.mjs rotation.json handoff --at 2026-06-25T09:00:00-07:00
|
||||
* node scripts/sre/oncall-rotation.mjs rotation.json range --from 2026-06-01 --to 2026-06-30
|
||||
*
|
||||
* Rotation file format:
|
||||
* {
|
||||
* "timezone": "America/Los_Angeles",
|
||||
* "shiftHours": 168, // 1 week
|
||||
* "startsAt": "2026-06-01T09:00:00-07:00",
|
||||
* "members": ["alice", "bob", "carol", "dave"]
|
||||
* }
|
||||
*
|
||||
* Salvaged from closed PR #5057 (base-stale; reimplemented on release).
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, existsSync } from "node:fs";
|
||||
import process from "node:process";
|
||||
|
||||
// ── Library API ──────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* @typedef {Object} Rotation
|
||||
* @property {string} timezone IANA timezone identifier (e.g. "America/Los_Angeles")
|
||||
* @property {number} shiftHours Hours per shift (168 = 1 week)
|
||||
* @property {string} startsAt ISO-8601 datetime (the start of member[0]'s first shift)
|
||||
* @property {string[]} members Ordered list of engineer handles
|
||||
*/
|
||||
|
||||
/**
|
||||
* Format a Date in the given IANA timezone using `Intl.DateTimeFormat`.
|
||||
* Returns an ISO-8601 string with the timezone offset, e.g.
|
||||
* `2026-06-25T09:00:00-07:00`.
|
||||
*
|
||||
* @param {Date} date
|
||||
* @param {string} timezone
|
||||
* @returns {string}
|
||||
*/
|
||||
export function formatInTimezone(date, timezone) {
|
||||
const dtf = new Intl.DateTimeFormat("en-US", {
|
||||
timeZone: timezone,
|
||||
year: "numeric",
|
||||
month: "2-digit",
|
||||
day: "2-digit",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
hour12: false,
|
||||
timeZoneName: "longOffset",
|
||||
});
|
||||
const parts = dtf.formatToParts(date);
|
||||
const get = (type) => parts.find((p) => p.type === type)?.value ?? "";
|
||||
const year = get("year");
|
||||
const month = get("month");
|
||||
const day = get("day");
|
||||
const hour = get("hour");
|
||||
const minute = get("minute");
|
||||
const second = get("second");
|
||||
// `timeZoneName: "longOffset"` yields "GMT-07:00" — extract the offset.
|
||||
const tzName = get("timeZoneName").replace(/^GMT/, "");
|
||||
return `${year}-${month}-${day}T${hour}:${minute}:${second}${tzName || "Z"}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a Date in the given timezone to the corresponding UTC ms.
|
||||
* We do this by formatting the date in the timezone, parsing the formatted
|
||||
* string as UTC, then computing the offset difference.
|
||||
*
|
||||
* @param {Date} utc
|
||||
* @param {string} timezone
|
||||
* @returns {number} UTC ms that, when formatted in `timezone`, equals `utc`
|
||||
*/
|
||||
export function utcForTimezone(utc, timezone) {
|
||||
const dtf = new Intl.DateTimeFormat("en-US", {
|
||||
timeZone: timezone,
|
||||
year: "numeric",
|
||||
month: "2-digit",
|
||||
day: "2-digit",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
hour12: false,
|
||||
});
|
||||
const parts = dtf.formatToParts(utc);
|
||||
const get = (type) => parts.find((p) => p.type === type)?.value ?? "";
|
||||
// Build a UTC date from the formatted parts, then derive the offset.
|
||||
const asUtc = Date.UTC(
|
||||
Number(get("year")),
|
||||
Number(get("month")) - 1,
|
||||
Number(get("day")),
|
||||
Number(get("hour")),
|
||||
Number(get("minute")),
|
||||
Number(get("second"))
|
||||
);
|
||||
// The difference between `asUtc` and the wall clock in UTC ms gives the
|
||||
// timezone offset (in ms). Adding that offset to the wall-clock-interpreted-
|
||||
// as-UTC time gives the real UTC instant.
|
||||
const offsetMs = asUtc - Date.UTC(
|
||||
utc.getUTCFullYear(),
|
||||
utc.getUTCMonth(),
|
||||
utc.getUTCDate(),
|
||||
utc.getUTCHours(),
|
||||
utc.getUTCMinutes(),
|
||||
utc.getUTCSeconds()
|
||||
);
|
||||
return asUtc - offsetMs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the rotation index for a given UTC instant.
|
||||
*
|
||||
* The rotation index is `floor((utcMs - startMs) / shiftMs) mod members.length`.
|
||||
* Negative mod in JS is tricky — we add `members.length` and mod again to keep
|
||||
* the index non-negative.
|
||||
*
|
||||
* @param {number} utcMs UTC millisecond timestamp
|
||||
* @param {Rotation} rotation
|
||||
* @returns {number} index into `rotation.members`
|
||||
*/
|
||||
export function rotationIndex(utcMs, rotation) {
|
||||
const startMs = utcForTimezone(new Date(rotation.startsAt), rotation.timezone);
|
||||
const shiftMs = rotation.shiftHours * 60 * 60 * 1000;
|
||||
const elapsed = utcMs - startMs;
|
||||
const rawIndex = Math.floor(elapsed / shiftMs);
|
||||
const len = rotation.members.length;
|
||||
return ((rawIndex % len) + len) % len;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the start and end UTC ms of the shift that contains `utcMs`.
|
||||
*
|
||||
* @param {number} utcMs
|
||||
* @param {Rotation} rotation
|
||||
* @returns {{ startsAt: string, endsAt: string, member: string, index: number }}
|
||||
*/
|
||||
export function shiftFor(utcMs, rotation) {
|
||||
const startMs = utcForTimezone(new Date(rotation.startsAt), rotation.timezone);
|
||||
const shiftMs = rotation.shiftHours * 60 * 60 * 1000;
|
||||
const elapsed = utcMs - startMs;
|
||||
const shiftIndex = Math.floor(elapsed / shiftMs);
|
||||
const shiftStart = startMs + shiftIndex * shiftMs;
|
||||
const shiftEnd = shiftStart + shiftMs;
|
||||
const memberIndex = ((shiftIndex % rotation.members.length) + rotation.members.length) % rotation.members.length;
|
||||
return {
|
||||
startsAt: new Date(shiftStart).toISOString(),
|
||||
endsAt: new Date(shiftEnd).toISOString(),
|
||||
member: rotation.members[memberIndex],
|
||||
index: memberIndex,
|
||||
shiftNumber: shiftIndex,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* List every shift in the rotation between two ISO-8601 datetimes.
|
||||
*
|
||||
* @param {string} fromIso
|
||||
* @param {string} toIso
|
||||
* @param {Rotation} rotation
|
||||
* @returns {Array<{ startsAt: string, endsAt: string, member: string, index: number, shiftNumber: number }>}
|
||||
*/
|
||||
export function shiftsInRange(fromIso, toIso, rotation) {
|
||||
const fromMs = Date.parse(fromIso);
|
||||
const toMs = Date.parse(toIso);
|
||||
if (!Number.isFinite(fromMs) || !Number.isFinite(toMs)) {
|
||||
throw new Error(`invalid date: from=${fromIso} to=${toIso}`);
|
||||
}
|
||||
if (toMs <= fromMs) {
|
||||
throw new Error(`range is empty or reversed: ${fromIso} >= ${toIso}`);
|
||||
}
|
||||
const startMs = utcForTimezone(new Date(rotation.startsAt), rotation.timezone);
|
||||
const shiftMs = rotation.shiftHours * 60 * 60 * 1000;
|
||||
const len = rotation.members.length;
|
||||
const halfShift = shiftMs / 2;
|
||||
const out = [];
|
||||
// Start at the first shift whose end is >= fromMs.
|
||||
const firstShift = Math.floor((fromMs - startMs) / shiftMs);
|
||||
const lastShift = Math.ceil((toMs - startMs) / shiftMs);
|
||||
for (let i = firstShift; i < lastShift; i += 1) {
|
||||
const sStart = startMs + i * shiftMs;
|
||||
const sEnd = sStart + shiftMs;
|
||||
// Skip shifts that start before the rotation was ever defined — they
|
||||
// would resolve to a member, but the rotation didn't exist yet so
|
||||
// there's no handoff record to point at.
|
||||
if (sStart < startMs) continue;
|
||||
// A shift is included if it overlaps the range AND either
|
||||
// - it started before the range and is still ongoing at fromMs, OR
|
||||
// - it started inside the range AND less than half of it extends past toMs.
|
||||
// The half-shift gate prevents a "next-cycle" shift from leaking into a
|
||||
// range that just barely clips its start.
|
||||
if (sEnd <= fromMs) continue;
|
||||
if (sStart >= toMs) break;
|
||||
const startedBeforeRange = sStart < fromMs;
|
||||
const overshoots = sEnd - toMs;
|
||||
if (!startedBeforeRange && overshoots >= halfShift) continue;
|
||||
const memberIndex = ((i % len) + len) % len;
|
||||
out.push({
|
||||
startsAt: new Date(sStart).toISOString(),
|
||||
endsAt: new Date(sEnd).toISOString(),
|
||||
member: rotation.members[memberIndex],
|
||||
index: memberIndex,
|
||||
shiftNumber: i,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── CLI ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
function loadRotation(path) {
|
||||
if (!existsSync(path)) {
|
||||
process.stderr.write(`oncall-rotation: file not found: ${path}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
const raw = readFileSync(path, "utf8");
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
} catch (err) {
|
||||
process.stderr.write(`oncall-rotation: invalid JSON in ${path}: ${err.message}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
// Minimal validation — keep the script forgiving but loud.
|
||||
for (const key of ["timezone", "shiftHours", "startsAt", "members"]) {
|
||||
if (!(key in parsed)) {
|
||||
process.stderr.write(`oncall-rotation: missing field "${key}" in ${path}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
}
|
||||
if (!Array.isArray(parsed.members) || parsed.members.length === 0) {
|
||||
process.stderr.write(`oncall-rotation: "members" must be a non-empty array\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
if (typeof parsed.shiftHours !== "number" || parsed.shiftHours <= 0) {
|
||||
process.stderr.write(`oncall-rotation: "shiftHours" must be a positive number\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
process.stdout.write(`Usage: oncall-rotation.mjs <rotation.json> <command> [options]
|
||||
|
||||
Commands:
|
||||
current Print the shift that contains "now" (UTC).
|
||||
handoff --at <iso> Print the shift containing the given instant.
|
||||
range --from <iso> --to <iso> List every shift overlapping the range.
|
||||
validate Validate the rotation file and print a summary.
|
||||
|
||||
Rotation file format (JSON):
|
||||
{
|
||||
"timezone": "America/Los_Angeles",
|
||||
"shiftHours": 168,
|
||||
"startsAt": "2026-06-01T09:00:00-07:00",
|
||||
"members": ["alice", "bob", "carol"]
|
||||
}
|
||||
`);
|
||||
}
|
||||
|
||||
function parseCommandArgs(argv) {
|
||||
const out = { command: null, from: null, at: null, to: null };
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const a = argv[i];
|
||||
if (a === "--from") {
|
||||
out.from = argv[++i];
|
||||
} else if (a === "--to") {
|
||||
out.to = argv[++i];
|
||||
} else if (a === "--at") {
|
||||
out.at = argv[++i];
|
||||
}
|
||||
}
|
||||
out.command = argv[0] ?? null;
|
||||
return out;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = process.argv.slice(2);
|
||||
if (args.length === 0 || args[0] === "--help" || args[0] === "-h") {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
const [rotationPath, ...rest] = args;
|
||||
if (!rotationPath || rest.length === 0) {
|
||||
printHelp();
|
||||
process.exit(2);
|
||||
}
|
||||
const rotation = loadRotation(rotationPath);
|
||||
const cmd = parseCommandArgs(rest);
|
||||
|
||||
if (cmd.command === "current") {
|
||||
const nowMs = Date.now();
|
||||
const shift = shiftFor(nowMs, rotation);
|
||||
process.stdout.write(`${JSON.stringify({ now: new Date(nowMs).toISOString(), ...shift }, null, 2)}\n`);
|
||||
return;
|
||||
}
|
||||
if (cmd.command === "handoff") {
|
||||
if (!cmd.at) {
|
||||
process.stderr.write("oncall-rotation: handoff requires --at <iso>\n");
|
||||
process.exit(2);
|
||||
}
|
||||
const atMs = Date.parse(cmd.at);
|
||||
if (!Number.isFinite(atMs)) {
|
||||
process.stderr.write(`oncall-rotation: invalid --at datetime: ${cmd.at}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
const shift = shiftFor(atMs, rotation);
|
||||
process.stdout.write(`${JSON.stringify({ at: new Date(atMs).toISOString(), ...shift }, null, 2)}\n`);
|
||||
return;
|
||||
}
|
||||
if (cmd.command === "range") {
|
||||
if (!cmd.from || !cmd.to) {
|
||||
process.stderr.write("oncall-rotation: range requires --from <iso> --to <iso>\n");
|
||||
process.exit(2);
|
||||
}
|
||||
const shifts = shiftsInRange(cmd.from, cmd.to, rotation);
|
||||
process.stdout.write(`${JSON.stringify({ from: cmd.from, to: cmd.to, shifts }, null, 2)}\n`);
|
||||
return;
|
||||
}
|
||||
if (cmd.command === "validate") {
|
||||
const summary = {
|
||||
timezone: rotation.timezone,
|
||||
shiftHours: rotation.shiftHours,
|
||||
startsAt: rotation.startsAt,
|
||||
members: rotation.members,
|
||||
firstShift: shiftFor(Date.parse(rotation.startsAt), rotation),
|
||||
lastShift: shiftFor(Date.parse(rotation.startsAt) + rotation.members.length * rotation.shiftHours * 3600_000 - 1, rotation),
|
||||
};
|
||||
process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`);
|
||||
return;
|
||||
}
|
||||
|
||||
process.stderr.write(`oncall-rotation: unknown command: ${cmd.command}\n`);
|
||||
printHelp();
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
import { pathToFileURL } from "node:url";
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main();
|
||||
}
|
||||
284
scripts/sre/redact-logs.mjs
Normal file
284
scripts/sre/redact-logs.mjs
Normal file
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* PII redaction for log shipping.
|
||||
*
|
||||
* Streams input (stdin or files) to stdout, replacing sensitive tokens
|
||||
* with stable redaction markers. Pure Node.js stdlib — no `npm install`.
|
||||
*
|
||||
* Recognised patterns (in order, longest match wins per position):
|
||||
*
|
||||
* 1. Anthropic API keys (sk-ant-...) → [REDACTED_API_KEY]
|
||||
* 2. Google API keys (AIza...) → [REDACTED_API_KEY]
|
||||
* 3. GitHub tokens (ghp_/gho_/ghu_/...) → [REDACTED_API_KEY]
|
||||
* 4. OpenAI keys (sk-..., sk-proj-...) → [REDACTED_API_KEY]
|
||||
* 5. AWS access keys (AKIA/ASIA) → [REDACTED_AWS_KEY]
|
||||
* 6. Bearer tokens → [REDACTED_BEARER]
|
||||
* 7. Email addresses → [REDACTED_EMAIL]
|
||||
* 8. Generic api_key=value pairs → [REDACTED_API_KEY]
|
||||
* 9. IPv4 addresses → [REDACTED_IPV4]
|
||||
* 10. IPv6 addresses → [REDACTED_IPV6]
|
||||
*
|
||||
* Provider-specific patterns are listed BEFORE the generic `sk-` rule so
|
||||
* that an `sk-ant-...` key counts as `ANTHROPIC_KEY` (not `OPENAI_KEY`)
|
||||
* for the per-call summary.
|
||||
*
|
||||
* Why stable markers: downstream log-search queries reference the
|
||||
* redaction markers (e.g. "show me all log lines with [REDACTED_IPV4]"),
|
||||
* which makes the redaction reversible by anyone with the original
|
||||
* vault lookup, but never by a log-search reader alone.
|
||||
*
|
||||
* CLI:
|
||||
* node scripts/sre/redact-logs.mjs < input.log > output.log
|
||||
* node scripts/sre/redact-logs.mjs --file access.log --output out.log
|
||||
* node scripts/sre/redact-logs.mjs --strict # exit non-zero on any match
|
||||
*
|
||||
* Library:
|
||||
* import { redact, redactString, RedactTransform } from "./scripts/sre/redact-logs.mjs";
|
||||
*
|
||||
* Salvaged from closed PR #5057 (base-stale; reimplemented on release).
|
||||
*/
|
||||
|
||||
import { createReadStream, createWriteStream } from "node:fs";
|
||||
import { TransformStream } from "node:stream/web";
|
||||
import process from "node:process";
|
||||
|
||||
// ── Pattern catalogue ─────────────────────────────────────────────────────────
|
||||
//
|
||||
// Each pattern is [name, regex, marker]. The regex uses the `g` flag so we can
|
||||
// iterate with `matchAll`. Order matters: longer / more specific patterns go
|
||||
// first so an `sk-ant-...` key counts as ANTHROPIC_KEY instead of OPENAI_KEY.
|
||||
|
||||
export const REDACT_PATTERNS = Object.freeze([
|
||||
// 1. Anthropic keys: sk-ant-api03-... / sk-ant-... (must beat the generic sk-)
|
||||
[
|
||||
"ANTHROPIC_KEY",
|
||||
/\bsk-ant-[A-Za-z0-9_\-]{20,}\b/g,
|
||||
"[REDACTED_API_KEY]",
|
||||
],
|
||||
// 2. Google API keys: AIza... (39 chars total)
|
||||
[
|
||||
"GOOGLE_KEY",
|
||||
/\bAIza[A-Za-z0-9_\-]{35}\b/g,
|
||||
"[REDACTED_API_KEY]",
|
||||
],
|
||||
// 3. GitHub tokens (classic + fine-grained + PAT prefixes)
|
||||
[
|
||||
"GITHUB_TOKEN",
|
||||
/\b(?:ghp|gho|ghu|ghs|ghr|github_pat)_[A-Za-z0-9]{30,}\b/g,
|
||||
"[REDACTED_API_KEY]",
|
||||
],
|
||||
// 4. OpenAI keys: sk-..., sk-proj-..., proj-... (after the more specific rules above)
|
||||
[
|
||||
"OPENAI_KEY",
|
||||
/\bsk-(?:proj-)?[A-Za-z0-9_\-]{20,}\b|\bproj-[A-Za-z0-9_\-]{20,}\b/g,
|
||||
"[REDACTED_API_KEY]",
|
||||
],
|
||||
// 5. AWS access keys — AKIA / ASIA prefixes, 20 chars total
|
||||
[
|
||||
"AWS_KEY",
|
||||
/\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g,
|
||||
"[REDACTED_AWS_KEY]",
|
||||
],
|
||||
// 6. Bearer tokens — Authorization: Bearer xxxx (16+ chars)
|
||||
[
|
||||
"BEARER",
|
||||
/(?:Bearer|Authorization:\s*Bearer)\s+([A-Za-z0-9._\-+/=]{16,})/g,
|
||||
"[REDACTED_BEARER]",
|
||||
],
|
||||
// 7. Email — RFC 5322-ish; rejects obvious junk but stays compact.
|
||||
[
|
||||
"EMAIL",
|
||||
/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,24}\b/g,
|
||||
"[REDACTED_EMAIL]",
|
||||
],
|
||||
// 8. Generic api_key / apiKey / password= value pairs (12+ char secret)
|
||||
[
|
||||
"GENERIC_KEY",
|
||||
/\b(?:api[_-]?key|apikey|password|passwd|pwd|secret|token|auth)\s*[:=]\s*['"]?([A-Za-z0-9._\-+/=]{12,})['"]?/gi,
|
||||
"[REDACTED_API_KEY]",
|
||||
],
|
||||
// 9. IPv4 (incl. port) — strict octet bounds
|
||||
[
|
||||
"IPV4",
|
||||
/\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d?\d)(?::\d{1,5})?\b/g,
|
||||
"[REDACTED_IPV4]",
|
||||
],
|
||||
// 10. IPv6 — full, compressed, ::1, ::ffff:1.2.3.4
|
||||
// Three alternative shapes:
|
||||
// (a) full 8-group form (no `::`),
|
||||
// (b) compressed form with `::` somewhere,
|
||||
// (c) `::1` / `::` alone anchored by a non-hex lookbehind so it
|
||||
// doesn't greedily extend `fe80::` into `fe80::foo`.
|
||||
[
|
||||
"IPV6",
|
||||
new RegExp(
|
||||
[
|
||||
// (a) Full 8-group: 1:2:3:4:5:6:7:8
|
||||
"\\b(?:[A-Fa-f0-9]{1,4}:){7}[A-Fa-f0-9]{1,4}\\b",
|
||||
// (b) Compressed with `::` somewhere in the middle (left side 1+ groups)
|
||||
"\\b(?:[A-Fa-f0-9]{1,4}:){1,6}[A-Fa-f0-9]{1,4}::[A-Fa-f0-9]{1,4}(?::[A-Fa-f0-9]{1,4}){0,6}\\b",
|
||||
"\\b(?:[A-Fa-f0-9]{1,4}:){1,7}:[A-Fa-f0-9]{1,4}(?::[A-Fa-f0-9]{1,4}){0,6}\\b",
|
||||
// (c) Leading `::` (no left side): ::1, ::1:2, ::ffff:1.2.3.4
|
||||
"(?<![A-Fa-f0-9:])::(?:[A-Fa-f0-9]{1,4}(?::[A-Fa-f0-9]{1,4}){0,6})?\\b",
|
||||
].join("|"),
|
||||
"g"
|
||||
),
|
||||
"[REDACTED_IPV6]",
|
||||
],
|
||||
]);
|
||||
|
||||
// ── Library API ──────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Redact all PII tokens in a string. Returns the redacted string and the
|
||||
* match counts so the caller can decide whether to fail in strict mode.
|
||||
*
|
||||
* @param {string} input
|
||||
* @returns {{ output: string, counts: Record<string, number> }}
|
||||
*/
|
||||
export function redactString(input) {
|
||||
if (typeof input !== "string" || input.length === 0) {
|
||||
return { output: input ?? "", counts: {} };
|
||||
}
|
||||
const counts = {};
|
||||
let output = input;
|
||||
for (const [name, regex, marker] of REDACT_PATTERNS) {
|
||||
output = output.replace(regex, () => {
|
||||
counts[name] = (counts[name] ?? 0) + 1;
|
||||
return marker;
|
||||
});
|
||||
}
|
||||
return { output, counts };
|
||||
}
|
||||
|
||||
/**
|
||||
* Redact a single line. Convenience wrapper around redactString that does
|
||||
* not allocate an intermediate object for the counts.
|
||||
*
|
||||
* @param {string} line
|
||||
* @returns {string}
|
||||
*/
|
||||
export function redact(line) {
|
||||
return redactString(line).output;
|
||||
}
|
||||
|
||||
/**
|
||||
* Redact TransformStream.
|
||||
*
|
||||
* Implements the WHATWG TransformStream API so callers can do:
|
||||
* await src.pipeThrough(new TextDecoderStream()).pipeThrough(new RedactTransform()).pipeTo(sink)
|
||||
*
|
||||
* Counts are accumulated on the stream instance (`.counts`).
|
||||
*/
|
||||
export class RedactTransform extends TransformStream {
|
||||
constructor() {
|
||||
const counts = {};
|
||||
super({
|
||||
transform(chunk, controller) {
|
||||
const text = typeof chunk === "string" ? chunk : new TextDecoder("utf-8").decode(chunk);
|
||||
const { output, counts: localCounts } = redactString(text);
|
||||
for (const [name, n] of Object.entries(localCounts)) {
|
||||
counts[name] = (counts[name] ?? 0) + n;
|
||||
}
|
||||
controller.enqueue(new TextEncoder().encode(output));
|
||||
},
|
||||
});
|
||||
// Attach counts as an enumerable own property so tests can read it.
|
||||
Object.defineProperty(this, "counts", {
|
||||
value: counts,
|
||||
writable: false,
|
||||
enumerable: true,
|
||||
configurable: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ── CLI ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = { files: [], output: null, strict: false, help: false };
|
||||
for (let i = 2; i < argv.length; i += 1) {
|
||||
const a = argv[i];
|
||||
if (a === "--file") {
|
||||
args.files.push(argv[++i]);
|
||||
} else if (a === "--output" || a === "-o") {
|
||||
args.output = argv[++i];
|
||||
} else if (a === "--strict") {
|
||||
args.strict = true;
|
||||
} else if (a === "--help" || a === "-h") {
|
||||
args.help = true;
|
||||
} else {
|
||||
process.stderr.write(`unknown argument: ${a}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
process.stdout.write(`Usage: redact-logs.mjs [options]
|
||||
|
||||
Options:
|
||||
--file <path> Read from file (repeatable). Defaults to stdin.
|
||||
--output, -o <p> Write to file. Defaults to stdout.
|
||||
--strict Exit non-zero if any PII is detected.
|
||||
--help, -h Show this help.
|
||||
|
||||
Library:
|
||||
import { redact, redactString, RedactTransform } from "./scripts/sre/redact-logs.mjs";
|
||||
`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv);
|
||||
if (args.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
const transform = new RedactTransform();
|
||||
|
||||
// Build a WHATWG ReadableStream from each input source. We open files /
|
||||
// stdin as a Node Readable and convert it via Readable.toWeb().
|
||||
const { Readable } = await import("node:stream");
|
||||
const { Writable: WritableStreamWeb } = await import("node:stream/web");
|
||||
const sources = args.files.length > 0 ? args.files : ["-"];
|
||||
|
||||
for (const source of sources) {
|
||||
const nodeSrc = source === "-" ? process.stdin : createReadStream(source, "utf8");
|
||||
const webSrc = Readable.toWeb(nodeSrc);
|
||||
const webDecoded = webSrc.pipeThrough(new TextDecoderStream("utf-8"));
|
||||
const webEncoded = webDecoded.pipeThrough(transform);
|
||||
|
||||
const sink = args.output
|
||||
? createWriteStream(args.output, "utf8")
|
||||
: process.stdout;
|
||||
const webSink = WritableStreamWeb.toWeb(sink);
|
||||
|
||||
try {
|
||||
await webEncoded.pipeTo(webSink);
|
||||
} catch (err) {
|
||||
process.stderr.write(`redact-logs: ${err.message}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
const totals = transform.counts;
|
||||
if (Object.keys(totals).length > 0) {
|
||||
const summary = Object.entries(totals)
|
||||
.map(([k, v]) => `${k}=${v}`)
|
||||
.join(" ");
|
||||
process.stderr.write(`redact-logs: redacted ${summary}\n`);
|
||||
if (args.strict) {
|
||||
process.exit(3);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only run as CLI when this module is the entrypoint (not when imported as a
|
||||
// library). `import.meta.url === pathToFileURL(process.argv[1]).href` is the
|
||||
// canonical ESM check.
|
||||
import { pathToFileURL } from "node:url";
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main();
|
||||
}
|
||||
319
scripts/test/_vpsClient.mjs
Normal file
319
scripts/test/_vpsClient.mjs
Normal file
@@ -0,0 +1,319 @@
|
||||
/**
|
||||
* scripts/test/_vpsClient.mjs
|
||||
*
|
||||
* Reusable Phase-3 VPS HTTP client for OmniRoute combo live-smoke tests.
|
||||
* NOT a test file — intentionally placed in scripts/test/ so check:test-discovery
|
||||
* does not scan it.
|
||||
*
|
||||
* Combo create/delete mechanism: SSH-sqlite fallback.
|
||||
* /api/combos requires management auth (returns 401 unauthenticated).
|
||||
* We insert/delete rows directly via:
|
||||
* execFileSync("ssh", ["root@192.168.0.15", "sqlite3", "/root/.omniroute/storage.sqlite", SQL])
|
||||
* Values are static test-scoped data — no untrusted interpolation.
|
||||
*
|
||||
* combos table schema (PRAGMA table_info):
|
||||
* id TEXT PK, name TEXT NOT NULL, data TEXT NOT NULL (JSON blob),
|
||||
* created_at TEXT NOT NULL, updated_at TEXT NOT NULL,
|
||||
* system_message TEXT, tool_filter_regex TEXT,
|
||||
* context_cache_protection INTEGER DEFAULT 0, sort_order INTEGER NOT NULL DEFAULT 0
|
||||
*
|
||||
* The `data` column stores the full combo as JSON (name, models[], strategy, config,
|
||||
* id, createdAt, updatedAt, version, sortOrder).
|
||||
*/
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Config
|
||||
// ---------------------------------------------------------------------------
|
||||
const BASE_URL = process.env.COMBO_LIVE_BASE_URL ?? "http://192.168.0.15:20128";
|
||||
const API_KEY = process.env.COMBO_LIVE_API_KEY ?? null;
|
||||
const VPS_SSH_HOST = "root@192.168.0.15";
|
||||
const VPS_DB_PATH = "/root/.omniroute/storage.sqlite";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Nonce counter — increments per call so semantic cache cannot serve stale
|
||||
// ---------------------------------------------------------------------------
|
||||
let _nonceCounter = 0;
|
||||
export function nonce() {
|
||||
return ++_nonceCounter;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared fetch helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
function authHeaders() {
|
||||
const h = { "Content-Type": "application/json" };
|
||||
if (API_KEY) h["Authorization"] = `Bearer ${API_KEY}`;
|
||||
return h;
|
||||
}
|
||||
|
||||
async function fetchJson(path, options = {}) {
|
||||
const url = `${BASE_URL}${path}`;
|
||||
const res = await fetch(url, { ...options, headers: { ...authHeaders(), ...(options.headers ?? {}) } });
|
||||
let body;
|
||||
try {
|
||||
body = await res.json();
|
||||
} catch {
|
||||
body = null;
|
||||
}
|
||||
return { status: res.status, ok: res.ok, body };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// health() — GET /api/monitoring/health
|
||||
// ---------------------------------------------------------------------------
|
||||
export async function health() {
|
||||
const { status, body } = await fetchJson("/api/monitoring/health");
|
||||
return {
|
||||
status,
|
||||
version: body?.version ?? null,
|
||||
uptime: body?.uptime ?? null,
|
||||
raw: body,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// chat() — POST /v1/chat/completions (non-streaming)
|
||||
// ---------------------------------------------------------------------------
|
||||
export async function chat(model, { maxTokens = 16, content } = {}) {
|
||||
const n = nonce();
|
||||
const userContent = content ?? `ping ${n}`;
|
||||
const payload = {
|
||||
model,
|
||||
stream: false,
|
||||
max_tokens: maxTokens,
|
||||
temperature: 0,
|
||||
messages: [{ role: "user", content: userContent }],
|
||||
};
|
||||
const { status, body } = await fetchJson("/v1/chat/completions", {
|
||||
method: "POST",
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const text = body?.choices?.[0]?.message?.content ?? null;
|
||||
return {
|
||||
status,
|
||||
model: body?.model ?? model,
|
||||
text,
|
||||
raw: body,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Combo create/delete via SSH sqlite
|
||||
// ---------------------------------------------------------------------------
|
||||
function sshSqlite(sql) {
|
||||
// Pipe SQL via stdin to avoid shell quoting issues with complex SQL statements.
|
||||
// ssh + sqlite3 reads from stdin when no trailing SQL arg is given.
|
||||
return execFileSync("ssh", [VPS_SSH_HOST, "sqlite3", VPS_DB_PATH], {
|
||||
input: sql,
|
||||
encoding: "utf8",
|
||||
timeout: 15_000,
|
||||
}).trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* createCombo(def) — inserts a combo row via SSH sqlite.
|
||||
*
|
||||
* def shape:
|
||||
* { name, strategy, models, config }
|
||||
*
|
||||
* models: array of "providerId/model" strings OR
|
||||
* array of { providerId, model, connectionId?, weight? } objects.
|
||||
*
|
||||
* config: optional object (judgeModel, fusionTuning, etc.)
|
||||
*
|
||||
* Returns the combo id string.
|
||||
*/
|
||||
export function createCombo(def) {
|
||||
const id = randomUUID();
|
||||
const now = new Date().toISOString();
|
||||
const strategy = def.strategy ?? "priority";
|
||||
const config = def.config ?? {};
|
||||
const name = def.name;
|
||||
|
||||
// Normalise models to the shape the DB expects
|
||||
const rawModels = def.models ?? [];
|
||||
const models = rawModels.map((m, idx) => {
|
||||
if (typeof m === "string") {
|
||||
// "providerId/model" shorthand
|
||||
const slashIdx = m.indexOf("/");
|
||||
const providerId = slashIdx >= 0 ? m.slice(0, slashIdx) : m;
|
||||
const model = slashIdx >= 0 ? m.slice(slashIdx + 1) : m;
|
||||
const slugName = name.toLowerCase().replace(/[^a-z0-9]/g, "-");
|
||||
const slugModel = model.toLowerCase().replace(/[^a-z0-9]/g, "-");
|
||||
return {
|
||||
id: `${slugName}-model-${idx + 1}-${providerId}-${slugModel}-${randomUUID()}`,
|
||||
kind: "model",
|
||||
model: `${providerId}/${model}`,
|
||||
providerId,
|
||||
weight: 1,
|
||||
};
|
||||
}
|
||||
// Already an object
|
||||
const providerId = m.providerId;
|
||||
const model = m.model;
|
||||
const slugName = name.toLowerCase().replace(/[^a-z0-9]/g, "-");
|
||||
const slugModel = (model ?? "").toLowerCase().replace(/[^a-z0-9]/g, "-");
|
||||
return {
|
||||
id: m.id ?? `${slugName}-model-${idx + 1}-${providerId}-${slugModel}-${randomUUID()}`,
|
||||
kind: "model",
|
||||
model: model.includes("/") ? model : `${providerId}/${model}`,
|
||||
providerId,
|
||||
...(m.connectionId ? { connectionId: m.connectionId } : {}),
|
||||
weight: m.weight ?? 1,
|
||||
};
|
||||
});
|
||||
|
||||
const dataObj = {
|
||||
name,
|
||||
models,
|
||||
strategy,
|
||||
config,
|
||||
id,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
version: 2,
|
||||
sortOrder: 9999,
|
||||
};
|
||||
|
||||
const dataJson = JSON.stringify(dataObj).replace(/'/g, "''");
|
||||
const nameSafe = name.replace(/'/g, "''");
|
||||
const sql = `INSERT INTO combos (id, name, data, created_at, updated_at, sort_order) VALUES ('${id}', '${nameSafe}', '${dataJson}', '${now}', '${now}', 9999);`;
|
||||
sshSqlite(sql);
|
||||
return id;
|
||||
}
|
||||
|
||||
/**
|
||||
* deleteCombo(nameOrId) — deletes a combo by name or id via SSH sqlite.
|
||||
* Only deletes __live_test__* prefixed combos as a safety guard.
|
||||
*/
|
||||
export function deleteCombo(nameOrId) {
|
||||
if (!nameOrId.startsWith("__live_test__")) {
|
||||
throw new Error(`deleteCombo safety guard: refusing to delete '${nameOrId}' — only __live_test__* combos allowed.`);
|
||||
}
|
||||
const safe = nameOrId.replace(/'/g, "''");
|
||||
// Try delete by name first, then by id
|
||||
sshSqlite(`DELETE FROM combos WHERE name='${safe}' OR id='${safe}';`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// listHealthyProviders(candidates) — probe each "provider/model" candidate
|
||||
// ---------------------------------------------------------------------------
|
||||
/**
|
||||
* For each "provider/model" string, fire a minimal chat() and keep those
|
||||
* returning HTTP 200 with non-empty text.
|
||||
*
|
||||
* @param {string[]} candidates - array of "provider/model" strings
|
||||
* @returns {Promise<string[]>} healthy candidates
|
||||
*/
|
||||
export async function listHealthyProviders(candidates) {
|
||||
const results = await Promise.allSettled(
|
||||
candidates.map(async (c) => {
|
||||
const r = await chat(c, { maxTokens: 16 });
|
||||
return r.status === 200 && r.text ? c : null;
|
||||
})
|
||||
);
|
||||
return results
|
||||
.map((r) => (r.status === "fulfilled" ? r.value : null))
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Preflight self-check (run directly: node scripts/test/_vpsClient.mjs)
|
||||
// ---------------------------------------------------------------------------
|
||||
const isMain = process.argv[1]?.endsWith("_vpsClient.mjs") ||
|
||||
import.meta.url === `file://${process.argv[1]}`;
|
||||
|
||||
if (isMain) {
|
||||
(async () => {
|
||||
console.log("=== OmniRoute VPS Phase-3 Preflight ===");
|
||||
console.log(`Base URL: ${BASE_URL}`);
|
||||
console.log(`API key: ${API_KEY ? "set (Bearer)" : "not set (REQUIRE_API_KEY=false)"}`);
|
||||
console.log();
|
||||
|
||||
// 1. Health
|
||||
console.log("--- health() ---");
|
||||
try {
|
||||
const h = await health();
|
||||
console.log(` status: ${h.status}`);
|
||||
console.log(` version: ${h.version}`);
|
||||
} catch (e) {
|
||||
console.error(` ERROR: ${e.message}`);
|
||||
}
|
||||
|
||||
// 2. Combo mechanism probe
|
||||
console.log();
|
||||
console.log("--- combo create/delete mechanism ---");
|
||||
console.log(" /api/combos GET (unauthenticated):", (() => {
|
||||
try {
|
||||
const r = execFileSync("curl", ["-s", "-o", "/dev/null", "-w", "%{http_code}", `${BASE_URL}/api/combos`], { encoding: "utf8", timeout: 5000 });
|
||||
return r.trim();
|
||||
} catch { return "error"; }
|
||||
})());
|
||||
console.log(" Mechanism: SSH sqlite fallback (management API requires auth)");
|
||||
console.log(" SSH host:", VPS_SSH_HOST);
|
||||
console.log(" DB path:", VPS_DB_PATH);
|
||||
|
||||
// 3. Create + delete a probe combo via SSH sqlite
|
||||
console.log();
|
||||
console.log("--- createCombo / deleteCombo (SSH sqlite) ---");
|
||||
const probeName = "__live_test__probe";
|
||||
let probeId;
|
||||
try {
|
||||
probeId = createCombo({
|
||||
name: probeName,
|
||||
strategy: "priority",
|
||||
models: ["groq/llama-3.1-8b-instant"],
|
||||
config: {},
|
||||
});
|
||||
console.log(` created id: ${probeId}`);
|
||||
deleteCombo(probeName);
|
||||
console.log(` deleted OK`);
|
||||
} catch (e) {
|
||||
console.error(` ERROR: ${e.message}`);
|
||||
// Attempt cleanup on error
|
||||
if (probeId) {
|
||||
try { deleteCombo(probeName); } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. ollama-cloud chat probe
|
||||
console.log();
|
||||
console.log("--- chat probe: ollama-cloud/glm-5.2 ---");
|
||||
try {
|
||||
const r = await chat("ollama-cloud/glm-5.2", { maxTokens: 16 });
|
||||
console.log(` status: ${r.status}`);
|
||||
console.log(` model: ${r.model}`);
|
||||
console.log(` text: ${r.text ? r.text.slice(0, 80) : "(empty)"}`);
|
||||
if (r.status !== 200 || !r.text) {
|
||||
console.log(" NOTE: ollama-cloud/glm-5.2 did not return a valid response.");
|
||||
console.log(" raw error:", JSON.stringify(r.raw?.error ?? r.raw).slice(0, 200));
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(` ERROR: ${e.message}`);
|
||||
}
|
||||
|
||||
// 5. Quick healthy provider scan over a small candidate set
|
||||
console.log();
|
||||
console.log("--- listHealthyProviders (subset scan) ---");
|
||||
const candidates = [
|
||||
"groq/llama-3.1-8b-instant",
|
||||
"gemini/gemini-2.0-flash",
|
||||
"deepseek/deepseek-chat",
|
||||
"cerebras/llama3.1-8b",
|
||||
"ollama-cloud/glm-5.2",
|
||||
];
|
||||
try {
|
||||
const healthy = await listHealthyProviders(candidates);
|
||||
console.log(` tested: ${candidates.join(", ")}`);
|
||||
console.log(` healthy (${healthy.length}/${candidates.length}): ${healthy.join(", ") || "(none)"}`);
|
||||
} catch (e) {
|
||||
console.error(` ERROR: ${e.message}`);
|
||||
}
|
||||
|
||||
console.log();
|
||||
console.log("=== Preflight complete ===");
|
||||
})();
|
||||
}
|
||||
656
scripts/test/combo-live-vps.mjs
Normal file
656
scripts/test/combo-live-vps.mjs
Normal file
@@ -0,0 +1,656 @@
|
||||
/**
|
||||
* scripts/test/combo-live-vps.mjs
|
||||
*
|
||||
* Phase-3 VPS HTTP scenario driver for OmniRoute combo routing.
|
||||
* Exercises 6 strategies (priority / round-robin / weighted / cost-optimized /
|
||||
* fusion / auto) against the live server at 192.168.0.15:20128.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/test/combo-live-vps.mjs
|
||||
* node scripts/test/combo-live-vps.mjs --only=round-robin
|
||||
* node scripts/test/combo-live-vps.mjs --failover # runs all 7 base scenarios + real failover
|
||||
*
|
||||
* Safety rules:
|
||||
* - Only creates/deletes __live_test__* combos
|
||||
* - Always cleans up in `finally` blocks
|
||||
* - Never stops services or touches other data
|
||||
*
|
||||
* Exit code: 0 if all scenarios PASS or SKIP; non-zero only on real FAIL.
|
||||
* Task 8 (--failover) can be appended after the main() call at the bottom.
|
||||
*/
|
||||
|
||||
import { chat, createCombo, deleteCombo, listHealthyProviders, nonce } from "./_vpsClient.mjs";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Candidate list — broad to maximise coverage across volatile provider health.
|
||||
// listHealthyProviders() probes each with a real chat() call.
|
||||
// ---------------------------------------------------------------------------
|
||||
const BROAD_CANDIDATES = [
|
||||
"groq/llama-3.1-8b-instant",
|
||||
"groq/llama-3.3-70b-versatile",
|
||||
"minimax/MiniMax-M3",
|
||||
"minimax/minimax-m3",
|
||||
"kimi-coding-apikey/moonshot-v1-8k",
|
||||
"openrouter/openai/gpt-3.5-turbo",
|
||||
"cerebras/llama-3.3-70b",
|
||||
"cerebras/llama3.1-8b",
|
||||
"deepseek/deepseek-chat",
|
||||
"ollama-cloud/glm-5.2",
|
||||
"glm/glm-4-flash",
|
||||
"gemini/gemini-2.0-flash",
|
||||
];
|
||||
|
||||
// Known approximate input cost ($/M tokens) from OmniRoute's default-pricing constants.
|
||||
// Used only to identify cheap vs pricey pairs for cost-optimized scenario.
|
||||
// Models absent from this map are treated as unknown cost (Infinity in the server's
|
||||
// sortModelsByCost, i.e. sorted last — effectively "most expensive").
|
||||
const KNOWN_INPUT_COST = {
|
||||
"groq/llama-3.1-8b-instant": 0, // inference-hosts.ts: price=0 (free tier)
|
||||
"groq/llama-3.3-70b-versatile": 0, // inference-hosts.ts: price=0 (free tier)
|
||||
"cerebras/llama3.1-8b": 0, // inference-hosts.ts: price=0
|
||||
"cerebras/llama-3.3-70b": 0, // inference-hosts.ts: price=0
|
||||
"deepseek/deepseek-chat": 0, // inference-hosts.ts: price=0
|
||||
"minimax/MiniMax-M3": 0.5, // regional.ts: $0.5/M input
|
||||
"minimax/minimax-m3": 0.5, // regional.ts: $0.5/M input
|
||||
"kimi-coding-apikey/moonshot-v1-8k": 1, // not in pricing table → Infinity on server → treated pricey
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CLI args
|
||||
// ---------------------------------------------------------------------------
|
||||
const onlyArg = process.argv.find((a) => a.startsWith("--only="));
|
||||
const onlyScenario = onlyArg ? onlyArg.slice(7) : null;
|
||||
// --failover: opt-in flag that appends a real-failover scenario (broken primary → healthy fallback)
|
||||
const failoverFlag = process.argv.includes("--failover");
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Result tracking
|
||||
// ---------------------------------------------------------------------------
|
||||
let exitCode = 0;
|
||||
const summary = [];
|
||||
|
||||
function pass(name, detail = "") {
|
||||
summary.push({ name, result: "PASS" });
|
||||
console.log(`PASS [${name}]${detail ? ": " + detail : ""}`);
|
||||
}
|
||||
|
||||
function skip(name, reason) {
|
||||
summary.push({ name, result: "SKIP" });
|
||||
console.log(`SKIP [${name}]: ${reason}`);
|
||||
}
|
||||
|
||||
function fail(name, reason, err = null) {
|
||||
summary.push({ name, result: "FAIL" });
|
||||
console.error(`FAIL [${name}]: ${reason}`);
|
||||
if (err) console.error(" caused by:", err?.message ?? String(err));
|
||||
exitCode = 1;
|
||||
}
|
||||
|
||||
async function runScenario(name, fn) {
|
||||
if (onlyScenario && onlyScenario !== name) return;
|
||||
try {
|
||||
await fn();
|
||||
} catch (err) {
|
||||
fail(name, `unexpected error: ${err?.message ?? String(err)}`, err);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helper: split "provider/model" → { providerId, modelPart }
|
||||
// For multi-segment paths like "openrouter/openai/gpt-3.5-turbo":
|
||||
// providerId = "openrouter", modelPart = "openai/gpt-3.5-turbo"
|
||||
// ---------------------------------------------------------------------------
|
||||
function splitProviderModel(full) {
|
||||
const idx = full.indexOf("/");
|
||||
if (idx < 0) return { providerId: full, modelPart: full };
|
||||
return { providerId: full.slice(0, idx), modelPart: full.slice(idx + 1) };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// STEP 0: Cache probe
|
||||
// Verifies that a sqlite-inserted combo is immediately routable.
|
||||
// getComboByName() in combos.ts does a direct DB read with no TTL cache,
|
||||
// so the combo should be visible as soon as the SSH INSERT commits.
|
||||
// If not (unexpected caching behaviour), we poll up to 12s before blocking.
|
||||
// ---------------------------------------------------------------------------
|
||||
async function step0CacheProbe(healthy) {
|
||||
console.log("\n=== STEP 0: sqlite → routable cache probe ===");
|
||||
const probe = healthy[0];
|
||||
const probeName = "__live_test__probe";
|
||||
let id;
|
||||
let blocked = false;
|
||||
|
||||
try {
|
||||
id = createCombo({ name: probeName, strategy: "priority", models: [probe] });
|
||||
console.log(` created combo id=${id} model=${probe}`);
|
||||
|
||||
// Attempt immediate chat — expect instant visibility (getComboByName bypasses cache)
|
||||
let r = await chat(probeName, { maxTokens: 4 });
|
||||
if (r.status === 200 && r.text) {
|
||||
console.log(
|
||||
` chat() → ${r.status} model=${r.model} text="${r.text.slice(0, 40)}"`
|
||||
);
|
||||
console.log(" PROBE RESULT: immediately routable — getComboByName bypasses in-memory cache");
|
||||
} else {
|
||||
console.log(
|
||||
` Immediate chat → status=${r.status} text=${r.text ?? "(empty)"}`
|
||||
);
|
||||
console.log(" Polling up to 12 s (TTL cache unexpectedly active)...");
|
||||
let resolved = false;
|
||||
for (let i = 0; i < 6; i++) {
|
||||
await new Promise((res) => setTimeout(res, 2000));
|
||||
r = await chat(probeName, { maxTokens: 4 });
|
||||
if (r.status === 200 && r.text) {
|
||||
console.log(` Resolved after ~${(i + 1) * 2}s: model=${r.model}`);
|
||||
console.log(" PROBE RESULT: visible after TTL expiry");
|
||||
resolved = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!resolved) {
|
||||
console.error(" PROBE RESULT: BLOCKER — combo not routable within 12s");
|
||||
blocked = true;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (id) {
|
||||
try {
|
||||
deleteCombo(probeName);
|
||||
console.log(` cleanup: ${probeName} deleted`);
|
||||
} catch (e) {
|
||||
console.error(` cleanup error: ${e?.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (blocked) {
|
||||
throw new Error("BLOCKER: sqlite-inserted combo not routable within 12s — cannot run scenarios");
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 1: priority
|
||||
// Two healthy models. Single chat() call → 200 + non-empty text.
|
||||
// ---------------------------------------------------------------------------
|
||||
async function scenarioPriority(healthy) {
|
||||
const name = "__live_test__priority";
|
||||
if (healthy.length < 1) {
|
||||
skip("priority", "no healthy providers");
|
||||
return;
|
||||
}
|
||||
const models = healthy.slice(0, 2);
|
||||
let id;
|
||||
try {
|
||||
id = createCombo({ name, strategy: "priority", models });
|
||||
const r = await chat(name, { maxTokens: 16 });
|
||||
if (r.status !== 200) {
|
||||
fail("priority", `status=${r.status} raw=${JSON.stringify(r.raw)?.slice(0, 120)}`);
|
||||
return;
|
||||
}
|
||||
if (!r.text) {
|
||||
fail("priority", "response text is empty");
|
||||
return;
|
||||
}
|
||||
pass("priority", `status=200 model=${r.model} text="${r.text.slice(0, 40)}"`);
|
||||
} finally {
|
||||
if (id) try { deleteCombo(name); } catch { /* best effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 2: round-robin
|
||||
// ≥2 healthy models. 5 calls with unique nonces → at least 2 distinct
|
||||
// response.model values (each call must return 200).
|
||||
// ---------------------------------------------------------------------------
|
||||
async function scenarioRoundRobin(healthy) {
|
||||
const name = "__live_test__round-robin";
|
||||
if (healthy.length < 2) {
|
||||
skip("round-robin", `need ≥2 healthy providers, found ${healthy.length}`);
|
||||
return;
|
||||
}
|
||||
const models = healthy.slice(0, Math.min(3, healthy.length));
|
||||
let id;
|
||||
try {
|
||||
id = createCombo({ name, strategy: "round-robin", models });
|
||||
const served = new Set();
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const r = await chat(name, { maxTokens: 16 });
|
||||
if (r.status !== 200) {
|
||||
fail("round-robin", `call ${i + 1} returned status=${r.status}`);
|
||||
return;
|
||||
}
|
||||
served.add(r.model);
|
||||
}
|
||||
if (served.size < 2) {
|
||||
fail(
|
||||
"round-robin",
|
||||
`only 1 distinct model across 5 calls: [${[...served].join(", ")}] — round-robin not distributing`
|
||||
);
|
||||
return;
|
||||
}
|
||||
pass("round-robin", `${served.size} distinct models across 5 calls: [${[...served].join(", ")}]`);
|
||||
} finally {
|
||||
if (id) try { deleteCombo(name); } catch { /* best effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 3: weighted
|
||||
// Two healthy models with equal weight (50/50). 8 calls → both appear at
|
||||
// least once (loose statistical check — P(only 1 model in 8 calls) ≈ 0.8%).
|
||||
// ---------------------------------------------------------------------------
|
||||
async function scenarioWeighted(healthy) {
|
||||
const name = "__live_test__weighted";
|
||||
if (healthy.length < 2) {
|
||||
skip("weighted", `need ≥2 healthy providers, found ${healthy.length}`);
|
||||
return;
|
||||
}
|
||||
const [m1, m2] = healthy.slice(0, 2);
|
||||
const { providerId: p1, modelPart: mp1 } = splitProviderModel(m1);
|
||||
const { providerId: p2, modelPart: mp2 } = splitProviderModel(m2);
|
||||
let id;
|
||||
try {
|
||||
id = createCombo({
|
||||
name,
|
||||
strategy: "weighted",
|
||||
models: [
|
||||
{ providerId: p1, model: mp1, weight: 50 },
|
||||
{ providerId: p2, model: mp2, weight: 50 },
|
||||
],
|
||||
});
|
||||
const tally = {};
|
||||
for (let i = 0; i < 8; i++) {
|
||||
const r = await chat(name, { maxTokens: 16 });
|
||||
if (r.status !== 200) {
|
||||
fail("weighted", `call ${i + 1} returned status=${r.status}`);
|
||||
return;
|
||||
}
|
||||
tally[r.model] = (tally[r.model] ?? 0) + 1;
|
||||
}
|
||||
const distinct = Object.keys(tally);
|
||||
if (distinct.length < 2) {
|
||||
fail(
|
||||
"weighted",
|
||||
`only 1 distinct model across 8 calls: ${JSON.stringify(tally)} — weighted routing not distributing`
|
||||
);
|
||||
return;
|
||||
}
|
||||
pass("weighted", `8 calls distribution: ${JSON.stringify(tally)}`);
|
||||
} finally {
|
||||
if (id) try { deleteCombo(name); } catch { /* best effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 4: cost-optimized
|
||||
// Find a cheap+pricey healthy pair (based on KNOWN_INPUT_COST).
|
||||
// Insert pricey first (position 0) so cost-optimized must reorder.
|
||||
// Assert: the served model matches the cheap provider.
|
||||
//
|
||||
// OmniRoute's sortModelsByCost uses getPricingForModel which merges
|
||||
// default-pricing constants. groq models have price=0; minimax M3 has $0.5.
|
||||
// Cost-optimized sorts ascending → groq (0) before minimax (0.5).
|
||||
// ---------------------------------------------------------------------------
|
||||
async function scenarioCostOptimized(healthy) {
|
||||
const name = "__live_test__cost-optimized";
|
||||
|
||||
// Partition healthy into cheap (price=0) and pricey (price>0 or unknown>0)
|
||||
const cheap = healthy.filter(
|
||||
(m) => KNOWN_INPUT_COST[m] !== undefined && KNOWN_INPUT_COST[m] === 0
|
||||
);
|
||||
const pricey = healthy.filter(
|
||||
(m) => KNOWN_INPUT_COST[m] !== undefined && KNOWN_INPUT_COST[m] > 0
|
||||
);
|
||||
|
||||
if (cheap.length === 0 || pricey.length === 0) {
|
||||
skip(
|
||||
"cost-optimized",
|
||||
`no distinguishable cheap+pricey pair among healthy=[${healthy.join(", ")}]`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const cheapModel = cheap[0];
|
||||
const priceyModel = pricey[0];
|
||||
let id;
|
||||
try {
|
||||
// Insert pricey first — cost-optimized should reorder to serve cheapModel first
|
||||
id = createCombo({
|
||||
name,
|
||||
strategy: "cost-optimized",
|
||||
models: [priceyModel, cheapModel],
|
||||
});
|
||||
const r = await chat(name, { maxTokens: 16 });
|
||||
if (r.status !== 200) {
|
||||
fail("cost-optimized", `status=${r.status}`);
|
||||
return;
|
||||
}
|
||||
if (!r.text) {
|
||||
fail("cost-optimized", "empty response text");
|
||||
return;
|
||||
}
|
||||
// Verify the cheap model was served (response.model contains cheap provider's model name)
|
||||
const cheapProvider = splitProviderModel(cheapModel).providerId;
|
||||
// Both direct match and provider-substring match are accepted since OmniRoute
|
||||
// returns the raw upstream model name (e.g. "llama-3.1-8b-instant" not "groq/...")
|
||||
const cheapModelPart = splitProviderModel(cheapModel).modelPart.toLowerCase();
|
||||
const servedModel = (r.model ?? "").toLowerCase();
|
||||
const isChapModel =
|
||||
servedModel === cheapModelPart ||
|
||||
servedModel.includes(cheapModelPart) ||
|
||||
servedModel === cheapModel.toLowerCase() ||
|
||||
// fallback: check provider header if available (response.model could be bare name)
|
||||
servedModel.includes(cheapProvider.toLowerCase());
|
||||
|
||||
if (!isChapModel) {
|
||||
fail(
|
||||
"cost-optimized",
|
||||
`expected cheaper model (${cheapModel}, price=${KNOWN_INPUT_COST[cheapModel]}) but got ${r.model} — cost-optimized may not have reordered`
|
||||
);
|
||||
return;
|
||||
}
|
||||
pass(
|
||||
"cost-optimized",
|
||||
`cheaper model served: ${r.model} (cheap=${cheapModel}@$${KNOWN_INPUT_COST[cheapModel]}, pricey=${priceyModel}@$${KNOWN_INPUT_COST[priceyModel]})`
|
||||
);
|
||||
} finally {
|
||||
if (id) try { deleteCombo(name); } catch { /* best effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 5: fusion
|
||||
// Panel of 2-3 healthy models fan out in parallel; judge synthesizes 1 answer.
|
||||
// Cost guard: panel ≤3, max_tokens=16, one call.
|
||||
// ---------------------------------------------------------------------------
|
||||
async function scenarioFusion(healthy) {
|
||||
const name = "__live_test__fusion";
|
||||
if (healthy.length < 2) {
|
||||
skip("fusion", `need ≥2 healthy providers, found ${healthy.length}`);
|
||||
return;
|
||||
}
|
||||
// Panel: up to 3 distinct models
|
||||
const panelModels = healthy.slice(0, Math.min(3, healthy.length));
|
||||
// Judge: reuse first healthy model (cheap, already warmed up)
|
||||
const judgeModel = panelModels[0];
|
||||
let id;
|
||||
try {
|
||||
id = createCombo({
|
||||
name,
|
||||
strategy: "fusion",
|
||||
models: panelModels,
|
||||
config: {
|
||||
judgeModel,
|
||||
fusionTuning: { minPanel: 2 },
|
||||
},
|
||||
});
|
||||
// Use a unique nonce in content to defeat semantic cache
|
||||
const r = await chat(name, {
|
||||
maxTokens: 16,
|
||||
content: `hi ${nonce()} answer in one word`,
|
||||
});
|
||||
if (r.status !== 200) {
|
||||
fail("fusion", `status=${r.status} raw=${JSON.stringify(r.raw)?.slice(0, 120)}`);
|
||||
return;
|
||||
}
|
||||
if (!r.text) {
|
||||
fail("fusion", "judge returned empty synthesized text");
|
||||
return;
|
||||
}
|
||||
pass("fusion", `synthesized text="${r.text.slice(0, 60)}" served-model=${r.model}`);
|
||||
} finally {
|
||||
if (id) try { deleteCombo(name); } catch { /* best effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 6: auto
|
||||
// model="auto" → virtual auto-combo bypasses DB lookup entirely.
|
||||
// Assert: status=200, non-empty text, response.model is a real model (not "auto").
|
||||
// Also test "auto/fast" variant (skip if it 400s as unknown).
|
||||
// ---------------------------------------------------------------------------
|
||||
async function scenarioAuto() {
|
||||
// 6a: bare "auto"
|
||||
const r = await chat("auto", { maxTokens: 16 });
|
||||
if (r.status !== 200) {
|
||||
fail("auto", `status=${r.status} raw=${JSON.stringify(r.raw)?.slice(0, 120)}`);
|
||||
return;
|
||||
}
|
||||
if (!r.text) {
|
||||
fail("auto", "empty response text");
|
||||
return;
|
||||
}
|
||||
const servedModel = r.model ?? "";
|
||||
if (servedModel === "auto" || !servedModel) {
|
||||
fail("auto", `response.model is still "auto" — pool was not resolved`);
|
||||
return;
|
||||
}
|
||||
pass("auto", `status=200 resolved-model=${servedModel} text="${r.text.slice(0, 40)}"`);
|
||||
|
||||
// 6b: "auto/fast" variant
|
||||
const r2 = await chat("auto/fast", { maxTokens: 16 });
|
||||
if (r2.status === 400 || r2.status === 404) {
|
||||
skip("auto/fast", `variant not recognised (${r2.status})`);
|
||||
} else if (r2.status !== 200 || !r2.text) {
|
||||
fail("auto/fast", `status=${r2.status} text=${r2.text ?? "(empty)"}`);
|
||||
} else {
|
||||
pass("auto/fast", `status=200 model=${r2.model} text="${r2.text.slice(0, 40)}"`);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 7: failover (opt-in via --failover)
|
||||
//
|
||||
// Approach: CROSS-PROVIDER BOGUS-MODEL (deterministic, no SSH crypto required).
|
||||
//
|
||||
// Why cross-provider?
|
||||
// A same-provider bogus-model fails silently: when target[0] (bogus) returns a
|
||||
// 404, OmniRoute calls recordProviderCooldown(providerA, undefined) — a provider-
|
||||
// wide key. The combo then pre-screens target[1] (same providerA, real model) and
|
||||
// finds providerA in cooldown → skips it → combo returns the 404 from target[0].
|
||||
// Using DIFFERENT providers avoids this: target[0] puts providerA in cooldown,
|
||||
// target[1] on providerB is unaffected, providerB serves the healthy response.
|
||||
//
|
||||
// Mechanism:
|
||||
// - Target[0]: <providerA>/__nonexistent_model_xyz__ → upstream 404 → providerA cooldown
|
||||
// - Target[1]: <providerB>/<realModel> → not in cooldown → 200 served
|
||||
// - config.maxRetries:0, retryDelayMs:0 → immediate fallover, no retry delay
|
||||
// - Bogus model can never return 200 → any 200 proves fallover to the real target
|
||||
//
|
||||
// If only 1 distinct provider is healthy: SKIP (BROKEN-CONNECTION approach needed; see
|
||||
// comment below for how to implement it using encrypted wrong API key via SSH sqlite).
|
||||
//
|
||||
// BROKEN-CONNECTION approach (for future reference or if cross-provider is unavailable):
|
||||
// 1. SSH to read STORAGE_ENCRYPTION_KEY from /root/.omniroute/.env
|
||||
// 2. Encrypt a wrong API key using scryptSync(key,"omniroute-field-encryption-v1",32)+AES-256-GCM
|
||||
// 3. INSERT broken provider_connection row into provider_connections table via SSH sqlite
|
||||
// 4. Combo: [{providerId:glm, model:glm/glm-4-flash, connectionId:brokenConnId}, realModel]
|
||||
// 5. Broken conn → 401 → recordProviderCooldown("glm",brokenConnId) → key "glm:brokenConnId"
|
||||
// 6. Target[1] on different provider → unaffected → 200
|
||||
// 7. Finally: DELETE combo AND broken connection
|
||||
// ---------------------------------------------------------------------------
|
||||
async function scenarioFailover(healthy) {
|
||||
const name = "__live_test__failover";
|
||||
|
||||
if (healthy.length < 1) {
|
||||
skip("failover", "no healthy providers — cannot run failover scenario");
|
||||
return;
|
||||
}
|
||||
|
||||
// Group healthy providers by providerId to find two distinct providers
|
||||
const byProvider = new Map();
|
||||
for (const m of healthy) {
|
||||
const { providerId } = splitProviderModel(m);
|
||||
if (!byProvider.has(providerId)) byProvider.set(providerId, []);
|
||||
byProvider.get(providerId).push(m);
|
||||
}
|
||||
const distinctProviders = [...byProvider.keys()];
|
||||
|
||||
if (distinctProviders.length < 2) {
|
||||
// Cannot use cross-provider approach with only one provider.
|
||||
// Same-provider bogus-model fails: 404 → recordProviderCooldown(provider, undefined) →
|
||||
// provider-wide cooldown blocks target[1] on the same provider.
|
||||
skip(
|
||||
"failover",
|
||||
`need ≥2 distinct healthy providers for cross-provider approach; ` +
|
||||
`found only 1 [${distinctProviders.join(", ")}]. ` +
|
||||
`Implement BROKEN-CONNECTION approach to run failover with a single provider.`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// CROSS-PROVIDER BOGUS-MODEL:
|
||||
// target[0] = <providerA>/__nonexistent_model_xyz__ (will get 404, puts providerA in cooldown)
|
||||
// target[1] = <providerB>/<realHealthyModel> (different provider, not in cooldown)
|
||||
const bogusProvider = distinctProviders[0];
|
||||
const realModel = byProvider.get(distinctProviders[1])[0];
|
||||
const bogusModel = `${bogusProvider}/__nonexistent_model_xyz__`;
|
||||
const { modelPart: realModelPart } = splitProviderModel(realModel);
|
||||
|
||||
let id;
|
||||
try {
|
||||
id = createCombo({
|
||||
name,
|
||||
strategy: "priority",
|
||||
models: [bogusModel, realModel],
|
||||
config: { maxRetries: 0, retryDelayMs: 0 },
|
||||
});
|
||||
|
||||
console.log(
|
||||
` failover combo (CROSS-PROVIDER BOGUS-MODEL):\n` +
|
||||
` [0] ${bogusModel} ← broken primary (will 404)\n` +
|
||||
` [1] ${realModel} ← healthy fallback (different provider)\n` +
|
||||
` strategy=priority, maxRetries=0`
|
||||
);
|
||||
|
||||
const r = await chat(name, { maxTokens: 16 });
|
||||
|
||||
// Assertions:
|
||||
// 1. status=200: the combo succeeded — ONLY possible if it fell over to target[1],
|
||||
// because target[0] (bogus model) can NEVER return 200 from the upstream.
|
||||
// 2. Non-empty text: real LLM content was returned (not an empty error body).
|
||||
// 3. Served model is NOT the bogus one (belt-and-suspenders; 200 already proves it).
|
||||
// 4. Served model matches the real healthy target (positive proof of which model served).
|
||||
|
||||
if (r.status !== 200) {
|
||||
fail(
|
||||
"failover",
|
||||
`expected status=200 after fallover (broken ${bogusModel} → ${realModel}), ` +
|
||||
`got status=${r.status}. ` +
|
||||
`raw=${JSON.stringify(r.raw)?.slice(0, 200)}`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!r.text) {
|
||||
fail("failover", `status=200 but empty text — fallover may have served a no-content response`);
|
||||
return;
|
||||
}
|
||||
|
||||
const bogusModelPart = "__nonexistent_model_xyz__";
|
||||
const servedModel = (r.model ?? "").toLowerCase();
|
||||
|
||||
// Negative proof: bogus model did NOT serve (should never happen, but guard anyway)
|
||||
if (servedModel.includes(bogusModelPart.toLowerCase())) {
|
||||
fail(
|
||||
"failover",
|
||||
`bogus model string "${bogusModelPart}" appears in served model field "${r.model}" — ` +
|
||||
`impossible 200 from a non-existent model; something is wrong`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Positive proof: served model matches the real healthy target
|
||||
const realModelLower = realModelPart.toLowerCase();
|
||||
const servedMatchesReal =
|
||||
servedModel === realModelLower ||
|
||||
servedModel.includes(realModelLower) ||
|
||||
servedModel === realModel.toLowerCase();
|
||||
|
||||
const proofNote = servedMatchesReal
|
||||
? ""
|
||||
: ` [NOTE: served="${r.model}" ≠ expected="${realModelPart}" ` +
|
||||
`— upstream alias likely; 200+text from bogus-primary is the failover proof]`;
|
||||
|
||||
pass(
|
||||
"failover",
|
||||
`CROSS-PROVIDER BOGUS-MODEL: broken primary (${bogusModel}) → ` +
|
||||
`fallover → served ${r.model} (real target: ${realModel}) ` +
|
||||
`text="${r.text.slice(0, 40)}"${proofNote}`
|
||||
);
|
||||
} finally {
|
||||
if (id) {
|
||||
try {
|
||||
deleteCombo(name);
|
||||
console.log(` cleanup: ${name} deleted`);
|
||||
} catch (e) {
|
||||
console.error(` cleanup error: ${e?.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
async function main() {
|
||||
console.log("=== OmniRoute combo-live-vps scenario driver ===");
|
||||
if (onlyScenario) console.log(`Filtering to scenario: ${onlyScenario}`);
|
||||
console.log();
|
||||
|
||||
// --- Health probe ---
|
||||
console.log("Probing healthy providers (broad candidate list)...");
|
||||
const healthy = await listHealthyProviders(BROAD_CANDIDATES);
|
||||
console.log(` healthy (${healthy.length}/${BROAD_CANDIDATES.length}): [${healthy.join(", ")}]`);
|
||||
|
||||
if (healthy.length === 0) {
|
||||
console.error("FATAL: no healthy providers — cannot run any scenarios");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// --- STEP 0: cache probe ---
|
||||
if (!onlyScenario) {
|
||||
// Run STEP 0 unconditionally unless --only is passed (targeted run skips housekeeping)
|
||||
try {
|
||||
await step0CacheProbe(healthy);
|
||||
} catch (err) {
|
||||
console.error(`\nBLOCKER: ${err.message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
console.log("\n=== Scenarios ===\n");
|
||||
|
||||
await runScenario("priority", () => scenarioPriority(healthy));
|
||||
await runScenario("round-robin", () => scenarioRoundRobin(healthy));
|
||||
await runScenario("weighted", () => scenarioWeighted(healthy));
|
||||
await runScenario("cost-optimized", () => scenarioCostOptimized(healthy));
|
||||
await runScenario("fusion", () => scenarioFusion(healthy));
|
||||
await runScenario("auto", () => scenarioAuto());
|
||||
|
||||
// --- Scenario 7: failover (opt-in) ---
|
||||
// Only runs when --failover is passed. Uses a bogus-model primary to force a real
|
||||
// combo failover to the healthy secondary, proving the priority fallover path works
|
||||
// against the live server without stopping any service.
|
||||
if (failoverFlag) {
|
||||
console.log("\n=== Failover scenario (--failover) ===\n");
|
||||
await runScenario("failover", () => scenarioFailover(healthy));
|
||||
}
|
||||
|
||||
// --- Summary ---
|
||||
console.log("\n=== Summary ===");
|
||||
for (const { name, result } of summary) {
|
||||
console.log(` ${result.padEnd(4)} [${name}]`);
|
||||
}
|
||||
const passed = summary.filter((s) => s.result === "PASS").length;
|
||||
const skipped = summary.filter((s) => s.result === "SKIP").length;
|
||||
const failed = summary.filter((s) => s.result === "FAIL").length;
|
||||
console.log(`\n ${passed} PASS ${skipped} SKIP ${failed} FAIL`);
|
||||
|
||||
process.exit(exitCode);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("FATAL:", err?.message ?? err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user