diff --git a/.env.example b/.env.example index fec3858f40..aee4c12a0d 100644 --- a/.env.example +++ b/.env.example @@ -1989,6 +1989,12 @@ APP_LOG_TO_FILE=true # Default: 8000 (8 seconds). On timeout, a last-good 200 is served when available. # CATALOG_BUILD_TIMEOUT_MS=8000 +# Age after which a connection's synced model list stops being authoritative for routing (#12849). +# A stale (or never-timestamped) synced catalog fails open to the provider registry. +# Used by: src/lib/db/models/activeSyncedCatalog.ts +# Default: 2592000000 (30 days) +# OMNIROUTE_SYNCED_CATALOG_STALE_AFTER_MS=2592000000 + # ── NanoBanana (Image Generation) ── # Polling config for async image generation jobs. # Used by: open-sse/handlers/imageGeneration.ts diff --git a/changelog.d/fixes/12732-catalog-auto-target-metadata-memo.md b/changelog.d/fixes/12732-catalog-auto-target-metadata-memo.md new file mode 100644 index 0000000000..60a0da256d --- /dev/null +++ b/changelog.d/fixes/12732-catalog-auto-target-metadata-memo.md @@ -0,0 +1 @@ +- **fix(models):** a cold `GET /v1/models` on a large deployment no longer blocks the event loop for about a second at a time or overruns the 8s cold-build bound: since #12046 the built-in `auto/*` combos resolved catalog metadata for every target of every combo without memoizing or yielding, and they all draw on the same candidate pool, so 720 synced models took the build from ~4s to ~18s. Each distinct target is now resolved once per build, with a yield between misses ([#12732](https://github.com/diegosouzapw/OmniRoute/issues/12732)) diff --git a/changelog.d/fixes/12732-quota-share-unweighted-drr.md b/changelog.d/fixes/12732-quota-share-unweighted-drr.md new file mode 100644 index 0000000000..6406d7fbe3 --- /dev/null +++ b/changelog.d/fixes/12732-quota-share-unweighted-drr.md @@ -0,0 +1 @@ +- **fix(combo):** a `quota-share` combo whose steps carry no weight now rotates across its targets again instead of sending every request to the first one — the resolver turns an unset weight into 0 and #10881 made 0 mean "disabled", so an all-unweighted combo had no quanta and fell back to definition order; an explicit 0 still disables a target next to weighted siblings ([#12732](https://github.com/diegosouzapw/OmniRoute/issues/12732)) diff --git a/changelog.d/maintenance/12732-env-synced-catalog-stale-after.md b/changelog.d/maintenance/12732-env-synced-catalog-stale-after.md new file mode 100644 index 0000000000..9a7b6c1ef8 --- /dev/null +++ b/changelog.d/maintenance/12732-env-synced-catalog-stale-after.md @@ -0,0 +1 @@ +- **docs(env):** document `OMNIROUTE_SYNCED_CATALOG_STALE_AFTER_MS` (#12849, default 30 days) in `.env.example` and `ENVIRONMENT.md`; the stale-synced-catalog fail-open shipped the override without either, which the env/docs contract gate reports as code-only ([#12732](https://github.com/diegosouzapw/OmniRoute/issues/12732)) diff --git a/changelog.d/maintenance/12732-release-green-pack-gate-provenance.md b/changelog.d/maintenance/12732-release-green-pack-gate-provenance.md new file mode 100644 index 0000000000..b582399352 --- /dev/null +++ b/changelog.d/maintenance/12732-release-green-pack-gate-provenance.md @@ -0,0 +1 @@ +- **fix(ci):** the release-green validator now runs its package-artifact gate the way `ci.yml` does — build, stamp `dist/BUILD_SHA`, then validate against the tree under test. `build:cli` never writes the stamp, so even with the provenance ref pointed at `HEAD` the gate could only ever report "dist/BUILD_SHA is missing" once the build itself compiled ([#12732](https://github.com/diegosouzapw/OmniRoute/issues/12732)) diff --git a/changelog.d/maintenance/12732-resilience-api-credential-health-key.md b/changelog.d/maintenance/12732-resilience-api-credential-health-key.md new file mode 100644 index 0000000000..0fbb216189 --- /dev/null +++ b/changelog.d/maintenance/12732-resilience-api-credential-health-key.md @@ -0,0 +1 @@ +- **test(resilience):** the `/api/resilience` configuration-only key-set assertion now lists `credentialHealthCheck`, the sweep-interval setting #12043 added to the projection, so the integration suite stops reading a documented configuration key as leaked runtime state ([#12732](https://github.com/diegosouzapw/OmniRoute/issues/12732)) diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index d7ab660823..5ac099334e 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -1054,6 +1054,7 @@ desktop install. | `OPENROUTER_CATALOG_TTL_MS` | `86400000` (24h) | `src/lib/catalog/openrouterCatalog.ts` | OpenRouter model catalog cache TTL. | | `MODEL_CATALOG_INCLUDE_NAMES` | `true` | `src/shared/constants/featureFlagDefinitions.ts` | Include display-friendly `name` fields in `/v1/models` responses. Disable for clients that expect IDs only. | | `CATALOG_BUILD_TIMEOUT_MS` | `8000` (8s) | `src/app/api/v1/models/catalogCache.ts` | Cold-path wait bound for a coalesced `GET /v1/models` catalog rebuild (#12627). On timeout, a last-good 200 is served when one exists. | +| `OMNIROUTE_SYNCED_CATALOG_STALE_AFTER_MS` | `2592000000` (30 days) | `src/lib/db/models/activeSyncedCatalog.ts` | Age after which a connection's synced model list stops being authoritative for routing and fails open to the registry (#12849). Never-timestamped rows count as stale. | | `NANOBANANA_POLL_TIMEOUT_MS` | `120000` | `open-sse/handlers/imageGeneration.ts` | Max wait for NanoBanana image generation jobs. | | `NANOBANANA_POLL_INTERVAL_MS` | `2500` | `open-sse/handlers/imageGeneration.ts` | NanoBanana job polling frequency. | | `ADOBE_FIREFLY_SUBMIT_BASE_DELAY_MS` | `8000` | `open-sse/services/adobeFireflyUpscale.ts` | Base delay for the Adobe Firefly upscale submit-retry exponential backoff. | diff --git a/open-sse/services/combo/quotaShareStrategy.ts b/open-sse/services/combo/quotaShareStrategy.ts index e12958042c..8be29743aa 100644 --- a/open-sse/services/combo/quotaShareStrategy.ts +++ b/open-sse/services/combo/quotaShareStrategy.ts @@ -180,12 +180,17 @@ function applyDrr(targets: ResolvedComboTarget[], comboName: string): ResolvedCo if (targets.length <= 1) return targets.slice(); const deficits = getDrrDeficits(comboName); - const totalWeight = targets.reduce((sum, t) => sum + normalizeWeight(t.weight), 0); - if (totalWeight <= 0) return targets.slice(); + const weights = targets.map((t) => normalizeWeight(t.weight)); + const weightedTotal = weights.reduce((sum, w) => sum + w, 0); + // A 0 disables a target only relative to weighted siblings. The combo resolver turns an + // unset step weight into 0, so an all-zero set is an unweighted combo, not an all-disabled + // one: share evenly instead of returning definition order, which pinned the first target. + const unweighted = weightedTotal <= 0; + const totalWeight = unweighted ? targets.length : weightedTotal; // Add each target's quantum (weight share) to its deficit. - for (const target of targets) { - const quantum = normalizeWeight(target.weight) / totalWeight; + for (const [index, target] of targets.entries()) { + const quantum = (unweighted ? 1 : weights[index]) / totalWeight; deficits.set(target.executionKey, (deficits.get(target.executionKey) ?? 0) + quantum); } diff --git a/scripts/quality/validate-release-green.mjs b/scripts/quality/validate-release-green.mjs index e5939214e2..d7d5143fa9 100644 --- a/scripts/quality/validate-release-green.mjs +++ b/scripts/quality/validate-release-green.mjs @@ -461,6 +461,37 @@ async function runAsync(cmd, cmdArgs, opts = {}) { } } +/** + * Package-artifact gate, run the way ci.yml's pack job runs it (#10427). + * + * `check:pack-artifact` assembles dist/ through `build:cli` when staging is missing, and + * `build:cli` never writes dist/BUILD_SHA — only `build:release` does. Pointing the ref at + * HEAD (PACK_GATE_ENV) is not enough on its own: the guard still stops at "dist/BUILD_SHA is + * missing". ci.yml builds, stamps, then validates; mirror that order here. The guard is not + * relaxed: an unstamped dist/ or one built from another commit still fails. + */ +async function runPackArtifactGate(timeoutMs) { + const deadline = Date.now() + timeoutMs; + const steps = [ + { cmd: npmCmd, args: ["run", "build:cli"] }, + { cmd: process.execPath, args: ["scripts/build/write-build-sha.mjs"] }, + { + cmd: npmCmd, + args: ["run", "check:pack-artifact"], + env: PACK_GATE_ENV, + }, + ]; + let out = ""; + for (const step of steps) { + const remaining = deadline - Date.now(); + if (remaining <= 0) return classifyRunError({ killed: true, signal: "SIGTERM" }, timeoutMs); + const result = await runAsync(step.cmd, step.args, { env: step.env, timeout: remaining }); + out += result.out; + if (result.code !== 0) return { code: result.code, out }; + } + return { code: 0, out }; +} + async function main() { const args = new Set(process.argv.slice(2)); const JSON_OUT = args.has("--json"); @@ -713,14 +744,15 @@ async function main() { slow.push({ id: "pack-artifact", label: "Package artifact (npm pack policy)", - args: ["run", "check:pack-artifact"], - env: PACK_GATE_ENV, + run: runPackArtifactGate, timeout: 20 * 60 * 1000, }); } slow.forEach((g) => announce(`${g.label} [parallel]`)); const slowResults = await Promise.all( - slow.map((g) => runAsync(npmCmd, g.args, { timeout: g.timeout, env: g.env })) + slow.map((g) => + g.run ? g.run(g.timeout) : runAsync(npmCmd, g.args, { timeout: g.timeout, env: g.env }) + ) ); slow.forEach((g, i) => { const { code, out } = slowResults[i]; @@ -770,10 +802,7 @@ async function main() { } } else if (WITH_BUILD) { // --with-build without the suites (--quick): still verify the package artifact. - const { code, out } = await runAsync(npmCmd, ["run", "check:pack-artifact"], { - env: PACK_GATE_ENV, - timeout: 20 * 60 * 1000, - }); + const { code, out } = await runPackArtifactGate(20 * 60 * 1000); saveGateLog("pack-artifact", out); record({ id: "pack-artifact", diff --git a/src/app/api/v1/models/catalog.ts b/src/app/api/v1/models/catalog.ts index 87991da4d3..0098980f46 100644 --- a/src/app/api/v1/models/catalog.ts +++ b/src/app/api/v1/models/catalog.ts @@ -104,6 +104,7 @@ import { mergeComboCapabilities, getConnectionScopedEffortTiers, type ConnectionScopedReasoningCatalog, + memoizeTargetMetadata, } from "./catalogHelpers"; import { qualifyOpenRouterModelId, @@ -848,6 +849,7 @@ async function buildUnifiedModelsResponseCore( // catalog build. Runtime auto routing still prepares fresh request-scoped inputs. let preparedAutoInputs: Awaited> | undefined; let materializedAutoCount = 0; + const autoMeta = memoizeTargetMetadata(getComboTargetCatalogMetadata, maybeYieldCatalogBuild); for (const autoId of [ ...Object.keys(AUTO_TEMPLATE_VARIANTS), ...AUTO_SUFFIX_VARIANTS, @@ -890,7 +892,7 @@ async function buildUnifiedModelsResponseCore( connectionId: m.connectionId, ...(m.allowedConnectionIds ? { allowedConnectionIds: m.allowedConnectionIds } : {}), })); - const autoTargetMetadata = autoTargets.map((t) => getComboTargetCatalogMetadata(t)); + const autoTargetMetadata = await autoMeta(autoTargets); // #9147: once per build const knownAutoMeta = autoTargetMetadata.filter( (m): m is ComboTargetCatalogMetadata => m !== null ); diff --git a/src/app/api/v1/models/catalogHelpers.ts b/src/app/api/v1/models/catalogHelpers.ts index 4973a37e89..12b9815b8a 100644 --- a/src/app/api/v1/models/catalogHelpers.ts +++ b/src/app/api/v1/models/catalogHelpers.ts @@ -222,3 +222,36 @@ export function mergeComboCapabilities( } return capabilities; } + +/** + * Memoize per-target catalog metadata for one catalog build, yielding between misses. + * #12046 resolves metadata for every target of every built-in `auto/*` combo, and those + * ~40 combos draw on the same candidate pool: unmemoized, the build repeated the same + * lookups tens of thousands of times without yielding (#9147 — 720 synced models took the + * cold build from ~4s to ~18s, past the 8s cold-build bound). Metadata depends only on + * the target fields in the key, so each distinct target is resolved once per build. + */ +export function memoizeTargetMetadata( + resolve: (target: ComboCatalogTarget) => T | null, + afterMiss: () => Promise +): (targets: ComboCatalogTarget[]) => Promise> { + const byKey = new Map(); + return async (targets) => { + const resolved: Array = []; + for (const target of targets) { + const key = JSON.stringify([ + target.providerId ?? null, + target.provider ?? null, + target.modelStr ?? null, + target.connectionId ?? null, + target.allowedConnectionIds ?? null, + ]); + if (!byKey.has(key)) { + byKey.set(key, resolve(target)); + await afterMiss(); + } + resolved.push(byKey.get(key) ?? null); + } + return resolved; + }; +} diff --git a/tests/integration/resilience-http-e2e.test.ts b/tests/integration/resilience-http-e2e.test.ts index ebcdb9c53a..280e597fb8 100644 --- a/tests/integration/resilience-http-e2e.test.ts +++ b/tests/integration/resilience-http-e2e.test.ts @@ -557,10 +557,12 @@ test("resilience API only exposes configuration, not runtime breaker state", asy assert.equal(response.status, 200); // Exact key set — this is the whole point of the test: configuration only. // `providerQuotaOverrides` joined the projection in #9871; - // `quotaPreflight` joined in #12014 (Settings → Routing Quota Preflight card). + // `quotaPreflight` joined in #12014 (Settings → Routing Quota Preflight card); + // `credentialHealthCheck` joined in #12043 (the sweep's intervalMinutes setting). assert.deepEqual(Object.keys(json).sort(), [ "comboCooldownWait", "connectionCooldown", + "credentialHealthCheck", "legacy", "providerBreaker", "providerCooldown", diff --git a/tests/unit/catalog-target-metadata-memo-9147.test.ts b/tests/unit/catalog-target-metadata-memo-9147.test.ts new file mode 100644 index 0000000000..3567d4f478 --- /dev/null +++ b/tests/unit/catalog-target-metadata-memo-9147.test.ts @@ -0,0 +1,60 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { memoizeTargetMetadata } from "../../src/app/api/v1/models/catalogHelpers.ts"; + +// #9147 / #12046: the built-in auto/* combos all resolve metadata for the same candidate +// pool, so the catalog build must resolve each distinct target once and yield between misses. + +test("memoizeTargetMetadata resolves each distinct target once across calls", async () => { + const resolved: string[] = []; + let yields = 0; + const resolveTargets = memoizeTargetMetadata( + (target) => { + resolved.push(`${target.providerId}/${target.modelStr}`); + return target.modelStr === "missing" ? null : { id: target.modelStr }; + }, + async () => { + yields++; + } + ); + const pool = [ + { providerId: "openai", modelStr: "gpt-a" }, + { providerId: "openai", modelStr: "missing" }, + { providerId: "claude", modelStr: "gpt-a" }, + ]; + + const first = await resolveTargets(pool); + const second = await resolveTargets([...pool].reverse()); + + assert.deepEqual(first, [{ id: "gpt-a" }, null, { id: "gpt-a" }]); + assert.deepEqual(second, [{ id: "gpt-a" }, null, { id: "gpt-a" }]); + assert.deepEqual(resolved, ["openai/gpt-a", "openai/missing", "claude/gpt-a"]); + assert.equal(yields, 3, "one yield per cache miss, none on hits"); +}); + +test("memoizeTargetMetadata keys on connection scope, not just provider/model", async () => { + let calls = 0; + const resolveTargets = memoizeTargetMetadata( + (target) => { + calls++; + return { scope: target.connectionId ?? target.allowedConnectionIds?.join(",") ?? "any" }; + }, + async () => {} + ); + + const result = await resolveTargets([ + { providerId: "openai", modelStr: "gpt-a" }, + { providerId: "openai", modelStr: "gpt-a", connectionId: "conn-1" }, + { providerId: "openai", modelStr: "gpt-a", allowedConnectionIds: ["conn-1", "conn-2"] }, + { providerId: "openai", modelStr: "gpt-a", connectionId: "conn-1" }, + ]); + + assert.deepEqual(result, [ + { scope: "any" }, + { scope: "conn-1" }, + { scope: "conn-1,conn-2" }, + { scope: "conn-1" }, + ]); + assert.equal(calls, 3); +}); diff --git a/tests/unit/quota-share-strategy.test.ts b/tests/unit/quota-share-strategy.test.ts index 57481c41a2..970ec2bd88 100644 --- a/tests/unit/quota-share-strategy.test.ts +++ b/tests/unit/quota-share-strategy.test.ts @@ -282,6 +282,48 @@ describe("DRR: deficit round robin", () => { ); }); + test("unweighted steps (weight 0 from the resolver) still alternate instead of pinning the first", () => { + // comboStructure resolves a step with no weight to 0, and #10881 made 0 mean "disabled". + // With every target at 0 the total weight is 0 and DRR returned definition order, so a + // quota-share combo without explicit weights sent every request to its first target. + const t1 = makeTarget("ek-unweighted-1", "conn-unweighted-1", 0); + const t2 = makeTarget("ek-unweighted-2", "conn-unweighted-2", 0); + + const selected: Array = []; + for (let i = 0; i < 4; i++) { + const r = selectQuotaShareTarget( + [t1, t2], + "combo-unweighted", + "anthropic/claude-sonnet-4-5", + NOW + ); + selected.push(r.target?.executionKey); + r.decrementInflight(); + } + assert.deepEqual(selected, [ + "ek-unweighted-1", + "ek-unweighted-2", + "ek-unweighted-1", + "ek-unweighted-2", + ]); + }); + + test("an explicit weight 0 still disables that target while another target is weighted", () => { + const weighted = makeTarget("ek-on", "conn-on", 100); + const disabled = makeTarget("ek-off", "conn-off", 0); + + for (let i = 0; i < 4; i++) { + const r = selectQuotaShareTarget( + [disabled, weighted], + "combo-disabled", + "anthropic/claude-sonnet-4-5", + NOW + ); + assert.equal(r.target?.executionKey, "ek-on"); + r.decrementInflight(); + } + }); + test("DRR state is isolated per comboName", () => { const t = makeTarget("ek-shared", "conn-shared", 100); selectQuotaShareTarget([t], "combo-A", "anthropic/claude-sonnet-4-5", NOW); diff --git a/tests/unit/validate-release-green.test.ts b/tests/unit/validate-release-green.test.ts index aae0570284..d116bf0722 100644 --- a/tests/unit/validate-release-green.test.ts +++ b/tests/unit/validate-release-green.test.ts @@ -277,6 +277,33 @@ test("pre-flight runs tarball boot only after the package artifact builder compl ); }); +test("pack gate builds, stamps dist/BUILD_SHA, then validates against the tree under test (#10427)", async () => { + const fs = await import("node:fs"); + const src = fs.readFileSync( + new URL("../../scripts/quality/validate-release-green.mjs", import.meta.url), + "utf8" + ); + const gate = src.slice(src.indexOf("async function runPackArtifactGate")); + assert.ok(gate.length > 0, "the pack gate runner must exist"); + const buildAt = gate.indexOf('"build:cli"'); + const stampAt = gate.indexOf("scripts/build/write-build-sha.mjs"); + const checkAt = gate.indexOf('"check:pack-artifact"'); + // `build:cli` never writes dist/BUILD_SHA, so a bare `check:pack-artifact` always failed + // the provenance guard with "dist/BUILD_SHA is missing" — the same trap ci.yml avoids. + assert.ok(buildAt >= 0 && stampAt > buildAt, "BUILD_SHA must be stamped after build:cli"); + assert.ok(checkAt > stampAt, "the artifact must be validated only after it is stamped"); + assert.match( + gate.slice(checkAt, checkAt + 200), + /env: PACK_GATE_ENV/, + "a release-branch tip is never an ancestor of origin/main mid-cycle" + ); + assert.match(src, /const PACK_GATE_ENV = \{ OMNIROUTE_RELEASE_REF: "HEAD" \}/); + // Both entry points (the parallel wave and --with-build --quick) must use it. + assert.equal(src.match(/runPackArtifactGate\b/g)?.length, 3); + assert.doesNotMatch(src, /runAsync\(npmCmd, \["run", "check:pack-artifact"\]/); + assert.doesNotMatch(src, /id: "pack-artifact",[^}]*args:/); +}); + // ─── --full-ci gate extraction (P0, v3.8.46 post-mortem) ───────────────────── const CI_FIXTURE = `