Compare commits

..

1 Commits

Author SHA1 Message Date
diegosouzapw
ed2bf5dfe9 fix(nvidia): fail open when a synced model catalog goes stale (#12849)
A connection's synced model catalog (populated via Import Models, or opt-in
autoFetchModels/autoSync) was treated as authoritative forever once populated.
lookupModelMeta (src/sse/services/model.ts) rejects any model absent from an
authoritative synced catalog, and nothing ever refreshed it automatically
(modelSyncScheduler only re-syncs autoSync:true connections, off by default).
A NVIDIA connection synced once therefore had routing permanently pinned to
that moment's catalog: live upstream models added afterwards (even ones
present in the current static registry, e.g. moonshotai/kimi-k3) were
rejected with 'not available in the active live catalog' indefinitely.

Add a per-connection synced_models_at timestamp (provider_connections,
migration 176), stamped by replaceSyncedAvailableModelsForConnection on every
sync. getActiveSyncedCatalog now only treats a provider's synced catalog as
authoritative while at least one active connection was synced within
OMNIROUTE_SYNCED_CATALOG_STALE_AFTER_MS (default 30 days); once every
connection is stale — or was never synced, e.g. pre-migration rows — it fails
open the same way an unsynced provider already does, instead of gating on a
frozen point-in-time snapshot forever.

Root cause confirmed via a TDD repro proven RED against src/lib/db/models.ts,
src/lib/db/models/activeSyncedCatalog.ts and src/lib/db/providers.ts as they
stood on release/v3.8.51 (all 5 new assertions failed — the DB did not even
have the synced_models_at column yet), then GREEN after the fix
(tests/unit/nvidia-stale-synced-catalog-12849.test.ts).

The narrower reporter-blamed cause (a stale hand-maintained
open-sse/config/nvidiaHostedModels.snapshot.json allowlist) was already fixed
by #12538 and is not read by any runtime routing path.

Refs #12849
2026-09-10 14:26:31 -03:00
13 changed files with 238 additions and 137 deletions

View File

@@ -1 +0,0 @@
- fix(providers): send `x-api-key` instead of `Authorization: Bearer` for OpenCode Zen's `/v1/responses` endpoint (Muse Spark Contributor models), fixing a 401 on OmniRoute's auth header (#12633)

View File

@@ -1 +0,0 @@
- fix(models): declare the real ~1M contextLength for OpenCode Zen's Muse Spark 1.2 models instead of falling back to the 200000 provider default (#12681)

View File

@@ -0,0 +1 @@
- fix(nvidia): fail open when a synced model catalog goes stale instead of gating forever (#12849)

View File

@@ -30,25 +30,17 @@ export const opencodeProvider: RegistryEntry = {
// content (see issue #10867). The opencode provider is passthrough, so
// declaring them here only sets the wire format / capability flags — the
// live upstream model list already advertises both ids.
// #12681: real window confirmed against the opencode-go registry's own
// muse-spark-1.2-contributor entries (contextLength: 1048576, maxOutputTokens:
// 131072) — without an explicit value here resolution fell back to the
// provider-wide defaultContextLength (200000), understating the real window.
{
id: "muse-spark-1.2",
name: "Muse Spark 1.2",
supportsReasoning: true,
targetFormat: "openai-responses",
contextLength: 1048576,
maxOutputTokens: 131072,
},
{
id: "muse-spark-1.2-contributor-free",
name: "Muse Spark 1.2 Contributor Free",
supportsReasoning: true,
targetFormat: "openai-responses",
contextLength: 1048576,
maxOutputTokens: 131072,
},
{ id: "deepseek-v4-flash-free", name: "DeepSeek V4 Flash Free", supportsReasoning: true },
// #6998: 2026-07-14 refresh — the upstream free tier rotated its lineup;

View File

@@ -63,17 +63,11 @@ export const opencode_zenProvider: RegistryEntry = {
// targetFormat declaration, so requests routed here still hit
// /chat/completions with a mismatched or unanswerable body and the
// upstream returns an empty message.
// #12681: real window confirmed against the opencode-go registry's own
// muse-spark-1.2-contributor entries (contextLength: 1048576, maxOutputTokens:
// 131072) — without an explicit value here resolution fell back to the
// provider-wide defaultContextLength (200000), understating the real window.
{
id: "muse-spark-1.2",
name: "Muse Spark 1.2",
supportsReasoning: true,
targetFormat: "openai-responses",
contextLength: 1048576,
maxOutputTokens: 131072,
},
// Explicit wire-format overlay of the base opencode provider's muse-spark entry
// (targetFormat: openai-responses). Keep in sync with base on catalog syncs.
@@ -82,8 +76,6 @@ export const opencode_zenProvider: RegistryEntry = {
name: "Muse Spark 1.2 Contributor Free",
supportsReasoning: true,
targetFormat: "openai-responses",
contextLength: 1048576,
maxOutputTokens: 131072,
},
// ── DeepSeek ────────────────────────────────────────────────

View File

@@ -31,13 +31,6 @@ import {
import { isOpencodeGeoBlocked, proxyKeyOf } from "./opencodeGeoBlock.ts";
import { isNetworkRotationSharedEgressGuardEnabled } from "@/shared/utils/featureFlags";
/**
* The main OpenCode Zen host, shared by the `opencode` and `opencode-zen`
* registry entries. Used to scope the `x-api-key` auth override (#12633) away
* from `opencode-go`, which serves a different upstream (`.../zen/go/v1`).
*/
const ZEN_BASE_URL = "https://opencode.ai/zen/v1";
/**
* Per-account proxy configuration, persisted by NoAuthAccountCard under
* `providerSpecificData.accountProxies` (keyed by the account id, which the UI
@@ -783,20 +776,6 @@ export class OpencodeExecutor extends BaseExecutor {
}
}
/**
* #12633: OpenCode Zen's `/v1/responses` endpoint (reached when
* `_requestFormat === "openai-responses"`, e.g. Muse Spark Contributor
* models) requires `x-api-key`, not `Authorization: Bearer` — unlike the
* default `/chat/completions` endpoint on the same host, which accepts
* Bearer. Scoped by baseUrl (not provider id/alias) so this only applies to
* the main Zen host (`opencode` / `opencode-zen`, both `https://opencode.ai/zen/v1`)
* and never to opencode-go, which serves Responses-format models from a
* different upstream (`https://opencode.ai/zen/go/v1`) that expects Bearer.
*/
private usesZenApiKeyAuth(): boolean {
return this._requestFormat === "openai-responses" && this.config?.baseUrl === ZEN_BASE_URL;
}
buildHeaders(
credentials: ProviderCredentials | null,
stream = true,
@@ -813,7 +792,7 @@ export class OpencodeExecutor extends BaseExecutor {
: undefined;
if (key) {
if (this._requestFormat === "claude" || this.usesZenApiKeyAuth()) {
if (this._requestFormat === "claude") {
headers["x-api-key"] = key;
} else {
headers["Authorization"] = `Bearer ${key}`;

View File

@@ -0,0 +1,6 @@
-- #12849: track when a connection's synced model catalog was last written so
-- getActiveSyncedCatalog can stop treating it as authoritative forever. Plain
-- TEXT column (ISO timestamp) — rowToCamel passes it through as-is;
-- NULL = never synced (pre-existing rows fail open, same as today's no-sync
-- state, rather than staying pinned to a frozen snapshot indefinitely).
ALTER TABLE provider_connections ADD COLUMN synced_models_at TEXT;

View File

@@ -8,7 +8,7 @@ import { isRetiredGitHubCopilotModelId } from "@omniroute/open-sse/config/provid
import type { SqliteAdapter } from "./adapters/types";
import { getDbInstance } from "./core";
import { getProviderConnectionsCount } from "./providers";
import { getProviderConnectionsCount, touchConnectionSyncedModelsAt } from "./providers";
import { type JsonRecord, getKeyValue } from "./models/shared";
import {
normalizeSyncedAvailableModels,
@@ -615,6 +615,10 @@ export async function replaceSyncedAvailableModelsForConnection(
const key = `${providerId}:${connectionId}`;
const normalizedModels = normalizeSyncedAvailableModels(models, providerId);
persistCanonicalSyncedAvailableModels(key, normalizedModels, normalizeSyncedAvailableModels);
// #12849: stamp the sync time on every successful sync — even a re-sync that
// returns an unchanged list proves the catalog is still current, so staleness
// gating in getActiveSyncedCatalog must not treat it as aging regardless.
if (connectionId) await touchConnectionSyncedModelsAt(connectionId);
// Return the full unioned list for the provider
return getSyncedAvailableModels(providerId);
}

View File

@@ -41,8 +41,30 @@ export type ProviderCatalogReconciliation = {
type ProviderConnectionRef = {
id: string;
provider: string;
syncedModelsAt: string | null;
};
// #12849: a connection synced once and never refreshed must not pin routing to
// that point-in-time snapshot forever — a live model the provider has since
// added would be rejected as "unavailable" indefinitely. Once the synced
// catalog exceeds this age (or was never timestamped — pre-migration rows),
// getActiveSyncedCatalog stops treating it as authoritative and fails open,
// matching the existing no-sync-yet behavior. Overridable for ops/testing.
const DEFAULT_SYNCED_CATALOG_STALE_AFTER_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
function getSyncedCatalogStaleAfterMs(): number {
const raw = process.env.OMNIROUTE_SYNCED_CATALOG_STALE_AFTER_MS;
const parsed = raw !== undefined ? Number(raw) : NaN;
return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_SYNCED_CATALOG_STALE_AFTER_MS;
}
function isSyncedAtFresh(syncedModelsAt: string | null): boolean {
if (!syncedModelsAt) return false;
const syncedAtMs = Date.parse(syncedModelsAt);
if (Number.isNaN(syncedAtMs)) return false;
return Date.now() - syncedAtMs <= getSyncedCatalogStaleAfterMs();
}
function resolveStoredProviderId(aliasOrId: string): string {
const normalized = aliasOrId.trim();
if (!normalized) return "";
@@ -92,6 +114,7 @@ function readConnectionRef(connection: unknown): ProviderConnectionRef | null {
const record = connection as {
id?: unknown;
provider?: unknown;
syncedModelsAt?: unknown;
};
if (
@@ -106,6 +129,7 @@ function readConnectionRef(connection: unknown): ProviderConnectionRef | null {
return {
id: record.id,
provider: record.provider,
syncedModelsAt: typeof record.syncedModelsAt === "string" ? record.syncedModelsAt : null,
};
}
@@ -179,24 +203,38 @@ async function unionCustomModels(
* Return the unioned synced catalog belonging only to active connections.
*
* A provider is authoritative only when at least one active connection has a
* non-empty usable catalog. Missing, empty, malformed, or unavailable state
* fails open to the static registry.
* non-empty usable catalog that was synced recently enough (#12849). Missing,
* empty, malformed, stale, or unavailable state fails open to the static
* registry instead of gating on a frozen point-in-time snapshot forever.
*/
async function loadConnectionCatalog(storedProviderId: string): Promise<SyncedAvailableModel[]> {
type ConnectionCatalog = {
models: SyncedAvailableModel[];
hasFreshConnection: boolean;
};
async function loadConnectionCatalog(storedProviderId: string): Promise<ConnectionCatalog> {
const [connections, modelsByConnection] = await Promise.all([
getRawProviderConnections({ provider: storedProviderId, isActive: true }, undefined, undefined, [
"id",
"provider",
"synced_models_at",
]),
getSyncedAvailableModelsByConnection(storedProviderId),
]);
const activeConnectionIds = connections
const activeConnections = connections
.map(readConnectionRef)
.filter((connection): connection is ProviderConnectionRef => connection !== null)
.map((connection) => connection.id);
.filter((connection): connection is ProviderConnectionRef => connection !== null);
return collectModelsForConnections(modelsByConnection, activeConnectionIds);
return {
models: collectModelsForConnections(
modelsByConnection,
activeConnections.map((connection) => connection.id)
),
hasFreshConnection: activeConnections.some((connection) =>
isSyncedAtFresh(connection.syncedModelsAt)
),
};
}
export async function getActiveSyncedCatalog(providerId: string): Promise<ActiveSyncedCatalog> {
@@ -212,11 +250,18 @@ export async function getActiveSyncedCatalog(providerId: string): Promise<Active
// picker-added customModels so dispatch admits the same rows the picker REST shows.
const models = enrichCursorCatalog(
storedProviderId,
await unionCustomModels(storedProviderId, unionModels(siblingCatalogs))
await unionCustomModels(
storedProviderId,
unionModels(siblingCatalogs.map((catalog) => catalog.models))
)
);
if (models.length > 0) {
// #12849: only gate on this catalog while at least one sibling connection
// was synced recently — otherwise a one-time historical sync would keep
// rejecting live models forever with no way to self-recover.
const hasFreshConnection = siblingCatalogs.some((catalog) => catalog.hasFreshConnection);
return {
authoritative: providerUsesAuthoritativeLiveCatalog(providerId),
authoritative: providerUsesAuthoritativeLiveCatalog(providerId) && hasFreshConnection,
models,
};
}

View File

@@ -229,6 +229,7 @@ export const PROVIDER_CONNECTIONS_COLUMNS = new Set([
"rate_limit_overrides_json",
"created_at",
"updated_at",
"synced_models_at",
]);
// ──────────────── Provider Connections ────────────────
@@ -1063,6 +1064,29 @@ export async function touchConnectionLastUsed(
});
}
/**
* #12849: stamp when a connection's synced model catalog was last written.
* getActiveSyncedCatalog reads this to stop treating a synced catalog as
* authoritative forever — a connection synced once and never refreshed
* silently pinned routing to that point-in-time snapshot with no staleness
* check. Lightweight targeted UPDATE, mirrors touchConnectionLastUsed.
*/
export async function touchConnectionSyncedModelsAt(id: string): Promise<void> {
if (!id) return;
const db = getDbInstance() as unknown as DbLike;
const now = new Date().toISOString();
db.prepare(
`UPDATE provider_connections SET
synced_models_at = @syncedModelsAt,
updated_at = @updatedAt
WHERE id = @id`
).run({
syncedModelsAt: now,
updatedAt: now,
id,
});
}
/**
* Lightweight backoff reset — runs a targeted UPDATE without SELECT or re-encrypt.
* Follows the `clearConnectionErrorIfUnchanged` pattern but without the CAS check,

View File

@@ -1,54 +0,0 @@
import test from "node:test";
import assert from "node:assert/strict";
import { OpencodeExecutor } from "../../open-sse/executors/opencode.ts";
test("#12633: openai-responses format on opencode-zen sends x-api-key, not Authorization Bearer", () => {
const executor = new OpencodeExecutor("opencode-zen");
executor._requestFormat = "openai-responses";
const headers = executor.buildHeaders(
{ apiKey: "sk-zen-test" },
true,
null,
"muse-spark-1.2-contributor-free"
);
assert.equal(headers["x-api-key"], "sk-zen-test");
assert.equal(headers["Authorization"], undefined);
});
test("#12633: openai-responses format on the base opencode (oc) provider also sends x-api-key", () => {
const executor = new OpencodeExecutor("opencode");
executor._requestFormat = "openai-responses";
const headers = executor.buildHeaders(
{ apiKey: "sk-oc-test" },
true,
null,
"muse-spark-1.2-contributor-free"
);
assert.equal(headers["x-api-key"], "sk-oc-test");
assert.equal(headers["Authorization"], undefined);
});
test("#12633: openai-responses format on opencode-go (different upstream endpoint) keeps Authorization Bearer", () => {
const executor = new OpencodeExecutor("opencode-go");
executor._requestFormat = "openai-responses";
const headers = executor.buildHeaders(
{ apiKey: "sk-go-test" },
true,
null,
"muse-spark-1.2-contributor"
);
assert.equal(headers["Authorization"], "Bearer sk-go-test");
assert.equal(headers["x-api-key"], undefined);
});
test("#12633: claude format keeps sending x-api-key (unchanged behavior)", () => {
const executor = new OpencodeExecutor("opencode-zen");
executor._requestFormat = "claude";
const headers = executor.buildHeaders({ apiKey: "sk-claude-test" }, true, null, "some-model");
assert.equal(headers["x-api-key"], "sk-claude-test");
assert.equal(headers["Authorization"], undefined);
});

View File

@@ -1,33 +0,0 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { REGISTRY } from "../../open-sse/config/providerRegistry.ts";
import { getTokenLimit } from "../../open-sse/services/contextManager.ts";
test("#12681: opencode registry declares an explicit real contextLength for muse-spark-1.2 models", () => {
const opencode = REGISTRY["opencode"];
const museSpark = opencode.models.find((m) => m.id === "muse-spark-1.2");
const museSparkFree = opencode.models.find((m) => m.id === "muse-spark-1.2-contributor-free");
assert.notEqual(
museSpark?.contextLength,
undefined,
"muse-spark-1.2 should declare its own real contextLength instead of relying on the 200000 provider default"
);
assert.notEqual(
museSparkFree?.contextLength,
undefined,
"muse-spark-1.2-contributor-free should declare its own real contextLength instead of relying on the 200000 provider default"
);
});
test("#12681: opencode-zen registry declares an explicit real contextLength for muse-spark-1.2 models", () => {
const zen = REGISTRY["opencode-zen"];
const museSpark = zen.models.find((m) => m.id === "muse-spark-1.2");
const museSparkFree = zen.models.find((m) => m.id === "muse-spark-1.2-contributor-free");
assert.notEqual(museSpark?.contextLength, undefined);
assert.notEqual(museSparkFree?.contextLength, undefined);
});
test("#12681: contextManager.getTokenLimit resolves muse-spark-1.2-contributor-free to its real 1M+ window, not the 200000 provider default", () => {
assert.equal(getTokenLimit("opencode", "muse-spark-1.2-contributor-free"), 1048576);
assert.equal(getTokenLimit("opencode-zen", "muse-spark-1.2-contributor-free"), 1048576);
});

View File

@@ -0,0 +1,147 @@
/**
* #12849: NVIDIA (and every other authoritative-live-catalog provider) treated a
* connection's *synced* model catalog as authoritative forever once populated —
* no staleness check, no default periodic refresh. A model that is live upstream
* and present in the current static registry was rejected as "not available in
* the active live catalog" indefinitely once any historical sync existed.
*
* getActiveSyncedCatalog now fails open once a connection's synced catalog
* exceeds a staleness threshold (default 30 days; overridable via
* OMNIROUTE_SYNCED_CATALOG_STALE_AFTER_MS), instead of gating on a frozen
* point-in-time snapshot forever.
*/
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-nvidia-stale-12849-"));
process.env.DATA_DIR = TEST_DATA_DIR;
process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "nvidia-stale-12849-test-secret";
const core = await import("../../src/lib/db/core.ts");
const { replaceSyncedAvailableModelsForConnection } = await import("../../src/lib/db/models.ts");
const { getModelInfo } = await import("../../src/sse/services/model.ts");
const { nvidiaProvider } = await import(
"../../open-sse/config/providers/registry/nvidia/index.ts"
);
const PROVIDER = "nvidia";
const CONNECTION_ID = "nvidia-stale-catalog-12849";
// Live upstream + present in the current static registry (asserted below), but
// deliberately absent from the small "historical sync" catalog seeded here.
const LIVE_MODEL = "moonshotai/kimi-k3";
const STALE_SYNC_ONLY_MODEL = "some-retired-model-that-no-longer-exists";
function connectionRow(): { syncedModelsAt: string | null } {
const db = core.getDbInstance();
const row = db
.prepare("SELECT synced_models_at AS syncedModelsAt FROM provider_connections WHERE id = ?")
.get(CONNECTION_ID) as { syncedModelsAt: string | null } | undefined;
if (!row) throw new Error(`connection ${CONNECTION_ID} not found`);
return row;
}
function ageConnectionSync(daysAgo: number): void {
const db = core.getDbInstance();
const agedTimestamp = new Date(Date.now() - daysAgo * 24 * 60 * 60 * 1000).toISOString();
db.prepare("UPDATE provider_connections SET synced_models_at = ? WHERE id = ?").run(
agedTimestamp,
CONNECTION_ID
);
}
async function seedHistoricalSync(): Promise<void> {
const db = core.getDbInstance();
const now = new Date().toISOString();
db.prepare(
`INSERT OR REPLACE INTO provider_connections (id, provider, is_active, created_at, updated_at)
VALUES (?, ?, 1, ?, ?)`
).run(CONNECTION_ID, PROVIDER, now, now);
await replaceSyncedAvailableModelsForConnection(PROVIDER, CONNECTION_ID, [
{ id: STALE_SYNC_ONLY_MODEL, name: STALE_SYNC_ONLY_MODEL, source: "imported" },
]);
}
test.beforeEach(async () => {
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
assert.ok(
nvidiaProvider.models.some((model) => model.id === LIVE_MODEL),
`precondition: ${LIVE_MODEL} must exist in the current NVIDIA static registry`
);
await seedHistoricalSync();
});
test.after(() => {
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
});
test("#12849: a fresh synced catalog still gates — a model missing from it is rejected", async () => {
// Sanity: touchConnectionSyncedModelsAt stamped this sync as fresh already.
const { syncedModelsAt } = connectionRow();
assert.ok(syncedModelsAt, "replaceSyncedAvailableModelsForConnection must stamp synced_models_at");
const resolved = await getModelInfo(`${PROVIDER}/${LIVE_MODEL}`);
assert.equal(resolved.provider, null);
assert.equal(resolved.errorType, "model_not_found");
assert.match(resolved.errorMessage, /active live catalog/i);
});
test("#12849: a stale synced catalog fails open — a live+registry model is no longer rejected", async () => {
ageConnectionSync(45); // past the 30-day default staleness threshold
const resolved = await getModelInfo(`${PROVIDER}/${LIVE_MODEL}`);
assert.equal(
resolved.provider,
PROVIDER,
`expected the stale catalog to fail open, got errorMessage=${resolved.errorMessage}`
);
assert.equal(resolved.model, LIVE_MODEL);
});
test("#12849: a stale synced catalog is treated as non-authoritative in getActiveSyncedCatalog", async () => {
const { getActiveSyncedCatalog } = await import("../../src/lib/db/models/activeSyncedCatalog.ts");
ageConnectionSync(45);
const catalog = await getActiveSyncedCatalog(PROVIDER);
assert.equal(catalog.authoritative, false);
});
test("#12849: a connection never synced (no timestamp) is non-authoritative, not gated forever", async () => {
const db = core.getDbInstance();
db.prepare("UPDATE provider_connections SET synced_models_at = NULL WHERE id = ?").run(
CONNECTION_ID
);
const resolved = await getModelInfo(`${PROVIDER}/${LIVE_MODEL}`);
assert.equal(resolved.provider, PROVIDER);
assert.equal(resolved.model, LIVE_MODEL);
});
test("#12849: OMNIROUTE_SYNCED_CATALOG_STALE_AFTER_MS overrides the default threshold", async () => {
const previous = process.env.OMNIROUTE_SYNCED_CATALOG_STALE_AFTER_MS;
process.env.OMNIROUTE_SYNCED_CATALOG_STALE_AFTER_MS = String(60 * 60 * 1000); // 1 hour
try {
ageConnectionSync(1); // 1 day old — stale under the 1-hour override, fresh under the 30-day default
const resolved = await getModelInfo(`${PROVIDER}/${LIVE_MODEL}`);
assert.equal(resolved.provider, PROVIDER);
} finally {
if (previous === undefined) delete process.env.OMNIROUTE_SYNCED_CATALOG_STALE_AFTER_MS;
else process.env.OMNIROUTE_SYNCED_CATALOG_STALE_AFTER_MS = previous;
}
});