fix(vision): honor Custom Models vision flag on advertised aliases (#12974)

`parseModel` splitting on the first `/` is the trap, and recovering the custom row for all three id forms without changing `parseModel` is the right way around it — that function has too many callers to widen. Lifting the lookup into `customVisionOverride.ts` because `models.ts` is frozen is the correct response to the cap, not a workaround.

---

Validated in one consolidated worktree cut from `release/v3.8.51`, boarded with the other 19 PRs of this batch. Two in-batch conflicts, both additive and resolved by keeping each side: the `ENVIRONMENT.md` table (#13035 + #13011) and the `chatHelpers.ts` import block (#12975 on the tip + #13017).

- `typecheck:core` clean; `check:dashboard-typecheck` OK (206 pre-existing, within baseline); `check:changelog-integrity` OK; `check:docs-counts` migrations ✓
- complexity 2816 / baseline 3218 and cognitive-complexity 1271 / baseline 1437 — both under baseline
- 531 of 532 focused assertions green across the batch's 46 test files
- `check-file-size` rebaselined for the batch's real growth (annotation `_rebaseline_2026_09_11_mergebatch_v3851_houminxi`, landed on #13038), attributed per PR

The single red is **not this batch**: `tests/unit/combo/quota-weighted-strategy.test.ts` → "A/B isolation: 7 hard-empty + 2 at 0.5% + 1 at 40%, floor=1" asserts an order between two connections of identical weight and flakes on the pure tip too — 2 failures in 4 runs at `origin/release/v3.8.51` with nothing from this batch applied.

⚠️ base-red inherited: #12732 — `Docs Gates`, `Merge integrity`, `No new ESLint warnings`, `Unit Tests fast-path` and `Fast Quality Gates` reproduce on the pure tip (provider count 356 vs the 358 the modules define, SKILL.md drift, and `open-sse/utils/stream.ts` at 3115 > frozen 3098, untouched here).

Thanks @HouMinXi — the live evidence on these (X500 logs, `storage.sqlite` state, real `/v1/models` probes, the 36-minute outage write-up) is what let a 20-PR batch be reviewed as a unit.
This commit is contained in:
Bob.Hou
2026-09-11 18:28:32 -04:00
committed by GitHub
parent 6788b437f5
commit 73dee8db43
5 changed files with 404 additions and 91 deletions

View File

@@ -0,0 +1 @@
- **fix(vision):** Custom Models with "Vision capable" checked no longer have image requests swapped to `glm/glm-4.6v` when the client sends the advertised alias (`vllm/path/...`) or the bare path-shaped id — Vision Bridge now matches the stored override for all three id forms ([#12758](https://github.com/diegosouzapw/OmniRoute/issues/12758))

View File

@@ -6,7 +6,6 @@
import { isRetiredGitHubCopilotModelId } from "@omniroute/open-sse/config/providers/registry/github/retiredModels.ts";
import type { SqliteAdapter } from "./adapters/types";
import { getDbInstance } from "./core";
import { getProviderConnectionsCount } from "./providers";
import { type JsonRecord, getKeyValue } from "./models/shared";
@@ -53,6 +52,13 @@ export {
} from "./models/aliases";
export { getMitmAlias, setMitmAliasAll } from "./models/mitmAlias";
export type { SyncedAvailableModel } from "./models/synced";
export {
getCustomModelVisionOverride,
listCustomModelVisionOverrides,
type CustomModelVisionOverrideMap,
type CustomModelVisionDatabase,
type CustomModelVisionOverrideReadOptions,
} from "./models/customVisionOverride";
// ──────────────── Custom Models ────────────────
@@ -91,93 +97,6 @@ export async function getAllCustomModels() {
return result;
}
/** Nested provider → model map of explicit custom-model vision overrides. */
export type CustomModelVisionOverrideMap = ReadonlyMap<string, ReadonlyMap<string, boolean>>;
export type CustomModelVisionDatabase = Pick<SqliteAdapter, "prepare">;
export interface CustomModelVisionOverrideReadOptions {
/** Narrow test seam; production uses the canonical DB singleton. */
getDatabase?: () => CustomModelVisionDatabase;
}
function readVisionOverrideFromModels(value: string | null, modelId: string): boolean | null {
if (!value) return null;
try {
const models = JSON.parse(value) as unknown;
if (!Array.isArray(models)) return null;
const entry = models.find(
(candidate): candidate is { id: string; supportsVision?: boolean } =>
candidate !== null &&
typeof candidate === "object" &&
!Array.isArray(candidate) &&
(candidate as { id?: unknown }).id === modelId
);
return entry && typeof entry.supportsVision === "boolean" ? entry.supportsVision : null;
} catch {
return null;
}
}
/**
* Resolve one explicit custom-model vision override. A supplied bulk map avoids
* SQLite reads for request/build-local capability resolution.
*/
export function getCustomModelVisionOverride(
providerId: string,
modelId: string,
bulk?: CustomModelVisionOverrideMap | null,
options: CustomModelVisionOverrideReadOptions = {}
): boolean | null {
try {
if (bulk) return bulk.get(providerId)?.get(modelId) ?? null;
const db = options.getDatabase?.() ?? getDbInstance();
const row = db
.prepare("SELECT value FROM key_value WHERE namespace = 'customModels' AND key = ?")
.get(providerId);
return readVisionOverrideFromModels(getKeyValue(row).value, modelId);
} catch {
return null;
}
}
/** Bulk-load explicit custom-model vision overrides with one SQLite query. */
export function listCustomModelVisionOverrides(
options: CustomModelVisionOverrideReadOptions = {}
): CustomModelVisionOverrideMap {
try {
const db = options.getDatabase?.() ?? getDbInstance();
const rows = db
.prepare("SELECT key, value FROM key_value WHERE namespace = 'customModels'")
.all();
const result = new Map<string, Map<string, boolean>>();
for (const row of rows) {
const { key, value } = getKeyValue(row);
if (!key || !value) continue;
try {
const models = JSON.parse(value) as unknown;
if (!Array.isArray(models)) continue;
const byModel = new Map<string, boolean>();
for (const candidate of models) {
if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) continue;
const { id, supportsVision } = candidate as {
id?: unknown;
supportsVision?: unknown;
};
if (typeof id === "string" && typeof supportsVision === "boolean") {
byModel.set(id, supportsVision);
}
}
if (byModel.size > 0) result.set(key, byModel);
} catch {
// Malformed custom-model rows do not participate in capability resolution.
}
}
return result;
} catch {
return new Map<string, Map<string, boolean>>();
}
}
export async function addCustomModel(
providerId: string,
modelId: string,

View File

@@ -0,0 +1,191 @@
/**
* Explicit Custom Models "Vision capable" lookup.
*
* Dashboard stores the flag under the connection id (often an
* openai-compatible-chat-* uuid) and the path-shaped model id. Clients send
* the advertised alias (`vllm/path/...`), the bare path, or the internal
* `providerId/modelPath`. parseModel splits on the first slash, so the
* advertised forms miss the stored pair. Match the stored row against all
* three forms before Vision Bridge substitutes a fallback VLM.
*/
import type { SqliteAdapter } from "../adapters/types";
import { getDbInstance } from "../core";
import { getKeyValue } from "./shared";
/** Nested provider → model map of explicit custom-model vision overrides. */
export type CustomModelVisionOverrideMap = ReadonlyMap<string, ReadonlyMap<string, boolean>>;
export type CustomModelVisionDatabase = Pick<SqliteAdapter, "prepare">;
export interface CustomModelVisionOverrideReadOptions {
/** Narrow test seam; production uses the canonical DB singleton. */
getDatabase?: () => CustomModelVisionDatabase;
/**
* Raw model string from the request (`vllm/orcarouter/Qwen...`, the bare
* path, or `providerId/modelPath`). Used when the parsed provider/model pair
* does not match the stored customModels key.
*/
lookupKey?: string;
}
type OverrideHit = { providerId: string; modelId: string; supportsVision: boolean };
function idsEqual(left: string, right: string): boolean {
return left === right || left.toLowerCase() === right.toLowerCase();
}
function isPathShaped(value: string | undefined): boolean {
return Boolean(value && value.includes("/"));
}
/** Stored custom-model id matches the parsed pair and/or the raw request string. */
export function customModelIdMatchesRequest(
storedId: string,
requestedModelId: string,
lookupKey?: string
): boolean {
if (!storedId) return false;
if (idsEqual(storedId, requestedModelId)) return true;
if (lookupKey && idsEqual(storedId, lookupKey)) return true;
// Suffix only when the stored id is itself path-shaped. A leaf like "4o"
// must not match lookupKey "openai/gpt-4o".
if (
isPathShaped(storedId) &&
lookupKey &&
lookupKey.toLowerCase().endsWith(`/${storedId.toLowerCase()}`)
) {
return true;
}
return false;
}
function readVisionOverrideFromModels(value: string | null, modelId: string): boolean | null {
if (!value) return null;
try {
const models = JSON.parse(value) as unknown;
if (!Array.isArray(models)) return null;
const entry = models.find(
(candidate): candidate is { id: string; supportsVision?: boolean } =>
candidate !== null &&
typeof candidate === "object" &&
!Array.isArray(candidate) &&
typeof (candidate as { id?: unknown }).id === "string" &&
idsEqual((candidate as { id: string }).id, modelId)
);
return entry && typeof entry.supportsVision === "boolean" ? entry.supportsVision : null;
} catch {
return null;
}
}
function collectOverrideHits(map: CustomModelVisionOverrideMap): OverrideHit[] {
const hits: OverrideHit[] = [];
for (const [providerId, byModel] of map) {
for (const [modelId, supportsVision] of byModel) {
hits.push({ providerId, modelId, supportsVision });
}
}
return hits;
}
function pickMatchingOverride(
hits: OverrideHit[],
providerId: string,
modelId: string,
lookupKey?: string
): boolean | null {
const exact = hits.find(
(hit) => idsEqual(hit.providerId, providerId) && idsEqual(hit.modelId, modelId)
);
if (exact) return exact.supportsVision;
const matches = hits.filter((hit) =>
customModelIdMatchesRequest(hit.modelId, modelId, lookupKey)
);
if (matches.length === 0) return null;
const first = matches[0].supportsVision;
if (!matches.every((hit) => hit.supportsVision === first)) return null;
if (providerId) {
const sameProvider = matches.filter((hit) => idsEqual(hit.providerId, providerId));
if (sameProvider.length === 1) return sameProvider[0].supportsVision;
}
return first;
}
/**
* Resolve one explicit custom-model vision override. A supplied bulk map avoids
* SQLite reads for request/build-local capability resolution.
*/
export function getCustomModelVisionOverride(
providerId: string,
modelId: string,
bulk?: CustomModelVisionOverrideMap | null,
options: CustomModelVisionOverrideReadOptions = {}
): boolean | null {
try {
const lookupKey = options.lookupKey;
const canScan = isPathShaped(modelId) || isPathShaped(lookupKey);
if (!providerId && !canScan) return null;
if (bulk) {
if (!canScan && providerId) {
return bulk.get(providerId)?.get(modelId) ?? null;
}
return pickMatchingOverride(collectOverrideHits(bulk), providerId, modelId, lookupKey);
}
const db = options.getDatabase?.() ?? getDbInstance();
if (providerId) {
const row = db
.prepare("SELECT value FROM key_value WHERE namespace = 'customModels' AND key = ?")
.get(providerId);
const exact = readVisionOverrideFromModels(getKeyValue(row).value, modelId);
if (exact !== null) return exact;
}
if (!canScan) return null;
return pickMatchingOverride(
collectOverrideHits(listCustomModelVisionOverrides({ getDatabase: () => db })),
providerId,
modelId,
lookupKey
);
} catch {
return null;
}
}
/** Bulk-load explicit custom-model vision overrides with one SQLite query. */
export function listCustomModelVisionOverrides(
options: CustomModelVisionOverrideReadOptions = {}
): CustomModelVisionOverrideMap {
try {
const db = options.getDatabase?.() ?? getDbInstance();
const rows = db
.prepare("SELECT key, value FROM key_value WHERE namespace = 'customModels'")
.all();
const result = new Map<string, Map<string, boolean>>();
for (const row of rows) {
const { key, value } = getKeyValue(row);
if (!key || !value) continue;
try {
const models = JSON.parse(value) as unknown;
if (!Array.isArray(models)) continue;
const byModel = new Map<string, boolean>();
for (const candidate of models) {
if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) continue;
const { id, supportsVision } = candidate as {
id?: unknown;
supportsVision?: unknown;
};
if (typeof id === "string" && typeof supportsVision === "boolean") {
byModel.set(id, supportsVision);
}
}
if (byModel.size > 0) result.set(key, byModel);
} catch {
// Malformed custom-model rows do not participate in capability resolution.
}
}
return result;
} catch {
return new Map<string, Map<string, boolean>>();
}
}

View File

@@ -845,14 +845,16 @@ export function getResolvedModelCapabilities(
// fields keep using the non-leaf `spec` from getStaticSpec() above.
const visionSpec = getVisionStaticSpec(resolved.model, resolved.rawModel);
// #9195: read the custom model's supportsVision override from the DB so the
// dashboard "Vision capable" toggle affects Combo routing.
// #9195 / #12758: keep the original provider&&model short-circuit. All
// three advertised id forms still parse to both halves; the matcher
// recovers the stored connection-id row via lookupKey / path leftover.
const customVisionOverride =
resolved.provider && resolved.model
? getCustomModelVisionOverride(
resolved.provider,
resolved.model,
snapshot?.customVisionOverrides
snapshot?.customVisionOverrides,
{ lookupKey: resolved.lookupKey ?? resolved.rawModel ?? lookupKey }
)
: null;

View File

@@ -0,0 +1,200 @@
/**
* #12758 — vision-bridge must honor Custom Models "Vision capable" for the
* same three id forms /v1/models advertises, not only the internal
* providerId/modelPath string.
*
* parseModel splits on the first slash, so a path-shaped custom id such as
* `orcarouter/Qwen3.8-27B-Uncensored-NVFP4` is looked up as provider
* `orcarouter` + model `Qwen3.8-...`. The override lives under the real
* openai-compatible connection id. Text chat already resolves that record;
* the bridge used a narrower pair and silently swapped in glm/glm-4.6v.
*/
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-12758-vision-"));
process.env.DATA_DIR = TEST_DATA_DIR;
process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "custom-vision-12758-test-secret";
const core = await import("../../src/lib/db/core.ts");
const { addCustomModel, getCustomModelVisionOverride } = await import("../../src/lib/db/models.ts");
const { getResolvedModelCapabilities } = await import("../../src/lib/modelCapabilities.ts");
const { VisionBridgeGuardrail } = await import("../../src/lib/guardrails/visionBridge.ts");
import type { GuardrailContext } from "../../src/lib/guardrails/base.ts";
import type { VisionModelConfig } from "../../src/lib/guardrails/visionBridgeHelpers.ts";
const CONNECTION_ID = "openai-compatible-chat-12758-vllm";
const CUSTOM_MODEL_ID = "orcarouter/Qwen3.8-27B-Uncensored-NVFP4";
const ADVERTISED_ALIAS = `vllm/${CUSTOM_MODEL_ID}`;
const FULL_INTERNAL = `${CONNECTION_ID}/${CUSTOM_MODEL_ID}`;
const FALLBACK_VISION_MODEL = "glm/glm-4.6v";
const ID_FORMS = [
{ name: "advertised alias vllm/path", model: ADVERTISED_ALIAS },
{ name: "bare path-shaped id", model: CUSTOM_MODEL_ID },
{ name: "full providerId/modelPath", model: FULL_INTERNAL },
] as const;
async function seedVisionCustomModel() {
await addCustomModel(
CONNECTION_ID,
CUSTOM_MODEL_ID,
"Qwen 3.8 vision",
"manual",
"chat-completions",
["chat"],
undefined,
{},
true
);
}
function imagePayload(model: string): Record<string, unknown> {
return {
model,
messages: [
{
role: "user",
content: [
{ type: "text", text: "What color is this image? One word." },
{
type: "image_url",
image_url: { url: "https://example.com/swatch.png" },
},
],
},
],
};
}
test.beforeEach(async () => {
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
await seedVisionCustomModel();
});
test.after(() => {
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
});
test("#12758 custom vision override matches all three advertised id forms", () => {
assert.equal(
getCustomModelVisionOverride(CONNECTION_ID, CUSTOM_MODEL_ID),
true,
"exact connection-id lookup is the control"
);
assert.equal(
getCustomModelVisionOverride("vllm", CUSTOM_MODEL_ID, undefined, {
lookupKey: ADVERTISED_ALIAS,
}),
true,
"vllm/path advertised alias must resolve the custom override"
);
assert.equal(
getCustomModelVisionOverride("orcarouter", "Qwen3.8-27B-Uncensored-NVFP4", undefined, {
lookupKey: CUSTOM_MODEL_ID,
}),
true,
"first-slash split of the path-shaped id must still hit the override"
);
});
for (const form of ID_FORMS) {
test(`#12758 capabilities.supportsVision is true for ${form.name}`, () => {
const caps = getResolvedModelCapabilities(form.model);
assert.equal(
caps.supportsVision,
true,
`${form.model} must inherit the Custom Models Vision capable flag`
);
});
test(`#12758 vision-bridge does not swap ${form.name} to ${FALLBACK_VISION_MODEL}`, async () => {
let visionCallCount = 0;
const guardrail = new VisionBridgeGuardrail({
deps: {
getSettings: async () => ({
visionBridgeEnabled: true,
visionBridgeModel: FALLBACK_VISION_MODEL,
}),
callVisionModel: async (_imageDataUri: string, _config: VisionModelConfig) => {
visionCallCount++;
return "should never describe";
},
hasUsableCredentials: async () => null,
},
});
const payload = imagePayload(form.model);
const result = await guardrail.preCall(payload, {
model: form.model,
log: { debug() {}, info() {}, warn() {}, error() {} },
} as unknown as GuardrailContext);
assert.equal(result.block, false);
assert.equal(visionCallCount, 0, "native vision must not call the describe model");
assert.equal(
result.modifiedPayload,
undefined,
`${form.model} must not be rewritten to ${FALLBACK_VISION_MODEL}`
);
const rewritten = (result.modifiedPayload as { model?: string } | undefined)?.model;
assert.notEqual(rewritten, FALLBACK_VISION_MODEL);
});
}
test("#12758 explicit supportsVision:false still wins on the advertised alias", async () => {
const textOnlyId = "orcarouter/text-only-qwen";
await addCustomModel(
CONNECTION_ID,
textOnlyId,
"Qwen text only",
"manual",
"chat-completions",
["chat"],
undefined,
{},
false
);
const advertised = `vllm/${textOnlyId}`;
assert.equal(
getCustomModelVisionOverride("vllm", textOnlyId, undefined, { lookupKey: advertised }),
false
);
assert.equal(getResolvedModelCapabilities(advertised).supportsVision, false);
});
test("#12758 bare registry id does not inherit an unrelated custom vision flag", () => {
assert.equal(
getCustomModelVisionOverride("", "gpt-4o", undefined, { lookupKey: "gpt-4o" }),
null,
"empty-provider gpt-4o must not scan customModels"
);
assert.equal(getResolvedModelCapabilities("gpt-4o").supportsVision, true);
});
test("#12758 a leaf stored id does not suffix-steal openai/gpt-4o", async () => {
await addCustomModel(
CONNECTION_ID,
"4o",
"stolen leaf",
"manual",
"chat-completions",
["chat"],
undefined,
{},
true
);
assert.equal(
getCustomModelVisionOverride("openai", "gpt-4o", undefined, { lookupKey: "openai/gpt-4o" }),
null,
"leaf '4o' must not match lookupKey openai/gpt-4o"
);
});