mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-03 13:52:09 +03:00
fix(vision-bridge): process images when vision-capable model has combo mapping
When a model-combo mapping routes a vision-capable model through a combo where some targets may NOT support vision, the vision bridge must process images so combo targets can describe them. Before: if body.model supports vision, the vision bridge skipped image processing entirely. Non-vision combo targets would receive raw images they can't handle. After: before skipping, check if the model has a model-combo mapping. If it does, process images through the vision bridge regardless of body.model's native vision support. - Add checkModelHasComboMapping() helper (dynamic import, failsafe) - Add checkModelHasComboMapping dep to VisionBridgeDependencies (testable) - Guardrail preCall: check combo mapping before early-return on vision support - Add VB-S11 / VB-S11b tests
This commit is contained in:
@@ -18,6 +18,28 @@ import {
|
||||
isVisionBridgeForcedModel,
|
||||
} from "@/shared/constants/visionBridgeDefaults";
|
||||
|
||||
/// Check if a model name has a model-combo mapping.
|
||||
/// When a user sends `model: "gpt-4o"` with a model-combo mapping,
|
||||
/// the actual execution model(s) might differ. Non-vision combo
|
||||
/// targets would fail with images they can't handle, so the
|
||||
/// vision bridge must process images even if body.model supports vision.
|
||||
async function checkModelHasComboMapping(model: string): Promise<boolean> {
|
||||
try {
|
||||
// 1. Check for exact combo name match
|
||||
const { getComboByName } = await import("@/lib/localDb");
|
||||
const exactCombo = await getComboByName(model);
|
||||
if (exactCombo) return true;
|
||||
|
||||
// 2. Check for model-combo mapping (glob pattern match)
|
||||
const { resolveComboForModel } = await import("@/lib/db/modelComboMappings");
|
||||
const mapping = await resolveComboForModel(model);
|
||||
return mapping !== null;
|
||||
} catch {
|
||||
// Tables may not exist (pre-migration), or DB not initialized
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export interface VisionBridgeDependencies {
|
||||
getSettings?: () => Promise<Record<string, unknown>>;
|
||||
callVisionModel?: (
|
||||
@@ -25,6 +47,8 @@ export interface VisionBridgeDependencies {
|
||||
config: import("./visionBridgeHelpers").VisionModelConfig,
|
||||
apiKey?: string
|
||||
) => Promise<string>;
|
||||
/** Skip real DB lookup — return true to test combo-mapping path, false for normal path. */
|
||||
checkModelHasComboMapping?: (model: string) => Promise<boolean>;
|
||||
}
|
||||
|
||||
export class VisionBridgeGuardrail extends BaseGuardrail {
|
||||
@@ -61,7 +85,17 @@ export class VisionBridgeGuardrail extends BaseGuardrail {
|
||||
// 4. Check if model supports vision
|
||||
const capabilities = getResolvedModelCapabilities(model);
|
||||
if (capabilities.supportsVision === true && !forceVisionBridge) {
|
||||
return { block: false };
|
||||
// The request model supports vision natively, but check if a
|
||||
// model-combo mapping routes this model through a combo where
|
||||
// some targets may NOT support vision. In that case, the vision
|
||||
// bridge must process images so combo targets can describe them.
|
||||
const hasMapping = this.deps.checkModelHasComboMapping
|
||||
? await this.deps.checkModelHasComboMapping(model)
|
||||
: await checkModelHasComboMapping(model);
|
||||
if (!hasMapping) {
|
||||
return { block: false };
|
||||
}
|
||||
// Combo mapping found — fall through to process images
|
||||
}
|
||||
|
||||
// 5. Get body and check for messages
|
||||
|
||||
@@ -38,6 +38,7 @@ function createGuardrail(options?: Parameters<typeof VisionBridgeGuardrail>[0])
|
||||
}
|
||||
return mockVisionResponse;
|
||||
},
|
||||
...(options?.deps ?? {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -501,3 +502,69 @@ test("VB-S10: returns meta with imagesProcessed count", async () => {
|
||||
assert.strictEqual(typeof meta.processingTimeMs, "number");
|
||||
assert.strictEqual(meta.visionModel, "openai/gpt-4o-mini");
|
||||
});
|
||||
|
||||
// ── VB-S11: Combo mapping forces vision processing despite vision-capable model ──
|
||||
|
||||
test("VB-S11: processes images when vision-capable model has combo mapping", async () => {
|
||||
mockVisionResponse = "A description from combo-mapped vision bridge";
|
||||
const guardrail = createGuardrail({
|
||||
deps: {
|
||||
checkModelHasComboMapping: async (_model: string) => true,
|
||||
},
|
||||
});
|
||||
|
||||
const payload = createPayload({
|
||||
model: "openai/gpt-4o",
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "What is this?" },
|
||||
{
|
||||
type: "image_url",
|
||||
image_url: { url: "https://example.com/image.png" },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const startCallCount = visionCallCount;
|
||||
const result = await guardrail.preCall(payload, createContext({ model: "openai/gpt-4o" }));
|
||||
|
||||
// Vision bridge should have processed the image
|
||||
assert.strictEqual(result.block, false);
|
||||
assert.ok(visionCallCount > startCallCount, "Expected vision model to be called");
|
||||
assert.ok(result.modifiedPayload !== undefined, "Expected modifiedPayload when combo mapping forces vision bridge");
|
||||
});
|
||||
|
||||
test("VB-S11b: passthroughs when vision-capable model has NO combo mapping", async () => {
|
||||
const guardrail = createGuardrail({
|
||||
deps: {
|
||||
checkModelHasComboMapping: async (_model: string) => false,
|
||||
},
|
||||
});
|
||||
|
||||
const payload = createPayload({
|
||||
model: "openai/gpt-4o",
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "What is this?" },
|
||||
{
|
||||
type: "image_url",
|
||||
image_url: { url: "https://example.com/image.png" },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const result = await guardrail.preCall(payload, createContext({ model: "openai/gpt-4o" }));
|
||||
|
||||
// Vision bridge should skip (passthrough) since model supports vision and no combo mapping
|
||||
assert.strictEqual(result.block, false);
|
||||
assert.strictEqual(result.modifiedPayload, undefined);
|
||||
assert.strictEqual(visionCallCount, 0);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user