diff --git a/docs/ops/VM_DEPLOYMENT_GUIDE.md b/docs/ops/VM_DEPLOYMENT_GUIDE.md index 69e3ac4209..c0e64c74e7 100644 --- a/docs/ops/VM_DEPLOYMENT_GUIDE.md +++ b/docs/ops/VM_DEPLOYMENT_GUIDE.md @@ -429,5 +429,7 @@ For deployments on small VPS instances (1 GB RAM or less): - **Disable background services** — set `OMNIROUTE_DISABLE_BACKGROUND_SERVICES=1` to skip scheduler, MCP server, and periodic maintenance tasks. See `docs/reference/ENVIRONMENT.md`. - **Use SQLite WAL mode** — enabled by default, reduces peak memory during concurrent reads. +- **Cap the V8 heap** — set `OMNIROUTE_MEMORY_MB` (e.g. `512`) so the runtime does not calibrate a ceiling larger than the VM. See `docs/reference/ENVIRONMENT.md`. +- **Limit concurrent heavy requests** — lower `OMNIROUTE_CHAT_MAX_HEAVY_IN_FLIGHT` (default `1`); excess requests get a retryable `503` with `Retry-After` instead of competing for memory. - **Avoid `next build` on the VPS** — build locally and deploy the standalone output (`.next/standalone/`). - **Monitor with `top` / `free -m`** — OmniRoute typically uses 200-400 MB RSS at idle on a 1 GB VM. diff --git a/open-sse/services/combo/comboStructure.ts b/open-sse/services/combo/comboStructure.ts index 76ef372dc2..0430f03482 100644 --- a/open-sse/services/combo/comboStructure.ts +++ b/open-sse/services/combo/comboStructure.ts @@ -18,6 +18,7 @@ import { getHiddenModelsByProvider } from "../../../src/lib/db/models"; import { getComboModelString, normalizeComboStep } from "../../../src/lib/combos/steps.ts"; import { getProviderByAlias, getProviderById } from "../../../src/shared/constants/providers.ts"; import { estimateTokens } from "../contextManager.ts"; +import { containsMediaKind } from "../../utils/mediaParts.ts"; import { getResolvedModelCapabilities } from "../modelCapabilities.ts"; import { parseModel, stripContextWindowSuffix } from "../model.ts"; import { dedupeTargetsByExecutionKey, isRecord } from "./comboData.ts"; @@ -481,21 +482,8 @@ function estimateRequestInputTokens(body: Record): number { return Object.keys(estimatePayload).length > 0 ? estimateTokens(estimatePayload) : 0; } -function valueContainsImagePart(value: unknown, depth = 0): boolean { - if (depth > 8 || value === null || value === undefined) return false; - if (typeof value === "string") return value.startsWith("data:image/"); - if (Array.isArray(value)) return value.some((entry) => valueContainsImagePart(entry, depth + 1)); - if (!isRecord(value)) return false; - - const type = typeof value.type === "string" ? value.type.toLowerCase() : null; - if (type === "image" || type === "image_url" || type === "input_image") return true; - if ("image_url" in value || "input_image" in value) return true; - - const source = isRecord(value.source) ? value.source : null; - const mediaType = typeof source?.media_type === "string" ? source.media_type.toLowerCase() : ""; - if (mediaType.startsWith("image/")) return true; - - return Object.values(value).some((entry) => valueContainsImagePart(entry, depth + 1)); +function valueContainsImagePart(value: unknown): boolean { + return containsMediaKind([{ content: [value] }], "image"); } export function deriveRequestCompatibilityRequirements( @@ -533,6 +521,8 @@ function hasKnownCompatibleContextLimit( return evaluateContextLimit(capabilities, requirements, target.modelStr) === true; } +const HARD_COMPAT_REASONS = new Set(["tools", "vision", "structured_output", "output_tokens"]); + /** * #8332: vision is a hard requirement, not a soft preference — a target whose vision * support is not confirmed can never succeed on an image_url request. Callers @@ -616,7 +606,7 @@ export type CompatFilterOptions = { failOpen?: boolean; }; -function hasHardCapabilityFailure(reasons: string[]): boolean { +export function hasHardCapabilityFailure(reasons: string[]): boolean { return reasons.some((reason) => HARD_COMPAT_REASONS.has(reason)); } diff --git a/tests/unit/combo-vision-aware-routing.test.ts b/tests/unit/combo-vision-aware-routing.test.ts index ae84b3ce08..8d5c186c3f 100644 --- a/tests/unit/combo-vision-aware-routing.test.ts +++ b/tests/unit/combo-vision-aware-routing.test.ts @@ -37,6 +37,8 @@ process.env.DATA_DIR = TEST_DATA_DIR; const core = await import("../../src/lib/db/core.ts"); const { getResolvedModelCapabilities } = await import("../../src/lib/modelCapabilities.ts"); const { filterTargetsByRequestCompatibility } = await import("../../open-sse/services/combo.ts"); +const { deriveRequestCompatibilityRequirements, hasHardCapabilityFailure } = + await import("../../open-sse/services/combo/comboStructure.ts"); test.after(() => { core.resetDbInstance(); @@ -100,6 +102,28 @@ test("image request: combo drops the non-vision target, keeps the vision target" assert.ok(!ids.includes("mistral/ministral-14b-latest"), "non-vision target must be dropped"); }); +test("nested case-insensitive image indicators still enforce hard vision compatibility", () => { + const requirements = deriveRequestCompatibilityRequirements({ + messages: [ + { + role: "user", + content: [ + { + payload: { + type: "IMAGE_URL", + image_url: { url: "data:image/png;base64,iVBOR" }, + }, + }, + ], + }, + ], + }); + + assert.equal(requirements.requiresVision, true); + assert.equal(hasHardCapabilityFailure(["vision"]), true); + assert.equal(hasHardCapabilityFailure(["context_window"]), false); +}); + test( "image request with NO confirmed-vision target: strip all (#8332 — never dispatch " + "an image body to a confirmed-non-vision target, even as a last resort)",