From ac20a3bd48eba4f0e86a4a1e633263391c2dea6a Mon Sep 17 00:00:00 2001 From: danscMax <153344025+danscMax@users.noreply.github.com> Date: Sat, 18 Jul 2026 21:14:57 +0300 Subject: [PATCH] fix(api): allow text-to-image on dual-modality models + revive HuggingFace image host (#7648) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(api): allow text-to-image on dual-modality models + revive HuggingFace image host Two image-generation regressions surfaced while testing /v1/images/generations: 1. Dual-modality models (inputModalities ["text","image"]) were rejected with "Image input is required" because the gate treated any "image" modality as mandatory. That blocked pure text-to-image on 41 models (Together x10, Stability x10, LMArena x15, NVIDIA x3, BFL x2, NanoGPT x1). Only edit-only models (modalities ["image"] with no "text") should require an image input; extract modalitiesRequireImageInput() and gate on that. 2. The HuggingFace image provider pointed at api-inference.huggingface.co, which HF retired (DNS-dead -> "fetch failed" 502). Route through router.huggingface.co/hf-inference/models, matching the chat provider which already migrated. Regression guard: tests/unit/image-text-to-image-modality.test.ts (fails on base -- the helper did not exist and the baseUrl was the retired host). * fix(api): keep Stability edit/control/upscale endpoints image-required modalitiesRequireImageInput() correctly stopped gating dual-modality (text+image) generation models on an image input, fixing pure text-to-image for 41 models. But 10 of those dual-modality entries are Stability AI's dedicated /v2beta/stable-image/{edit,control,upscale}/* endpoints (inpaint, outpaint, search-and-replace, search-and-recolor, replace-background-and-relight, creative, sketch, structure, style, style-transfer) — they accept a text prompt too, but mechanically require an input image upstream. The blanket modality-based inference silently dropped OmniRoute's client-side gate for exactly those 10, trading a clean 400 for a confusing upstream Stability error. Add an explicit `imageRequired` override on the registry entry, decided by the model's actual endpoint rather than inferred from its listed modalities, and combine it with modalitiesRequireImageInput() at the route gate: `imageModelEntry?.imageRequired || modalitiesRequireImageInput(...)`. Extracted the Stability AI model list into providers/registry/stability-ai/imageModels.ts (mirroring the existing kie/segmind pattern) — imageRegistry.ts sits right at the 800-line file-size cap and the extra flags would have pushed it over. Extended tests/unit/image-text-to-image-modality.test.ts: the previous "no dual-modality model is gated as image-required" assertion was exactly the bug (it would have passed even with the regression); new assertions cover the 10 Stability edit/control/upscale models by id (still require an image) alongside the true dual-modality generation models (BFL Kontext, NVIDIA, NanoGPT — still accept text-only). Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --------- Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --- open-sse/config/imageRegistry.ts | 56 +++---- .../registry/stability-ai/imageModels.ts | 76 +++++++++ src/app/api/v1/images/generations/route.ts | 9 +- .../unit/image-text-to-image-modality.test.ts | 145 ++++++++++++++++++ 4 files changed, 258 insertions(+), 28 deletions(-) create mode 100644 open-sse/config/providers/registry/stability-ai/imageModels.ts create mode 100644 tests/unit/image-text-to-image-modality.test.ts diff --git a/open-sse/config/imageRegistry.ts b/open-sse/config/imageRegistry.ts index d044eaf629..45803f4122 100644 --- a/open-sse/config/imageRegistry.ts +++ b/open-sse/config/imageRegistry.ts @@ -9,12 +9,16 @@ import { LMARENA_DIRECT_IMAGE_MODELS } from "./providers/registry/lmarena/direct import { SEGMIND_IMAGE_PROVIDER } from "./providers/registry/segmind/imageModels.ts"; import { KIE_IMAGE_MODELS } from "./providers/registry/kie/imageModels.ts"; import { FREEPIK_IMAGE_PROVIDER } from "./providers/registry/freepik/index.ts"; +import { STABILITY_AI_IMAGE_MODELS } from "./providers/registry/stability-ai/imageModels.ts"; import { GEMINI_IMAGEN_PROVIDER } from "./providers/registry/gemini/imageModels.ts"; interface ImageModelEntry { id: string; name: string; inputModalities?: string[]; + // See STABILITY_AI_IMAGE_MODELS for why this exists: some models accept "text" + // but mechanically require an image regardless. + imageRequired?: boolean; description?: string; isMarket?: boolean; } @@ -39,6 +43,7 @@ interface ImageModelAliasEntry { name: string; listInCatalog: boolean; inputModalities?: string[]; + imageRequired?: boolean; description?: string; } @@ -127,6 +132,13 @@ function findImageModelConfig(providerId, modelId) { return provider.models.find((model) => model.id === modelId) || null; } +// Kept out of getImageModelEntry() (which sits at the complexity-ratchet cap) — an +// alias can override imageRequired directly, else it falls back to its target +// model's own flag. Consumers coerce the result with Boolean(), so no `?? false`. +function resolveAliasImageRequired(alias, modelConfig) { + return alias.imageRequired ?? modelConfig?.imageRequired; +} + export const IMAGE_PROVIDERS: Record = { openai: { id: "openai", @@ -477,32 +489,7 @@ export const IMAGE_PROVIDERS: Record = { authType: "apikey", authHeader: "bearer", format: "stability-ai", - models: [ - { id: "stable-image-ultra", name: "Stable Image Ultra" }, - { id: "stable-image-core", name: "Stable Image Core" }, - { id: "sd3.5-large-turbo", name: "sd3.5-large-turbo" }, - { id: "sd3.5-large", name: "sd3.5-large" }, - { id: "sd3.5-medium", name: "sd3.5-medium" }, - { id: "sd3.5-flash", name: "sd3.5-flash" }, - { id: "erase", name: "Erase", inputModalities: ["image"] }, - { id: "inpaint", name: "Inpaint", inputModalities: ["text", "image"] }, - { id: "outpaint", name: "Outpaint", inputModalities: ["text", "image"] }, - { id: "remove-background", name: "Remove Background", inputModalities: ["image"] }, - { id: "search-and-replace", name: "Search and Replace", inputModalities: ["text", "image"] }, - { id: "search-and-recolor", name: "Search and Recolor", inputModalities: ["text", "image"] }, - { - id: "replace-background-and-relight", - name: "Replace Background and Relight", - inputModalities: ["text", "image"], - }, - { id: "creative", name: "Creative Upscale", inputModalities: ["text", "image"] }, - { id: "fast", name: "Fast Upscale", inputModalities: ["image"] }, - { id: "conservative", name: "Conservative Upscale", inputModalities: ["image"] }, - { id: "sketch", name: "Sketch Control", inputModalities: ["text", "image"] }, - { id: "structure", name: "Structure Control", inputModalities: ["text", "image"] }, - { id: "style", name: "Style Control", inputModalities: ["text", "image"] }, - { id: "style-transfer", name: "Style Transfer", inputModalities: ["text", "image"] }, - ], + models: STABILITY_AI_IMAGE_MODELS, supportedSizes: ["1024x1024", "1024x1280", "1280x1024"], }, @@ -625,7 +612,9 @@ export const IMAGE_PROVIDERS: Record = { // beyond this seed list. huggingface: { id: "huggingface", - baseUrl: "https://api-inference.huggingface.co/models", + // HF retired api-inference.huggingface.co; text-to-image now routes through + // router.huggingface.co with the hf-inference provider pinned in the path. + baseUrl: "https://router.huggingface.co/hf-inference/models", authType: "apikey", authHeader: "bearer", format: "huggingface-image", @@ -771,6 +760,7 @@ export function getImageModelEntry(modelStr) { provider: alias.provider, model: alias.model, inputModalities: alias.inputModalities || modelConfig?.inputModalities || ["text"], + imageRequired: resolveAliasImageRequired(alias, modelConfig), description: alias.description || modelConfig?.description || undefined, }; } @@ -785,6 +775,18 @@ export function getImageModelEntry(modelStr) { provider, model, inputModalities: modelConfig.inputModalities || ["text"], + imageRequired: modelConfig.imageRequired, description: modelConfig.description || undefined, }; } + +/** + * An image input is only MANDATORY for edit-only models — those whose modalities + * are `["image"]` with no `"text"`. Models listing both `["text", "image"]` accept + * an image but can also run pure text-to-image, so they must NOT be gated on an + * image input (that gate previously blocked 41 dual-modality t2i models). + */ +export function modalitiesRequireImageInput(inputModalities) { + const list = Array.isArray(inputModalities) ? inputModalities : ["text"]; + return list.includes("image") && !list.includes("text"); +} diff --git a/open-sse/config/providers/registry/stability-ai/imageModels.ts b/open-sse/config/providers/registry/stability-ai/imageModels.ts new file mode 100644 index 0000000000..4a5b20623e --- /dev/null +++ b/open-sse/config/providers/registry/stability-ai/imageModels.ts @@ -0,0 +1,76 @@ +/** + * Stability AI image-generation model catalog. + * + * Extracted out of imageRegistry.ts (which sits right at the 800-line file-size + * cap) so the catalog lives in its own semantic family module, following the same + * pattern as `providers/registry/kie/imageModels.ts` and + * `providers/registry/segmind/imageModels.ts`. See `imageRegistry.ts`'s + * `stability-ai` entry for baseUrl/auth/format wiring. + * + * `imageRequired: true` marks the dedicated edit/control/upscale endpoints + * (STABILITY_EDIT_ENDPOINTS in open-sse/handlers/imageGeneration.ts) that accept a + * text prompt but mechanically require an input image regardless — + * modalitiesRequireImageInput() alone can't tell them apart from flexible + * dual-modality generation models (BFL Kontext, Together, NVIDIA, LMArena, + * NanoGPT), which correctly allow pure text-to-image. + */ + +export interface StabilityImageModelEntry { + id: string; + name: string; + inputModalities?: string[]; + imageRequired?: boolean; +} + +export const STABILITY_AI_IMAGE_MODELS: StabilityImageModelEntry[] = [ + { id: "stable-image-ultra", name: "Stable Image Ultra" }, + { id: "stable-image-core", name: "Stable Image Core" }, + { id: "sd3.5-large-turbo", name: "sd3.5-large-turbo" }, + { id: "sd3.5-large", name: "sd3.5-large" }, + { id: "sd3.5-medium", name: "sd3.5-medium" }, + { id: "sd3.5-flash", name: "sd3.5-flash" }, + { id: "erase", name: "Erase", inputModalities: ["image"] }, + { id: "inpaint", name: "Inpaint", inputModalities: ["text", "image"], imageRequired: true }, + { id: "outpaint", name: "Outpaint", inputModalities: ["text", "image"], imageRequired: true }, + { id: "remove-background", name: "Remove Background", inputModalities: ["image"] }, + { + id: "search-and-replace", + name: "Search and Replace", + inputModalities: ["text", "image"], + imageRequired: true, + }, + { + id: "search-and-recolor", + name: "Search and Recolor", + inputModalities: ["text", "image"], + imageRequired: true, + }, + { + id: "replace-background-and-relight", + name: "Replace Background and Relight", + inputModalities: ["text", "image"], + imageRequired: true, + }, + { + id: "creative", + name: "Creative Upscale", + inputModalities: ["text", "image"], + imageRequired: true, + }, + { id: "fast", name: "Fast Upscale", inputModalities: ["image"] }, + { id: "conservative", name: "Conservative Upscale", inputModalities: ["image"] }, + { id: "sketch", name: "Sketch Control", inputModalities: ["text", "image"], imageRequired: true }, + { + id: "structure", + name: "Structure Control", + inputModalities: ["text", "image"], + imageRequired: true, + }, + { id: "style", name: "Style Control", inputModalities: ["text", "image"], imageRequired: true }, + { + id: "style-transfer", + name: "Style Transfer", + inputModalities: ["text", "image"], + imageRequired: true, + }, +]; diff --git a/src/app/api/v1/images/generations/route.ts b/src/app/api/v1/images/generations/route.ts index 162bacc1a5..857aeb3f04 100644 --- a/src/app/api/v1/images/generations/route.ts +++ b/src/app/api/v1/images/generations/route.ts @@ -10,6 +10,7 @@ import { parseImageModel, getImageProvider, getImageModelEntry, + modalitiesRequireImageInput, } from "@omniroute/open-sse/config/imageRegistry.ts"; import { errorResponse, unavailableResponse } from "@omniroute/open-sse/utils/error.ts"; import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts"; @@ -151,7 +152,13 @@ async function postHandler(request, context) { const imageModelEntry = getImageModelEntry(body.model); const inputModalities = imageModelEntry?.inputModalities || ["text"]; const requiresPrompt = inputModalities.includes("text"); - const requiresImageInput = inputModalities.includes("image"); + // imageRequired is an explicit registry override for models that list "text" among + // their modalities (they accept a prompt) but mechanically require an input image + // regardless — e.g. Stability AI's dedicated edit/control/upscale endpoints. Without + // it, modalitiesRequireImageInput() would infer "image optional" for any model that + // also lists "text", which is wrong for those. + const requiresImageInput = + Boolean(imageModelEntry?.imageRequired) || modalitiesRequireImageInput(inputModalities); const hasPrompt = typeof body.prompt === "string" && body.prompt.trim().length > 0; const hasImageInput = hasImageGenerationInput(body); diff --git a/tests/unit/image-text-to-image-modality.test.ts b/tests/unit/image-text-to-image-modality.test.ts new file mode 100644 index 0000000000..edeba35af7 --- /dev/null +++ b/tests/unit/image-text-to-image-modality.test.ts @@ -0,0 +1,145 @@ +/** + * Two image-generation regressions: + * + * 1. Dual-modality text-to-image models (inputModalities === ["text","image"]) were + * rejected on /v1/images/generations with "Image input is required" because the + * gate treated any "image" modality as mandatory. 41 models (Together, Stability, + * LMArena, NVIDIA, BFL, NanoGPT) could not do pure text-to-image as a result. + * + * Follow-up carve-out: 10 of those Stability AI models (inpaint, outpaint, + * search-and-replace, search-and-recolor, replace-background-and-relight, + * creative, sketch, structure, style, style-transfer) are Stability's dedicated + * /v2beta/stable-image/{edit,control,upscale}/* endpoints (STABILITY_EDIT_ENDPOINTS + * in open-sse/handlers/imageGeneration.ts) — they accept a text prompt too, but + * mechanically REQUIRE an input image. modalitiesRequireImageInput() alone can't + * distinguish them from the flexible dual-modality generation models (BFL Kontext, + * Together, NVIDIA, LMArena, NanoGPT), so the registry carries an explicit + * `imageRequired: true` flag on exactly those 10 entries, and the route gate + * (src/app/api/v1/images/generations/route.ts) combines both signals: + * `imageModelEntry?.imageRequired || modalitiesRequireImageInput(inputModalities)`. + * + * 2. The HuggingFace image provider pointed at the retired api-inference.huggingface.co + * host (DNS-dead → "fetch failed" 502). Text-to-image now routes through + * router.huggingface.co with the hf-inference provider pinned in the path. + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + IMAGE_PROVIDERS, + getImageModelEntry, + modalitiesRequireImageInput, +} from "../../open-sse/config/imageRegistry.ts"; + +// The 10 Stability AI edit/control/upscale endpoints that mechanically require an +// input image even though they also accept a text prompt — STABILITY_EDIT_ENDPOINTS +// in open-sse/handlers/imageGeneration.ts, minus the image-only entries (erase, +// remove-background, fast, conservative) which were never affected by this bug +// because their inputModalities never included "text" in the first place. +const STABILITY_IMAGE_REQUIRED_MODELS = [ + "inpaint", + "outpaint", + "search-and-replace", + "search-and-recolor", + "replace-background-and-relight", + "creative", + "sketch", + "structure", + "style", + "style-transfer", +]; + +// Mirrors the combined gate in src/app/api/v1/images/generations/route.ts. +function effectiveRequiresImageInput(entry, inputModalities) { + return Boolean(entry?.imageRequired) || modalitiesRequireImageInput(inputModalities); +} + +test("modalitiesRequireImageInput: only edit-only models require an image input", () => { + // Edit-only → image is mandatory. + assert.equal(modalitiesRequireImageInput(["image"]), true); + // Dual text-to-image + image-to-image → image optional (the bug: these were blocked). + assert.equal(modalitiesRequireImageInput(["text", "image"]), false); + // Text-only → never requires an image. + assert.equal(modalitiesRequireImageInput(["text"]), false); + // Defensive: undefined/non-array defaults to text-only behavior. + assert.equal(modalitiesRequireImageInput(undefined), false); + assert.equal(modalitiesRequireImageInput(null), false); +}); + +test("dual-modality (text+image) registry models allow pure text-to-image, EXCEPT the Stability edit/control/upscale set flagged imageRequired", () => { + const wronglyBlocked = []; + const wronglyAllowed = []; + for (const [providerId, config] of Object.entries(IMAGE_PROVIDERS)) { + for (const model of config.models || []) { + const im = model.inputModalities || ["text"]; + if (!im.includes("text") || !im.includes("image")) continue; + const effective = effectiveRequiresImageInput(model, im); + if (model.imageRequired) { + if (!effective) wronglyAllowed.push(`${providerId}/${model.id}`); + } else if (effective) { + wronglyBlocked.push(`${providerId}/${model.id}`); + } + } + } + assert.deepEqual( + wronglyBlocked, + [], + "dual text+image models without imageRequired must allow pure text-to-image" + ); + assert.deepEqual( + wronglyAllowed, + [], + "dual text+image models flagged imageRequired must still gate on image input" + ); +}); + +test("stability-ai edit/control/upscale models still REQUIRE an image despite accepting a text prompt", () => { + for (const modelId of STABILITY_IMAGE_REQUIRED_MODELS) { + const entry = getImageModelEntry(`stability-ai/${modelId}`); + assert.ok(entry, `stability-ai/${modelId} must resolve to a registry entry`); + assert.equal( + entry.imageRequired, + true, + `stability-ai/${modelId} must be flagged imageRequired` + ); + assert.equal( + effectiveRequiresImageInput(entry, entry.inputModalities), + true, + `stability-ai/${modelId} must still gate on image input (mechanically requires one upstream)` + ); + } +}); + +test("true dual-modality generation models (BFL Kontext, NVIDIA, NanoGPT) accept text-only", () => { + const dualModalitySamples = [ + "black-forest-labs/flux-kontext-pro", + "nvidia/black-forest-labs/flux.1-dev", + "nanogpt/qwen-image", + ]; + for (const modelStr of dualModalitySamples) { + const entry = getImageModelEntry(modelStr); + assert.ok(entry, `${modelStr} must resolve to a registry entry`); + assert.ok(!entry.imageRequired, `${modelStr} must not be flagged imageRequired`); + assert.equal( + effectiveRequiresImageInput(entry, entry.inputModalities), + false, + `${modelStr} must accept pure text-to-image` + ); + } +}); + +test("HuggingFace image provider uses the live router host, not the retired api-inference host", () => { + const hf = IMAGE_PROVIDERS.huggingface; + assert.ok(hf, "huggingface image provider must exist"); + assert.equal(hf.baseUrl, "https://router.huggingface.co/hf-inference/models"); + assert.ok( + !hf.baseUrl.includes("api-inference.huggingface.co"), + "must not use the DNS-dead api-inference.huggingface.co host" + ); + // The handler builds `${baseUrl}/${model}` — assert the resulting URL is the router form. + const model = hf.models[0].id; + assert.equal( + `${hf.baseUrl}/${model}`, + `https://router.huggingface.co/hf-inference/models/${model}` + ); +});