diff --git a/changelog.d/fixes/12492-devin-literal-effort-ids.md b/changelog.d/fixes/12492-devin-literal-effort-ids.md new file mode 100644 index 0000000000..831ed6046b --- /dev/null +++ b/changelog.d/fixes/12492-devin-literal-effort-ids.md @@ -0,0 +1 @@ +- **fix(devin):** treat Devin CLI model ids as literal — never strip or synthesize effort suffixes ([#12492](https://github.com/diegosouzapw/OmniRoute/pull/12492) — thanks @Neuron-Mr-White) diff --git a/open-sse/handlers/chatCore/claudeEffortVariant.ts b/open-sse/handlers/chatCore/claudeEffortVariant.ts index 2da837a4f6..5e5f9a7792 100644 --- a/open-sse/handlers/chatCore/claudeEffortVariant.ts +++ b/open-sse/handlers/chatCore/claudeEffortVariant.ts @@ -16,6 +16,7 @@ import { splitClaudeEffortSuffix } from "../../config/providerModels.ts"; import { isClaudeCodeCompatibleProvider } from "../../services/claudeCodeCompatible.ts"; import { FORMATS } from "../../translator/formats.ts"; import { isKnownClaudeEffortBaseModel } from "../../utils/claudeEffortVariants.ts"; +import { isDevinLiteralModelIdProvider } from "../../utils/devinLiteralModelIds.ts"; /** * True when the client already supplied an explicit reasoning effort (top-level reasoning_effort, @@ -52,6 +53,14 @@ export function applyClaudeEffortVariant(opts: { let effectiveModel = opts.effectiveModel; let log: string | null = null; + // Devin CLI catalogs embed the effort tier in the model id itself + // (`claude-opus-5-low` is a distinct upstream model). Stripping the suffix + // would dispatch a base id that does not exist upstream, so keep the id + // literal for these providers regardless of the Claude-family name. + if (isDevinLiteralModelIdProvider(provider)) { + return { effectiveModel, log: null }; + } + if (typeof effectiveModel === "string") { const { baseModel, effort } = splitClaudeEffortSuffix(effectiveModel); const isDirectClaudeLane = provider === "claude" || isClaudeCodeCompatibleProvider(provider); diff --git a/open-sse/utils/claudeEffortVariants.ts b/open-sse/utils/claudeEffortVariants.ts index 78dcad34c0..3b1fa5bc21 100644 --- a/open-sse/utils/claudeEffortVariants.ts +++ b/open-sse/utils/claudeEffortVariants.ts @@ -30,6 +30,7 @@ */ import { getModelSpec } from "@/shared/constants/modelSpecs"; import { supportsXHighEffort } from "../config/providerModels.ts"; +import { isDevinLiteralModelIdProvider } from "./devinLiteralModelIds.ts"; /** Base reasoning-effort levels advertised for every effort-capable Claude model. */ export const CLAUDE_EFFORT_VARIANT_LEVELS = ["low", "medium", "high"] as const; @@ -94,6 +95,16 @@ export function shouldExposeClaudeEffortVariants( if (id.startsWith(NO_THINKING_PREFIX)) return false; if (CLAUDE_EFFORT_SUFFIX_RE.test(id)) return false; + // Devin CLI catalogs (devin-cli / devin-cli-agentic / devin-desktop, aliases + // dv / dva) embed the tier in the model id itself — every tier is already a + // distinct catalog id, and the gateway keeps those ids literal (see + // devinLiteralModelIds.ts). Synthesizing `-` variants on top of them + // would advertise unroutable phantom ids like `dva/claude-opus-5-max-low`. + const providerSlash = id.indexOf("/"); + if (providerSlash > 0 && isDevinLiteralModelIdProvider(id.slice(0, providerSlash))) { + return false; + } + const name = bareModelName(id); return isKnownClaudeEffortBaseModel(name); } diff --git a/open-sse/utils/devinLiteralModelIds.ts b/open-sse/utils/devinLiteralModelIds.ts new file mode 100644 index 0000000000..3dc92dfcc7 --- /dev/null +++ b/open-sse/utils/devinLiteralModelIds.ts @@ -0,0 +1,40 @@ +/** + * Devin CLI providers whose upstream catalog embeds the reasoning tier IN the + * model id itself: `claude-opus-5-low`, `claude-opus-5-medium`, … and + * `gpt-5-6-sol-max` / `gpt-5-6-sol-low` are distinct upstream models + * (see `config/providers/registry/devin/catalog.ts`). For these providers a + * trailing `-{effort}` suffix is NOT a client-side effort variant: + * + * - stripping it (`applyClaudeEffortVariant`) would dispatch a base id that + * does not exist upstream — e.g. `dva/claude-opus-5-low` became + * `claude-opus-5` and the executor rejected it with + * "Model is not present in the current Devin catalog"; + * - synthesizing variants on top of tier-embedded ids produces phantom ids + * (`claude-opus-5-max-low`) that cannot route once the strip is fixed. + * + * Ids here cover the provider id and its routing alias, so both canonical and + * alias-prefixed qualified model ids are recognized. + */ + +const DEVIN_LITERAL_MODEL_ID_PROVIDERS = new Set([ + "devin-cli", + "devin-cli-agentic", + "devin-desktop", +]); +const DEVIN_LITERAL_MODEL_ID_ALIASES = new Set(["dv", "dva"]); + +function bareProviderToken(value: string): string { + const slash = value.indexOf("/"); + return slash >= 0 ? value.slice(0, slash) : value; +} + +/** + * True when `provider` (a provider id or alias, optionally `provider/model` + * qualified) serves a Devin catalog whose model ids embed the effort tier and + * must therefore be treated as literal ids. + */ +export function isDevinLiteralModelIdProvider(provider: string | null | undefined): boolean { + if (typeof provider !== "string" || provider.length === 0) return false; + const token = bareProviderToken(provider); + return DEVIN_LITERAL_MODEL_ID_PROVIDERS.has(token) || DEVIN_LITERAL_MODEL_ID_ALIASES.has(token); +} diff --git a/open-sse/utils/syncedEffortVariants.ts b/open-sse/utils/syncedEffortVariants.ts index 213128cb38..da47678dfe 100644 --- a/open-sse/utils/syncedEffortVariants.ts +++ b/open-sse/utils/syncedEffortVariants.ts @@ -27,6 +27,7 @@ * model that legitimately ends in an effort-like token (e.g. a model named "...-high"). */ import { CANONICAL_EFFORT_VALUES } from "@/shared/reasoning/effortStandardization.ts"; +import { isDevinLiteralModelIdProvider } from "./devinLiteralModelIds.ts"; /** Provider ids with dedicated `-{effort}` aliases — never synthesize another suffix layer. */ export const SYNCED_EFFORT_SKIP_PROVIDERS = new Set(["codex", "glm", "glm-cn", "glmt"]); @@ -37,7 +38,10 @@ const SYNCED_EFFORT_SKIP_PROVIDER_PREFIXES = ["kimi"]; export function isSkippedEffortProvider(ownedBy: string): boolean { return ( SYNCED_EFFORT_SKIP_PROVIDERS.has(ownedBy) || - SYNCED_EFFORT_SKIP_PROVIDER_PREFIXES.some((prefix) => ownedBy.startsWith(prefix)) + SYNCED_EFFORT_SKIP_PROVIDER_PREFIXES.some((prefix) => ownedBy.startsWith(prefix)) || + // Devin CLI catalogs (devin-cli / devin-cli-agentic / devin-desktop, aliases + // dv / dva) embed the tier in the id itself — no variant layer on top. + isDevinLiteralModelIdProvider(ownedBy) ); } diff --git a/tests/unit/chatcore-claude-effort-variant.test.ts b/tests/unit/chatcore-claude-effort-variant.test.ts index 4cda1b2a47..7ea9c1f515 100644 --- a/tests/unit/chatcore-claude-effort-variant.test.ts +++ b/tests/unit/chatcore-claude-effort-variant.test.ts @@ -215,3 +215,62 @@ test("no-think alias's explicit reasoning_effort:none is not overwritten by a st assert.equal(body.model, "claude-sonnet-5"); assert.equal(body.reasoning_effort, "none"); }); + +// ── Devin CLI providers: model ids embed the tier and must stay literal ───────── +// Regression for `dva/claude-opus-5-low` → stripped to `claude-opus-5` → executor +// rejected "Model is not present in the current Devin catalog" (400). The Devin +// catalog (devin/catalog.ts) has one id per tier; only the accidental +// double-suffixed ids (`claude-opus-5-max-low`) survived the old behavior. + +test("devin-cli-agentic provider keeps a tier-embedded id literal (no strip, no body mutation)", () => { + const body: Record = { model: "claude-opus-5-low", messages: [] }; + const r = applyClaudeEffortVariant({ + provider: "devin-cli-agentic", + effectiveModel: "claude-opus-5-low", + body, + sourceFormat: FORMATS.OPENAI, + }); + assert.equal(r.effectiveModel, "claude-opus-5-low"); + assert.equal(body.model, "claude-opus-5-low"); + assert.equal(body.reasoning_effort, undefined); + assert.equal(r.log, null); +}); + +test("devin provider alias (dva) is covered too", () => { + const body: Record = { model: "claude-opus-5-medium", messages: [] }; + const r = applyClaudeEffortVariant({ + provider: "dva", + effectiveModel: "claude-opus-5-medium", + body, + sourceFormat: FORMATS.OPENAI, + }); + assert.equal(r.effectiveModel, "claude-opus-5-medium"); + assert.equal(body.model, "claude-opus-5-medium"); + assert.equal(r.log, null); +}); + +test("devin-cli (text bridge) and devin-desktop keep literal ids as well", () => { + for (const provider of ["devin-cli", "devin-desktop", "dv"]) { + const body: Record = { model: "claude-sonnet-5-low", messages: [] }; + const r = applyClaudeEffortVariant({ + provider, + effectiveModel: "claude-sonnet-5-low", + body, + sourceFormat: FORMATS.OPENAI, + }); + assert.equal(r.effectiveModel, "claude-sonnet-5-low", provider); + assert.equal(body.reasoning_effort, undefined, provider); + } +}); + +test("a claude-lane strip still happens for the same model name (control)", () => { + const body: Record = { model: "claude-opus-5-low", messages: [] }; + const r = applyClaudeEffortVariant({ + provider: "claude", + effectiveModel: "claude-opus-5-low", + body, + sourceFormat: FORMATS.OPENAI, + }); + assert.equal(r.effectiveModel, "claude-opus-5"); + assert.equal(body.reasoning_effort, "low"); +}); diff --git a/tests/unit/devin-literal-effort-ids.test.ts b/tests/unit/devin-literal-effort-ids.test.ts new file mode 100644 index 0000000000..d54f451687 --- /dev/null +++ b/tests/unit/devin-literal-effort-ids.test.ts @@ -0,0 +1,75 @@ +// tests/unit/devin-literal-effort-ids.test.ts +// Devin CLI providers (devin-cli / devin-cli-agentic / devin-desktop, aliases dv / dva) +// serve catalogs whose model ids EMBED the reasoning tier (`claude-opus-5-low`, +// `gpt-5-6-sol-max` are distinct upstream models — see +// open-sse/config/providers/registry/devin/catalog.ts). Locks the three gates that +// must treat those ids as literal: +// 1. applyClaudeEffortVariant never strips the suffix for devin providers; +// 2. appendClaudeEffortVariants never synthesizes `-` variants on top of +// them (no phantom `dva/claude-opus-5-max-low` ids in /v1/models); +// 3. appendSyncedEffortVariants / isSkippedEffortProvider treat devin providers +// as owning their own suffix mechanism (no second variant layer). +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { isDevinLiteralModelIdProvider } from "../../open-sse/utils/devinLiteralModelIds.ts"; +import { appendClaudeEffortVariants } from "../../open-sse/utils/claudeEffortVariants.ts"; +import { + appendSyncedEffortVariants, + isSkippedEffortProvider, +} from "../../open-sse/utils/syncedEffortVariants.ts"; + +test("isDevinLiteralModelIdProvider matches ids, aliases, and qualified prefixes", () => { + for (const hit of [ + "devin-cli", + "devin-cli-agentic", + "devin-desktop", + "dv", + "dva", + "dva/claude-opus-5-low", + "devin-cli-agentic/claude-opus-5-max", + ]) { + assert.equal(isDevinLiteralModelIdProvider(hit), true, hit); + } + for (const miss of ["claude", "cc", "vertex", "", null, undefined, "codex", "deepseek"]) { + assert.equal( + isDevinLiteralModelIdProvider(miss as string | null | undefined), + false, + String(miss) + ); + } +}); + +test("appendClaudeEffortVariants adds no tier variants for devin-prefixed models", () => { + const models = [ + { id: "dva/claude-opus-5-max", root: "claude-opus-5-max" }, + { id: "devin-cli-agentic/claude-5-fable-max", root: "claude-5-fable-max" }, + ]; + const out = appendClaudeEffortVariants(models); + assert.equal(out.length, models.length); + assert.deepEqual( + out.map((m) => m.id), + ["dva/claude-opus-5-max", "devin-cli-agentic/claude-5-fable-max"] + ); +}); + +test("appendSyncedEffortVariants adds no tier variants for devin-owned models", () => { + const models = [ + { + id: "dva/gpt-5-6-sol-max", + owned_by: "devin-cli-agentic", + capabilities: { effort_tiers: ["low", "medium", "high", "xhigh"] }, + }, + ]; + const out = appendSyncedEffortVariants(models as never); + assert.equal(out.length, 1); + assert.equal(out[0].id, "dva/gpt-5-6-sol-max"); +}); + +test("isSkippedEffortProvider covers devin providers and aliases", () => { + for (const provider of ["devin-cli", "devin-cli-agentic", "devin-desktop", "dva", "dv"]) { + assert.equal(isSkippedEffortProvider(provider), true, provider); + } + assert.equal(isSkippedEffortProvider("claude"), false); + assert.equal(isSkippedEffortProvider("codex"), true); // pre-existing skip stays +});