mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-24 08:02:14 +03:00
feat(models): surface learned reasoning_effort sets in catalog, variants, and dispatch (#11252)
Validated on the combined 12-PR batch board + the resolved merge against the post-#11232 tip: focused suites 73/73 (learned-reasoning-effort-caps, synced-capabilities-learned-effort-override, synced-effort-suffix-learned-validation, effort-tiers-loop-catalog-e2e, reasoning-effort-clamp-and-retry, reasoning-effort-learned-capability) + opencode-plugin effort-tier-variants 4/4, typecheck:core clean, gates within baseline. The stacked-branch conflict after #11232 squash-landed was resolved by hand (the learned-caps module keeps both the Set API and the new model-scoped lookup). The effort_tiers loop is closed end-to-end: catalog advertises exactly what the upstream accepts, and -<tier> suffix variants resolve against the learned set. Thank you @maxmad64bis!
This commit is contained in:
@@ -23,7 +23,7 @@
|
||||
"scripts": {
|
||||
"build": "tsup",
|
||||
"clean": "rm -rf dist",
|
||||
"test": "node --import tsx/esm --test tests/scaffold.test.ts tests/auth.test.ts tests/options-schema.test.ts tests/multi-instance.test.ts tests/fetch-interceptor.test.ts tests/provider.test.ts tests/gemini-sanitize.test.ts tests/combos.test.ts tests/config-shim.test.ts tests/features.test.ts tests/feature-defaults.test.ts tests/usable-combo.test.ts tests/disk-snapshot-perms.test.ts tests/fork-features.test.ts tests/auto-combo-context.test.ts tests/provider-id-routing.test.ts tests/management-read-token.test.ts tests/auto-sync.test.ts tests/model-allowlist.test.ts tests/log-level.test.ts",
|
||||
"test": "node --import tsx/esm --test tests/scaffold.test.ts tests/auth.test.ts tests/options-schema.test.ts tests/multi-instance.test.ts tests/fetch-interceptor.test.ts tests/provider.test.ts tests/gemini-sanitize.test.ts tests/combos.test.ts tests/config-shim.test.ts tests/features.test.ts tests/feature-defaults.test.ts tests/usable-combo.test.ts tests/disk-snapshot-perms.test.ts tests/fork-features.test.ts tests/auto-combo-context.test.ts tests/provider-id-routing.test.ts tests/management-read-token.test.ts tests/auto-sync.test.ts tests/model-allowlist.test.ts tests/log-level.test.ts tests/effort-tier-variants.test.ts",
|
||||
"prepublishOnly": "npm run clean && npm run build && npm test"
|
||||
},
|
||||
"keywords": [
|
||||
|
||||
@@ -1161,6 +1161,8 @@ export interface OmniRouteRawModelEntry {
|
||||
attachment?: boolean;
|
||||
structured_output?: boolean;
|
||||
temperature?: boolean;
|
||||
/** Runtime-learned or synced reasoning tiers (server-gated, blind-mapped). */
|
||||
effort_tiers?: string[];
|
||||
};
|
||||
release_date?: string;
|
||||
last_updated?: string;
|
||||
@@ -1302,6 +1304,18 @@ export function mapRawModelToModelV2(
|
||||
ctx: { providerId: string; baseURL: string; apiFormat?: { anthropicPrefixes?: string[] } }
|
||||
): ModelV2 {
|
||||
const caps = raw.capabilities ?? {};
|
||||
// effort_tiers loop: server-declared tiers become ModelV2 variants so the
|
||||
// UI offers exactly the tiers OmniRoute vouches for (instead of opencode's
|
||||
// invented [low, medium, high] fallback). Blind: filtering/exclusion rules
|
||||
// live server-side. Absent/empty/malformed => key omitted ENTIRELY (an
|
||||
// empty variants object would suppress opencode's fallback for this model).
|
||||
const declaredTiers = Array.isArray(caps.effort_tiers)
|
||||
? caps.effort_tiers.filter((t): t is string => typeof t === "string" && t.length > 0)
|
||||
: [];
|
||||
const variants =
|
||||
declaredTiers.length > 0
|
||||
? Object.fromEntries(declaredTiers.map((tier) => [tier, { reasoningEffort: tier }]))
|
||||
: undefined;
|
||||
const inMods = new Set(raw.input_modalities ?? ["text"]);
|
||||
const outMods = new Set(raw.output_modalities ?? ["text"]);
|
||||
|
||||
@@ -1315,10 +1329,7 @@ export function mapRawModelToModelV2(
|
||||
// OpenCode looks up `-m <plugin>/<combo>` as model id `<combo>` under
|
||||
// the plugin provider (#10345). Other bare ids still prefix with
|
||||
// `providerId` so credentials resolve as `(omniroute, model)`.
|
||||
id:
|
||||
raw.id.includes("/") || raw.owned_by === "combo"
|
||||
? raw.id
|
||||
: `${ctx.providerId}/${raw.id}`,
|
||||
id: raw.id.includes("/") || raw.owned_by === "combo" ? raw.id : `${ctx.providerId}/${raw.id}`,
|
||||
/**
|
||||
* Display name. Falls back to raw.id when no enrichment is available;
|
||||
* the caller (`createOmniRouteProviderHook`) overlays
|
||||
@@ -1357,6 +1368,7 @@ export function mapRawModelToModelV2(
|
||||
...(typeof raw.max_input_tokens === "number" ? { input: raw.max_input_tokens } : {}),
|
||||
output: typeof raw.max_output_tokens === "number" ? raw.max_output_tokens : 0,
|
||||
},
|
||||
...(variants ? { variants } : {}),
|
||||
status: "active",
|
||||
options: {},
|
||||
headers: {},
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
/**
|
||||
* effort_tiers loop — plugin maps server-declared tiers to ModelV2 variants.
|
||||
* Blind mapping (I3): no owned_by/provider knowledge here — the SERVER gates
|
||||
* eligibility (shouldExposeSyncedEffortVariants). Absence semantics (M3):
|
||||
* no tiers => NO variants key at all (an empty object would also kill
|
||||
* opencode's own fallback for non-tiered models).
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mapRawModelToModelV2, type OmniRouteRawModelEntry } from "../src/index.js";
|
||||
|
||||
const CTX = { providerId: "omniroute", baseURL: "http://127.0.0.1:20128" } as const;
|
||||
|
||||
test("maps declared tiers to reasoningEffort variants", () => {
|
||||
const raw: OmniRouteRawModelEntry = {
|
||||
id: "oc/x-preview-f-free",
|
||||
owned_by: "opencode",
|
||||
capabilities: { reasoning: true, effort_tiers: ["low", "high", "max"] },
|
||||
};
|
||||
const model = mapRawModelToModelV2(raw, { ...CTX });
|
||||
const variants = (model as unknown as Record<string, unknown>).variants as
|
||||
Record<string, Record<string, unknown>> | undefined;
|
||||
assert.ok(variants, "variants key present when tiers declared");
|
||||
assert.deepEqual(Object.keys(variants).sort(), ["high", "low", "max"]);
|
||||
assert.deepEqual(variants.max, { reasoningEffort: "max" });
|
||||
assert.deepEqual(variants.low, { reasoningEffort: "low" });
|
||||
});
|
||||
|
||||
test("no tiers => NO variants key (not an empty object)", () => {
|
||||
const raw: OmniRouteRawModelEntry = {
|
||||
id: "plain-model",
|
||||
capabilities: { reasoning: true },
|
||||
};
|
||||
const model = mapRawModelToModelV2(raw, { ...CTX }) as unknown as Record<string, unknown>;
|
||||
assert.equal("variants" in model, false);
|
||||
});
|
||||
|
||||
test("empty or malformed tiers array => NO variants key", () => {
|
||||
const empty = mapRawModelToModelV2(
|
||||
{ id: "m", capabilities: { effort_tiers: [] } },
|
||||
{ ...CTX }
|
||||
) as unknown as Record<string, unknown>;
|
||||
assert.equal("variants" in empty, false);
|
||||
|
||||
const junk = mapRawModelToModelV2(
|
||||
{ id: "m", capabilities: { effort_tiers: [42, null, "ok"] as unknown as string[] } },
|
||||
{ ...CTX }
|
||||
) as unknown as Record<string, unknown>;
|
||||
const variants = junk.variants as Record<string, Record<string, unknown>> | undefined;
|
||||
assert.deepEqual(Object.keys(variants ?? {}), ["ok"], "non-string tokens dropped");
|
||||
});
|
||||
|
||||
test("static registry entry WITH tiers also gets variants (N1 blast radius)", () => {
|
||||
const raw: OmniRouteRawModelEntry = {
|
||||
id: "some-static-model",
|
||||
owned_by: "registry",
|
||||
capabilities: { effort_tiers: ["minimal", "high"] },
|
||||
};
|
||||
const model = mapRawModelToModelV2(raw, { ...CTX }) as unknown as Record<string, unknown>;
|
||||
const variants = model.variants as Record<string, Record<string, unknown>> | undefined;
|
||||
assert.deepEqual(Object.keys(variants ?? {}).sort(), ["high", "minimal"]);
|
||||
});
|
||||
1
changelog.d/features/effort-tiers-loop-learned-sets.md
Normal file
1
changelog.d/features/effort-tiers-loop-learned-sets.md
Normal file
@@ -0,0 +1 @@
|
||||
- **feat(catalog):** surface runtime-learned `reasoning_effort` tiers in `/v1/models` `capabilities.effort_tiers` (learned set replaces synced metadata when present), map them to OpenCode `ModelV2.variants` in the OmniRoute plugin, and align dispatch `-<tier>` suffix validation to the effective (learned ?? synced) set — so the UI offers exactly the tiers the upstream accepts (e.g. `{low, high, max}` for `oc/x-preview-f-free`) and each advertised variant completes. Excludes codex/glm/kimi, which keep their own dedicated `-{effort}` suffix mechanism and never gain `effort_tiers` from this path (related to #7694, builds on #11232)
|
||||
@@ -63,6 +63,30 @@ export function getLearnedReasoningEffort(
|
||||
return v ? new Set(v) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Model-scoped lookup bridging the key-space gap between executors and the
|
||||
* catalog: executors record under their CONNECTION id
|
||||
* (`openai-compatible-chat-<uuid>:<model>`, cf. compatibleProviderId.ts),
|
||||
* while the catalog loops on provider ids (`opencode`, …) — an exact
|
||||
* `${provider}:${model}` lookup would always miss. Scans by model segment
|
||||
* instead. Multiple connections teaching different sets for the same model
|
||||
* name intersect (most restrictive proven set wins — conservative across
|
||||
* connections sharing one catalog entry).
|
||||
*/
|
||||
export function getLearnedReasoningEffortForModel(
|
||||
model: string | null | undefined
|
||||
): Set<string> | null {
|
||||
const m = typeof model === "string" ? model.trim().toLowerCase() : "";
|
||||
if (!m || learnedCaps.size === 0) return null;
|
||||
let result: Set<string> | null = null;
|
||||
for (const [key, value] of learnedCaps) {
|
||||
const colon = key.indexOf(":");
|
||||
if (colon === -1 || key.slice(colon + 1) !== m) continue;
|
||||
result = result ? new Set([...result].filter((v) => value.has(v))) : new Set(value);
|
||||
}
|
||||
return result && result.size > 0 ? result : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record that `acceptedValues` is the enum the upstream advertised for
|
||||
* provider+model, and store the accepted set. Returns the stored set, or null
|
||||
@@ -85,7 +109,13 @@ export function recordLearnedReasoningEffort(
|
||||
const lowered = typeof raw === "string" ? raw.trim().toLowerCase() : "";
|
||||
if (lowered && REASONING_EFFORT_ORDER.includes(lowered)) newSet.add(lowered);
|
||||
}
|
||||
if (newSet.size === 0) return null;
|
||||
if (newSet.size === 0) {
|
||||
// OBS2/M5: a 4xx advertised an enum we cannot map — say so, never learn silently.
|
||||
console.warn(
|
||||
`[learnedReasoningEffortCaps] unrecognized reasoning_effort enum for ${key}: ${acceptedValues.join(", ")} — nothing learned`
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
const existing = learnedCaps.get(key);
|
||||
if (existing !== undefined) {
|
||||
|
||||
@@ -33,7 +33,8 @@ export const SYNCED_EFFORT_SKIP_PROVIDERS = new Set(["codex", "glm", "glm-cn", "
|
||||
/** Provider-id prefixes covering that mechanism's multiple connection variants (kimi-coding, kimi-coding-apikey). */
|
||||
const SYNCED_EFFORT_SKIP_PROVIDER_PREFIXES = ["kimi"];
|
||||
|
||||
function isSkippedEffortProvider(ownedBy: string): boolean {
|
||||
/** Whether `ownedBy` already owns its own `-{effort}` suffix mechanism (never synthesize/expose another). */
|
||||
export function isSkippedEffortProvider(ownedBy: string): boolean {
|
||||
return (
|
||||
SYNCED_EFFORT_SKIP_PROVIDERS.has(ownedBy) ||
|
||||
SYNCED_EFFORT_SKIP_PROVIDER_PREFIXES.some((prefix) => ownedBy.startsWith(prefix))
|
||||
|
||||
@@ -1120,6 +1120,9 @@ async function buildUnifiedModelsResponseCore(
|
||||
else if (endpoints.includes("rerank")) modelType = "rerank";
|
||||
else if (endpoints.includes("images")) modelType = "image";
|
||||
else if (endpoints.includes("audio")) modelType = "audio";
|
||||
// Same owned_by the alias/canonical entries below will carry — computed once
|
||||
// so the effort_tiers exclusion (codex/glm/kimi) and the entries agree.
|
||||
const syncedOwnedBy = resolvePublicOwnerId(providerId, canonicalProviderId);
|
||||
const syncedFields = {
|
||||
...(modelType ? { type: modelType } : {}),
|
||||
...(apiFormat !== "chat-completions" ? { api_format: apiFormat } : {}),
|
||||
@@ -1133,12 +1136,19 @@ async function buildUnifiedModelsResponseCore(
|
||||
: {}),
|
||||
// #4264/#7694: vision + reasoning-effort-tier flags captured at sync time,
|
||||
// merged into a single capabilities object (see ./syncedCapabilities.ts).
|
||||
...(buildSyncedCapabilities(sm) ? { capabilities: buildSyncedCapabilities(sm) } : {}),
|
||||
// ownedBy gates effort_tiers off for codex/glm/kimi (own suffix mechanism).
|
||||
...(buildSyncedCapabilities(sm, syncedOwnedBy)
|
||||
? { capabilities: buildSyncedCapabilities(sm, syncedOwnedBy) }
|
||||
: {}),
|
||||
};
|
||||
|
||||
const existingAliasModel = models.find((model) => model.id === aliasId);
|
||||
if (existingAliasModel) {
|
||||
const mergedCapabilities = mergeSyncedCapabilities(existingAliasModel.capabilities, sm);
|
||||
const mergedCapabilities = mergeSyncedCapabilities(
|
||||
existingAliasModel.capabilities,
|
||||
sm,
|
||||
syncedOwnedBy
|
||||
);
|
||||
Object.assign(existingAliasModel, syncedFields);
|
||||
if (mergedCapabilities) existingAliasModel.capabilities = mergedCapabilities;
|
||||
continue;
|
||||
|
||||
@@ -5,26 +5,54 @@
|
||||
* to keep the vision (#4264) and reasoning-effort-tier (#7694) flags merged into a SINGLE
|
||||
* `capabilities` object rather than two separate spreads that would silently overwrite one
|
||||
* another via object-spread order. A model can be both vision- and reasoning-capable.
|
||||
*
|
||||
* effort_tiers loop (2026-08-23): a runtime-learned accepted set (#11232,
|
||||
* learnedReasoningEffortCaps) REPLACES the synced `supportedThinkingEfforts`
|
||||
* when one exists — the proven contract beats the advertised one. Lookup is
|
||||
* model-scoped: executors record under connection ids while this module sees
|
||||
* provider ids, so an exact provider:model key would always miss.
|
||||
*
|
||||
* Exclusion gate: `ownedBy` is REQUIRED and checked against
|
||||
* `isSkippedEffortProvider` (codex/glm/kimi — providers that already own a
|
||||
* conflicting `-{effort}` suffix mechanism, see syncedEffortVariants.ts, #7694).
|
||||
* Without this, the blind opencode-plugin mapping (`capabilities.effort_tiers`
|
||||
* -> ModelV2 `variants`) would double-handle those providers' native suffix
|
||||
* ids. `shouldExposeSyncedEffortVariants` gates only the *synthetic*
|
||||
* `<id>-<tier>` catalog entries (open-sse/utils/syncedEffortVariants.ts) — it
|
||||
* never runs over the base entry's `capabilities`, so it cannot substitute
|
||||
* for this check. Required (not optional) so no call site can silently skip it.
|
||||
*/
|
||||
// Use the same canonical alias as catalogModelPolicy.ts (l.1) — a relative path from
|
||||
// src/app/api/v1/models/ to open-sse/ would need 5 `../` and silently breaks under
|
||||
// refactors. (Confirmed convention: grep "from \"@omniroute/open-sse" src/app/api/v1/models/)
|
||||
import { getLearnedReasoningEffortForModel } from "@omniroute/open-sse/services/learnedReasoningEffortCaps.ts";
|
||||
import { isSkippedEffortProvider } from "@omniroute/open-sse/utils/syncedEffortVariants.ts";
|
||||
|
||||
interface SyncedCapabilityFlags {
|
||||
id?: string;
|
||||
supportsVision?: boolean;
|
||||
supportedThinkingEfforts?: string[];
|
||||
}
|
||||
|
||||
function hasEffortTiers(sm: SyncedCapabilityFlags): boolean {
|
||||
return Array.isArray(sm.supportedThinkingEfforts) && sm.supportedThinkingEfforts.length > 0;
|
||||
function effectiveEffortTiers(sm: SyncedCapabilityFlags, ownedBy: string): string[] | undefined {
|
||||
if (isSkippedEffortProvider(ownedBy)) return undefined;
|
||||
const learned = sm.id ? getLearnedReasoningEffortForModel(sm.id) : null;
|
||||
if (learned) return [...learned];
|
||||
return Array.isArray(sm.supportedThinkingEfforts) && sm.supportedThinkingEfforts.length > 0
|
||||
? sm.supportedThinkingEfforts
|
||||
: undefined;
|
||||
}
|
||||
|
||||
/** Build the `capabilities` object for a fresh synced-model catalog entry, or `undefined` when neither flag applies. */
|
||||
export function buildSyncedCapabilities(
|
||||
sm: SyncedCapabilityFlags
|
||||
sm: SyncedCapabilityFlags,
|
||||
ownedBy: string
|
||||
): Record<string, boolean | string[]> | undefined {
|
||||
const effortTiers = hasEffortTiers(sm);
|
||||
if (!sm.supportsVision && !effortTiers) return undefined;
|
||||
const tiers = effectiveEffortTiers(sm, ownedBy);
|
||||
if (!sm.supportsVision && !tiers) return undefined;
|
||||
return {
|
||||
...(sm.supportsVision ? { vision: true } : {}),
|
||||
...(effortTiers ? { effort_tiers: sm.supportedThinkingEfforts! } : {}),
|
||||
...(tiers ? { effort_tiers: tiers } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -35,13 +63,14 @@ export function buildSyncedCapabilities(
|
||||
*/
|
||||
export function mergeSyncedCapabilities(
|
||||
existing: Record<string, unknown> | undefined,
|
||||
sm: SyncedCapabilityFlags
|
||||
sm: SyncedCapabilityFlags,
|
||||
ownedBy: string
|
||||
): Record<string, unknown> | undefined {
|
||||
const effortTiers = hasEffortTiers(sm);
|
||||
if (!sm.supportsVision && !effortTiers && !existing) return undefined;
|
||||
const tiers = effectiveEffortTiers(sm, ownedBy);
|
||||
if (!sm.supportsVision && !tiers && !existing) return undefined;
|
||||
return {
|
||||
...(existing || {}),
|
||||
...(sm.supportsVision ? { vision: true } : {}),
|
||||
...(effortTiers ? { effort_tiers: sm.supportedThinkingEfforts! } : {}),
|
||||
...(tiers ? { effort_tiers: tiers } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
splitSyncedEffortSuffix,
|
||||
stripContextWindowSuffix,
|
||||
} from "@omniroute/open-sse/services/model.ts";
|
||||
import { getLearnedReasoningEffortForModel } from "@omniroute/open-sse/services/learnedReasoningEffortCaps.ts";
|
||||
import { REGISTRY } from "@omniroute/open-sse/config/providerRegistry.ts";
|
||||
import { getRegisteredProviderEffortBaseModelId } from "@omniroute/open-sse/utils/registeredEffortVariants.ts";
|
||||
|
||||
@@ -124,6 +125,20 @@ function isSyncedEffortSkippedProvider(providerId: string): boolean {
|
||||
return SYNCED_EFFORT_SKIP_PROVIDER_PREFIXES.some((prefix) => providerId.startsWith(prefix));
|
||||
}
|
||||
|
||||
/**
|
||||
* C1: effective tier set for suffix validation = learned ?? sync. The catalog
|
||||
* advertises variants from the learned set; validating the suffix against raw
|
||||
* synced metadata would strand learned-only tiers (dead-on-arrival ids).
|
||||
*/
|
||||
function effectiveKnownEfforts(
|
||||
modelId: string,
|
||||
syncedEfforts: readonly string[] | null | undefined
|
||||
): string[] {
|
||||
const learned = getLearnedReasoningEffortForModel(modelId);
|
||||
if (learned) return [...learned];
|
||||
return Array.isArray(syncedEfforts) ? [...syncedEfforts] : [];
|
||||
}
|
||||
|
||||
/** Resolve a suffix against an explicitly tiered static registry model. */
|
||||
function resolveRegistryModelIdAndEffort(
|
||||
providerId: string,
|
||||
@@ -139,7 +154,10 @@ function resolveRegistryModelIdAndEffort(
|
||||
|
||||
for (const candidate of registryModels) {
|
||||
if (!Array.isArray(candidate?.supportedThinkingEfforts)) continue;
|
||||
const attempt = splitSyncedEffortSuffix(modelId, candidate.supportedThinkingEfforts);
|
||||
const attempt = splitSyncedEffortSuffix(
|
||||
modelId,
|
||||
effectiveKnownEfforts(candidate.id, candidate.supportedThinkingEfforts)
|
||||
);
|
||||
if (attempt.effort && attempt.baseModel === candidate.id) {
|
||||
return { modelId: attempt.baseModel, effort: attempt.effort };
|
||||
}
|
||||
@@ -183,7 +201,7 @@ function resolveSyncedModelIdAndEffort(
|
||||
}
|
||||
const attempt = splitSyncedEffortSuffix(
|
||||
modelId,
|
||||
candidate.supportedThinkingEfforts as string[]
|
||||
effectiveKnownEfforts(candidate.id, candidate.supportedThinkingEfforts as string[])
|
||||
);
|
||||
if (attempt.effort && attempt.baseModel === candidate.id) {
|
||||
return { modelId: attempt.baseModel, effort: attempt.effort };
|
||||
@@ -232,7 +250,9 @@ function resolveRuntimeFormats(
|
||||
): RuntimeModelMeta {
|
||||
const apiFormat =
|
||||
(typeof customMatch?.apiFormat === "string" ? customMatch.apiFormat : undefined) ||
|
||||
(typeof compatOverrideMatch?.apiFormat === "string" ? compatOverrideMatch.apiFormat : undefined) ||
|
||||
(typeof compatOverrideMatch?.apiFormat === "string"
|
||||
? compatOverrideMatch.apiFormat
|
||||
: undefined) ||
|
||||
(syncedMatch?.apiFormat === "responses" ? "responses" : undefined);
|
||||
const targetFormat =
|
||||
typeof customMatch?.targetFormat === "string"
|
||||
@@ -359,7 +379,12 @@ async function lookupModelMeta(
|
||||
const available =
|
||||
!liveCatalog.authoritative || Boolean(customMatch || syncedMatch || liveBackedEffortVariant);
|
||||
|
||||
const metadata = buildRuntimeModelMeta(customMatch, syncedMatch, registryMatch, compatOverrideMatch);
|
||||
const metadata = buildRuntimeModelMeta(
|
||||
customMatch,
|
||||
syncedMatch,
|
||||
registryMatch,
|
||||
compatOverrideMatch
|
||||
);
|
||||
if (effort) metadata.resolvedThinkingEffort = effort;
|
||||
|
||||
return { modelId: resolvedModelId, metadata, available };
|
||||
|
||||
112
tests/unit/effort-tiers-loop-catalog-e2e.test.ts
Normal file
112
tests/unit/effort-tiers-loop-catalog-e2e.test.ts
Normal file
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* effort_tiers loop — I1 end-to-end proof: a set recorded through the REAL
|
||||
* record path (executor-style connection key) surfaces in the REAL catalog
|
||||
* response (/api/v1/models), including the learned-only variant entry.
|
||||
* Never "fix" this test by injecting the same string on both sides.
|
||||
*/
|
||||
import { test, after, beforeEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-effort-loop-e2e-"));
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "loop-e2e-secret";
|
||||
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const providersDb = await import("../../src/lib/db/providers.ts");
|
||||
const modelsDb = await import("../../src/lib/db/models.ts");
|
||||
const v1ModelsCatalog = await import("../../src/app/api/v1/models/catalog.ts");
|
||||
const { recordLearnedReasoningEffort, __test_resetLearnedReasoningEffortCaps } =
|
||||
await import("../../open-sse/services/learnedReasoningEffortCaps.ts");
|
||||
|
||||
async function resetStorage() {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
// Copied verbatim from sync-reasoning-supported-efforts-7694.test.ts
|
||||
async function seedProviderConnection(provider: string) {
|
||||
return providersDb.createProviderConnection({
|
||||
provider,
|
||||
authType: "apikey",
|
||||
name: `${provider}-${Math.random().toString(16).slice(2, 8)}`,
|
||||
apiKey: `${provider}-key`,
|
||||
isActive: true,
|
||||
testStatus: "active",
|
||||
});
|
||||
}
|
||||
|
||||
test.beforeEach(async () => {
|
||||
__test_resetLearnedReasoningEffortCaps();
|
||||
await resetStorage();
|
||||
});
|
||||
|
||||
test.after(async () => {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("learned set flows end-to-end into /v1/models capabilities and variant entries", async () => {
|
||||
// Non-namespaced id on purpose: mirrors the real incident model
|
||||
// (x-preview-f-free) where sm.id === the executor-visible post-strip id.
|
||||
const MODEL_ID = "loop-model-e2e";
|
||||
const connection = await seedProviderConnection("huggingface");
|
||||
await modelsDb.replaceSyncedAvailableModelsForConnection("huggingface", connection.id, [
|
||||
{
|
||||
id: MODEL_ID,
|
||||
name: "Loop Model E2E",
|
||||
supportedThinkingEfforts: ["none", "low", "medium", "high"],
|
||||
},
|
||||
]);
|
||||
|
||||
// Simulate the real 400 learning path (base.ts calls exactly this, with the
|
||||
// executor's CONNECTION id as provider key):
|
||||
recordLearnedReasoningEffort("openai-compatible-chat-eaff6869", MODEL_ID, ["low", "high", "max"]);
|
||||
|
||||
const response = await v1ModelsCatalog.getUnifiedModelsResponse(
|
||||
new Request("http://localhost/api/v1/models")
|
||||
);
|
||||
assert.equal(response.status, 200);
|
||||
const body = (await response.json()) as {
|
||||
data: Array<{ id: string; capabilities?: { effort_tiers?: string[] } }>;
|
||||
};
|
||||
|
||||
const baseEntry = body.data.find((m) => m.id.endsWith(MODEL_ID));
|
||||
assert.ok(baseEntry, "base entry present");
|
||||
assert.deepEqual(baseEntry!.capabilities?.effort_tiers, ["low", "high", "max"]);
|
||||
|
||||
const maxVariant = body.data.find((m) => m.id === `${baseEntry!.id}-max`);
|
||||
assert.ok(maxVariant, "learned-only tier synthesized as a variant entry");
|
||||
});
|
||||
|
||||
test("excluded provider (glm) never surfaces effort_tiers, learned or synced", async () => {
|
||||
const MODEL_ID = "glm-4-flash";
|
||||
const connection = await seedProviderConnection("glm");
|
||||
await modelsDb.replaceSyncedAvailableModelsForConnection("glm", connection.id, [
|
||||
{
|
||||
id: MODEL_ID,
|
||||
name: "GLM 4 Flash",
|
||||
supportedThinkingEfforts: ["none", "low", "medium", "high"],
|
||||
},
|
||||
]);
|
||||
recordLearnedReasoningEffort("glm-connection-1", MODEL_ID, ["low", "high"]);
|
||||
|
||||
const response = await v1ModelsCatalog.getUnifiedModelsResponse(
|
||||
new Request("http://localhost/api/v1/models")
|
||||
);
|
||||
assert.equal(response.status, 200);
|
||||
const body = (await response.json()) as {
|
||||
data: Array<{ id: string; capabilities?: { effort_tiers?: string[] } }>;
|
||||
};
|
||||
|
||||
const baseEntry = body.data.find((m) => m.id.endsWith(MODEL_ID));
|
||||
assert.ok(baseEntry, "base entry present");
|
||||
assert.equal(
|
||||
baseEntry!.capabilities?.effort_tiers,
|
||||
undefined,
|
||||
"glm owns its own -{effort} suffix mechanism — the catalog must not also expose effort_tiers"
|
||||
);
|
||||
});
|
||||
@@ -91,7 +91,12 @@ test("records the highest recognized value from the accepted list", () => {
|
||||
]) as unknown as Set<string>;
|
||||
assert.ok(learned instanceof Set);
|
||||
assert.ok(learned.has("high"));
|
||||
assert.equal((getLearnedReasoningEffort("ovh", "qwen3-coder-30b-a3b-instruct") as unknown as Set<string>).has("high"), true);
|
||||
assert.equal(
|
||||
(
|
||||
getLearnedReasoningEffort("ovh", "qwen3-coder-30b-a3b-instruct") as unknown as Set<string>
|
||||
).has("high"),
|
||||
true
|
||||
);
|
||||
});
|
||||
|
||||
test("returns null and stores nothing when acceptedValues has no recognized token", () => {
|
||||
@@ -116,7 +121,10 @@ test("monotonic decrease: a later, higher accepted-list never ratchets the cap b
|
||||
|
||||
test("a later, lower accepted-list does ratchet the cap down", () => {
|
||||
recordLearnedReasoningEffort("acme", "model-x", ["none", "low", "medium", "high"]);
|
||||
const learned = recordLearnedReasoningEffort("acme", "model-x", ["none", "low"]) as unknown as Set<string>;
|
||||
const learned = recordLearnedReasoningEffort("acme", "model-x", [
|
||||
"none",
|
||||
"low",
|
||||
]) as unknown as Set<string>;
|
||||
assert.equal(learned.size, 2);
|
||||
assert.ok(learned.has("low"));
|
||||
assert.equal((getLearnedReasoningEffort("acme", "model-x") as unknown as Set<string>).size, 2);
|
||||
@@ -177,8 +185,12 @@ test("getLearnedReasoningEffort returns null for unknown provider+model", () =>
|
||||
|
||||
test("getLearnedReasoningEffort is keyed case-insensitively on provider+model", () => {
|
||||
recordLearnedReasoningEffort("OVH", "Qwen3-Coder-30B", ["none", "high"]);
|
||||
assert.ok((getLearnedReasoningEffort("ovh", "qwen3-coder-30b") as unknown as Set<string>).has("high"));
|
||||
assert.ok((getLearnedReasoningEffort("OVH", "QWEN3-CODER-30B") as unknown as Set<string>).has("high"));
|
||||
assert.ok(
|
||||
(getLearnedReasoningEffort("ovh", "qwen3-coder-30b") as unknown as Set<string>).has("high")
|
||||
);
|
||||
assert.ok(
|
||||
(getLearnedReasoningEffort("OVH", "QWEN3-CODER-30B") as unknown as Set<string>).has("high")
|
||||
);
|
||||
});
|
||||
|
||||
test("different providers for the same model id have independent caps", () => {
|
||||
@@ -194,3 +206,45 @@ test("handles empty/null provider or model gracefully", () => {
|
||||
assert.equal(recordLearnedReasoningEffort("", "m", ["high"]), null);
|
||||
assert.equal(recordLearnedReasoningEffort("p", "", ["high"]), null);
|
||||
});
|
||||
|
||||
// ── getLearnedReasoningEffortForModel ────────────────────────────────────────
|
||||
|
||||
import { getLearnedReasoningEffortForModel } from "../../open-sse/services/learnedReasoningEffortCaps.ts";
|
||||
|
||||
test("getLearnedReasoningEffortForModel finds a set recorded under any provider key", () => {
|
||||
recordLearnedReasoningEffort("openai-compatible-chat-eaff6869", "X-Preview-F-Free", [
|
||||
"low",
|
||||
"high",
|
||||
"max",
|
||||
]);
|
||||
const set = getLearnedReasoningEffortForModel("x-preview-f-free");
|
||||
assert.ok(set);
|
||||
assert.deepEqual([...set].sort(), ["high", "low", "max"]);
|
||||
});
|
||||
|
||||
test("getLearnedReasoningEffortForModel intersects when multiple providers disagree", () => {
|
||||
recordLearnedReasoningEffort("conn-a", "shared-model", ["low", "high", "max"]);
|
||||
recordLearnedReasoningEffort("conn-b", "shared-model", ["low"]);
|
||||
const set = getLearnedReasoningEffortForModel("shared-model");
|
||||
assert.ok(set);
|
||||
assert.deepEqual([...set], ["low"]);
|
||||
});
|
||||
|
||||
test("getLearnedReasoningEffortForModel returns null when nothing learned or empty model", () => {
|
||||
assert.equal(getLearnedReasoningEffortForModel("never-learned"), null);
|
||||
assert.equal(getLearnedReasoningEffortForModel(""), null);
|
||||
assert.equal(getLearnedReasoningEffortForModel(undefined), null);
|
||||
});
|
||||
|
||||
test("recordLearnedReasoningEffort warns when every token is unrecognized", () => {
|
||||
const warnings: string[] = [];
|
||||
const orig = console.warn;
|
||||
console.warn = (msg: string) => warnings.push(msg);
|
||||
try {
|
||||
const result = recordLearnedReasoningEffort("p", "m", ["bogus-one", "bogus-two"]);
|
||||
assert.equal(result, null);
|
||||
assert.ok(warnings.some((w) => w.includes("reasoning_effort") && w.includes("bogus-one")));
|
||||
} finally {
|
||||
console.warn = orig;
|
||||
}
|
||||
});
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
/**
|
||||
* effort_tiers loop — learned set overrides synced metadata in catalog
|
||||
* capabilities (design 2026-08-23, decisions: appris > sync, in-memory).
|
||||
* Records go through the REAL record path (executor-style connection keys)
|
||||
* then read back through the catalog builders — proves the key-space bridge,
|
||||
* unlike a unit injection of the same string on both sides.
|
||||
*/
|
||||
import { test, after, beforeEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
recordLearnedReasoningEffort,
|
||||
__test_resetLearnedReasoningEffortCaps,
|
||||
} from "../../open-sse/services/learnedReasoningEffortCaps.ts";
|
||||
import {
|
||||
buildSyncedCapabilities,
|
||||
mergeSyncedCapabilities,
|
||||
} from "../../src/app/api/v1/models/syncedCapabilities.ts";
|
||||
|
||||
beforeEach(() => __test_resetLearnedReasoningEffortCaps());
|
||||
after(() => __test_resetLearnedReasoningEffortCaps());
|
||||
|
||||
const SYNC_TIERS = ["none", "low", "medium", "high", "xhigh"];
|
||||
|
||||
test("learned set replaces synced effort_tiers", () => {
|
||||
recordLearnedReasoningEffort("openai-compatible-chat-eaff6869", "x-preview-f-free", [
|
||||
"low",
|
||||
"high",
|
||||
"max",
|
||||
]);
|
||||
const caps = buildSyncedCapabilities(
|
||||
{ id: "x-preview-f-free", supportedThinkingEfforts: SYNC_TIERS },
|
||||
"huggingface"
|
||||
);
|
||||
assert.deepEqual(caps?.effort_tiers, ["low", "high", "max"]);
|
||||
});
|
||||
|
||||
test("nothing learned keeps synced metadata untouched", () => {
|
||||
const caps = buildSyncedCapabilities(
|
||||
{ id: "some-synced-model", supportedThinkingEfforts: SYNC_TIERS },
|
||||
"huggingface"
|
||||
);
|
||||
assert.deepEqual(caps?.effort_tiers, SYNC_TIERS);
|
||||
});
|
||||
|
||||
test("neither learned nor synced yields undefined", () => {
|
||||
const caps = buildSyncedCapabilities({ id: "plain-model" }, "huggingface");
|
||||
assert.equal(caps, undefined);
|
||||
});
|
||||
|
||||
test("merge path keeps vision AND applies the learned override", () => {
|
||||
recordLearnedReasoningEffort("conn-a", "vision-model", ["low", "max"]);
|
||||
const merged = mergeSyncedCapabilities(
|
||||
{ tool_calling: true },
|
||||
{ id: "vision-model", supportsVision: true, supportedThinkingEfforts: SYNC_TIERS },
|
||||
"huggingface"
|
||||
);
|
||||
assert.equal(merged?.vision, true);
|
||||
assert.equal(merged?.tool_calling, true);
|
||||
assert.deepEqual(merged?.effort_tiers, ["low", "max"]);
|
||||
});
|
||||
|
||||
// Exclusion gate (#7694): codex/glm/kimi already own a conflicting
|
||||
// `-{effort}` suffix mechanism — the blind opencode-plugin mapping must never
|
||||
// see effort_tiers for them, learned or synced, or it double-handles the suffix.
|
||||
for (const ownedBy of ["codex", "glm", "glm-cn", "glmt", "kimi", "kimi-coding-apikey"]) {
|
||||
test(`build: excluded provider "${ownedBy}" never gets effort_tiers (synced)`, () => {
|
||||
const caps = buildSyncedCapabilities(
|
||||
{ id: "excluded-model", supportedThinkingEfforts: SYNC_TIERS },
|
||||
ownedBy
|
||||
);
|
||||
assert.equal(caps?.effort_tiers, undefined);
|
||||
});
|
||||
|
||||
test(`build: excluded provider "${ownedBy}" never gets effort_tiers (learned)`, () => {
|
||||
recordLearnedReasoningEffort(`conn-${ownedBy}`, "excluded-model", ["low", "max"]);
|
||||
const caps = buildSyncedCapabilities(
|
||||
{ id: "excluded-model", supportedThinkingEfforts: SYNC_TIERS },
|
||||
ownedBy
|
||||
);
|
||||
assert.equal(caps?.effort_tiers, undefined);
|
||||
});
|
||||
}
|
||||
|
||||
test("excluded provider still gets vision through buildSyncedCapabilities", () => {
|
||||
const caps = buildSyncedCapabilities({ id: "codex-vision-model", supportsVision: true }, "codex");
|
||||
assert.deepEqual(caps, { vision: true });
|
||||
});
|
||||
|
||||
test("merge path also excludes codex/glm/kimi from effort_tiers", () => {
|
||||
recordLearnedReasoningEffort("conn-glm", "glm-model", ["low", "max"]);
|
||||
const merged = mergeSyncedCapabilities(
|
||||
{ tool_calling: true },
|
||||
{ id: "glm-model", supportsVision: true, supportedThinkingEfforts: SYNC_TIERS },
|
||||
"glm"
|
||||
);
|
||||
assert.equal(merged?.vision, true);
|
||||
assert.equal(merged?.effort_tiers, undefined);
|
||||
});
|
||||
80
tests/unit/synced-effort-suffix-learned-validation.test.ts
Normal file
80
tests/unit/synced-effort-suffix-learned-validation.test.ts
Normal file
@@ -0,0 +1,80 @@
|
||||
/**
|
||||
* C1 — the `-<tier>` suffix resolver validates against the EFFECTIVE tier set
|
||||
* (learned ?? sync), not raw synced metadata. Without this, the catalog
|
||||
* advertises <alias>/<model>-max (learned set) but dispatch refuses to strip
|
||||
* `-max` because sync metadata lacks the tier — dead-on-arrival variant.
|
||||
* Harness mirrors deepseek-thinking-efforts.test.ts (custom provider +
|
||||
* persistDiscoveredModels + async getModelInfo).
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-c1-effort-dispatch-"));
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "c1-test-secret";
|
||||
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const providersDb = await import("../../src/lib/db/providers.ts");
|
||||
const modelDiscovery = await import("../../src/lib/providerModels/modelDiscovery.ts");
|
||||
const { getModelInfo } = await import("../../src/sse/services/model.ts");
|
||||
const { recordLearnedReasoningEffort, __test_resetLearnedReasoningEffortCaps } =
|
||||
await import("@omniroute/open-sse/services/learnedReasoningEffortCaps.ts");
|
||||
|
||||
async function resetStorage() {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
const PROVIDER = "c1prov";
|
||||
const MODEL_ID = "c1-model";
|
||||
|
||||
async function seed() {
|
||||
const connection = await providersDb.createProviderConnection({
|
||||
provider: PROVIDER,
|
||||
authType: "apikey",
|
||||
name: "c1-runtime-efforts",
|
||||
apiKey: `${PROVIDER}-key`,
|
||||
isActive: true,
|
||||
testStatus: "active",
|
||||
});
|
||||
// Sync tiers deliberately EXCLUDE max — only the learned set will vouch for it.
|
||||
await modelDiscovery.persistDiscoveredModels(PROVIDER, connection.id, [
|
||||
{ id: MODEL_ID, reasoning: { supported_efforts: ["none", "low", "medium", "high"] } },
|
||||
]);
|
||||
}
|
||||
|
||||
test.beforeEach(async () => {
|
||||
__test_resetLearnedReasoningEffortCaps();
|
||||
await resetStorage();
|
||||
await seed();
|
||||
});
|
||||
|
||||
test.after(async () => {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("-max resolves once the learned set advertises it (sync metadata does not)", async () => {
|
||||
// Real record path, executor-style CONNECTION key — NOT the provider alias.
|
||||
recordLearnedReasoningEffort("openai-compatible-chat-eaff6869", MODEL_ID, ["low", "high", "max"]);
|
||||
const info = await getModelInfo(`${PROVIDER}/${MODEL_ID}-max`);
|
||||
assert.equal(info.provider, PROVIDER);
|
||||
assert.equal(info.model, MODEL_ID);
|
||||
assert.equal(info.resolvedThinkingEffort, "max");
|
||||
});
|
||||
|
||||
test("-medium still resolves via sync tiers even before anything is learned", async () => {
|
||||
const info = await getModelInfo(`${PROVIDER}/${MODEL_ID}-medium`);
|
||||
assert.equal(info.model, MODEL_ID);
|
||||
assert.equal(info.resolvedThinkingEffort, "medium");
|
||||
});
|
||||
|
||||
test("a tier neither learned nor synced is left untouched (literal id)", async () => {
|
||||
recordLearnedReasoningEffort("conn-a", MODEL_ID, ["low"]);
|
||||
const info = await getModelInfo(`${PROVIDER}/${MODEL_ID}-ultra`);
|
||||
assert.equal(info.resolvedThinkingEffort, undefined);
|
||||
});
|
||||
Reference in New Issue
Block a user