feat(radar): surface verified model metadata

This commit is contained in:
backryun
2026-08-13 11:34:34 -03:00
parent d2cda9651d
commit 537d19bb72
8 changed files with 407 additions and 37 deletions

View File

@@ -105,8 +105,10 @@ Opt-in false → { status: "opt_out" } — no network call
When both are on, the sync path is:
1. `GET <feed base URL>/v1/catalog/latest` with an optional `Authorization: Bearer
<supporter key>` header (see below).
1. `GET <feed base URL>/v1/catalog/latest` with `x-omniroute-radar-schema: 2` and an optional
`Authorization: Bearer <supporter key>` header (see below). Servers default to the separately
signed v1 transition artifact when the schema header is absent, so older installed clients keep
receiving updates.
2. Nothing about the request, the operator, or their traffic is uploaded — it is a
plain, unauthenticated-by-default GET. OmniRoute never posts usage data, provider
configuration, or model traffic to the feed service.
@@ -334,6 +336,16 @@ The dashboard exposes four local actions:
A feed `enabled: false` remains the safety exception: it wins over a stale local
`enabled: true`, keeps the merged entry disabled, and records `disabledBy: "radar"`.
Catalog publications use `schemaVersion: 2`. `contextWindow` and each of `tools`, `vision`, and
`thinking` are independently `number | null` / `boolean | null`: `null` means unknown, while
`false` means a D16-confirmed official provider source explicitly says the capability is absent.
Internal OmniRoute registry/model-spec flags are never promoted directly to feed facts. The client
still accepts v1 snapshots; because the old builder used `false` as an absence placeholder, v1 `false` is
normalized to unknown while v1 `true` remains factual. Unknown schema versions fail closed and the
last valid cache remains available. Every v2 model with a non-null context/capability must carry a
credential-free HTTPS `metadataEvidenceUrls[]`; otherwise schema validation fails and the cache is
not replaced. The catalog table renders all three states as ``, ``, and `?`.
### Guided combos and MCP access
Confirmed `familyId` values survive the read-time overlay and drive the pure
@@ -619,8 +631,9 @@ service without touching client code:
1. Serve a `GET /v1/catalog/latest` endpoint returning a JSON body that satisfies
`RadarFeedSchema` (`src/lib/radar/feedSchema.ts`) — top-level `feed:
"omniroute-radar"`, `schemaVersion: 1`, `version`, `tier`, `providers`, `models`,
`quirks`, and `totals`.
"omniroute-radar"`, `schemaVersion: 2`, `version`, `tier`, `providers`, `models`,
`quirks`, and `totals`. Honor `x-omniroute-radar-schema: 2`; a transition-compatible server
should default requests without it to a separately signed v1 artifact.
2. Sign the exact response bytes with an Ed25519 key pair and return the base64
signature in the `x-omniroute-feed-signature` response header.
3. Set `RADAR_FEED_URL` to the new base URL and `RADAR_FEED_PUBKEY` to the matching

View File

@@ -19,7 +19,12 @@ export interface RadarMergedEntry {
origin: "baseline" | "radar" | "local";
disabledBy?: "radar";
contextWindow?: number | null;
capabilities?: { tools: boolean; vision: boolean; thinking: boolean };
capabilities?: {
tools: boolean | null;
vision: boolean | null;
thinking: boolean | null;
};
metadataEvidenceUrls?: string[];
budget?: { kind: string; tokensPerMonth?: number; poolId?: string };
limits?: { rpm: number | null; rpd: number | null; tpm: number | null; tpd: number | null };
setup?: { keyUrl: string | null; steps: string[] } | null;
@@ -55,6 +60,21 @@ function budgetLabel(entry: RadarMergedEntry): string {
return `${formatTokens(entry.monthlyTokens)}/mo`;
}
function capabilityBadge(label: string, value: boolean | null | undefined, trueClass: string) {
const state = value === true ? "✓" : value === false ? "✕" : "?";
const stateClass =
value === true
? trueClass
: value === false
? "bg-red-500/10 text-red-400"
: "bg-gray-500/10 text-gray-400";
return (
<span className={`text-[10px] px-1.5 py-0.5 rounded ${stateClass}`}>
{label} {state}
</span>
);
}
export function RadarCatalogTable({ entries, refreshCatalog, onError }: RadarCatalogTableProps) {
const t = useTranslations("radarPage");
const [states, setStates] = useState<RadarLocalModelState[]>([]);
@@ -227,21 +247,21 @@ export function RadarCatalogTable({ entries, refreshCatalog, onError }: RadarCat
{entry.contextWindow ? `${(entry.contextWindow / 1000).toFixed(0)}K` : "—"}
</td>
<td className="py-3">
<div className="flex gap-1">
{entry.capabilities?.tools && (
<span className="text-[10px] px-1.5 py-0.5 rounded bg-blue-500/10 text-blue-400">
{t("capTools")}
</span>
<div className="flex flex-wrap gap-1">
{capabilityBadge(
t("capTools"),
entry.capabilities?.tools,
"bg-blue-500/10 text-blue-400"
)}
{entry.capabilities?.vision && (
<span className="text-[10px] px-1.5 py-0.5 rounded bg-purple-500/10 text-purple-400">
{t("capVision")}
</span>
{capabilityBadge(
t("capVision"),
entry.capabilities?.vision,
"bg-purple-500/10 text-purple-400"
)}
{entry.capabilities?.thinking && (
<span className="text-[10px] px-1.5 py-0.5 rounded bg-amber-500/10 text-amber-400">
{t("capThinking")}
</span>
{capabilityBadge(
t("capThinking"),
entry.capabilities?.thinking,
"bg-amber-500/10 text-amber-400"
)}
</div>
</td>

View File

@@ -64,10 +64,12 @@ export interface MergedEntry {
contextWindow?: number | null;
/** Capability flags reported by the feed. Undefined for baseline-only entries. */
capabilities?: {
tools: boolean;
vision: boolean;
thinking: boolean;
tools: boolean | null;
vision: boolean | null;
thinking: boolean | null;
};
/** Credential-free HTTPS evidence for non-null context/capability facts. */
metadataEvidenceUrls?: string[];
/** Rate/quota limits reported by the feed. Undefined for baseline-only entries. */
limits?: {
rpm: number | null;
@@ -104,10 +106,11 @@ export interface FeedModel {
};
contextWindow: number | null;
capabilities: {
tools: boolean;
vision: boolean;
thinking: boolean;
tools: boolean | null;
vision: boolean | null;
thinking: boolean | null;
};
metadataEvidenceUrls?: string[];
trainsOnPrompts: boolean | null;
tosRisk: MergedEntry["tos"];
setup: {
@@ -288,6 +291,10 @@ function mergeOne(
if (!overriddenKeys.has("capabilities")) {
result.capabilities = feed.capabilities;
}
result.metadataEvidenceUrls =
overriddenKeys.has("contextWindow") || overriddenKeys.has("capabilities")
? []
: (feed.metadataEvidenceUrls ?? []);
if (!overriddenKeys.has("limits")) {
result.limits = feed.limits;
}
@@ -332,6 +339,9 @@ function feedModelToMerged(
feed: FeedModel,
overrides: Partial<MergedEntry> | undefined
): MergedEntry {
const metadataOverridden =
overrides !== undefined &&
(Object.hasOwn(overrides, "contextWindow") || Object.hasOwn(overrides, "capabilities"));
const entry: MergedEntry = {
provider: feed.provider,
modelId: feed.modelId,
@@ -345,8 +355,15 @@ function feedModelToMerged(
trainsOnPrompts: overrides?.trainsOnPrompts ?? feed.trainsOnPrompts ?? undefined,
enabled: feed.enabled ? (overrides?.enabled ?? true) : false,
origin: overrides ? "local" : "radar",
contextWindow: overrides?.contextWindow ?? feed.contextWindow,
capabilities: overrides?.capabilities ?? feed.capabilities,
contextWindow:
overrides !== undefined && Object.hasOwn(overrides, "contextWindow")
? (overrides.contextWindow ?? null)
: feed.contextWindow,
capabilities:
overrides !== undefined && Object.hasOwn(overrides, "capabilities")
? (overrides.capabilities ?? feed.capabilities)
: feed.capabilities,
metadataEvidenceUrls: metadataOverridden ? [] : (feed.metadataEvidenceUrls ?? []),
limits: overrides?.limits ?? feed.limits,
setup: overrides?.setup ?? feed.setup,
};

View File

@@ -5,7 +5,7 @@
* is the source of truth; this schema validates whatever we downloaded
* before caching it locally.
*
* Schema version: 1
* Schema versions: v1 legacy + v2 with explicit unknown model metadata.
*/
import { z } from "zod";
@@ -81,12 +81,28 @@ const LimitsSchema = z.object({
tpd: IntNullable,
});
const CapabilitiesSchema = z.object({
const CapabilitiesV1Schema = z.object({
tools: z.boolean(),
vision: z.boolean(),
thinking: z.boolean(),
});
const CapabilitiesV2Schema = z.object({
tools: z.boolean().nullable(),
vision: z.boolean().nullable(),
thinking: z.boolean().nullable(),
});
const MetadataEvidenceUrlSchema = z
.string()
.url()
.superRefine((value, ctx) => {
const parsed = new URL(value);
if (parsed.protocol !== "https:" || parsed.username || parsed.password) {
ctx.addIssue({ code: "custom", message: "metadata evidence must use credential-free HTTPS" });
}
});
const SetupSchema = z
.object({
keyUrl: z.string().url().nullable(),
@@ -141,7 +157,7 @@ const RadarReferralsSchema = z
// Model
// ---------------------------------------------------------------------------
const ModelSchema = z.object({
const ModelV1Schema = z.object({
provider: z.string(),
modelId: z.string(),
displayName: z.string(),
@@ -150,13 +166,26 @@ const ModelSchema = z.object({
budget: BudgetSchema,
limits: LimitsSchema,
contextWindow: z.number().int().nullable(),
capabilities: CapabilitiesSchema,
capabilities: CapabilitiesV1Schema,
trainsOnPrompts: z.boolean().nullable(),
tosRisk: TosRiskEnum,
setup: SetupSchema,
enabled: z.boolean(),
});
const ModelV2Schema = ModelV1Schema.extend({
contextWindow: z.number().int().positive().nullable(),
capabilities: CapabilitiesV2Schema,
metadataEvidenceUrls: z.array(MetadataEvidenceUrlSchema),
}).superRefine((model, ctx) => {
const hasMetadata =
model.contextWindow !== null ||
Object.values(model.capabilities).some((capability) => capability !== null);
if (hasMetadata && model.metadataEvidenceUrls.length === 0) {
ctx.addIssue({ code: "custom", message: "known model metadata requires evidence" });
}
});
// ---------------------------------------------------------------------------
// Provider
// ---------------------------------------------------------------------------
@@ -187,7 +216,7 @@ const QuirkSchema = z.object({
// Top-level feed schema
// ---------------------------------------------------------------------------
export const RadarFeedSchema = z.object({
const RadarFeedV1Schema = z.object({
feed: z.literal("omniroute-radar"),
schemaVersion: z.literal(1),
version: z.string(),
@@ -206,7 +235,7 @@ export const RadarFeedSchema = z.object({
models: z.number().int(),
}),
providers: z.array(ProviderSchema),
models: z.array(ModelSchema),
models: z.array(ModelV1Schema),
quirks: z.array(QuirkSchema),
referrals: RadarReferralsSchema,
totals: z.object({
@@ -216,12 +245,40 @@ export const RadarFeedSchema = z.object({
}),
});
const RadarFeedV2Schema = RadarFeedV1Schema.extend({
schemaVersion: z.literal(2),
models: z.array(ModelV2Schema),
});
const RawRadarFeedSchema = z.discriminatedUnion("schemaVersion", [
RadarFeedV1Schema,
RadarFeedV2Schema,
]);
export const RadarFeedSchema = RawRadarFeedSchema.transform((feed) => {
if (feed.schemaVersion === 2) return feed;
return {
...feed,
models: feed.models.map((model) => ({
...model,
// The v1 builder used false as an absence placeholder. True was never
// a default, so it remains factual; false is normalized to unknown.
capabilities: {
tools: model.capabilities.tools || null,
vision: model.capabilities.vision || null,
thinking: model.capabilities.thinking || null,
},
metadataEvidenceUrls: [],
})),
};
});
// ---------------------------------------------------------------------------
// Inferred types
// ---------------------------------------------------------------------------
export type RadarFeed = z.infer<typeof RadarFeedSchema>;
export type RadarModel = z.infer<typeof ModelSchema>;
export type RadarModel = z.infer<typeof ModelV2Schema>;
export type RadarProvider = z.infer<typeof ProviderSchema>;
export type RadarQuirk = z.infer<typeof QuirkSchema>;
export type RadarBudget = z.infer<typeof BudgetSchema>;

View File

@@ -121,6 +121,15 @@ export function compareVersions(a: string, b: string): number {
return 0;
}
function cachedSchemaVersion(cache: RadarCacheEntry): 1 | 2 | null {
try {
const parsed = RadarFeedSchema.safeParse(JSON.parse(cache.payload) as unknown);
return parsed.success ? parsed.data.schemaVersion : null;
} catch {
return null;
}
}
// ---------------------------------------------------------------------------
// Scheduling helper (exported for UI/route wiring later)
// ---------------------------------------------------------------------------
@@ -187,7 +196,7 @@ export async function syncRadar(deps: SyncDeps = {}): Promise<SyncStatus> {
const baseUrl = (process.env.RADAR_FEED_URL || DEFAULT_FEED_BASE_URL).replace(/\/+$/, "");
const url = `${baseUrl}/v1/catalog/latest`;
const headers: Record<string, string> = {};
const headers: Record<string, string> = { "x-omniroute-radar-schema": "2" };
if (settings.supporterKey) {
headers["Authorization"] = `Bearer ${settings.supporterKey}`;
}
@@ -283,10 +292,20 @@ export async function syncRadar(deps: SyncDeps = {}): Promise<SyncStatus> {
}
const isEntitlementDowngrade = existingCache?.tier === "live" && servedTier === "community";
const versionComparison = existingCache
? compareVersions(feed.version, existingCache.version)
: 1;
const isSameVersionSchemaUpgrade =
existingCache !== null &&
existingCache.tier === servedTier &&
versionComparison === 0 &&
feed.schemaVersion === 2 &&
cachedSchemaVersion(existingCache) === 1;
if (
existingCache &&
!isEntitlementDowngrade &&
compareVersions(feed.version, existingCache.version) <= 0
!isSameVersionSchemaUpgrade &&
versionComparison <= 0
) {
return { status: "stale" };
}

View File

@@ -746,6 +746,53 @@ test("FIX2 feedModelToMerged path: contextWindow/capabilities/limits/setup survi
});
});
test("metadata evidence is removed when a baseline model overrides feed metadata", () => {
const evidence = "https://provider.example/docs/model";
const baseline = makeBaseline();
const feed = [
makeFeedModel({
provider: "groq",
modelId: "llama-3.3-70b-versatile",
contextWindow: 100,
capabilities: { tools: true, vision: false, thinking: null },
metadataEvidenceUrls: [evidence],
}),
];
const key = "groq:llama-3.3-70b-versatile";
const [entry] = applyFeed({
baseline,
feed,
localOverrides: new Map([
[key, { contextWindow: 999, capabilities: { tools: false, vision: null, thinking: null } }],
]),
tombstones: new Set(),
});
assert.equal(entry.contextWindow, 999);
assert.deepEqual(entry.metadataEvidenceUrls, []);
});
test("metadata evidence is removed when a feed-only model overrides metadata with null", () => {
const key = "new-provider:new-model";
const [entry] = applyFeed({
baseline: [],
feed: [
makeFeedModel({
provider: "new-provider",
modelId: "new-model",
contextWindow: 100,
metadataEvidenceUrls: ["https://provider.example/docs/model"],
}),
],
localOverrides: new Map([[key, { contextWindow: null }]]),
tombstones: new Set(),
});
assert.equal(entry.contextWindow, null);
assert.deepEqual(entry.metadataEvidenceUrls, []);
});
test("F3 mergeOne path: familyId survives the feed merge over a baseline entry", () => {
const result = applyFeed({
baseline: makeBaseline(),

View File

@@ -0,0 +1,68 @@
// @vitest-environment jsdom
import React, { act } from "react";
import { createRoot, type Root } from "react-dom/client";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("next-intl", () => ({ useTranslations: () => (key: string) => key }));
vi.mock("next/link", () => ({
default: ({ children, ...props }: React.AnchorHTMLAttributes<HTMLAnchorElement>) => (
<a {...props}>{children}</a>
),
}));
import { RadarCatalogTable } from "../../src/app/(dashboard)/dashboard/radar/RadarCatalogTable";
describe("Radar catalog capability knowledge", () => {
let root: Root | undefined;
let container: HTMLElement;
beforeEach(() => {
(
globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT?: boolean }
).IS_REACT_ACT_ENVIRONMENT = true;
container = document.createElement("div");
document.body.appendChild(container);
vi.stubGlobal(
"fetch",
vi.fn(async () => new Response(null, { status: 503 }))
);
});
afterEach(() => {
if (root) act(() => root!.unmount());
root = undefined;
container.remove();
vi.unstubAllGlobals();
});
it("distinguishes true, false, and unknown for every capability", async () => {
root = createRoot(container);
act(() => {
root!.render(
<RadarCatalogTable
entries={[
{
provider: "groq",
modelId: "llama",
displayName: "Llama",
monthlyTokens: 0,
creditTokens: 0,
freeType: "rate_only",
poolKey: null,
tos: "ok",
origin: "radar",
contextWindow: null,
capabilities: { tools: true, vision: false, thinking: null },
},
]}
refreshCatalog={async () => undefined}
onError={() => undefined}
/>
);
});
expect(container.textContent).toContain("capTools ✓");
expect(container.textContent).toContain("capVision ✕");
expect(container.textContent).toContain("capThinking ?");
});
});

View File

@@ -39,6 +39,16 @@ const FIXTURE_PATH = path.resolve(import.meta.dirname!, "../fixtures/radar-feed-
const FIXTURE_BYTES = fs.readFileSync(FIXTURE_PATH);
const FIXTURE_STRING = FIXTURE_BYTES.toString("utf-8");
function v2FixtureBytes(): Buffer {
const parsed = JSON.parse(FIXTURE_STRING);
parsed.schemaVersion = 2;
parsed.models = parsed.models.map((model: Record<string, unknown>) => ({
...model,
metadataEvidenceUrls: ["https://console.groq.com/docs/models"],
}));
return Buffer.from(JSON.stringify(parsed), "utf8");
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
@@ -188,11 +198,50 @@ test("feedSchema: rejects wrong feed literal", () => {
assert.equal(result.success, false, "must reject wrong feed literal");
});
test("feedSchema: rejects wrong schemaVersion", () => {
test("feedSchema: accepts v2 nullable capabilities and preserves explicit false", () => {
const parsed = JSON.parse(FIXTURE_STRING);
parsed.schemaVersion = 2;
for (const model of parsed.models) {
model.metadataEvidenceUrls = ["https://example.test/official-model-docs"];
}
parsed.models[0].capabilities = { tools: true, vision: false, thinking: null };
parsed.models[0].metadataEvidenceUrls = ["https://console.groq.com/docs/models"];
const result = feedSchema.RadarFeedSchema.safeParse(parsed);
assert.equal(result.success, false, "must reject schemaVersion != 1");
assert.equal(result.success, true);
assert.deepEqual(result.data?.models[0].capabilities, {
tools: true,
vision: false,
thinking: null,
});
assert.deepEqual(result.data?.models[0].metadataEvidenceUrls, [
"https://console.groq.com/docs/models",
]);
});
test("feedSchema: normalizes ambiguous v1 false placeholders to unknown", () => {
const parsed = JSON.parse(FIXTURE_STRING);
const result = feedSchema.RadarFeedSchema.parse(parsed);
assert.deepEqual(result.models[0].capabilities, {
tools: true,
vision: null,
thinking: null,
});
});
test("feedSchema: rejects unknown schemaVersion", () => {
const parsed = JSON.parse(FIXTURE_STRING);
parsed.schemaVersion = 3;
assert.equal(feedSchema.RadarFeedSchema.safeParse(parsed).success, false);
});
test("feedSchema: rejects known v2 metadata without evidence", () => {
const parsed = JSON.parse(FIXTURE_STRING);
parsed.schemaVersion = 2;
parsed.models = parsed.models.map((model: Record<string, unknown>) => ({
...model,
metadataEvidenceUrls: [],
}));
assert.equal(feedSchema.RadarFeedSchema.safeParse(parsed).success, false);
});
test("feedSchema: budget per_model requires positive tokensPerMonth", () => {
@@ -383,6 +432,69 @@ test("syncRadar: version floor — same version => stale, cache untouched", asyn
assert.equal(cacheWritten, false, "cache must NOT be overwritten with same version");
});
test("syncRadar: same version upgrades a validated v1 cache to the negotiated v2 artifact", async () => {
const v2Bytes = v2FixtureBytes();
const sig = signBytes(v2Bytes);
const cacheStore: syncMod.RadarCacheEntry[] = [];
const result = await syncMod.syncRadar({
getFlag: () => true,
getSettings: () => ({ optIn: true, supporterKey: null }),
getCache: () => ({
version: "2026.08.01.1",
tier: "community",
payload: FIXTURE_STRING,
signature: "previous-v1-signature",
}),
setCache: (entry) => cacheStore.push(entry),
fetch: (() =>
Promise.resolve(
mockResponse(v2Bytes, {
"x-omniroute-feed-signature": sig,
"x-omniroute-feed-tier": "community",
})
)) as unknown as typeof globalThis.fetch,
});
assert.deepEqual(result, {
status: "updated",
version: "2026.08.01.1",
tier: "community",
});
assert.equal(cacheStore.length, 1);
assert.equal(JSON.parse(cacheStore[0].payload).schemaVersion, 2);
});
test("syncRadar: same-version v2 cannot replace an existing validated v2 cache", async () => {
const v2Bytes = v2FixtureBytes();
const sig = signBytes(v2Bytes);
let cacheWritten = false;
const result = await syncMod.syncRadar({
getFlag: () => true,
getSettings: () => ({ optIn: true, supporterKey: null }),
getCache: () => ({
version: "2026.08.01.1",
tier: "community",
payload: v2Bytes.toString("utf8"),
signature: "previous-v2-signature",
}),
setCache: () => {
cacheWritten = true;
},
fetch: (() =>
Promise.resolve(
mockResponse(v2Bytes, {
"x-omniroute-feed-signature": sig,
"x-omniroute-feed-tier": "community",
})
)) as unknown as typeof globalThis.fetch,
});
assert.equal(result.status, "stale");
assert.equal(cacheWritten, false);
});
test("syncRadar: version floor — incoming older => stale", async () => {
const sig = signBytes(FIXTURE_BYTES);
let cacheWritten = false;
@@ -589,6 +701,23 @@ test("syncRadar: sends Authorization header when supporter key exists", async ()
);
});
test("syncRadar negotiates schema v2 so legacy clients can keep the default v1 artifact", async () => {
let requestHeaders: Record<string, string> = {};
const sig = signBytes(FIXTURE_BYTES);
await syncMod.syncRadar({
getFlag: () => true,
getSettings: () => ({ optIn: true, supporterKey: null }),
getCache: () => null,
setCache: () => {},
fetch: ((_url: string, init: RequestInit) => {
requestHeaders = init.headers as Record<string, string>;
return Promise.resolve(mockResponse(FIXTURE_BYTES, { "x-omniroute-feed-signature": sig }));
}) as unknown as typeof globalThis.fetch,
});
assert.equal(requestHeaders?.["x-omniroute-radar-schema"], "2");
});
test("syncRadar: no Authorization header when no supporter key", async () => {
let capturedHeaders: Record<string, string> = {};
const sig = signBytes(FIXTURE_BYTES);