diff --git a/src/app/api/v1/models/catalogCache.ts b/src/app/api/v1/models/catalogCache.ts index a7f8d0aa51..aa585b86b1 100644 --- a/src/app/api/v1/models/catalogCache.ts +++ b/src/app/api/v1/models/catalogCache.ts @@ -19,6 +19,7 @@ import { after } from "next/server"; import { getModelCatalogCacheVersion } from "@/lib/db/readCache"; import { extractApiKey } from "@/sse/services/auth"; +import { catalogPageCacheKey, catalogStringResponse, parseCatalogPage } from "./catalogPagination"; import { isCodexModelCatalogClient } from "./catalogRequest"; /** Fingerprint an API key for the catalog memo Map. Never store the raw secret. */ @@ -151,8 +152,14 @@ function withTimeout(promise: Promise, ms: number, label: string): Promise return new Promise((resolve, reject) => { const timer = setTimeout(() => reject(new Error(label)), ms); promise.then( - (value) => { clearTimeout(timer); resolve(value); }, - (err) => { clearTimeout(timer); reject(err); } + (value) => { + clearTimeout(timer); + resolve(value); + }, + (err) => { + clearTimeout(timer); + reject(err); + } ); }); } @@ -180,7 +187,8 @@ function buildCatalogCacheKey(request: Request, catalogSettings?: CatalogCacheOp const configuredOnly = url.searchParams.get("configuredOnly") === "true" ? "1" : "0"; const hideAuto = catalogSettings?.hideAutoCombos ? "1" : "0"; const hideNoThink = catalogSettings?.hideNoThinkVariants ? "1" : "0"; - return `${prefix}|${isCodex}|${fingerprintCatalogAuthKey(apiKey)}|${configuredOnly}|${hideAuto}|${hideNoThink}`; + const page = catalogPageCacheKey(parseCatalogPage(request)); + return `${prefix}|${isCodex}|${fingerprintCatalogAuthKey(apiKey)}|${configuredOnly}|${hideAuto}|${hideNoThink}|${page}`; } // Tracks the model-catalog cache version (src/lib/db/readCache.ts) as of the last @@ -327,19 +335,21 @@ async function awaitCatalogInFlight( } const lastGood = catalogLastGood.get(cacheKey); if (msg === "catalog_build_timeout" && lastGood) { - return new Response(lastGood.body, { - status: lastGood.status, - headers: mergeCatalogHeaders(corsHeaders, lastGood.headers, diagnosticHeaders, { + return catalogStringResponse( + lastGood.body, + mergeCatalogHeaders(corsHeaders, lastGood.headers, diagnosticHeaders, { "x-omniroute-catalog": "last-good", }), - }); + lastGood.status + ); } throw err; } - return new Response(payload.body, { - status: payload.status, - headers: mergeCatalogHeaders(corsHeaders, payload.headers, diagnosticHeaders), - }); + return catalogStringResponse( + payload.body, + mergeCatalogHeaders(corsHeaders, payload.headers, diagnosticHeaders), + payload.status + ); } /** @@ -363,10 +373,11 @@ export async function resolveCachedCatalogResponse( const cached = catalogCache.get(cacheKey); if (cached && cached.expiresAt > now) { - return new Response(cached.body, { - status: cached.status, - headers: mergeCatalogHeaders(corsHeaders, cached.headers, diagnosticHeaders), - }); + return catalogStringResponse( + cached.body, + mergeCatalogHeaders(corsHeaders, cached.headers, diagnosticHeaders), + cached.status + ); } // Stale-while-revalidate: an expired entry is still served immediately as long as @@ -383,10 +394,11 @@ export async function resolveCachedCatalogResponse( buildPayload, catalogSettings?.scheduleBackgroundRefresh ?? defaultBackgroundRefreshScheduler ); - return new Response(cached.body, { - status: cached.status, - headers: mergeCatalogHeaders(corsHeaders, cached.headers, diagnosticHeaders), - }); + return catalogStringResponse( + cached.body, + mergeCatalogHeaders(corsHeaders, cached.headers, diagnosticHeaders), + cached.status + ); } const currentGeneration = getModelCatalogCacheVersion(); diff --git a/src/app/api/v1/models/catalogPagination.ts b/src/app/api/v1/models/catalogPagination.ts new file mode 100644 index 0000000000..1fbecde59f --- /dev/null +++ b/src/app/api/v1/models/catalogPagination.ts @@ -0,0 +1,107 @@ +/** + * OpenAI-compatible `limit` / `after` paging plus a complete JSON body writer + * for GET /v1/models. Large catalogs (~1MB+) were returned as one + * `Response.json()` write without Content-Length, which cut mid-body for + * Desktop and other buffered clients. + */ + +export const CATALOG_PAGE_LIMIT_MAX = 10_000; +export const CATALOG_BODY_CHUNK_BYTES = 64 * 1024; + +export type CatalogPage = { + after: string | null; + limit: number | null; +}; + +export type CatalogPageResult = { + models: T[]; + hasMore: boolean; + lastId: string | null; +}; + +/** + * Parse OpenAI-style `limit` and `after` query params. Missing or invalid + * values mean "return the full list" so existing clients stay byte-compatible + * aside from the added Content-Length header. + */ +export function parseCatalogPage(request: Request): CatalogPage { + const url = new URL(request.url); + const afterRaw = url.searchParams.get("after"); + const after = afterRaw && afterRaw.trim().length > 0 ? afterRaw.trim() : null; + const limitRaw = url.searchParams.get("limit"); + let limit: number | null = null; + if (limitRaw != null && limitRaw.trim() !== "") { + const parsed = Number.parseInt(limitRaw, 10); + if (Number.isFinite(parsed) && parsed > 0) { + limit = Math.min(parsed, CATALOG_PAGE_LIMIT_MAX); + } + } + return { after, limit }; +} + +export function catalogPageCacheKey(page: CatalogPage): string { + return `after=${page.after ?? ""};limit=${page.limit ?? ""}`; +} + +export function applyCatalogPage( + models: T[], + page: CatalogPage +): CatalogPageResult { + if (!page.after && page.limit == null) { + return { models, hasMore: false, lastId: null }; + } + + let start = 0; + if (page.after) { + const idx = models.findIndex((model) => model.id === page.after); + start = idx >= 0 ? idx + 1 : 0; + } + + const end = page.limit != null ? start + page.limit : models.length; + const sliced = models.slice(start, end); + const last = sliced[sliced.length - 1]; + return { + models: sliced, + hasMore: end < models.length, + lastId: typeof last?.id === "string" ? last.id : null, + }; +} + +/** + * Serialize a catalog JSON object with Content-Length and chunked enqueue so + * the body cannot be cut mid-write on large payloads. + */ +export function catalogJsonResponse( + body: Record, + headers: Record = {}, + status = 200 +): Response { + const payload = JSON.stringify(body); + return catalogStringResponse(payload, headers, status); +} + +export function catalogStringResponse( + payload: string, + headers: Headers | Record = {}, + status = 200 +): Response { + const bytes = Buffer.byteLength(payload); + const merged = headers instanceof Headers ? new Headers(headers) : new Headers(headers); + merged.set("content-type", "application/json"); + merged.set("content-length", String(bytes)); + + if (bytes <= CATALOG_BODY_CHUNK_BYTES) { + return new Response(payload, { status, headers: merged }); + } + + const stream = new ReadableStream({ + start(controller) { + const buf = Buffer.from(payload); + for (let offset = 0; offset < buf.length; offset += CATALOG_BODY_CHUNK_BYTES) { + controller.enqueue(buf.subarray(offset, offset + CATALOG_BODY_CHUNK_BYTES)); + } + controller.close(); + }, + }); + return new Response(stream, { status, headers: merged }); +} diff --git a/src/app/api/v1/models/catalogResponse.ts b/src/app/api/v1/models/catalogResponse.ts index b1d5a032d7..9c76b32cbf 100644 --- a/src/app/api/v1/models/catalogResponse.ts +++ b/src/app/api/v1/models/catalogResponse.ts @@ -40,6 +40,7 @@ import { } from "@/shared/utils/featureFlags"; import { extractApiKey } from "@/sse/services/auth"; import { maybeOmitCatalogModelName } from "./catalogHelpers"; +import { applyCatalogPage, catalogJsonResponse, parseCatalogPage } from "./catalogPagination"; import { isCodexModelCatalogClient } from "./catalogRequest"; /** @@ -283,13 +284,19 @@ export async function finalizeCatalogResponse( // empty/foreign `base_instructions` would drop codex's agent prompt to nothing and // break its agent behavior (verified empirically against codex 0.137). An empty array // keeps codex on its built-in model info — same inference as today, minus the error. + const page = parseCatalogPage(request); + const paged = applyCatalogPage(orderedModels, page); const responseBody: Record = { object: "list", - data: orderedModels, + data: paged.models, }; + if (page.limit != null || page.after) { + responseBody.has_more = paged.hasMore; + if (paged.lastId) responseBody.last_id = paged.lastId; + } if (isCodexModelCatalogClient(request)) { responseBody.models = []; } - return Response.json(responseBody, { headers }); + return catalogJsonResponse(responseBody, headers); } diff --git a/tests/unit/10313-catalog-cache-key-hashing.test.ts b/tests/unit/10313-catalog-cache-key-hashing.test.ts index ca5b4d1947..fa1922b965 100644 --- a/tests/unit/10313-catalog-cache-key-hashing.test.ts +++ b/tests/unit/10313-catalog-cache-key-hashing.test.ts @@ -45,10 +45,11 @@ function captureMapKeys(): { keys: string[]; restore: () => void } { }; } -// buildCatalogCacheKey emits `prefix|isCodex|apiKeyFingerprint|configuredOnly|hideAuto|hideNoThink` -// (6 pipe-delimited fields). Other in-flight keys (e.g. `x-request-id`) don't match. +// buildCatalogCacheKey emits +// `prefix|isCodex|apiKeyFingerprint|configuredOnly|hideAuto|hideNoThink|page` +// (7 pipe-delimited fields). Other in-flight keys (e.g. `x-request-id`) don't match. function isCatalogCacheKey(k: string): boolean { - return k.split("|").length === 6; + return k.split("|").length === 7; } test("catalog cache Map keys must not contain the raw bearer API key (#10313)", async () => { diff --git a/tests/unit/models-catalog-pagination.test.ts b/tests/unit/models-catalog-pagination.test.ts new file mode 100644 index 0000000000..8bf903e7de --- /dev/null +++ b/tests/unit/models-catalog-pagination.test.ts @@ -0,0 +1,79 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + applyCatalogPage, + CATALOG_BODY_CHUNK_BYTES, + catalogJsonResponse, + catalogPageCacheKey, + catalogStringResponse, + parseCatalogPage, +} from "../../src/app/api/v1/models/catalogPagination.ts"; + +function req(url: string): Request { + return new Request(url); +} + +test("parseCatalogPage treats missing params as a full list", () => { + assert.deepEqual(parseCatalogPage(req("http://local/v1/models")), { after: null, limit: null }); +}); + +test("parseCatalogPage accepts OpenAI-compatible limit and after", () => { + assert.deepEqual(parseCatalogPage(req("http://local/v1/models?limit=20&after=gpt-4o")), { + after: "gpt-4o", + limit: 20, + }); +}); + +test("parseCatalogPage ignores non-positive limits", () => { + assert.deepEqual(parseCatalogPage(req("http://local/v1/models?limit=0&after=")), { + after: null, + limit: null, + }); +}); + +test("applyCatalogPage slices after a cursor and reports has_more", () => { + const models = [{ id: "a" }, { id: "b" }, { id: "c" }, { id: "d" }]; + const page = applyCatalogPage(models, { after: "b", limit: 1 }); + assert.deepEqual(page, { models: [{ id: "c" }], hasMore: true, lastId: "c" }); +}); + +test("applyCatalogPage returns the remainder when limit exceeds the tail", () => { + const models = [{ id: "a" }, { id: "b" }, { id: "c" }]; + const page = applyCatalogPage(models, { after: "b", limit: 10 }); + assert.deepEqual(page, { models: [{ id: "c" }], hasMore: false, lastId: "c" }); +}); + +test("catalogPageCacheKey distinguishes pages so cached bodies do not mix", () => { + assert.notEqual( + catalogPageCacheKey({ after: null, limit: 10 }), + catalogPageCacheKey({ after: "gpt-4o", limit: 10 }) + ); +}); + +test("catalogJsonResponse sets Content-Length to the full UTF-8 byte count", async () => { + const body = { object: "list", data: [{ id: "x", name: "café" }] }; + const response = catalogJsonResponse(body, {}); + const payload = JSON.stringify(body); + assert.equal(response.headers.get("content-length"), String(Buffer.byteLength(payload))); + assert.equal(await response.text(), payload); +}); + +test("catalogStringResponse streams large bodies without cutting mid-JSON", async () => { + const payload = JSON.stringify({ + object: "list", + data: Array.from({ length: 80 }, (_, i) => ({ + id: `model-${i}`, + filler: "x".repeat(2_000), + })), + }); + assert.ok(Buffer.byteLength(payload) > CATALOG_BODY_CHUNK_BYTES); + + const response = catalogStringResponse(payload, {}); + assert.equal(response.headers.get("content-length"), String(Buffer.byteLength(payload))); + const text = await response.text(); + assert.equal(text.length, payload.length); + assert.equal(text, payload); + const parsed = JSON.parse(text) as { data: unknown[] }; + assert.equal(parsed.data.length, 80); +});