From 45fde4fd05a9ee89cd9b186bf72a96e6c8a3383e Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Tue, 25 Aug 2026 19:21:46 -0300 Subject: [PATCH] fix(api): restore the after() injection point for the /v1/models SWR refresh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `/v1/models` has passed a third argument to `getUnifiedModelsResponse()` (`{ scheduleBackgroundRefresh: (task) => after(task) }`) ever since #10198, but #9199 had already removed the parameter: the function takes two, so the object was silently dropped and `catalogCache` kept scheduling the stale-while- revalidate rebuild with `setTimeout(..., 0)`. The builder is overwhelmingly synchronous under the single-threaded App Router, so it pinned the event loop before the stale response was flushed — the #8728 guarantee did not exist. - `catalogCache` now imports `after` from `next/server` and exposes `defaultBackgroundRefreshScheduler`, which defers to `after()` and falls back to a macrotask outside a Next request scope (instrumentation warm-up, tests). - `resolveCachedCatalogResponse` takes `scheduleBackgroundRefresh` and `getStaleWhileRevalidateMs` on its existing options object. - `getUnifiedModelsResponse` accepts the options object the route already passes and propagates the scheduler. The excess argument was invisible to CI: `tsconfig.typecheck-core.json` is a curated 27-file allowlist, `check:dashboard-typecheck` only covers `src/app/(dashboard)`, and `next.config.mjs` sets `ignoreBuildErrors: true`. Closes #11551 --- src/app/api/v1/models/catalog.ts | 27 ++++++++- src/app/api/v1/models/catalogCache.ts | 80 +++++++++++++++++++++------ 2 files changed, 88 insertions(+), 19 deletions(-) diff --git a/src/app/api/v1/models/catalog.ts b/src/app/api/v1/models/catalog.ts index d70a275e3d..ca421e2eea 100644 --- a/src/app/api/v1/models/catalog.ts +++ b/src/app/api/v1/models/catalog.ts @@ -133,7 +133,11 @@ export { getCustomVisionCapabilityFields }; // lives in ./catalogCache. Re-exported here because the existing tests import the // hooks from this module, and CATALOG_STALE_WHILE_REVALIDATE_MS is part of the // documented behavior of this endpoint. -import { CATALOG_CACHE_TTL_MS_DEFAULT, resolveCachedCatalogResponse } from "./catalogCache"; +import { + CATALOG_CACHE_TTL_MS_DEFAULT, + resolveCachedCatalogResponse, + type BackgroundRefreshScheduler, +} from "./catalogCache"; export { CATALOG_STALE_WHILE_REVALIDATE_MS, @@ -144,7 +148,19 @@ export { __flushCatalogBackgroundRefreshForTest, __forceCatalogInFlightRejectionForTest, } from "./catalogCache"; -export type { CachedCatalog } from "./catalogCache"; +export type { CachedCatalog, BackgroundRefreshScheduler } from "./catalogCache"; + +/** + * Per-call options for {@link getUnifiedModelsResponse}. + * + * Restored in #11551: `/v1/models` passes Next's `after()` so the stale-while- + * revalidate rebuild is deferred until after the response flush. #9199 had removed + * the injection point while the route kept passing it, so the argument was silently + * dropped and the refresh ran on a plain `setTimeout`. + */ +export type CatalogResponseOptions = { + scheduleBackgroundRefresh?: BackgroundRefreshScheduler; +}; const BUILTIN_AUTO_YIELD_INTERVAL = 2; @@ -158,7 +174,8 @@ function yieldCatalogBuildTurn(): Promise { */ export async function getUnifiedModelsResponse( request: Request, - corsHeaders: Record = {} + corsHeaders: Record = {}, + options: CatalogResponseOptions = {} ) { const diagnosticHeaders = getCatalogDiagnosticsHeaders({ request }); @@ -200,6 +217,10 @@ export async function getUnifiedModelsResponse( hideAutoCombos: settingsForAuth?.hideAutoCombos === true || settingsForAuth?.autoRoutingEnabled === false, hideNoThinkVariants: settingsForAuth?.hideNoThinkVariants === true, + // #11551: the route injects Next's `after()` here so the SWR background + // rebuild starts only once the stale response has been flushed. Without a + // scheduler the cache falls back to defaultBackgroundRefreshScheduler. + scheduleBackgroundRefresh: options.scheduleBackgroundRefresh, } ); } catch (err) { diff --git a/src/app/api/v1/models/catalogCache.ts b/src/app/api/v1/models/catalogCache.ts index f2aa2973bd..5f674ff9f5 100644 --- a/src/app/api/v1/models/catalogCache.ts +++ b/src/app/api/v1/models/catalogCache.ts @@ -14,6 +14,8 @@ */ import { createHmac } from "node:crypto"; +import { after } from "next/server"; + import { getModelCatalogCacheVersion } from "@/lib/db/readCache"; import { extractApiKey } from "@/sse/services/auth"; @@ -78,6 +80,49 @@ export const CATALOG_STALE_WHILE_REVALIDATE_MS = 30_000; */ export const CATALOG_CACHE_TTL_MS_DEFAULT = 60_000; +/** + * Per-call knobs for {@link resolveCachedCatalogResponse}. + * + * `hideAutoCombos` / `hideNoThinkVariants` are catalog-shape dimensions folded into + * the cache key. `getStaleWhileRevalidateMs` and `scheduleBackgroundRefresh` are the + * injection points restored in #11551: the route wires Next's `after()` so the + * background refresh runs only once the response has been flushed to the client. + */ +export type CatalogResolveOptions = { + hideAutoCombos?: boolean; + hideNoThinkVariants?: boolean; + /** Overrides {@link CATALOG_STALE_WHILE_REVALIDATE_MS} for this call. */ + getStaleWhileRevalidateMs?: () => number; + /** Defers a background refresh; defaults to {@link defaultBackgroundRefreshScheduler}. */ + scheduleBackgroundRefresh?: BackgroundRefreshScheduler; +}; + +/** Defers `task` until it is safe to run without delaying the current response. */ +export type BackgroundRefreshScheduler = (task: () => Promise) => void; + +/** + * Default scheduler (#8728 / #11551). + * + * Next's `after()` runs the task once the response has been flushed, which is the + * whole point of the stale-while-revalidate path: the builder is overwhelmingly + * synchronous under the single-threaded App Router, so running it before the flush + * pins the event loop and the "served immediately" stale body only reaches the + * client after the rebuild finishes. + * + * `after()` requires a Next request scope. Callers outside one (instrumentation + * warm-up, direct unit-test imports) fall back to a macrotask, which preserves the + * "hand the response back first" ordering within the same process. + */ +export function defaultBackgroundRefreshScheduler(task: () => Promise): void { + try { + after(task); + } catch { + setTimeout(() => { + void task(); + }, 0); + } +} + type CatalogInFlight = { version: number; promise: Promise; @@ -98,10 +143,7 @@ const catalogInFlight = new Map(); let _catalogBuilderRuns = 0; -function buildCatalogCacheKey( - request: Request, - catalogSettings?: { hideAutoCombos?: boolean; hideNoThinkVariants?: boolean } -): string { +function buildCatalogCacheKey(request: Request, catalogSettings?: CatalogResolveOptions): string { const url = new URL(request.url); const prefix = url.searchParams.get("prefix") || ""; const apiKey = extractApiKey(request) || ""; @@ -195,23 +237,26 @@ function storePayload( function scheduleBackgroundRefresh( cacheKey: string, request: Request, - buildPayload: (request: Request) => Promise + buildPayload: (request: Request) => Promise, + schedule: BackgroundRefreshScheduler = defaultBackgroundRefreshScheduler ): void { if (catalogInFlight.has(cacheKey)) return; // a refresh for this key is already running const generation = getModelCatalogCacheVersion(); const refreshPromise: Promise = new Promise((resolve, reject) => { - setTimeout(() => { + schedule(() => runBuilder(buildPayload, request) - .then((payload) => resolve(storePayload(cacheKey, payload, generation))) + .then((payload) => { + resolve(storePayload(cacheKey, payload, generation)); + }) .catch((err) => { console.error( `[catalog] Background stale-while-revalidate refresh failed for key "${cacheKey}":`, err ); reject(err); - }); - }, 0); + }) + ); }); // Nobody on the stale path awaits this, so pre-handle the rejection; a cold-path // caller that joins it via catalogInFlight attaches its own handler and still @@ -246,7 +291,7 @@ export async function resolveCachedCatalogResponse( request: Request, headerSources: { corsHeaders: Record; diagnosticHeaders: Record }, buildPayload: (request: Request) => Promise, - catalogSettings?: { hideAutoCombos?: boolean; hideNoThinkVariants?: boolean } + catalogSettings?: CatalogResolveOptions ): Promise { const { corsHeaders, diagnosticHeaders } = headerSources; dropCatalogCacheIfStateChanged(); @@ -267,12 +312,15 @@ export async function resolveCachedCatalogResponse( // intermittent failure behind a fake success forever — and (b) it is within the // staleness window, so a refresh that keeps failing eventually falls through to the // cold-path wait instead of pinning ancient data. - if ( - cached && - cached.status === 200 && - now - cached.expiresAt <= CATALOG_STALE_WHILE_REVALIDATE_MS - ) { - scheduleBackgroundRefresh(cacheKey, request, buildPayload); + const staleWindowMs = + catalogSettings?.getStaleWhileRevalidateMs?.() ?? CATALOG_STALE_WHILE_REVALIDATE_MS; + if (cached && cached.status === 200 && now - cached.expiresAt <= staleWindowMs) { + scheduleBackgroundRefresh( + cacheKey, + request, + buildPayload, + catalogSettings?.scheduleBackgroundRefresh + ); return new Response(cached.body, { status: cached.status, headers: mergeCatalogHeaders(corsHeaders, cached.headers, diagnosticHeaders),