From b61a530a3322e11b239bf8ec3dfb176daa927cb4 Mon Sep 17 00:00:00 2001 From: Nguyen Thanh Dat Date: Wed, 26 Aug 2026 18:10:47 +0700 Subject: [PATCH] fix(api): reconnect the /v1/models background-refresh scheduler (#11551) (#11574) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merged via /merge-batch (lote 2026-08-26, v3.8.51). Boarded no worktree combinado junto com outras ~30 PRs; validação única: typecheck/complexity/cognitive-complexity/changelog-integrity verdes, file-size rebaseado onde necessário (crescimento legítimo), lint com os mesmos 228 achados pré-existentes confirmados via sonda contra o tip puro (não introduzidos por este lote), e ~370 testes focados (unit + vitest) passando. Obrigado pela contribuição. --- .../fixes/11574-v1-models-after-scheduler.md | 1 + src/app/api/v1/models/catalog.ts | 15 ++- src/app/api/v1/models/catalogCache.ts | 92 ++++++++++++++++--- 3 files changed, 91 insertions(+), 17 deletions(-) create mode 100644 changelog.d/fixes/11574-v1-models-after-scheduler.md diff --git a/changelog.d/fixes/11574-v1-models-after-scheduler.md b/changelog.d/fixes/11574-v1-models-after-scheduler.md new file mode 100644 index 0000000000..532ba971c0 --- /dev/null +++ b/changelog.d/fixes/11574-v1-models-after-scheduler.md @@ -0,0 +1 @@ +- **fix(api):** `/v1/models` schedules its stale-while-revalidate rebuild through Next's `after()` again, so a stale catalog reaches the client before the rebuild blocks the event loop ([#11574](https://github.com/diegosouzapw/OmniRoute/pull/11574)) diff --git a/src/app/api/v1/models/catalog.ts b/src/app/api/v1/models/catalog.ts index 7a513943b6..3847813c19 100644 --- a/src/app/api/v1/models/catalog.ts +++ b/src/app/api/v1/models/catalog.ts @@ -133,7 +133,11 @@ export { getCustomVisionCapabilityFields }; // lives in ./catalogCache. Re-exported here because the existing tests import the // hooks from this module, and CATALOG_STALE_WHILE_REVALIDATE_MS is part of the // documented behavior of this endpoint. -import { CATALOG_CACHE_TTL_MS_DEFAULT, resolveCachedCatalogResponse } from "./catalogCache"; +import { + CATALOG_CACHE_TTL_MS_DEFAULT, + resolveCachedCatalogResponse, + type BackgroundRefreshScheduler, +} from "./catalogCache"; export { CATALOG_STALE_WHILE_REVALIDATE_MS, @@ -155,10 +159,16 @@ function yieldCatalogBuildTurn(): Promise { /** * Build unified OpenAI-compatible model catalog response. * Reused by `/api/v1/models` and `/api/v1` to avoid semantic drift (T09). + * + * `options.scheduleBackgroundRefresh` is the App Router's injection point for the + * stale-while-revalidate rebuild (#8728): the route passes Next's `after()` so the + * rebuild starts only once the stale body has been flushed. Omitted by non-route + * callers, which fall back to the cache module's own default. */ export async function getUnifiedModelsResponse( request: Request, - corsHeaders: Record = {} + corsHeaders: Record = {}, + options: { scheduleBackgroundRefresh?: BackgroundRefreshScheduler } = {} ) { const diagnosticHeaders = getCatalogDiagnosticsHeaders({ request }); @@ -200,6 +210,7 @@ export async function getUnifiedModelsResponse( hideAutoCombos: settingsForAuth?.hideAutoCombos === true || settingsForAuth?.autoRoutingEnabled === false, hideNoThinkVariants: settingsForAuth?.hideNoThinkVariants === true, + scheduleBackgroundRefresh: options.scheduleBackgroundRefresh, } ); } catch (err) { diff --git a/src/app/api/v1/models/catalogCache.ts b/src/app/api/v1/models/catalogCache.ts index f2aa2973bd..79ef366f1b 100644 --- a/src/app/api/v1/models/catalogCache.ts +++ b/src/app/api/v1/models/catalogCache.ts @@ -14,6 +14,8 @@ */ import { createHmac } from "node:crypto"; +import { after } from "next/server"; + import { getModelCatalogCacheVersion } from "@/lib/db/readCache"; import { extractApiKey } from "@/sse/services/auth"; @@ -54,6 +56,61 @@ export type CatalogPayload = { */ export const CATALOG_STALE_WHILE_REVALIDATE_MS = 30_000; +/** + * Schedules the stale-while-revalidate rebuild. Injected so the App Router route can + * hand over Next's `after()` and a test can hand over a deterministic hook. + * + * The task returns a promise, so a scheduler that awaits it (as `after()` does) keeps + * the runtime alive until the rebuild finishes. + */ +export type BackgroundRefreshScheduler = (task: () => Promise) => void; + +/** + * Default scheduler: Next's `after()`, which runs the task once the response has been + * flushed to the client. + * + * That flush guarantee is the whole point of #8728. The builder is overwhelmingly + * synchronous under the single-threaded App Router, so a rebuild that starts before the + * flush pins the event loop and the "served immediately" stale body only reaches the + * client once the rebuild has finished — the stale path stops being cheap, which is what + * it exists for. `setTimeout(…, 0)` defers by a macrotask but does not wait for the + * flush, so it never delivered that guarantee. + * + * `after()` throws outside a Next request scope — the CLI/Electron server, unit tests — + * so fall back to the macrotask there. Those callers have no response being flushed, so + * the deferral is all they ever needed. + */ +function defaultBackgroundRefreshScheduler(task: () => Promise): void { + try { + after(task); + } catch { + setTimeout(() => { + void task(); + }, 0); + } +} + +/** + * Per-call knobs for `resolveCachedCatalogResponse`. The first two are cache-key + * dimensions; the last two are injection points with production defaults. + */ +export type CatalogCacheOptions = { + hideAutoCombos?: boolean; + hideNoThinkVariants?: boolean; + /** + * Overrides `CATALOG_STALE_WHILE_REVALIDATE_MS` for this call only. + * + * Deliberately NOT the module-level policy accessor #9199 removed: there is no + * setter, no module state, and no production caller passes it, so the bound stays + * fixed at 30 s for every real request and a failing refresh still cannot pin an old + * catalog forever. It exists so a test can hold an entry in the stale branch while it + * measures when the rebuild starts, instead of racing the real window. + */ + getStaleWhileRevalidateMs?: () => number; + /** Overrides `defaultBackgroundRefreshScheduler` for this call. */ + scheduleBackgroundRefresh?: BackgroundRefreshScheduler; +}; + /** * Fallback memoization window; overridden by `settings.cache.modelCatalogCacheTtlMs`. * @@ -181,9 +238,10 @@ function storePayload( * no second coalescing mechanism — so a concurrent cold/stale request for the * same key joins this refresh instead of starting another. * - * The builder runs one macrotask later so the stale response that triggered this - * call is handed back before the builder's synchronous prologue runs; the whole - * point of this path is that the caller does not pay for the rebuild. + * The builder runs through `schedule` so the stale response that triggered this call + * is handed back — flushed to the client, under the production scheduler — before the + * builder's synchronous prologue runs; the whole point of this path is that the caller + * does not pay for the rebuild. * * The tracked promise **rejects** on failure. catalogInFlight is shared with the * cold path: a caller whose entry aged past the stale window skips the stale @@ -192,16 +250,17 @@ function storePayload( * build failure as a 200. The rejection is pre-handled so this path can never * raise an unhandledRejection; a failed refresh simply never overwrites the entry. */ -function scheduleBackgroundRefresh( +function startBackgroundRefresh( cacheKey: string, request: Request, - buildPayload: (request: Request) => Promise + buildPayload: (request: Request) => Promise, + schedule: BackgroundRefreshScheduler ): void { if (catalogInFlight.has(cacheKey)) return; // a refresh for this key is already running const generation = getModelCatalogCacheVersion(); const refreshPromise: Promise = new Promise((resolve, reject) => { - setTimeout(() => { + schedule(() => runBuilder(buildPayload, request) .then((payload) => resolve(storePayload(cacheKey, payload, generation))) .catch((err) => { @@ -210,8 +269,8 @@ function scheduleBackgroundRefresh( err ); reject(err); - }); - }, 0); + }) + ); }); // Nobody on the stale path awaits this, so pre-handle the rejection; a cold-path // caller that joins it via catalogInFlight attaches its own handler and still @@ -246,7 +305,7 @@ export async function resolveCachedCatalogResponse( request: Request, headerSources: { corsHeaders: Record; diagnosticHeaders: Record }, buildPayload: (request: Request) => Promise, - catalogSettings?: { hideAutoCombos?: boolean; hideNoThinkVariants?: boolean } + catalogSettings?: CatalogCacheOptions ): Promise { const { corsHeaders, diagnosticHeaders } = headerSources; dropCatalogCacheIfStateChanged(); @@ -267,12 +326,15 @@ export async function resolveCachedCatalogResponse( // intermittent failure behind a fake success forever — and (b) it is within the // staleness window, so a refresh that keeps failing eventually falls through to the // cold-path wait instead of pinning ancient data. - if ( - cached && - cached.status === 200 && - now - cached.expiresAt <= CATALOG_STALE_WHILE_REVALIDATE_MS - ) { - scheduleBackgroundRefresh(cacheKey, request, buildPayload); + const staleWindowMs = + catalogSettings?.getStaleWhileRevalidateMs?.() ?? CATALOG_STALE_WHILE_REVALIDATE_MS; + if (cached && cached.status === 200 && now - cached.expiresAt <= staleWindowMs) { + startBackgroundRefresh( + cacheKey, + request, + buildPayload, + catalogSettings?.scheduleBackgroundRefresh ?? defaultBackgroundRefreshScheduler + ); return new Response(cached.body, { status: cached.status, headers: mergeCatalogHeaders(corsHeaders, cached.headers, diagnosticHeaders),