diff --git a/.env.example b/.env.example index 1a11f36562..67775b6725 100644 --- a/.env.example +++ b/.env.example @@ -2724,6 +2724,16 @@ APP_LOG_TO_FILE=true # When enabled, the node authenticates with the API key stored on its connection. # AUDIO_REMOTE_PROVIDER_NODES=false +# Used by: src/app/api/v1/_shared/rerankProviderNodes.ts — lets POST /v1/rerank (and +# the memory engine's loopback rerank step) use an OpenAI-compatible provider node +# hosted outside localhost, e.g. a LAN box or Tailscale peer running TEI/Infinity/vLLM. +# OFF by default: routing to a remote host changes egress identity, so it must be an +# explicit operator decision. Loopback/private nodes (localhost, 127.0.0.1, +# 172.16-31.x) are always allowed and unaffected by this flag. Remote nodes must also +# pass the provider outbound URL policy (see OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS); +# cloud-metadata hosts are never routed to. +# RERANK_REMOTE_PROVIDER_NODES=false + # ── Free Proxy Pool (auto-sync scheduler) ── # Background refresh of the free-proxy pool. Opt-in, OFF by default (parallels # Hard Rule #20's default-off posture for data-mutating background features). diff --git a/changelog.d/features/13732-rerank-remote-provider-nodes.md b/changelog.d/features/13732-rerank-remote-provider-nodes.md new file mode 100644 index 0000000000..419f19d06c --- /dev/null +++ b/changelog.d/features/13732-rerank-remote-provider-nodes.md @@ -0,0 +1 @@ +- **feat(api):** `POST /v1/rerank` (and the memory engine's loopback rerank step) can route to OpenAI-compatible provider nodes on a LAN/Tailscale host — not only loopback — behind the new `RERANK_REMOTE_PROVIDER_NODES` feature flag (default off), subject to the provider outbound URL policy; the loopback host check is consolidated into `@/shared/network/loopbackNodeHost` shared by rerank, audio, and the local health checker ([#13732](https://github.com/diegosouzapw/OmniRoute/pull/13732)) — thanks @seanford diff --git a/changelog.d/fixes/13733-rerank-local-node-tei-infinity-shapes.md b/changelog.d/fixes/13733-rerank-local-node-tei-infinity-shapes.md new file mode 100644 index 0000000000..4924a190c3 --- /dev/null +++ b/changelog.d/fixes/13733-rerank-local-node-tei-infinity-shapes.md @@ -0,0 +1 @@ +- **fix(api):** `POST /v1/rerank` now actually works against native TEI / Infinity provider nodes: the `/rerank` fallback sends `texts` + `return_text` alongside `documents`, and bare-array or `score`-only upstream responses are normalized to the Cohere `{results: [{index, relevance_score, document?}]}` envelope (sorted, `top_n`-capped) so clients and the memory engine's rerank step see real scores ([#13733](https://github.com/diegosouzapw/OmniRoute/pull/13733)) — thanks @seanford diff --git a/docs/frameworks/MEMORY.md b/docs/frameworks/MEMORY.md index 17e3db7b4c..e3ff58e99d 100644 --- a/docs/frameworks/MEMORY.md +++ b/docs/frameworks/MEMORY.md @@ -177,7 +177,9 @@ Nine embedding and vector fields are available in `MemorySettingsExtended` in | `staticEnabled` | `boolean` | `false` | Opt-in for static potion-base-8M local model | | `rerankEnabled` | `boolean` | `false` | Enable reranking step (adds +200-500ms/req) | | `rerankProviderModel` | `string \| null` | `null` | Rerank provider/model in `provider/model` format | -| `vectorStore` | `"sqlite-vec" \| "qdrant" \| "auto"` | `"auto"` | Which vector backend to use | + +`rerankProviderModel` is resolved by `POST /v1/rerank` (called over loopback), so it accepts anything that route accepts: a curated cloud rerank model (`cohere/rerank-v3.5`, `jina-ai/jina-reranker-v3.5`, …) or an OpenAI-compatible provider node as `/` (e.g. `skilled-mini/bge-reranker-v2-m3` for a TEI/Infinity box). Loopback nodes are always eligible; a node on another host (LAN, Tailscale) additionally requires the `RERANK_REMOTE_PROVIDER_NODES` feature flag and must pass the provider outbound URL policy — see [Feature Flags](../reference/FEATURE_FLAGS.md). The dashboard selector lists curated providers plus local nodes; any valid `provider/model` string can be set directly via `PUT /api/settings/memory`. +| `vectorStore` | `"sqlite-vec" \| "qdrant" \| "auto"` | `"auto"` | Which vector backend to use | These are exposed via `GET /PUT /api/settings/memory` (schema `MemorySettingsExtendedSchema`). diff --git a/docs/reference/API_REFERENCE.md b/docs/reference/API_REFERENCE.md index 79c7bf22ed..8b3e111d97 100644 --- a/docs/reference/API_REFERENCE.md +++ b/docs/reference/API_REFERENCE.md @@ -480,7 +480,7 @@ All POST routes follow the same shape: `Bearer your-api-key` + Zod-validated JSO For clients that cannot attach `Authorization: Bearer ...`, OmniRoute also accepts API keys in the URL via either query-string compatibility (`?token=...`, `?apiKey=...`, `?api_key=...`, `?key=...`) or the dedicated `/api/v1/vscode/{token}/...` endpoints documented below. ```bash -# Rerank +# Rerank (cloud registry provider, or an OpenAI-compatible provider node as "/") POST /v1/rerank { "model": "jina-ai/jina-reranker-v3.5", "query": "...", "documents": ["..."] } # Jina classify (Foundation API credentials) @@ -506,6 +506,22 @@ POST /v1/videos/generations { "model": "runway/gen-3", "prompt": "..." } POST /v1/music/generations { "model": "suno/v3.5", "prompt": "..." } ``` +> **Rerank provider nodes:** `POST /v1/rerank` also routes to OpenAI-compatible provider nodes +> (oMLX, vLLM, Infinity, TEI behind a gateway, …) addressed as `/`. Loopback +> nodes (`localhost`, `127.0.0.1`, `172.16.0.0/12`) are always eligible. Nodes on any other +> host — a LAN box or Tailscale peer — are eligible only when the operator enables the +> `RERANK_REMOTE_PROVIDER_NODES` feature flag **and** the node's base URL passes the provider +> outbound URL policy (`OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS` / `OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS`); +> cloud-metadata hosts are never routed to. The memory engine's rerank step calls this route over +> loopback, so the same rule governs `rerankProviderModel` in the Memory settings. +> +> **Local server shapes:** the node is called at `/v1/rerank` and, on 404, at `/rerank` +> (Infinity, TEI). The upstream body carries both the Cohere/OpenAI spelling (`documents`, +> `return_documents`) and the TEI spelling (`texts`, `return_text`), and the upstream response is +> normalized to the Cohere envelope: TEI's bare `[{index, score, text}]`, `{results: [{index, score}]}` +> from thin gateways, and Voyage-style `{data: [...]}` all come back to the client as +> `{results: [{index, relevance_score, document?}]}`, sorted by score and capped at `top_n`. + > **Provider-node discovery:** models on an OpenAI-compatible provider node appear in `GET /v1/models` > under the node prefix. Rows that carry no endpoint metadata (typical for local `/v1/models` listings) > inherit the node's `apiType`, so an `embeddings` node's models are `type: "embedding"` and a diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index a87038f5f7..5a992a66b7 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -243,6 +243,7 @@ OmniRoute uses **SQLite** (via `better-sqlite3`) for all persistence. These vari | `OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS` | `false` | `src/shared/network/outboundUrlGuard.ts` | Allow provider URLs pointing to private/local networks (localhost, 192.168.x.x, 10.x.x.x, etc.). **REQUIRED for self-hosted providers** (LM Studio, Ollama, vLLM, Llamafile, Triton, SearXNG). When `false`, the dashboard rejects validation of local URLs. | | `OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS` | `true` | `src/shared/network/outboundUrlGuard.ts` | Allow adding/validating providers on local/private addresses (127.0.0.1, localhost, LAN, private ranges) — scoped to the provider validation path. **Default `true`** (local-first); set `false` to enforce strict public-only blocking. Cloud-metadata endpoints (169.254.169.254, metadata.google.internal) stay blocked regardless. (#5066) | | `AUDIO_REMOTE_PROVIDER_NODES` | `false` | `src/app/api/v1/_shared/audioProviderNodes.ts` | Let the `/v1/audio/*` routes (transcriptions, speech, translations) use an OpenAI-compatible provider node hosted outside localhost. Off by default — routing audio to a remote host changes egress identity and must be an explicit operator decision. Loopback/private nodes (localhost, 127.0.0.1, 172.16-31.x) are always allowed and unaffected. (#3963) | +| `RERANK_REMOTE_PROVIDER_NODES` | `false` | `src/app/api/v1/_shared/rerankProviderNodes.ts` | Let `POST /v1/rerank` (and the memory engine's loopback rerank step) use an OpenAI-compatible provider node hosted outside localhost — a LAN box or Tailscale peer running TEI, Infinity, vLLM, etc. Off by default — routing to a remote host changes egress identity and must be an explicit operator decision. Loopback nodes (localhost, 127.0.0.1, 172.16-31.x) are always allowed and unaffected. Remote nodes must also pass the provider outbound URL policy (`OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS` / `OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS`); cloud-metadata hosts are never routed to. | | `OMNIROUTE_OIDC_DISABLE_PASSWORD_LOGIN` | `false` | `src/app/api/auth/login/route.ts` | When OIDC is enabled, disable password login so users can only authenticate via OIDC Single Sign-On. The bare alias `OIDC_DISABLE_PASSWORD_LOGIN` is also accepted; the Dashboard Feature Flag of the same key takes precedence. (#10889) | | `OIDC_DISABLE_PASSWORD_LOGIN` | `false` | `src/app/api/auth/login/route.ts` | Bare alias of `OMNIROUTE_OIDC_DISABLE_PASSWORD_LOGIN` (#10889). | diff --git a/docs/reference/FEATURE_FLAGS.md b/docs/reference/FEATURE_FLAGS.md index 77db1c0831..6c60166405 100644 --- a/docs/reference/FEATURE_FLAGS.md +++ b/docs/reference/FEATURE_FLAGS.md @@ -46,7 +46,7 @@ A boolean flag is considered **enabled** when its effective value is `"true"`, ## Flag Catalog -73 flags across 6 categories. **Default** is the definition default — the value +74 flags across 6 categories. **Default** is the definition default — the value used when neither a DB override nor an environment variable is present. ### Security (10) @@ -64,12 +64,13 @@ used when neither a DB override nor an environment variable is present. | `AUTH_LOG_INCLUDE_ACCOUNT_ID` | boolean | `false` | Include account prefix in AUTH log lines (e.g. "Using account: abc12345..."). Disabled by default so account identifiers are redacted from shared/multi-tenant process logs. Independent from Debug Mode; flipping Debug Mode does not reveal this. | | `OMNIROUTE_OIDC_DISABLE_PASSWORD_LOGIN` | boolean | `false` | When OIDC is enabled, disable password login so users can only authenticate via OIDC Single Sign-On. When disabled (default), both password login and OIDC are available. | -### Network (15) +### Network (16) | Key | Type | Default | Restart | Description | | ----------------------------------------------- | ------- | ------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `ENABLE_TLS_FINGERPRINT` | boolean | `false` | ✓ | Enable TLS fingerprint stealth mode. | | `AUDIO_REMOTE_PROVIDER_NODES` | boolean | `false` | | Allow the /v1/audio/* routes to use OpenAI-compatible provider nodes hosted outside localhost. Off by default — routing audio to a remote host changes egress identity and must be an explicit operator decision. Loopback nodes are always allowed and unaffected. | +| `RERANK_REMOTE_PROVIDER_NODES` | boolean | `false` | | Allow POST /v1/rerank (and the memory engine's loopback rerank step) to use OpenAI-compatible provider nodes hosted outside localhost. Off by default — routing to a remote host changes egress identity and must be an explicit operator decision. Loopback nodes are always allowed; remote nodes must also pass the provider outbound URL policy. | | `PROXY_AUTO_SELECT_ENABLED` | boolean | `false` | | When no proxy is assigned to a connection, auto-select the first working proxy from the registry. Off by default (otherwise any registry proxy becomes a global fallback — #3332). | | `OMNIROUTE_CONTROL_PLANE_PROXY_DIRECT_FALLBACK` | boolean | `false` | | Allow OAuth and provider validation flows to bypass a pinned proxy and connect directly when proxy reachability pre-checks fail. Off by default because this can change egress IP. | | `NETWORK_ROTATION_SHARED_EGRESS_GUARD` | boolean | `true` | | On a network exception (timeout, connection refused/reset) for a multi-account rotation executor, when the failing account has no dedicated proxy, apply a short cooldown and skip other proxy-less accounts for the rest of the request instead of retrying each one. On by default (safe: no egress IP change, only reduces latency/cooldown risk on shared-egress accounts). Disable to restore immediate propagation on the first proxy-less throw. | @@ -213,7 +214,7 @@ Returns every flag with its effective value, source, and a summary. "requiresRestart": false, "warningLevel": "caution", }, - // ... all 72 flags + // ... all 74 flags ], "summary": { "total": 56, diff --git a/open-sse/config/audioRegistry.ts b/open-sse/config/audioRegistry.ts index 7ee1796c94..39672e535b 100644 --- a/open-sse/config/audioRegistry.ts +++ b/open-sse/config/audioRegistry.ts @@ -8,6 +8,7 @@ */ import { getProviderAlias } from "@/shared/constants/providers"; +import { isLoopbackNodeHost } from "@/shared/network/loopbackNodeHost"; interface AudioModel { id: string; @@ -633,19 +634,12 @@ export interface ProviderNodeRow { apiType?: string; } -/** Hosts reachable only from the operator's machine/Docker network. */ -export function isLoopbackNodeHost(baseUrl: string): boolean { - try { - const hostname = new URL(baseUrl).hostname; - return ( - hostname === "localhost" || - hostname === "127.0.0.1" || - /^172\.(1[6-9]|2[0-9]|3[0-1])\.\d{1,3}\.\d{1,3}$/.test(hostname) - ); - } catch { - return false; - } -} +/** + * Hosts reachable only from the operator's machine/Docker network. + * Re-exported from the shared module so the audio, rerank, and local-health-check paths + * agree on one definition (the shared version additionally rejects `user@host` URLs). + */ +export { isLoopbackNodeHost }; /** * Build a dynamic AudioProvider from a provider_node DB entry. diff --git a/src/app/api/v1/_shared/rerankLocalNodeShapes.ts b/src/app/api/v1/_shared/rerankLocalNodeShapes.ts new file mode 100644 index 0000000000..f6440ba07e --- /dev/null +++ b/src/app/api/v1/_shared/rerankLocalNodeShapes.ts @@ -0,0 +1,171 @@ +/** + * Request/response shape adapters for OpenAI-compatible rerank provider nodes. + * + * `POST /v1/rerank` speaks the Cohere shape to clients (`{model, query, documents, top_n, + * return_documents}` → `{results: [{index, relevance_score, document?}]}`). Most local + * servers (vLLM, llama.cpp, Infinity behind its OpenAI facade, oMLX) speak that shape + * too, but two popular self-hosted rerankers do not: + * + * - Hugging Face **text-embeddings-inference (TEI)** mounts `POST /rerank` (no `/v1`), + * requires the candidate list as `texts` (not `documents`), takes `return_text` + * instead of `return_documents`, and answers with a bare array + * `[{index, score, text?}]` — no `results` envelope, `score` not `relevance_score`. + * - **Infinity** in native mode likewise mounts `/rerank`, accepts `documents`, and + * returns `{results: [{index, relevance_score, document?}]}` — Cohere-shaped, but a + * few thin gateways in front of it (and of TEI) emit `score` instead of + * `relevance_score`. + * + * The route already fell back from `/v1/rerank` to `/rerank` on 404 "for + * Infinity / TEI", but it kept sending `documents` and returned the upstream body verbatim, + * so the TEI fallback could never have worked (HTTP 422: missing field `texts`) and a + * `score`-only response left the memory engine — which reads `relevance_score` — with + * undefined scores. These two pure helpers close that gap: + * + * - {@link buildLocalRerankRequestBody} sends both spellings (`documents` + `texts`, + * `return_documents` + `return_text`). Every server named above ignores the fields it + * does not know (TEI's request struct is not `deny_unknown_fields`; the OpenAI-shaped + * servers tolerate extras), so one body serves all of them. + * - {@link normalizeLocalRerankResponse} folds any of the observed response shapes into + * the Cohere envelope, backfilling `relevance_score` from `score`, sorting by score, + * honouring `top_n`, and attaching `document.text` when the caller asked for documents + * and the upstream did not echo them. + */ + +import { toNumberOrNull } from "@/shared/utils/numeric"; + +export interface LocalRerankRequestInput { + model: string; + query: string; + documents: unknown[]; + top_n?: number | null; + return_documents?: boolean | null; +} + +export interface CohereRerankResult { + index: number; + relevance_score: number; + document?: { text: string } | Record; +} + +export interface CohereRerankResponse { + results: CohereRerankResult[]; + [extra: string]: unknown; +} + +/** Cohere allows documents to be strings or `{text}` objects; every local server wants strings. */ +export function documentText(doc: unknown): string { + if (typeof doc === "string") return doc; + if (doc && typeof doc === "object") { + const text = (doc as { text?: unknown }).text; + if (typeof text === "string") return text; + } + return doc === null || doc === undefined ? "" : String(doc); +} + +/** + * Build the upstream request body for a local rerank node. Includes both the + * Cohere/OpenAI spelling (`documents`, `return_documents`) and the TEI spelling + * (`texts`, `return_text`) so the same body works against every known local server. + */ +export function buildLocalRerankRequestBody(input: LocalRerankRequestInput) { + const texts = input.documents.map(documentText); + const topN = input.top_n || input.documents.length; + // Cohere default: documents are only echoed back when the caller asks for them. + const returnDocuments = input.return_documents === true; + return { + model: input.model, + query: input.query, + documents: input.documents, + top_n: topN, + return_documents: returnDocuments, + // TEI spelling + texts, + return_text: returnDocuments, + }; +} + +interface RawResult { + index?: unknown; + relevance_score?: unknown; + score?: unknown; + text?: unknown; + document?: unknown; +} + +function pickResultsArray(data: unknown): { list: RawResult[]; envelope: Record } { + if (Array.isArray(data)) { + return { list: data as RawResult[], envelope: {} }; + } + if (data && typeof data === "object") { + const obj = data as Record; + for (const key of ["results", "data"]) { + if (Array.isArray(obj[key])) { + const { [key]: _list, ...rest } = obj; + return { list: obj[key] as RawResult[], envelope: rest }; + } + } + // An object with no recognisable results list: keep its fields, report no results. + return { list: [], envelope: obj }; + } + return { list: [], envelope: {} }; +} + +function resolveResultDocument(raw: RawResult, fallback: unknown): Record { + if (raw.document && typeof raw.document === "object") { + return raw.document as Record; + } + if (typeof raw.document === "string") return { text: raw.document }; + if (typeof raw.text === "string") return { text: raw.text }; + return { text: documentText(fallback) }; +} + +/** One raw entry → Cohere result, or `null` when its index or score is unusable. */ +function toCohereResult( + raw: RawResult, + documents: unknown[], + returnDocuments: boolean +): CohereRerankResult | null { + if (!raw || typeof raw !== "object") return null; + const index = toNumberOrNull(raw.index); + if (index === null || !Number.isInteger(index) || index < 0 || index >= documents.length) { + return null; + } + const score = toNumberOrNull(raw.relevance_score) ?? toNumberOrNull(raw.score); + if (score === null) return null; + + const result: CohereRerankResult = { index, relevance_score: score }; + if (returnDocuments) result.document = resolveResultDocument(raw, documents[index]); + return result; +} + +/** + * Normalize a local rerank node's response into the Cohere envelope. + * + * Accepts: `{results: [...]}` (Cohere / Infinity / vLLM / llama.cpp), `{data: [...]}` + * (Voyage-style), or a bare `[...]` (TEI). Each entry may carry `relevance_score` or + * `score`, and optionally `document` (`{text}` or string) or `text` (TEI). + * + * Entries without a usable integer `index` in range, or without a numeric score, are + * dropped rather than passed through malformed. Results are sorted by score descending + * (TEI already does; Cohere clients expect it) and capped at `top_n`. Top-level fields + * other than the results array (`model`, `usage`, `id`, …) are preserved. + */ +export function normalizeLocalRerankResponse( + data: unknown, + documents: unknown[], + options: { top_n?: number | null; return_documents?: boolean | null } = {} +): CohereRerankResponse { + const { list, envelope } = pickResultsArray(data); + const returnDocuments = options.return_documents === true; + const results: CohereRerankResult[] = []; + + for (const raw of list) { + const result = toCohereResult(raw, documents, returnDocuments); + if (result) results.push(result); + } + + results.sort((a, b) => b.relevance_score - a.relevance_score); + + const topN = options.top_n && options.top_n > 0 ? options.top_n : results.length; + return { ...envelope, results: results.slice(0, topN) }; +} diff --git a/src/app/api/v1/_shared/rerankProviderNodes.ts b/src/app/api/v1/_shared/rerankProviderNodes.ts new file mode 100644 index 0000000000..b11052c283 --- /dev/null +++ b/src/app/api/v1/_shared/rerankProviderNodes.ts @@ -0,0 +1,104 @@ +/** + * Provider-node selection for `POST /v1/rerank`. + * + * Mirrors `audioProviderNodes.ts`: the pure selection step takes the node rows plus an + * explicit `allowRemote` decision so the policy is directly testable, and the thin + * `loadRerankProviderNodes()` wrapper resolves the DB rows and the feature flag. + * + * Eligibility (see `@/shared/network/providerNodeHost`): + * - loopback nodes (localhost / 127.0.0.1 / 172.16.0.0/12) are always eligible — unchanged + * from the original hardcoded filter; + * - remote nodes (a LAN box, a Tailscale peer, a public host) are eligible only when the + * operator opted in via `RERANK_REMOTE_PROVIDER_NODES` (default OFF, like + * `AUDIO_REMOTE_PROVIDER_NODES`, #3963) AND the base URL passes the provider outbound + * URL policy (`getProviderOutboundGuard()`, #5066 / #9123) — so cloud-metadata hosts + * are never routed to, and strict `public-only` deployments never route to private hosts. + */ + +import { getCachedProviderNodes } from "@/lib/db/readCache"; +import { isFeatureFlagEnabled } from "@/shared/utils/featureFlags"; +import { isEligibleProviderNodeHost } from "@/shared/network/providerNodeHost"; + +/** Feature flag gating remote (non-loopback) rerank provider nodes. Default OFF. */ +export const RERANK_REMOTE_NODES_FLAG = "RERANK_REMOTE_PROVIDER_NODES"; + +export interface RerankProviderNodeRow { + id?: string; + prefix?: string | null; + baseUrl?: string | null; + apiType?: string | null; +} + +export interface DynamicRerankProvider { + id: string; + baseUrl: string; + authType: "apikey"; + authHeader: "bearer"; + /** Full provider connection id for credential lookup. */ + providerId: string; +} + +/** + * Build a dynamic rerank provider from a provider_node. OpenAI-compatible backends + * (oMLX, vLLM, Infinity, TEI behind a gateway, …) expose `/v1/rerank` under the same base + * URL as chat/embeddings. + */ +export function buildDynamicRerankProvider(node: RerankProviderNodeRow): DynamicRerankProvider { + if (!node.prefix || !node.baseUrl || !node.id) { + throw new Error("Invalid provider_node: missing id, prefix or baseUrl"); + } + // Strip trailing /v1 if present — we'll add /rerank + let base = node.baseUrl.replace(/\/+$/, ""); + if (base.endsWith("/v1")) base = base.slice(0, -3); + return { + id: node.prefix, + baseUrl: `${base}/v1/rerank`, + authType: "apikey", + authHeader: "bearer", + providerId: node.id, + }; +} + +/** + * Pure selection step — no DB, no flag lookup. + * + * @param nodes provider_node rows + * @param allowRemote whether non-loopback nodes are eligible (feature-flagged) + */ +export function selectRerankProviderNodes( + nodes: RerankProviderNodeRow[], + { allowRemote }: { allowRemote: boolean } +): DynamicRerankProvider[] { + const providers: DynamicRerankProvider[] = []; + for (const node of nodes) { + if (!node?.baseUrl) continue; + if (!isEligibleProviderNodeHost(node.baseUrl, { allowRemote })) continue; + try { + providers.push(buildDynamicRerankProvider(node)); + } catch { + // Malformed row — skip, never fail the request. + } + } + return providers; +} + +/** Resolve the eligible rerank provider nodes for the current request. */ +export async function loadRerankProviderNodes(): Promise { + let nodes: RerankProviderNodeRow[] = []; + try { + const rows = await getCachedProviderNodes(); + nodes = (Array.isArray(rows) ? rows : []).filter( + (n): n is RerankProviderNodeRow => n !== null && typeof n === "object" + ); + } catch { + // Non-critical — continue with cloud providers only + return []; + } + let allowRemote = false; + try { + allowRemote = isFeatureFlagEnabled(RERANK_REMOTE_NODES_FLAG); + } catch { + allowRemote = false; + } + return selectRerankProviderNodes(nodes, { allowRemote }); +} diff --git a/src/app/api/v1/rerank/route.ts b/src/app/api/v1/rerank/route.ts index aee7894889..064b81137b 100644 --- a/src/app/api/v1/rerank/route.ts +++ b/src/app/api/v1/rerank/route.ts @@ -10,7 +10,11 @@ import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts"; import { enforceApiKeyPolicy } from "@/shared/utils/apiKeyPolicy"; import { v1RerankSchema } from "@/shared/validation/schemas"; import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; -import { getCachedProviderNodes } from "@/lib/db/readCache"; +import { loadRerankProviderNodes } from "@/app/api/v1/_shared/rerankProviderNodes"; +import { + buildLocalRerankRequestBody, + normalizeLocalRerankResponse, +} from "@/app/api/v1/_shared/rerankLocalNodeShapes"; import { isAllRateLimitedCredentials, rateLimitedProviderResponse, @@ -34,29 +38,14 @@ export async function OPTIONS() { }); } -/** - * Build dynamic rerank provider from a local provider_node. - * Local OpenAI-compatible backends (oMLX, vLLM, etc.) expose /v1/rerank - * under the same base URL as chat. - */ -function buildDynamicRerankProvider(node: any) { - // Strip trailing /v1 if present — we'll add /rerank - let base = node.baseUrl || ""; - if (base.endsWith("/v1")) base = base.slice(0, -3); - return { - id: node.prefix, - baseUrl: `${base}/v1/rerank`, - authType: "apikey", - authHeader: "bearer", - providerId: node.id, // full provider connection ID for credential lookup - }; -} - /** * POST /v1/rerank - Cohere-compatible rerank endpoint * * Supports cloud providers (Cohere, Together, NVIDIA, Fireworks) - * and local provider_nodes (oMLX, vLLM, etc.) via dynamic routing. + * and OpenAI-compatible provider_nodes (oMLX, vLLM, Infinity, TEI behind a gateway, …) + * via dynamic routing. Loopback nodes are always eligible; remote nodes require the + * `RERANK_REMOTE_PROVIDER_NODES` opt-in and must pass the provider outbound URL policy + * (see `_shared/rerankProviderNodes.ts`). */ async function postHandler(request, context) { let rawBody; @@ -76,35 +65,9 @@ async function postHandler(request, context) { const policy = await enforceApiKeyPolicy(request, body.model); if (policy.rejection) return policy.rejection; - // Load local provider_nodes for rerank routing (localhost only) - let localProviders: ReturnType[] = []; - try { - const nodes = await getCachedProviderNodes(); - localProviders = (Array.isArray(nodes) ? nodes : []) - .filter((n: any) => { - try { - const hostname = new URL(n.baseUrl).hostname; - // Strictly matching 172.16.0.0/12 (Docker/local) and explicitly blocking ::1 per SSRF hardening - return ( - hostname === "localhost" || - hostname === "127.0.0.1" || - /^172\.(1[6-9]|2[0-9]|3[0-1])\.\d{1,3}\.\d{1,3}$/.test(hostname) - ); - } catch { - return false; - } - }) - .map((n) => { - try { - return buildDynamicRerankProvider(n); - } catch { - return null; - } - }) - .filter((p): p is NonNullable => p !== null); - } catch { - // Non-critical — continue with cloud providers only - } + // Load eligible provider_nodes for rerank routing (loopback always; remote when + // RERANK_REMOTE_PROVIDER_NODES is on and the URL passes the outbound policy). + const localProviders = await loadRerankProviderNodes(); // Try cloud registry first const { provider, model: modelId } = parseRerankModel(body.model); @@ -222,40 +185,34 @@ async function postHandler(request, context) { const token = credentials?.apiKey || credentials?.accessToken; const startTime = Date.now(); + // One body serves every known local server: Cohere/OpenAI spelling (`documents`, + // `return_documents`) plus the TEI spelling (`texts`, `return_text`). See + // `_shared/rerankLocalNodeShapes.ts`. + const upstreamBody = JSON.stringify( + buildLocalRerankRequestBody({ + model: localModel, + query: body.query, + documents: body.documents, + top_n: body.top_n as number | undefined, + return_documents: body.return_documents as boolean | undefined, + }) + ); + const upstreamInit: RequestInit = { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${token}`, + }, + body: upstreamBody, + }; try { - let res = await fetch(localProvider.baseUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${token}`, - }, - body: JSON.stringify({ - model: localModel, - query: body.query, - documents: body.documents, - top_n: body.top_n || body.documents.length, - return_documents: body.return_documents !== false, - }), - }); + let res = await fetch(localProvider.baseUrl, upstreamInit); // Some local providers (e.g. Infinity, TEI) mount at /rerank rather than /v1/rerank if (res.status === 404 && localProvider.baseUrl.endsWith("/v1/rerank")) { const fallbackUrl = localProvider.baseUrl.replace(/\/v1\/rerank$/, "/rerank"); try { - const fallbackRes = await fetch(fallbackUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${token}`, - }, - body: JSON.stringify({ - model: localModel, - query: body.query, - documents: body.documents, - top_n: body.top_n || body.documents.length, - return_documents: body.return_documents !== false, - }), - }); + const fallbackRes = await fetch(fallbackUrl, upstreamInit); if (fallbackRes.ok || fallbackRes.status !== 404) { res = fallbackRes; } @@ -292,7 +249,13 @@ async function postHandler(request, context) { return errorResponse(res.status, errorMessage); } - const data = await res.json(); + // Fold TEI's bare `[{index, score, text}]`, `score`-only gateways, and + // Voyage-style `{data: [...]}` into the Cohere envelope clients (and the + // memory engine, which reads `relevance_score`) expect. + const data = normalizeLocalRerankResponse(await res.json(), body.documents, { + top_n: body.top_n as number | undefined, + return_documents: body.return_documents as boolean | undefined, + }); const latencyMs = Date.now() - startTime; saveCallLog({ method: "POST", diff --git a/src/lib/localHealthCheck.ts b/src/lib/localHealthCheck.ts index 1853fc910c..203b0bddd8 100644 --- a/src/lib/localHealthCheck.ts +++ b/src/lib/localHealthCheck.ts @@ -12,6 +12,7 @@ */ import { getCachedProviderNodes } from "@/lib/db/readCache"; +import { isLoopbackNodeHost } from "@/shared/network/loopbackNodeHost"; import { isAutomatedTestProcess } from "@/shared/utils/testProcess"; // ── Types ──────────────────────────────────────────────────────────────── @@ -81,23 +82,8 @@ function isLocalHealthCheckDisabled(): boolean { ); } -function isLocalhostUrl(baseUrl: string): boolean { - try { - const u = new URL(baseUrl); - // Block credentials in URL to prevent SSRF via user@host (e.g., http://localhost@evil.com) - if (u.username || u.password) return false; - // Note: URL.hostname returns "[::1]" WITH brackets for IPv6 — both forms checked. - // Verified: node -e "new URL('http://[::1]:8080').hostname" → "[::1]" - // Strictly matching 172.16.0.0/12 (Docker/local) and explicitly blocking ::1 per SSRF hardening - return ( - u.hostname === "localhost" || - u.hostname === "127.0.0.1" || - /^172\.(1[6-9]|2[0-9]|3[0-1])\.\d{1,3}\.\d{1,3}$/.test(u.hostname) - ); - } catch { - return false; - } -} +/** Loopback/private-range hosts — shared definition (see `@/shared/network/loopbackNodeHost`). */ +const isLocalhostUrl = isLoopbackNodeHost; function getNextInterval(failures: number): number { return BACKOFF_SCHEDULE[Math.min(failures, BACKOFF_SCHEDULE.length - 1)]; diff --git a/src/shared/constants/featureFlagDefinitions.ts b/src/shared/constants/featureFlagDefinitions.ts index 5103642028..0cbc7c1508 100644 --- a/src/shared/constants/featureFlagDefinitions.ts +++ b/src/shared/constants/featureFlagDefinitions.ts @@ -155,6 +155,18 @@ export const FEATURE_FLAG_DEFINITIONS: FeatureFlagDefinition[] = [ requiresRestart: false, warningLevel: "danger", }, + { + key: "RERANK_REMOTE_PROVIDER_NODES", + label: "Remote Rerank Provider Nodes", + description: + "Allow POST /v1/rerank (and the memory engine's rerank step, which calls it over loopback) to use OpenAI-compatible provider nodes hosted outside localhost — a LAN box or Tailscale peer running TEI, Infinity, vLLM, etc. Off by default — routing to a remote host changes egress identity and must be an explicit operator decision. Loopback nodes are always allowed and unaffected. Remote nodes must also pass the provider outbound URL policy (cloud-metadata hosts are never routed to).", + descriptionI18nKey: "settings.featureFlags.rerankRemoteProviderNodes", + category: "network", + defaultValue: "false", + type: "boolean", + requiresRestart: false, + warningLevel: "danger", + }, { key: "PROXY_AUTO_SELECT_ENABLED", label: "Proxy Auto-Selection Fallback", diff --git a/src/shared/network/loopbackNodeHost.ts b/src/shared/network/loopbackNodeHost.ts new file mode 100644 index 0000000000..9f64c05ec7 --- /dev/null +++ b/src/shared/network/loopbackNodeHost.ts @@ -0,0 +1,33 @@ +/** + * Loopback classification for OpenAI-compatible `provider_nodes`. + * + * "Loopback" here means the operator's own machine or Docker network: `localhost`, + * `127.0.0.1`, and `172.16.0.0/12`. These hosts never leave the box, so every modality + * route (rerank, audio) admits them unconditionally and the local health checker probes + * them. `::1` stays excluded, matching the SSRF hardening that introduced the check. + * + * This module is deliberately pure — no DB, no env, no `node:` imports — so registries + * under `open-sse/config/` that may be reached from a browser bundle can import it + * (cf. #11122 for why `privateHost.ts` is split the same way). Policy-aware helpers that + * consult feature flags live in `./providerNodeHost.ts`. + */ + +const DOCKER_PRIVATE_RANGE = /^172\.(1[6-9]|2[0-9]|3[0-1])\.\d{1,3}\.\d{1,3}$/; + +/** Hosts reachable only from the operator's machine/Docker network. */ +export function isLoopbackNodeHost(baseUrl: string): boolean { + try { + const u = new URL(baseUrl); + // Block credentials in URL to prevent SSRF via user@host (e.g., http://localhost@evil.com) + if (u.username || u.password) return false; + // Note: URL.hostname returns "[::1]" WITH brackets for IPv6 — both forms stay excluded. + // Strictly matching 172.16.0.0/12 (Docker/local) and explicitly blocking ::1 per SSRF hardening + return ( + u.hostname === "localhost" || + u.hostname === "127.0.0.1" || + DOCKER_PRIVATE_RANGE.test(u.hostname) + ); + } catch { + return false; + } +} diff --git a/src/shared/network/providerNodeHost.ts b/src/shared/network/providerNodeHost.ts new file mode 100644 index 0000000000..5666911089 --- /dev/null +++ b/src/shared/network/providerNodeHost.ts @@ -0,0 +1,66 @@ +/** + * Policy-aware host eligibility for OpenAI-compatible `provider_nodes`. + * + * Modality routes that dispatch to provider nodes (rerank today, audio via its own + * selection step) answer two questions before forwarding a request: + * + * 1. Is the node loopback (operator's machine / Docker network)? Always eligible. + * See `./loopbackNodeHost.ts`. + * + * 2. Otherwise, may the route dispatch to a REMOTE node (a LAN box, a Tailscale peer, + * a public host)? Two conditions, both required: + * - the route's operator opt-in flag is set (e.g. `RERANK_REMOTE_PROVIDER_NODES`, + * mirroring `AUDIO_REMOTE_PROVIDER_NODES` — routing to a remote host changes + * egress identity and must be an explicit decision, cf. #3963), and + * - the node's base URL passes the same outbound URL policy that already governs + * provider-node creation and use (`getProviderOutboundGuard()`, #5066 / #9123). + * A node that was allowed to be created is therefore allowed to be routed to, + * and a URL the policy blocks (cloud-metadata always; any private host under + * strict `public-only`) is never routed to, whatever the flag says. + */ + +import { isLoopbackNodeHost } from "./loopbackNodeHost"; +import { + parseAndValidateNonMetadataUrl, + parseAndValidatePublicUrl, + parseOutboundUrl, +} from "./outboundUrlGuard"; +import { getProviderOutboundGuard } from "./outboundUrlGuardPolicy"; + +export { isLoopbackNodeHost }; + +/** + * Whether a non-loopback provider-node base URL passes the provider outbound URL policy. + * Mirrors `guardProviderNodeBaseUrl()` on the node-creation path so routing and creation + * agree: `none` → protocol/credential checks only; `block-metadata` (local-first default) + * → LAN allowed, cloud-metadata blocked; `public-only` → private hosts blocked. + */ +export function isRemoteNodeHostAllowedByPolicy(baseUrl: string): boolean { + try { + const guard = getProviderOutboundGuard(); + if (guard === "none") { + parseOutboundUrl(baseUrl); + } else if (guard === "block-metadata") { + parseAndValidateNonMetadataUrl(baseUrl); + } else { + parseAndValidatePublicUrl(baseUrl); + } + return true; + } catch { + return false; + } +} + +/** + * Combined eligibility used by modality routes: loopback nodes are always eligible; + * remote nodes are eligible only when the route's operator opt-in is set AND the URL + * passes the outbound policy. + */ +export function isEligibleProviderNodeHost( + baseUrl: string, + { allowRemote }: { allowRemote: boolean } +): boolean { + if (isLoopbackNodeHost(baseUrl)) return true; + if (!allowRemote) return false; + return isRemoteNodeHostAllowedByPolicy(baseUrl); +} diff --git a/tests/unit/feature-flags-settings.test.ts b/tests/unit/feature-flags-settings.test.ts index fc74e0f4ad..39651ee70a 100644 --- a/tests/unit/feature-flags-settings.test.ts +++ b/tests/unit/feature-flags-settings.test.ts @@ -40,7 +40,7 @@ const { // the dead ONEPROXY_ENABLED (readerless since the 1proxy purge, #12091) // brought it back to 53. UNIVERSAL_CONTEXT_HANDOFF_ENABLED bumped it to 54. // #13641 added SEARCH_STATS_HIDE_DELETED_CONNECTIONS, bumping the count to 56. -const EXPECTED_FEATURE_FLAG_COUNT = 73; +const EXPECTED_FEATURE_FLAG_COUNT = 74; // ────────────────────────────────────────────────────── // Test group 1 — Flag definitions registry diff --git a/tests/unit/rerank-local-node-shapes.test.ts b/tests/unit/rerank-local-node-shapes.test.ts new file mode 100644 index 0000000000..3d11b75b4e --- /dev/null +++ b/tests/unit/rerank-local-node-shapes.test.ts @@ -0,0 +1,297 @@ +// Regression tests for the TEI / Infinity request+response shape adapters on the +// POST /v1/rerank provider-node path. +// +// The route fell back from /v1/rerank to /rerank on 404 "for Infinity / +// TEI", but still sent `documents` (TEI requires `texts` → HTTP 422) and returned the +// upstream body verbatim (TEI answers a bare `[{index, score, text}]`, some gateways +// `{results:[{index, score}]}`), so clients — and the memory engine, which reads +// `relevance_score` — got either an error or undefined scores. +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-rerank-shapes-test-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const { buildLocalRerankRequestBody, documentText, normalizeLocalRerankResponse } = + await import("../../src/app/api/v1/_shared/rerankLocalNodeShapes.ts"); +const core = await import("../../src/lib/db/core.ts"); +const { invalidateDbCache } = await import("../../src/lib/db/readCache.ts"); +const { createProviderNode, createProviderConnection } = + await import("../../src/lib/db/providers.ts"); +const { POST } = await import("../../src/app/api/v1/rerank/route.ts"); + +const DOCS = ["a cat is a small animal", "the stock market fell", { text: "cats purr" }]; + +test.describe("buildLocalRerankRequestBody", () => { + test("sends both the Cohere/OpenAI and the TEI spellings", () => { + const body = buildLocalRerankRequestBody({ + model: "bge-reranker-v2-m3", + query: "what is a cat", + documents: DOCS, + top_n: 2, + return_documents: false, + }); + assert.equal(body.model, "bge-reranker-v2-m3"); + assert.equal(body.query, "what is a cat"); + assert.deepEqual(body.documents, DOCS); + assert.deepEqual(body.texts, ["a cat is a small animal", "the stock market fell", "cats purr"]); + assert.equal(body.top_n, 2); + assert.equal(body.return_documents, false); + assert.equal(body.return_text, false); + }); + + test("defaults top_n to the document count and return_documents to false (Cohere default)", () => { + // Clients that did not ask for documents must keep getting the bare + // {index, relevance_score} rows the /v1/rerank contract always returned + // (local-rerank-logging.test.ts); documents are echoed back only on request. + const body = buildLocalRerankRequestBody({ model: "m", query: "q", documents: DOCS }); + assert.equal(body.top_n, 3); + assert.equal(body.return_documents, false); + assert.equal(body.return_text, false); + const explicit = buildLocalRerankRequestBody({ + model: "m", + query: "q", + documents: DOCS, + return_documents: true, + }); + assert.equal(explicit.return_documents, true); + assert.equal(explicit.return_text, true); + }); + + test("documentText flattens strings, {text} objects, and other values", () => { + assert.equal(documentText("plain"), "plain"); + assert.equal(documentText({ text: "obj" }), "obj"); + assert.equal(documentText(42), "42"); + assert.equal(documentText(null), ""); + }); +}); + +test.describe("normalizeLocalRerankResponse", () => { + test("TEI bare array with `score` and `text` → Cohere envelope", () => { + const out = normalizeLocalRerankResponse( + [ + { index: 1, score: 0.01, text: "the stock market fell" }, + { index: 0, score: 0.98, text: "a cat is a small animal" }, + ], + DOCS, + { return_documents: true } + ); + assert.deepEqual(out, { + results: [ + { index: 0, relevance_score: 0.98, document: { text: "a cat is a small animal" } }, + { index: 1, relevance_score: 0.01, document: { text: "the stock market fell" } }, + ], + }); + }); + + test("gateway `{results:[{index, score}]}` gains relevance_score and keeps extras", () => { + const out = normalizeLocalRerankResponse( + { + model: "bge-reranker-v2-m3", + usage: { total_tokens: 12 }, + results: [ + { index: 0, score: 0.9 }, + { index: 2, score: 0.5 }, + ], + }, + DOCS, + { return_documents: false } + ); + assert.deepEqual(out, { + model: "bge-reranker-v2-m3", + usage: { total_tokens: 12 }, + results: [ + { index: 0, relevance_score: 0.9 }, + { index: 2, relevance_score: 0.5 }, + ], + }); + }); + + test("already-Cohere responses pass through unchanged apart from ordering/top_n", () => { + const out = normalizeLocalRerankResponse( + { + results: [ + { index: 1, relevance_score: 0.2, document: { text: "the stock market fell" } }, + { index: 0, relevance_score: 0.7, document: { text: "a cat is a small animal" } }, + { index: 2, relevance_score: 0.6, document: { text: "cats purr" } }, + ], + }, + DOCS, + { top_n: 2, return_documents: true } + ); + assert.deepEqual( + out.results.map((r) => [r.index, r.relevance_score]), + [ + [0, 0.7], + [2, 0.6], + ] + ); + }); + + test("Voyage-style `{data:[...]}` is accepted", () => { + const out = normalizeLocalRerankResponse( + { data: [{ index: 0, relevance_score: 0.3 }], object: "list" }, + DOCS, + { return_documents: false } + ); + assert.deepEqual(out, { object: "list", results: [{ index: 0, relevance_score: 0.3 }] }); + }); + + test("attaches document text from the request when the upstream did not echo it", () => { + const out = normalizeLocalRerankResponse({ results: [{ index: 2, score: 1 }] }, DOCS, { + return_documents: true, + }); + assert.deepEqual(out.results[0].document, { text: "cats purr" }); + }); + + test("drops malformed entries instead of forwarding them", () => { + const out = normalizeLocalRerankResponse( + [ + { index: 0, score: "0.5" }, // numeric string → accepted + { index: 9, score: 0.9 }, // out of range → dropped + { index: 1 }, // no score → dropped + { score: 0.1 }, // no index → dropped + "junk", + null, + ], + DOCS, + { return_documents: false } + ); + assert.deepEqual(out, { results: [{ index: 0, relevance_score: 0.5 }] }); + }); + + test("unrecognised payloads yield an empty results list rather than throwing", () => { + assert.deepEqual(normalizeLocalRerankResponse({ ok: true }, DOCS), { ok: true, results: [] }); + assert.deepEqual(normalizeLocalRerankResponse("nope", DOCS), { results: [] }); + assert.deepEqual(normalizeLocalRerankResponse(null, DOCS), { results: [] }); + }); +}); + +test.describe("POST /v1/rerank against a TEI-shaped local node", () => { + const originalFetch = globalThis.fetch; + + test.before(async () => { + const now = new Date().toISOString(); + await createProviderNode({ + id: "openai-compatible-rerank-tei", + name: "tei", + type: "openai", + prefix: "tei", + baseUrl: "http://127.0.0.1:8081/v1", + apiType: "rerank", + createdAt: now, + updatedAt: now, + }); + await createProviderConnection({ + id: "conn-tei-1", + provider: "openai-compatible-rerank-tei", + authType: "apikey", + name: "tei", + apiKey: "tei-token", + createdAt: now, + updatedAt: now, + }); + invalidateDbCache("nodes"); + invalidateDbCache("connections"); + }); + + test.afterEach(() => { + globalThis.fetch = originalFetch; + }); + + test.after(() => { + core.resetDbInstance(); + try { + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } catch { + // ignore + } + }); + + test("404 on /v1/rerank → /rerank with `texts`; bare TEI array is normalized", async () => { + const calls: Array<{ url: string; body: Record }> = []; + globalThis.fetch = async (url: string | URL | Request, init?: RequestInit) => { + const body = JSON.parse(String(init?.body || "{}")); + calls.push({ url: String(url), body }); + if (String(url).endsWith("/v1/rerank")) { + return new Response("Not Found", { status: 404 }); + } + // TEI: requires `texts`, answers a bare array sorted by score, `text` when asked. + if (!Array.isArray(body.texts)) { + return new Response(JSON.stringify({ error: "missing field `texts`" }), { status: 422 }); + } + return new Response( + JSON.stringify([ + { index: 0, score: 0.9875887, text: body.texts[0] }, + { index: 1, score: 0.000016, text: body.texts[1] }, + ]), + { status: 200, headers: { "content-type": "application/json" } } + ); + }; + + const res = await POST( + new Request("http://localhost/v1/rerank", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "tei/bge-reranker-v2-m3", + query: "what is a cat", + documents: ["a cat is a small animal", "the stock market fell"], + top_n: 1, + }), + }), + {} + ); + + assert.equal(res.status, 200); + const data = (await res.json()) as { + results: Array<{ index: number; relevance_score: number; document?: { text: string } }>; + }; + // No return_documents in the request → Cohere default (false): bare rows only. + assert.deepEqual(data, { results: [{ index: 0, relevance_score: 0.9875887 }] }); + assert.deepEqual( + calls.map((c) => c.url), + ["http://127.0.0.1:8081/v1/rerank", "http://127.0.0.1:8081/rerank"] + ); + assert.deepEqual(calls[1].body.texts, ["a cat is a small animal", "the stock market fell"]); + assert.deepEqual(calls[1].body.documents, ["a cat is a small animal", "the stock market fell"]); + assert.equal(calls[1].body.return_text, false); + assert.equal(calls[1].body.model, "bge-reranker-v2-m3"); + }); + + test("a `score`-only gateway on /v1/rerank yields relevance_score for clients", async () => { + globalThis.fetch = async () => + new Response( + JSON.stringify({ + results: [ + { index: 0, score: 0.98, text: "a cat is a small animal" }, + { index: 1, score: 0.02, text: "the stock market fell" }, + ], + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + + const res = await POST( + new Request("http://localhost/v1/rerank", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "tei/bge-reranker-v2-m3", + query: "what is a cat", + documents: ["a cat is a small animal", "the stock market fell"], + return_documents: false, + }), + }), + {} + ); + assert.equal(res.status, 200); + const data = (await res.json()) as { results: Array> }; + assert.deepEqual(data.results, [ + { index: 0, relevance_score: 0.98 }, + { index: 1, relevance_score: 0.02 }, + ]); + }); +}); diff --git a/tests/unit/rerank-remote-provider-nodes.test.ts b/tests/unit/rerank-remote-provider-nodes.test.ts new file mode 100644 index 0000000000..0642b43ed4 --- /dev/null +++ b/tests/unit/rerank-remote-provider-nodes.test.ts @@ -0,0 +1,308 @@ +// Regression tests for provider-node eligibility on POST /v1/rerank. +// +// The route used to carry its own hardcoded host filter (localhost / 127.0.0.1 / +// 172.16.0.0/12), so a rerank node on any other host — a LAN box or Tailscale peer +// running TEI, Infinity, or vLLM — was silently dropped and the request fell through to +// "Invalid rerank model", even though the same node served /v1/embeddings without +// complaint and had passed the provider outbound URL policy at creation time. +// +// Eligibility now mirrors the audio routes (#3963): loopback nodes are always eligible, +// remote nodes are opt-in via RERANK_REMOTE_PROVIDER_NODES (default OFF) and must still +// pass the provider outbound URL policy (#5066 / #9123) — cloud-metadata hosts are never +// routed to, and strict `public-only` deployments never route to private hosts. +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-rerank-remote-test-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const { isLoopbackNodeHost } = await import("../../src/shared/network/loopbackNodeHost.ts"); +const { isEligibleProviderNodeHost, isRemoteNodeHostAllowedByPolicy } = + await import("../../src/shared/network/providerNodeHost.ts"); +const { RERANK_REMOTE_NODES_FLAG, selectRerankProviderNodes } = + await import("../../src/app/api/v1/_shared/rerankProviderNodes.ts"); +const { isLoopbackNodeHost: audioLoopback } = + await import("../../open-sse/config/audioRegistry.ts"); +const core = await import("../../src/lib/db/core.ts"); +const { invalidateDbCache } = await import("../../src/lib/db/readCache.ts"); +const { createProviderNode, createProviderConnection } = + await import("../../src/lib/db/providers.ts"); +const { POST } = await import("../../src/app/api/v1/rerank/route.ts"); + +const LOOPBACK_NODE = { + id: "openai-compatible-rerank-loop", + prefix: "loop", + baseUrl: "http://127.0.0.1:8000/v1", + apiType: "rerank", +}; +const DOCKER_NODE = { + id: "openai-compatible-rerank-docker", + prefix: "dockernode", + baseUrl: "http://172.18.0.5:8000/v1", + apiType: "embeddings", +}; +const LAN_NODE = { + id: "openai-compatible-embeddings-lan", + prefix: "skilled-mini", + baseUrl: "http://10.10.50.19:8888/v1", + apiType: "embeddings", +}; +const METADATA_NODE = { + id: "openai-compatible-rerank-imds", + prefix: "imds", + baseUrl: "http://169.254.169.254/v1", + apiType: "rerank", +}; +const PUBLIC_NODE = { + id: "openai-compatible-rerank-public", + prefix: "pub", + baseUrl: "https://rerank.example.com/v1", + apiType: "rerank", +}; + +const ENV_KEYS = [ + RERANK_REMOTE_NODES_FLAG, + "OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS", + "OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS", + "OUTBOUND_SSRF_GUARD_ENABLED", +] as const; +const savedEnv: Record = {}; +for (const k of ENV_KEYS) savedEnv[k] = process.env[k]; + +function resetEnv() { + for (const k of ENV_KEYS) { + if (savedEnv[k] === undefined) delete process.env[k]; + else process.env[k] = savedEnv[k]; + } +} + +test.describe("loopback host classification (shared)", () => { + test("accepts localhost, 127.0.0.1 and 172.16/12; rejects ::1, LAN, public, user@host", () => { + assert.equal(isLoopbackNodeHost("http://localhost:8080/v1"), true); + assert.equal(isLoopbackNodeHost("http://127.0.0.1:8080/v1"), true); + assert.equal(isLoopbackNodeHost("http://172.31.255.1:8080/v1"), true); + assert.equal(isLoopbackNodeHost("http://[::1]:8080/v1"), false); + assert.equal(isLoopbackNodeHost("http://10.10.50.19:8888/v1"), false); + assert.equal(isLoopbackNodeHost("http://192.168.1.10:8888/v1"), false); + assert.equal(isLoopbackNodeHost("https://rerank.example.com/v1"), false); + assert.equal(isLoopbackNodeHost("http://localhost@evil.com/v1"), false); + assert.equal(isLoopbackNodeHost("not a url"), false); + }); + + test("audio registry re-exports the shared definition", () => { + assert.equal(audioLoopback, isLoopbackNodeHost); + }); +}); + +test.describe("remote node policy", () => { + test.afterEach(() => resetEnv()); + + test("local-first default (block-metadata): LAN allowed, cloud-metadata blocked", () => { + delete process.env.OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS; + delete process.env.OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS; + assert.equal(isRemoteNodeHostAllowedByPolicy(LAN_NODE.baseUrl), true); + assert.equal(isRemoteNodeHostAllowedByPolicy(PUBLIC_NODE.baseUrl), true); + assert.equal(isRemoteNodeHostAllowedByPolicy(METADATA_NODE.baseUrl), false); + assert.equal(isRemoteNodeHostAllowedByPolicy("http://user:pw@10.10.50.19/v1"), false); + assert.equal(isRemoteNodeHostAllowedByPolicy("ftp://10.10.50.19/v1"), false); + }); + + test("strict public-only: private hosts blocked, public allowed", () => { + process.env.OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS = "false"; + delete process.env.OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS; + assert.equal(isRemoteNodeHostAllowedByPolicy(LAN_NODE.baseUrl), false); + assert.equal(isRemoteNodeHostAllowedByPolicy(PUBLIC_NODE.baseUrl), true); + assert.equal(isRemoteNodeHostAllowedByPolicy(METADATA_NODE.baseUrl), false); + }); + + test("full opt-in (none): protocol/credential checks only", () => { + process.env.OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS = "true"; + assert.equal(isRemoteNodeHostAllowedByPolicy(LAN_NODE.baseUrl), true); + assert.equal(isRemoteNodeHostAllowedByPolicy("http://user:pw@10.10.50.19/v1"), false); + }); + + test("isEligibleProviderNodeHost: loopback always, remote only with allowRemote", () => { + delete process.env.OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS; + assert.equal(isEligibleProviderNodeHost(LOOPBACK_NODE.baseUrl, { allowRemote: false }), true); + assert.equal(isEligibleProviderNodeHost(LAN_NODE.baseUrl, { allowRemote: false }), false); + assert.equal(isEligibleProviderNodeHost(LAN_NODE.baseUrl, { allowRemote: true }), true); + assert.equal(isEligibleProviderNodeHost(METADATA_NODE.baseUrl, { allowRemote: true }), false); + }); +}); + +test.describe("selectRerankProviderNodes", () => { + test.afterEach(() => resetEnv()); + + test("flag off: only loopback/Docker nodes are selected (previous behavior)", () => { + const selected = selectRerankProviderNodes( + [LOOPBACK_NODE, DOCKER_NODE, LAN_NODE, METADATA_NODE, PUBLIC_NODE], + { allowRemote: false } + ); + assert.deepEqual( + selected.map((p) => p.id), + ["loop", "dockernode"] + ); + assert.equal(selected[0].baseUrl, "http://127.0.0.1:8000/v1/rerank"); + assert.equal(selected[0].providerId, LOOPBACK_NODE.id); + }); + + test("flag on: LAN and public nodes join; cloud-metadata never does", () => { + delete process.env.OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS; + const selected = selectRerankProviderNodes( + [LOOPBACK_NODE, DOCKER_NODE, LAN_NODE, METADATA_NODE, PUBLIC_NODE], + { allowRemote: true } + ); + assert.deepEqual( + selected.map((p) => p.id), + ["loop", "dockernode", "skilled-mini", "pub"] + ); + const lan = selected.find((p) => p.id === "skilled-mini"); + assert.equal(lan?.baseUrl, "http://10.10.50.19:8888/v1/rerank"); + }); + + test("flag on under strict public-only policy: LAN node still excluded", () => { + process.env.OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS = "false"; + const selected = selectRerankProviderNodes([LOOPBACK_NODE, LAN_NODE, PUBLIC_NODE], { + allowRemote: true, + }); + assert.deepEqual( + selected.map((p) => p.id), + ["loop", "pub"] + ); + }); + + test("skips rows without a base URL or prefix instead of throwing", () => { + const selected = selectRerankProviderNodes( + [ + { id: "x", prefix: "", baseUrl: "http://127.0.0.1:1/v1" }, + { id: "y", prefix: "y" }, + ], + { allowRemote: true } + ); + assert.deepEqual(selected, []); + }); +}); + +test.describe("POST /v1/rerank routes to a LAN provider node only when opted in", () => { + const originalFetch = globalThis.fetch; + + test.before(async () => { + const now = new Date().toISOString(); + await createProviderNode({ + id: LAN_NODE.id, + name: "skilled-mini", + type: "openai", + prefix: LAN_NODE.prefix, + baseUrl: LAN_NODE.baseUrl, + apiType: LAN_NODE.apiType, + createdAt: now, + updatedAt: now, + }); + await createProviderConnection({ + id: "conn-skilled-mini-1", + provider: LAN_NODE.id, + authType: "apikey", + name: "skilled-mini", + apiKey: "test-token", + createdAt: now, + updatedAt: now, + }); + invalidateDbCache("nodes"); + invalidateDbCache("connections"); + }); + + test.afterEach(() => { + globalThis.fetch = originalFetch; + resetEnv(); + }); + + test.after(() => { + core.resetDbInstance(); + try { + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } catch { + // ignore + } + }); + + function rerankRequest() { + return new Request("http://localhost/v1/rerank", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "skilled-mini/bge-reranker-v2-m3", + query: "what is a cat", + documents: ["a cat is a small animal", "the stock market fell"], + }), + }); + } + + test("flag off: LAN node is invisible and the request fails as an invalid model", async () => { + delete process.env[RERANK_REMOTE_NODES_FLAG]; + let upstreamCalled = false; + globalThis.fetch = async () => { + upstreamCalled = true; + return new Response("{}", { status: 200 }); + }; + + const res = await POST(rerankRequest(), {}); + assert.equal(res.status, 400); + const body = await res.json(); + assert.match(String(body?.error?.message ?? ""), /Invalid rerank model/); + assert.equal(upstreamCalled, false, "must not contact the remote node when opted out"); + }); + + test("flag on: request is forwarded to the LAN node's /v1/rerank with the node credential", async () => { + process.env[RERANK_REMOTE_NODES_FLAG] = "true"; + delete process.env.OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS; + const calls: Array<{ url: string; auth: string | null; body: Record }> = []; + globalThis.fetch = async (url: string | URL | Request, init?: RequestInit) => { + const headers = new Headers(init?.headers); + calls.push({ + url: String(url), + auth: headers.get("authorization"), + body: JSON.parse(String(init?.body || "{}")), + }); + return new Response( + JSON.stringify({ + results: [ + { index: 0, relevance_score: 0.98 }, + { index: 1, relevance_score: 0.01 }, + ], + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + }; + + const res = await POST(rerankRequest(), {}); + assert.equal(res.status, 200); + const data = (await res.json()) as { results: Array<{ index: number }> }; + assert.deepEqual( + data.results.map((r) => r.index), + [0, 1] + ); + assert.equal(calls.length, 1); + assert.equal(calls[0].url, "http://10.10.50.19:8888/v1/rerank"); + assert.equal(calls[0].auth, "Bearer test-token"); + assert.equal(calls[0].body.model, "bge-reranker-v2-m3"); + assert.deepEqual(calls[0].body.documents, ["a cat is a small animal", "the stock market fell"]); + assert.equal(res.headers.get("X-OmniRoute-Provider"), "skilled-mini"); + }); + + test("flag on but strict public-only policy: LAN node stays excluded", async () => { + process.env[RERANK_REMOTE_NODES_FLAG] = "true"; + process.env.OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS = "false"; + let upstreamCalled = false; + globalThis.fetch = async () => { + upstreamCalled = true; + return new Response("{}", { status: 200 }); + }; + + const res = await POST(rerankRequest(), {}); + assert.equal(res.status, 400); + assert.equal(upstreamCalled, false); + }); +}); diff --git a/tests/unit/server-owned-tool-loop-flag.test.ts b/tests/unit/server-owned-tool-loop-flag.test.ts index f58aa042f9..631095e190 100644 --- a/tests/unit/server-owned-tool-loop-flag.test.ts +++ b/tests/unit/server-owned-tool-loop-flag.test.ts @@ -68,7 +68,7 @@ describe("isServerOwnedToolLoopEnabled wrapper", () => { describe("feature-flags-settings count update", () => { it("flag count matches updated expected value", () => { - assert.equal(FEATURE_FLAG_DEFINITIONS.length, 73); + assert.equal(FEATURE_FLAG_DEFINITIONS.length, 74); }); });