diff --git a/open-sse/handlers/embeddings.ts b/open-sse/handlers/embeddings.ts index 72d98c3b2d..b66fff98b2 100644 --- a/open-sse/handlers/embeddings.ts +++ b/open-sse/handlers/embeddings.ts @@ -41,6 +41,31 @@ interface ClientRawRequest { headers: Record; } +/** + * Flatten a single embedding item's vector to the OpenAI-spec `number[]` shape. + * + * Some OpenAI-compatible embedding backends — notably a llama.cpp + * `llama-server --embedding --pooling ...` instance — return each vector wrapped in one + * extra array level: `[[...floats]]` instead of `[...floats]` for a single input. That + * extra level is silently spec-breaking, since a standard OpenAI-SDK consumer reading + * `response.data[i].embedding` gets a length-1 array holding the real vector instead of + * the vector itself. Unwrap only that single redundant level; vectors that are already + * flat (or genuinely multi-row) are left untouched. See issue #9089. + */ +function flattenSingleRowEmbedding(item: unknown): void { + if (!item || typeof item !== "object" || !("embedding" in item)) return; + const record = item as { embedding: unknown }; + const embedding = record.embedding; + if ( + Array.isArray(embedding) && + embedding.length === 1 && + Array.isArray(embedding[0]) && + typeof embedding[0][0] === "number" + ) { + record.embedding = embedding[0]; + } +} + /** * Handle embedding request. * Supports both hardcoded cloud providers and dynamic local provider_nodes. @@ -359,6 +384,19 @@ export async function handleEmbedding({ // Log provider response reqLogger.logProviderResponse(response.status, "", response.headers, data); + // OpenAI-spec compliance (#9089): each item's `embedding` must be a flat number[]. + // Some OpenAI-compatible backends (e.g. a llama.cpp `llama-server --embedding` + // instance) return the vector wrapped in one extra array level — `[[...floats]]` + // instead of `[...floats]` — for a single input, which silently breaks any standard + // OpenAI-SDK consumer doing `response.data[i].embedding`. Flatten that one redundant + // level without touching providers that already return flat vectors. + const responseItems = data.data || data; + if (Array.isArray(responseItems)) { + for (const item of responseItems) { + flattenSingleRowEmbedding(item); + } + } + // Normalize response to OpenAI format const normalizedResponse = { object: "list", diff --git a/tests/unit/embeddings-flatten-single-row-9089.test.ts b/tests/unit/embeddings-flatten-single-row-9089.test.ts new file mode 100644 index 0000000000..3424546c8c --- /dev/null +++ b/tests/unit/embeddings-flatten-single-row-9089.test.ts @@ -0,0 +1,106 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +process.env.DATA_DIR = mkdtempSync(join(tmpdir(), "omniroute-embeddings-9089-")); + +const { handleEmbedding } = await import("../../open-sse/handlers/embeddings.ts"); + +const localProvider = { + id: "localembed", + baseUrl: "http://localhost:8080/embeddings", + authType: "none" as const, + authHeader: "none" as const, + models: [], +}; + +function mockUpstream(payload: unknown): () => void { + const original = globalThis.fetch; + globalThis.fetch = async () => + new Response(JSON.stringify(payload), { + status: 200, + headers: { "content-type": "application/json" }, + }); + return () => { + globalThis.fetch = original; + }; +} + +// #9089: a custom "OpenAI Compatible" (Embeddings) provider pointed at a llama.cpp +// `llama-server --embedding --pooling cls` backend returns each vector wrapped in one +// extra array level — `[[...floats]]` instead of `[...floats]`. The OpenAI spec requires a +// flat `number[]`; the extra level silently breaks any SDK consumer doing +// `response.data[0].embedding` (it gets a length-1 array holding the real vector). +test("handleEmbedding flattens a single-row 2D embedding vector (#9089)", async () => { + const restore = mockUpstream({ + data: [{ object: "embedding", embedding: [[0.1, 0.2, 0.3]], index: 0 }], + usage: { prompt_tokens: 2, total_tokens: 2 }, + }); + try { + const result = await handleEmbedding({ + body: { model: "localembed/bge-m3", input: "test" }, + credentials: null, + resolvedProvider: localProvider, + resolvedModel: "bge-m3", + log: null, + }); + + assert.equal(result.success, true); + const rows = result.data.data as Array<{ embedding: number[] }>; + assert.deepEqual(rows[0].embedding, [0.1, 0.2, 0.3]); + assert.equal(rows[0].embedding.length, 3); + assert.equal(typeof rows[0].embedding[0], "number"); + } finally { + restore(); + } +}); + +test("handleEmbedding leaves an already-flat embedding untouched (#9089 regression guard)", async () => { + const restore = mockUpstream({ + data: [{ object: "embedding", embedding: [0.1, 0.2, 0.3], index: 0 }], + usage: { prompt_tokens: 2, total_tokens: 2 }, + }); + try { + const result = await handleEmbedding({ + body: { model: "localembed/bge-m3", input: "test" }, + credentials: null, + resolvedProvider: localProvider, + resolvedModel: "bge-m3", + log: null, + }); + + assert.equal(result.success, true); + const rows = result.data.data as Array<{ embedding: number[] }>; + assert.deepEqual(rows[0].embedding, [0.1, 0.2, 0.3]); + } finally { + restore(); + } +}); + +test("handleEmbedding flattens single-row vectors for every item in a batch (#9089)", async () => { + const restore = mockUpstream({ + data: [ + { object: "embedding", embedding: [[1, 2]], index: 0 }, + { object: "embedding", embedding: [[3, 4]], index: 1 }, + ], + usage: { total_tokens: 4 }, + }); + try { + const result = await handleEmbedding({ + body: { model: "localembed/bge-m3", input: ["a", "b"] }, + credentials: null, + resolvedProvider: localProvider, + resolvedModel: "bge-m3", + log: null, + }); + + assert.equal(result.success, true); + const rows = result.data.data as Array<{ embedding: number[] }>; + assert.deepEqual(rows[0].embedding, [1, 2]); + assert.deepEqual(rows[1].embedding, [3, 4]); + } finally { + restore(); + } +});