feat(providers): complete Jina + Gemini Embedding 2 multimodal via OmniRoute (#10581)

* feat(providers): complete Jina AI via OmniRoute including Omni multimodal

Dashboard and env keys share one Jina credential pool, native v5 Omni
{text}/{image}/{content} docs pass through /v1/embeddings intact, and
classify/segment/search are proxied without a third unused Jina card.

* chore(changelog): name Jina complete-provider fragment for #10581

* feat(providers): make Gemini Embedding 2 multimodal work via OmniRoute

Route gemini-embedding-2 through embedContent/batchEmbedContents so N
OpenAI input items become N vectors, pass through native multimodal
parts, and use dashboard Gemini keys (GEMINI_API_KEY only as fallback).

* fix(providers): resolve rebase fallout for Jina/Gemini embeddings

- narrow the two new no-explicit-any violations introduced by this PR
  (validateJinaFoundationProvider's params + catch, search.ts's
  normalizeJinaSearchResponse data param)
- cast credentials to Record<string, unknown> at the two quota-preflight
  call sites in src/sse/services/auth.ts so the new JinaEnvCredentials /
  GeminiEnvCredentials union members type-check without loosening the
  allRateLimited narrowing used elsewhere in the same function

Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>

---------

Co-authored-by: Ravi Tharuma <RaviTharuma@users.noreply.github.com>
Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
This commit is contained in:
Ravi Tharuma
2026-08-18 15:52:43 +02:00
committed by GitHub
parent 228ef6fba9
commit 3d0ffb49a4
41 changed files with 2416 additions and 91 deletions

View File

@@ -152,12 +152,13 @@ export const APIKEY_PROVIDERS_SPECIALTY = {
"jina-ai": {
id: "jina-ai",
alias: "jina",
name: "Jina AI",
name: "Jina AI (Foundation API)",
icon: "sort",
color: "#2563EB",
textIcon: "JA",
website: "https://jina.ai",
authHint: "Bearer API key for the Jina AI rerank API.",
authHint:
"Bearer API key for api.jina.ai — embeddings, rerank, classify, segment, and search. Dashboard keys take precedence over JINA_AI_API_KEY. This is not the Reader / r.jina.ai card and does not fetch URLs.",
hasFree: true,
freeNote: "10M free tokens on signup (non-commercial), no credit card required",
},
@@ -262,14 +263,16 @@ export const APIKEY_PROVIDERS_SPECIALTY = {
"jina-reader": {
id: "jina-reader",
alias: "jr",
name: "Jina Reader",
name: "Jina Reader (r.jina.ai)",
icon: "menu_book",
color: "#0EA5E9",
textIcon: "JR",
website: "https://jina.ai/reader",
authHint:
"Bearer API key for r.jina.ai URL-to-markdown (/v1/web/fetch only). Does not serve /v1/embeddings or /v1/rerank. The same Jina token as Foundation API works; OmniRoute reuses a jina-ai dashboard key or JINA_AI_API_KEY when this card is empty.",
hasFree: true,
notice: {
text: "Free tier: 1M fetches/month.",
text: "Reader / r.jina.ai only — not embeddings or rerank. Free tier: 1M fetches/month.",
apiKeyUrl: "https://jina.ai/api-dashboard",
},
serviceKinds: ["webFetch"],

View File

@@ -0,0 +1,126 @@
/**
* Gemini Embedding 2 native items (Google AI Studio embedContent / batchEmbedContents).
*
* Official 2026 contract (ai.google.dev/gemini-api/docs/embeddings):
* - Model id: gemini-embedding-2 (GA April 2026). Legacy text-only: gemini-embedding-001.
* - One Content (parts[]) → one embedding. Multiple parts in one Content fuse.
* - N Content objects / N batchEmbedContents requests → N embeddings.
* - Parts: { text }, { inline_data: { mime_type, data } }, { file_data: { mime_type, file_uri } }.
* CamelCase SDK spellings (inlineData / fileData) are accepted and forwarded.
*
* These are not OmniRoute's canonical `{ type, source }` items. For gemini
* they must reach generativelanguage.googleapis.com as Content parts — do
* not collapse the OpenAI `input` array to string[].
*/
import { isCanonicalEmbeddingItem, isPlainObject } from "./jinaNativeEmbeddingInput";
export type GeminiEmbeddingModality = "text" | "image" | "audio" | "video" | "document";
const GEMINI_EMBEDDING_2_IDS = new Set(["gemini-embedding-2", "gemini-embedding-2-preview"]);
export function isGeminiEmbedding2Family(modelId: string | null | undefined): boolean {
return typeof modelId === "string" && GEMINI_EMBEDDING_2_IDS.has(modelId);
}
function asRecord(value: unknown): Record<string, unknown> | null {
return isPlainObject(value) ? value : null;
}
function mimeFromInline(value: Record<string, unknown>): string | null {
const snake = asRecord(value.inline_data);
if (typeof snake?.mime_type === "string") return snake.mime_type;
const camel = asRecord(value.inlineData);
if (typeof camel?.mimeType === "string") return camel.mimeType;
return null;
}
function mimeFromFile(value: Record<string, unknown>): string | null {
const snake = asRecord(value.file_data);
if (typeof snake?.mime_type === "string") return snake.mime_type;
const camel = asRecord(value.fileData);
if (typeof camel?.mimeType === "string") return camel.mimeType;
return null;
}
export function modalityFromGeminiMime(mimeType: string): GeminiEmbeddingModality {
const mime = mimeType.trim().toLowerCase();
if (mime.startsWith("image/")) return "image";
if (mime.startsWith("audio/")) return "audio";
if (mime.startsWith("video/")) return "video";
if (mime === "application/pdf" || mime.startsWith("application/pdf")) return "document";
return "document";
}
export function isGeminiNativePart(value: unknown): boolean {
const record = asRecord(value);
if (!record || isCanonicalEmbeddingItem(record)) return false;
if (typeof record.text === "string" && record.text.trim().length > 0) {
return !("image" in record) && !("audio" in record) && !("video" in record) && !("pdf" in record);
}
if (asRecord(record.inline_data)?.data || asRecord(record.inlineData)?.data) return true;
if (asRecord(record.file_data)?.file_uri || asRecord(record.fileData)?.fileUri) return true;
return false;
}
export function isGeminiNativeContent(value: unknown): boolean {
const record = asRecord(value);
if (!record || isCanonicalEmbeddingItem(record)) return false;
if (!Array.isArray(record.parts) || record.parts.length === 0) return false;
return record.parts.every((part) => isGeminiNativePart(part));
}
export function isGeminiNativeEmbedRequest(value: unknown): boolean {
const record = asRecord(value);
if (!record || isCanonicalEmbeddingItem(record)) return false;
const content = record.content;
if (Array.isArray(content)) return false;
return isGeminiNativeContent(content);
}
export function isGeminiNativeEmbeddingItem(value: unknown): boolean {
return isGeminiNativePart(value) || isGeminiNativeContent(value) || isGeminiNativeEmbedRequest(value);
}
/**
* True when the request already uses Gemini's documented multimodal contract
* (a part, a Content with parts, or an EmbedContentRequest).
*/
export function isGeminiNativeEmbeddingInput(input: unknown): boolean {
if (isGeminiNativeEmbeddingItem(input)) return true;
if (!Array.isArray(input)) return false;
return input.some((item) => isGeminiNativeEmbeddingItem(item));
}
export function collectGeminiNativeModalities(input: unknown): GeminiEmbeddingModality[] {
const found = new Set<GeminiEmbeddingModality>();
const visitPart = (value: unknown) => {
const record = asRecord(value);
if (!record) return;
if (typeof record.text === "string" && record.text.trim().length > 0) found.add("text");
const inlineMime = mimeFromInline(record);
if (inlineMime) found.add(modalityFromGeminiMime(inlineMime));
const fileMime = mimeFromFile(record);
if (fileMime) found.add(modalityFromGeminiMime(fileMime));
};
const visit = (value: unknown) => {
if (isGeminiNativeEmbedRequest(value)) {
visit((value as { content: unknown }).content);
return;
}
if (isGeminiNativeContent(value)) {
for (const part of (value as { parts: unknown[] }).parts) visitPart(part);
return;
}
if (isGeminiNativePart(value)) visitPart(value);
};
if (Array.isArray(input)) {
for (const item of input) visit(item);
} else {
visit(input);
}
return [...found];
}

View File

@@ -0,0 +1,87 @@
/**
* Jina Search Foundation native embedding items (api.jina.ai EmbeddingsV5Request).
*
* Official 2026 input shapes (OpenAPI 2026.07.27):
* TextDoc { text }
* ImageDoc { image } URL or base64 / data URI
* AudioDoc { audio }
* VideoDoc { video }
* PDFDoc { pdf } single input only upstream; we still accept it in a list
* MergedContentGroup { content: [TextDoc|ImageDoc|AudioDoc|VideoDoc, ...] }
*
* These are not OmniRoute's canonical `{ type, source }` items. For jina-ai
* they must be forwarded intact — do not stringify, do not fetch image URLs
* into data URIs. Jina fetches public media itself.
*/
export const JINA_NATIVE_MEDIA_KEYS = ["text", "image", "audio", "video", "pdf"] as const;
export type JinaNativeMediaKey = (typeof JINA_NATIVE_MEDIA_KEYS)[number];
const NATIVE_KEY_TO_MODALITY: Record<JinaNativeMediaKey, "text" | "image" | "audio" | "video" | "document"> =
{
text: "text",
image: "image",
audio: "audio",
video: "video",
pdf: "document",
};
export function isPlainObject(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
/** OmniRoute canonical structured item — leave those on the translator path. */
export function isCanonicalEmbeddingItem(value: unknown): boolean {
return isPlainObject(value) && "type" in value && typeof value.type === "string";
}
export function isJinaNativeDoc(value: unknown): boolean {
if (!isPlainObject(value) || isCanonicalEmbeddingItem(value)) return false;
if ("content" in value && Array.isArray(value.content)) return false;
const present = JINA_NATIVE_MEDIA_KEYS.filter((key) => key in value);
if (present.length !== 1) return false;
return typeof value[present[0]] === "string" && String(value[present[0]]).trim().length > 0;
}
export function isJinaMergedContentGroup(value: unknown): boolean {
if (!isPlainObject(value) || isCanonicalEmbeddingItem(value)) return false;
if (!Array.isArray(value.content) || value.content.length === 0) return false;
return value.content.every((item) => isJinaNativeDoc(item) && !("pdf" in (item as object)));
}
export function isJinaNativeEmbeddingItem(value: unknown): boolean {
return isJinaNativeDoc(value) || isJinaMergedContentGroup(value);
}
/**
* True when the request already uses Jina's documented multimodal contract
* (single doc, mixed string+doc batch, or fused content groups).
*/
export function isJinaNativeEmbeddingInput(input: unknown): boolean {
if (isJinaNativeEmbeddingItem(input)) return true;
if (!Array.isArray(input)) return false;
return input.some((item) => isJinaNativeEmbeddingItem(item));
}
export function collectJinaNativeModalities(
input: unknown
): Array<"text" | "image" | "audio" | "video" | "document"> {
const found = new Set<"text" | "image" | "audio" | "video" | "document">();
const visit = (value: unknown) => {
if (isJinaMergedContentGroup(value)) {
for (const item of (value as { content: unknown[] }).content) visit(item);
return;
}
if (!isJinaNativeDoc(value)) return;
const key = JINA_NATIVE_MEDIA_KEYS.find((mediaKey) => mediaKey in (value as object));
if (key) found.add(NATIVE_KEY_TO_MODALITY[key]);
};
if (Array.isArray(input)) {
for (const item of input) visit(item);
} else {
visit(input);
}
return [...found];
}

View File

@@ -20,6 +20,11 @@ import {
} from "@/shared/reasoning/effortStandardization";
import { modelIdSchema, nonEmptyStringSchema } from "./misc.ts";
import {
isCanonicalEmbeddingItem,
JINA_NATIVE_MEDIA_KEYS,
} from "../jinaNativeEmbeddingInput.ts";
import { isGeminiNativeEmbeddingItem } from "../geminiNativeEmbeddingInput.ts";
export const embeddingTokenArraySchema = z
.array(z.number().int().min(0))
@@ -110,15 +115,260 @@ export const embeddingMultimodalItemSchema = z.discriminatedUnion("type", [
),
]);
function decodedInlineBytesFromEmbeddingItem(item: unknown): number {
if (!item || typeof item !== "object") return 0;
const record = item as Record<string, unknown>;
if (
"type" in record &&
record.type !== "text" &&
record.source &&
typeof record.source === "object"
) {
const source = record.source as { type?: string; data?: string };
if (source.type === "base64" && typeof source.data === "string") {
return decodedBase64Bytes(source.data);
}
}
for (const key of JINA_NATIVE_MEDIA_KEYS) {
if (key === "text" || typeof record[key] !== "string") continue;
const value = String(record[key]);
const dataUri = /^data:([^;,]+);base64,(.+)$/i.exec(value);
if (dataUri) return decodedBase64Bytes(dataUri[2]);
if (/^https:\/\//i.test(value)) return 0;
return decodedBase64Bytes(value);
}
if (Array.isArray(record.content)) {
return record.content.reduce(
(total, chunk) => total + decodedInlineBytesFromEmbeddingItem(chunk),
0
);
}
if (record.content && typeof record.content === "object" && !Array.isArray(record.content)) {
return decodedInlineBytesFromEmbeddingItem(record.content);
}
if (Array.isArray(record.parts)) {
return record.parts.reduce(
(total, chunk) => total + decodedInlineBytesFromEmbeddingItem(chunk),
0
);
}
const inline = record.inline_data ?? record.inlineData;
if (inline && typeof inline === "object") {
const data = (inline as { data?: unknown }).data;
if (typeof data === "string") return decodedBase64Bytes(data);
}
return 0;
}
const embeddingMultimodalInputSchema = z
.array(embeddingMultimodalItemSchema)
.min(1, "input must contain at least one item")
.max(MAX_EMBEDDING_INPUT_ITEMS, `input must contain at most ${MAX_EMBEDDING_INPUT_ITEMS} items`)
.superRefine((items, context) => {
const totalBytes = items.reduce((total, item) => {
if (item.type === "text" || item.source.type !== "base64") return total;
return total + decodedBase64Bytes(item.source.data);
}, 0);
const totalBytes = items.reduce(
(total, item) => total + decodedInlineBytesFromEmbeddingItem(item),
0
);
if (totalBytes > MAX_EMBEDDING_INLINE_TOTAL_BYTES) {
context.addIssue({
code: "custom",
message: "decoded inline media must not exceed 16 MiB per request",
});
}
});
function refineJinaMediaString(value: string, context: z.RefinementCtx) {
const trimmed = value.trim();
if (/^https:\/\//i.test(trimmed)) {
if (trimmed.length > MAX_EMBEDDING_URL_LENGTH) {
context.addIssue({ code: "custom", message: "media URL is too long" });
return;
}
try {
const url = parseAndValidatePublicUrl(trimmed);
if (url.protocol !== "https:") {
context.addIssue({ code: "custom", message: "media URLs must use HTTPS" });
}
} catch {
context.addIssue({ code: "custom", message: "media URL must be a safe public HTTPS URL" });
}
return;
}
if (/^(https?:|file:|data:text\/html)/i.test(trimmed) && !trimmed.startsWith("data:")) {
context.addIssue({ code: "custom", message: "media URL must be a safe public HTTPS URL" });
return;
}
const dataUri = /^data:([^;,]+);base64,(.+)$/i.exec(trimmed);
const payload = dataUri ? dataUri[2] : trimmed;
if (
payload.length > MAX_EMBEDDING_INLINE_ITEM_BASE64_LENGTH ||
decodedBase64Bytes(payload) > MAX_EMBEDDING_INLINE_ITEM_BYTES
) {
context.addIssue({
code: "custom",
message: "decoded inline media must not exceed 8 MiB",
});
}
}
const jinaNativeMediaStringSchema = z.string().trim().min(1).superRefine(refineJinaMediaString);
function exactlyOneJinaMediaKey(value: Record<string, unknown>, key: string): boolean {
if (isCanonicalEmbeddingItem(value)) return false;
return JINA_NATIVE_MEDIA_KEYS.filter((mediaKey) => mediaKey in value).length === 1 && key in value;
}
const jinaTextDocSchema = z
.object({ text: z.string().trim().min(1).max(MAX_EMBEDDING_TEXT_LENGTH) })
.passthrough()
.refine((value) => exactlyOneJinaMediaKey(value, "text"), {
message: "Jina TextDoc must be { text }",
});
const jinaImageDocSchema = z
.object({ image: jinaNativeMediaStringSchema })
.passthrough()
.refine((value) => exactlyOneJinaMediaKey(value, "image"), {
message: "Jina ImageDoc must be { image }",
});
const jinaAudioDocSchema = z
.object({ audio: jinaNativeMediaStringSchema })
.passthrough()
.refine((value) => exactlyOneJinaMediaKey(value, "audio"), {
message: "Jina AudioDoc must be { audio }",
});
const jinaVideoDocSchema = z
.object({ video: jinaNativeMediaStringSchema })
.passthrough()
.refine((value) => exactlyOneJinaMediaKey(value, "video"), {
message: "Jina VideoDoc must be { video }",
});
const jinaPdfDocSchema = z
.object({ pdf: jinaNativeMediaStringSchema })
.passthrough()
.refine((value) => exactlyOneJinaMediaKey(value, "pdf"), {
message: "Jina PDFDoc must be { pdf }",
});
export const jinaNativeDocSchema = z.union([
jinaTextDocSchema,
jinaImageDocSchema,
jinaAudioDocSchema,
jinaVideoDocSchema,
jinaPdfDocSchema,
]);
export const jinaMergedContentGroupSchema = z
.object({
content: z
.array(z.union([jinaTextDocSchema, jinaImageDocSchema, jinaAudioDocSchema, jinaVideoDocSchema]))
.min(1, "content must contain at least one chunk"),
})
.passthrough();
const geminiInlineBlobSchema = z
.object({
mime_type: z.string().trim().min(1).max(MAX_MEDIA_TYPE_LENGTH).optional(),
mimeType: z.string().trim().min(1).max(MAX_MEDIA_TYPE_LENGTH).optional(),
data: z.string().min(1),
})
.passthrough()
.superRefine((value, context) => {
if (!value.mime_type && !value.mimeType) {
context.addIssue({ code: "custom", message: "Gemini inline_data requires mime_type" });
}
const data = value.data;
if (
data.length > MAX_EMBEDDING_INLINE_ITEM_BASE64_LENGTH ||
decodedBase64Bytes(data) > MAX_EMBEDDING_INLINE_ITEM_BYTES
) {
context.addIssue({
code: "custom",
message: "decoded inline media must not exceed 8 MiB",
});
}
});
const geminiFileUriSchema = z
.string()
.trim()
.min(1)
.max(MAX_EMBEDDING_URL_LENGTH)
.superRefine((value, context) => {
if (value.startsWith("files/")) return;
try {
const url = parseAndValidatePublicUrl(value);
if (url.protocol !== "https:") {
context.addIssue({ code: "custom", message: "media URLs must use HTTPS" });
}
} catch {
context.addIssue({ code: "custom", message: "media URL must be a safe public HTTPS URL" });
}
});
const geminiFileDataSchema = z
.object({
mime_type: z.string().trim().min(1).max(MAX_MEDIA_TYPE_LENGTH).optional(),
mimeType: z.string().trim().min(1).max(MAX_MEDIA_TYPE_LENGTH).optional(),
file_uri: geminiFileUriSchema.optional(),
fileUri: geminiFileUriSchema.optional(),
})
.passthrough()
.refine((value) => Boolean(value.file_uri || value.fileUri), {
message: "Gemini file_data requires file_uri",
});
export const geminiNativePartSchema = z
.object({
text: z.string().trim().min(1).max(MAX_EMBEDDING_TEXT_LENGTH).optional(),
inline_data: geminiInlineBlobSchema.optional(),
inlineData: geminiInlineBlobSchema.optional(),
file_data: geminiFileDataSchema.optional(),
fileData: geminiFileDataSchema.optional(),
})
.passthrough()
.refine((value) => isGeminiNativeEmbeddingItem(value) && !("parts" in value) && !("content" in value), {
message: "Gemini part must be { text }, { inline_data }, or { file_data }",
});
export const geminiNativeContentSchema = z
.object({
parts: z.array(geminiNativePartSchema).min(1, "parts must contain at least one part"),
})
.passthrough();
export const geminiNativeEmbedRequestSchema = z
.object({
content: geminiNativeContentSchema,
})
.passthrough();
export const geminiNativeItemSchema = z.union([
geminiNativePartSchema,
geminiNativeContentSchema,
geminiNativeEmbedRequestSchema,
]);
const jinaNativeOrCanonicalArraySchema = z
.array(
z.union([
nonEmptyStringSchema,
embeddingMultimodalItemSchema,
jinaNativeDocSchema,
jinaMergedContentGroupSchema,
geminiNativeItemSchema,
])
)
.min(1, "input must contain at least one item")
.max(MAX_EMBEDDING_INPUT_ITEMS, `input must contain at most ${MAX_EMBEDDING_INPUT_ITEMS} items`)
.superRefine((items, context) => {
const totalBytes = items.reduce(
(total, item) => total + decodedInlineBytesFromEmbeddingItem(item),
0
);
if (totalBytes > MAX_EMBEDDING_INLINE_TOTAL_BYTES) {
context.addIssue({
code: "custom",
@@ -133,6 +383,10 @@ export const embeddingInputSchema = z.union([
embeddingTokenArraySchema,
z.array(embeddingTokenArraySchema).min(1, "input must contain at least one item"),
embeddingMultimodalInputSchema,
jinaNativeDocSchema,
jinaMergedContentGroupSchema,
geminiNativeItemSchema,
jinaNativeOrCanonicalArraySchema,
]);
export type EmbeddingMultimodalItem = z.infer<typeof embeddingMultimodalItemSchema>;
@@ -244,6 +498,30 @@ export const v1RerankSchema = z
})
.catchall(z.unknown());
// POST /v1/classify — Jina zero/few-shot classification (api.jina.ai).
export const v1ClassifySchema = z
.object({
model: modelIdSchema.optional(),
classifier_id: z.string().trim().min(1).optional(),
input: z.union([
nonEmptyStringSchema,
z.array(z.unknown()).min(1, "input must contain at least one item"),
]),
labels: z.array(z.string().trim().min(1)).min(1).optional(),
})
.catchall(z.unknown());
// POST /v1/segment — Jina segmenter (segment.jina.ai).
export const v1SegmentSchema = z
.object({
content: nonEmptyStringSchema,
tokenizer: z.string().trim().min(1).optional(),
return_tokens: z.boolean().optional(),
return_chunks: z.boolean().optional(),
max_chunk_length: z.coerce.number().positive().optional(),
})
.catchall(z.unknown());
export const providerChatCompletionSchema = z
.object({
model: modelIdSchema,
@@ -305,6 +583,9 @@ export const v1SearchSchema = z
"youcom-search",
"searxng-search",
"zai-search",
"jina-search",
"jina-ai",
"jina",
"duckduckgo-free",
])
.optional(),