diff --git a/open-sse/executors/index.ts b/open-sse/executors/index.ts index f6af45bb5e..f67c2f63e1 100644 --- a/open-sse/executors/index.ts +++ b/open-sse/executors/index.ts @@ -49,6 +49,7 @@ import { QwenWebExecutor } from "./qwen-web.ts"; import { KimiExecutor } from "./kimi.ts" import { TheOldLlmExecutor } from "./theoldllm.ts"; import { ChipotleExecutor } from "./chipotle.ts"; +import { LMArenaExecutor } from "./lmarena.ts"; const executors = { antigravity: new AntigravityExecutor(), @@ -140,6 +141,8 @@ const executors = { tllm: new TheOldLlmExecutor(), // Alias chipotle: new ChipotleExecutor(), pepper: new ChipotleExecutor(), // Alias + lmarena: new LMArenaExecutor(), + lma: new LMArenaExecutor(), // Alias }; const defaultCache = new Map(); @@ -198,3 +201,4 @@ export { InnerAiExecutor } from "./inner-ai.ts"; export { QwenWebExecutor } from "./qwen-web.ts"; export { TheOldLlmExecutor } from "./theoldllm.ts"; export { ChipotleExecutor } from "./chipotle.ts"; +export { LMArenaExecutor } from "./lmarena.ts"; diff --git a/open-sse/executors/lmarena.ts b/open-sse/executors/lmarena.ts new file mode 100644 index 0000000000..de4c18c3d0 --- /dev/null +++ b/open-sse/executors/lmarena.ts @@ -0,0 +1,414 @@ +/** + * LMArenaExecutor — LMArena Web Session Provider + * + * Routes requests through LMArena's web API using session credentials. + * LMArena is a model comparison platform with 100+ models (GPT, Claude, Gemini, Llama). + * + * API Structure: + * Endpoint: https://arena.ai/nextjs-api/stream + * Method: POST + * Content-Type: application/json + * Accept: text/event-stream + * + * Auth pipeline (per request): + * 1. Extract session cookie from credentials + * 2. Build request with model and messages + * 3. Make authenticated POST request to LMArena API + * 4. Handle SSE response stream with custom prefixes (a0:, ag:, a3:, ae:, ad:) + * + * SSE Format: + * a0: - Text content (concatenate) + * ag: - Thinking/reasoning content + * a2: - Heartbeat (ignore) + * a3: - Model error + * ae: - Platform error + * ad: - Done marker + */ +import { BaseExecutor, type ExecuteInput } from "./base.ts"; +import { sanitizeErrorMessage } from "../utils/error.ts"; + +const LMARENA_API_BASE = "https://arena.ai"; +const LMARENA_STREAM_URL = `${LMARENA_API_BASE}/nextjs-api/stream`; + +const LMARENA_USER_AGENT = + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"; + +function readLMArenaCookie(credentials: unknown): string { + if (!credentials || typeof credentials !== "object") return ""; + const c = credentials as Record; + const direct = typeof c.cookie === "string" ? c.cookie : ""; + if (direct.trim()) return direct; + const apiKey = typeof c.apiKey === "string" ? c.apiKey : ""; + if (apiKey.trim()) return apiKey; + const psd = c.providerSpecificData; + if (psd && typeof psd === "object") { + const nested = (psd as Record).cookie; + if (typeof nested === "string" && nested.trim()) return nested; + } + return ""; +} + +interface ArenaSSEEvent { + type: "text" | "thinking" | "error" | "done" | "heartbeat"; + content?: string; +} + +export function parseArenaSSE(line: string): ArenaSSEEvent | null { + if (line.startsWith("a0:")) { + try { + const content = JSON.parse(line.substring(3)); + return { type: "text", content: typeof content === "string" ? content : content.text || "" }; + } catch { + return null; + } + } else if (line.startsWith("ag:")) { + try { + const content = JSON.parse(line.substring(3)); + return { type: "thinking", content: typeof content === "string" ? content : content.thinking || "" }; + } catch { + return null; + } + } else if (line.startsWith("a3:") || line.startsWith("ae:")) { + try { + const content = JSON.parse(line.substring(3)); + return { type: "error", content: typeof content === "string" ? content : content.error || JSON.stringify(content) }; + } catch { + return { type: "error", content: line.substring(3) }; + } + } else if (line.startsWith("ad:")) { + return { type: "done" }; + } else if (line.startsWith("a2:")) { + return { type: "heartbeat" }; + } + return null; +} + +export class LMArenaExecutor extends BaseExecutor { + constructor(providerConfig = {}) { + super("lmarena", { format: "openai", ...providerConfig }); + } + + protected buildUrl(_model: string, _credentials: unknown): string { + return LMARENA_STREAM_URL; + } + + protected buildHeaders( + _model: string, + credentials: unknown, + _body: unknown + ): Record { + const cookie = readLMArenaCookie(credentials); + const headers: Record = { + "Content-Type": "application/json", + Accept: "text/event-stream", + "User-Agent": LMARENA_USER_AGENT, + Origin: LMARENA_API_BASE, + Referer: `${LMARENA_API_BASE}/`, + }; + + if (cookie) { + headers.Cookie = cookie; + } + + return headers; + } + + protected transformRequest(body: unknown, model: string): unknown { + const openaiBody = body as Record; + const messages = openaiBody.messages as Array<{ role: string; content: string }>; + + return { + messages: messages.map(m => ({ + role: m.role, + content: m.content, + })), + model, + stream: openaiBody.stream || false, + }; + } + + async execute(input: ExecuteInput): Promise { + const { model, body, stream, credentials, signal, log } = input; + + const cookie = readLMArenaCookie(credentials); + if (!cookie) { + return new Response( + JSON.stringify({ + error: { + message: "LMArena requires a session cookie. Please provide cookie in credentials.", + type: "authentication_error", + code: "missing_cookie", + }, + }), + { + status: 401, + headers: { "Content-Type": "application/json" }, + } + ); + } + + const url = this.buildUrl(model, credentials); + const headers = this.buildHeaders(model, credentials, body); + const transformedBody = this.transformRequest(body, model); + + log?.info?.("LMArenaExecutor", `Executing request for model: ${model}`); + + try { + const response = await fetch(url, { + method: "POST", + headers, + body: JSON.stringify(transformedBody), + signal, + }); + + if (!response.ok) { + const errorText = await response.text(); + let errorMessage = `LMArena API error: ${response.status}`; + try { + const errorJson = JSON.parse(errorText); + errorMessage = errorJson.error?.message || errorJson.message || errorMessage; + } catch { + errorMessage = errorText || errorMessage; + } + + return new Response( + JSON.stringify({ + error: { + message: sanitizeErrorMessage(errorMessage), + type: "api_error", + code: String(response.status), + }, + }), + { + status: response.status, + headers: { "Content-Type": "application/json" }, + } + ); + } + + if (stream) { + return this.handleStreamingResponse(response, model, log); + } else { + return this.handleNonStreamingResponse(response, model, log); + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + log?.error?.("LMArenaExecutor", `Request failed: ${message}`); + + return new Response( + JSON.stringify({ + error: { + message: sanitizeErrorMessage(message), + type: "network_error", + code: "request_failed", + }, + }), + { + status: 502, + headers: { "Content-Type": "application/json" }, + } + ); + } + } + + private async handleStreamingResponse( + response: Response, + model: string, + log?: ExecuteInput["log"] + ): Promise { + const reader = response.body?.getReader(); + if (!reader) { + throw new Error("No response body for streaming"); + } + + const decoder = new TextDecoder(); + let buffer = ""; + let fullText = ""; + let fullThinking = ""; + + const stream = new ReadableStream({ + async start(controller) { + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + + for (const line of lines) { + if (!line.trim()) continue; + + const sseLine = line.startsWith("data: ") ? line.substring(6) : line; + const event = parseArenaSSE(sseLine); + + if (!event) continue; + + if (event.type === "text" && event.content) { + fullText += event.content; + const chunk = { + id: `chatcmpl-${Date.now()}`, + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model, + choices: [ + { + index: 0, + delta: { content: event.content }, + finish_reason: null, + }, + ], + }; + controller.enqueue(`data: ${JSON.stringify(chunk)}\n\n`); + } else if (event.type === "thinking" && event.content) { + fullThinking += event.content; + } else if (event.type === "error") { + const errorChunk = { + id: `chatcmpl-${Date.now()}`, + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model, + choices: [ + { + index: 0, + delta: {}, + finish_reason: "stop", + }, + ], + error: { message: event.content }, + }; + controller.enqueue(`data: ${JSON.stringify(errorChunk)}\n\n`); + controller.close(); + return; + } else if (event.type === "done") { + const finalChunk = { + id: `chatcmpl-${Date.now()}`, + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model, + choices: [ + { + index: 0, + delta: {}, + finish_reason: "stop", + }, + ], + }; + controller.enqueue(`data: ${JSON.stringify(finalChunk)}\n\n`); + controller.enqueue("data: [DONE]\n\n"); + controller.close(); + return; + } + } + } + + const finalChunk = { + id: `chatcmpl-${Date.now()}`, + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model, + choices: [ + { + index: 0, + delta: {}, + finish_reason: "stop", + }, + ], + }; + controller.enqueue(`data: ${JSON.stringify(finalChunk)}\n\n`); + controller.enqueue("data: [DONE]\n\n"); + controller.close(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + log?.error?.("LMArenaExecutor", `Streaming error: ${message}`); + controller.error(error); + } + }, + }); + + return new Response(stream, { + status: 200, + headers: { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }, + }); + } + + private async handleNonStreamingResponse( + response: Response, + model: string, + log?: ExecuteInput["log"] + ): Promise { + const text = await response.text(); + const lines = text.split("\n"); + let fullText = ""; + let fullThinking = ""; + let error: string | null = null; + + for (const line of lines) { + if (!line.trim()) continue; + + const sseLine = line.startsWith("data: ") ? line.substring(6) : line; + const event = parseArenaSSE(sseLine); + + if (!event) continue; + + if (event.type === "text" && event.content) { + fullText += event.content; + } else if (event.type === "thinking" && event.content) { + fullThinking += event.content; + } else if (event.type === "error") { + error = event.content || "Unknown error"; + break; + } else if (event.type === "done") { + break; + } + } + + if (error) { + return new Response( + JSON.stringify({ + error: { + message: sanitizeErrorMessage(error), + type: "api_error", + code: "lmarena_error", + }, + }), + { + status: 502, + headers: { "Content-Type": "application/json" }, + } + ); + } + + const result = { + id: `chatcmpl-${Date.now()}`, + object: "chat.completion", + created: Math.floor(Date.now() / 1000), + model, + choices: [ + { + index: 0, + message: { + role: "assistant", + content: fullText, + }, + finish_reason: "stop", + }, + ], + usage: { + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + }, + }; + + return new Response(JSON.stringify(result), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } +} diff --git a/src/shared/constants/providers.ts b/src/shared/constants/providers.ts index de54566eee..b937becf80 100644 --- a/src/shared/constants/providers.ts +++ b/src/shared/constants/providers.ts @@ -470,6 +470,20 @@ export const WEB_COOKIE_PROVIDERS = { authHint: "Paste your __client cookie value from .clerk.agent.adapta.one (DevTools → Application → Cookies)", }, + lmarena: { + id: "lmarena", + alias: "lma", + name: "LMArena (Free)", + icon: "auto_awesome", + color: "#FF6B6B", + textIcon: "LMA", + website: "https://lmarena.ai", + hasFree: true, + freeNote: "Free model comparison platform — 40+ models (GPT, Claude, Gemini, Llama). No subscription required.", + authHint: + "Paste your session cookie from lmarena.ai (DevTools → Application → Cookies). Optional — works with free tier for basic comparisons.", + riskNoticeVariant: "webCookie", + }, huggingchat: { id: "huggingchat", // "hc" belongs to the hackclub provider; huggingchat uses its own id as alias. diff --git a/src/shared/providers/webSessionCredentials.ts b/src/shared/providers/webSessionCredentials.ts index bf4c4a190a..f3f159c554 100644 --- a/src/shared/providers/webSessionCredentials.ts +++ b/src/shared/providers/webSessionCredentials.ts @@ -192,6 +192,13 @@ export const WEB_SESSION_CREDENTIAL_REQUIREMENTS = { acceptsFullCookieHeader: true, storageKeys: ["cookie", "manus_session"], }, + lmarena: { + kind: "cookie", + credentialName: "session", + placeholder: "session=... or full Cookie header from lmarena.ai", + acceptsFullCookieHeader: true, + storageKeys: ["cookie", "session"], + }, } satisfies Record; export function getWebSessionCredentialRequirement( diff --git a/tests/unit/lmarena-provider.test.ts b/tests/unit/lmarena-provider.test.ts new file mode 100644 index 0000000000..72982bfda7 --- /dev/null +++ b/tests/unit/lmarena-provider.test.ts @@ -0,0 +1,280 @@ +/** + * LMArena Provider — Unit Tests (Phase 2A of issue #3368) + * + * Run: node --import tsx/esm --test tests/unit/lmarena-provider.test.ts + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { WEB_COOKIE_PROVIDERS } from "../../src/shared/constants/providers.ts"; +import { + getWebSessionCredentialRequirement, + requiresWebSessionCredential, + hasUsableWebSessionCredential, +} from "../../src/shared/providers/webSessionCredentials.ts"; +import { LMArenaExecutor, parseArenaSSE } from "../../open-sse/executors/lmarena.ts"; + +describe("LMArena Provider Definition", () => { + it("is registered in WEB_COOKIE_PROVIDERS", () => { + assert.ok(WEB_COOKIE_PROVIDERS.lmarena, "lmarena should be in WEB_COOKIE_PROVIDERS"); + assert.equal(WEB_COOKIE_PROVIDERS.lmarena.id, "lmarena"); + assert.equal(WEB_COOKIE_PROVIDERS.lmarena.alias, "lma"); + assert.equal(WEB_COOKIE_PROVIDERS.lmarena.name, "LMArena (Free)"); + assert.equal(WEB_COOKIE_PROVIDERS.lmarena.website, "https://lmarena.ai"); + assert.equal(WEB_COOKIE_PROVIDERS.lmarena.hasFree, true); + assert.equal(WEB_COOKIE_PROVIDERS.lmarena.riskNoticeVariant, "webCookie"); + }); + + it("has correct metadata", () => { + const provider = WEB_COOKIE_PROVIDERS.lmarena; + assert.ok(provider.freeNote, "Should have freeNote"); + assert.ok(provider.authHint, "Should have authHint"); + assert.ok(provider.icon, "Should have icon"); + assert.ok(provider.color, "Should have color"); + assert.ok(provider.textIcon, "Should have textIcon"); + }); +}); + +describe("LMArena Credential Requirements", () => { + it("requires web session credential", () => { + assert.equal(requiresWebSessionCredential("lmarena"), true); + }); + + it("has correct credential requirement", () => { + const req = getWebSessionCredentialRequirement("lmarena"); + assert.ok(req, "Should have credential requirement"); + assert.equal(req.kind, "cookie"); + assert.equal(req.credentialName, "session"); + assert.ok(req.placeholder.includes("lmarena.ai")); + assert.equal(req.acceptsFullCookieHeader, true); + assert.ok(req.storageKeys.includes("cookie")); + assert.ok(req.storageKeys.includes("session")); + }); + + it("validates usable credentials correctly", () => { + assert.equal( + hasUsableWebSessionCredential("lmarena", { cookie: "session=abc123" }), + true + ); + assert.equal( + hasUsableWebSessionCredential("lmarena", { session: "abc123" }), + true + ); + assert.equal( + hasUsableWebSessionCredential("lmarena", { cookie: "" }), + false + ); + assert.equal( + hasUsableWebSessionCredential("lmarena", {}), + false + ); + }); +}); + +describe("LMArena Executor", () => { + it("can be instantiated", () => { + const executor = new LMArenaExecutor(); + assert.ok(executor, "Executor should be instantiated"); + }); + + it("has correct provider ID", () => { + const executor = new LMArenaExecutor(); + assert.equal((executor as any).provider, "lmarena"); + }); + + it("builds correct URL (arena.ai/nextjs-api/stream)", () => { + const executor = new LMArenaExecutor(); + const url = (executor as any).buildUrl("gpt-4", {}); + assert.ok(url.includes("arena.ai"), "URL should include arena.ai"); + assert.ok(url.includes("/nextjs-api/stream"), "URL should include /nextjs-api/stream"); + }); + + it("builds headers with cookie", () => { + const executor = new LMArenaExecutor(); + const headers = (executor as any).buildHeaders("gpt-4", { cookie: "session=abc123" }, {}); + assert.ok(headers.Cookie, "Should have Cookie header"); + assert.equal(headers.Cookie, "session=abc123"); + assert.equal(headers["Content-Type"], "application/json"); + assert.equal(headers.Accept, "text/event-stream"); + }); + + it("builds headers without cookie when not provided", () => { + const executor = new LMArenaExecutor(); + const headers = (executor as any).buildHeaders("gpt-4", {}, {}); + assert.ok(!headers.Cookie, "Should not have Cookie header when no cookie provided"); + }); + + it("reads cookie from credentials correctly", () => { + const executor = new LMArenaExecutor(); + + // Direct cookie field + let headers = (executor as any).buildHeaders("gpt-4", { cookie: "session=abc" }, {}); + assert.equal(headers.Cookie, "session=abc"); + + // apiKey field (dashboard form) + headers = (executor as any).buildHeaders("gpt-4", { apiKey: "session=def" }, {}); + assert.equal(headers.Cookie, "session=def"); + + // providerSpecificData.cookie + headers = (executor as any).buildHeaders( + "gpt-4", + { providerSpecificData: { cookie: "session=ghi" } }, + {} + ); + assert.equal(headers.Cookie, "session=ghi"); + + // Priority: direct > apiKey > providerSpecificData + headers = (executor as any).buildHeaders( + "gpt-4", + { cookie: "session=abc", apiKey: "session=def" }, + {} + ); + assert.equal(headers.Cookie, "session=abc"); + }); + + it("parses LMArena SSE text events (a0: prefix)", () => { + const textEvent = 'a0:{"text":"Hello, world!"}'; + const result = parseArenaSSE(textEvent); + + assert.ok(result, "Should parse text event"); + assert.equal(result.type, "text"); + assert.equal(result.content, "Hello, world!"); + }); + + it("parses LMArena SSE thinking events (ag: prefix)", () => { + const thinkingEvent = 'ag:{"thinking":"Let me analyze this..."}'; + const result = parseArenaSSE(thinkingEvent); + + assert.ok(result, "Should parse thinking event"); + assert.equal(result.type, "thinking"); + assert.equal(result.content, "Let me analyze this..."); + }); + + it("parses LMArena SSE error events (a3: and ae: prefixes)", () => { + const errorEvent1 = 'a3:{"error":"Rate limit exceeded"}'; + const result1 = parseArenaSSE(errorEvent1); + assert.ok(result1, "Should parse a3: error event"); + assert.equal(result1.type, "error"); + assert.equal(result1.content, "Rate limit exceeded"); + + const errorEvent2 = 'ae:{"error":"Invalid session"}'; + const result2 = parseArenaSSE(errorEvent2); + assert.ok(result2, "Should parse ae: error event"); + assert.equal(result2.type, "error"); + assert.equal(result2.content, "Invalid session"); + }); + + it("parses LMArena SSE done event (ad: prefix)", () => { + const doneEvent = 'ad:{}'; + const result = parseArenaSSE(doneEvent); + + assert.ok(result, "Should parse done event"); + assert.equal(result.type, "done"); + }); + + it("handles malformed SSE events gracefully", () => { + const malformedEvent = 'invalid:data'; + const result = parseArenaSSE(malformedEvent); + + assert.equal(result, null, "Should return null for malformed events"); + }); + + it("transforms OpenAI messages to LMArena format", () => { + const executor = new LMArenaExecutor(); + const transformRequest = (executor as any).transformRequest.bind(executor); + + const openaiBody = { + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "Hello!" }, + { role: "assistant", content: "Hi there!" }, + { role: "user", content: "How are you?" } + ], + model: "gpt-4", + stream: true + }; + + const arenaBody = transformRequest(openaiBody, "gpt-4"); + + assert.ok(arenaBody, "Should transform request body"); + assert.ok(arenaBody.messages, "Should have messages array"); + assert.equal(arenaBody.model, "gpt-4", "Should preserve model"); + assert.equal(arenaBody.stream, true, "Should preserve stream flag"); + }); + + it("returns 401 when cookie is missing", async () => { + const executor = new LMArenaExecutor(); + + const response = await executor.execute({ + model: "gpt-4", + body: { messages: [{ role: "user", content: "Hello" }] }, + credentials: {}, + signal: new AbortController().signal, + log: console + }); + + assert.equal(response.status, 401, "Should return 401 for missing cookie"); + const errorBody = await response.json(); + assert.ok(errorBody.error, "Should have error object"); + assert.ok(errorBody.error.message.includes("cookie"), "Error should mention cookie"); + }); + + it("handles streaming response correctly", async () => { + const executor = new LMArenaExecutor(); + + const mockSSE = [ + 'data: a0:{"text":"Hello"}\n\n', + 'data: a0:{"text":", world!"}\n\n', + 'data: ad:{}\n\n' + ].join(''); + + const originalFetch = global.fetch; + global.fetch = async () => new Response(mockSSE, { + status: 200, + headers: { "Content-Type": "text/event-stream" } + }); + + try { + const response = await executor.execute({ + model: "gpt-4", + body: { messages: [{ role: "user", content: "Hello" }], stream: true }, + credentials: { cookie: "session=test" }, + signal: new AbortController().signal, + log: console + }); + + assert.equal(response.status, 200, "Should return 200 for successful streaming"); + assert.ok(response.body, "Should have response body for streaming"); + } finally { + global.fetch = originalFetch; + } + }); + + it("handles error response from LMArena API", async () => { + const executor = new LMArenaExecutor(); + + const originalFetch = global.fetch; + global.fetch = async () => new Response(JSON.stringify({ + error: { message: "Rate limit exceeded" } + }), { + status: 429, + headers: { "Content-Type": "application/json" } + }); + + try { + const response = await executor.execute({ + model: "gpt-4", + body: { messages: [{ role: "user", content: "Hello" }] }, + credentials: { cookie: "session=test" }, + signal: new AbortController().signal, + log: console + }); + + assert.equal(response.status, 429, "Should return 429 for rate limit"); + const errorBody = await response.json(); + assert.ok(errorBody.error, "Should have error object"); + } finally { + global.fetch = originalFetch; + } + }); +});