feat: add LMArena provider (Phase 2A of #3368) (#3421)

Integrated into release/v3.8.17
This commit is contained in:
Paijo
2026-06-09 04:43:10 +07:00
committed by GitHub
parent ea0c0d8499
commit 07a81c8a40
5 changed files with 719 additions and 0 deletions

View File

@@ -49,6 +49,7 @@ import { QwenWebExecutor } from "./qwen-web.ts";
import { KimiExecutor } from "./kimi.ts"
import { TheOldLlmExecutor } from "./theoldllm.ts";
import { ChipotleExecutor } from "./chipotle.ts";
import { LMArenaExecutor } from "./lmarena.ts";
const executors = {
antigravity: new AntigravityExecutor(),
@@ -140,6 +141,8 @@ const executors = {
tllm: new TheOldLlmExecutor(), // Alias
chipotle: new ChipotleExecutor(),
pepper: new ChipotleExecutor(), // Alias
lmarena: new LMArenaExecutor(),
lma: new LMArenaExecutor(), // Alias
};
const defaultCache = new Map();
@@ -198,3 +201,4 @@ export { InnerAiExecutor } from "./inner-ai.ts";
export { QwenWebExecutor } from "./qwen-web.ts";
export { TheOldLlmExecutor } from "./theoldllm.ts";
export { ChipotleExecutor } from "./chipotle.ts";
export { LMArenaExecutor } from "./lmarena.ts";

View File

@@ -0,0 +1,414 @@
/**
* LMArenaExecutor — LMArena Web Session Provider
*
* Routes requests through LMArena's web API using session credentials.
* LMArena is a model comparison platform with 100+ models (GPT, Claude, Gemini, Llama).
*
* API Structure:
* Endpoint: https://arena.ai/nextjs-api/stream
* Method: POST
* Content-Type: application/json
* Accept: text/event-stream
*
* Auth pipeline (per request):
* 1. Extract session cookie from credentials
* 2. Build request with model and messages
* 3. Make authenticated POST request to LMArena API
* 4. Handle SSE response stream with custom prefixes (a0:, ag:, a3:, ae:, ad:)
*
* SSE Format:
* a0: - Text content (concatenate)
* ag: - Thinking/reasoning content
* a2: - Heartbeat (ignore)
* a3: - Model error
* ae: - Platform error
* ad: - Done marker
*/
import { BaseExecutor, type ExecuteInput } from "./base.ts";
import { sanitizeErrorMessage } from "../utils/error.ts";
const LMARENA_API_BASE = "https://arena.ai";
const LMARENA_STREAM_URL = `${LMARENA_API_BASE}/nextjs-api/stream`;
const LMARENA_USER_AGENT =
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
function readLMArenaCookie(credentials: unknown): string {
if (!credentials || typeof credentials !== "object") return "";
const c = credentials as Record<string, unknown>;
const direct = typeof c.cookie === "string" ? c.cookie : "";
if (direct.trim()) return direct;
const apiKey = typeof c.apiKey === "string" ? c.apiKey : "";
if (apiKey.trim()) return apiKey;
const psd = c.providerSpecificData;
if (psd && typeof psd === "object") {
const nested = (psd as Record<string, unknown>).cookie;
if (typeof nested === "string" && nested.trim()) return nested;
}
return "";
}
interface ArenaSSEEvent {
type: "text" | "thinking" | "error" | "done" | "heartbeat";
content?: string;
}
export function parseArenaSSE(line: string): ArenaSSEEvent | null {
if (line.startsWith("a0:")) {
try {
const content = JSON.parse(line.substring(3));
return { type: "text", content: typeof content === "string" ? content : content.text || "" };
} catch {
return null;
}
} else if (line.startsWith("ag:")) {
try {
const content = JSON.parse(line.substring(3));
return { type: "thinking", content: typeof content === "string" ? content : content.thinking || "" };
} catch {
return null;
}
} else if (line.startsWith("a3:") || line.startsWith("ae:")) {
try {
const content = JSON.parse(line.substring(3));
return { type: "error", content: typeof content === "string" ? content : content.error || JSON.stringify(content) };
} catch {
return { type: "error", content: line.substring(3) };
}
} else if (line.startsWith("ad:")) {
return { type: "done" };
} else if (line.startsWith("a2:")) {
return { type: "heartbeat" };
}
return null;
}
export class LMArenaExecutor extends BaseExecutor {
constructor(providerConfig = {}) {
super("lmarena", { format: "openai", ...providerConfig });
}
protected buildUrl(_model: string, _credentials: unknown): string {
return LMARENA_STREAM_URL;
}
protected buildHeaders(
_model: string,
credentials: unknown,
_body: unknown
): Record<string, string> {
const cookie = readLMArenaCookie(credentials);
const headers: Record<string, string> = {
"Content-Type": "application/json",
Accept: "text/event-stream",
"User-Agent": LMARENA_USER_AGENT,
Origin: LMARENA_API_BASE,
Referer: `${LMARENA_API_BASE}/`,
};
if (cookie) {
headers.Cookie = cookie;
}
return headers;
}
protected transformRequest(body: unknown, model: string): unknown {
const openaiBody = body as Record<string, unknown>;
const messages = openaiBody.messages as Array<{ role: string; content: string }>;
return {
messages: messages.map(m => ({
role: m.role,
content: m.content,
})),
model,
stream: openaiBody.stream || false,
};
}
async execute(input: ExecuteInput): Promise<Response> {
const { model, body, stream, credentials, signal, log } = input;
const cookie = readLMArenaCookie(credentials);
if (!cookie) {
return new Response(
JSON.stringify({
error: {
message: "LMArena requires a session cookie. Please provide cookie in credentials.",
type: "authentication_error",
code: "missing_cookie",
},
}),
{
status: 401,
headers: { "Content-Type": "application/json" },
}
);
}
const url = this.buildUrl(model, credentials);
const headers = this.buildHeaders(model, credentials, body);
const transformedBody = this.transformRequest(body, model);
log?.info?.("LMArenaExecutor", `Executing request for model: ${model}`);
try {
const response = await fetch(url, {
method: "POST",
headers,
body: JSON.stringify(transformedBody),
signal,
});
if (!response.ok) {
const errorText = await response.text();
let errorMessage = `LMArena API error: ${response.status}`;
try {
const errorJson = JSON.parse(errorText);
errorMessage = errorJson.error?.message || errorJson.message || errorMessage;
} catch {
errorMessage = errorText || errorMessage;
}
return new Response(
JSON.stringify({
error: {
message: sanitizeErrorMessage(errorMessage),
type: "api_error",
code: String(response.status),
},
}),
{
status: response.status,
headers: { "Content-Type": "application/json" },
}
);
}
if (stream) {
return this.handleStreamingResponse(response, model, log);
} else {
return this.handleNonStreamingResponse(response, model, log);
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
log?.error?.("LMArenaExecutor", `Request failed: ${message}`);
return new Response(
JSON.stringify({
error: {
message: sanitizeErrorMessage(message),
type: "network_error",
code: "request_failed",
},
}),
{
status: 502,
headers: { "Content-Type": "application/json" },
}
);
}
}
private async handleStreamingResponse(
response: Response,
model: string,
log?: ExecuteInput["log"]
): Promise<Response> {
const reader = response.body?.getReader();
if (!reader) {
throw new Error("No response body for streaming");
}
const decoder = new TextDecoder();
let buffer = "";
let fullText = "";
let fullThinking = "";
const stream = new ReadableStream({
async start(controller) {
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
buffer += decoder.decode(value, { stream: true });
const lines = buffer.split("\n");
buffer = lines.pop() || "";
for (const line of lines) {
if (!line.trim()) continue;
const sseLine = line.startsWith("data: ") ? line.substring(6) : line;
const event = parseArenaSSE(sseLine);
if (!event) continue;
if (event.type === "text" && event.content) {
fullText += event.content;
const chunk = {
id: `chatcmpl-${Date.now()}`,
object: "chat.completion.chunk",
created: Math.floor(Date.now() / 1000),
model,
choices: [
{
index: 0,
delta: { content: event.content },
finish_reason: null,
},
],
};
controller.enqueue(`data: ${JSON.stringify(chunk)}\n\n`);
} else if (event.type === "thinking" && event.content) {
fullThinking += event.content;
} else if (event.type === "error") {
const errorChunk = {
id: `chatcmpl-${Date.now()}`,
object: "chat.completion.chunk",
created: Math.floor(Date.now() / 1000),
model,
choices: [
{
index: 0,
delta: {},
finish_reason: "stop",
},
],
error: { message: event.content },
};
controller.enqueue(`data: ${JSON.stringify(errorChunk)}\n\n`);
controller.close();
return;
} else if (event.type === "done") {
const finalChunk = {
id: `chatcmpl-${Date.now()}`,
object: "chat.completion.chunk",
created: Math.floor(Date.now() / 1000),
model,
choices: [
{
index: 0,
delta: {},
finish_reason: "stop",
},
],
};
controller.enqueue(`data: ${JSON.stringify(finalChunk)}\n\n`);
controller.enqueue("data: [DONE]\n\n");
controller.close();
return;
}
}
}
const finalChunk = {
id: `chatcmpl-${Date.now()}`,
object: "chat.completion.chunk",
created: Math.floor(Date.now() / 1000),
model,
choices: [
{
index: 0,
delta: {},
finish_reason: "stop",
},
],
};
controller.enqueue(`data: ${JSON.stringify(finalChunk)}\n\n`);
controller.enqueue("data: [DONE]\n\n");
controller.close();
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
log?.error?.("LMArenaExecutor", `Streaming error: ${message}`);
controller.error(error);
}
},
});
return new Response(stream, {
status: 200,
headers: {
"Content-Type": "text/event-stream",
"Cache-Control": "no-cache",
Connection: "keep-alive",
},
});
}
private async handleNonStreamingResponse(
response: Response,
model: string,
log?: ExecuteInput["log"]
): Promise<Response> {
const text = await response.text();
const lines = text.split("\n");
let fullText = "";
let fullThinking = "";
let error: string | null = null;
for (const line of lines) {
if (!line.trim()) continue;
const sseLine = line.startsWith("data: ") ? line.substring(6) : line;
const event = parseArenaSSE(sseLine);
if (!event) continue;
if (event.type === "text" && event.content) {
fullText += event.content;
} else if (event.type === "thinking" && event.content) {
fullThinking += event.content;
} else if (event.type === "error") {
error = event.content || "Unknown error";
break;
} else if (event.type === "done") {
break;
}
}
if (error) {
return new Response(
JSON.stringify({
error: {
message: sanitizeErrorMessage(error),
type: "api_error",
code: "lmarena_error",
},
}),
{
status: 502,
headers: { "Content-Type": "application/json" },
}
);
}
const result = {
id: `chatcmpl-${Date.now()}`,
object: "chat.completion",
created: Math.floor(Date.now() / 1000),
model,
choices: [
{
index: 0,
message: {
role: "assistant",
content: fullText,
},
finish_reason: "stop",
},
],
usage: {
prompt_tokens: 0,
completion_tokens: 0,
total_tokens: 0,
},
};
return new Response(JSON.stringify(result), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
}

View File

@@ -470,6 +470,20 @@ export const WEB_COOKIE_PROVIDERS = {
authHint:
"Paste your __client cookie value from .clerk.agent.adapta.one (DevTools → Application → Cookies)",
},
lmarena: {
id: "lmarena",
alias: "lma",
name: "LMArena (Free)",
icon: "auto_awesome",
color: "#FF6B6B",
textIcon: "LMA",
website: "https://lmarena.ai",
hasFree: true,
freeNote: "Free model comparison platform — 40+ models (GPT, Claude, Gemini, Llama). No subscription required.",
authHint:
"Paste your session cookie from lmarena.ai (DevTools → Application → Cookies). Optional — works with free tier for basic comparisons.",
riskNoticeVariant: "webCookie",
},
huggingchat: {
id: "huggingchat",
// "hc" belongs to the hackclub provider; huggingchat uses its own id as alias.

View File

@@ -192,6 +192,13 @@ export const WEB_SESSION_CREDENTIAL_REQUIREMENTS = {
acceptsFullCookieHeader: true,
storageKeys: ["cookie", "manus_session"],
},
lmarena: {
kind: "cookie",
credentialName: "session",
placeholder: "session=... or full Cookie header from lmarena.ai",
acceptsFullCookieHeader: true,
storageKeys: ["cookie", "session"],
},
} satisfies Record<keyof typeof WEB_COOKIE_PROVIDERS, WebSessionCredentialRequirement>;
export function getWebSessionCredentialRequirement(

View File

@@ -0,0 +1,280 @@
/**
* LMArena Provider — Unit Tests (Phase 2A of issue #3368)
*
* Run: node --import tsx/esm --test tests/unit/lmarena-provider.test.ts
*/
import { describe, it } from "node:test";
import assert from "node:assert/strict";
import { WEB_COOKIE_PROVIDERS } from "../../src/shared/constants/providers.ts";
import {
getWebSessionCredentialRequirement,
requiresWebSessionCredential,
hasUsableWebSessionCredential,
} from "../../src/shared/providers/webSessionCredentials.ts";
import { LMArenaExecutor, parseArenaSSE } from "../../open-sse/executors/lmarena.ts";
describe("LMArena Provider Definition", () => {
it("is registered in WEB_COOKIE_PROVIDERS", () => {
assert.ok(WEB_COOKIE_PROVIDERS.lmarena, "lmarena should be in WEB_COOKIE_PROVIDERS");
assert.equal(WEB_COOKIE_PROVIDERS.lmarena.id, "lmarena");
assert.equal(WEB_COOKIE_PROVIDERS.lmarena.alias, "lma");
assert.equal(WEB_COOKIE_PROVIDERS.lmarena.name, "LMArena (Free)");
assert.equal(WEB_COOKIE_PROVIDERS.lmarena.website, "https://lmarena.ai");
assert.equal(WEB_COOKIE_PROVIDERS.lmarena.hasFree, true);
assert.equal(WEB_COOKIE_PROVIDERS.lmarena.riskNoticeVariant, "webCookie");
});
it("has correct metadata", () => {
const provider = WEB_COOKIE_PROVIDERS.lmarena;
assert.ok(provider.freeNote, "Should have freeNote");
assert.ok(provider.authHint, "Should have authHint");
assert.ok(provider.icon, "Should have icon");
assert.ok(provider.color, "Should have color");
assert.ok(provider.textIcon, "Should have textIcon");
});
});
describe("LMArena Credential Requirements", () => {
it("requires web session credential", () => {
assert.equal(requiresWebSessionCredential("lmarena"), true);
});
it("has correct credential requirement", () => {
const req = getWebSessionCredentialRequirement("lmarena");
assert.ok(req, "Should have credential requirement");
assert.equal(req.kind, "cookie");
assert.equal(req.credentialName, "session");
assert.ok(req.placeholder.includes("lmarena.ai"));
assert.equal(req.acceptsFullCookieHeader, true);
assert.ok(req.storageKeys.includes("cookie"));
assert.ok(req.storageKeys.includes("session"));
});
it("validates usable credentials correctly", () => {
assert.equal(
hasUsableWebSessionCredential("lmarena", { cookie: "session=abc123" }),
true
);
assert.equal(
hasUsableWebSessionCredential("lmarena", { session: "abc123" }),
true
);
assert.equal(
hasUsableWebSessionCredential("lmarena", { cookie: "" }),
false
);
assert.equal(
hasUsableWebSessionCredential("lmarena", {}),
false
);
});
});
describe("LMArena Executor", () => {
it("can be instantiated", () => {
const executor = new LMArenaExecutor();
assert.ok(executor, "Executor should be instantiated");
});
it("has correct provider ID", () => {
const executor = new LMArenaExecutor();
assert.equal((executor as any).provider, "lmarena");
});
it("builds correct URL (arena.ai/nextjs-api/stream)", () => {
const executor = new LMArenaExecutor();
const url = (executor as any).buildUrl("gpt-4", {});
assert.ok(url.includes("arena.ai"), "URL should include arena.ai");
assert.ok(url.includes("/nextjs-api/stream"), "URL should include /nextjs-api/stream");
});
it("builds headers with cookie", () => {
const executor = new LMArenaExecutor();
const headers = (executor as any).buildHeaders("gpt-4", { cookie: "session=abc123" }, {});
assert.ok(headers.Cookie, "Should have Cookie header");
assert.equal(headers.Cookie, "session=abc123");
assert.equal(headers["Content-Type"], "application/json");
assert.equal(headers.Accept, "text/event-stream");
});
it("builds headers without cookie when not provided", () => {
const executor = new LMArenaExecutor();
const headers = (executor as any).buildHeaders("gpt-4", {}, {});
assert.ok(!headers.Cookie, "Should not have Cookie header when no cookie provided");
});
it("reads cookie from credentials correctly", () => {
const executor = new LMArenaExecutor();
// Direct cookie field
let headers = (executor as any).buildHeaders("gpt-4", { cookie: "session=abc" }, {});
assert.equal(headers.Cookie, "session=abc");
// apiKey field (dashboard form)
headers = (executor as any).buildHeaders("gpt-4", { apiKey: "session=def" }, {});
assert.equal(headers.Cookie, "session=def");
// providerSpecificData.cookie
headers = (executor as any).buildHeaders(
"gpt-4",
{ providerSpecificData: { cookie: "session=ghi" } },
{}
);
assert.equal(headers.Cookie, "session=ghi");
// Priority: direct > apiKey > providerSpecificData
headers = (executor as any).buildHeaders(
"gpt-4",
{ cookie: "session=abc", apiKey: "session=def" },
{}
);
assert.equal(headers.Cookie, "session=abc");
});
it("parses LMArena SSE text events (a0: prefix)", () => {
const textEvent = 'a0:{"text":"Hello, world!"}';
const result = parseArenaSSE(textEvent);
assert.ok(result, "Should parse text event");
assert.equal(result.type, "text");
assert.equal(result.content, "Hello, world!");
});
it("parses LMArena SSE thinking events (ag: prefix)", () => {
const thinkingEvent = 'ag:{"thinking":"Let me analyze this..."}';
const result = parseArenaSSE(thinkingEvent);
assert.ok(result, "Should parse thinking event");
assert.equal(result.type, "thinking");
assert.equal(result.content, "Let me analyze this...");
});
it("parses LMArena SSE error events (a3: and ae: prefixes)", () => {
const errorEvent1 = 'a3:{"error":"Rate limit exceeded"}';
const result1 = parseArenaSSE(errorEvent1);
assert.ok(result1, "Should parse a3: error event");
assert.equal(result1.type, "error");
assert.equal(result1.content, "Rate limit exceeded");
const errorEvent2 = 'ae:{"error":"Invalid session"}';
const result2 = parseArenaSSE(errorEvent2);
assert.ok(result2, "Should parse ae: error event");
assert.equal(result2.type, "error");
assert.equal(result2.content, "Invalid session");
});
it("parses LMArena SSE done event (ad: prefix)", () => {
const doneEvent = 'ad:{}';
const result = parseArenaSSE(doneEvent);
assert.ok(result, "Should parse done event");
assert.equal(result.type, "done");
});
it("handles malformed SSE events gracefully", () => {
const malformedEvent = 'invalid:data';
const result = parseArenaSSE(malformedEvent);
assert.equal(result, null, "Should return null for malformed events");
});
it("transforms OpenAI messages to LMArena format", () => {
const executor = new LMArenaExecutor();
const transformRequest = (executor as any).transformRequest.bind(executor);
const openaiBody = {
messages: [
{ role: "system", content: "You are a helpful assistant." },
{ role: "user", content: "Hello!" },
{ role: "assistant", content: "Hi there!" },
{ role: "user", content: "How are you?" }
],
model: "gpt-4",
stream: true
};
const arenaBody = transformRequest(openaiBody, "gpt-4");
assert.ok(arenaBody, "Should transform request body");
assert.ok(arenaBody.messages, "Should have messages array");
assert.equal(arenaBody.model, "gpt-4", "Should preserve model");
assert.equal(arenaBody.stream, true, "Should preserve stream flag");
});
it("returns 401 when cookie is missing", async () => {
const executor = new LMArenaExecutor();
const response = await executor.execute({
model: "gpt-4",
body: { messages: [{ role: "user", content: "Hello" }] },
credentials: {},
signal: new AbortController().signal,
log: console
});
assert.equal(response.status, 401, "Should return 401 for missing cookie");
const errorBody = await response.json();
assert.ok(errorBody.error, "Should have error object");
assert.ok(errorBody.error.message.includes("cookie"), "Error should mention cookie");
});
it("handles streaming response correctly", async () => {
const executor = new LMArenaExecutor();
const mockSSE = [
'data: a0:{"text":"Hello"}\n\n',
'data: a0:{"text":", world!"}\n\n',
'data: ad:{}\n\n'
].join('');
const originalFetch = global.fetch;
global.fetch = async () => new Response(mockSSE, {
status: 200,
headers: { "Content-Type": "text/event-stream" }
});
try {
const response = await executor.execute({
model: "gpt-4",
body: { messages: [{ role: "user", content: "Hello" }], stream: true },
credentials: { cookie: "session=test" },
signal: new AbortController().signal,
log: console
});
assert.equal(response.status, 200, "Should return 200 for successful streaming");
assert.ok(response.body, "Should have response body for streaming");
} finally {
global.fetch = originalFetch;
}
});
it("handles error response from LMArena API", async () => {
const executor = new LMArenaExecutor();
const originalFetch = global.fetch;
global.fetch = async () => new Response(JSON.stringify({
error: { message: "Rate limit exceeded" }
}), {
status: 429,
headers: { "Content-Type": "application/json" }
});
try {
const response = await executor.execute({
model: "gpt-4",
body: { messages: [{ role: "user", content: "Hello" }] },
credentials: { cookie: "session=test" },
signal: new AbortController().signal,
log: console
});
assert.equal(response.status, 429, "Should return 429 for rate limit");
const errorBody = await response.json();
assert.ok(errorBody.error, "Should have error object");
} finally {
global.fetch = originalFetch;
}
});
});