diff --git a/changelog.d/fixes/10601-xai-800-message-limit.md b/changelog.d/fixes/10601-xai-800-message-limit.md new file mode 100644 index 0000000000..3dcd33ab8e --- /dev/null +++ b/changelog.d/fixes/10601-xai-800-message-limit.md @@ -0,0 +1 @@ +- **fix(xai):** trim Chat Completions `messages` and Responses `input` to xAI's 800-item history cap before dispatch, so long tool loops no longer die on `413 Chat history exceeds the 800-message limit` ([#10601](https://github.com/diegosouzapw/OmniRoute/pull/10601)) diff --git a/open-sse/executors/xai.ts b/open-sse/executors/xai.ts index de8fcf7425..3b5a5eef03 100644 --- a/open-sse/executors/xai.ts +++ b/open-sse/executors/xai.ts @@ -3,6 +3,7 @@ import { PROVIDERS } from "../config/constants.ts"; import { getModelTargetFormat } from "../config/providerModels.ts"; import { isResponsesEndpointPath } from "../utils/responsesEndpoint.ts"; import { chatRequestToXaiResponses } from "@/lib/providers/xai/translators/openai-chat.ts"; +import { capXaiRequestHistory } from "../services/xaiMessageCap.ts"; type JsonRecord = Record; @@ -157,7 +158,8 @@ export class XaiExecutor extends BaseExecutor { } // Keep model id from the routed request when the translator left it empty. if (out.model == null && model) out.model = model; - return out; + // After chat→Responses expansion, `input` is what xAI counts toward 800. + return capXaiRequestHistory(out); } let modelId = typeof out.model === "string" ? out.model : model; @@ -185,7 +187,7 @@ export class XaiExecutor extends BaseExecutor { if (effort) out.reasoning_effort = effort; } - return out; + return capXaiRequestHistory(out); } } diff --git a/open-sse/services/xaiMessageCap.ts b/open-sse/services/xaiMessageCap.ts new file mode 100644 index 0000000000..92886e68cd --- /dev/null +++ b/open-sse/services/xaiMessageCap.ts @@ -0,0 +1,129 @@ +/** + * xAI rejects a request with HTTP 413 when chat history exceeds 800 items: + * "Chat history exceeds the 800-message limit; compact the conversation and retry." + * + * Token-based compression does not catch this: a long agent loop of tiny + * tool calls still fits a 256k–500k window. Cap the arrays xAI actually + * counts — Chat Completions `messages` and Responses `input` — at the + * executor edge, after any chat→Responses expansion. + */ +import { + fixToolAdjacency, + fixToolPairs, + stripTrailingAssistantOrphanToolUse, +} from "./contextManager.ts"; + +export const XAI_CHAT_HISTORY_LIMIT = 800; + +type HistoryItem = Record; + +function isSystemRole(item: HistoryItem): boolean { + return item.role === "system" || item.role === "developer"; +} + +function repairChatMessages(messages: HistoryItem[]): HistoryItem[] { + let result = fixToolPairs(messages); + result = fixToolAdjacency(result); + result = fixToolPairs(result); + return stripTrailingAssistantOrphanToolUse(result); +} + +/** + * Keep system/developer messages plus the newest tail, then drop tool-call + * orphans created by the cut. If the repaired list is still over the limit + * (lots of system messages), take the newest `limit` items and repair again. + */ +export function capXaiChatMessages( + messages: HistoryItem[], + limit = XAI_CHAT_HISTORY_LIMIT +): HistoryItem[] { + if (!Array.isArray(messages) || messages.length <= limit) return messages; + + const system = messages.filter(isSystemRole); + const nonSystem = messages.filter((item) => !isSystemRole(item)); + const budget = Math.max(2, limit - system.length); + let result = repairChatMessages([...system, ...nonSystem.slice(-budget)]); + + if (result.length > limit) { + result = repairChatMessages(result.slice(-limit)); + } + return result; +} + +function lastUserIndex(items: HistoryItem[]): number { + for (let i = items.length - 1; i >= 0; i--) { + if (items[i].role === "user") return i; + } + return -1; +} + +/** + * Responses `input` expands one assistant+tools chat turn into many items + * (`function_call` + `function_call_output`). Drop orphans left by a tail cut: + * outputs whose call was dropped, and mid-history calls whose output was + * dropped. Trailing unmatched `function_call`s (the in-flight turn) stay. + */ +export function repairXaiResponsesInput(items: HistoryItem[]): HistoryItem[] { + const callIds = new Set(); + const outputIds = new Set(); + for (const item of items) { + if (typeof item.call_id !== "string") continue; + if (item.type === "function_call") callIds.add(item.call_id); + if (item.type === "function_call_output") outputIds.add(item.call_id); + } + + const lastUser = lastUserIndex(items); + return items.filter((item, idx) => { + if (item.type === "function_call_output") { + return typeof item.call_id === "string" && callIds.has(item.call_id); + } + if (item.type === "function_call") { + if (typeof item.call_id === "string" && outputIds.has(item.call_id)) return true; + return lastUser < 0 || idx > lastUser; + } + return true; + }); +} + +export function capXaiResponsesInput( + input: HistoryItem[], + limit = XAI_CHAT_HISTORY_LIMIT +): HistoryItem[] { + if (!Array.isArray(input) || input.length <= limit) return input; + + let result = repairXaiResponsesInput(input.slice(-limit)); + if (result.length > limit) { + result = repairXaiResponsesInput(result.slice(-limit)); + } + return result; +} + +/** + * Cap whichever history array the body is using. No-op (same object / + * same array refs) when already within the limit. + */ +export function capXaiRequestHistory( + body: Record +): Record { + if (!body || typeof body !== "object") return body; + + const next: Record = { ...body }; + let changed = false; + + if (Array.isArray(body.messages)) { + const messages = capXaiChatMessages(body.messages as HistoryItem[]); + if (messages !== body.messages) { + next.messages = messages; + changed = true; + } + } + if (Array.isArray(body.input)) { + const input = capXaiResponsesInput(body.input as HistoryItem[]); + if (input !== body.input) { + next.input = input; + changed = true; + } + } + + return changed ? next : body; +} diff --git a/tests/unit/xai-message-cap.test.ts b/tests/unit/xai-message-cap.test.ts new file mode 100644 index 0000000000..d40b4f7236 --- /dev/null +++ b/tests/unit/xai-message-cap.test.ts @@ -0,0 +1,194 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + XAI_CHAT_HISTORY_LIMIT, + capXaiChatMessages, + capXaiRequestHistory, + capXaiResponsesInput, + repairXaiResponsesInput, +} from "../../open-sse/services/xaiMessageCap.ts"; +import { XaiExecutor } from "../../open-sse/executors/xai.ts"; + +const credentials = { apiKey: "test-key" }; + +function chatTurn(i: number) { + return [ + { role: "user", content: `u${i}` }, + { role: "assistant", content: `a${i}` }, + ]; +} + +test("XAI_CHAT_HISTORY_LIMIT matches the upstream 413", () => { + assert.equal(XAI_CHAT_HISTORY_LIMIT, 800); +}); + +test("capXaiChatMessages is a no-op at or under the limit", () => { + const messages = [ + { role: "system", content: "sys" }, + ...Array.from({ length: 400 }, (_, i) => chatTurn(i)).flat(), + ]; + assert.equal(messages.length, 801); + const under = messages.slice(0, 800); + assert.equal(capXaiChatMessages(under), under); + assert.equal(capXaiChatMessages(under).length, 800); +}); + +test("capXaiChatMessages keeps system plus the newest tail", () => { + const messages = [ + { role: "system", content: "sys" }, + ...Array.from({ length: 450 }, (_, i) => chatTurn(i)).flat(), + ]; + assert.ok(messages.length > 800); + const capped = capXaiChatMessages(messages); + assert.ok(capped.length <= 800); + assert.equal(capped[0].role, "system"); + assert.equal(capped[0].content, "sys"); + assert.equal(capped[capped.length - 1].content, "a449"); + assert.equal(capped[capped.length - 2].content, "u449"); + assert.equal( + capped.some((m) => m.content === "u0"), + false + ); +}); + +test("capXaiChatMessages drops a tool_result whose tool_use was cut", () => { + const messages = [ + { role: "system", content: "sys" }, + { + role: "assistant", + content: null, + tool_calls: [{ id: "old", type: "function", function: { name: "search" } }], + }, + { role: "tool", tool_call_id: "old", content: "stale" }, + ...Array.from({ length: 420 }, (_, i) => chatTurn(i)).flat(), + { + role: "assistant", + content: null, + tool_calls: [{ id: "kept", type: "function", function: { name: "read" } }], + }, + { role: "tool", tool_call_id: "kept", content: "file" }, + { role: "user", content: "go" }, + ]; + const capped = capXaiChatMessages(messages); + assert.ok(capped.length <= 800); + const toolIds = capped.filter((m) => m.role === "tool").map((m) => m.tool_call_id); + assert.deepEqual(toolIds, ["kept"]); +}); + +test("repairXaiResponsesInput drops orphaned function_call_output", () => { + const items = [ + { type: "function_call_output", call_id: "missing", output: "nope" }, + { role: "user", content: [{ type: "input_text", text: "hi" }] }, + { type: "function_call", call_id: "ok", name: "read", arguments: "{}" }, + { type: "function_call_output", call_id: "ok", output: "file" }, + ]; + const repaired = repairXaiResponsesInput(items); + assert.equal( + repaired.some((item) => item.call_id === "missing"), + false + ); + assert.equal(repaired.length, 3); +}); + +test("repairXaiResponsesInput keeps a trailing in-flight function_call", () => { + const items = [ + { role: "user", content: [{ type: "input_text", text: "hi" }] }, + { type: "function_call", call_id: "pending", name: "read", arguments: "{}" }, + ]; + assert.deepEqual(repairXaiResponsesInput(items), items); +}); + +test("capXaiResponsesInput keeps the newest 800 items and repairs the cut", () => { + const input = []; + for (let i = 0; i < 500; i++) { + input.push({ role: "user", content: [{ type: "input_text", text: `u${i}` }] }); + input.push({ type: "function_call", call_id: `c${i}`, name: "t", arguments: "{}" }); + input.push({ type: "function_call_output", call_id: `c${i}`, output: `o${i}` }); + } + assert.ok(input.length > 800); + const capped = capXaiResponsesInput(input); + assert.ok(capped.length <= 800); + assert.equal(capped[capped.length - 1].output, "o499"); + const outputs = capped.filter((item) => item.type === "function_call_output"); + const calls = new Set( + capped.filter((item) => item.type === "function_call").map((item) => item.call_id) + ); + for (const item of outputs) { + assert.ok(calls.has(item.call_id), `output ${item.call_id} has no matching call`); + } +}); + +test("capXaiRequestHistory is a no-op when both arrays already fit", () => { + const body = { + model: "grok-4.3", + messages: [{ role: "user", content: "hi" }], + }; + assert.equal(capXaiRequestHistory(body), body); + assert.equal("input" in capXaiRequestHistory(body), false); +}); + +test("capXaiRequestHistory caps messages and input independently", () => { + const messages = Array.from({ length: 801 }, (_, i) => ({ + role: i % 2 === 0 ? "user" : "assistant", + content: String(i), + })); + const input = Array.from({ length: 801 }, (_, i) => ({ + role: "user", + content: [{ type: "input_text", text: String(i) }], + })); + const out = capXaiRequestHistory({ model: "grok-4.3", messages, input }); + assert.ok((out.messages as unknown[]).length <= 800); + assert.ok((out.input as unknown[]).length <= 800); + assert.equal((out.messages as { content: string }[]).at(-1)?.content, "800"); + assert.equal((out.input as { content: { text: string }[] }[]).at(-1)?.content[0].text, "800"); +}); + +test("XaiExecutor caps Chat Completions history before grok-4.3 leaves the executor", () => { + const executor = new XaiExecutor(); + const messages = [ + { role: "system", content: "sys" }, + ...Array.from({ length: 450 }, (_, i) => chatTurn(i)).flat(), + ]; + const out = executor.transformRequest( + "grok-4.3", + { model: "grok-4.3", messages }, + false, + credentials + ) as Record; + assert.ok(Array.isArray(out.messages)); + assert.ok((out.messages as unknown[]).length <= 800); + assert.equal((out.messages as { content: string }[])[0].content, "sys"); + assert.equal((out.messages as { content: string }[]).at(-1)?.content, "a449"); +}); + +test("XaiExecutor caps Responses input after expanding a long chat history", () => { + const executor = new XaiExecutor(); + const messages = []; + for (let i = 0; i < 300; i++) { + messages.push({ role: "user", content: `u${i}` }); + messages.push({ + role: "assistant", + content: null, + tool_calls: [ + { id: `c${i}a`, type: "function", function: { name: "a", arguments: "{}" } }, + { id: `c${i}b`, type: "function", function: { name: "b", arguments: "{}" } }, + ], + }); + messages.push({ role: "tool", tool_call_id: `c${i}a`, content: "ra" }); + messages.push({ role: "tool", tool_call_id: `c${i}b`, content: "rb" }); + } + // 300 turns × (1 user + 2 function_call + 2 function_call_output) = 1500 input items + const out = executor.transformRequest( + "grok-4.6", + { model: "grok-4.6", messages }, + false, + credentials + ) as Record; + assert.equal(out.messages, undefined); + assert.ok(Array.isArray(out.input)); + assert.ok((out.input as unknown[]).length <= 800); + const last = (out.input as { type?: string; call_id?: string }[]).at(-1); + assert.equal(last?.type, "function_call_output"); + assert.equal(last?.call_id, "c299b"); +});