From 0c3498e724ef6e68c2bfeb330e92695b29f0edc9 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Thu, 2 Jul 2026 22:29:37 -0300 Subject: [PATCH] refactor(executors): extract pure Meta AI response parser from muse-spark-web (#6017) Extract the pure Meta AI SSE/JSON response parsing + content/reasoning/error extraction (parseMetaSseFrames, readMetaJsonPayloads, collect*/extract*/classify* helpers, parseMetaAiResponseText, isRecord, the reasoning/renderer key arrays, MetaSseFrame/ ParsedMetaAiResponse types) verbatim into the leaf muse-spark-web/response-parser.ts. Host imports back the 3 it uses; all module-private (no re-export). Host 1301 -> 925 LOC. Byte-identical bodies (verbatim), leaf has zero imports (no cycle). Conversation cache, cookie/auth, fetch, executor class untouched. Adds a split-guard; consumer tests stay green (muse-spark-cookie-copy-5449 2, muse-spark-web-continuation 6). --- open-sse/executors/muse-spark-web.ts | 383 +----------------- .../muse-spark-web/response-parser.ts | 383 ++++++++++++++++++ .../muse-spark-response-parser-split.test.ts | 33 ++ 3 files changed, 421 insertions(+), 378 deletions(-) create mode 100644 open-sse/executors/muse-spark-web/response-parser.ts create mode 100644 tests/unit/muse-spark-response-parser-split.test.ts diff --git a/open-sse/executors/muse-spark-web.ts b/open-sse/executors/muse-spark-web.ts index 6a7cac1f1d..106c28fb9d 100644 --- a/open-sse/executors/muse-spark-web.ts +++ b/open-sse/executors/muse-spark-web.ts @@ -13,6 +13,11 @@ import { normalizeSessionCookieHeader, normalizeSessionCookieHeaders, } from "@/lib/providers/webCookieAuth"; +import { + type ParsedMetaAiResponse, + isRecord, + parseMetaAiResponseText, +} from "./muse-spark-web/response-parser.ts"; const META_AI_GRAPHQL_API = "https://www.meta.ai/api/graphql"; // Meta rebranded the chat product from "Abra" to "Ecto"; the session cookie @@ -48,25 +53,6 @@ const MODEL_MAP: Record = { "muse-spark-contemplating": { mode: "think_hard", isThinking: true }, }; -type MetaSseFrame = { - event: string; - data: string; -}; - -type ParsedMetaAiResponse = { - content: string; - deltas: string[]; - reasoningContent: string; - reasoningDeltas: string[]; - errorCode: string | null; - errorMessage: string | null; - status: number; -}; - -function isRecord(value: unknown): value is Record { - return !!value && typeof value === "object" && !Array.isArray(value); -} - function extractMessageText(content: unknown): string { if (typeof content === "string") { return content.trim(); @@ -387,365 +373,6 @@ function buildMetaAiRequestBody(prompt: string, model: string, conversation: Con }; } -function parseMetaSseFrames(text: string): MetaSseFrame[] { - const frames: MetaSseFrame[] = []; - const lines = text.split(/\r?\n/); - let currentEvent = "message"; - let dataLines: string[] = []; - - const flush = () => { - if (dataLines.length === 0 && currentEvent === "message") { - return; - } - - frames.push({ - event: currentEvent, - data: dataLines.join("\n").trim(), - }); - - currentEvent = "message"; - dataLines = []; - }; - - for (const line of lines) { - if (!line) { - flush(); - continue; - } - - if (line.startsWith(":")) { - continue; - } - - if (line.startsWith("event:")) { - currentEvent = line.slice("event:".length).trim() || "message"; - continue; - } - - if (line.startsWith("data:")) { - dataLines.push(line.slice("data:".length).trimStart()); - } - } - - flush(); - return frames; -} - -function readMetaJsonPayloads(text: string): Array> { - const trimmed = text.trim(); - if (trimmed.startsWith("{")) { - try { - const parsed = JSON.parse(trimmed); - return isRecord(parsed) ? [parsed] : []; - } catch { - return []; - } - } - - return parseMetaSseFrames(text) - .filter((frame) => frame.data) - .map((frame) => { - try { - const parsed = JSON.parse(frame.data); - return isRecord(parsed) ? parsed : null; - } catch { - return null; - } - }) - .filter((frame): frame is Record => !!frame); -} - -const META_AI_REASONING_KEYS = [ - "reasoning", - "reasoningContent", - "reasoning_content", - "reasoningText", - "thinking", - "thinkingContent", - "thinkingText", - "thought", - "thoughtText", - "thoughts", - "internalThoughts", - "chainOfThought", - "thinkingTrace", - "thinking_trace", -] as const; - -const META_AI_NESTED_RENDERER_KEYS = [ - "contentRenderer", - "textContent", - "message", - "mediaContent", - "unified_response", - "unifiedResponseContent", - "sections", - "view_model", - "primitive", - "primitives", - "nested_responses", -] as const; - -function collectRendererTexts(value: unknown, seen: Set, depth = 0): string[] { - if (depth > 8) { - return []; - } - - if (typeof value === "string") { - const normalized = value.trim(); - if (!normalized || seen.has(normalized)) { - return []; - } - seen.add(normalized); - return [normalized]; - } - - if (Array.isArray(value)) { - return value.flatMap((item) => collectRendererTexts(item, seen, depth + 1)); - } - - if (!isRecord(value)) { - return []; - } - - const parts: string[] = []; - if (typeof value.text === "string") { - parts.push(...collectRendererTexts(value.text, seen, depth + 1)); - } - - for (const key of [ - "contentRenderer", - "textContent", - "message", - "mediaContent", - "unified_response", - "unifiedResponseContent", - "sections", - "view_model", - "primitive", - "primitives", - "nested_responses", - ]) { - if (key in value) { - parts.push(...collectRendererTexts(value[key], seen, depth + 1)); - } - } - - return parts; -} - -function collectReasoningTexts( - value: unknown, - seen: Set, - depth = 0, - force = false -): string[] { - if (depth > 8) { - return []; - } - - if (typeof value === "string") { - const normalized = value.trim(); - if (!force || !normalized || seen.has(normalized)) { - return []; - } - seen.add(normalized); - return [normalized]; - } - - if (Array.isArray(value)) { - return value.flatMap((item) => collectReasoningTexts(item, seen, depth + 1, force)); - } - - if (!isRecord(value)) { - return []; - } - - const typename = typeof value.__typename === "string" ? value.__typename : ""; - const localForce = force || /reasoning|thinking|thought/i.test(typename); - const parts: string[] = []; - - if (typeof value.text === "string" && localForce) { - parts.push(...collectReasoningTexts(value.text, seen, depth + 1, true)); - } - - for (const key of META_AI_REASONING_KEYS) { - if (key in value) { - parts.push(...collectReasoningTexts(value[key], seen, depth + 1, true)); - } - } - - for (const key of META_AI_NESTED_RENDERER_KEYS) { - if (key in value) { - parts.push(...collectReasoningTexts(value[key], seen, depth + 1, localForce)); - } - } - - return parts; -} - -function extractAssistantContent(message: Record): string { - if (typeof message.content === "string" && message.content.length > 0) { - return message.content; - } - - const contentRenderer = isRecord(message.contentRenderer) ? message.contentRenderer : null; - if (!contentRenderer) { - return ""; - } - - const parts = collectRendererTexts(contentRenderer, new Set()); - return parts.join("\n\n").trim(); -} - -function extractAssistantReasoning(message: Record): string { - const parts = collectReasoningTexts(message, new Set()); - return parts.join("\n\n").trim(); -} - -function extractAssistantError(message: Record) { - const error = isRecord(message.error) ? message.error : null; - const streamingState = - typeof message.streamingState === "string" ? message.streamingState.toUpperCase() : null; - return { - code: typeof error?.code === "string" ? error.code : null, - message: - typeof error?.message === "string" - ? error.message.trim() - : streamingState === "ERROR" && - typeof message.content === "string" && - message.content.trim() - ? message.content.trim() - : null, - }; -} - -function classifyMetaAiError(errorMessage: string | null, content: string) { - const combined = `${errorMessage || ""}\n${content}`.trim(); - if (!combined) { - return null; - } - - if (/authentication required to send messages|login is required|sign in/i.test(combined)) { - return { - status: 401, - message: "Meta AI auth failed — your meta.ai ecto_1_sess cookie may be missing or expired.", - }; - } - - if (/limit exceeded|rate limit|too many requests/i.test(combined)) { - return { - status: 429, - message: "Meta AI rate limited the session. Wait a moment and retry.", - }; - } - - if (/blocked by our security system|security system/i.test(combined)) { - return { - status: 403, - message: - "Meta AI blocked the request through its web security checks. Refresh the session cookie and retry.", - }; - } - - return null; -} - -function parseMetaAiResponseText(text: string, isThinkingModel: boolean): ParsedMetaAiResponse { - let lastContent = ""; - const deltas: string[] = []; - let lastReasoning = ""; - const reasoningDeltas: string[] = []; - let errorCode: string | null = null; - let errorMessage: string | null = null; - - for (const payload of readMetaJsonPayloads(text)) { - if (Array.isArray(payload.errors) && payload.errors.length > 0) { - const firstError = payload.errors.find( - (item) => isRecord(item) && typeof item.message === "string" - ); - if (isRecord(firstError) && typeof firstError.message === "string") { - errorMessage = firstError.message.trim(); - } - } - - const data = isRecord(payload.data) ? payload.data : null; - const sendMessageStream = isRecord(data?.sendMessageStream) ? data?.sendMessageStream : null; - if (!sendMessageStream || sendMessageStream.__typename !== "AssistantMessage") { - continue; - } - - const content = extractAssistantContent(sendMessageStream); - if (content && content !== lastContent) { - deltas.push(content.startsWith(lastContent) ? content.slice(lastContent.length) : content); - lastContent = content; - } - - if (isThinkingModel) { - const reasoning = extractAssistantReasoning(sendMessageStream); - if (reasoning && reasoning !== content && reasoning !== lastReasoning) { - reasoningDeltas.push( - reasoning.startsWith(lastReasoning) ? reasoning.slice(lastReasoning.length) : reasoning - ); - lastReasoning = reasoning; - } - } - - const upstreamError = extractAssistantError(sendMessageStream); - if (upstreamError.message) { - errorMessage = upstreamError.message; - errorCode = upstreamError.code; - } - } - - const classifiedError = classifyMetaAiError(errorMessage, lastContent); - if (classifiedError) { - return { - content: lastContent, - deltas, - reasoningContent: lastReasoning, - reasoningDeltas, - errorCode, - errorMessage: classifiedError.message, - status: classifiedError.status, - }; - } - - if (errorMessage) { - return { - content: lastContent, - deltas, - reasoningContent: lastReasoning, - reasoningDeltas, - errorCode, - errorMessage: `Meta AI returned an error: ${errorMessage}`, - status: 502, - }; - } - - if (!lastContent) { - return { - content: "", - deltas: [], - reasoningContent: lastReasoning, - reasoningDeltas, - errorCode: null, - errorMessage: "Meta AI returned no assistant content", - status: 502, - }; - } - - return { - content: lastContent, - deltas: deltas.filter((delta) => delta.length > 0), - reasoningContent: lastReasoning, - reasoningDeltas: reasoningDeltas.filter((delta) => delta.length > 0), - errorCode: null, - errorMessage: null, - status: 200, - }; -} - function sseChunk(data: unknown): string { return `data: ${JSON.stringify(data)}\n\n`; } diff --git a/open-sse/executors/muse-spark-web/response-parser.ts b/open-sse/executors/muse-spark-web/response-parser.ts new file mode 100644 index 0000000000..2e80a10447 --- /dev/null +++ b/open-sse/executors/muse-spark-web/response-parser.ts @@ -0,0 +1,383 @@ +// Pure Meta AI SSE/JSON response parsing + content/reasoning/error extraction. +// Extracted verbatim from muse-spark-web.ts. No host state/fetch/auth. + +export type MetaSseFrame = { + event: string; + data: string; +}; + +export type ParsedMetaAiResponse = { + content: string; + deltas: string[]; + reasoningContent: string; + reasoningDeltas: string[]; + errorCode: string | null; + errorMessage: string | null; + status: number; +}; + +export function isRecord(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value); +} + +export function parseMetaSseFrames(text: string): MetaSseFrame[] { + const frames: MetaSseFrame[] = []; + const lines = text.split(/\r?\n/); + let currentEvent = "message"; + let dataLines: string[] = []; + + const flush = () => { + if (dataLines.length === 0 && currentEvent === "message") { + return; + } + + frames.push({ + event: currentEvent, + data: dataLines.join("\n").trim(), + }); + + currentEvent = "message"; + dataLines = []; + }; + + for (const line of lines) { + if (!line) { + flush(); + continue; + } + + if (line.startsWith(":")) { + continue; + } + + if (line.startsWith("event:")) { + currentEvent = line.slice("event:".length).trim() || "message"; + continue; + } + + if (line.startsWith("data:")) { + dataLines.push(line.slice("data:".length).trimStart()); + } + } + + flush(); + return frames; +} + +export function readMetaJsonPayloads(text: string): Array> { + const trimmed = text.trim(); + if (trimmed.startsWith("{")) { + try { + const parsed = JSON.parse(trimmed); + return isRecord(parsed) ? [parsed] : []; + } catch { + return []; + } + } + + return parseMetaSseFrames(text) + .filter((frame) => frame.data) + .map((frame) => { + try { + const parsed = JSON.parse(frame.data); + return isRecord(parsed) ? parsed : null; + } catch { + return null; + } + }) + .filter((frame): frame is Record => !!frame); +} + +export const META_AI_REASONING_KEYS = [ + "reasoning", + "reasoningContent", + "reasoning_content", + "reasoningText", + "thinking", + "thinkingContent", + "thinkingText", + "thought", + "thoughtText", + "thoughts", + "internalThoughts", + "chainOfThought", + "thinkingTrace", + "thinking_trace", +] as const; + +export const META_AI_NESTED_RENDERER_KEYS = [ + "contentRenderer", + "textContent", + "message", + "mediaContent", + "unified_response", + "unifiedResponseContent", + "sections", + "view_model", + "primitive", + "primitives", + "nested_responses", +] as const; + +export function collectRendererTexts(value: unknown, seen: Set, depth = 0): string[] { + if (depth > 8) { + return []; + } + + if (typeof value === "string") { + const normalized = value.trim(); + if (!normalized || seen.has(normalized)) { + return []; + } + seen.add(normalized); + return [normalized]; + } + + if (Array.isArray(value)) { + return value.flatMap((item) => collectRendererTexts(item, seen, depth + 1)); + } + + if (!isRecord(value)) { + return []; + } + + const parts: string[] = []; + if (typeof value.text === "string") { + parts.push(...collectRendererTexts(value.text, seen, depth + 1)); + } + + for (const key of [ + "contentRenderer", + "textContent", + "message", + "mediaContent", + "unified_response", + "unifiedResponseContent", + "sections", + "view_model", + "primitive", + "primitives", + "nested_responses", + ]) { + if (key in value) { + parts.push(...collectRendererTexts(value[key], seen, depth + 1)); + } + } + + return parts; +} + +export function collectReasoningTexts( + value: unknown, + seen: Set, + depth = 0, + force = false +): string[] { + if (depth > 8) { + return []; + } + + if (typeof value === "string") { + const normalized = value.trim(); + if (!force || !normalized || seen.has(normalized)) { + return []; + } + seen.add(normalized); + return [normalized]; + } + + if (Array.isArray(value)) { + return value.flatMap((item) => collectReasoningTexts(item, seen, depth + 1, force)); + } + + if (!isRecord(value)) { + return []; + } + + const typename = typeof value.__typename === "string" ? value.__typename : ""; + const localForce = force || /reasoning|thinking|thought/i.test(typename); + const parts: string[] = []; + + if (typeof value.text === "string" && localForce) { + parts.push(...collectReasoningTexts(value.text, seen, depth + 1, true)); + } + + for (const key of META_AI_REASONING_KEYS) { + if (key in value) { + parts.push(...collectReasoningTexts(value[key], seen, depth + 1, true)); + } + } + + for (const key of META_AI_NESTED_RENDERER_KEYS) { + if (key in value) { + parts.push(...collectReasoningTexts(value[key], seen, depth + 1, localForce)); + } + } + + return parts; +} + +export function extractAssistantContent(message: Record): string { + if (typeof message.content === "string" && message.content.length > 0) { + return message.content; + } + + const contentRenderer = isRecord(message.contentRenderer) ? message.contentRenderer : null; + if (!contentRenderer) { + return ""; + } + + const parts = collectRendererTexts(contentRenderer, new Set()); + return parts.join("\n\n").trim(); +} + +export function extractAssistantReasoning(message: Record): string { + const parts = collectReasoningTexts(message, new Set()); + return parts.join("\n\n").trim(); +} + +export function extractAssistantError(message: Record) { + const error = isRecord(message.error) ? message.error : null; + const streamingState = + typeof message.streamingState === "string" ? message.streamingState.toUpperCase() : null; + return { + code: typeof error?.code === "string" ? error.code : null, + message: + typeof error?.message === "string" + ? error.message.trim() + : streamingState === "ERROR" && + typeof message.content === "string" && + message.content.trim() + ? message.content.trim() + : null, + }; +} + +export function classifyMetaAiError(errorMessage: string | null, content: string) { + const combined = `${errorMessage || ""}\n${content}`.trim(); + if (!combined) { + return null; + } + + if (/authentication required to send messages|login is required|sign in/i.test(combined)) { + return { + status: 401, + message: "Meta AI auth failed — your meta.ai ecto_1_sess cookie may be missing or expired.", + }; + } + + if (/limit exceeded|rate limit|too many requests/i.test(combined)) { + return { + status: 429, + message: "Meta AI rate limited the session. Wait a moment and retry.", + }; + } + + if (/blocked by our security system|security system/i.test(combined)) { + return { + status: 403, + message: + "Meta AI blocked the request through its web security checks. Refresh the session cookie and retry.", + }; + } + + return null; +} + +export function parseMetaAiResponseText( + text: string, + isThinkingModel: boolean +): ParsedMetaAiResponse { + let lastContent = ""; + const deltas: string[] = []; + let lastReasoning = ""; + const reasoningDeltas: string[] = []; + let errorCode: string | null = null; + let errorMessage: string | null = null; + + for (const payload of readMetaJsonPayloads(text)) { + if (Array.isArray(payload.errors) && payload.errors.length > 0) { + const firstError = payload.errors.find( + (item) => isRecord(item) && typeof item.message === "string" + ); + if (isRecord(firstError) && typeof firstError.message === "string") { + errorMessage = firstError.message.trim(); + } + } + + const data = isRecord(payload.data) ? payload.data : null; + const sendMessageStream = isRecord(data?.sendMessageStream) ? data?.sendMessageStream : null; + if (!sendMessageStream || sendMessageStream.__typename !== "AssistantMessage") { + continue; + } + + const content = extractAssistantContent(sendMessageStream); + if (content && content !== lastContent) { + deltas.push(content.startsWith(lastContent) ? content.slice(lastContent.length) : content); + lastContent = content; + } + + if (isThinkingModel) { + const reasoning = extractAssistantReasoning(sendMessageStream); + if (reasoning && reasoning !== content && reasoning !== lastReasoning) { + reasoningDeltas.push( + reasoning.startsWith(lastReasoning) ? reasoning.slice(lastReasoning.length) : reasoning + ); + lastReasoning = reasoning; + } + } + + const upstreamError = extractAssistantError(sendMessageStream); + if (upstreamError.message) { + errorMessage = upstreamError.message; + errorCode = upstreamError.code; + } + } + + const classifiedError = classifyMetaAiError(errorMessage, lastContent); + if (classifiedError) { + return { + content: lastContent, + deltas, + reasoningContent: lastReasoning, + reasoningDeltas, + errorCode, + errorMessage: classifiedError.message, + status: classifiedError.status, + }; + } + + if (errorMessage) { + return { + content: lastContent, + deltas, + reasoningContent: lastReasoning, + reasoningDeltas, + errorCode, + errorMessage: `Meta AI returned an error: ${errorMessage}`, + status: 502, + }; + } + + if (!lastContent) { + return { + content: "", + deltas: [], + reasoningContent: lastReasoning, + reasoningDeltas, + errorCode: null, + errorMessage: "Meta AI returned no assistant content", + status: 502, + }; + } + + return { + content: lastContent, + deltas: deltas.filter((delta) => delta.length > 0), + reasoningContent: lastReasoning, + reasoningDeltas: reasoningDeltas.filter((delta) => delta.length > 0), + errorCode: null, + errorMessage: null, + status: 200, + }; +} diff --git a/tests/unit/muse-spark-response-parser-split.test.ts b/tests/unit/muse-spark-response-parser-split.test.ts new file mode 100644 index 0000000000..719dcff33e --- /dev/null +++ b/tests/unit/muse-spark-response-parser-split.test.ts @@ -0,0 +1,33 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; + +// Split-guard for the muse-spark-web Meta AI response-parser extraction. +// The pure SSE/JSON parsing + content/reasoning/error extraction lives in +// muse-spark-web/response-parser.ts (no host state/fetch/auth). Host imports it back. +const HERE = dirname(fileURLToPath(import.meta.url)); +const EXE = join(HERE, "../../open-sse/executors"); +const HOST = join(EXE, "muse-spark-web.ts"); +const LEAF = join(EXE, "muse-spark-web/response-parser.ts"); + +test("leaf hosts the parser and does not import the host", () => { + const src = readFileSync(LEAF, "utf8"); + for (const sym of ["parseMetaAiResponseText", "parseMetaSseFrames", "isRecord"]) { + assert.match(src, new RegExp(`export function ${sym}\\b`)); + } + assert.doesNotMatch(src, /from "\.\.\/muse-spark-web\.ts"/); +}); + +test("host imports the parser back from the leaf", () => { + const host = readFileSync(HOST, "utf8"); + assert.match(host, /from "\.\/muse-spark-web\/response-parser\.ts"/); +}); + +test("parseMetaAiResponseText tolerates empty input", async () => { + const { parseMetaAiResponseText } = + await import("../../open-sse/executors/muse-spark-web/response-parser.ts"); + const out = parseMetaAiResponseText("", false); + assert.equal(typeof out, "object"); +});