diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b450a87be..18a93318d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ _Living section — bullets land here as PRs merge into `release/v3.8.47` (paral - **fix(cli):** compression CLI REST fallback now reads/writes the canonical `defaultMode` field (surfaced as `strategy`) instead of a nonexistent `engine` key, and table output renders nested objects as JSON instead of `[object Object]` (#6571 — thanks @charleszolot) - **fix(providers):** web-cookie providers without a `providerRegistry.ts` entry (`lmarena`, `gemini-business`, `poe-web`, `venice-web`, `v0-vercel-web`) now report `unsupported: true` instead of silently "OK" ([#6309](https://github.com/diegosouzapw/OmniRoute/pull/6309)) — `validateWebCookieProvider()` (`src/lib/providers/validation.ts`) previously required a registry entry and returned "Provider not found in registry" for these; a fallback to `WEB_COOKIE_PROVIDERS[provider].website` was proposed, but live verification showed the `${website}/models` probe does not reliably signal session validity for these providers (redirects/SPA 200s regardless of cookie validity — e.g. lmarena's real API is `arena.ai`, not `lmarena.ai`; Poe's real endpoint is a GraphQL POST, not a REST `/models`), so it would report an expired or garbage cookie as valid. Until each provider has a verified, side-effect-free auth probe against its real API host, the fallback now returns `unsupported` (no network call) instead of a false positive. Regression guard: `tests/unit/web-cookie-validation-fallback.test.ts`. (thanks @oyi77) - **fix(api):** `POST /api/middleware/hooks` and `PUT /api/middleware/hooks/[name]` no longer leak raw internal error messages in their 500 responses (#6645 — thanks @chirag127) — both catch blocks returned `error?.message` directly (Hard Rule #12), which could surface internal SQLite path fragments on a DB failure; both now route through `sanitizeErrorMessage()` from `open-sse/utils/error.ts`. Regression guard: `tests/unit/middleware-hooks-error-sanitization.test.ts`. +- **fix(providers):** ChatGPT Web (`chatgpt-web`) responses rendered raw ChatGPT UI citation markup — private-use marker tokens (e.g. `citeturn0search0`) and `url…` inline-link markers — instead of real Markdown links, since these only ever get resolved client-side by chatgpt.com's own JS using `message.metadata.content_references` ([#6635](https://github.com/diegosouzapw/OmniRoute/pull/6635)) — `cleanChatGptText()` now resolves `content_references` (grouped webpages, footnote sources, inline `webpage`/`url` mentions) into `[label](url)` Markdown links for both the streaming and non-streaming response builders, and for the GPT-5.5 Pro `stream_handoff` polled-answer path, falling back to stripping any marker that has no resolvable source instead of leaking the raw private-use bytes. The citation parsing/rendering logic was extracted into a new pure sibling module (`open-sse/executors/chatgpt-web/citations.ts`) to keep the executor under the frozen file-size cap. Regression guard: `tests/unit/chatgpt-web-citations.test.ts` (non-streaming citation resolution, streaming marker buffering across split SSE chunks, and the Pro-handoff polled-answer path). (thanks @Thinkscape) ### 📝 Maintenance diff --git a/config/quality/eslint-suppressions.json b/config/quality/eslint-suppressions.json index 1481b37aa9..46aaffb606 100644 --- a/config/quality/eslint-suppressions.json +++ b/config/quality/eslint-suppressions.json @@ -737,11 +737,6 @@ "count": 6 } }, - "tests/unit/chatgpt-web.test.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 6 - } - }, "tests/unit/chipotle-executor.test.ts": { "@typescript-eslint/no-explicit-any": { "count": 3 diff --git a/open-sse/executors/chatgpt-web.ts b/open-sse/executors/chatgpt-web.ts index 17125d4fcb..ed4480857a 100644 --- a/open-sse/executors/chatgpt-web.ts +++ b/open-sse/executors/chatgpt-web.ts @@ -33,6 +33,7 @@ import { type ChatGptImageConversationContext, } from "../services/chatgptImageCache.ts"; import { isThinkingCapableModel, resolveChatGptModel } from "./chatgpt-web/models.ts"; +import { cleanChatGptText } from "./chatgpt-web/citations.ts"; // ─── Constants ────────────────────────────────────────────────────────────── @@ -1165,6 +1166,7 @@ interface ContentChunk { answer?: string; conversationId?: string; messageId?: string; + metadata?: Record; error?: string; done?: boolean; /** Image asset pointers seen on the current message (e.g. file-service://file-abc). */ @@ -1226,6 +1228,7 @@ async function* extractContent( let conversationId: string | null = null; let currentId: string | null = null; let currentParts = ""; + let currentMetadata: Record | undefined; let emittedLen = 0; let isLive = false; // Dedupe pointers across echoes / repeated events. Order-preserving Set. @@ -1270,7 +1273,8 @@ async function* extractContent( // on a tool-role message (handled below). if (event.type === "server_ste_metadata") { const meta = (event as Record).metadata as - Record | undefined; + | Record + | undefined; if (meta && meta.turn_use_case === "image gen") { imageGenAsync = true; } @@ -1295,10 +1299,15 @@ async function* extractContent( if (id && id !== currentId) { currentId = id; currentParts = ""; + currentMetadata = undefined; emittedLen = 0; isLive = false; } + if (m.metadata && typeof m.metadata === "object") { + currentMetadata = m.metadata; + } + if (status === "in_progress") { isLive = true; } @@ -1337,6 +1346,7 @@ async function* extractContent( answer: currentParts, conversationId: conversationId ?? undefined, messageId: currentId ?? undefined, + metadata: currentMetadata, }; } } @@ -1350,6 +1360,7 @@ async function* extractContent( answer: currentParts, conversationId: conversationId ?? undefined, messageId: currentId ?? undefined, + metadata: currentMetadata, }; } @@ -1358,6 +1369,7 @@ async function* extractContent( answer: currentParts, conversationId: conversationId ?? undefined, messageId: currentId ?? undefined, + metadata: currentMetadata, imagePointers: imagePointers.size > 0 ? Array.from(imagePointers.values()) : undefined, imageGenAsync, handoff, @@ -1389,6 +1401,7 @@ interface ChatGptConversationDetail { interface FinalAssistantAnswer { text: string; messageId?: string; + metadata?: Record; finished: boolean; } @@ -1433,12 +1446,17 @@ function extractFinalAssistantAnswer( (finished && (!best.finished || sort >= best.sort)) || (!finished && !best.finished && sort >= best.sort) ) { - best = { text, messageId: message.id, finished, sort }; + best = { text, messageId: message.id, metadata: message.metadata, finished, sort }; } } if (!best) return null; - return { text: best.text, messageId: best.messageId, finished: best.finished }; + return { + text: best.text, + messageId: best.messageId, + metadata: best.metadata, + finished: best.finished, + }; } function delayWithAbort(ms: number, signal?: AbortSignal | null): Promise { @@ -1587,10 +1605,7 @@ type ImageResolver = ( * "no image was produced". Escalated mesh report: image visible in the ChatGPT * chat but returned to OmniRoute as a bare "completed without image markdown". */ -export function detectImageResolutionFailure( - pointerCount: number, - resolvedCount: number -): boolean { +export function detectImageResolutionFailure(pointerCount: number, resolvedCount: number): boolean { return pointerCount > 0 && resolvedCount === 0; } @@ -1674,13 +1689,12 @@ function buildStreamingResponse( let imageGenAsync = false; let handoff = false; let emittedText = ""; - let polledFinalAnswer = ""; + let polledFinalAnswer: FinalAssistantAnswer | null = null; let parentCandidateMessageId: string | null = null; - const emitTextDelta = (content: string): void => { - const cleaned = cleanChatGptText(content); - if (!cleaned) return; - emittedText += cleaned; + const emitRenderedDelta = (content: string): void => { + if (!content) return; + emittedText += content; controller.enqueue( encoder.encode( sseChunk({ @@ -1692,7 +1706,7 @@ function buildStreamingResponse( choices: [ { index: 0, - delta: { content: cleaned }, + delta: { content }, finish_reason: null, logprobs: null, }, @@ -1702,14 +1716,29 @@ function buildStreamingResponse( ); }; - const appendFinalAnswer = (text: string): void => { - const cleaned = cleanChatGptText(text); + const emitRenderedAnswer = ( + rawText: string, + metadata?: Record + ): void => { + const rendered = cleanChatGptText(rawText, metadata); + if (!rendered || rendered.length <= emittedText.length) return; + if (!rendered.startsWith(emittedText)) { + // We cannot retract bytes already streamed. This should be rare; + // it mainly protects clients if ChatGPT rewrites earlier text. + const common = commonPrefixLength(rendered, emittedText); + if (common < emittedText.length) return; + } + emitRenderedDelta(rendered.slice(emittedText.length)); + }; + + const appendFinalAnswer = (text: string, metadata?: Record): void => { + const cleaned = cleanChatGptText(text, metadata); const finalTrimmed = cleaned.trim(); if (!finalTrimmed) return; const emittedTrimmed = emittedText.trim(); if (emittedTrimmed === finalTrimmed || emittedTrimmed.endsWith(finalTrimmed)) return; const prefix = emittedTrimmed && !emittedText.endsWith("\n") ? "\n\n" : ""; - emitTextDelta(`${prefix}${cleaned}`); + emitRenderedDelta(`${prefix}${cleaned}`); }; // Heartbeat: long async work (Pro polling, WebSocket image-gen, @@ -1776,8 +1805,8 @@ function buildStreamingResponse( break; } - if (chunk.delta) { - emitTextDelta(chunk.delta); + if (chunk.answer) { + emitRenderedAnswer(chunk.answer, chunk.metadata); } } @@ -1786,7 +1815,7 @@ function buildStreamingResponse( try { const polled = await pollFinalAnswer(conversationId); if (polled?.text) { - polledFinalAnswer = polled.text; + polledFinalAnswer = polled; if (polled.messageId) parentCandidateMessageId = polled.messageId; } } finally { @@ -1795,7 +1824,7 @@ function buildStreamingResponse( } if (polledFinalAnswer) { - appendFinalAnswer(polledFinalAnswer); + appendFinalAnswer(polledFinalAnswer.text, polledFinalAnswer.metadata); } // Async image_gen ends the SSE with a "Processing image..." @@ -1971,6 +2000,7 @@ async function buildNonStreamingResponse( let imagePointers: ImagePointerRef[] | undefined; let imageGenAsync = false; let handoff = false; + let answerMetadata: Record | undefined; let parentCandidateMessageId: string | null = null; for await (const chunk of extractContent(eventStream, signal)) { @@ -1987,24 +2017,29 @@ async function buildNonStreamingResponse( } if (chunk.done) { fullAnswer = chunk.answer || fullAnswer; + answerMetadata = chunk.metadata ?? answerMetadata; imagePointers = chunk.imagePointers; imageGenAsync = chunk.imageGenAsync ?? false; handoff = handoff || (chunk.handoff ?? false); if (chunk.messageId) parentCandidateMessageId = chunk.messageId; break; } - if (chunk.answer) fullAnswer = chunk.answer; + if (chunk.answer) { + fullAnswer = chunk.answer; + answerMetadata = chunk.metadata ?? answerMetadata; + } } if (pollFinalAnswer && conversationId && (handoff || !fullAnswer.trim())) { const polled = await pollFinalAnswer(conversationId); if (polled?.text) { fullAnswer = polled.text; + answerMetadata = polled.metadata ?? answerMetadata; if (polled.messageId) parentCandidateMessageId = polled.messageId; } } - fullAnswer = cleanChatGptText(fullAnswer); + fullAnswer = cleanChatGptText(fullAnswer, answerMetadata); // Async image gen: SSE ended with "Processing image..." — poll for the // final pointer the same way the streaming path does. @@ -2683,7 +2718,8 @@ export class ChatGptWebExecutor extends BaseExecutor { clientHeaders, }: ExecuteInput) { const messages = (body as Record | null)?.messages as - Array> | undefined; + | Array> + | undefined; if (!messages || !Array.isArray(messages) || messages.length === 0) { return { response: errorResponse(400, "Missing or empty messages array"), @@ -3073,15 +3109,11 @@ export class ChatGptWebExecutor extends BaseExecutor { } } -// Strip ChatGPT's internal entity markup. The browser renders these as proper -// inline citations / chips via JS; for a plain text completion we just want -// the human-readable form. -// entity["city","Paris","capital of France"] → Paris -// entity["…","value", …] → value -const ENTITY_RE = /entity\["[^"]*","([^"]*)"[^\]]*\]/g; - -function cleanChatGptText(text: string): string { - return text.replace(ENTITY_RE, "$1"); +function commonPrefixLength(a: string, b: string): number { + const n = Math.min(a.length, b.length); + let i = 0; + while (i < n && a.charCodeAt(i) === b.charCodeAt(i)) i++; + return i; } function stringToStream(text: string): ReadableStream { diff --git a/open-sse/executors/chatgpt-web/citations.ts b/open-sse/executors/chatgpt-web/citations.ts new file mode 100644 index 0000000000..395a18317c --- /dev/null +++ b/open-sse/executors/chatgpt-web/citations.ts @@ -0,0 +1,387 @@ +// Pure ChatGPT-web citation-marker parsing/rendering, extracted verbatim from +// chatgpt-web.ts (no module state — safe to unit test in isolation). +// +// Strip ChatGPT's internal entity/citation markup. The browser renders these +// private-use markers (for example `citeturn0search0`) with metadata from +// `message.metadata.content_references`; API clients need plain Markdown with +// real links instead of raw ChatGPT UI tokens. +// entity["city","Paris","capital of France"] → Paris +// entity["…","value", …] → value +const ENTITY_RE = /entity\["[^"]*","([^"]*)"[^\]]*\]/g; +const CHATGPT_MARKER_START = "\uE200"; +const CHATGPT_MARKER_SEP = "\uE202"; +const CHATGPT_MARKER_END = "\uE201"; +const CHATGPT_REF_TOKEN_RE = /turn\d+(?:search|product|news|image|webpage)\d+/g; + +type ChatGptCitationSource = { + title: string; + url: string; + attribution: string; +}; + +type ChatGptCitationMention = { + start?: number; + end?: number; + markerText?: string; + replacement: string; +}; + +type ChatGptCitationData = { + sources: ChatGptCitationSource[]; + mentions: ChatGptCitationMention[]; + refTokenToSourceNumber: Map; +}; + +function asRecord(value: unknown): Record | null { + return value && typeof value === "object" ? (value as Record) : null; +} + +function asString(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value : null; +} + +function asNumber(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function asArray(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} + +function markdownLinkText(value: string): string { + return value.replace(/\[/g, "\\[").replace(/\]/g, "\\]").replace(/\n/g, " ").trim(); +} + +function markdownUrl(value: string): string { + return value.replace(/\(/g, "%28").replace(/\)/g, "%29"); +} + +function canonicalCitationUrl(value: string): string { + try { + const url = new URL(value); + url.searchParams.delete("utm_source"); + return url.toString(); + } catch { + return value; + } +} + +function referenceUrls(ref: Record): string[] { + const urls: string[] = []; + for (const key of ["url", "safe_url", "link"]) { + const url = asString(ref[key]); + if (url) urls.push(url); + } + for (const url of asArray(ref.safe_urls)) { + if (typeof url === "string" && url.trim()) urls.push(url); + } + return [...new Set(urls)]; +} + +function refTokenFromStructuredRef(ref: Record): string | null { + const turn = asNumber(ref.turn_index); + const refType = asString(ref.ref_type); + const refIndex = asNumber(ref.ref_index); + if (turn == null || refIndex == null || !refType) return null; + return `turn${turn}${refType}${refIndex}`; +} + +function mapStructuredRefs( + refs: unknown, + sourceNumber: number, + refTokenToSourceNumber: Map +): void { + for (const refValue of asArray(refs)) { + const ref = asRecord(refValue); + if (!ref) continue; + const token = refTokenFromStructuredRef(ref); + if (token && !refTokenToSourceNumber.has(token)) { + refTokenToSourceNumber.set(token, sourceNumber); + } + } +} + +function formatCitationLinks(numbers: number[], sources: ChatGptCitationSource[]): string { + return [...new Set(numbers)] + .sort((a, b) => a - b) + .map((num) => { + const source = sources[num - 1]; + return source ? `[${num}](${markdownUrl(source.url)})` : ""; + }) + .filter(Boolean) + .join(""); +} + +function urlMarkerLabel(markerText?: string | null): string | null { + if (!markerText) return null; + const privateMatch = markerText.match(/\uE200url\uE202([^\uE201\uE202]+)/u); + if (privateMatch?.[1]) return privateMatch[1].trim(); + const plainMatch = markerText.match(/^url[:\s]+(.+)$/i); + return plainMatch?.[1]?.trim() || null; +} + +function citationMarkerCandidates(markerText?: string): string[] { + if (!markerText) return []; + const candidates = [markerText]; + const tokens = markerText.match(CHATGPT_REF_TOKEN_RE) ?? []; + if (tokens.length > 0 && markerText.includes("cite")) { + candidates.push( + `${CHATGPT_MARKER_START}cite${tokens.map((token) => CHATGPT_MARKER_SEP + token).join("")}${CHATGPT_MARKER_END}` + ); + } + return [...new Set(candidates)]; +} + +type AddCitationSourceFn = ( + titleValue: unknown, + urlValue: unknown, + attributionValue?: unknown +) => number; +type AddCitationMentionFn = (ref: Record, replacement: string) => void; + +/** Supporting-website sources nested under one `grouped_webpages` item. */ +function collectSupportingWebsiteNumbers( + item: Record, + addSource: AddCitationSourceFn, + refTokenToSourceNumber: Map +): number[] { + const numbers: number[] = []; + for (const supportingValue of asArray(item.supporting_websites)) { + const supporting = asRecord(supportingValue); + if (!supporting) continue; + const supportingNumber = addSource(supporting.title, supporting.url, supporting.attribution); + if (supportingNumber) { + numbers.push(supportingNumber); + mapStructuredRefs(supporting.refs, supportingNumber, refTokenToSourceNumber); + } + } + return numbers; +} + +/** One `grouped_webpages` item — its own primary source plus any supporting-website + * sources nested under it. */ +function collectGroupedWebpageItemNumbers( + itemValue: unknown, + addSource: AddCitationSourceFn, + refTokenToSourceNumber: Map +): number[] { + const item = asRecord(itemValue); + if (!item) return []; + const numbers: number[] = []; + const mainNumber = addSource(item.title, item.url, item.attribution); + if (mainNumber) { + numbers.push(mainNumber); + mapStructuredRefs(item.refs, mainNumber, refTokenToSourceNumber); + } + numbers.push(...collectSupportingWebsiteNumbers(item, addSource, refTokenToSourceNumber)); + return numbers; +} + +/** Fallback when no `grouped_webpages` item yielded a usable source — fall back to + * the ref's own URLs directly. */ +function collectGroupedWebpagesFallbackNumbers( + ref: Record, + addSource: AddCitationSourceFn +): number[] { + const numbers: number[] = []; + for (const url of referenceUrls(ref)) { + const fallbackNumber = addSource(ref.title, url, ref.attribution); + if (fallbackNumber) numbers.push(fallbackNumber); + } + return numbers; +} + +/** `content_references[].type === "grouped_webpages"` — a primary source per item, + * each optionally paired with supporting-website sources; falls back to the ref's + * own URLs when no item yielded a usable source. */ +function collectGroupedWebpagesRef( + ref: Record, + sources: ChatGptCitationSource[], + addSource: AddCitationSourceFn, + addMention: AddCitationMentionFn, + refTokenToSourceNumber: Map +): void { + let numbers: number[] = []; + for (const itemValue of asArray(ref.items)) { + numbers.push(...collectGroupedWebpageItemNumbers(itemValue, addSource, refTokenToSourceNumber)); + } + + if (numbers.length === 0) { + numbers = collectGroupedWebpagesFallbackNumbers(ref, addSource); + } + + addMention(ref, formatCitationLinks(numbers, sources)); +} + +/** `content_references[].type === "sources_footnote"` — a flat list of sources with + * no inline mention to replace (the footnote itself carries no marker text). */ +function collectSourcesFootnoteRef( + ref: Record, + addSource: AddCitationSourceFn +): void { + for (const sourceValue of asArray(ref.sources)) { + const source = asRecord(sourceValue); + if (source) addSource(source.title, source.url, source.attribution); + } +} + +/** Any other reference type — a direct `webpage`/`url` marker with an inline label + * renders as `[label](url)`; everything else falls back to numbered source links. */ +function collectDefaultRef( + ref: Record, + type: string, + sources: ChatGptCitationSource[], + addSource: AddCitationSourceFn, + addMention: AddCitationMentionFn, + refTokenToSourceNumber: Map +): void { + const urls = referenceUrls(ref); + const label = urlMarkerLabel(asString(ref.matched_text)); + if ((type === "webpage" || type === "url") && label && urls[0]) { + addMention(ref, `[${markdownLinkText(label)}](${markdownUrl(urls[0])})`); + return; + } + + const numbers = urls + .map((url) => addSource(ref.title ?? ref.alt, url, ref.attribution)) + .filter((num) => num > 0); + if (numbers.length === 0) return; + + mapStructuredRefs(ref.refs, numbers[0], refTokenToSourceNumber); + addMention(ref, formatCitationLinks(numbers, sources)); +} + +function collectChatGptCitationData(metadata?: Record): ChatGptCitationData { + const refs = asArray(metadata?.content_references); + const sources: ChatGptCitationSource[] = []; + const mentions: ChatGptCitationMention[] = []; + const sourceIndexByCanonicalUrl = new Map(); + const refTokenToSourceNumber = new Map(); + + const addSource: AddCitationSourceFn = (titleValue, urlValue, attributionValue) => { + const url = asString(urlValue); + if (!url) return 0; + const canonical = canonicalCitationUrl(url); + const existing = sourceIndexByCanonicalUrl.get(canonical); + if (existing) return existing; + + const title = asString(titleValue) ?? url; + const attribution = asString(attributionValue) ?? ""; + const idx = sources.length + 1; + sources.push({ title: title.replace(/\n/g, " ").trim(), url, attribution }); + sourceIndexByCanonicalUrl.set(canonical, idx); + return idx; + }; + + const addMention: AddCitationMentionFn = (ref, replacement) => { + if (!replacement) return; + const start = asNumber(ref.start_idx); + const end = asNumber(ref.end_idx); + const markerText = asString(ref.matched_text) ?? undefined; + if (markerText || (start != null && end != null)) { + mentions.push({ + ...(start != null ? { start } : {}), + ...(end != null ? { end } : {}), + ...(markerText ? { markerText } : {}), + replacement, + }); + } + }; + + for (const refValue of refs) { + const ref = asRecord(refValue); + if (!ref) continue; + const type = asString(ref.type) ?? ""; + + if (type === "grouped_webpages") { + collectGroupedWebpagesRef(ref, sources, addSource, addMention, refTokenToSourceNumber); + continue; + } + if (type === "sources_footnote") { + collectSourcesFootnoteRef(ref, addSource); + continue; + } + collectDefaultRef(ref, type, sources, addSource, addMention, refTokenToSourceNumber); + } + + return { sources, mentions, refTokenToSourceNumber }; +} + +function replacePrivateCitationMarkers(text: string, citationData: ChatGptCitationData): string { + const replaceTokens = (tokens: string[]): string => { + const numbers = tokens + .map((token) => citationData.refTokenToSourceNumber.get(token)) + .filter((num): num is number => typeof num === "number"); + return numbers.length > 0 ? formatCitationLinks(numbers, citationData.sources) : ""; + }; + + return text + .replace(/\uE200cite((?:\uE202[^\uE201\uE202]+)+)\uE201/gu, (_all, body: string) => { + const tokens = [...body.matchAll(/\uE202([^\uE201\uE202]+)/gu)].map((match) => match[1]); + return replaceTokens(tokens); + }) + .replace( + /\bcite((?:turn\d+(?:search|product|news|image|webpage)\d+)+)\b/g, + (_all, body: string) => { + return replaceTokens(body.match(CHATGPT_REF_TOKEN_RE) ?? []); + } + ); +} + +function stripDanglingChatGptMarkers(text: string, citationData: ChatGptCitationData): string { + return replacePrivateCitationMarkers(text, citationData) + .replace( + /\uE200url\uE202([^\uE201\uE202]+)\uE202(https?:\/\/[^\uE201]+)\uE201/gu, + (_all, label: string, url: string) => { + return `[${markdownLinkText(label)}](${markdownUrl(url)})`; + } + ) + .replace( + /\uE200url\uE202([^\uE201\uE202]+)\uE202(?:[^\uE201]*\uE201)?/gu, + (_all, label: string) => { + return label.trim(); + } + ) + .replace(/\uE200cite(?:\uE202[^\uE201\uE202]*)*$/gu, "") + .replace(/\uE200[a-z_]+(?:\uE202[^\uE201\uE202]*)*\uE201/giu, "") + .replace(/\uE200[a-z_]+(?:\uE202[^\uE201\uE202]*)*$/giu, "") + .replace(/\uE202?turn\d+(?:search|product|news|image|webpage)\d+\uE201?/gu, "") + .replace(/[\uE200\uE201\uE202]/gu, ""); +} + +function applyChatGptCitations(text: string, metadata?: Record): string { + const citationData = collectChatGptCitationData(metadata); + let rendered = text; + + for (const mention of [...citationData.mentions].sort( + (a, b) => (b.start ?? -1) - (a.start ?? -1) + )) { + let replaced = false; + for (const markerText of citationMarkerCandidates(mention.markerText)) { + const limit = + mention.start != null + ? Math.min(rendered.length, mention.start + markerText.length) + : rendered.length; + let pos = rendered.lastIndexOf(markerText, limit); + if (pos < 0) pos = rendered.indexOf(markerText); + if (pos >= 0) { + rendered = + rendered.slice(0, pos) + mention.replacement + rendered.slice(pos + markerText.length); + replaced = true; + break; + } + } + + if (!replaced && mention.start != null && mention.end != null) { + const start = Math.max(0, Math.min(mention.start, rendered.length)); + const end = Math.max(start, Math.min(mention.end, rendered.length)); + rendered = rendered.slice(0, start) + mention.replacement + rendered.slice(end); + } + } + + return stripDanglingChatGptMarkers(rendered, citationData); +} + +export function cleanChatGptText(text: string, metadata?: Record): string { + return applyChatGptCitations(text.replace(ENTITY_RE, "$1"), metadata); +} diff --git a/tests/unit/chatgpt-web-citations.test.ts b/tests/unit/chatgpt-web-citations.test.ts new file mode 100644 index 0000000000..9141d2755c --- /dev/null +++ b/tests/unit/chatgpt-web-citations.test.ts @@ -0,0 +1,392 @@ +// ChatGPT-web citation-marker → Markdown link rendering (#6635). +// +// content_references metadata (grouped_webpages, sources_footnote, webpage/url +// mentions) is resolved into real Markdown links instead of raw ChatGPT UI +// private-use marker tokens (citeturn0search0, entity[...], etc.). +// These tests live in a dedicated file (chatgpt-web.test.ts is a frozen +// god-file at the file-size cap and cannot grow) — mirrors the minimal-mock +// pattern already used by chatgpt-web-tools-5240.test.ts. + +import test from "node:test"; +import assert from "node:assert/strict"; + +const { ChatGptWebExecutor, __resetChatGptWebCachesForTesting } = await import( + "../../open-sse/executors/chatgpt-web.ts" +); +const { __setTlsFetchOverrideForTesting } = await import( + "../../open-sse/services/chatgptTlsClient.ts" +); + +// ─── Minimal TLS-fetch mock ────────────────────────────────────────────────── +// Tailored to the citation flow: root/DPL, session→accessToken, sentinel→token +// (no PoW), conv→SSE built from the caller-supplied events. Warmup GETs fall +// through to 404, which the executor tolerates. + +function makeHeaders(map: Record = {}) { + const h = new Headers(); + for (const [k, v] of Object.entries(map)) h.set(k, String(v)); + return h; +} + +function sseText(events: unknown[]): string { + const chunks: string[] = []; + for (const evt of events) { + const { __event, ...payload } = evt as Record & { __event?: string }; + if (__event) chunks.push(`event: ${__event}\r\n`); + chunks.push(`data: ${JSON.stringify(payload)}\r\n\r\n`); + } + chunks.push("data: [DONE]\r\n\r\n"); + return chunks.join(""); +} + +function installMockFetch({ + conv, + conversationDetail, +}: { + conv: { status: number; events: unknown[] }; + conversationDetail?: { status: number; body: unknown }; +}) { + const calls = { urls: [] as string[], bodies: [] as unknown[], conversationDetail: 0 }; + + __setTlsFetchOverrideForTesting( + async (url: string, opts: { method?: string; body?: unknown } = {}) => { + const u = String(url); + calls.urls.push(u); + calls.bodies.push(opts.body); + const json = (body: unknown, status = 200) => ({ + status, + headers: makeHeaders({ "Content-Type": "application/json" }), + text: JSON.stringify(body), + body: null, + }); + + if ( + (u === "https://chatgpt.com/" || u === "https://chatgpt.com") && + (opts.method || "GET") === "GET" + ) { + return { + status: 200, + headers: makeHeaders({ "Content-Type": "text/html" }), + text: '', + body: null, + }; + } + if (u.includes("/api/auth/session")) { + return json({ + accessToken: "jwt-abc", + expires: new Date(Date.now() + 3600_000).toISOString(), + user: { id: "user-1" }, + }); + } + if (u.includes("/sentinel/chat-requirements")) { + return json({ token: "req-token", proofofwork: { required: false } }); + } + // /backend-api/conversation/ — detail poll used by GPT-5.5 Pro handoff. + if (conversationDetail) { + const m1 = u.match(/\/backend-api\/conversation\/([^/?#]+)$/); + if (m1) { + calls.conversationDetail++; + return json(conversationDetail.body, conversationDetail.status); + } + } + if ( + u.endsWith("/backend-api/f/conversation") || + u.endsWith("/backend-api/conversation") || + /\/backend-api\/(f\/)?conversation\?/.test(u) + ) { + return { + status: conv.status, + headers: makeHeaders({ "Content-Type": "text/event-stream" }), + text: sseText(conv.events), + body: null, + }; + } + // Warmup (/me, /conversations, /models) — tolerated. + return { status: 404, headers: makeHeaders(), text: "not mocked", body: null }; + } + ); + + return { + calls, + restore() { + __setTlsFetchOverrideForTesting(null); + }, + }; +} + +test("Non-streaming: resolves ChatGPT web citation markers into markdown links", async () => { + __resetChatGptWebCachesForTesting(); + const urlMarker = "urlTesla"; + const citationMarker = "citeturn0search0turn0search3"; + const answerPrefix = `${urlMarker} FSD v14 is rolling out `; + const m = installMockFetch({ + conv: { + status: 200, + events: [ + { + conversation_id: "c1", + message: { + id: "m1", + author: { role: "assistant" }, + content: { + content_type: "text", + parts: [`${answerPrefix}${citationMarker}`], + }, + status: "finished_successfully", + metadata: { + content_references: [ + { + type: "webpage", + title: "Tesla", + matched_text: urlMarker, + start_idx: 0, + end_idx: urlMarker.length, + safe_urls: ["https://www.tesla.com/en_au/support/autopilot"], + }, + { + type: "grouped_webpages", + matched_text: citationMarker, + start_idx: answerPrefix.length, + end_idx: answerPrefix.length + citationMarker.length, + items: [ + { + title: "Tesla FSD v14 release notes", + url: "https://www.tesla.com/support/fsd-v14?utm_source=chatgpt.com", + attribution: "tesla.com", + }, + { + title: "Owner discussion", + url: "https://example.com/owners/fsd-v14", + attribution: "example.com", + }, + ], + }, + ], + }, + }, + }, + ], + }, + }); + try { + const executor = new ChatGptWebExecutor(); + const result = await executor.execute({ + model: "gpt-5.5-pro-extended", + body: { messages: [{ role: "user", content: "latest Tesla FSD in Australia" }] }, + stream: false, + credentials: { apiKey: "test" }, + signal: AbortSignal.timeout(10_000), + log: null, + }); + + assert.equal(result.response.status, 200); + const json = await result.response.json(); + const content = json.choices[0].message.content; + assert.match( + content, + /\[Tesla\]\(https:\/\/www\.tesla\.com\/en_au\/support\/autopilot\) FSD v14 is rolling out/ + ); + assert.match( + content, + /\[1\]\(https:\/\/www\.tesla\.com\/support\/fsd-v14\?utm_source=chatgpt\.com\)/ + ); + assert.match(content, /\[2\]\(https:\/\/example\.com\/owners\/fsd-v14\)/); + assert.doesNotMatch(content, /|||turn0search/); + } finally { + m.restore(); + } +}); + +test("Streaming: buffers split ChatGPT citation markers until metadata can link them", async () => { + __resetChatGptWebCachesForTesting(); + const citationMarker = "citeturn0search0turn0search3"; + const prefix = "Tesla FSD v14 is rolling out "; + const m = installMockFetch({ + conv: { + status: 200, + events: [ + { + conversation_id: "c1", + message: { + id: "m1", + author: { role: "assistant" }, + content: { content_type: "text", parts: [prefix] }, + status: "in_progress", + }, + }, + { + conversation_id: "c1", + message: { + id: "m1", + author: { role: "assistant" }, + content: { content_type: "text", parts: [prefix + "citeturn0search0"] }, + status: "in_progress", + }, + }, + { + conversation_id: "c1", + message: { + id: "m1", + author: { role: "assistant" }, + content: { content_type: "text", parts: [prefix + citationMarker + "."] }, + status: "finished_successfully", + metadata: { + content_references: [ + { + type: "grouped_webpages", + matched_text: citationMarker, + start_idx: prefix.length, + end_idx: prefix.length + citationMarker.length, + items: [ + { + title: "Tesla source", + url: "https://www.tesla.com/fsd", + attribution: "tesla.com", + }, + { + title: "Owners source", + url: "https://example.com/owners", + attribution: "example.com", + }, + ], + }, + ], + }, + }, + }, + ], + }, + }); + try { + const executor = new ChatGptWebExecutor(); + const result = await executor.execute({ + model: "gpt-5.5-pro-extended", + body: { + messages: [{ role: "user", content: "latest Tesla FSD in Australia" }], + stream: true, + }, + stream: true, + credentials: { apiKey: "test" }, + signal: AbortSignal.timeout(10_000), + log: null, + }); + + assert.equal(result.response.status, 200); + const text = await result.response.text(); + const content = text + .split("\n") + .filter((l) => l.startsWith("data: ") && l !== "data: [DONE]") + .map((l) => { + try { + return JSON.parse(l.slice(6)); + } catch { + return null; + } + }) + .filter((j) => j?.choices?.[0]?.delta?.content) + .map((j) => j.choices[0].delta.content) + .join(""); + + assert.equal( + content, + "Tesla FSD v14 is rolling out [1](https://www.tesla.com/fsd)[2](https://example.com/owners)." + ); + assert.doesNotMatch(content, /|||turn0search/); + } finally { + m.restore(); + } +}); + +test("GPT-5.5 Pro non-streaming: stream_handoff polls conversation detail for final answer", async () => { + __resetChatGptWebCachesForTesting(); + const citationMarker = "citeturn0search0"; + const m = installMockFetch({ + conv: { + status: 200, + events: [ + { + conversation_id: "conv-pro", + message: { + id: "progress-1", + author: { role: "assistant" }, + content: { content_type: "text", parts: ["Working on it…"] }, + status: "in_progress", + }, + }, + { __event: "stream_handoff", conversation_id: "conv-pro" }, + ], + }, + conversationDetail: { + status: 200, + body: { + mapping: { + thought: { + message: { + id: "thought", + author: { role: "assistant" }, + content: { content_type: "thoughts", parts: ["hidden thinking"] }, + status: "finished_successfully", + end_turn: true, + create_time: 1, + update_time: 1, + }, + }, + final: { + message: { + id: "final", + author: { role: "assistant" }, + content: { + content_type: "text", + parts: [`👉 Final full Pro answer. ${citationMarker}`], + }, + status: "finished_successfully", + end_turn: true, + create_time: 2, + update_time: 2, + metadata: { + content_references: [ + { + type: "grouped_webpages", + matched_text: citationMarker, + start_idx: "👉 Final full Pro answer. ".length, + end_idx: "👉 Final full Pro answer. ".length + citationMarker.length, + items: [ + { + title: "Polled Pro source", + url: "https://example.com/pro-source", + attribution: "example.com", + }, + ], + }, + ], + }, + }, + }, + }, + }, + }, + }); + try { + const executor = new ChatGptWebExecutor(); + const result = await executor.execute({ + model: "gpt-5.5-pro-extended", + body: { messages: [{ role: "user", content: "hard problem" }] }, + stream: false, + credentials: { apiKey: "cookie-pro-poll" }, + signal: AbortSignal.timeout(10_000), + log: null, + }); + + const json = await result.response.json(); + assert.equal( + json.choices[0].message.content, + "👉 Final full Pro answer. [1](https://example.com/pro-source)" + ); + assert.equal(m.calls.conversationDetail, 1); + const convIdx = m.calls.urls.findIndex((u) => u.endsWith("/backend-api/f/conversation")); + const sentBody = JSON.parse(m.calls.bodies[convIdx]); + assert.equal(sentBody.history_and_training_disabled, true); + } finally { + m.restore(); + } +}); diff --git a/tests/unit/chatgpt-web.test.ts b/tests/unit/chatgpt-web.test.ts index 7a206e7e5c..10ec9abec5 100644 --- a/tests/unit/chatgpt-web.test.ts +++ b/tests/unit/chatgpt-web.test.ts @@ -1,6 +1,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import { writeFile } from "node:fs/promises"; +import type { TlsFetchOptions } from "../../open-sse/services/chatgptTlsClient.ts"; const { ChatGptWebExecutor, __derivePublicBaseUrlForTesting, __resetChatGptWebCachesForTesting } = await import("../../open-sse/executors/chatgpt-web.ts"); @@ -63,6 +64,49 @@ async function withEnv(overrides, fn) { } } +type MockTlsConfig = { + status: number; + body?: unknown; + setCookie?: string; + error?: unknown; + events?: unknown[]; +}; + +type MockFetchOptions = { + session?: MockTlsConfig; + sentinel?: MockTlsConfig; + conv?: MockTlsConfig; + dpl?: MockTlsConfig; + fileDownload?: MockTlsConfig; + attachmentDownload?: MockTlsConfig; + conversationDetail?: MockTlsConfig | MockTlsConfig[]; + signedDownload?: MockTlsConfig; + userConfig?: MockTlsConfig; + onSession?: (opts: TlsFetchOptions) => void; + onSentinel?: (opts: TlsFetchOptions) => void; + onConv?: (opts: TlsFetchOptions) => void; + onFileDownload?: (opts: TlsFetchOptions, fileId: string) => void; + onAttachmentDownload?: (opts: TlsFetchOptions, fileId: string) => void; + onUserConfig?: (opts: TlsFetchOptions, url: string) => void; +}; + +type MockFetchCalls = { + session: number; + dpl: number; + sentinel: number; + conv: number; + fileDownload: number; + attachmentDownload: number; + conversationDetail: number; + signedDownload: number; + userConfig: number; + userConfigUrls: string[]; + userConfigMethods: string[]; + urls: string[]; + headers: Array | undefined>; + bodies: Array; +}; + /** Dispatch the TLS-impersonating fetch by URL pathname. * Default: session 200 with accessToken, sentinel 200 no PoW, conv 200 empty stream. */ function installMockFetch({ @@ -81,8 +125,8 @@ function installMockFetch({ onFileDownload, onAttachmentDownload, onUserConfig, -}: any = {}) { - const calls = { +}: MockFetchOptions = {}) { + const calls: MockFetchCalls = { session: 0, dpl: 0, sentinel: 0, @@ -791,76 +835,6 @@ test("Streaming: cumulative parts are diffed into non-overlapping deltas", async } }); -test("GPT-5.5 Pro non-streaming: stream_handoff polls conversation detail for final answer", async () => { - reset(); - const m = installMockFetch({ - conv: { - status: 200, - events: [ - { - conversation_id: "conv-pro", - message: { - id: "progress-1", - author: { role: "assistant" }, - content: { content_type: "text", parts: ["Working on it…"] }, - status: "in_progress", - }, - }, - { __event: "stream_handoff", conversation_id: "conv-pro" }, - ], - }, - conversationDetail: { - status: 200, - body: { - mapping: { - thought: { - message: { - id: "thought", - author: { role: "assistant" }, - content: { content_type: "thoughts", parts: ["hidden thinking"] }, - status: "finished_successfully", - end_turn: true, - create_time: 1, - update_time: 1, - }, - }, - final: { - message: { - id: "final", - author: { role: "assistant" }, - content: { content_type: "text", parts: ["👉 Final full Pro answer."] }, - status: "finished_successfully", - end_turn: true, - create_time: 2, - update_time: 2, - }, - }, - }, - }, - }, - }); - try { - const executor = new ChatGptWebExecutor(); - const result = await executor.execute({ - model: "gpt-5.5-pro-extended", - body: { messages: [{ role: "user", content: "hard problem" }] }, - stream: false, - credentials: { apiKey: "cookie-pro-poll" }, - signal: AbortSignal.timeout(10_000), - log: null, - }); - - const json = await result.response.json(); - assert.equal(json.choices[0].message.content, "👉 Final full Pro answer."); - assert.equal(m.calls.conversationDetail, 1); - const convIdx = m.calls.urls.findIndex((u) => u.endsWith("/backend-api/f/conversation")); - const sentBody = JSON.parse(m.calls.bodies[convIdx]); - assert.equal(sentBody.history_and_training_disabled, true); - } finally { - m.restore(); - } -}); - test("GPT-5.5 Pro streaming: preserves interim reasoning and appends final polled answer", async () => { reset(); const m = installMockFetch({ @@ -3050,7 +3024,11 @@ test("Image edit handler: bytes-hash match drives executor with cached conversat credentials: { apiKey: "test" }, log: null, }); - assert.equal(result.success, true, `expected success, got error: ${(result as any).error}`); + assert.equal( + result.success, + true, + `expected success, got error: ${(result as { error?: unknown }).error}` + ); const convIdx = m.calls.urls.findIndex((u) => u.endsWith("/backend-api/f/conversation")); assert.ok(convIdx >= 0, "conversation request was sent"); const sentBody = JSON.parse(m.calls.bodies[convIdx]); @@ -3085,8 +3063,11 @@ test("Image edit handler: no cached match returns 400 (does not silently generat log: null, }); assert.equal(result.success, false); - assert.equal((result as any).status, 400); - assert.match(String((result as any).error), /generated through this OmniRoute instance/); + assert.equal((result as { status?: unknown }).status, 400); + assert.match( + String((result as { error?: unknown }).error), + /generated through this OmniRoute instance/ + ); assert.equal(m.calls.session, 0, "no upstream calls were attempted"); assert.equal(m.calls.conv, 0, "no chat-completion was attempted"); } finally { @@ -3109,8 +3090,8 @@ test("Image gen handler: n>4 is rejected before any upstream call", async () => log: null, }); assert.equal(result.success, false); - assert.equal((result as any).status, 400); - assert.match(String((result as any).error), /n=1\.\.4/); + assert.equal((result as { status?: unknown }).status, 400); + assert.match(String((result as { error?: unknown }).error), /n=1\.\.4/); assert.equal(m.calls.session, 0, "no session exchange was attempted"); assert.equal(m.calls.conv, 0, "no conversation request was attempted"); } finally {