From f851374e62cd67ff1a24910c4d53d583356d3567 Mon Sep 17 00:00:00 2001 From: Markus Hartung Date: Thu, 6 Aug 2026 05:59:30 +0200 Subject: [PATCH] feat(conversations): redesign to no-forking model with pagination and duplicate-anchor fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every conversation is now a single straight line: when a request's turn history diverges from what's on file (real OpenClaw traffic edits/ duplicates turns to keep provider-side prompt caches warm), the diverging history mints its own independent conversation instead of forking a branch inside the old one. Distinguished via anchorHasChild — whether the reconnect anchor already has a recorded child. Also fixes the actual production-blocking bug this surfaced: real agentic traffic is full of byte-identical repeated turns (tool-polling loop output, heartbeat acks — one real conversation had 28 duplicates of a single turn). findReconnectMatch used to return on the first candidate anchor found for a repeated turn's content hash — in practice the oldest, stalest occurrence — whose recorded next-turn differs from the current request, so it looked like a divergence on every single request instead of ever reconnecting. Now every candidate anchor is evaluated and the one that verifiably extends furthest wins (ties break toward the anchor with no recorded child). Dashboard: /dashboard/conversations lists conversations by actual turn-node count instead of request-touch count (a freshly-forked conversation can carry hundreds of turns from a single insert but start at turn_count=1, which wrongly excluded it from the old turn_count>=2 filter). The conversation view loads the last 20 turns with a "Load more" button, scrolls to bottom on open and stays pinned there via a ResizeObserver while large/late-settling content keeps growing (a single requestAnimationFrame undershoots for a page containing multi-KB tool-output turns), and resyncs its "Goto latest request"/summary fields from the background list poll so they don't go stale while the modal stays open (keyed off the id, not the whole row object, so the poll-for-new-turns interval isn't reset every tick by that resync). X-ConversationId threading: chat.ts now passes the request's own correlationId into resolveConversationId so new turn-chain nodes can be tagged with a request identifier that exists before the call_logs row itself does. usageHistory's in-memory pending-request state (byModel/ byAccount/details/pendingById) is reused across Next.js dev HMR module re-evaluations via a globalThis singleton (same pattern as db/core.ts), so a live poll against a request that started before a hot-reload doesn't silently lose its partialAssistantText/isActive tracking. --- open-sse/services/conversationTracker.ts | 401 +++++++++++--- .../dashboard/conversations/page.tsx | 505 +++++++++++++++++- src/app/api/conversations/[id]/tree/route.ts | 62 +++ src/app/api/conversations/route.ts | 15 +- src/lib/db/agenticConversations.ts | 377 ++++++++++--- .../136_conversation_turn_nodes.sql | 61 +++ src/lib/usage/usageHistory.ts | 42 +- src/sse/handlers/chat.ts | 11 +- tests/unit/agenticConversations.test.ts | 417 ++++++++++++--- tests/unit/conversationTracker.test.ts | 493 +++++++++++++---- ...conversations-tree-route-seq-param.test.ts | 30 ++ 11 files changed, 2026 insertions(+), 388 deletions(-) create mode 100644 src/app/api/conversations/[id]/tree/route.ts create mode 100644 src/lib/db/migrations/136_conversation_turn_nodes.sql create mode 100644 tests/unit/conversations-tree-route-seq-param.test.ts diff --git a/open-sse/services/conversationTracker.ts b/open-sse/services/conversationTracker.ts index dd69dc3c00..47ae6224d2 100644 --- a/open-sse/services/conversationTracker.ts +++ b/open-sse/services/conversationTracker.ts @@ -3,9 +3,28 @@ * HTTP requests that are turns of the same multi-turn agentic conversation. * * Clients resend the full growing message/input history on every turn (no - * server-side state dependency), so continuation is detected by checking - * whether an earlier request's history is a strict prefix of the current - * request's history. This is a new, persisted mechanism — separate from + * server-side state dependency). Continuation is detected with a per-turn + * hash chain (each turn's id = sha256(parentId, role, sha256(text)), the + * same idea as a git commit graph): a new request's turns are walked from + * the start against the candidate conversation's existing chain, matching as + * far as they agree. Real agentic-CLI traffic (OpenClaw and similar) often + * edits or duplicates a turn mid-history to keep provider-side prompt caches + * warm — e.g. request 1 has turns `a b c … h i`, request 2 has + * `a b c′ … h i′ i j k`. A whole-history hash (the original approach) breaks + * on any such edit and never reconnects. + * + * Every OmniRoute conversation is a single straight line — it never forks. + * When a turn diverges from what's already on file (`c` became `c'`), that + * diverging history becomes its OWN independent conversation, with its own + * id, built fresh from this request's full turn list — not a branch grafted + * onto the old chain (2026-08-06 redesign; the branching model's real + * traffic accumulated dozens of edits per session, and indenting one more + * tree level per edit eventually left no room to show content at all). + * `a b c d` and `a b c' d'` end up as two distinct conversations, sharing no + * further storage after the point they diverge — simpler to store, query, + * and render than a tree, and it matches how the data is actually used: a + * "conversation" here is one continuous transcript, not a version-control + * graph. This is a new, persisted mechanism — separate from * `sessionManager.ts`'s `generateSessionId()` (in-memory, routing/latency * only) even though it uses the same sha256-fingerprint style. * @@ -16,8 +35,11 @@ import { createHash, randomUUID } from "node:crypto"; import { createAgenticConversation, findAgenticConversationsByFingerprint, + getConversationTurnIndex, + insertConversationTurnNodes, touchOrCreateExternalConversation, updateAgenticConversation, + type ConversationTurnIndex, } from "../../src/lib/db/agenticConversations.ts"; type JsonRecord = Record; @@ -25,6 +47,16 @@ type JsonRecord = Record; interface CanonicalTurn { role: "system" | "user" | "assistant" | "tool"; text: string; + /** 'text' | 'tool_use' | 'tool_result' — carried through to + * conversation_turn_nodes so the tree view (and any other consumer) can + * build the exact NormalizedBlock (src/mitm/inspector/types.ts) the + * request-detail panel already builds from buildRequestTurns/ + * buildResponseTurns, rendering tool calls/results through the same + * ChatBubble/MessageContent/ToolCallBlock/ToolResultBlock components + * everywhere instead of a parallel tree-only implementation. */ + blockKind: "text" | "tool_use" | "tool_result"; + /** Set only when blockKind === "tool_use". */ + toolName: string | null; } export interface ResolveConversationIdInput { @@ -33,6 +65,15 @@ export interface ResolveConversationIdInput { apiKeyId: string | null; /** Raw `x-omniroute-session-id` header value, if the client supplied one. */ clientSessionIdHeader: string | null; + /** + * call_logs.correlation_id for this request (109_call_logs_correlation_id) + * — generated earlier in the request lifecycle, well before this request's + * own call_logs row/id exists, so it's the only stable identifier + * available here to tag new turn nodes with. The tree API route + * (src/app/api/conversations/[id]/tree/route.ts) joins through it to + * resolve a navigable call_logs.id. + */ + correlationId: string | null; } export interface ResolveConversationIdResult { @@ -50,9 +91,47 @@ function normalizeRole(raw: unknown): CanonicalTurn["role"] { return "user"; } +/** + * Extract human-readable text from an OpenAI/Anthropic/Responses-API + * `content` value. Chat Completions sends a plain string; Responses API and + * Anthropic send an array of typed blocks (`{type:"text"|"input_text"| + * "output_text", text}`, `tool_use`, `tool_result`, ...) — collapsing that + * array to its text (rather than `JSON.stringify`-ing the whole thing) is + * what feeds both the turn-hash-chain (so the same underlying text chains + * identically regardless of which block-array shape a client used to send + * it) and `text_preview`, which the /dashboard/conversations tree view + * renders directly as markdown — a raw JSON blob there was a real bug, not a + * cosmetic one. + */ function stringifyContent(content: unknown): string { if (typeof content === "string") return content; if (content == null) return ""; + if (Array.isArray(content)) { + const parts: string[] = []; + for (const item of content) { + if (typeof item === "string") { + parts.push(item); + continue; + } + const block = item && typeof item === "object" ? (item as JsonRecord) : null; + if (!block) continue; + const type = block.type; + if ( + (type === "text" || type === "input_text" || type === "output_text") && + typeof block.text === "string" + ) { + parts.push(block.text); + } else if (type === "tool_use" || type === "function_call") { + const name = typeof block.name === "string" ? block.name : ""; + parts.push(`[tool_use ${name}]`); + } else if (type === "tool_result" || type === "function_call_output") { + parts.push(stringifyContent(block.content ?? block.output ?? "")); + } else if (typeof block.text === "string") { + parts.push(block.text); + } + } + return parts.join("\n"); + } try { return JSON.stringify(content); } catch { @@ -96,7 +175,25 @@ export function extractCanonicalTurns(body: JsonRecord | null | undefined): Cano : null; if (!role) continue; const text = stringifyContent(rec.content ?? rec.text ?? rec.arguments ?? rec.output); - if (text) turns.push({ role, text }); + if (!text) continue; + + // Chat Completions tool-result messages (role: "tool"/"function") and + // Responses API function_call/function_call_output items are the only + // two shapes this canonicalizer sees for tool activity — everything + // else (including plain assistant/user/system text) is "text". + let blockKind: CanonicalTurn["blockKind"] = "text"; + let toolName: string | null = null; + if (rec.type === "function_call") { + blockKind = "tool_use"; + toolName = typeof rec.name === "string" ? rec.name : null; + } else if (rec.type === "function_call_output") { + blockKind = "tool_result"; + } else if (rec.role === "tool" || rec.role === "function") { + blockKind = "tool_result"; + toolName = typeof rec.name === "string" ? rec.name : null; + } + + turns.push({ role, text, blockKind, toolName }); } return turns; } @@ -107,24 +204,6 @@ function hashHex(text: string): string { return createHash("sha256").update(text).digest("hex"); } -function hashShort(text: string): string { - return hashHex(text).slice(0, 16); -} - -// Deliberately ignores the system message. Real coding-agent CLIs (Claude -// Code, opencode, etc.) commonly regenerate the system prompt on EVERY -// request with live context (timestamp, cwd, git status...) — anchoring -// identity to it would mint a brand new conversation on every single turn -// for exactly that kind of real traffic, even though the actual -// user/assistant history is a genuine, unbroken continuation. Discovered -// live on a real deployment: 28 consecutive requests from one growing -// session, each recorded as its own turn_count=1 conversation (issue #9315 -// follow-up). -function extractFirstNonSystemText(turns: CanonicalTurn[]): string | null { - const first = turns.find((t) => t.role !== "system"); - return first ? first.text : null; -} - function extractToolNames(body: JsonRecord | null | undefined): string[] { if (!body || !Array.isArray(body.tools)) return []; const names: string[] = []; @@ -138,53 +217,177 @@ function extractToolNames(body: JsonRecord | null | undefined): string[] { return names.sort(); } +// Deliberately excludes any message text — both the system prompt (real +// coding-agent CLIs like Claude Code/opencode regenerate it every request +// with live context: timestamp, cwd, git status...) AND, discovered live on +// a real OmniRoute deployment running OpenClaw, the first non-system turn +// too: OpenClaw's sliding context window drops/summarizes the EARLIEST +// turns as a session grows, so `firstNonSystemText` never stays stable +// across requests either — anchoring identity to either one mints a brand +// new conversation (or, worse, finds zero fingerprint candidates at all, so +// the turn-chain match in resolveConversationId never even runs) on every +// single turn for exactly this kind of real traffic, even though the actual +// history is a genuine, unbroken continuation. The bucket only needs to be +// small enough to bound candidate lookup — apiKeyId + model + toolNames is +// stable across a whole session and still narrow in practice; actual +// identity is decided by the turn-chain walk (real content overlap), not by +// this bucket, so widening it here cannot cause a false merge on its own. export function computeFingerprintHash(input: { apiKeyId: string | null; model: string | null; - turns: CanonicalTurn[]; toolNames: string[]; }): string { - const firstText = extractFirstNonSystemText(input.turns); - const parts = [ - input.apiKeyId ?? "", - input.model ?? "", - firstText ? hashShort(firstText) : "", - input.toolNames.join(","), - ]; + const parts = [input.apiKeyId ?? "", input.model ?? "", input.toolNames.join(",")]; // NOTE: no connectionId — conversation identity must not depend on which // upstream connection this particular turn happened to be routed to. return hashHex(parts.join("|")); } -// ── Bounded turn hash (continuation check, O(1) on very long histories) ── +// ── Turn hash chain (continuation + branch detection) ──────────────────── // -// A history of 100k+ tokens (observed in real traffic) makes hashing the -// FULL turn array on every request expensive. Instead hash a bounded -// projection: total turn count, the role sequence (cheap - one char per -// turn), and the text of the first 2 and last 3 turns. This is a documented -// heuristic trade-off: two histories differing ONLY in an untouched middle -// section, with the same length, would collide. Accepted for the O(1) cost -// bound; see conversationTracker.test.ts for the explicit test pinning this -// known limitation. -const HEAD_TURNS = 2; -const TAIL_TURNS = 3; +// Each turn gets a stable id chained to its predecessor, the same idea as a +// git commit graph: id = sha256(parentId, role, sha256(text)). A brand-new +// tree's first turn chains off the conversation root id itself (not off +// `null`) so two different, unrelated conversation trees whose first turn +// happens to be byte-identical (e.g. two sessions that both open with "hi") +// never compute the same node id — `conversation_turn_nodes.id` is a global +// primary key, not scoped per conversation_id. +// Bounded well past a UI "preview" length: the /dashboard/conversations tree +// view renders this as full markdown per turn (matching the request detail +// panel's rendering), not a truncated one-liner — 8000 chars comfortably +// covers a real single turn (a system prompt or a long assistant reply) +// while still bounding pathological outliers. +const TEXT_PREVIEW_LENGTH = 8000; -export function hashTurnsBounded(turns: CanonicalTurn[]): string { - // Same reasoning as extractFirstNonSystemText: a regenerated-every-turn - // system prompt must not be part of the continuation signal, or it alone - // breaks the head-hash for real CLI traffic even once the fingerprint - // bucket lookup (above) correctly finds the right candidate. - const relevant = turns.filter((t) => t.role !== "system"); - const head = relevant.slice(0, HEAD_TURNS); - const tail = relevant.length > HEAD_TURNS ? relevant.slice(-TAIL_TURNS) : []; - const roleSequence = relevant.map((t) => t.role[0]).join(""); - const parts = [ - String(relevant.length), - roleSequence, - ...head.map((t) => t.text), - ...tail.map((t) => t.text), - ]; - return hashHex(parts.join("")); +function hashTurnContent(turn: CanonicalTurn): string { + return hashHex(`${turn.role}${turn.text}`); +} + +function chainNodeId(parentId: string, turn: CanonicalTurn): string { + return hashHex(`${parentId}${hashTurnContent(turn)}`); +} + +interface NewTurnNode { + id: string; + parentId: string | null; + role: string; + contentHash: string; + textPreview: string; + blockKind: string; + toolName: string | null; +} + +/** Build the new-node run for turns[fromIndex:], chained off `chainAnchor`. */ +function buildNewNodes( + turns: CanonicalTurn[], + fromIndex: number, + chainAnchor: string, + rootId: string +): NewTurnNode[] { + const nodes: NewTurnNode[] = []; + let parent = chainAnchor; + for (let i = fromIndex; i < turns.length; i++) { + const turn = turns[i]; + const nodeId = chainNodeId(parent, turn); + nodes.push({ + id: nodeId, + // The root anchor is a hashing seed, not a real node — the first turn + // of a tree has no parent turn. + parentId: parent === rootId ? null : parent, + role: turn.role, + contentHash: hashTurnContent(turn), + textPreview: turn.text.slice(0, TEXT_PREVIEW_LENGTH), + blockKind: turn.blockKind, + toolName: turn.toolName, + }); + parent = nodeId; + } + return nodes; +} + +interface ReconnectMatch { + /** Index into `chainTurns` where the reconnection was found (turns before + * this index were dropped from the chain's view — a compacted summary the + * client sent instead of resending them verbatim — and are not inserted + * as nodes). */ + startIndex: number; + /** How far the match extends past startIndex (>= startIndex + 1). */ + matchEndIndex: number; + /** Node id to chain new nodes off (the last matched node). */ + anchorNodeId: string; + /** True when `anchorNodeId` already has a recorded child in this chain — + * i.e. turns[matchEndIndex] (if any) would collide with an existing, + * DIFFERENT turn rather than simply being new. See resolveConversationId's + * doc comment for what this distinction now controls. */ + anchorHasChild: boolean; +} + +/** + * Find where `chainTurns` reconnects to an existing chain, trying the + * leftmost turn first (so a still-fully-present prefix — the common case — + * matches immediately at the start) and falling back to later turns only + * when earlier ones aren't found anywhere in the chain. This is what makes + * continuation detection survive OpenClaw's sliding context window: once + * the earliest turns are compacted away, turn 0 of a new request is some + * turn from the MIDDLE of the existing chain, not its start — a start-only + * walk (checking only whether turn 0 is the chain's own first turn) would + * find nothing. + * + * Real agentic traffic is full of byte-identical repeated turns — a tool + * polling loop's "Process still running." output, a heartbeat ack, a + * one-word "ok" — so `byContentHash.get(...)` routinely returns MANY + * candidate anchors for the same turn (one real conversation observed 28 + * duplicates of a single OpenClaw runtime-context turn). Evaluating only the + * first candidate (as this used to do) meant returning whichever occurrence + * SQLite happened to list first — in practice the OLDEST, most stale one — + * whose recorded next-turn almost never matches the current request, so the + * walk stalled a few turns in and (worse) that stale anchor already has a + * DIFFERENT recorded child, tripping `anchorHasChild` and making + * resolveConversationId treat a genuine continuation as a divergence. Live + * result: a real conversation minted a brand-new copy of its ENTIRE history + * on every single request instead of ever reconnecting (2026-08-06). Every + * candidate anchor for every prefix start is now tried, and the one that + * verifiably extends furthest into the actual request wins — the only + * reliable signal of genuine continuation when content repeats. + */ +function findReconnectMatch( + chainTurns: CanonicalTurn[], + index: ConversationTurnIndex +): ReconnectMatch | null { + let best: ReconnectMatch | null = null; + + for (let s = 0; s < chainTurns.length; s++) { + const anchors = index.byContentHash.get(hashTurnContent(chainTurns[s])); + if (!anchors) continue; + for (const anchorNodeId of anchors) { + let parent = anchorNodeId; + let matchEndIndex = s + 1; + for (let i = s + 1; i < chainTurns.length; i++) { + const nodeId = chainNodeId(parent, chainTurns[i]); + if (!index.nodeIds.has(nodeId)) break; + parent = nodeId; + matchEndIndex++; + } + const anchorHasChild = index.parentsWithChildren.has(parent); + // Longest verified run wins outright. An equal-length run breaks + // toward anchorHasChild===false: a tie means both candidate anchors' + // recorded next-turn already differs from what's being requested (the + // walk stopped for the same reason on both), so the anchor with NO + // established child is the safe, unambiguous "just append here" — the + // other, having a different recorded child already, would incorrectly + // read as a divergence purely because it happened to be tried first. + const isBetter = + !best || + matchEndIndex > best.matchEndIndex || + (matchEndIndex === best.matchEndIndex && !anchorHasChild && best.anchorHasChild); + if (isBetter) { + best = { startIndex: s, matchEndIndex, anchorNodeId: parent, anchorHasChild }; + } + // Can't do better than matching every turn through to the end. + if (matchEndIndex === chainTurns.length) return best; + } + } + return best; } // ── Orchestration ───────────────────────────────────────────────────────── @@ -208,36 +411,78 @@ export async function resolveConversationId( const fingerprintHash = computeFingerprintHash({ apiKeyId: input.apiKeyId, model: input.model, - turns, toolNames, }); + // The turn CHAIN excludes the system message entirely, same reasoning as + // extractFirstNonSystemText above: real coding-agent CLIs regenerate the + // system prompt (timestamp/cwd/git status...) on every single request, so + // treating it as an ordinary chained turn would make turn-0 (or wherever + // it sits) fail to match on every request — reintroducing the exact + // always-new-conversation bug this chain design exists to fix. + const chainTurns = turns.filter((t) => t.role !== "system"); + const candidates = findAgenticConversationsByFingerprint(fingerprintHash); for (const candidate of candidates) { - // A genuine continuation always strictly grows the history (the client - // appends at least the assistant's reply plus a new turn) — same-length - // must never match, or two independent single-shot requests with - // byte-identical content (a client retry, or two unrelated conversations - // that both just say "hi") would merge into one conversation. - if (turns.length <= candidate.lastMessageCount) continue; - const truncated = turns.slice(0, candidate.lastMessageCount); - if (hashTurnsBounded(truncated) === candidate.lastMessagesHash) { - updateAgenticConversation(candidate.id, { - lastMessageCount: turns.length, - lastMessagesHash: hashTurnsBounded(turns), - turnCount: candidate.turnCount + 1, - }); + const index = getConversationTurnIndex(candidate.id); + if (index.nodeIds.size === 0) continue; + + const match = findReconnectMatch(chainTurns, index); + // No match anywhere in the chain means this candidate isn't actually + // this conversation's lineage — it only shares the coarse fingerprint + // bucket (apiKeyId/model/toolNames), which real traffic proves is not + // enough to assume overlap on its own (see computeFingerprintHash's doc + // comment) — try the next candidate rather than attaching a completely + // unrelated turn. + if (!match) continue; + + if (match.matchEndIndex === chainTurns.length) { + // Every turn from the reconnect point onward already exists on this + // chain (e.g. an exact retry, or the whole request is already fully + // recorded) — a real continuation, nothing new to insert. + updateAgenticConversation(candidate.id, { turnCount: candidate.turnCount + 1 }); return { conversationId: candidate.id, isNewConversation: false }; } + + if (!match.anchorHasChild) { + // Genuine tail growth: the reconnect point has no recorded child yet, + // so turns[matchEndIndex:] are simply turns this conversation hasn't + // seen before — append them to this SAME chain. Turns before + // startIndex (a compacted-away prefix, if any) are never inserted — + // they don't represent new content, just the client's own context + // management. + const newNodes = buildNewNodes( + chainTurns, + match.matchEndIndex, + match.anchorNodeId, + candidate.id + ); + insertConversationTurnNodes(candidate.id, input.correlationId, newNodes); + updateAgenticConversation(candidate.id, { turnCount: candidate.turnCount + 1 }); + return { conversationId: candidate.id, isNewConversation: false }; + } + + // The reconnect point already has a DIFFERENT recorded child — this + // request's turn at that position diverges from what's on file (a real + // OpenClaw cache-aware-context edit: turn `c` became `c'`). As of the + // 2026-08-06 redesign, an edited/duplicated turn no longer forks a + // branch inside this conversation's own chain — every OmniRoute + // conversation is now a single straight line, never a tree. The + // diverging history becomes its own independent conversation instead + // (built fresh below, from this request's full turn list) — distinct + // conversation ids for `a b c d` and `a b c' d'`, not one tree with two + // branches. This is both simpler to store/query and fixes a real UX + // problem the branching model had: real OpenClaw traffic accumulates + // dozens of edits per session, and indenting one more level per fork + // eventually left no horizontal space for content at all. Keep checking + // remaining candidates first, though — a later candidate may already BE + // that independent conversation from a previous edit at this same spot + // (e.g. a repeated retry of the edited turn), which should continue + // that one rather than minting yet another new id for it. } const id = `conv_${randomUUID()}`; - createAgenticConversation({ - id, - apiKeyId: input.apiKeyId, - fingerprintHash, - lastMessageCount: turns.length, - lastMessagesHash: hashTurnsBounded(turns), - }); + createAgenticConversation({ id, apiKeyId: input.apiKeyId, fingerprintHash }); + insertConversationTurnNodes(id, input.correlationId, buildNewNodes(chainTurns, 0, id, id)); return { conversationId: id, isNewConversation: true }; } diff --git a/src/app/(dashboard)/dashboard/conversations/page.tsx b/src/app/(dashboard)/dashboard/conversations/page.tsx index acb4954442..eaa9346dd7 100644 --- a/src/app/(dashboard)/dashboard/conversations/page.tsx +++ b/src/app/(dashboard)/dashboard/conversations/page.tsx @@ -7,6 +7,8 @@ import { formatTime } from "@/shared/utils/formatting"; import { copyToClipboard } from "@/shared/utils/clipboard"; import RequestLoggerDetail from "@/shared/components/RequestLoggerDetail"; import useEmailPrivacyStore from "@/store/emailPrivacyStore"; +import { ChatBubble } from "@/app/(dashboard)/dashboard/tools/traffic-inspector/components/chat/ChatBubble"; +import type { NormalizedBlock, NormalizedTurn } from "@/mitm/inspector/types"; interface ConversationRow { id: string; @@ -17,8 +19,41 @@ interface ConversationRow { lastModel: string | null; lastProvider: string | null; lastStatus: number | null; + isActive: boolean; } +// Same spinner used for an in-flight request on /dashboard/logs +// (RequestLoggerV2) — reused here so "in progress" reads the same way in +// both places. +function ActiveSpinner() { + return ( + + + + ); +} + +interface ConversationTurn { + seq: number; + id: string; + parentId: string | null; + role: string; + textPreview: string; + blockKind: string; + toolName: string | null; + firstSeenAt: string; +} + +interface ConversationTurnsPage { + nodes: ConversationTurn[]; + hasMore: boolean; +} + +const CONVERSATION_PAGE_SIZE = 20; + const DEFAULT_POLL_SECONDS = 5; const POLL_STORAGE_KEY = "conversationsListPollSeconds"; @@ -57,6 +92,89 @@ function StatusBadge({ status }: { status: number | null }) { ); } +/** + * Builds the exact NormalizedBlock (src/mitm/inspector/types.ts) the + * request-detail panel already builds from buildRequestTurns/ + * buildResponseTurns, so a tool call/result renders through the very same + * ChatBubble → MessageContent → ToolCallBlock/ToolResultBlock pipeline as + * the detail view — not a parallel implementation. `textPreview` round- + * tripped through JSON for a structured tool_use/tool_result turn; parse it + * best-effort so the block gets a real object, not a JSON string. + */ +function toTurn(node: ConversationTurn): NormalizedTurn { + const role: NormalizedTurn["role"] = + node.role === "system" || node.role === "user" || node.role === "assistant" + ? node.role + : "tool"; + + let block: NormalizedBlock; + if (node.blockKind === "tool_use") { + let input: unknown = node.textPreview; + try { + input = JSON.parse(node.textPreview); + } catch { + // Arguments weren't valid JSON — show the raw string. + } + block = { type: "tool_use", id: node.id.slice(0, 12), name: node.toolName ?? "tool", input }; + } else if (node.blockKind === "tool_result") { + let content: unknown = node.textPreview; + try { + content = JSON.parse(node.textPreview); + } catch { + // Not JSON — show the raw string. + } + block = { type: "tool_result", tool_use_id: node.id.slice(0, 12), content }; + } else { + block = { type: "text", text: node.textPreview || "_(empty)_" }; + } + + return { role, blocks: [block], timestamp: node.firstSeenAt }; +} + +/** + * Renders a conversation's turns top to bottom, oldest first — always a + * flat, chronological list. Every OmniRoute conversation is a single + * straight line (an edited/duplicated turn mints its own independent + * conversation instead of branching this one — see conversationTracker.ts's + * 2026-08-06 redesign), so there is no fork/indentation logic here at all + * anymore. `onLoadOlder` renders as a button above the turns when more + * (older) history exists than the current page. + */ +function ConversationLogView({ + nodes, + hasMore, + loadingMore, + onLoadOlder, +}: { + nodes: ConversationTurn[]; + hasMore: boolean; + loadingMore: boolean; + onLoadOlder: () => void; +}) { + if (nodes.length === 0) { + return ( +
No turns recorded for this conversation.
+ ); + } + return ( +
+ {hasMore && ( + + )} + {nodes.map((node) => ( + + ))} +
+ ); +} + function ConversationsPageContent() { const router = useRouter(); const searchParams = useSearchParams(); @@ -64,6 +182,9 @@ function ConversationsPageContent() { // live searchParams on every render re-fires the deep-link open effect right when the // panel closes and router.replace() strips the ?id= param. const [initialId] = useState(() => searchParams.get("id")); + // Deep link for the conversation modal — separate param from `id` (the + // request-detail panel) so either overlay can be linked independently. + const [initialConversationParam] = useState(() => searchParams.get("tree")); const [conversations, setConversations] = useState([]); const [total, setTotal] = useState(0); @@ -74,6 +195,17 @@ function ConversationsPageContent() { const [detailData, setDetailData] = useState(null); const [detailLoading, setDetailLoading] = useState(false); const [detailLoggingEnabled, setDetailLoggingEnabled] = useState(false); + const [activeConversation, setActiveConversation] = useState(null); + // Extracted so effects that only care "which conversation" (not its + // summary fields) can depend on this stable primitive instead of the + // whole activeConversation object — that object gets a fresh reference + // every list-poll tick once opened (see the resync effect below), which + // would otherwise rebind timers/listeners on every poll tick. + const activeConversationId = activeConversation?.id ?? null; + const [conversationNodes, setConversationNodes] = useState([]); + const [conversationLoading, setConversationLoading] = useState(false); + const [conversationHasMore, setConversationHasMore] = useState(false); + const [loadingOlder, setLoadingOlder] = useState(false); const [pollSeconds, setPollSeconds] = useState(() => { try { const saved = localStorage.getItem(POLL_STORAGE_KEY); @@ -84,6 +216,25 @@ function ConversationsPageContent() { } }); const initialOpenedRef = useRef(false); + const initialConversationOpenedRef = useRef(false); + const conversationPanelRef = useRef(null); + const conversationContentRef = useRef(null); + // True right after opening a conversation (or clicking "Go to bottom"), + // cleared once the user scrolls away from the bottom themselves. A large + // conversation's last page can include multi-KB tool-output/context turns + // whose markdown takes more than one animation frame to lay out, so a + // single scrollTop=scrollHeight right after fetch can undershoot — the + // ResizeObserver below re-pins on every subsequent layout change while + // this stays true, instead of a one-shot scroll that races the render. + const pinnedToBottomRef = useRef(false); + // Set right before prepending an older page, so the effect below can + // adjust scrollTop by exactly how much content grew above the fold — + // otherwise "Load more" would visually yank the view to the top. + const prependAdjustRef = useRef<{ prevScrollHeight: number; prevScrollTop: number } | null>(null); + // Mirrors the newest loaded turn's seq without needing conversationNodes + // itself in the poll effect's dependency array (which would tear down and + // restart the interval on every single appended turn). + const newestSeqRef = useRef(null); useEffect(() => { let cancelled = false; @@ -111,6 +262,21 @@ function ConversationsPageContent() { }; }, [pollSeconds]); + // activeConversation is a snapshot taken once at openConversation() time — + // it's never touched again while the modal stays open (the turns-poll + // effect below only appends conversationNodes). Without this, "Goto latest + // request" and any other displayed summary field (lastModel/lastStatus/ + // turnCount) go stale the moment a new request lands in this conversation + // while you're still reading it, even though the list poll above (which + // runs regardless of whether the modal is open) already has the fresh + // row. Re-sync from it whenever the list refreshes. + useEffect(() => { + if (!activeConversationId) return; + const fresh = conversations.find((c) => c.id === activeConversationId); + if (!fresh) return; + setActiveConversation((prev) => (prev && prev.id === fresh.id ? fresh : prev)); + }, [conversations, activeConversationId]); + useEffect(() => { fetch("/api/logs/detail?limit=1") .then((res) => (res.ok ? res.json() : null)) @@ -180,9 +346,224 @@ function ConversationsPageContent() { openById(initialId).catch(() => {}); }, [initialId, openById]); - const openConversation = (row: ConversationRow) => { - if (!row.lastCallLogId) return; - openById(row.lastCallLogId).catch(() => {}); + const scrollToBottom = useCallback(() => { + pinnedToBottomRef.current = true; + const el = conversationPanelRef.current; + if (!el) return; + requestAnimationFrame(() => { + try { + el.scrollTop = el.scrollHeight; + } catch {} + }); + }, []); + + // Keeps the panel pinned to its bottom while conversationContentRef's + // height keeps changing (initial render of a large page, late-settling + // markdown/tool-output layout, a new turn arriving via poll) — see + // pinnedToBottomRef's comment above for why a single scrollToBottom call + // isn't enough on its own for a heavy page. + useEffect(() => { + const content = conversationContentRef.current; + const panel = conversationPanelRef.current; + if (!content || !panel) return; + const observer = new ResizeObserver(() => { + if (!pinnedToBottomRef.current) return; + panel.scrollTop = panel.scrollHeight; + }); + observer.observe(content); + return () => observer.disconnect(); + // Keyed on the id, not the whole object: activeConversation's summary + // fields (lastCallLogId etc.) get resynced from the list poll while the + // modal stays open (see that effect's comment), which would otherwise + // tear down and recreate this observer on every poll tick. + }, [activeConversation?.id]); + + // Un-pin as soon as the user scrolls away from the bottom themselves (e.g. + // to read earlier turns or click "Load more"), so later content growth + // doesn't yank them back down against their will. Re-pins automatically if + // they scroll back down to the bottom on their own. + useEffect(() => { + const panel = conversationPanelRef.current; + if (!panel) return; + const NEAR_BOTTOM_PX = 24; + const onScroll = () => { + const distanceFromBottom = panel.scrollHeight - panel.scrollTop - panel.clientHeight; + pinnedToBottomRef.current = distanceFromBottom <= NEAR_BOTTOM_PX; + }; + panel.addEventListener("scroll", onScroll, { passive: true }); + return () => panel.removeEventListener("scroll", onScroll); + }, [activeConversation?.id]); + + const fetchConversationPage = useCallback( + (id: string, params: string): Promise => + fetch(`/api/conversations/${id}/tree?${params}`, { cache: "no-store" }) + .then((res) => (res.ok ? res.json() : null)) + .then((data) => + data && Array.isArray(data.nodes) + ? { nodes: data.nodes, hasMore: Boolean(data.hasMore) } + : null + ) + .catch(() => null), + [] + ); + + const openConversation = useCallback( + (row: ConversationRow) => { + setActiveConversation(row); + setConversationNodes([]); + setConversationHasMore(false); + setConversationLoading(true); + try { + const url = new URL(globalThis.location.href); + url.searchParams.set("tree", row.id); + router.replace(url.pathname + url.search); + } catch { + // ignore navigation errors + } + fetchConversationPage(row.id, `limit=${CONVERSATION_PAGE_SIZE}`) + .then((page) => { + setConversationNodes(page?.nodes ?? []); + setConversationHasMore(page?.hasMore ?? false); + }) + .finally(() => { + setConversationLoading(false); + // A freshly-opened conversation should start scrolled to the + // latest (bottom-most) turn, not the oldest one on the page. + scrollToBottom(); + }); + }, + [router, fetchConversationPage, scrollToBottom] + ); + + const closeConversation = useCallback(() => { + setActiveConversation(null); + try { + const url = new URL(globalThis.location.href); + url.searchParams.delete("tree"); + router.replace(url.pathname + url.search); + } catch { + // ignore navigation errors + } + }, [router]); + + const loadOlderTurns = useCallback(() => { + const panel = conversationPanelRef.current; + const oldestSeq = conversationNodes[0]?.seq; + if (!activeConversation || !panel || oldestSeq == null || loadingOlder) return; + setLoadingOlder(true); + prependAdjustRef.current = { + prevScrollHeight: panel.scrollHeight, + prevScrollTop: panel.scrollTop, + }; + fetchConversationPage( + activeConversation.id, + `limit=${CONVERSATION_PAGE_SIZE}&beforeSeq=${oldestSeq}` + ) + .then((page) => { + if (page && page.nodes.length > 0) { + setConversationNodes((prev) => [...page.nodes, ...prev]); + } + setConversationHasMore(page?.hasMore ?? false); + }) + .finally(() => setLoadingOlder(false)); + }, [activeConversation, conversationNodes, loadingOlder, fetchConversationPage]); + + // Preserve scroll position across a "load more" prepend — otherwise + // adding older turns above the fold visually yanks the view to the top. + useEffect(() => { + const adjust = prependAdjustRef.current; + if (!adjust) return; + prependAdjustRef.current = null; + const panel = conversationPanelRef.current; + if (!panel) return; + requestAnimationFrame(() => { + panel.scrollTop = adjust.prevScrollTop + (panel.scrollHeight - adjust.prevScrollHeight); + }); + }, [conversationNodes]); + + useEffect(() => { + newestSeqRef.current = + conversationNodes.length > 0 ? conversationNodes[conversationNodes.length - 1].seq : null; + }, [conversationNodes]); + + useEffect(() => { + if (!activeConversationId) return; + const onKeyDown = (e: KeyboardEvent) => { + if (e.key === "Escape") closeConversation(); + }; + document.addEventListener("keydown", onKeyDown); + return () => document.removeEventListener("keydown", onKeyDown); + // activeConversationId, not the whole activeConversation object: it + // gets resynced (new object reference) from the list poll while the + // modal stays open (see that effect's comment) — depending on the + // object here would rebind this listener on every poll tick for no + // reason. + }, [activeConversationId, closeConversation]); + + // While the conversation is open, keep polling for turns that arrive + // later (the request that opened it may not be the last one — OpenClaw + // can send another turn while you're still reading). Reuses the same + // "Auto-refresh Xs" setting as the list, rather than a separate interval, + // so there's one poll cadence to reason about on this page. Only ever + // APPENDS newer turns (via afterSeq) — it never re-fetches or replaces + // the whole page, so a "Load more" page loaded earlier stays put, and it + // deliberately does NOT re-scroll on every refresh (only the initial open + // does that), so it doesn't yank the view mid-read. + // + // Depends on activeConversationId, NOT the whole activeConversation + // object: activeConversation gets a fresh object reference every list-poll + // tick (see the resync effect above, needed so "Goto latest request" + // doesn't go stale) — on the SAME poll cadence as this effect's own + // interval. Depending on the object would tear down and recreate this + // setInterval every single tick, resetting its countdown each time and + // starving it of ever actually firing — silently breaking the exact + // "keep filling in new turns while open" behavior this effect exists for. + useEffect(() => { + if (!activeConversationId) return; + const tick = () => { + if (document.visibilityState !== "visible") return; + if (newestSeqRef.current == null) return; + fetchConversationPage(activeConversationId, `afterSeq=${newestSeqRef.current}`).then( + (page) => { + if (page && page.nodes.length > 0) { + setConversationNodes((prev) => [...prev, ...page.nodes]); + } + } + ); + }; + const interval = setInterval(tick, pollSeconds * 1000); + return () => clearInterval(interval); + }, [activeConversationId, pollSeconds, fetchConversationPage]); + + // Deep link: /dashboard/conversations?tree= opens that conversation. + // Prefer the already-loaded row (has lastCallLogId for "Goto latest + // request"); fall back to a minimal row if the conversation isn't in the + // current page of the list (still fully works — the API only needs the + // id). + useEffect(() => { + if (!initialConversationParam || initialConversationOpenedRef.current || loading) return; + initialConversationOpenedRef.current = true; + const found = conversations.find((c) => c.id === initialConversationParam); + openConversation( + found ?? { + id: initialConversationParam, + turnCount: 0, + firstSeenAt: "", + lastSeenAt: "", + lastCallLogId: null, + lastModel: null, + lastProvider: null, + lastStatus: null, + isActive: false, + } + ); + }, [initialConversationParam, loading, conversations, openConversation]); + + const gotoLatestRequest = () => { + const id = activeConversation?.lastCallLogId; + if (!id) return; + closeConversation(); + openById(id).catch(() => {}); }; return ( @@ -241,15 +622,18 @@ function ConversationsPageContent() { className="rounded-xl border border-border p-3 flex flex-col gap-2 active:bg-bg-subtle cursor-pointer" >
- { - e.stopPropagation(); - copyToClipboard(row.id); - }} - className="font-mono text-[11px] text-text-main hover:underline truncate" - > - {row.id.slice(0, 16)}… + + {row.isActive && } + { + e.stopPropagation(); + copyToClipboard(row.id); + }} + className="font-mono text-[11px] text-text-main hover:underline truncate" + > + {row.id.slice(0, 16)}… + {row.turnCount} turns @@ -257,9 +641,7 @@ function ConversationsPageContent() {
- - {row.lastModel ?? "—"} - + {row.lastModel ?? "—"}
@@ -292,15 +674,18 @@ function ConversationsPageContent() { onClick={() => openConversation(row)} > - { - e.stopPropagation(); - copyToClipboard(row.id); - }} - className="hover:underline" - > - {row.id.slice(0, 16)}… + + {row.isActive && } + { + e.stopPropagation(); + copyToClipboard(row.id); + }} + className="hover:underline" + > + {row.id.slice(0, 16)}… + @@ -324,6 +709,77 @@ function ConversationsPageContent() { )} + {activeConversation && ( +
+
+
e.stopPropagation()} + > +
+
+

Conversation

+ + {activeConversation.id.slice(0, 24)}… + +
+
+ {activeConversation.lastCallLogId && ( + + )} + + +
+
+
+ {conversationLoading ? ( +
Loading…
+ ) : ( + + )} +
+
+
+ )} + {selectedLog && ( )}
diff --git a/src/app/api/conversations/[id]/tree/route.ts b/src/app/api/conversations/[id]/tree/route.ts new file mode 100644 index 0000000000..100fb4403c --- /dev/null +++ b/src/app/api/conversations/[id]/tree/route.ts @@ -0,0 +1,62 @@ +import { NextResponse } from "next/server"; +import { requireManagementAuth } from "@/lib/api/requireManagementAuth"; +import { getConversationTurnPage } from "@/lib/db/agenticConversations"; + +export const dynamic = "force-dynamic"; + +// Number(null) is 0, not NaN — so naively doing Number(searchParams.get(x)) +// turns an ABSENT query param into a real 0 instead of "not provided", +// which made beforeSeq/afterSeq always look present (0 != null) and forced +// the query into the afterSeq branch (uncapped, all rows) on every request, +// even ones with no beforeSeq/afterSeq at all. +export function parseSeqParam(raw: string | null): number | undefined { + if (raw === null || raw === "") return undefined; + const n = Number(raw); + return Number.isFinite(n) ? n : undefined; +} + +// Every OmniRoute conversation is a single straight line (see +// conversationTracker.ts's 2026-08-06 redesign — an edited/duplicated turn +// mints its own independent conversation instead of forking this one), so +// this always returns a flat, chronological page — never a tree — capped +// at `limit` (default 20) since a real OpenClaw conversation can run to +// hundreds of turns. `beforeSeq`/`afterSeq` page backward/forward from a +// previously-returned node's `seq`. +export async function GET( + req: Request, + { params }: { params: Promise<{ id: string }> | { id: string } } +) { + const authError = await requireManagementAuth(req); + if (authError) return authError; + + try { + const { id } = await params; + if (!id) return NextResponse.json({ error: "Missing id" }, { status: 400 }); + + const { searchParams } = new URL(req.url); + const limitParam = parseSeqParam(searchParams.get("limit")); + + const { nodes, hasMore } = getConversationTurnPage(id, { + limit: limitParam != null && limitParam > 0 ? limitParam : undefined, + beforeSeq: parseSeqParam(searchParams.get("beforeSeq")), + afterSeq: parseSeqParam(searchParams.get("afterSeq")), + }); + + return NextResponse.json({ + nodes: nodes.map((n) => ({ + seq: n.seq, + id: n.id, + parentId: n.parentId, + role: n.role, + textPreview: n.textPreview, + blockKind: n.blockKind, + toolName: n.toolName, + firstSeenAt: n.firstSeenAt, + })), + hasMore, + }); + } catch (err) { + console.error("[API ERROR] /api/conversations/[id]/tree failed:", err); + return NextResponse.json({ error: "Failed to fetch conversation" }, { status: 500 }); + } +} diff --git a/src/app/api/conversations/route.ts b/src/app/api/conversations/route.ts index e8e8d2959b..7996c2a5ec 100644 --- a/src/app/api/conversations/route.ts +++ b/src/app/api/conversations/route.ts @@ -1,6 +1,7 @@ import { NextResponse } from "next/server"; import { requireManagementAuth } from "@/lib/api/requireManagementAuth"; import { listMultiTurnConversations } from "@/lib/db/agenticConversations"; +import { getPendingById } from "@/lib/usage/usageHistory"; export const dynamic = "force-dynamic"; @@ -18,7 +19,19 @@ export async function GET(req: Request) { offset: Number.isFinite(offset) ? offset : undefined, }); - return NextResponse.json({ conversations: rows, total }); + // A pending (still-streaming) request's sessionTag is the conversation's + // own id (agentic_conversations.id === call_logs.session_tag) — cross + // reference so the list can show "in progress" without a separate poll. + const activeConversationIds = new Set(); + for (const pending of getPendingById().values()) { + if (pending.sessionTag) activeConversationIds.add(pending.sessionTag); + } + const conversations = rows.map((row) => ({ + ...row, + isActive: activeConversationIds.has(row.id), + })); + + return NextResponse.json({ conversations, total }); } catch (err) { console.error("[API ERROR] /api/conversations failed:", err); return NextResponse.json({ error: "Failed to fetch conversations" }, { status: 500 }); diff --git a/src/lib/db/agenticConversations.ts b/src/lib/db/agenticConversations.ts index 0eeceee689..8086059be6 100644 --- a/src/lib/db/agenticConversations.ts +++ b/src/lib/db/agenticConversations.ts @@ -1,9 +1,14 @@ /** - * db/agenticConversations.ts — CRUD for the agentic conversation-tracking table. + * db/agenticConversations.ts — CRUD for the agentic conversation-tracking + * tables: `agentic_conversations` (one row per conversation tree root) and + * `conversation_turn_nodes` (one row per distinct turn instance, chained to + * its predecessor — see open-sse/services/conversationTracker.ts for how the + * chain hash is computed and walked to detect continuations/forks). * - * See open-sse/services/conversationTracker.ts for how `fingerprint_hash` and - * `last_messages_hash` are computed and used to detect that a new request is a - * continuation of a previous one. + * `last_message_count`/`last_messages_hash` on `agentic_conversations` are + * dead columns kept only for backward on-disk compatibility (superseded by + * `conversation_turn_nodes`, migration 136) — never read, written as + * placeholders. */ import { v4 as uuidv4 } from "uuid"; @@ -13,8 +18,6 @@ export interface AgenticConversationRow { id: string; apiKeyId: string | null; fingerprintHash: string; - lastMessageCount: number; - lastMessagesHash: string; turnCount: number; firstSeenAt: string; lastSeenAt: string; @@ -32,8 +35,6 @@ function toRow(value: unknown): AgenticConversationRow { id: String(r.id ?? ""), apiKeyId: typeof r.api_key_id === "string" ? r.api_key_id : null, fingerprintHash: String(r.fingerprint_hash ?? ""), - lastMessageCount: Number(r.last_message_count ?? 0), - lastMessagesHash: String(r.last_messages_hash ?? ""), turnCount: Number(r.turn_count ?? 1), firstSeenAt: String(r.first_seen_at ?? ""), lastSeenAt: String(r.last_seen_at ?? ""), @@ -44,33 +45,23 @@ export function createAgenticConversation(input: { id?: string; apiKeyId: string | null; fingerprintHash: string; - lastMessageCount: number; - lastMessagesHash: string; }): AgenticConversationRow { const db = getDbInstance(); const now = new Date().toISOString(); const id = input.id || `conv_${uuidv4()}`; + // last_message_count/last_messages_hash are dead columns (superseded by + // conversation_turn_nodes, migration 136) — 0/'' placeholders only. db.prepare( `INSERT INTO agentic_conversations (id, api_key_id, fingerprint_hash, last_message_count, last_messages_hash, turn_count, first_seen_at, last_seen_at) - VALUES (?, ?, ?, ?, ?, 1, ?, ?)` - ).run( - id, - input.apiKeyId, - input.fingerprintHash, - input.lastMessageCount, - input.lastMessagesHash, - now, - now - ); + VALUES (?, ?, ?, 0, '', 1, ?, ?)` + ).run(id, input.apiKeyId, input.fingerprintHash, now, now); return { id, apiKeyId: input.apiKeyId, fingerprintHash: input.fingerprintHash, - lastMessageCount: input.lastMessageCount, - lastMessagesHash: input.lastMessagesHash, turnCount: 1, firstSeenAt: now, lastSeenAt: now, @@ -89,16 +80,265 @@ export function findAgenticConversationsByFingerprint( return rows.map(toRow); } -export function updateAgenticConversation( - id: string, - patch: { lastMessageCount: number; lastMessagesHash: string; turnCount: number } -): void { +export function updateAgenticConversation(id: string, patch: { turnCount: number }): void { const db = getDbInstance(); - db.prepare( - `UPDATE agentic_conversations - SET last_message_count = ?, last_messages_hash = ?, turn_count = ?, last_seen_at = ? - WHERE id = ?` - ).run(patch.lastMessageCount, patch.lastMessagesHash, patch.turnCount, new Date().toISOString(), id); + db.prepare(`UPDATE agentic_conversations SET turn_count = ?, last_seen_at = ? WHERE id = ?`).run( + patch.turnCount, + new Date().toISOString(), + id + ); +} + +// ── Turn-node tree (migration 136) ─────────────────────────────────────── + +export interface ConversationTurnNode { + id: string; + conversationId: string; + parentId: string | null; + role: string; + textPreview: string; + /** 'text' | 'tool_use' | 'tool_result' — lets a consumer build the exact + * NormalizedBlock (src/mitm/inspector/types.ts) the request-detail panel + * already builds, so tool calls/results render through the same + * ChatBubble/MessageContent/ToolCallBlock/ToolResultBlock components + * everywhere instead of a parallel implementation. */ + blockKind: string; + /** Set only when blockKind === 'tool_use'. */ + toolName: string | null; + lastCorrelationId: string | null; + firstSeenAt: string; + lastSeenAt: string; +} + +function toTurnNode(value: unknown): ConversationTurnNode { + const r = asRecord(value); + return { + id: String(r.id ?? ""), + conversationId: String(r.conversation_id ?? ""), + parentId: typeof r.parent_id === "string" ? r.parent_id : null, + role: String(r.role ?? ""), + textPreview: String(r.text_preview ?? ""), + blockKind: String(r.block_kind ?? "text"), + toolName: typeof r.tool_name === "string" ? r.tool_name : null, + lastCorrelationId: typeof r.last_correlation_id === "string" ? r.last_correlation_id : null, + firstSeenAt: String(r.first_seen_at ?? ""), + lastSeenAt: String(r.last_seen_at ?? ""), + }; +} + +export interface ConversationTurnIndex { + /** Every existing node id for the chain — O(1) forward-walk membership checks. */ + nodeIds: Set; + /** + * contentHash (sha256 of just a turn's own role+text, independent of + * parent) -> node ids sharing that content. Lets resolveConversationId + * find a reconnection point ANYWHERE in the chain, not only at its start — + * real OpenClaw traffic drops/summarizes the earliest turns as a session + * grows, so a new request's turn 0 is often not the chain's own first turn. + */ + byContentHash: Map; + /** + * Node ids that already have at least one recorded child. Lets + * resolveConversationId distinguish "this turn is genuinely new" (the + * reconnect anchor has no child yet — safe to extend this SAME + * conversation) from "a turn already exists at this position and this + * request's turn differs from it" (an edit — becomes its own independent + * conversation as of the 2026-08-06 redesign; see resolveConversationId's + * doc comment for why conversations no longer fork in place). + */ + parentsWithChildren: Set; +} + +/** + * Bulk-load a conversation chain's node ids and content-hash index in one + * query, for the reconnect-anchor search in resolveConversationId + * (conversationTracker.ts). Chains are small in practice (tens to low + * hundreds of turns), so one bulk load per candidate is cheap. + */ +export function getConversationTurnIndex(conversationId: string): ConversationTurnIndex { + const db = getDbInstance(); + const rows = db + .prepare( + `SELECT id, parent_id, content_hash FROM conversation_turn_nodes WHERE conversation_id = ?` + ) + .all(conversationId); + const nodeIds = new Set(); + const byContentHash = new Map(); + const parentsWithChildren = new Set(); + for (const r of rows) { + const rec = asRecord(r); + const id = String(rec.id ?? ""); + const parentId = typeof rec.parent_id === "string" ? rec.parent_id : null; + const contentHash = String(rec.content_hash ?? ""); + nodeIds.add(id); + if (parentId) parentsWithChildren.add(parentId); + if (!contentHash) continue; + const bucket = byContentHash.get(contentHash); + if (bucket) bucket.push(id); + else byContentHash.set(contentHash, [id]); + } + return { nodeIds, byContentHash, parentsWithChildren }; +} + +/** + * Insert a run of new turn nodes (a fresh branch, or the whole chain for a + * brand-new conversation). Nodes already existing on this chain are never + * re-inserted or touched here — only the newly-diverging tail from the + * fork/reconnect point onward reaches this function (see conversationTracker.ts). + */ +export function insertConversationTurnNodes( + conversationId: string, + correlationId: string | null, + nodes: Array<{ + id: string; + parentId: string | null; + role: string; + contentHash: string; + textPreview: string; + blockKind: string; + toolName: string | null; + }> +): void { + if (nodes.length === 0) return; + const db = getDbInstance(); + const now = new Date().toISOString(); + const insert = db.prepare( + `INSERT OR IGNORE INTO conversation_turn_nodes + (id, conversation_id, parent_id, role, content_hash, text_preview, block_kind, tool_name, last_correlation_id, first_seen_at, last_seen_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ); + const insertMany = db.transaction((rows: typeof nodes) => { + for (const node of rows) { + insert.run( + node.id, + conversationId, + node.parentId, + node.role, + node.contentHash, + node.textPreview, + node.blockKind, + node.toolName, + correlationId, + now, + now + ); + } + }); + insertMany(nodes); +} + +/** All turn nodes for a conversation, in order — for tests and any caller + * that genuinely needs the whole chain in one shot. The dashboard itself + * uses the paginated getConversationTurnPage below instead, since a long + * real OpenClaw conversation can run to hundreds of turns. */ +export function getConversationTurnTree(conversationId: string): ConversationTurnNode[] { + const db = getDbInstance(); + const rows = db + .prepare( + `SELECT * FROM conversation_turn_nodes WHERE conversation_id = ? ORDER BY first_seen_at ASC` + ) + .all(conversationId); + return rows.map(toTurnNode); +} + +export interface ConversationTurnNodeWithSeq extends ConversationTurnNode { + /** SQLite rowid — a stable, monotonically-increasing insertion-order + * cursor for pagination (first_seen_at can tie within the same request's + * batch insert; rowid never does). */ + seq: number; +} + +export interface ConversationTurnPage { + /** Ascending order (oldest first) within the page. */ + nodes: ConversationTurnNodeWithSeq[]; + /** True when older turns exist beyond this page (only meaningful for the + * initial load / beforeSeq cases — always false for afterSeq/poll). */ + hasMore: boolean; +} + +/** + * Paginated turn fetch for the /dashboard/conversations view: a real + * OpenClaw conversation can run to hundreds of turns, so the page always + * loads the most recent `limit` (default 20) rather than everything. + * - No cursor: initial load — the LAST `limit` turns. + * - `beforeSeq`: "load more" (older) — the `limit` turns immediately before it. + * - `afterSeq`: poll for new turns since the last load — everything newer, + * uncapped (a handful of turns in practice). + * Only one of beforeSeq/afterSeq is meaningful per call; afterSeq wins if + * both are somehow given. + */ +export function getConversationTurnPage( + conversationId: string, + opts: { limit?: number; beforeSeq?: number; afterSeq?: number } = {} +): ConversationTurnPage { + const db = getDbInstance(); + const limit = Math.max(1, Math.min(opts.limit ?? 20, 500)); + + if (opts.afterSeq != null) { + const rows = db + .prepare( + `SELECT rowid as seq, * FROM conversation_turn_nodes + WHERE conversation_id = ? AND rowid > ? ORDER BY rowid ASC` + ) + .all(conversationId, opts.afterSeq); + return { nodes: rows.map(toTurnNodeWithSeq), hasMore: false }; + } + + const rows = + opts.beforeSeq != null + ? db + .prepare( + `SELECT rowid as seq, * FROM conversation_turn_nodes + WHERE conversation_id = ? AND rowid < ? ORDER BY rowid DESC LIMIT ?` + ) + .all(conversationId, opts.beforeSeq, limit + 1) + : db + .prepare( + `SELECT rowid as seq, * FROM conversation_turn_nodes + WHERE conversation_id = ? ORDER BY rowid DESC LIMIT ?` + ) + .all(conversationId, limit + 1); + + const hasMore = rows.length > limit; + const page = (hasMore ? rows.slice(0, limit) : rows).reverse(); + return { nodes: page.map(toTurnNodeWithSeq), hasMore }; +} + +function toTurnNodeWithSeq(value: unknown): ConversationTurnNodeWithSeq { + const rec = asRecord(value); + return { ...toTurnNode(value), seq: Number(rec.seq ?? 0) }; +} + +/** + * Turn nodes are tagged with `last_correlation_id` (see + * conversationTracker.ts's doc comment for why — the request's own + * call_logs.id doesn't exist yet when a node is created), not a call_logs.id + * directly. Resolves the set in one bulk query (not one lookup per node) to + * a `correlation_id -> call_logs.id` map so the tree view can link each node + * to a navigable request. + */ +export function resolveCallLogIdsByCorrelationIds(correlationIds: string[]): Map { + const unique = [...new Set(correlationIds.filter(Boolean))]; + const result = new Map(); + if (unique.length === 0) return result; + + const db = getDbInstance(); + const placeholders = unique.map(() => "?").join(","); + const rows = db + .prepare(`SELECT id, correlation_id FROM call_logs WHERE correlation_id IN (${placeholders})`) + .all(...unique); + for (const r of rows) { + const rec = asRecord(r); + const correlationId = typeof rec.correlation_id === "string" ? rec.correlation_id : null; + const callLogId = typeof rec.id === "string" ? rec.id : null; + // Keep the first match per correlation_id — a retry/combo-fallback can + // theoretically share one correlation_id across a couple of call_logs + // rows; any of them is a valid navigation target. + if (correlationId && callLogId && !result.has(correlationId)) { + result.set(correlationId, callLogId); + } + } + return result; } /** @@ -130,45 +370,6 @@ export function touchOrCreateExternalConversation( ).run(id, ctx.apiKeyId, now, now); } -/** - * Exact-match lookup of the most recent call_logs row tagged with this - * conversation id — used to build the "Full Conversation" transcript panel. - * Deliberately NOT `pushLikeFilter`-based (that's substring/LIKE matching, - * wrong for an exact id join). - */ -export function getLatestCallLogForConversation(conversationId: string): JsonRecord | null { - const db = getDbInstance(); - const row = db - .prepare(`SELECT * FROM call_logs WHERE session_tag = ? ORDER BY timestamp DESC LIMIT 1`) - .get(conversationId); - return row ? asRecord(row) : null; -} - -export interface CallLogRef { - id: string; - timestamp: string; -} - -/** - * Every call_logs row for a conversation, ascending by timestamp — the sibling - * of getLatestCallLogForConversation (DESC/LIMIT 1) used by the multi-row - * transcript builder (src/mitm/inspector/multiRowConversation.ts). Only - * id/timestamp are needed here; full bodies are loaded per-row via the - * existing getCallLogById (src/lib/usage/callLogs.ts), matching the - * established exportCallLogsSince loop pattern rather than a bespoke bulk - * loader. - */ -export function getAllCallLogsForConversation(conversationId: string): CallLogRef[] { - const db = getDbInstance(); - const rows = db - .prepare(`SELECT id, timestamp FROM call_logs WHERE session_tag = ? ORDER BY timestamp ASC`) - .all(conversationId); - return rows.map((r) => { - const rec = asRecord(r); - return { id: String(rec.id ?? ""), timestamp: String(rec.timestamp ?? "") }; - }); -} - export interface MultiTurnConversationRow extends AgenticConversationRow { lastCallLogId: string | null; lastModel: string | null; @@ -177,22 +378,42 @@ export interface MultiTurnConversationRow extends AgenticConversationRow { } /** - * Conversations with turn_count >= 2 — filters out one-shot, non-agentic - * traffic for the /dashboard/conversations list page. Joined to each + * Conversations with >= 2 actual turn nodes — filters out one-shot, + * non-agentic traffic for the /dashboard/conversations list page. + * + * Deliberately filters on conversation_turn_nodes COUNT, not `turn_count`: + * `turn_count` tracks how many separate REQUESTS have touched this + * conversation (see createAgenticConversation/updateAgenticConversation), + * not how many turns it contains. A brand-new conversation minted by + * resolveConversationId's divergence path (see conversationTracker.ts) + * starts at turn_count=1 even though its first insert can carry the + * conversation's entire prior history (hundreds of nodes) — real OpenClaw + * traffic diverges/edits turns often enough that most conversations never + * accumulate a second touching request, so a turn_count-based filter left + * them permanently invisible despite having a rich multi-turn transcript + * (2026-08-06, request 1785975096139-6627d2 / conv_36fff6fa...: turn_count=1, + * 398 real conversation_turn_nodes rows). Joined to each * conversation's most recent call_logs row (by MAX(timestamp) per * session_tag) in a single query rather than one lookup per row, since this * is a list view that can have many conversations. */ -export function listMultiTurnConversations(filter: { - limit?: number; - offset?: number; -} = {}): { rows: MultiTurnConversationRow[]; total: number } { +export function listMultiTurnConversations( + filter: { + limit?: number; + offset?: number; + } = {} +): { rows: MultiTurnConversationRow[]; total: number } { const db = getDbInstance(); const limit = Math.max(1, Math.min(filter.limit ?? 50, 200)); const offset = Math.max(0, filter.offset ?? 0); const total = asRecord( - db.prepare(`SELECT COUNT(*) as c FROM agentic_conversations WHERE turn_count >= 2`).get() + db + .prepare( + `SELECT COUNT(*) as c FROM agentic_conversations ac + WHERE (SELECT COUNT(*) FROM conversation_turn_nodes n WHERE n.conversation_id = ac.id) >= 2` + ) + .get() ).c as number; const rows = db @@ -207,7 +428,7 @@ export function listMultiTurnConversations(filter: { SELECT MAX(cl2.timestamp) FROM call_logs cl2 WHERE cl2.session_tag = cl1.session_tag ) ) latest ON latest.session_tag = ac.id - WHERE ac.turn_count >= 2 + WHERE (SELECT COUNT(*) FROM conversation_turn_nodes n WHERE n.conversation_id = ac.id) >= 2 ORDER BY ac.last_seen_at DESC LIMIT ? OFFSET ?` ) diff --git a/src/lib/db/migrations/136_conversation_turn_nodes.sql b/src/lib/db/migrations/136_conversation_turn_nodes.sql new file mode 100644 index 0000000000..16718ec454 --- /dev/null +++ b/src/lib/db/migrations/136_conversation_turn_nodes.sql @@ -0,0 +1,61 @@ +-- 136_conversation_turn_nodes.sql +-- Per-turn hash-chain nodes backing a conversation's linear transcript +-- (open-sse/services/conversationTracker.ts). Each row is one distinct +-- turn instance, chained to its predecessor the same way a git commit +-- graph chains commits: id = sha256(parentId ?? conversationId, role, +-- sha256(text)). A request whose turns all match existing nodes just +-- extends the chain; a request whose Nth turn differs from what's on file +-- (a real-world OpenClaw pattern: cache-aware context injection edits/ +-- duplicates a turn mid-history to keep provider-side prompt caches warm) +-- becomes its OWN independent conversation instead (2026-08-06 — every +-- OmniRoute conversation is a single straight line, it never forks; see +-- resolveConversationId's doc comment). See agentic_conversations (135) for +-- the conversation-root record; its last_message_count/last_messages_hash +-- columns are superseded by this table and no longer written to. +-- +-- content_hash (sha256 of just this turn's own role+text, independent of +-- parent) lets resolveConversationId reconnect from ANY existing node, not +-- only the chain's start: real OpenClaw traffic drops/summarizes the +-- EARLIEST turns as a session grows (a sliding context window), so the turn +-- at index 0 of a new request is often not the chain's first turn at all — +-- a start-only walk would find zero match and mint a spurious new +-- conversation despite a real, unbroken tail overlap. + +CREATE TABLE IF NOT EXISTS conversation_turn_nodes ( + id TEXT PRIMARY KEY, -- chain hash, see above + conversation_id TEXT NOT NULL, -- agentic_conversations.id (this chain's conversation) + parent_id TEXT, -- NULL for the first turn of the chain + role TEXT NOT NULL, + content_hash TEXT NOT NULL DEFAULT '', -- sha256(role+text) alone, for reconnect-anchor lookup + text_preview TEXT NOT NULL, -- extracted turn text (bounded, see + -- conversationTracker.ts's + -- TEXT_PREVIEW_LENGTH) — rendered as + -- full markdown in the + -- /dashboard/conversations view, not + -- truncated to a one-line label + block_kind TEXT NOT NULL DEFAULT 'text', -- 'text' | 'tool_use' | 'tool_result' — lets + -- the conversation view build the exact same + -- NormalizedBlock (src/mitm/inspector/types.ts) the + -- request-detail panel already builds from + -- buildRequestTurns/buildResponseTurns, so both render + -- tool calls/results through the SAME ChatBubble/ + -- MessageContent/ToolCallBlock/ToolResultBlock + -- components — not a parallel implementation + tool_name TEXT, -- set only for block_kind='tool_use' + last_correlation_id TEXT, -- call_logs.correlation_id of the request that + -- (re)touched this node — resolveConversationId + -- runs before the call_logs row's own id exists, + -- but correlation_id (109_call_logs_correlation_id) + -- is already generated earlier in the same request + -- and is available synchronously; the API route + -- joins through it to resolve a navigable call_logs.id + first_seen_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_turn_nodes_conversation + ON conversation_turn_nodes(conversation_id); +CREATE INDEX IF NOT EXISTS idx_turn_nodes_parent + ON conversation_turn_nodes(parent_id); +CREATE INDEX IF NOT EXISTS idx_turn_nodes_content_hash + ON conversation_turn_nodes(conversation_id, content_hash); diff --git a/src/lib/usage/usageHistory.ts b/src/lib/usage/usageHistory.ts index 0ed7ef451e..3d9b0dfa68 100644 --- a/src/lib/usage/usageHistory.ts +++ b/src/lib/usage/usageHistory.ts @@ -145,21 +145,43 @@ function normalizePendingMetadata(metadata?: PendingRequestMetadata): PendingReq // ──────────────── Pending Requests (in-memory) ──────────────── -const pendingRequests: { - byModel: Record; - byAccount: Record>; - details: Record>; -} = { - byModel: Object.create(null) as Record, - byAccount: Object.create(null) as Record>, - details: Object.create(null) as Record>, -}; +declare global { + var __omnirouteUsageHistoryPendingState: + | { + pendingRequests: { + byModel: Record; + byAccount: Record>; + details: Record>; + }; + pendingById: Map; + } + | undefined; +} + +// Reuse the SAME object/Map across Next.js dev HMR module re-evaluations — +// same pattern (and reason) as src/lib/db/core.ts's `globalThis.__omnirouteDb`. +// Without this, an edit anywhere in this module's dependency graph resets +// in-flight request tracking to empty mid-stream, so a live poll against +// getPendingById() (RequestLoggerDetail.tsx's Conversation Context section) +// silently stops seeing partialAssistantText for a request that started +// before the reload — the request keeps streaming fine, but the *next* +// module instance's pendingById has never heard of it. +const pendingState = (globalThis.__omnirouteUsageHistoryPendingState ??= { + pendingRequests: { + byModel: Object.create(null) as Record, + byAccount: Object.create(null) as Record>, + details: Object.create(null) as Record>, + }, + pendingById: new Map(), +}); + +const pendingRequests = pendingState.pendingRequests; /** * O(1) ID → PendingRequestDetail lookup map. * Populated when a detail is created and cleaned up when it is removed/finalized. */ -const pendingById = new Map(); +const pendingById = pendingState.pendingById; const DEFAULT_MAX_PENDING_REQUEST_AGE_MS = 60 * 60 * 1000; const MAX_PENDING_DETAILS = 5000; diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 97905d92f2..3d5d6f6909 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -543,6 +543,7 @@ async function handleChatImplementation( model: modelStr, apiKeyId: apiKeyInfo?.id ?? null, clientSessionIdHeader: clientConversationHeader, + correlationId: reqId, }); // T08: per-key active session limit (0 = unlimited). @@ -946,7 +947,10 @@ async function handleChatImplementation( }); } catch {} } - return withConversationId(withCorrelationId(withSessionHeader(response, sessionId), reqId), conversationId); + return withConversationId( + withCorrelationId(withSessionHeader(response, sessionId), reqId), + conversationId + ); } telemetry.endPhase(); @@ -989,7 +993,10 @@ async function handleChatImplementation( false ); recordTelemetry(telemetry); - return withConversationId(withCorrelationId(withSessionHeader(response, sessionId), reqId), conversationId); + return withConversationId( + withCorrelationId(withSessionHeader(response, sessionId), reqId), + conversationId + ); } export const handleChat = chatAdmission.withChatAdmission(handleChatImplementation); diff --git a/tests/unit/agenticConversations.test.ts b/tests/unit/agenticConversations.test.ts index 54fbbe1f00..ec6908df9d 100644 --- a/tests/unit/agenticConversations.test.ts +++ b/tests/unit/agenticConversations.test.ts @@ -17,9 +17,12 @@ import { findAgenticConversationsByFingerprint, updateAgenticConversation, touchOrCreateExternalConversation, - getLatestCallLogForConversation, - getAllCallLogsForConversation, listMultiTurnConversations, + getConversationTurnIndex, + insertConversationTurnNodes, + getConversationTurnTree, + getConversationTurnPage, + resolveCallLogIdsByCorrelationIds, } from "../../src/lib/db/agenticConversations.ts"; import { getDbInstance } from "../../src/lib/db/core.ts"; @@ -27,8 +30,6 @@ test("createAgenticConversation + findAgenticConversationsByFingerprint round-tr const row = createAgenticConversation({ apiKeyId: "key-a", fingerprintHash: "fp-round-trip", - lastMessageCount: 1, - lastMessagesHash: "hash-1", }); assert.match(row.id, /^conv_/); @@ -41,43 +42,282 @@ test("createAgenticConversation + findAgenticConversationsByFingerprint round-tr }); test("findAgenticConversationsByFingerprint returns multiple rows for a shared fingerprint", () => { - createAgenticConversation({ - apiKeyId: "key-b", - fingerprintHash: "fp-shared", - lastMessageCount: 1, - lastMessagesHash: "hash-b1", - }); - createAgenticConversation({ - apiKeyId: "key-b", - fingerprintHash: "fp-shared", - lastMessageCount: 1, - lastMessagesHash: "hash-b2", - }); + createAgenticConversation({ apiKeyId: "key-b", fingerprintHash: "fp-shared" }); + createAgenticConversation({ apiKeyId: "key-b", fingerprintHash: "fp-shared" }); const found = findAgenticConversationsByFingerprint("fp-shared"); assert.equal(found.length, 2); }); -test("updateAgenticConversation updates message count/hash/turn count", () => { - const row = createAgenticConversation({ - apiKeyId: "key-c", - fingerprintHash: "fp-update", - lastMessageCount: 1, - lastMessagesHash: "hash-c1", - }); +test("updateAgenticConversation updates turn count", () => { + const row = createAgenticConversation({ apiKeyId: "key-c", fingerprintHash: "fp-update" }); - updateAgenticConversation(row.id, { - lastMessageCount: 5, - lastMessagesHash: "hash-c5", - turnCount: 3, - }); + updateAgenticConversation(row.id, { turnCount: 3 }); const found = findAgenticConversationsByFingerprint("fp-update"); - assert.equal(found[0].lastMessageCount, 5); - assert.equal(found[0].lastMessagesHash, "hash-c5"); assert.equal(found[0].turnCount, 3); }); +test("insertConversationTurnNodes + getConversationTurnIndex round-trip", () => { + const row = createAgenticConversation({ apiKeyId: "key-nodes", fingerprintHash: "fp-nodes" }); + + insertConversationTurnNodes(row.id, "corr-1", [ + { + id: "node-a", + parentId: null, + role: "user", + contentHash: "hash-a", + textPreview: "a", + blockKind: "text", + toolName: null, + }, + { + id: "node-b", + parentId: "node-a", + role: "assistant", + contentHash: "hash-b", + textPreview: "b", + blockKind: "text", + toolName: null, + }, + ]); + + const index = getConversationTurnIndex(row.id); + assert.equal(index.nodeIds.size, 2); + assert.ok(index.nodeIds.has("node-a")); + assert.ok(index.nodeIds.has("node-b")); + assert.deepEqual(index.byContentHash.get("hash-a"), ["node-a"]); + assert.deepEqual(index.byContentHash.get("hash-b"), ["node-b"]); + + // A different conversation's nodes must never leak into this index. + const other = createAgenticConversation({ + apiKeyId: "key-nodes-2", + fingerprintHash: "fp-nodes-2", + }); + insertConversationTurnNodes(other.id, "corr-2", [ + { + id: "node-c", + parentId: null, + role: "user", + contentHash: "hash-c", + textPreview: "c", + blockKind: "text", + toolName: null, + }, + ]); + const reReadIndex = getConversationTurnIndex(row.id); + assert.equal(reReadIndex.nodeIds.size, 2); + assert.equal(reReadIndex.byContentHash.has("hash-c"), false); +}); + +test("getConversationTurnIndex groups multiple node ids under the same content hash (duplicate turn text at different tree positions)", () => { + const row = createAgenticConversation({ apiKeyId: "key-dup-content", fingerprintHash: "fp-dup" }); + + insertConversationTurnNodes(row.id, "corr-1", [ + { + id: "node-1", + parentId: null, + role: "user", + contentHash: "hash-ok", + textPreview: "ok", + blockKind: "text", + toolName: null, + }, + { + id: "node-2", + parentId: "node-1", + role: "assistant", + contentHash: "hash-reply", + textPreview: "reply", + blockKind: "text", + toolName: null, + }, + // Same content ("ok") recurs later in the same tree, at a different node. + { + id: "node-3", + parentId: "node-2", + role: "user", + contentHash: "hash-ok", + textPreview: "ok", + blockKind: "text", + toolName: null, + }, + ]); + + const index = getConversationTurnIndex(row.id); + const matches = index.byContentHash.get("hash-ok"); + assert.equal(matches?.length, 2); + assert.deepEqual([...matches!].sort(), ["node-1", "node-3"]); +}); + +test("insertConversationTurnNodes is idempotent for already-existing node ids (INSERT OR IGNORE)", () => { + const row = createAgenticConversation({ apiKeyId: "key-idem", fingerprintHash: "fp-idem" }); + + insertConversationTurnNodes(row.id, "corr-1", [ + { + id: "node-dup", + parentId: null, + role: "user", + contentHash: "hash-dup", + textPreview: "first", + blockKind: "text", + toolName: null, + }, + ]); + // Re-insert the same id — must not throw, must not duplicate. + insertConversationTurnNodes(row.id, "corr-2", [ + { + id: "node-dup", + parentId: null, + role: "user", + contentHash: "hash-dup", + textPreview: "first", + blockKind: "text", + toolName: null, + }, + ]); + + const tree = getConversationTurnTree(row.id); + assert.equal(tree.length, 1); +}); + +test("getConversationTurnTree returns nodes with parent/child structure and text preview", () => { + const row = createAgenticConversation({ apiKeyId: "key-tree", fingerprintHash: "fp-tree" }); + + insertConversationTurnNodes(row.id, "corr-tree", [ + { + id: "root-turn", + parentId: null, + role: "user", + contentHash: "hash-hello", + textPreview: "hello", + blockKind: "text", + toolName: null, + }, + { + id: "child-turn", + parentId: "root-turn", + role: "assistant", + contentHash: "hash-hi", + textPreview: "hi there", + blockKind: "text", + toolName: null, + }, + ]); + // A sibling branch off the same parent. + insertConversationTurnNodes(row.id, "corr-tree-2", [ + { + id: "sibling-turn", + parentId: "root-turn", + role: "assistant", + contentHash: "hash-hey", + textPreview: "hey", + blockKind: "text", + toolName: null, + }, + ]); + + const tree = getConversationTurnTree(row.id); + assert.equal(tree.length, 3); + + const root = tree.find((n) => n.id === "root-turn"); + const children = tree.filter((n) => n.parentId === "root-turn"); + assert.equal(root?.parentId, null); + assert.equal(root?.textPreview, "hello"); + assert.equal(children.length, 2); + assert.deepEqual(children.map((c) => c.id).sort(), ["child-turn", "sibling-turn"]); +}); + +test("getConversationTurnPage: initial load returns only the last `limit` turns, oldest-first, with hasMore", () => { + const row = createAgenticConversation({ apiKeyId: "key-page", fingerprintHash: "fp-page" }); + const nodes = Array.from({ length: 25 }, (_, i) => ({ + id: `n${i}`, + parentId: i === 0 ? null : `n${i - 1}`, + role: i % 2 === 0 ? "user" : "assistant", + contentHash: `hash-${i}`, + textPreview: `turn-${i}`, + blockKind: "text", + toolName: null, + })); + insertConversationTurnNodes(row.id, "corr-page", nodes); + + const page = getConversationTurnPage(row.id, { limit: 20 }); + assert.equal(page.nodes.length, 20); + assert.equal(page.hasMore, true); + // Oldest-first within the page, and it's the LAST 20 (turn-5..turn-24). + assert.equal(page.nodes[0].textPreview, "turn-5"); + assert.equal(page.nodes[19].textPreview, "turn-24"); +}); + +test("getConversationTurnPage: beforeSeq loads the previous page (older turns), with correct hasMore", () => { + const row = createAgenticConversation({ apiKeyId: "key-page-2", fingerprintHash: "fp-page-2" }); + const nodes = Array.from({ length: 25 }, (_, i) => ({ + id: `m${i}`, + parentId: i === 0 ? null : `m${i - 1}`, + role: "user", + contentHash: `hash-m${i}`, + textPreview: `turn-${i}`, + blockKind: "text", + toolName: null, + })); + insertConversationTurnNodes(row.id, "corr-page-2", nodes); + + const firstPage = getConversationTurnPage(row.id, { limit: 20 }); + const oldestSeqInFirstPage = firstPage.nodes[0].seq; + + const olderPage = getConversationTurnPage(row.id, { limit: 20, beforeSeq: oldestSeqInFirstPage }); + assert.equal(olderPage.nodes.length, 5, "only 5 turns (0-4) exist before the first page"); + assert.equal(olderPage.hasMore, false); + assert.equal(olderPage.nodes[0].textPreview, "turn-0"); + assert.equal(olderPage.nodes[4].textPreview, "turn-4"); +}); + +test("getConversationTurnPage: afterSeq returns only turns newer than the cursor (for polling), uncapped", () => { + const row = createAgenticConversation({ apiKeyId: "key-page-3", fingerprintHash: "fp-page-3" }); + insertConversationTurnNodes(row.id, "corr-page-3", [ + { + id: "p0", + parentId: null, + role: "user", + contentHash: "h0", + textPreview: "a", + blockKind: "text", + toolName: null, + }, + { + id: "p1", + parentId: "p0", + role: "assistant", + contentHash: "h1", + textPreview: "b", + blockKind: "text", + toolName: null, + }, + ]); + const firstPage = getConversationTurnPage(row.id, { limit: 20 }); + const newestSeq = firstPage.nodes[firstPage.nodes.length - 1].seq; + + // Nothing new yet. + assert.equal(getConversationTurnPage(row.id, { afterSeq: newestSeq }).nodes.length, 0); + + // A new turn arrives (e.g. a later request continuing this conversation). + insertConversationTurnNodes(row.id, "corr-page-3b", [ + { + id: "p2", + parentId: "p1", + role: "user", + contentHash: "h2", + textPreview: "c", + blockKind: "text", + toolName: null, + }, + ]); + const polled = getConversationTurnPage(row.id, { afterSeq: newestSeq }); + assert.equal(polled.nodes.length, 1); + assert.equal(polled.nodes[0].textPreview, "c"); + assert.equal(polled.hasMore, false); +}); + test("touchOrCreateExternalConversation creates then increments turn_count on repeat calls", () => { const id = "ext-conv-test-id"; touchOrCreateExternalConversation(id, { apiKeyId: "key-d" }); @@ -95,76 +335,56 @@ test("touchOrCreateExternalConversation creates then increments turn_count on re assert.equal(afterTouch.turn_count, 2); }); -test("getLatestCallLogForConversation returns the most recent row with an exact session_tag match", () => { - const db = getDbInstance(); - const conversationId = "conv-for-latest-lookup"; - - db.prepare( - `INSERT INTO call_logs (id, timestamp, method, path, status, model, session_tag) - VALUES (?, ?, 'POST', '/v1/chat/completions', 200, 'big-pickle', ?)` - ).run("call-older", "2026-01-01T00:00:00.000Z", conversationId); - db.prepare( - `INSERT INTO call_logs (id, timestamp, method, path, status, model, session_tag) - VALUES (?, ?, 'POST', '/v1/chat/completions', 200, 'big-pickle', ?)` - ).run("call-newer", "2026-01-01T00:05:00.000Z", conversationId); - // A prefix-matching but NOT exact tag must never match (guards against - // accidentally reusing a LIKE-based filter for this exact-match lookup). - db.prepare( - `INSERT INTO call_logs (id, timestamp, method, path, status, model, session_tag) - VALUES (?, ?, 'POST', '/v1/chat/completions', 200, 'big-pickle', ?)` - ).run("call-prefix-decoy", "2026-01-01T00:10:00.000Z", conversationId + "-extra"); - - const latest = getLatestCallLogForConversation(conversationId); - assert.equal(latest?.id, "call-newer"); -}); - -test("getAllCallLogsForConversation returns every row ascending, exact-match only", () => { - const db = getDbInstance(); - const conversationId = "conv-for-all-rows"; - - db.prepare( - `INSERT INTO call_logs (id, timestamp, method, path, status, model, session_tag) - VALUES (?, ?, 'POST', '/v1/chat/completions', 200, 'big-pickle', ?)` - ).run("all-newer", "2026-02-01T00:05:00.000Z", conversationId); - db.prepare( - `INSERT INTO call_logs (id, timestamp, method, path, status, model, session_tag) - VALUES (?, ?, 'POST', '/v1/chat/completions', 200, 'big-pickle', ?)` - ).run("all-older", "2026-02-01T00:00:00.000Z", conversationId); - db.prepare( - `INSERT INTO call_logs (id, timestamp, method, path, status, model, session_tag) - VALUES (?, ?, 'POST', '/v1/chat/completions', 200, 'big-pickle', ?)` - ).run("all-decoy", "2026-02-01T00:03:00.000Z", conversationId + "-extra"); - - const rows = getAllCallLogsForConversation(conversationId); - assert.deepEqual( - rows.map((r) => r.id), - ["all-older", "all-newer"] - ); -}); - -test("listMultiTurnConversations only returns conversations with turn_count >= 2, joined to their latest call_logs row", () => { +test("listMultiTurnConversations only returns conversations with >= 2 actual turn nodes, joined to their latest call_logs row", () => { const db = getDbInstance(); createAgenticConversation({ id: "conv-single-turn", apiKeyId: null, fingerprintHash: "fp-single", - lastMessageCount: 1, - lastMessagesHash: "h1", }); + insertConversationTurnNodes("conv-single-turn", "corr-single", [ + { + id: "single-node-1", + parentId: null, + role: "user", + contentHash: "hash-single-1", + textPreview: "hi", + blockKind: "text", + toolName: null, + }, + ]); const multi = createAgenticConversation({ id: "conv-multi-turn", apiKeyId: null, fingerprintHash: "fp-multi", - lastMessageCount: 3, - lastMessagesHash: "h2", - }); - updateAgenticConversation(multi.id, { - lastMessageCount: 3, - lastMessagesHash: "h2", - turnCount: 2, }); + // turn_count deliberately left at its default of 1 here: it tracks + // requests-touched, not node count, and a freshly-minted conversation can + // already carry many turn nodes from a single insert (see the doc comment + // on listMultiTurnConversations) — the filter must key off actual node + // count, not turn_count, for this conversation to be listed at all. + insertConversationTurnNodes(multi.id, "corr-multi", [ + { + id: "multi-node-1", + parentId: null, + role: "user", + contentHash: "hash-multi-1", + textPreview: "hi", + blockKind: "text", + toolName: null, + }, + { + id: "multi-node-2", + parentId: "multi-node-1", + role: "assistant", + contentHash: "hash-multi-2", + textPreview: "hello", + blockKind: "text", + toolName: null, + }, + ]); db.prepare( `INSERT INTO call_logs (id, timestamp, method, path, status, model, provider, session_tag) @@ -186,3 +406,26 @@ test("listMultiTurnConversations only returns conversations with turn_count >= 2 assert.equal(found?.lastModel, "gemma-4"); assert.equal(found?.lastProvider, "gemini"); }); + +test("resolveCallLogIdsByCorrelationIds bulk-resolves correlation_id to call_logs.id", () => { + const db = getDbInstance(); + + db.prepare( + `INSERT INTO call_logs (id, timestamp, method, path, status, model, correlation_id) + VALUES (?, ?, 'POST', '/v1/chat/completions', 200, 'big-pickle', ?)` + ).run("call-corr-1", "2026-04-01T00:00:00.000Z", "corr-a"); + db.prepare( + `INSERT INTO call_logs (id, timestamp, method, path, status, model, correlation_id) + VALUES (?, ?, 'POST', '/v1/chat/completions', 200, 'big-pickle', ?)` + ).run("call-corr-2", "2026-04-01T00:01:00.000Z", "corr-b"); + + const resolved = resolveCallLogIdsByCorrelationIds(["corr-a", "corr-b", "corr-missing"]); + assert.equal(resolved.get("corr-a"), "call-corr-1"); + assert.equal(resolved.get("corr-b"), "call-corr-2"); + assert.equal(resolved.has("corr-missing"), false); +}); + +test("resolveCallLogIdsByCorrelationIds returns an empty map for an empty/all-falsy input", () => { + assert.equal(resolveCallLogIdsByCorrelationIds([]).size, 0); + assert.equal(resolveCallLogIdsByCorrelationIds(["", ""]).size, 0); +}); diff --git a/tests/unit/conversationTracker.test.ts b/tests/unit/conversationTracker.test.ts index 4529f01b6a..51a9508749 100644 --- a/tests/unit/conversationTracker.test.ts +++ b/tests/unit/conversationTracker.test.ts @@ -16,10 +16,15 @@ process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "conversation-tracker import { extractCanonicalTurns, computeFingerprintHash, - hashTurnsBounded, resolveConversationId, } from "../../open-sse/services/conversationTracker.ts"; -import { findAgenticConversationsByFingerprint } from "../../src/lib/db/agenticConversations.ts"; +import { getConversationTurnTree } from "../../src/lib/db/agenticConversations.ts"; + +let correlationCounter = 0; +function nextCorrelationId(): string { + correlationCounter += 1; + return `corr-${correlationCounter}`; +} test("extractCanonicalTurns: OpenAI messages array", () => { const turns = extractCanonicalTurns({ @@ -46,8 +51,55 @@ test("extractCanonicalTurns: Responses API input array", () => { }); assert.equal(turns.length, 3); assert.equal(turns[0].role, "user"); + // Regression: content-block arrays (Responses API's `input_text`/ + // `output_text` shape) must extract their `.text`, not JSON.stringify the + // whole block array — a raw JSON blob here directly becomes what + // /dashboard/conversations renders as a turn's text. + assert.equal(turns[0].text, "check the file"); assert.equal(turns[1].role, "tool"); + // `arguments` here is already a JSON string (how OpenAI/Responses API send + // tool-call arguments) — stringifyContent passes strings through as-is, + // only the content-BLOCK-ARRAY case (turns[0] above) needed the fix. + assert.equal(turns[1].text, '{"command":"ls"}'); assert.equal(turns[2].role, "tool"); + assert.equal(turns[2].text, "ok"); + + // blockKind/toolName let a consumer (the /dashboard/conversations tree) + // build the same NormalizedBlock shape the request-detail panel already + // builds, so tool calls/results render through the same ChatBubble/ + // MessageContent/ToolCallBlock/ToolResultBlock components everywhere. + assert.equal(turns[0].blockKind, "text"); + assert.equal(turns[0].toolName, null); + assert.equal(turns[1].blockKind, "tool_use"); + assert.equal(turns[1].toolName, "exec"); + assert.equal(turns[2].blockKind, "tool_result"); + assert.equal(turns[2].toolName, null); +}); + +test("extractCanonicalTurns: Chat Completions tool-result message (role: tool) classifies as tool_result", () => { + const turns = extractCanonicalTurns({ + messages: [ + { role: "user", content: "what's the weather?" }, + { role: "tool", tool_call_id: "c1", content: '{"tempC":21}' }, + ], + }); + assert.equal(turns[0].blockKind, "text"); + assert.equal(turns[1].role, "tool"); + assert.equal(turns[1].blockKind, "tool_result"); + assert.equal(turns[1].text, '{"tempC":21}'); +}); + +test("extractCanonicalTurns: content-block arrays (Anthropic/Responses-API shape) extract text, not raw JSON", () => { + const turns = extractCanonicalTurns({ + messages: [ + { role: "user", content: [{ type: "text", text: "hello there" }] }, + { role: "assistant", content: [{ type: "output_text", text: "hi back" }] }, + ], + }); + assert.equal(turns[0].text, "hello there"); + assert.equal(turns[1].text, "hi back"); + assert.ok(!turns[0].text.includes("{"), "must not contain raw JSON"); + assert.ok(!turns[1].text.includes("{"), "must not contain raw JSON"); }); test("extractCanonicalTurns: Responses API bare-string input", () => { @@ -58,31 +110,51 @@ test("extractCanonicalTurns: Responses API bare-string input", () => { }); test("computeFingerprintHash: same inputs produce the same hash", () => { - const turns = extractCanonicalTurns({ messages: [{ role: "user", content: "hi" }] }); - const a = computeFingerprintHash({ apiKeyId: "key1", model: "gpt-4o", turns, toolNames: [] }); - const b = computeFingerprintHash({ apiKeyId: "key1", model: "gpt-4o", turns, toolNames: [] }); + const a = computeFingerprintHash({ apiKeyId: "key1", model: "gpt-4o", toolNames: [] }); + const b = computeFingerprintHash({ apiKeyId: "key1", model: "gpt-4o", toolNames: [] }); assert.equal(a, b); }); test("computeFingerprintHash: different apiKeyId or model changes the hash", () => { - const turns = extractCanonicalTurns({ messages: [{ role: "user", content: "hi" }] }); - const base = computeFingerprintHash({ apiKeyId: "key1", model: "gpt-4o", turns, toolNames: [] }); - const diffKey = computeFingerprintHash({ - apiKeyId: "key2", - model: "gpt-4o", - turns, - toolNames: [], - }); - const diffModel = computeFingerprintHash({ - apiKeyId: "key1", - model: "gpt-5", - turns, - toolNames: [], - }); + const base = computeFingerprintHash({ apiKeyId: "key1", model: "gpt-4o", toolNames: [] }); + const diffKey = computeFingerprintHash({ apiKeyId: "key2", model: "gpt-4o", toolNames: [] }); + const diffModel = computeFingerprintHash({ apiKeyId: "key1", model: "gpt-5", toolNames: [] }); assert.notEqual(base, diffKey); assert.notEqual(base, diffModel); }); +test("computeFingerprintHash: identical apiKeyId/model/toolNames produce the same hash regardless of message content", () => { + // The whole point of the fix: real OpenClaw traffic rotates its earliest + // turns out of a sliding context window, so the bucket key must not + // depend on message text at all — actual identity is decided later by the + // turn-chain walk (real content overlap), not by this coarse bucket. + const a = computeFingerprintHash({ apiKeyId: "key1", model: "gpt-4o", toolNames: ["exec"] }); + const b = computeFingerprintHash({ apiKeyId: "key1", model: "gpt-4o", toolNames: ["exec"] }); + assert.equal(a, b); +}); + +test("resolveConversationId: stores clean extracted text_preview for content-block turns, not raw JSON (real OpenClaw/Responses-API traffic shape)", async () => { + const apiKeyId = "key-preview-json-bug"; + const turn = await resolveConversationId({ + body: { + model: "big-pickle", + input: [{ role: "user", content: [{ type: "input_text", text: "hello there" }] }], + }, + model: "big-pickle", + apiKeyId, + clientSessionIdHeader: null, + correlationId: nextCorrelationId(), + }); + + const tree = getConversationTurnTree(turn.conversationId); + assert.equal(tree.length, 1); + assert.equal(tree[0].textPreview, "hello there"); + assert.ok( + !tree[0].textPreview.includes("{"), + "text_preview must be the extracted turn text, not a JSON-stringified content-block array" + ); +}); + test("resolveConversationId: exact-match continuation reuses the same id", async () => { const apiKeyId = "key-exact"; const turn1 = await resolveConversationId({ @@ -90,6 +162,7 @@ test("resolveConversationId: exact-match continuation reuses the same id", async model: "big-pickle", apiKeyId, clientSessionIdHeader: null, + correlationId: nextCorrelationId(), }); assert.equal(turn1.isNewConversation, true); @@ -105,6 +178,7 @@ test("resolveConversationId: exact-match continuation reuses the same id", async model: "big-pickle", apiKeyId, clientSessionIdHeader: null, + correlationId: nextCorrelationId(), }); assert.equal(turn2.conversationId, turn1.conversationId); assert.equal(turn2.isNewConversation, false); @@ -117,6 +191,7 @@ test("resolveConversationId: prefix-match continuation across a longer history", model: "big-pickle", apiKeyId, clientSessionIdHeader: null, + correlationId: nextCorrelationId(), }); // Turn 3 resends the full history including turn 2's exchange — still a @@ -135,83 +210,138 @@ test("resolveConversationId: prefix-match continuation across a longer history", model: "big-pickle", apiKeyId, clientSessionIdHeader: null, + correlationId: nextCorrelationId(), }); assert.equal(turn3.conversationId, turn1.conversationId); }); -test("resolveConversationId: divergent history mints a new id despite a shared fingerprint", async () => { - const apiKeyId = "key-divergent"; - - // Establish a real 2-turn conversation: turn1 (1 msg) then turn2 (3 msgs, - // extending turn1's history) — this brings the stored candidate up to - // lastMessageCount=3 with a real, specific history hash. - const turn1 = await resolveConversationId({ - body: { model: "big-pickle", messages: [{ role: "user", content: "same first message" }] }, - model: "big-pickle", - apiKeyId, - clientSessionIdHeader: null, - }); - const turn2 = await resolveConversationId({ +test("resolveConversationId: an edited/duplicated mid-history turn mints its own independent conversation (2026-08-06 redesign — no forking)", async () => { + // The scenario that originally motivated the hash-chain rewrite, and now + // motivates the no-forking redesign: OpenClaw-style cache-aware context + // injection edits turn `c` to `c'` and duplicates turn `i` with an + // injected variant `i'` ahead of it, between two otherwise-related + // requests: + // request 1: a b c d e f g h i + // request 2: a b c' d e f g h i' i j k + // `a`/`b` are byte-identical, but every OmniRoute conversation is a single + // straight line — it never forks. So request 2 must become its OWN + // independent conversation (not request1's), with its OWN complete chain + // (a b c' d e f g h i' i j k), and request1's chain must stay untouched. + const apiKeyId = "key-fork"; + const request1 = await resolveConversationId({ body: { model: "big-pickle", messages: [ - { role: "user", content: "same first message" }, - { role: "assistant", content: "real reply" }, - { role: "user", content: "real follow-up" }, + { role: "user", content: "a" }, + { role: "assistant", content: "b" }, + { role: "user", content: "c" }, + { role: "assistant", content: "d" }, + { role: "user", content: "e" }, + { role: "assistant", content: "f" }, + { role: "user", content: "g" }, + { role: "assistant", content: "h" }, + { role: "user", content: "i" }, ], }, model: "big-pickle", apiKeyId, clientSessionIdHeader: null, + correlationId: nextCorrelationId(), }); - assert.equal(turn2.conversationId, turn1.conversationId); + assert.equal(request1.isNewConversation, true); - // Same fingerprint inputs (model/apiKeyId/first message/no tools) and the - // SAME message count as the established candidate, but the actual content - // beyond the shared first message never matches turn2's real history — - // must NOT be merged into that conversation despite the length/fingerprint - // match, proving the prefix-hash check (not just length/fingerprint) gates - // identity. - const divergent = await resolveConversationId({ + const request2 = await resolveConversationId({ body: { model: "big-pickle", messages: [ - { role: "user", content: "same first message" }, - { role: "assistant", content: "a totally different reply than what actually happened" }, - { role: "user", content: "a follow-up that never occurred in the real history" }, + { role: "user", content: "a" }, + { role: "assistant", content: "b" }, + { role: "user", content: "c'" }, + { role: "assistant", content: "d" }, + { role: "user", content: "e" }, + { role: "assistant", content: "f" }, + { role: "user", content: "g" }, + { role: "assistant", content: "h" }, + { role: "user", content: "i'" }, + { role: "assistant", content: "i" }, + { role: "user", content: "j" }, + { role: "assistant", content: "k" }, ], }, model: "big-pickle", apiKeyId, clientSessionIdHeader: null, + correlationId: nextCorrelationId(), }); - assert.notEqual(divergent.conversationId, turn1.conversationId); - assert.equal(divergent.isNewConversation, true); + // A distinct, brand-new conversation — not request1's. + assert.notEqual(request2.conversationId, request1.conversationId); + assert.equal(request2.isNewConversation, true); - // Both conversations really do share one fingerprint bucket — proves the - // prefix check, not just the fingerprint, is what kept them separate. - const turns = extractCanonicalTurns({ - messages: [{ role: "user", content: "same first message" }], - }); - const fingerprint = computeFingerprintHash({ - apiKeyId, + // request1's chain is completely untouched: still exactly its own 9 turns. + const tree1 = getConversationTurnTree(request1.conversationId); + assert.equal(tree1.length, 9); + assert.deepEqual( + tree1.map((n) => n.textPreview).sort(), + ["a", "b", "c", "d", "e", "f", "g", "h", "i"].sort() + ); + + // request2's chain is its own complete, independent 12-turn history — + // including its OWN copies of "a" and "b" (different node ids than + // request1's, since each conversation's chain hashing is scoped to its + // own conversation id), not references into request1's chain. + const tree2 = getConversationTurnTree(request2.conversationId); + assert.equal(tree2.length, 12); + assert.deepEqual( + tree2.map((n) => n.textPreview).sort(), + ["a", "b", "c'", "d", "e", "f", "g", "h", "i'", "i", "j", "k"].sort() + ); + + const ids1 = new Set(tree1.map((n) => n.id)); + const ids2 = new Set(tree2.map((n) => n.id)); + for (const id of ids2) { + assert.ok(!ids1.has(id), "the two conversations must not share any node ids"); + } + + // A repeat of request2's exact history continues request2 (not a THIRD + // conversation) — the redesign doesn't mint a new id on every retry of an + // already-diverged chain. + const request2Retry = await resolveConversationId({ + body: { + model: "big-pickle", + messages: [ + { role: "user", content: "a" }, + { role: "assistant", content: "b" }, + { role: "user", content: "c'" }, + { role: "assistant", content: "d" }, + { role: "user", content: "e" }, + { role: "assistant", content: "f" }, + { role: "user", content: "g" }, + { role: "assistant", content: "h" }, + { role: "user", content: "i'" }, + { role: "assistant", content: "i" }, + { role: "user", content: "j" }, + { role: "assistant", content: "k" }, + { role: "user", content: "l" }, + ], + }, model: "big-pickle", - turns, - toolNames: [], + apiKeyId, + clientSessionIdHeader: null, + correlationId: nextCorrelationId(), }); - const candidates = findAgenticConversationsByFingerprint(fingerprint); - assert.ok(candidates.length >= 2); + assert.equal(request2Retry.conversationId, request2.conversationId); + assert.equal(request2Retry.isNewConversation, false); }); test("resolveConversationId: continuation is detected even when the system prompt is regenerated every turn (dynamic CLI boilerplate)", async () => { // Real coding-agent CLIs (Claude Code, opencode, etc.) commonly regenerate // the system prompt on EVERY request with live context (timestamp, cwd, - // git status...). If the fingerprint/prefix-hash treat the system message - // like any other turn, that volatility alone breaks continuation detection - // for real traffic — every turn mints a brand new conversation id, even - // though apiKeyId/model/toolNames and the actual user/assistant history are + // git status...). The chain must exclude the system message entirely, or + // that volatility alone breaks continuation detection for real traffic — + // every turn would mint a brand new conversation id, even though + // apiKeyId/model/toolNames and the actual user/assistant history are // unchanged. Discovered live on a real deployment (#9315 follow-up): 28 // consecutive requests from one growing session, each with turn_count=1. const apiKeyId = "key-volatile-system"; @@ -229,6 +359,7 @@ test("resolveConversationId: continuation is detected even when the system promp model: "big-pickle", apiKeyId, clientSessionIdHeader: null, + correlationId: nextCorrelationId(), }); assert.equal(turn1.isNewConversation, true); @@ -248,6 +379,7 @@ test("resolveConversationId: continuation is detected even when the system promp model: "big-pickle", apiKeyId, clientSessionIdHeader: null, + correlationId: nextCorrelationId(), }); assert.equal( @@ -256,18 +388,190 @@ test("resolveConversationId: continuation is detected even when the system promp "expected turn2 to be recognized as a continuation despite the regenerated system prompt" ); assert.equal(turn2.isNewConversation, false); + + // The regenerated system prompt must never appear as a chain node. + const tree = getConversationTurnTree(turn1.conversationId); + for (const node of tree) { + assert.notEqual(node.role, "system"); + } }); -test("resolveConversationId: two independent single-message requests must NOT merge, even with byte-identical content", async () => { - // Regression: a client retrying a failed request (or two genuinely separate - // conversations opening with the same line, e.g. "hi") both arrive as a - // single-message request. Before this fix, a same-length request whose - // truncated-to-candidate-length hash trivially matched (since it IS the - // candidate, verbatim) was accepted as a "continuation" — merging two - // unrelated single-shot requests under one conversation id. A real - // continuation always strictly grows the history (assistant reply + more), - // so same-length must never match. - const apiKeyId = "key-identical-singleshot"; +test("resolveConversationId: continuation is detected even when the earliest turns rotate out of a sliding context window (live OpenClaw traffic pattern)", async () => { + // Discovered live on a real deployment: OpenClaw drops/summarizes the + // EARLIEST turns as a session grows (to bound context size), so the + // request's first non-system turn is a DIFFERENT piece of text on every + // single request — not just an edited/duplicated turn somewhere in the + // middle (that's the fork scenario above), but the very first turn the + // fingerprint bucket used to anchor on. If the bucket depends on that text + // at all, findAgenticConversationsByFingerprint returns zero candidates + // and the turn-chain match never even runs — the conversation looks + // "new" forever, the exact symptom this whole test file guards against. + const apiKeyId = "key-sliding-window"; + const toolNames = ["exec"]; + + const turn1 = await resolveConversationId({ + body: { + model: "big-pickle", + tools: [{ name: "exec" }], + messages: [ + { role: "user", content: "turn-A-oldest" }, + { role: "assistant", content: "turn-B" }, + { role: "user", content: "turn-C-shared-tail" }, + ], + }, + model: "big-pickle", + apiKeyId, + clientSessionIdHeader: null, + correlationId: nextCorrelationId(), + }); + assert.equal(turn1.isNewConversation, true); + + // Turn 2: the oldest turns ("turn-A-oldest", "turn-B") are gone, replaced + // by an unrelated summary — only "turn-C-shared-tail" onward survived. + const turn2 = await resolveConversationId({ + body: { + model: "big-pickle", + tools: [{ name: "exec" }], + messages: [ + { role: "user", content: "[context summary, unrelated to turn-A/turn-B text]" }, + { role: "user", content: "turn-C-shared-tail" }, + { role: "assistant", content: "turn-D" }, + { role: "user", content: "turn-E" }, + ], + }, + model: "big-pickle", + apiKeyId, + clientSessionIdHeader: null, + correlationId: nextCorrelationId(), + }); + + assert.equal( + turn2.conversationId, + turn1.conversationId, + "expected turn2 to be recognized as a continuation despite the first turn's text changing entirely" + ); + assert.equal(turn2.isNewConversation, false); + + // Confirmed via the fingerprint itself: identical apiKeyId/model/toolNames + // (the only inputs to computeFingerprintHash now) despite completely + // different message content between the two requests. + const fp1 = computeFingerprintHash({ apiKeyId, model: "big-pickle", toolNames }); + const fp2 = computeFingerprintHash({ apiKeyId, model: "big-pickle", toolNames }); + assert.equal(fp1, fp2); +}); + +test("resolveConversationId: continuation is detected even when the reconnect turn's content is duplicated earlier in the chain (tool-polling loop)", async () => { + // Discovered live: real agentic traffic (a tool-polling loop, "ack"/"poll" + // repeated many times — one real conversation had 28 byte-identical copies + // of a single turn) leaves MANY existing nodes sharing the same content + // hash. When a sliding context window means the new request's earliest + // retained turn is one of these repeated turns, findReconnectMatch must + // not just grab whichever occurrence happens to be tried first (the + // oldest, per SQLite's insertion-order return) — that stale occurrence's + // recorded next-turn differs from the new content, so it looks like a + // divergence even though the TRUE tail occurrence (no recorded child yet) + // would extend cleanly. This is what made a real conversation mint a + // brand-new copy of its entire history on every single request instead of + // ever reconnecting (2026-08-06). + const apiKeyId = "key-dup-content"; + + const turn1 = await resolveConversationId({ + body: { + model: "big-pickle", + messages: [ + { role: "user", content: "start" }, + { role: "assistant", content: "a1" }, + { role: "user", content: "ack" }, + { role: "assistant", content: "poll" }, + { role: "user", content: "ack" }, + { role: "assistant", content: "poll" }, + { role: "user", content: "ack" }, + { role: "assistant", content: "poll" }, + ], + }, + model: "big-pickle", + apiKeyId, + clientSessionIdHeader: null, + correlationId: nextCorrelationId(), + }); + assert.equal(turn1.isNewConversation, true); + + // Sliding window: only the last "ack"/"poll" pair survived, followed by + // genuinely new content. "ack" and "poll" each match 3 existing nodes. + const turn2 = await resolveConversationId({ + body: { + model: "big-pickle", + messages: [ + { role: "user", content: "ack" }, + { role: "assistant", content: "poll" }, + { role: "user", content: "brand new turn" }, + ], + }, + model: "big-pickle", + apiKeyId, + clientSessionIdHeader: null, + correlationId: nextCorrelationId(), + }); + + assert.equal( + turn2.conversationId, + turn1.conversationId, + "expected turn2 to reconnect to turn1's conversation via the TRUE tail occurrence of the repeated ack/poll turns, not mint a new one" + ); + assert.equal(turn2.isNewConversation, false); + + const tree = getConversationTurnTree(turn1.conversationId); + assert.equal( + tree.length, + 9, + "the new turn should be appended, not a whole new duplicate history" + ); + assert.ok(tree.some((n) => n.textPreview === "brand new turn")); +}); + +test("resolveConversationId: different api keys never merge, even with byte-identical content", async () => { + // Fingerprint isolation (apiKeyId is part of computeFingerprintHash) is + // the actual multi-tenant boundary — must hold regardless of the turn + // chain's own content-addressing. + const body = { model: "big-pickle", messages: [{ role: "user", content: "hi" }] }; + + const first = await resolveConversationId({ + body, + model: "big-pickle", + apiKeyId: "key-tenant-a", + clientSessionIdHeader: null, + correlationId: nextCorrelationId(), + }); + const second = await resolveConversationId({ + body, + model: "big-pickle", + apiKeyId: "key-tenant-b", + clientSessionIdHeader: null, + correlationId: nextCorrelationId(), + }); + + assert.notEqual(second.conversationId, first.conversationId); + + const fingerprintA = computeFingerprintHash({ + apiKeyId: "key-tenant-a", + model: "big-pickle", + toolNames: [], + }); + const fingerprintB = computeFingerprintHash({ + apiKeyId: "key-tenant-b", + model: "big-pickle", + toolNames: [], + }); + assert.notEqual(fingerprintA, fingerprintB); +}); + +test("resolveConversationId: a byte-identical repeat of a single-turn request continues the same conversation", async () => { + // Content-addressed nodes mean a byte-identical opener from the SAME + // apiKey/model (a client retry, or a genuinely separate session that also + // just says "hi") fully matches the existing 1-turn chain — nothing + // diverges (there's no turn afterward to disagree on yet), so this is a + // real continuation, not a fork candidate at all. + const apiKeyId = "key-repeated-singleshot"; const body = { model: "big-pickle", messages: [{ role: "user", content: "hi" }] }; const first = await resolveConversationId({ @@ -275,26 +579,22 @@ test("resolveConversationId: two independent single-message requests must NOT me model: "big-pickle", apiKeyId, clientSessionIdHeader: null, + correlationId: nextCorrelationId(), }); const second = await resolveConversationId({ body, model: "big-pickle", apiKeyId, clientSessionIdHeader: null, - }); - const third = await resolveConversationId({ - body, - model: "big-pickle", - apiKeyId, - clientSessionIdHeader: null, + correlationId: nextCorrelationId(), }); - assert.notEqual(second.conversationId, first.conversationId); - assert.notEqual(third.conversationId, first.conversationId); - assert.notEqual(third.conversationId, second.conversationId); assert.equal(first.isNewConversation, true); - assert.equal(second.isNewConversation, true); - assert.equal(third.isNewConversation, true); + assert.equal(second.conversationId, first.conversationId); + assert.equal(second.isNewConversation, false); + + const tree = getConversationTurnTree(first.conversationId); + assert.equal(tree.length, 1); }); test("resolveConversationId: client-supplied X-Omniroute-Session-Id wins outright", async () => { @@ -304,6 +604,7 @@ test("resolveConversationId: client-supplied X-Omniroute-Session-Id wins outrigh model: "big-pickle", apiKeyId: "key-header", clientSessionIdHeader: headerValue, + correlationId: nextCorrelationId(), }); assert.equal(first.conversationId, headerValue); @@ -315,29 +616,7 @@ test("resolveConversationId: client-supplied X-Omniroute-Session-Id wins outrigh model: "gpt-4o", apiKeyId: "key-header-2", clientSessionIdHeader: headerValue, + correlationId: nextCorrelationId(), }); assert.equal(second.conversationId, headerValue); }); - -test("hashTurnsBounded: documented blind spot — a genuinely untouched middle turn is invisible", () => { - // hashTurnsBounded only inspects: total length, the role sequence, the - // first 2 turns, and the last 3 turns. With 8 turns, index 3 falls in - // neither the head (0-1) nor the tail (5-7) — changing ONLY that turn's - // text, with everything else (including the role sequence) identical, - // must produce an identical bounded hash. This pins the accepted - // trade-off explicitly rather than relying on it silently. - const build = (middleText: string) => [ - { role: "user" as const, text: "start-1" }, - { role: "assistant" as const, text: "start-2" }, - { role: "assistant" as const, text: "unchanged-a" }, - { role: "assistant" as const, text: middleText }, - { role: "assistant" as const, text: "unchanged-b" }, - { role: "assistant" as const, text: "tail-1" }, - { role: "assistant" as const, text: "tail-2" }, - { role: "assistant" as const, text: "tail-3" }, - ]; - assert.equal( - hashTurnsBounded(build("middle-A")), - hashTurnsBounded(build("middle-B-completely-different")) - ); -}); diff --git a/tests/unit/conversations-tree-route-seq-param.test.ts b/tests/unit/conversations-tree-route-seq-param.test.ts new file mode 100644 index 0000000000..d01ff1df7f --- /dev/null +++ b/tests/unit/conversations-tree-route-seq-param.test.ts @@ -0,0 +1,30 @@ +/** + * Regression test for /api/conversations/[id]/tree's query-param parsing. + * + * Real bug: `Number(searchParams.get("beforeSeq"))` is 0 (not NaN) when the + * param is absent, since `Number(null) === 0`. That made an ABSENT + * beforeSeq/afterSeq look like "beforeSeq=0"/"afterSeq=0" was explicitly + * given, which — because the DB layer checks `opts.afterSeq != null` (true + * for 0) BEFORE checking limit — forced every single request into the + * uncapped "poll for new turns" branch, ignoring `limit` entirely and + * returning the conversation's ENTIRE history on every load. + */ + +import test from "node:test"; +import assert from "node:assert/strict"; + +import { parseSeqParam } from "../../src/app/api/conversations/[id]/tree/route.ts"; + +test("parseSeqParam: an absent query param returns undefined, not 0", () => { + assert.equal(parseSeqParam(null), undefined); + assert.equal(parseSeqParam(""), undefined); +}); + +test("parseSeqParam: a real numeric string parses to that number, including a literal '0'", () => { + assert.equal(parseSeqParam("0"), 0); + assert.equal(parseSeqParam("42"), 42); +}); + +test("parseSeqParam: a non-numeric string returns undefined rather than NaN", () => { + assert.equal(parseSeqParam("not-a-number"), undefined); +});