diff --git a/open-sse/config/providers/registry/devin-cli-agentic/index.ts b/open-sse/config/providers/registry/devin-cli-agentic/index.ts index 5d71378b1c..3001936e25 100644 --- a/open-sse/config/providers/registry/devin-cli-agentic/index.ts +++ b/open-sse/config/providers/registry/devin-cli-agentic/index.ts @@ -7,9 +7,10 @@ export const devin_cli_agenticProvider: RegistryEntry = { format: "claude", executor: "devin-cli-agentic", baseUrl: "devin://acp/stdio", - authType: "oauth", - authHeader: "Authorization", - authPrefix: "Bearer ", + // Authentication is owned exclusively by the official Devin CLI inside its + // isolated volume. OmniRoute must not import or persist a host credential. + authType: "none", + authHeader: "none", defaultContextLength: 200000, models: DEVIN_MODEL_CATALOG.map((model) => ({ ...model, @@ -18,4 +19,3 @@ export const devin_cli_agenticProvider: RegistryEntry = { supportsVision: false, })), }; - diff --git a/open-sse/executors/devin-agentic/anthropicResponse.ts b/open-sse/executors/devin-agentic/anthropicResponse.ts index add93e7612..d637ab1612 100644 --- a/open-sse/executors/devin-agentic/anthropicResponse.ts +++ b/open-sse/executors/devin-agentic/anthropicResponse.ts @@ -1,4 +1,9 @@ -import { estimateTokens, type ClaudeResponseArgs, type ClaudeToolUseArgs, type JsonRecord } from "./types.ts"; +import { + estimateTokens, + type ClaudeResponseArgs, + type ClaudeToolUseArgs, + type JsonRecord, +} from "./types.ts"; function usage(inputTokens: number, outputTokens: number) { return { @@ -97,4 +102,3 @@ export function buildClaudeSseFrames(message: JsonRecord): string { out += frame("message_stop", { type: "message_stop" }); return out; } - diff --git a/open-sse/executors/devin-agentic/serializer.ts b/open-sse/executors/devin-agentic/serializer.ts index a5c430acd2..f993025fcb 100644 --- a/open-sse/executors/devin-agentic/serializer.ts +++ b/open-sse/executors/devin-agentic/serializer.ts @@ -5,6 +5,9 @@ import { type AnthropicTool, type DevinPrompt, } from "./types.ts"; +import { createHash } from "node:crypto"; + +export const MAX_TOOL_RESULT_CHARS = 65536; function stringifyContentValue(value: unknown): string { if (typeof value === "string") return value; @@ -12,6 +15,13 @@ function stringifyContentValue(value: unknown): string { return JSON.stringify(value); } +function boundedToolResult(value: unknown): string { + const text = stringifyContentValue(value); + if (text.length <= MAX_TOOL_RESULT_CHARS) return text; + const removed = text.length - MAX_TOOL_RESULT_CHARS; + return `${text.slice(0, MAX_TOOL_RESULT_CHARS)}\n[TRUNCATED ${removed} CHARACTERS BY OMNIROUTE]`; +} + function serializeSystem(system: unknown): string[] { if (typeof system === "string" && system.trim()) return [`[System]\n${system}`]; if (!Array.isArray(system)) return []; @@ -31,7 +41,11 @@ function serializeSystem(system: unknown): string[] { return parts.length > 0 ? [`[System]\n${parts.join("\n")}`] : []; } -function serializeBlock(block: unknown): string { +function serializeBlock( + block: unknown, + knownToolUses: Set, + tools: AnthropicTool[] +): string { const record = asRecord(block); const type = String(record.type || ""); @@ -39,21 +53,44 @@ function serializeBlock(block: unknown): string { if (type === "thinking") return `[Thinking]\n${String(record.thinking || "")}`; if (type === "redacted_thinking") return "[Redacted Thinking]"; if (type === "tool_use") { + const id = String(record.id || "").trim(); + const name = String(record.name || "").trim(); + if (!id || knownToolUses.has(id)) { + throw new DevinAgenticBridgeError( + id ? `Duplicate Anthropic tool_use id: ${id}` : "Anthropic tool_use is missing id", + id ? "duplicate_tool_use_id" : "missing_tool_use_id" + ); + } + const declared = tools.find((tool) => tool.name === name); + if (!declared) { + throw new DevinAgenticBridgeError( + `Historical tool_use references undeclared tool: ${name || "unknown"}`, + "undeclared_historical_tool" + ); + } + knownToolUses.add(id); return [ "[Assistant Tool Use]", - `id: ${String(record.id || "")}`, - `name: ${String(record.name || "")}`, + `id: ${id}`, + `name: ${name}`, "arguments:", JSON.stringify(record.input || {}, null, 2), ].join("\n"); } if (type === "tool_result") { + const toolUseId = String(record.tool_use_id || "").trim(); + if (!toolUseId || !knownToolUses.has(toolUseId)) { + throw new DevinAgenticBridgeError( + `Anthropic tool_result references unknown tool_use id: ${toolUseId || "missing"}`, + "orphan_tool_result" + ); + } return [ "[Tool Result]", - `tool_use_id: ${String(record.tool_use_id || "")}`, + `tool_use_id: ${toolUseId}`, `is_error: ${record.is_error === true ? "true" : "false"}`, "content:", - stringifyContentValue(record.content), + boundedToolResult(record.content), ].join("\n"); } if (type === "image") { @@ -69,16 +106,28 @@ function serializeBlock(block: unknown): string { ); } -function serializeMessage(message: unknown): string { +function serializeMessage( + message: unknown, + knownToolUses: Set, + tools: AnthropicTool[] +): string { const record = asRecord(message); const role = String(record.role || "user"); + if (role !== "user" && role !== "assistant") { + throw new DevinAgenticBridgeError( + `Unsupported Anthropic message role: ${role}`, + "unsupported_role" + ); + } const label = role === "assistant" ? "Assistant" : role === "system" ? "System" : "User"; const content = record.content; if (typeof content === "string") return `[${label}]\n${content}`; if (!Array.isArray(content)) return `[${label}]\n${stringifyContentValue(content)}`; - return `[${label}]\n${content.map((block) => serializeBlock(block)).join("\n\n")}`; + return `[${label}]\n${content + .map((block) => serializeBlock(block, knownToolUses, tools)) + .join("\n\n")}`; } function normalizeTools(tools: unknown): AnthropicTool[] { @@ -121,14 +170,39 @@ function serializeToolCatalog(tools: AnthropicTool[]): string[] { ]; } +function serializeToolChoice(value: unknown, tools: AnthropicTool[]): string[] { + if (value == null) return []; + const choice = asRecord(value); + const type = String(choice.type || ""); + if (type === "auto") return ["[Tool Choice]\nauto"]; + if (type === "any") return ["[Tool Choice]\nA tool call is required."]; + if (type === "none") return ["[Tool Choice]\nDo not call a tool."]; + if (type === "tool") { + const name = String(choice.name || "").trim(); + if (!tools.some((tool) => tool.name === name)) { + throw new DevinAgenticBridgeError( + `tool_choice references unknown tool: ${name}`, + "invalid_tool_choice" + ); + } + return [`[Tool Choice]\nCall exactly this tool: ${name}`]; + } + throw new DevinAgenticBridgeError( + `Unsupported Anthropic tool_choice type: ${type || "missing"}`, + "invalid_tool_choice" + ); +} + export function serializeAnthropicForDevin(body: unknown): DevinPrompt { const record = asRecord(body); const messages = Array.isArray(record.messages) ? record.messages : []; const tools = normalizeTools(record.tools); + const knownToolUses = new Set(); const sections: string[] = [ ...serializeSystem(record.system), ...serializeToolCatalog(tools), - ...messages.map((message) => serializeMessage(message)), + ...serializeToolChoice(record.tool_choice, tools), + ...messages.map((message) => serializeMessage(message, knownToolUses, tools)), ].filter((section) => section.trim().length > 0); if (sections.length === 0) { @@ -136,5 +210,6 @@ export function serializeAnthropicForDevin(body: unknown): DevinPrompt { } const text = sections.join("\n\n---\n\n"); - return { text, tools, inputTokensEstimate: estimateTokens(text) }; + const idSeed = createHash("sha256").update(text).digest("hex").slice(0, 24); + return { text, tools, inputTokensEstimate: estimateTokens(text), idSeed }; } diff --git a/open-sse/executors/devin-agentic/toolParser.ts b/open-sse/executors/devin-agentic/toolParser.ts index 10e087d905..fcebe8a1a8 100644 --- a/open-sse/executors/devin-agentic/toolParser.ts +++ b/open-sse/executors/devin-agentic/toolParser.ts @@ -66,7 +66,7 @@ function validateSchema(value: unknown, schema: JsonRecord, path: string): strin return errors; } -export function parseDevinToolRequest(text: string, tools: AnthropicTool[]) { +export function parseDevinToolRequest(text: string, tools: AnthropicTool[], idSeed = "") { const matches = [...text.matchAll(/\s*([\s\S]*?)\s*<\/tool>/g)]; if (matches.length === 0) return null; if (matches.length > 1) { @@ -110,7 +110,7 @@ export function parseDevinToolRequest(text: string, tools: AnthropicTool[]) { } const digest = createHash("sha256") - .update(`${name}:${stableJson(input)}`) + .update(`${idSeed}:${name}:${stableJson(input)}`) .digest("hex") .slice(0, 16); return { id: `tool_devin_${digest}`, name, input }; diff --git a/open-sse/executors/devin-agentic/types.ts b/open-sse/executors/devin-agentic/types.ts index 690b04bd40..8cde997407 100644 --- a/open-sse/executors/devin-agentic/types.ts +++ b/open-sse/executors/devin-agentic/types.ts @@ -10,6 +10,7 @@ export type DevinPrompt = { text: string; tools: AnthropicTool[]; inputTokensEstimate: number; + idSeed: string; }; export type ParsedToolRequest = { @@ -53,4 +54,3 @@ export function asRecord(value: unknown): JsonRecord { export function estimateTokens(text: string): number { return Math.max(1, Math.ceil(text.length / 4)); } - diff --git a/open-sse/executors/devin-cli-agentic.ts b/open-sse/executors/devin-cli-agentic.ts index ceccf1edbd..281715528d 100644 --- a/open-sse/executors/devin-cli-agentic.ts +++ b/open-sse/executors/devin-cli-agentic.ts @@ -2,7 +2,10 @@ import { spawn } from "node:child_process"; import path from "node:path"; import os from "node:os"; import fs from "node:fs"; +import { randomUUID } from "node:crypto"; import { BaseExecutor, type ExecuteInput } from "./base.ts"; +import { DEVIN_MODEL_CATALOG } from "../config/providers/registry/devin/catalog.ts"; +import { buildErrorBody, sanitizeErrorMessage } from "../utils/error.ts"; import { buildClaudeSseFrames, buildClaudeTextResponse, @@ -21,6 +24,17 @@ type AcpMessage = { error?: { code: number; message: string }; }; +const ACP_PROTOCOL_VERSION = 1; +const MAX_ACP_OUTPUT_CHARS = 1024 * 1024; +const REPAIRABLE_TOOL_ERRORS = new Set([ + "invalid_tool_json", + "missing_tool_name", + "unknown_tool", + "invalid_tool_arguments", + "multiple_tool_requests", + "mixed_tool_narrative", +]); + const CLAUDE_ENV_BLOCKLIST = [ "ANTHROPIC_API_KEY", "CLAUDE_CODE_OAUTH_TOKEN", @@ -96,6 +110,9 @@ export function buildDevinChildEnv( }; if (source.LC_ALL) env.LC_ALL = source.LC_ALL; if (source.TERM) env.TERM = source.TERM; + if (source.DEVIN_BRIDGE_MOCK_LOG === "/evidence/mock-acp.jsonl") { + env.DEVIN_BRIDGE_MOCK_LOG = source.DEVIN_BRIDGE_MOCK_LOG; + } for (const key of CLAUDE_ENV_BLOCKLIST) delete env[key]; return env; @@ -103,16 +120,15 @@ export function buildDevinChildEnv( function errorBody(error: unknown) { const bridge = error instanceof DevinAgenticBridgeError ? error : null; - return { - error: { - message: bridge?.message || (error instanceof Error ? error.message : String(error)), - type: "devin_agentic_error", - code: bridge?.code || "devin_agentic_error", - }, - }; + const status = bridge?.status || 500; + const message = bridge?.message || (error instanceof Error ? error.message : String(error)); + return buildErrorBody(status, sanitizeErrorMessage(message), undefined, { + type: "devin_agentic_error", + code: bridge?.code || "devin_agentic_error", + }); } -async function runAcpTurn(args: { +export async function runAcpTurn(args: { devinBin: string; env: NodeJS.ProcessEnv; model: string; @@ -121,8 +137,9 @@ async function runAcpTurn(args: { log?: ExecuteInput["log"]; }) { const timeoutMs = Number(process.env.DEVIN_AGENTIC_ACP_TIMEOUT_MS || 120000); - const child = spawn(args.devinBin, ["acp"], { + const child = spawn(args.devinBin, ["acp", "--agent-type", "summarizer"], { env: args.env, + cwd: args.env.HOME, stdio: ["pipe", "pipe", "pipe"], shell: false, }); @@ -130,17 +147,23 @@ async function runAcpTurn(args: { let nextId = 1; let buffer = ""; let text = ""; - let initialized = false; - let sessionCreated = false; + let phase: "initialize" | "session" | "prompt" = "initialize"; let sessionId = ""; + let initializeRequestId = 0; + let sessionRequestId = 0; let promptRequestId = 0; let settled = false; return await new Promise((resolve, reject) => { + const abortHandler = () => { + finish(new DevinAgenticBridgeError("Devin ACP request was cancelled", "acp_cancelled", 499)); + }; + const finish = (err: Error | null, value = "") => { if (settled) return; settled = true; clearTimeout(timer); + args.signal?.removeEventListener("abort", abortHandler); try { child.stdin.end(); } catch {} @@ -162,9 +185,8 @@ async function runAcpTurn(args: { return id; }; - args.signal?.addEventListener("abort", () => { - finish(new DevinAgenticBridgeError("Devin ACP request was cancelled", "acp_cancelled", 499)); - }); + if (args.signal?.aborted) return abortHandler(); + args.signal?.addEventListener("abort", abortHandler, { once: true }); child.on("error", (err) => { const message = @@ -180,6 +202,16 @@ async function runAcpTurn(args: { child.stdout.on("data", (chunk: Buffer) => { buffer += chunk.toString("utf8"); + if (buffer.length + text.length > MAX_ACP_OUTPUT_CHARS) { + finish( + new DevinAgenticBridgeError( + "Devin ACP output exceeded the bridge limit", + "acp_output_too_large", + 502 + ) + ); + return; + } let nl: number; while ((nl = buffer.indexOf("\n")) !== -1) { const line = buffer.slice(0, nl).trim(); @@ -190,7 +222,14 @@ async function runAcpTurn(args: { try { msg = JSON.parse(line); } catch { - continue; + finish( + new DevinAgenticBridgeError( + "Devin ACP emitted invalid JSON on stdout", + "invalid_acp_frame", + 502 + ) + ); + return; } if (msg.error) { @@ -204,17 +243,28 @@ async function runAcpTurn(args: { return; } - if (!initialized && msg.result !== undefined && !msg.method) { - initialized = true; - send("session/new", { - cwd: process.env.DEVIN_BRIDGE_WORKSPACE || process.cwd(), + if (phase === "initialize" && msg.id === initializeRequestId && msg.result !== undefined) { + const protocolVersion = Number(asRecord(msg.result).protocolVersion); + if (protocolVersion !== ACP_PROTOCOL_VERSION) { + finish( + new DevinAgenticBridgeError( + `Devin ACP negotiated unsupported protocol version: ${String(protocolVersion)}`, + "unsupported_acp_version", + 502 + ) + ); + return; + } + phase = "session"; + sessionRequestId = send("session/new", { + cwd: args.env.HOME, mcpServers: [], model: args.model || undefined, }); continue; } - if (initialized && !sessionCreated && msg.result !== undefined && !msg.method) { + if (phase === "session" && msg.id === sessionRequestId && msg.result !== undefined) { sessionId = String(asRecord(msg.result).sessionId || ""); if (!sessionId) { finish( @@ -226,7 +276,7 @@ async function runAcpTurn(args: { ); return; } - sessionCreated = true; + phase = "prompt"; promptRequestId = send("session/prompt", { sessionId, prompt: [{ type: "text", text: args.promptText }], @@ -236,6 +286,17 @@ async function runAcpTurn(args: { if (msg.method === "session/update" || msg.method === "$/update") { const params = asRecord(msg.params); + const updateSessionId = String(params.sessionId || ""); + if (updateSessionId && sessionId && updateSessionId !== sessionId) { + finish( + new DevinAgenticBridgeError( + "Devin ACP update referenced a different session", + "acp_session_mismatch", + 502 + ) + ); + return; + } const update = asRecord(params.update); const kind = String(update.sessionUpdate || params.type || ""); if (kind === "agent_message_chunk") { @@ -250,10 +311,40 @@ async function runAcpTurn(args: { continue; } - if (sessionCreated && msg.id === promptRequestId && msg.result !== undefined) { + if (phase === "prompt" && msg.id === promptRequestId && msg.result !== undefined) { + const stopReason = String(asRecord(msg.result).stopReason || ""); + if (stopReason === "cancelled") { + finish( + new DevinAgenticBridgeError("Devin ACP cancelled the turn", "acp_cancelled", 502) + ); + return; + } const resultText = extractText(asRecord(msg.result).content) || extractText(asRecord(msg.result).message); - finish(null, text || resultText); + const finalText = text || resultText; + if (!finalText) { + finish( + new DevinAgenticBridgeError( + `Devin ACP completed without model output (stopReason=${stopReason || "missing"})`, + "empty_acp_output", + 502 + ) + ); + return; + } + finish(null, finalText); + continue; + } + + if (msg.id !== undefined && msg.id !== null && !msg.method) { + finish( + new DevinAgenticBridgeError( + `Devin ACP returned an unexpected response id: ${String(msg.id)}`, + "unexpected_acp_response", + 502 + ) + ); + return; } } }); @@ -271,14 +362,32 @@ async function runAcpTurn(args: { ); }); - send("initialize", { - protocolVersion: "0.3", + initializeRequestId = send("initialize", { + protocolVersion: ACP_PROTOCOL_VERSION, clientInfo: { name: "omniroute-devin-cli-agentic", version: "1.0" }, - capabilities: {}, + clientCapabilities: {}, }); }); } +function assertKnownDevinModel(model: string): void { + if (!DEVIN_MODEL_CATALOG.some((entry) => entry.id === model)) { + throw new DevinAgenticBridgeError( + `Model is not present in the current Devin catalog: ${model}`, + "unknown_devin_model", + 400 + ); + } +} + +async function generateAgenticOutput( + args: Omit[0], "promptText">, + promptText: string +) { + const first = await runAcpTurn({ ...args, promptText }); + return first; +} + function extractText(value: unknown): string { if (typeof value === "string") return value; if (Array.isArray(value)) return value.map((item) => extractText(item)).join(""); @@ -309,21 +418,45 @@ export class DevinCliAgenticExecutor extends BaseExecutor { async execute({ model, body, stream, credentials, signal, log }: ExecuteInput) { try { + assertKnownDevinModel(model); const prompt = serializeAnthropicForDevin(body); const devinBin = resolveDevinBin(); log?.info?.("DEVIN_AGENTIC", `devin acp → model=${model}, bin=${devinBin}`); - const text = await runAcpTurn({ + const turnArgs = { devinBin, env: buildDevinChildEnv(credentials), model, - promptText: prompt.text, signal, log, - }); + }; - const tool = parseDevinToolRequest(text, prompt.tools); - const id = `msg_devin_${Date.now()}`; + let text = await generateAgenticOutput(turnArgs, prompt.text); + let tool; + try { + tool = parseDevinToolRequest(text, prompt.tools, prompt.idSeed); + } catch (error) { + if ( + !(error instanceof DevinAgenticBridgeError) || + !REPAIRABLE_TOOL_ERRORS.has(error.code) + ) { + throw error; + } + const repairPrompt = [ + prompt.text, + "", + "---", + "", + "[Single Repair Attempt]", + `The previous output was rejected: ${sanitizeErrorMessage(error.message)}`, + "Return either plain final text or exactly one standalone JSON envelope.", + "Do not narrate a tool action.", + ].join("\n"); + text = await generateAgenticOutput(turnArgs, repairPrompt); + tool = parseDevinToolRequest(text, prompt.tools, prompt.idSeed); + } + + const id = `msg_devin_${randomUUID().replaceAll("-", "")}`; const outputTokens = estimateTokens(text); const message = tool ? buildClaudeToolUseResponse({ diff --git a/src/shared/constants/providers/noauth.ts b/src/shared/constants/providers/noauth.ts index e7c0dda99e..7b7246beba 100644 --- a/src/shared/constants/providers/noauth.ts +++ b/src/shared/constants/providers/noauth.ts @@ -3,6 +3,24 @@ * Pure data literal; re-exported by the providers.ts barrel. No behavior change. */ export const NOAUTH_PROVIDERS = { + "devin-cli-agentic": { + id: "devin-cli-agentic", + alias: "dva", + name: "Devin CLI Agentic Bridge", + icon: "terminal", + color: "#635BFF", + textIcon: "DV", + website: "https://docs.devin.ai/work-with-devin/devin-cli", + noAuth: true, + hasFree: false, + serviceKinds: ["llm"], + isLocalCli: true, + toolCalling: "emulated", + authHint: "Authentication is owned by the official Devin CLI in its isolated bridge volume.", + notice: { + text: "This provider accepts only the official Devin CLI over local ACP stdio and never falls back to another provider.", + }, + }, opencode: { id: "opencode", alias: "oc", diff --git a/tests/unit/executor-devin-cli-agentic-acp.test.ts b/tests/unit/executor-devin-cli-agentic-acp.test.ts index 49f668806c..141471526d 100644 --- a/tests/unit/executor-devin-cli-agentic-acp.test.ts +++ b/tests/unit/executor-devin-cli-agentic-acp.test.ts @@ -1,7 +1,6 @@ import test from "node:test"; import assert from "node:assert/strict"; import fs from "node:fs"; -import os from "node:os"; import path from "node:path"; import { writeFileSync } from "node:fs"; @@ -10,13 +9,22 @@ import { buildDevinChildEnv, DevinCliAgenticExecutor, } from "../../open-sse/executors/devin-cli-agentic.ts"; +import { devin_cli_agenticProvider } from "../../open-sse/config/providers/registry/devin-cli-agentic/index.ts"; +import { getProviderCredentials } from "../../src/sse/services/auth.ts"; process.env.DEVIN_AGENTIC_HOME = path.join(process.cwd(), ".sandbox", "unit-home"); +fs.mkdirSync(process.env.DEVIN_AGENTIC_HOME, { recursive: true }); async function readResponseText(response: Response) { return await response.text(); } +function sandboxTmp(prefix: string) { + const root = path.join(process.cwd(), ".sandbox", "unit-processes"); + fs.mkdirSync(root, { recursive: true }); + return fs.mkdtempSync(path.join(root, prefix)); +} + test("Devin child environment is allowlisted and requires an isolated home", () => { const isolatedHome = path.join(process.cwd(), ".sandbox", "unit-home"); const env = buildDevinChildEnv( @@ -28,6 +36,7 @@ test("Devin child environment is allowlisted and requires an isolated home", () AWS_ACCESS_KEY_ID: "must-not-leak", GITHUB_TOKEN: "must-not-leak", DEVIN_AGENTIC_HOME: isolatedHome, + DEVIN_BRIDGE_MOCK_LOG: "/evidence/mock-acp.jsonl", } ); @@ -37,6 +46,18 @@ test("Devin child environment is allowlisted and requires an isolated home", () assert.equal(env.ANTHROPIC_AUTH_TOKEN, undefined); assert.equal(env.AWS_ACCESS_KEY_ID, undefined); assert.equal(env.GITHUB_TOKEN, undefined); + assert.equal(env.DEVIN_BRIDGE_MOCK_LOG, "/evidence/mock-acp.jsonl"); + assert.equal( + buildDevinChildEnv( + {}, + { + PATH: "/usr/bin", + DEVIN_AGENTIC_HOME: isolatedHome, + DEVIN_BRIDGE_MOCK_LOG: "/tmp/unsafe.jsonl", + } + ).DEVIN_BRIDGE_MOCK_LOG, + undefined + ); assert.throws( () => buildDevinChildEnv({}, { PATH: "/usr/bin", DEVIN_AGENTIC_HOME: "/tmp/outside" }), /inside the bridge sandbox/ @@ -49,13 +70,27 @@ test("Devin agentic upstream is fixed to local ACP stdio", () => { assert.throws(() => assertLocalAcpUrl("http://localhost:9999"), /ACP stdio/); }); +test("Devin agentic provider delegates auth only to the isolated CLI", () => { + assert.equal(devin_cli_agenticProvider.authType, "none"); + assert.equal(devin_cli_agenticProvider.baseUrl, "devin://acp/stdio"); + assert.equal(devin_cli_agenticProvider.baseUrls, undefined); +}); + +test("Devin agentic provider resolves synthetic no-auth credentials without a DB row", async () => { + const credentials = await getProviderCredentials("devin-cli-agentic"); + assert.equal(credentials?.connectionId, "noauth"); + assert.equal(credentials?.apiKey, null); +}); + function writeMockDevin(tmpDir: string, responseText: string) { const framesFile = path.join(tmpDir, "frames.json"); - const scriptFile = path.join(tmpDir, "mock-devin"); + const argsFile = path.join(tmpDir, "args.json"); + const scriptFile = path.join(tmpDir, "mock-devin.cjs"); const script = `#!/usr/bin/env node const fs = require("fs"); const readline = require("readline"); const frames = []; +fs.writeFileSync(${JSON.stringify(argsFile)}, JSON.stringify(process.argv.slice(2))); const rl = readline.createInterface({ input: process.stdin }); rl.on("line", (line) => { if (!line.trim()) return; @@ -63,7 +98,7 @@ rl.on("line", (line) => { frames.push(msg); fs.writeFileSync(${JSON.stringify(framesFile)}, JSON.stringify(frames, null, 2)); if (msg.method === "initialize") { - process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id: msg.id, result: {} }) + "\\n"); + process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id: msg.id, result: { protocolVersion: 1 } }) + "\\n"); } else if (msg.method === "session/new") { process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id: msg.id, result: { sessionId: "sess_agentic" } }) + "\\n"); } else if (msg.method === "session/prompt") { @@ -77,13 +112,45 @@ rl.on("line", (line) => { }); `; writeFileSync(scriptFile, script, { mode: 0o755 }); - return { scriptFile, framesFile }; + return { scriptFile, framesFile, argsFile }; +} + +function writeScenarioMock(tmpDir: string, body: string) { + const scriptFile = path.join(tmpDir, "mock-devin.cjs"); + writeFileSync( + scriptFile, + `#!/usr/bin/env node +const readline = require("readline"); +const rl = readline.createInterface({ input: process.stdin }); +const send = (value) => process.stdout.write(JSON.stringify(value) + "\\n"); +${body} +`, + { mode: 0o755 } + ); + return scriptFile; +} + +async function executeTextRequest(scriptFile: string, signal?: AbortSignal) { + const oldBin = process.env.CLI_DEVIN_AGENTIC_BIN; + process.env.CLI_DEVIN_AGENTIC_BIN = scriptFile; + try { + return await new DevinCliAgenticExecutor().execute({ + model: "swe-1-7", + stream: false, + credentials: {}, + signal, + body: { messages: [{ role: "user", content: "Say hello" }] }, + }); + } finally { + if (oldBin === undefined) delete process.env.CLI_DEVIN_AGENTIC_BIN; + else process.env.CLI_DEVIN_AGENTIC_BIN = oldBin; + } } test("DevinCliAgenticExecutor returns Anthropic tool_use JSON and sends ACP frames", async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "devin-agentic-")); + const tmpDir = sandboxTmp("devin-agentic-"); const oldBin = process.env.CLI_DEVIN_AGENTIC_BIN; - const { scriptFile, framesFile } = writeMockDevin( + const { scriptFile, framesFile, argsFile } = writeMockDevin( tmpDir, '{"name":"Read","arguments":{"file_path":"src/index.ts"}}' ); @@ -111,7 +178,7 @@ test("DevinCliAgenticExecutor returns Anthropic tool_use JSON and sends ACP fram }, }); - assert.equal(result.response.status, 200); + assert.equal(result.response.status, 200, await result.response.clone().text()); const json = JSON.parse(await readResponseText(result.response)); assert.equal(json.stop_reason, "tool_use"); assert.equal(json.content[0].type, "tool_use"); @@ -122,6 +189,14 @@ test("DevinCliAgenticExecutor returns Anthropic tool_use JSON and sends ACP fram assert.ok(frames.some((frame: { method?: string }) => frame.method === "initialize")); assert.ok(frames.some((frame: { method?: string }) => frame.method === "session/new")); assert.ok(frames.some((frame: { method?: string }) => frame.method === "session/prompt")); + const initialize = frames.find((frame: { method?: string }) => frame.method === "initialize"); + assert.equal(initialize.params.protocolVersion, 1); + assert.deepEqual(initialize.params.clientCapabilities, {}); + assert.deepEqual(JSON.parse(fs.readFileSync(argsFile, "utf8")), [ + "acp", + "--agent-type", + "summarizer", + ]); } finally { if (oldBin === undefined) delete process.env.CLI_DEVIN_AGENTIC_BIN; else process.env.CLI_DEVIN_AGENTIC_BIN = oldBin; @@ -130,7 +205,7 @@ test("DevinCliAgenticExecutor returns Anthropic tool_use JSON and sends ACP fram }); test("DevinCliAgenticExecutor returns Anthropic SSE for streaming Claude clients", async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "devin-agentic-sse-")); + const tmpDir = sandboxTmp("devin-agentic-sse-"); const oldBin = process.env.CLI_DEVIN_AGENTIC_BIN; const { scriptFile } = writeMockDevin(tmpDir, "Done"); process.env.CLI_DEVIN_AGENTIC_BIN = scriptFile; @@ -144,7 +219,7 @@ test("DevinCliAgenticExecutor returns Anthropic SSE for streaming Claude clients body: { messages: [{ role: "user", content: [{ type: "text", text: "Say done" }] }] }, }); - assert.equal(result.response.status, 200); + assert.equal(result.response.status, 200, await result.response.clone().text()); const sse = await readResponseText(result.response); assert.match(sse, /event: message_start/); assert.match(sse, /event: content_block_delta/); @@ -156,3 +231,141 @@ test("DevinCliAgenticExecutor returns Anthropic SSE for streaming Claude clients fs.rmSync(tmpDir, { recursive: true, force: true }); } }); + +test("ACP client handles fragmented frames, multiple chunks, and stderr", async () => { + const tmpDir = sandboxTmp("devin-agentic-fragmented-"); + const scriptFile = writeScenarioMock( + tmpDir, + `rl.on("line", (line) => { + const msg = JSON.parse(line); + if (msg.method === "initialize") send({ jsonrpc: "2.0", id: msg.id, result: { protocolVersion: 1 } }); + if (msg.method === "session/new") send({ jsonrpc: "2.0", id: msg.id, result: { sessionId: "fragmented" } }); + if (msg.method === "session/prompt") { + process.stderr.write("bounded diagnostic\\n"); + const first = JSON.stringify({ jsonrpc: "2.0", method: "session/update", params: { sessionId: "fragmented", update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text: "Hel" } } } }); + process.stdout.write(first.slice(0, 13)); + setTimeout(() => { + process.stdout.write(first.slice(13) + "\\n"); + send({ jsonrpc: "2.0", method: "session/update", params: { sessionId: "fragmented", update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text: "lo" } } } }); + send({ jsonrpc: "2.0", id: msg.id, result: { stopReason: "end_turn" } }); + }, 10); + } +});` + ); + try { + const result = await executeTextRequest(scriptFile); + assert.equal(result.response.status, 200, await result.response.clone().text()); + const body = JSON.parse(await result.response.text()); + assert.equal(body.content[0].text, "Hello"); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } +}); + +test("ACP client fails closed on protocol errors and early exit", async () => { + const cases = [ + { + name: "invalid-frame", + code: "invalid_acp_frame", + body: `rl.on("line", () => process.stdout.write("not-json\\n"));`, + }, + { + name: "rpc-error", + code: "acp_error", + body: `rl.on("line", (line) => { const msg = JSON.parse(line); send({ jsonrpc: "2.0", id: msg.id, error: { code: -32602, message: "bad request" } }); });`, + }, + { + name: "early-exit", + code: "acp_early_exit", + body: `rl.on("line", () => process.exit(7));`, + }, + ]; + + for (const scenario of cases) { + const tmpDir = sandboxTmp(`devin-agentic-${scenario.name}-`); + try { + const result = await executeTextRequest(writeScenarioMock(tmpDir, scenario.body)); + assert.equal(result.response.status, 502, scenario.name); + const body = JSON.parse(await result.response.text()); + assert.equal(body.error.code, scenario.code, scenario.name); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + } +}); + +test("ACP client times out, cancels, and terminates a stuck process", async () => { + const tmpDir = sandboxTmp("devin-agentic-stuck-"); + const scriptFile = writeScenarioMock(tmpDir, `rl.on("line", () => {});`); + const oldTimeout = process.env.DEVIN_AGENTIC_ACP_TIMEOUT_MS; + try { + process.env.DEVIN_AGENTIC_ACP_TIMEOUT_MS = "80"; + const timeoutResult = await executeTextRequest(scriptFile); + assert.equal(timeoutResult.response.status, 504); + assert.equal(JSON.parse(await timeoutResult.response.text()).error.code, "acp_timeout"); + + process.env.DEVIN_AGENTIC_ACP_TIMEOUT_MS = "1000"; + const controller = new AbortController(); + setTimeout(() => controller.abort(), 30); + const cancelled = await executeTextRequest(scriptFile, controller.signal); + assert.equal(cancelled.response.status, 499); + assert.equal(JSON.parse(await cancelled.response.text()).error.code, "acp_cancelled"); + } finally { + if (oldTimeout === undefined) delete process.env.DEVIN_AGENTIC_ACP_TIMEOUT_MS; + else process.env.DEVIN_AGENTIC_ACP_TIMEOUT_MS = oldTimeout; + fs.rmSync(tmpDir, { recursive: true, force: true }); + } +}); + +test("tool repair is attempted once and produces a validated tool_use", async () => { + const tmpDir = sandboxTmp("devin-agentic-repair-"); + const stateFile = path.join(tmpDir, "spawn-count"); + const scriptFile = writeScenarioMock( + tmpDir, + `const fs = require("fs"); +const stateFile = ${JSON.stringify(stateFile)}; +const count = Number(fs.existsSync(stateFile) ? fs.readFileSync(stateFile, "utf8") : "0") + 1; +fs.writeFileSync(stateFile, String(count)); +rl.on("line", (line) => { + const msg = JSON.parse(line); + if (msg.method === "initialize") send({ jsonrpc: "2.0", id: msg.id, result: { protocolVersion: 1 } }); + if (msg.method === "session/new") send({ jsonrpc: "2.0", id: msg.id, result: { sessionId: "repair" } }); + if (msg.method === "session/prompt") { + const text = count === 1 + ? 'I will read it. {"name":"Read","arguments":{"file_path":"a.ts"}}' + : '{"name":"Read","arguments":{"file_path":"a.ts"}}'; + send({ jsonrpc: "2.0", method: "session/update", params: { sessionId: "repair", update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text } } } }); + send({ jsonrpc: "2.0", id: msg.id, result: { stopReason: "end_turn" } }); + } +});` + ); + const oldBin = process.env.CLI_DEVIN_AGENTIC_BIN; + process.env.CLI_DEVIN_AGENTIC_BIN = scriptFile; + try { + const result = await new DevinCliAgenticExecutor().execute({ + model: "swe-1-7", + stream: false, + credentials: {}, + body: { + tools: [ + { + name: "Read", + input_schema: { + type: "object", + required: ["file_path"], + properties: { file_path: { type: "string" } }, + }, + }, + ], + messages: [{ role: "user", content: "Read a.ts" }], + }, + }); + assert.equal(result.response.status, 200, await result.response.clone().text()); + assert.equal(JSON.parse(await result.response.text()).stop_reason, "tool_use"); + assert.equal(fs.readFileSync(stateFile, "utf8"), "2"); + } finally { + if (oldBin === undefined) delete process.env.CLI_DEVIN_AGENTIC_BIN; + else process.env.CLI_DEVIN_AGENTIC_BIN = oldBin; + fs.rmSync(tmpDir, { recursive: true, force: true }); + } +}); diff --git a/tests/unit/executor-devin-cli-agentic-core.test.ts b/tests/unit/executor-devin-cli-agentic-core.test.ts index 701e18756e..5b65465415 100644 --- a/tests/unit/executor-devin-cli-agentic-core.test.ts +++ b/tests/unit/executor-devin-cli-agentic-core.test.ts @@ -2,7 +2,10 @@ import test from "node:test"; import assert from "node:assert/strict"; import { buildClaudeSseFrames } from "../../open-sse/executors/devin-agentic/anthropicResponse.ts"; -import { serializeAnthropicForDevin } from "../../open-sse/executors/devin-agentic/serializer.ts"; +import { + MAX_TOOL_RESULT_CHARS, + serializeAnthropicForDevin, +} from "../../open-sse/executors/devin-agentic/serializer.ts"; import { parseDevinToolRequest } from "../../open-sse/executors/devin-agentic/toolParser.ts"; const readTool = { @@ -40,6 +43,52 @@ test("devin agentic serializer preserves Anthropic tool history and schemas", () assert.match(prompt.text, /\[Assistant Tool Use\]/); assert.match(prompt.text, /\[Tool Result\]/); assert.equal(prompt.tools[0].name, "Read"); + assert.match(prompt.idSeed, /^[a-f0-9]{24}$/); +}); + +test("devin agentic serializer preserves tool choice and validates tool-result association", () => { + const prompt = serializeAnthropicForDevin({ + tools: [readTool], + tool_choice: { type: "tool", name: "Read" }, + messages: [{ role: "user", content: "Read it" }], + }); + assert.match(prompt.text, /Call exactly this tool: Read/); + assert.throws( + () => + serializeAnthropicForDevin({ + tools: [readTool], + messages: [ + { + role: "user", + content: [{ type: "tool_result", tool_use_id: "toolu_missing", content: "nope" }], + }, + ], + }), + /unknown tool_use id/ + ); +}); + +test("devin agentic serializer marks bounded tool-result truncation explicitly", () => { + const prompt = serializeAnthropicForDevin({ + tools: [readTool], + messages: [ + { + role: "assistant", + content: [{ type: "tool_use", id: "toolu_big", name: "Read", input: { file_path: "a" } }], + }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "toolu_big", + content: "x".repeat(MAX_TOOL_RESULT_CHARS + 9), + }, + ], + }, + ], + }); + assert.match(prompt.text, /\[TRUNCATED 9 CHARACTERS BY OMNIROUTE\]/); }); test("devin agentic serializer rejects images explicitly", () => { @@ -55,7 +104,8 @@ test("devin agentic serializer rejects images explicitly", () => { test("devin agentic parser validates known tool and arguments", () => { const parsed = parseDevinToolRequest( '{"name":"Read","arguments":{"file_path":"src/index.ts"}}', - [readTool] + [readTool], + "request-a" ); assert.equal(parsed?.name, "Read"); @@ -63,6 +113,15 @@ test("devin agentic parser validates known tool and arguments", () => { assert.match(parsed?.id || "", /^tool_devin_/); }); +test("devin agentic tool ids are stable per request and distinct across turns", () => { + const text = '{"name":"Read","arguments":{"file_path":"src/index.ts"}}'; + const first = parseDevinToolRequest(text, [readTool], "request-a"); + const retry = parseDevinToolRequest(text, [readTool], "request-a"); + const laterTurn = parseDevinToolRequest(text, [readTool], "request-b"); + assert.equal(first?.id, retry?.id); + assert.notEqual(first?.id, laterTurn?.id); +}); + test("devin agentic parser rejects unknown tools and invalid arguments", () => { assert.throws( () => parseDevinToolRequest('{"name":"Write","arguments":{}}', [readTool]),