mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-14 10:52:17 +03:00
Two shards on release/v3.8.51 went red in one day with the same signature —
"ENOTEMPTY, Directory not empty: /tmp/omniroute-<test>-XXXXXX" — from
combo-same-provider-cascade (Unit Tests fast-path 4/4, on a PR that touches only
.github/) and auth-policy-embeddings-webfetch-7785 (the 20k-test TIA step). Both pass
alone and on re-run: the cleanup races something still writing into the directory
(SQLite WAL/-shm checkpoint, a worker, the backup) and under a loaded hosted runner
the window opens. 1154 test files do their own cleanup with
fs.rmSync(dir, { recursive: true, force: true }); 57 already asked for retries.
One-shot codemod (scripts/ad-hoc/codemod-rm-maxretries.mjs, kept for the record):
every rm / rmSync / rmdirSync option object with `recursive: true` and no
`maxRetries` gains `maxRetries: 5, retryDelay: 100` — Node itself then retries
ENOTEMPTY/EBUSY/EPERM for up to ~0.5 s before giving up. 2243 call sites in 1292
files under tests/, the shared tests/_setup/isolateDataDir.ts exit hook included.
Only the option object changes: no call site, assertion or import is touched.
Validation: prettier and ESLint (with the frozen suppressions) clean on all 1292
files; a random 20-file sample runs green (quota-redis-store hangs identically on
the untouched tree — it needs a Redis on localhost, an environment matter). The
four unit shards on this PR are the full run.
578 lines
24 KiB
TypeScript
578 lines
24 KiB
TypeScript
import test from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import { writeFileSync } from "node:fs";
|
|
|
|
const isolatedRoot = path.join(process.cwd(), ".sandbox", "direct-acp-test");
|
|
process.env.HOME = path.join(isolatedRoot, "home");
|
|
process.env.DATA_DIR = path.join(isolatedRoot, "data");
|
|
process.env.SQLITE_FILE = path.join(isolatedRoot, "data", "storage.sqlite");
|
|
process.env.DEVIN_AGENTIC_HOME = process.env.HOME;
|
|
fs.mkdirSync(process.env.HOME, { recursive: true });
|
|
fs.mkdirSync(process.env.DATA_DIR, { recursive: true });
|
|
|
|
const { assertLocalAcpUrl, buildDevinChildEnv, DevinCliAgenticExecutor } =
|
|
await import("../../open-sse/executors/devin-cli-agentic.ts");
|
|
const { devin_cli_agenticProvider } =
|
|
await import("../../open-sse/config/providers/registry/devin-cli-agentic/index.ts");
|
|
const { getProviderCredentials } = await import("../../src/sse/services/auth.ts");
|
|
|
|
async function readResponseText(response: Response) {
|
|
return await response.text();
|
|
}
|
|
|
|
function sandboxTmp(prefix: string) {
|
|
const root = path.join(process.cwd(), ".sandbox", "unit-processes");
|
|
fs.mkdirSync(root, { recursive: true });
|
|
return fs.mkdtempSync(path.join(root, prefix));
|
|
}
|
|
|
|
test("Devin child environment is allowlisted and requires an isolated home", () => {
|
|
const isolatedHome = path.join(process.cwd(), ".sandbox", "unit-home");
|
|
const env = buildDevinChildEnv(
|
|
{ apiKey: "devin-test" },
|
|
{
|
|
HOME: "/Users/example",
|
|
PATH: "/usr/bin:/bin",
|
|
ANTHROPIC_AUTH_TOKEN: "must-not-leak",
|
|
AWS_ACCESS_KEY_ID: "must-not-leak",
|
|
GITHUB_TOKEN: "must-not-leak",
|
|
DEVIN_AGENTIC_HOME: isolatedHome,
|
|
DEVIN_BRIDGE_MOCK_LOG: "/evidence/mock-acp.jsonl",
|
|
}
|
|
);
|
|
|
|
assert.equal(env.HOME, isolatedHome);
|
|
assert.equal(env.PATH, "/usr/bin:/bin");
|
|
assert.equal(env.WINDSURF_API_KEY, undefined);
|
|
assert.equal(env.ANTHROPIC_AUTH_TOKEN, undefined);
|
|
assert.equal(env.AWS_ACCESS_KEY_ID, undefined);
|
|
assert.equal(env.GITHUB_TOKEN, undefined);
|
|
assert.equal(env.DEVIN_BRIDGE_MOCK_LOG, "/evidence/mock-acp.jsonl");
|
|
assert.equal(
|
|
buildDevinChildEnv(
|
|
{},
|
|
{
|
|
PATH: "/usr/bin",
|
|
DEVIN_AGENTIC_HOME: isolatedHome,
|
|
DEVIN_BRIDGE_MOCK_LOG: "/tmp/unsafe.jsonl",
|
|
}
|
|
).DEVIN_BRIDGE_MOCK_LOG,
|
|
undefined
|
|
);
|
|
assert.throws(
|
|
() => buildDevinChildEnv({}, { PATH: "/usr/bin", DEVIN_AGENTIC_HOME: "/tmp/outside" }),
|
|
/inside the bridge sandbox/
|
|
);
|
|
});
|
|
|
|
test("Devin child environment derives only the trusted bridge proxy", () => {
|
|
const isolatedHome = path.join(process.cwd(), ".sandbox", "unit-home");
|
|
const trustedProxy = "http://network-guard:8080";
|
|
const trusted = buildDevinChildEnv(
|
|
{},
|
|
{
|
|
DEVIN_AGENTIC_HOME: isolatedHome,
|
|
DEVIN_BRIDGE_PROXY_URL: trustedProxy,
|
|
HTTP_PROXY: "http://user:password@host-proxy.example:3128",
|
|
HTTPS_PROXY: "http://user:password@host-proxy.example:3128",
|
|
ALL_PROXY: "socks5://host-proxy.example:1080",
|
|
NO_PROXY: "metadata.internal",
|
|
}
|
|
);
|
|
|
|
assert.equal(trusted.HTTP_PROXY, trustedProxy);
|
|
assert.equal(trusted.HTTPS_PROXY, trustedProxy);
|
|
assert.equal(trusted.ALL_PROXY, undefined);
|
|
assert.equal(trusted.NO_PROXY, undefined);
|
|
assert.equal(trusted.DEVIN_BRIDGE_PROXY_URL, undefined);
|
|
|
|
const untrusted = buildDevinChildEnv(
|
|
{},
|
|
{
|
|
DEVIN_AGENTIC_HOME: isolatedHome,
|
|
DEVIN_BRIDGE_PROXY_URL: "http://user:password@network-guard:8080",
|
|
HTTP_PROXY: "http://host-proxy.example:3128",
|
|
HTTPS_PROXY: "http://host-proxy.example:3128",
|
|
}
|
|
);
|
|
assert.equal(untrusted.HTTP_PROXY, undefined);
|
|
assert.equal(untrusted.HTTPS_PROXY, undefined);
|
|
});
|
|
|
|
test("Devin agentic upstream is fixed to local ACP stdio", () => {
|
|
assert.doesNotThrow(() => assertLocalAcpUrl("devin://acp/stdio"));
|
|
assert.throws(() => assertLocalAcpUrl("https://api.anthropic.com"), /ACP stdio/);
|
|
assert.throws(() => assertLocalAcpUrl("http://localhost:9999"), /ACP stdio/);
|
|
});
|
|
|
|
test("Devin agentic provider delegates auth only to the isolated CLI", () => {
|
|
assert.equal(devin_cli_agenticProvider.authType, "none");
|
|
assert.equal(devin_cli_agenticProvider.baseUrl, "devin://acp/stdio");
|
|
assert.equal(devin_cli_agenticProvider.baseUrls, undefined);
|
|
});
|
|
|
|
test("Devin agentic provider resolves synthetic no-auth credentials without a DB row", async () => {
|
|
const credentials = await getProviderCredentials("devin-cli-agentic");
|
|
assert.equal(credentials?.connectionId, "noauth");
|
|
assert.equal(credentials?.apiKey, null);
|
|
});
|
|
|
|
function writeMockDevin(tmpDir: string, responseText: string) {
|
|
const framesFile = path.join(tmpDir, "frames.json");
|
|
const argsFile = path.join(tmpDir, "args.json");
|
|
const scriptFile = path.join(tmpDir, "mock-devin.cjs");
|
|
const script = `#!/usr/bin/env node
|
|
const fs = require("fs");
|
|
const readline = require("readline");
|
|
const frames = [];
|
|
fs.writeFileSync(${JSON.stringify(argsFile)}, JSON.stringify(process.argv.slice(2)));
|
|
const rl = readline.createInterface({ input: process.stdin });
|
|
rl.on("line", (line) => {
|
|
if (!line.trim()) return;
|
|
const msg = JSON.parse(line);
|
|
frames.push(msg);
|
|
fs.writeFileSync(${JSON.stringify(framesFile)}, JSON.stringify(frames, null, 2));
|
|
if (msg.method === "initialize") {
|
|
process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id: msg.id, result: { protocolVersion: 1 } }) + "\\n");
|
|
} else if (msg.method === "session/new") {
|
|
process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id: msg.id, result: {
|
|
sessionId: "sess_agentic",
|
|
modes: { currentModeId: "accept-edits", availableModes: [{ id: "accept-edits" }, { id: "ask" }] }
|
|
} }) + "\\n");
|
|
} else if (msg.method === "session/set_config_option") {
|
|
process.stdout.write(JSON.stringify({
|
|
jsonrpc: "2.0",
|
|
id: msg.id,
|
|
result: { configOptions: [{ id: "mode", currentValue: "ask" }] }
|
|
}) + "\\n");
|
|
} else if (msg.method === "session/set_mode") {
|
|
process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id: msg.id, result: {} }) + "\\n");
|
|
} else if (msg.method === "session/prompt") {
|
|
process.stdout.write(JSON.stringify({
|
|
jsonrpc: "2.0",
|
|
method: "session/update",
|
|
params: { update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text: ${JSON.stringify(responseText)} } } }
|
|
}) + "\\n");
|
|
process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id: msg.id, result: { stopReason: "end_turn" } }) + "\\n");
|
|
}
|
|
});
|
|
`;
|
|
writeFileSync(scriptFile, script, { mode: 0o755 });
|
|
return { scriptFile, framesFile, argsFile };
|
|
}
|
|
|
|
function writeScenarioMock(tmpDir: string, body: string) {
|
|
const scriptFile = path.join(tmpDir, "mock-devin.cjs");
|
|
writeFileSync(
|
|
scriptFile,
|
|
`#!/usr/bin/env node
|
|
const readline = require("readline");
|
|
const rl = readline.createInterface({ input: process.stdin });
|
|
const send = (value) => process.stdout.write(JSON.stringify(value) + "\\n");
|
|
const safeModes = { currentModeId: "accept-edits", availableModes: [{ id: "accept-edits" }, { id: "ask" }] };
|
|
const sendSession = (msg, sessionId) => send({ jsonrpc: "2.0", id: msg.id, result: { sessionId, modes: safeModes } });
|
|
const acceptAskMode = (msg) => {
|
|
if (msg.method === "session/set_mode") {
|
|
send({ jsonrpc: "2.0", id: msg.id, result: {} });
|
|
return true;
|
|
}
|
|
if (msg.method !== "session/set_config_option") return false;
|
|
if (msg.params?.configId !== "mode" || msg.params?.type !== "id" || msg.params?.value !== "ask") {
|
|
send({ jsonrpc: "2.0", id: msg.id, error: { code: -32602, message: "unsafe mode" } });
|
|
} else {
|
|
send({ jsonrpc: "2.0", id: msg.id, result: { configOptions: [{ id: "mode", currentValue: "ask" }] } });
|
|
}
|
|
return true;
|
|
};
|
|
${body}
|
|
`,
|
|
{ mode: 0o755 }
|
|
);
|
|
return scriptFile;
|
|
}
|
|
|
|
async function executeTextRequest(scriptFile: string, signal?: AbortSignal) {
|
|
const oldBin = process.env.CLI_DEVIN_AGENTIC_BIN;
|
|
process.env.CLI_DEVIN_AGENTIC_BIN = scriptFile;
|
|
try {
|
|
return await new DevinCliAgenticExecutor().execute({
|
|
model: "swe-1-7",
|
|
stream: false,
|
|
credentials: {},
|
|
signal,
|
|
body: { messages: [{ role: "user", content: "Say hello" }] },
|
|
});
|
|
} finally {
|
|
if (oldBin === undefined) delete process.env.CLI_DEVIN_AGENTIC_BIN;
|
|
else process.env.CLI_DEVIN_AGENTIC_BIN = oldBin;
|
|
}
|
|
}
|
|
|
|
test("DevinCliAgenticExecutor returns Anthropic tool_use JSON and sends ACP frames", async () => {
|
|
const tmpDir = sandboxTmp("devin-agentic-");
|
|
const oldBin = process.env.CLI_DEVIN_AGENTIC_BIN;
|
|
const { scriptFile, framesFile, argsFile } = writeMockDevin(
|
|
tmpDir,
|
|
'<tool>{"name":"Read","arguments":{"file_path":"src/index.ts"}}</tool>'
|
|
);
|
|
process.env.CLI_DEVIN_AGENTIC_BIN = scriptFile;
|
|
|
|
try {
|
|
const executor = new DevinCliAgenticExecutor();
|
|
const result = await executor.execute({
|
|
model: "swe-1-7",
|
|
stream: false,
|
|
credentials: { apiKey: "devin-test" },
|
|
body: {
|
|
messages: [{ role: "user", content: [{ type: "text", text: "Read src/index.ts" }] }],
|
|
tools: [
|
|
{
|
|
name: "Read",
|
|
input_schema: {
|
|
type: "object",
|
|
required: ["file_path"],
|
|
properties: { file_path: { type: "string" } },
|
|
additionalProperties: false,
|
|
},
|
|
},
|
|
],
|
|
},
|
|
});
|
|
|
|
assert.equal(result.response.status, 200, await result.response.clone().text());
|
|
const json = JSON.parse(await readResponseText(result.response));
|
|
assert.equal(json.stop_reason, "tool_use");
|
|
assert.equal(json.content[0].type, "tool_use");
|
|
assert.equal(json.content[0].name, "Read");
|
|
assert.deepEqual(json.content[0].input, { file_path: "src/index.ts" });
|
|
|
|
const frames = JSON.parse(fs.readFileSync(framesFile, "utf8"));
|
|
assert.ok(frames.some((frame: { method?: string }) => frame.method === "initialize"));
|
|
assert.ok(frames.some((frame: { method?: string }) => frame.method === "session/new"));
|
|
assert.ok(
|
|
!frames.some((frame: { method?: string }) => frame.method === "session/set_config_option")
|
|
);
|
|
assert.ok(frames.some((frame: { method?: string }) => frame.method === "session/prompt"));
|
|
const initialize = frames.find((frame: { method?: string }) => frame.method === "initialize");
|
|
assert.equal(initialize.params.protocolVersion, 1);
|
|
assert.deepEqual(initialize.params.clientCapabilities, {});
|
|
const prompt = frames.find((frame: { method?: string }) => frame.method === "session/prompt");
|
|
const promptText = prompt.params.prompt[0].text;
|
|
assert.match(promptText, /Devin Summarizer Bridge/);
|
|
assert.match(promptText, /execution trace/i);
|
|
assert.match(promptText, /client workspace is \/workspace/i);
|
|
assert.match(promptText, /Read src\/index\.ts/);
|
|
assert.deepEqual(JSON.parse(fs.readFileSync(argsFile, "utf8")), [
|
|
"acp",
|
|
"--agent-type",
|
|
"summarizer",
|
|
]);
|
|
} finally {
|
|
if (oldBin === undefined) delete process.env.CLI_DEVIN_AGENTIC_BIN;
|
|
else process.env.CLI_DEVIN_AGENTIC_BIN = oldBin;
|
|
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
}
|
|
});
|
|
|
|
test("no-tools summarizer does not depend on mutable ACP permission modes", async () => {
|
|
const tmpDir = sandboxTmp("devin-agentic-no-ask-mode-");
|
|
const scriptFile = writeScenarioMock(
|
|
tmpDir,
|
|
`rl.on("line", (line) => {
|
|
const msg = JSON.parse(line);
|
|
if (msg.method === "initialize") send({ jsonrpc: "2.0", id: msg.id, result: { protocolVersion: 1 } });
|
|
if (msg.method === "session/new") send({
|
|
jsonrpc: "2.0",
|
|
id: msg.id,
|
|
result: {
|
|
sessionId: "no-ask",
|
|
modes: { currentModeId: "accept-edits", availableModes: [{ id: "accept-edits", name: "Code" }] }
|
|
}
|
|
});
|
|
if (msg.method === "session/prompt") {
|
|
send({ jsonrpc: "2.0", method: "session/update", params: { sessionId: "no-ask", update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text: "unsafe" } } } });
|
|
send({ jsonrpc: "2.0", id: msg.id, result: { stopReason: "end_turn" } });
|
|
}
|
|
});`
|
|
);
|
|
try {
|
|
const result = await executeTextRequest(scriptFile);
|
|
assert.equal(result.response.status, 200);
|
|
const body = JSON.parse(await result.response.text());
|
|
assert.equal(body.content[0].text, "unsafe");
|
|
} finally {
|
|
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
}
|
|
});
|
|
|
|
test("ACP client fails closed when session/new omits the session id", async () => {
|
|
const tmpDir = sandboxTmp("devin-agentic-unconfirmed-ask-mode-");
|
|
const scriptFile = writeScenarioMock(
|
|
tmpDir,
|
|
`rl.on("line", (line) => {
|
|
const msg = JSON.parse(line);
|
|
if (msg.method === "initialize") send({ jsonrpc: "2.0", id: msg.id, result: { protocolVersion: 1 } });
|
|
if (msg.method === "session/new") send({ jsonrpc: "2.0", id: msg.id, result: {} });
|
|
});`
|
|
);
|
|
try {
|
|
const result = await executeTextRequest(scriptFile);
|
|
assert.equal(result.response.status, 502);
|
|
const body = JSON.parse(await result.response.text());
|
|
assert.equal(body.error.code, "missing_session_id");
|
|
} finally {
|
|
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
}
|
|
});
|
|
|
|
test("DevinCliAgenticExecutor returns Anthropic SSE for streaming Claude clients", async () => {
|
|
const tmpDir = sandboxTmp("devin-agentic-sse-");
|
|
const oldBin = process.env.CLI_DEVIN_AGENTIC_BIN;
|
|
const { scriptFile } = writeMockDevin(tmpDir, "Done");
|
|
process.env.CLI_DEVIN_AGENTIC_BIN = scriptFile;
|
|
|
|
try {
|
|
const executor = new DevinCliAgenticExecutor();
|
|
const result = await executor.execute({
|
|
model: "swe-1-7",
|
|
stream: true,
|
|
credentials: {},
|
|
body: { messages: [{ role: "user", content: [{ type: "text", text: "Say done" }] }] },
|
|
});
|
|
|
|
assert.equal(result.response.status, 200, await result.response.clone().text());
|
|
const sse = await readResponseText(result.response);
|
|
assert.match(sse, /event: message_start/);
|
|
assert.match(sse, /event: content_block_delta/);
|
|
assert.match(sse, /Done/);
|
|
assert.match(sse, /event: message_stop/);
|
|
} finally {
|
|
if (oldBin === undefined) delete process.env.CLI_DEVIN_AGENTIC_BIN;
|
|
else process.env.CLI_DEVIN_AGENTIC_BIN = oldBin;
|
|
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
}
|
|
});
|
|
|
|
test("ACP client handles fragmented frames, multiple chunks, and stderr", async () => {
|
|
const tmpDir = sandboxTmp("devin-agentic-fragmented-");
|
|
const scriptFile = writeScenarioMock(
|
|
tmpDir,
|
|
`rl.on("line", (line) => {
|
|
const msg = JSON.parse(line);
|
|
if (acceptAskMode(msg)) return;
|
|
if (msg.method === "initialize") send({ jsonrpc: "2.0", id: msg.id, result: { protocolVersion: 1 } });
|
|
if (msg.method === "session/new") sendSession(msg, "fragmented");
|
|
if (msg.method === "session/prompt") {
|
|
process.stderr.write("bounded diagnostic\\n");
|
|
const first = JSON.stringify({ jsonrpc: "2.0", method: "session/update", params: { sessionId: "fragmented", update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text: "Hel" } } } });
|
|
process.stdout.write(first.slice(0, 13));
|
|
setTimeout(() => {
|
|
process.stdout.write(first.slice(13) + "\\n");
|
|
send({ jsonrpc: "2.0", method: "session/update", params: { sessionId: "fragmented", update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text: "lo" } } } });
|
|
send({ jsonrpc: "2.0", id: msg.id, result: { stopReason: "end_turn" } });
|
|
}, 10);
|
|
}
|
|
});`
|
|
);
|
|
try {
|
|
const result = await executeTextRequest(scriptFile);
|
|
assert.equal(result.response.status, 200, await result.response.clone().text());
|
|
const body = JSON.parse(await result.response.text());
|
|
assert.equal(body.content[0].text, "Hello");
|
|
} finally {
|
|
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
}
|
|
});
|
|
|
|
test("ACP client fails closed when Devin attempts an internal tool call", async () => {
|
|
const tmpDir = sandboxTmp("devin-agentic-internal-tool-");
|
|
const scriptFile = writeScenarioMock(
|
|
tmpDir,
|
|
`rl.on("line", (line) => {
|
|
const msg = JSON.parse(line);
|
|
if (acceptAskMode(msg)) return;
|
|
if (msg.method === "initialize") send({ jsonrpc: "2.0", id: msg.id, result: { protocolVersion: 1 } });
|
|
if (msg.method === "session/new") sendSession(msg, "internal-tool");
|
|
if (msg.method === "session/prompt") {
|
|
send({ jsonrpc: "2.0", method: "session/update", params: { sessionId: "internal-tool", update: { sessionUpdate: "tool_call", toolCallId: "internal-1", title: "Read a.ts" } } });
|
|
send({ jsonrpc: "2.0", method: "session/update", params: { sessionId: "internal-tool", update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text: "Done" } } } });
|
|
send({ jsonrpc: "2.0", id: msg.id, result: { stopReason: "end_turn" } });
|
|
}
|
|
});`
|
|
);
|
|
try {
|
|
const result = await executeTextRequest(scriptFile);
|
|
assert.equal(result.response.status, 502);
|
|
const body = JSON.parse(await result.response.text());
|
|
assert.equal(body.error.code, "devin_internal_tool_execution");
|
|
} finally {
|
|
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
}
|
|
});
|
|
|
|
test("ACP client fails closed on protocol errors and early exit", async () => {
|
|
const cases = [
|
|
{
|
|
name: "invalid-frame",
|
|
code: "invalid_acp_frame",
|
|
body: `rl.on("line", () => process.stdout.write("not-json\\n"));`,
|
|
},
|
|
{
|
|
name: "rpc-error",
|
|
code: "acp_error",
|
|
body: `rl.on("line", (line) => { const msg = JSON.parse(line); send({ jsonrpc: "2.0", id: msg.id, error: { code: -32602, message: "bad request" } }); });`,
|
|
},
|
|
{
|
|
name: "early-exit",
|
|
code: "acp_early_exit",
|
|
body: `rl.on("line", () => process.exit(7));`,
|
|
},
|
|
];
|
|
|
|
for (const scenario of cases) {
|
|
const tmpDir = sandboxTmp(`devin-agentic-${scenario.name}-`);
|
|
try {
|
|
const result = await executeTextRequest(writeScenarioMock(tmpDir, scenario.body));
|
|
assert.equal(result.response.status, 502, scenario.name);
|
|
const body = JSON.parse(await result.response.text());
|
|
assert.equal(body.error.code, scenario.code, scenario.name);
|
|
} finally {
|
|
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
}
|
|
}
|
|
});
|
|
|
|
test("ACP client times out, cancels, and terminates a stuck process", async () => {
|
|
const tmpDir = sandboxTmp("devin-agentic-stuck-");
|
|
const scriptFile = writeScenarioMock(tmpDir, `rl.on("line", () => {});`);
|
|
const oldTimeout = process.env.DEVIN_AGENTIC_ACP_TIMEOUT_MS;
|
|
try {
|
|
process.env.DEVIN_AGENTIC_ACP_TIMEOUT_MS = "80";
|
|
const timeoutResult = await executeTextRequest(scriptFile);
|
|
assert.equal(timeoutResult.response.status, 504);
|
|
assert.equal(JSON.parse(await timeoutResult.response.text()).error.code, "acp_timeout");
|
|
|
|
process.env.DEVIN_AGENTIC_ACP_TIMEOUT_MS = "1000";
|
|
const controller = new AbortController();
|
|
setTimeout(() => controller.abort(), 30);
|
|
const cancelled = await executeTextRequest(scriptFile, controller.signal);
|
|
assert.equal(cancelled.response.status, 499);
|
|
assert.equal(JSON.parse(await cancelled.response.text()).error.code, "acp_cancelled");
|
|
} finally {
|
|
if (oldTimeout === undefined) delete process.env.DEVIN_AGENTIC_ACP_TIMEOUT_MS;
|
|
else process.env.DEVIN_AGENTIC_ACP_TIMEOUT_MS = oldTimeout;
|
|
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
}
|
|
});
|
|
|
|
test("premature tool narration is repaired once into a validated tool_use", async () => {
|
|
const tmpDir = sandboxTmp("devin-agentic-repair-");
|
|
const stateFile = path.join(tmpDir, "spawn-count");
|
|
const scriptFile = writeScenarioMock(
|
|
tmpDir,
|
|
`const fs = require("fs");
|
|
const stateFile = ${JSON.stringify(stateFile)};
|
|
const count = Number(fs.existsSync(stateFile) ? fs.readFileSync(stateFile, "utf8") : "0") + 1;
|
|
fs.writeFileSync(stateFile, String(count));
|
|
rl.on("line", (line) => {
|
|
const msg = JSON.parse(line);
|
|
if (acceptAskMode(msg)) return;
|
|
if (msg.method === "initialize") send({ jsonrpc: "2.0", id: msg.id, result: { protocolVersion: 1 } });
|
|
if (msg.method === "session/new") sendSession(msg, "repair");
|
|
if (msg.method === "session/prompt") {
|
|
const text = count === 1
|
|
? '<summary>Current State: inspected math.js. The immediate next step was to read math.test.js, then fix the bug and run npm test.</summary>'
|
|
: '<tool>{"name":"Read","arguments":{"file_path":"a.ts"}}</tool>';
|
|
send({ jsonrpc: "2.0", method: "session/update", params: { sessionId: "repair", update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text } } } });
|
|
send({ jsonrpc: "2.0", id: msg.id, result: { stopReason: "end_turn" } });
|
|
}
|
|
});`
|
|
);
|
|
const oldBin = process.env.CLI_DEVIN_AGENTIC_BIN;
|
|
process.env.CLI_DEVIN_AGENTIC_BIN = scriptFile;
|
|
try {
|
|
const result = await new DevinCliAgenticExecutor().execute({
|
|
model: "swe-1-7",
|
|
stream: false,
|
|
credentials: {},
|
|
body: {
|
|
tools: [
|
|
{
|
|
name: "Read",
|
|
input_schema: {
|
|
type: "object",
|
|
required: ["file_path"],
|
|
properties: { file_path: { type: "string" } },
|
|
},
|
|
},
|
|
],
|
|
messages: [{ role: "user", content: "Read a.ts" }],
|
|
},
|
|
});
|
|
assert.equal(result.response.status, 200, await result.response.clone().text());
|
|
assert.equal(JSON.parse(await result.response.text()).stop_reason, "tool_use");
|
|
assert.equal(fs.readFileSync(stateFile, "utf8"), "2");
|
|
} finally {
|
|
if (oldBin === undefined) delete process.env.CLI_DEVIN_AGENTIC_BIN;
|
|
else process.env.CLI_DEVIN_AGENTIC_BIN = oldBin;
|
|
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
}
|
|
});
|
|
|
|
test("premature tool narration fails closed when the single repair is still narrative", async () => {
|
|
const tmpDir = sandboxTmp("devin-agentic-repair-narrative-");
|
|
const stateFile = path.join(tmpDir, "spawn-count");
|
|
const scriptFile = writeScenarioMock(
|
|
tmpDir,
|
|
`const fs = require("fs");
|
|
const stateFile = ${JSON.stringify(stateFile)};
|
|
const count = Number(fs.existsSync(stateFile) ? fs.readFileSync(stateFile, "utf8") : "0") + 1;
|
|
fs.writeFileSync(stateFile, String(count));
|
|
rl.on("line", (line) => {
|
|
const msg = JSON.parse(line);
|
|
if (acceptAskMode(msg)) return;
|
|
if (msg.method === "initialize") send({ jsonrpc: "2.0", id: msg.id, result: { protocolVersion: 1 } });
|
|
if (msg.method === "session/new") sendSession(msg, "repair-narrative");
|
|
if (msg.method === "session/prompt") {
|
|
const text = count === 1
|
|
? "I'll start by reading the file."
|
|
: "I'll read the file now, then run the tests.";
|
|
send({ jsonrpc: "2.0", method: "session/update", params: { sessionId: "repair-narrative", update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text } } } });
|
|
send({ jsonrpc: "2.0", id: msg.id, result: { stopReason: "end_turn" } });
|
|
}
|
|
});`
|
|
);
|
|
const oldBin = process.env.CLI_DEVIN_AGENTIC_BIN;
|
|
process.env.CLI_DEVIN_AGENTIC_BIN = scriptFile;
|
|
try {
|
|
const result = await new DevinCliAgenticExecutor().execute({
|
|
model: "swe-1-7",
|
|
stream: false,
|
|
credentials: {},
|
|
body: {
|
|
tools: [
|
|
{
|
|
name: "Read",
|
|
input_schema: {
|
|
type: "object",
|
|
required: ["file_path"],
|
|
properties: { file_path: { type: "string" } },
|
|
},
|
|
},
|
|
],
|
|
messages: [{ role: "user", content: "Read a.ts" }],
|
|
},
|
|
});
|
|
assert.equal(result.response.status, 502);
|
|
const body = JSON.parse(await result.response.text());
|
|
assert.equal(body.error.code, "unexecuted_tool_intent");
|
|
assert.equal(fs.readFileSync(stateFile, "utf8"), "2");
|
|
} finally {
|
|
if (oldBin === undefined) delete process.env.CLI_DEVIN_AGENTIC_BIN;
|
|
else process.env.CLI_DEVIN_AGENTIC_BIN = oldBin;
|
|
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
}
|
|
});
|