mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-14 19:02:17 +03:00
`main` has been red since b342c1a361 on the vitest and integration gates:
✖ tests/unit/autoCombo/provider-family-combos.test.ts > auto/<family>
✖ chat pipeline applies Codex OAuth fingerprint and priority tier inside combos
Both call resetStorage() from beforeEach, which does an fs.rmSync(TEST_DATA_DIR,
{recursive: true, force: true}) with no retry, and intermittently loses the race
with a not-yet-released SQLite handle (ENOTEMPTY).
release/v3.8.51 fixed this in #11968 with a mechanical codemod adding
maxRetries/retryDelay to every recursive rm/rmSync/rmdirSync under tests/, but
that PR landed only on the release branch. Because main only receives work at
the release squash, it stayed broken for the whole cycle — and repo-wide gates
then turn every open PR into main red on checks unrelated to their diff.
This is the --base main twin: re-runs the same codemod that already shipped on
the release branch (scripts/ad-hoc/codemod-rm-maxretries.mjs), so the two
branches converge on identical test-teardown semantics. Test-only; no product
logic is touched.
The remaining three failures reported on #12133 (unit full suite exceeding its
4800s ceiling, package-artifact exceeding 1200s, and the boot-smoke that is
skipped as a consequence) are runner-contention timeouts, not code defects —
validate-release-green.mjs runs those heavy gates concurrently on one shared
hosted runner. There is no fix to port for those.
578 lines
24 KiB
TypeScript
578 lines
24 KiB
TypeScript
import test from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import { writeFileSync } from "node:fs";
|
|
|
|
const isolatedRoot = path.join(process.cwd(), ".sandbox", "direct-acp-test");
|
|
process.env.HOME = path.join(isolatedRoot, "home");
|
|
process.env.DATA_DIR = path.join(isolatedRoot, "data");
|
|
process.env.SQLITE_FILE = path.join(isolatedRoot, "data", "storage.sqlite");
|
|
process.env.DEVIN_AGENTIC_HOME = process.env.HOME;
|
|
fs.mkdirSync(process.env.HOME, { recursive: true });
|
|
fs.mkdirSync(process.env.DATA_DIR, { recursive: true });
|
|
|
|
const { assertLocalAcpUrl, buildDevinChildEnv, DevinCliAgenticExecutor } =
|
|
await import("../../open-sse/executors/devin-cli-agentic.ts");
|
|
const { devin_cli_agenticProvider } =
|
|
await import("../../open-sse/config/providers/registry/devin-cli-agentic/index.ts");
|
|
const { getProviderCredentials } = await import("../../src/sse/services/auth.ts");
|
|
|
|
async function readResponseText(response: Response) {
|
|
return await response.text();
|
|
}
|
|
|
|
function sandboxTmp(prefix: string) {
|
|
const root = path.join(process.cwd(), ".sandbox", "unit-processes");
|
|
fs.mkdirSync(root, { recursive: true });
|
|
return fs.mkdtempSync(path.join(root, prefix));
|
|
}
|
|
|
|
test("Devin child environment is allowlisted and requires an isolated home", () => {
|
|
const isolatedHome = path.join(process.cwd(), ".sandbox", "unit-home");
|
|
const env = buildDevinChildEnv(
|
|
{ apiKey: "devin-test" },
|
|
{
|
|
HOME: "/Users/example",
|
|
PATH: "/usr/bin:/bin",
|
|
ANTHROPIC_AUTH_TOKEN: "must-not-leak",
|
|
AWS_ACCESS_KEY_ID: "must-not-leak",
|
|
GITHUB_TOKEN: "must-not-leak",
|
|
DEVIN_AGENTIC_HOME: isolatedHome,
|
|
DEVIN_BRIDGE_MOCK_LOG: "/evidence/mock-acp.jsonl",
|
|
}
|
|
);
|
|
|
|
assert.equal(env.HOME, isolatedHome);
|
|
assert.equal(env.PATH, "/usr/bin:/bin");
|
|
assert.equal(env.WINDSURF_API_KEY, undefined);
|
|
assert.equal(env.ANTHROPIC_AUTH_TOKEN, undefined);
|
|
assert.equal(env.AWS_ACCESS_KEY_ID, undefined);
|
|
assert.equal(env.GITHUB_TOKEN, undefined);
|
|
assert.equal(env.DEVIN_BRIDGE_MOCK_LOG, "/evidence/mock-acp.jsonl");
|
|
assert.equal(
|
|
buildDevinChildEnv(
|
|
{},
|
|
{
|
|
PATH: "/usr/bin",
|
|
DEVIN_AGENTIC_HOME: isolatedHome,
|
|
DEVIN_BRIDGE_MOCK_LOG: "/tmp/unsafe.jsonl",
|
|
}
|
|
).DEVIN_BRIDGE_MOCK_LOG,
|
|
undefined
|
|
);
|
|
assert.throws(
|
|
() => buildDevinChildEnv({}, { PATH: "/usr/bin", DEVIN_AGENTIC_HOME: "/tmp/outside" }),
|
|
/inside the bridge sandbox/
|
|
);
|
|
});
|
|
|
|
test("Devin child environment derives only the trusted bridge proxy", () => {
|
|
const isolatedHome = path.join(process.cwd(), ".sandbox", "unit-home");
|
|
const trustedProxy = "http://network-guard:8080";
|
|
const trusted = buildDevinChildEnv(
|
|
{},
|
|
{
|
|
DEVIN_AGENTIC_HOME: isolatedHome,
|
|
DEVIN_BRIDGE_PROXY_URL: trustedProxy,
|
|
HTTP_PROXY: "http://user:password@host-proxy.example:3128",
|
|
HTTPS_PROXY: "http://user:password@host-proxy.example:3128",
|
|
ALL_PROXY: "socks5://host-proxy.example:1080",
|
|
NO_PROXY: "metadata.internal",
|
|
}
|
|
);
|
|
|
|
assert.equal(trusted.HTTP_PROXY, trustedProxy);
|
|
assert.equal(trusted.HTTPS_PROXY, trustedProxy);
|
|
assert.equal(trusted.ALL_PROXY, undefined);
|
|
assert.equal(trusted.NO_PROXY, undefined);
|
|
assert.equal(trusted.DEVIN_BRIDGE_PROXY_URL, undefined);
|
|
|
|
const untrusted = buildDevinChildEnv(
|
|
{},
|
|
{
|
|
DEVIN_AGENTIC_HOME: isolatedHome,
|
|
DEVIN_BRIDGE_PROXY_URL: "http://user:password@network-guard:8080",
|
|
HTTP_PROXY: "http://host-proxy.example:3128",
|
|
HTTPS_PROXY: "http://host-proxy.example:3128",
|
|
}
|
|
);
|
|
assert.equal(untrusted.HTTP_PROXY, undefined);
|
|
assert.equal(untrusted.HTTPS_PROXY, undefined);
|
|
});
|
|
|
|
test("Devin agentic upstream is fixed to local ACP stdio", () => {
|
|
assert.doesNotThrow(() => assertLocalAcpUrl("devin://acp/stdio"));
|
|
assert.throws(() => assertLocalAcpUrl("https://api.anthropic.com"), /ACP stdio/);
|
|
assert.throws(() => assertLocalAcpUrl("http://localhost:9999"), /ACP stdio/);
|
|
});
|
|
|
|
test("Devin agentic provider delegates auth only to the isolated CLI", () => {
|
|
assert.equal(devin_cli_agenticProvider.authType, "none");
|
|
assert.equal(devin_cli_agenticProvider.baseUrl, "devin://acp/stdio");
|
|
assert.equal(devin_cli_agenticProvider.baseUrls, undefined);
|
|
});
|
|
|
|
test("Devin agentic provider resolves synthetic no-auth credentials without a DB row", async () => {
|
|
const credentials = await getProviderCredentials("devin-cli-agentic");
|
|
assert.equal(credentials?.connectionId, "noauth");
|
|
assert.equal(credentials?.apiKey, null);
|
|
});
|
|
|
|
function writeMockDevin(tmpDir: string, responseText: string) {
|
|
const framesFile = path.join(tmpDir, "frames.json");
|
|
const argsFile = path.join(tmpDir, "args.json");
|
|
const scriptFile = path.join(tmpDir, "mock-devin.cjs");
|
|
const script = `#!/usr/bin/env node
|
|
const fs = require("fs");
|
|
const readline = require("readline");
|
|
const frames = [];
|
|
fs.writeFileSync(${JSON.stringify(argsFile)}, JSON.stringify(process.argv.slice(2)));
|
|
const rl = readline.createInterface({ input: process.stdin });
|
|
rl.on("line", (line) => {
|
|
if (!line.trim()) return;
|
|
const msg = JSON.parse(line);
|
|
frames.push(msg);
|
|
fs.writeFileSync(${JSON.stringify(framesFile)}, JSON.stringify(frames, null, 2));
|
|
if (msg.method === "initialize") {
|
|
process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id: msg.id, result: { protocolVersion: 1 } }) + "\\n");
|
|
} else if (msg.method === "session/new") {
|
|
process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id: msg.id, result: {
|
|
sessionId: "sess_agentic",
|
|
modes: { currentModeId: "accept-edits", availableModes: [{ id: "accept-edits" }, { id: "ask" }] }
|
|
} }) + "\\n");
|
|
} else if (msg.method === "session/set_config_option") {
|
|
process.stdout.write(JSON.stringify({
|
|
jsonrpc: "2.0",
|
|
id: msg.id,
|
|
result: { configOptions: [{ id: "mode", currentValue: "ask" }] }
|
|
}) + "\\n");
|
|
} else if (msg.method === "session/set_mode") {
|
|
process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id: msg.id, result: {} }) + "\\n");
|
|
} else if (msg.method === "session/prompt") {
|
|
process.stdout.write(JSON.stringify({
|
|
jsonrpc: "2.0",
|
|
method: "session/update",
|
|
params: { update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text: ${JSON.stringify(responseText)} } } }
|
|
}) + "\\n");
|
|
process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id: msg.id, result: { stopReason: "end_turn" } }) + "\\n");
|
|
}
|
|
});
|
|
`;
|
|
writeFileSync(scriptFile, script, { mode: 0o755 });
|
|
return { scriptFile, framesFile, argsFile };
|
|
}
|
|
|
|
function writeScenarioMock(tmpDir: string, body: string) {
|
|
const scriptFile = path.join(tmpDir, "mock-devin.cjs");
|
|
writeFileSync(
|
|
scriptFile,
|
|
`#!/usr/bin/env node
|
|
const readline = require("readline");
|
|
const rl = readline.createInterface({ input: process.stdin });
|
|
const send = (value) => process.stdout.write(JSON.stringify(value) + "\\n");
|
|
const safeModes = { currentModeId: "accept-edits", availableModes: [{ id: "accept-edits" }, { id: "ask" }] };
|
|
const sendSession = (msg, sessionId) => send({ jsonrpc: "2.0", id: msg.id, result: { sessionId, modes: safeModes } });
|
|
const acceptAskMode = (msg) => {
|
|
if (msg.method === "session/set_mode") {
|
|
send({ jsonrpc: "2.0", id: msg.id, result: {} });
|
|
return true;
|
|
}
|
|
if (msg.method !== "session/set_config_option") return false;
|
|
if (msg.params?.configId !== "mode" || msg.params?.type !== "id" || msg.params?.value !== "ask") {
|
|
send({ jsonrpc: "2.0", id: msg.id, error: { code: -32602, message: "unsafe mode" } });
|
|
} else {
|
|
send({ jsonrpc: "2.0", id: msg.id, result: { configOptions: [{ id: "mode", currentValue: "ask" }] } });
|
|
}
|
|
return true;
|
|
};
|
|
${body}
|
|
`,
|
|
{ mode: 0o755 }
|
|
);
|
|
return scriptFile;
|
|
}
|
|
|
|
async function executeTextRequest(scriptFile: string, signal?: AbortSignal) {
|
|
const oldBin = process.env.CLI_DEVIN_AGENTIC_BIN;
|
|
process.env.CLI_DEVIN_AGENTIC_BIN = scriptFile;
|
|
try {
|
|
return await new DevinCliAgenticExecutor().execute({
|
|
model: "swe-1-7",
|
|
stream: false,
|
|
credentials: {},
|
|
signal,
|
|
body: { messages: [{ role: "user", content: "Say hello" }] },
|
|
});
|
|
} finally {
|
|
if (oldBin === undefined) delete process.env.CLI_DEVIN_AGENTIC_BIN;
|
|
else process.env.CLI_DEVIN_AGENTIC_BIN = oldBin;
|
|
}
|
|
}
|
|
|
|
test("DevinCliAgenticExecutor returns Anthropic tool_use JSON and sends ACP frames", async () => {
|
|
const tmpDir = sandboxTmp("devin-agentic-");
|
|
const oldBin = process.env.CLI_DEVIN_AGENTIC_BIN;
|
|
const { scriptFile, framesFile, argsFile } = writeMockDevin(
|
|
tmpDir,
|
|
'<tool>{"name":"Read","arguments":{"file_path":"src/index.ts"}}</tool>'
|
|
);
|
|
process.env.CLI_DEVIN_AGENTIC_BIN = scriptFile;
|
|
|
|
try {
|
|
const executor = new DevinCliAgenticExecutor();
|
|
const result = await executor.execute({
|
|
model: "swe-1-7",
|
|
stream: false,
|
|
credentials: { apiKey: "devin-test" },
|
|
body: {
|
|
messages: [{ role: "user", content: [{ type: "text", text: "Read src/index.ts" }] }],
|
|
tools: [
|
|
{
|
|
name: "Read",
|
|
input_schema: {
|
|
type: "object",
|
|
required: ["file_path"],
|
|
properties: { file_path: { type: "string" } },
|
|
additionalProperties: false,
|
|
},
|
|
},
|
|
],
|
|
},
|
|
});
|
|
|
|
assert.equal(result.response.status, 200, await result.response.clone().text());
|
|
const json = JSON.parse(await readResponseText(result.response));
|
|
assert.equal(json.stop_reason, "tool_use");
|
|
assert.equal(json.content[0].type, "tool_use");
|
|
assert.equal(json.content[0].name, "Read");
|
|
assert.deepEqual(json.content[0].input, { file_path: "src/index.ts" });
|
|
|
|
const frames = JSON.parse(fs.readFileSync(framesFile, "utf8"));
|
|
assert.ok(frames.some((frame: { method?: string }) => frame.method === "initialize"));
|
|
assert.ok(frames.some((frame: { method?: string }) => frame.method === "session/new"));
|
|
assert.ok(
|
|
!frames.some((frame: { method?: string }) => frame.method === "session/set_config_option")
|
|
);
|
|
assert.ok(frames.some((frame: { method?: string }) => frame.method === "session/prompt"));
|
|
const initialize = frames.find((frame: { method?: string }) => frame.method === "initialize");
|
|
assert.equal(initialize.params.protocolVersion, 1);
|
|
assert.deepEqual(initialize.params.clientCapabilities, {});
|
|
const prompt = frames.find((frame: { method?: string }) => frame.method === "session/prompt");
|
|
const promptText = prompt.params.prompt[0].text;
|
|
assert.match(promptText, /Devin Summarizer Bridge/);
|
|
assert.match(promptText, /execution trace/i);
|
|
assert.match(promptText, /client workspace is \/workspace/i);
|
|
assert.match(promptText, /Read src\/index\.ts/);
|
|
assert.deepEqual(JSON.parse(fs.readFileSync(argsFile, "utf8")), [
|
|
"acp",
|
|
"--agent-type",
|
|
"summarizer",
|
|
]);
|
|
} finally {
|
|
if (oldBin === undefined) delete process.env.CLI_DEVIN_AGENTIC_BIN;
|
|
else process.env.CLI_DEVIN_AGENTIC_BIN = oldBin;
|
|
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
}
|
|
});
|
|
|
|
test("no-tools summarizer does not depend on mutable ACP permission modes", async () => {
|
|
const tmpDir = sandboxTmp("devin-agentic-no-ask-mode-");
|
|
const scriptFile = writeScenarioMock(
|
|
tmpDir,
|
|
`rl.on("line", (line) => {
|
|
const msg = JSON.parse(line);
|
|
if (msg.method === "initialize") send({ jsonrpc: "2.0", id: msg.id, result: { protocolVersion: 1 } });
|
|
if (msg.method === "session/new") send({
|
|
jsonrpc: "2.0",
|
|
id: msg.id,
|
|
result: {
|
|
sessionId: "no-ask",
|
|
modes: { currentModeId: "accept-edits", availableModes: [{ id: "accept-edits", name: "Code" }] }
|
|
}
|
|
});
|
|
if (msg.method === "session/prompt") {
|
|
send({ jsonrpc: "2.0", method: "session/update", params: { sessionId: "no-ask", update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text: "unsafe" } } } });
|
|
send({ jsonrpc: "2.0", id: msg.id, result: { stopReason: "end_turn" } });
|
|
}
|
|
});`
|
|
);
|
|
try {
|
|
const result = await executeTextRequest(scriptFile);
|
|
assert.equal(result.response.status, 200);
|
|
const body = JSON.parse(await result.response.text());
|
|
assert.equal(body.content[0].text, "unsafe");
|
|
} finally {
|
|
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
}
|
|
});
|
|
|
|
test("ACP client fails closed when session/new omits the session id", async () => {
|
|
const tmpDir = sandboxTmp("devin-agentic-unconfirmed-ask-mode-");
|
|
const scriptFile = writeScenarioMock(
|
|
tmpDir,
|
|
`rl.on("line", (line) => {
|
|
const msg = JSON.parse(line);
|
|
if (msg.method === "initialize") send({ jsonrpc: "2.0", id: msg.id, result: { protocolVersion: 1 } });
|
|
if (msg.method === "session/new") send({ jsonrpc: "2.0", id: msg.id, result: {} });
|
|
});`
|
|
);
|
|
try {
|
|
const result = await executeTextRequest(scriptFile);
|
|
assert.equal(result.response.status, 502);
|
|
const body = JSON.parse(await result.response.text());
|
|
assert.equal(body.error.code, "missing_session_id");
|
|
} finally {
|
|
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
}
|
|
});
|
|
|
|
test("DevinCliAgenticExecutor returns Anthropic SSE for streaming Claude clients", async () => {
|
|
const tmpDir = sandboxTmp("devin-agentic-sse-");
|
|
const oldBin = process.env.CLI_DEVIN_AGENTIC_BIN;
|
|
const { scriptFile } = writeMockDevin(tmpDir, "Done");
|
|
process.env.CLI_DEVIN_AGENTIC_BIN = scriptFile;
|
|
|
|
try {
|
|
const executor = new DevinCliAgenticExecutor();
|
|
const result = await executor.execute({
|
|
model: "swe-1-7",
|
|
stream: true,
|
|
credentials: {},
|
|
body: { messages: [{ role: "user", content: [{ type: "text", text: "Say done" }] }] },
|
|
});
|
|
|
|
assert.equal(result.response.status, 200, await result.response.clone().text());
|
|
const sse = await readResponseText(result.response);
|
|
assert.match(sse, /event: message_start/);
|
|
assert.match(sse, /event: content_block_delta/);
|
|
assert.match(sse, /Done/);
|
|
assert.match(sse, /event: message_stop/);
|
|
} finally {
|
|
if (oldBin === undefined) delete process.env.CLI_DEVIN_AGENTIC_BIN;
|
|
else process.env.CLI_DEVIN_AGENTIC_BIN = oldBin;
|
|
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
}
|
|
});
|
|
|
|
test("ACP client handles fragmented frames, multiple chunks, and stderr", async () => {
|
|
const tmpDir = sandboxTmp("devin-agentic-fragmented-");
|
|
const scriptFile = writeScenarioMock(
|
|
tmpDir,
|
|
`rl.on("line", (line) => {
|
|
const msg = JSON.parse(line);
|
|
if (acceptAskMode(msg)) return;
|
|
if (msg.method === "initialize") send({ jsonrpc: "2.0", id: msg.id, result: { protocolVersion: 1 } });
|
|
if (msg.method === "session/new") sendSession(msg, "fragmented");
|
|
if (msg.method === "session/prompt") {
|
|
process.stderr.write("bounded diagnostic\\n");
|
|
const first = JSON.stringify({ jsonrpc: "2.0", method: "session/update", params: { sessionId: "fragmented", update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text: "Hel" } } } });
|
|
process.stdout.write(first.slice(0, 13));
|
|
setTimeout(() => {
|
|
process.stdout.write(first.slice(13) + "\\n");
|
|
send({ jsonrpc: "2.0", method: "session/update", params: { sessionId: "fragmented", update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text: "lo" } } } });
|
|
send({ jsonrpc: "2.0", id: msg.id, result: { stopReason: "end_turn" } });
|
|
}, 10);
|
|
}
|
|
});`
|
|
);
|
|
try {
|
|
const result = await executeTextRequest(scriptFile);
|
|
assert.equal(result.response.status, 200, await result.response.clone().text());
|
|
const body = JSON.parse(await result.response.text());
|
|
assert.equal(body.content[0].text, "Hello");
|
|
} finally {
|
|
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
}
|
|
});
|
|
|
|
test("ACP client fails closed when Devin attempts an internal tool call", async () => {
|
|
const tmpDir = sandboxTmp("devin-agentic-internal-tool-");
|
|
const scriptFile = writeScenarioMock(
|
|
tmpDir,
|
|
`rl.on("line", (line) => {
|
|
const msg = JSON.parse(line);
|
|
if (acceptAskMode(msg)) return;
|
|
if (msg.method === "initialize") send({ jsonrpc: "2.0", id: msg.id, result: { protocolVersion: 1 } });
|
|
if (msg.method === "session/new") sendSession(msg, "internal-tool");
|
|
if (msg.method === "session/prompt") {
|
|
send({ jsonrpc: "2.0", method: "session/update", params: { sessionId: "internal-tool", update: { sessionUpdate: "tool_call", toolCallId: "internal-1", title: "Read a.ts" } } });
|
|
send({ jsonrpc: "2.0", method: "session/update", params: { sessionId: "internal-tool", update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text: "Done" } } } });
|
|
send({ jsonrpc: "2.0", id: msg.id, result: { stopReason: "end_turn" } });
|
|
}
|
|
});`
|
|
);
|
|
try {
|
|
const result = await executeTextRequest(scriptFile);
|
|
assert.equal(result.response.status, 502);
|
|
const body = JSON.parse(await result.response.text());
|
|
assert.equal(body.error.code, "devin_internal_tool_execution");
|
|
} finally {
|
|
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
}
|
|
});
|
|
|
|
test("ACP client fails closed on protocol errors and early exit", async () => {
|
|
const cases = [
|
|
{
|
|
name: "invalid-frame",
|
|
code: "invalid_acp_frame",
|
|
body: `rl.on("line", () => process.stdout.write("not-json\\n"));`,
|
|
},
|
|
{
|
|
name: "rpc-error",
|
|
code: "acp_error",
|
|
body: `rl.on("line", (line) => { const msg = JSON.parse(line); send({ jsonrpc: "2.0", id: msg.id, error: { code: -32602, message: "bad request" } }); });`,
|
|
},
|
|
{
|
|
name: "early-exit",
|
|
code: "acp_early_exit",
|
|
body: `rl.on("line", () => process.exit(7));`,
|
|
},
|
|
];
|
|
|
|
for (const scenario of cases) {
|
|
const tmpDir = sandboxTmp(`devin-agentic-${scenario.name}-`);
|
|
try {
|
|
const result = await executeTextRequest(writeScenarioMock(tmpDir, scenario.body));
|
|
assert.equal(result.response.status, 502, scenario.name);
|
|
const body = JSON.parse(await result.response.text());
|
|
assert.equal(body.error.code, scenario.code, scenario.name);
|
|
} finally {
|
|
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
}
|
|
}
|
|
});
|
|
|
|
test("ACP client times out, cancels, and terminates a stuck process", async () => {
|
|
const tmpDir = sandboxTmp("devin-agentic-stuck-");
|
|
const scriptFile = writeScenarioMock(tmpDir, `rl.on("line", () => {});`);
|
|
const oldTimeout = process.env.DEVIN_AGENTIC_ACP_TIMEOUT_MS;
|
|
try {
|
|
process.env.DEVIN_AGENTIC_ACP_TIMEOUT_MS = "80";
|
|
const timeoutResult = await executeTextRequest(scriptFile);
|
|
assert.equal(timeoutResult.response.status, 504);
|
|
assert.equal(JSON.parse(await timeoutResult.response.text()).error.code, "acp_timeout");
|
|
|
|
process.env.DEVIN_AGENTIC_ACP_TIMEOUT_MS = "1000";
|
|
const controller = new AbortController();
|
|
setTimeout(() => controller.abort(), 30);
|
|
const cancelled = await executeTextRequest(scriptFile, controller.signal);
|
|
assert.equal(cancelled.response.status, 499);
|
|
assert.equal(JSON.parse(await cancelled.response.text()).error.code, "acp_cancelled");
|
|
} finally {
|
|
if (oldTimeout === undefined) delete process.env.DEVIN_AGENTIC_ACP_TIMEOUT_MS;
|
|
else process.env.DEVIN_AGENTIC_ACP_TIMEOUT_MS = oldTimeout;
|
|
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
}
|
|
});
|
|
|
|
test("premature tool narration is repaired once into a validated tool_use", async () => {
|
|
const tmpDir = sandboxTmp("devin-agentic-repair-");
|
|
const stateFile = path.join(tmpDir, "spawn-count");
|
|
const scriptFile = writeScenarioMock(
|
|
tmpDir,
|
|
`const fs = require("fs");
|
|
const stateFile = ${JSON.stringify(stateFile)};
|
|
const count = Number(fs.existsSync(stateFile) ? fs.readFileSync(stateFile, "utf8") : "0") + 1;
|
|
fs.writeFileSync(stateFile, String(count));
|
|
rl.on("line", (line) => {
|
|
const msg = JSON.parse(line);
|
|
if (acceptAskMode(msg)) return;
|
|
if (msg.method === "initialize") send({ jsonrpc: "2.0", id: msg.id, result: { protocolVersion: 1 } });
|
|
if (msg.method === "session/new") sendSession(msg, "repair");
|
|
if (msg.method === "session/prompt") {
|
|
const text = count === 1
|
|
? '<summary>Current State: inspected math.js. The immediate next step was to read math.test.js, then fix the bug and run npm test.</summary>'
|
|
: '<tool>{"name":"Read","arguments":{"file_path":"a.ts"}}</tool>';
|
|
send({ jsonrpc: "2.0", method: "session/update", params: { sessionId: "repair", update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text } } } });
|
|
send({ jsonrpc: "2.0", id: msg.id, result: { stopReason: "end_turn" } });
|
|
}
|
|
});`
|
|
);
|
|
const oldBin = process.env.CLI_DEVIN_AGENTIC_BIN;
|
|
process.env.CLI_DEVIN_AGENTIC_BIN = scriptFile;
|
|
try {
|
|
const result = await new DevinCliAgenticExecutor().execute({
|
|
model: "swe-1-7",
|
|
stream: false,
|
|
credentials: {},
|
|
body: {
|
|
tools: [
|
|
{
|
|
name: "Read",
|
|
input_schema: {
|
|
type: "object",
|
|
required: ["file_path"],
|
|
properties: { file_path: { type: "string" } },
|
|
},
|
|
},
|
|
],
|
|
messages: [{ role: "user", content: "Read a.ts" }],
|
|
},
|
|
});
|
|
assert.equal(result.response.status, 200, await result.response.clone().text());
|
|
assert.equal(JSON.parse(await result.response.text()).stop_reason, "tool_use");
|
|
assert.equal(fs.readFileSync(stateFile, "utf8"), "2");
|
|
} finally {
|
|
if (oldBin === undefined) delete process.env.CLI_DEVIN_AGENTIC_BIN;
|
|
else process.env.CLI_DEVIN_AGENTIC_BIN = oldBin;
|
|
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
}
|
|
});
|
|
|
|
test("premature tool narration fails closed when the single repair is still narrative", async () => {
|
|
const tmpDir = sandboxTmp("devin-agentic-repair-narrative-");
|
|
const stateFile = path.join(tmpDir, "spawn-count");
|
|
const scriptFile = writeScenarioMock(
|
|
tmpDir,
|
|
`const fs = require("fs");
|
|
const stateFile = ${JSON.stringify(stateFile)};
|
|
const count = Number(fs.existsSync(stateFile) ? fs.readFileSync(stateFile, "utf8") : "0") + 1;
|
|
fs.writeFileSync(stateFile, String(count));
|
|
rl.on("line", (line) => {
|
|
const msg = JSON.parse(line);
|
|
if (acceptAskMode(msg)) return;
|
|
if (msg.method === "initialize") send({ jsonrpc: "2.0", id: msg.id, result: { protocolVersion: 1 } });
|
|
if (msg.method === "session/new") sendSession(msg, "repair-narrative");
|
|
if (msg.method === "session/prompt") {
|
|
const text = count === 1
|
|
? "I'll start by reading the file."
|
|
: "I'll read the file now, then run the tests.";
|
|
send({ jsonrpc: "2.0", method: "session/update", params: { sessionId: "repair-narrative", update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text } } } });
|
|
send({ jsonrpc: "2.0", id: msg.id, result: { stopReason: "end_turn" } });
|
|
}
|
|
});`
|
|
);
|
|
const oldBin = process.env.CLI_DEVIN_AGENTIC_BIN;
|
|
process.env.CLI_DEVIN_AGENTIC_BIN = scriptFile;
|
|
try {
|
|
const result = await new DevinCliAgenticExecutor().execute({
|
|
model: "swe-1-7",
|
|
stream: false,
|
|
credentials: {},
|
|
body: {
|
|
tools: [
|
|
{
|
|
name: "Read",
|
|
input_schema: {
|
|
type: "object",
|
|
required: ["file_path"],
|
|
properties: { file_path: { type: "string" } },
|
|
},
|
|
},
|
|
],
|
|
messages: [{ role: "user", content: "Read a.ts" }],
|
|
},
|
|
});
|
|
assert.equal(result.response.status, 502);
|
|
const body = JSON.parse(await result.response.text());
|
|
assert.equal(body.error.code, "unexecuted_tool_intent");
|
|
assert.equal(fs.readFileSync(stateFile, "utf8"), "2");
|
|
} finally {
|
|
if (oldBin === undefined) delete process.env.CLI_DEVIN_AGENTIC_BIN;
|
|
else process.env.CLI_DEVIN_AGENTIC_BIN = oldBin;
|
|
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
}
|
|
});
|