fix(providers): devin-cli-agentic no longer forwards bare <summary> envelopes (#13691)

Devin's summarizer agent (heavily used by SWE-2 models) can answer a turn with a
bare <summary>...</summary> envelope and no <tool> tag. Previously that text was
passed straight through as the final assistant message with stop_reason
end_turn (finish_reason stop downstream), so agent loops treated an internal
progress report as the answer and stopped.

DevinCliAgenticExecutor now detects a whole-response <summary> envelope with no
tool call, issues one bounded repair prompt asking for either a <tool> call or
a plain final answer, and if the repair still comes back wrapped, strips the
envelope and uses the inner body as the final text so the turn always
terminates with something usable.

Also adds the swe-2/swe-2-medium/swe-2-high/swe-2-max entries that were
missing from DEVIN_MODEL_CATALOG (and their pricing table entries), so
devin-cli-agentic stops rejecting them with unknown_devin_model.

changelog.d/fixes/13691-devin-summary-envelope.md

@@ -0,0 +1 @@
+- fix(providers): devin-cli-agentic no longer forwards a bare `<summary>` envelope as the final answer for SWE-2 models; add missing swe-2 catalog entries (#13691)

open-sse/config/providers/registry/devin/catalog.ts

@@ -107,6 +107,10 @@ export const DEVIN_MODEL_CATALOG: RegistryModel[] = [
   model("swe-1-7-medium", "SWE-1.7 Medium", 128_000, 262_000),
   model("swe-1-7-lightning", "SWE-1.7 Lightning Max", 96_000, 202_752),
   model("swe-1-7-lightning-medium", "SWE-1.7 Lightning Medium", 96_000, 202_752),
+  model("swe-2", "SWE-2", 128_000, 262_000),
+  model("swe-2-medium", "SWE-2 Medium", 128_000, 262_000),
+  model("swe-2-high", "SWE-2 High", 128_000, 262_000),
+  model("swe-2-max", "SWE-2 Max", 128_000, 262_000),
   model("adaptive", "Adaptive"),
 
   ...effortModels("grok-4-6", "Grok 4.6", 100_000, 500_000, [

open-sse/executors/devin-agentic/toolParser.ts

@@ -66,6 +66,21 @@ function validateSchema(value: unknown, schema: JsonRecord, path: string): strin
   return errors;
 }
 
+const BARE_SUMMARY_ENVELOPE_RE = /^<summary>\s*([\s\S]*?)\s*<\/summary>$/i;
+
+/**
+ * Detects a whole-response Devin "summarizer" progress report: an internal
+ * `<summary>...</summary>` envelope with no `<tool>` request alongside it.
+ * Mirrors the strict whole-string match used for `<tool>` above — a
+ * narrative block that merely mentions "summary" inline must not match.
+ * Returns the inner body, or null when the text is not a bare envelope.
+ */
+export function extractBareSummaryEnvelope(text: string): string | null {
+  const trimmed = text.trim();
+  const match = trimmed.match(BARE_SUMMARY_ENVELOPE_RE);
+  return match ? match[1].trim() : null;
+}
+
 export function parseDevinToolRequest(text: string, tools: AnthropicTool[], idSeed = "") {
   const matches = [...text.matchAll(/<tool>\s*([\s\S]*?)\s*<\/tool>/g)];
   if (matches.length === 0) return null;

open-sse/executors/devin-cli-agentic.ts

@@ -12,7 +12,7 @@ import {
   buildClaudeToolUseResponse,
 } from "./devin-agentic/anthropicResponse.ts";
 import { serializeAnthropicForDevin } from "./devin-agentic/serializer.ts";
-import { parseDevinToolRequest } from "./devin-agentic/toolParser.ts";
+import { extractBareSummaryEnvelope, parseDevinToolRequest } from "./devin-agentic/toolParser.ts";
 import { asRecord, DevinAgenticBridgeError, estimateTokens } from "./devin-agentic/types.ts";
 
 type AcpMessage = {
@@ -35,6 +35,7 @@ const REPAIRABLE_TOOL_ERRORS = new Set([
   "multiple_tool_requests",
   "mixed_tool_narrative",
   "unexecuted_tool_intent",
+  "bare_summary_envelope",
 ]);
 
 function describesUnexecutedToolIntent(text: string): boolean {
@@ -444,6 +445,20 @@ async function generateAgenticOutput(
   return first;
 }
 
+function buildRepairInstruction(errorCode: string): string {
+  if (errorCode === "unexecuted_tool_intent") {
+    return "Plain text is not accepted for this repair. Return exactly one standalone <tool> JSON envelope now.";
+  }
+  if (errorCode === "bare_summary_envelope") {
+    return [
+      "Do not return a <summary> progress report.",
+      "If more work remains, return exactly one <tool> envelope now.",
+      "Otherwise, return the final user-facing answer as plain text, with no <summary> tags.",
+    ].join(" ");
+  }
+  return "Return either plain final text or exactly one standalone <tool> JSON envelope.";
+}
+
 function extractText(value: unknown): string {
   if (typeof value === "string") return value;
   if (Array.isArray(value)) return value.map((item) => extractText(item)).join("");
@@ -497,6 +512,12 @@ export class DevinCliAgenticExecutor extends BaseExecutor {
           );
         }
         tool = parseDevinToolRequest(text, prompt.tools, prompt.idSeed);
+        if (!tool && extractBareSummaryEnvelope(text) !== null) {
+          throw new DevinAgenticBridgeError(
+            "The response was a bare <summary> progress report with no tool call or final answer",
+            "bare_summary_envelope"
+          );
+        }
       } catch (error) {
         if (
           !(error instanceof DevinAgenticBridgeError) ||
@@ -512,9 +533,7 @@ export class DevinCliAgenticExecutor extends BaseExecutor {
           "",
           "[Single Repair Attempt]",
           `The previous output was rejected: ${sanitizeErrorMessage(error.message)}`,
-          requiresToolOnRepair
-            ? "Plain text is not accepted for this repair. Return exactly one standalone <tool> JSON envelope now."
-            : "Return either plain final text or exactly one standalone <tool> JSON envelope.",
+          buildRepairInstruction(error.code),
           "Do not narrate a tool action.",
         ].join("\n");
         text = await generateAgenticOutput(turnArgs, repairPrompt);
@@ -526,6 +545,15 @@ export class DevinCliAgenticExecutor extends BaseExecutor {
             502
           );
         }
+        if (!tool && error.code === "bare_summary_envelope") {
+          const stillBare = extractBareSummaryEnvelope(text);
+          if (stillBare !== null) {
+            // Bounded retry exhausted — strip the wrapper and use the inner
+            // body as the final answer rather than surfacing an internal
+            // progress-report format to the caller.
+            text = stillBare;
+          }
+        }
       }
 
       const id = `msg_devin_${randomUUID().replaceAll("-", "")}`;

src/shared/constants/pricing/devin.ts

@@ -93,6 +93,13 @@ export const DEVIN_MODEL_PRICING: Record<string, DevinTokenPricing> = {
     cached: 1,
     output: 12.5,
   }),
+  // swe-2 family (#13691): same rate as swe-1-7 pending a confirmed live-catalog
+  // price for the newer tier — kept provider-bound like the rest of this table.
+  ...priced(["swe-2", "swe-2-medium", "swe-2-high", "swe-2-max"], {
+    input: 0.5,
+    cached: 0.2,
+    output: 2.5,
+  }),
   adaptive: { input: 0.5, cached: 0.1, output: 2 },
   ...priced(variantIds("grok-4-6", ["xhigh", "high", "medium", "low"]), {
     input: 2,

tests/unit/devin-cli-agentic-summary-passthrough.test.ts

@@ -0,0 +1,105 @@
+// Regression test for issue #13691: devin-cli-agentic forwarded a bare
+// <summary>...</summary> envelope (Devin's internal "summarizer" progress
+// report) as the final assistant text verbatim, with stop_reason "end_turn"
+// (translated to OpenAI finish_reason "stop" downstream). SWE-2 models are
+// heavily tuned to emit this envelope even when no further tool call is
+// needed, so agent loops treated the internal report as the answer and
+// stopped.
+//
+// This test drives the real DevinCliAgenticExecutor against a fake local ACP
+// bridge (tests/unit/fake-devin-acp-summarizer.mjs) that reproduces exactly
+// that behavior: it answers a session/prompt with a <summary> envelope and no
+// <tool> tag. No network/devin binary is involved.
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { DevinCliAgenticExecutor } from "../../open-sse/executors/devin-cli-agentic.ts";
+import { DEVIN_MODEL_CATALOG } from "../../open-sse/config/providers/registry/devin/catalog.ts";
+import { extractBareSummaryEnvelope } from "../../open-sse/executors/devin-agentic/toolParser.ts";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const FAKE_BIN = path.join(__dirname, "fake-devin-acp-summarizer.mjs");
+
+test("devin-cli-agentic must not forward a bare <summary> envelope as the final answer (#13691)", async () => {
+  const sandboxHome = fs.mkdtempSync(path.join(os.tmpdir(), "devin-agentic-"));
+  const home = path.join(sandboxHome, ".sandbox", "home");
+  fs.mkdirSync(home, { recursive: true });
+
+  const previousBin = process.env.CLI_DEVIN_AGENTIC_BIN;
+  const previousHome = process.env.DEVIN_AGENTIC_HOME;
+  process.env.CLI_DEVIN_AGENTIC_BIN = FAKE_BIN;
+  process.env.DEVIN_AGENTIC_HOME = home;
+
+  try {
+    const executor = new DevinCliAgenticExecutor();
+    const result = await executor.execute({
+      model: "claude-sonnet-4-6",
+      stream: false,
+      credentials: {},
+      body: {
+        model: "claude-sonnet-4-6",
+        max_tokens: 1024,
+        messages: [{ role: "user", content: "Look up the record for me." }],
+        tools: [
+          {
+            name: "lookup",
+            description: "Look up a record",
+            input_schema: { type: "object", properties: {}, additionalProperties: false },
+          },
+        ],
+      },
+    });
+
+    assert.equal(result.response.status, 200, "executor must not error out on this input");
+    const payload = await result.response.json();
+    assert.equal(payload.type, "message");
+
+    const textBlock = (payload.content || []).find(
+      (block: { type?: string }) => block.type === "text"
+    );
+    assert.ok(textBlock, "expected a text content block in the response");
+    const finalText: string = textBlock.text;
+
+    // The fake bridge always answers with the same <summary> envelope, no
+    // matter what repair prompt is sent, so the bounded-retry path exhausts
+    // and the executor must fall back to stripping the wrapper.
+    assert.ok(
+      !/<summary>/i.test(finalText),
+      `expected the <summary> envelope to be stripped/handled before being returned as the final answer, got:\n${finalText}`
+    );
+    assert.ok(
+      finalText.includes("Lookup finished and the record was located."),
+      "expected the stripped inner body to still carry the useful content"
+    );
+  } finally {
+    if (previousBin === undefined) delete process.env.CLI_DEVIN_AGENTIC_BIN;
+    else process.env.CLI_DEVIN_AGENTIC_BIN = previousBin;
+    if (previousHome === undefined) delete process.env.DEVIN_AGENTIC_HOME;
+    else process.env.DEVIN_AGENTIC_HOME = previousHome;
+    fs.rmSync(sandboxHome, { recursive: true, force: true });
+  }
+});
+
+test("extractBareSummaryEnvelope only matches a whole-string <summary> wrapper (#13691)", () => {
+  assert.equal(extractBareSummaryEnvelope("<summary>inner text</summary>"), "inner text");
+  assert.equal(extractBareSummaryEnvelope("  <summary>\ninner text\n</summary>  "), "inner text");
+  // A narrative block that merely mentions "summary" or has a <summary> tag
+  // alongside other prose must not be treated as a bare envelope.
+  assert.equal(extractBareSummaryEnvelope("Here is a summary of what I did."), null);
+  assert.equal(
+    extractBareSummaryEnvelope("<summary>partial</summary>\nSome trailing narrative."),
+    null
+  );
+});
+
+test("swe-2 model family is present in the Devin catalog (#13691)", () => {
+  const ids = DEVIN_MODEL_CATALOG.map((entry) => entry.id);
+  for (const id of ["swe-2", "swe-2-medium", "swe-2-high", "swe-2-max"]) {
+    assert.ok(ids.includes(id), `expected ${id} to be in DEVIN_MODEL_CATALOG`);
+  }
+});

tests/unit/devin-cli-catalog.test.ts

@@ -17,7 +17,8 @@ test("Devin transports expose the same curated catalog without duplicate ids", (
     devin_cli_agenticProvider.models.map((model) => model.id),
     catalogIds
   );
-  assert.equal(catalogIds.length, 110);
+  // +4 for the swe-2 family added in #13691 (swe-2, swe-2-medium, swe-2-high, swe-2-max).
+  assert.equal(catalogIds.length, 114);
   assert.equal(new Set(catalogIds).size, catalogIds.length);
   assert.ok(catalogIds.every((id) => !id.toLowerCase().includes("byok")));
 });
@@ -39,6 +40,8 @@ test("Devin catalog contains only the operator-selected model families", () => {
     "glm-5-3-flash-max",
     "swe-1-7",
     "swe-1-7-lightning",
+    "swe-2",
+    "swe-2-max",
     "adaptive",
     "grok-4-6-xhigh",
     "inkling-max",

tests/unit/fake-devin-acp-summarizer.mjs

@@ -0,0 +1,60 @@
+#!/usr/bin/env node
+// Fake local ACP bridge used by tests/unit/devin-cli-agentic-summary-passthrough.test.ts
+// (issue #13691). Reproduces the exact SWE-2 behavior: answers a session/prompt
+// with a bare <summary>...</summary> envelope and no <tool> tag.
+import readline from "node:readline";
+
+const SUMMARY_TEXT = [
+  "<summary>",
+  "Overview",
+  "Searched the knowledge base for the requested record.",
+  "",
+  "Key Details & Breadcrumbs",
+  "- Found the relevant entry in the catalog table.",
+  "",
+  "Current State",
+  "Lookup finished and the record was located.",
+  "</summary>",
+].join("\n");
+
+const rl = readline.createInterface({ input: process.stdin });
+
+function send(obj) {
+  process.stdout.write(JSON.stringify(obj) + "\n");
+}
+
+const sessionId = "fake-session-1";
+
+rl.on("line", (line) => {
+  const trimmed = line.trim();
+  if (!trimmed) return;
+  let msg;
+  try {
+    msg = JSON.parse(trimmed);
+  } catch {
+    return;
+  }
+  if (msg.method === "initialize") {
+    send({ jsonrpc: "2.0", id: msg.id, result: { protocolVersion: 1 } });
+    return;
+  }
+  if (msg.method === "session/new") {
+    send({ jsonrpc: "2.0", id: msg.id, result: { sessionId } });
+    return;
+  }
+  if (msg.method === "session/prompt") {
+    send({
+      jsonrpc: "2.0",
+      method: "session/update",
+      params: {
+        sessionId,
+        update: {
+          sessionUpdate: "agent_message_chunk",
+          content: { type: "text", text: SUMMARY_TEXT },
+        },
+      },
+    });
+    send({ jsonrpc: "2.0", id: msg.id, result: { stopReason: "end_turn" } });
+    return;
+  }
+});