diff --git a/scripts/devin-bridge/common b/scripts/devin-bridge/common index a8e6baef51..2b6b1f3b92 100755 --- a/scripts/devin-bridge/common +++ b/scripts/devin-bridge/common @@ -10,6 +10,11 @@ bridge_prepare_sandbox() { chmod 0777 "$BRIDGE_SANDBOX/e2e-workspace" "$BRIDGE_SANDBOX/live-workspace" \ "$BRIDGE_SANDBOX/evidence" } +bridge_reset_claude_egress_audit() { + bridge_prepare_sandbox + : >"$BRIDGE_SANDBOX/evidence/claude-egress.jsonl" + chmod 0666 "$BRIDGE_SANDBOX/evidence/claude-egress.jsonl" +} bridge_reset_e2e_fixture() { bridge_prepare_sandbox cp -R "$BRIDGE_ROOT/tests/fixtures/devin-bridge/e2e-workspace/." \ @@ -17,6 +22,7 @@ bridge_reset_e2e_fixture() { rm -f "$BRIDGE_SANDBOX/e2e-workspace/.e2e-hook.log" \ "$BRIDGE_SANDBOX/evidence/claude-stream.jsonl" \ "$BRIDGE_SANDBOX/evidence/mock-acp.jsonl" + bridge_reset_claude_egress_audit } bridge_reset_live_fixture() { bridge_prepare_sandbox @@ -27,8 +33,8 @@ bridge_reset_live_fixture() { "$BRIDGE_SANDBOX/evidence/live-fix.jsonl" \ "$BRIDGE_SANDBOX/evidence/live-command.jsonl" \ "$BRIDGE_SANDBOX/evidence/live-models.json" \ - "$BRIDGE_SANDBOX/evidence/egress.jsonl" \ - "$BRIDGE_SANDBOX/evidence/claude-egress.jsonl" + "$BRIDGE_SANDBOX/evidence/egress.jsonl" + bridge_reset_claude_egress_audit } bridge_test_env() { bridge_prepare_sandbox @@ -73,3 +79,37 @@ bridge_check_devin_auth() { printf '%s\n' "$output" bridge_assert_devin_auth_status "$exit_status" "$output" } + +bridge_assert_zero_claude_egress() { + local audit_path="$1" + [[ -f "$audit_path" ]] || { + printf 'FAIL: Claude egress audit file is missing\n' >&2 + return 1 + } + [[ ! -s "$audit_path" ]] || { + printf 'FAIL: Claude attempted external egress during the real run\n' >&2 + return 1 + } +} + +bridge_assert_claude_guard_denials() { + local audit_path="$1" + [[ -s "$audit_path" ]] || { + printf 'FAIL: Claude egress denial audit is missing or empty\n' >&2 + return 1 + } + node -e ' + const fs = require("node:fs"); + const entries = fs.readFileSync(process.argv[1], "utf8") + .trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)); + if (!entries.length) throw new Error("Claude egress audit has no records"); + if (entries.some((entry) => entry.decision !== "deny")) { + throw new Error("Claude egress audit contains a non-deny decision"); + } + for (const hostname of ["api.anthropic.com", "claude.ai"]) { + if (!entries.some((entry) => entry.hostname === hostname && entry.decision === "deny")) { + throw new Error(`Claude egress audit is missing deny for ${hostname}`); + } + } + ' "$audit_path" +} diff --git a/scripts/devin-bridge/test-e2e-mock b/scripts/devin-bridge/test-e2e-mock index 14d2e9d8b8..344cbef1e8 100755 --- a/scripts/devin-bridge/test-e2e-mock +++ b/scripts/devin-bridge/test-e2e-mock @@ -8,4 +8,5 @@ docker compose -f "$BRIDGE_COMPOSE" --profile offline up --abort-on-container-ex --exit-code-from claude claude grep -q '"action":"final"' "$BRIDGE_SANDBOX/evidence/mock-acp.jsonl" grep -q 'BRIDGE_E2E_COMPLETE' "$BRIDGE_SANDBOX/evidence/claude-stream.jsonl" +bridge_assert_zero_claude_egress "$BRIDGE_SANDBOX/evidence/claude-egress.jsonl" printf 'PASS: real Claude Code completed the offline agentic fixture\n' diff --git a/scripts/devin-bridge/test-live-devin b/scripts/devin-bridge/test-live-devin index b0d6ffaab3..5948978076 100755 --- a/scripts/devin-bridge/test-live-devin +++ b/scripts/devin-bridge/test-live-devin @@ -29,15 +29,5 @@ node -e ' } } ' "$BRIDGE_SANDBOX/evidence/egress.jsonl" -node -e ' - const fs = require("node:fs"); - const path = process.argv[1]; - if (!fs.existsSync(path)) process.exit(0); - for (const line of fs.readFileSync(path, "utf8").trim().split("\n").filter(Boolean)) { - const entry = JSON.parse(line); - if (entry.decision === "allow") { - throw new Error(`Claude guard allowed egress: ${entry.hostname}`); - } - } -' "$BRIDGE_SANDBOX/evidence/claude-egress.jsonl" +bridge_assert_zero_claude_egress "$BRIDGE_SANDBOX/evidence/claude-egress.jsonl" printf 'PASS: live model %s was discovered and validated by three scenarios\n' "$devin_model" diff --git a/scripts/devin-bridge/verify-anthropic-isolation b/scripts/devin-bridge/verify-anthropic-isolation index 101a572014..4db231e001 100755 --- a/scripts/devin-bridge/verify-anthropic-isolation +++ b/scripts/devin-bridge/verify-anthropic-isolation @@ -187,6 +187,7 @@ grep -q 'bridge_check_devin_auth' "$BRIDGE_ROOT/scripts/devin-bridge/launch" || grep -q 'up -d network-guard claude-egress-guard' "$BRIDGE_ROOT/scripts/devin-bridge/launch" || \ fail "normal launch does not start the audited Claude egress guard" if [[ "${1:-}" == --static ]]; then printf 'PASS: static bridge isolation checks passed\n'; exit 0; fi +bridge_reset_claude_egress_audit docker compose -f "$BRIDGE_COMPOSE" --profile offline up -d claude-egress-guard cleanup_claude_guard() { docker compose -f "$BRIDGE_COMPOSE" --profile offline stop claude-egress-guard >/dev/null 2>&1 || true @@ -209,6 +210,30 @@ docker compose -f "$BRIDGE_COMPOSE" --profile offline run --rm --no-deps claude fi done ' +docker compose -f "$BRIDGE_COMPOSE" --profile offline run --rm --no-deps claude \ + node --input-type=module -e ' + async function expectProxyDenial(request) { + try { + const response = await request; + if (response.status !== 403) { + throw new Error(`unexpected proxy response: ${response.status}`); + } + } catch (error) { + if (error instanceof Error && error.message.startsWith("unexpected proxy response:")) { + throw error; + } + } + } + await expectProxyDenial(fetch("https://api.anthropic.com", { + signal: AbortSignal.timeout(3000), + })); + await expectProxyDenial(fetch("https://claude.ai", { + signal: AbortSignal.timeout(3000), + })); +' +bridge_assert_claude_guard_denials "$BRIDGE_SANDBOX/evidence/claude-egress.jsonl" || \ + fail "Claude proxy denial audit proof failed" cleanup_claude_guard trap - EXIT +bridge_reset_claude_egress_audit printf 'PASS: runtime bridge isolation checks passed\n' diff --git a/tests/unit/devin-bridge-live-runtime.test.ts b/tests/unit/devin-bridge-live-runtime.test.ts index a183cc9fe3..a23ef72bee 100644 --- a/tests/unit/devin-bridge-live-runtime.test.ts +++ b/tests/unit/devin-bridge-live-runtime.test.ts @@ -7,9 +7,12 @@ import test from "node:test"; const root = process.cwd(); const composePath = path.join(root, "docker", "devin-bridge", "compose.yml"); const commonPath = path.join(root, "scripts", "devin-bridge", "common"); +const mockE2ePath = path.join(root, "scripts", "devin-bridge", "test-e2e-mock"); const launchPath = path.join(root, "scripts", "devin-bridge", "launch"); const loginPath = path.join(root, "scripts", "devin-bridge", "login-devin"); const selectorPath = path.join(root, "scripts", "devin-bridge", "select-live-model.mjs"); +const liveE2ePath = path.join(root, "scripts", "devin-bridge", "test-live-devin"); +const verifierPath = path.join(root, "scripts", "devin-bridge", "verify-anthropic-isolation"); function composeConfig() { const result = spawnSync( @@ -67,6 +70,14 @@ function selectModel(document: unknown, env: NodeJS.ProcessEnv = {}) { }); } +function runCommon(functionName: string, filePath: string) { + return spawnSync( + "bash", + ["-c", 'source "$1"; "$2" "$3"', "bridge-audit-test", commonPath, functionName, filePath], + { cwd: root, encoding: "utf8" } + ); +} + test("network policy permits only Devin/Cognition and exact Codeium control-plane hosts", async () => { const { isAllowedGuardHostname } = await import("../../docker/devin-bridge/network-guard/policy.mjs"); @@ -183,3 +194,57 @@ test("normal live launch uses the strict auth and proxied Devin helpers", () => assert.match(launch, /bridge_run_devin models list --format json/); assert.doesNotMatch(launch, /\bdevin auth status\b/); }); + +test("Claude audit helpers fail closed for missing, empty, partial, or any real-run record", () => { + const sandboxRoot = path.join(root, ".sandbox"); + fs.mkdirSync(sandboxRoot, { recursive: true }); + const auditRoot = fs.mkdtempSync(path.join(sandboxRoot, "audit-unit-")); + const auditPath = path.join(auditRoot, "claude-egress.jsonl"); + try { + assert.notEqual(runCommon("bridge_assert_zero_claude_egress", auditPath).status, 0); + fs.writeFileSync(auditPath, ""); + assert.equal(runCommon("bridge_assert_zero_claude_egress", auditPath).status, 0); + fs.writeFileSync(auditPath, '{"hostname":"api.anthropic.com","decision":"deny"}\n'); + assert.notEqual(runCommon("bridge_assert_zero_claude_egress", auditPath).status, 0); + + assert.notEqual(runCommon("bridge_assert_claude_guard_denials", auditPath).status, 0); + fs.appendFileSync(auditPath, '{"hostname":"claude.ai","decision":"deny"}\n'); + assert.equal(runCommon("bridge_assert_claude_guard_denials", auditPath).status, 0); + fs.appendFileSync(auditPath, '{"hostname":"example.com","decision":"allow"}\n'); + assert.notEqual(runCommon("bridge_assert_claude_guard_denials", auditPath).status, 0); + } finally { + fs.rmSync(auditRoot, { recursive: true, force: true }); + } +}); + +test("offline and live resets precreate an empty world-writable Claude audit", () => { + const auditPath = path.join(root, ".sandbox", "evidence", "claude-egress.jsonl"); + for (const resetFunction of ["bridge_reset_e2e_fixture", "bridge_reset_live_fixture"]) { + const reset = spawnSync( + "bash", + ["-c", 'source "$1"; "$2"', "bridge-reset-test", commonPath, resetFunction], + { cwd: root, encoding: "utf8" } + ); + assert.equal(reset.status, 0, reset.stderr); + assert.equal(fs.existsSync(auditPath), true, resetFunction); + assert.equal(fs.statSync(auditPath).size, 0, resetFunction); + assert.equal(fs.statSync(auditPath).mode & 0o777, 0o666, resetFunction); + fs.writeFileSync(auditPath, "record"); + } +}); + +test("verifier proves audited denials while real E2E gates require zero Claude attempts", () => { + const verifier = fs.readFileSync(verifierPath, "utf8"); + const mockE2e = fs.readFileSync(mockE2ePath, "utf8"); + const liveE2e = fs.readFileSync(liveE2ePath, "utf8"); + + assert.match(verifier, /fetch\("https:\/\/api\.anthropic\.com"/); + assert.match(verifier, /fetch\("https:\/\/claude\.ai"/); + assert.match(verifier, /bridge_assert_claude_guard_denials/); + assert.ok( + [...verifier.matchAll(/bridge_reset_claude_egress_audit/g)].length >= 2, + "deliberate proof must reset the audit before and after its own requests" + ); + assert.match(mockE2e, /bridge_assert_zero_claude_egress/); + assert.match(liveE2e, /bridge_assert_zero_claude_egress/); +});