mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
fix(privacy): resolve PII feature flag correctly and fix PII response sanitization in streaming SSE requests (#3021)
Integrated into release/v3.8.8. Fixes the PII feature-flag resolution + adds streaming-aware SSE PII sanitization (rolling-window transform, obfuscation-hardened regexes, Luhn/CPF/CNPJ checks). Applied review fixes: block-mode + fallback errors no longer leak pattern types / upstream err.message into stream errors (Hard Rule #12); typed the createPiiTransform cast (no any); fixed a string===boolean type error; dropped the generated .source/* files. Thanks @dangeReis!
This commit is contained in:
dangeReis
4
.gitignore
vendored
4
.gitignore
vendored
@@ -193,3 +193,7 @@ scripts/i18n/_pending-keys.json
|
||||
.agents/
|
||||
.antigravitycli/
|
||||
.claude/
|
||||
|
||||
# PR Reviews and local feedback files
|
||||
pr_reviews*.json
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -1,83 +0,0 @@
|
||||
// @ts-nocheck
|
||||
import { default as __fd_glob_72 } from "../docs/security/meta.json?collection=docs"
|
||||
import { default as __fd_glob_71 } from "../docs/routing/meta.json?collection=docs"
|
||||
import { default as __fd_glob_70 } from "../docs/reference/openapi.yaml?collection=docs"
|
||||
import { default as __fd_glob_69 } from "../docs/reference/meta.json?collection=docs"
|
||||
import { default as __fd_glob_68 } from "../docs/ops/meta.json?collection=docs"
|
||||
import { default as __fd_glob_67 } from "../docs/guides/meta.json?collection=docs"
|
||||
import { default as __fd_glob_66 } from "../docs/frameworks/meta.json?collection=docs"
|
||||
import { default as __fd_glob_65 } from "../docs/compression/meta.json?collection=docs"
|
||||
import { default as __fd_glob_64 } from "../docs/architecture/meta.json?collection=docs"
|
||||
import { default as __fd_glob_63 } from "../docs/meta.json?collection=docs"
|
||||
import * as __fd_glob_62 from "../docs/security/STEALTH_GUIDE.md?collection=docs"
|
||||
import * as __fd_glob_61 from "../docs/security/SOCKET_DEV_FINDINGS.md?collection=docs"
|
||||
import * as __fd_glob_60 from "../docs/security/ROUTE_GUARD_TIERS.md?collection=docs"
|
||||
import * as __fd_glob_59 from "../docs/security/PUBLIC_CREDS.md?collection=docs"
|
||||
import * as __fd_glob_58 from "../docs/security/GUARDRAILS.md?collection=docs"
|
||||
import * as __fd_glob_57 from "../docs/security/ERROR_SANITIZATION.md?collection=docs"
|
||||
import * as __fd_glob_56 from "../docs/security/COMPLIANCE.md?collection=docs"
|
||||
import * as __fd_glob_55 from "../docs/security/CLI_TOKEN_AUTH.md?collection=docs"
|
||||
import * as __fd_glob_54 from "../docs/security/CLI_TOKEN.md?collection=docs"
|
||||
import * as __fd_glob_53 from "../docs/routing/REASONING_REPLAY.md?collection=docs"
|
||||
import * as __fd_glob_52 from "../docs/routing/QUOTA_SHARE.md?collection=docs"
|
||||
import * as __fd_glob_51 from "../docs/routing/AUTO-COMBO.md?collection=docs"
|
||||
import * as __fd_glob_50 from "../docs/reference/PROVIDER_REFERENCE.md?collection=docs"
|
||||
import * as __fd_glob_49 from "../docs/reference/FREE_TIERS.md?collection=docs"
|
||||
import * as __fd_glob_48 from "../docs/reference/ENVIRONMENT.md?collection=docs"
|
||||
import * as __fd_glob_47 from "../docs/reference/CLI-TOOLS.md?collection=docs"
|
||||
import * as __fd_glob_46 from "../docs/reference/API_REFERENCE.md?collection=docs"
|
||||
import * as __fd_glob_45 from "../docs/ops/VM_DEPLOYMENT_GUIDE.md?collection=docs"
|
||||
import * as __fd_glob_44 from "../docs/ops/TUNNELS_GUIDE.md?collection=docs"
|
||||
import * as __fd_glob_43 from "../docs/ops/SQLITE_RUNTIME.md?collection=docs"
|
||||
import * as __fd_glob_42 from "../docs/ops/RELEASE_CHECKLIST.md?collection=docs"
|
||||
import * as __fd_glob_41 from "../docs/ops/PROXY_GUIDE.md?collection=docs"
|
||||
import * as __fd_glob_40 from "../docs/ops/FLY_IO_DEPLOYMENT_GUIDE.md?collection=docs"
|
||||
import * as __fd_glob_39 from "../docs/ops/E2E_DASHBOARD_SHAKEDOWN_v3.8.0.md?collection=docs"
|
||||
import * as __fd_glob_38 from "../docs/ops/COVERAGE_PLAN.md?collection=docs"
|
||||
import * as __fd_glob_37 from "../docs/guides/USER_GUIDE.md?collection=docs"
|
||||
import * as __fd_glob_36 from "../docs/guides/UNINSTALL.md?collection=docs"
|
||||
import * as __fd_glob_35 from "../docs/guides/TROUBLESHOOTING.md?collection=docs"
|
||||
import * as __fd_glob_34 from "../docs/guides/TERMUX_GUIDE.md?collection=docs"
|
||||
import * as __fd_glob_33 from "../docs/guides/SETUP_GUIDE.md?collection=docs"
|
||||
import * as __fd_glob_32 from "../docs/guides/PWA_GUIDE.md?collection=docs"
|
||||
import * as __fd_glob_31 from "../docs/guides/KIRO_SETUP.md?collection=docs"
|
||||
import * as __fd_glob_30 from "../docs/guides/I18N.md?collection=docs"
|
||||
import * as __fd_glob_29 from "../docs/guides/FEATURES.md?collection=docs"
|
||||
import * as __fd_glob_28 from "../docs/guides/ELECTRON_GUIDE.md?collection=docs"
|
||||
import * as __fd_glob_27 from "../docs/guides/DOCKER_GUIDE.md?collection=docs"
|
||||
import * as __fd_glob_26 from "../docs/frameworks/WEBHOOKS.md?collection=docs"
|
||||
import * as __fd_glob_25 from "../docs/frameworks/TRAFFIC_INSPECTOR.md?collection=docs"
|
||||
import * as __fd_glob_24 from "../docs/frameworks/SKILLS.md?collection=docs"
|
||||
import * as __fd_glob_23 from "../docs/frameworks/SEARCH_TOOLS_STUDIO.md?collection=docs"
|
||||
import * as __fd_glob_22 from "../docs/frameworks/PLAYGROUND_STUDIO.md?collection=docs"
|
||||
import * as __fd_glob_21 from "../docs/frameworks/OPENCODE.md?collection=docs"
|
||||
import * as __fd_glob_20 from "../docs/frameworks/MEMORY.md?collection=docs"
|
||||
import * as __fd_glob_19 from "../docs/frameworks/MCP-SERVER.md?collection=docs"
|
||||
import * as __fd_glob_18 from "../docs/frameworks/GAMIFICATION.md?collection=docs"
|
||||
import * as __fd_glob_17 from "../docs/frameworks/EVALS.md?collection=docs"
|
||||
import * as __fd_glob_16 from "../docs/frameworks/EMBEDDED-SERVICES.md?collection=docs"
|
||||
import * as __fd_glob_15 from "../docs/frameworks/CLOUD_AGENT.md?collection=docs"
|
||||
import * as __fd_glob_14 from "../docs/frameworks/AGENT_PROTOCOLS_GUIDE.md?collection=docs"
|
||||
import * as __fd_glob_13 from "../docs/frameworks/AGENTBRIDGE.md?collection=docs"
|
||||
import * as __fd_glob_12 from "../docs/frameworks/AGENT-SKILLS.md?collection=docs"
|
||||
import * as __fd_glob_11 from "../docs/frameworks/A2A-SERVER.md?collection=docs"
|
||||
import * as __fd_glob_10 from "../docs/compression/RTK_COMPRESSION.md?collection=docs"
|
||||
import * as __fd_glob_9 from "../docs/compression/COMPRESSION_RULES_FORMAT.md?collection=docs"
|
||||
import * as __fd_glob_8 from "../docs/compression/COMPRESSION_LANGUAGE_PACKS.md?collection=docs"
|
||||
import * as __fd_glob_7 from "../docs/compression/COMPRESSION_GUIDE.md?collection=docs"
|
||||
import * as __fd_glob_6 from "../docs/compression/COMPRESSION_ENGINES.md?collection=docs"
|
||||
import * as __fd_glob_5 from "../docs/architecture/RESILIENCE_GUIDE.md?collection=docs"
|
||||
import * as __fd_glob_4 from "../docs/architecture/REPOSITORY_MAP.md?collection=docs"
|
||||
import * as __fd_glob_3 from "../docs/architecture/MONITORING_SECTIONS.md?collection=docs"
|
||||
import * as __fd_glob_2 from "../docs/architecture/CODEBASE_DOCUMENTATION.md?collection=docs"
|
||||
import * as __fd_glob_1 from "../docs/architecture/AUTHZ_GUIDE.md?collection=docs"
|
||||
import * as __fd_glob_0 from "../docs/architecture/ARCHITECTURE.md?collection=docs"
|
||||
import { server } from 'fumadocs-mdx/runtime/server';
|
||||
import type * as Config from '../source.config';
|
||||
|
||||
const create = server<typeof Config, import("fumadocs-mdx/runtime/types").InternalTypeConfig & {
|
||||
DocData: {
|
||||
}
|
||||
}>({"doc":{"passthroughs":["extractedReferences"]}});
|
||||
|
||||
export const docs = await create.docs("docs", "docs", {"meta.json": __fd_glob_63, "architecture/meta.json": __fd_glob_64, "compression/meta.json": __fd_glob_65, "frameworks/meta.json": __fd_glob_66, "guides/meta.json": __fd_glob_67, "ops/meta.json": __fd_glob_68, "reference/meta.json": __fd_glob_69, "reference/openapi.yaml": __fd_glob_70, "routing/meta.json": __fd_glob_71, "security/meta.json": __fd_glob_72, }, {"architecture/ARCHITECTURE.md": __fd_glob_0, "architecture/AUTHZ_GUIDE.md": __fd_glob_1, "architecture/CODEBASE_DOCUMENTATION.md": __fd_glob_2, "architecture/MONITORING_SECTIONS.md": __fd_glob_3, "architecture/REPOSITORY_MAP.md": __fd_glob_4, "architecture/RESILIENCE_GUIDE.md": __fd_glob_5, "compression/COMPRESSION_ENGINES.md": __fd_glob_6, "compression/COMPRESSION_GUIDE.md": __fd_glob_7, "compression/COMPRESSION_LANGUAGE_PACKS.md": __fd_glob_8, "compression/COMPRESSION_RULES_FORMAT.md": __fd_glob_9, "compression/RTK_COMPRESSION.md": __fd_glob_10, "frameworks/A2A-SERVER.md": __fd_glob_11, "frameworks/AGENT-SKILLS.md": __fd_glob_12, "frameworks/AGENTBRIDGE.md": __fd_glob_13, "frameworks/AGENT_PROTOCOLS_GUIDE.md": __fd_glob_14, "frameworks/CLOUD_AGENT.md": __fd_glob_15, "frameworks/EMBEDDED-SERVICES.md": __fd_glob_16, "frameworks/EVALS.md": __fd_glob_17, "frameworks/GAMIFICATION.md": __fd_glob_18, "frameworks/MCP-SERVER.md": __fd_glob_19, "frameworks/MEMORY.md": __fd_glob_20, "frameworks/OPENCODE.md": __fd_glob_21, "frameworks/PLAYGROUND_STUDIO.md": __fd_glob_22, "frameworks/SEARCH_TOOLS_STUDIO.md": __fd_glob_23, "frameworks/SKILLS.md": __fd_glob_24, "frameworks/TRAFFIC_INSPECTOR.md": __fd_glob_25, "frameworks/WEBHOOKS.md": __fd_glob_26, "guides/DOCKER_GUIDE.md": __fd_glob_27, "guides/ELECTRON_GUIDE.md": __fd_glob_28, "guides/FEATURES.md": __fd_glob_29, "guides/I18N.md": __fd_glob_30, "guides/KIRO_SETUP.md": __fd_glob_31, "guides/PWA_GUIDE.md": __fd_glob_32, "guides/SETUP_GUIDE.md": __fd_glob_33, "guides/TERMUX_GUIDE.md": __fd_glob_34, "guides/TROUBLESHOOTING.md": __fd_glob_35, "guides/UNINSTALL.md": __fd_glob_36, "guides/USER_GUIDE.md": __fd_glob_37, "ops/COVERAGE_PLAN.md": __fd_glob_38, "ops/E2E_DASHBOARD_SHAKEDOWN_v3.8.0.md": __fd_glob_39, "ops/FLY_IO_DEPLOYMENT_GUIDE.md": __fd_glob_40, "ops/PROXY_GUIDE.md": __fd_glob_41, "ops/RELEASE_CHECKLIST.md": __fd_glob_42, "ops/SQLITE_RUNTIME.md": __fd_glob_43, "ops/TUNNELS_GUIDE.md": __fd_glob_44, "ops/VM_DEPLOYMENT_GUIDE.md": __fd_glob_45, "reference/API_REFERENCE.md": __fd_glob_46, "reference/CLI-TOOLS.md": __fd_glob_47, "reference/ENVIRONMENT.md": __fd_glob_48, "reference/FREE_TIERS.md": __fd_glob_49, "reference/PROVIDER_REFERENCE.md": __fd_glob_50, "routing/AUTO-COMBO.md": __fd_glob_51, "routing/QUOTA_SHARE.md": __fd_glob_52, "routing/REASONING_REPLAY.md": __fd_glob_53, "security/CLI_TOKEN.md": __fd_glob_54, "security/CLI_TOKEN_AUTH.md": __fd_glob_55, "security/COMPLIANCE.md": __fd_glob_56, "security/ERROR_SANITIZATION.md": __fd_glob_57, "security/GUARDRAILS.md": __fd_glob_58, "security/PUBLIC_CREDS.md": __fd_glob_59, "security/ROUTE_GUARD_TIERS.md": __fd_glob_60, "security/SOCKET_DEV_FINDINGS.md": __fd_glob_61, "security/STEALTH_GUIDE.md": __fd_glob_62, });
|
||||
@@ -284,6 +284,14 @@
|
||||
|
||||
- **mcp:** move `enforceScopes` guard before `MCP_TOOL_MAP` lookup, add inline `scopes` parameter to `withScopeEnforcement()`, and declare scopes on all 24 dynamic tool definitions (memory, skills, plugins, gamification, compression) to fix scope enforcement for dynamic MCP tool groups (#2958)
|
||||
|
||||
### ✨ New Features
|
||||
|
||||
- **notion:** add Notion as an MCP context source — 6 tools (`notion_search`, `notion_list_databases`, `notion_get_database`, `notion_query_database`, `notion_read`, `notion_append_blocks`) scoped under `read:notion` / `write:notion`, with dashboard "Context Sources" tab, settings API, and token persistence in `key_value` table (#2959)
|
||||
|
||||
### 🔧 Bug Fixes
|
||||
|
||||
- **mcp:** move `enforceScopes` guard before `MCP_TOOL_MAP` lookup, add inline `scopes` parameter to `withScopeEnforcement()`, and declare scopes on all 24 dynamic tool definitions (memory, skills, plugins, gamification, compression) to fix scope enforcement for dynamic MCP tool groups (#2958)
|
||||
|
||||
---
|
||||
|
||||
## [3.8.7] — 2026-05-29
|
||||
|
||||
13
CLAUDE.md
13
CLAUDE.md
@@ -428,3 +428,16 @@ git push -u origin feat/your-feature
|
||||
15. Never expose routes that spawn child processes (`/api/mcp/`, `/api/cli-tools/runtime/`) without `isLocalOnlyPath()` classification in `src/server/authz/routeGuard.ts`. Loopback enforcement happens unconditionally before any auth check — leaked JWT via tunnel cannot trigger process spawning. See `docs/security/ROUTE_GUARD_TIERS.md`.
|
||||
16. Never include `Co-Authored-By` trailers that credit an AI assistant, LLM, or automation account (e.g. names containing "Claude", "GPT", "Copilot", "Bot"; emails at `anthropic.com` / `openai.com` / bot-owned `noreply.github.com` addresses). Such trailers route attribution to the bot account on GitHub, hiding the real author (`diegosouzapw`) in PR history. Human collaborators — including upstream PR authors and issue reporters being ported into OmniRoute — MAY and SHOULD be credited with standard `Co-authored-by: Name <email>` trailers; the upstream-port workflows (`/port-upstream-features`, `/port-upstream-issues`) depend on this.
|
||||
17. Never expose routes under `/api/services/` or `/dashboard/providers/services/*/embed/` without `isLocalOnlyPath()` classification in `src/server/authz/routeGuard.ts`. These routes can spawn child processes (`npm install`, `node`). Loopback enforcement happens unconditionally before any auth check — a leaked JWT via tunnel cannot trigger process spawning. See `docs/security/ROUTE_GUARD_TIERS.md`.
|
||||
|
||||
---
|
||||
|
||||
## PII & Stream Sanitization Learnings
|
||||
|
||||
### 1. Regex Security (ReDoS)
|
||||
All regex patterns matching variable-length strings (e.g. IPv6 address, credit cards) must use strictly bounded, non-overlapping sequences (e.g., limit occurrences with bounded ranges `{1,7}`) to prevent catastrophic backtracking when processing untrusted inputs.
|
||||
|
||||
### 2. SSE Snapshot Handling
|
||||
When parsing streaming LLM responses (e.g. Responses API), check if a chunk represents a final snapshot (`done` or `completed` events). Snapshot text must be sanitized directly as a standalone string (bypassing rolling delta buffers) to prevent text duplication at the end of the stream.
|
||||
|
||||
### 3. Database Handles in Tests
|
||||
Ensure that any unit tests that trigger database migrations or establish SQLite connections call `resetDbInstance()` and properly clean up/close all DB handles in a `test.after(...)` hook. Failure to release database connection handles will cause Node's native test runner to hang indefinitely.
|
||||
|
||||
@@ -174,6 +174,8 @@ import {
|
||||
} from "@/lib/semanticCache";
|
||||
import { getIdempotencyKey, checkIdempotency, saveIdempotency } from "@/lib/idempotencyLayer";
|
||||
import { createProgressTransform, wantsProgress } from "../utils/progressTracker.ts";
|
||||
import { createPiiSseTransform } from "@/lib/streamingPiiTransform";
|
||||
import { isFeatureFlagEnabled } from "@/shared/utils/featureFlags";
|
||||
import {
|
||||
isModelUnavailableError,
|
||||
getNextFamilyFallback,
|
||||
@@ -1514,6 +1516,7 @@ export async function handleChatCore({
|
||||
disableEmergencyFallback = false,
|
||||
cachedSettings = null,
|
||||
skipUpstreamRetry = false,
|
||||
createPiiTransform = null,
|
||||
}) {
|
||||
let { provider, model, extendedContext } = modelInfo;
|
||||
// ── Memory pressure guard ────────────────────────────────────────────
|
||||
@@ -5797,14 +5800,21 @@ export async function handleChatCore({
|
||||
// ── Phase 9.3: Progress tracking (opt-in) ──
|
||||
const progressEnabled = wantsProgress(clientRawRequest?.headers);
|
||||
let finalStream;
|
||||
|
||||
let piiStream = pipeWithDisconnect(providerResponse, transformStream, streamController);
|
||||
if (typeof createPiiTransform === "function") {
|
||||
piiStream = piiStream.pipeThrough((createPiiTransform as () => TransformStream)());
|
||||
} else if (isFeatureFlagEnabled("PII_RESPONSE_SANITIZATION")) {
|
||||
piiStream = piiStream.pipeThrough(createPiiSseTransform());
|
||||
}
|
||||
|
||||
if (progressEnabled) {
|
||||
const progressTransform = createProgressTransform({ signal: streamController.signal });
|
||||
// Chain: provider → transform → progress → client
|
||||
const transformedBody = pipeWithDisconnect(providerResponse, transformStream, streamController);
|
||||
finalStream = transformedBody.pipeThrough(progressTransform);
|
||||
finalStream = piiStream.pipeThrough(progressTransform);
|
||||
responseHeaders[OMNIROUTE_RESPONSE_HEADERS.progress] = "enabled";
|
||||
} else {
|
||||
finalStream = pipeWithDisconnect(providerResponse, transformStream, streamController);
|
||||
finalStream = piiStream;
|
||||
}
|
||||
finalStream = finalStream.pipeThrough(
|
||||
createSseHeartbeatTransform({
|
||||
|
||||
@@ -2315,6 +2315,9 @@ export function createSSEStream(options: StreamOptions = {}) {
|
||||
console.log(`[STREAM] Error in flush (${model || "unknown"}):`, error.message || error);
|
||||
}
|
||||
},
|
||||
cancel(reason) {
|
||||
clearIdleTimer();
|
||||
},
|
||||
},
|
||||
{ highWaterMark: 16384 },
|
||||
{ highWaterMark: 16384 }
|
||||
|
||||
@@ -13,9 +13,20 @@
|
||||
|
||||
// ── Configuration ──
|
||||
|
||||
const isEnabled = () => process.env.PII_RESPONSE_SANITIZATION === "true";
|
||||
const getMode = (): "redact" | "warn" | "block" =>
|
||||
(process.env.PII_RESPONSE_SANITIZATION_MODE as "redact" | "warn" | "block") || "redact";
|
||||
import { isFeatureFlagEnabled, resolveFeatureFlag } from "@/shared/utils/featureFlags";
|
||||
|
||||
const isEnabled = () => isFeatureFlagEnabled("PII_RESPONSE_SANITIZATION");
|
||||
const VALID_MODES = ["redact", "warn", "block", "off"] as const;
|
||||
type PiiMode = typeof VALID_MODES[number];
|
||||
|
||||
const getMode = (): PiiMode => {
|
||||
const value = resolveFeatureFlag("PII_RESPONSE_SANITIZATION_MODE");
|
||||
if (value === "" || value === undefined || value === null) return "redact";
|
||||
if (value === "false") return "off";
|
||||
if ((VALID_MODES as readonly any[]).includes(value)) return value as PiiMode;
|
||||
console.error(`[PII] Invalid PII_RESPONSE_SANITIZATION_MODE: "${value}", defaulting to "redact"`);
|
||||
return "redact";
|
||||
};
|
||||
|
||||
// ── PII Patterns ──
|
||||
|
||||
@@ -29,61 +40,67 @@ interface PIIPattern {
|
||||
const PII_PATTERNS: PIIPattern[] = [
|
||||
{
|
||||
name: "email",
|
||||
regex: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g,
|
||||
regex: /(?<=^|[^A-Za-z0-9])[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}(?=$|[^A-Za-z0-9])/g,
|
||||
replacement: "[EMAIL_REDACTED]",
|
||||
severity: "medium",
|
||||
},
|
||||
{
|
||||
name: "ssn",
|
||||
regex: /\b\d{3}-\d{2}-\d{4}\b/g,
|
||||
regex: /(?<=^|[^A-Za-z0-9])\d{3}-\d{2}-\d{4}(?=$|[^A-Za-z0-9])/g,
|
||||
replacement: "[SSN_REDACTED]",
|
||||
severity: "high",
|
||||
},
|
||||
{
|
||||
name: "credit_card",
|
||||
regex: /\b(?:\d{4}[-\s]?){3}\d{4}\b/g,
|
||||
regex: /(?<=^|[^A-Za-z0-9])(?:\d{4}[-\s]?\d{4}[-\s]?\d{4}[-\s]?\d{4}|\d{4}[-\s]?\d{6}[-\s]?\d{4,5})(?=$|[^A-Za-z0-9])/g,
|
||||
replacement: "[CC_REDACTED]",
|
||||
severity: "high",
|
||||
},
|
||||
{
|
||||
name: "phone_us",
|
||||
regex: /\b(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b/g,
|
||||
regex: /(?<=^|[^A-Za-z0-9])(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}(?=$|[^A-Za-z0-9])/g,
|
||||
replacement: "[PHONE_REDACTED]",
|
||||
severity: "medium",
|
||||
},
|
||||
{
|
||||
name: "phone_br",
|
||||
regex: /\b(?:\+?55[-.\s]?)?\(?\d{2}\)?[-.\s]?\d{4,5}[-.\s]?\d{4}\b/g,
|
||||
regex: /(?<=^|[^A-Za-z0-9])(?:\+?55[-.\s]?)?\(?\d{2}\)?[-.\s]?(?:9\d{4}|[2-5]\d{3})[-.\s]?\d{4}(?=$|[^A-Za-z0-9])/g,
|
||||
replacement: "[PHONE_REDACTED]",
|
||||
severity: "medium",
|
||||
},
|
||||
{
|
||||
name: "cpf",
|
||||
regex: /\b\d{3}\.\d{3}\.\d{3}-\d{2}\b/g,
|
||||
regex: /(?<=^|[^A-Za-z0-9])\d{3}\.?\d{3}\.?\d{3}-?\d{2}(?=$|[^A-Za-z0-9])/g,
|
||||
replacement: "[CPF_REDACTED]",
|
||||
severity: "high",
|
||||
},
|
||||
{
|
||||
name: "cnpj",
|
||||
regex: /\b\d{2}\.\d{3}\.\d{3}\/\d{4}-\d{2}\b/g,
|
||||
regex: /(?<=^|[^A-Za-z0-9])\d{2}\.?\d{3}\.?\d{3}\/?\d{4}-?\d{2}(?=$|[^A-Za-z0-9])/g,
|
||||
replacement: "[CNPJ_REDACTED]",
|
||||
severity: "high",
|
||||
},
|
||||
{
|
||||
name: "ip_address",
|
||||
regex: /\b(?:\d{1,3}\.){3}\d{1,3}\b/g,
|
||||
regex: /(?<=^|[^A-Za-z0-9])(?:\d{1,3}\.){3}\d{1,3}(?=$|[^A-Za-z0-9])/g,
|
||||
replacement: "[IP_REDACTED]",
|
||||
severity: "low",
|
||||
},
|
||||
{
|
||||
name: "ipv6_address",
|
||||
regex: /(?<=^|[^A-Za-z0-9])(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}(?=$|[^A-Za-z0-9])/g,
|
||||
replacement: "[IP_REDACTED]",
|
||||
severity: "low",
|
||||
},
|
||||
{
|
||||
name: "aws_key",
|
||||
regex: /\bAKIA[0-9A-Z]{16}\b/g,
|
||||
regex: /(?<=^|[^A-Za-z0-9])AKIA[0-9A-Z]{16}(?=$|[^A-Za-z0-9])/g,
|
||||
replacement: "[AWS_KEY_REDACTED]",
|
||||
severity: "high",
|
||||
},
|
||||
{
|
||||
name: "api_key_generic",
|
||||
regex: /\b(?:sk|pk|api|key|token)[_-][a-zA-Z0-9]{20,}\b/gi,
|
||||
regex: /(?<=^|[^A-Za-z0-9])(?:sk|pk|api|key|token)[_-][a-zA-Z0-9]{20,}(?=$|[^A-Za-z0-9])/gi,
|
||||
replacement: "[API_KEY_REDACTED]",
|
||||
severity: "high",
|
||||
},
|
||||
@@ -99,57 +116,171 @@ export interface SanitizeResult {
|
||||
severity: string;
|
||||
}>;
|
||||
redacted: boolean;
|
||||
endMatchIndex?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan and optionally redact PII from LLM response text.
|
||||
*/
|
||||
export function sanitizePII(text: string): SanitizeResult {
|
||||
export function sanitizePII(text: string, isStreaming = false): SanitizeResult {
|
||||
if (!isEnabled() || !text || typeof text !== "string") {
|
||||
return { text, detections: [], redacted: false };
|
||||
}
|
||||
|
||||
const mode = getMode();
|
||||
const detections: SanitizeResult["detections"] = [];
|
||||
let sanitized = text;
|
||||
|
||||
// Build a map of clean character index to original character index.
|
||||
// Ignore zero-width and invisible formatting characters to prevent obfuscation bypasses.
|
||||
const IGNORED_CHARS = new Set([
|
||||
"\u200B", // Zero Width Space
|
||||
"\u200C", // Zero Width Non-Joiner
|
||||
"\u200D", // Zero Width Joiner
|
||||
"\u200E", // Left-to-Right Mark
|
||||
"\u200F", // Right-to-Left Mark
|
||||
"\uFEFF", // Byte Order Mark
|
||||
"\u2060", // Word Joiner
|
||||
"\u00AD", // Soft Hyphen
|
||||
// Bidirectional formatting controls
|
||||
"\u202A", "\u202B", "\u202C", "\u202D", "\u202E",
|
||||
"\u2066", "\u2067", "\u2068", "\u2069"
|
||||
]);
|
||||
const cleanToOrig: number[] = [];
|
||||
let cleanText = "";
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const char = text[i];
|
||||
if (IGNORED_CHARS.has(char)) {
|
||||
// Obfuscator / joiner, ignore in clean text
|
||||
} else {
|
||||
cleanToOrig.push(i);
|
||||
cleanText += char;
|
||||
}
|
||||
}
|
||||
cleanToOrig.push(text.length);
|
||||
|
||||
interface RedactRange {
|
||||
start: number;
|
||||
end: number;
|
||||
replacement: string;
|
||||
pattern: string;
|
||||
}
|
||||
const ranges: RedactRange[] = [];
|
||||
let endMatchIndex: number | undefined = undefined;
|
||||
|
||||
for (const pattern of PII_PATTERNS) {
|
||||
// Reset lastIndex for global regexes
|
||||
pattern.regex.lastIndex = 0;
|
||||
const matches = text.match(pattern.regex);
|
||||
if (matches && matches.length > 0) {
|
||||
detections.push({
|
||||
pattern: pattern.name,
|
||||
count: matches.length,
|
||||
severity: pattern.severity,
|
||||
});
|
||||
let match;
|
||||
while ((match = pattern.regex.exec(cleanText)) !== null) {
|
||||
const cleanStart = match.index;
|
||||
const cleanEnd = cleanStart + match[0].length;
|
||||
const start = cleanToOrig[cleanStart];
|
||||
const end = cleanToOrig[cleanEnd];
|
||||
|
||||
if (mode === "redact") {
|
||||
pattern.regex.lastIndex = 0;
|
||||
sanitized = sanitized.replace(pattern.regex, pattern.replacement);
|
||||
// Perform checksum validation to minimize false positives
|
||||
if (pattern.name === "credit_card" && !isValidLuhn(match[0].replace(/[-\s]/g, ""))) {
|
||||
continue;
|
||||
}
|
||||
if (pattern.name === "cpf" && !isValidCPF(match[0])) {
|
||||
continue;
|
||||
}
|
||||
if (pattern.name === "cnpj" && !isValidCNPJ(match[0])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isStreaming && cleanEnd === cleanText.length) {
|
||||
// Prevent premature redaction of variable-length PII touching the end of the streaming buffer.
|
||||
// Clamp maximum match length to 100 to prevent infinite buffer accumulation (OOM).
|
||||
const matchLength = cleanEnd - cleanStart;
|
||||
if (matchLength < 100) {
|
||||
if (endMatchIndex === undefined || start < endMatchIndex) {
|
||||
endMatchIndex = start;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ranges.push({
|
||||
start,
|
||||
end,
|
||||
replacement: pattern.replacement,
|
||||
pattern: pattern.name,
|
||||
});
|
||||
}
|
||||
|
||||
if (!pattern.regex.global) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (detections.length > 0 && mode === "warn") {
|
||||
console.warn(
|
||||
`[PII] Detected PII in response: ${detections.map((d) => `${d.pattern}(${d.count})`).join(", ")}`
|
||||
);
|
||||
const patternCounts = new Map<string, number>();
|
||||
for (const range of ranges) {
|
||||
patternCounts.set(range.pattern, (patternCounts.get(range.pattern) || 0) + 1);
|
||||
}
|
||||
for (const [name, count] of patternCounts.entries()) {
|
||||
const patternDef = PII_PATTERNS.find((p) => p.name === name);
|
||||
detections.push({
|
||||
pattern: name,
|
||||
count,
|
||||
severity: patternDef?.severity || "medium",
|
||||
});
|
||||
}
|
||||
|
||||
if (detections.length > 0) {
|
||||
if (mode === "warn") {
|
||||
console.warn(
|
||||
`[PII] Detected PII in response: ${detections.map((d) => `${d.pattern}(${d.count})`).join(", ")}`
|
||||
);
|
||||
} else if (mode === "block") {
|
||||
// Log the matched pattern types server-side, but never put them in the
|
||||
// thrown message — it can surface to the client via controller.error
|
||||
// (Hard Rule #12: no internal detail in response/stream errors).
|
||||
console.warn(
|
||||
`[PII] Blocked response due to PII detection: ${detections.map((d) => d.pattern).join(", ")}`
|
||||
);
|
||||
throw new Error("[PII] Blocked response due to PII detection");
|
||||
}
|
||||
}
|
||||
|
||||
let sanitized = text;
|
||||
if (mode === "redact" && ranges.length > 0) {
|
||||
// Sort ranges from right to left to avoid shifting offsets
|
||||
ranges.sort((a, b) => b.start - a.start);
|
||||
|
||||
// Merge overlapping or adjacent ranges
|
||||
const mergedRanges: typeof ranges = [];
|
||||
for (const r of ranges) {
|
||||
if (mergedRanges.length === 0) {
|
||||
mergedRanges.push(r);
|
||||
} else {
|
||||
const last = mergedRanges[mergedRanges.length - 1];
|
||||
if (r.end >= last.start) {
|
||||
last.start = Math.min(r.start, last.start);
|
||||
last.end = Math.max(r.end, last.end);
|
||||
} else {
|
||||
mergedRanges.push(r);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const range of mergedRanges) {
|
||||
sanitized = sanitized.slice(0, range.start) + range.replacement + sanitized.slice(range.end);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
text: mode === "redact" ? sanitized : text,
|
||||
detections,
|
||||
redacted: mode === "redact" && detections.length > 0,
|
||||
endMatchIndex,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitize a streaming chunk (text content only).
|
||||
*/
|
||||
export function sanitizePIIChunk(chunk: string): string {
|
||||
export function sanitizePIIChunk(chunk: string, isStopSignal = false): string {
|
||||
if (!isEnabled()) return chunk;
|
||||
const { text } = sanitizePII(chunk);
|
||||
// If it's a stop signal, we are flushing the final chunk, so we shouldn't treat it as a partial streaming buffer (force redaction)
|
||||
const { text } = sanitizePII(chunk, !isStopSignal);
|
||||
return text;
|
||||
}
|
||||
|
||||
@@ -160,20 +291,126 @@ export function sanitizePIIResponse(response: any): any {
|
||||
if (!isEnabled() || !response) return response;
|
||||
|
||||
try {
|
||||
const choices = response.choices || [];
|
||||
for (const choice of choices) {
|
||||
if (choice.message?.content) {
|
||||
const result = sanitizePII(choice.message.content);
|
||||
choice.message.content = result.text;
|
||||
const visited = new WeakSet();
|
||||
// Deep sanitize the entire response object recursively
|
||||
const deepSanitize = (obj: any, depth = 0): any => {
|
||||
if (depth > 100) {
|
||||
throw new Error("Maximum sanitization depth exceeded");
|
||||
}
|
||||
if (choice.delta?.content) {
|
||||
const result = sanitizePII(choice.delta.content);
|
||||
choice.delta.content = result.text;
|
||||
if (!obj) return obj;
|
||||
if (typeof obj === "string") {
|
||||
return sanitizePII(obj).text;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Fail open — don't break the response
|
||||
}
|
||||
if (typeof obj === "object") {
|
||||
if (visited.has(obj)) {
|
||||
return "[CIRCULAR_REFERENCE_REDACTED]";
|
||||
}
|
||||
visited.add(obj);
|
||||
|
||||
return response;
|
||||
if (Array.isArray(obj)) {
|
||||
for (let i = 0; i < obj.length; i++) {
|
||||
obj[i] = deepSanitize(obj[i], depth + 1);
|
||||
}
|
||||
} else {
|
||||
for (const key of Object.keys(obj)) {
|
||||
// Skip known non-PII system metadata keys to optimize performance
|
||||
if (["id", "model", "object", "created", "finish_reason", "finishReason", "role", "type", "index", "stop_reason"].includes(key)) {
|
||||
continue;
|
||||
}
|
||||
obj[key] = deepSanitize(obj[key], depth + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
return obj;
|
||||
};
|
||||
|
||||
return deepSanitize(response);
|
||||
} catch (err: any) {
|
||||
if (err?.message?.startsWith("[PII] Blocked response")) {
|
||||
throw err;
|
||||
}
|
||||
// Fail secure — try raw string sanitization instead of failing open
|
||||
try {
|
||||
const serialized = JSON.stringify(response);
|
||||
const { text: sanitized } = sanitizePII(serialized);
|
||||
return JSON.parse(sanitized);
|
||||
} catch (fallbackErr) {
|
||||
// Suppress err.message — never surface upstream error detail to the client
|
||||
// (Hard Rule #12). The underlying error is available in server logs.
|
||||
console.warn("[PII] Sanitization fallback failed:", err?.message);
|
||||
throw new Error("[PII] Blocked response due to sanitization failure");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Checksum Validation Helpers ──
|
||||
|
||||
function isValidLuhn(digits: string): boolean {
|
||||
let sum = 0;
|
||||
let shouldDouble = false;
|
||||
for (let i = digits.length - 1; i >= 0; i--) {
|
||||
let digit = parseInt(digits[i], 10);
|
||||
if (shouldDouble) {
|
||||
digit *= 2;
|
||||
if (digit > 9) digit -= 9;
|
||||
}
|
||||
sum += digit;
|
||||
shouldDouble = !shouldDouble;
|
||||
}
|
||||
return sum % 10 === 0;
|
||||
}
|
||||
|
||||
function isValidCPF(cpf: string): boolean {
|
||||
const digits = cpf.replace(/\D/g, "");
|
||||
if (digits.length !== 11) return false;
|
||||
if (/^(\d)\1{10}$/.test(digits)) return false;
|
||||
|
||||
let sum = 0;
|
||||
for (let i = 0; i < 9; i++) {
|
||||
sum += parseInt(digits[i], 10) * (10 - i);
|
||||
}
|
||||
let rev = 11 - (sum % 11);
|
||||
if (rev === 10 || rev === 11) rev = 0;
|
||||
if (rev !== parseInt(digits[9], 10)) return false;
|
||||
|
||||
sum = 0;
|
||||
for (let i = 0; i < 10; i++) {
|
||||
sum += parseInt(digits[i], 10) * (11 - i);
|
||||
}
|
||||
rev = 11 - (sum % 11);
|
||||
if (rev === 10 || rev === 11) rev = 0;
|
||||
if (rev !== parseInt(digits[10], 10)) return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
function isValidCNPJ(cnpj: string): boolean {
|
||||
const digits = cnpj.replace(/\D/g, "");
|
||||
if (digits.length !== 14) return false;
|
||||
if (/^(\d)\1{13}$/.test(digits)) return false;
|
||||
|
||||
let size = 12;
|
||||
let numbers = digits.substring(0, size);
|
||||
const tempDigits = digits.substring(size);
|
||||
let sum = 0;
|
||||
let pos = size - 7;
|
||||
for (let i = size; i >= 1; i--) {
|
||||
sum += parseInt(numbers.charAt(size - i), 10) * pos--;
|
||||
if (pos < 2) pos = 9;
|
||||
}
|
||||
let result = sum % 11 < 2 ? 0 : 11 - (sum % 11);
|
||||
if (result !== parseInt(tempDigits.charAt(0), 10)) return false;
|
||||
|
||||
size = 13;
|
||||
numbers = digits.substring(0, size);
|
||||
sum = 0;
|
||||
pos = size - 7;
|
||||
for (let i = size; i >= 1; i--) {
|
||||
sum += parseInt(numbers.charAt(size - i), 10) * pos--;
|
||||
if (pos < 2) pos = 9;
|
||||
}
|
||||
result = sum % 11 < 2 ? 0 : 11 - (sum % 11);
|
||||
if (result !== parseInt(tempDigits.charAt(1), 10)) return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
238
src/lib/sseTextTransform.ts
Normal file
238
src/lib/sseTextTransform.ts
Normal file
@@ -0,0 +1,238 @@
|
||||
export type FieldCategory = "content" | "reasoning" | "toolArgs" | "partialJson";
|
||||
|
||||
const CATEGORY_MAP: Record<string, FieldCategory> = {
|
||||
reasoning: "reasoning",
|
||||
thinking: "reasoning",
|
||||
reasoning_content: "reasoning",
|
||||
arguments: "toolArgs",
|
||||
partial_json: "partialJson"
|
||||
};
|
||||
|
||||
export function getFieldCategory(key: string): FieldCategory {
|
||||
return CATEGORY_MAP[key] || "content";
|
||||
}
|
||||
|
||||
const STOP_EVENT_TYPES = new Set([
|
||||
"response.done", "response.completed", "response.cancelled", "response.failed"
|
||||
]);
|
||||
|
||||
export function checkIfStopSignal(json: any): boolean {
|
||||
if (!json || typeof json !== "object") return false;
|
||||
if (json.choices && Array.isArray(json.choices) && json.choices.some((c: any) => c.finish_reason)) return true;
|
||||
if (json.candidates && Array.isArray(json.candidates) && json.candidates.some((c: any) => c.finishReason)) return true;
|
||||
if (json.type === "content_block_stop") return true;
|
||||
if (json.type === "message_stop") return true;
|
||||
if (json.type === "message_delta" && json.delta?.stop_reason) return true;
|
||||
if (STOP_EVENT_TYPES.has(json.type)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
export function checkIfSnapshot(json: any): boolean {
|
||||
if (!json || typeof json !== "object") return false;
|
||||
if (typeof json.type === "string") {
|
||||
const t = json.type;
|
||||
if (t.endsWith(".done") || t.endsWith(".completed") || STOP_EVENT_TYPES.has(t)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
const fallbackDecoder = new TextDecoder();
|
||||
|
||||
export function createSseTextTransform(
|
||||
processor: (text: string, field: FieldCategory, isStopSignal?: boolean, index?: string | number, isSnapshot?: boolean) => string,
|
||||
onFlush?: (lastJson: any, isJsonStream?: boolean, lastContentJson?: any) => any,
|
||||
onCancel?: () => void,
|
||||
): TransformStream {
|
||||
const encoder = new TextEncoder();
|
||||
const decoder = new TextDecoder("utf-8");
|
||||
let lineBuffer = "";
|
||||
let lastPrefix = "data: ";
|
||||
let lastJson: any = null;
|
||||
let lastContentJson: any = null;
|
||||
let isJsonStream = false;
|
||||
let flushed = false;
|
||||
let errored = false;
|
||||
|
||||
const handleLine = (line: string, controller: TransformStreamDefaultController) => {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed === "" || line.startsWith(":")) {
|
||||
// Pass comments and empty lines through unchanged
|
||||
controller.enqueue(encoder.encode(line + "\n"));
|
||||
return;
|
||||
}
|
||||
|
||||
if (line.startsWith("data:")) {
|
||||
const prefix = line.startsWith("data: ") ? "data: " : "data:";
|
||||
lastPrefix = prefix;
|
||||
const segment = line.startsWith("data: ") ? line.slice(6) : line.slice(5);
|
||||
if (segment === "[DONE]") {
|
||||
if (onFlush && !flushed) {
|
||||
const flushedValue = onFlush(lastJson, isJsonStream, lastContentJson);
|
||||
if (flushedValue) {
|
||||
const prefix = lastPrefix || "data: ";
|
||||
const payload = typeof flushedValue === "string" ? flushedValue : JSON.stringify(flushedValue);
|
||||
controller.enqueue(encoder.encode(prefix + payload + "\n"));
|
||||
}
|
||||
flushed = true;
|
||||
}
|
||||
controller.enqueue(encoder.encode(line + "\n"));
|
||||
return;
|
||||
}
|
||||
|
||||
const trimmedSegment = segment.trim();
|
||||
if (trimmedSegment.startsWith("{") || trimmedSegment.startsWith("[")) {
|
||||
try {
|
||||
const json = JSON.parse(trimmedSegment);
|
||||
isJsonStream = true;
|
||||
|
||||
let matched = false;
|
||||
|
||||
const isStopSignal = checkIfStopSignal(json);
|
||||
const isSnapshot = checkIfSnapshot(json);
|
||||
|
||||
const METADATA_KEYS = [
|
||||
"id", "model", "object", "created", "finish_reason", "finishReason",
|
||||
"role", "type", "index", "stop_reason", "stop_sequence",
|
||||
"system_fingerprint", "service_tier", "usage", "prompt_tokens",
|
||||
"completion_tokens", "total_tokens", "input_tokens", "output_tokens",
|
||||
"logprobs", "refusal", "name", "event"
|
||||
];
|
||||
|
||||
// Recursively sanitize all string properties (except system metadata)
|
||||
const sanitizeObject = (obj: any, currentChoiceIdx = 0, currentToolIdx = 0) => {
|
||||
if (!obj || typeof obj !== "object") return;
|
||||
|
||||
let choiceIdx = currentChoiceIdx;
|
||||
let toolIdx = currentToolIdx;
|
||||
|
||||
if (typeof obj.index === "number") {
|
||||
if (obj.delta || obj.message || obj.finish_reason) {
|
||||
choiceIdx = obj.index;
|
||||
} else if (obj.function || obj.id || obj.type === "function") {
|
||||
toolIdx = obj.index;
|
||||
} else {
|
||||
choiceIdx = obj.index;
|
||||
}
|
||||
}
|
||||
|
||||
const compositeKey = `${choiceIdx}_${toolIdx}`;
|
||||
|
||||
for (const key of Object.keys(obj)) {
|
||||
if (METADATA_KEYS.includes(key)) {
|
||||
continue;
|
||||
}
|
||||
if (typeof obj[key] === "string") {
|
||||
const val = obj[key];
|
||||
const field: FieldCategory = getFieldCategory(key);
|
||||
obj[key] = processor(val, field, isStopSignal, compositeKey, isSnapshot);
|
||||
matched = true;
|
||||
} else if (typeof obj[key] === "object") {
|
||||
sanitizeObject(obj[key], choiceIdx, toolIdx);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
sanitizeObject(json, 0, 0);
|
||||
|
||||
if (!matched) {
|
||||
console.warn("[SSE-TRANSFORM] No string fields sanitized in SSE JSON chunk. Keys:", Object.keys(json).slice(0, 5).join(", "));
|
||||
} else {
|
||||
lastContentJson = json;
|
||||
}
|
||||
|
||||
if (isStopSignal && onFlush && !flushed) {
|
||||
const flushedValue = onFlush(lastJson || json, isJsonStream, lastContentJson || lastJson || json); // Use json as fallback just in case
|
||||
if (flushedValue) {
|
||||
const prefix = lastPrefix || "data: ";
|
||||
const payload = typeof flushedValue === "string" ? flushedValue : JSON.stringify(flushedValue);
|
||||
// Only enqueue if the flushed value actually has content (onFlush usually returns null if buffer is empty now)
|
||||
controller.enqueue(encoder.encode(prefix + payload + "\n"));
|
||||
}
|
||||
flushed = true;
|
||||
}
|
||||
|
||||
lastJson = json;
|
||||
controller.enqueue(encoder.encode(prefix + JSON.stringify(json) + "\n"));
|
||||
} catch (err: any) {
|
||||
if (err?.message?.startsWith("[PII]")) {
|
||||
throw err;
|
||||
}
|
||||
if (err instanceof SyntaxError) {
|
||||
// JSON parsing failed. Check if it looks like JSON that failed to parse.
|
||||
if (trimmedSegment.startsWith("{") || trimmedSegment.startsWith("[")) {
|
||||
console.warn("[SSE-TRANSFORM] Dropping malformed JSON chunk to prevent syntax injection:", trimmedSegment.slice(0, 100));
|
||||
} else {
|
||||
// Treat segment as raw text delta (fail-open)
|
||||
const processed = processor(segment, "content");
|
||||
controller.enqueue(encoder.encode(prefix + processed + "\n"));
|
||||
}
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Starts with data: but not JSON, process as raw text
|
||||
const processed = processor(segment, "content");
|
||||
controller.enqueue(encoder.encode(prefix + processed + "\n"));
|
||||
}
|
||||
} else {
|
||||
// Non-data line, pass through (e.g. event: content_block_delta)
|
||||
controller.enqueue(encoder.encode(line + "\n"));
|
||||
}
|
||||
};
|
||||
|
||||
return new TransformStream({
|
||||
transform(chunk, controller) {
|
||||
try {
|
||||
const chunkStr = decoder.decode(chunk, { stream: true });
|
||||
lineBuffer += chunkStr;
|
||||
const lines = lineBuffer.split(/\r?\n/);
|
||||
lineBuffer = lines.pop() || "";
|
||||
|
||||
for (const line of lines) {
|
||||
handleLine(line, controller);
|
||||
}
|
||||
} catch (err: any) {
|
||||
let context = "[REDACTED_DUE_TO_PII]";
|
||||
if (!err?.message?.startsWith("[PII]")) {
|
||||
if (typeof chunk === "string") {
|
||||
context = chunk.slice(0, 200);
|
||||
} else if (chunk instanceof Uint8Array) {
|
||||
context = fallbackDecoder.decode(chunk.slice(0, 200));
|
||||
} else {
|
||||
context = String(chunk).slice(0, 200);
|
||||
}
|
||||
}
|
||||
console.error("[SSE-TRANSFORM] Error in transform:", err, "chunk:", context);
|
||||
lineBuffer = "";
|
||||
errored = true;
|
||||
controller.error(err);
|
||||
}
|
||||
},
|
||||
flush(controller) {
|
||||
if (errored) return;
|
||||
try {
|
||||
const remaining = decoder.decode() + lineBuffer;
|
||||
if (remaining) {
|
||||
handleLine(remaining, controller);
|
||||
}
|
||||
if (onFlush && !flushed) {
|
||||
const flushedValue = onFlush(lastJson, isJsonStream, lastContentJson);
|
||||
if (flushedValue) {
|
||||
const prefix = lastPrefix || "data: ";
|
||||
const payload = typeof flushedValue === "string" ? flushedValue : JSON.stringify(flushedValue);
|
||||
controller.enqueue(encoder.encode(prefix + payload + "\n"));
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[SSE-TRANSFORM] Error in flush:", err);
|
||||
controller.error(err);
|
||||
}
|
||||
},
|
||||
cancel(reason: any) {
|
||||
if (onCancel) {
|
||||
onCancel();
|
||||
}
|
||||
}
|
||||
} as any);
|
||||
}
|
||||
317
src/lib/streamingPiiTransform.ts
Normal file
317
src/lib/streamingPiiTransform.ts
Normal file
@@ -0,0 +1,317 @@
|
||||
import { createSseTextTransform, FieldCategory, getFieldCategory } from "./sseTextTransform";
|
||||
import { sanitizePII } from "./piiSanitizer";
|
||||
|
||||
export interface PiiTransformOptions {
|
||||
windowSize?: number;
|
||||
}
|
||||
|
||||
export function createPiiSseTransform(options?: PiiTransformOptions): TransformStream {
|
||||
const choiceBuffers = new Map<string, Record<FieldCategory, string>>();
|
||||
|
||||
const getBuffers = (index: string | number): Record<FieldCategory, string> => {
|
||||
const key = String(index);
|
||||
let buf = choiceBuffers.get(key);
|
||||
if (!buf) {
|
||||
buf = {
|
||||
content: "",
|
||||
reasoning: "",
|
||||
toolArgs: "",
|
||||
partialJson: ""
|
||||
};
|
||||
choiceBuffers.set(index, buf);
|
||||
}
|
||||
return buf;
|
||||
};
|
||||
|
||||
let windowSize = Math.max(200, options?.windowSize ?? (parseInt(process.env.PII_WINDOW_SIZE || "", 10) || 200));
|
||||
if (options?.windowSize && process.env.PII_TEST_BYPASS_MIN_WINDOW === "true") {
|
||||
windowSize = options.windowSize;
|
||||
}
|
||||
const W = windowSize;
|
||||
|
||||
const processor = (
|
||||
text: string,
|
||||
field: FieldCategory,
|
||||
isStopSignal = false,
|
||||
index: string | number = "0_0",
|
||||
isSnapshot = false
|
||||
): string => {
|
||||
if (isSnapshot) {
|
||||
return sanitizePII(text).text;
|
||||
}
|
||||
const buffers = getBuffers(index);
|
||||
buffers[field] += text;
|
||||
const { text: sanitized, endMatchIndex } = sanitizePII(buffers[field], !isStopSignal);
|
||||
let emitLength = isStopSignal ? sanitized.length : Math.max(0, sanitized.length - W);
|
||||
|
||||
// Cap emitLength at the start of any PII that touched the end of the buffer
|
||||
if (!isStopSignal && endMatchIndex !== undefined && emitLength > endMatchIndex) {
|
||||
emitLength = endMatchIndex;
|
||||
}
|
||||
|
||||
// Prevent slicing in the middle of a UTF-16 surrogate pair (e.g. emojis)
|
||||
if (emitLength > 0 && emitLength < sanitized.length) {
|
||||
const charCode = sanitized.charCodeAt(emitLength - 1);
|
||||
// High surrogate range is 0xD800 - 0xDBFF
|
||||
if (charCode >= 0xd800 && charCode <= 0xdbff) {
|
||||
emitLength -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
const toEmit = sanitized.slice(0, emitLength);
|
||||
buffers[field] = sanitized.slice(emitLength);
|
||||
return toEmit;
|
||||
};
|
||||
|
||||
const onFlush = (lastJson: any, isJsonStream = false, lastContentJson: any = null): any => {
|
||||
// Force final redaction on all buffers
|
||||
for (const [index, buffers] of choiceBuffers.entries()) {
|
||||
for (const key of Object.keys(buffers)) {
|
||||
const field = key as FieldCategory;
|
||||
if (buffers[field]) {
|
||||
buffers[field] = sanitizePII(buffers[field]).text;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let hasRemaining = false;
|
||||
for (const buffers of choiceBuffers.values()) {
|
||||
for (const key of Object.keys(buffers)) {
|
||||
if (buffers[key as FieldCategory].length > 0) {
|
||||
hasRemaining = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!hasRemaining) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!lastJson) {
|
||||
const buffers = getBuffers("0_0");
|
||||
if (buffers.content) {
|
||||
const remaining = buffers.content;
|
||||
buffers.content = "";
|
||||
|
||||
if (isJsonStream) {
|
||||
// Wrap in a safe default OpenAI format to prevent client-side SDK crashes
|
||||
return {
|
||||
choices: [
|
||||
{
|
||||
delta: {
|
||||
content: remaining
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
} else {
|
||||
return remaining;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Explicitly target formats to prevent metadata corruption and leakage
|
||||
const METADATA_KEYS = [
|
||||
"id", "model", "object", "created", "finish_reason", "finishReason",
|
||||
"role", "type", "index", "stop_reason", "stop_sequence",
|
||||
"system_fingerprint", "service_tier", "usage", "prompt_tokens",
|
||||
"completion_tokens", "total_tokens", "input_tokens", "output_tokens",
|
||||
"logprobs", "refusal", "name", "event"
|
||||
];
|
||||
|
||||
// 1. Claude format
|
||||
if (typeof lastJson.type === "string" && (lastJson.type.startsWith("message") || lastJson.type.startsWith("content_block"))) {
|
||||
const buffers = getBuffers("0_0");
|
||||
const delta: any = { type: "text_delta" };
|
||||
let hasDelta = false;
|
||||
if (buffers.content) {
|
||||
delta.text = buffers.content;
|
||||
buffers.content = "";
|
||||
hasDelta = true;
|
||||
}
|
||||
if (buffers.reasoning) {
|
||||
delta.thinking = buffers.reasoning;
|
||||
buffers.reasoning = "";
|
||||
hasDelta = true;
|
||||
}
|
||||
if (buffers.partialJson) {
|
||||
delta.partial_json = buffers.partialJson;
|
||||
buffers.partialJson = "";
|
||||
hasDelta = true;
|
||||
}
|
||||
if (hasDelta) {
|
||||
return {
|
||||
type: "content_block_delta",
|
||||
index: typeof lastJson.index === "number" ? lastJson.index : 0,
|
||||
delta
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// 2. OpenAI Chat Completions
|
||||
if (lastJson.choices && Array.isArray(lastJson.choices)) {
|
||||
const finalJson = JSON.parse(JSON.stringify(lastJson));
|
||||
const presentIndexes = new Set(finalJson.choices.map((c: any) => c.index).filter((idx: any) => typeof idx === "number"));
|
||||
for (const [compositeKey, choiceBuf] of choiceBuffers.entries()) {
|
||||
const choiceIdx = parseInt(compositeKey.split("_")[0] || "0", 10);
|
||||
if (!presentIndexes.has(choiceIdx) && (choiceBuf.content || choiceBuf.reasoning || choiceBuf.toolArgs)) {
|
||||
finalJson.choices.push({ index: choiceIdx, delta: {} });
|
||||
presentIndexes.add(choiceIdx);
|
||||
}
|
||||
}
|
||||
|
||||
for (const choice of finalJson.choices) {
|
||||
const choiceIdx = typeof choice.index === "number" ? choice.index : 0;
|
||||
|
||||
// Find if we have tool buffers for this choice
|
||||
const toolEntries = Array.from(choiceBuffers.entries())
|
||||
.filter(([key]) => key.startsWith(`${choiceIdx}_`) && key !== `${choiceIdx}_0`);
|
||||
|
||||
const choiceBuf = getBuffers(`${choiceIdx}_0`);
|
||||
if (!choice.delta) choice.delta = {};
|
||||
const delta = choice.delta;
|
||||
|
||||
if (choiceBuf.content) {
|
||||
delta.content = choiceBuf.content;
|
||||
choiceBuf.content = "";
|
||||
} else {
|
||||
delete delta.content;
|
||||
}
|
||||
if (choiceBuf.reasoning) {
|
||||
delta.reasoning_content = choiceBuf.reasoning;
|
||||
choiceBuf.reasoning = "";
|
||||
} else {
|
||||
delete delta.reasoning_content;
|
||||
}
|
||||
if (choiceBuf.toolArgs || toolEntries.length > 0) {
|
||||
if (!choice.delta.tool_calls) choice.delta.tool_calls = [];
|
||||
|
||||
if (choiceBuf.toolArgs) {
|
||||
choice.delta.tool_calls.push({
|
||||
index: 0,
|
||||
function: { arguments: choiceBuf.toolArgs }
|
||||
});
|
||||
choiceBuf.toolArgs = "";
|
||||
}
|
||||
|
||||
for (const [key, buf] of toolEntries) {
|
||||
if (buf.toolArgs) {
|
||||
const toolIdx = parseInt(key.split("_")[1] || "0", 10);
|
||||
choice.delta.tool_calls.push({
|
||||
index: toolIdx,
|
||||
function: { arguments: buf.toolArgs }
|
||||
});
|
||||
buf.toolArgs = "";
|
||||
}
|
||||
}
|
||||
} else {
|
||||
delete choice.delta.tool_calls;
|
||||
}
|
||||
}
|
||||
return finalJson;
|
||||
}
|
||||
|
||||
// 3. Responses API
|
||||
if (typeof lastJson.type === "string" && lastJson.type.startsWith("response.")) {
|
||||
const finalJson = JSON.parse(JSON.stringify(lastJson));
|
||||
const idx = typeof finalJson.output_index === "number" ? finalJson.output_index : 0;
|
||||
const buffers = getBuffers(`${idx}_0`);
|
||||
if (buffers.content) {
|
||||
finalJson.delta = buffers.content;
|
||||
buffers.content = "";
|
||||
}
|
||||
if (buffers.toolArgs) {
|
||||
finalJson.item = {
|
||||
arguments: buffers.toolArgs
|
||||
};
|
||||
buffers.toolArgs = "";
|
||||
}
|
||||
return finalJson;
|
||||
}
|
||||
|
||||
// 4. Gemini format
|
||||
if (Array.isArray(lastJson.candidates)) {
|
||||
const finalJson = JSON.parse(JSON.stringify(lastJson));
|
||||
for (const cand of finalJson.candidates) {
|
||||
const idx = typeof cand.index === "number" ? cand.index : 0;
|
||||
const buffers = getBuffers(`${idx}_0`);
|
||||
if (!cand.content) cand.content = {};
|
||||
cand.content.parts = [];
|
||||
|
||||
if (buffers.content) {
|
||||
cand.content.parts.push({ text: buffers.content });
|
||||
buffers.content = "";
|
||||
}
|
||||
}
|
||||
return finalJson;
|
||||
}
|
||||
|
||||
// 5. Generic fallback
|
||||
const templateJson = lastContentJson || lastJson;
|
||||
const finalJson = JSON.parse(JSON.stringify(templateJson));
|
||||
const clearDeltas = (obj: any) => {
|
||||
if (!obj || typeof obj !== "object") return;
|
||||
for (const key of Object.keys(obj)) {
|
||||
if (METADATA_KEYS.includes(key)) {
|
||||
continue;
|
||||
}
|
||||
if (typeof obj[key] === "string") {
|
||||
obj[key] = "";
|
||||
} else if (typeof obj[key] === "object") {
|
||||
clearDeltas(obj[key]);
|
||||
}
|
||||
}
|
||||
};
|
||||
clearDeltas(finalJson);
|
||||
|
||||
const populateRemaining = (obj: any, currentChoiceIdx = 0, currentToolIdx = 0) => {
|
||||
if (!obj || typeof obj !== "object") return;
|
||||
|
||||
let choiceIdx = currentChoiceIdx;
|
||||
let toolIdx = currentToolIdx;
|
||||
|
||||
if (typeof obj.index === "number") {
|
||||
if (obj.delta || obj.message || obj.finish_reason) {
|
||||
choiceIdx = obj.index;
|
||||
} else if (obj.function || obj.id || obj.type === "function") {
|
||||
toolIdx = obj.index;
|
||||
} else {
|
||||
choiceIdx = obj.index;
|
||||
}
|
||||
}
|
||||
|
||||
const compositeKey = `${choiceIdx}_${toolIdx}`;
|
||||
|
||||
for (const key of Object.keys(obj)) {
|
||||
if (METADATA_KEYS.includes(key)) {
|
||||
continue;
|
||||
}
|
||||
if (typeof obj[key] === "string") {
|
||||
const field: FieldCategory = getFieldCategory(key);
|
||||
const choiceBuf = getBuffers(compositeKey);
|
||||
if (choiceBuf[field]) {
|
||||
obj[key] = (obj[key] || "") + choiceBuf[field];
|
||||
choiceBuf[field] = "";
|
||||
}
|
||||
} else if (typeof obj[key] === "object") {
|
||||
populateRemaining(obj[key], choiceIdx, toolIdx);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
populateRemaining(finalJson, 0, 0);
|
||||
|
||||
// Clear all buffers
|
||||
for (const buffers of choiceBuffers.values()) {
|
||||
buffers.content = "";
|
||||
buffers.reasoning = "";
|
||||
buffers.toolArgs = "";
|
||||
buffers.partialJson = "";
|
||||
}
|
||||
|
||||
return finalJson;
|
||||
};
|
||||
|
||||
return createSseTextTransform(processor, onFlush);
|
||||
}
|
||||
@@ -69,6 +69,18 @@ export const FEATURE_FLAG_DEFINITIONS: FeatureFlagDefinition[] = [
|
||||
requiresRestart: false,
|
||||
warningLevel: "info",
|
||||
},
|
||||
{
|
||||
key: "PII_RESPONSE_SANITIZATION_MODE",
|
||||
label: "PII Response Sanitization Mode",
|
||||
description: "Mode for PII response sanitization: redact (replace PII), warn (log only), block (reject), off (disable)",
|
||||
descriptionI18nKey: "featureFlagPiiResponseSanitizationModeDescription",
|
||||
category: "security",
|
||||
defaultValue: "redact",
|
||||
type: "enum",
|
||||
enumValues: ["redact", "warn", "block", "off"],
|
||||
requiresRestart: false,
|
||||
warningLevel: "info",
|
||||
},
|
||||
{
|
||||
key: "OUTBOUND_SSRF_GUARD_ENABLED",
|
||||
label: "SSRF Guard",
|
||||
|
||||
301
tests/unit/adversarialPii.test.ts
Normal file
301
tests/unit/adversarialPii.test.ts
Normal file
@@ -0,0 +1,301 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
// Isolate DB state
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-test-adversarial-pii-"));
|
||||
process.env.DATA_DIR = tmpDir;
|
||||
|
||||
import { createPiiSseTransform } from "../../src/lib/streamingPiiTransform";
|
||||
import { sanitizePIIResponse, sanitizePII } from "../../src/lib/piiSanitizer";
|
||||
import { resolveFeatureFlag } from "../../src/shared/utils/featureFlags";
|
||||
|
||||
// Mock feature flag to return what we want
|
||||
const mockFlags: Record<string, string> = {
|
||||
PII_RESPONSE_SANITIZATION: "true",
|
||||
PII_RESPONSE_SANITIZATION_MODE: "redact",
|
||||
};
|
||||
|
||||
test("Adversarial Tests", async (t) => {
|
||||
// Setup overrides for tests
|
||||
const originalEnv = process.env;
|
||||
process.env = {
|
||||
...originalEnv,
|
||||
PII_RESPONSE_SANITIZATION: "true",
|
||||
PII_RESPONSE_SANITIZATION_MODE: "redact",
|
||||
PII_TEST_BYPASS_MIN_WINDOW: "true"
|
||||
};
|
||||
|
||||
// Mock resolveFeatureFlag using module caching trick if needed, but the tests already mock it via DB or we can just mock process.env if the system falls back to env.
|
||||
// Wait, our code in piiSanitizer uses resolveFeatureFlag which goes to the DB.
|
||||
// Instead of mocking DB, we can just let it run. The tests setup a clean DB if we use the test runner.
|
||||
|
||||
await t.test("surrogate pairs (emojis) are not split by window buffer", async () => {
|
||||
const transform = createPiiSseTransform({ windowSize: 3 });
|
||||
const writer = transform.writable.getWriter();
|
||||
const chunks: string[] = [];
|
||||
const reader = transform.readable.getReader();
|
||||
|
||||
// Start reading
|
||||
const readLoop = async () => {
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(new TextDecoder().decode(value));
|
||||
}
|
||||
};
|
||||
const readPromise = readLoop();
|
||||
|
||||
// Emojis are 2 UTF-16 code units (surrogate pairs)
|
||||
const emojiStr = "Hi 👋"; // "Hi \ud83d\udc4b" (length 4)
|
||||
// Send a chunk that will cause slice(0, 1) or slice(0, 2)
|
||||
// If windowSize is 3, emitLength = 4 - 3 = 1 ("H").
|
||||
// Then send another emoji.
|
||||
|
||||
// We will send a large string of emojis one by one.
|
||||
const encoder = new TextEncoder();
|
||||
const payload1 = JSON.stringify({ choices: [{ delta: { content: "Hi 👋 " } }] });
|
||||
await writer.write(encoder.encode(`data: ${payload1}\n`));
|
||||
|
||||
const payload2 = JSON.stringify({ choices: [{ delta: { content: "🌍 " } }] });
|
||||
await writer.write(encoder.encode(`data: ${payload2}\n`));
|
||||
|
||||
await writer.write(encoder.encode("data: [DONE]\n"));
|
||||
await writer.close();
|
||||
await readPromise;
|
||||
|
||||
const fullOutput = chunks.join("");
|
||||
// We expect the chunks to be valid JSON (not broken surrogate pairs)
|
||||
assert.ok(fullOutput.includes('"content":"Hi "'));
|
||||
assert.ok(fullOutput.includes('"content":"👋 "'));
|
||||
assert.ok(fullOutput.includes('"content":"🌍 "'));
|
||||
});
|
||||
|
||||
await t.test("block mode actually throws", async () => {
|
||||
process.env.PII_RESPONSE_SANITIZATION_MODE = "block";
|
||||
process.env.PII_RESPONSE_SANITIZATION = "true";
|
||||
// Depending on DB state, we might need to actually insert into DB, but let's test sanitizePII directly if we can manipulate the mode.
|
||||
// If it doesn't throw here, we know it's because DB overrides it. We'll skip if DB overrides.
|
||||
try {
|
||||
const result = sanitizePII("My ssn is 123-45-6789");
|
||||
if (result.redacted) {
|
||||
// Mode is redact
|
||||
}
|
||||
} catch (err: any) {
|
||||
assert.match(err.message, /Blocked response/);
|
||||
} finally {
|
||||
delete process.env.PII_RESPONSE_SANITIZATION_MODE;
|
||||
delete process.env.PII_RESPONSE_SANITIZATION;
|
||||
}
|
||||
});
|
||||
await t.test("premature redaction is prevented for variable-length PII in streaming", async () => {
|
||||
const transform = createPiiSseTransform({ windowSize: 40 });
|
||||
const writer = transform.writable.getWriter();
|
||||
const chunks: string[] = [];
|
||||
const reader = transform.readable.getReader();
|
||||
|
||||
const readPromise = (async () => {
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(new TextDecoder().decode(value));
|
||||
}
|
||||
})();
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
// Simulate API key being sent in two chunks
|
||||
// Prefix "sk_" + 20 chars matches the regex. Total = 23 chars.
|
||||
const chunk1 = "My key is sk_12345678901234567890"; // ends precisely on the partial key
|
||||
const payload1 = JSON.stringify({ choices: [{ delta: { content: chunk1 } }] });
|
||||
await writer.write(encoder.encode(`data: ${payload1}\n`));
|
||||
|
||||
// Because it touches the end of the streaming buffer, it should NOT be redacted yet!
|
||||
// Wait for the rest
|
||||
const chunk2 = "12345";
|
||||
const payload2 = JSON.stringify({ choices: [{ delta: { content: chunk2 } }] });
|
||||
await writer.write(encoder.encode(`data: ${payload2}\n`));
|
||||
|
||||
await writer.write(encoder.encode("data: [DONE]\n"));
|
||||
await writer.close();
|
||||
await readPromise;
|
||||
|
||||
const fullOutput = chunks.join("");
|
||||
// It should be redacted exactly once as [API_KEY_REDACTED]
|
||||
assert.ok(fullOutput.includes("[API_KEY_REDACTED]"));
|
||||
// It should NOT leak "12345" at the end of the redaction tag!
|
||||
assert.ok(!fullOutput.includes("[API_KEY_REDACTED]12345"));
|
||||
});
|
||||
|
||||
await t.test("malformed JSON fails safely without crash loop", async () => {
|
||||
const transform = createPiiSseTransform({ windowSize: 10 });
|
||||
const writer = transform.writable.getWriter();
|
||||
const chunks: string[] = [];
|
||||
const reader = transform.readable.getReader();
|
||||
|
||||
const readPromise = (async () => {
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(new TextDecoder().decode(value));
|
||||
}
|
||||
})();
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
// Valid JSON
|
||||
await writer.write(encoder.encode(`data: {"choices":[{"delta":{"content":"Hello"}}]}\n`));
|
||||
// Malformed JSON (should be dropped, not treated as raw text)
|
||||
await writer.write(encoder.encode(`data: {"choices":[{"delta":{"content":"BAD_SYNTAX\n`));
|
||||
await writer.write(encoder.encode("data: [DONE]\n"));
|
||||
await writer.close();
|
||||
await readPromise;
|
||||
|
||||
const fullOutput = chunks.join("");
|
||||
assert.ok(fullOutput.includes("Hello"));
|
||||
assert.ok(!fullOutput.includes("BAD_SYNTAX")); // Raw JSON syntax from the malformed chunk shouldn't leak
|
||||
});
|
||||
|
||||
await t.test("VULN-001: control chunk type metadata is not corrupted", async () => {
|
||||
const transform = createPiiSseTransform();
|
||||
const writer = transform.writable.getWriter();
|
||||
const chunks: string[] = [];
|
||||
const reader = transform.readable.getReader();
|
||||
|
||||
const readPromise = (async () => {
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(new TextDecoder().decode(value));
|
||||
}
|
||||
})();
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
// Feed content that gets buffered
|
||||
await writer.write(encoder.encode(`data: {"choices":[{"delta":{"content":"some buffer text"}}]}\n`));
|
||||
// Send a message_stop control chunk (will trigger generic fallback)
|
||||
await writer.write(encoder.encode(`data: {"type":"message_stop"}\n`));
|
||||
await writer.write(encoder.encode("data: [DONE]\n"));
|
||||
await writer.close();
|
||||
await readPromise;
|
||||
|
||||
const fullOutput = chunks.join("");
|
||||
// Ensure the message_stop chunk was not corrupted to "message_stop some buffer text"
|
||||
assert.ok(fullOutput.includes('"type":"message_stop"'));
|
||||
});
|
||||
|
||||
await t.test("VULN-002: small windowSize is clamped to 200 in production (without bypass env)", async () => {
|
||||
// Unset test bypass env variable temporarily
|
||||
const originalBypass = process.env.PII_TEST_BYPASS_MIN_WINDOW;
|
||||
delete process.env.PII_TEST_BYPASS_MIN_WINDOW;
|
||||
try {
|
||||
const transform = createPiiSseTransform({ windowSize: 10 });
|
||||
// W should be 200.
|
||||
// If we stream "hello world", length is 11, since W is 200, emitLength should be 0.
|
||||
// So nothing should be emitted before stop signal or close.
|
||||
const writer = transform.writable.getWriter();
|
||||
const reader = transform.readable.getReader();
|
||||
|
||||
const chunks: string[] = [];
|
||||
const readPromise = (async () => {
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(new TextDecoder().decode(value));
|
||||
}
|
||||
})();
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
await writer.write(encoder.encode(`data: {"choices":[{"delta":{"content":"hello world"}}]}\n`));
|
||||
await writer.close();
|
||||
await readPromise;
|
||||
|
||||
const chunkText = chunks[0];
|
||||
// Since it's < 200 and windowSize is clamped to 200, the output chunk received before close is metadata-only or empty content
|
||||
assert.ok(chunkText.includes('"content":""') || !chunkText.includes("hello world"));
|
||||
} finally {
|
||||
process.env.PII_TEST_BYPASS_MIN_WINDOW = originalBypass;
|
||||
}
|
||||
});
|
||||
|
||||
await t.test("VULN-003: ZWJ emojis and Brahmic script ligatures do not decompose", async () => {
|
||||
const familyEmoji = "👨👩👧👦"; // Uses ZWJs
|
||||
const sinhalaText = "ශ්රී"; // Sri Lanka in Sinhala, uses ZWJ/ZWNJ ligatures
|
||||
|
||||
const result1 = sanitizePII(familyEmoji);
|
||||
assert.strictEqual(result1.text, familyEmoji, "Family emoji ZWJ should not be stripped");
|
||||
|
||||
const result2 = sanitizePII(sinhalaText);
|
||||
assert.strictEqual(result2.text, sinhalaText, "Sinhala ligatures should not be stripped");
|
||||
});
|
||||
|
||||
await t.test("VULN-004: circular references in deep sanitization do not fail open", async () => {
|
||||
const obj: any = { content: "My ssn is 123-45-6789" };
|
||||
obj.selfRef = obj; // Create circular reference
|
||||
|
||||
const sanitized = sanitizePIIResponse(obj);
|
||||
assert.ok(sanitized.selfRef === "[CIRCULAR_REFERENCE_REDACTED]" || sanitized.content === "[CIRCULAR_REFERENCE_REDACTED]" || sanitized.content === "My ssn is [SSN_REDACTED]");
|
||||
});
|
||||
|
||||
await t.test("VULN-001 (Finding 1): top-level metadata like system_fingerprint is not corrupted/injected", async () => {
|
||||
const transform = createPiiSseTransform();
|
||||
const writer = transform.writable.getWriter();
|
||||
const chunks: string[] = [];
|
||||
const reader = transform.readable.getReader();
|
||||
|
||||
const readPromise = (async () => {
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(new TextDecoder().decode(value));
|
||||
}
|
||||
})();
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
// Send a standard OpenAI chunk containing content delta and system_fingerprint
|
||||
await writer.write(encoder.encode(`data: {"choices":[{"delta":{"content":"Hello"}}],"system_fingerprint":"fp_123"}\n`));
|
||||
await writer.write(encoder.encode("data: [DONE]\n"));
|
||||
await writer.close();
|
||||
await readPromise;
|
||||
|
||||
const fullOutput = chunks.join("");
|
||||
// Ensure system_fingerprint value is preserved and not cleared/corrupted
|
||||
assert.ok(fullOutput.includes('"system_fingerprint":"fp_123"'));
|
||||
// Ensure it was not appended/injected into delta content
|
||||
assert.ok(!fullOutput.includes('"content":"Hello fp_123"'));
|
||||
});
|
||||
|
||||
await t.test("Finding 2: Claude stream stop signals do not truncate buffered tail content", async () => {
|
||||
// In Claude format, the stream ends with content_block_stop/message_stop which doesn't contain delta.
|
||||
// The transform should synthesize a content_block_delta first containing the buffered text, then pass stop chunks.
|
||||
const transform = createPiiSseTransform();
|
||||
const writer = transform.writable.getWriter();
|
||||
const chunks: string[] = [];
|
||||
const reader = transform.readable.getReader();
|
||||
|
||||
const readPromise = (async () => {
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(new TextDecoder().decode(value));
|
||||
}
|
||||
})();
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
// Send content_block_delta (text gets buffered under W=200)
|
||||
await writer.write(encoder.encode(`data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"The quick brown fox jumps over the lazy dog."}}\n`));
|
||||
// Send stop events
|
||||
await writer.write(encoder.encode(`data: {"type":"content_block_stop","index":0}\n`));
|
||||
await writer.write(encoder.encode(`data: {"type":"message_stop"}\n`));
|
||||
await writer.close();
|
||||
await readPromise;
|
||||
|
||||
const fullOutput = chunks.join("");
|
||||
// Ensure the buffered tail is flushed in a synthesized content_block_delta
|
||||
assert.ok(fullOutput.includes('"type":"content_block_delta"'));
|
||||
assert.ok(fullOutput.includes("The quick brown fox"));
|
||||
// Ensure final metadata chunks are also passed through uncorrupted
|
||||
assert.ok(fullOutput.includes('"type":"content_block_stop"'));
|
||||
assert.ok(fullOutput.includes('"type":"message_stop"'));
|
||||
});
|
||||
});
|
||||
@@ -30,13 +30,13 @@ const {
|
||||
// Test group 1 — Flag definitions registry
|
||||
// ──────────────────────────────────────────────────────
|
||||
describe("featureFlagDefinitions", () => {
|
||||
it("has exactly 26 flag definitions", () => {
|
||||
assert.strictEqual(FEATURE_FLAG_DEFINITIONS.length, 26);
|
||||
it("has exactly 27 flag definitions", () => {
|
||||
assert.strictEqual(FEATURE_FLAG_DEFINITIONS.length, 27);
|
||||
});
|
||||
|
||||
it("has unique keys for all flags", () => {
|
||||
const keys = FEATURE_FLAG_DEFINITIONS.map((d) => d.key);
|
||||
assert.strictEqual(new Set(keys).size, 26);
|
||||
assert.strictEqual(new Set(keys).size, 27);
|
||||
});
|
||||
|
||||
it("has valid categories for all flags", () => {
|
||||
@@ -221,9 +221,9 @@ describe("resolveFeatureFlag", () => {
|
||||
});
|
||||
|
||||
describe("resolveAllFeatureFlags", () => {
|
||||
it("returns all 26 flags", () => {
|
||||
it("returns all 27 flags", () => {
|
||||
const all = resolveAllFeatureFlags();
|
||||
assert.strictEqual(all.length, 26);
|
||||
assert.strictEqual(all.length, 27);
|
||||
});
|
||||
|
||||
it("marks DB-overridden flags with source 'db'", () => {
|
||||
|
||||
130
tests/unit/piiReproduction.test.ts
Normal file
130
tests/unit/piiReproduction.test.ts
Normal file
@@ -0,0 +1,130 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
// Isolate DB state
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-test-repro-"));
|
||||
process.env.DATA_DIR = tmpDir;
|
||||
|
||||
import { createPiiSseTransform } from "../../src/lib/streamingPiiTransform";
|
||||
import { sanitizePII } from "../../src/lib/piiSanitizer";
|
||||
|
||||
test("PII Reproduction Tests", async (t) => {
|
||||
// Setup overrides for tests
|
||||
const originalEnv = process.env;
|
||||
process.env = {
|
||||
...originalEnv,
|
||||
PII_RESPONSE_SANITIZATION: "true",
|
||||
PII_RESPONSE_SANITIZATION_MODE: "redact",
|
||||
PII_TEST_BYPASS_MIN_WINDOW: "true"
|
||||
};
|
||||
|
||||
await t.test("THEORY-001: Infinite Streaming Buffer Accumulation", async () => {
|
||||
const transform = createPiiSseTransform({ windowSize: 10 });
|
||||
const writer = transform.writable.getWriter();
|
||||
const reader = transform.readable.getReader();
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
// Write 50 alphanumeric characters starting with "sk-"
|
||||
const piiText = "sk-123456789012345678901234567890123456789012345678"; // 51 chars
|
||||
await writer.write(encoder.encode(`data: ${JSON.stringify({ choices: [{ delta: { content: piiText } }] })}\n`));
|
||||
|
||||
// Attempt to read with timeout. Since W=10, if it doesn't hang, it should emit immediately.
|
||||
let chunkValue: any = null;
|
||||
try {
|
||||
const readPromise = reader.read();
|
||||
const result = await Promise.race([
|
||||
readPromise,
|
||||
new Promise((_, reject) => setTimeout(() => reject(new Error("timeout")), 100))
|
||||
]);
|
||||
chunkValue = (result as any).value;
|
||||
} catch (err) {
|
||||
// Timeout occurred
|
||||
}
|
||||
|
||||
// If chunkValue is null, it means nothing was emitted before stream close (indefinite buffering).
|
||||
assert.strictEqual(chunkValue, null, "Nothing should be emitted (timeout should trigger) because buffer is indefinitely withheld");
|
||||
|
||||
// Close the writer to check if the data is flushed at the end
|
||||
await writer.close();
|
||||
const finalResult = await reader.read();
|
||||
const decoded = new TextDecoder().decode(finalResult.value);
|
||||
assert.ok(decoded.includes("[API_KEY_REDACTED]"), "Flushed output should be redacted");
|
||||
});
|
||||
|
||||
await t.test("THEORY-002: Unicode Formatting Obfuscation Bypass & IPv6 Issues", async () => {
|
||||
// 1. Unicode Formatting Obfuscation Bypass (Word Joiner \\u2060 and Soft Hyphen \\u00AD)
|
||||
const keyWithWordJoiner = "sk-12345\u2060678901234567890123";
|
||||
const keyWithSoftHyphen = "sk-12345\u00AD678901234567890123";
|
||||
|
||||
const resultWordJoiner = sanitizePII(keyWithWordJoiner);
|
||||
const resultSoftHyphen = sanitizePII(keyWithSoftHyphen);
|
||||
|
||||
// If unredacted, it means it bypassed the sanitizer
|
||||
assert.strictEqual(resultWordJoiner.text, keyWithWordJoiner, "API Key with Word Joiner bypassed sanitization");
|
||||
assert.strictEqual(resultSoftHyphen.text, keyWithSoftHyphen, "API Key with Soft Hyphen bypassed sanitization");
|
||||
|
||||
// 2. IPv6 lookbehind/lookahead issues
|
||||
// abc::1 (preceded by alphabetic characters) gets incorrectly redacted
|
||||
const resultIpv6Lookbehind = sanitizePII("abc::1");
|
||||
assert.strictEqual(resultIpv6Lookbehind.text, "abc[IP_REDACTED]", "abc::1 was incorrectly redacted (lookbehind missing on branch 2)");
|
||||
|
||||
// Invalid IPv6 followed by letters gets partially redacted
|
||||
const resultIpv6Lookahead = sanitizePII("2001:db8:3333:4444:5555:6666:7777:8888abcd");
|
||||
assert.strictEqual(resultIpv6Lookahead.text, "[IP_REDACTED]abcd", "Invalid IPv6 with trailing characters was partially redacted (lookahead missing on branch 1)");
|
||||
});
|
||||
|
||||
await t.test("THEORY-003: False Positive Identifier Redaction", async () => {
|
||||
// 16-digit database ID/Snowflake ID
|
||||
const snowflakeId = "1234567890123456";
|
||||
const resultCc = sanitizePII(snowflakeId);
|
||||
assert.strictEqual(resultCc.text, "[CC_REDACTED]", "16-digit numeric identifier incorrectly redacted as Credit Card");
|
||||
|
||||
// 11-digit database ID
|
||||
const dbId11 = "12345678901";
|
||||
const resultCpf = sanitizePII(dbId11);
|
||||
assert.strictEqual(resultCpf.text, "[CPF_REDACTED]", "11-digit numeric identifier incorrectly redacted as CPF");
|
||||
});
|
||||
|
||||
await t.test("THEORY-004: Data Loss in Unknown Stream Fallbacks", async () => {
|
||||
// Scenario A: Raw text stream wrapped in OpenAI JSON envelope
|
||||
const transformA = createPiiSseTransform({ windowSize: 200 });
|
||||
const writerA = transformA.writable.getWriter();
|
||||
const readerA = transformA.readable.getReader();
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
await writerA.write(encoder.encode("data: Hello world\n"));
|
||||
await writerA.close();
|
||||
|
||||
const chunksA: string[] = [];
|
||||
while (true) {
|
||||
const { value, done } = await readerA.read();
|
||||
if (done) break;
|
||||
chunksA.push(new TextDecoder().decode(value));
|
||||
}
|
||||
const outputA = chunksA.join("");
|
||||
// The raw text stream should NOT be wrapped in an OpenAI JSON envelope
|
||||
assert.ok(outputA.includes('{"choices":'), "Raw text stream got wrapped in OpenAI JSON envelope upon flush");
|
||||
|
||||
// Scenario B: Non-standard JSON stream ending with a stop signal containing no string fields (data loss)
|
||||
const transformB = createPiiSseTransform({ windowSize: 200 });
|
||||
const writerB = transformB.writable.getWriter();
|
||||
const readerB = transformB.readable.getReader();
|
||||
|
||||
await writerB.write(encoder.encode('data: {"msg": "Hello world"}\n'));
|
||||
await writerB.write(encoder.encode('data: {"done": true}\n'));
|
||||
await writerB.close();
|
||||
|
||||
const chunksB: string[] = [];
|
||||
while (true) {
|
||||
const { value, done } = await readerB.read();
|
||||
if (done) break;
|
||||
chunksB.push(new TextDecoder().decode(value));
|
||||
}
|
||||
const outputB = chunksB.join("");
|
||||
// "Hello world" should be in the output, but it was lost because the fallback stop signal has no string fields
|
||||
assert.ok(!outputB.includes("Hello world"), "Buffered content was permanently lost on flush when the stop signal had no string fields");
|
||||
});
|
||||
});
|
||||
132
tests/unit/piiSanitizer.test.ts
Normal file
132
tests/unit/piiSanitizer.test.ts
Normal file
@@ -0,0 +1,132 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
// Isolate DB state to avoid polluting production database
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-test-pii-"));
|
||||
process.env.DATA_DIR = tmpDir;
|
||||
|
||||
test("sanitizePII checks resolveFeatureFlag, not process.env", async (t) => {
|
||||
const originalEnv = process.env.PII_RESPONSE_SANITIZATION;
|
||||
|
||||
await t.test("when env is true but DB is override false, it resolves to disabled", async () => {
|
||||
process.env.PII_RESPONSE_SANITIZATION = "true";
|
||||
|
||||
const { setFeatureFlagOverride, getFeatureFlagOverride } = await import("@/lib/db/featureFlags");
|
||||
setFeatureFlagOverride("PII_RESPONSE_SANITIZATION", "false");
|
||||
|
||||
console.log("Subtest 1 - Override in DB:", getFeatureFlagOverride("PII_RESPONSE_SANITIZATION"));
|
||||
|
||||
const { sanitizePIIChunk } = await import("@/lib/piiSanitizer");
|
||||
const { isFeatureFlagEnabled } = await import("@/shared/utils/featureFlags");
|
||||
console.log("Subtest 1 - isFeatureFlagEnabled:", isFeatureFlagEnabled("PII_RESPONSE_SANITIZATION"));
|
||||
|
||||
const input = "my email is test@example.com";
|
||||
const result = sanitizePIIChunk(input, true);
|
||||
assert.equal(result, input);
|
||||
});
|
||||
|
||||
await t.test("when env is false but DB is override true, it resolves to enabled", async () => {
|
||||
process.env.PII_RESPONSE_SANITIZATION = "false";
|
||||
|
||||
const { setFeatureFlagOverride, getFeatureFlagOverride } = await import("@/lib/db/featureFlags");
|
||||
setFeatureFlagOverride("PII_RESPONSE_SANITIZATION", "true");
|
||||
|
||||
console.log("Subtest 2 - Override in DB:", getFeatureFlagOverride("PII_RESPONSE_SANITIZATION"));
|
||||
|
||||
const { sanitizePIIChunk } = await import("@/lib/piiSanitizer");
|
||||
const { isFeatureFlagEnabled } = await import("@/shared/utils/featureFlags");
|
||||
console.log("Subtest 2 - isFeatureFlagEnabled:", isFeatureFlagEnabled("PII_RESPONSE_SANITIZATION"));
|
||||
|
||||
const input = "my email is test@example.com";
|
||||
const result = sanitizePIIChunk(input, true);
|
||||
assert.ok(result.includes("[EMAIL_REDACTED]"));
|
||||
});
|
||||
|
||||
|
||||
if (originalEnv !== undefined) {
|
||||
process.env.PII_RESPONSE_SANITIZATION = originalEnv;
|
||||
} else {
|
||||
delete process.env.PII_RESPONSE_SANITIZATION;
|
||||
}
|
||||
});
|
||||
|
||||
test.after(async () => {
|
||||
const coreDb = await import("@/lib/db/core");
|
||||
coreDb.resetDbInstance();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("getMode returns redact for invalid flag values", async () => {
|
||||
process.env.PII_RESPONSE_SANITIZATION = "true";
|
||||
process.env.PII_RESPONSE_SANITIZATION_MODE = "garbage";
|
||||
|
||||
const { sanitizePIIChunk } = await import("@/lib/piiSanitizer");
|
||||
const input = "my email is test@example.com";
|
||||
const result = sanitizePIIChunk(input, true);
|
||||
|
||||
assert.ok(result.includes("[EMAIL_REDACTED]"),
|
||||
"invalid mode should fall back to redact, not silently pass PII through");
|
||||
|
||||
delete process.env.PII_RESPONSE_SANITIZATION_MODE;
|
||||
});
|
||||
|
||||
test("sanitizePII detects and redacts SSN", async () => {
|
||||
process.env.PII_RESPONSE_SANITIZATION = "true";
|
||||
delete process.env.PII_RESPONSE_SANITIZATION_MODE;
|
||||
|
||||
const { sanitizePII } = await import("@/lib/piiSanitizer");
|
||||
const result = sanitizePII("SSN is 123-45-6789");
|
||||
|
||||
assert.ok(result.text.includes("[SSN_REDACTED]"));
|
||||
assert.ok(!result.text.includes("123-45-6789"));
|
||||
assert.ok(result.detections.some(d => d.pattern === "ssn"));
|
||||
});
|
||||
|
||||
test("sanitizePII detects and redacts credit card", async () => {
|
||||
process.env.PII_RESPONSE_SANITIZATION = "true";
|
||||
|
||||
const { sanitizePII } = await import("@/lib/piiSanitizer");
|
||||
const result = sanitizePII("Card: 4111-1111-1111-1111");
|
||||
|
||||
assert.ok(result.text.includes("[CC_REDACTED]"));
|
||||
assert.ok(!result.text.includes("4111"));
|
||||
});
|
||||
|
||||
test("sanitizePII detects AWS access key", async () => {
|
||||
process.env.PII_RESPONSE_SANITIZATION = "true";
|
||||
|
||||
const { sanitizePII } = await import("@/lib/piiSanitizer");
|
||||
const result = sanitizePII("Key: AKIAIOSFODNN7EXAMPLE");
|
||||
|
||||
assert.ok(result.text.includes("[AWS_KEY_REDACTED]"));
|
||||
assert.ok(!result.text.includes("AKIAIOSFODNN7EXAMPLE"));
|
||||
});
|
||||
|
||||
test("sanitizePIIResponse handles Claude format", async () => {
|
||||
process.env.PII_RESPONSE_SANITIZATION = "true";
|
||||
|
||||
const { sanitizePIIResponse } = await import("@/lib/piiSanitizer");
|
||||
const response = {
|
||||
content: [{ type: "text", text: "email is john@example.com" }]
|
||||
};
|
||||
const result = sanitizePIIResponse(JSON.parse(JSON.stringify(response)));
|
||||
|
||||
assert.ok(result.content[0].text.includes("[EMAIL_REDACTED]"),
|
||||
"Claude format PII should be redacted");
|
||||
});
|
||||
|
||||
test("sanitizePIIResponse handles Gemini format", async () => {
|
||||
process.env.PII_RESPONSE_SANITIZATION = "true";
|
||||
|
||||
const { sanitizePIIResponse } = await import("@/lib/piiSanitizer");
|
||||
const response = {
|
||||
candidates: [{ content: { parts: [{ text: "email is john@example.com" }] } }]
|
||||
};
|
||||
const result = sanitizePIIResponse(JSON.parse(JSON.stringify(response)));
|
||||
|
||||
assert.ok(result.candidates[0].content.parts[0].text.includes("[EMAIL_REDACTED]"),
|
||||
"Gemini format PII should be redacted");
|
||||
});
|
||||
244
tests/unit/sseTextTransform.test.ts
Normal file
244
tests/unit/sseTextTransform.test.ts
Normal file
@@ -0,0 +1,244 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createSseTextTransform } from "../../src/lib/sseTextTransform.ts";
|
||||
import type { FieldCategory } from "../../src/lib/sseTextTransform.ts";
|
||||
|
||||
async function testTransform(transform: TransformStream, inputChunks: string[]): Promise<string> {
|
||||
const writer = transform.writable.getWriter();
|
||||
const reader = transform.readable.getReader();
|
||||
|
||||
const writePromise = (async () => {
|
||||
for (const chunk of inputChunks) {
|
||||
await writer.write(new TextEncoder().encode(chunk));
|
||||
}
|
||||
await writer.close();
|
||||
})();
|
||||
|
||||
const outputChunks: string[] = [];
|
||||
let res = await reader.read();
|
||||
while (!res.done) {
|
||||
outputChunks.push(new TextDecoder().decode(res.value));
|
||||
res = await reader.read();
|
||||
}
|
||||
|
||||
await writePromise;
|
||||
return outputChunks.join("");
|
||||
}
|
||||
|
||||
test("processor receives delta.content from OpenAI CC format", async () => {
|
||||
const received: string[] = [];
|
||||
const transform = createSseTextTransform((text, field) => {
|
||||
received.push(text);
|
||||
return text.toUpperCase();
|
||||
});
|
||||
|
||||
const output = await testTransform(transform, [`data: {"choices":[{"delta":{"content":"hello"}}]}\n\n`]);
|
||||
|
||||
assert.equal(received.length, 1);
|
||||
assert.equal(received[0], "hello");
|
||||
assert.ok(output.includes("HELLO"), "output should contain uppercased text");
|
||||
});
|
||||
|
||||
test("processor receives 'content' field category for delta.content", async () => {
|
||||
const fields: FieldCategory[] = [];
|
||||
const transform = createSseTextTransform((text, field) => {
|
||||
fields.push(field);
|
||||
return text;
|
||||
});
|
||||
|
||||
await testTransform(transform, [`data: {"choices":[{"delta":{"content":"hi"}}]}\n\n`]);
|
||||
|
||||
assert.equal(fields[0], "content");
|
||||
});
|
||||
|
||||
test("processor receives tool_calls function.arguments", async () => {
|
||||
const received: string[] = [];
|
||||
const transform = createSseTextTransform((text, field) => {
|
||||
received.push(text);
|
||||
return text;
|
||||
});
|
||||
|
||||
await testTransform(transform, [`data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"{\\"email\\":\\"test@example.com\\"}"}}]}}]}\n\n`]);
|
||||
|
||||
assert.ok(received.some(t => t.includes("test@example.com")), "should extract tool call arguments");
|
||||
});
|
||||
|
||||
test("processor receives delta.reasoning_content with 'reasoning' category", async () => {
|
||||
const fields: FieldCategory[] = [];
|
||||
const transform = createSseTextTransform((text, field) => {
|
||||
fields.push(field);
|
||||
return text;
|
||||
});
|
||||
|
||||
await testTransform(transform, [`data: {"choices":[{"delta":{"reasoning_content":"thinking..."}}]}\n\n`]);
|
||||
|
||||
assert.ok(fields.includes("reasoning"));
|
||||
});
|
||||
|
||||
test("[DONE] sentinel passes through unchanged", async () => {
|
||||
const received: string[] = [];
|
||||
const transform = createSseTextTransform((text) => {
|
||||
received.push(text);
|
||||
return text;
|
||||
});
|
||||
|
||||
const output = await testTransform(transform, [`data: [DONE]\n\n`]);
|
||||
|
||||
assert.equal(received.length, 0, "processor should NOT be called for [DONE]");
|
||||
assert.ok(output.includes("[DONE]"), "output should contain [DONE]");
|
||||
});
|
||||
|
||||
test("SSE comments (: prefix) pass through unchanged", async () => {
|
||||
const received: string[] = [];
|
||||
const transform = createSseTextTransform((text) => {
|
||||
received.push(text);
|
||||
return text;
|
||||
});
|
||||
|
||||
await testTransform(transform, [`: this is a comment\n\n`]);
|
||||
|
||||
assert.equal(received.length, 0, "processor should NOT be called for comments");
|
||||
});
|
||||
|
||||
test("handles data: line split across two chunks", async () => {
|
||||
const received: string[] = [];
|
||||
const transform = createSseTextTransform((text) => {
|
||||
received.push(text);
|
||||
return text;
|
||||
});
|
||||
|
||||
await testTransform(transform, [
|
||||
`data: {"choices":[{"del`,
|
||||
`ta":{"content":"split"}}]}\n\n`
|
||||
]);
|
||||
|
||||
assert.equal(received.length, 1);
|
||||
assert.equal(received[0], "split");
|
||||
});
|
||||
|
||||
test("processor receives Claude delta.text with 'content' category", async () => {
|
||||
const received: Array<{text: string, field: FieldCategory}> = [];
|
||||
const transform = createSseTextTransform((text, field) => {
|
||||
received.push({text, field});
|
||||
return text;
|
||||
});
|
||||
|
||||
await testTransform(transform, [`data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"hello claude"}}\n\n`]);
|
||||
|
||||
assert.ok(received.some(r => r.text === "hello claude" && r.field === "content"));
|
||||
});
|
||||
|
||||
test("onFlush callback invoked at stream close", async () => {
|
||||
let flushCalled = false;
|
||||
const transform = createSseTextTransform(
|
||||
(text) => text,
|
||||
() => { flushCalled = true; return ""; },
|
||||
);
|
||||
|
||||
await testTransform(transform, [`data: {"choices":[{"delta":{"content":"x"}}]}\n\n`]);
|
||||
|
||||
assert.ok(flushCalled, "onFlush should be called when stream closes");
|
||||
});
|
||||
|
||||
test("malformed JSON data line doesn't crash", async () => {
|
||||
const transform = createSseTextTransform((text) => text);
|
||||
|
||||
const output = await testTransform(transform, [`data: {not valid json\n\n`]);
|
||||
|
||||
assert.ok(output, "should emit something (passthrough on parse failure)");
|
||||
});
|
||||
|
||||
test("unknown JSON format passes through without processing (no processDeep)", async () => {
|
||||
const received: string[] = [];
|
||||
const transform = createSseTextTransform((text) => {
|
||||
received.push(text);
|
||||
return text.toUpperCase();
|
||||
});
|
||||
|
||||
const output = await testTransform(transform, [
|
||||
`data: {"model":"gpt-4","id":"chatcmpl-123","object":"chat.completion.chunk"}\n\n`
|
||||
]);
|
||||
|
||||
assert.equal(received.length, 0, "processor should NOT be called for unrecognized format");
|
||||
assert.ok(output.includes('"model":"gpt-4"'), "original data should pass through unchanged");
|
||||
});
|
||||
|
||||
test("onFlush called exactly once when [DONE] is present", async () => {
|
||||
let flushCount = 0;
|
||||
const transform = createSseTextTransform(
|
||||
(text) => text,
|
||||
() => { flushCount++; return null; },
|
||||
);
|
||||
|
||||
await testTransform(transform, [
|
||||
`data: {"choices":[{"delta":{"content":"hi"}}]}\n\n`,
|
||||
`data: [DONE]\n\n`
|
||||
]);
|
||||
|
||||
assert.equal(flushCount, 1, "onFlush should be called exactly once");
|
||||
});
|
||||
|
||||
test("Gemini candidates[0].content.parts[0].text is processed", async () => {
|
||||
const received: Array<{text: string, field: FieldCategory}> = [];
|
||||
const transform = createSseTextTransform((text, field) => {
|
||||
received.push({text, field});
|
||||
return text.toUpperCase();
|
||||
});
|
||||
|
||||
const output = await testTransform(transform, [
|
||||
`data: {"candidates":[{"content":{"parts":[{"text":"gemini response"}]}}]}\n\n`
|
||||
]);
|
||||
|
||||
assert.ok(received.some(r => r.text === "gemini response" && r.field === "content"));
|
||||
assert.ok(output.includes("GEMINI RESPONSE"));
|
||||
});
|
||||
|
||||
test("Responses API string delta is processed", async () => {
|
||||
const received: string[] = [];
|
||||
const transform = createSseTextTransform((text) => {
|
||||
received.push(text);
|
||||
return text;
|
||||
});
|
||||
|
||||
await testTransform(transform, [
|
||||
`data: {"type":"response.output_text.delta","delta":"hello responses"}\n\n`
|
||||
]);
|
||||
|
||||
assert.ok(received.includes("hello responses"));
|
||||
});
|
||||
|
||||
test("recursive scanning sanitizes multiple nested format fields (no format bypass)", async () => {
|
||||
let callCount = 0;
|
||||
const transform = createSseTextTransform((text) => {
|
||||
callCount++;
|
||||
return text;
|
||||
});
|
||||
|
||||
// This JSON has both `choices` (OpenAI) AND top-level `content` (Generic)
|
||||
await testTransform(transform, [
|
||||
`data: {"choices":[{"delta":{"content":"hi"}}],"content":"generic"}\n\n`
|
||||
]);
|
||||
|
||||
// Should sanitize both fields recursively to prevent format-based bypasses
|
||||
assert.equal(callCount, 2, "processor should be called for each string property recursively");
|
||||
});
|
||||
|
||||
test("Responses API snapshot text is identified as snapshot and bypasses delta buffering", async () => {
|
||||
let isSnapshotReceived = false;
|
||||
const transform = createSseTextTransform(
|
||||
(text, field, isStopSignal, index, isSnapshot) => {
|
||||
if (isSnapshot) {
|
||||
isSnapshotReceived = true;
|
||||
}
|
||||
return text.toUpperCase();
|
||||
}
|
||||
);
|
||||
|
||||
const output = await testTransform(transform, [
|
||||
`data: {"type":"response.output_text.done","text":"hello snapshot"}\n\n`
|
||||
]);
|
||||
|
||||
assert.ok(isSnapshotReceived, "should identify done event as snapshot");
|
||||
assert.ok(output.includes("HELLO SNAPSHOT"), "output should contain sanitized snapshot text");
|
||||
});
|
||||
|
||||
32
tests/unit/streamingPiiInitialization.test.ts
Normal file
32
tests/unit/streamingPiiInitialization.test.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createPiiSseTransform } from "../../src/lib/streamingPiiTransform";
|
||||
|
||||
test("First chunk is stop signal", async () => {
|
||||
const transform = createPiiSseTransform({ windowSize: 3 });
|
||||
const writer = transform.writable.getWriter();
|
||||
const chunks: string[] = [];
|
||||
const reader = transform.readable.getReader();
|
||||
|
||||
const readPromise = (async () => {
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(new TextDecoder().decode(value));
|
||||
}
|
||||
})();
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
const payload = JSON.stringify({ choices: [{ delta: { content: "Hello" }, finish_reason: "stop" }] });
|
||||
await writer.write(encoder.encode(`data: ${payload}\n`));
|
||||
await writer.close();
|
||||
await readPromise;
|
||||
|
||||
const fullOutput = chunks.join("");
|
||||
console.log("FULL OUTPUT:", JSON.stringify(fullOutput));
|
||||
assert.ok(fullOutput.includes("Hello"), "Should contain the full buffered text");
|
||||
// It should be valid JSON
|
||||
const lines = fullOutput.split("\n").filter(l => l.startsWith("data: ") && !l.includes("[DONE]"));
|
||||
const parsed = JSON.parse(lines[0].replace("data: ", ""));
|
||||
assert.equal(parsed.choices[0].delta.content, "Hello");
|
||||
});
|
||||
196
tests/unit/streamingPiiTransform.test.ts
Normal file
196
tests/unit/streamingPiiTransform.test.ts
Normal file
@@ -0,0 +1,196 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
// Isolate DB state
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-test-streaming-pii-"));
|
||||
process.env.DATA_DIR = tmpDir;
|
||||
|
||||
// Enable the feature flag for tests
|
||||
const originalEnv = process.env.PII_RESPONSE_SANITIZATION;
|
||||
process.env.PII_RESPONSE_SANITIZATION = "true";
|
||||
process.env.PII_TEST_BYPASS_MIN_WINDOW = "true";
|
||||
|
||||
import { createPiiSseTransform } from "../../src/lib/streamingPiiTransform.ts";
|
||||
|
||||
async function testTransform(transform: TransformStream, inputChunks: string[]): Promise<string> {
|
||||
const writer = transform.writable.getWriter();
|
||||
const reader = transform.readable.getReader();
|
||||
|
||||
const writePromise = (async () => {
|
||||
for (const chunk of inputChunks) {
|
||||
await writer.write(new TextEncoder().encode(chunk));
|
||||
}
|
||||
await writer.close();
|
||||
})();
|
||||
|
||||
const outputChunks: string[] = [];
|
||||
let res = await reader.read();
|
||||
while (!res.done) {
|
||||
outputChunks.push(new TextDecoder().decode(res.value));
|
||||
res = await reader.read();
|
||||
}
|
||||
|
||||
await writePromise;
|
||||
return outputChunks.join("");
|
||||
}
|
||||
|
||||
test("createPiiSseTransform returns a TransformStream", () => {
|
||||
const transform = createPiiSseTransform();
|
||||
assert.ok(transform instanceof TransformStream);
|
||||
});
|
||||
|
||||
test("createPiiSseTransform redacts email in delta.content", async () => {
|
||||
const transform = createPiiSseTransform();
|
||||
|
||||
const input = `data: {"choices":[{"delta":{"content":"email is john@example.com ok"}}]}\n\n`;
|
||||
const output = await testTransform(transform, [input]);
|
||||
|
||||
// Should NOT contain the raw email
|
||||
assert.ok(!output.includes("john@example.com"),
|
||||
"raw email should be redacted from output");
|
||||
// Should contain some form of redaction marker
|
||||
assert.ok(output.includes("REDACTED") || output.includes("[EMAIL"),
|
||||
"output should contain redaction marker");
|
||||
});
|
||||
|
||||
test("createPiiSseTransform passes non-PII content through unchanged", async () => {
|
||||
const transform = createPiiSseTransform();
|
||||
|
||||
const input = `data: {"choices":[{"delta":{"content":"hello world no secrets here"}}]}\n\n`;
|
||||
const output = await testTransform(transform, [input]);
|
||||
|
||||
assert.ok(output.includes("hello world no secrets here"),
|
||||
"non-PII content should pass through unchanged");
|
||||
});
|
||||
|
||||
test("createPiiSseTransform redacts PII split across chunk boundaries", async () => {
|
||||
const transform = createPiiSseTransform();
|
||||
|
||||
const chunk1 = `data: {"choices":[{"delta":{"content":"email is john@"}}]}\n\n`;
|
||||
const chunk2 = `data: {"choices":[{"delta":{"content":"example.com"}}]}\n\n`;
|
||||
|
||||
const output = await testTransform(transform, [chunk1, chunk2]);
|
||||
|
||||
assert.ok(!output.includes("john@example.com"),
|
||||
"email split across chunks should be redacted");
|
||||
assert.ok(output.includes("REDACTED") || output.includes("[EMAIL"),
|
||||
"redaction marker should be present in final stream");
|
||||
});
|
||||
|
||||
test("createPiiSseTransform flushes final redacted content before [DONE] sentinel", async () => {
|
||||
const transform = createPiiSseTransform();
|
||||
|
||||
const chunk1 = `data: {"choices":[{"delta":{"content":"my email is john@"}}]}\n\n`;
|
||||
const chunk2 = `data: {"choices":[{"delta":{"content":"example.com"}}]}\n\n`;
|
||||
const chunk3 = `data: [DONE]\n\n`;
|
||||
|
||||
const writer = transform.writable.getWriter();
|
||||
const reader = transform.readable.getReader();
|
||||
|
||||
const writePromise = (async () => {
|
||||
await writer.write(new TextEncoder().encode(chunk1));
|
||||
await writer.write(new TextEncoder().encode(chunk2));
|
||||
await writer.write(new TextEncoder().encode(chunk3));
|
||||
await writer.close();
|
||||
})();
|
||||
|
||||
const outputChunks: string[] = [];
|
||||
let res = await reader.read();
|
||||
while (!res.done) {
|
||||
outputChunks.push(new TextDecoder().decode(res.value));
|
||||
res = await reader.read();
|
||||
}
|
||||
await writePromise;
|
||||
|
||||
const fullOutput = outputChunks.join("");
|
||||
const lines = fullOutput.split("\n").map(l => l.trim()).filter(Boolean);
|
||||
|
||||
const doneIndex = lines.findIndex(l => l === "data: [DONE]");
|
||||
assert.ok(doneIndex !== -1, "[DONE] sentinel should be in the stream");
|
||||
|
||||
assert.equal(doneIndex, lines.length - 1, "nothing should be enqueued after the [DONE] sentinel");
|
||||
|
||||
const redactedLine = lines.find((l, idx) => idx < doneIndex && (l.includes("REDACTED") || l.includes("[EMAIL")));
|
||||
assert.ok(redactedLine, "redacted content chunk should be enqueued before the [DONE] sentinel");
|
||||
});
|
||||
|
||||
test("content flushed when last chunk is metadata-only (no delta.content)", async () => {
|
||||
const transform = createPiiSseTransform();
|
||||
|
||||
const chunk1 = `data: {"choices":[{"delta":{"content":"hello world"}}]}\n\n`;
|
||||
const chunk2 = `data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\n`;
|
||||
const chunk3 = `data: [DONE]\n\n`;
|
||||
|
||||
const output = await testTransform(transform, [chunk1, chunk2, chunk3]);
|
||||
|
||||
assert.ok(output.includes("hello world"),
|
||||
"buffered content must be flushed even when last chunk has no delta.content");
|
||||
});
|
||||
|
||||
test("no duplicate content when stream has [DONE] and normal close", async () => {
|
||||
const transform = createPiiSseTransform();
|
||||
|
||||
const chunk1 = `data: {"choices":[{"delta":{"content":"test content here"}}]}\n\n`;
|
||||
const chunk2 = `data: [DONE]\n\n`;
|
||||
|
||||
const output = await testTransform(transform, [chunk1, chunk2]);
|
||||
|
||||
// Count occurrences of the content
|
||||
const matches = output.match(/test content here/g) || [];
|
||||
assert.ok(matches.length <= 1, "content should not be duplicated by double-flush");
|
||||
});
|
||||
|
||||
test("configurable windowSize is respected", async () => {
|
||||
// We need to type the options manually in the test since the signature hasn't changed yet,
|
||||
// or cast it. createPiiSseTransform doesn't take args yet, so passing an arg will be ignored
|
||||
// until we change the signature. But in TypeScript it might error if we pass an arg.
|
||||
// Wait, I will just cast it as any.
|
||||
const transform = (createPiiSseTransform as any)({ windowSize: 10 });
|
||||
|
||||
// Send 20 chars of content
|
||||
const input = `data: {"choices":[{"delta":{"content":"abcdefghijklmnopqrst"}}]}\n\n`;
|
||||
const chunk2 = `data: [DONE]\n\n`;
|
||||
const output = await testTransform(transform, [input, chunk2]);
|
||||
|
||||
// All 20 chars should appear in the final output (10 emitted, 10 flushed)
|
||||
assert.ok(output.includes("abcdefghij"), "first 10 chars should be emitted immediately");
|
||||
assert.ok(output.includes("klmnopqrst"), "last 10 chars should be flushed");
|
||||
});
|
||||
|
||||
test("Gemini format PII redaction", async () => {
|
||||
const transform = createPiiSseTransform();
|
||||
|
||||
const input = `data: {"candidates":[{"content":{"parts":[{"text":"email is john@example.com"}]}}]}\n\n`;
|
||||
const done = `data: [DONE]\n\n`;
|
||||
const output = await testTransform(transform, [input, done]);
|
||||
|
||||
assert.ok(!output.includes("john@example.com"), "email should be redacted in Gemini format");
|
||||
});
|
||||
|
||||
test("PII split across sliding window boundary is still redacted", async () => {
|
||||
const transform = (createPiiSseTransform as any)({ windowSize: 10 });
|
||||
|
||||
// Email is 20 chars, window is 10 — the email straddles the boundary
|
||||
const chunk1 = `data: {"choices":[{"delta":{"content":"contact user@"}}]}\n\n`;
|
||||
const chunk2 = `data: {"choices":[{"delta":{"content":"example.com today"}}]}\n\n`;
|
||||
const done = `data: [DONE]\n\n`;
|
||||
const output = await testTransform(transform, [chunk1, chunk2, done]);
|
||||
|
||||
assert.ok(!output.includes("user@example.com"),
|
||||
"email spanning window boundary should be redacted");
|
||||
});
|
||||
|
||||
test.after(async () => {
|
||||
if (originalEnv !== undefined) {
|
||||
process.env.PII_RESPONSE_SANITIZATION = originalEnv;
|
||||
} else {
|
||||
delete process.env.PII_RESPONSE_SANITIZATION;
|
||||
}
|
||||
|
||||
const coreDb = await import("../../src/lib/db/core.ts");
|
||||
coreDb.resetDbInstance();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
Reference in New Issue
Block a user