diff --git a/scripts/sre/redact-logs.mjs b/scripts/sre/redact-logs.mjs index 88f9806234..6b1384529b 100644 --- a/scripts/sre/redact-logs.mjs +++ b/scripts/sre/redact-logs.mjs @@ -1,284 +1,289 @@ -#!/usr/bin/env node -/** - * PII redaction for log shipping. - * - * Streams input (stdin or files) to stdout, replacing sensitive tokens - * with stable redaction markers. Pure Node.js stdlib — no `npm install`. - * - * Recognised patterns (in order, longest match wins per position): - * - * 1. Anthropic API keys (sk-ant-...) → [REDACTED_API_KEY] - * 2. Google API keys (AIza...) → [REDACTED_API_KEY] - * 3. GitHub tokens (ghp_/gho_/ghu_/...) → [REDACTED_API_KEY] - * 4. OpenAI keys (sk-..., sk-proj-...) → [REDACTED_API_KEY] - * 5. AWS access keys (AKIA/ASIA) → [REDACTED_AWS_KEY] - * 6. Bearer tokens → [REDACTED_BEARER] - * 7. Email addresses → [REDACTED_EMAIL] - * 8. Generic api_key=value pairs → [REDACTED_API_KEY] - * 9. IPv4 addresses → [REDACTED_IPV4] - * 10. IPv6 addresses → [REDACTED_IPV6] - * - * Provider-specific patterns are listed BEFORE the generic `sk-` rule so - * that an `sk-ant-...` key counts as `ANTHROPIC_KEY` (not `OPENAI_KEY`) - * for the per-call summary. - * - * Why stable markers: downstream log-search queries reference the - * redaction markers (e.g. "show me all log lines with [REDACTED_IPV4]"), - * which makes the redaction reversible by anyone with the original - * vault lookup, but never by a log-search reader alone. - * - * CLI: - * node scripts/sre/redact-logs.mjs < input.log > output.log - * node scripts/sre/redact-logs.mjs --file access.log --output out.log - * node scripts/sre/redact-logs.mjs --strict # exit non-zero on any match - * - * Library: - * import { redact, redactString, RedactTransform } from "./scripts/sre/redact-logs.mjs"; - * - * Salvaged from closed PR #5057 (base-stale; reimplemented on release). - */ - -import { createReadStream, createWriteStream } from "node:fs"; -import { TransformStream } from "node:stream/web"; -import process from "node:process"; - -// ── Pattern catalogue ───────────────────────────────────────────────────────── -// -// Each pattern is [name, regex, marker]. The regex uses the `g` flag so we can -// iterate with `matchAll`. Order matters: longer / more specific patterns go -// first so an `sk-ant-...` key counts as ANTHROPIC_KEY instead of OPENAI_KEY. - -export const REDACT_PATTERNS = Object.freeze([ - // 1. Anthropic keys: sk-ant-api03-... / sk-ant-... (must beat the generic sk-) - [ - "ANTHROPIC_KEY", - /\bsk-ant-[A-Za-z0-9_\-]{20,}\b/g, - "[REDACTED_API_KEY]", - ], - // 2. Google API keys: AIza... (39 chars total) - [ - "GOOGLE_KEY", - /\bAIza[A-Za-z0-9_\-]{35}\b/g, - "[REDACTED_API_KEY]", - ], - // 3. GitHub tokens (classic + fine-grained + PAT prefixes) - [ - "GITHUB_TOKEN", - /\b(?:ghp|gho|ghu|ghs|ghr|github_pat)_[A-Za-z0-9]{30,}\b/g, - "[REDACTED_API_KEY]", - ], - // 4. OpenAI keys: sk-..., sk-proj-..., proj-... (after the more specific rules above) - [ - "OPENAI_KEY", - /\bsk-(?:proj-)?[A-Za-z0-9_\-]{20,}\b|\bproj-[A-Za-z0-9_\-]{20,}\b/g, - "[REDACTED_API_KEY]", - ], - // 5. AWS access keys — AKIA / ASIA prefixes, 20 chars total - [ - "AWS_KEY", - /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g, - "[REDACTED_AWS_KEY]", - ], - // 6. Bearer tokens — Authorization: Bearer xxxx (16+ chars) - [ - "BEARER", - /(?:Bearer|Authorization:\s*Bearer)\s+([A-Za-z0-9._\-+/=]{16,})/g, - "[REDACTED_BEARER]", - ], - // 7. Email — RFC 5322-ish; rejects obvious junk but stays compact. - [ - "EMAIL", - /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,24}\b/g, - "[REDACTED_EMAIL]", - ], - // 8. Generic api_key / apiKey / password= value pairs (12+ char secret) - [ - "GENERIC_KEY", - /\b(?:api[_-]?key|apikey|password|passwd|pwd|secret|token|auth)\s*[:=]\s*['"]?([A-Za-z0-9._\-+/=]{12,})['"]?/gi, - "[REDACTED_API_KEY]", - ], - // 9. IPv4 (incl. port) — strict octet bounds - [ - "IPV4", - /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d?\d)(?::\d{1,5})?\b/g, - "[REDACTED_IPV4]", - ], - // 10. IPv6 — full, compressed, ::1, ::ffff:1.2.3.4 - // Three alternative shapes: - // (a) full 8-group form (no `::`), - // (b) compressed form with `::` somewhere, - // (c) `::1` / `::` alone anchored by a non-hex lookbehind so it - // doesn't greedily extend `fe80::` into `fe80::foo`. - [ - "IPV6", - new RegExp( - [ - // (a) Full 8-group: 1:2:3:4:5:6:7:8 - "\\b(?:[A-Fa-f0-9]{1,4}:){7}[A-Fa-f0-9]{1,4}\\b", - // (b) Compressed with `::` somewhere in the middle (left side 1+ groups) - "\\b(?:[A-Fa-f0-9]{1,4}:){1,6}[A-Fa-f0-9]{1,4}::[A-Fa-f0-9]{1,4}(?::[A-Fa-f0-9]{1,4}){0,6}\\b", - "\\b(?:[A-Fa-f0-9]{1,4}:){1,7}:[A-Fa-f0-9]{1,4}(?::[A-Fa-f0-9]{1,4}){0,6}\\b", - // (c) Leading `::` (no left side): ::1, ::1:2, ::ffff:1.2.3.4 - "(? }} - */ -export function redactString(input) { - if (typeof input !== "string" || input.length === 0) { - return { output: input ?? "", counts: {} }; - } - const counts = {}; - let output = input; - for (const [name, regex, marker] of REDACT_PATTERNS) { - output = output.replace(regex, () => { - counts[name] = (counts[name] ?? 0) + 1; - return marker; - }); - } - return { output, counts }; -} - -/** - * Redact a single line. Convenience wrapper around redactString that does - * not allocate an intermediate object for the counts. - * - * @param {string} line - * @returns {string} - */ -export function redact(line) { - return redactString(line).output; -} - -/** - * Redact TransformStream. - * - * Implements the WHATWG TransformStream API so callers can do: - * await src.pipeThrough(new TextDecoderStream()).pipeThrough(new RedactTransform()).pipeTo(sink) - * - * Counts are accumulated on the stream instance (`.counts`). - */ -export class RedactTransform extends TransformStream { - constructor() { - const counts = {}; - super({ - transform(chunk, controller) { - const text = typeof chunk === "string" ? chunk : new TextDecoder("utf-8").decode(chunk); - const { output, counts: localCounts } = redactString(text); - for (const [name, n] of Object.entries(localCounts)) { - counts[name] = (counts[name] ?? 0) + n; - } - controller.enqueue(new TextEncoder().encode(output)); - }, - }); - // Attach counts as an enumerable own property so tests can read it. - Object.defineProperty(this, "counts", { - value: counts, - writable: false, - enumerable: true, - configurable: false, - }); - } -} - -// ── CLI ────────────────────────────────────────────────────────────────────── - -function parseArgs(argv) { - const args = { files: [], output: null, strict: false, help: false }; - for (let i = 2; i < argv.length; i += 1) { - const a = argv[i]; - if (a === "--file") { - args.files.push(argv[++i]); - } else if (a === "--output" || a === "-o") { - args.output = argv[++i]; - } else if (a === "--strict") { - args.strict = true; - } else if (a === "--help" || a === "-h") { - args.help = true; - } else { - process.stderr.write(`unknown argument: ${a}\n`); - process.exit(2); - } - } - return args; -} - -function printHelp() { - process.stdout.write(`Usage: redact-logs.mjs [options] - -Options: - --file Read from file (repeatable). Defaults to stdin. - --output, -o

Write to file. Defaults to stdout. - --strict Exit non-zero if any PII is detected. - --help, -h Show this help. - -Library: - import { redact, redactString, RedactTransform } from "./scripts/sre/redact-logs.mjs"; -`); -} - -async function main() { - const args = parseArgs(process.argv); - if (args.help) { - printHelp(); - return; - } - - const transform = new RedactTransform(); - - // Build a WHATWG ReadableStream from each input source. We open files / - // stdin as a Node Readable and convert it via Readable.toWeb(). - const { Readable } = await import("node:stream"); - const { Writable: WritableStreamWeb } = await import("node:stream/web"); - const sources = args.files.length > 0 ? args.files : ["-"]; - - for (const source of sources) { - const nodeSrc = source === "-" ? process.stdin : createReadStream(source, "utf8"); - const webSrc = Readable.toWeb(nodeSrc); - const webDecoded = webSrc.pipeThrough(new TextDecoderStream("utf-8")); - const webEncoded = webDecoded.pipeThrough(transform); - - const sink = args.output - ? createWriteStream(args.output, "utf8") - : process.stdout; - const webSink = WritableStreamWeb.toWeb(sink); - - try { - await webEncoded.pipeTo(webSink); - } catch (err) { - process.stderr.write(`redact-logs: ${err.message}\n`); - process.exit(1); - } - } - - const totals = transform.counts; - if (Object.keys(totals).length > 0) { - const summary = Object.entries(totals) - .map(([k, v]) => `${k}=${v}`) - .join(" "); - process.stderr.write(`redact-logs: redacted ${summary}\n`); - if (args.strict) { - process.exit(3); - } - } -} - -// Only run as CLI when this module is the entrypoint (not when imported as a -// library). `import.meta.url === pathToFileURL(process.argv[1]).href` is the -// canonical ESM check. -import { pathToFileURL } from "node:url"; -if (import.meta.url === pathToFileURL(process.argv[1]).href) { - main(); -} +#!/usr/bin/env node +/** + * PII redaction for log shipping. + * + * Streams input (stdin or files) to stdout, replacing sensitive tokens + * with stable redaction markers. Pure Node.js stdlib — no `npm install`. + * + * Recognised patterns (in order, longest match wins per position): + * + * 1. Anthropic API keys (sk-ant-...) → [REDACTED_API_KEY] + * 2. Google API keys (AIza...) → [REDACTED_API_KEY] + * 3. GitHub tokens (ghp_/gho_/ghu_/...) → [REDACTED_API_KEY] + * 4. OpenAI keys (sk-..., sk-proj-...) → [REDACTED_API_KEY] + * 5. AWS access keys (AKIA/ASIA) → [REDACTED_AWS_KEY] + * 6. Bearer tokens → [REDACTED_BEARER] + * 7. Email addresses → [REDACTED_EMAIL] + * 8. Generic api_key=value pairs → [REDACTED_API_KEY] + * 9. IPv4 addresses → [REDACTED_IPV4] + * 10. IPv6 addresses → [REDACTED_IPV6] + * + * Provider-specific patterns are listed BEFORE the generic `sk-` rule so + * that an `sk-ant-...` key counts as `ANTHROPIC_KEY` (not `OPENAI_KEY`) + * for the per-call summary. + * + * Why stable markers: downstream log-search queries reference the + * redaction markers (e.g. "show me all log lines with [REDACTED_IPV4]"), + * which makes the redaction reversible by anyone with the original + * vault lookup, but never by a log-search reader alone. + * + * CLI: + * node scripts/sre/redact-logs.mjs < input.log > output.log + * node scripts/sre/redact-logs.mjs --file access.log --output out.log + * node scripts/sre/redact-logs.mjs --strict # exit non-zero on any match + * + * Library: + * import { redact, redactString, RedactTransform } from "./scripts/sre/redact-logs.mjs"; + * + * Salvaged from closed PR #5057 (base-stale; reimplemented on release). + */ + +import { createReadStream, createWriteStream } from "node:fs"; +import { TransformStream } from "node:stream/web"; +import process from "node:process"; + +// ── Pattern catalogue ───────────────────────────────────────────────────────── +// +// Each pattern is [name, regex, marker]. The regex uses the `g` flag so we can +// iterate with `matchAll`. Order matters: longer / more specific patterns go +// first so an `sk-ant-...` key counts as ANTHROPIC_KEY instead of OPENAI_KEY. + +export const REDACT_PATTERNS = Object.freeze([ + // 1. Anthropic keys: sk-ant-api03-... / sk-ant-... (must beat the generic sk-) + ["ANTHROPIC_KEY", /\bsk-ant-[A-Za-z0-9_\-]{20,}\b/g, "[REDACTED_API_KEY]"], + // 2. Google API keys: AIza... (39 chars total) + ["GOOGLE_KEY", /\bAIza[A-Za-z0-9_\-]{35}\b/g, "[REDACTED_API_KEY]"], + // 3. GitHub tokens (classic + fine-grained + PAT prefixes) + [ + "GITHUB_TOKEN", + /\b(?:ghp|gho|ghu|ghs|ghr|github_pat)_[A-Za-z0-9]{30,}\b/g, + "[REDACTED_API_KEY]", + ], + // 4. OpenAI keys: sk-..., sk-proj-..., proj-... (after the more specific rules above) + [ + "OPENAI_KEY", + /\bsk-(?:proj-)?[A-Za-z0-9_\-]{20,}\b|\bproj-[A-Za-z0-9_\-]{20,}\b/g, + "[REDACTED_API_KEY]", + ], + // 5. AWS access keys — AKIA / ASIA prefixes, 20 chars total + ["AWS_KEY", /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g, "[REDACTED_AWS_KEY]"], + // 6. Bearer tokens — Authorization: Bearer xxxx (16+ chars) + [ + "BEARER", + /(?:Bearer|Authorization:\s*Bearer)\s+([A-Za-z0-9._\-+/=]{16,})/g, + "[REDACTED_BEARER]", + ], + // 7. Email — RFC 5322-ish; rejects obvious junk but stays compact. + ["EMAIL", /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,24}\b/g, "[REDACTED_EMAIL]"], + // 8. Generic api_key / apiKey / password= value pairs (12+ char secret) + [ + "GENERIC_KEY", + /\b(?:api[_-]?key|apikey|password|passwd|pwd|secret|token|auth)\s*[:=]\s*['"]?([A-Za-z0-9._\-+/=]{12,})['"]?/gi, + "[REDACTED_API_KEY]", + ], + // 9. IPv4 (incl. port) — strict octet bounds + [ + "IPV4", + /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d?\d)(?::\d{1,5})?\b/g, + "[REDACTED_IPV4]", + ], + // 10. IPv6 — full, compressed, ::1, ::ffff:1.2.3.4 + // Three alternative shapes: + // (a) full 8-group form (no `::`), + // (b) compressed form with `::` somewhere, + // (c) `::1` / `::` alone anchored by a non-hex lookbehind so it + // doesn't greedily extend `fe80::` into `fe80::foo`. + [ + "IPV6", + new RegExp( + [ + // (a) Full 8-group: 1:2:3:4:5:6:7:8 + "\\b(?:[A-Fa-f0-9]{1,4}:){7}[A-Fa-f0-9]{1,4}\\b", + // (b) Compressed with `::` somewhere in the middle (left side 1+ groups) + "\\b(?:[A-Fa-f0-9]{1,4}:){1,6}[A-Fa-f0-9]{1,4}::[A-Fa-f0-9]{1,4}(?::[A-Fa-f0-9]{1,4}){0,6}\\b", + "\\b(?:[A-Fa-f0-9]{1,4}:){1,7}:[A-Fa-f0-9]{1,4}(?::[A-Fa-f0-9]{1,4}){0,6}\\b", + // (c) Leading `::` (no left side): ::1, ::1:2, ::ffff:1.2.3.4 + "(? }} + */ +export function redactString(input) { + if (typeof input !== "string" || input.length === 0) { + return { output: input ?? "", counts: {} }; + } + const counts = {}; + let output = input; + for (const [name, regex, marker] of REDACT_PATTERNS) { + output = output.replace(regex, () => { + counts[name] = (counts[name] ?? 0) + 1; + return marker; + }); + } + return { output, counts }; +} + +/** + * Redact a single line. Convenience wrapper around redactString that does + * not allocate an intermediate object for the counts. + * + * @param {string} line + * @returns {string} + */ +export function redact(line) { + return redactString(line).output; +} + +/** + * Redact TransformStream. + * + * Implements the WHATWG TransformStream API so callers can do: + * await src.pipeThrough(new TextDecoderStream()).pipeThrough(new RedactTransform()).pipeTo(sink) + * + * Counts are accumulated on the stream instance (`.counts`). + */ +export class RedactTransform extends TransformStream { + constructor() { + const counts = {}; + const decoder = new TextDecoder("utf-8"); + const encoder = new TextEncoder(); + let pendingLine = ""; + + const emitRedacted = (text, controller) => { + if (text.length === 0) return; + + const { output, counts: localCounts } = redactString(text); + for (const [name, n] of Object.entries(localCounts)) { + counts[name] = (counts[name] ?? 0) + n; + } + controller.enqueue(encoder.encode(output)); + }; + + super({ + transform(chunk, controller) { + const text = + typeof chunk === "string" + ? decoder.decode() + chunk + : decoder.decode(chunk, { stream: true }); + pendingLine += text; + + const lastNewline = pendingLine.lastIndexOf("\n"); + if (lastNewline >= 0) { + emitRedacted(pendingLine.slice(0, lastNewline + 1), controller); + pendingLine = pendingLine.slice(lastNewline + 1); + } + }, + flush(controller) { + pendingLine += decoder.decode(); + emitRedacted(pendingLine, controller); + }, + }); + // Attach counts as an enumerable own property so tests can read it. + Object.defineProperty(this, "counts", { + value: counts, + writable: false, + enumerable: true, + configurable: false, + }); + } +} + +// ── CLI ────────────────────────────────────────────────────────────────────── + +function parseArgs(argv) { + const args = { files: [], output: null, strict: false, help: false }; + for (let i = 2; i < argv.length; i += 1) { + const a = argv[i]; + if (a === "--file") { + args.files.push(argv[++i]); + } else if (a === "--output" || a === "-o") { + args.output = argv[++i]; + } else if (a === "--strict") { + args.strict = true; + } else if (a === "--help" || a === "-h") { + args.help = true; + } else { + process.stderr.write(`unknown argument: ${a}\n`); + process.exit(2); + } + } + return args; +} + +function printHelp() { + process.stdout.write(`Usage: redact-logs.mjs [options] + +Options: + --file Read from file (repeatable). Defaults to stdin. + --output, -o

Write to file. Defaults to stdout. + --strict Exit non-zero if any PII is detected. + --help, -h Show this help. + +Library: + import { redact, redactString, RedactTransform } from "./scripts/sre/redact-logs.mjs"; +`); +} + +async function main() { + const args = parseArgs(process.argv); + if (args.help) { + printHelp(); + return; + } + + const transform = new RedactTransform(); + + // Build a WHATWG ReadableStream from each input source. We open files / + // stdin as a Node Readable and convert it via Readable.toWeb(). + const { Readable } = await import("node:stream"); + const { Writable: WritableStreamWeb } = await import("node:stream/web"); + const sources = args.files.length > 0 ? args.files : ["-"]; + + for (const source of sources) { + const nodeSrc = source === "-" ? process.stdin : createReadStream(source, "utf8"); + const webSrc = Readable.toWeb(nodeSrc); + const webDecoded = webSrc.pipeThrough(new TextDecoderStream("utf-8")); + const webEncoded = webDecoded.pipeThrough(transform); + + const sink = args.output ? createWriteStream(args.output, "utf8") : process.stdout; + const webSink = WritableStreamWeb.toWeb(sink); + + try { + await webEncoded.pipeTo(webSink); + } catch (err) { + process.stderr.write(`redact-logs: ${err.message}\n`); + process.exit(1); + } + } + + const totals = transform.counts; + if (Object.keys(totals).length > 0) { + const summary = Object.entries(totals) + .map(([k, v]) => `${k}=${v}`) + .join(" "); + process.stderr.write(`redact-logs: redacted ${summary}\n`); + if (args.strict) { + process.exit(3); + } + } +} + +// Only run as CLI when this module is the entrypoint (not when imported as a +// library). `import.meta.url === pathToFileURL(process.argv[1]).href` is the +// canonical ESM check. +import { pathToFileURL } from "node:url"; +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main(); +} diff --git a/tests/unit/sre-redact-logs.test.ts b/tests/unit/sre-redact-logs.test.ts index 6c180fe792..f5fdf7b289 100644 --- a/tests/unit/sre-redact-logs.test.ts +++ b/tests/unit/sre-redact-logs.test.ts @@ -250,6 +250,46 @@ test("RedactTransform: streams input chunks to output, redacting as it goes", as assert.equal(t.counts.IPV4 ?? 0, 1); }); +test("RedactTransform: redacts every sensitive pattern across every chunk boundary", async () => { + const samples = [ + ["ANTHROPIC_KEY", `sk-ant-${"a".repeat(24)}`, "[REDACTED_API_KEY]"], + ["GOOGLE_KEY", `AIza${"A".repeat(35)}`, "[REDACTED_API_KEY]"], + ["GITHUB_TOKEN", `ghp_${"A".repeat(36)}`, "[REDACTED_API_KEY]"], + ["OPENAI_KEY", `sk-proj-${"A".repeat(24)}`, "[REDACTED_API_KEY]"], + ["AWS_KEY", `AKIA${"A".repeat(16)}`, "[REDACTED_AWS_KEY]"], + ["BEARER", `Bearer ${"A".repeat(20)}`, "[REDACTED_BEARER]"], + ["EMAIL", "alice@example.com", "[REDACTED_EMAIL]"], + ["GENERIC_KEY", `password=${"x".repeat(16)}`, "[REDACTED_API_KEY]"], + ["IPV4", "192.168.1.42", "[REDACTED_IPV4]"], + ["IPV6", "2001:0db8:85a3:0000:0000:8a2e:0370:7334", "[REDACTED_IPV6]"], + ]; + + for (const [name, sensitive, marker] of samples) { + for (let offset = 1; offset < sensitive.length; offset += 1) { + const transform = new RedactTransform(); + const output = []; + const source = new ReadableStream({ + start(controller) { + controller.enqueue(sensitive.slice(0, offset)); + controller.enqueue(`${sensitive.slice(offset)}\n`); + controller.close(); + }, + }); + + await source.pipeThrough(transform).pipeTo( + new WritableStream({ + write(chunk) { + output.push(new TextDecoder().decode(chunk)); + }, + }) + ); + + assert.equal(output.join(""), `${marker}\n`, `${name} split at ${offset}`); + assert.equal(transform.counts[name], 1, `${name} count split at ${offset}`); + } + } +}); + // ─── 13. Counts are independent between calls ─────────────────────────────── test("redactString: counts do not bleed across calls", () => {