Test: Scan compose secrets by data shape

tests/server/scan-tracked-secrets.py

@@ -0,0 +1,140 @@
+"""Scan tracked Compose and env files for literal secrets.
+
+The parser intentionally understands only the assignment forms this repository
+uses. Markdown and other prose are only checked for unmistakable token and PEM
+private-key signatures.
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+from typing import Iterable
+
+
+ENV_FILENAMES = {".env", ".env.example"}
+YAML_SUFFIXES = {".yaml", ".yml"}
+IGNORED_KEY_SUFFIXES = ("_FILE", "_PATH", "_NAME", "_KEY_NAME")
+SECRET_KEY_PARTS = ("PASSWORD", "SECRET", "TOKEN", "API_KEY", "PRIVATE_KEY", "ACCESS_KEY")
+BOOLEAN_VALUES = {"true", "false", "yes", "no", "0", "1"}
+ENV_ASSIGNMENT = re.compile(r"^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$")
+YAML_MAPPING = re.compile(r"^(\s*)([A-Za-z_][A-Za-z0-9_.-]*)\s*:\s*(.*?)\s*$")
+YAML_ENV_ITEM = re.compile(r"^(\s*)-\s+([A-Za-z_][A-Za-z0-9_]*)=(.*?)\s*$")
+PRIVATE_KEY_HEADER = re.compile(r"BEGIN [A-Z ]*PRIVATE KEY")
+PROVIDER_TOKEN = re.compile(r"sk-ant-[A-Za-z0-9]|ghp_[A-Za-z0-9]{20}|xox[baprs]-[A-Za-z0-9]")
+
+
+def strip_comment(value: str) -> str:
+    """Remove a YAML-style comment while preserving quoted values."""
+    quote = ""
+    escaped = False
+    for index, character in enumerate(value):
+        if escaped:
+            escaped = False
+            continue
+        if quote == '"' and character == "\\":
+            escaped = True
+            continue
+        if character in {"'", '"'}:
+            if not quote:
+                quote = character
+            elif quote == character:
+                quote = ""
+            continue
+        if character == "#" and not quote and (index == 0 or value[index - 1].isspace()):
+            return value[:index].rstrip()
+    return value.rstrip()
+
+
+def normalize_value(value: str) -> str:
+    value = strip_comment(value).strip()
+    if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
+        return value[1:-1].strip()
+    return value
+
+
+def is_secret_key(key: str) -> bool:
+    normalized = key.upper()
+    if normalized.endswith(IGNORED_KEY_SUFFIXES):
+        return False
+    return any(part in normalized for part in SECRET_KEY_PARTS)
+
+
+def is_placeholder(value: str) -> bool:
+    normalized = normalize_value(value)
+    if not normalized or normalized.upper() == "CHANGE_ME":
+        return True
+    if normalized.lower() in BOOLEAN_VALUES | {"null", "~"}:
+        return True
+    return bool(re.fullmatch(r"\$\{[^}\n]+\}", normalized))
+
+
+def semantic_findings(path: str, content: str) -> Iterable[str]:
+    if Path(path).name in ENV_FILENAMES:
+        for line_number, line in enumerate(content.splitlines(), start=1):
+            match = ENV_ASSIGNMENT.match(strip_comment(line))
+            if match and is_secret_key(match.group(1)) and not is_placeholder(match.group(2)):
+                yield f"{path}:{line_number}: {match.group(1)}"
+        return
+
+    if Path(path).suffix.lower() not in YAML_SUFFIXES:
+        return
+
+    environment_indents: list[int] = []
+    for line_number, line in enumerate(content.splitlines(), start=1):
+        statement = strip_comment(line)
+        if not statement.strip():
+            continue
+
+        mapping = YAML_MAPPING.match(statement)
+        item = YAML_ENV_ITEM.match(statement)
+        indent = len((mapping or item).group(1)) if mapping or item else len(statement) - len(statement.lstrip())
+        environment_indents = [depth for depth in environment_indents if indent > depth]
+
+        if mapping:
+            key, value = mapping.group(2), mapping.group(3)
+            if key.lower() == "environment" and not value.strip():
+                environment_indents.append(indent)
+            if is_secret_key(key) and not is_placeholder(value):
+                yield f"{path}:{line_number}: {key}"
+        elif item and environment_indents:
+            key, value = item.group(2), item.group(3)
+            if is_secret_key(key) and not is_placeholder(value):
+                yield f"{path}:{line_number}: {key}"
+
+
+def signature_findings(path: str, content: str) -> Iterable[str]:
+    for line_number, line in enumerate(content.splitlines(), start=1):
+        if PRIVATE_KEY_HEADER.search(line):
+            yield f"{path}:{line_number}: private key"
+        if PROVIDER_TOKEN.search(line):
+            yield f"{path}:{line_number}: provider token"
+
+
+def scan(root: Path, paths: Iterable[str]) -> list[str]:
+    findings: list[str] = []
+    for path in paths:
+        candidate = Path(path)
+        file_path = candidate if candidate.is_absolute() else root / candidate
+        if not file_path.is_file():
+            continue
+        content = file_path.read_text(encoding="utf-8", errors="replace")
+        findings.extend(semantic_findings(path, content))
+        findings.extend(signature_findings(path, content))
+    return findings
+
+
+def main(argv: list[str]) -> int:
+    if len(argv) < 2:
+        print("usage: scan-tracked-secrets.py ROOT PATH [PATH ...]", file=sys.stderr)
+        return 2
+
+    findings = scan(Path(argv[0]), argv[1:])
+    for finding in findings:
+        print(finding)
+    return 1 if findings else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))