diff --git a/tests/server/compose-secrets-contract b/tests/server/compose-secrets-contract index 9ed6a44..92c0f46 100755 --- a/tests/server/compose-secrets-contract +++ b/tests/server/compose-secrets-contract @@ -24,36 +24,31 @@ note() { findings+=("$1"); } [[ -d "$server_dir" ]] || { printf 'compose secrets contract: no server/ directory\n' >&2; exit 1; } -# ── 1. Tracked content is clean ────────────────────────────────────────────── -# -# Only tracked files: the live .env a cutover briefly leaves in a service -# directory is exactly what the gitignore exists for, and flagging it here -# would punish the ignore for working. +# ── 1. Scanner fixtures and tracked content ────────────────────────────────── -while IFS= read -r file; do - path="$repo_dir/$file" - [[ -f "$path" ]] || continue +scanner="$repo_dir/tests/server/scan-tracked-secrets.py" +fixtures_dir="$repo_dir/tests/server/fixtures/secrets" - # A secret-bearing key with a literal value. ${VAR} interpolations, empty - # values, the CHANGE_ME placeholder, and booleans (ALLOW_EMPTY_PASSWORD=yes - # is a switch, not a credential) are the allowed shapes; anything else - # after PASSWORD/SECRET/TOKEN/KEY is treated as a leak. Keys that merely - # configure where a secret lives (a *_FILE path, a key NAME) are not - # values. +if ! python3 "$scanner" "$fixtures_dir/clean" compose.yml .env.example README.md; then + note 'the clean secret-scanning fixture was rejected' +fi + +for fixture in compose.yml .env.example; do + if output="$(python3 "$scanner" "$fixtures_dir/leaked" "$fixture" 2>&1)"; then + note "the leaked $fixture fixture was accepted" + elif [[ "$fixture" == compose.yml && "$output" != *'POSTGRES_PASSWORD'* ]]; then + note 'the leaked compose fixture did not name POSTGRES_PASSWORD' + elif [[ "$fixture" == .env.example && "$output" != *'API_TOKEN'* ]]; then + note 'the leaked env fixture did not name API_TOKEN' + fi +done + +mapfile -t tracked_server_files < <(git -C "$repo_dir" ls-files 'server/**' 'server/*') +if ! output="$(python3 "$scanner" "$repo_dir" "${tracked_server_files[@]}" 2>&1)"; then while IFS= read -r hit; do - note "$file looks like it carries a secret: ${hit%%[=:]*}" - done < <(grep -inE '(password|secret|token|api_key|private_key|access_key)[a-z0-9_]*[[:space:]]*[:=]' "$path" 2>/dev/null \ - | grep -vE '[:=][[:space:]]*["'"'"']?(\$\{|CHANGE_ME|(true|false|yes|no|[01])["'"'"']?[[:space:]]*$|["'"'"']?[[:space:]]*$)' \ - | grep -viE '(_file|_path|_name|_key_name)[[:space:]]*[:=]' \ - | grep -vE '^[0-9]+:[[:space:]]*#') - - if grep -qE 'BEGIN [A-Z ]*PRIVATE KEY' "$path" 2>/dev/null; then - note "$file contains a private key" - fi - if grep -qE 'sk-ant-[A-Za-z0-9]|ghp_[A-Za-z0-9]{20}|xox[baprs]-[A-Za-z0-9]' "$path" 2>/dev/null; then - note "$file contains something that looks like an API token" - fi -done < <(git -C "$repo_dir" ls-files 'server/') + [[ -n "$hit" ]] && note "$hit" + done <<< "$output" +fi # ── 2. The ignore still stands ─────────────────────────────────────────────── # diff --git a/tests/server/fixtures/secrets/clean/.env.example b/tests/server/fixtures/secrets/clean/.env.example new file mode 100644 index 0000000..cc6a71d --- /dev/null +++ b/tests/server/fixtures/secrets/clean/.env.example @@ -0,0 +1,5 @@ +POSTGRES_PASSWORD=CHANGE_ME +API_TOKEN= +ALLOW_EMPTY_PASSWORD=yes +FEATURE_SECRET_ENABLED=false +PRIVATE_KEY_PATH=/run/secrets/private_key diff --git a/tests/server/fixtures/secrets/clean/README.md b/tests/server/fixtures/secrets/clean/README.md new file mode 100644 index 0000000..2d2f203 --- /dev/null +++ b/tests/server/fixtures/secrets/clean/README.md @@ -0,0 +1,5 @@ +This prose is not a Compose assignment: password: example. +It also says secret key without defining one. +The server README sentence that triggered the audit is reproduced here: +under `server/` carries anything that looks like a secret: this repository is +public, and the gitignore is a seatbelt, not the brakes. diff --git a/tests/server/fixtures/secrets/clean/compose.yml b/tests/server/fixtures/secrets/clean/compose.yml new file mode 100644 index 0000000..2674c9a --- /dev/null +++ b/tests/server/fixtures/secrets/clean/compose.yml @@ -0,0 +1,8 @@ +services: + database: + environment: + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + API_TOKEN: ${API_TOKEN:-CHANGE_ME} + ALLOW_EMPTY_PASSWORD: yes + DATABASE_NAME: application diff --git a/tests/server/fixtures/secrets/leaked/.env.example b/tests/server/fixtures/secrets/leaked/.env.example new file mode 100644 index 0000000..9250b00 --- /dev/null +++ b/tests/server/fixtures/secrets/leaked/.env.example @@ -0,0 +1 @@ +API_TOKEN=fixture-should-be-rejected diff --git a/tests/server/fixtures/secrets/leaked/compose.yml b/tests/server/fixtures/secrets/leaked/compose.yml new file mode 100644 index 0000000..206202c --- /dev/null +++ b/tests/server/fixtures/secrets/leaked/compose.yml @@ -0,0 +1,4 @@ +services: + database: + environment: + POSTGRES_PASSWORD: fixture-should-be-rejected diff --git a/tests/server/scan-tracked-secrets.py b/tests/server/scan-tracked-secrets.py new file mode 100644 index 0000000..8d80177 --- /dev/null +++ b/tests/server/scan-tracked-secrets.py @@ -0,0 +1,140 @@ +"""Scan tracked Compose and env files for literal secrets. + +The parser intentionally understands only the assignment forms this repository +uses. Markdown and other prose are only checked for unmistakable token and PEM +private-key signatures. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path +from typing import Iterable + + +ENV_FILENAMES = {".env", ".env.example"} +YAML_SUFFIXES = {".yaml", ".yml"} +IGNORED_KEY_SUFFIXES = ("_FILE", "_PATH", "_NAME", "_KEY_NAME") +SECRET_KEY_PARTS = ("PASSWORD", "SECRET", "TOKEN", "API_KEY", "PRIVATE_KEY", "ACCESS_KEY") +BOOLEAN_VALUES = {"true", "false", "yes", "no", "0", "1"} +ENV_ASSIGNMENT = re.compile(r"^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$") +YAML_MAPPING = re.compile(r"^(\s*)([A-Za-z_][A-Za-z0-9_.-]*)\s*:\s*(.*?)\s*$") +YAML_ENV_ITEM = re.compile(r"^(\s*)-\s+([A-Za-z_][A-Za-z0-9_]*)=(.*?)\s*$") +PRIVATE_KEY_HEADER = re.compile(r"BEGIN [A-Z ]*PRIVATE KEY") +PROVIDER_TOKEN = re.compile(r"sk-ant-[A-Za-z0-9]|ghp_[A-Za-z0-9]{20}|xox[baprs]-[A-Za-z0-9]") + + +def strip_comment(value: str) -> str: + """Remove a YAML-style comment while preserving quoted values.""" + quote = "" + escaped = False + for index, character in enumerate(value): + if escaped: + escaped = False + continue + if quote == '"' and character == "\\": + escaped = True + continue + if character in {"'", '"'}: + if not quote: + quote = character + elif quote == character: + quote = "" + continue + if character == "#" and not quote and (index == 0 or value[index - 1].isspace()): + return value[:index].rstrip() + return value.rstrip() + + +def normalize_value(value: str) -> str: + value = strip_comment(value).strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + return value[1:-1].strip() + return value + + +def is_secret_key(key: str) -> bool: + normalized = key.upper() + if normalized.endswith(IGNORED_KEY_SUFFIXES): + return False + return any(part in normalized for part in SECRET_KEY_PARTS) + + +def is_placeholder(value: str) -> bool: + normalized = normalize_value(value) + if not normalized or normalized.upper() == "CHANGE_ME": + return True + if normalized.lower() in BOOLEAN_VALUES | {"null", "~"}: + return True + return bool(re.fullmatch(r"\$\{[^}\n]+\}", normalized)) + + +def semantic_findings(path: str, content: str) -> Iterable[str]: + if Path(path).name in ENV_FILENAMES: + for line_number, line in enumerate(content.splitlines(), start=1): + match = ENV_ASSIGNMENT.match(strip_comment(line)) + if match and is_secret_key(match.group(1)) and not is_placeholder(match.group(2)): + yield f"{path}:{line_number}: {match.group(1)}" + return + + if Path(path).suffix.lower() not in YAML_SUFFIXES: + return + + environment_indents: list[int] = [] + for line_number, line in enumerate(content.splitlines(), start=1): + statement = strip_comment(line) + if not statement.strip(): + continue + + mapping = YAML_MAPPING.match(statement) + item = YAML_ENV_ITEM.match(statement) + indent = len((mapping or item).group(1)) if mapping or item else len(statement) - len(statement.lstrip()) + environment_indents = [depth for depth in environment_indents if indent > depth] + + if mapping: + key, value = mapping.group(2), mapping.group(3) + if key.lower() == "environment" and not value.strip(): + environment_indents.append(indent) + if is_secret_key(key) and not is_placeholder(value): + yield f"{path}:{line_number}: {key}" + elif item and environment_indents: + key, value = item.group(2), item.group(3) + if is_secret_key(key) and not is_placeholder(value): + yield f"{path}:{line_number}: {key}" + + +def signature_findings(path: str, content: str) -> Iterable[str]: + for line_number, line in enumerate(content.splitlines(), start=1): + if PRIVATE_KEY_HEADER.search(line): + yield f"{path}:{line_number}: private key" + if PROVIDER_TOKEN.search(line): + yield f"{path}:{line_number}: provider token" + + +def scan(root: Path, paths: Iterable[str]) -> list[str]: + findings: list[str] = [] + for path in paths: + candidate = Path(path) + file_path = candidate if candidate.is_absolute() else root / candidate + if not file_path.is_file(): + continue + content = file_path.read_text(encoding="utf-8", errors="replace") + findings.extend(semantic_findings(path, content)) + findings.extend(signature_findings(path, content)) + return findings + + +def main(argv: list[str]) -> int: + if len(argv) < 2: + print("usage: scan-tracked-secrets.py ROOT PATH [PATH ...]", file=sys.stderr) + return 2 + + findings = scan(Path(argv[0]), argv[1:]) + for finding in findings: + print(finding) + return 1 if findings else 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:]))