Test: Scan compose secrets by data shape
This commit is contained in:
Gabriel Brown
@@ -24,36 +24,31 @@ note() { findings+=("$1"); }
|
||||
|
||||
[[ -d "$server_dir" ]] || { printf 'compose secrets contract: no server/ directory\n' >&2; exit 1; }
|
||||
|
||||
# ── 1. Tracked content is clean ──────────────────────────────────────────────
|
||||
#
|
||||
# Only tracked files: the live .env a cutover briefly leaves in a service
|
||||
# directory is exactly what the gitignore exists for, and flagging it here
|
||||
# would punish the ignore for working.
|
||||
# ── 1. Scanner fixtures and tracked content ──────────────────────────────────
|
||||
|
||||
while IFS= read -r file; do
|
||||
path="$repo_dir/$file"
|
||||
[[ -f "$path" ]] || continue
|
||||
scanner="$repo_dir/tests/server/scan-tracked-secrets.py"
|
||||
fixtures_dir="$repo_dir/tests/server/fixtures/secrets"
|
||||
|
||||
# A secret-bearing key with a literal value. ${VAR} interpolations, empty
|
||||
# values, the CHANGE_ME placeholder, and booleans (ALLOW_EMPTY_PASSWORD=yes
|
||||
# is a switch, not a credential) are the allowed shapes; anything else
|
||||
# after PASSWORD/SECRET/TOKEN/KEY is treated as a leak. Keys that merely
|
||||
# configure where a secret lives (a *_FILE path, a key NAME) are not
|
||||
# values.
|
||||
if ! python3 "$scanner" "$fixtures_dir/clean" compose.yml .env.example README.md; then
|
||||
note 'the clean secret-scanning fixture was rejected'
|
||||
fi
|
||||
|
||||
for fixture in compose.yml .env.example; do
|
||||
if output="$(python3 "$scanner" "$fixtures_dir/leaked" "$fixture" 2>&1)"; then
|
||||
note "the leaked $fixture fixture was accepted"
|
||||
elif [[ "$fixture" == compose.yml && "$output" != *'POSTGRES_PASSWORD'* ]]; then
|
||||
note 'the leaked compose fixture did not name POSTGRES_PASSWORD'
|
||||
elif [[ "$fixture" == .env.example && "$output" != *'API_TOKEN'* ]]; then
|
||||
note 'the leaked env fixture did not name API_TOKEN'
|
||||
fi
|
||||
done
|
||||
|
||||
mapfile -t tracked_server_files < <(git -C "$repo_dir" ls-files 'server/**' 'server/*')
|
||||
if ! output="$(python3 "$scanner" "$repo_dir" "${tracked_server_files[@]}" 2>&1)"; then
|
||||
while IFS= read -r hit; do
|
||||
note "$file looks like it carries a secret: ${hit%%[=:]*}"
|
||||
done < <(grep -inE '(password|secret|token|api_key|private_key|access_key)[a-z0-9_]*[[:space:]]*[:=]' "$path" 2>/dev/null \
|
||||
| grep -vE '[:=][[:space:]]*["'"'"']?(\$\{|CHANGE_ME|(true|false|yes|no|[01])["'"'"']?[[:space:]]*$|["'"'"']?[[:space:]]*$)' \
|
||||
| grep -viE '(_file|_path|_name|_key_name)[[:space:]]*[:=]' \
|
||||
| grep -vE '^[0-9]+:[[:space:]]*#')
|
||||
|
||||
if grep -qE 'BEGIN [A-Z ]*PRIVATE KEY' "$path" 2>/dev/null; then
|
||||
note "$file contains a private key"
|
||||
fi
|
||||
if grep -qE 'sk-ant-[A-Za-z0-9]|ghp_[A-Za-z0-9]{20}|xox[baprs]-[A-Za-z0-9]' "$path" 2>/dev/null; then
|
||||
note "$file contains something that looks like an API token"
|
||||
fi
|
||||
done < <(git -C "$repo_dir" ls-files 'server/')
|
||||
[[ -n "$hit" ]] && note "$hit"
|
||||
done <<< "$output"
|
||||
fi
|
||||
|
||||
# ── 2. The ignore still stands ───────────────────────────────────────────────
|
||||
#
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
POSTGRES_PASSWORD=CHANGE_ME
|
||||
API_TOKEN=
|
||||
ALLOW_EMPTY_PASSWORD=yes
|
||||
FEATURE_SECRET_ENABLED=false
|
||||
PRIVATE_KEY_PATH=/run/secrets/private_key
|
||||
@@ -0,0 +1,5 @@
|
||||
This prose is not a Compose assignment: password: example.
|
||||
It also says secret key without defining one.
|
||||
The server README sentence that triggered the audit is reproduced here:
|
||||
under `server/` carries anything that looks like a secret: this repository is
|
||||
public, and the gitignore is a seatbelt, not the brakes.
|
||||
@@ -0,0 +1,8 @@
|
||||
services:
|
||||
database:
|
||||
environment:
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
API_TOKEN: ${API_TOKEN:-CHANGE_ME}
|
||||
ALLOW_EMPTY_PASSWORD: yes
|
||||
DATABASE_NAME: application
|
||||
@@ -0,0 +1 @@
|
||||
API_TOKEN=fixture-should-be-rejected
|
||||
@@ -0,0 +1,4 @@
|
||||
services:
|
||||
database:
|
||||
environment:
|
||||
POSTGRES_PASSWORD: fixture-should-be-rejected
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Scan tracked Compose and env files for literal secrets.
|
||||
|
||||
The parser intentionally understands only the assignment forms this repository
|
||||
uses. Markdown and other prose are only checked for unmistakable token and PEM
|
||||
private-key signatures.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
|
||||
ENV_FILENAMES = {".env", ".env.example"}
|
||||
YAML_SUFFIXES = {".yaml", ".yml"}
|
||||
IGNORED_KEY_SUFFIXES = ("_FILE", "_PATH", "_NAME", "_KEY_NAME")
|
||||
SECRET_KEY_PARTS = ("PASSWORD", "SECRET", "TOKEN", "API_KEY", "PRIVATE_KEY", "ACCESS_KEY")
|
||||
BOOLEAN_VALUES = {"true", "false", "yes", "no", "0", "1"}
|
||||
ENV_ASSIGNMENT = re.compile(r"^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$")
|
||||
YAML_MAPPING = re.compile(r"^(\s*)([A-Za-z_][A-Za-z0-9_.-]*)\s*:\s*(.*?)\s*$")
|
||||
YAML_ENV_ITEM = re.compile(r"^(\s*)-\s+([A-Za-z_][A-Za-z0-9_]*)=(.*?)\s*$")
|
||||
PRIVATE_KEY_HEADER = re.compile(r"BEGIN [A-Z ]*PRIVATE KEY")
|
||||
PROVIDER_TOKEN = re.compile(r"sk-ant-[A-Za-z0-9]|ghp_[A-Za-z0-9]{20}|xox[baprs]-[A-Za-z0-9]")
|
||||
|
||||
|
||||
def strip_comment(value: str) -> str:
|
||||
"""Remove a YAML-style comment while preserving quoted values."""
|
||||
quote = ""
|
||||
escaped = False
|
||||
for index, character in enumerate(value):
|
||||
if escaped:
|
||||
escaped = False
|
||||
continue
|
||||
if quote == '"' and character == "\\":
|
||||
escaped = True
|
||||
continue
|
||||
if character in {"'", '"'}:
|
||||
if not quote:
|
||||
quote = character
|
||||
elif quote == character:
|
||||
quote = ""
|
||||
continue
|
||||
if character == "#" and not quote and (index == 0 or value[index - 1].isspace()):
|
||||
return value[:index].rstrip()
|
||||
return value.rstrip()
|
||||
|
||||
|
||||
def normalize_value(value: str) -> str:
|
||||
value = strip_comment(value).strip()
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
|
||||
return value[1:-1].strip()
|
||||
return value
|
||||
|
||||
|
||||
def is_secret_key(key: str) -> bool:
|
||||
normalized = key.upper()
|
||||
if normalized.endswith(IGNORED_KEY_SUFFIXES):
|
||||
return False
|
||||
return any(part in normalized for part in SECRET_KEY_PARTS)
|
||||
|
||||
|
||||
def is_placeholder(value: str) -> bool:
|
||||
normalized = normalize_value(value)
|
||||
if not normalized or normalized.upper() == "CHANGE_ME":
|
||||
return True
|
||||
if normalized.lower() in BOOLEAN_VALUES | {"null", "~"}:
|
||||
return True
|
||||
return bool(re.fullmatch(r"\$\{[^}\n]+\}", normalized))
|
||||
|
||||
|
||||
def semantic_findings(path: str, content: str) -> Iterable[str]:
|
||||
if Path(path).name in ENV_FILENAMES:
|
||||
for line_number, line in enumerate(content.splitlines(), start=1):
|
||||
match = ENV_ASSIGNMENT.match(strip_comment(line))
|
||||
if match and is_secret_key(match.group(1)) and not is_placeholder(match.group(2)):
|
||||
yield f"{path}:{line_number}: {match.group(1)}"
|
||||
return
|
||||
|
||||
if Path(path).suffix.lower() not in YAML_SUFFIXES:
|
||||
return
|
||||
|
||||
environment_indents: list[int] = []
|
||||
for line_number, line in enumerate(content.splitlines(), start=1):
|
||||
statement = strip_comment(line)
|
||||
if not statement.strip():
|
||||
continue
|
||||
|
||||
mapping = YAML_MAPPING.match(statement)
|
||||
item = YAML_ENV_ITEM.match(statement)
|
||||
indent = len((mapping or item).group(1)) if mapping or item else len(statement) - len(statement.lstrip())
|
||||
environment_indents = [depth for depth in environment_indents if indent > depth]
|
||||
|
||||
if mapping:
|
||||
key, value = mapping.group(2), mapping.group(3)
|
||||
if key.lower() == "environment" and not value.strip():
|
||||
environment_indents.append(indent)
|
||||
if is_secret_key(key) and not is_placeholder(value):
|
||||
yield f"{path}:{line_number}: {key}"
|
||||
elif item and environment_indents:
|
||||
key, value = item.group(2), item.group(3)
|
||||
if is_secret_key(key) and not is_placeholder(value):
|
||||
yield f"{path}:{line_number}: {key}"
|
||||
|
||||
|
||||
def signature_findings(path: str, content: str) -> Iterable[str]:
|
||||
for line_number, line in enumerate(content.splitlines(), start=1):
|
||||
if PRIVATE_KEY_HEADER.search(line):
|
||||
yield f"{path}:{line_number}: private key"
|
||||
if PROVIDER_TOKEN.search(line):
|
||||
yield f"{path}:{line_number}: provider token"
|
||||
|
||||
|
||||
def scan(root: Path, paths: Iterable[str]) -> list[str]:
|
||||
findings: list[str] = []
|
||||
for path in paths:
|
||||
candidate = Path(path)
|
||||
file_path = candidate if candidate.is_absolute() else root / candidate
|
||||
if not file_path.is_file():
|
||||
continue
|
||||
content = file_path.read_text(encoding="utf-8", errors="replace")
|
||||
findings.extend(semantic_findings(path, content))
|
||||
findings.extend(signature_findings(path, content))
|
||||
return findings
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) < 2:
|
||||
print("usage: scan-tracked-secrets.py ROOT PATH [PATH ...]", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
findings = scan(Path(argv[0]), argv[1:])
|
||||
for finding in findings:
|
||||
print(finding)
|
||||
return 1 if findings else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
Reference in New Issue
Block a user