From d267da58ad9280e55e508b1051b5da4fcd7e6fda Mon Sep 17 00:00:00 2001 From: Gabriel Brown Date: Thu, 27 Aug 2026 03:44:57 -0400 Subject: [PATCH] Fix: Roll back failed SSH hardening --- boot | 135 ++++++++++- tests/setup/root-server-bootstrap-contract | 246 ++++++++++++++++++++- 2 files changed, 363 insertions(+), 18 deletions(-) diff --git a/boot b/boot index 676dfa1..2cfefeb 100755 --- a/boot +++ b/boot @@ -68,23 +68,131 @@ safe_root_authorized_keys() { grep -qEv '^[[:space:]]*(#|$)' "$keys" } -harden_server_ssh() { - local username="$1" user_home="$2" sshd_dir sshd_dropin harden - sshd_dir="$(system_path /etc/ssh/sshd_config.d)" || return 1 - sshd_dropin="$sshd_dir/90-panama.conf" +detect_ssh_unit() { + local unit + for unit in sshd.service ssh.service; do + systemctl cat "$unit" >/dev/null 2>&1 && { + printf '%s\n' "$unit" + return 0 + } + done + return 1 +} - if [[ -f "$sshd_dropin" ]]; then - echo "sshd is already hardened ($sshd_dropin)" - return 0 +restore_ssh_dropin() { + local restore + if [[ -n "${ssh_backup:-}" && -e "$ssh_backup" ]]; then + restore="$(mktemp --tmpdir="$sshd_dir" .90-panama.XXXXXX.restore)" || return 1 + if ! cp -a -- "$ssh_backup" "$restore"; then + rm -f -- "$restore" + return 1 + fi + if ! mv -f -- "$restore" "$ssh_dropin"; then + rm -f -- "$restore" + return 1 + fi + else + rm -f -- "$ssh_dropin" fi +} + +restore_ssh_transaction_traps() { + trap - EXIT INT TERM + [[ -n "${ssh_saved_exit_trap:-}" ]] && eval "$ssh_saved_exit_trap" + [[ -n "${ssh_saved_int_trap:-}" ]] && eval "$ssh_saved_int_trap" + [[ -n "${ssh_saved_term_trap:-}" ]] && eval "$ssh_saved_term_trap" + return 0 +} + +harden_server_ssh() { + local username="$1" user_home="$2" sshd_dir ssh_dropin harden ssh_unit + local ssh_candidate="" ssh_backup="" rollback_failed=0 + local ssh_transaction_active=0 + local ssh_saved_exit_trap ssh_saved_int_trap ssh_saved_term_trap + sshd_dir="$(system_path /etc/ssh/sshd_config.d)" || return 1 + ssh_dropin="$sshd_dir/90-panama.conf" printf 'Harden sshd (disable root login and password auth)? [Y/n]: ' read -r harden "$sshd_dropin" - systemctl reload sshd 2>/dev/null || systemctl reload ssh 2>/dev/null || true - echo "Wrote $sshd_dropin; make sure your key works before logging out." + if [[ "$harden" =~ ^[Nn] ]]; then + return 0 fi + + if ! ssh_unit="$(detect_ssh_unit)"; then + echo "SSH hardening failed: neither sshd.service nor ssh.service exists" >&2 + return 1 + fi + + ssh_candidate="$(umask 077; mktemp --tmpdir="$sshd_dir" .90-panama.XXXXXX.tmp)" || return 1 + if ! printf 'PermitRootLogin no\nPasswordAuthentication no\n' >"$ssh_candidate"; then + rm -f -- "$ssh_candidate" + return 1 + fi + + if [[ -e "$ssh_dropin" ]]; then + ssh_backup="$(umask 077; mktemp --tmpdir="$sshd_dir" .90-panama.XXXXXX.backup)" || { + rm -f -- "$ssh_candidate" + return 1 + } + if ! cat -- "$ssh_dropin" >"$ssh_backup"; then + rm -f -- "$ssh_candidate" "$ssh_backup" + return 1 + fi + fi + + ssh_saved_exit_trap="$(trap -p EXIT)" + ssh_saved_int_trap="$(trap -p INT)" + ssh_saved_term_trap="$(trap -p TERM)" + trap 'if [[ "${ssh_transaction_active:-0}" == 1 ]]; then restore_ssh_dropin || true; fi' EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + ssh_transaction_active=1 + + if ! mv -f -- "$ssh_candidate" "$ssh_dropin"; then + ssh_transaction_active=0 + restore_ssh_transaction_traps + rm -f -- "$ssh_candidate" "$ssh_backup" + return 1 + fi + ssh_candidate="" + + if ! sshd -t; then + restore_ssh_dropin || rollback_failed=1 + sshd -t || rollback_failed=1 + ssh_transaction_active=0 + restore_ssh_transaction_traps + if (( rollback_failed )); then + printf 'SSH rollback needs manual recovery. Backup: %s\n' "$ssh_backup" >&2 + printf ' cp -a -- %q %q\n' "$ssh_backup" "$ssh_dropin" >&2 + printf ' sshd -t\n' >&2 + printf ' systemctl reload %s\n' "$ssh_unit" >&2 + else + [[ -z "$ssh_backup" ]] || rm -f -- "$ssh_backup" + fi + return 1 + fi + + if ! systemctl reload "$ssh_unit"; then + restore_ssh_dropin || rollback_failed=1 + sshd -t || rollback_failed=1 + systemctl reload "$ssh_unit" || rollback_failed=1 + ssh_transaction_active=0 + restore_ssh_transaction_traps + if (( rollback_failed )); then + printf 'SSH rollback needs manual recovery. Backup: %s\n' "$ssh_backup" >&2 + printf ' cp -a -- %q %q\n' "$ssh_backup" "$ssh_dropin" >&2 + printf ' sshd -t\n' >&2 + printf ' systemctl reload %s\n' "$ssh_unit" >&2 + else + [[ -z "$ssh_backup" ]] || rm -f -- "$ssh_backup" + fi + return 1 + fi + + ssh_transaction_active=0 + restore_ssh_transaction_traps + [[ -z "$ssh_backup" ]] || rm -f -- "$ssh_backup" + echo "Wrote $ssh_dropin; make sure your key works before logging out." } # Panama assumes Fedora's repositories and package names. @@ -175,7 +283,10 @@ if [[ "$(id -u)" -eq 0 ]]; then fi if safe_authorized_keys "$username" "$user_home"; then - harden_server_ssh "$username" "$user_home" + if ! harden_server_ssh "$username" "$user_home"; then + echo "SSH hardening failed; stopping before install handoff." >&2 + exit 1 + fi else echo "SSH hardening unavailable: $username has no safe authorized_keys" >&2 fi diff --git a/tests/setup/root-server-bootstrap-contract b/tests/setup/root-server-bootstrap-contract index b33f2cb..2ab3ae7 100755 --- a/tests/setup/root-server-bootstrap-contract +++ b/tests/setup/root-server-bootstrap-contract @@ -20,6 +20,7 @@ import errno import fcntl import os import pty +import re import shutil import subprocess import sys @@ -50,6 +51,17 @@ log() {{ local argument {{ for argument in "$@"; do printf '%q ' "$argument"; done; printf '\\n'; }} >>"$calls" }} +consume_result() {{ + local name="$1" results result + results="$PANAMA_BOOT_FIXTURE_ROOT/state/$name" + if ! IFS= read -r result <"$results"; then + return 0 + fi + /usr/bin/tail -n +2 "$results" >"$results.next" + /usr/bin/mv -f -- "$results.next" "$results" + [[ "$result" =~ ^[0-9]+$ ]] || exit 97 + return "$result" +}} ''' write_executable(stub_dir / "id", common + r''' @@ -112,20 +124,39 @@ log dnf "$@" ''') write_executable(stub_dir / "sshd", common + r''' log sshd "$@" -exit 97 +[[ "$#" -eq 1 && "$1" == -t ]] || exit 97 +for artifact in "$PANAMA_BOOT_FIXTURE_ROOT/etc/ssh/sshd_config.d"/.90-panama.*; do + [[ -e "$artifact" ]] || continue + log ssh-artifact "$artifact" "$(/usr/bin/stat -c %a -- "$artifact")" +done +consume_result SSHD_RESULTS ''') write_executable(stub_dir / "systemctl", common + r''' log systemctl "$@" case "${1:-}:${2:-}" in - reload:sshd|reload:ssh) exit 0 ;; + cat:sshd.service) [[ "$(<"$PANAMA_BOOT_FIXTURE_ROOT/state/SSHD_UNIT")" == present ]] ;; + cat:ssh.service) [[ "$(<"$PANAMA_BOOT_FIXTURE_ROOT/state/SSH_UNIT")" == present ]] ;; + reload:sshd.service|reload:ssh.service) consume_result RELOAD_RESULTS ;; *) exit 97 ;; esac +''') + write_executable(stub_dir / "mv", common + r''' +log mv "$@" "source-mode=$(/usr/bin/stat -c %a -- "${3:-}")" +exec /usr/bin/mv "$@" ''') for command in ("useradd", "usermod"): write_executable(stub_dir / command, common + f'''\nlog {command} "$@"\nexit 97\n''') -def configure_case(name: str) -> tuple[Path, Path]: +def configure_case( + name: str, + *, + sshd_results: tuple[int, ...] = (), + reload_results: tuple[int, ...] = (), + prior_dropin: bytes | None = None, + sshd_unit: bool = True, + ssh_unit: bool = True, +) -> tuple[Path, Path]: fixture_root = work / name / "root" stub_dir = work / name / "bin" calls = fixture_root / "calls" @@ -143,12 +174,21 @@ def configure_case(name: str) -> tuple[Path, Path]: (state / "target-dir-meta").write_text("1000:700\n") (state / "target-key-meta").write_text("1000:600\n") (state / "root-key-meta").write_text("0:600\n") + (state / "SSHD_RESULTS").write_text("".join(f"{result}\n" for result in sshd_results)) + (state / "RELOAD_RESULTS").write_text("".join(f"{result}\n" for result in reload_results)) + (state / "SSHD_UNIT").write_text("present\n" if sshd_unit else "absent\n") + (state / "SSH_UNIT").write_text("present\n" if ssh_unit else "absent\n") (fixture_root / "stub-install").write_text( "#!/usr/bin/env bash\nprintf 'install-handoff %s\\n' \"${PANAMA_PATH:-unset}\" >> \"$PANAMA_BOOT_FIXTURE_ROOT/calls\"\n" ) (fixture_root / "stub-install").chmod(0o755) make_stubs(stub_dir, fixture_root, calls) + if prior_dropin is not None: + dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf" + dropin.write_bytes(prior_dropin) + dropin.chmod(0o600) + target_keys = ssh_dir / "authorized_keys" root_keys = root_ssh_dir / "authorized_keys" if name == "missing": @@ -187,7 +227,15 @@ def configure_case(name: str) -> tuple[Path, Path]: elif name == "relative-home": target_keys.write_text("ssh-ed25519 target\n") (state / "home").write_text("home/gib\n") - elif name == "safe-existing-key": + elif name in ( + "safe-existing-key", + "success-without-prior-dropin", + "success-replaces-prior-dropin", + "candidate-invalid", + "candidate-reload-fails", + "rollback-validation-fails", + "rollback-reload-fails", + ): target_keys.write_text("ssh-ed25519 target\n") elif name == "safe-root-key-copy": root_keys.write_text("ssh-ed25519 root\n") @@ -196,8 +244,8 @@ def configure_case(name: str) -> tuple[Path, Path]: return fixture_root, stub_dir -def run_case(name: str) -> tuple[int, str, str, Path]: - fixture_root, stub_dir = configure_case(name) +def run_case(name: str, **configuration: object) -> tuple[int, str, str, Path]: + fixture_root, stub_dir = configure_case(name, **configuration) master, slave = pty.openpty() def attach_terminal() -> None: @@ -290,6 +338,192 @@ for case in ("safe-existing-key", "safe-root-key-copy"): if not keys.exists() or keys.read_text() != "ssh-ed25519 root\n": note("safe-root-key-copy: root key was not copied to the target account") +desired_dropin = b"PermitRootLogin no\nPasswordAuthentication no\n" +prior_dropin = b"# prior Panama settings\nPasswordAuthentication yes\n" +transaction_cases = { + "success-without-prior-dropin": { + "sshd_results": (0,), + "reload_results": (0,), + "prior_dropin": None, + "sshd_unit": True, + "ssh_unit": True, + "succeeds": True, + }, + "success-replaces-prior-dropin": { + "sshd_results": (0,), + "reload_results": (0,), + "prior_dropin": prior_dropin, + "sshd_unit": False, + "ssh_unit": True, + "succeeds": True, + }, + "candidate-invalid": { + "sshd_results": (1, 0), + "reload_results": (), + "prior_dropin": prior_dropin, + "sshd_unit": True, + "ssh_unit": True, + "succeeds": False, + }, + "candidate-reload-fails": { + "sshd_results": (0, 0), + "reload_results": (1, 0), + "prior_dropin": prior_dropin, + "sshd_unit": True, + "ssh_unit": True, + "succeeds": False, + }, + "rollback-validation-fails": { + "sshd_results": (0, 1), + "reload_results": (1,), + "prior_dropin": prior_dropin, + "sshd_unit": True, + "ssh_unit": True, + "succeeds": False, + "rollback_fails": True, + }, + "rollback-reload-fails": { + "sshd_results": (0, 0), + "reload_results": (1, 1), + "prior_dropin": prior_dropin, + "sshd_unit": True, + "ssh_unit": True, + "succeeds": False, + "rollback_fails": True, + }, +} + +for case, expected in transaction_cases.items(): + configuration = { + key: value + for key, value in expected.items() + if key not in {"succeeds", "rollback_fails"} + } + status, output, calls, fixture_root = run_case(case, **configuration) + call_lines = calls.splitlines() + dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf" + sshd_dir = dropin.parent + validations = [index for index, line in enumerate(call_lines) if line.startswith("sshd -t ")] + reloads = [ + index + for index, line in enumerate(call_lines) + if line.startswith("systemctl reload ") + ] + activation_lines = [ + (index, line) + for index, line in enumerate(call_lines) + if re.fullmatch( + rf"mv -f -- {re.escape(str(sshd_dir))}/\.90-panama\.[A-Za-z0-9]+\.tmp " + rf"{re.escape(str(dropin))} source-mode=600 ", + line, + ) + ] + restore_lines = [ + index + for index, line in enumerate(call_lines) + if re.fullmatch( + rf"mv -f -- {re.escape(str(sshd_dir))}/\.90-panama\.[A-Za-z0-9]+\.restore " + rf"{re.escape(str(dropin))} source-mode=600 ", + line, + ) + ] + + succeeds = bool(expected["succeeds"]) + if succeeds and status != 0: + note(f"{case}: transaction stopped with status {status}: {output.strip()}") + if not succeeds and status == 0: + note(f"{case}: failed transaction returned success") + if succeeds and "install-handoff " not in calls: + note(f"{case}: successful transaction did not hand off to install") + if not succeeds and "install-handoff " in calls: + note(f"{case}: failed transaction handed off to install") + + wanted_contents = desired_dropin if succeeds else prior_dropin + actual_contents = dropin.read_bytes() if dropin.exists() else None + if actual_contents != wanted_contents: + note(f"{case}: SSH drop-in contents were not {'activated' if succeeds else 'restored'}") + + if len(activation_lines) != 1: + note(f"{case}: candidate was not activated once through a restrictive same-directory rename") + if succeeds: + if len(validations) != 1 or len(reloads) != 1: + note(f"{case}: success did not validate once and reload once") + elif activation_lines and not activation_lines[0][0] < validations[0] < reloads[0]: + note(f"{case}: success did not activate, validate, then reload") + elif case == "candidate-invalid": + if len(validations) != 2 or reloads: + note(f"{case}: invalid candidate did not validate candidate and restoration without reload") + elif activation_lines and restore_lines and not ( + activation_lines[0][0] < validations[0] < restore_lines[0] < validations[1] + ): + note(f"{case}: rollback command order was wrong") + else: + if len(validations) != 2 or len(reloads) != 2: + note(f"{case}: reload failure did not validate and reload the restored configuration") + elif activation_lines and restore_lines and not ( + activation_lines[0][0] + < validations[0] + < reloads[0] + < restore_lines[0] + < validations[1] + < reloads[1] + ): + note(f"{case}: rollback command order was wrong") + + if succeeds and restore_lines: + note(f"{case}: successful transaction performed a rollback") + if not succeeds and len(restore_lines) != 1: + note(f"{case}: prior drop-in was not restored exactly once") + + detected_unit = "ssh.service" if case == "success-replaces-prior-dropin" else "sshd.service" + other_unit = "sshd.service" if detected_unit == "ssh.service" else "ssh.service" + reload_lines = [call_lines[index] for index in reloads] + if reload_lines and any(line != f"systemctl reload {detected_unit} " for line in reload_lines): + note(f"{case}: reloaded a unit other than detected {detected_unit}") + if any(line == f"systemctl reload {other_unit} " for line in call_lines): + note(f"{case}: guessed {other_unit} after reload failure") + if detected_unit == "sshd.service": + if "systemctl cat sshd.service " not in call_lines: + note(f"{case}: did not detect sshd.service") + if "systemctl cat ssh.service " in call_lines: + note(f"{case}: probed ssh.service after finding sshd.service") + elif not ( + "systemctl cat sshd.service " in call_lines + and "systemctl cat ssh.service " in call_lines + and call_lines.index("systemctl cat sshd.service ") + < call_lines.index("systemctl cat ssh.service ") + ): + note(f"{case}: did not fall back from absent sshd.service to ssh.service") + + artifacts = list(sshd_dir.glob(".90-panama.*")) + rollback_fails = bool(expected.get("rollback_fails", False)) + if not rollback_fails and artifacts: + note(f"{case}: successful or cleanly rolled-back transaction left temporary artifacts") + if rollback_fails: + backups = [artifact for artifact in artifacts if artifact.name.endswith(".backup")] + if len(backups) != 1: + note(f"{case}: rollback failure did not retain exactly one backup") + else: + backup = backups[0] + if backup.parent != sshd_dir or backup.read_bytes() != prior_dropin: + note(f"{case}: retained backup was not a same-directory byte copy") + if backup.stat().st_mode & 0o777 != 0o600: + note(f"{case}: retained backup permissions were not restrictive") + if str(backup.resolve()) not in output: + note(f"{case}: recovery output omitted the absolute backup path") + if "sshd -t" not in output or f"systemctl reload {detected_unit}" not in output: + note(f"{case}: recovery output omitted validation or reload commands") + + artifact_logs = [line for line in call_lines if line.startswith("ssh-artifact ")] + if expected["prior_dropin"] is not None and not any( + re.fullmatch( + rf"ssh-artifact {re.escape(str(sshd_dir))}/\.90-panama\.[A-Za-z0-9]+\.backup 600 ", + line, + ) + for line in artifact_logs + ): + note(f"{case}: backup was not collision-safe, same-directory, non-.conf, and restrictive") + if findings: print(f"root server bootstrap: {len(findings)} finding(s)", file=sys.stderr) for finding in findings: