From 51e2c8418aa473e375db5a85ebd9ecb811b4f11a Mon Sep 17 00:00:00 2001 From: Gabriel Brown Date: Thu, 27 Aug 2026 04:06:54 -0400 Subject: [PATCH] Fix: Complete SSH hardening recovery --- boot | 57 ++++- tests/setup/root-server-bootstrap-contract | 270 +++++++++++++++++++-- 2 files changed, 300 insertions(+), 27 deletions(-) diff --git a/boot b/boot index 2cfefeb..d3503e5 100755 --- a/boot +++ b/boot @@ -84,15 +84,15 @@ restore_ssh_dropin() { if [[ -n "${ssh_backup:-}" && -e "$ssh_backup" ]]; then restore="$(mktemp --tmpdir="$sshd_dir" .90-panama.XXXXXX.restore)" || return 1 if ! cp -a -- "$ssh_backup" "$restore"; then - rm -f -- "$restore" + remove_ssh_artifact "$restore" || true return 1 fi if ! mv -f -- "$restore" "$ssh_dropin"; then - rm -f -- "$restore" + remove_ssh_artifact "$restore" || true return 1 fi else - rm -f -- "$ssh_dropin" + remove_ssh_artifact "$ssh_dropin" fi } @@ -104,6 +104,37 @@ restore_ssh_transaction_traps() { return 0 } +remove_ssh_artifact() { + local artifact="$1" + [[ -n "$artifact" && -e "$artifact" ]] || return 0 + if rm -f -- "$artifact"; then + return 0 + fi + printf 'SSH transaction cleanup failed. Retained artifact: %s\n' "$artifact" >&2 + printf ' rm -f -- %q\n' "$artifact" >&2 + return 1 +} + +handle_ssh_transaction_signal() { + local signal_status="$1" + trap - INT TERM + rollback_failed=0 + restore_ssh_dropin || rollback_failed=1 + sshd -t || rollback_failed=1 + systemctl reload "$ssh_unit" || rollback_failed=1 + ssh_transaction_active=0 + restore_ssh_transaction_traps + if (( rollback_failed )); then + printf 'SSH rollback needs manual recovery. Backup: %s\n' "$ssh_backup" >&2 + printf ' cp -a -- %q %q\n' "$ssh_backup" "$ssh_dropin" >&2 + printf ' sshd -t\n' >&2 + printf ' systemctl reload %s\n' "$ssh_unit" >&2 + else + remove_ssh_artifact "$ssh_backup" || true + fi + exit "$signal_status" +} + harden_server_ssh() { local username="$1" user_home="$2" sshd_dir ssh_dropin harden ssh_unit local ssh_candidate="" ssh_backup="" rollback_failed=0 @@ -125,17 +156,18 @@ harden_server_ssh() { ssh_candidate="$(umask 077; mktemp --tmpdir="$sshd_dir" .90-panama.XXXXXX.tmp)" || return 1 if ! printf 'PermitRootLogin no\nPasswordAuthentication no\n' >"$ssh_candidate"; then - rm -f -- "$ssh_candidate" + remove_ssh_artifact "$ssh_candidate" || true return 1 fi if [[ -e "$ssh_dropin" ]]; then ssh_backup="$(umask 077; mktemp --tmpdir="$sshd_dir" .90-panama.XXXXXX.backup)" || { - rm -f -- "$ssh_candidate" + remove_ssh_artifact "$ssh_candidate" || true return 1 } if ! cat -- "$ssh_dropin" >"$ssh_backup"; then - rm -f -- "$ssh_candidate" "$ssh_backup" + remove_ssh_artifact "$ssh_candidate" || true + remove_ssh_artifact "$ssh_backup" || true return 1 fi fi @@ -144,14 +176,15 @@ harden_server_ssh() { ssh_saved_int_trap="$(trap -p INT)" ssh_saved_term_trap="$(trap -p TERM)" trap 'if [[ "${ssh_transaction_active:-0}" == 1 ]]; then restore_ssh_dropin || true; fi' EXIT - trap 'exit 130' INT - trap 'exit 143' TERM + trap 'handle_ssh_transaction_signal 130' INT + trap 'handle_ssh_transaction_signal 143' TERM ssh_transaction_active=1 if ! mv -f -- "$ssh_candidate" "$ssh_dropin"; then ssh_transaction_active=0 restore_ssh_transaction_traps - rm -f -- "$ssh_candidate" "$ssh_backup" + remove_ssh_artifact "$ssh_candidate" || true + remove_ssh_artifact "$ssh_backup" || true return 1 fi ssh_candidate="" @@ -167,7 +200,7 @@ harden_server_ssh() { printf ' sshd -t\n' >&2 printf ' systemctl reload %s\n' "$ssh_unit" >&2 else - [[ -z "$ssh_backup" ]] || rm -f -- "$ssh_backup" + remove_ssh_artifact "$ssh_backup" || true fi return 1 fi @@ -184,14 +217,14 @@ harden_server_ssh() { printf ' sshd -t\n' >&2 printf ' systemctl reload %s\n' "$ssh_unit" >&2 else - [[ -z "$ssh_backup" ]] || rm -f -- "$ssh_backup" + remove_ssh_artifact "$ssh_backup" || true fi return 1 fi ssh_transaction_active=0 restore_ssh_transaction_traps - [[ -z "$ssh_backup" ]] || rm -f -- "$ssh_backup" + remove_ssh_artifact "$ssh_backup" || return 1 echo "Wrote $ssh_dropin; make sure your key works before logging out." } diff --git a/tests/setup/root-server-bootstrap-contract b/tests/setup/root-server-bootstrap-contract index 2ab3ae7..357e806 100755 --- a/tests/setup/root-server-bootstrap-contract +++ b/tests/setup/root-server-bootstrap-contract @@ -21,11 +21,13 @@ import fcntl import os import pty import re +import signal import shutil import subprocess import sys import tempfile import termios +import time from pathlib import Path boot = sys.argv[1] @@ -142,7 +144,32 @@ esac ''') write_executable(stub_dir / "mv", common + r''' log mv "$@" "source-mode=$(/usr/bin/stat -c %a -- "${3:-}")" -exec /usr/bin/mv "$@" +if [[ "${3:-}" == *.tmp ]]; then + consume_result MV_ACTIVATION_RESULTS + result=$? + (( result == 0 )) || exit "$result" +fi +/usr/bin/mv "$@" +if [[ "${3:-}" == *.tmp && -e "$PANAMA_BOOT_FIXTURE_ROOT/state/HOLD_ACTIVATION" ]]; then + : >"$PANAMA_BOOT_FIXTURE_ROOT/state/ACTIVATED" + while [[ ! -e "$PANAMA_BOOT_FIXTURE_ROOT/state/RELEASE_ACTIVATION" ]]; do + /usr/bin/sleep 0.01 + done +fi +''') + write_executable(stub_dir / "rm", common + r''' +log rm "$@" +if [[ "${1:-}" == -f && "${2:-}" == -- && "${3:-}" == *.backup ]]; then + consume_result RM_BACKUP_RESULTS + result=$? + (( result == 0 )) || exit "$result" +fi +if [[ "${1:-}" == -f && "${2:-}" == -- && "${3:-}" == *.tmp ]]; then + consume_result RM_CANDIDATE_RESULTS + result=$? + (( result == 0 )) || exit "$result" +fi +exec /usr/bin/rm "$@" ''') for command in ("useradd", "usermod"): write_executable(stub_dir / command, common + f'''\nlog {command} "$@"\nexit 97\n''') @@ -153,9 +180,13 @@ def configure_case( *, sshd_results: tuple[int, ...] = (), reload_results: tuple[int, ...] = (), + mv_activation_results: tuple[int, ...] = (), + rm_backup_results: tuple[int, ...] = (), + rm_candidate_results: tuple[int, ...] = (), prior_dropin: bytes | None = None, sshd_unit: bool = True, ssh_unit: bool = True, + hold_activation: bool = False, ) -> tuple[Path, Path]: fixture_root = work / name / "root" stub_dir = work / name / "bin" @@ -176,8 +207,19 @@ def configure_case( (state / "root-key-meta").write_text("0:600\n") (state / "SSHD_RESULTS").write_text("".join(f"{result}\n" for result in sshd_results)) (state / "RELOAD_RESULTS").write_text("".join(f"{result}\n" for result in reload_results)) + (state / "MV_ACTIVATION_RESULTS").write_text( + "".join(f"{result}\n" for result in mv_activation_results) + ) + (state / "RM_BACKUP_RESULTS").write_text( + "".join(f"{result}\n" for result in rm_backup_results) + ) + (state / "RM_CANDIDATE_RESULTS").write_text( + "".join(f"{result}\n" for result in rm_candidate_results) + ) (state / "SSHD_UNIT").write_text("present\n" if sshd_unit else "absent\n") (state / "SSH_UNIT").write_text("present\n" if ssh_unit else "absent\n") + if hold_activation: + (state / "HOLD_ACTIVATION").touch() (fixture_root / "stub-install").write_text( "#!/usr/bin/env bash\nprintf 'install-handoff %s\\n' \"${PANAMA_PATH:-unset}\" >> \"$PANAMA_BOOT_FIXTURE_ROOT/calls\"\n" ) @@ -232,9 +274,16 @@ def configure_case( "success-without-prior-dropin", "success-replaces-prior-dropin", "candidate-invalid", + "candidate-invalid-without-prior", "candidate-reload-fails", + "candidate-reload-fails-without-prior", "rollback-validation-fails", "rollback-reload-fails", + "success-backup-cleanup-fails", + "rollback-backup-cleanup-fails", + "signal-int-restores-prior", + "signal-term-removes-new-dropin", + "candidate-cleanup-fails", ): target_keys.write_text("ssh-ed25519 target\n") elif name == "safe-root-key-copy": @@ -244,11 +293,25 @@ def configure_case( return fixture_root, stub_dir -def run_case(name: str, **configuration: object) -> tuple[int, str, str, Path]: - fixture_root, stub_dir = configure_case(name, **configuration) +def run_case( + name: str, + *, + signal_after_activation: int | None = None, + prior_traps: bool = False, + **configuration: object, +) -> tuple[int, str, str, Path, int]: + fixture_root, stub_dir = configure_case( + name, + hold_activation=signal_after_activation is not None, + **configuration, + ) master, slave = pty.openpty() def attach_terminal() -> None: + # The test runner launches contracts as background jobs, which inherit + # SIGINT ignored. A real interactive bootstrap starts with SIGINT at + # its default disposition, so restore that state before exec. + signal.signal(signal.SIGINT, signal.SIG_DFL) fcntl.ioctl(0, termios.TIOCSCTTY, 0) env = { @@ -258,6 +321,18 @@ def run_case(name: str, **configuration: object) -> tuple[int, str, str, Path]: "PANAMA_PATH": f"{fixture_root}/home/gib/.local/share/Panama", "HOME": f"{fixture_root}/root", } + if prior_traps: + bash_env = fixture_root / "prior-traps" + bash_env.write_text( + '''if [[ "$0" == "$PANAMA_BOOT_SCRIPT" ]]; then +trap 'printf "prior-exit %s\\n" "$BASHPID" >>"$PANAMA_BOOT_FIXTURE_ROOT/calls"' EXIT +trap 'printf "prior-int %s\\n" "$BASHPID" >>"$PANAMA_BOOT_FIXTURE_ROOT/calls"' INT +trap 'printf "prior-term %s\\n" "$BASHPID" >>"$PANAMA_BOOT_FIXTURE_ROOT/calls"' TERM +fi +''' + ) + env["BASH_ENV"] = str(bash_env) + env["PANAMA_BOOT_SCRIPT"] = boot process = subprocess.Popen( ["bash", boot, "--server"], stdin=slave, @@ -269,6 +344,16 @@ def run_case(name: str, **configuration: object) -> tuple[int, str, str, Path]: ) os.close(slave) os.write(master, b"gib\nY\n") + if signal_after_activation is not None: + activation = fixture_root / "state/ACTIVATED" + deadline = time.monotonic() + 5 + while not activation.exists() and process.poll() is None and time.monotonic() < deadline: + time.sleep(0.01) + if not activation.exists(): + note(f"{name}: fixture did not observe atomic activation before signaling") + else: + os.kill(process.pid, signal_after_activation) + (fixture_root / "state/RELEASE_ACTIVATION").touch() chunks: list[bytes] = [] while True: try: @@ -284,7 +369,7 @@ def run_case(name: str, **configuration: object) -> tuple[int, str, str, Path]: status = process.wait() calls = (fixture_root / "calls").read_text() output = b"".join(chunks).decode(errors="replace") - return status, output, calls, fixture_root + return status, output, calls, fixture_root, process.pid unsafe_cases = ( @@ -302,7 +387,7 @@ unsafe_cases = ( "relative-home", ) for case in unsafe_cases: - status, output, calls, fixture_root = run_case(case) + status, output, calls, fixture_root, _ = run_case(case) if status != 0: note(f"{case}: bootstrap stopped with status {status}: {output.strip()}") if "SSH hardening unavailable" not in output: @@ -321,7 +406,7 @@ for case in unsafe_cases: note(f"{case}: unsafe login path did not hand off to install") for case in ("safe-existing-key", "safe-root-key-copy"): - status, output, calls, fixture_root = run_case(case) + status, output, calls, fixture_root, _ = run_case(case) if status != 0: note(f"{case}: safe login path stopped with status {status}: {output.strip()}") if "SSH hardening unavailable" in output: @@ -365,6 +450,14 @@ transaction_cases = { "ssh_unit": True, "succeeds": False, }, + "candidate-invalid-without-prior": { + "sshd_results": (1, 0), + "reload_results": (), + "prior_dropin": None, + "sshd_unit": True, + "ssh_unit": True, + "succeeds": False, + }, "candidate-reload-fails": { "sshd_results": (0, 0), "reload_results": (1, 0), @@ -373,6 +466,14 @@ transaction_cases = { "ssh_unit": True, "succeeds": False, }, + "candidate-reload-fails-without-prior": { + "sshd_results": (0, 0), + "reload_results": (1, 0), + "prior_dropin": None, + "sshd_unit": True, + "ssh_unit": True, + "succeeds": False, + }, "rollback-validation-fails": { "sshd_results": (0, 1), "reload_results": (1,), @@ -399,7 +500,7 @@ for case, expected in transaction_cases.items(): for key, value in expected.items() if key not in {"succeeds", "rollback_fails"} } - status, output, calls, fixture_root = run_case(case, **configuration) + status, output, calls, fixture_root, _ = run_case(case, **configuration) call_lines = calls.splitlines() dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf" sshd_dir = dropin.parent @@ -427,6 +528,12 @@ for case, expected in transaction_cases.items(): line, ) ] + removal_lines = [ + index + for index, line in enumerate(call_lines) + if line == f"rm -f -- {dropin} " + ] + rollback_lines = restore_lines if expected["prior_dropin"] is not None else removal_lines succeeds = bool(expected["succeeds"]) if succeeds and status != 0: @@ -438,7 +545,7 @@ for case, expected in transaction_cases.items(): if not succeeds and "install-handoff " in calls: note(f"{case}: failed transaction handed off to install") - wanted_contents = desired_dropin if succeeds else prior_dropin + wanted_contents = desired_dropin if succeeds else expected["prior_dropin"] actual_contents = dropin.read_bytes() if dropin.exists() else None if actual_contents != wanted_contents: note(f"{case}: SSH drop-in contents were not {'activated' if succeeds else 'restored'}") @@ -450,21 +557,21 @@ for case, expected in transaction_cases.items(): note(f"{case}: success did not validate once and reload once") elif activation_lines and not activation_lines[0][0] < validations[0] < reloads[0]: note(f"{case}: success did not activate, validate, then reload") - elif case == "candidate-invalid": + elif case in {"candidate-invalid", "candidate-invalid-without-prior"}: if len(validations) != 2 or reloads: note(f"{case}: invalid candidate did not validate candidate and restoration without reload") - elif activation_lines and restore_lines and not ( - activation_lines[0][0] < validations[0] < restore_lines[0] < validations[1] + elif activation_lines and rollback_lines and not ( + activation_lines[0][0] < validations[0] < rollback_lines[0] < validations[1] ): note(f"{case}: rollback command order was wrong") else: if len(validations) != 2 or len(reloads) != 2: note(f"{case}: reload failure did not validate and reload the restored configuration") - elif activation_lines and restore_lines and not ( + elif activation_lines and rollback_lines and not ( activation_lines[0][0] < validations[0] < reloads[0] - < restore_lines[0] + < rollback_lines[0] < validations[1] < reloads[1] ): @@ -472,8 +579,9 @@ for case, expected in transaction_cases.items(): if succeeds and restore_lines: note(f"{case}: successful transaction performed a rollback") - if not succeeds and len(restore_lines) != 1: - note(f"{case}: prior drop-in was not restored exactly once") + if not succeeds: + if len(rollback_lines) != 1: + note(f"{case}: pre-transaction SSH state was not restored exactly once") detected_unit = "ssh.service" if case == "success-replaces-prior-dropin" else "sshd.service" other_unit = "sshd.service" if detected_unit == "ssh.service" else "ssh.service" @@ -524,6 +632,138 @@ for case, expected in transaction_cases.items(): ): note(f"{case}: backup was not collision-safe, same-directory, non-.conf, and restrictive") +cleanup_failure_cases = { + "success-backup-cleanup-fails": { + "sshd_results": (0,), + "reload_results": (0,), + "rm_backup_results": (1,), + "expected_dropin": desired_dropin, + "expected_validations": 1, + "expected_reloads": 1, + }, + "rollback-backup-cleanup-fails": { + "sshd_results": (1, 0), + "reload_results": (), + "rm_backup_results": (1,), + "expected_dropin": prior_dropin, + "expected_validations": 2, + "expected_reloads": 0, + }, +} + +for case, expected in cleanup_failure_cases.items(): + status, output, calls, fixture_root, _ = run_case( + case, + sshd_results=expected["sshd_results"], + reload_results=expected["reload_results"], + rm_backup_results=expected["rm_backup_results"], + prior_dropin=prior_dropin, + ) + dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf" + backups = list(dropin.parent.glob(".90-panama.*.backup")) + if status == 0: + note(f"{case}: cleanup failure returned success") + if "install-handoff " in calls: + note(f"{case}: cleanup failure handed off to install") + if dropin.read_bytes() != expected["expected_dropin"]: + note(f"{case}: cleanup failure changed the settled SSH drop-in") + if calls.count("sshd -t \n") != expected["expected_validations"]: + note(f"{case}: cleanup failure validation count was wrong") + if calls.count("systemctl reload sshd.service \n") != expected["expected_reloads"]: + note(f"{case}: cleanup failure reload count was wrong") + if len(backups) != 1: + note(f"{case}: failed cleanup did not retain exactly one backup") + else: + backup = backups[0] + if str(backup.resolve()) not in output or "rm -f --" not in output: + note(f"{case}: retained backup was not reported with an actionable cleanup command") + +status, output, calls, fixture_root, _ = run_case( + "candidate-cleanup-fails", + mv_activation_results=(1,), + rm_candidate_results=(1,), +) +dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf" +candidates = list(dropin.parent.glob(".90-panama.*.tmp")) +if status == 0: + note("candidate-cleanup-fails: activation cleanup failure returned success") +if dropin.exists(): + note("candidate-cleanup-fails: failed activation changed the final drop-in") +if "install-handoff " in calls: + note("candidate-cleanup-fails: failed activation reached install handoff") +if len(candidates) != 1: + note("candidate-cleanup-fails: failed cleanup did not retain exactly one candidate") +else: + candidate = candidates[0] + if str(candidate.resolve()) not in output or "rm -f --" not in output: + note("candidate-cleanup-fails: retained candidate lacked an actionable cleanup command") + +signal_cases = { + "signal-int-restores-prior": { + "signal": signal.SIGINT, + "status": 130, + "prior_dropin": prior_dropin, + }, + "signal-term-removes-new-dropin": { + "signal": signal.SIGTERM, + "status": 143, + "prior_dropin": None, + }, +} + +for case, expected in signal_cases.items(): + status, output, calls, fixture_root, boot_pid = run_case( + case, + signal_after_activation=expected["signal"], + prior_traps=True, + sshd_results=(0,), + reload_results=(0,), + prior_dropin=expected["prior_dropin"], + ) + dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf" + actual_dropin = dropin.read_bytes() if dropin.exists() else None + if status != expected["status"]: + note(f"{case}: signal returned status {status}, expected {expected['status']}") + if actual_dropin != expected["prior_dropin"]: + note(f"{case}: signal did not restore the pre-transaction SSH state") + if list(dropin.parent.glob(".90-panama.*")): + note(f"{case}: signal left transaction residue") + if "install-handoff " in calls: + note(f"{case}: signal reached install handoff") + if f"prior-exit {boot_pid}\n" not in calls: + note(f"{case}: signal suppressed the saved EXIT trap") + if calls.count("sshd -t \n") != 1: + note(f"{case}: signal did not validate restored configuration once") + if calls.count("systemctl reload sshd.service \n") != 1: + note(f"{case}: signal did not reload restored configuration once") + call_lines = calls.splitlines() + rollback_indices = [ + index + for index, line in enumerate(call_lines) + if ( + expected["prior_dropin"] is not None + and re.fullmatch( + rf"mv -f -- {re.escape(str(dropin.parent))}/\.90-panama\.[A-Za-z0-9]+\.restore " + rf"{re.escape(str(dropin))} source-mode=600 ", + line, + ) + ) + or (expected["prior_dropin"] is None and line == f"rm -f -- {dropin} ") + ] + validation_indices = [ + index for index, line in enumerate(call_lines) if line == "sshd -t " + ] + reload_indices = [ + index + for index, line in enumerate(call_lines) + if line == "systemctl reload sshd.service " + ] + if not ( + len(rollback_indices) == len(validation_indices) == len(reload_indices) == 1 + and rollback_indices[0] < validation_indices[0] < reload_indices[0] + ): + note(f"{case}: signal did not restore, validate, then reload in order") + if findings: print(f"root server bootstrap: {len(findings)} finding(s)", file=sys.stderr) for finding in findings: