Fix: Roll back failed SSH hardening

boot

@@ -68,23 +68,131 @@ safe_root_authorized_keys() {
   grep -qEv '^[[:space:]]*(#|$)' "$keys"
 }
 
-harden_server_ssh() {
+detect_ssh_unit() {
-  local username="$1" user_home="$2" sshd_dir sshd_dropin harden
+  local unit
-  sshd_dir="$(system_path /etc/ssh/sshd_config.d)" || return 1
+  for unit in sshd.service ssh.service; do
-  sshd_dropin="$sshd_dir/90-panama.conf"
+    systemctl cat "$unit" >/dev/null 2>&1 && {
+      printf '%s\n' "$unit"
+      return 0
+    }
+  done
+  return 1
+}
 
-  if [[ -f "$sshd_dropin" ]]; then
+restore_ssh_dropin() {
-    echo "sshd is already hardened ($sshd_dropin)"
+  local restore
-    return 0
+  if [[ -n "${ssh_backup:-}" && -e "$ssh_backup" ]]; then
+    restore="$(mktemp --tmpdir="$sshd_dir" .90-panama.XXXXXX.restore)" || return 1
+    if ! cp -a -- "$ssh_backup" "$restore"; then
+      rm -f -- "$restore"
+      return 1
+    fi
+    if ! mv -f -- "$restore" "$ssh_dropin"; then
+      rm -f -- "$restore"
+      return 1
+    fi
+  else
+    rm -f -- "$ssh_dropin"
   fi
+}
+
+restore_ssh_transaction_traps() {
+  trap - EXIT INT TERM
+  [[ -n "${ssh_saved_exit_trap:-}" ]] && eval "$ssh_saved_exit_trap"
+  [[ -n "${ssh_saved_int_trap:-}" ]] && eval "$ssh_saved_int_trap"
+  [[ -n "${ssh_saved_term_trap:-}" ]] && eval "$ssh_saved_term_trap"
+  return 0
+}
+
+harden_server_ssh() {
+  local username="$1" user_home="$2" sshd_dir ssh_dropin harden ssh_unit
+  local ssh_candidate="" ssh_backup="" rollback_failed=0
+  local ssh_transaction_active=0
+  local ssh_saved_exit_trap ssh_saved_int_trap ssh_saved_term_trap
+  sshd_dir="$(system_path /etc/ssh/sshd_config.d)" || return 1
+  ssh_dropin="$sshd_dir/90-panama.conf"
 
   printf 'Harden sshd (disable root login and password auth)? [Y/n]: '
   read -r harden </dev/tty || harden=""
-  if [[ ! "$harden" =~ ^[Nn] ]]; then
+  if [[ "$harden" =~ ^[Nn] ]]; then
-    printf 'PermitRootLogin no\nPasswordAuthentication no\n' >"$sshd_dropin"
+    return 0
-    systemctl reload sshd 2>/dev/null || systemctl reload ssh 2>/dev/null || true
-    echo "Wrote $sshd_dropin; make sure your key works before logging out."
   fi
+
+  if ! ssh_unit="$(detect_ssh_unit)"; then
+    echo "SSH hardening failed: neither sshd.service nor ssh.service exists" >&2
+    return 1
+  fi
+
+  ssh_candidate="$(umask 077; mktemp --tmpdir="$sshd_dir" .90-panama.XXXXXX.tmp)" || return 1
+  if ! printf 'PermitRootLogin no\nPasswordAuthentication no\n' >"$ssh_candidate"; then
+    rm -f -- "$ssh_candidate"
+    return 1
+  fi
+
+  if [[ -e "$ssh_dropin" ]]; then
+    ssh_backup="$(umask 077; mktemp --tmpdir="$sshd_dir" .90-panama.XXXXXX.backup)" || {
+      rm -f -- "$ssh_candidate"
+      return 1
+    }
+    if ! cat -- "$ssh_dropin" >"$ssh_backup"; then
+      rm -f -- "$ssh_candidate" "$ssh_backup"
+      return 1
+    fi
+  fi
+
+  ssh_saved_exit_trap="$(trap -p EXIT)"
+  ssh_saved_int_trap="$(trap -p INT)"
+  ssh_saved_term_trap="$(trap -p TERM)"
+  trap 'if [[ "${ssh_transaction_active:-0}" == 1 ]]; then restore_ssh_dropin || true; fi' EXIT
+  trap 'exit 130' INT
+  trap 'exit 143' TERM
+  ssh_transaction_active=1
+
+  if ! mv -f -- "$ssh_candidate" "$ssh_dropin"; then
+    ssh_transaction_active=0
+    restore_ssh_transaction_traps
+    rm -f -- "$ssh_candidate" "$ssh_backup"
+    return 1
+  fi
+  ssh_candidate=""
+
+  if ! sshd -t; then
+    restore_ssh_dropin || rollback_failed=1
+    sshd -t || rollback_failed=1
+    ssh_transaction_active=0
+    restore_ssh_transaction_traps
+    if (( rollback_failed )); then
+      printf 'SSH rollback needs manual recovery. Backup: %s\n' "$ssh_backup" >&2
+      printf '  cp -a -- %q %q\n' "$ssh_backup" "$ssh_dropin" >&2
+      printf '  sshd -t\n' >&2
+      printf '  systemctl reload %s\n' "$ssh_unit" >&2
+    else
+      [[ -z "$ssh_backup" ]] || rm -f -- "$ssh_backup"
+    fi
+    return 1
+  fi
+
+  if ! systemctl reload "$ssh_unit"; then
+    restore_ssh_dropin || rollback_failed=1
+    sshd -t || rollback_failed=1
+    systemctl reload "$ssh_unit" || rollback_failed=1
+    ssh_transaction_active=0
+    restore_ssh_transaction_traps
+    if (( rollback_failed )); then
+      printf 'SSH rollback needs manual recovery. Backup: %s\n' "$ssh_backup" >&2
+      printf '  cp -a -- %q %q\n' "$ssh_backup" "$ssh_dropin" >&2
+      printf '  sshd -t\n' >&2
+      printf '  systemctl reload %s\n' "$ssh_unit" >&2
+    else
+      [[ -z "$ssh_backup" ]] || rm -f -- "$ssh_backup"
+    fi
+    return 1
+  fi
+
+  ssh_transaction_active=0
+  restore_ssh_transaction_traps
+  [[ -z "$ssh_backup" ]] || rm -f -- "$ssh_backup"
+  echo "Wrote $ssh_dropin; make sure your key works before logging out."
 }
 
 # Panama assumes Fedora's repositories and package names.
@@ -175,7 +283,10 @@ if [[ "$(id -u)" -eq 0 ]]; then
   fi
 
   if safe_authorized_keys "$username" "$user_home"; then
-    harden_server_ssh "$username" "$user_home"
+    if ! harden_server_ssh "$username" "$user_home"; then
+      echo "SSH hardening failed; stopping before install handoff." >&2
+      exit 1
+    fi
   else
     echo "SSH hardening unavailable: $username has no safe authorized_keys" >&2
   fi

tests/setup/root-server-bootstrap-contract

@@ -20,6 +20,7 @@ import errno
 import fcntl
 import os
 import pty
+import re
 import shutil
 import subprocess
 import sys
@@ -50,6 +51,17 @@ log() {{
   local argument
   {{ for argument in "$@"; do printf '%q ' "$argument"; done; printf '\\n'; }} >>"$calls"
 }}
+consume_result() {{
+  local name="$1" results result
+  results="$PANAMA_BOOT_FIXTURE_ROOT/state/$name"
+  if ! IFS= read -r result <"$results"; then
+    return 0
+  fi
+  /usr/bin/tail -n +2 "$results" >"$results.next"
+  /usr/bin/mv -f -- "$results.next" "$results"
+  [[ "$result" =~ ^[0-9]+$ ]] || exit 97
+  return "$result"
+}}
 '''
 
     write_executable(stub_dir / "id", common + r'''
@@ -112,20 +124,39 @@ log dnf "$@"
 ''')
     write_executable(stub_dir / "sshd", common + r'''
 log sshd "$@"
-exit 97
+[[ "$#" -eq 1 && "$1" == -t ]] || exit 97
+for artifact in "$PANAMA_BOOT_FIXTURE_ROOT/etc/ssh/sshd_config.d"/.90-panama.*; do
+  [[ -e "$artifact" ]] || continue
+  log ssh-artifact "$artifact" "$(/usr/bin/stat -c %a -- "$artifact")"
+done
+consume_result SSHD_RESULTS
 ''')
     write_executable(stub_dir / "systemctl", common + r'''
 log systemctl "$@"
 case "${1:-}:${2:-}" in
-  reload:sshd|reload:ssh) exit 0 ;;
+  cat:sshd.service) [[ "$(<"$PANAMA_BOOT_FIXTURE_ROOT/state/SSHD_UNIT")" == present ]] ;;
+  cat:ssh.service) [[ "$(<"$PANAMA_BOOT_FIXTURE_ROOT/state/SSH_UNIT")" == present ]] ;;
+  reload:sshd.service|reload:ssh.service) consume_result RELOAD_RESULTS ;;
   *) exit 97 ;;
 esac
+''')
+    write_executable(stub_dir / "mv", common + r'''
+log mv "$@" "source-mode=$(/usr/bin/stat -c %a -- "${3:-}")"
+exec /usr/bin/mv "$@"
 ''')
     for command in ("useradd", "usermod"):
         write_executable(stub_dir / command, common + f'''\nlog {command} "$@"\nexit 97\n''')
 
 
-def configure_case(name: str) -> tuple[Path, Path]:
+def configure_case(
+    name: str,
+    *,
+    sshd_results: tuple[int, ...] = (),
+    reload_results: tuple[int, ...] = (),
+    prior_dropin: bytes | None = None,
+    sshd_unit: bool = True,
+    ssh_unit: bool = True,
+) -> tuple[Path, Path]:
     fixture_root = work / name / "root"
     stub_dir = work / name / "bin"
     calls = fixture_root / "calls"
@@ -143,12 +174,21 @@ def configure_case(name: str) -> tuple[Path, Path]:
     (state / "target-dir-meta").write_text("1000:700\n")
     (state / "target-key-meta").write_text("1000:600\n")
     (state / "root-key-meta").write_text("0:600\n")
+    (state / "SSHD_RESULTS").write_text("".join(f"{result}\n" for result in sshd_results))
+    (state / "RELOAD_RESULTS").write_text("".join(f"{result}\n" for result in reload_results))
+    (state / "SSHD_UNIT").write_text("present\n" if sshd_unit else "absent\n")
+    (state / "SSH_UNIT").write_text("present\n" if ssh_unit else "absent\n")
     (fixture_root / "stub-install").write_text(
         "#!/usr/bin/env bash\nprintf 'install-handoff %s\\n' \"${PANAMA_PATH:-unset}\" >> \"$PANAMA_BOOT_FIXTURE_ROOT/calls\"\n"
     )
     (fixture_root / "stub-install").chmod(0o755)
     make_stubs(stub_dir, fixture_root, calls)
 
+    if prior_dropin is not None:
+        dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf"
+        dropin.write_bytes(prior_dropin)
+        dropin.chmod(0o600)
+
     target_keys = ssh_dir / "authorized_keys"
     root_keys = root_ssh_dir / "authorized_keys"
     if name == "missing":
@@ -187,7 +227,15 @@ def configure_case(name: str) -> tuple[Path, Path]:
     elif name == "relative-home":
         target_keys.write_text("ssh-ed25519 target\n")
         (state / "home").write_text("home/gib\n")
-    elif name == "safe-existing-key":
+    elif name in (
+        "safe-existing-key",
+        "success-without-prior-dropin",
+        "success-replaces-prior-dropin",
+        "candidate-invalid",
+        "candidate-reload-fails",
+        "rollback-validation-fails",
+        "rollback-reload-fails",
+    ):
         target_keys.write_text("ssh-ed25519 target\n")
     elif name == "safe-root-key-copy":
         root_keys.write_text("ssh-ed25519 root\n")
@@ -196,8 +244,8 @@ def configure_case(name: str) -> tuple[Path, Path]:
     return fixture_root, stub_dir
 
 
-def run_case(name: str) -> tuple[int, str, str, Path]:
+def run_case(name: str, **configuration: object) -> tuple[int, str, str, Path]:
-    fixture_root, stub_dir = configure_case(name)
+    fixture_root, stub_dir = configure_case(name, **configuration)
     master, slave = pty.openpty()
 
     def attach_terminal() -> None:
@@ -290,6 +338,192 @@ for case in ("safe-existing-key", "safe-root-key-copy"):
         if not keys.exists() or keys.read_text() != "ssh-ed25519 root\n":
             note("safe-root-key-copy: root key was not copied to the target account")
 
+desired_dropin = b"PermitRootLogin no\nPasswordAuthentication no\n"
+prior_dropin = b"# prior Panama settings\nPasswordAuthentication yes\n"
+transaction_cases = {
+    "success-without-prior-dropin": {
+        "sshd_results": (0,),
+        "reload_results": (0,),
+        "prior_dropin": None,
+        "sshd_unit": True,
+        "ssh_unit": True,
+        "succeeds": True,
+    },
+    "success-replaces-prior-dropin": {
+        "sshd_results": (0,),
+        "reload_results": (0,),
+        "prior_dropin": prior_dropin,
+        "sshd_unit": False,
+        "ssh_unit": True,
+        "succeeds": True,
+    },
+    "candidate-invalid": {
+        "sshd_results": (1, 0),
+        "reload_results": (),
+        "prior_dropin": prior_dropin,
+        "sshd_unit": True,
+        "ssh_unit": True,
+        "succeeds": False,
+    },
+    "candidate-reload-fails": {
+        "sshd_results": (0, 0),
+        "reload_results": (1, 0),
+        "prior_dropin": prior_dropin,
+        "sshd_unit": True,
+        "ssh_unit": True,
+        "succeeds": False,
+    },
+    "rollback-validation-fails": {
+        "sshd_results": (0, 1),
+        "reload_results": (1,),
+        "prior_dropin": prior_dropin,
+        "sshd_unit": True,
+        "ssh_unit": True,
+        "succeeds": False,
+        "rollback_fails": True,
+    },
+    "rollback-reload-fails": {
+        "sshd_results": (0, 0),
+        "reload_results": (1, 1),
+        "prior_dropin": prior_dropin,
+        "sshd_unit": True,
+        "ssh_unit": True,
+        "succeeds": False,
+        "rollback_fails": True,
+    },
+}
+
+for case, expected in transaction_cases.items():
+    configuration = {
+        key: value
+        for key, value in expected.items()
+        if key not in {"succeeds", "rollback_fails"}
+    }
+    status, output, calls, fixture_root = run_case(case, **configuration)
+    call_lines = calls.splitlines()
+    dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf"
+    sshd_dir = dropin.parent
+    validations = [index for index, line in enumerate(call_lines) if line.startswith("sshd -t ")]
+    reloads = [
+        index
+        for index, line in enumerate(call_lines)
+        if line.startswith("systemctl reload ")
+    ]
+    activation_lines = [
+        (index, line)
+        for index, line in enumerate(call_lines)
+        if re.fullmatch(
+            rf"mv -f -- {re.escape(str(sshd_dir))}/\.90-panama\.[A-Za-z0-9]+\.tmp "
+            rf"{re.escape(str(dropin))} source-mode=600 ",
+            line,
+        )
+    ]
+    restore_lines = [
+        index
+        for index, line in enumerate(call_lines)
+        if re.fullmatch(
+            rf"mv -f -- {re.escape(str(sshd_dir))}/\.90-panama\.[A-Za-z0-9]+\.restore "
+            rf"{re.escape(str(dropin))} source-mode=600 ",
+            line,
+        )
+    ]
+
+    succeeds = bool(expected["succeeds"])
+    if succeeds and status != 0:
+        note(f"{case}: transaction stopped with status {status}: {output.strip()}")
+    if not succeeds and status == 0:
+        note(f"{case}: failed transaction returned success")
+    if succeeds and "install-handoff " not in calls:
+        note(f"{case}: successful transaction did not hand off to install")
+    if not succeeds and "install-handoff " in calls:
+        note(f"{case}: failed transaction handed off to install")
+
+    wanted_contents = desired_dropin if succeeds else prior_dropin
+    actual_contents = dropin.read_bytes() if dropin.exists() else None
+    if actual_contents != wanted_contents:
+        note(f"{case}: SSH drop-in contents were not {'activated' if succeeds else 'restored'}")
+
+    if len(activation_lines) != 1:
+        note(f"{case}: candidate was not activated once through a restrictive same-directory rename")
+    if succeeds:
+        if len(validations) != 1 or len(reloads) != 1:
+            note(f"{case}: success did not validate once and reload once")
+        elif activation_lines and not activation_lines[0][0] < validations[0] < reloads[0]:
+            note(f"{case}: success did not activate, validate, then reload")
+    elif case == "candidate-invalid":
+        if len(validations) != 2 or reloads:
+            note(f"{case}: invalid candidate did not validate candidate and restoration without reload")
+        elif activation_lines and restore_lines and not (
+            activation_lines[0][0] < validations[0] < restore_lines[0] < validations[1]
+        ):
+            note(f"{case}: rollback command order was wrong")
+    else:
+        if len(validations) != 2 or len(reloads) != 2:
+            note(f"{case}: reload failure did not validate and reload the restored configuration")
+        elif activation_lines and restore_lines and not (
+            activation_lines[0][0]
+            < validations[0]
+            < reloads[0]
+            < restore_lines[0]
+            < validations[1]
+            < reloads[1]
+        ):
+            note(f"{case}: rollback command order was wrong")
+
+    if succeeds and restore_lines:
+        note(f"{case}: successful transaction performed a rollback")
+    if not succeeds and len(restore_lines) != 1:
+        note(f"{case}: prior drop-in was not restored exactly once")
+
+    detected_unit = "ssh.service" if case == "success-replaces-prior-dropin" else "sshd.service"
+    other_unit = "sshd.service" if detected_unit == "ssh.service" else "ssh.service"
+    reload_lines = [call_lines[index] for index in reloads]
+    if reload_lines and any(line != f"systemctl reload {detected_unit} " for line in reload_lines):
+        note(f"{case}: reloaded a unit other than detected {detected_unit}")
+    if any(line == f"systemctl reload {other_unit} " for line in call_lines):
+        note(f"{case}: guessed {other_unit} after reload failure")
+    if detected_unit == "sshd.service":
+        if "systemctl cat sshd.service " not in call_lines:
+            note(f"{case}: did not detect sshd.service")
+        if "systemctl cat ssh.service " in call_lines:
+            note(f"{case}: probed ssh.service after finding sshd.service")
+    elif not (
+        "systemctl cat sshd.service " in call_lines
+        and "systemctl cat ssh.service " in call_lines
+        and call_lines.index("systemctl cat sshd.service ")
+        < call_lines.index("systemctl cat ssh.service ")
+    ):
+        note(f"{case}: did not fall back from absent sshd.service to ssh.service")
+
+    artifacts = list(sshd_dir.glob(".90-panama.*"))
+    rollback_fails = bool(expected.get("rollback_fails", False))
+    if not rollback_fails and artifacts:
+        note(f"{case}: successful or cleanly rolled-back transaction left temporary artifacts")
+    if rollback_fails:
+        backups = [artifact for artifact in artifacts if artifact.name.endswith(".backup")]
+        if len(backups) != 1:
+            note(f"{case}: rollback failure did not retain exactly one backup")
+        else:
+            backup = backups[0]
+            if backup.parent != sshd_dir or backup.read_bytes() != prior_dropin:
+                note(f"{case}: retained backup was not a same-directory byte copy")
+            if backup.stat().st_mode & 0o777 != 0o600:
+                note(f"{case}: retained backup permissions were not restrictive")
+            if str(backup.resolve()) not in output:
+                note(f"{case}: recovery output omitted the absolute backup path")
+        if "sshd -t" not in output or f"systemctl reload {detected_unit}" not in output:
+            note(f"{case}: recovery output omitted validation or reload commands")
+
+    artifact_logs = [line for line in call_lines if line.startswith("ssh-artifact ")]
+    if expected["prior_dropin"] is not None and not any(
+        re.fullmatch(
+            rf"ssh-artifact {re.escape(str(sshd_dir))}/\.90-panama\.[A-Za-z0-9]+\.backup 600 ",
+            line,
+        )
+        for line in artifact_logs
+    ):
+        note(f"{case}: backup was not collision-safe, same-directory, non-.conf, and restrictive")
+
 if findings:
     print(f"root server bootstrap: {len(findings)} finding(s)", file=sys.stderr)
     for finding in findings: