gib/Panama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix: Roll back failed SSH hardening

Browse Source
This commit is contained in:
Gabriel Brown
2026-08-27 03:44:57 -04:00
parent 20b41c3976
commit d267da58ad
2 changed files with 363 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
boot
+123 -12
View File
@@ -68,23 +68,131 @@ safe_root_authorized_keys() {
grep -qEv '^[[:space:]]*(#|$)' "$keys"
}
harden_server_ssh() {
local username="$1" user_home="$2" sshd_dir sshd_dropin harden
sshd_dir="$(system_path /etc/ssh/sshd_config.d)" || return 1
sshd_dropin="$sshd_dir/90-panama.conf"
if [[ -f "$sshd_dropin" ]]; then
echo "sshd is already hardened ($sshd_dropin)"
detect_ssh_unit() {
local unit
for unit in sshd.service ssh.service; do
systemctl cat "$unit" >/dev/null 2>&1 && {
printf '%s\n' "$unit"
return 0
}
done
return 1
}
restore_ssh_dropin() {
local restore
if [[ -n "${ssh_backup:-}" && -e "$ssh_backup" ]]; then
restore="$(mktemp --tmpdir="$sshd_dir" .90-panama.XXXXXX.restore)" || return 1
if ! cp -a -- "$ssh_backup" "$restore"; then
rm -f -- "$restore"
return 1
fi
if ! mv -f -- "$restore" "$ssh_dropin"; then
rm -f -- "$restore"
return 1
fi
else
rm -f -- "$ssh_dropin"
fi
}
restore_ssh_transaction_traps() {
trap - EXIT INT TERM
[[ -n "${ssh_saved_exit_trap:-}" ]] && eval "$ssh_saved_exit_trap"
[[ -n "${ssh_saved_int_trap:-}" ]] && eval "$ssh_saved_int_trap"
[[ -n "${ssh_saved_term_trap:-}" ]] && eval "$ssh_saved_term_trap"
return 0
}
harden_server_ssh() {
local username="$1" user_home="$2" sshd_dir ssh_dropin harden ssh_unit
local ssh_candidate="" ssh_backup="" rollback_failed=0
local ssh_transaction_active=0
local ssh_saved_exit_trap ssh_saved_int_trap ssh_saved_term_trap
sshd_dir="$(system_path /etc/ssh/sshd_config.d)" || return 1
ssh_dropin="$sshd_dir/90-panama.conf"
printf 'Harden sshd (disable root login and password auth)? [Y/n]: '
read -r harden </dev/tty || harden=""
if [[ ! "$harden" =~ ^[Nn] ]]; then
printf 'PermitRootLogin no\nPasswordAuthentication no\n' >"$sshd_dropin"
systemctl reload sshd 2>/dev/null || systemctl reload ssh 2>/dev/null || true
echo "Wrote $sshd_dropin; make sure your key works before logging out."
if [[ "$harden" =~ ^[Nn] ]]; then
return 0
fi
if ! ssh_unit="$(detect_ssh_unit)"; then
echo "SSH hardening failed: neither sshd.service nor ssh.service exists" >&2
return 1
fi
ssh_candidate="$(umask 077; mktemp --tmpdir="$sshd_dir" .90-panama.XXXXXX.tmp)" || return 1
if ! printf 'PermitRootLogin no\nPasswordAuthentication no\n' >"$ssh_candidate"; then
rm -f -- "$ssh_candidate"
return 1
fi
if [[ -e "$ssh_dropin" ]]; then
ssh_backup="$(umask 077; mktemp --tmpdir="$sshd_dir" .90-panama.XXXXXX.backup)" || {
rm -f -- "$ssh_candidate"
return 1
}
if ! cat -- "$ssh_dropin" >"$ssh_backup"; then
rm -f -- "$ssh_candidate" "$ssh_backup"
return 1
fi
fi
ssh_saved_exit_trap="$(trap -p EXIT)"
ssh_saved_int_trap="$(trap -p INT)"
ssh_saved_term_trap="$(trap -p TERM)"
trap 'if [[ "${ssh_transaction_active:-0}" == 1 ]]; then restore_ssh_dropin || true; fi' EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
ssh_transaction_active=1
if ! mv -f -- "$ssh_candidate" "$ssh_dropin"; then
ssh_transaction_active=0
restore_ssh_transaction_traps
rm -f -- "$ssh_candidate" "$ssh_backup"
return 1
fi
ssh_candidate=""
if ! sshd -t; then
restore_ssh_dropin || rollback_failed=1
sshd -t || rollback_failed=1
ssh_transaction_active=0
restore_ssh_transaction_traps
if (( rollback_failed )); then
printf 'SSH rollback needs manual recovery. Backup: %s\n' "$ssh_backup" >&2
printf ' cp -a -- %q %q\n' "$ssh_backup" "$ssh_dropin" >&2
printf ' sshd -t\n' >&2
printf ' systemctl reload %s\n' "$ssh_unit" >&2
else
[[ -z "$ssh_backup" ]] || rm -f -- "$ssh_backup"
fi
return 1
fi
if ! systemctl reload "$ssh_unit"; then
restore_ssh_dropin || rollback_failed=1
sshd -t || rollback_failed=1
systemctl reload "$ssh_unit" || rollback_failed=1
ssh_transaction_active=0
restore_ssh_transaction_traps
if (( rollback_failed )); then
printf 'SSH rollback needs manual recovery. Backup: %s\n' "$ssh_backup" >&2
printf ' cp -a -- %q %q\n' "$ssh_backup" "$ssh_dropin" >&2
printf ' sshd -t\n' >&2
printf ' systemctl reload %s\n' "$ssh_unit" >&2
else
[[ -z "$ssh_backup" ]] || rm -f -- "$ssh_backup"
fi
return 1
fi
ssh_transaction_active=0
restore_ssh_transaction_traps
[[ -z "$ssh_backup" ]] || rm -f -- "$ssh_backup"
echo "Wrote $ssh_dropin; make sure your key works before logging out."
}
# Panama assumes Fedora's repositories and package names.
@@ -175,7 +283,10 @@ if [[ "$(id -u)" -eq 0 ]]; then
fi
if safe_authorized_keys "$username" "$user_home"; then
harden_server_ssh "$username" "$user_home"
if ! harden_server_ssh "$username" "$user_home"; then
echo "SSH hardening failed; stopping before install handoff." >&2
exit 1
fi
else
echo "SSH hardening unavailable: $username has no safe authorized_keys" >&2
fi
tests/setup/root-server-bootstrap-contract
+240 -6
View File
@@ -20,6 +20,7 @@ import errno
import fcntl
import os
import pty
import re
import shutil
import subprocess
import sys
@@ -50,6 +51,17 @@ log() {{
local argument
{{ for argument in "$@"; do printf '%q ' "$argument"; done; printf '\\n'; }} >>"$calls"
}}
consume_result() {{
local name="$1" results result
results="$PANAMA_BOOT_FIXTURE_ROOT/state/$name"
if ! IFS= read -r result <"$results"; then
return 0
fi
/usr/bin/tail -n +2 "$results" >"$results.next"
/usr/bin/mv -f -- "$results.next" "$results"
[[ "$result" =~ ^[0-9]+$ ]] || exit 97
return "$result"
}}
'''
write_executable(stub_dir / "id", common + r'''
@@ -112,20 +124,39 @@ log dnf "$@"
''')
write_executable(stub_dir / "sshd", common + r'''
log sshd "$@"
exit 97
[[ "$#" -eq 1 && "$1" == -t ]] || exit 97
for artifact in "$PANAMA_BOOT_FIXTURE_ROOT/etc/ssh/sshd_config.d"/.90-panama.*; do
[[ -e "$artifact" ]] || continue
log ssh-artifact "$artifact" "$(/usr/bin/stat -c %a -- "$artifact")"
done
consume_result SSHD_RESULTS
''')
write_executable(stub_dir / "systemctl", common + r'''
log systemctl "$@"
case "${1:-}:${2:-}" in
reload:sshd|reload:ssh) exit 0 ;;
cat:sshd.service) [[ "$(<"$PANAMA_BOOT_FIXTURE_ROOT/state/SSHD_UNIT")" == present ]] ;;
cat:ssh.service) [[ "$(<"$PANAMA_BOOT_FIXTURE_ROOT/state/SSH_UNIT")" == present ]] ;;
reload:sshd.service|reload:ssh.service) consume_result RELOAD_RESULTS ;;
*) exit 97 ;;
esac
''')
write_executable(stub_dir / "mv", common + r'''
log mv "$@" "source-mode=$(/usr/bin/stat -c %a -- "${3:-}")"
exec /usr/bin/mv "$@"
''')
for command in ("useradd", "usermod"):
write_executable(stub_dir / command, common + f'''\nlog {command} "$@"\nexit 97\n''')
def configure_case(name: str) -> tuple[Path, Path]:
def configure_case(
name: str,
*,
sshd_results: tuple[int, ...] = (),
reload_results: tuple[int, ...] = (),
prior_dropin: bytes | None = None,
sshd_unit: bool = True,
ssh_unit: bool = True,
) -> tuple[Path, Path]:
fixture_root = work / name / "root"
stub_dir = work / name / "bin"
calls = fixture_root / "calls"
@@ -143,12 +174,21 @@ def configure_case(name: str) -> tuple[Path, Path]:
(state / "target-dir-meta").write_text("1000:700\n")
(state / "target-key-meta").write_text("1000:600\n")
(state / "root-key-meta").write_text("0:600\n")
(state / "SSHD_RESULTS").write_text("".join(f"{result}\n" for result in sshd_results))
(state / "RELOAD_RESULTS").write_text("".join(f"{result}\n" for result in reload_results))
(state / "SSHD_UNIT").write_text("present\n" if sshd_unit else "absent\n")
(state / "SSH_UNIT").write_text("present\n" if ssh_unit else "absent\n")
(fixture_root / "stub-install").write_text(
"#!/usr/bin/env bash\nprintf 'install-handoff %s\\n' \"${PANAMA_PATH:-unset}\" >> \"$PANAMA_BOOT_FIXTURE_ROOT/calls\"\n"
)
(fixture_root / "stub-install").chmod(0o755)
make_stubs(stub_dir, fixture_root, calls)
if prior_dropin is not None:
dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf"
dropin.write_bytes(prior_dropin)
dropin.chmod(0o600)
target_keys = ssh_dir / "authorized_keys"
root_keys = root_ssh_dir / "authorized_keys"
if name == "missing":
@@ -187,7 +227,15 @@ def configure_case(name: str) -> tuple[Path, Path]:
elif name == "relative-home":
target_keys.write_text("ssh-ed25519 target\n")
(state / "home").write_text("home/gib\n")
elif name == "safe-existing-key":
elif name in (
"safe-existing-key",
"success-without-prior-dropin",
"success-replaces-prior-dropin",
"candidate-invalid",
"candidate-reload-fails",
"rollback-validation-fails",
"rollback-reload-fails",
):
target_keys.write_text("ssh-ed25519 target\n")
elif name == "safe-root-key-copy":
root_keys.write_text("ssh-ed25519 root\n")
@@ -196,8 +244,8 @@ def configure_case(name: str) -> tuple[Path, Path]:
return fixture_root, stub_dir
def run_case(name: str) -> tuple[int, str, str, Path]:
fixture_root, stub_dir = configure_case(name)
def run_case(name: str, **configuration: object) -> tuple[int, str, str, Path]:
fixture_root, stub_dir = configure_case(name, **configuration)
master, slave = pty.openpty()
def attach_terminal() -> None:
@@ -290,6 +338,192 @@ for case in ("safe-existing-key", "safe-root-key-copy"):
if not keys.exists() or keys.read_text() != "ssh-ed25519 root\n":
note("safe-root-key-copy: root key was not copied to the target account")
desired_dropin = b"PermitRootLogin no\nPasswordAuthentication no\n"
prior_dropin = b"# prior Panama settings\nPasswordAuthentication yes\n"
transaction_cases = {
"success-without-prior-dropin": {
"sshd_results": (0,),
"reload_results": (0,),
"prior_dropin": None,
"sshd_unit": True,
"ssh_unit": True,
"succeeds": True,
},
"success-replaces-prior-dropin": {
"sshd_results": (0,),
"reload_results": (0,),
"prior_dropin": prior_dropin,
"sshd_unit": False,
"ssh_unit": True,
"succeeds": True,
},
"candidate-invalid": {
"sshd_results": (1, 0),
"reload_results": (),
"prior_dropin": prior_dropin,
"sshd_unit": True,
"ssh_unit": True,
"succeeds": False,
},
"candidate-reload-fails": {
"sshd_results": (0, 0),
"reload_results": (1, 0),
"prior_dropin": prior_dropin,
"sshd_unit": True,
"ssh_unit": True,
"succeeds": False,
},
"rollback-validation-fails": {
"sshd_results": (0, 1),
"reload_results": (1,),
"prior_dropin": prior_dropin,
"sshd_unit": True,
"ssh_unit": True,
"succeeds": False,
"rollback_fails": True,
},
"rollback-reload-fails": {
"sshd_results": (0, 0),
"reload_results": (1, 1),
"prior_dropin": prior_dropin,
"sshd_unit": True,
"ssh_unit": True,
"succeeds": False,
"rollback_fails": True,
},
}
for case, expected in transaction_cases.items():
configuration = {
key: value
for key, value in expected.items()
if key not in {"succeeds", "rollback_fails"}
}
status, output, calls, fixture_root = run_case(case, **configuration)
call_lines = calls.splitlines()
dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf"
sshd_dir = dropin.parent
validations = [index for index, line in enumerate(call_lines) if line.startswith("sshd -t ")]
reloads = [
index
for index, line in enumerate(call_lines)
if line.startswith("systemctl reload ")
]
activation_lines = [
(index, line)
for index, line in enumerate(call_lines)
if re.fullmatch(
rf"mv -f -- {re.escape(str(sshd_dir))}/\.90-panama\.[A-Za-z0-9]+\.tmp "
rf"{re.escape(str(dropin))} source-mode=600 ",
line,
)
]
restore_lines = [
index
for index, line in enumerate(call_lines)
if re.fullmatch(
rf"mv -f -- {re.escape(str(sshd_dir))}/\.90-panama\.[A-Za-z0-9]+\.restore "
rf"{re.escape(str(dropin))} source-mode=600 ",
line,
)
]
succeeds = bool(expected["succeeds"])
if succeeds and status != 0:
note(f"{case}: transaction stopped with status {status}: {output.strip()}")
if not succeeds and status == 0:
note(f"{case}: failed transaction returned success")
if succeeds and "install-handoff " not in calls:
note(f"{case}: successful transaction did not hand off to install")
if not succeeds and "install-handoff " in calls:
note(f"{case}: failed transaction handed off to install")
wanted_contents = desired_dropin if succeeds else prior_dropin
actual_contents = dropin.read_bytes() if dropin.exists() else None
if actual_contents != wanted_contents:
note(f"{case}: SSH drop-in contents were not {'activated' if succeeds else 'restored'}")
if len(activation_lines) != 1:
note(f"{case}: candidate was not activated once through a restrictive same-directory rename")
if succeeds:
if len(validations) != 1 or len(reloads) != 1:
note(f"{case}: success did not validate once and reload once")
elif activation_lines and not activation_lines[0][0] < validations[0] < reloads[0]:
note(f"{case}: success did not activate, validate, then reload")
elif case == "candidate-invalid":
if len(validations) != 2 or reloads:
note(f"{case}: invalid candidate did not validate candidate and restoration without reload")
elif activation_lines and restore_lines and not (
activation_lines[0][0] < validations[0] < restore_lines[0] < validations[1]
):
note(f"{case}: rollback command order was wrong")
else:
if len(validations) != 2 or len(reloads) != 2:
note(f"{case}: reload failure did not validate and reload the restored configuration")
elif activation_lines and restore_lines and not (
activation_lines[0][0]
< validations[0]
< reloads[0]
< restore_lines[0]
< validations[1]
< reloads[1]
):
note(f"{case}: rollback command order was wrong")
if succeeds and restore_lines:
note(f"{case}: successful transaction performed a rollback")
if not succeeds and len(restore_lines) != 1:
note(f"{case}: prior drop-in was not restored exactly once")
detected_unit = "ssh.service" if case == "success-replaces-prior-dropin" else "sshd.service"
other_unit = "sshd.service" if detected_unit == "ssh.service" else "ssh.service"
reload_lines = [call_lines[index] for index in reloads]
if reload_lines and any(line != f"systemctl reload {detected_unit} " for line in reload_lines):
note(f"{case}: reloaded a unit other than detected {detected_unit}")
if any(line == f"systemctl reload {other_unit} " for line in call_lines):
note(f"{case}: guessed {other_unit} after reload failure")
if detected_unit == "sshd.service":
if "systemctl cat sshd.service " not in call_lines:
note(f"{case}: did not detect sshd.service")
if "systemctl cat ssh.service " in call_lines:
note(f"{case}: probed ssh.service after finding sshd.service")
elif not (
"systemctl cat sshd.service " in call_lines
and "systemctl cat ssh.service " in call_lines
and call_lines.index("systemctl cat sshd.service ")
< call_lines.index("systemctl cat ssh.service ")
):
note(f"{case}: did not fall back from absent sshd.service to ssh.service")
artifacts = list(sshd_dir.glob(".90-panama.*"))
rollback_fails = bool(expected.get("rollback_fails", False))
if not rollback_fails and artifacts:
note(f"{case}: successful or cleanly rolled-back transaction left temporary artifacts")
if rollback_fails:
backups = [artifact for artifact in artifacts if artifact.name.endswith(".backup")]
if len(backups) != 1:
note(f"{case}: rollback failure did not retain exactly one backup")
else:
backup = backups[0]
if backup.parent != sshd_dir or backup.read_bytes() != prior_dropin:
note(f"{case}: retained backup was not a same-directory byte copy")
if backup.stat().st_mode & 0o777 != 0o600:
note(f"{case}: retained backup permissions were not restrictive")
if str(backup.resolve()) not in output:
note(f"{case}: recovery output omitted the absolute backup path")
if "sshd -t" not in output or f"systemctl reload {detected_unit}" not in output:
note(f"{case}: recovery output omitted validation or reload commands")
artifact_logs = [line for line in call_lines if line.startswith("ssh-artifact ")]
if expected["prior_dropin"] is not None and not any(
re.fullmatch(
rf"ssh-artifact {re.escape(str(sshd_dir))}/\.90-panama\.[A-Za-z0-9]+\.backup 600 ",
line,
)
for line in artifact_logs
):
note(f"{case}: backup was not collision-safe, same-directory, non-.conf, and restrictive")
if findings:
print(f"root server bootstrap: {len(findings)} finding(s)", file=sys.stderr)
for finding in findings: