#!/usr/bin/env bash

# `boot --server` is deliberately public and must be safe before it reaches the
# cloned repository. Exercise its root branch through a PTY, against only a
# temporary filesystem and PATH adapters.

set -uo pipefail

repo_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
boot="$repo_dir/boot"

[[ -x "$boot" ]] || {
    printf 'root server bootstrap: %s is not executable\n' "$boot" >&2
    exit 1
}

python3 - "$boot" <<'PY'
import atexit
import errno
import fcntl
import os
import pty
import re
import signal
import shutil
import subprocess
import sys
import tempfile
import termios
import time
from pathlib import Path

boot = sys.argv[1]
work = Path(tempfile.mkdtemp())
atexit.register(shutil.rmtree, work, ignore_errors=True)
findings: list[str] = []


def note(message: str) -> None:
    findings.append(message)


def write_executable(path: Path, contents: str) -> None:
    path.write_text(contents)
    path.chmod(0o755)


def make_stubs(stub_dir: Path, fixture_root: Path, calls: Path) -> None:
    common = f'''#!/usr/bin/env bash
set -u
calls={str(calls)!r}
log() {{
  local argument
  {{ for argument in "$@"; do printf '%q ' "$argument"; done; printf '\\n'; }} >>"$calls"
}}
consume_result() {{
  local name="$1" results result
  results="$PANAMA_BOOT_FIXTURE_ROOT/state/$name"
  if ! IFS= read -r result <"$results"; then
    return 0
  fi
  /usr/bin/tail -n +2 "$results" >"$results.next"
  /usr/bin/mv -f -- "$results.next" "$results"
  [[ "$result" =~ ^[0-9]+$ ]] || exit 97
  return "$result"
}}
'''

    write_executable(stub_dir / "id", common + r'''
log id "$@"
case "${1:-}" in
  -u)
    case "${2:-}" in
      '') printf '0\n' ;;
      root) printf '0\n' ;;
      gib) cat "$PANAMA_BOOT_FIXTURE_ROOT/state/target-uid" ;;
      *) exit 97 ;;
    esac
    ;;
  -nG) [[ "${2:-}" == gib ]] || exit 97; printf 'gib wheel\n' ;;
  *) exit 97 ;;
esac
''')
    write_executable(stub_dir / "passwd", common + r'''
log passwd "$@"
[[ "${1:-}" == -S && "${2:-}" == gib ]] || exit 97
printf 'gib PS\n'
''')
    write_executable(stub_dir / "getent", common + r'''
log getent "$@"
[[ "${1:-}" == passwd && "${2:-}" == gib ]] || exit 97
home="$(<"$PANAMA_BOOT_FIXTURE_ROOT/state/home")"
printf 'gib:x:1000:1000::%s:/bin/bash\n' "$home"
''')
    write_executable(stub_dir / "stat", common + r'''
log stat "$@"
[[ "${1:-}" == -Lc && "${2:-}" == '%u:%a' ]] || exit 97
case "${3:-}" in
  "$PANAMA_BOOT_FIXTURE_ROOT/home/gib/.ssh") cat "$PANAMA_BOOT_FIXTURE_ROOT/state/target-dir-meta" ;;
  "$PANAMA_BOOT_FIXTURE_ROOT/home/gib/.ssh/authorized_keys") cat "$PANAMA_BOOT_FIXTURE_ROOT/state/target-key-meta" ;;
  "$PANAMA_BOOT_FIXTURE_ROOT/root/.ssh/authorized_keys") cat "$PANAMA_BOOT_FIXTURE_ROOT/state/root-key-meta" ;;
  *) exit 97 ;;
esac
''')
    write_executable(stub_dir / "runuser", common + r'''
log runuser "$@"
[[ "${1:-}" == -u && "${2:-}" == gib && "${3:-}" == -- ]] || exit 97
shift 3
"$@"
''')
    write_executable(stub_dir / "git", common + r'''
log git "$@"
case "${1:-}" in
  clone)
    mkdir -p "$3/.git"
    cp "$PANAMA_BOOT_FIXTURE_ROOT/stub-install" "$3/install"
    chmod +x "$3/install"
    ;;
  -C) [[ "${3:-}" == pull && "${4:-}" == --ff-only ]] || exit 97 ;;
  *) exit 97 ;;
esac
''')
    write_executable(stub_dir / "dnf", common + r'''
log dnf "$@"
[[ "${1:-}" == install && "${2:-}" == -y && "${3:-}" == git ]] || exit 97
''')
    write_executable(stub_dir / "sshd", common + r'''
log sshd "$@"
[[ "$#" -eq 1 && "$1" == -t ]] || exit 97
for artifact in "$PANAMA_BOOT_FIXTURE_ROOT/etc/ssh/sshd_config.d"/.90-panama.*; do
  [[ -e "$artifact" ]] || continue
  log ssh-artifact "$artifact" "$(/usr/bin/stat -c %a -- "$artifact")"
done
consume_result SSHD_RESULTS
''')
    write_executable(stub_dir / "systemctl", common + r'''
log systemctl "$@"
case "${1:-}:${2:-}" in
  cat:sshd.service) [[ "$(<"$PANAMA_BOOT_FIXTURE_ROOT/state/SSHD_UNIT")" == present ]] ;;
  cat:ssh.service) [[ "$(<"$PANAMA_BOOT_FIXTURE_ROOT/state/SSH_UNIT")" == present ]] ;;
  reload:sshd.service|reload:ssh.service) consume_result RELOAD_RESULTS ;;
  *) exit 97 ;;
esac
''')
    write_executable(stub_dir / "mv", common + r'''
log mv "$@" "source-mode=$(/usr/bin/stat -c %a -- "${3:-}")"
if [[ "${3:-}" == *.tmp ]]; then
  consume_result MV_ACTIVATION_RESULTS
  result=$?
  (( result == 0 )) || exit "$result"
fi
/usr/bin/mv "$@"
if [[ "${3:-}" == *.tmp && -e "$PANAMA_BOOT_FIXTURE_ROOT/state/HOLD_ACTIVATION" ]]; then
  : >"$PANAMA_BOOT_FIXTURE_ROOT/state/ACTIVATED"
  while [[ ! -e "$PANAMA_BOOT_FIXTURE_ROOT/state/RELEASE_ACTIVATION" ]]; do
    /usr/bin/sleep 0.01
  done
fi
''')
    write_executable(stub_dir / "rm", common + r'''
log rm "$@"
if [[ "${1:-}" == -f && "${2:-}" == -- && "${3:-}" == *.backup ]]; then
  consume_result RM_BACKUP_RESULTS
  result=$?
  (( result == 0 )) || exit "$result"
fi
if [[ "${1:-}" == -f && "${2:-}" == -- && "${3:-}" == *.tmp ]]; then
  consume_result RM_CANDIDATE_RESULTS
  result=$?
  (( result == 0 )) || exit "$result"
fi
exec /usr/bin/rm "$@"
''')
    for command in ("useradd", "usermod"):
        write_executable(stub_dir / command, common + f'''\nlog {command} "$@"\nexit 97\n''')


def configure_case(
    name: str,
    *,
    sshd_results: tuple[int, ...] = (),
    reload_results: tuple[int, ...] = (),
    mv_activation_results: tuple[int, ...] = (),
    rm_backup_results: tuple[int, ...] = (),
    rm_candidate_results: tuple[int, ...] = (),
    prior_dropin: bytes | None = None,
    sshd_unit: bool = True,
    ssh_unit: bool = True,
    hold_activation: bool = False,
) -> tuple[Path, Path]:
    fixture_root = work / name / "root"
    stub_dir = work / name / "bin"
    calls = fixture_root / "calls"
    state = fixture_root / "state"
    ssh_dir = fixture_root / "home/gib/.ssh"
    root_ssh_dir = fixture_root / "root/.ssh"
    (fixture_root / "etc/ssh/sshd_config.d").mkdir(parents=True)
    ssh_dir.mkdir(parents=True)
    root_ssh_dir.mkdir(parents=True)
    stub_dir.mkdir(parents=True)
    state.mkdir()
    calls.touch()
    (state / "target-uid").write_text("1000\n")
    (state / "home").write_text("/home/gib\n")
    (state / "target-dir-meta").write_text("1000:700\n")
    (state / "target-key-meta").write_text("1000:600\n")
    (state / "root-key-meta").write_text("0:600\n")
    (state / "SSHD_RESULTS").write_text("".join(f"{result}\n" for result in sshd_results))
    (state / "RELOAD_RESULTS").write_text("".join(f"{result}\n" for result in reload_results))
    (state / "MV_ACTIVATION_RESULTS").write_text(
        "".join(f"{result}\n" for result in mv_activation_results)
    )
    (state / "RM_BACKUP_RESULTS").write_text(
        "".join(f"{result}\n" for result in rm_backup_results)
    )
    (state / "RM_CANDIDATE_RESULTS").write_text(
        "".join(f"{result}\n" for result in rm_candidate_results)
    )
    (state / "SSHD_UNIT").write_text("present\n" if sshd_unit else "absent\n")
    (state / "SSH_UNIT").write_text("present\n" if ssh_unit else "absent\n")
    if hold_activation:
        (state / "HOLD_ACTIVATION").touch()
    (fixture_root / "stub-install").write_text(
        "#!/usr/bin/env bash\nprintf 'install-handoff %s\\n' \"${PANAMA_PATH:-unset}\" >> \"$PANAMA_BOOT_FIXTURE_ROOT/calls\"\n"
    )
    (fixture_root / "stub-install").chmod(0o755)
    make_stubs(stub_dir, fixture_root, calls)

    if prior_dropin is not None:
        dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf"
        dropin.write_bytes(prior_dropin)
        dropin.chmod(0o600)

    target_keys = ssh_dir / "authorized_keys"
    root_keys = root_ssh_dir / "authorized_keys"
    if name == "missing":
        pass
    elif name == "empty":
        target_keys.touch()
    elif name == "comment-only":
        target_keys.write_text("# no usable key\n\n")
    elif name == "ssh-directory-symlink":
        shutil.rmtree(ssh_dir)
        alternate = fixture_root / "unsafe-ssh"
        alternate.mkdir()
        (fixture_root / "home/gib/.ssh").symlink_to(alternate)
    elif name == "authorized-keys-symlink":
        alternate = fixture_root / "unsafe-authorized-keys"
        alternate.write_text("ssh-ed25519 unsafe\n")
        target_keys.symlink_to(alternate)
    elif name == "directory-wrong-mode":
        target_keys.write_text("ssh-ed25519 target\n")
        (state / "target-dir-meta").write_text("1000:755\n")
    elif name == "root-copy-directory-wrong-mode":
        root_keys.write_text("ssh-ed25519 root\n")
        (state / "target-dir-meta").write_text("1000:755\n")
    elif name == "file-wrong-mode":
        target_keys.write_text("ssh-ed25519 target\n")
        (state / "target-key-meta").write_text("1000:644\n")
    elif name == "directory-wrong-owner":
        target_keys.write_text("ssh-ed25519 target\n")
        (state / "target-dir-meta").write_text("0:700\n")
    elif name == "file-wrong-owner":
        target_keys.write_text("ssh-ed25519 target\n")
        (state / "target-key-meta").write_text("0:600\n")
    elif name == "root-target-account":
        target_keys.write_text("ssh-ed25519 target\n")
        (state / "target-uid").write_text("0\n")
    elif name == "relative-home":
        target_keys.write_text("ssh-ed25519 target\n")
        (state / "home").write_text("home/gib\n")
    elif name in (
        "safe-existing-key",
        "success-without-prior-dropin",
        "success-replaces-prior-dropin",
        "candidate-invalid",
        "candidate-invalid-without-prior",
        "candidate-reload-fails",
        "candidate-reload-fails-without-prior",
        "rollback-validation-fails",
        "rollback-reload-fails",
        "success-backup-cleanup-fails",
        "rollback-backup-cleanup-fails",
        "signal-int-restores-prior",
        "signal-term-removes-new-dropin",
        "candidate-cleanup-fails",
    ):
        target_keys.write_text("ssh-ed25519 target\n")
    elif name == "safe-root-key-copy":
        root_keys.write_text("ssh-ed25519 root\n")
    else:
        raise ValueError(name)
    return fixture_root, stub_dir


def run_case(
    name: str,
    *,
    signal_after_activation: int | None = None,
    prior_traps: bool = False,
    **configuration: object,
) -> tuple[int, str, str, Path, int]:
    fixture_root, stub_dir = configure_case(
        name,
        hold_activation=signal_after_activation is not None,
        **configuration,
    )
    master, slave = pty.openpty()

    def attach_terminal() -> None:
        # The test runner launches contracts as background jobs, which inherit
        # SIGINT ignored. A real interactive bootstrap starts with SIGINT at
        # its default disposition, so restore that state before exec.
        signal.signal(signal.SIGINT, signal.SIG_DFL)
        fcntl.ioctl(0, termios.TIOCSCTTY, 0)

    env = {
        **os.environ,
        "PATH": f"{stub_dir}:/usr/bin:/bin",
        "PANAMA_BOOT_FIXTURE_ROOT": str(fixture_root),
        "PANAMA_PATH": f"{fixture_root}/home/gib/.local/share/Panama",
        "HOME": f"{fixture_root}/root",
    }
    if prior_traps:
        bash_env = fixture_root / "prior-traps"
        bash_env.write_text(
            '''if [[ "$0" == "$PANAMA_BOOT_SCRIPT" ]]; then
trap 'printf "prior-exit %s\\n" "$BASHPID" >>"$PANAMA_BOOT_FIXTURE_ROOT/calls"' EXIT
trap 'printf "prior-int %s\\n" "$BASHPID" >>"$PANAMA_BOOT_FIXTURE_ROOT/calls"' INT
trap 'printf "prior-term %s\\n" "$BASHPID" >>"$PANAMA_BOOT_FIXTURE_ROOT/calls"' TERM
fi
'''
        )
        env["BASH_ENV"] = str(bash_env)
        env["PANAMA_BOOT_SCRIPT"] = boot
    process = subprocess.Popen(
        ["bash", boot, "--server"],
        stdin=slave,
        stdout=slave,
        stderr=slave,
        env=env,
        start_new_session=True,
        preexec_fn=attach_terminal,
    )
    os.close(slave)
    os.write(master, b"gib\nY\n")
    if signal_after_activation is not None:
        activation = fixture_root / "state/ACTIVATED"
        deadline = time.monotonic() + 5
        while not activation.exists() and process.poll() is None and time.monotonic() < deadline:
            time.sleep(0.01)
        if not activation.exists():
            note(f"{name}: fixture did not observe atomic activation before signaling")
        else:
            os.kill(process.pid, signal_after_activation)
        (fixture_root / "state/RELEASE_ACTIVATION").touch()
    chunks: list[bytes] = []
    while True:
        try:
            chunk = os.read(master, 4096)
        except OSError as error:
            if error.errno == errno.EIO:
                break
            raise
        if not chunk:
            break
        chunks.append(chunk)
    os.close(master)
    status = process.wait()
    calls = (fixture_root / "calls").read_text()
    output = b"".join(chunks).decode(errors="replace")
    return status, output, calls, fixture_root, process.pid


unsafe_cases = (
    "missing",
    "empty",
    "comment-only",
    "ssh-directory-symlink",
    "authorized-keys-symlink",
    "directory-wrong-mode",
    "root-copy-directory-wrong-mode",
    "file-wrong-mode",
    "directory-wrong-owner",
    "file-wrong-owner",
    "root-target-account",
    "relative-home",
)
for case in unsafe_cases:
    status, output, calls, fixture_root, _ = run_case(case)
    if status != 0:
        note(f"{case}: bootstrap stopped with status {status}: {output.strip()}")
    if "SSH hardening unavailable" not in output:
        note(f"{case}: unsafe login path did not explain why hardening was unavailable")
    if "sshd -t" in calls:
        note(f"{case}: unsafe login path validated sshd")
    if "systemctl reload" in calls:
        note(f"{case}: unsafe login path reloaded SSH")
    if (fixture_root / "etc/ssh/sshd_config.d/90-panama.conf").exists():
        note(f"{case}: unsafe login path changed the SSH drop-in")
    if case == "root-copy-directory-wrong-mode" and (
        fixture_root / "home/gib/.ssh/authorized_keys"
    ).exists():
        note("root-copy-directory-wrong-mode: copied a root key into an unsafe SSH directory")
    if "install-handoff " not in calls:
        note(f"{case}: unsafe login path did not hand off to install")

for case in ("safe-existing-key", "safe-root-key-copy"):
    status, output, calls, fixture_root, _ = run_case(case)
    if status != 0:
        note(f"{case}: safe login path stopped with status {status}: {output.strip()}")
    if "SSH hardening unavailable" in output:
        note(f"{case}: safe login path was rejected")
    if "systemctl reload" not in calls:
        note(f"{case}: safe login path did not reach SSH hardening")
    if "install-handoff " not in calls:
        note(f"{case}: safe login path did not hand off to install")
    dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf"
    if (dropin.read_text() if dropin.exists() else "") != "PermitRootLogin no\nPasswordAuthentication no\n":
        note(f"{case}: safe login path did not write the expected SSH drop-in")
    if case == "safe-root-key-copy":
        keys = fixture_root / "home/gib/.ssh/authorized_keys"
        if not keys.exists() or keys.read_text() != "ssh-ed25519 root\n":
            note("safe-root-key-copy: root key was not copied to the target account")

desired_dropin = b"PermitRootLogin no\nPasswordAuthentication no\n"
prior_dropin = b"# prior Panama settings\nPasswordAuthentication yes\n"
transaction_cases = {
    "success-without-prior-dropin": {
        "sshd_results": (0,),
        "reload_results": (0,),
        "prior_dropin": None,
        "sshd_unit": True,
        "ssh_unit": True,
        "succeeds": True,
    },
    "success-replaces-prior-dropin": {
        "sshd_results": (0,),
        "reload_results": (0,),
        "prior_dropin": prior_dropin,
        "sshd_unit": False,
        "ssh_unit": True,
        "succeeds": True,
    },
    "candidate-invalid": {
        "sshd_results": (1, 0),
        "reload_results": (),
        "prior_dropin": prior_dropin,
        "sshd_unit": True,
        "ssh_unit": True,
        "succeeds": False,
    },
    "candidate-invalid-without-prior": {
        "sshd_results": (1, 0),
        "reload_results": (),
        "prior_dropin": None,
        "sshd_unit": True,
        "ssh_unit": True,
        "succeeds": False,
    },
    "candidate-reload-fails": {
        "sshd_results": (0, 0),
        "reload_results": (1, 0),
        "prior_dropin": prior_dropin,
        "sshd_unit": True,
        "ssh_unit": True,
        "succeeds": False,
    },
    "candidate-reload-fails-without-prior": {
        "sshd_results": (0, 0),
        "reload_results": (1, 0),
        "prior_dropin": None,
        "sshd_unit": True,
        "ssh_unit": True,
        "succeeds": False,
    },
    "rollback-validation-fails": {
        "sshd_results": (0, 1),
        "reload_results": (1,),
        "prior_dropin": prior_dropin,
        "sshd_unit": True,
        "ssh_unit": True,
        "succeeds": False,
        "rollback_fails": True,
    },
    "rollback-reload-fails": {
        "sshd_results": (0, 0),
        "reload_results": (1, 1),
        "prior_dropin": prior_dropin,
        "sshd_unit": True,
        "ssh_unit": True,
        "succeeds": False,
        "rollback_fails": True,
    },
}

for case, expected in transaction_cases.items():
    configuration = {
        key: value
        for key, value in expected.items()
        if key not in {"succeeds", "rollback_fails"}
    }
    status, output, calls, fixture_root, _ = run_case(case, **configuration)
    call_lines = calls.splitlines()
    dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf"
    sshd_dir = dropin.parent
    validations = [index for index, line in enumerate(call_lines) if line.startswith("sshd -t ")]
    reloads = [
        index
        for index, line in enumerate(call_lines)
        if line.startswith("systemctl reload ")
    ]
    activation_lines = [
        (index, line)
        for index, line in enumerate(call_lines)
        if re.fullmatch(
            rf"mv -f -- {re.escape(str(sshd_dir))}/\.90-panama\.[A-Za-z0-9]+\.tmp "
            rf"{re.escape(str(dropin))} source-mode=600 ",
            line,
        )
    ]
    restore_lines = [
        index
        for index, line in enumerate(call_lines)
        if re.fullmatch(
            rf"mv -f -- {re.escape(str(sshd_dir))}/\.90-panama\.[A-Za-z0-9]+\.restore "
            rf"{re.escape(str(dropin))} source-mode=600 ",
            line,
        )
    ]
    removal_lines = [
        index
        for index, line in enumerate(call_lines)
        if line == f"rm -f -- {dropin} "
    ]
    rollback_lines = restore_lines if expected["prior_dropin"] is not None else removal_lines

    succeeds = bool(expected["succeeds"])
    if succeeds and status != 0:
        note(f"{case}: transaction stopped with status {status}: {output.strip()}")
    if not succeeds and status == 0:
        note(f"{case}: failed transaction returned success")
    if succeeds and "install-handoff " not in calls:
        note(f"{case}: successful transaction did not hand off to install")
    if not succeeds and "install-handoff " in calls:
        note(f"{case}: failed transaction handed off to install")

    wanted_contents = desired_dropin if succeeds else expected["prior_dropin"]
    actual_contents = dropin.read_bytes() if dropin.exists() else None
    if actual_contents != wanted_contents:
        note(f"{case}: SSH drop-in contents were not {'activated' if succeeds else 'restored'}")

    if len(activation_lines) != 1:
        note(f"{case}: candidate was not activated once through a restrictive same-directory rename")
    if succeeds:
        if len(validations) != 1 or len(reloads) != 1:
            note(f"{case}: success did not validate once and reload once")
        elif activation_lines and not activation_lines[0][0] < validations[0] < reloads[0]:
            note(f"{case}: success did not activate, validate, then reload")
    elif case in {"candidate-invalid", "candidate-invalid-without-prior"}:
        if len(validations) != 2 or reloads:
            note(f"{case}: invalid candidate did not validate candidate and restoration without reload")
        elif activation_lines and rollback_lines and not (
            activation_lines[0][0] < validations[0] < rollback_lines[0] < validations[1]
        ):
            note(f"{case}: rollback command order was wrong")
    else:
        if len(validations) != 2 or len(reloads) != 2:
            note(f"{case}: reload failure did not validate and reload the restored configuration")
        elif activation_lines and rollback_lines and not (
            activation_lines[0][0]
            < validations[0]
            < reloads[0]
            < rollback_lines[0]
            < validations[1]
            < reloads[1]
        ):
            note(f"{case}: rollback command order was wrong")

    if succeeds and restore_lines:
        note(f"{case}: successful transaction performed a rollback")
    if not succeeds:
        if len(rollback_lines) != 1:
            note(f"{case}: pre-transaction SSH state was not restored exactly once")

    detected_unit = "ssh.service" if case == "success-replaces-prior-dropin" else "sshd.service"
    other_unit = "sshd.service" if detected_unit == "ssh.service" else "ssh.service"
    reload_lines = [call_lines[index] for index in reloads]
    if reload_lines and any(line != f"systemctl reload {detected_unit} " for line in reload_lines):
        note(f"{case}: reloaded a unit other than detected {detected_unit}")
    if any(line == f"systemctl reload {other_unit} " for line in call_lines):
        note(f"{case}: guessed {other_unit} after reload failure")
    if detected_unit == "sshd.service":
        if "systemctl cat sshd.service " not in call_lines:
            note(f"{case}: did not detect sshd.service")
        if "systemctl cat ssh.service " in call_lines:
            note(f"{case}: probed ssh.service after finding sshd.service")
    elif not (
        "systemctl cat sshd.service " in call_lines
        and "systemctl cat ssh.service " in call_lines
        and call_lines.index("systemctl cat sshd.service ")
        < call_lines.index("systemctl cat ssh.service ")
    ):
        note(f"{case}: did not fall back from absent sshd.service to ssh.service")

    artifacts = list(sshd_dir.glob(".90-panama.*"))
    rollback_fails = bool(expected.get("rollback_fails", False))
    if not rollback_fails and artifacts:
        note(f"{case}: successful or cleanly rolled-back transaction left temporary artifacts")
    if rollback_fails:
        backups = [artifact for artifact in artifacts if artifact.name.endswith(".backup")]
        if len(backups) != 1:
            note(f"{case}: rollback failure did not retain exactly one backup")
        else:
            backup = backups[0]
            if backup.parent != sshd_dir or backup.read_bytes() != prior_dropin:
                note(f"{case}: retained backup was not a same-directory byte copy")
            if backup.stat().st_mode & 0o777 != 0o600:
                note(f"{case}: retained backup permissions were not restrictive")
            if str(backup.resolve()) not in output:
                note(f"{case}: recovery output omitted the absolute backup path")
        if "sshd -t" not in output or f"systemctl reload {detected_unit}" not in output:
            note(f"{case}: recovery output omitted validation or reload commands")

    artifact_logs = [line for line in call_lines if line.startswith("ssh-artifact ")]
    if expected["prior_dropin"] is not None and not any(
        re.fullmatch(
            rf"ssh-artifact {re.escape(str(sshd_dir))}/\.90-panama\.[A-Za-z0-9]+\.backup 600 ",
            line,
        )
        for line in artifact_logs
    ):
        note(f"{case}: backup was not collision-safe, same-directory, non-.conf, and restrictive")

cleanup_failure_cases = {
    "success-backup-cleanup-fails": {
        "sshd_results": (0,),
        "reload_results": (0,),
        "rm_backup_results": (1,),
        "expected_dropin": desired_dropin,
        "expected_validations": 1,
        "expected_reloads": 1,
    },
    "rollback-backup-cleanup-fails": {
        "sshd_results": (1, 0),
        "reload_results": (),
        "rm_backup_results": (1,),
        "expected_dropin": prior_dropin,
        "expected_validations": 2,
        "expected_reloads": 0,
    },
}

for case, expected in cleanup_failure_cases.items():
    status, output, calls, fixture_root, _ = run_case(
        case,
        sshd_results=expected["sshd_results"],
        reload_results=expected["reload_results"],
        rm_backup_results=expected["rm_backup_results"],
        prior_dropin=prior_dropin,
    )
    dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf"
    backups = list(dropin.parent.glob(".90-panama.*.backup"))
    if status == 0:
        note(f"{case}: cleanup failure returned success")
    if "install-handoff " in calls:
        note(f"{case}: cleanup failure handed off to install")
    if dropin.read_bytes() != expected["expected_dropin"]:
        note(f"{case}: cleanup failure changed the settled SSH drop-in")
    if calls.count("sshd -t \n") != expected["expected_validations"]:
        note(f"{case}: cleanup failure validation count was wrong")
    if calls.count("systemctl reload sshd.service \n") != expected["expected_reloads"]:
        note(f"{case}: cleanup failure reload count was wrong")
    if len(backups) != 1:
        note(f"{case}: failed cleanup did not retain exactly one backup")
    else:
        backup = backups[0]
        if str(backup.resolve()) not in output or "rm -f --" not in output:
            note(f"{case}: retained backup was not reported with an actionable cleanup command")

status, output, calls, fixture_root, _ = run_case(
    "candidate-cleanup-fails",
    mv_activation_results=(1,),
    rm_candidate_results=(1,),
)
dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf"
candidates = list(dropin.parent.glob(".90-panama.*.tmp"))
if status == 0:
    note("candidate-cleanup-fails: activation cleanup failure returned success")
if dropin.exists():
    note("candidate-cleanup-fails: failed activation changed the final drop-in")
if "install-handoff " in calls:
    note("candidate-cleanup-fails: failed activation reached install handoff")
if len(candidates) != 1:
    note("candidate-cleanup-fails: failed cleanup did not retain exactly one candidate")
else:
    candidate = candidates[0]
    if str(candidate.resolve()) not in output or "rm -f --" not in output:
        note("candidate-cleanup-fails: retained candidate lacked an actionable cleanup command")

signal_cases = {
    "signal-int-restores-prior": {
        "signal": signal.SIGINT,
        "status": 130,
        "prior_dropin": prior_dropin,
    },
    "signal-term-removes-new-dropin": {
        "signal": signal.SIGTERM,
        "status": 143,
        "prior_dropin": None,
    },
}

for case, expected in signal_cases.items():
    status, output, calls, fixture_root, boot_pid = run_case(
        case,
        signal_after_activation=expected["signal"],
        prior_traps=True,
        sshd_results=(0,),
        reload_results=(0,),
        prior_dropin=expected["prior_dropin"],
    )
    dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf"
    actual_dropin = dropin.read_bytes() if dropin.exists() else None
    if status != expected["status"]:
        note(f"{case}: signal returned status {status}, expected {expected['status']}")
    if actual_dropin != expected["prior_dropin"]:
        note(f"{case}: signal did not restore the pre-transaction SSH state")
    if list(dropin.parent.glob(".90-panama.*")):
        note(f"{case}: signal left transaction residue")
    if "install-handoff " in calls:
        note(f"{case}: signal reached install handoff")
    if f"prior-exit {boot_pid}\n" not in calls:
        note(f"{case}: signal suppressed the saved EXIT trap")
    if calls.count("sshd -t \n") != 1:
        note(f"{case}: signal did not validate restored configuration once")
    if calls.count("systemctl reload sshd.service \n") != 1:
        note(f"{case}: signal did not reload restored configuration once")
    call_lines = calls.splitlines()
    rollback_indices = [
        index
        for index, line in enumerate(call_lines)
        if (
            expected["prior_dropin"] is not None
            and re.fullmatch(
                rf"mv -f -- {re.escape(str(dropin.parent))}/\.90-panama\.[A-Za-z0-9]+\.restore "
                rf"{re.escape(str(dropin))} source-mode=600 ",
                line,
            )
        )
        or (expected["prior_dropin"] is None and line == f"rm -f -- {dropin} ")
    ]
    validation_indices = [
        index for index, line in enumerate(call_lines) if line == "sshd -t "
    ]
    reload_indices = [
        index
        for index, line in enumerate(call_lines)
        if line == "systemctl reload sshd.service "
    ]
    if not (
        len(rollback_indices) == len(validation_indices) == len(reload_indices) == 1
        and rollback_indices[0] < validation_indices[0] < reload_indices[0]
    ):
        note(f"{case}: signal did not restore, validate, then reload in order")

if findings:
    print(f"root server bootstrap: {len(findings)} finding(s)", file=sys.stderr)
    for finding in findings:
        print(f"  - {finding}", file=sys.stderr)
    raise SystemExit(1)

print("root server bootstrap: PASS")
PY