gib/Panama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix: Complete SSH bootstrap hardening

Browse Source
This commit is contained in:
Gabriel Brown
2026-08-27 05:19:58 -04:00
parent 98e29eb8f3
commit fc8f226747
9 changed files with 852 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/contracts.manifest
+2 -1
View File
@@ -271,7 +271,8 @@ hermetic tests/setup/package-lists-contract
hermetic tests/setup/projects-contract
hermetic tests/setup/readme-contract
hermetic tests/setup/role-contract
# Root bootstrap runs entirely against a temporary filesystem and PATH adapters.
# Root bootstrap uses a temporary filesystem, PATH adapters, real public-key
# parsing, and a user namespace for the actual-root fixture guard.
hermetic tests/setup/root-server-bootstrap-contract
hermetic tests/setup/skills-contract
hermetic tests/setup/test-runner-contract
tests/setup/readme-contract
+16
View File
@@ -57,18 +57,28 @@ assert_bootstrap_probe_rejected() {
}
target_key_requirement='target user owns[^.]*\.ssh[^.]*mode[^.]*([^0-9]|^)0700([^0-9]|$)[^.]*authorized_keys[^.]*mode[^.]*([^0-9]|^)0600([^0-9]|$)'
parseable_key_requirement='(every|each)[^.]*non-?comment[^.]*authorized_keys[^.]*(OpenSSH|ssh-keygen)[^.]*(parse|valid)|(OpenSSH|ssh-keygen)[^-]*parse[^.]*every[^.]*non-?comment'
hardening_continues_requirement='hardening[[:space:]]+is[[:space:]]+unavailable[^.]*without[^.]*verified[^.]*key[^.]*install[[:space:]]+continues[^.]*without[[:space:]]+(it|SSH[[:space:]]+hardening)'
atomic_dropin_requirement='atomic[[:space:]]+same-directory[[:space:]]+drop-in'
rollback_requirement='validation[^.]*reload[^.]*fail[^.]*(restor|rollback)[^.]*previous[[:space:]]+drop-in'
effective_policy_requirement='sshd -T[^.]*root[^.]*target|sshd -T[^.]*target[^.]*root'
require_bootstrap_doc "$target_key_requirement" \
'the root bootstrap docs do not require target-user ownership with exact 0700/0600 SSH modes'
require_bootstrap_doc "$parseable_key_requirement" \
'the root bootstrap docs do not require OpenSSH to parse every non-comment key entry'
require_bootstrap_doc "$hardening_continues_requirement" \
'the root bootstrap docs do not say bootstrap continues without unavailable SSH hardening'
require_bootstrap_doc 'sshd -t' \
'the root bootstrap docs do not name sshd -t validation'
require_bootstrap_doc "$effective_policy_requirement" \
'the root bootstrap docs do not name sshd -T checks for root and target contexts'
require_bootstrap_doc "$atomic_dropin_requirement" \
'the root bootstrap docs do not describe the atomic same-directory drop-in'
require_bootstrap_doc '00-panama\.conf' \
'the root bootstrap docs do not name the precedence-safe 00-panama.conf drop-in'
require_bootstrap_doc 'PermitRootLogin[^.]*no[^.]*PasswordAuthentication[^.]*no[^.]*KbdInteractiveAuthentication[^.]*no' \
'the root bootstrap docs do not state all three effective authentication denials'
require_bootstrap_doc 'detected (SSH )?unit.*reload|reload.*detected (SSH )?unit' \
'the root bootstrap docs do not describe reloading the detected SSH unit'
require_bootstrap_doc "$rollback_requirement" \
@@ -85,6 +95,10 @@ assert_bootstrap_probe_rejected 'exact SSH modes' "$target_key_requirement" \
"${bootstrap_doc//0700/700}"
assert_bootstrap_probe_rejected 'target-user ownership' "$target_key_requirement" \
"${bootstrap_doc//target user owns/someone owns}"
weakened_key_doc="${bootstrap_doc//OpenSSH/text tooling}"
weakened_key_doc="${weakened_key_doc//ssh-keygen/text parser}"
assert_bootstrap_probe_rejected 'OpenSSH key parsing' "$parseable_key_requirement" \
"$weakened_key_doc"
assert_bootstrap_probe_rejected 'hardening availability' "$hardening_continues_requirement" \
"${bootstrap_doc//unavailable/available}"
assert_bootstrap_probe_rejected 'hardening continuation' "$hardening_continues_requirement" \
@@ -95,6 +109,8 @@ assert_bootstrap_probe_rejected 'rollback after failure' "$rollback_requirement"
"${bootstrap_doc//restores /keeps }"
assert_bootstrap_probe_rejected 'rollback trigger' "$rollback_requirement" \
"${bootstrap_doc//fails/works}"
assert_bootstrap_probe_rejected 'effective target policy' "$effective_policy_requirement" \
"${bootstrap_doc//target/root}"
if grep -qiE 'merely writes? (the )?(SSH )?(drop-in|file)|reload failure.*ignored|ignores? .*reload failure' <<<"$bootstrap_doc"; then
note 'the root bootstrap docs weaken the transaction by treating the write or reload failure as harmless'
tests/setup/root-server-bootstrap-contract
+542 -63
View File
@@ -21,8 +21,10 @@ import fcntl
import os
import pty
import re
import select
import signal
import shutil
import stat as stat_module
import subprocess
import sys
import tempfile
@@ -45,6 +47,22 @@ def write_executable(path: Path, contents: str) -> None:
path.chmod(0o755)
def generate_public_key(label: str) -> str:
key_path = work / label
subprocess.run(
["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", label, "-f", key_path],
check=True,
)
public_key = key_path.with_suffix(".pub").read_text()
key_path.unlink()
key_path.with_suffix(".pub").unlink()
return public_key
TARGET_PUBLIC_KEY = generate_public_key("panama-target-fixture")
ROOT_PUBLIC_KEY = generate_public_key("panama-root-fixture")
def make_stubs(stub_dir: Path, fixture_root: Path, calls: Path) -> None:
common = f'''#!/usr/bin/env bash
set -u
@@ -77,7 +95,7 @@ case "${1:-}" in
*) exit 97 ;;
esac
;;
-nG) [[ "${2:-}" == gib ]] || exit 97; printf 'gib wheel\n' ;;
-nG) [[ "${2:-}" == gib ]] || exit 97; printf 'operators wheel\n' ;;
*) exit 97 ;;
esac
''')
@@ -90,7 +108,7 @@ printf 'gib PS\n'
log getent "$@"
[[ "${1:-}" == passwd && "${2:-}" == gib ]] || exit 97
home="$(<"$PANAMA_BOOT_FIXTURE_ROOT/state/home")"
printf 'gib:x:1000:1000::%s:/bin/bash\n' "$home"
printf 'gib:x:1000:2000::%s:/bin/bash\n' "$home"
''')
write_executable(stub_dir / "stat", common + r'''
log stat "$@"
@@ -106,7 +124,23 @@ esac
log runuser "$@"
[[ "${1:-}" == -u && "${2:-}" == gib && "${3:-}" == -- ]] || exit 97
shift 3
if [[ "${1:-}" == install && "${2:-}" == -d && "${3:-}" == -m && "${4:-}" == 0700 && "${5:-}" == -- ]]; then
/usr/bin/install "${@:2}"
printf '%s:700\n' "$(<"$PANAMA_BOOT_FIXTURE_ROOT/state/target-uid")" \
>"$PANAMA_BOOT_FIXTURE_ROOT/state/target-dir-meta"
exit 0
fi
if [[ "${1:-}" == install && "${2:-}" == -m && "${3:-}" == 0600 && "${4:-}" == -- ]]; then
/usr/bin/install "${@:2}"
printf '%s:600\n' "$(<"$PANAMA_BOOT_FIXTURE_ROOT/state/target-uid")" \
>"$PANAMA_BOOT_FIXTURE_ROOT/state/target-key-meta"
exit 0
fi
"$@"
''')
write_executable(stub_dir / "ssh-keygen", common + r'''
log ssh-keygen "$@"
exec /usr/bin/ssh-keygen "$@"
''')
write_executable(stub_dir / "git", common + r'''
log git "$@"
@@ -126,12 +160,29 @@ log dnf "$@"
''')
write_executable(stub_dir / "sshd", common + r'''
log sshd "$@"
[[ "$#" -eq 1 && "$1" == -t ]] || exit 97
for artifact in "$PANAMA_BOOT_FIXTURE_ROOT/etc/ssh/sshd_config.d"/.90-panama.*; do
[[ -e "$artifact" ]] || continue
log ssh-artifact "$artifact" "$(/usr/bin/stat -c %a -- "$artifact")"
done
consume_result SSHD_RESULTS
case "${1:-}" in
-t)
[[ "$#" -eq 1 ]] || exit 97
for artifact in "$PANAMA_BOOT_FIXTURE_ROOT/etc/ssh/sshd_config.d"/.00-panama.*; do
[[ -e "$artifact" ]] || continue
log ssh-artifact "$artifact" "$(/usr/bin/stat -c %a -- "$artifact")"
done
consume_result SSHD_RESULTS
;;
-T)
[[ "$#" -eq 3 && "$2" == -C ]] || exit 97
case "$3" in
user=root,host=localhost,addr=127.0.0.1)
cat "$PANAMA_BOOT_FIXTURE_ROOT/state/ROOT_POLICY"
;;
user=gib,host=localhost,addr=127.0.0.1)
cat "$PANAMA_BOOT_FIXTURE_ROOT/state/TARGET_POLICY"
;;
*) exit 97 ;;
esac
;;
*) exit 97 ;;
esac
''')
write_executable(stub_dir / "systemctl", common + r'''
log systemctl "$@"
@@ -141,6 +192,40 @@ case "${1:-}:${2:-}" in
reload:sshd.service|reload:ssh.service) consume_result RELOAD_RESULTS ;;
*) exit 97 ;;
esac
''')
write_executable(stub_dir / "mktemp", common + r'''
log mktemp "$@"
artifact="$(/usr/bin/mktemp "$@")" || exit
case "$artifact" in
*.tmp)
if [[ -e "$PANAMA_BOOT_FIXTURE_ROOT/state/HOLD_CANDIDATE_PREPARATION" ]]; then
: >"$PANAMA_BOOT_FIXTURE_ROOT/state/CANDIDATE_PREPARING"
while [[ ! -e "$PANAMA_BOOT_FIXTURE_ROOT/state/RELEASE_CANDIDATE_PREPARATION" ]]; do
/usr/bin/sleep 0.01
done
fi
;;
*.backup)
if [[ -e "$PANAMA_BOOT_FIXTURE_ROOT/state/HOLD_BACKUP_MKTEMP" ]]; then
: >"$PANAMA_BOOT_FIXTURE_ROOT/state/BACKUP_MKTEMP_RUNNING"
while [[ ! -e "$PANAMA_BOOT_FIXTURE_ROOT/state/RELEASE_BACKUP_MKTEMP" ]]; do
/usr/bin/sleep 0.01
done
fi
;;
esac
printf '%s\n' "$artifact"
''')
write_executable(stub_dir / "cp", common + r'''
log cp "$@"
destination="${@: -1}"
if [[ "$destination" == *.backup && -e "$PANAMA_BOOT_FIXTURE_ROOT/state/HOLD_BACKUP_PREPARATION" ]]; then
: >"$PANAMA_BOOT_FIXTURE_ROOT/state/BACKUP_PREPARING"
while [[ ! -e "$PANAMA_BOOT_FIXTURE_ROOT/state/RELEASE_BACKUP_PREPARATION" ]]; do
/usr/bin/sleep 0.01
done
fi
exec /usr/bin/cp "$@"
''')
write_executable(stub_dir / "mv", common + r'''
log mv "$@" "source-mode=$(/usr/bin/stat -c %a -- "${3:-}")"
@@ -176,9 +261,14 @@ if [[ "${1:-}" == -f && "${2:-}" == -- && "${3:-}" == *.tmp ]]; then
result=$?
(( result == 0 )) || exit "$result"
fi
if [[ "${1:-}" == -f && "${2:-}" == -- && "${3:-}" == */00-panama.conf ]]; then
consume_result RM_DROPIN_RESULTS
result=$?
(( result == 0 )) || exit "$result"
fi
exec /usr/bin/rm "$@"
''')
for command in ("useradd", "usermod"):
for command in ("chown", "useradd", "usermod"):
write_executable(stub_dir / command, common + f'''\nlog {command} "$@"\nexit 97\n''')
@@ -190,11 +280,24 @@ def configure_case(
mv_activation_results: tuple[int, ...] = (),
rm_backup_results: tuple[int, ...] = (),
rm_candidate_results: tuple[int, ...] = (),
rm_dropin_results: tuple[int, ...] = (),
prior_dropin: bytes | None = None,
prior_dropin_kind: str = "regular",
root_policy: str = (
"permitrootlogin no\n"
"passwordauthentication no\n"
"kbdinteractiveauthentication no\n"
),
target_policy: str = (
"passwordauthentication no\n"
"kbdinteractiveauthentication no\n"
),
sshd_unit: bool = True,
ssh_unit: bool = True,
hold_activation: bool = False,
hold_before_activation: bool = False,
hold_candidate_preparation: bool = False,
hold_backup_preparation: bool = False,
) -> tuple[Path, Path]:
fixture_root = work / name / "root"
stub_dir = work / name / "bin"
@@ -224,22 +327,45 @@ def configure_case(
(state / "RM_CANDIDATE_RESULTS").write_text(
"".join(f"{result}\n" for result in rm_candidate_results)
)
(state / "RM_DROPIN_RESULTS").write_text(
"".join(f"{result}\n" for result in rm_dropin_results)
)
(state / "ROOT_POLICY").write_text(root_policy)
(state / "TARGET_POLICY").write_text(target_policy)
(state / "SSHD_UNIT").write_text("present\n" if sshd_unit else "absent\n")
(state / "SSH_UNIT").write_text("present\n" if ssh_unit else "absent\n")
if hold_activation:
(state / "HOLD_ACTIVATION").touch()
if hold_before_activation:
(state / "HOLD_BEFORE_ACTIVATION").touch()
if hold_candidate_preparation:
(state / "HOLD_CANDIDATE_PREPARATION").touch()
if hold_backup_preparation:
(state / "HOLD_BACKUP_PREPARATION").touch()
(fixture_root / "stub-install").write_text(
"#!/usr/bin/env bash\nprintf 'install-handoff %s\\n' \"${PANAMA_PATH:-unset}\" >> \"$PANAMA_BOOT_FIXTURE_ROOT/calls\"\n"
)
(fixture_root / "stub-install").chmod(0o755)
make_stubs(stub_dir, fixture_root, calls)
if prior_dropin is not None:
dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf"
dropin = fixture_root / "etc/ssh/sshd_config.d/00-panama.conf"
if prior_dropin_kind != "regular":
if prior_dropin_kind == "symlink":
symlink_target = fixture_root / "unsupported-panama-dropin"
symlink_target.write_bytes(prior_dropin or b"unsupported symlink target\n")
dropin.symlink_to(symlink_target)
elif prior_dropin_kind == "directory":
dropin.mkdir()
elif prior_dropin_kind == "fifo":
os.mkfifo(dropin)
else:
raise ValueError(prior_dropin_kind)
elif prior_dropin is not None:
dropin.write_bytes(prior_dropin)
dropin.chmod(0o600)
dropin.chmod(0o640)
os.utime(dropin, ns=(1_700_000_000_123_456_789, 1_700_000_000_123_456_789))
os.setxattr(dropin, b"user.panama-contract", b"preserve-me")
subprocess.run(["setfacl", "-m", "u:65534:r--", dropin], check=True)
target_keys = ssh_dir / "authorized_keys"
root_keys = root_ssh_dir / "authorized_keys"
@@ -256,31 +382,42 @@ def configure_case(
(fixture_root / "home/gib/.ssh").symlink_to(alternate)
elif name == "authorized-keys-symlink":
alternate = fixture_root / "unsafe-authorized-keys"
alternate.write_text("ssh-ed25519 unsafe\n")
alternate.write_text(TARGET_PUBLIC_KEY)
target_keys.symlink_to(alternate)
elif name == "malformed-key":
target_keys.write_text("this is not OpenSSH key material\n")
elif name == "mixed-valid-and-malformed-key":
target_keys.write_text(TARGET_PUBLIC_KEY + "this is not OpenSSH key material\n")
elif name == "malformed-root-key":
root_keys.write_text("this is not OpenSSH key material\n")
elif name == "directory-wrong-mode":
target_keys.write_text("ssh-ed25519 target\n")
target_keys.write_text(TARGET_PUBLIC_KEY)
(state / "target-dir-meta").write_text("1000:755\n")
elif name == "root-copy-directory-wrong-mode":
root_keys.write_text("ssh-ed25519 root\n")
root_keys.write_text(ROOT_PUBLIC_KEY)
(state / "target-dir-meta").write_text("1000:755\n")
elif name == "file-wrong-mode":
target_keys.write_text("ssh-ed25519 target\n")
target_keys.write_text(TARGET_PUBLIC_KEY)
(state / "target-key-meta").write_text("1000:644\n")
elif name == "directory-wrong-owner":
target_keys.write_text("ssh-ed25519 target\n")
target_keys.write_text(TARGET_PUBLIC_KEY)
(state / "target-dir-meta").write_text("0:700\n")
elif name == "file-wrong-owner":
target_keys.write_text("ssh-ed25519 target\n")
target_keys.write_text(TARGET_PUBLIC_KEY)
(state / "target-key-meta").write_text("0:600\n")
elif name == "root-target-account":
target_keys.write_text("ssh-ed25519 target\n")
target_keys.write_text(TARGET_PUBLIC_KEY)
(state / "target-uid").write_text("0\n")
elif name == "relative-home":
target_keys.write_text("ssh-ed25519 target\n")
target_keys.write_text(TARGET_PUBLIC_KEY)
(state / "home").write_text("home/gib\n")
elif name in (
"safe-existing-key",
"declines-hardening",
"missing-ssh-unit",
"preexisting-dropin-symlink",
"preexisting-dropin-directory",
"preexisting-dropin-fifo",
"success-without-prior-dropin",
"success-replaces-prior-dropin",
"candidate-invalid",
@@ -289,6 +426,9 @@ def configure_case(
"candidate-reload-fails-without-prior",
"rollback-validation-fails",
"rollback-reload-fails",
"rollback-removal-fails-without-prior",
"effective-root-policy-conflict",
"effective-target-policy-conflict",
"success-backup-cleanup-fails",
"rollback-backup-cleanup-fails",
"signal-int-restores-prior",
@@ -297,10 +437,15 @@ def configure_case(
"signal-int-before-activation-prior",
"signal-term-before-activation-no-prior",
"signal-int-before-activation-cleanup-fails",
"signal-int-during-candidate-preparation",
"signal-term-during-backup-preparation",
):
target_keys.write_text("ssh-ed25519 target\n")
target_keys.write_text(TARGET_PUBLIC_KEY)
elif name == "safe-root-key-copy":
root_keys.write_text("ssh-ed25519 root\n")
shutil.rmtree(ssh_dir)
(state / "target-dir-meta").write_text("missing\n")
(state / "target-key-meta").write_text("missing\n")
root_keys.write_text(ROOT_PUBLIC_KEY)
else:
raise ValueError(name)
return fixture_root, stub_dir
@@ -311,6 +456,9 @@ def run_case(
*,
signal_after_activation: int | None = None,
signal_before_activation: int | None = None,
signal_during_candidate_preparation: int | None = None,
signal_during_backup_preparation: int | None = None,
harden_answer: str = "Y",
prior_traps: bool = False,
**configuration: object,
) -> tuple[int, str, str, Path, int]:
@@ -318,6 +466,8 @@ def run_case(
name,
hold_activation=signal_after_activation is not None,
hold_before_activation=signal_before_activation is not None,
hold_candidate_preparation=signal_during_candidate_preparation is not None,
hold_backup_preparation=signal_during_backup_preparation is not None,
**configuration,
)
master, slave = pty.openpty()
@@ -358,8 +508,28 @@ fi
preexec_fn=attach_terminal,
)
os.close(slave)
os.write(master, b"gib\nY\n")
if signal_before_activation is not None:
os.write(master, f"gib\n{harden_answer}\n".encode())
if signal_during_candidate_preparation is not None:
marker = fixture_root / "state/CANDIDATE_PREPARING"
deadline = time.monotonic() + 5
while not marker.exists() and process.poll() is None and time.monotonic() < deadline:
time.sleep(0.01)
if not marker.exists():
note(f"{name}: fixture did not observe candidate preparation before signaling")
else:
os.kill(process.pid, signal_during_candidate_preparation)
(fixture_root / "state/RELEASE_CANDIDATE_PREPARATION").touch()
elif signal_during_backup_preparation is not None:
marker = fixture_root / "state/BACKUP_PREPARING"
deadline = time.monotonic() + 5
while not marker.exists() and process.poll() is None and time.monotonic() < deadline:
time.sleep(0.01)
if not marker.exists():
note(f"{name}: fixture did not observe backup preparation before signaling")
else:
os.kill(process.pid, signal_during_backup_preparation)
(fixture_root / "state/RELEASE_BACKUP_PREPARATION").touch()
elif signal_before_activation is not None:
armed = fixture_root / "state/TRANSACTION_ARMED"
deadline = time.monotonic() + 5
while not armed.exists() and process.poll() is None and time.monotonic() < deadline:
@@ -380,7 +550,19 @@ fi
os.kill(process.pid, signal_after_activation)
(fixture_root / "state/RELEASE_ACTIVATION").touch()
chunks: list[bytes] = []
deadline = time.monotonic() + 8
timed_out = False
while True:
readable, _, _ = select.select([master], [], [], 0.1)
if not readable:
if process.poll() is not None:
break
if time.monotonic() >= deadline:
timed_out = True
os.killpg(process.pid, signal.SIGKILL)
process.wait()
continue
continue
try:
chunk = os.read(master, 4096)
except OSError as error:
@@ -392,15 +574,53 @@ fi
chunks.append(chunk)
os.close(master)
status = process.wait()
if timed_out:
note(f"{name}: bootstrap timed out, likely while reading an unsupported object")
calls = (fixture_root / "calls").read_text()
output = b"".join(chunks).decode(errors="replace")
return status, output, calls, fixture_root, process.pid
guard_root = work / "actual-root-fixture-guard"
guard_root.mkdir()
guard_env = {
**os.environ,
"PANAMA_BOOT_FIXTURE_ROOT": str(guard_root),
"HOME": str(guard_root),
}
if os.geteuid() == 0:
guard_command = ["bash", boot, "--server"]
else:
guard_command = ["unshare", "--user", "--map-root-user", "--", "bash", boot, "--server"]
try:
guard_result = subprocess.run(
guard_command,
env=guard_env,
capture_output=True,
text=True,
timeout=5,
)
except (FileNotFoundError, subprocess.TimeoutExpired) as error:
note(f"actual-root-fixture-guard: could not create a hermetic root process: {error}")
else:
if guard_result.returncode != 1:
note(
"actual-root-fixture-guard: actual root did not reject "
f"PANAMA_BOOT_FIXTURE_ROOT with status 1: {guard_result.returncode}"
)
if "PANAMA_BOOT_FIXTURE_ROOT is test-only" not in guard_result.stderr:
note("actual-root-fixture-guard: rejection diagnostic was missing")
if any(guard_root.iterdir()):
note("actual-root-fixture-guard: boot mutated its rejected fixture root")
unsafe_cases = (
"missing",
"empty",
"comment-only",
"malformed-key",
"mixed-valid-and-malformed-key",
"malformed-root-key",
"ssh-directory-symlink",
"authorized-keys-symlink",
"directory-wrong-mode",
@@ -421,7 +641,7 @@ for case in unsafe_cases:
note(f"{case}: unsafe login path validated sshd")
if "systemctl reload" in calls:
note(f"{case}: unsafe login path reloaded SSH")
if (fixture_root / "etc/ssh/sshd_config.d/90-panama.conf").exists():
if (fixture_root / "etc/ssh/sshd_config.d/00-panama.conf").exists():
note(f"{case}: unsafe login path changed the SSH drop-in")
if case == "root-copy-directory-wrong-mode" and (
fixture_root / "home/gib/.ssh/authorized_keys"
@@ -430,26 +650,132 @@ for case in unsafe_cases:
if "install-handoff " not in calls:
note(f"{case}: unsafe login path did not hand off to install")
desired_dropin = (
b"PermitRootLogin no\n"
b"PasswordAuthentication no\n"
b"KbdInteractiveAuthentication no\n"
)
prior_dropin = b"# prior Panama settings\nPasswordAuthentication yes\n"
for case in ("safe-existing-key", "safe-root-key-copy"):
status, output, calls, fixture_root, _ = run_case(case)
if status != 0:
note(f"{case}: safe login path stopped with status {status}: {output.strip()}")
if "SSH hardening unavailable" in output:
note(f"{case}: safe login path was rejected")
note(f"{case}: safe login path was rejected: {output.strip()} | {calls.strip()}")
if "systemctl reload" not in calls:
note(f"{case}: safe login path did not reach SSH hardening")
if "install-handoff " not in calls:
note(f"{case}: safe login path did not hand off to install")
dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf"
if (dropin.read_text() if dropin.exists() else "") != "PermitRootLogin no\nPasswordAuthentication no\n":
dropin = fixture_root / "etc/ssh/sshd_config.d/00-panama.conf"
if (dropin.read_bytes() if dropin.exists() else None) != desired_dropin:
note(f"{case}: safe login path did not write the expected SSH drop-in")
if case == "safe-root-key-copy":
keys = fixture_root / "home/gib/.ssh/authorized_keys"
if not keys.exists() or keys.read_text() != "ssh-ed25519 root\n":
ssh_dir = keys.parent
if not keys.exists() or keys.read_text() != ROOT_PUBLIC_KEY:
note("safe-root-key-copy: root key was not copied to the target account")
if keys.exists() and (
ssh_dir.stat().st_mode & 0o777 != 0o700
or keys.stat().st_mode & 0o777 != 0o600
):
note("safe-root-key-copy: destination modes were not normalized to 0700/0600")
call_lines = calls.splitlines()
install_dir = (
f"runuser -u gib -- install -d -m 0700 -- {ssh_dir} "
)
install_key = (
"runuser -u gib -- install -m 0600 -- "
f"/dev/stdin {keys} "
)
if install_dir not in call_lines or install_key not in call_lines:
note("safe-root-key-copy: destination creation and writing did not run as the target user")
else:
validation_indices = [
index
for index, line in enumerate(call_lines)
if line.startswith("ssh-keygen -l -f ")
]
if not validation_indices or max(validation_indices) < call_lines.index(install_key):
note("safe-root-key-copy: copied key validity was not rechecked after installation")
if any(line.startswith("chown ") for line in call_lines):
note("safe-root-key-copy: bootstrap still assumes the primary group matches the username")
status, output, calls, fixture_root, _ = run_case(
"declines-hardening",
harden_answer="n",
)
declined_dropin = fixture_root / "etc/ssh/sshd_config.d/00-panama.conf"
if status != 0 or "install-handoff " not in calls:
note("declines-hardening: declining did not continue to install")
if declined_dropin.exists() or "sshd " in calls or "systemctl reload " in calls:
note("declines-hardening: declining changed or validated SSH state")
status, output, calls, fixture_root, _ = run_case(
"missing-ssh-unit",
sshd_unit=False,
ssh_unit=False,
)
missing_unit_dropin = fixture_root / "etc/ssh/sshd_config.d/00-panama.conf"
if status != 0 or "install-handoff " not in calls:
note("missing-ssh-unit: unavailable hardening did not continue to install")
if "SSH hardening unavailable" not in output:
note("missing-ssh-unit: missing units did not explain that hardening was unavailable")
if missing_unit_dropin.exists() or "sshd " in calls or "systemctl reload " in calls:
note("missing-ssh-unit: unavailable hardening changed or validated SSH state")
unsupported_dropins = {
"preexisting-dropin-symlink": "symlink",
"preexisting-dropin-directory": "directory",
"preexisting-dropin-fifo": "fifo",
}
for case, kind in unsupported_dropins.items():
status, output, calls, fixture_root, _ = run_case(
case,
prior_dropin=prior_dropin,
prior_dropin_kind=kind,
)
dropin = fixture_root / "etc/ssh/sshd_config.d/00-panama.conf"
if status != 0 or "install-handoff " not in calls:
note(f"{case}: unsupported object did not continue to install")
if "SSH hardening unavailable" not in output:
note(f"{case}: unsupported object did not explain that hardening was unavailable")
if "sshd " in calls or "systemctl reload " in calls:
note(f"{case}: unsupported object reached SSH validation or reload")
if kind == "symlink" and not dropin.is_symlink():
note(f"{case}: pre-existing symlink was changed")
if kind == "directory" and not dropin.is_dir():
note(f"{case}: pre-existing directory was changed")
if kind == "fifo" and not stat_module.S_ISFIFO(dropin.lstat().st_mode):
note(f"{case}: pre-existing FIFO was changed")
def regular_metadata(path: Path) -> tuple[object, ...]:
metadata = path.stat()
xattrs = tuple((name, os.getxattr(path, name)) for name in sorted(os.listxattr(path)))
acl = subprocess.check_output(["getfacl", "-cp", path])
return (
stat_module.S_IMODE(metadata.st_mode),
metadata.st_uid,
metadata.st_gid,
metadata.st_mtime_ns,
xattrs,
acl,
)
metadata_reference = work / "prior-dropin-metadata-reference"
metadata_reference.write_bytes(prior_dropin)
metadata_reference.chmod(0o640)
os.utime(
metadata_reference,
ns=(1_700_000_000_123_456_789, 1_700_000_000_123_456_789),
)
os.setxattr(metadata_reference, b"user.panama-contract", b"preserve-me")
subprocess.run(["setfacl", "-m", "u:65534:r--", metadata_reference], check=True)
expected_prior_metadata = regular_metadata(metadata_reference)
desired_dropin = b"PermitRootLogin no\nPasswordAuthentication no\n"
prior_dropin = b"# prior Panama settings\nPasswordAuthentication yes\n"
transaction_cases = {
"success-without-prior-dropin": {
"sshd_results": (0,),
@@ -517,17 +843,53 @@ transaction_cases = {
"succeeds": False,
"rollback_fails": True,
},
"rollback-removal-fails-without-prior": {
"sshd_results": (1, 1),
"reload_results": (),
"rm_dropin_results": (1,),
"prior_dropin": None,
"sshd_unit": True,
"ssh_unit": True,
"succeeds": False,
"rollback_fails": True,
"settled_dropin": desired_dropin,
},
"effective-root-policy-conflict": {
"sshd_results": (0, 0),
"reload_results": (),
"prior_dropin": prior_dropin,
"root_policy": (
"permitrootlogin yes\n"
"passwordauthentication no\n"
"kbdinteractiveauthentication no\n"
),
"sshd_unit": True,
"ssh_unit": True,
"succeeds": False,
},
"effective-target-policy-conflict": {
"sshd_results": (0, 0),
"reload_results": (),
"prior_dropin": prior_dropin,
"target_policy": (
"passwordauthentication no\n"
"kbdinteractiveauthentication yes\n"
),
"sshd_unit": True,
"ssh_unit": True,
"succeeds": False,
},
}
for case, expected in transaction_cases.items():
configuration = {
key: value
for key, value in expected.items()
if key not in {"succeeds", "rollback_fails"}
if key not in {"succeeds", "rollback_fails", "settled_dropin"}
}
status, output, calls, fixture_root, _ = run_case(case, **configuration)
call_lines = calls.splitlines()
dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf"
dropin = fixture_root / "etc/ssh/sshd_config.d/00-panama.conf"
sshd_dir = dropin.parent
validations = [index for index, line in enumerate(call_lines) if line.startswith("sshd -t ")]
reloads = [
@@ -539,7 +901,7 @@ for case, expected in transaction_cases.items():
(index, line)
for index, line in enumerate(call_lines)
if re.fullmatch(
rf"mv -f -- {re.escape(str(sshd_dir))}/\.90-panama\.[A-Za-z0-9]+\.tmp "
rf"mv -f -- {re.escape(str(sshd_dir))}/\.00-panama\.[A-Za-z0-9]+\.tmp "
rf"{re.escape(str(dropin))} source-mode=600 ",
line,
)
@@ -548,8 +910,8 @@ for case, expected in transaction_cases.items():
index
for index, line in enumerate(call_lines)
if re.fullmatch(
rf"mv -f -- {re.escape(str(sshd_dir))}/\.90-panama\.[A-Za-z0-9]+\.restore "
rf"{re.escape(str(dropin))} source-mode=600 ",
rf"mv -f -- {re.escape(str(sshd_dir))}/\.00-panama\.[A-Za-z0-9]+\.restore "
rf"{re.escape(str(dropin))} source-mode=640 ",
line,
)
]
@@ -570,31 +932,86 @@ for case, expected in transaction_cases.items():
if not succeeds and "install-handoff " in calls:
note(f"{case}: failed transaction handed off to install")
wanted_contents = desired_dropin if succeeds else expected["prior_dropin"]
wanted_contents = (
desired_dropin
if succeeds
else expected.get("settled_dropin", expected["prior_dropin"])
)
actual_contents = dropin.read_bytes() if dropin.exists() else None
if actual_contents != wanted_contents:
note(f"{case}: SSH drop-in contents were not {'activated' if succeeds else 'restored'}")
if len(activation_lines) != 1:
note(f"{case}: candidate was not activated once through a restrictive same-directory rename")
root_policy_lines = [
index
for index, line in enumerate(call_lines)
if line == r"sshd -T -C user=root\,host=localhost\,addr=127.0.0.1 "
]
target_policy_lines = [
index
for index, line in enumerate(call_lines)
if line == r"sshd -T -C user=gib\,host=localhost\,addr=127.0.0.1 "
]
if case in {
"candidate-invalid",
"candidate-invalid-without-prior",
"rollback-removal-fails-without-prior",
}:
expected_policy_users: tuple[str, ...] = ()
elif case == "effective-root-policy-conflict":
expected_policy_users = ("root",)
else:
expected_policy_users = ("root", "gib")
if len(root_policy_lines) != (1 if "root" in expected_policy_users else 0):
note(f"{case}: effective root policy validation count was wrong")
if len(target_policy_lines) != (1 if "gib" in expected_policy_users else 0):
note(f"{case}: effective target policy validation count was wrong")
if succeeds:
if len(validations) != 1 or len(reloads) != 1:
note(f"{case}: success did not validate once and reload once")
elif activation_lines and not activation_lines[0][0] < validations[0] < reloads[0]:
note(f"{case}: success did not activate, validate, then reload")
elif case in {"candidate-invalid", "candidate-invalid-without-prior"}:
if len(validations) != 2 or reloads:
note(f"{case}: invalid candidate did not validate candidate and restoration without reload")
elif activation_lines and rollback_lines and not (
activation_lines[0][0] < validations[0] < rollback_lines[0] < validations[1]
elif root_policy_lines and target_policy_lines and activation_lines and not (
activation_lines[0][0]
< validations[0]
< root_policy_lines[0]
< target_policy_lines[0]
< reloads[0]
):
note(f"{case}: rollback command order was wrong")
note(f"{case}: success did not activate, validate syntax and effective policy, then reload")
elif case in {
"candidate-invalid",
"candidate-invalid-without-prior",
"rollback-removal-fails-without-prior",
"effective-root-policy-conflict",
"effective-target-policy-conflict",
}:
if len(validations) != 2 or reloads:
note(f"{case}: rejected candidate did not validate candidate and restoration without reload")
elif activation_lines and rollback_lines:
policy_order = [
*root_policy_lines,
*target_policy_lines,
]
if not (
activation_lines[0][0]
< validations[0]
< (policy_order[0] if policy_order else rollback_lines[0])
and all(
left < right
for left, right in zip(policy_order, [*policy_order[1:], rollback_lines[0]])
)
and rollback_lines[0] < validations[1]
):
note(f"{case}: rollback command order was wrong")
else:
if len(validations) != 2 or len(reloads) != 2:
note(f"{case}: reload failure did not validate and reload the restored configuration")
elif activation_lines and rollback_lines and not (
elif activation_lines and rollback_lines and root_policy_lines and target_policy_lines and not (
activation_lines[0][0]
< validations[0]
< root_policy_lines[0]
< target_policy_lines[0]
< reloads[0]
< rollback_lines[0]
< validations[1]
@@ -607,6 +1024,9 @@ for case, expected in transaction_cases.items():
if not succeeds:
if len(rollback_lines) != 1:
note(f"{case}: pre-transaction SSH state was not restored exactly once")
if expected["prior_dropin"] is not None and dropin.exists():
if regular_metadata(dropin) != expected_prior_metadata:
note(f"{case}: rollback did not restore complete regular-file metadata")
detected_unit = "ssh.service" if case == "success-replaces-prior-dropin" else "sshd.service"
other_unit = "sshd.service" if detected_unit == "ssh.service" else "ssh.service"
@@ -628,20 +1048,26 @@ for case, expected in transaction_cases.items():
):
note(f"{case}: did not fall back from absent sshd.service to ssh.service")
artifacts = list(sshd_dir.glob(".90-panama.*"))
artifacts = list(sshd_dir.glob(".00-panama.*"))
rollback_fails = bool(expected.get("rollback_fails", False))
if not rollback_fails and artifacts:
note(f"{case}: successful or cleanly rolled-back transaction left temporary artifacts")
if rollback_fails:
backups = [artifact for artifact in artifacts if artifact.name.endswith(".backup")]
if len(backups) != 1:
if expected["prior_dropin"] is None:
if backups:
note(f"{case}: no-prior-file recovery retained a nonexistent backup")
expected_remove = f"rm -f -- {dropin.resolve()}"
if expected_remove not in output or "cp -a --" in output:
note(f"{case}: no-prior-file recovery did not instruct removal of the installed drop-in")
elif len(backups) != 1:
note(f"{case}: rollback failure did not retain exactly one backup")
else:
backup = backups[0]
if backup.parent != sshd_dir or backup.read_bytes() != prior_dropin:
note(f"{case}: retained backup was not a same-directory byte copy")
if backup.stat().st_mode & 0o777 != 0o600:
note(f"{case}: retained backup permissions were not restrictive")
note(f"{case}: retained backup was not a same-directory copy")
if regular_metadata(backup) != expected_prior_metadata:
note(f"{case}: retained backup did not preserve complete regular-file metadata")
if str(backup.resolve()) not in output:
note(f"{case}: recovery output omitted the absolute backup path")
if "sshd -t" not in output or f"systemctl reload {detected_unit}" not in output:
@@ -650,7 +1076,7 @@ for case, expected in transaction_cases.items():
artifact_logs = [line for line in call_lines if line.startswith("ssh-artifact ")]
if expected["prior_dropin"] is not None and not any(
re.fullmatch(
rf"ssh-artifact {re.escape(str(sshd_dir))}/\.90-panama\.[A-Za-z0-9]+\.backup 600 ",
rf"ssh-artifact {re.escape(str(sshd_dir))}/\.00-panama\.[A-Za-z0-9]+\.backup 640 ",
line,
)
for line in artifact_logs
@@ -684,8 +1110,8 @@ for case, expected in cleanup_failure_cases.items():
rm_backup_results=expected["rm_backup_results"],
prior_dropin=prior_dropin,
)
dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf"
backups = list(dropin.parent.glob(".90-panama.*.backup"))
dropin = fixture_root / "etc/ssh/sshd_config.d/00-panama.conf"
backups = list(dropin.parent.glob(".00-panama.*.backup"))
if status == 0:
note(f"{case}: cleanup failure returned success")
if "install-handoff " in calls:
@@ -696,20 +1122,27 @@ for case, expected in cleanup_failure_cases.items():
note(f"{case}: cleanup failure validation count was wrong")
if calls.count("systemctl reload sshd.service \n") != expected["expected_reloads"]:
note(f"{case}: cleanup failure reload count was wrong")
expected_policy_checks = 1 if case == "success-backup-cleanup-fails" else 0
if calls.count("sshd -T -C user=root\\,host=localhost\\,addr=127.0.0.1 \n") != expected_policy_checks:
note(f"{case}: cleanup failure root policy validation count was wrong")
if calls.count("sshd -T -C user=gib\\,host=localhost\\,addr=127.0.0.1 \n") != expected_policy_checks:
note(f"{case}: cleanup failure target policy validation count was wrong")
if len(backups) != 1:
note(f"{case}: failed cleanup did not retain exactly one backup")
else:
backup = backups[0]
if str(backup.resolve()) not in output or "rm -f --" not in output:
note(f"{case}: retained backup was not reported with an actionable cleanup command")
if regular_metadata(backup) != expected_prior_metadata:
note(f"{case}: cleanup failure backup lost regular-file metadata")
status, output, calls, fixture_root, _ = run_case(
"candidate-cleanup-fails",
mv_activation_results=(1,),
rm_candidate_results=(1,),
)
dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf"
candidates = list(dropin.parent.glob(".90-panama.*.tmp"))
dropin = fixture_root / "etc/ssh/sshd_config.d/00-panama.conf"
candidates = list(dropin.parent.glob(".00-panama.*.tmp"))
if status == 0:
note("candidate-cleanup-fails: activation cleanup failure returned success")
if dropin.exists():
@@ -745,13 +1178,13 @@ for case, expected in signal_cases.items():
reload_results=(0,),
prior_dropin=expected["prior_dropin"],
)
dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf"
dropin = fixture_root / "etc/ssh/sshd_config.d/00-panama.conf"
actual_dropin = dropin.read_bytes() if dropin.exists() else None
if status != expected["status"]:
note(f"{case}: signal returned status {status}, expected {expected['status']}")
if actual_dropin != expected["prior_dropin"]:
note(f"{case}: signal did not restore the pre-transaction SSH state")
if list(dropin.parent.glob(".90-panama.*")):
if list(dropin.parent.glob(".00-panama.*")):
note(f"{case}: signal left transaction residue")
if "install-handoff " in calls:
note(f"{case}: signal reached install handoff")
@@ -768,8 +1201,8 @@ for case, expected in signal_cases.items():
if (
expected["prior_dropin"] is not None
and re.fullmatch(
rf"mv -f -- {re.escape(str(dropin.parent))}/\.90-panama\.[A-Za-z0-9]+\.restore "
rf"{re.escape(str(dropin))} source-mode=600 ",
rf"mv -f -- {re.escape(str(dropin.parent))}/\.00-panama\.[A-Za-z0-9]+\.restore "
rf"{re.escape(str(dropin))} source-mode=640 ",
line,
)
)
@@ -788,6 +1221,8 @@ for case, expected in signal_cases.items():
and rollback_indices[0] < validation_indices[0] < reload_indices[0]
):
note(f"{case}: signal did not restore, validate, then reload in order")
if expected["prior_dropin"] is not None and regular_metadata(dropin) != expected_prior_metadata:
note(f"{case}: signal rollback did not restore complete regular-file metadata")
pre_activation_signal_cases = {
"signal-int-before-activation-prior": {
@@ -819,10 +1254,10 @@ for case, expected in pre_activation_signal_cases.items():
prior_dropin=expected["prior_dropin"],
rm_candidate_results=rm_candidate_results,
)
dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf"
dropin = fixture_root / "etc/ssh/sshd_config.d/00-panama.conf"
actual_dropin = dropin.read_bytes() if dropin.exists() else None
candidates = list(dropin.parent.glob(".90-panama.*.tmp"))
backups = list(dropin.parent.glob(".90-panama.*.backup"))
candidates = list(dropin.parent.glob(".00-panama.*.tmp"))
backups = list(dropin.parent.glob(".00-panama.*.backup"))
if status != expected["status"]:
note(f"{case}: signal returned status {status}, expected {expected['status']}")
if actual_dropin != expected["prior_dropin"]:
@@ -848,6 +1283,50 @@ for case, expected in pre_activation_signal_cases.items():
if ".restore " in calls or f"rm -f -- {dropin} " in calls:
note(f"{case}: pre-activation signal rewrote the unchanged final drop-in")
preparation_signal_cases = {
"signal-int-during-candidate-preparation": {
"signal": signal.SIGINT,
"status": 130,
"prior_dropin": None,
"phase": "candidate",
},
"signal-term-during-backup-preparation": {
"signal": signal.SIGTERM,
"status": 143,
"prior_dropin": prior_dropin,
"phase": "backup",
},
}
for case, expected in preparation_signal_cases.items():
signal_arguments = (
{"signal_during_candidate_preparation": expected["signal"]}
if expected["phase"] == "candidate"
else {"signal_during_backup_preparation": expected["signal"]}
)
status, output, calls, fixture_root, boot_pid = run_case(
case,
prior_traps=True,
prior_dropin=expected["prior_dropin"],
**signal_arguments,
)
dropin = fixture_root / "etc/ssh/sshd_config.d/00-panama.conf"
actual_dropin = dropin.read_bytes() if dropin.exists() else None
if status != expected["status"]:
note(f"{case}: signal returned status {status}, expected {expected['status']}")
if actual_dropin != expected["prior_dropin"]:
note(f"{case}: preparation signal changed the final drop-in")
if list(dropin.parent.glob(".00-panama.*")):
note(f"{case}: preparation signal left candidate or backup residue")
if "install-handoff " in calls:
note(f"{case}: preparation signal reached install handoff")
if f"prior-exit {boot_pid}\n" not in calls:
note(f"{case}: preparation signal suppressed the saved EXIT trap")
if "sshd " in calls or "systemctl reload " in calls:
note(f"{case}: preparation signal validated or reloaded unchanged SSH state")
if expected["prior_dropin"] is not None and regular_metadata(dropin) != expected_prior_metadata:
note(f"{case}: preparation signal changed prior regular-file metadata")
if findings:
print(f"root server bootstrap: {len(findings)} finding(s)", file=sys.stderr)
for finding in findings: