Fix: Clean interrupted SSH candidates

This commit is contained in:
Gabriel Brown
2026-08-27 04:12:45 -04:00
parent 51e2c8418a
commit ac25fd776c
2 changed files with 92 additions and 1 deletions
+7
View File
@@ -118,6 +118,13 @@ remove_ssh_artifact() {
handle_ssh_transaction_signal() { handle_ssh_transaction_signal() {
local signal_status="$1" local signal_status="$1"
trap - INT TERM trap - INT TERM
if [[ -n "$ssh_candidate" && -e "$ssh_candidate" ]]; then
ssh_transaction_active=0
restore_ssh_transaction_traps
remove_ssh_artifact "$ssh_candidate" || true
remove_ssh_artifact "$ssh_backup" || true
exit "$signal_status"
fi
rollback_failed=0 rollback_failed=0
restore_ssh_dropin || rollback_failed=1 restore_ssh_dropin || rollback_failed=1
sshd -t || rollback_failed=1 sshd -t || rollback_failed=1
+85 -1
View File
@@ -144,6 +144,13 @@ esac
''') ''')
write_executable(stub_dir / "mv", common + r''' write_executable(stub_dir / "mv", common + r'''
log mv "$@" "source-mode=$(/usr/bin/stat -c %a -- "${3:-}")" log mv "$@" "source-mode=$(/usr/bin/stat -c %a -- "${3:-}")"
if [[ "${3:-}" == *.tmp && -e "$PANAMA_BOOT_FIXTURE_ROOT/state/HOLD_BEFORE_ACTIVATION" ]]; then
: >"$PANAMA_BOOT_FIXTURE_ROOT/state/TRANSACTION_ARMED"
while [[ ! -e "$PANAMA_BOOT_FIXTURE_ROOT/state/RELEASE_BEFORE_ACTIVATION" ]]; do
/usr/bin/sleep 0.01
done
exit 98
fi
if [[ "${3:-}" == *.tmp ]]; then if [[ "${3:-}" == *.tmp ]]; then
consume_result MV_ACTIVATION_RESULTS consume_result MV_ACTIVATION_RESULTS
result=$? result=$?
@@ -187,6 +194,7 @@ def configure_case(
sshd_unit: bool = True, sshd_unit: bool = True,
ssh_unit: bool = True, ssh_unit: bool = True,
hold_activation: bool = False, hold_activation: bool = False,
hold_before_activation: bool = False,
) -> tuple[Path, Path]: ) -> tuple[Path, Path]:
fixture_root = work / name / "root" fixture_root = work / name / "root"
stub_dir = work / name / "bin" stub_dir = work / name / "bin"
@@ -220,6 +228,8 @@ def configure_case(
(state / "SSH_UNIT").write_text("present\n" if ssh_unit else "absent\n") (state / "SSH_UNIT").write_text("present\n" if ssh_unit else "absent\n")
if hold_activation: if hold_activation:
(state / "HOLD_ACTIVATION").touch() (state / "HOLD_ACTIVATION").touch()
if hold_before_activation:
(state / "HOLD_BEFORE_ACTIVATION").touch()
(fixture_root / "stub-install").write_text( (fixture_root / "stub-install").write_text(
"#!/usr/bin/env bash\nprintf 'install-handoff %s\\n' \"${PANAMA_PATH:-unset}\" >> \"$PANAMA_BOOT_FIXTURE_ROOT/calls\"\n" "#!/usr/bin/env bash\nprintf 'install-handoff %s\\n' \"${PANAMA_PATH:-unset}\" >> \"$PANAMA_BOOT_FIXTURE_ROOT/calls\"\n"
) )
@@ -284,6 +294,9 @@ def configure_case(
"signal-int-restores-prior", "signal-int-restores-prior",
"signal-term-removes-new-dropin", "signal-term-removes-new-dropin",
"candidate-cleanup-fails", "candidate-cleanup-fails",
"signal-int-before-activation-prior",
"signal-term-before-activation-no-prior",
"signal-int-before-activation-cleanup-fails",
): ):
target_keys.write_text("ssh-ed25519 target\n") target_keys.write_text("ssh-ed25519 target\n")
elif name == "safe-root-key-copy": elif name == "safe-root-key-copy":
@@ -297,12 +310,14 @@ def run_case(
name: str, name: str,
*, *,
signal_after_activation: int | None = None, signal_after_activation: int | None = None,
signal_before_activation: int | None = None,
prior_traps: bool = False, prior_traps: bool = False,
**configuration: object, **configuration: object,
) -> tuple[int, str, str, Path, int]: ) -> tuple[int, str, str, Path, int]:
fixture_root, stub_dir = configure_case( fixture_root, stub_dir = configure_case(
name, name,
hold_activation=signal_after_activation is not None, hold_activation=signal_after_activation is not None,
hold_before_activation=signal_before_activation is not None,
**configuration, **configuration,
) )
master, slave = pty.openpty() master, slave = pty.openpty()
@@ -344,7 +359,17 @@ fi
) )
os.close(slave) os.close(slave)
os.write(master, b"gib\nY\n") os.write(master, b"gib\nY\n")
if signal_after_activation is not None: if signal_before_activation is not None:
armed = fixture_root / "state/TRANSACTION_ARMED"
deadline = time.monotonic() + 5
while not armed.exists() and process.poll() is None and time.monotonic() < deadline:
time.sleep(0.01)
if not armed.exists():
note(f"{name}: fixture did not observe transaction arming before signaling")
else:
os.kill(process.pid, signal_before_activation)
(fixture_root / "state/RELEASE_BEFORE_ACTIVATION").touch()
elif signal_after_activation is not None:
activation = fixture_root / "state/ACTIVATED" activation = fixture_root / "state/ACTIVATED"
deadline = time.monotonic() + 5 deadline = time.monotonic() + 5
while not activation.exists() and process.poll() is None and time.monotonic() < deadline: while not activation.exists() and process.poll() is None and time.monotonic() < deadline:
@@ -764,6 +789,65 @@ for case, expected in signal_cases.items():
): ):
note(f"{case}: signal did not restore, validate, then reload in order") note(f"{case}: signal did not restore, validate, then reload in order")
pre_activation_signal_cases = {
"signal-int-before-activation-prior": {
"signal": signal.SIGINT,
"status": 130,
"prior_dropin": prior_dropin,
"cleanup_fails": False,
},
"signal-term-before-activation-no-prior": {
"signal": signal.SIGTERM,
"status": 143,
"prior_dropin": None,
"cleanup_fails": False,
},
"signal-int-before-activation-cleanup-fails": {
"signal": signal.SIGINT,
"status": 130,
"prior_dropin": None,
"cleanup_fails": True,
},
}
for case, expected in pre_activation_signal_cases.items():
rm_candidate_results = (1,) if expected["cleanup_fails"] else ()
status, output, calls, fixture_root, boot_pid = run_case(
case,
signal_before_activation=expected["signal"],
prior_traps=True,
prior_dropin=expected["prior_dropin"],
rm_candidate_results=rm_candidate_results,
)
dropin = fixture_root / "etc/ssh/sshd_config.d/90-panama.conf"
actual_dropin = dropin.read_bytes() if dropin.exists() else None
candidates = list(dropin.parent.glob(".90-panama.*.tmp"))
backups = list(dropin.parent.glob(".90-panama.*.backup"))
if status != expected["status"]:
note(f"{case}: signal returned status {status}, expected {expected['status']}")
if actual_dropin != expected["prior_dropin"]:
note(f"{case}: pre-activation signal changed the final drop-in state")
if backups:
note(f"{case}: pre-activation signal left backup residue")
if expected["cleanup_fails"]:
if len(candidates) != 1:
note(f"{case}: injected cleanup failure did not retain exactly one candidate")
else:
candidate = candidates[0]
expected_command = f"rm -f -- {candidate.resolve()}"
if str(candidate.resolve()) not in output or expected_command not in output:
note(f"{case}: retained candidate lacked its absolute cleanup command")
elif candidates:
note(f"{case}: pre-activation signal left candidate residue")
if "install-handoff " in calls:
note(f"{case}: pre-activation signal reached install handoff")
if f"prior-exit {boot_pid}\n" not in calls:
note(f"{case}: pre-activation signal suppressed the saved EXIT trap")
if "sshd -t " in calls or "systemctl reload " in calls:
note(f"{case}: pre-activation signal validated or reloaded unchanged SSH state")
if ".restore " in calls or f"rm -f -- {dropin} " in calls:
note(f"{case}: pre-activation signal rewrote the unchanged final drop-in")
if findings: if findings:
print(f"root server bootstrap: {len(findings)} finding(s)", file=sys.stderr) print(f"root server bootstrap: {len(findings)} finding(s)", file=sys.stderr)
for finding in findings: for finding in findings: