#!/usr/bin/env bash # Post-reboot verification for the rootless podman stack. # Run as gib. Compares against ~/premigration_backup_2026-08-10/post_migration_baseline.txt set -uo pipefail fail=0 ok() { printf ' \033[32mOK\033[0m %s\n' "$1"; } bad() { printf ' \033[31mFAIL\033[0m %s\n' "$1"; fail=$((fail+1)); } echo "== linger ==" [[ "$(loginctl show-user gib --property=Linger --value 2>/dev/null)" == "yes" ]] \ && ok "Linger=yes" || bad "linger is OFF - user units will not start at boot" echo "== unprivileged port start ==" p=$(sysctl -n net.ipv4.ip_unprivileged_port_start 2>/dev/null) [[ "$p" == "80" ]] && ok "ip_unprivileged_port_start=80" \ || bad "ip_unprivileged_port_start=$p (NPM cannot bind :80/:443)" echo "== user units ==" for s in postgresql bang completeuphoria sierraandtyler uptime portainer beszel authentik tools isponsorblocktv vaultwarden infisical rustdesk nginx-proxy-manager gbrown-agent wristcode; do st=$(systemctl --user is-active "podman-$s.service" 2>/dev/null) [[ "$st" == "active" ]] && ok "$s" || bad "$s is $st" done echo "== containers ==" for c in postgresql nginx-proxy-manager uptime portainer bang completeuphoria sierraandtyler beszel beszel-agent authentik-server authentik-worker convertx it-tools pdf-tools bento-pdf iSponsorBlockTV vaultwarden infisical-redis infisical-backend rustdesk-server gbrown-agent; do if podman ps --format '{{.Names}}' 2>/dev/null | grep -qx "$c"; then ok "$c"; else bad "$c not running"; fi done echo "== postgres health ==" h=$(podman inspect postgresql --format '{{.State.Health.Status}}' 2>/dev/null) [[ "$h" == "healthy" ]] && ok "postgres healthy" || bad "postgres health=$h" echo "== endpoints ==" chk() { local n=$1 exp=$2; shift 2; c=$(curl -sk -o /dev/null -w '%{http_code}' "$@" 2>/dev/null) [[ "$c" == "$exp" ]] && ok "$n -> $c" || bad "$n -> $c (expected $exp)"; } # Admin ports moved to the WireGuard address 2026-08-12 (AGENTS.md §13.2). # They no longer listen on localhost, so probe 192.168.2.2. chk "NPM :81" 200 http://192.168.2.2:81 chk "NPM :80" 200 http://localhost:80 chk "NPM :443" 200 --resolve nginx.gibbyb.com:443:127.0.0.1 https://nginx.gibbyb.com chk "Uptime" 302 http://192.168.2.2:3001 chk "Portainer" 200 https://192.168.2.2:9443 # ───────────────────────────────────────────────────────────────────────────── # Boot-enablement check (added 2026-08-12) # # Five stacks were once found RUNNING but DISABLED — healthy in `podman ps`, yet they # would have disappeared on the next reboot. Starting a stack and enabling it are # separate actions; this catches the gap. See AGENTS.md §13.3. # ───────────────────────────────────────────────────────────────────────────── echo echo "== boot-enablement ==" # podman's own units are meant to stay disabled; n8n is intentionally down. # podman's own units stay disabled by design; n8n is intentionally down; # podman-update.service is `static` (the timer activates it, so it is never enabled); # watchtower was RETIRED 2026-08-17 and must stay disabled (AGENTS.md §13.15). SKIP='podman-auto-update|podman-restart|podman-clean-transient|podman-kube@|podman-user-wait-network-online|podman-n8n|podman-update|podman-watchtower' bad=0 for u in $(systemctl --user list-unit-files 'podman-*.service' --no-legend 2>/dev/null | awk '{print $1}'); do echo "$u" | grep -qE "$SKIP" && continue state=$(systemctl --user is-enabled "$u" 2>/dev/null) if [ "$state" != "enabled" ]; then echo " WILL NOT SURVIVE REBOOT: $u ($state) -> systemctl --user enable $u" bad=$((bad+1)) fi done [ "$bad" -eq 0 ] && echo " all service units enabled" echo echo "== nightly image updates (replaces watchtower — AGENTS.md §13.15) ==" if [ "$(systemctl --user is-enabled podman-update.timer 2>/dev/null)" = "enabled" ]; then echo " timer enabled, next: $(systemctl --user list-timers podman-update.timer --no-pager 2>/dev/null | awk 'NR==2{print $1,$2,$3,$4}')" else echo " WARNING: podman-update.timer is NOT enabled — nothing will update" fi if [ "$(systemctl --user is-active podman-watchtower.service 2>/dev/null)" = "active" ]; then echo " WARNING: watchtower is RUNNING — it recreates containers outside their pod (§13.13)" fi # A container outside its compose pod is the signature of a watchtower-style recreate. podless="" for c in $(podman ps --format '{{.Names}}' 2>/dev/null); do [ -z "$(podman inspect "$c" --format '{{.Pod}}' 2>/dev/null)" ] && podless="$podless $c" done [ -n "$podless" ] && echo " PODLESS (rebuild with systemctl --user restart):$podless" || echo " all containers in their pod" echo echo "== admin ports bound to 0.0.0.0 (should be 192.168.2.2 — AGENTS.md §13.2) ==" podman ps --format '{{.Names}} {{.Ports}}' | grep -E '0\.0\.0\.0:(81|80-81|3001|9443|5432)(->|-)' \ | sed 's/^/ EXPOSED: /' || echo " none" echo if [[ $fail -eq 0 ]]; then printf '\033[32mALL CHECKS PASSED\033[0m\n' else printf '\033[31m%d CHECK(S) FAILED\033[0m\n' "$fail" echo "Investigate: journalctl --user -u podman-.service -n 50" fi exit $fail