Sync both server snapshots with what the live machines run, and keep their backup tooling

2026-10-30 13:44:29 -04:00
1 parent 95fa480469
commit cb089b39ea
@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+# Post-reboot verification for the rootless podman stack.
+# Run as gib. Compares against ~/premigration_backup_2026-08-10/post_migration_baseline.txt
+set -uo pipefail
+
+fail=0
+ok()   { printf '  \033[32mOK\033[0m   %s\n' "$1"; }
+bad()  { printf '  \033[31mFAIL\033[0m %s\n' "$1"; fail=$((fail+1)); }
+
+echo "== linger =="
+[[ "$(loginctl show-user gib --property=Linger --value 2>/dev/null)" == "yes" ]] \
+  && ok "Linger=yes" || bad "linger is OFF - user units will not start at boot"
+
+echo "== unprivileged port start =="
+p=$(sysctl -n net.ipv4.ip_unprivileged_port_start 2>/dev/null)
+[[ "$p" == "80" ]] && ok "ip_unprivileged_port_start=80" \
+  || bad "ip_unprivileged_port_start=$p (NPM cannot bind :80/:443)"
+
+echo "== user units =="
+for s in postgresql bang completeuphoria sierraandtyler uptime portainer beszel authentik tools isponsorblocktv vaultwarden infisical rustdesk nginx-proxy-manager gbrown-agent; do
+  st=$(systemctl --user is-active "podman-$s.service" 2>/dev/null)
+  [[ "$st" == "active" ]] && ok "$s" || bad "$s is $st"
+done
+
+echo "== containers =="
+for c in postgresql nginx-proxy-manager uptime portainer bang completeuphoria sierraandtyler beszel beszel-agent authentik-server authentik-worker convertx it-tools pdf-tools bento-pdf iSponsorBlockTV vaultwarden infisical-redis infisical-backend rustdesk-server gbrown-agent; do
+  if podman ps --format '{{.Names}}' 2>/dev/null | grep -qx "$c"; then ok "$c"; else bad "$c not running"; fi
+done
+
+echo "== postgres health =="
+h=$(podman inspect postgresql --format '{{.State.Health.Status}}' 2>/dev/null)
+[[ "$h" == "healthy" ]] && ok "postgres healthy" || bad "postgres health=$h"
+
+echo "== endpoints =="
+chk() { local n=$1 exp=$2; shift 2; c=$(curl -sk -o /dev/null -w '%{http_code}' "$@" 2>/dev/null)
+        [[ "$c" == "$exp" ]] && ok "$n -> $c" || bad "$n -> $c (expected $exp)"; }
+# Admin ports moved to the WireGuard address 2026-08-12 (AGENTS.md §13.2).
+# They no longer listen on localhost, so probe 192.168.2.2.
+chk "NPM :81"   200 http://192.168.2.2:81
+chk "NPM :80"   200 http://localhost:80
+chk "NPM :443"  200 --resolve nginx.gibbyb.com:443:127.0.0.1 https://nginx.gibbyb.com
+chk "Uptime"    302 http://192.168.2.2:3001
+chk "Portainer" 200 https://192.168.2.2:9443
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Boot-enablement check (added 2026-08-12)
+#
+# Five stacks were once found RUNNING but DISABLED — healthy in `podman ps`, yet they
+# would have disappeared on the next reboot. Starting a stack and enabling it are
+# separate actions; this catches the gap. See AGENTS.md §13.3.
+# ─────────────────────────────────────────────────────────────────────────────
+echo
+echo "== boot-enablement =="
+# podman's own units are meant to stay disabled; n8n is intentionally down.
+# podman's own units stay disabled by design; n8n is intentionally down;
+# podman-update.service is `static` (the timer activates it, so it is never enabled);
+# watchtower was RETIRED 2026-08-17 and must stay disabled (AGENTS.md §13.15).
+SKIP='podman-auto-update|podman-restart|podman-clean-transient|podman-kube@|podman-user-wait-network-online|podman-n8n|podman-update|podman-watchtower'
+bad=0
+for u in $(systemctl --user list-unit-files 'podman-*.service' --no-legend 2>/dev/null | awk '{print $1}'); do
+    echo "$u" | grep -qE "$SKIP" && continue
+    state=$(systemctl --user is-enabled "$u" 2>/dev/null)
+    if [ "$state" != "enabled" ]; then
+        echo "  WILL NOT SURVIVE REBOOT: $u ($state)  -> systemctl --user enable $u"
+        bad=$((bad+1))
+    fi
+done
+[ "$bad" -eq 0 ] && echo "  all service units enabled"
+
+echo
+echo "== nightly image updates (replaces watchtower — AGENTS.md §13.15) =="
+if [ "$(systemctl --user is-enabled podman-update.timer 2>/dev/null)" = "enabled" ]; then
+    echo "  timer enabled, next: $(systemctl --user list-timers podman-update.timer --no-pager 2>/dev/null | awk 'NR==2{print $1,$2,$3,$4}')"
+else
+    echo "  WARNING: podman-update.timer is NOT enabled — nothing will update"
+fi
+if [ "$(systemctl --user is-active podman-watchtower.service 2>/dev/null)" = "active" ]; then
+    echo "  WARNING: watchtower is RUNNING — it recreates containers outside their pod (§13.13)"
+fi
+# A container outside its compose pod is the signature of a watchtower-style recreate.
+podless=""
+for c in $(podman ps --format '{{.Names}}' 2>/dev/null); do
+    [ -z "$(podman inspect "$c" --format '{{.Pod}}' 2>/dev/null)" ] && podless="$podless $c"
+done
+[ -n "$podless" ] && echo "  PODLESS (rebuild with systemctl --user restart):$podless" || echo "  all containers in their pod"
+
+echo
+echo "== admin ports bound to 0.0.0.0 (should be 192.168.2.2 — AGENTS.md §13.2) =="
+podman ps --format '{{.Names}} {{.Ports}}' | grep -E '0\.0\.0\.0:(81|80-81|3001|9443|5432)(->|-)' \
+    | sed 's/^/  EXPOSED: /' || echo "  none"
+
+echo
+if [[ $fail -eq 0 ]]; then
+  printf '\033[32mALL CHECKS PASSED\033[0m\n'
+else
+  printf '\033[31m%d CHECK(S) FAILED\033[0m\n' "$fail"
+  echo "Investigate: journalctl --user -u podman-<name>.service -n 50"
+fi
+exit $fail