100 lines
5.3 KiB
Bash
Executable File
100 lines
5.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Post-reboot verification for the rootless podman stack.
|
|
# Run as gib. Compares against ~/premigration_backup_2026-08-10/post_migration_baseline.txt
|
|
set -uo pipefail
|
|
|
|
fail=0
|
|
ok() { printf ' \033[32mOK\033[0m %s\n' "$1"; }
|
|
bad() { printf ' \033[31mFAIL\033[0m %s\n' "$1"; fail=$((fail+1)); }
|
|
|
|
echo "== linger =="
|
|
[[ "$(loginctl show-user gib --property=Linger --value 2>/dev/null)" == "yes" ]] \
|
|
&& ok "Linger=yes" || bad "linger is OFF - user units will not start at boot"
|
|
|
|
echo "== unprivileged port start =="
|
|
p=$(sysctl -n net.ipv4.ip_unprivileged_port_start 2>/dev/null)
|
|
[[ "$p" == "80" ]] && ok "ip_unprivileged_port_start=80" \
|
|
|| bad "ip_unprivileged_port_start=$p (NPM cannot bind :80/:443)"
|
|
|
|
echo "== user units =="
|
|
for s in postgresql bang completeuphoria sierraandtyler uptime portainer beszel authentik tools isponsorblocktv vaultwarden infisical rustdesk nginx-proxy-manager gbrown-agent; do
|
|
st=$(systemctl --user is-active "podman-$s.service" 2>/dev/null)
|
|
[[ "$st" == "active" ]] && ok "$s" || bad "$s is $st"
|
|
done
|
|
|
|
echo "== containers =="
|
|
for c in postgresql nginx-proxy-manager uptime portainer bang completeuphoria sierraandtyler beszel beszel-agent authentik-server authentik-worker convertx it-tools pdf-tools bento-pdf iSponsorBlockTV vaultwarden infisical-redis infisical-backend rustdesk-server gbrown-agent; do
|
|
if podman ps --format '{{.Names}}' 2>/dev/null | grep -qx "$c"; then ok "$c"; else bad "$c not running"; fi
|
|
done
|
|
|
|
echo "== postgres health =="
|
|
h=$(podman inspect postgresql --format '{{.State.Health.Status}}' 2>/dev/null)
|
|
[[ "$h" == "healthy" ]] && ok "postgres healthy" || bad "postgres health=$h"
|
|
|
|
echo "== endpoints =="
|
|
chk() { local n=$1 exp=$2; shift 2; c=$(curl -sk -o /dev/null -w '%{http_code}' "$@" 2>/dev/null)
|
|
[[ "$c" == "$exp" ]] && ok "$n -> $c" || bad "$n -> $c (expected $exp)"; }
|
|
# Admin ports moved to the WireGuard address 2026-08-12 (AGENTS.md §13.2).
|
|
# They no longer listen on localhost, so probe 192.168.2.2.
|
|
chk "NPM :81" 200 http://192.168.2.2:81
|
|
chk "NPM :80" 200 http://localhost:80
|
|
chk "NPM :443" 200 --resolve nginx.gibbyb.com:443:127.0.0.1 https://nginx.gibbyb.com
|
|
chk "Uptime" 302 http://192.168.2.2:3001
|
|
chk "Portainer" 200 https://192.168.2.2:9443
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Boot-enablement check (added 2026-08-12)
|
|
#
|
|
# Five stacks were once found RUNNING but DISABLED — healthy in `podman ps`, yet they
|
|
# would have disappeared on the next reboot. Starting a stack and enabling it are
|
|
# separate actions; this catches the gap. See AGENTS.md §13.3.
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
echo
|
|
echo "== boot-enablement =="
|
|
# podman's own units are meant to stay disabled; n8n is intentionally down.
|
|
# podman's own units stay disabled by design; n8n is intentionally down;
|
|
# podman-update.service is `static` (the timer activates it, so it is never enabled);
|
|
# watchtower was RETIRED 2026-08-17 and must stay disabled (AGENTS.md §13.15).
|
|
SKIP='podman-auto-update|podman-restart|podman-clean-transient|podman-kube@|podman-user-wait-network-online|podman-n8n|podman-update|podman-watchtower'
|
|
bad=0
|
|
for u in $(systemctl --user list-unit-files 'podman-*.service' --no-legend 2>/dev/null | awk '{print $1}'); do
|
|
echo "$u" | grep -qE "$SKIP" && continue
|
|
state=$(systemctl --user is-enabled "$u" 2>/dev/null)
|
|
if [ "$state" != "enabled" ]; then
|
|
echo " WILL NOT SURVIVE REBOOT: $u ($state) -> systemctl --user enable $u"
|
|
bad=$((bad+1))
|
|
fi
|
|
done
|
|
[ "$bad" -eq 0 ] && echo " all service units enabled"
|
|
|
|
echo
|
|
echo "== nightly image updates (replaces watchtower — AGENTS.md §13.15) =="
|
|
if [ "$(systemctl --user is-enabled podman-update.timer 2>/dev/null)" = "enabled" ]; then
|
|
echo " timer enabled, next: $(systemctl --user list-timers podman-update.timer --no-pager 2>/dev/null | awk 'NR==2{print $1,$2,$3,$4}')"
|
|
else
|
|
echo " WARNING: podman-update.timer is NOT enabled — nothing will update"
|
|
fi
|
|
if [ "$(systemctl --user is-active podman-watchtower.service 2>/dev/null)" = "active" ]; then
|
|
echo " WARNING: watchtower is RUNNING — it recreates containers outside their pod (§13.13)"
|
|
fi
|
|
# A container outside its compose pod is the signature of a watchtower-style recreate.
|
|
podless=""
|
|
for c in $(podman ps --format '{{.Names}}' 2>/dev/null); do
|
|
[ -z "$(podman inspect "$c" --format '{{.Pod}}' 2>/dev/null)" ] && podless="$podless $c"
|
|
done
|
|
[ -n "$podless" ] && echo " PODLESS (rebuild with systemctl --user restart):$podless" || echo " all containers in their pod"
|
|
|
|
echo
|
|
echo "== admin ports bound to 0.0.0.0 (should be 192.168.2.2 — AGENTS.md §13.2) =="
|
|
podman ps --format '{{.Names}} {{.Ports}}' | grep -E '0\.0\.0\.0:(81|80-81|3001|9443|5432)(->|-)' \
|
|
| sed 's/^/ EXPOSED: /' || echo " none"
|
|
|
|
echo
|
|
if [[ $fail -eq 0 ]]; then
|
|
printf '\033[32mALL CHECKS PASSED\033[0m\n'
|
|
else
|
|
printf '\033[31m%d CHECK(S) FAILED\033[0m\n' "$fail"
|
|
echo "Investigate: journalctl --user -u podman-<name>.service -n 50"
|
|
fi
|
|
exit $fail
|