Sync both server snapshots with what the live machines run, and keep their backup tooling

2026-10-30 13:44:29 -04:00
1 parent 95fa480469
commit cb089b39ea
@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: Authentik
 After=network-online.target podman.socket podman-postgresql.service
 Wants=network-online.target podman-postgresql.service
@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: Beszel
 After=network-online.target podman.socket
 Wants=network-online.target
@@ -32,7 +32,7 @@ services:
    restart: unless-stopped

  bill-tracker-backend:
-    image: ${CONVEX_BACKEND_IMAGE:-ghcr.io/get-convex/convex-backend@sha256:1f2044e3eac463ac78973b136c0baf72d4ada602611d853d6f99f280e29e0a98}
+    image: ${CONVEX_BACKEND_IMAGE:-ghcr.io/convex/convex-backend@sha256:207cf95def42f01614b6e3af0af5c20d0dcaa5ff2adf258ee7bc9555576644fa}
    container_name: bill-tracker-backend
    hostname: bill-tracker-backend
    domainname: api.billtracker.gbrown.org
@@ -56,7 +56,7 @@ services:
    stop_signal: SIGINT

  bill-tracker-dashboard:
-    image: ${CONVEX_DASHBOARD_IMAGE:-ghcr.io/get-convex/convex-dashboard@sha256:284a2638e0c1a4ec0c2327d8219776f3a426ca5824b81686ae4d9454dc0ce8ed}
+    image: ${CONVEX_DASHBOARD_IMAGE:-ghcr.io/convex/convex-dashboard@sha256:a55bf8abca1332f787bc4d4a79850182175027be7b70aa230be4389a1d3268e2}
    container_name: bill-tracker-dashboard
    hostname: bill-tracker-dashboard
    domainname: dashboard.billtracker.gbrown.org
@@ -29,7 +29,7 @@ services:
    restart: unless-stopped

  convexmonorepo-backend:
-    image: ghcr.io/get-convex/convex-backend:${BACKEND_TAG:-latest}
+    image: ghcr.io/convex/convex-backend:${BACKEND_TAG:-latest}
    container_name: ${BACKEND_CONTAINER_NAME:-convex-backend}
    hostname: ${BACKEND_CONTAINER_NAME:-convex-backend}
    domainname: ${BACKEND_DOMAIN:-convex.gbrown.org}
@@ -58,7 +58,7 @@ services:
    stop_signal: SIGINT

  convexmonorepo-dashboard:
-    image: ghcr.io/get-convex/convex-dashboard:${DASHBOARD_TAG:-latest}
+    image: ghcr.io/convex/convex-dashboard:${DASHBOARD_TAG:-latest}
    container_name: ${DASHBOARD_CONTAINER_NAME:-convex-dashboard}
    hostname: ${DASHBOARD_CONTAINER_NAME:-convex-dashboard}
    domainname: ${DASHBOARD_DOMAIN:-dashboard.${BACKEND_DOMAIN:-convex.gbrown.org}}
@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: iSponsorBlockTV
 After=network-online.target podman.socket
 Wants=network-online.target
@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: Infisical
 After=network-online.target podman.socket podman-postgresql.service
 Wants=network-online.target podman-postgresql.service
@@ -19,7 +19,7 @@ services:
    - NODE_ENV=${NODE_ENV}
    - SENTRY_AUTH_TOKEN=${SENTRY_AUTH_TOKEN}
    - NEXT_PUBLIC_SITE_URL=${NEXT_PUBLIC_SITE_URL:-http://localhost:${NEXT_PORT:-3000}}
-    - NEXT_PUBLIC_CONVEX_URL=${NEXT_PUBLIC_CONVEX_URL:-http://${BACKEND_CONTAINER_NAME:-lashaddict-backend}:${BACKEND_PORT:-3210}}
+    - NEXT_PUBLIC_CONVEX_URL=${CONVEX_SERVER_URL:-http://${BACKEND_CONTAINER_NAME:-lashaddict-backend}:${BACKEND_PORT:-3210}}
    - NEXT_PUBLIC_PLAUSIBLE_URL=${NEXT_PUBLIC_PLAUSIBLE_URL:-https://plausible.gbrown.org}
    - NEXT_PUBLIC_SENTRY_DSN=${NEXT_PUBLIC_SENTRY_DSN}
    - NEXT_PUBLIC_SENTRY_URL=${NEXT_PUBLIC_SENTRY_URL}
@@ -33,7 +33,7 @@ services:
    stdin_open: true
    restart: unless-stopped
  lashaddict-backend:
-    image: ghcr.io/get-convex/convex-backend:${BACKEND_TAG:-latest}
+    image: ghcr.io/convex/convex-backend:${BACKEND_TAG:-latest}
    container_name: ${BACKEND_CONTAINER_NAME:-lashaddict-backend}
    # Same Cloudflare DNS pin as lashaddict-next (see its comment): the backend
    # fetches its own auth discovery URL, which Hetzner resolvers failed.
@@ -67,7 +67,7 @@ services:
    stop_grace_period: 10s
    stop_signal: SIGINT
  lashaddict-dashboard:
-    image: ghcr.io/get-convex/convex-dashboard:${DASHBOARD_TAG:-latest}
+    image: ghcr.io/convex/convex-dashboard:${DASHBOARD_TAG:-latest}
    container_name: ${DASHBOARD_CONTAINER_NAME:-lashaddict-dashboard}
    # Same Cloudflare DNS pin as lashaddict-next (see its comment): the backend
    # fetches its own auth discovery URL, which Hetzner resolvers failed.
@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: N8n
 After=network-online.target podman-postgresql.service
 Wants=network-online.target
@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: Nginx Proxy Manager
 After=network-online.target podman-postgresql.service
 Wants=network-online.target
@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: Portainer
 After=network-online.target
 Wants=network-online.target
@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: PostgreSQL
 After=network-online.target
 Wants=network-online.target
@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: Rustdesk
 After=network-online.target podman.socket
 Wants=network-online.target
@@ -64,7 +64,7 @@ services:
    security_opt:
    - label:disable
  spoon-backend:
-    image: ghcr.io/get-convex/convex-backend:${BACKEND_TAG:-latest}
+    image: ghcr.io/convex/convex-backend:${BACKEND_TAG:-latest}
    container_name: ${BACKEND_CONTAINER_NAME:-spoon-backend}
    hostname: ${BACKEND_CONTAINER_NAME:-spoon-backend}
    domainname: ${BACKEND_DOMAIN:-convex.spoon.gbrown.org}
@@ -93,7 +93,7 @@ services:
    stop_grace_period: 10s
    stop_signal: SIGINT
  spoon-dashboard:
-    image: ghcr.io/get-convex/convex-dashboard:${DASHBOARD_TAG:-latest}
+    image: ghcr.io/convex/convex-dashboard:${DASHBOARD_TAG:-latest}
    container_name: ${DASHBOARD_CONTAINER_NAME:-spoon-dashboard}
    hostname: ${DASHBOARD_CONTAINER_NAME:-spoon-dashboard}
    domainname: ${DASHBOARD_DOMAIN:-dashboard.${BACKEND_DOMAIN:-spoon.gbrown.org}}
@@ -29,7 +29,7 @@ services:
    restart: unless-stopped

  stpeteit-backend:
-    image: ghcr.io/get-convex/convex-backend:${BACKEND_TAG:-latest}
+    image: ghcr.io/convex/convex-backend:${BACKEND_TAG:-latest}
    container_name: ${BACKEND_CONTAINER_NAME:-stpeteit-backend}
    hostname: ${BACKEND_CONTAINER_NAME:-stpeteit-backend}
    domainname: ${BACKEND_DOMAIN:-convex.stpeteit.com}
@@ -58,7 +58,7 @@ services:
    stop_signal: SIGINT

  stpeteit-dashboard:
-    image: ghcr.io/get-convex/convex-dashboard:${DASHBOARD_TAG:-latest}
+    image: ghcr.io/convex/convex-dashboard:${DASHBOARD_TAG:-latest}
    container_name: ${DASHBOARD_CONTAINER_NAME:-stpeteit-dashboard}
    hostname: ${DASHBOARD_CONTAINER_NAME:-stpeteit-dashboard}
    domainname: ${DASHBOARD_DOMAIN:-dashboard.${BACKEND_DOMAIN:-convex.stpete.com}}
@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: Tools
 After=network-online.target podman.socket
 Wants=network-online.target
@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: Uptime
 After=network-online.target
 Wants=network-online.target
@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: Vaultwarden
 After=network-online.target podman.socket
 Wants=network-online.target
@@ -0,0 +1,194 @@
+#!/usr/bin/env bash
+#
+# Daily backup of ~/Server to the home server, as hardlinked dated snapshots.
+#
+# WHAT THIS FIXES vs. the old version (2026-09-17):
+#   1. OWNERSHIP. Rootless podman leaves most container data owned by subuids
+#      (gitea = 525288, postgres data = the userns root mapping). Plain `rsync -a`
+#      running as gib cannot chown on the far side, so every previous backup landed
+#      100% owned by uid 1000. Restoring that gives you the right bytes with the wrong
+#      owners: postgres refuses to start unless PGDATA is owned correctly, and gitea
+#      fails on its own data dir. Fixed with `-M--fake-super`, which makes the REMOTE
+#      rsync stash mode/uid/gid in a user.rsync.%stat xattr (verified working on the
+#      home server's ZFS). Restore with --fake-super on the read side to put them back.
+#   2. CONSISTENCY. 13 SQLite DBs and a live postgres were copied hot, mid-write. Now
+#      dumped properly first (sqlite3 .backup / pg_dumpall) into _backup_state/.
+#   3. SPACE / HISTORY. Every run was a full ~19G copy into a new dated directory.
+#      Now it rsyncs into ONE stable directory, `current/`, and versioning is left to
+#      the home server: Media/Server_Backup is its own ZFS dataset with zfs-auto-snapshot
+#      keeping 14 dailies. ZFS snapshots are block-level, so a database that changes a
+#      few pages costs a few pages -- not a whole new copy the way dated dirs (or even
+#      rsync --link-dest hardlinks, which are per-file) would.
+#
+# Containers keep running throughout. That is safe *because* of the dumps above --
+# the dumps, not the raw data dirs, are the authoritative restore path.
+#
+# See RESTORE.md for how to bring this back on a new machine.
+
+set -uo pipefail
+
+REMOTE_HOST="[email protected]"
+REMOTE_ROOT="/home/gib/Public/Server_Backup/Containers/lilman.gib"
+SOURCE="$HOME/Server/"
+STATE_DIR="$HOME/Server/_backup_state"
+DEST_NAME="current"                # One stable directory. ZFS snapshots provide history.
+LOG_DIR="$HOME/Server/logs"
+LOG="$LOG_DIR/backup.log"
+LOCK="${XDG_RUNTIME_DIR:-/tmp}/server-backup.lock"
+
+# Versioning is ZFS's job, not ours -- see the header. We only sanity-check that it is
+# actually still happening, so this can never silently degrade into a single mirror with
+# no history. Warn if the newest snapshot of the dataset is older than this.
+REMOTE_DATASET="Media/Server_Backup"
+SNAPSHOT_MAX_AGE_HOURS=48
+
+mkdir -p "$LOG_DIR"
+log() { printf '%s  %s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG"; }
+die() { log "FATAL: $*"; exit 1; }
+
+# Refuse to overlap with a still-running backup rather than corrupting a snapshot.
+exec 9>"$LOCK" || die "cannot open lock $LOCK"
+flock -n 9 || die "another backup is already running"
+
+EXCLUSIONS=(
+    # --- build noise -------------------------------------------------------
+    # NOTE: these are bare patterns, so they match at ANY depth. Today they only
+    # catch genuine scratch dirs (checked 2026-09-17). If you add a service whose
+    # real data lives in a dir called cache/build/dist/tmp, it will be silently
+    # skipped -- re-check this list when adding services.
+    "node_modules" ".git" "*.log" "*.tmp" ".cache" "cache" "tmp" "temp"
+    ".npm" ".yarn" "dist" "build" "target" ".venv" "venv" "__pycache__"
+    "*.pyc" ".DS_Store" "Thumbs.db" "vendor" "*.mp4" "*.mkv"
+
+    # --- unreadable to gib, and genuinely disposable ------------------------
+    # Redis/valkey cache dumps: mode 0600 under a subuid because the container
+    # chowns its own data. Pure cache (usesend + infisical), regenerated on boot.
+    "dump.rdb"
+    # Gitea runs WITHOUT userns_mode (s6 needs container-root -- see AGENTS.md §3),
+    # so these are subuid-owned and mode 0700. Excluding them is what lets this run
+    # without sudo. Nothing here is needed to restore gitea:
+    #   sessions/     ephemeral web logins; worst case everyone logs in again
+    #   issues.bleve  search index, rebuilt from the DB on demand
+    #   Gitea/ssh     a 25-byte `environment` file for the git shell
+    #
+    # ⚠️ The SSH HOST KEYS are NOT here -- they live in Gitea/volume/ssh, are
+    # gib-owned, and DO get backed up. Losing those = host-key mismatch for every client.
+    "sessions" "issues.bleve" "Gitea/ssh"
+
+    # Mode 0600 under subuid 525288, so unreadable to gib -- this single file is what
+    # made every previous run exit 23. It is NOT disposable (it holds the MCP bridge's
+    # registered OAuth clients), so it is captured via `podman exec` into
+    # _backup_state/files/ instead of being read off disk.
+    "BillTracker/data/mcp/bridge.json"
+
+    # Stirling-PDF writes this 0600 under subuid 525287 (since its 2026-09-25 update),
+    # which made every run since exit 23. Excluded rather than captured: losing it only
+    # means re-entering any credentials stored in Stirling-PDF.
+    "pdf-tools/extraConfigs/credential-encryption.key"
+)
+
+EXCLUDE_ARGS=()
+for e in "${EXCLUSIONS[@]}"; do EXCLUDE_ARGS+=( "--exclude=$e" ); done
+
+# ---------------------------------------------------------------------------
+# Phase 1: consistent dumps of everything transactional.
+# ---------------------------------------------------------------------------
+dump_state() {
+    log "Dumping databases to _backup_state/ ..."
+    rm -rf "$STATE_DIR"
+    mkdir -p "$STATE_DIR"/{postgres,sqlite,files}
+    printf 'Generated %s by ~/Server/backup. Authoritative restore source -- see RESTORE.md\n' \
+        "$(date -Is)" > "$STATE_DIR/README"
+
+    # Live postgres (serves npm, authentik, usesend, ... -- pg_dumpall gets all of them).
+    if podman ps --format '{{.Names}}' | grep -qx postgresql; then
+        if podman exec postgresql pg_dumpall -U npm 2>>"$LOG" | gzip -9 > "$STATE_DIR/postgres/all.sql.gz"; then
+            log "  postgres: $(du -h "$STATE_DIR/postgres/all.sql.gz" | cut -f1)"
+        else
+            log "  WARNING: pg_dumpall failed -- raw pg_data is still copied, but is a hot copy"
+        fi
+    else
+        log "  WARNING: postgresql container not running, skipping pg_dumpall"
+    fi
+
+    # SQLite: .backup uses the online backup API, so this is safe while containers write.
+    local n=0
+    while IFS= read -r db; do
+        # *.db also matches stirling-pdf's H2 database, which is not SQLite. Check magic.
+        head -c 16 "$db" 2>/dev/null | grep -q "SQLite format 3" || continue
+        local rel="${db#"$HOME"/Server/}" out
+        out="$STATE_DIR/sqlite/$rel"
+        mkdir -p "$(dirname "$out")"
+        if sqlite3 -cmd ".timeout 30000" "$db" ".backup '$out'" 2>>"$LOG"; then n=$((n+1)); else log "  WARNING: sqlite dump failed: $rel"; fi
+    done < <(find "$HOME/Server" -path "$STATE_DIR" -prune -o \
+                  -type f \( -name '*.db' -o -name '*.sqlite' -o -name '*.sqlite3' \) -print 2>/dev/null)
+    log "  sqlite: $n databases"
+
+    # Files gib cannot read off disk, pulled through their container instead.
+    if podman ps --format '{{.Names}}' | grep -qx bill-tracker-mcp; then
+        podman exec bill-tracker-mcp cat /data/bridge.json \
+            > "$STATE_DIR/files/BillTracker--data--mcp--bridge.json" 2>>"$LOG" \
+            && log "  bridge.json captured" \
+            || log "  WARNING: could not capture bridge.json"
+    fi
+}
+
+# ---------------------------------------------------------------------------
+# Phase 2: confirm ZFS is still providing history.
+#
+# We deliberately overwrite one directory instead of keeping dated copies, which
+# means ZFS snapshots are the ONLY thing standing between us and "a mirror of a
+# mistake". If auto-snapshot ever stops, that must be loud, not silent.
+# ---------------------------------------------------------------------------
+check_zfs_history() {
+    local newest now age_h
+    newest=$(ssh -o BatchMode=yes "$REMOTE_HOST" \
+        "zfs list -H -p -o creation -t snapshot -s creation -d 1 '$REMOTE_DATASET' 2>/dev/null | tail -1" || true)
+    if [ -z "$newest" ]; then
+        log "  ⚠️  WARNING: no ZFS snapshots on $REMOTE_DATASET -- this backup has NO history."
+        log "      It is a plain mirror right now. Check zfs-auto-snapshot on the home server."
+        return
+    fi
+    now=$(date +%s)
+    age_h=$(( (now - newest) / 3600 ))
+    if [ "$age_h" -gt "$SNAPSHOT_MAX_AGE_HOURS" ]; then
+        log "  ⚠️  WARNING: newest ZFS snapshot is ${age_h}h old (limit ${SNAPSHOT_MAX_AGE_HOURS}h) -- versioning may be broken."
+    else
+        log "  ZFS history OK (newest snapshot ${age_h}h old, 14 dailies retained)"
+    fi
+}
+
+# ---------------------------------------------------------------------------
+# Phase 3: mirror.
+# ---------------------------------------------------------------------------
+log "=== backup start -> $REMOTE_HOST:$REMOTE_ROOT/$DEST_NAME ==="
+ssh -o BatchMode=yes -o ConnectTimeout=15 "$REMOTE_HOST" "mkdir -p '$REMOTE_ROOT/$DEST_NAME'" \
+    || die "cannot reach $REMOTE_HOST"
+
+dump_state
+check_zfs_history
+
+# --delete/--delete-excluded keep `current` an exact mirror of the source. Anything
+# removed here still exists in yesterday's ZFS snapshot, so this is not destructive.
+rsync -a --numeric-ids -M--fake-super --delete --delete-excluded \
+      --human-readable --stats \
+      "${EXCLUDE_ARGS[@]}" \
+      --rsync-path="mkdir -p '$REMOTE_ROOT/$DEST_NAME' && rsync" \
+      "$SOURCE" "$REMOTE_HOST:$REMOTE_ROOT/$DEST_NAME/" 2>&1 | tee -a "$LOG"
+status=${PIPESTATUS[0]}
+
+# 24 = "source files vanished during transfer". Normal on a live system, not a failure.
+if [ "$status" -ne 0 ] && [ "$status" -ne 24 ]; then
+    die "rsync failed with exit $status -- 'current' may be half-updated; yesterday's ZFS snapshot is still good"
+fi
+[ "$status" -eq 24 ] && log "NOTE: rsync exit 24 (files vanished mid-run) -- expected on a live system"
+
+# Optional heartbeat so a silent failure cannot go unnoticed. Create the file with an
+# uptime-kuma push URL in it and this pings on success only; absent, nothing happens.
+if [ -r "$HOME/Server/.backup-heartbeat-url" ]; then
+    url=$(< "$HOME/Server/.backup-heartbeat-url")
+    curl -fsS -m 20 "$url" >/dev/null 2>&1 \
+        && log "Heartbeat sent" || log "NOTE: heartbeat ping failed (backup itself was fine)"
+fi
+
+log "=== backup complete ==="
@@ -0,0 +1,22 @@
+[Unit]
+Description=Back up ~/Server to the home server
+Documentation=file:///home/gib/Server/RESTORE.md
+After=network-online.target
+Wants=network-online.target
+# The backup reads podman-managed data and shells into containers for dumps.
+RequiresMountsFor=/home/gib/Media
+
+[Service]
+Type=oneshot
+ExecStart=%h/Server/backup
+# A stalled rsync should fail loudly, not hang until the next timer fires.
+TimeoutStartSec=6h
+# This box is IO-bound before it is CPU-bound (~18%% avg60 pressure), so stay
+# out of the way of the containers. best-effort/7 rather than idle, which can
+# starve the transfer outright when the volume is busy.
+Nice=10
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7
+
+[Install]
+WantedBy=default.target
@@ -0,0 +1,18 @@
+[Unit]
+Description=Daily backup of ~/Server to the home server
+
+[Timer]
+# MUST finish before the home server's cron.daily fires zfs-auto-snapshot, which has
+# been landing between 03:22 and 03:52. If the snapshot catches a half-finished rsync,
+# that day's history entry is a partial backup. 01:00 leaves >2h of margin; the first
+# full run took well under that and daily deltas are far smaller.
+OnCalendar=*-*-* 01:00:00
+# Small jitter only -- must not push us into the snapshot window.
+RandomizedDelaySec=10m
+# If the VPS was down at 01:00, run once after it comes back rather than
+# silently skipping a day.
+Persistent=true
+Unit=server-backup.service
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+# Post-reboot verification for the rootless podman stack.
+# Run as gib. Compares against ~/premigration_backup_2026-08-10/post_migration_baseline.txt
+set -uo pipefail
+
+fail=0
+ok()   { printf '  \033[32mOK\033[0m   %s\n' "$1"; }
+bad()  { printf '  \033[31mFAIL\033[0m %s\n' "$1"; fail=$((fail+1)); }
+
+echo "== linger =="
+[[ "$(loginctl show-user gib --property=Linger --value 2>/dev/null)" == "yes" ]] \
+  && ok "Linger=yes" || bad "linger is OFF - user units will not start at boot"
+
+echo "== unprivileged port start =="
+p=$(sysctl -n net.ipv4.ip_unprivileged_port_start 2>/dev/null)
+[[ "$p" == "80" ]] && ok "ip_unprivileged_port_start=80" \
+  || bad "ip_unprivileged_port_start=$p (NPM cannot bind :80/:443)"
+
+echo "== user units =="
+for s in postgresql bang completeuphoria sierraandtyler uptime portainer beszel authentik tools isponsorblocktv vaultwarden infisical rustdesk nginx-proxy-manager gbrown-agent; do
+  st=$(systemctl --user is-active "podman-$s.service" 2>/dev/null)
+  [[ "$st" == "active" ]] && ok "$s" || bad "$s is $st"
+done
+
+echo "== containers =="
+for c in postgresql nginx-proxy-manager uptime portainer bang completeuphoria sierraandtyler beszel beszel-agent authentik-server authentik-worker convertx it-tools pdf-tools bento-pdf iSponsorBlockTV vaultwarden infisical-redis infisical-backend rustdesk-server gbrown-agent; do
+  if podman ps --format '{{.Names}}' 2>/dev/null | grep -qx "$c"; then ok "$c"; else bad "$c not running"; fi
+done
+
+echo "== postgres health =="
+h=$(podman inspect postgresql --format '{{.State.Health.Status}}' 2>/dev/null)
+[[ "$h" == "healthy" ]] && ok "postgres healthy" || bad "postgres health=$h"
+
+echo "== endpoints =="
+chk() { local n=$1 exp=$2; shift 2; c=$(curl -sk -o /dev/null -w '%{http_code}' "$@" 2>/dev/null)
+        [[ "$c" == "$exp" ]] && ok "$n -> $c" || bad "$n -> $c (expected $exp)"; }
+# Admin ports moved to the WireGuard address 2026-08-12 (AGENTS.md §13.2).
+# They no longer listen on localhost, so probe 192.168.2.2.
+chk "NPM :81"   200 http://192.168.2.2:81
+chk "NPM :80"   200 http://localhost:80
+chk "NPM :443"  200 --resolve nginx.gibbyb.com:443:127.0.0.1 https://nginx.gibbyb.com
+chk "Uptime"    302 http://192.168.2.2:3001
+chk "Portainer" 200 https://192.168.2.2:9443
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Boot-enablement check (added 2026-08-12)
+#
+# Five stacks were once found RUNNING but DISABLED — healthy in `podman ps`, yet they
+# would have disappeared on the next reboot. Starting a stack and enabling it are
+# separate actions; this catches the gap. See AGENTS.md §13.3.
+# ─────────────────────────────────────────────────────────────────────────────
+echo
+echo "== boot-enablement =="
+# podman's own units are meant to stay disabled; n8n is intentionally down.
+# podman's own units stay disabled by design; n8n is intentionally down;
+# podman-update.service is `static` (the timer activates it, so it is never enabled);
+# watchtower was RETIRED 2026-08-17 and must stay disabled (AGENTS.md §13.15).
+SKIP='podman-auto-update|podman-restart|podman-clean-transient|podman-kube@|podman-user-wait-network-online|podman-n8n|podman-update|podman-watchtower'
+bad=0
+for u in $(systemctl --user list-unit-files 'podman-*.service' --no-legend 2>/dev/null | awk '{print $1}'); do
+    echo "$u" | grep -qE "$SKIP" && continue
+    state=$(systemctl --user is-enabled "$u" 2>/dev/null)
+    if [ "$state" != "enabled" ]; then
+        echo "  WILL NOT SURVIVE REBOOT: $u ($state)  -> systemctl --user enable $u"
+        bad=$((bad+1))
+    fi
+done
+[ "$bad" -eq 0 ] && echo "  all service units enabled"
+
+echo
+echo "== nightly image updates (replaces watchtower — AGENTS.md §13.15) =="
+if [ "$(systemctl --user is-enabled podman-update.timer 2>/dev/null)" = "enabled" ]; then
+    echo "  timer enabled, next: $(systemctl --user list-timers podman-update.timer --no-pager 2>/dev/null | awk 'NR==2{print $1,$2,$3,$4}')"
+else
+    echo "  WARNING: podman-update.timer is NOT enabled — nothing will update"
+fi
+if [ "$(systemctl --user is-active podman-watchtower.service 2>/dev/null)" = "active" ]; then
+    echo "  WARNING: watchtower is RUNNING — it recreates containers outside their pod (§13.13)"
+fi
+# A container outside its compose pod is the signature of a watchtower-style recreate.
+podless=""
+for c in $(podman ps --format '{{.Names}}' 2>/dev/null); do
+    [ -z "$(podman inspect "$c" --format '{{.Pod}}' 2>/dev/null)" ] && podless="$podless $c"
+done
+[ -n "$podless" ] && echo "  PODLESS (rebuild with systemctl --user restart):$podless" || echo "  all containers in their pod"
+
+echo
+echo "== admin ports bound to 0.0.0.0 (should be 192.168.2.2 — AGENTS.md §13.2) =="
+podman ps --format '{{.Names}} {{.Ports}}' | grep -E '0\.0\.0\.0:(81|80-81|3001|9443|5432)(->|-)' \
+    | sed 's/^/  EXPOSED: /' || echo "  none"
+
+echo
+if [[ $fail -eq 0 ]]; then
+  printf '\033[32mALL CHECKS PASSED\033[0m\n'
+else
+  printf '\033[31m%d CHECK(S) FAILED\033[0m\n' "$fail"
+  echo "Investigate: journalctl --user -u podman-<name>.service -n 50"
+fi
+exit $fail