#!/usr/bin/env bash
#
# Daily backup of ~/Server (NVMe) to the ZFS pool on this same machine.
# Purpose: survive the NVMe/LUKS volume dying. Not an offsite backup.
#
# Runs as ROOT via server-backup.service. That is deliberate: this box runs Docker,
# so most container data under ~/Server is root-owned. The old script used `sudo rsync`,
# which worked but could never be automated without a sudoers rule. A systemd system
# unit is already root, so ownership is preserved natively and no sudoers entry is
# needed at runtime. (Note this differs from the VPS, which is rootless podman and
# needs rsync --fake-super to carry subuid ownership; nothing like that applies here.)
#
# History is ZFS's job: Media/Server_Backup has zfs-auto-snapshot keeping 14 dailies,
# so this writes ONE directory and never makes dated copies. Snapshots are block-level,
# so a 1.6 GB database that changes a few pages costs a few pages.
#
# See RESTORE.md for how to bring this back.

set -uo pipefail

SOURCE="/home/gib/Server/"
DEST="/home/gib/Public/Server_Backup/Containers/server.gib/current"
STATE_DIR="/home/gib/Server/_backup_state"
LOG_DIR="/home/gib/Server/logs"
LOG="$LOG_DIR/backup.log"
LOCK="/run/server-backup.lock"
REMOTE_DATASET="Media/Server_Backup"
SNAPSHOT_MAX_AGE_HOURS=48

mkdir -p "$LOG_DIR"
log() { printf '%s  %s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG"; }
die() { log "FATAL: $*"; exit 1; }

exec 9>"$LOCK" || die "cannot open lock $LOCK"
flock -n 9 || die "another backup is already running"

EXCLUSIONS=(
    # --- build noise (inherited from the original script) -------------------
    "node_modules" ".git" "*.log" "*.tmp" ".cache" "cache" "tmp" "temp"
    ".npm" ".yarn" "dist" "build" "target" ".venv" "venv" "__pycache__"
    "*.pyc" ".DS_Store" "Thumbs.db" "vendor" "*.mp4" "*.mkv"

    # --- the big one: Gitea CI runner Docker-in-Docker storage --------------
    # 203 GB across runner-1/2/3 (72 GB of containerd image layers each), and it
    # churns constantly as CI builds run. It is pure cache: a rebuilt runner just
    # re-pulls everything. Excluding dind/ specifically KEEPS runner-N/data/.runner
    # (the 648-byte registration token) and certs/, which are the only parts that
    # would actually be painful to recreate.
    #
    # Measured as root on 2026-09-17 -- a plain `du` as gib reports these as ~13 MB
    # because it cannot descend into root-owned dirs. Do not trust non-root sizes here.
    "/Agents/Gitea/volumes/*/dind"

    # Regenerated every run; no reason to ship the previous run's copy.
    "/_backup_state/*.partial"
)

EXCLUDE_ARGS=()
for e in "${EXCLUSIONS[@]}"; do EXCLUDE_ARGS+=( "--exclude=$e" ); done

# Convex deployments backed by SQLite, as "container:sqlite path under ~/Server".
# Convex keeps that file in rollback-journal mode and writes to it constantly, so
# sqlite3 .backup either hits "database is locked" or restarts forever. Instead,
# dump_convex takes Convex's own snapshot export (`npx convex export`, restored with
# `npx convex import`), and dump_state skips the raw file.
CONVEX_EXPORTS=(
    "gbrown-backend:Gbrown/data/convex/db.sqlite3"
)

# ---------------------------------------------------------------------------
# Consistent dumps. Containers keep running; these are what make that safe.
# ---------------------------------------------------------------------------
dump_state() {
    log "Dumping databases to _backup_state/ ..."
    rm -rf "$STATE_DIR"
    mkdir -p "$STATE_DIR/sqlite"
    printf 'Generated %s by /usr/local/sbin/server-backup. Authoritative restore source.\n' \
        "$(date -Is)" > "$STATE_DIR/README"

    local n=0 skipped=0
    while IFS= read -r db; do
        # Only real SQLite. Docker/containerd/buildkit scatter BoltDB files named *.db
        # (portainer.db, metadata_v2.db, meta.db ...) which sqlite3 cannot read.
        read -r -n 15 magic < "$db" 2>/dev/null
        [ "$magic" = "SQLite format 3" ] || { skipped=$((skipped+1)); continue; }

        local rel="${db#/home/gib/Server/}" out
        [[ " ${CONVEX_EXPORTS[*]#*:} " == *" $rel "* ]] && continue   # dump_convex covers it
        out="$STATE_DIR/sqlite/$rel"
        mkdir -p "$(dirname "$out")"
        # .timeout: wait up to 30s for a writer's lock instead of failing on first contact.
        if sqlite3 -cmd ".timeout 30000" "$db" ".backup '$out'" 2>>"$LOG"; then
            n=$((n+1))
        else
            log "  WARNING: sqlite dump failed: $rel"
        fi
    done < <(find /home/gib/Server \
                  -path "$STATE_DIR" -prune -o \
                  -path '*/dind' -prune -o \
                  -path '*/other/firefox' -prune -o \
                  -path '*/node_modules' -prune -o \
                  -type f \( -name '*.db' -o -name '*.sqlite' -o -name '*.sqlite3' \) \
                  ! -name '*-wal' ! -name '*-shm' \
                  ! -name '*logs.db' ! -name '*cache*.db' -print 2>/dev/null)
    log "  sqlite: $n dumped, $skipped skipped (not SQLite)"
    dump_convex
}

# Snapshot-exports each CONVEX_EXPORTS deployment to _backup_state/convex/<container>.zip
# through a throwaway node container, so nothing is installed on the host. The admin key
# is derived from the instance secret on every run rather than stored. Takes ~8 min.
dump_convex() {
    mkdir -p "$STATE_DIR/convex"
    local entry c key
    for entry in "${CONVEX_EXPORTS[@]}"; do
        c="${entry%%:*}"
        key=$(docker exec "$c" ./generate_admin_key.sh 2>/dev/null | tail -1)
        if [ -z "$key" ]; then
            log "  WARNING: convex export failed: $c (could not generate admin key -- is it running?)"
            continue
        fi
        if CONVEX_SELF_HOSTED_ADMIN_KEY="$key" timeout 1800 docker run --rm \
               --network nginx-bridge -v "$STATE_DIR/convex:/out" \
               -e CONVEX_SELF_HOSTED_URL="http://$c:3210" -e CONVEX_SELF_HOSTED_ADMIN_KEY \
               -e npm_config_update_notifier=false -e npm_config_fund=false -e npm_config_audit=false \
               docker.io/library/node:22-alpine sh -ec '
                   # The CLI only runs inside a project that depends on convex.
                   mkdir /work && cd /work && echo "{}" > package.json
                   npm install --silent convex@latest
                   npx convex export --include-file-storage --path "/out/'"$c"'.zip"' \
               >>"$LOG" 2>&1; then
            log "  convex: exported $c ($(du -h "$STATE_DIR/convex/$c.zip" | cut -f1))"
        else
            log "  WARNING: convex export failed: $c (see npx output above)"
        fi
    done
}

# ---------------------------------------------------------------------------
# ZFS is the only thing giving us history, so make its absence loud.
# ---------------------------------------------------------------------------
check_zfs_history() {
    local newest now age_h
    newest=$(zfs list -H -p -o creation -t snapshot -s creation -d 1 "$REMOTE_DATASET" 2>/dev/null | tail -1)
    if [ -z "$newest" ]; then
        log "  ⚠️  WARNING: no ZFS snapshots on $REMOTE_DATASET -- this backup has NO history."
        log "      It is a plain mirror right now. Check zfs-auto-snapshot."
        return
    fi
    now=$(date +%s); age_h=$(( (now - newest) / 3600 ))
    if [ "$age_h" -gt "$SNAPSHOT_MAX_AGE_HOURS" ]; then
        log "  ⚠️  WARNING: newest ZFS snapshot is ${age_h}h old (limit ${SNAPSHOT_MAX_AGE_HOURS}h) -- versioning may be broken."
    else
        log "  ZFS history OK (newest snapshot ${age_h}h old, 14 dailies retained)"
    fi
}

log "=== backup start -> $DEST ==="
[ -d "$(dirname "$DEST")" ] || die "destination parent missing -- is the ZFS pool mounted?"
mountpoint -q /home/gib/Public || die "/home/gib/Public is not a mountpoint -- refusing to write into an unmounted path"
mkdir -p "$DEST"

dump_state
check_zfs_history

rsync -a --numeric-ids --delete --delete-excluded --human-readable --stats \
      "${EXCLUDE_ARGS[@]}" "$SOURCE" "$DEST/" 2>&1 | tee -a "$LOG"
status=${PIPESTATUS[0]}

if [ "$status" -ne 0 ] && [ "$status" -ne 24 ]; then
    die "rsync failed with exit $status -- 'current' may be half-updated; yesterday's ZFS snapshot is still good"
fi
[ "$status" -eq 24 ] && log "NOTE: rsync exit 24 (files vanished mid-run) -- expected on a live system"

if [ -r /home/gib/Server/.backup-heartbeat-url ]; then
    url=$(< /home/gib/Server/.backup-heartbeat-url)
    curl -fsS -m 20 "$url" >/dev/null 2>&1 \
        && log "Heartbeat sent" || log "NOTE: heartbeat ping failed (backup itself was fine)"
fi

log "=== backup complete ==="