#!/usr/bin/env bash
#
# Daily backup of ~/Server to the home server, as hardlinked dated snapshots.
#
# WHAT THIS FIXES vs. the old version (2026-09-17):
#   1. OWNERSHIP. Rootless podman leaves most container data owned by subuids
#      (gitea = 525288, postgres data = the userns root mapping). Plain `rsync -a`
#      running as gib cannot chown on the far side, so every previous backup landed
#      100% owned by uid 1000. Restoring that gives you the right bytes with the wrong
#      owners: postgres refuses to start unless PGDATA is owned correctly, and gitea
#      fails on its own data dir. Fixed with `-M--fake-super`, which makes the REMOTE
#      rsync stash mode/uid/gid in a user.rsync.%stat xattr (verified working on the
#      home server's ZFS). Restore with --fake-super on the read side to put them back.
#   2. CONSISTENCY. 13 SQLite DBs and a live postgres were copied hot, mid-write. Now
#      dumped properly first (sqlite3 .backup / pg_dumpall) into _backup_state/.
#   3. SPACE / HISTORY. Every run was a full ~19G copy into a new dated directory.
#      Now it rsyncs into ONE stable directory, `current/`, and versioning is left to
#      the home server: Media/Server_Backup is its own ZFS dataset with zfs-auto-snapshot
#      keeping 14 dailies. ZFS snapshots are block-level, so a database that changes a
#      few pages costs a few pages -- not a whole new copy the way dated dirs (or even
#      rsync --link-dest hardlinks, which are per-file) would.
#
# Containers keep running throughout. That is safe *because* of the dumps above --
# the dumps, not the raw data dirs, are the authoritative restore path.
#
# See RESTORE.md for how to bring this back on a new machine.

set -uo pipefail

REMOTE_HOST="gib@server.gib"
REMOTE_ROOT="/home/gib/Public/Server_Backup/Containers/lilman.gib"
SOURCE="$HOME/Server/"
STATE_DIR="$HOME/Server/_backup_state"
DEST_NAME="current"                # One stable directory. ZFS snapshots provide history.
LOG_DIR="$HOME/Server/logs"
LOG="$LOG_DIR/backup.log"
LOCK="${XDG_RUNTIME_DIR:-/tmp}/server-backup.lock"

# Versioning is ZFS's job, not ours -- see the header. We only sanity-check that it is
# actually still happening, so this can never silently degrade into a single mirror with
# no history. Warn if the newest snapshot of the dataset is older than this.
REMOTE_DATASET="Media/Server_Backup"
SNAPSHOT_MAX_AGE_HOURS=48

mkdir -p "$LOG_DIR"
log() { printf '%s  %s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG"; }
die() { log "FATAL: $*"; exit 1; }

# Refuse to overlap with a still-running backup rather than corrupting a snapshot.
exec 9>"$LOCK" || die "cannot open lock $LOCK"
flock -n 9 || die "another backup is already running"

EXCLUSIONS=(
    # --- build noise -------------------------------------------------------
    # NOTE: these are bare patterns, so they match at ANY depth. Today they only
    # catch genuine scratch dirs (checked 2026-09-17). If you add a service whose
    # real data lives in a dir called cache/build/dist/tmp, it will be silently
    # skipped -- re-check this list when adding services.
    "node_modules" ".git" "*.log" "*.tmp" ".cache" "cache" "tmp" "temp"
    ".npm" ".yarn" "dist" "build" "target" ".venv" "venv" "__pycache__"
    "*.pyc" ".DS_Store" "Thumbs.db" "vendor" "*.mp4" "*.mkv"

    # --- unreadable to gib, and genuinely disposable ------------------------
    # Redis/valkey cache dumps: mode 0600 under a subuid because the container
    # chowns its own data. Pure cache (usesend + infisical), regenerated on boot.
    "dump.rdb"
    # Gitea runs WITHOUT userns_mode (s6 needs container-root -- see AGENTS.md §3),
    # so these are subuid-owned and mode 0700. Excluding them is what lets this run
    # without sudo. Nothing here is needed to restore gitea:
    #   sessions/     ephemeral web logins; worst case everyone logs in again
    #   issues.bleve  search index, rebuilt from the DB on demand
    #   Gitea/ssh     a 25-byte `environment` file for the git shell
    #
    # ⚠️ The SSH HOST KEYS are NOT here -- they live in Gitea/volume/ssh, are
    # gib-owned, and DO get backed up. Losing those = host-key mismatch for every client.
    "sessions" "issues.bleve" "Gitea/ssh"

    # Mode 0600 under subuid 525288, so unreadable to gib -- this single file is what
    # made every previous run exit 23. It is NOT disposable (it holds the MCP bridge's
    # registered OAuth clients), so it is captured via `podman exec` into
    # _backup_state/files/ instead of being read off disk.
    "BillTracker/data/mcp/bridge.json"

    # Stirling-PDF writes this 0600 under subuid 525287 (since its 2026-09-25 update),
    # which made every run since exit 23. Excluded rather than captured: losing it only
    # means re-entering any credentials stored in Stirling-PDF.
    "pdf-tools/extraConfigs/credential-encryption.key"
)

EXCLUDE_ARGS=()
for e in "${EXCLUSIONS[@]}"; do EXCLUDE_ARGS+=( "--exclude=$e" ); done

# ---------------------------------------------------------------------------
# Phase 1: consistent dumps of everything transactional.
# ---------------------------------------------------------------------------
dump_state() {
    log "Dumping databases to _backup_state/ ..."
    rm -rf "$STATE_DIR"
    mkdir -p "$STATE_DIR"/{postgres,sqlite,files}
    printf 'Generated %s by ~/Server/backup. Authoritative restore source -- see RESTORE.md\n' \
        "$(date -Is)" > "$STATE_DIR/README"

    # Live postgres (serves npm, authentik, usesend, ... -- pg_dumpall gets all of them).
    if podman ps --format '{{.Names}}' | grep -qx postgresql; then
        if podman exec postgresql pg_dumpall -U npm 2>>"$LOG" | gzip -9 > "$STATE_DIR/postgres/all.sql.gz"; then
            log "  postgres: $(du -h "$STATE_DIR/postgres/all.sql.gz" | cut -f1)"
        else
            log "  WARNING: pg_dumpall failed -- raw pg_data is still copied, but is a hot copy"
        fi
    else
        log "  WARNING: postgresql container not running, skipping pg_dumpall"
    fi

    # SQLite: .backup uses the online backup API, so this is safe while containers write.
    local n=0
    while IFS= read -r db; do
        # *.db also matches stirling-pdf's H2 database, which is not SQLite. Check magic.
        head -c 16 "$db" 2>/dev/null | grep -q "SQLite format 3" || continue
        local rel="${db#"$HOME"/Server/}" out
        out="$STATE_DIR/sqlite/$rel"
        mkdir -p "$(dirname "$out")"
        if sqlite3 -cmd ".timeout 30000" "$db" ".backup '$out'" 2>>"$LOG"; then n=$((n+1)); else log "  WARNING: sqlite dump failed: $rel"; fi
    done < <(find "$HOME/Server" -path "$STATE_DIR" -prune -o \
                  -type f \( -name '*.db' -o -name '*.sqlite' -o -name '*.sqlite3' \) -print 2>/dev/null)
    log "  sqlite: $n databases"

    # Files gib cannot read off disk, pulled through their container instead.
    if podman ps --format '{{.Names}}' | grep -qx bill-tracker-mcp; then
        podman exec bill-tracker-mcp cat /data/bridge.json \
            > "$STATE_DIR/files/BillTracker--data--mcp--bridge.json" 2>>"$LOG" \
            && log "  bridge.json captured" \
            || log "  WARNING: could not capture bridge.json"
    fi
}

# ---------------------------------------------------------------------------
# Phase 2: confirm ZFS is still providing history.
#
# We deliberately overwrite one directory instead of keeping dated copies, which
# means ZFS snapshots are the ONLY thing standing between us and "a mirror of a
# mistake". If auto-snapshot ever stops, that must be loud, not silent.
# ---------------------------------------------------------------------------
check_zfs_history() {
    local newest now age_h
    newest=$(ssh -o BatchMode=yes "$REMOTE_HOST" \
        "zfs list -H -p -o creation -t snapshot -s creation -d 1 '$REMOTE_DATASET' 2>/dev/null | tail -1" || true)
    if [ -z "$newest" ]; then
        log "  ⚠️  WARNING: no ZFS snapshots on $REMOTE_DATASET -- this backup has NO history."
        log "      It is a plain mirror right now. Check zfs-auto-snapshot on the home server."
        return
    fi
    now=$(date +%s)
    age_h=$(( (now - newest) / 3600 ))
    if [ "$age_h" -gt "$SNAPSHOT_MAX_AGE_HOURS" ]; then
        log "  ⚠️  WARNING: newest ZFS snapshot is ${age_h}h old (limit ${SNAPSHOT_MAX_AGE_HOURS}h) -- versioning may be broken."
    else
        log "  ZFS history OK (newest snapshot ${age_h}h old, 14 dailies retained)"
    fi
}

# ---------------------------------------------------------------------------
# Phase 3: mirror.
# ---------------------------------------------------------------------------
log "=== backup start -> $REMOTE_HOST:$REMOTE_ROOT/$DEST_NAME ==="
ssh -o BatchMode=yes -o ConnectTimeout=15 "$REMOTE_HOST" "mkdir -p '$REMOTE_ROOT/$DEST_NAME'" \
    || die "cannot reach $REMOTE_HOST"

dump_state
check_zfs_history

# --delete/--delete-excluded keep `current` an exact mirror of the source. Anything
# removed here still exists in yesterday's ZFS snapshot, so this is not destructive.
rsync -a --numeric-ids -M--fake-super --delete --delete-excluded \
      --human-readable --stats \
      "${EXCLUDE_ARGS[@]}" \
      --rsync-path="mkdir -p '$REMOTE_ROOT/$DEST_NAME' && rsync" \
      "$SOURCE" "$REMOTE_HOST:$REMOTE_ROOT/$DEST_NAME/" 2>&1 | tee -a "$LOG"
status=${PIPESTATUS[0]}

# 24 = "source files vanished during transfer". Normal on a live system, not a failure.
if [ "$status" -ne 0 ] && [ "$status" -ne 24 ]; then
    die "rsync failed with exit $status -- 'current' may be half-updated; yesterday's ZFS snapshot is still good"
fi
[ "$status" -eq 24 ] && log "NOTE: rsync exit 24 (files vanished mid-run) -- expected on a live system"

# Optional heartbeat so a silent failure cannot go unnoticed. Create the file with an
# uptime-kuma push URL in it and this pings on success only; absent, nothing happens.
if [ -r "$HOME/Server/.backup-heartbeat-url" ]; then
    url=$(< "$HOME/Server/.backup-heartbeat-url")
    curl -fsS -m 20 "$url" >/dev/null 2>&1 \
        && log "Heartbeat sent" || log "NOTE: heartbeat ping failed (backup itself was fine)"
fi

log "=== backup complete ==="