#!/usr/bin/env bash
#
# Weekly copy of the Media pool -> the Backup pool, both on this machine.
#
# Purpose: survive losing the Media pool itself (or a bad delete on it). Both pools are
# in this one box, so this is NOT an offsite backup and never will be.
#
#   source  /home/gib/Public                     (Media, 50.9 T raidz)
#   dest    /home/gib/.local/share/Backup/Public (Backup, 14.5 T raidz1)
#
# Runs as ROOT via media-backup.service. Not optional: /home/gib/Public/Nextcloud/Server is
# uid 33 mode 0750 and the borg repo is root:root mode 0600. The historical hand-run rsync
# ran as gib, could not read those, skipped them -- AND EXITED 0. That is how 442 G of
# Nextcloud ended up in the Backup pool while nobody could tell it was the wrong 442 G.
#
# HISTORY IS TAKEN BY THIS SCRIPT, NOT BY A TIMER. zfs-auto-snapshot is disabled on the
# Backup pool (com.sun:auto-snapshot=false) because a timer-driven snapshot can land
# mid-rsync and record a half-written backup as that week's restore point. Here the snapshot
# is taken only AFTER a successful sync, so every snapshot on that pool is, by construction,
# a complete backup.

set -uo pipefail

SOURCE="/home/gib/Public/"
DEST="/home/gib/.local/share/Backup/Public"
POOL="Backup"
BORG_LOCK="/home/gib/Public/Nextcloud/Backups/borg/lock.exclusive"
KEEP_SNAPSHOTS=8            # ~2 months at weekly
MIN_FREE_GB=200             # refuse to start if the pool is already this tight
LOG="/var/log/media-backup.log"
LOCK="/run/media-backup.lock"

log() { printf '%s  %s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG"; }
die() { log "FATAL: $*"; exit 1; }

exec 9>"$LOCK" || die "cannot open lock $LOCK"
flock -n 9 || die "another run is already in progress"

EXCLUDES=(
    # --- too big, deliberately not backed up -------------------------------
    # 23.2 T. Larger on its own than the entire Backup pool. gib's explicit decision:
    # if Media is lost, the TV library gets rebuilt.
    "/TV"

    # Torrent scratch. qBittorrent manages it (ratio 2, ~14.7 d inactive, then
    # RemoveWithContent), and anything worth keeping has already been imported.
    "/Downloads"

    # --- already self-versioning, so a second copy buys nothing ------------
    # TimeMachine is its own backup format with its own history.
    "/Mac"

    # 586 G of raw Nextcloud data. Covered by AIO's borg repo below, which is
    # deduplicated, encrypted, and carries 7d/4w/6m of history instead of one stale
    # point-in-time. Verified: the borgbackup container mounts this exact path as
    # /nextcloud_aio_volumes/nextcloud_aio_nextcloud_data and borg walks the whole tree.
    # We keep Nextcloud/Backups (the repo). We do NOT keep Nextcloud/Server.
    "/Nextcloud/Server"

    # --- regenerable ------------------------------------------------------
    "/Server_Backup/Ollama_Models"     # re-pullable model weights
    "/Server_Backup/ComfyUI"           # re-downloadable checkpoints
    "/Server_Backup/frigate"           # camera footage; container is decommissioned
    "/Server_Backup/.Trash-1000"       # desktop trash
)
# Kept on purpose, so nobody "tidies" them later: Server_Backup/Containers (the only copy
# of both machines' ~/Server backups outside the Media pool), immich (irreplaceable photos),
# Games and ROMs (gib: "they can be hard to get").

EXCLUDE_ARGS=()
for e in "${EXCLUDES[@]}"; do EXCLUDE_ARGS+=( "--exclude=$e" ); done

log "=== media-backup start ==="

# --- preflight ------------------------------------------------------------
mountpoint -q /home/gib/Public \
    || die "/home/gib/Public is not a mountpoint -- Media pool not imported, refusing to run"
mountpoint -q /home/gib/.local/share/Backup \
    || die "Backup pool is not mounted -- refusing to write into a plain directory on the NVMe"
mkdir -p "$DEST"

# A borg run holds this lock for 2-4 hours and rewrites index/hints/integrity at the END.
# Copying mid-run pairs new data chunks with a stale index: a repo that looks fine and
# fails at restore. AIO's borg runs daily from 07:30 UTC for about 2 hours, so it can
# still be going when the timer fires. Wait for it to finish, checking every
# BORG_WAIT_INTERVAL, and defer only if it is still held after BORG_WAIT_MAX. The unit's
# TimeoutStartSec=24h covers the wait plus the ~50 min sync.
BORG_WAIT_INTERVAL=300      # 5 min
BORG_WAIT_MAX=21600         # 6 h
waited=0
while [ -e "$BORG_LOCK" ] && [ "$waited" -lt "$BORG_WAIT_MAX" ]; do
    log "borg is mid-run (lock held at $BORG_LOCK). Waited $((waited / 60)) of $((BORG_WAIT_MAX / 60)) min, checking again in $((BORG_WAIT_INTERVAL / 60)) min"
    sleep "$BORG_WAIT_INTERVAL"
    waited=$(( waited + BORG_WAIT_INTERVAL ))
done
if [ -e "$BORG_LOCK" ]; then
    log "borg is mid-run (lock held at $BORG_LOCK)"
    log "DEFERRING this week's run rather than copying a torn repo. Next fire: next Saturday."
    log "=== media-backup deferred ==="
    exit 0
fi
[ "$waited" -gt 0 ] && log "borg lock released after waiting $((waited / 60)) min, continuing"

avail_gb=$(zfs get -Hp -o value available "$POOL" 2>/dev/null)
avail_gb=$(( ${avail_gb:-0} / 1024 / 1024 / 1024 ))
log "Backup pool free: ${avail_gb} GB"
[ "$avail_gb" -lt "$MIN_FREE_GB" ] \
    && die "only ${avail_gb} GB free (floor ${MIN_FREE_GB} GB) -- refusing to start"

# --- sync -----------------------------------------------------------------
# --numeric-ids is mandatory: `getent passwd 33` returns nothing on this host (uid 33 is
# www-data inside a container), so rsync has no name to map ownership by.
# --delete/--delete-excluded keep the copy exact; anything wrongly removed is still in
# last week's snapshot.
log "Syncing ${SOURCE} -> ${DEST}/ ..."
rsync -a --numeric-ids --delete --delete-excluded --human-readable --stats \
      "${EXCLUDE_ARGS[@]}" "$SOURCE" "$DEST/" 2>&1 | tee -a "$LOG"
status=${PIPESTATUS[0]}

# 24 = "source files vanished during transfer". Normal on a live system.
if [ "$status" -ne 0 ] && [ "$status" -ne 24 ]; then
    die "rsync exit $status -- NOT snapshotting. Last good snapshot is still the newest."
fi
[ "$status" -eq 24 ] && log "NOTE: rsync exit 24 (files vanished mid-run) -- expected"

# --- snapshot, only now that the sync succeeded ---------------------------
SNAP="${POOL}@backup-$(date +%Y-%m-%d)"
if zfs list -t snapshot -H "$SNAP" >/dev/null 2>&1; then
    log "snapshot $SNAP already exists (second run today), skipping"
else
    zfs snapshot "$SNAP" && log "snapshot taken: $SNAP" || log "WARNING: snapshot failed"
fi

# Keep the newest N. These are ours, so nothing else prunes them.
mapfile -t snaps < <(zfs list -t snapshot -H -o name -s creation -d 1 "$POOL" 2>/dev/null | grep "@backup-")
if [ "${#snaps[@]}" -gt "$KEEP_SNAPSHOTS" ]; then
    drop=$(( ${#snaps[@]} - KEEP_SNAPSHOTS ))
    for s in "${snaps[@]:0:$drop}"; do
        zfs destroy "$s" && log "pruned old snapshot: $s"
    done
fi
log "snapshots retained: $(zfs list -t snapshot -H -o name -d 1 "$POOL" 2>/dev/null | grep -c '@backup-')"

# Heartbeat for Uptime Kuma ("Media Backup (server.gib)"), sent only after a completed
# sync and snapshot. A deferred week exits before this, so it does not ping.
if [ -r /home/gib/Server/.media-backup-heartbeat-url ]; then
    url=$(< /home/gib/Server/.media-backup-heartbeat-url)
    curl -fsS -m 20 "$url" >/dev/null 2>&1 \
        && log "Heartbeat sent" || log "NOTE: heartbeat ping failed (backup itself was fine)"
fi

log "=== media-backup complete ==="