gibandClaude Opus 5.5
86ea1a3374
AIO's borg now runs daily from 07:30 UTC for about 2 hours, so the Saturday 05:00 fire can find the lock held and skip the whole week, which trips Uptime Kuma's 8-day push window. Poll the lock every 5 minutes for up to 6 hours and defer only if it is still held after that. Co-Authored-By: Claude Opus 5.5 <[email protected]>
153 lines
7.3 KiB
Bash
Executable File
153 lines
7.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Weekly copy of the Media pool -> the Backup pool, both on this machine.
|
|
#
|
|
# Purpose: survive losing the Media pool itself (or a bad delete on it). Both pools are
|
|
# in this one box, so this is NOT an offsite backup and never will be.
|
|
#
|
|
# source /home/gib/Public (Media, 50.9 T raidz)
|
|
# dest /home/gib/.local/share/Backup/Public (Backup, 14.5 T raidz1)
|
|
#
|
|
# Runs as ROOT via media-backup.service. Not optional: /home/gib/Public/Nextcloud/Server is
|
|
# uid 33 mode 0750 and the borg repo is root:root mode 0600. The historical hand-run rsync
|
|
# ran as gib, could not read those, skipped them -- AND EXITED 0. That is how 442 G of
|
|
# Nextcloud ended up in the Backup pool while nobody could tell it was the wrong 442 G.
|
|
#
|
|
# HISTORY IS TAKEN BY THIS SCRIPT, NOT BY A TIMER. zfs-auto-snapshot is disabled on the
|
|
# Backup pool (com.sun:auto-snapshot=false) because a timer-driven snapshot can land
|
|
# mid-rsync and record a half-written backup as that week's restore point. Here the snapshot
|
|
# is taken only AFTER a successful sync, so every snapshot on that pool is, by construction,
|
|
# a complete backup.
|
|
|
|
set -uo pipefail
|
|
|
|
SOURCE="/home/gib/Public/"
|
|
DEST="/home/gib/.local/share/Backup/Public"
|
|
POOL="Backup"
|
|
BORG_LOCK="/home/gib/Public/Nextcloud/Backups/borg/lock.exclusive"
|
|
KEEP_SNAPSHOTS=8 # ~2 months at weekly
|
|
MIN_FREE_GB=200 # refuse to start if the pool is already this tight
|
|
LOG="/var/log/media-backup.log"
|
|
LOCK="/run/media-backup.lock"
|
|
|
|
log() { printf '%s %s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG"; }
|
|
die() { log "FATAL: $*"; exit 1; }
|
|
|
|
exec 9>"$LOCK" || die "cannot open lock $LOCK"
|
|
flock -n 9 || die "another run is already in progress"
|
|
|
|
EXCLUDES=(
|
|
# --- too big, deliberately not backed up -------------------------------
|
|
# 23.2 T. Larger on its own than the entire Backup pool. gib's explicit decision:
|
|
# if Media is lost, the TV library gets rebuilt.
|
|
"/TV"
|
|
|
|
# Torrent scratch. qBittorrent manages it (ratio 2, ~14.7 d inactive, then
|
|
# RemoveWithContent), and anything worth keeping has already been imported.
|
|
"/Downloads"
|
|
|
|
# --- already self-versioning, so a second copy buys nothing ------------
|
|
# TimeMachine is its own backup format with its own history.
|
|
"/Mac"
|
|
|
|
# 586 G of raw Nextcloud data. Covered by AIO's borg repo below, which is
|
|
# deduplicated, encrypted, and carries 7d/4w/6m of history instead of one stale
|
|
# point-in-time. Verified: the borgbackup container mounts this exact path as
|
|
# /nextcloud_aio_volumes/nextcloud_aio_nextcloud_data and borg walks the whole tree.
|
|
# We keep Nextcloud/Backups (the repo). We do NOT keep Nextcloud/Server.
|
|
"/Nextcloud/Server"
|
|
|
|
# --- regenerable ------------------------------------------------------
|
|
"/Server_Backup/Ollama_Models" # re-pullable model weights
|
|
"/Server_Backup/ComfyUI" # re-downloadable checkpoints
|
|
"/Server_Backup/frigate" # camera footage; container is decommissioned
|
|
"/Server_Backup/.Trash-1000" # desktop trash
|
|
)
|
|
# Kept on purpose, so nobody "tidies" them later: Server_Backup/Containers (the only copy
|
|
# of both machines' ~/Server backups outside the Media pool), immich (irreplaceable photos),
|
|
# Games and ROMs (gib: "they can be hard to get").
|
|
|
|
EXCLUDE_ARGS=()
|
|
for e in "${EXCLUDES[@]}"; do EXCLUDE_ARGS+=( "--exclude=$e" ); done
|
|
|
|
log "=== media-backup start ==="
|
|
|
|
# --- preflight ------------------------------------------------------------
|
|
mountpoint -q /home/gib/Public \
|
|
|| die "/home/gib/Public is not a mountpoint -- Media pool not imported, refusing to run"
|
|
mountpoint -q /home/gib/.local/share/Backup \
|
|
|| die "Backup pool is not mounted -- refusing to write into a plain directory on the NVMe"
|
|
mkdir -p "$DEST"
|
|
|
|
# A borg run holds this lock for 2-4 hours and rewrites index/hints/integrity at the END.
|
|
# Copying mid-run pairs new data chunks with a stale index: a repo that looks fine and
|
|
# fails at restore. AIO's borg runs daily from 07:30 UTC for about 2 hours, so it can
|
|
# still be going when the timer fires. Wait for it to finish, checking every
|
|
# BORG_WAIT_INTERVAL, and defer only if it is still held after BORG_WAIT_MAX. The unit's
|
|
# TimeoutStartSec=24h covers the wait plus the ~50 min sync.
|
|
BORG_WAIT_INTERVAL=300 # 5 min
|
|
BORG_WAIT_MAX=21600 # 6 h
|
|
waited=0
|
|
while [ -e "$BORG_LOCK" ] && [ "$waited" -lt "$BORG_WAIT_MAX" ]; do
|
|
log "borg is mid-run (lock held at $BORG_LOCK). Waited $((waited / 60)) of $((BORG_WAIT_MAX / 60)) min, checking again in $((BORG_WAIT_INTERVAL / 60)) min"
|
|
sleep "$BORG_WAIT_INTERVAL"
|
|
waited=$(( waited + BORG_WAIT_INTERVAL ))
|
|
done
|
|
if [ -e "$BORG_LOCK" ]; then
|
|
log "borg is mid-run (lock held at $BORG_LOCK)"
|
|
log "DEFERRING this week's run rather than copying a torn repo. Next fire: next Saturday."
|
|
log "=== media-backup deferred ==="
|
|
exit 0
|
|
fi
|
|
[ "$waited" -gt 0 ] && log "borg lock released after waiting $((waited / 60)) min, continuing"
|
|
|
|
avail_gb=$(zfs get -Hp -o value available "$POOL" 2>/dev/null)
|
|
avail_gb=$(( ${avail_gb:-0} / 1024 / 1024 / 1024 ))
|
|
log "Backup pool free: ${avail_gb} GB"
|
|
[ "$avail_gb" -lt "$MIN_FREE_GB" ] \
|
|
&& die "only ${avail_gb} GB free (floor ${MIN_FREE_GB} GB) -- refusing to start"
|
|
|
|
# --- sync -----------------------------------------------------------------
|
|
# --numeric-ids is mandatory: `getent passwd 33` returns nothing on this host (uid 33 is
|
|
# www-data inside a container), so rsync has no name to map ownership by.
|
|
# --delete/--delete-excluded keep the copy exact; anything wrongly removed is still in
|
|
# last week's snapshot.
|
|
log "Syncing ${SOURCE} -> ${DEST}/ ..."
|
|
rsync -a --numeric-ids --delete --delete-excluded --human-readable --stats \
|
|
"${EXCLUDE_ARGS[@]}" "$SOURCE" "$DEST/" 2>&1 | tee -a "$LOG"
|
|
status=${PIPESTATUS[0]}
|
|
|
|
# 24 = "source files vanished during transfer". Normal on a live system.
|
|
if [ "$status" -ne 0 ] && [ "$status" -ne 24 ]; then
|
|
die "rsync exit $status -- NOT snapshotting. Last good snapshot is still the newest."
|
|
fi
|
|
[ "$status" -eq 24 ] && log "NOTE: rsync exit 24 (files vanished mid-run) -- expected"
|
|
|
|
# --- snapshot, only now that the sync succeeded ---------------------------
|
|
SNAP="${POOL}@backup-$(date +%Y-%m-%d)"
|
|
if zfs list -t snapshot -H "$SNAP" >/dev/null 2>&1; then
|
|
log "snapshot $SNAP already exists (second run today), skipping"
|
|
else
|
|
zfs snapshot "$SNAP" && log "snapshot taken: $SNAP" || log "WARNING: snapshot failed"
|
|
fi
|
|
|
|
# Keep the newest N. These are ours, so nothing else prunes them.
|
|
mapfile -t snaps < <(zfs list -t snapshot -H -o name -s creation -d 1 "$POOL" 2>/dev/null | grep "@backup-")
|
|
if [ "${#snaps[@]}" -gt "$KEEP_SNAPSHOTS" ]; then
|
|
drop=$(( ${#snaps[@]} - KEEP_SNAPSHOTS ))
|
|
for s in "${snaps[@]:0:$drop}"; do
|
|
zfs destroy "$s" && log "pruned old snapshot: $s"
|
|
done
|
|
fi
|
|
log "snapshots retained: $(zfs list -t snapshot -H -o name -d 1 "$POOL" 2>/dev/null | grep -c '@backup-')"
|
|
|
|
# Heartbeat for Uptime Kuma ("Media Backup (server.gib)"), sent only after a completed
|
|
# sync and snapshot. A deferred week exits before this, so it does not ping.
|
|
if [ -r /home/gib/Server/.media-backup-heartbeat-url ]; then
|
|
url=$(< /home/gib/Server/.media-backup-heartbeat-url)
|
|
curl -fsS -m 20 "$url" >/dev/null 2>&1 \
|
|
&& log "Heartbeat sent" || log "NOTE: heartbeat ping failed (backup itself was fine)"
|
|
fi
|
|
|
|
log "=== media-backup complete ==="
|