Keep the home server's root backup scripts with the rest of its tooling
This commit is contained in:
Gabriel Brown committed
1 parent
7cc98cd318
commit
042885f113
3 files changed
+326
-3
No files matched your search
Executable
+140
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Weekly copy of the Media pool -> the Backup pool, both on this machine.
|
||||
#
|
||||
# Purpose: survive losing the Media pool itself (or a bad delete on it). Both pools are
|
||||
# in this one box, so this is NOT an offsite backup and never will be.
|
||||
#
|
||||
# source /home/gib/Public (Media, 50.9 T raidz)
|
||||
# dest /home/gib/.local/share/Backup/Public (Backup, 14.5 T raidz1)
|
||||
#
|
||||
# Runs as ROOT via media-backup.service. Not optional: /home/gib/Public/Nextcloud/Server is
|
||||
# uid 33 mode 0750 and the borg repo is root:root mode 0600. The historical hand-run rsync
|
||||
# ran as gib, could not read those, skipped them -- AND EXITED 0. That is how 442 G of
|
||||
# Nextcloud ended up in the Backup pool while nobody could tell it was the wrong 442 G.
|
||||
#
|
||||
# HISTORY IS TAKEN BY THIS SCRIPT, NOT BY A TIMER. zfs-auto-snapshot is disabled on the
|
||||
# Backup pool (com.sun:auto-snapshot=false) because a timer-driven snapshot can land
|
||||
# mid-rsync and record a half-written backup as that week's restore point. Here the snapshot
|
||||
# is taken only AFTER a successful sync, so every snapshot on that pool is, by construction,
|
||||
# a complete backup.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SOURCE="/home/gib/Public/"
|
||||
DEST="/home/gib/.local/share/Backup/Public"
|
||||
POOL="Backup"
|
||||
BORG_LOCK="/home/gib/Public/Nextcloud/Backups/borg/lock.exclusive"
|
||||
KEEP_SNAPSHOTS=8 # ~2 months at weekly
|
||||
MIN_FREE_GB=200 # refuse to start if the pool is already this tight
|
||||
LOG="/var/log/media-backup.log"
|
||||
LOCK="/run/media-backup.lock"
|
||||
|
||||
log() { printf '%s %s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG"; }
|
||||
die() { log "FATAL: $*"; exit 1; }
|
||||
|
||||
exec 9>"$LOCK" || die "cannot open lock $LOCK"
|
||||
flock -n 9 || die "another run is already in progress"
|
||||
|
||||
EXCLUDES=(
|
||||
# --- too big, deliberately not backed up -------------------------------
|
||||
# 23.2 T. Larger on its own than the entire Backup pool. gib's explicit decision:
|
||||
# if Media is lost, the TV library gets rebuilt.
|
||||
"/TV"
|
||||
|
||||
# Torrent scratch. qBittorrent manages it (ratio 2, ~14.7 d inactive, then
|
||||
# RemoveWithContent), and anything worth keeping has already been imported.
|
||||
"/Downloads"
|
||||
|
||||
# --- already self-versioning, so a second copy buys nothing ------------
|
||||
# TimeMachine is its own backup format with its own history.
|
||||
"/Mac"
|
||||
|
||||
# 586 G of raw Nextcloud data. Covered by AIO's borg repo below, which is
|
||||
# deduplicated, encrypted, and carries 7d/4w/6m of history instead of one stale
|
||||
# point-in-time. Verified: the borgbackup container mounts this exact path as
|
||||
# /nextcloud_aio_volumes/nextcloud_aio_nextcloud_data and borg walks the whole tree.
|
||||
# We keep Nextcloud/Backups (the repo). We do NOT keep Nextcloud/Server.
|
||||
"/Nextcloud/Server"
|
||||
|
||||
# --- regenerable ------------------------------------------------------
|
||||
"/Server_Backup/Ollama_Models" # re-pullable model weights
|
||||
"/Server_Backup/ComfyUI" # re-downloadable checkpoints
|
||||
"/Server_Backup/frigate" # camera footage; container is decommissioned
|
||||
"/Server_Backup/.Trash-1000" # desktop trash
|
||||
)
|
||||
# Kept on purpose, so nobody "tidies" them later: Server_Backup/Containers (the only copy
|
||||
# of both machines' ~/Server backups outside the Media pool), immich (irreplaceable photos),
|
||||
# Games and ROMs (gib: "they can be hard to get").
|
||||
|
||||
EXCLUDE_ARGS=()
|
||||
for e in "${EXCLUDES[@]}"; do EXCLUDE_ARGS+=( "--exclude=$e" ); done
|
||||
|
||||
log "=== media-backup start ==="
|
||||
|
||||
# --- preflight ------------------------------------------------------------
|
||||
mountpoint -q /home/gib/Public \
|
||||
|| die "/home/gib/Public is not a mountpoint -- Media pool not imported, refusing to run"
|
||||
mountpoint -q /home/gib/.local/share/Backup \
|
||||
|| die "Backup pool is not mounted -- refusing to write into a plain directory on the NVMe"
|
||||
mkdir -p "$DEST"
|
||||
|
||||
# A borg run holds this lock for 2-4 hours and rewrites index/hints/integrity at the END.
|
||||
# Copying mid-run pairs new data chunks with a stale index: a repo that looks fine and
|
||||
# fails at restore. Defer rather than queue -- the next weekly run will catch it.
|
||||
if [ -e "$BORG_LOCK" ]; then
|
||||
log "borg is mid-run (lock held at $BORG_LOCK)"
|
||||
log "DEFERRING this week's run rather than copying a torn repo. Next fire: next Saturday."
|
||||
log "=== media-backup deferred ==="
|
||||
exit 0
|
||||
fi
|
||||
|
||||
avail_gb=$(zfs get -Hp -o value available "$POOL" 2>/dev/null)
|
||||
avail_gb=$(( ${avail_gb:-0} / 1024 / 1024 / 1024 ))
|
||||
log "Backup pool free: ${avail_gb} GB"
|
||||
[ "$avail_gb" -lt "$MIN_FREE_GB" ] \
|
||||
&& die "only ${avail_gb} GB free (floor ${MIN_FREE_GB} GB) -- refusing to start"
|
||||
|
||||
# --- sync -----------------------------------------------------------------
|
||||
# --numeric-ids is mandatory: `getent passwd 33` returns nothing on this host (uid 33 is
|
||||
# www-data inside a container), so rsync has no name to map ownership by.
|
||||
# --delete/--delete-excluded keep the copy exact; anything wrongly removed is still in
|
||||
# last week's snapshot.
|
||||
log "Syncing ${SOURCE} -> ${DEST}/ ..."
|
||||
rsync -a --numeric-ids --delete --delete-excluded --human-readable --stats \
|
||||
"${EXCLUDE_ARGS[@]}" "$SOURCE" "$DEST/" 2>&1 | tee -a "$LOG"
|
||||
status=${PIPESTATUS[0]}
|
||||
|
||||
# 24 = "source files vanished during transfer". Normal on a live system.
|
||||
if [ "$status" -ne 0 ] && [ "$status" -ne 24 ]; then
|
||||
die "rsync exit $status -- NOT snapshotting. Last good snapshot is still the newest."
|
||||
fi
|
||||
[ "$status" -eq 24 ] && log "NOTE: rsync exit 24 (files vanished mid-run) -- expected"
|
||||
|
||||
# --- snapshot, only now that the sync succeeded ---------------------------
|
||||
SNAP="${POOL}@backup-$(date +%Y-%m-%d)"
|
||||
if zfs list -t snapshot -H "$SNAP" >/dev/null 2>&1; then
|
||||
log "snapshot $SNAP already exists (second run today), skipping"
|
||||
else
|
||||
zfs snapshot "$SNAP" && log "snapshot taken: $SNAP" || log "WARNING: snapshot failed"
|
||||
fi
|
||||
|
||||
# Keep the newest N. These are ours, so nothing else prunes them.
|
||||
mapfile -t snaps < <(zfs list -t snapshot -H -o name -s creation -d 1 "$POOL" 2>/dev/null | grep "@backup-")
|
||||
if [ "${#snaps[@]}" -gt "$KEEP_SNAPSHOTS" ]; then
|
||||
drop=$(( ${#snaps[@]} - KEEP_SNAPSHOTS ))
|
||||
for s in "${snaps[@]:0:$drop}"; do
|
||||
zfs destroy "$s" && log "pruned old snapshot: $s"
|
||||
done
|
||||
fi
|
||||
log "snapshots retained: $(zfs list -t snapshot -H -o name -d 1 "$POOL" 2>/dev/null | grep -c '@backup-')"
|
||||
|
||||
# Heartbeat for Uptime Kuma ("Media Backup (server.gib)"), sent only after a completed
|
||||
# sync and snapshot. A deferred week exits before this, so it does not ping.
|
||||
if [ -r /home/gib/Server/.media-backup-heartbeat-url ]; then
|
||||
url=$(< /home/gib/Server/.media-backup-heartbeat-url)
|
||||
curl -fsS -m 20 "$url" >/dev/null 2>&1 \
|
||||
&& log "Heartbeat sent" || log "NOTE: heartbeat ping failed (backup itself was fine)"
|
||||
fi
|
||||
|
||||
log "=== media-backup complete ==="
|
||||
Reference in new issue
Block a user