Keep the home server's root backup scripts with the rest of its tooling
This commit is contained in:
Gabriel Brown committed
1 parent
7cc98cd318
commit
042885f113
3 files changed
+326
-3
No files matched your search
+5
-3
@@ -129,9 +129,11 @@ them.
|
||||
- `home/RESTORE.md`: how to restore the home server's `~/Server` from the
|
||||
ZFS backup.
|
||||
- `home/backup` and `home/systemd/`: the home server's backup trigger and its
|
||||
root system units (`server-backup` nightly, `media-backup` weekly). The
|
||||
scripts those units run live in `/usr/local/sbin` on that machine. They are
|
||||
readable by root alone, so this tree has no copy of them.
|
||||
root system units (`server-backup` nightly, `media-backup` weekly).
|
||||
- `home/server-backup` and `home/media-backup`: the scripts those units run.
|
||||
On that machine they live in `/usr/local/sbin`, root-owned, because a
|
||||
root-run script must not sit in a directory the user can write to. Restore
|
||||
them there, not into `~/Server`.
|
||||
|
||||
The VPS units carry `RequiresMountsFor=/home/gib/Media` because that host
|
||||
keeps its podman graphroot on an attached volume mounted there. The setting
|
||||
|
||||
Executable
+140
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Weekly copy of the Media pool -> the Backup pool, both on this machine.
|
||||
#
|
||||
# Purpose: survive losing the Media pool itself (or a bad delete on it). Both pools are
|
||||
# in this one box, so this is NOT an offsite backup and never will be.
|
||||
#
|
||||
# source /home/gib/Public (Media, 50.9 T raidz)
|
||||
# dest /home/gib/.local/share/Backup/Public (Backup, 14.5 T raidz1)
|
||||
#
|
||||
# Runs as ROOT via media-backup.service. Not optional: /home/gib/Public/Nextcloud/Server is
|
||||
# uid 33 mode 0750 and the borg repo is root:root mode 0600. The historical hand-run rsync
|
||||
# ran as gib, could not read those, skipped them -- AND EXITED 0. That is how 442 G of
|
||||
# Nextcloud ended up in the Backup pool while nobody could tell it was the wrong 442 G.
|
||||
#
|
||||
# HISTORY IS TAKEN BY THIS SCRIPT, NOT BY A TIMER. zfs-auto-snapshot is disabled on the
|
||||
# Backup pool (com.sun:auto-snapshot=false) because a timer-driven snapshot can land
|
||||
# mid-rsync and record a half-written backup as that week's restore point. Here the snapshot
|
||||
# is taken only AFTER a successful sync, so every snapshot on that pool is, by construction,
|
||||
# a complete backup.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SOURCE="/home/gib/Public/"
|
||||
DEST="/home/gib/.local/share/Backup/Public"
|
||||
POOL="Backup"
|
||||
BORG_LOCK="/home/gib/Public/Nextcloud/Backups/borg/lock.exclusive"
|
||||
KEEP_SNAPSHOTS=8 # ~2 months at weekly
|
||||
MIN_FREE_GB=200 # refuse to start if the pool is already this tight
|
||||
LOG="/var/log/media-backup.log"
|
||||
LOCK="/run/media-backup.lock"
|
||||
|
||||
log() { printf '%s %s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG"; }
|
||||
die() { log "FATAL: $*"; exit 1; }
|
||||
|
||||
exec 9>"$LOCK" || die "cannot open lock $LOCK"
|
||||
flock -n 9 || die "another run is already in progress"
|
||||
|
||||
EXCLUDES=(
|
||||
# --- too big, deliberately not backed up -------------------------------
|
||||
# 23.2 T. Larger on its own than the entire Backup pool. gib's explicit decision:
|
||||
# if Media is lost, the TV library gets rebuilt.
|
||||
"/TV"
|
||||
|
||||
# Torrent scratch. qBittorrent manages it (ratio 2, ~14.7 d inactive, then
|
||||
# RemoveWithContent), and anything worth keeping has already been imported.
|
||||
"/Downloads"
|
||||
|
||||
# --- already self-versioning, so a second copy buys nothing ------------
|
||||
# TimeMachine is its own backup format with its own history.
|
||||
"/Mac"
|
||||
|
||||
# 586 G of raw Nextcloud data. Covered by AIO's borg repo below, which is
|
||||
# deduplicated, encrypted, and carries 7d/4w/6m of history instead of one stale
|
||||
# point-in-time. Verified: the borgbackup container mounts this exact path as
|
||||
# /nextcloud_aio_volumes/nextcloud_aio_nextcloud_data and borg walks the whole tree.
|
||||
# We keep Nextcloud/Backups (the repo). We do NOT keep Nextcloud/Server.
|
||||
"/Nextcloud/Server"
|
||||
|
||||
# --- regenerable ------------------------------------------------------
|
||||
"/Server_Backup/Ollama_Models" # re-pullable model weights
|
||||
"/Server_Backup/ComfyUI" # re-downloadable checkpoints
|
||||
"/Server_Backup/frigate" # camera footage; container is decommissioned
|
||||
"/Server_Backup/.Trash-1000" # desktop trash
|
||||
)
|
||||
# Kept on purpose, so nobody "tidies" them later: Server_Backup/Containers (the only copy
|
||||
# of both machines' ~/Server backups outside the Media pool), immich (irreplaceable photos),
|
||||
# Games and ROMs (gib: "they can be hard to get").
|
||||
|
||||
EXCLUDE_ARGS=()
|
||||
for e in "${EXCLUDES[@]}"; do EXCLUDE_ARGS+=( "--exclude=$e" ); done
|
||||
|
||||
log "=== media-backup start ==="
|
||||
|
||||
# --- preflight ------------------------------------------------------------
|
||||
mountpoint -q /home/gib/Public \
|
||||
|| die "/home/gib/Public is not a mountpoint -- Media pool not imported, refusing to run"
|
||||
mountpoint -q /home/gib/.local/share/Backup \
|
||||
|| die "Backup pool is not mounted -- refusing to write into a plain directory on the NVMe"
|
||||
mkdir -p "$DEST"
|
||||
|
||||
# A borg run holds this lock for 2-4 hours and rewrites index/hints/integrity at the END.
|
||||
# Copying mid-run pairs new data chunks with a stale index: a repo that looks fine and
|
||||
# fails at restore. Defer rather than queue -- the next weekly run will catch it.
|
||||
if [ -e "$BORG_LOCK" ]; then
|
||||
log "borg is mid-run (lock held at $BORG_LOCK)"
|
||||
log "DEFERRING this week's run rather than copying a torn repo. Next fire: next Saturday."
|
||||
log "=== media-backup deferred ==="
|
||||
exit 0
|
||||
fi
|
||||
|
||||
avail_gb=$(zfs get -Hp -o value available "$POOL" 2>/dev/null)
|
||||
avail_gb=$(( ${avail_gb:-0} / 1024 / 1024 / 1024 ))
|
||||
log "Backup pool free: ${avail_gb} GB"
|
||||
[ "$avail_gb" -lt "$MIN_FREE_GB" ] \
|
||||
&& die "only ${avail_gb} GB free (floor ${MIN_FREE_GB} GB) -- refusing to start"
|
||||
|
||||
# --- sync -----------------------------------------------------------------
|
||||
# --numeric-ids is mandatory: `getent passwd 33` returns nothing on this host (uid 33 is
|
||||
# www-data inside a container), so rsync has no name to map ownership by.
|
||||
# --delete/--delete-excluded keep the copy exact; anything wrongly removed is still in
|
||||
# last week's snapshot.
|
||||
log "Syncing ${SOURCE} -> ${DEST}/ ..."
|
||||
rsync -a --numeric-ids --delete --delete-excluded --human-readable --stats \
|
||||
"${EXCLUDE_ARGS[@]}" "$SOURCE" "$DEST/" 2>&1 | tee -a "$LOG"
|
||||
status=${PIPESTATUS[0]}
|
||||
|
||||
# 24 = "source files vanished during transfer". Normal on a live system.
|
||||
if [ "$status" -ne 0 ] && [ "$status" -ne 24 ]; then
|
||||
die "rsync exit $status -- NOT snapshotting. Last good snapshot is still the newest."
|
||||
fi
|
||||
[ "$status" -eq 24 ] && log "NOTE: rsync exit 24 (files vanished mid-run) -- expected"
|
||||
|
||||
# --- snapshot, only now that the sync succeeded ---------------------------
|
||||
SNAP="${POOL}@backup-$(date +%Y-%m-%d)"
|
||||
if zfs list -t snapshot -H "$SNAP" >/dev/null 2>&1; then
|
||||
log "snapshot $SNAP already exists (second run today), skipping"
|
||||
else
|
||||
zfs snapshot "$SNAP" && log "snapshot taken: $SNAP" || log "WARNING: snapshot failed"
|
||||
fi
|
||||
|
||||
# Keep the newest N. These are ours, so nothing else prunes them.
|
||||
mapfile -t snaps < <(zfs list -t snapshot -H -o name -s creation -d 1 "$POOL" 2>/dev/null | grep "@backup-")
|
||||
if [ "${#snaps[@]}" -gt "$KEEP_SNAPSHOTS" ]; then
|
||||
drop=$(( ${#snaps[@]} - KEEP_SNAPSHOTS ))
|
||||
for s in "${snaps[@]:0:$drop}"; do
|
||||
zfs destroy "$s" && log "pruned old snapshot: $s"
|
||||
done
|
||||
fi
|
||||
log "snapshots retained: $(zfs list -t snapshot -H -o name -d 1 "$POOL" 2>/dev/null | grep -c '@backup-')"
|
||||
|
||||
# Heartbeat for Uptime Kuma ("Media Backup (server.gib)"), sent only after a completed
|
||||
# sync and snapshot. A deferred week exits before this, so it does not ping.
|
||||
if [ -r /home/gib/Server/.media-backup-heartbeat-url ]; then
|
||||
url=$(< /home/gib/Server/.media-backup-heartbeat-url)
|
||||
curl -fsS -m 20 "$url" >/dev/null 2>&1 \
|
||||
&& log "Heartbeat sent" || log "NOTE: heartbeat ping failed (backup itself was fine)"
|
||||
fi
|
||||
|
||||
log "=== media-backup complete ==="
|
||||
Executable
+181
@@ -0,0 +1,181 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Daily backup of ~/Server (NVMe) to the ZFS pool on this same machine.
|
||||
# Purpose: survive the NVMe/LUKS volume dying. Not an offsite backup.
|
||||
#
|
||||
# Runs as ROOT via server-backup.service. That is deliberate: this box runs Docker,
|
||||
# so most container data under ~/Server is root-owned. The old script used `sudo rsync`,
|
||||
# which worked but could never be automated without a sudoers rule. A systemd system
|
||||
# unit is already root, so ownership is preserved natively and no sudoers entry is
|
||||
# needed at runtime. (Note this differs from the VPS, which is rootless podman and
|
||||
# needs rsync --fake-super to carry subuid ownership; nothing like that applies here.)
|
||||
#
|
||||
# History is ZFS's job: Media/Server_Backup has zfs-auto-snapshot keeping 14 dailies,
|
||||
# so this writes ONE directory and never makes dated copies. Snapshots are block-level,
|
||||
# so a 1.6 GB database that changes a few pages costs a few pages.
|
||||
#
|
||||
# See RESTORE.md for how to bring this back.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SOURCE="/home/gib/Server/"
|
||||
DEST="/home/gib/Public/Server_Backup/Containers/server.gib/current"
|
||||
STATE_DIR="/home/gib/Server/_backup_state"
|
||||
LOG_DIR="/home/gib/Server/logs"
|
||||
LOG="$LOG_DIR/backup.log"
|
||||
LOCK="/run/server-backup.lock"
|
||||
REMOTE_DATASET="Media/Server_Backup"
|
||||
SNAPSHOT_MAX_AGE_HOURS=48
|
||||
|
||||
mkdir -p "$LOG_DIR"
|
||||
log() { printf '%s %s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG"; }
|
||||
die() { log "FATAL: $*"; exit 1; }
|
||||
|
||||
exec 9>"$LOCK" || die "cannot open lock $LOCK"
|
||||
flock -n 9 || die "another backup is already running"
|
||||
|
||||
EXCLUSIONS=(
|
||||
# --- build noise (inherited from the original script) -------------------
|
||||
"node_modules" ".git" "*.log" "*.tmp" ".cache" "cache" "tmp" "temp"
|
||||
".npm" ".yarn" "dist" "build" "target" ".venv" "venv" "__pycache__"
|
||||
"*.pyc" ".DS_Store" "Thumbs.db" "vendor" "*.mp4" "*.mkv"
|
||||
|
||||
# --- the big one: Gitea CI runner Docker-in-Docker storage --------------
|
||||
# 203 GB across runner-1/2/3 (72 GB of containerd image layers each), and it
|
||||
# churns constantly as CI builds run. It is pure cache: a rebuilt runner just
|
||||
# re-pulls everything. Excluding dind/ specifically KEEPS runner-N/data/.runner
|
||||
# (the 648-byte registration token) and certs/, which are the only parts that
|
||||
# would actually be painful to recreate.
|
||||
#
|
||||
# Measured as root on 2026-09-17 -- a plain `du` as gib reports these as ~13 MB
|
||||
# because it cannot descend into root-owned dirs. Do not trust non-root sizes here.
|
||||
"/Agents/Gitea/volumes/*/dind"
|
||||
|
||||
# Regenerated every run; no reason to ship the previous run's copy.
|
||||
"/_backup_state/*.partial"
|
||||
)
|
||||
|
||||
EXCLUDE_ARGS=()
|
||||
for e in "${EXCLUSIONS[@]}"; do EXCLUDE_ARGS+=( "--exclude=$e" ); done
|
||||
|
||||
# Convex deployments backed by SQLite, as "container:sqlite path under ~/Server".
|
||||
# Convex keeps that file in rollback-journal mode and writes to it constantly, so
|
||||
# sqlite3 .backup either hits "database is locked" or restarts forever. Instead,
|
||||
# dump_convex takes Convex's own snapshot export (`npx convex export`, restored with
|
||||
# `npx convex import`), and dump_state skips the raw file.
|
||||
CONVEX_EXPORTS=(
|
||||
"gbrown-backend:Gbrown/data/convex/db.sqlite3"
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Consistent dumps. Containers keep running; these are what make that safe.
|
||||
# ---------------------------------------------------------------------------
|
||||
dump_state() {
|
||||
log "Dumping databases to _backup_state/ ..."
|
||||
rm -rf "$STATE_DIR"
|
||||
mkdir -p "$STATE_DIR/sqlite"
|
||||
printf 'Generated %s by /usr/local/sbin/server-backup. Authoritative restore source.\n' \
|
||||
"$(date -Is)" > "$STATE_DIR/README"
|
||||
|
||||
local n=0 skipped=0
|
||||
while IFS= read -r db; do
|
||||
# Only real SQLite. Docker/containerd/buildkit scatter BoltDB files named *.db
|
||||
# (portainer.db, metadata_v2.db, meta.db ...) which sqlite3 cannot read.
|
||||
read -r -n 15 magic < "$db" 2>/dev/null
|
||||
[ "$magic" = "SQLite format 3" ] || { skipped=$((skipped+1)); continue; }
|
||||
|
||||
local rel="${db#/home/gib/Server/}" out
|
||||
[[ " ${CONVEX_EXPORTS[*]#*:} " == *" $rel "* ]] && continue # dump_convex covers it
|
||||
out="$STATE_DIR/sqlite/$rel"
|
||||
mkdir -p "$(dirname "$out")"
|
||||
# .timeout: wait up to 30s for a writer's lock instead of failing on first contact.
|
||||
if sqlite3 -cmd ".timeout 30000" "$db" ".backup '$out'" 2>>"$LOG"; then
|
||||
n=$((n+1))
|
||||
else
|
||||
log " WARNING: sqlite dump failed: $rel"
|
||||
fi
|
||||
done < <(find /home/gib/Server \
|
||||
-path "$STATE_DIR" -prune -o \
|
||||
-path '*/dind' -prune -o \
|
||||
-path '*/other/firefox' -prune -o \
|
||||
-path '*/node_modules' -prune -o \
|
||||
-type f \( -name '*.db' -o -name '*.sqlite' -o -name '*.sqlite3' \) \
|
||||
! -name '*-wal' ! -name '*-shm' \
|
||||
! -name '*logs.db' ! -name '*cache*.db' -print 2>/dev/null)
|
||||
log " sqlite: $n dumped, $skipped skipped (not SQLite)"
|
||||
dump_convex
|
||||
}
|
||||
|
||||
# Snapshot-exports each CONVEX_EXPORTS deployment to _backup_state/convex/<container>.zip
|
||||
# through a throwaway node container, so nothing is installed on the host. The admin key
|
||||
# is derived from the instance secret on every run rather than stored. Takes ~8 min.
|
||||
dump_convex() {
|
||||
mkdir -p "$STATE_DIR/convex"
|
||||
local entry c key
|
||||
for entry in "${CONVEX_EXPORTS[@]}"; do
|
||||
c="${entry%%:*}"
|
||||
key=$(docker exec "$c" ./generate_admin_key.sh 2>/dev/null | tail -1)
|
||||
if [ -z "$key" ]; then
|
||||
log " WARNING: convex export failed: $c (could not generate admin key -- is it running?)"
|
||||
continue
|
||||
fi
|
||||
if CONVEX_SELF_HOSTED_ADMIN_KEY="$key" timeout 1800 docker run --rm \
|
||||
--network nginx-bridge -v "$STATE_DIR/convex:/out" \
|
||||
-e CONVEX_SELF_HOSTED_URL="http://$c:3210" -e CONVEX_SELF_HOSTED_ADMIN_KEY \
|
||||
-e npm_config_update_notifier=false -e npm_config_fund=false -e npm_config_audit=false \
|
||||
docker.io/library/node:22-alpine sh -ec '
|
||||
# The CLI only runs inside a project that depends on convex.
|
||||
mkdir /work && cd /work && echo "{}" > package.json
|
||||
npm install --silent convex@latest
|
||||
npx convex export --include-file-storage --path "/out/'"$c"'.zip"' \
|
||||
>>"$LOG" 2>&1; then
|
||||
log " convex: exported $c ($(du -h "$STATE_DIR/convex/$c.zip" | cut -f1))"
|
||||
else
|
||||
log " WARNING: convex export failed: $c (see npx output above)"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ZFS is the only thing giving us history, so make its absence loud.
|
||||
# ---------------------------------------------------------------------------
|
||||
check_zfs_history() {
|
||||
local newest now age_h
|
||||
newest=$(zfs list -H -p -o creation -t snapshot -s creation -d 1 "$REMOTE_DATASET" 2>/dev/null | tail -1)
|
||||
if [ -z "$newest" ]; then
|
||||
log " ⚠️ WARNING: no ZFS snapshots on $REMOTE_DATASET -- this backup has NO history."
|
||||
log " It is a plain mirror right now. Check zfs-auto-snapshot."
|
||||
return
|
||||
fi
|
||||
now=$(date +%s); age_h=$(( (now - newest) / 3600 ))
|
||||
if [ "$age_h" -gt "$SNAPSHOT_MAX_AGE_HOURS" ]; then
|
||||
log " ⚠️ WARNING: newest ZFS snapshot is ${age_h}h old (limit ${SNAPSHOT_MAX_AGE_HOURS}h) -- versioning may be broken."
|
||||
else
|
||||
log " ZFS history OK (newest snapshot ${age_h}h old, 14 dailies retained)"
|
||||
fi
|
||||
}
|
||||
|
||||
log "=== backup start -> $DEST ==="
|
||||
[ -d "$(dirname "$DEST")" ] || die "destination parent missing -- is the ZFS pool mounted?"
|
||||
mountpoint -q /home/gib/Public || die "/home/gib/Public is not a mountpoint -- refusing to write into an unmounted path"
|
||||
mkdir -p "$DEST"
|
||||
|
||||
dump_state
|
||||
check_zfs_history
|
||||
|
||||
rsync -a --numeric-ids --delete --delete-excluded --human-readable --stats \
|
||||
"${EXCLUDE_ARGS[@]}" "$SOURCE" "$DEST/" 2>&1 | tee -a "$LOG"
|
||||
status=${PIPESTATUS[0]}
|
||||
|
||||
if [ "$status" -ne 0 ] && [ "$status" -ne 24 ]; then
|
||||
die "rsync failed with exit $status -- 'current' may be half-updated; yesterday's ZFS snapshot is still good"
|
||||
fi
|
||||
[ "$status" -eq 24 ] && log "NOTE: rsync exit 24 (files vanished mid-run) -- expected on a live system"
|
||||
|
||||
if [ -r /home/gib/Server/.backup-heartbeat-url ]; then
|
||||
url=$(< /home/gib/Server/.backup-heartbeat-url)
|
||||
curl -fsS -m 20 "$url" >/dev/null 2>&1 \
|
||||
&& log "Heartbeat sent" || log "NOTE: heartbeat ping failed (backup itself was fine)"
|
||||
fi
|
||||
|
||||
log "=== backup complete ==="
|
||||
Reference in new issue
Block a user