From 042885f11343c83effee2b18573ff644bbf97ff4 Mon Sep 17 00:00:00 2001 From: Gabriel Brown Date: Fri, 30 Oct 2026 13:59:29 -0400 Subject: [PATCH] Keep the home server's root backup scripts with the rest of its tooling --- server/README.md | 8 +- server/home/media-backup | 140 +++++++++++++++++++++++++++++ server/home/server-backup | 181 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 326 insertions(+), 3 deletions(-) create mode 100755 server/home/media-backup create mode 100755 server/home/server-backup diff --git a/server/README.md b/server/README.md index 15072dc..1309de2 100644 --- a/server/README.md +++ b/server/README.md @@ -129,9 +129,11 @@ them. - `home/RESTORE.md`: how to restore the home server's `~/Server` from the ZFS backup. - `home/backup` and `home/systemd/`: the home server's backup trigger and its - root system units (`server-backup` nightly, `media-backup` weekly). The - scripts those units run live in `/usr/local/sbin` on that machine. They are - readable by root alone, so this tree has no copy of them. + root system units (`server-backup` nightly, `media-backup` weekly). +- `home/server-backup` and `home/media-backup`: the scripts those units run. + On that machine they live in `/usr/local/sbin`, root-owned, because a + root-run script must not sit in a directory the user can write to. Restore + them there, not into `~/Server`. The VPS units carry `RequiresMountsFor=/home/gib/Media` because that host keeps its podman graphroot on an attached volume mounted there. The setting diff --git a/server/home/media-backup b/server/home/media-backup new file mode 100755 index 0000000..d6bf013 --- /dev/null +++ b/server/home/media-backup @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# +# Weekly copy of the Media pool -> the Backup pool, both on this machine. +# +# Purpose: survive losing the Media pool itself (or a bad delete on it). Both pools are +# in this one box, so this is NOT an offsite backup and never will be. +# +# source /home/gib/Public (Media, 50.9 T raidz) +# dest /home/gib/.local/share/Backup/Public (Backup, 14.5 T raidz1) +# +# Runs as ROOT via media-backup.service. Not optional: /home/gib/Public/Nextcloud/Server is +# uid 33 mode 0750 and the borg repo is root:root mode 0600. The historical hand-run rsync +# ran as gib, could not read those, skipped them -- AND EXITED 0. That is how 442 G of +# Nextcloud ended up in the Backup pool while nobody could tell it was the wrong 442 G. +# +# HISTORY IS TAKEN BY THIS SCRIPT, NOT BY A TIMER. zfs-auto-snapshot is disabled on the +# Backup pool (com.sun:auto-snapshot=false) because a timer-driven snapshot can land +# mid-rsync and record a half-written backup as that week's restore point. Here the snapshot +# is taken only AFTER a successful sync, so every snapshot on that pool is, by construction, +# a complete backup. + +set -uo pipefail + +SOURCE="/home/gib/Public/" +DEST="/home/gib/.local/share/Backup/Public" +POOL="Backup" +BORG_LOCK="/home/gib/Public/Nextcloud/Backups/borg/lock.exclusive" +KEEP_SNAPSHOTS=8 # ~2 months at weekly +MIN_FREE_GB=200 # refuse to start if the pool is already this tight +LOG="/var/log/media-backup.log" +LOCK="/run/media-backup.lock" + +log() { printf '%s %s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG"; } +die() { log "FATAL: $*"; exit 1; } + +exec 9>"$LOCK" || die "cannot open lock $LOCK" +flock -n 9 || die "another run is already in progress" + +EXCLUDES=( + # --- too big, deliberately not backed up ------------------------------- + # 23.2 T. Larger on its own than the entire Backup pool. gib's explicit decision: + # if Media is lost, the TV library gets rebuilt. + "/TV" + + # Torrent scratch. qBittorrent manages it (ratio 2, ~14.7 d inactive, then + # RemoveWithContent), and anything worth keeping has already been imported. + "/Downloads" + + # --- already self-versioning, so a second copy buys nothing ------------ + # TimeMachine is its own backup format with its own history. + "/Mac" + + # 586 G of raw Nextcloud data. Covered by AIO's borg repo below, which is + # deduplicated, encrypted, and carries 7d/4w/6m of history instead of one stale + # point-in-time. Verified: the borgbackup container mounts this exact path as + # /nextcloud_aio_volumes/nextcloud_aio_nextcloud_data and borg walks the whole tree. + # We keep Nextcloud/Backups (the repo). We do NOT keep Nextcloud/Server. + "/Nextcloud/Server" + + # --- regenerable ------------------------------------------------------ + "/Server_Backup/Ollama_Models" # re-pullable model weights + "/Server_Backup/ComfyUI" # re-downloadable checkpoints + "/Server_Backup/frigate" # camera footage; container is decommissioned + "/Server_Backup/.Trash-1000" # desktop trash +) +# Kept on purpose, so nobody "tidies" them later: Server_Backup/Containers (the only copy +# of both machines' ~/Server backups outside the Media pool), immich (irreplaceable photos), +# Games and ROMs (gib: "they can be hard to get"). + +EXCLUDE_ARGS=() +for e in "${EXCLUDES[@]}"; do EXCLUDE_ARGS+=( "--exclude=$e" ); done + +log "=== media-backup start ===" + +# --- preflight ------------------------------------------------------------ +mountpoint -q /home/gib/Public \ + || die "/home/gib/Public is not a mountpoint -- Media pool not imported, refusing to run" +mountpoint -q /home/gib/.local/share/Backup \ + || die "Backup pool is not mounted -- refusing to write into a plain directory on the NVMe" +mkdir -p "$DEST" + +# A borg run holds this lock for 2-4 hours and rewrites index/hints/integrity at the END. +# Copying mid-run pairs new data chunks with a stale index: a repo that looks fine and +# fails at restore. Defer rather than queue -- the next weekly run will catch it. +if [ -e "$BORG_LOCK" ]; then + log "borg is mid-run (lock held at $BORG_LOCK)" + log "DEFERRING this week's run rather than copying a torn repo. Next fire: next Saturday." + log "=== media-backup deferred ===" + exit 0 +fi + +avail_gb=$(zfs get -Hp -o value available "$POOL" 2>/dev/null) +avail_gb=$(( ${avail_gb:-0} / 1024 / 1024 / 1024 )) +log "Backup pool free: ${avail_gb} GB" +[ "$avail_gb" -lt "$MIN_FREE_GB" ] \ + && die "only ${avail_gb} GB free (floor ${MIN_FREE_GB} GB) -- refusing to start" + +# --- sync ----------------------------------------------------------------- +# --numeric-ids is mandatory: `getent passwd 33` returns nothing on this host (uid 33 is +# www-data inside a container), so rsync has no name to map ownership by. +# --delete/--delete-excluded keep the copy exact; anything wrongly removed is still in +# last week's snapshot. +log "Syncing ${SOURCE} -> ${DEST}/ ..." +rsync -a --numeric-ids --delete --delete-excluded --human-readable --stats \ + "${EXCLUDE_ARGS[@]}" "$SOURCE" "$DEST/" 2>&1 | tee -a "$LOG" +status=${PIPESTATUS[0]} + +# 24 = "source files vanished during transfer". Normal on a live system. +if [ "$status" -ne 0 ] && [ "$status" -ne 24 ]; then + die "rsync exit $status -- NOT snapshotting. Last good snapshot is still the newest." +fi +[ "$status" -eq 24 ] && log "NOTE: rsync exit 24 (files vanished mid-run) -- expected" + +# --- snapshot, only now that the sync succeeded --------------------------- +SNAP="${POOL}@backup-$(date +%Y-%m-%d)" +if zfs list -t snapshot -H "$SNAP" >/dev/null 2>&1; then + log "snapshot $SNAP already exists (second run today), skipping" +else + zfs snapshot "$SNAP" && log "snapshot taken: $SNAP" || log "WARNING: snapshot failed" +fi + +# Keep the newest N. These are ours, so nothing else prunes them. +mapfile -t snaps < <(zfs list -t snapshot -H -o name -s creation -d 1 "$POOL" 2>/dev/null | grep "@backup-") +if [ "${#snaps[@]}" -gt "$KEEP_SNAPSHOTS" ]; then + drop=$(( ${#snaps[@]} - KEEP_SNAPSHOTS )) + for s in "${snaps[@]:0:$drop}"; do + zfs destroy "$s" && log "pruned old snapshot: $s" + done +fi +log "snapshots retained: $(zfs list -t snapshot -H -o name -d 1 "$POOL" 2>/dev/null | grep -c '@backup-')" + +# Heartbeat for Uptime Kuma ("Media Backup (server.gib)"), sent only after a completed +# sync and snapshot. A deferred week exits before this, so it does not ping. +if [ -r /home/gib/Server/.media-backup-heartbeat-url ]; then + url=$(< /home/gib/Server/.media-backup-heartbeat-url) + curl -fsS -m 20 "$url" >/dev/null 2>&1 \ + && log "Heartbeat sent" || log "NOTE: heartbeat ping failed (backup itself was fine)" +fi + +log "=== media-backup complete ===" diff --git a/server/home/server-backup b/server/home/server-backup new file mode 100755 index 0000000..92eee66 --- /dev/null +++ b/server/home/server-backup @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# +# Daily backup of ~/Server (NVMe) to the ZFS pool on this same machine. +# Purpose: survive the NVMe/LUKS volume dying. Not an offsite backup. +# +# Runs as ROOT via server-backup.service. That is deliberate: this box runs Docker, +# so most container data under ~/Server is root-owned. The old script used `sudo rsync`, +# which worked but could never be automated without a sudoers rule. A systemd system +# unit is already root, so ownership is preserved natively and no sudoers entry is +# needed at runtime. (Note this differs from the VPS, which is rootless podman and +# needs rsync --fake-super to carry subuid ownership; nothing like that applies here.) +# +# History is ZFS's job: Media/Server_Backup has zfs-auto-snapshot keeping 14 dailies, +# so this writes ONE directory and never makes dated copies. Snapshots are block-level, +# so a 1.6 GB database that changes a few pages costs a few pages. +# +# See RESTORE.md for how to bring this back. + +set -uo pipefail + +SOURCE="/home/gib/Server/" +DEST="/home/gib/Public/Server_Backup/Containers/server.gib/current" +STATE_DIR="/home/gib/Server/_backup_state" +LOG_DIR="/home/gib/Server/logs" +LOG="$LOG_DIR/backup.log" +LOCK="/run/server-backup.lock" +REMOTE_DATASET="Media/Server_Backup" +SNAPSHOT_MAX_AGE_HOURS=48 + +mkdir -p "$LOG_DIR" +log() { printf '%s %s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG"; } +die() { log "FATAL: $*"; exit 1; } + +exec 9>"$LOCK" || die "cannot open lock $LOCK" +flock -n 9 || die "another backup is already running" + +EXCLUSIONS=( + # --- build noise (inherited from the original script) ------------------- + "node_modules" ".git" "*.log" "*.tmp" ".cache" "cache" "tmp" "temp" + ".npm" ".yarn" "dist" "build" "target" ".venv" "venv" "__pycache__" + "*.pyc" ".DS_Store" "Thumbs.db" "vendor" "*.mp4" "*.mkv" + + # --- the big one: Gitea CI runner Docker-in-Docker storage -------------- + # 203 GB across runner-1/2/3 (72 GB of containerd image layers each), and it + # churns constantly as CI builds run. It is pure cache: a rebuilt runner just + # re-pulls everything. Excluding dind/ specifically KEEPS runner-N/data/.runner + # (the 648-byte registration token) and certs/, which are the only parts that + # would actually be painful to recreate. + # + # Measured as root on 2026-09-17 -- a plain `du` as gib reports these as ~13 MB + # because it cannot descend into root-owned dirs. Do not trust non-root sizes here. + "/Agents/Gitea/volumes/*/dind" + + # Regenerated every run; no reason to ship the previous run's copy. + "/_backup_state/*.partial" +) + +EXCLUDE_ARGS=() +for e in "${EXCLUSIONS[@]}"; do EXCLUDE_ARGS+=( "--exclude=$e" ); done + +# Convex deployments backed by SQLite, as "container:sqlite path under ~/Server". +# Convex keeps that file in rollback-journal mode and writes to it constantly, so +# sqlite3 .backup either hits "database is locked" or restarts forever. Instead, +# dump_convex takes Convex's own snapshot export (`npx convex export`, restored with +# `npx convex import`), and dump_state skips the raw file. +CONVEX_EXPORTS=( + "gbrown-backend:Gbrown/data/convex/db.sqlite3" +) + +# --------------------------------------------------------------------------- +# Consistent dumps. Containers keep running; these are what make that safe. +# --------------------------------------------------------------------------- +dump_state() { + log "Dumping databases to _backup_state/ ..." + rm -rf "$STATE_DIR" + mkdir -p "$STATE_DIR/sqlite" + printf 'Generated %s by /usr/local/sbin/server-backup. Authoritative restore source.\n' \ + "$(date -Is)" > "$STATE_DIR/README" + + local n=0 skipped=0 + while IFS= read -r db; do + # Only real SQLite. Docker/containerd/buildkit scatter BoltDB files named *.db + # (portainer.db, metadata_v2.db, meta.db ...) which sqlite3 cannot read. + read -r -n 15 magic < "$db" 2>/dev/null + [ "$magic" = "SQLite format 3" ] || { skipped=$((skipped+1)); continue; } + + local rel="${db#/home/gib/Server/}" out + [[ " ${CONVEX_EXPORTS[*]#*:} " == *" $rel "* ]] && continue # dump_convex covers it + out="$STATE_DIR/sqlite/$rel" + mkdir -p "$(dirname "$out")" + # .timeout: wait up to 30s for a writer's lock instead of failing on first contact. + if sqlite3 -cmd ".timeout 30000" "$db" ".backup '$out'" 2>>"$LOG"; then + n=$((n+1)) + else + log " WARNING: sqlite dump failed: $rel" + fi + done < <(find /home/gib/Server \ + -path "$STATE_DIR" -prune -o \ + -path '*/dind' -prune -o \ + -path '*/other/firefox' -prune -o \ + -path '*/node_modules' -prune -o \ + -type f \( -name '*.db' -o -name '*.sqlite' -o -name '*.sqlite3' \) \ + ! -name '*-wal' ! -name '*-shm' \ + ! -name '*logs.db' ! -name '*cache*.db' -print 2>/dev/null) + log " sqlite: $n dumped, $skipped skipped (not SQLite)" + dump_convex +} + +# Snapshot-exports each CONVEX_EXPORTS deployment to _backup_state/convex/.zip +# through a throwaway node container, so nothing is installed on the host. The admin key +# is derived from the instance secret on every run rather than stored. Takes ~8 min. +dump_convex() { + mkdir -p "$STATE_DIR/convex" + local entry c key + for entry in "${CONVEX_EXPORTS[@]}"; do + c="${entry%%:*}" + key=$(docker exec "$c" ./generate_admin_key.sh 2>/dev/null | tail -1) + if [ -z "$key" ]; then + log " WARNING: convex export failed: $c (could not generate admin key -- is it running?)" + continue + fi + if CONVEX_SELF_HOSTED_ADMIN_KEY="$key" timeout 1800 docker run --rm \ + --network nginx-bridge -v "$STATE_DIR/convex:/out" \ + -e CONVEX_SELF_HOSTED_URL="http://$c:3210" -e CONVEX_SELF_HOSTED_ADMIN_KEY \ + -e npm_config_update_notifier=false -e npm_config_fund=false -e npm_config_audit=false \ + docker.io/library/node:22-alpine sh -ec ' + # The CLI only runs inside a project that depends on convex. + mkdir /work && cd /work && echo "{}" > package.json + npm install --silent convex@latest + npx convex export --include-file-storage --path "/out/'"$c"'.zip"' \ + >>"$LOG" 2>&1; then + log " convex: exported $c ($(du -h "$STATE_DIR/convex/$c.zip" | cut -f1))" + else + log " WARNING: convex export failed: $c (see npx output above)" + fi + done +} + +# --------------------------------------------------------------------------- +# ZFS is the only thing giving us history, so make its absence loud. +# --------------------------------------------------------------------------- +check_zfs_history() { + local newest now age_h + newest=$(zfs list -H -p -o creation -t snapshot -s creation -d 1 "$REMOTE_DATASET" 2>/dev/null | tail -1) + if [ -z "$newest" ]; then + log " ⚠️ WARNING: no ZFS snapshots on $REMOTE_DATASET -- this backup has NO history." + log " It is a plain mirror right now. Check zfs-auto-snapshot." + return + fi + now=$(date +%s); age_h=$(( (now - newest) / 3600 )) + if [ "$age_h" -gt "$SNAPSHOT_MAX_AGE_HOURS" ]; then + log " ⚠️ WARNING: newest ZFS snapshot is ${age_h}h old (limit ${SNAPSHOT_MAX_AGE_HOURS}h) -- versioning may be broken." + else + log " ZFS history OK (newest snapshot ${age_h}h old, 14 dailies retained)" + fi +} + +log "=== backup start -> $DEST ===" +[ -d "$(dirname "$DEST")" ] || die "destination parent missing -- is the ZFS pool mounted?" +mountpoint -q /home/gib/Public || die "/home/gib/Public is not a mountpoint -- refusing to write into an unmounted path" +mkdir -p "$DEST" + +dump_state +check_zfs_history + +rsync -a --numeric-ids --delete --delete-excluded --human-readable --stats \ + "${EXCLUDE_ARGS[@]}" "$SOURCE" "$DEST/" 2>&1 | tee -a "$LOG" +status=${PIPESTATUS[0]} + +if [ "$status" -ne 0 ] && [ "$status" -ne 24 ]; then + die "rsync failed with exit $status -- 'current' may be half-updated; yesterday's ZFS snapshot is still good" +fi +[ "$status" -eq 24 ] && log "NOTE: rsync exit 24 (files vanished mid-run) -- expected on a live system" + +if [ -r /home/gib/Server/.backup-heartbeat-url ]; then + url=$(< /home/gib/Server/.backup-heartbeat-url) + curl -fsS -m 20 "$url" >/dev/null 2>&1 \ + && log "Heartbeat sent" || log "NOTE: heartbeat ping failed (backup itself was fine)" +fi + +log "=== backup complete ==="