182 lines
8.4 KiB
Bash
Executable File
182 lines
8.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Daily backup of ~/Server (NVMe) to the ZFS pool on this same machine.
|
|
# Purpose: survive the NVMe/LUKS volume dying. Not an offsite backup.
|
|
#
|
|
# Runs as ROOT via server-backup.service. That is deliberate: this box runs Docker,
|
|
# so most container data under ~/Server is root-owned. The old script used `sudo rsync`,
|
|
# which worked but could never be automated without a sudoers rule. A systemd system
|
|
# unit is already root, so ownership is preserved natively and no sudoers entry is
|
|
# needed at runtime. (Note this differs from the VPS, which is rootless podman and
|
|
# needs rsync --fake-super to carry subuid ownership; nothing like that applies here.)
|
|
#
|
|
# History is ZFS's job: Media/Server_Backup has zfs-auto-snapshot keeping 14 dailies,
|
|
# so this writes ONE directory and never makes dated copies. Snapshots are block-level,
|
|
# so a 1.6 GB database that changes a few pages costs a few pages.
|
|
#
|
|
# See RESTORE.md for how to bring this back.
|
|
|
|
set -uo pipefail
|
|
|
|
SOURCE="/home/gib/Server/"
|
|
DEST="/home/gib/Public/Server_Backup/Containers/server.gib/current"
|
|
STATE_DIR="/home/gib/Server/_backup_state"
|
|
LOG_DIR="/home/gib/Server/logs"
|
|
LOG="$LOG_DIR/backup.log"
|
|
LOCK="/run/server-backup.lock"
|
|
REMOTE_DATASET="Media/Server_Backup"
|
|
SNAPSHOT_MAX_AGE_HOURS=48
|
|
|
|
mkdir -p "$LOG_DIR"
|
|
log() { printf '%s %s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG"; }
|
|
die() { log "FATAL: $*"; exit 1; }
|
|
|
|
exec 9>"$LOCK" || die "cannot open lock $LOCK"
|
|
flock -n 9 || die "another backup is already running"
|
|
|
|
EXCLUSIONS=(
|
|
# --- build noise (inherited from the original script) -------------------
|
|
"node_modules" ".git" "*.log" "*.tmp" ".cache" "cache" "tmp" "temp"
|
|
".npm" ".yarn" "dist" "build" "target" ".venv" "venv" "__pycache__"
|
|
"*.pyc" ".DS_Store" "Thumbs.db" "vendor" "*.mp4" "*.mkv"
|
|
|
|
# --- the big one: Gitea CI runner Docker-in-Docker storage --------------
|
|
# 203 GB across runner-1/2/3 (72 GB of containerd image layers each), and it
|
|
# churns constantly as CI builds run. It is pure cache: a rebuilt runner just
|
|
# re-pulls everything. Excluding dind/ specifically KEEPS runner-N/data/.runner
|
|
# (the 648-byte registration token) and certs/, which are the only parts that
|
|
# would actually be painful to recreate.
|
|
#
|
|
# Measured as root on 2026-09-17 -- a plain `du` as gib reports these as ~13 MB
|
|
# because it cannot descend into root-owned dirs. Do not trust non-root sizes here.
|
|
"/Agents/Gitea/volumes/*/dind"
|
|
|
|
# Regenerated every run; no reason to ship the previous run's copy.
|
|
"/_backup_state/*.partial"
|
|
)
|
|
|
|
EXCLUDE_ARGS=()
|
|
for e in "${EXCLUSIONS[@]}"; do EXCLUDE_ARGS+=( "--exclude=$e" ); done
|
|
|
|
# Convex deployments backed by SQLite, as "container:sqlite path under ~/Server".
|
|
# Convex keeps that file in rollback-journal mode and writes to it constantly, so
|
|
# sqlite3 .backup either hits "database is locked" or restarts forever. Instead,
|
|
# dump_convex takes Convex's own snapshot export (`npx convex export`, restored with
|
|
# `npx convex import`), and dump_state skips the raw file.
|
|
CONVEX_EXPORTS=(
|
|
"gbrown-backend:Gbrown/data/convex/db.sqlite3"
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Consistent dumps. Containers keep running; these are what make that safe.
|
|
# ---------------------------------------------------------------------------
|
|
dump_state() {
|
|
log "Dumping databases to _backup_state/ ..."
|
|
rm -rf "$STATE_DIR"
|
|
mkdir -p "$STATE_DIR/sqlite"
|
|
printf 'Generated %s by /usr/local/sbin/server-backup. Authoritative restore source.\n' \
|
|
"$(date -Is)" > "$STATE_DIR/README"
|
|
|
|
local n=0 skipped=0
|
|
while IFS= read -r db; do
|
|
# Only real SQLite. Docker/containerd/buildkit scatter BoltDB files named *.db
|
|
# (portainer.db, metadata_v2.db, meta.db ...) which sqlite3 cannot read.
|
|
read -r -n 15 magic < "$db" 2>/dev/null
|
|
[ "$magic" = "SQLite format 3" ] || { skipped=$((skipped+1)); continue; }
|
|
|
|
local rel="${db#/home/gib/Server/}" out
|
|
[[ " ${CONVEX_EXPORTS[*]#*:} " == *" $rel "* ]] && continue # dump_convex covers it
|
|
out="$STATE_DIR/sqlite/$rel"
|
|
mkdir -p "$(dirname "$out")"
|
|
# .timeout: wait up to 30s for a writer's lock instead of failing on first contact.
|
|
if sqlite3 -cmd ".timeout 30000" "$db" ".backup '$out'" 2>>"$LOG"; then
|
|
n=$((n+1))
|
|
else
|
|
log " WARNING: sqlite dump failed: $rel"
|
|
fi
|
|
done < <(find /home/gib/Server \
|
|
-path "$STATE_DIR" -prune -o \
|
|
-path '*/dind' -prune -o \
|
|
-path '*/other/firefox' -prune -o \
|
|
-path '*/node_modules' -prune -o \
|
|
-type f \( -name '*.db' -o -name '*.sqlite' -o -name '*.sqlite3' \) \
|
|
! -name '*-wal' ! -name '*-shm' \
|
|
! -name '*logs.db' ! -name '*cache*.db' -print 2>/dev/null)
|
|
log " sqlite: $n dumped, $skipped skipped (not SQLite)"
|
|
dump_convex
|
|
}
|
|
|
|
# Snapshot-exports each CONVEX_EXPORTS deployment to _backup_state/convex/<container>.zip
|
|
# through a throwaway node container, so nothing is installed on the host. The admin key
|
|
# is derived from the instance secret on every run rather than stored. Takes ~8 min.
|
|
dump_convex() {
|
|
mkdir -p "$STATE_DIR/convex"
|
|
local entry c key
|
|
for entry in "${CONVEX_EXPORTS[@]}"; do
|
|
c="${entry%%:*}"
|
|
key=$(docker exec "$c" ./generate_admin_key.sh 2>/dev/null | tail -1)
|
|
if [ -z "$key" ]; then
|
|
log " WARNING: convex export failed: $c (could not generate admin key -- is it running?)"
|
|
continue
|
|
fi
|
|
if CONVEX_SELF_HOSTED_ADMIN_KEY="$key" timeout 1800 docker run --rm \
|
|
--network nginx-bridge -v "$STATE_DIR/convex:/out" \
|
|
-e CONVEX_SELF_HOSTED_URL="http://$c:3210" -e CONVEX_SELF_HOSTED_ADMIN_KEY \
|
|
-e npm_config_update_notifier=false -e npm_config_fund=false -e npm_config_audit=false \
|
|
docker.io/library/node:22-alpine sh -ec '
|
|
# The CLI only runs inside a project that depends on convex.
|
|
mkdir /work && cd /work && echo "{}" > package.json
|
|
npm install --silent convex@latest
|
|
npx convex export --include-file-storage --path "/out/'"$c"'.zip"' \
|
|
>>"$LOG" 2>&1; then
|
|
log " convex: exported $c ($(du -h "$STATE_DIR/convex/$c.zip" | cut -f1))"
|
|
else
|
|
log " WARNING: convex export failed: $c (see npx output above)"
|
|
fi
|
|
done
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ZFS is the only thing giving us history, so make its absence loud.
|
|
# ---------------------------------------------------------------------------
|
|
check_zfs_history() {
|
|
local newest now age_h
|
|
newest=$(zfs list -H -p -o creation -t snapshot -s creation -d 1 "$REMOTE_DATASET" 2>/dev/null | tail -1)
|
|
if [ -z "$newest" ]; then
|
|
log " ⚠️ WARNING: no ZFS snapshots on $REMOTE_DATASET -- this backup has NO history."
|
|
log " It is a plain mirror right now. Check zfs-auto-snapshot."
|
|
return
|
|
fi
|
|
now=$(date +%s); age_h=$(( (now - newest) / 3600 ))
|
|
if [ "$age_h" -gt "$SNAPSHOT_MAX_AGE_HOURS" ]; then
|
|
log " ⚠️ WARNING: newest ZFS snapshot is ${age_h}h old (limit ${SNAPSHOT_MAX_AGE_HOURS}h) -- versioning may be broken."
|
|
else
|
|
log " ZFS history OK (newest snapshot ${age_h}h old, 14 dailies retained)"
|
|
fi
|
|
}
|
|
|
|
log "=== backup start -> $DEST ==="
|
|
[ -d "$(dirname "$DEST")" ] || die "destination parent missing -- is the ZFS pool mounted?"
|
|
mountpoint -q /home/gib/Public || die "/home/gib/Public is not a mountpoint -- refusing to write into an unmounted path"
|
|
mkdir -p "$DEST"
|
|
|
|
dump_state
|
|
check_zfs_history
|
|
|
|
rsync -a --numeric-ids --delete --delete-excluded --human-readable --stats \
|
|
"${EXCLUDE_ARGS[@]}" "$SOURCE" "$DEST/" 2>&1 | tee -a "$LOG"
|
|
status=${PIPESTATUS[0]}
|
|
|
|
if [ "$status" -ne 0 ] && [ "$status" -ne 24 ]; then
|
|
die "rsync failed with exit $status -- 'current' may be half-updated; yesterday's ZFS snapshot is still good"
|
|
fi
|
|
[ "$status" -eq 24 ] && log "NOTE: rsync exit 24 (files vanished mid-run) -- expected on a live system"
|
|
|
|
if [ -r /home/gib/Server/.backup-heartbeat-url ]; then
|
|
url=$(< /home/gib/Server/.backup-heartbeat-url)
|
|
curl -fsS -m 20 "$url" >/dev/null 2>&1 \
|
|
&& log "Heartbeat sent" || log "NOTE: heartbeat ping failed (backup itself was fine)"
|
|
fi
|
|
|
|
log "=== backup complete ==="
|