Sync both server snapshots with what the live machines run, and keep their backup tooling
This commit is contained in:
Gabriel Brown committed
1 parent
95fa480469
commit
cb089b39ea
42 files changed
+3099
-21
No files matched your search
Executable
+194
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Daily backup of ~/Server to the home server, as hardlinked dated snapshots.
|
||||
#
|
||||
# WHAT THIS FIXES vs. the old version (2026-09-17):
|
||||
# 1. OWNERSHIP. Rootless podman leaves most container data owned by subuids
|
||||
# (gitea = 525288, postgres data = the userns root mapping). Plain `rsync -a`
|
||||
# running as gib cannot chown on the far side, so every previous backup landed
|
||||
# 100% owned by uid 1000. Restoring that gives you the right bytes with the wrong
|
||||
# owners: postgres refuses to start unless PGDATA is owned correctly, and gitea
|
||||
# fails on its own data dir. Fixed with `-M--fake-super`, which makes the REMOTE
|
||||
# rsync stash mode/uid/gid in a user.rsync.%stat xattr (verified working on the
|
||||
# home server's ZFS). Restore with --fake-super on the read side to put them back.
|
||||
# 2. CONSISTENCY. 13 SQLite DBs and a live postgres were copied hot, mid-write. Now
|
||||
# dumped properly first (sqlite3 .backup / pg_dumpall) into _backup_state/.
|
||||
# 3. SPACE / HISTORY. Every run was a full ~19G copy into a new dated directory.
|
||||
# Now it rsyncs into ONE stable directory, `current/`, and versioning is left to
|
||||
# the home server: Media/Server_Backup is its own ZFS dataset with zfs-auto-snapshot
|
||||
# keeping 14 dailies. ZFS snapshots are block-level, so a database that changes a
|
||||
# few pages costs a few pages -- not a whole new copy the way dated dirs (or even
|
||||
# rsync --link-dest hardlinks, which are per-file) would.
|
||||
#
|
||||
# Containers keep running throughout. That is safe *because* of the dumps above --
|
||||
# the dumps, not the raw data dirs, are the authoritative restore path.
|
||||
#
|
||||
# See RESTORE.md for how to bring this back on a new machine.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
REMOTE_HOST="[email protected]"
|
||||
REMOTE_ROOT="/home/gib/Public/Server_Backup/Containers/lilman.gib"
|
||||
SOURCE="$HOME/Server/"
|
||||
STATE_DIR="$HOME/Server/_backup_state"
|
||||
DEST_NAME="current" # One stable directory. ZFS snapshots provide history.
|
||||
LOG_DIR="$HOME/Server/logs"
|
||||
LOG="$LOG_DIR/backup.log"
|
||||
LOCK="${XDG_RUNTIME_DIR:-/tmp}/server-backup.lock"
|
||||
|
||||
# Versioning is ZFS's job, not ours -- see the header. We only sanity-check that it is
|
||||
# actually still happening, so this can never silently degrade into a single mirror with
|
||||
# no history. Warn if the newest snapshot of the dataset is older than this.
|
||||
REMOTE_DATASET="Media/Server_Backup"
|
||||
SNAPSHOT_MAX_AGE_HOURS=48
|
||||
|
||||
mkdir -p "$LOG_DIR"
|
||||
log() { printf '%s %s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG"; }
|
||||
die() { log "FATAL: $*"; exit 1; }
|
||||
|
||||
# Refuse to overlap with a still-running backup rather than corrupting a snapshot.
|
||||
exec 9>"$LOCK" || die "cannot open lock $LOCK"
|
||||
flock -n 9 || die "another backup is already running"
|
||||
|
||||
EXCLUSIONS=(
|
||||
# --- build noise -------------------------------------------------------
|
||||
# NOTE: these are bare patterns, so they match at ANY depth. Today they only
|
||||
# catch genuine scratch dirs (checked 2026-09-17). If you add a service whose
|
||||
# real data lives in a dir called cache/build/dist/tmp, it will be silently
|
||||
# skipped -- re-check this list when adding services.
|
||||
"node_modules" ".git" "*.log" "*.tmp" ".cache" "cache" "tmp" "temp"
|
||||
".npm" ".yarn" "dist" "build" "target" ".venv" "venv" "__pycache__"
|
||||
"*.pyc" ".DS_Store" "Thumbs.db" "vendor" "*.mp4" "*.mkv"
|
||||
|
||||
# --- unreadable to gib, and genuinely disposable ------------------------
|
||||
# Redis/valkey cache dumps: mode 0600 under a subuid because the container
|
||||
# chowns its own data. Pure cache (usesend + infisical), regenerated on boot.
|
||||
"dump.rdb"
|
||||
# Gitea runs WITHOUT userns_mode (s6 needs container-root -- see AGENTS.md §3),
|
||||
# so these are subuid-owned and mode 0700. Excluding them is what lets this run
|
||||
# without sudo. Nothing here is needed to restore gitea:
|
||||
# sessions/ ephemeral web logins; worst case everyone logs in again
|
||||
# issues.bleve search index, rebuilt from the DB on demand
|
||||
# Gitea/ssh a 25-byte `environment` file for the git shell
|
||||
#
|
||||
# ⚠️ The SSH HOST KEYS are NOT here -- they live in Gitea/volume/ssh, are
|
||||
# gib-owned, and DO get backed up. Losing those = host-key mismatch for every client.
|
||||
"sessions" "issues.bleve" "Gitea/ssh"
|
||||
|
||||
# Mode 0600 under subuid 525288, so unreadable to gib -- this single file is what
|
||||
# made every previous run exit 23. It is NOT disposable (it holds the MCP bridge's
|
||||
# registered OAuth clients), so it is captured via `podman exec` into
|
||||
# _backup_state/files/ instead of being read off disk.
|
||||
"BillTracker/data/mcp/bridge.json"
|
||||
|
||||
# Stirling-PDF writes this 0600 under subuid 525287 (since its 2026-09-25 update),
|
||||
# which made every run since exit 23. Excluded rather than captured: losing it only
|
||||
# means re-entering any credentials stored in Stirling-PDF.
|
||||
"pdf-tools/extraConfigs/credential-encryption.key"
|
||||
)
|
||||
|
||||
EXCLUDE_ARGS=()
|
||||
for e in "${EXCLUSIONS[@]}"; do EXCLUDE_ARGS+=( "--exclude=$e" ); done
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 1: consistent dumps of everything transactional.
|
||||
# ---------------------------------------------------------------------------
|
||||
dump_state() {
|
||||
log "Dumping databases to _backup_state/ ..."
|
||||
rm -rf "$STATE_DIR"
|
||||
mkdir -p "$STATE_DIR"/{postgres,sqlite,files}
|
||||
printf 'Generated %s by ~/Server/backup. Authoritative restore source -- see RESTORE.md\n' \
|
||||
"$(date -Is)" > "$STATE_DIR/README"
|
||||
|
||||
# Live postgres (serves npm, authentik, usesend, ... -- pg_dumpall gets all of them).
|
||||
if podman ps --format '{{.Names}}' | grep -qx postgresql; then
|
||||
if podman exec postgresql pg_dumpall -U npm 2>>"$LOG" | gzip -9 > "$STATE_DIR/postgres/all.sql.gz"; then
|
||||
log " postgres: $(du -h "$STATE_DIR/postgres/all.sql.gz" | cut -f1)"
|
||||
else
|
||||
log " WARNING: pg_dumpall failed -- raw pg_data is still copied, but is a hot copy"
|
||||
fi
|
||||
else
|
||||
log " WARNING: postgresql container not running, skipping pg_dumpall"
|
||||
fi
|
||||
|
||||
# SQLite: .backup uses the online backup API, so this is safe while containers write.
|
||||
local n=0
|
||||
while IFS= read -r db; do
|
||||
# *.db also matches stirling-pdf's H2 database, which is not SQLite. Check magic.
|
||||
head -c 16 "$db" 2>/dev/null | grep -q "SQLite format 3" || continue
|
||||
local rel="${db#"$HOME"/Server/}" out
|
||||
out="$STATE_DIR/sqlite/$rel"
|
||||
mkdir -p "$(dirname "$out")"
|
||||
if sqlite3 -cmd ".timeout 30000" "$db" ".backup '$out'" 2>>"$LOG"; then n=$((n+1)); else log " WARNING: sqlite dump failed: $rel"; fi
|
||||
done < <(find "$HOME/Server" -path "$STATE_DIR" -prune -o \
|
||||
-type f \( -name '*.db' -o -name '*.sqlite' -o -name '*.sqlite3' \) -print 2>/dev/null)
|
||||
log " sqlite: $n databases"
|
||||
|
||||
# Files gib cannot read off disk, pulled through their container instead.
|
||||
if podman ps --format '{{.Names}}' | grep -qx bill-tracker-mcp; then
|
||||
podman exec bill-tracker-mcp cat /data/bridge.json \
|
||||
> "$STATE_DIR/files/BillTracker--data--mcp--bridge.json" 2>>"$LOG" \
|
||||
&& log " bridge.json captured" \
|
||||
|| log " WARNING: could not capture bridge.json"
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 2: confirm ZFS is still providing history.
|
||||
#
|
||||
# We deliberately overwrite one directory instead of keeping dated copies, which
|
||||
# means ZFS snapshots are the ONLY thing standing between us and "a mirror of a
|
||||
# mistake". If auto-snapshot ever stops, that must be loud, not silent.
|
||||
# ---------------------------------------------------------------------------
|
||||
check_zfs_history() {
|
||||
local newest now age_h
|
||||
newest=$(ssh -o BatchMode=yes "$REMOTE_HOST" \
|
||||
"zfs list -H -p -o creation -t snapshot -s creation -d 1 '$REMOTE_DATASET' 2>/dev/null | tail -1" || true)
|
||||
if [ -z "$newest" ]; then
|
||||
log " ⚠️ WARNING: no ZFS snapshots on $REMOTE_DATASET -- this backup has NO history."
|
||||
log " It is a plain mirror right now. Check zfs-auto-snapshot on the home server."
|
||||
return
|
||||
fi
|
||||
now=$(date +%s)
|
||||
age_h=$(( (now - newest) / 3600 ))
|
||||
if [ "$age_h" -gt "$SNAPSHOT_MAX_AGE_HOURS" ]; then
|
||||
log " ⚠️ WARNING: newest ZFS snapshot is ${age_h}h old (limit ${SNAPSHOT_MAX_AGE_HOURS}h) -- versioning may be broken."
|
||||
else
|
||||
log " ZFS history OK (newest snapshot ${age_h}h old, 14 dailies retained)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 3: mirror.
|
||||
# ---------------------------------------------------------------------------
|
||||
log "=== backup start -> $REMOTE_HOST:$REMOTE_ROOT/$DEST_NAME ==="
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=15 "$REMOTE_HOST" "mkdir -p '$REMOTE_ROOT/$DEST_NAME'" \
|
||||
|| die "cannot reach $REMOTE_HOST"
|
||||
|
||||
dump_state
|
||||
check_zfs_history
|
||||
|
||||
# --delete/--delete-excluded keep `current` an exact mirror of the source. Anything
|
||||
# removed here still exists in yesterday's ZFS snapshot, so this is not destructive.
|
||||
rsync -a --numeric-ids -M--fake-super --delete --delete-excluded \
|
||||
--human-readable --stats \
|
||||
"${EXCLUDE_ARGS[@]}" \
|
||||
--rsync-path="mkdir -p '$REMOTE_ROOT/$DEST_NAME' && rsync" \
|
||||
"$SOURCE" "$REMOTE_HOST:$REMOTE_ROOT/$DEST_NAME/" 2>&1 | tee -a "$LOG"
|
||||
status=${PIPESTATUS[0]}
|
||||
|
||||
# 24 = "source files vanished during transfer". Normal on a live system, not a failure.
|
||||
if [ "$status" -ne 0 ] && [ "$status" -ne 24 ]; then
|
||||
die "rsync failed with exit $status -- 'current' may be half-updated; yesterday's ZFS snapshot is still good"
|
||||
fi
|
||||
[ "$status" -eq 24 ] && log "NOTE: rsync exit 24 (files vanished mid-run) -- expected on a live system"
|
||||
|
||||
# Optional heartbeat so a silent failure cannot go unnoticed. Create the file with an
|
||||
# uptime-kuma push URL in it and this pings on success only; absent, nothing happens.
|
||||
if [ -r "$HOME/Server/.backup-heartbeat-url" ]; then
|
||||
url=$(< "$HOME/Server/.backup-heartbeat-url")
|
||||
curl -fsS -m 20 "$url" >/dev/null 2>&1 \
|
||||
&& log "Heartbeat sent" || log "NOTE: heartbeat ping failed (backup itself was fine)"
|
||||
fi
|
||||
|
||||
log "=== backup complete ==="
|
||||
Reference in new issue
Block a user