195 lines
9.9 KiB
Bash
Executable File
195 lines
9.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Daily backup of ~/Server to the home server, as hardlinked dated snapshots.
|
|
#
|
|
# WHAT THIS FIXES vs. the old version (2026-09-17):
|
|
# 1. OWNERSHIP. Rootless podman leaves most container data owned by subuids
|
|
# (gitea = 525288, postgres data = the userns root mapping). Plain `rsync -a`
|
|
# running as gib cannot chown on the far side, so every previous backup landed
|
|
# 100% owned by uid 1000. Restoring that gives you the right bytes with the wrong
|
|
# owners: postgres refuses to start unless PGDATA is owned correctly, and gitea
|
|
# fails on its own data dir. Fixed with `-M--fake-super`, which makes the REMOTE
|
|
# rsync stash mode/uid/gid in a user.rsync.%stat xattr (verified working on the
|
|
# home server's ZFS). Restore with --fake-super on the read side to put them back.
|
|
# 2. CONSISTENCY. 13 SQLite DBs and a live postgres were copied hot, mid-write. Now
|
|
# dumped properly first (sqlite3 .backup / pg_dumpall) into _backup_state/.
|
|
# 3. SPACE / HISTORY. Every run was a full ~19G copy into a new dated directory.
|
|
# Now it rsyncs into ONE stable directory, `current/`, and versioning is left to
|
|
# the home server: Media/Server_Backup is its own ZFS dataset with zfs-auto-snapshot
|
|
# keeping 14 dailies. ZFS snapshots are block-level, so a database that changes a
|
|
# few pages costs a few pages -- not a whole new copy the way dated dirs (or even
|
|
# rsync --link-dest hardlinks, which are per-file) would.
|
|
#
|
|
# Containers keep running throughout. That is safe *because* of the dumps above --
|
|
# the dumps, not the raw data dirs, are the authoritative restore path.
|
|
#
|
|
# See RESTORE.md for how to bring this back on a new machine.
|
|
|
|
set -uo pipefail
|
|
|
|
REMOTE_HOST="[email protected]"
|
|
REMOTE_ROOT="/home/gib/Public/Server_Backup/Containers/lilman.gib"
|
|
SOURCE="$HOME/Server/"
|
|
STATE_DIR="$HOME/Server/_backup_state"
|
|
DEST_NAME="current" # One stable directory. ZFS snapshots provide history.
|
|
LOG_DIR="$HOME/Server/logs"
|
|
LOG="$LOG_DIR/backup.log"
|
|
LOCK="${XDG_RUNTIME_DIR:-/tmp}/server-backup.lock"
|
|
|
|
# Versioning is ZFS's job, not ours -- see the header. We only sanity-check that it is
|
|
# actually still happening, so this can never silently degrade into a single mirror with
|
|
# no history. Warn if the newest snapshot of the dataset is older than this.
|
|
REMOTE_DATASET="Media/Server_Backup"
|
|
SNAPSHOT_MAX_AGE_HOURS=48
|
|
|
|
mkdir -p "$LOG_DIR"
|
|
log() { printf '%s %s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG"; }
|
|
die() { log "FATAL: $*"; exit 1; }
|
|
|
|
# Refuse to overlap with a still-running backup rather than corrupting a snapshot.
|
|
exec 9>"$LOCK" || die "cannot open lock $LOCK"
|
|
flock -n 9 || die "another backup is already running"
|
|
|
|
EXCLUSIONS=(
|
|
# --- build noise -------------------------------------------------------
|
|
# NOTE: these are bare patterns, so they match at ANY depth. Today they only
|
|
# catch genuine scratch dirs (checked 2026-09-17). If you add a service whose
|
|
# real data lives in a dir called cache/build/dist/tmp, it will be silently
|
|
# skipped -- re-check this list when adding services.
|
|
"node_modules" ".git" "*.log" "*.tmp" ".cache" "cache" "tmp" "temp"
|
|
".npm" ".yarn" "dist" "build" "target" ".venv" "venv" "__pycache__"
|
|
"*.pyc" ".DS_Store" "Thumbs.db" "vendor" "*.mp4" "*.mkv"
|
|
|
|
# --- unreadable to gib, and genuinely disposable ------------------------
|
|
# Redis/valkey cache dumps: mode 0600 under a subuid because the container
|
|
# chowns its own data. Pure cache (usesend + infisical), regenerated on boot.
|
|
"dump.rdb"
|
|
# Gitea runs WITHOUT userns_mode (s6 needs container-root -- see AGENTS.md §3),
|
|
# so these are subuid-owned and mode 0700. Excluding them is what lets this run
|
|
# without sudo. Nothing here is needed to restore gitea:
|
|
# sessions/ ephemeral web logins; worst case everyone logs in again
|
|
# issues.bleve search index, rebuilt from the DB on demand
|
|
# Gitea/ssh a 25-byte `environment` file for the git shell
|
|
#
|
|
# ⚠️ The SSH HOST KEYS are NOT here -- they live in Gitea/volume/ssh, are
|
|
# gib-owned, and DO get backed up. Losing those = host-key mismatch for every client.
|
|
"sessions" "issues.bleve" "Gitea/ssh"
|
|
|
|
# Mode 0600 under subuid 525288, so unreadable to gib -- this single file is what
|
|
# made every previous run exit 23. It is NOT disposable (it holds the MCP bridge's
|
|
# registered OAuth clients), so it is captured via `podman exec` into
|
|
# _backup_state/files/ instead of being read off disk.
|
|
"BillTracker/data/mcp/bridge.json"
|
|
|
|
# Stirling-PDF writes this 0600 under subuid 525287 (since its 2026-09-25 update),
|
|
# which made every run since exit 23. Excluded rather than captured: losing it only
|
|
# means re-entering any credentials stored in Stirling-PDF.
|
|
"pdf-tools/extraConfigs/credential-encryption.key"
|
|
)
|
|
|
|
EXCLUDE_ARGS=()
|
|
for e in "${EXCLUSIONS[@]}"; do EXCLUDE_ARGS+=( "--exclude=$e" ); done
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 1: consistent dumps of everything transactional.
|
|
# ---------------------------------------------------------------------------
|
|
dump_state() {
|
|
log "Dumping databases to _backup_state/ ..."
|
|
rm -rf "$STATE_DIR"
|
|
mkdir -p "$STATE_DIR"/{postgres,sqlite,files}
|
|
printf 'Generated %s by ~/Server/backup. Authoritative restore source -- see RESTORE.md\n' \
|
|
"$(date -Is)" > "$STATE_DIR/README"
|
|
|
|
# Live postgres (serves npm, authentik, usesend, ... -- pg_dumpall gets all of them).
|
|
if podman ps --format '{{.Names}}' | grep -qx postgresql; then
|
|
if podman exec postgresql pg_dumpall -U npm 2>>"$LOG" | gzip -9 > "$STATE_DIR/postgres/all.sql.gz"; then
|
|
log " postgres: $(du -h "$STATE_DIR/postgres/all.sql.gz" | cut -f1)"
|
|
else
|
|
log " WARNING: pg_dumpall failed -- raw pg_data is still copied, but is a hot copy"
|
|
fi
|
|
else
|
|
log " WARNING: postgresql container not running, skipping pg_dumpall"
|
|
fi
|
|
|
|
# SQLite: .backup uses the online backup API, so this is safe while containers write.
|
|
local n=0
|
|
while IFS= read -r db; do
|
|
# *.db also matches stirling-pdf's H2 database, which is not SQLite. Check magic.
|
|
head -c 16 "$db" 2>/dev/null | grep -q "SQLite format 3" || continue
|
|
local rel="${db#"$HOME"/Server/}" out
|
|
out="$STATE_DIR/sqlite/$rel"
|
|
mkdir -p "$(dirname "$out")"
|
|
if sqlite3 -cmd ".timeout 30000" "$db" ".backup '$out'" 2>>"$LOG"; then n=$((n+1)); else log " WARNING: sqlite dump failed: $rel"; fi
|
|
done < <(find "$HOME/Server" -path "$STATE_DIR" -prune -o \
|
|
-type f \( -name '*.db' -o -name '*.sqlite' -o -name '*.sqlite3' \) -print 2>/dev/null)
|
|
log " sqlite: $n databases"
|
|
|
|
# Files gib cannot read off disk, pulled through their container instead.
|
|
if podman ps --format '{{.Names}}' | grep -qx bill-tracker-mcp; then
|
|
podman exec bill-tracker-mcp cat /data/bridge.json \
|
|
> "$STATE_DIR/files/BillTracker--data--mcp--bridge.json" 2>>"$LOG" \
|
|
&& log " bridge.json captured" \
|
|
|| log " WARNING: could not capture bridge.json"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 2: confirm ZFS is still providing history.
|
|
#
|
|
# We deliberately overwrite one directory instead of keeping dated copies, which
|
|
# means ZFS snapshots are the ONLY thing standing between us and "a mirror of a
|
|
# mistake". If auto-snapshot ever stops, that must be loud, not silent.
|
|
# ---------------------------------------------------------------------------
|
|
check_zfs_history() {
|
|
local newest now age_h
|
|
newest=$(ssh -o BatchMode=yes "$REMOTE_HOST" \
|
|
"zfs list -H -p -o creation -t snapshot -s creation -d 1 '$REMOTE_DATASET' 2>/dev/null | tail -1" || true)
|
|
if [ -z "$newest" ]; then
|
|
log " ⚠️ WARNING: no ZFS snapshots on $REMOTE_DATASET -- this backup has NO history."
|
|
log " It is a plain mirror right now. Check zfs-auto-snapshot on the home server."
|
|
return
|
|
fi
|
|
now=$(date +%s)
|
|
age_h=$(( (now - newest) / 3600 ))
|
|
if [ "$age_h" -gt "$SNAPSHOT_MAX_AGE_HOURS" ]; then
|
|
log " ⚠️ WARNING: newest ZFS snapshot is ${age_h}h old (limit ${SNAPSHOT_MAX_AGE_HOURS}h) -- versioning may be broken."
|
|
else
|
|
log " ZFS history OK (newest snapshot ${age_h}h old, 14 dailies retained)"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 3: mirror.
|
|
# ---------------------------------------------------------------------------
|
|
log "=== backup start -> $REMOTE_HOST:$REMOTE_ROOT/$DEST_NAME ==="
|
|
ssh -o BatchMode=yes -o ConnectTimeout=15 "$REMOTE_HOST" "mkdir -p '$REMOTE_ROOT/$DEST_NAME'" \
|
|
|| die "cannot reach $REMOTE_HOST"
|
|
|
|
dump_state
|
|
check_zfs_history
|
|
|
|
# --delete/--delete-excluded keep `current` an exact mirror of the source. Anything
|
|
# removed here still exists in yesterday's ZFS snapshot, so this is not destructive.
|
|
rsync -a --numeric-ids -M--fake-super --delete --delete-excluded \
|
|
--human-readable --stats \
|
|
"${EXCLUDE_ARGS[@]}" \
|
|
--rsync-path="mkdir -p '$REMOTE_ROOT/$DEST_NAME' && rsync" \
|
|
"$SOURCE" "$REMOTE_HOST:$REMOTE_ROOT/$DEST_NAME/" 2>&1 | tee -a "$LOG"
|
|
status=${PIPESTATUS[0]}
|
|
|
|
# 24 = "source files vanished during transfer". Normal on a live system, not a failure.
|
|
if [ "$status" -ne 0 ] && [ "$status" -ne 24 ]; then
|
|
die "rsync failed with exit $status -- 'current' may be half-updated; yesterday's ZFS snapshot is still good"
|
|
fi
|
|
[ "$status" -eq 24 ] && log "NOTE: rsync exit 24 (files vanished mid-run) -- expected on a live system"
|
|
|
|
# Optional heartbeat so a silent failure cannot go unnoticed. Create the file with an
|
|
# uptime-kuma push URL in it and this pings on success only; absent, nothing happens.
|
|
if [ -r "$HOME/Server/.backup-heartbeat-url" ]; then
|
|
url=$(< "$HOME/Server/.backup-heartbeat-url")
|
|
curl -fsS -m 20 "$url" >/dev/null 2>&1 \
|
|
&& log "Heartbeat sent" || log "NOTE: heartbeat ping failed (backup itself was fine)"
|
|
fi
|
|
|
|
log "=== backup complete ==="
|