diff --git a/server/home/media-backup b/server/home/media-backup index d6bf013..79e0596 100755 --- a/server/home/media-backup +++ b/server/home/media-backup @@ -81,13 +81,25 @@ mkdir -p "$DEST" # A borg run holds this lock for 2-4 hours and rewrites index/hints/integrity at the END. # Copying mid-run pairs new data chunks with a stale index: a repo that looks fine and -# fails at restore. Defer rather than queue -- the next weekly run will catch it. +# fails at restore. AIO's borg runs daily from 07:30 UTC for about 2 hours, so it can +# still be going when the timer fires. Wait for it to finish, checking every +# BORG_WAIT_INTERVAL, and defer only if it is still held after BORG_WAIT_MAX. The unit's +# TimeoutStartSec=24h covers the wait plus the ~50 min sync. +BORG_WAIT_INTERVAL=300 # 5 min +BORG_WAIT_MAX=21600 # 6 h +waited=0 +while [ -e "$BORG_LOCK" ] && [ "$waited" -lt "$BORG_WAIT_MAX" ]; do + log "borg is mid-run (lock held at $BORG_LOCK). Waited $((waited / 60)) of $((BORG_WAIT_MAX / 60)) min, checking again in $((BORG_WAIT_INTERVAL / 60)) min" + sleep "$BORG_WAIT_INTERVAL" + waited=$(( waited + BORG_WAIT_INTERVAL )) +done if [ -e "$BORG_LOCK" ]; then log "borg is mid-run (lock held at $BORG_LOCK)" log "DEFERRING this week's run rather than copying a torn repo. Next fire: next Saturday." log "=== media-backup deferred ===" exit 0 fi +[ "$waited" -gt 0 ] && log "borg lock released after waiting $((waited / 60)) min, continuing" avail_gb=$(zfs get -Hp -o value available "$POOL" 2>/dev/null) avail_gb=$(( ${avail_gb:-0} / 1024 / 1024 / 1024 )) diff --git a/server/home/systemd/media-backup.timer b/server/home/systemd/media-backup.timer index cc5d05a..5875c64 100644 --- a/server/home/systemd/media-backup.timer +++ b/server/home/systemd/media-backup.timer @@ -6,8 +6,9 @@ Description=Weekly Media -> Backup pool copy # timeshift (04:00), arr-queue-reconcile (04:31), and both ~/Server backups (01:00, 02:00). # Saturday specifically because Monday carries the Media scrub and Sunday has # raid-check (01:00), docker-prune (05:33) and dind-prune (06:03). -# gib also runs Nextcloud's borg by hand roughly monthly and is, in his words, -# not going to be doing that at 5 AM on a Saturday -- so the borg lock should be free. +# Nextcloud AIO's borg runs daily from 07:30 UTC for about 2 hours, so its lock can +# overlap this fire. The script waits up to 6 h for the lock, checking every 5 min, +# and defers the week only if borg is still running after that. OnCalendar=Sat *-*-* 05:00:00 RandomizedDelaySec=15m Persistent=true