Gabriel Brown
f33da41cc6
A machine's role is now the interview's first question and the one answer Panama records. Servers get the same shell minus the screen: core packages, nvm, Bun, Claude Code and Codex (desktops get Codex too), linger, rootless ports from 80, firewalld, the nginx-bridge network, and a nightly image updater that replaced watchtower for cause. server/containers/ carries junior's 23 compose services -- secrets moved to per-machine .env files that never enter this public repo, every transformed compose proven to render byte-identical to what is live. 'panama server' enables, disables and relinks them; nothing here restarts a running service. 'boot --server' walks a fresh VPS from its root login to a normal install. Five new contracts pin the secrets rule, the catalog's shape, panama-server's behavior, the role plumbing, and the dotfile classification. Claude-Session: https://claude.ai/code/session_01NU5JGiN3JfzqrLQB6wmJ1E
239 lines
12 KiB
Bash
Executable File
239 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# update-containers — pull new images and restart the services that changed.
|
|
#
|
|
# REPLACES WATCHTOWER. Written 2026-08-17 after watchtower took gitea down for
|
|
# three days.
|
|
#
|
|
# WHY NOT WATCHTOWER: podman-compose puts every project in a POD. Watchtower does
|
|
# not update through compose — it builds a replacement container from the image
|
|
# plus the old container's config, and that replacement lands OUTSIDE the pod.
|
|
# On 2026-08-14 it did exactly that to gitea, reported `failed=0 updated=1`, and
|
|
# left it returning 502 until someone noticed three days later.
|
|
#
|
|
# This script never touches containers directly. It pulls the image and then
|
|
# restarts the SYSTEMD UNIT, so podman-compose rebuilds the project the same way
|
|
# it does at boot — pod, network aliases, published ports and all.
|
|
#
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# USAGE
|
|
# update-containers # every service except SKIP (what the timer runs)
|
|
# update-containers gitea # one service, e.g. from CI after a push
|
|
# update-containers --dry-run # show what WOULD update, change nothing
|
|
# update-containers --list # show services and their images, then exit
|
|
#
|
|
# Runs nightly at 00:00 via podman-update.timer (server/systemd/, enabled by
|
|
# setup-server). Log: ~/Server/logs/update-containers.log
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
set -uo pipefail
|
|
|
|
# ═════════════════════════════════════════════════════════════════════════════
|
|
# CONFIG — this is the part you edit
|
|
# ═════════════════════════════════════════════════════════════════════════════
|
|
|
|
# Services that must NEVER update unattended. Add a service here to freeze it.
|
|
#
|
|
# postgresql — restarting it drops all databases at once, and every service
|
|
# with them. Minor 17.x bumps are safe in principle but not worth
|
|
# doing at midnight unobserved. Also: the image tag is pinned to
|
|
# pgvector/pgvector:pg17-trixie for collation reasons — an
|
|
# unattended change of base could corrupt text indexes.
|
|
# authentik — runs IRREVERSIBLE database migrations on startup. Rolling back
|
|
# the image does not roll back the schema; you would need a restore.
|
|
#
|
|
# Update these two by hand, after a pg_dump:
|
|
# podman pull <image> && systemctl --user restart podman-<name>.service
|
|
SKIP=(postgresql authentik watchtower)
|
|
|
|
# Services built locally rather than pulled. There is no registry image to check,
|
|
# so pulling is pointless — they are skipped automatically when no `image:` line
|
|
# resolves, but listing them here keeps the log quiet and the intent obvious.
|
|
LOCAL_BUILD=(bang completeuphoria sierraandtyler)
|
|
|
|
# ⚠️ NOTE ON GITEA: it is deliberately NOT in SKIP. The three-day outage was caused
|
|
# by watchtower's recreate mechanism, not by the act of updating — restarting the
|
|
# unit is the safe path this script uses. But gitea DOES run database migrations on
|
|
# version upgrades. If you would rather approve those yourself, move `gitea` into
|
|
# SKIP above; nothing else needs changing.
|
|
|
|
SERVER_DIR="$HOME/Server"
|
|
LOG_DIR="$SERVER_DIR/logs"
|
|
LOG="$LOG_DIR/update-containers.log"
|
|
|
|
# ═════════════════════════════════════════════════════════════════════════════
|
|
|
|
mkdir -p "$LOG_DIR"
|
|
|
|
DRY_RUN=0
|
|
ONLY=""
|
|
case "${1:-}" in
|
|
--dry-run) DRY_RUN=1 ;;
|
|
--list) LIST=1 ;;
|
|
--help|-h) sed -n '19,27p' "$0"; exit 0 ;;
|
|
"") ;;
|
|
-*) echo "unknown option: $1" >&2; exit 2 ;;
|
|
*) ONLY="$1" ;;
|
|
esac
|
|
|
|
log() { printf '%s %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG"; }
|
|
|
|
in_list() { local n="$1"; shift; local x; for x in "$@"; do [ "$x" = "$n" ] && return 0; done; return 1; }
|
|
|
|
# Units that are podman-compose services. Excludes podman's own helper units,
|
|
# which are not services and must stay disabled.
|
|
list_units() {
|
|
systemctl --user list-unit-files 'podman-*.service' --no-legend 2>/dev/null \
|
|
| awk '{print $1}' \
|
|
| grep -vE 'podman-(auto-update|restart|clean-transient|kube@|user-wait-network-online|update)' \
|
|
| sed 's/^podman-//; s/\.service$//' \
|
|
| sort
|
|
}
|
|
|
|
# Images declared in a service's compose.yml. Ignores commented lines and the
|
|
# `build:` stanza, which has no pullable image.
|
|
#
|
|
# Image tags frequently reference variables from the service's .env, e.g.
|
|
# ghcr.io/get-convex/convex-backend:${BACKEND_TAG:-latest}
|
|
# git.gbrown.org/gib/${NEXT_CONTAINER_NAME}:latest
|
|
# Those must be expanded or the pull is attempted against a literal "${...}" and
|
|
# always fails. We expand in a SUBSHELL so the .env cannot leak into this script's
|
|
# environment (these files contain database passwords and API keys).
|
|
images_for() {
|
|
local dir="$1"
|
|
[ -f "$dir/compose.yml" ] || return 0
|
|
grep -oE '^[[:space:]]*image:[[:space:]]*[^[:space:]#]+' "$dir/compose.yml" 2>/dev/null \
|
|
| sed -E 's/^[[:space:]]*image:[[:space:]]*//' \
|
|
| while IFS= read -r img; do
|
|
case "$img" in
|
|
*'${'*) ( set -a; [ -f "$dir/.env" ] && . "$dir/.env" >/dev/null 2>&1; set +a
|
|
eval "printf '%s\\n' \"$img\"" 2>/dev/null ) ;;
|
|
*) printf '%s\n' "$img" ;;
|
|
esac
|
|
done \
|
|
| grep -v '\${' \
|
|
| sort -u
|
|
}
|
|
|
|
workdir_for() {
|
|
systemctl --user show "podman-$1.service" -p WorkingDirectory --value 2>/dev/null
|
|
}
|
|
|
|
# ── --list ───────────────────────────────────────────────────────────────────
|
|
if [ "${LIST:-0}" = "1" ]; then
|
|
for svc in $(list_units); do
|
|
printf '%-22s' "$svc"
|
|
if in_list "$svc" "${SKIP[@]}"; then echo "SKIP (frozen)"; continue; fi
|
|
if in_list "$svc" "${LOCAL_BUILD[@]}"; then echo "SKIP (built locally)"; continue; fi
|
|
echo "$(images_for "$(workdir_for "$svc")" | tr '\n' ' ')"
|
|
done
|
|
exit 0
|
|
fi
|
|
|
|
# ── main ─────────────────────────────────────────────────────────────────────
|
|
# ${DRY_RUN:+...} would expand even when DRY_RUN=0, because "0" is a non-empty
|
|
# string. Build the label explicitly instead.
|
|
dry_label=""; [ "$DRY_RUN" -eq 1 ] && dry_label=" [DRY RUN]"
|
|
log "=== update run started${ONLY:+ (single service: $ONLY)}$dry_label ==="
|
|
|
|
updated=(); failed=(); checked=0
|
|
|
|
for svc in $(list_units); do
|
|
[ -n "$ONLY" ] && [ "$svc" != "$ONLY" ] && continue
|
|
if [ -z "$ONLY" ]; then
|
|
in_list "$svc" "${SKIP[@]}" && { log " $svc: frozen (SKIP list)"; continue; }
|
|
in_list "$svc" "${LOCAL_BUILD[@]}" && continue
|
|
fi
|
|
|
|
# ⚠️ NEVER resurrect a service that is deliberately stopped. `systemctl restart`
|
|
# STARTS an inactive unit — without this guard, a new n8n image would silently
|
|
# bring n8n back up months after it was intentionally shut down.
|
|
if [ "$(systemctl --user is-active "podman-$svc.service")" != "active" ]; then
|
|
[ -n "$ONLY" ] && log " $svc: unit is not active — refusing to start it"
|
|
continue
|
|
fi
|
|
|
|
dir="$(workdir_for "$svc")"
|
|
[ -d "$dir" ] || { log " $svc: no working directory, skipped"; continue; }
|
|
|
|
mapfile -t imgs < <(images_for "$dir")
|
|
[ "${#imgs[@]}" -eq 0 ] && continue
|
|
|
|
checked=$((checked+1))
|
|
changed=0
|
|
|
|
for img in "${imgs[@]}"; do
|
|
# A digest-pinned image can never change; pulling it is wasted work.
|
|
case "$img" in *@sha256:*) continue ;; esac
|
|
|
|
if ! podman pull -q "$img" >/dev/null 2>&1; then
|
|
log " $svc: pull failed for $img (private registry? built locally?)"
|
|
continue
|
|
fi
|
|
disk_id="$(podman image inspect "$img" --format '{{.Id}}' 2>/dev/null)"
|
|
[ -n "$disk_id" ] || continue
|
|
|
|
# Compare the image on disk against what each RUNNING CONTAINER is actually
|
|
# using — not against the pre-pull image ID.
|
|
#
|
|
# Comparing before/after the pull looks equivalent but is subtly broken: a
|
|
# --dry-run (or any earlier pull) fetches the new image, so a later real run
|
|
# sees before == after, concludes "no change", and never restarts. The
|
|
# service then runs the OLD image forever with the new one sitting on disk,
|
|
# invisible. Anchoring on the running container makes the check idempotent
|
|
# and self-healing — it reports drift no matter who pulled, or when.
|
|
#
|
|
# ⚠️ Use `inspect .Image`, NOT `ps --format {{.ImageID}}`. The former returns
|
|
# the full 64-char ID matching `image inspect .Id`; the latter returns a
|
|
# 12-char short ID, so comparing them is ALWAYS unequal and every service
|
|
# looks like it needs an update.
|
|
while IFS='|' read -r cname cimgname cimgid; do
|
|
[ "$cimgname" = "$img" ] || continue
|
|
if [ "$cimgid" != "$disk_id" ]; then
|
|
log " $svc: $cname on ${cimgid:0:12} -> new image ${disk_id:0:12} ($img)"
|
|
changed=1
|
|
fi
|
|
done < <(podman ps --filter "label=com.docker.compose.project=$svc" --format '{{.Names}}' 2>/dev/null \
|
|
| while IFS= read -r n; do
|
|
podman inspect "$n" --format '{{.Name}}|{{.ImageName}}|{{.Image}}' 2>/dev/null
|
|
done)
|
|
done
|
|
|
|
[ "$changed" -eq 0 ] && continue
|
|
|
|
if [ "$DRY_RUN" -eq 1 ]; then
|
|
log " $svc: would restart (dry run)"
|
|
updated+=("$svc")
|
|
continue
|
|
fi
|
|
|
|
# THE IMPORTANT LINE: restart the unit so podman-compose rebuilds the project
|
|
# (pod, networks, aliases, ports). Never recreate the container directly.
|
|
if systemctl --user restart "podman-$svc.service" 2>>"$LOG"; then
|
|
sleep 5
|
|
if [ "$(systemctl --user is-active "podman-$svc.service")" = "active" ]; then
|
|
log " $svc: RESTARTED ok"
|
|
updated+=("$svc")
|
|
else
|
|
log " $svc: ⚠ RESTARTED BUT UNIT NOT ACTIVE"
|
|
failed+=("$svc")
|
|
fi
|
|
else
|
|
log " $svc: ⚠ RESTART FAILED"
|
|
failed+=("$svc")
|
|
fi
|
|
done
|
|
|
|
# Reclaim layers the old images left behind. Dangling only — never touches an
|
|
# image a container still references.
|
|
if [ "$DRY_RUN" -eq 0 ] && [ "${#updated[@]}" -gt 0 ]; then
|
|
freed="$(podman image prune -f 2>/dev/null | tail -1)"
|
|
[ -n "$freed" ] && log " pruned: $freed"
|
|
fi
|
|
|
|
log "=== done: $checked checked, ${#updated[@]} updated${updated:+ (${updated[*]})}, ${#failed[@]} failed${failed:+ (${failed[*]})} ==="
|
|
|
|
# Non-zero if anything failed, so the systemd unit shows as failed and
|
|
# `systemctl --user --failed` surfaces it.
|
|
[ "${#failed[@]}" -eq 0 ]
|