#!/usr/bin/env bash
# ─────────────────────────────────────────────────────────────────────────────
# update-containers — pull new images and restart the services that changed.
#
# REPLACES WATCHTOWER. Written 2026-08-17 after watchtower took gitea down for
# three days.
#
# WHY NOT WATCHTOWER: podman-compose puts every project in a POD. Watchtower does
# not update through compose — it builds a replacement container from the image
# plus the old container's config, and that replacement lands OUTSIDE the pod.
# On 2026-08-14 it did exactly that to gitea, reported `failed=0 updated=1`, and
# left it returning 502 until someone noticed three days later.
#
# This script never touches containers directly. It pulls the image and then
# restarts the SYSTEMD UNIT, so podman-compose rebuilds the project the same way
# it does at boot — pod, network aliases, published ports and all.
#
# ─────────────────────────────────────────────────────────────────────────────
# USAGE
#   update-containers              # every service except SKIP (what the timer runs)
#   update-containers gitea        # one service, e.g. from CI after a push
#   update-containers --dry-run    # show what WOULD update, change nothing
#   update-containers --list       # show services and their images, then exit
#
# Runs nightly at 00:00 via podman-update.timer (server/systemd/, enabled by
# setup-server). Log: ~/Server/logs/update-containers.log
# ─────────────────────────────────────────────────────────────────────────────

set -uo pipefail

# ═════════════════════════════════════════════════════════════════════════════
# CONFIG — this is the part you edit
# ═════════════════════════════════════════════════════════════════════════════

# Services that must NEVER update unattended. Add a service here to freeze it.
#
#   postgresql  — restarting it drops all databases at once, and every service
#                 with them. Minor 17.x bumps are safe in principle but not worth
#                 doing at midnight unobserved. Also: the image tag is pinned to
#                 pgvector/pgvector:pg17-trixie for collation reasons — an
#                 unattended change of base could corrupt text indexes.
#   authentik   — runs IRREVERSIBLE database migrations on startup. Rolling back
#                 the image does not roll back the schema; you would need a restore.
#
# Update these two by hand, after a pg_dump:
#   podman pull <image> && systemctl --user restart podman-<name>.service
SKIP=(postgresql authentik watchtower)

# Services built locally rather than pulled. There is no registry image to check,
# so pulling is pointless — they are skipped automatically when no `image:` line
# resolves, but listing them here keeps the log quiet and the intent obvious.
LOCAL_BUILD=(bang completeuphoria sierraandtyler)

# ⚠️ NOTE ON GITEA: it is deliberately NOT in SKIP. The three-day outage was caused
# by watchtower's recreate mechanism, not by the act of updating — restarting the
# unit is the safe path this script uses. But gitea DOES run database migrations on
# version upgrades. If you would rather approve those yourself, move `gitea` into
# SKIP above; nothing else needs changing.

SERVER_DIR="$HOME/Server"
LOG_DIR="$SERVER_DIR/logs"
LOG="$LOG_DIR/update-containers.log"

# ═════════════════════════════════════════════════════════════════════════════

mkdir -p "$LOG_DIR"

DRY_RUN=0
ONLY=""
case "${1:-}" in
    --dry-run) DRY_RUN=1 ;;
    --list)    LIST=1 ;;
    --help|-h) sed -n '19,27p' "$0"; exit 0 ;;
    "")        ;;
    -*)        echo "unknown option: $1" >&2; exit 2 ;;
    *)         ONLY="$1" ;;
esac

log() { printf '%s  %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG"; }

in_list() { local n="$1"; shift; local x; for x in "$@"; do [ "$x" = "$n" ] && return 0; done; return 1; }

# Units that are podman-compose services. Excludes podman's own helper units,
# which are not services and must stay disabled.
list_units() {
    systemctl --user list-unit-files 'podman-*.service' --no-legend 2>/dev/null \
        | awk '{print $1}' \
        | grep -vE 'podman-(auto-update|restart|clean-transient|kube@|user-wait-network-online|update)' \
        | sed 's/^podman-//; s/\.service$//' \
        | sort
}

# Images declared in a service's compose.yml. Ignores commented lines and the
# `build:` stanza, which has no pullable image.
#
# Image tags frequently reference variables from the service's .env, e.g.
#   ghcr.io/get-convex/convex-backend:${BACKEND_TAG:-latest}
#   git.gbrown.org/gib/${NEXT_CONTAINER_NAME}:latest
# Those must be expanded or the pull is attempted against a literal "${...}" and
# always fails. We expand in a SUBSHELL so the .env cannot leak into this script's
# environment (these files contain database passwords and API keys).
images_for() {
    local dir="$1"
    [ -f "$dir/compose.yml" ] || return 0
    grep -oE '^[[:space:]]*image:[[:space:]]*[^[:space:]#]+' "$dir/compose.yml" 2>/dev/null \
        | sed -E 's/^[[:space:]]*image:[[:space:]]*//' \
        | while IFS= read -r img; do
              case "$img" in
                  *'${'*) ( set -a; [ -f "$dir/.env" ] && . "$dir/.env" >/dev/null 2>&1; set +a
                            eval "printf '%s\\n' \"$img\"" 2>/dev/null ) ;;
                  *)      printf '%s\n' "$img" ;;
              esac
          done \
        | grep -v '\${' \
        | sort -u
}

workdir_for() {
    systemctl --user show "podman-$1.service" -p WorkingDirectory --value 2>/dev/null
}

# ── --list ───────────────────────────────────────────────────────────────────
if [ "${LIST:-0}" = "1" ]; then
    for svc in $(list_units); do
        printf '%-22s' "$svc"
        if in_list "$svc" "${SKIP[@]}";        then echo "SKIP (frozen)"; continue; fi
        if in_list "$svc" "${LOCAL_BUILD[@]}"; then echo "SKIP (built locally)"; continue; fi
        echo "$(images_for "$(workdir_for "$svc")" | tr '\n' ' ')"
    done
    exit 0
fi

# ── main ─────────────────────────────────────────────────────────────────────
# ${DRY_RUN:+...} would expand even when DRY_RUN=0, because "0" is a non-empty
# string. Build the label explicitly instead.
dry_label=""; [ "$DRY_RUN" -eq 1 ] && dry_label=" [DRY RUN]"
log "=== update run started${ONLY:+ (single service: $ONLY)}$dry_label ==="

updated=(); failed=(); checked=0

for svc in $(list_units); do
    [ -n "$ONLY" ] && [ "$svc" != "$ONLY" ] && continue
    if [ -z "$ONLY" ]; then
        in_list "$svc" "${SKIP[@]}"        && { log "  $svc: frozen (SKIP list)"; continue; }
        in_list "$svc" "${LOCAL_BUILD[@]}" && continue
    fi

    # ⚠️ NEVER resurrect a service that is deliberately stopped. `systemctl restart`
    # STARTS an inactive unit — without this guard, a new n8n image would silently
    # bring n8n back up months after it was intentionally shut down.
    if [ "$(systemctl --user is-active "podman-$svc.service")" != "active" ]; then
        [ -n "$ONLY" ] && log "  $svc: unit is not active — refusing to start it"
        continue
    fi

    dir="$(workdir_for "$svc")"
    [ -d "$dir" ] || { log "  $svc: no working directory, skipped"; continue; }

    mapfile -t imgs < <(images_for "$dir")
    [ "${#imgs[@]}" -eq 0 ] && continue

    checked=$((checked+1))
    changed=0

    for img in "${imgs[@]}"; do
        # A digest-pinned image can never change; pulling it is wasted work.
        case "$img" in *@sha256:*) continue ;; esac

        if ! podman pull -q "$img" >/dev/null 2>&1; then
            log "  $svc: pull failed for $img (private registry? built locally?)"
            continue
        fi
        disk_id="$(podman image inspect "$img" --format '{{.Id}}' 2>/dev/null)"
        [ -n "$disk_id" ] || continue

        # Compare the image on disk against what each RUNNING CONTAINER is actually
        # using — not against the pre-pull image ID.
        #
        # Comparing before/after the pull looks equivalent but is subtly broken: a
        # --dry-run (or any earlier pull) fetches the new image, so a later real run
        # sees before == after, concludes "no change", and never restarts. The
        # service then runs the OLD image forever with the new one sitting on disk,
        # invisible. Anchoring on the running container makes the check idempotent
        # and self-healing — it reports drift no matter who pulled, or when.
        #
        # ⚠️ Use `inspect .Image`, NOT `ps --format {{.ImageID}}`. The former returns
        # the full 64-char ID matching `image inspect .Id`; the latter returns a
        # 12-char short ID, so comparing them is ALWAYS unequal and every service
        # looks like it needs an update.
        while IFS='|' read -r cname cimgname cimgid; do
            [ "$cimgname" = "$img" ] || continue
            if [ "$cimgid" != "$disk_id" ]; then
                log "  $svc: $cname on ${cimgid:0:12} -> new image ${disk_id:0:12} ($img)"
                changed=1
            fi
        done < <(podman ps --filter "label=com.docker.compose.project=$svc" --format '{{.Names}}' 2>/dev/null \
                 | while IFS= read -r n; do
                       podman inspect "$n" --format '{{.Name}}|{{.ImageName}}|{{.Image}}' 2>/dev/null
                   done)
    done

    [ "$changed" -eq 0 ] && continue

    if [ "$DRY_RUN" -eq 1 ]; then
        log "  $svc: would restart (dry run)"
        updated+=("$svc")
        continue
    fi

    # THE IMPORTANT LINE: restart the unit so podman-compose rebuilds the project
    # (pod, networks, aliases, ports). Never recreate the container directly.
    if systemctl --user restart "podman-$svc.service" 2>>"$LOG"; then
        sleep 5
        if [ "$(systemctl --user is-active "podman-$svc.service")" = "active" ]; then
            log "  $svc: RESTARTED ok"
            updated+=("$svc")
        else
            log "  $svc: ⚠ RESTARTED BUT UNIT NOT ACTIVE"
            failed+=("$svc")
        fi
    else
        log "  $svc: ⚠ RESTART FAILED"
        failed+=("$svc")
    fi
done

# Reclaim layers the old images left behind. Dangling only — never touches an
# image a container still references.
if [ "$DRY_RUN" -eq 0 ] && [ "${#updated[@]}" -gt 0 ]; then
    freed="$(podman image prune -f 2>/dev/null | tail -1)"
    [ -n "$freed" ] && log "  pruned: $freed"
fi

log "=== done: $checked checked, ${#updated[@]} updated${updated:+ (${updated[*]})}, ${#failed[@]} failed${failed:+ (${failed[*]})} ==="

# Non-zero if anything failed, so the systemd unit shows as failed and
# `systemctl --user --failed` surfaces it.
[ "${#failed[@]}" -eq 0 ]