Sync both server snapshots with what the live machines run, and keep their backup tooling

README.md

@@ -250,7 +250,7 @@ docs/           Settings reference, and the design specs behind the work
 
 ## Tests
 
-192 of them, under `tests/`. `tests/contracts.manifest` classifies every
+193 of them, under `tests/`. `tests/contracts.manifest` classifies every
 contract by the capabilities it needs. Run the hermetic set, or grant a
 specific external capability when automation needs it:
 

bin/panama-server

@@ -4,8 +4,10 @@
 #
 # The repository carries the catalog: server/vps/<Name>/ holds a
 # compose.yml, a podman-<name>.service user unit, and an .env.example naming
-# what the service needs told. The machine carries the rest: ~/Server/<Name>/
-# is a real directory owning the .env (secrets never sit inside the checkout,
+# what the service needs told. A service may also carry helper units (any
+# other *.service or *.timer, e.g. a cert refresh) and helper files (scripts,
+# a README) beside them. The machine carries the rest: ~/Server/<Name>/ is a
+# real directory owning the .env (secrets never sit inside the checkout,
 # gitignored or not) and the bind-mounted data. Only the tracked files are
 # symlinked, so a `git clean` in the repo can never reach a database.
 #
@@ -83,12 +85,12 @@ require_service() {
   fi
 }
 
-# The one unit file a service directory carries. Its filename is its identity
-# -- podman-<name>.service, exactly what lands in the systemd user directory --
-# so nothing here invents a name that could drift from the file's.
+# The one compose unit a service directory carries. Its filename is its
+# identity -- podman-<name>.service, exactly what lands in the systemd user
+# directory -- so nothing here invents a name that could drift from the file's.
 unit_path_for() {
   local name="$1" unit
-  for unit in "$SERVER_DIR/$name"/*.service; do
+  for unit in "$SERVER_DIR/$name"/podman-*.service; do
     [[ -e "$unit" ]] || continue
     printf '%s' "$unit"
     return 0
@@ -96,6 +98,44 @@ unit_path_for() {
   return 1
 }
 
+# Every other unit in the directory: helper services and the timers that fire
+# them. They are linked beside the compose unit, and each timer is enabled.
+helper_units_for() {
+  local name="$1" unit
+  for unit in "$SERVER_DIR/$name"/*.service "$SERVER_DIR/$name"/*.timer; do
+    [[ -e "$unit" ]] || continue
+    [[ "$(basename "$unit")" == podman-*.service ]] && continue
+    printf '%s\n' "$unit"
+  done
+}
+
+# The files linked into ~/Server/<Name>/: every visible top-level file that is
+# not a unit. That is compose.yml plus any helper script or README; dotfiles
+# are skipped so .env.example (and a stray .env) never become links.
+live_files_for() {
+  local name="$1" file
+  for file in "$SERVER_DIR/$name"/*; do
+    [[ -f "$file" ]] || continue
+    case "$file" in *.service|*.timer) continue ;; esac
+    printf '%s\n' "$file"
+  done
+}
+
+link_service_files() {
+  local name="$1" file unit
+  mkdir -p "$TARGET_DIR/$name" "$UNIT_DIR"
+  while IFS= read -r file; do
+    link_file "$file" "$TARGET_DIR/$name/$(basename "$file")"
+  done < <(live_files_for "$name")
+  while IFS= read -r unit; do
+    link_file "$unit" "$UNIT_DIR/$(basename "$unit")"
+  done < <(helper_units_for "$name")
+}
+
+helper_timers_for() {
+  helper_units_for "$1" | grep '\.timer$' | xargs -r -n1 basename
+}
+
 unit_name_for() {
   local unit
   unit="$(unit_path_for "$1")" || return 1
@@ -111,10 +151,12 @@ unit_installed() {
   [[ -L "$UNIT_DIR/$unit_name" ]]
 }
 
+# Everything this machine links for the service, so a change to a helper
+# script or timer is reported by relink just like a compose change.
 definition_hash() {
-  local name="$1" unit
-  unit="$(unit_path_for "$name")" || unit=/dev/null
-  cat "$SERVER_DIR/$name/compose.yml" "$unit" 2>/dev/null | sha256sum | cut -d' ' -f1
+  local name="$1"
+  { live_files_for "$name"; unit_path_for "$name" && echo; helper_units_for "$name"; } \
+    | sort | xargs -r -d '\n' cat 2>/dev/null | sha256sum | cut -d' ' -f1
 }
 
 recorded_hash() {
@@ -198,6 +240,7 @@ cmd_enable() {
   fi
 
   mkdir -p "$UNIT_DIR"
+  link_service_files "$name"
   link_file "$unit_path" "$UNIT_DIR/$unit_name"
   systemctl --user daemon-reload
   if systemctl --user enable --now "$unit_name"; then
@@ -207,6 +250,10 @@ cmd_enable() {
     err "$unit_name did not start; see: systemctl --user status $unit_name"
     exit 1
   fi
+  local timer
+  for timer in $(helper_timers_for "$name"); do
+    systemctl --user enable --now "$timer" && ok "Enabled $timer"
+  done
 }
 
 cmd_disable() {
@@ -216,11 +263,15 @@ cmd_disable() {
   local unit_name
   unit_name="$(unit_name_for "$name")" || { err "$name has no unit file in the catalog"; exit 1; }
 
+  local timer unit
+  for timer in $(helper_timers_for "$name"); do
+    systemctl --user disable --now "$timer" 2>/dev/null || true
+  done
   systemctl --user disable --now "$unit_name" 2>/dev/null || true
-  if [[ -L "$UNIT_DIR/$unit_name" ]]; then
-    rm "$UNIT_DIR/$unit_name"
+  for unit in "$UNIT_DIR/$unit_name" $(helper_units_for "$name" | xargs -r -n1 basename | sed "s|^|$UNIT_DIR/|"); do
+    [[ -L "$unit" ]] && rm "$unit"
+  done
   systemctl --user daemon-reload
-  fi
   ok "$name disabled. ~/Server/$name, its .env and its data were not touched."
 }
 
@@ -267,8 +318,7 @@ cmd_relink() {
     unit_installed "$name" || continue
 
     link_file "$unit_path" "$UNIT_DIR/$unit_name"
-    mkdir -p "$TARGET_DIR/$name"
-    link_file "$SERVER_DIR/$name/compose.yml" "$TARGET_DIR/$name/compose.yml"
+    link_service_files "$name"
     refreshed=$((refreshed + 1))
 
     # Changed since the last time this machine looked, which is what makes a

server/README.md

@@ -8,7 +8,9 @@ screen, plus rootless podman compose services managed as systemd user units.
 server/
   vps/            The catalog panama-server installs from, one directory
                   per service: compose.yml, the podman-<name>.service user
-                  unit, and an .env.example naming what the service needs told
+                  unit, and an .env.example naming what the service needs told.
+                  Loose files beside the directories are the VPS's own
+                  host tooling (see "Host tooling" below)
   home/           The home server's services in the same shape (see below)
   scripts/        update-containers, the nightly image updater
   systemd/        The units behind it, linked by setup-server
@@ -25,6 +27,8 @@ installable: panama-server reads `server/vps/` only, so a service moves into
 the catalog when it actually moves to podman.
 
 Paths keep their nesting (`server/home/Agents/Gitea` is `~/Server/Agents/Gitea`).
+`Media/compact-tv/` is the nightly TV re-encode job that runs beside the Media
+stack, with its own README.
 Three services are checkouts of upstream repositories. For those, only the
 local files are kept, and an `UPSTREAM` file names the repository and the exact
 commit the checkout sits at: `PostHog`, `Sentry` and `Plausible`.
@@ -38,7 +42,8 @@ compose and its copy here can differ on those lines alone.
 
 The repository carries the definitions; the machine carries the state.
 `panama server enable <Name>` creates `~/Server/<Name>/` as a **real
-directory** and symlinks only `compose.yml` into it; the unit symlinks into
+directory** and symlinks the service's visible top-level files into it:
+`compose.yml`, plus any helper script or README. The units symlink into
 `~/.config/systemd/user/`. The `.env` (seeded once from `.env.example`, then
 yours) and the bind-mounted `data/` live in `~/Server/<Name>/` and are never
 inside the checkout — so nothing git can do, `clean -fdx` included, can reach
@@ -99,6 +104,39 @@ existing service. The unit is the standard oneshot `podman compose up -d`
 shape (`WorkingDirectory=%h/Server/<Name>`); the shape contract pins the
 details. Secrets go in `.env.example` as `KEY=CHANGE_ME` lines.
 
+A service that needs a scheduled job of its own carries it beside the
+compose unit: a helper `<job>.service` and a `<job>.timer` with the same
+name, plus the script it runs. Enable links all three and enables the timer.
+Disable stops the timer and unlinks the units. `UseSend` is the example: its
+`usesend-cert-refresh` timer keeps the SMTP proxy's certificate in step with
+Nginx Proxy Manager's.
+
+## Host tooling
+
+Each server's own scripts and runbooks sit as loose files at the top of its
+tree. They are reference copies for rebuilding a machine. Nothing installs
+them.
+
+- `vps/backup`, `vps/server-backup.service`, `vps/server-backup.timer`: the
+  VPS's nightly backup, run as a user unit. It dumps every database into
+  `~/Server/_backup_state/`, then rsyncs `~/Server` to the home server's ZFS
+  pool with `--fake-super`, so subuid ownership survives the trip.
+- `vps/verify-rootless`: the post-reboot health check for the rootless
+  stack. It covers linger, the unprivileged port floor, unit state and boot
+  enablement, postgres, endpoints, and admin ports bound to the WireGuard
+  address.
+- `vps/AGENTS.md`: the VPS runbook, with both public IPs redacted.
+- `home/RESTORE.md`: how to restore the home server's `~/Server` from the
+  ZFS backup.
+- `home/backup` and `home/systemd/`: the home server's backup trigger and its
+  root system units (`server-backup` nightly, `media-backup` weekly). The
+  scripts those units run live in `/usr/local/sbin` on that machine. They are
+  readable by root alone, so this tree has no copy of them.
+
+The VPS units carry `RequiresMountsFor=/home/gib/Media` because that host
+keeps its podman graphroot on an attached volume mounted there. The setting
+is harmless on a machine without the mount.
+
 ## Nightly updates
 
 `server/scripts/update-containers` pulls images and restarts changed

server/home/Agents/Gitea/compose.yml

@@ -19,7 +19,7 @@ x-runner-env: &runner-env
   TZ: America/New_York
   GITEA_INSTANCE_URL: https://git.gbrown.org
   GITEA_RUNNER_REGISTRATION_TOKEN: ${GITEA_RUNNER_REGISTRATION_TOKEN}
-  GITEA_RUNNER_LABELS: "ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest,ubuntu-24.04:docker://docker.gitea.com/runner-images:ubuntu-24.04,ubuntu-22.04:docker://docker.gitea.com/runner-images:ubuntu-22.04"
+  GITEA_RUNNER_LABELS: "ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest,ubuntu-24.04:docker://docker.gitea.com/runner-images:ubuntu-24.04,ubuntu-22.04:docker://docker.gitea.com/runner-images:ubuntu-22.04,build:docker://docker.gitea.com/runner-images:ubuntu-latest"
   DOCKER_TLS_VERIFY: "1"
   DOCKER_CERT_PATH: /certs/client
   CONFIG_FILE: /data/config.yml

server/home/Agents/Portainer/compose.yml

@@ -18,7 +18,7 @@ services:
   #   restart: unless-stopped
 
   portainer-agent:
-    image: portainer/agent:2.33.2
+    image: portainer/agent:latest
     container_name: portainer-agent
     hostname: portainer-agent
     networks:

server/home/Gbrown/compose.yml

@@ -43,8 +43,8 @@ volumes:
 
 services:
   gbrown-backend:
-    # Pinned to the build the VPS Convex stacks already run. Bump on purpose.
-    image: ghcr.io/get-convex/convex-backend@sha256:d715e9ec088784407ca4ba2d3db592702cd328d02c76cdca3852c0018f2a76b4
+    # Pinned to the build the VPS Convex stacks run (bumped 2026-10-28). Bump on purpose.
+    image: ghcr.io/convex/convex-backend@sha256:207cf95def42f01614b6e3af0af5c20d0dcaa5ff2adf258ee7bc9555576644fa
     container_name: gbrown-backend
     hostname: gbrown-backend
     networks: ['nginx-bridge']
@@ -70,7 +70,7 @@ services:
 
   gbrown-dashboard:
     # Released alongside the backend; pinned with it.
-    image: ghcr.io/get-convex/convex-dashboard@sha256:f85cf0d0448b9c835ae3df5c7c1c6f0145dd4c8f0f92eb0245daf304efdd9f1a
+    image: ghcr.io/convex/convex-dashboard@sha256:a55bf8abca1332f787bc4d4a79850182175027be7b70aa230be4389a1d3268e2
     container_name: gbrown-dashboard
     hostname: gbrown-dashboard
     networks: ['nginx-bridge']

server/home/Media/compact-tv/README.md

@@ -0,0 +1,109 @@
+# compact-tv
+
+Sonarr and Radarr each have exactly four quality profiles. Two of them (**Standard** and
+**Mini**) also have their files shrunk by `compact-tv.py`, which runs every night.
+
+| Profile | Used for | What gets kept |
+|---|---|---|
+| **Mini** | shows I demote hard | TV: 720p at 800 kbps (~0.42 GB per hour). Movies: 720p at 1500 kbps |
+| **Standard** | **every request from other users (Seerr default)**, and anything I demote | TV: 1080p at 1200 kbps (~0.6 GB per hour). Movies: 1080p at 2500 kbps (~1.2 GB per hour) |
+| **HD** | my own library, unchanged | whatever downloads, up to 1080p |
+| **UHD** | only me, through Seerr's 4K servers | 4K |
+
+SD sources keep their resolution, at 600 kbps for TV and 900 kbps for movies. Movies get
+higher rates because dark scenes, grain and action break down first at low bitrates. Nothing is ever
+upscaled. Audio becomes Opus: 96 kbps for stereo, 160 for 5.1, 192 for 7.1. Subtitles,
+chapters and attachments are copied as they are. I picked the rates from sample encodes of
+Shameless S05E06 (VMAF ~91 for Standard, ~86 for Mini).
+
+## How the profiles work
+
+- **Standard and Mini** are copies of HD that upgrade only toward small releases (see below),
+  so Sonarr and Radarr never replace a file with a bigger one.
+- **Small downloads first.** Both profiles prefer small releases when they grab:
+  - The "Small Encode" format (MeGusta, PSA, ELiTE, GalaxyTV/RG, YTS/YIFY) scores +3000.
+  - LQ scores 0. In HD and UHD it scores -10000, which keeps those groups out of HD.
+  - Standard merges HDTV/WEB/Bluray at each resolution, so the format score decides between
+    sources rather than the source type.
+  - Mini merges 720p and 1080p into one group and adds +1000 for 720p.
+  - Many files therefore arrive already small, and the job skips them.
+- **Minimum sizes are shared by every profile.** For 720p I lowered them to 5 MB per minute
+  and for 1080p to 6 MB per minute (8 for Bluray-1080p), so small encodes are not rejected.
+  HD still never takes the LQ groups.
+- **Seerr:**
+  - The "Sonarr - HD" and "Radarr" servers default to Standard. Only my admin account sees
+    the profile picker, so other users always get Standard. I pick HD or UHD per request.
+  - Other users no longer have "Request 4K".
+- **Configarr is opt-in** (`docker compose --profile configarr up configarr`). Its templates
+  would recreate the old TRaSH profiles and size limits.
+- Backups from before all this: `backups/pre-four-profiles-20261028/`. That covers both
+  apps' profiles, custom formats and size definitions, which title was on which profile,
+  and Seerr's settings. `setup-small-profiles.py` reapplies the Standard and Mini settings.
+
+## How the job works
+
+The nightly run (00:30 to 08:00) has two phases.
+
+**1. Search first.** It asks Sonarr and Radarr to search titles on Standard or Mini that it
+hasn't searched before, biggest first: 5 shows and 20 movies a night, so downloads and
+indexer use stay spread out. Standard and Mini upgrade only toward small releases:
+- Every allowed quality sits in one group, so no resolution or source upgrade can happen.
+- An upgrade needs a custom-format score jump of at least 2500, and upgrades stop at 3000.
+  Only gaining "Small Encode" clears that bar.
+
+So a search replaces a big file only with a small encode, never with a big HEVC release.
+After that, the apps' regular RSS checks keep picking up small releases on their own.
+
+**2. Encode what's left.** A title becomes eligible for encoding 7 days after its search,
+and only its files that are still big get encoded. Files that arrive after the title's
+search work differently:
+- This covers new episodes of ongoing shows, and fresh requests.
+- The first night they're seen, they get their own search (one episode or movie at a time).
+- They're encoded only if they're still big 3 days later.
+
+Downloads never wait for any of this, so new episodes and requests land in the library as
+soon as they download. Only encoding waits. For each file, the job encodes to
+`~/.cache/compact-tv/work.mkv` on the NVMe and then checks the result:
+- the duration matches the original
+- the audio and subtitle track counts match
+- the file is smaller
+
+It then renames the new file over the original, keeping the same name with `.mkv`, and asks
+Sonarr or Radarr to rescan. Plex picks the change up through its folder watcher, and the
+episode keeps its Plex ID, so watch history stays. If a small release shows up later, the
+app may still replace an encoded file with it, which costs bandwidth, not GPU.
+
+Other details:
+- Each output carries a `COMPACT_TIER` tag. A file is skipped if re-encoding would save less
+  than 30%. Ruled-out files are cached by size in `state.json`, so they aren't probed every
+  night. Search dates and failures are kept there too.
+- A downloaded HEVC file no bigger than twice the target size counts as done. That is the
+  small release a search brought in (a MeGusta 1080p episode runs about 2.3 Mbps against
+  Standard's 1.2), and it is neither re-encoded nor searched again. The job's own tagged
+  outputs are exempt, so Standard to Mini still works.
+- **Skipped:** HDR and Dolby Vision sources (they need tone mapping), subtitle tracks ffprobe
+  can't identify, and files that have failed twice.
+- Speed is about 0.19 hours of GPU per hour of 1080p video (NVENC is the bottleneck). The job
+  stops starting new encodes at 08:00.
+- `cost-report.py` estimates, per title, the GPU hours the job would need against the size of
+  a fresh small download.
+- A `PAUSE` file in this folder makes the nightly run do nothing.
+
+## Going back up
+
+Moving down works: Standard to Mini re-encodes the Standard copy. Moving up does not
+restore quality. To get quality back, I set the title to HD and search, which redownloads
+it at full size.
+
+## Running it
+
+    ./compact-tv.py --dry-run                      # what tonight would do
+    ./compact-tv.py --search-all --limit 0         # search every unsearched title now, encode nothing
+    touch PAUSE                                    # skip nightly runs until PAUSE is deleted
+    ./compact-tv.py --series "Shameless (US)" --limit 1
+    systemctl status compact-tv.timer compact-tv.service
+    tail -f compact-tv.log
+
+The installed units are copies of `compact-tv.service` and `compact-tv.timer` from this
+folder. `ExecStart` goes through `/usr/bin/python3` because SELinux won't let systemd
+execute a file labeled as home content.

server/home/Media/compact-tv/compact-tv.py

@@ -0,0 +1,348 @@
+#!/usr/bin/env python3
+"""Shrink shows and movies that Sonarr/Radarr have on the "Standard" or "Mini" profile.
+
+Demoting something is just picking one of those profiles. Each night this job first asks
+Sonarr/Radarr to search a few titles it hasn't searched before. Both profiles upgrade only
+toward small releases, so the search swaps whatever big files have a small release for
+bandwidth instead of GPU. A week after a title's search, the job re-encodes whatever is
+still big to HEVC on the GPU (NVENC) at the tier's bitrate, verifies the result, swaps it in
+place of the original, and has Sonarr/Radarr rescan. Plex and Jellyfin pick up changes
+through their own folder watchers.
+
+Moving something back up means switching it to HD and searching, which redownloads it at
+full size. See README.md.
+
+Usage:
+  compact-tv.py [--until HH:MM] [--series TITLE] [--limit N] [--dry-run]
+
+--series matches a show or movie title. Runs as gib (files are gib-owned) from
+compact-tv.service, nightly.
+"""
+import argparse, datetime, fcntl, json, os, pathlib, re, shutil, subprocess, sys, time, urllib.request
+
+HERE = pathlib.Path(__file__).resolve().parent
+STATE = HERE / "state.json"
+LOG = HERE / "compact-tv.log"
+WORK = pathlib.Path.home() / ".cache" / "compact-tv"   # NVMe, so half-done encodes never land in a ZFS snapshot
+VOLUMES = pathlib.Path.home() / "Server/Media/volumes"
+
+# Per app: API port, container path -> host path, and how to list one title's files.
+APPS = {
+    "sonarr": {"port": 8989, "paths": ("/tv/", "/home/gib/Public/TV/"), "titles": "/series",
+               "files": lambda call, t: call(f"/episodefile?seriesId={t['id']}"),
+               "rescan": lambda t: {"name": "RescanSeries", "seriesId": t["id"]}},
+    "radarr": {"port": 7878, "paths": ("/movies/", "/home/gib/Public/Movies/"), "titles": "/movie",
+               "files": lambda call, t: [t["movieFile"]] if t.get("movieFile") else [],
+               "rescan": lambda t: {"name": "RescanMovie", "movieId": t["id"]}},
+}
+
+# Video bitrate (kbps) per output class, and the widest output each tier allows.
+# TV floors come from watching samples: 1200 kbps at 1080p, 800 at 720p, 600 at SD.
+# Movies get roughly double: dark scenes, grain and action break down first at low rates,
+# and a 2-hour movie at 2500 kbps is still ~2.3 GB (a quarter of a typical 1080p file).
+# That also leaves already-small releases (YTS-style, ~1.5-2 GB) under the 30% bar.
+# Nothing is ever upscaled: a 720p or SD source keeps its size and uses that class's rate.
+TIERS = {
+    "Standard": {"max_width": 1920, "kbps": {"sonarr": {"1080": 1200, "720": 1200, "sd": 600},
+                                             "radarr": {"1080": 2500, "720": 1800, "sd": 900}}},
+    "Mini":     {"max_width": 1280, "kbps": {"sonarr": {"1080": 800,  "720": 800,  "sd": 600},
+                                             "radarr": {"1080": 1500, "720": 1500, "sd": 900}}},
+}
+MIN_SAVING = 0.30            # leave a file alone unless re-encoding saves at least this much
+MAX_FAILURES = 2             # after this many failed attempts a file is skipped for good
+
+# Search first, encode later. Standard/Mini upgrade only toward small releases (see
+# setup-small-profiles.py), so asking the *arr to search a title swaps whatever big files
+# have a small release, for bandwidth instead of GPU. Each night searches a few titles not
+# searched before, biggest first, so downloads and indexer use stay spread out. A title's
+# files are only encoded once its search is SEARCH_GRACE_DAYS old.
+#
+# Files that arrive after their title's search (new episodes, fresh requests) were never
+# searched for a small copy, so the job searches that episode or movie on its own first,
+# and encodes it only if it is still big FILE_SEARCH_GRACE_DAYS later. Downloads are never
+# delayed by any of this; only encoding waits.
+SEARCHES_PER_NIGHT = {"sonarr": 5, "radarr": 20}
+SEARCH_GRACE_DAYS = 7
+FILE_SEARCH_GRACE_DAYS = 3        # small encodes usually appear within hours to a day; 3 leaves room
+HDR_TRANSFERS = {"smpte2084", "arib-std-b67"}
+
+
+def log(msg):
+    line = f"{datetime.datetime.now():%Y-%m-%d %H:%M:%S}  {msg}"
+    print(line, flush=True)
+    with open(LOG, "a") as f:
+        f.write(line + "\n")
+
+
+# ── Sonarr / Radarr ─────────────────────────────────────────────────────────
+def arr(app):
+    """Return a call(path, body=None) for the app's v3 API."""
+    key = re.search(r"<ApiKey>([^<]+)", (VOLUMES / app / "config.xml").read_text()).group(1)
+    ip = subprocess.run(["docker", "inspect", app, "--format",
+                         "{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}"],
+                        capture_output=True, text=True, check=True).stdout.split()[0]
+    base = f"http://{ip}:{APPS[app]['port']}/api/v3"
+
+    def call(path, body=None):
+        req = urllib.request.Request(base + path, headers={"X-Api-Key": key, "Content-Type": "application/json"},
+                                     data=json.dumps(body).encode() if body is not None else None)
+        with urllib.request.urlopen(req, timeout=60) as r:
+            return json.load(r)
+    return call
+
+
+# ── probing and planning ────────────────────────────────────────────────────
+def probe(path):
+    out = subprocess.run(["ffprobe", "-v", "error", "-print_format", "json", "-show_format", "-show_streams", str(path)],
+                         capture_output=True, text=True)
+    return json.loads(out.stdout) if out.returncode == 0 else None
+
+
+def subtitles(info):
+    """Subtitle tracks ffmpeg can carry over. Tracks ffprobe can't identify are dropped."""
+    return [s for s in info["streams"] if s["codec_type"] == "subtitle" and s.get("codec_name")]
+
+
+def audio_kbps(channels):
+    return 96 if channels <= 2 else 160 if channels <= 6 else 192
+
+
+def plan(info, tier, app):
+    """Return (ffmpeg args, estimated output bytes) for one file, or (None, reason) to skip it."""
+    t = TIERS[tier]
+    video = next((s for s in info["streams"] if s["codec_type"] == "video"
+                  and not s.get("disposition", {}).get("attached_pic")), None)
+    if not video:
+        return None, "no video stream"
+    if video.get("color_transfer") in HDR_TRANSFERS or any("Dolby Vision" in str(sd) for sd in video.get("side_data_list", [])):
+        return None, "HDR/Dolby Vision source (not handled yet)"
+    if (info["format"].get("tags") or {}).get("COMPACT_TIER") == tier:
+        return None, f"already {tier}"
+
+    src_w = int(video["width"])
+    out_w = min(src_w, t["max_width"])
+    cls = "1080" if out_w >= 1700 else "720" if out_w >= 1200 else "sd"
+    vkbps = t["kbps"][app][cls]
+    duration = float(info["format"]["duration"])
+    audios = [s for s in info["streams"] if s["codec_type"] == "audio"]
+    est = int((vkbps + sum(audio_kbps(int(a.get("channels") or 2)) for a in audios)) * 1000 / 8 * duration * 1.03)
+    if int(info["format"]["size"]) * (1 - MIN_SAVING) < est:
+        return None, f"already small ({int(info['format']['size']) / 1e9 * 3600 / duration:.2f} GB/h)"
+    # A downloaded HEVC file within 2x the target is a small release that already replaced a
+    # big one. Re-encoding it would cost GPU and quality for little space. Our own outputs
+    # (tagged) still go through, so moving a title from Standard to Mini works.
+    tags = info["format"].get("tags") or {}
+    if video.get("codec_name") == "hevc" and "COMPACT_TIER" not in tags and int(info["format"]["size"]) <= 2 * est:
+        return None, f"small HEVC release ({int(info['format']['size']) / 1e9 * 3600 / duration:.2f} GB/h)"
+
+    vf = []
+    if video.get("field_order") not in (None, "progressive", "unknown"):
+        vf.append("bwdif")
+    if out_w < src_w:
+        vf.append(f"scale={out_w}:-2:flags=lanczos")
+    args = ["-map", f"0:{video['index']}"]
+    if vf:
+        args += ["-filter:v", ",".join(vf)]
+    args += ["-c:v", "hevc_nvenc", "-preset", "p7", "-tune", "hq", "-profile:v", "main10", "-pix_fmt", "p010le",
+             "-rc", "vbr", "-multipass", "fullres", "-spatial-aq", "1", "-temporal-aq", "1", "-rc-lookahead", "32",
+             "-bf", "4", "-b_ref_mode", "middle",
+             "-b:v", f"{vkbps}k", "-maxrate", f"{vkbps * 2}k", "-bufsize", f"{vkbps * 4}k"]
+    for i, a in enumerate(audios):
+        ch = int(a.get("channels") or 2)
+        layout = {1: "mono", 2: "stereo", 6: "5.1", 8: "7.1"}.get(ch)
+        args += ["-map", f"0:{a['index']}", f"-c:a:{i}", "libopus", f"-b:a:{i}", f"{audio_kbps(ch)}k",
+                 f"-filter:a:{i}", f"aformat=channel_layouts={layout or 'stereo'}"]
+    args += ["-mapping_family", "1"]
+    for i, s in enumerate(subtitles(info)):
+        args += ["-map", f"0:{s['index']}", f"-c:s:{i}", "srt" if s["codec_name"] == "mov_text" else "copy"]
+    args += ["-map", "0:t?", "-c:t", "copy", "-metadata", f"COMPACT_TIER={tier}"]
+    return args, est
+
+
+# ── encode, verify, swap ────────────────────────────────────────────────────
+def encode(src, args, tier):
+    WORK.mkdir(parents=True, exist_ok=True)
+    tmp = WORK / "work.mkv"
+    tmp.unlink(missing_ok=True)
+    cmd = ["ffmpeg", "-hide_banner", "-loglevel", "error", "-nostdin", "-y", "-hwaccel", "cuda", "-i", str(src),
+           *args, "-metadata", f"COMPACT_SOURCE={src.name} ({src.stat().st_size} bytes)", str(tmp)]
+    r = subprocess.run(cmd, capture_output=True, text=True)
+    if r.returncode != 0:
+        tmp.unlink(missing_ok=True)
+        raise RuntimeError("ffmpeg failed: " + r.stderr.strip()[-300:])
+    return tmp
+
+
+def verify(src_info, tmp):
+    new = probe(tmp)
+    if not new:
+        raise RuntimeError("output does not probe")
+    a, b = float(src_info["format"]["duration"]), float(new["format"]["duration"])
+    if abs(a - b) > max(2.0, a * 0.005):
+        raise RuntimeError(f"duration mismatch {a:.1f}s vs {b:.1f}s")
+    counts = lambda i: (sum(1 for s in i["streams"] if s["codec_type"] == "audio"), len(subtitles(i)))
+    if counts(src_info) != counts(new):
+        raise RuntimeError(f"audio/subtitle track counts {counts(src_info)} -> {counts(new)}")
+    if int(new["format"]["size"]) >= int(src_info["format"]["size"]):
+        raise RuntimeError("output is not smaller")
+    return int(new["format"]["size"])
+
+
+def swap(src, tmp):
+    """Copy the encode next to the original, then rename it over the original in one step."""
+    final = src.with_suffix(".mkv")
+    staged = src.with_name(f".{final.name}.compact-tmp")
+    shutil.copyfile(tmp, staged)
+    os.replace(staged, final)
+    if final != src:
+        src.unlink()
+    tmp.unlink(missing_ok=True)
+    return final
+
+
+# ── main ────────────────────────────────────────────────────────────────────
+def main():
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--until", help="stop starting new encodes after this local time (HH:MM)")
+    ap.add_argument("--series", help="only this series title")
+    ap.add_argument("--limit", type=int, help="encode at most N files")
+    ap.add_argument("--dry-run", action="store_true", help="plan only, change nothing")
+    ap.add_argument("--search-all", action="store_true",
+                    help="search every not-yet-searched title now instead of a few per night")
+    opt = ap.parse_args()
+
+    if (HERE / "PAUSE").exists() and not opt.dry_run:
+        log("PAUSE file present, not encoding anything (delete it to resume)")
+        return
+
+    lock = open(HERE / ".lock", "w")
+    try:
+        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except BlockingIOError:
+        sys.exit("another compact-tv run is active")
+
+    deadline = None
+    if opt.until:
+        h, m = map(int, opt.until.split(":"))
+        now = datetime.datetime.now()
+        deadline = now.replace(hour=h, minute=m, second=0, microsecond=0)
+        if deadline <= now:
+            deadline += datetime.timedelta(days=1)
+
+    state = json.loads(STATE.read_text()) if STATE.exists() else {}
+    for k in ("failures", "searched", "skip"):
+        state.setdefault(k, {})
+    save = lambda: STATE.write_text(json.dumps(state, indent=1))
+    today = datetime.date.today()
+    work = []                                  # (app, call, title, tier); TV first, then movies
+    for app, a in APPS.items():
+        call = arr(app)
+        profiles = {p["id"]: p["name"] for p in call("/qualityprofile")}
+        work += [(app, call, t, profiles[t["qualityProfileId"]]) for t in call(a["titles"])
+                 if profiles.get(t["qualityProfileId"]) in TIERS and (not opt.series or t["title"] == opt.series)]
+    log(f"=== run start{' [dry run]' if opt.dry_run else ''}: {len(work)} title(s) on "
+        f"{'/'.join(TIERS)}{f', until {deadline:%H:%M}' if deadline else ''} ===")
+
+    # ── 1. search: ask the *arr for small replacements before spending GPU ──
+    size_of = lambda t: (t.get("statistics") or {}).get("sizeOnDisk") or t.get("sizeOnDisk") or 0
+    for app in APPS:
+        fresh = sorted((w for w in work if w[0] == app and f"{app}:{w[2]['id']}" not in state["searched"]
+                        and size_of(w[2]) > 0), key=lambda w: -size_of(w[2]))
+        fresh = fresh if opt.search_all else fresh[:SEARCHES_PER_NIGHT[app]]
+        for _, call, t, tier in fresh:
+            log(f"  {'would search' if opt.dry_run else 'search'} {t['title']} ({size_of(t) / 1e9:.0f} GB, {tier})")
+        if fresh and not opt.dry_run:
+            call = fresh[0][1]
+            if app == "sonarr":
+                for _, _, t, _ in fresh:
+                    call("/command", {"name": "SeriesSearch", "seriesId": t["id"]})
+            else:
+                call("/command", {"name": "MoviesSearch", "movieIds": [t["id"] for _, _, t, _ in fresh]})
+            for _, _, t, _ in fresh:
+                state["searched"][f"{app}:{t['id']}"] = today.isoformat()
+            save()
+
+    # ── 2. encode what is still big after its search ────────────────────────
+    age = lambda iso: (today - datetime.date.fromisoformat(iso)).days
+
+    def ready(app, ef, title_searched, to_search):
+        """True if this big file has had its chance to be replaced by a small download."""
+        if (ef.get("dateAdded") or "")[:10] <= title_searched:   # was on disk when the title was searched
+            return age(title_searched) >= SEARCH_GRACE_DAYS
+        key = f"{app}:file:{ef['id']}"                            # arrived later: search it on its own first
+        if key not in state["searched"]:
+            to_search.append(ef)
+            return False
+        return age(state["searched"][key]) >= FILE_SEARCH_GRACE_DAYS
+
+    def search_files(app, call, show, files):
+        if opt.dry_run:
+            log(f"  would search {len(files)} newer file(s) of {show['title']}"); return
+        if app == "sonarr":
+            ids = {f["id"] for f in files}
+            eps = [e["id"] for e in call(f"/episode?seriesId={show['id']}") if e.get("episodeFileId") in ids]
+            call("/command", {"name": "EpisodeSearch", "episodeIds": eps})
+        else:
+            call("/command", {"name": "MoviesSearch", "movieIds": [show["id"]]})
+        for f in files:
+            state["searched"][f"{app}:file:{f['id']}"] = today.isoformat()
+        log(f"  search {len(files)} newer file(s) of {show['title']} before encoding"); save()
+
+    done = saved = 0
+    for app, call, show, tier in work:
+        title_searched = state["searched"].get(f"{app}:{show['id']}")
+        if not title_searched:
+            continue                          # its title search comes first, on a later night
+        container, host = APPS[app]["paths"]
+        files = sorted(APPS[app]["files"](call, show), key=lambda f: f["path"])
+        changed, to_search = False, []
+        for ef in files:
+            if deadline and datetime.datetime.now() >= deadline:
+                log("deadline reached, stopping"); break
+            if opt.limit is not None and done >= opt.limit:
+                break
+            src = pathlib.Path(ef["path"].replace(container, host, 1))
+            if state["failures"].get(str(src), 0) >= MAX_FAILURES or not src.exists():
+                continue
+            # Files already ruled out at this tier and size need no re-probe (most of them, most nights).
+            seen = f"{tier}:{src.stat().st_size}"
+            if state["skip"].get(str(src)) == seen:
+                continue
+            info = probe(src)
+            if not info:
+                log(f"  SKIP {src.name}: does not probe"); continue
+            args, est = plan(info, tier, app)
+            if args is None:
+                state["skip"][str(src)] = seen  # reason in `est`; quiet, since most nights most files are done
+                continue
+            if not ready(app, ef, title_searched, to_search):
+                continue
+            old = int(info["format"]["size"])
+            if opt.dry_run:
+                log(f"  would encode {src.name}: {old / 1e6:.0f} MB -> ~{est / 1e6:.0f} MB ({tier})")
+                done += 1; continue
+            t0 = time.monotonic()
+            try:
+                tmp = encode(src, args, tier)
+                new = verify(info, tmp)
+                final = swap(src, tmp)
+            except Exception as e:
+                state["failures"][str(src)] = state["failures"].get(str(src), 0) + 1
+                save()
+                log(f"  FAIL {src.name}: {e}"); continue
+            done += 1; saved += old - new; changed = True
+            log(f"  {final.name}: {old / 1e6:.0f} MB -> {new / 1e6:.0f} MB "
+                f"(-{(1 - new / old) * 100:.0f}%, {time.monotonic() - t0:.0f}s, {tier})")
+        if to_search:
+            search_files(app, call, show, to_search)
+        if changed:
+            call("/command", APPS[app]["rescan"](show))
+        if (deadline and datetime.datetime.now() >= deadline) or (opt.limit is not None and done >= opt.limit):
+            break
+    if not opt.dry_run:
+        save()
+    log(f"=== run end: {done} file(s){' planned' if opt.dry_run else ''}, saved {saved / 1e9:.1f} GB ===")
+
+
+if __name__ == "__main__":
+    main()

server/home/Media/compact-tv/compact-tv.service

@@ -0,0 +1,17 @@
+# Installed to /etc/systemd/system/. Runs as gib because the TV files are gib-owned,
+# so the job needs no root. A system unit (rather than a user unit) because gib has
+# no lingering session on this headless box. ExecStart goes through python3 because
+# SELinux will not let systemd execute a file labeled as home content (user_home_t).
+[Unit]
+Description=Re-encode TV shows on Sonarr's Compact/Supercompact profiles
+Documentation=file:///home/gib/Server/Media/compact-tv/README.md
+After=docker.service
+Requires=docker.service
+
+[Service]
+Type=oneshot
+User=gib
+Group=gib
+ExecStart=/usr/bin/python3 /home/gib/Server/Media/compact-tv/compact-tv.py --until 08:00
+Nice=15
+IOSchedulingClass=idle

server/home/Media/compact-tv/compact-tv.timer

@@ -0,0 +1,11 @@
+# Starts after the 00:00 hour and stops starting new encodes at 08:00 (see --until in
+# the service), so Plex and Frigate keep the GPU's encoder during the day.
+[Unit]
+Description=Nightly TV re-encode for Compact/Supercompact shows
+
+[Timer]
+OnCalendar=*-*-* 00:30:00
+Persistent=false
+
+[Install]
+WantedBy=timers.target

server/home/Media/compact-tv/cost-report.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Per-title cost of shrinking everything on Standard/Mini: re-encode vs redownload.
+
+For each show or movie on those profiles, probes every file with compact-tv's own planner
+and reports what the nightly job would do (encode, skip as already small, skip as HDR),
+the space it would save, and the GPU time it would take. Alongside that it estimates what a
+fresh small download would cost instead, using typical small-encode sizes.
+
+GPU time is measured, not guessed: the Shameless pilot took 669 s for 57.5 minutes of
+1080p (about 0.19 h of GPU per hour of video). 4K sources decode slower; the 1.5x for them
+is an estimate until the first 4K night is logged.
+
+Usage: cost-report.py > report.tsv     (also prints a summary to stderr)
+"""
+import importlib.util, json, pathlib, sys
+
+HERE = pathlib.Path(__file__).resolve().parent
+spec = importlib.util.spec_from_file_location("ct", HERE / "compact-tv.py")
+ct = importlib.util.module_from_spec(spec); spec.loader.exec_module(ct)
+
+GPU_H_PER_H = 669 / 3450                 # pilot: hours of GPU per hour of 1080p video
+UHD_FACTOR = 1.5                         # estimate for 2160p sources
+SMALL_GB_PER_H = {"Standard": 0.85, "Mini": 0.57}   # MeGusta-class 1080p / 720p HEVC, measured on S.W.A.T.
+
+rows = []
+for app, a in ct.APPS.items():
+    call = ct.arr(app)
+    profiles = {p["id"]: p["name"] for p in call("/qualityprofile")}
+    container, host = a["paths"]
+    for t in call(a["titles"]):
+        tier = profiles.get(t["qualityProfileId"])
+        if tier not in ct.TIERS:
+            continue
+        r = {"app": app, "title": t["title"], "tier": tier, "files": 0, "hours": 0.0, "size": 0,
+             "enc_files": 0, "enc_size": 0, "enc_est": 0, "gpu_h": 0.0, "hdr_size": 0, "small_size": 0}
+        for f in a["files"](call, t):
+            p = pathlib.Path(f["path"].replace(container, host, 1))
+            info = ct.probe(p) if p.exists() else None
+            if not info:
+                continue
+            size, dur = int(info["format"]["size"]), float(info["format"]["duration"]) / 3600
+            r["files"] += 1; r["hours"] += dur; r["size"] += size
+            args, why = ct.plan(info, tier, app)
+            if args:
+                w = max((s.get("width") or 0) for s in info["streams"] if s["codec_type"] == "video")
+                r["enc_files"] += 1; r["enc_size"] += size; r["enc_est"] += why
+                r["gpu_h"] += dur * GPU_H_PER_H * (UHD_FACTOR if w > 1920 else 1)
+            elif "HDR" in str(why):
+                r["hdr_size"] += size
+            else:
+                r["small_size"] += size
+        if r["files"]:
+            r["redownload_gb"] = round(r["hours"] * SMALL_GB_PER_H[tier], 1)
+            rows.append(r)
+
+rows.sort(key=lambda r: -(r["enc_size"] - r["enc_est"]))
+cols = ["app", "title", "tier", "files", "hours", "size", "enc_files", "enc_size", "enc_est", "gpu_h", "hdr_size",
+        "small_size", "redownload_gb"]
+print("\t".join(cols))
+for r in rows:
+    print("\t".join(str(round(r[c], 2) if isinstance(r[c], float) else r[c]) for c in cols))
+tot = lambda k: sum(r[k] for r in rows)
+print(f"{len(rows)} titles; encode {tot('enc_files')} files {tot('enc_size')/1e12:.2f} TB -> ~{tot('enc_est')/1e12:.2f} TB, "
+      f"{tot('gpu_h'):.0f} GPU hours; HDR skipped {tot('hdr_size')/1e12:.2f} TB; already small {tot('small_size')/1e12:.2f} TB",
+      file=sys.stderr)

server/home/Media/compact-tv/setup-small-profiles.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""Make Standard and Mini prefer small releases, and upgrade only toward small ones.
+
+Run once per app; it is idempotent, so re-running updates everything in place.
+
+- Adds "Small Encode" (release groups known for small HEVC/x264 encodes) and "720p" /
+  "1080p" resolution formats. They score 0 in every profile except Standard and Mini.
+- HD and UHD are otherwise untouched: they keep LQ at -10000, so they never pick these.
+- Standard and Mini:
+  - All allowed qualities share ONE group, so no "better resolution" or "better source"
+    upgrade can ever download a big file. Only the custom-format score decides.
+  - Scores: Small Encode +3000, x265 +2000 (so first grabs still prefer HEVC), LQ 0,
+    release-tier formats 0 (one big release must never "beat" another big one).
+    Standard adds +500 for 1080p and +250 for 720p; Mini adds +1000 for 720p and +500 for
+    1080p. Those preferences stay below the upgrade threshold, so they never cause churn.
+  - Upgrades are on, but an upgrade needs a score jump of at least 2500, and upgrading
+    stops at 3000. In practice only gaining "Small Encode" clears that bar: a big H.264
+    file gets replaced by a small encode, but never by a big HEVC one.
+
+Usage: setup-small-profiles.py <sonarr|radarr>
+"""
+import json, re, subprocess, sys, urllib.request, pathlib
+
+APP = sys.argv[1]
+cfg = pathlib.Path.home() / f"Server/Media/volumes/{APP}/config.xml"
+key = re.search(r"<ApiKey>([^<]+)", cfg.read_text()).group(1)
+ip = subprocess.run(["docker", "inspect", APP, "--format", "{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}"],
+                    capture_output=True, text=True, check=True).stdout.split()[0]
+base = f"http://{ip}:{8989 if APP == 'sonarr' else 7878}/api/v3"
+
+def call(path, body=None, method=None):
+    req = urllib.request.Request(base + path, method=method, headers={"X-Api-Key": key, "Content-Type": "application/json"},
+                                 data=json.dumps(body).encode() if body is not None else None)
+    with urllib.request.urlopen(req, timeout=60) as r:
+        return json.load(r)
+
+GROUPS = r"^(MeGusta|PSA|ELiTE|GalaxyTV|GalaxyRG|GalaxyRG265|YIFY|YTS(\.(MX|LT|AG))?)$"
+resolution = lambda r: [{"name": f"{r}p", "implementation": "ResolutionSpecification",
+                         "negate": False, "required": True, "fields": [{"name": "value", "value": r}]}]
+WANTED = {
+    "Small Encode": [{"name": "Small encode groups", "implementation": "ReleaseGroupSpecification",
+                      "negate": False, "required": False, "fields": [{"name": "value", "value": GROUPS}]}],
+    "720p": resolution(720),
+    "1080p": resolution(1080),
+}
+TIERS = {
+    "Standard": {"Small Encode": 3000, "x265 (HD)": 2000, "LQ": 0, "LQ (Release Title)": 0, "1080p": 500, "720p": 250},
+    "Mini":     {"Small Encode": 3000, "x265 (HD)": 2000, "LQ": 0, "LQ (Release Title)": 0, "720p": 1000, "1080p": 500},
+}
+RELEASE_TIER = re.compile(r"Tier \d+|WEB Scene")
+MIN_UPGRADE_JUMP, UPGRADE_UNTIL = 2500, 3000
+
+cfs = {c["name"]: c for c in call("/customformat")}
+for name, specs in WANTED.items():
+    body = {"name": name, "includeCustomFormatWhenRenaming": False, "specifications": specs}
+    if name in cfs:
+        body["id"] = cfs[name]["id"]; call(f"/customformat/{body['id']}", body, "PUT")
+    else:
+        call("/customformat", body)
+cfs = {c["name"]: c["id"] for c in call("/customformat")}
+
+
+def flatten(items):
+    """Every quality in the profile with whether it is allowed, groups expanded, lowest rank first."""
+    for it in items:
+        for sub in (it["items"] or [it]):
+            yield sub["quality"], it["allowed"]
+
+
+def one_group(profile):
+    """Put every allowed quality into a single top-ranked group; leave disallowed ones as they are."""
+    flat = list(flatten(profile["items"]))
+    group = {"id": 1100, "name": "Any allowed", "allowed": True,
+             "items": [{"quality": q, "items": [], "allowed": True} for q, ok in flat if ok]}
+    profile["items"] = [{"quality": q, "items": [], "allowed": False} for q, ok in flat if not ok] + [group]
+    profile["cutoff"] = group["id"]
+    return profile
+
+
+for prof in call("/qualityprofile"):
+    have = {f["format"] for f in prof["formatItems"]}
+    for name in WANTED:
+        if cfs[name] not in have:
+            prof["formatItems"].append({"format": cfs[name], "name": name, "score": 0})
+    scores = TIERS.get(prof["name"])
+    if scores:
+        prof = one_group(prof)
+        prof.update(upgradeAllowed=True, minUpgradeFormatScore=MIN_UPGRADE_JUMP, cutoffFormatScore=UPGRADE_UNTIL)
+        for f in prof["formatItems"]:
+            if f["name"] in scores:
+                f["score"] = scores[f["name"]]
+            elif RELEASE_TIER.search(f["name"]):
+                f["score"] = 0
+    call(f"/qualityprofile/{prof['id']}", prof, "PUT")
+    if scores:
+        allowed = [q["quality"]["name"] for it in prof["items"] if it["allowed"] for q in it["items"]]
+        print(f"{APP} {prof['name']}: one group of {len(allowed)} qualities ({allowed[0]} .. {allowed[-1]}); "
+              f"upgrades on, jump >= {MIN_UPGRADE_JUMP}, until {UPGRADE_UNTIL}")
+        print("   nonzero scores: " + ", ".join(f"{f['name']}={f['score']}" for f in prof["formatItems"] if f["score"]))

server/home/Media/compose.yml

@@ -427,3 +427,8 @@ services:
     # meant a completed sync was immediately restarted -- a hot loop of ~9
     # full syncs every 3 minutes. `no` lets it run once per `up` and stop.
     restart: "no"
+    # Opt-in only (`docker compose --profile configarr up configarr`). Its templates
+    # create the TRaSH profiles and size limits that were removed on 2026-10-28 in
+    # favour of four hand-managed profiles (Mini/Standard/HD/UHD), so a plain `up`
+    # must never run it. See Media/compact-tv/README.md.
+    profiles: [configarr]

server/home/Minecraft/Java/compose.yml

@@ -41,8 +41,36 @@ services:
       # world is in backups/; upgrades rewrite the world and cannot be undone.
       - VERSION=26.1.2
       - DIFFICULTY=normal
-      - MAX_PLAYERS=10
-      - TYPE=VANILLA
+      - MAX_PLAYERS=30
+      # Paper since 2026-10-13. The last vanilla state is backups/gib-26.1.2-vanilla-*.
+      - TYPE=PAPER
+      # DonutSMP-style plugin set, pulled from Modrinth on each start. Every entry
+      # needs a 26.1.2 Paper build; check Modrinth before adding or bumping VERSION.
+      # Config lives in servers/gib/plugins/<Plugin>/. GrimAC only publishes alpha builds.
+      - MODRINTH_DOWNLOAD_DEPENDENCIES=required
+      - |
+        MODRINTH_PROJECTS=
+        luckperms
+        vaultunlocked
+        essentialsx
+        essentialsx-spawn
+        placeholderapi
+        playerpoints
+        zmenu
+        zauctionhouse
+        ultimateshop
+        smartspawner
+        wildrtp
+        ultimate-teams
+        asbounty
+        axafkzone
+        tab-was-taken
+        deluxemenus
+        coreprotect
+        worldedit
+        worldguard
+        grimac:alpha
+        chunky
       - MEMORY=12G
       - SIMULATION_DISTANCE=16
       - VIEW_DISTANCE=16

server/home/OpenWebUI/compose.yml

@@ -38,6 +38,7 @@ services:
       - ENABLE_OAUTH_SIGNUP=true
       - OAUTH_MERGE_ACCOUNTS_BY_EMAIL=true
       #- ENABLE_OAUTH_ROLE_MANAGEMENT=true
+      #- OAUTH_ROLES_CLAIM=roles
       #- OAUTH_ADMIN_ROLES=admin,internal,superuser
       - OAUTH_CLIENT_ID=${OAUTH_CLIENT_ID}
       - OAUTH_CLIENT_SECRET=${OAUTH_CLIENT_SECRET}

server/home/RESTORE.md

@@ -0,0 +1,140 @@
+# Restoring ~/Server on server.gib from the ZFS-pool backup
+
+Written 2026-09-17. Read this *before* you need it.
+
+## What this protects against
+
+The NVMe/LUKS btrfs volume (`/`, `/home`, and `/var/lib/docker`) dying. The backup lives on
+the ZFS pool in the same machine, so it does **not** protect against fire, theft, or the
+whole box failing. It is a disk-failure backup, not an offsite one.
+
+## Layout
+
+    /home/gib/Public/Server_Backup/Containers/server.gib/current/
+
+**One directory, overwritten daily at 02:00** by `server-backup.timer`. History lives in ZFS:
+`Media/Server_Backup` has `zfs-auto-snapshot` keeping **14 dailies**, taken by `cron.daily`
+between roughly 03:22 and 03:52. The 02:00 schedule is deliberate — it must finish before
+that window, or a snapshot captures a half-written backup as that day's history. It also
+staggers against the VPS backup, which pushes into this same pool at 01:00.
+
+Older copies are under the hidden snapshot dir (`snapdir=hidden`, so `ls` will not show it,
+but you can `cd` in):
+
+    ls /home/gib/Public/Server_Backup/.zfs/snapshot/
+    ls /home/gib/Public/Server_Backup/.zfs/snapshot/<snap>/Containers/server.gib/current/
+
+The `03-16-2026` and `03-31-2026` directories are leftovers from the old manual script.
+
+## Why this runs as root
+
+This box runs Docker, so most container data under `~/Server` is root-owned. The old script
+used `sudo rsync`, which worked but meant it could never be automated without a sudoers
+rule. The replacement runs as root via `server-backup.service`, a systemd **system** unit —
+already root, so ownership is preserved natively and **no sudoers entry is needed at
+runtime**. If you see `/etc/sudoers.d/99-claude-setup` on this machine, it was temporary
+setup scaffolding and should be deleted.
+
+(This differs from the VPS, which is rootless podman and needs `rsync --fake-super` to carry
+subuid ownership through. Nothing like that is needed here.)
+
+The script lives at `/usr/local/sbin/server-backup`, deliberately **not** in `~/Server` — a
+root-executed script inside a user-writable directory is a footgun. `~/Server/backup` is a
+thin wrapper that triggers the systemd unit so manual runs behave identically to nightly ones.
+
+## What is in a backup
+
+| Path | What it is |
+|---|---|
+| `Media/`, `Frigate/`, `Home/`, `OpenWebUI/`, ... | Raw bind-mount data per service |
+| `_backup_state/sqlite/<original path>` | Online `.backup` of every SQLite DB — **authoritative** |
+| `_backup_state/convex/<container>.zip` | `npx convex export` snapshot of each SQLite-backed Convex deployment (gbrown) — **authoritative** |
+
+> **Use `_backup_state/` for databases, not the raw copies.** Containers keep running during
+> the backup, so the raw files are crash-consistent at best. The dumps use SQLite's online
+> backup API and are what will actually restore cleanly. That covers the ones that matter:
+> Plex's `com.plexapp.plugins.library.db` (watch history, ratings), Jellyfin, Frigate, Home
+> Assistant, OpenWebUI, Audiobookshelf, and every *arr config.
+>
+> **Convex is the exception.** It writes to its SQLite file constantly, so a raw `.backup`
+> never completes. Its `_backup_state/convex/` export is the restore source, and the raw
+> `db.sqlite3` is not dumped at all.
+
+## What is deliberately NOT backed up
+
+- **`Agents/Gitea/volumes/runner-{1,2,3}/dind` — ~203 GB.** Docker-in-Docker containerd
+  image layers for the CI runners. Pure cache that churns constantly; a rebuilt runner just
+  re-pulls it. The parts that matter — `runner-N/data/.runner` (the 648-byte registration
+  token) and `certs/` — **are** backed up.
+  > Measured as root. A plain `du` as `gib` reports these as ~13 MB because gib cannot
+  > descend into them. Do not trust non-root sizes here.
+- Docker **named volumes** (`/var/lib/docker/volumes`, 47 in use, ~61 GB): Sentry (13) and
+  Plausible (4) were consciously excluded — Sentry's event data is not worth restoring and
+  Plausible had nothing in it. Romm looked like a gap but is not: its anonymous `/romm`
+  volume is only a parent directory, with `assets/`, `config/` and `resources/` bind-mounted
+  from `~/Server` and `library/` from the pool.
+- **Nextcloud AIO** — has its own borg backup at `~/Public/Nextcloud/Backups`, run by
+  hand from the AIO admin UI. Last run 2026-09-17 (finished 09-18). Runs are slow and
+  AIO stops the Nextcloud containers while one is going.
+- `node_modules`, build output, `*.log`, `cache`/`tmp` dirs, `*.mp4`/`*.mkv`.
+
+## Restoring
+
+Stop the stack first so nothing overwrites what you restore:
+
+    cd ~/Server/<service> && docker compose down     # per service, or stop docker entirely
+
+Pull it back — as root, so ownership is restored:
+
+    sudo rsync -a --numeric-ids --delete \
+      /home/gib/Public/Server_Backup/Containers/server.gib/current/ \
+      /home/gib/Server/
+
+For an older day, use a snapshot path instead:
+
+    /home/gib/Public/Server_Backup/.zfs/snapshot/<snap>/Containers/server.gib/current/
+
+Then put the databases back from the dumps. For each one, stop the owning container, copy the
+dump over the live file, and match the original ownership. Plex, for example:
+
+    docker compose -f ~/Server/Media/compose.yml stop plex
+    D=~/Server/_backup_state/sqlite/Media/volumes/plex/config/Library/Application\ Support/Plex\ Media\ Server/Plug-in\ Support/Databases
+    L=~/Server/Media/volumes/plex/config/Library/Application\ Support/Plex\ Media\ Server/Plug-in\ Support/Databases
+    sudo cp "$D/com.plexapp.plugins.library.db" "$L/"
+    sudo chown --reference="$L/com.plexapp.dlna.db" "$L/com.plexapp.plugins.library.db"
+    docker compose -f ~/Server/Media/compose.yml start plex
+
+`chown --reference` copies ownership from a neighbouring file, which saves looking up what
+uid each container expects.
+
+For Convex, start the backend (it can be empty) and import the export through it. This
+replaces every table with the snapshot:
+
+    Z=~/Server/_backup_state/convex/gbrown-backend.zip
+    KEY=$(docker exec gbrown-backend ./generate_admin_key.sh | tail -1)
+    docker run --rm -it --network nginx-bridge -v "$Z:/backup.zip:ro" \
+      -e CONVEX_SELF_HOSTED_URL=http://gbrown-backend:3210 -e CONVEX_SELF_HOSTED_ADMIN_KEY="$KEY" \
+      docker.io/library/node:22-alpine sh -ec 'mkdir /w && cd /w && echo "{}" > package.json &&
+        npm install --silent convex && npx convex import --replace-all /backup.zip'
+
+Functions and schema are not in the export. Redeploy them from the app's repo as usual.
+
+After a full restore the CI runners will have no `dind` storage. That is fine — they re-pull
+images on the next job. The registration in `runner-N/data/.runner` is intact, so they
+reconnect to Gitea without re-registering.
+
+## Checking it is working
+
+    systemctl list-timers server-backup.timer
+    systemctl status server-backup.service
+    tail -40 ~/Server/logs/backup.log
+
+A healthy run ends with `=== backup complete ===`. Watch for:
+
+- `ZFS history OK` — confirms snapshots are still being taken. If this ever says **WARNING**,
+  versioning is broken and `current/` is just a mirror: a mistake would propagate with
+  nothing to roll back to. Fix `zfs-auto-snapshot` before anything else.
+- `rsync exit 24` is a NOTE, not a failure — a file changed mid-transfer, normal on a live box.
+
+Optional: put an uptime-kuma push URL in `~/Server/.backup-heartbeat-url` and successful runs
+will ping it, turning a silent failure into a down monitor.

server/home/backup

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Manual trigger for the daily backup.
+#
+# The real script lives at /usr/local/sbin/server-backup and runs as ROOT via
+# server-backup.service. It is kept out of ~/Server on purpose: ~/Server is
+# user-writable, and a root-executed script sitting in a user-writable directory
+# is an obvious footgun. Run it through systemd so a manual run behaves exactly
+# like the nightly one (same environment, same logging, same lock).
+#
+#   status:  systemctl status server-backup.service
+#   log:     tail -f ~/Server/logs/backup.log
+#   timer:   systemctl list-timers server-backup.timer
+#   restore: see ~/Server/RESTORE.md
+exec sudo systemctl start --wait server-backup.service

server/home/systemd/media-backup.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=Weekly copy of the Media pool to the Backup pool
+Documentation=file:///home/gib/Server/RESTORE.md
+After=zfs-mount.service
+Wants=zfs-mount.service
+RequiresMountsFor=/home/gib/Public /home/gib/.local/share/Backup
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/media-backup
+# 8.6 TB on the first run, deltas after. Generous, but bounded so a hung rsync
+# fails loudly instead of overlapping next week's fire.
+TimeoutStartSec=24h
+Nice=10
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7
+
+[Install]
+WantedBy=multi-user.target

server/home/systemd/media-backup.timer

@@ -0,0 +1,17 @@
+[Unit]
+Description=Weekly Media -> Backup pool copy
+
+[Timer]
+# Saturday 05:00. Clear of: the cron.daily zfs-auto-snapshot window (03:22-03:52),
+# timeshift (04:00), arr-queue-reconcile (04:31), and both ~/Server backups (01:00, 02:00).
+# Saturday specifically because Monday carries the Media scrub and Sunday has
+# raid-check (01:00), docker-prune (05:33) and dind-prune (06:03).
+# gib also runs Nextcloud's borg by hand roughly monthly and is, in his words,
+# not going to be doing that at 5 AM on a Saturday -- so the borg lock should be free.
+OnCalendar=Sat *-*-* 05:00:00
+RandomizedDelaySec=15m
+Persistent=true
+Unit=media-backup.service
+
+[Install]
+WantedBy=timers.target

server/home/systemd/server-backup.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=Back up /home/gib/Server to the ZFS pool
+Documentation=file:///home/gib/Server/RESTORE.md
+# The destination is on ZFS; do not run before the pool is up.
+After=zfs-mount.service docker.service
+Wants=zfs-mount.service
+RequiresMountsFor=/home/gib/Public
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/server-backup
+TimeoutStartSec=8h
+# This box is a daily driver running 145 containers. Stay out of the way.
+Nice=10
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7
+
+[Install]
+WantedBy=multi-user.target

server/home/systemd/server-backup.timer

@@ -0,0 +1,14 @@
+[Unit]
+Description=Daily backup of /home/gib/Server to the ZFS pool
+
+[Timer]
+# MUST finish before cron.daily fires zfs-auto-snapshot (~03:22-03:52); a snapshot that
+# catches a running rsync stores a partial backup as that day's history.
+# 02:00 also staggers against the VPS backup, which pushes into this same pool at 01:00.
+OnCalendar=*-*-* 02:00:00
+RandomizedDelaySec=10m
+Persistent=true
+Unit=server-backup.service
+
+[Install]
+WantedBy=timers.target

server/scripts/update-containers

@@ -92,7 +92,7 @@ list_units() {
 # services therefore resolve no images and are skipped without a log line.
 #
 # Image tags frequently reference variables from the service's .env, e.g.
-#   ghcr.io/get-convex/convex-backend:${BACKEND_TAG:-latest}
+#   ghcr.io/convex/convex-backend:${BACKEND_TAG:-latest}
 #   git.gbrown.org/gib/${NEXT_CONTAINER_NAME}:latest
 # Those must be expanded or the pull is attempted against a literal "${...}" and
 # always fails. We expand in a SUBSHELL so the .env cannot leak into this script's

server/vps/Authentik/podman-authentik.service

@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: Authentik
 After=network-online.target podman.socket podman-postgresql.service
 Wants=network-online.target podman-postgresql.service

server/vps/Beszel/podman-beszel.service

@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: Beszel
 After=network-online.target podman.socket
 Wants=network-online.target

server/vps/BillTracker/compose.yml

@@ -32,7 +32,7 @@ services:
     restart: unless-stopped
 
   bill-tracker-backend:
-    image: ${CONVEX_BACKEND_IMAGE:-ghcr.io/get-convex/convex-backend@sha256:1f2044e3eac463ac78973b136c0baf72d4ada602611d853d6f99f280e29e0a98}
+    image: ${CONVEX_BACKEND_IMAGE:-ghcr.io/convex/convex-backend@sha256:207cf95def42f01614b6e3af0af5c20d0dcaa5ff2adf258ee7bc9555576644fa}
     container_name: bill-tracker-backend
     hostname: bill-tracker-backend
     domainname: api.billtracker.gbrown.org
@@ -56,7 +56,7 @@ services:
     stop_signal: SIGINT
 
   bill-tracker-dashboard:
-    image: ${CONVEX_DASHBOARD_IMAGE:-ghcr.io/get-convex/convex-dashboard@sha256:284a2638e0c1a4ec0c2327d8219776f3a426ca5824b81686ae4d9454dc0ce8ed}
+    image: ${CONVEX_DASHBOARD_IMAGE:-ghcr.io/convex/convex-dashboard@sha256:a55bf8abca1332f787bc4d4a79850182175027be7b70aa230be4389a1d3268e2}
     container_name: bill-tracker-dashboard
     hostname: bill-tracker-dashboard
     domainname: dashboard.billtracker.gbrown.org

server/vps/ConvexMonorepo/compose.yml

@@ -29,7 +29,7 @@ services:
     restart: unless-stopped
 
   convexmonorepo-backend:
-    image: ghcr.io/get-convex/convex-backend:${BACKEND_TAG:-latest}
+    image: ghcr.io/convex/convex-backend:${BACKEND_TAG:-latest}
     container_name: ${BACKEND_CONTAINER_NAME:-convex-backend}
     hostname: ${BACKEND_CONTAINER_NAME:-convex-backend}
     domainname: ${BACKEND_DOMAIN:-convex.gbrown.org}
@@ -58,7 +58,7 @@ services:
     stop_signal: SIGINT
 
   convexmonorepo-dashboard:
-    image: ghcr.io/get-convex/convex-dashboard:${DASHBOARD_TAG:-latest}
+    image: ghcr.io/convex/convex-dashboard:${DASHBOARD_TAG:-latest}
     container_name: ${DASHBOARD_CONTAINER_NAME:-convex-dashboard}
     hostname: ${DASHBOARD_CONTAINER_NAME:-convex-dashboard}
     domainname: ${DASHBOARD_DOMAIN:-dashboard.${BACKEND_DOMAIN:-convex.gbrown.org}}

server/vps/ISponsorBlockTV/podman-isponsorblocktv.service

@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: iSponsorBlockTV
 After=network-online.target podman.socket
 Wants=network-online.target

server/vps/Infisical/podman-infisical.service

@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: Infisical
 After=network-online.target podman.socket podman-postgresql.service
 Wants=network-online.target podman-postgresql.service

server/vps/LashAddict/compose.yml

@@ -19,7 +19,7 @@ services:
     - NODE_ENV=${NODE_ENV}
     - SENTRY_AUTH_TOKEN=${SENTRY_AUTH_TOKEN}
     - NEXT_PUBLIC_SITE_URL=${NEXT_PUBLIC_SITE_URL:-http://localhost:${NEXT_PORT:-3000}}
-    - NEXT_PUBLIC_CONVEX_URL=${NEXT_PUBLIC_CONVEX_URL:-http://${BACKEND_CONTAINER_NAME:-lashaddict-backend}:${BACKEND_PORT:-3210}}
+    - NEXT_PUBLIC_CONVEX_URL=${CONVEX_SERVER_URL:-http://${BACKEND_CONTAINER_NAME:-lashaddict-backend}:${BACKEND_PORT:-3210}}
     - NEXT_PUBLIC_PLAUSIBLE_URL=${NEXT_PUBLIC_PLAUSIBLE_URL:-https://plausible.gbrown.org}
     - NEXT_PUBLIC_SENTRY_DSN=${NEXT_PUBLIC_SENTRY_DSN}
     - NEXT_PUBLIC_SENTRY_URL=${NEXT_PUBLIC_SENTRY_URL}
@@ -33,7 +33,7 @@ services:
     stdin_open: true
     restart: unless-stopped
   lashaddict-backend:
-    image: ghcr.io/get-convex/convex-backend:${BACKEND_TAG:-latest}
+    image: ghcr.io/convex/convex-backend:${BACKEND_TAG:-latest}
     container_name: ${BACKEND_CONTAINER_NAME:-lashaddict-backend}
     # Same Cloudflare DNS pin as lashaddict-next (see its comment): the backend
     # fetches its own auth discovery URL, which Hetzner resolvers failed.
@@ -67,7 +67,7 @@ services:
     stop_grace_period: 10s
     stop_signal: SIGINT
   lashaddict-dashboard:
-    image: ghcr.io/get-convex/convex-dashboard:${DASHBOARD_TAG:-latest}
+    image: ghcr.io/convex/convex-dashboard:${DASHBOARD_TAG:-latest}
     container_name: ${DASHBOARD_CONTAINER_NAME:-lashaddict-dashboard}
     # Same Cloudflare DNS pin as lashaddict-next (see its comment): the backend
     # fetches its own auth discovery URL, which Hetzner resolvers failed.

server/vps/N8n/podman-n8n.service

@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: N8n
 After=network-online.target podman-postgresql.service
 Wants=network-online.target

server/vps/Nginx_Proxy_Manager/podman-nginx-proxy-manager.service

@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: Nginx Proxy Manager
 After=network-online.target podman-postgresql.service
 Wants=network-online.target

server/vps/Portainer/podman-portainer.service

@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: Portainer
 After=network-online.target
 Wants=network-online.target

server/vps/PostgreSQL/podman-postgresql.service

@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: PostgreSQL
 After=network-online.target
 Wants=network-online.target

server/vps/Rustdesk/podman-rustdesk.service

@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: Rustdesk
 After=network-online.target podman.socket
 Wants=network-online.target

server/vps/Spoon/compose.yml

@@ -64,7 +64,7 @@ services:
     security_opt:
     - label:disable
   spoon-backend:
-    image: ghcr.io/get-convex/convex-backend:${BACKEND_TAG:-latest}
+    image: ghcr.io/convex/convex-backend:${BACKEND_TAG:-latest}
     container_name: ${BACKEND_CONTAINER_NAME:-spoon-backend}
     hostname: ${BACKEND_CONTAINER_NAME:-spoon-backend}
     domainname: ${BACKEND_DOMAIN:-convex.spoon.gbrown.org}
@@ -93,7 +93,7 @@ services:
     stop_grace_period: 10s
     stop_signal: SIGINT
   spoon-dashboard:
-    image: ghcr.io/get-convex/convex-dashboard:${DASHBOARD_TAG:-latest}
+    image: ghcr.io/convex/convex-dashboard:${DASHBOARD_TAG:-latest}
     container_name: ${DASHBOARD_CONTAINER_NAME:-spoon-dashboard}
     hostname: ${DASHBOARD_CONTAINER_NAME:-spoon-dashboard}
     domainname: ${DASHBOARD_DOMAIN:-dashboard.${BACKEND_DOMAIN:-spoon.gbrown.org}}

server/vps/StPeteIT/compose.yml

@@ -29,7 +29,7 @@ services:
     restart: unless-stopped
 
   stpeteit-backend:
-    image: ghcr.io/get-convex/convex-backend:${BACKEND_TAG:-latest}
+    image: ghcr.io/convex/convex-backend:${BACKEND_TAG:-latest}
     container_name: ${BACKEND_CONTAINER_NAME:-stpeteit-backend}
     hostname: ${BACKEND_CONTAINER_NAME:-stpeteit-backend}
     domainname: ${BACKEND_DOMAIN:-convex.stpeteit.com}
@@ -58,7 +58,7 @@ services:
     stop_signal: SIGINT
 
   stpeteit-dashboard:
-    image: ghcr.io/get-convex/convex-dashboard:${DASHBOARD_TAG:-latest}
+    image: ghcr.io/convex/convex-dashboard:${DASHBOARD_TAG:-latest}
     container_name: ${DASHBOARD_CONTAINER_NAME:-stpeteit-dashboard}
     hostname: ${DASHBOARD_CONTAINER_NAME:-stpeteit-dashboard}
     domainname: ${DASHBOARD_DOMAIN:-dashboard.${BACKEND_DOMAIN:-convex.stpete.com}}

server/vps/Tools/podman-tools.service

@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: Tools
 After=network-online.target podman.socket
 Wants=network-online.target

server/vps/Uptime/podman-uptime.service

@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: Uptime
 After=network-online.target
 Wants=network-online.target

server/vps/UseSend/.env.example

@@ -8,14 +8,17 @@ AWS_DEFAULT_REGION=us-east-2
 DATABASE_URL=CHANGE_ME
 GITHUB_ID=Ov23liz34colyJed9t9S
 GITHUB_SECRET=CHANGE_ME
-MINIO_ROOT_USER=usesend
-MINIO_ROOT_PASSWORD=CHANGE_ME
 NEXTAUTH_SECRET=CHANGE_ME
 NEXT_PUBLIC_IS_CLOUD=false
 POSTGRES_USER=usesend
 POSTGRES_PASSWORD=CHANGE_ME
 POSTGRES_DB=usesend
 REDIS_URL=redis://usesend-redis:6379
+RUSTFS_ACCESS_KEY=CHANGE_ME
+RUSTFS_SECRET_KEY=CHANGE_ME
+S3_COMPATIBLE_API_URL=https://storage.usesend.gbrown.org
+S3_COMPATIBLE_PUBLIC_URL=https://storage.usesend.gbrown.org/unsend
+S3_COMPATIBLE_BUCKET=unsend
 SMTP_HOST=smtp.usesend.gbrown.org
 SMTP_USER=usesend
 USESEND_PORT=3000

server/vps/UseSend/README.md

@@ -0,0 +1,39 @@
+# TLS certificate for the UseSend SMTP proxy
+
+The `usesend/smtp-proxy` image ships a default certificate that **cannot complete a
+TLS 1.3 handshake** — OpenSSL rejects it with `no suitable signature algorithm`.
+Because the proxy also refuses `AUTH` until `STARTTLS` has succeeded, that default
+cert made *all* inbound SMTP impossible. These files replace it.
+
+## What lives in certs/
+
+`fullchain.pem` and `privkey.pem`, copied from NPM's Let's Encrypt store:
+
+    ~/Server/Nginx_Proxy_Manager/volumes/letsencrypt/live/npm-43/
+
+That certificate carries `*.usesend.gbrown.org`, which is what lets clients validate
+the proxy as `smtp.usesend.gbrown.org`. A wildcard matches exactly one label, so
+`*.gbrown.org` alone would not cover it.
+
+Clients reach that name two ways, neither of them public DNS:
+
+- containers on `nginx-bridge` get the network alias set in `compose.yml`
+- LAN and home-server clients get a UniFi DNS record pointing it at `192.168.2.2`
+
+## Why copies instead of a bind mount
+
+NPM mounts that letsencrypt tree with `:Z`, which gives it a *private* SELinux MCS
+category. A second container mounting the same path would relabel it and the two
+would fight over it. Copying keeps the stacks independent.
+
+## Keeping the copies fresh
+
+NPM renews on its own schedule, so `usesend-cert-refresh.timer` runs
+`refresh-smtp-certs` daily. It copies the cert when NPM's fingerprint changes,
+refuses a half-renewed cert/key pair, and restarts the proxy.
+
+If the copies ever expire, every app sending mail through the proxy fails at once,
+with a TLS error rather than anything that mentions certificates. Check with:
+
+    openssl x509 -in ~/Server/UseSend/certs/fullchain.pem -noout -enddate
+    systemctl --user list-timers usesend-cert-refresh.timer

server/vps/UseSend/compose.yml

@@ -21,25 +21,69 @@ services:
     tty: true
     stdin_open: true
     restart: unless-stopped
-  minio:
-    image: minio/minio
+  # S3-compatible storage for images uploaded in the email editor, following
+  # upstream's docker/prod/compose.yml (RustFS replaced MinIO, whose images are
+  # no longer published). The browser uploads straight to S3_COMPATIBLE_API_URL
+  # and sent emails link to S3_COMPATIBLE_PUBLIC_URL, so NPM proxies
+  # storage.usesend.gbrown.org -> http://usesend-storage:9000. No host ports:
+  # the console (9001) is only reachable from inside nginx-bridge.
+  # Pinned like upstream. Bump the tag deliberately when upstream does.
+  rustfs:
+    image: docker.io/rustfs/rustfs:1.0.4
     container_name: usesend-storage
     hostname: usesend-storage
     networks:
     - nginx-bridge
-    env_file: .env
+    # RustFS runs as uid 10001; keep-id keeps ./volumes/rustfs gib-owned (AGENTS.md §3.2)
+    userns_mode: keep-id:uid=10001,gid=10001
+    security_opt:
+    - no-new-privileges:true
     environment:
-      MINIO_ROOT_USER: ${MINIO_ROOT_USER}
-      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
-    labels:
-      com.centurylinklabs.watchtower.enable: 'true'
+      RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY:?err}
+      RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY:?err}
+      RUSTFS_ADDRESS: 0.0.0.0:9000
+      RUSTFS_CONSOLE_ADDRESS: 0.0.0.0:9001
+      RUSTFS_CONSOLE_ENABLE: 'true'
+      RUSTFS_OBS_LOGGER_LEVEL: error
+      TZ: America/New_York
     volumes:
-    - ./volumes/minio:/data:Z
-    entrypoint: sh
-    command: -c 'mkdir -p /data/unsend && minio server /data --console-address ':9001' --address ':9002''
-    tty: true
-    stdin_open: true
+    - ./volumes/rustfs:/data:Z
+    command: ["/data"]
+    healthcheck:
+      test: ["CMD", "curl", "-fsS", "http://127.0.0.1:9000/health"]
+      interval: 20s
+      timeout: 10s
+      retries: 5
+      start_period: 40s
     restart: unless-stopped
+  # One-shot: creates the bucket, allows browser uploads from the app (CORS) and
+  # makes objects publicly readable so images load in sent emails. Reruns on
+  # every unit start, which is harmless.
+  rustfs-init:
+    image: docker.io/amazon/aws-cli:2.34.30
+    container_name: usesend-storage-init
+    networks:
+    - nginx-bridge
+    entrypoint: ["/bin/sh", "-ec"]
+    environment:
+      AWS_ACCESS_KEY_ID: ${RUSTFS_ACCESS_KEY:?err}
+      AWS_SECRET_ACCESS_KEY: ${RUSTFS_SECRET_KEY:?err}
+      AWS_DEFAULT_REGION: us-east-1
+      AWS_EC2_METADATA_DISABLED: 'true'
+      BUCKET: ${S3_COMPATIBLE_BUCKET:-unsend}
+      APP_ORIGIN: ${USESEND_URL:?err}
+    depends_on:
+      rustfs:
+        condition: service_healthy
+    command:
+    - |
+      s3="aws --endpoint-url http://usesend-storage:9000 s3api"
+      $$s3 head-bucket --bucket "$$BUCKET" >/dev/null 2>&1 || $$s3 create-bucket --bucket "$$BUCKET"
+      $$s3 put-bucket-cors --bucket "$$BUCKET" --cors-configuration \
+        '{"CORSRules":[{"AllowedOrigins":["'"$$APP_ORIGIN"'"],"AllowedMethods":["GET","HEAD","PUT"],"AllowedHeaders":["*"],"ExposeHeaders":["ETag"],"MaxAgeSeconds":3600}]}'
+      $$s3 put-bucket-policy --bucket "$$BUCKET" --policy \
+        '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::'"$$BUCKET"'/*"]}]}'
+    restart: 'no'
   smtp-server:
     image: usesend/smtp-proxy:latest
     container_name: usesend-smtp-server
@@ -47,26 +91,15 @@ services:
     networks:
       nginx-bridge:
         # The proxy REQUIRES STARTTLS before it will accept AUTH, so clients must
-        # reach it by a name the certificate actually covers. The wildcard is
-        # *.gbrown.org, which matches ONE label -- hence smtp-usesend, not
-        # smtp.usesend.
+        # reach it by a name the certificate actually covers: smtp.usesend.gbrown.org,
+        # validated by the *.usesend.gbrown.org SAN on npm-43 (see certs/README.md).
         #
         # Resolution: podman's aardvark-dns answers this name authoritatively for
         # containers ON nginx-bridge, handing back the container IP. NPM is not
-        # involved at all -- it only reverse-proxies HTTP(S), and SMTP ports are
-        # never published to the host, so the proxy is unreachable from outside.
-        #
-        # NOTE: there IS a wildcard *.gbrown.org record in Cloudflare, so this name
-        # also resolves publicly (to Cloudflare, which serves nothing for it). The
-        # consequence: if this container is STOPPED, aardvark-dns stops answering,
-        # the lookup falls through to that public record, and clients dial
-        # Cloudflare on 587 and hang until timeout instead of failing fast.
+        # involved at all -- it only reverse-proxies HTTP(S). Cloudflare is not
+        # involved either: there is deliberately no public DNS record for this name,
+        # so if the container is stopped, lookups fail fast with NXDOMAIN.
         aliases:
-        # Both names resolve to this container and BOTH validate against npm-43,
-        # which carries *.gbrown.org and *.usesend.gbrown.org. smtp.usesend is the
-        # name UseSend's own UI advertises and is the target to standardise on;
-        # smtp-usesend stays until every consumer has been moved across.
-        - smtp-usesend.gbrown.org
         - smtp.usesend.gbrown.org
     env_file: .env
     environment:
@@ -83,8 +116,8 @@ services:
     # brute-force it, and contradicts the Cloudflare-only posture in §7.
     # The `Home` wg interface sits in firewalld's `trusted` zone, so no rule is needed.
     # Rootless can bind these because net.ipv4.ip_unprivileged_port_start = 80.
-    # Consumers on the home server reach it as smtp-usesend.gbrown.org (UniFi DNS maps
-    # that name to 192.168.2.2) so the *.gbrown.org cert still validates by hostname.
+    # Consumers on the home server reach it as smtp.usesend.gbrown.org (a UniFi DNS record
+    # maps that name to 192.168.2.2) so the cert still validates by hostname.
     - "192.168.2.2:587:587"
     - "192.168.2.2:465:465"
     volumes:
@@ -121,6 +154,12 @@ services:
     - API_RATE_LIMIT=${API_RATE_LIMIT:-1}
     - SMTP_HOST=${SMTP_HOST:-smtp.usesend.gbrown.org}
     - SMTP_USER=${SMTP_USER:-usesend}
+    # Editor image uploads (the rustfs service above). Uploads are disabled if any is unset.
+    - S3_COMPATIBLE_ACCESS_KEY=${RUSTFS_ACCESS_KEY:?err}
+    - S3_COMPATIBLE_SECRET_KEY=${RUSTFS_SECRET_KEY:?err}
+    - S3_COMPATIBLE_API_URL=${S3_COMPATIBLE_API_URL:?err}
+    - S3_COMPATIBLE_PUBLIC_URL=${S3_COMPATIBLE_PUBLIC_URL:?err}
+    - S3_COMPATIBLE_BUCKET=${S3_COMPATIBLE_BUCKET:-unsend}
     labels:
       com.centurylinklabs.watchtower.enable: 'true'
     depends_on:

server/vps/UseSend/refresh-smtp-certs

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Keep the UseSend SMTP proxy's TLS certificate in step with NPM's Let's Encrypt store.
+#
+# The proxy refuses AUTH until STARTTLS succeeds, so an expired or mismatched cert
+# takes down ALL outbound mail for every app that sends through it -- and it fails
+# with a bare TLS error that never mentions certificates. Hence this timer.
+#
+# Copies rather than bind-mounts because NPM mounts its letsencrypt tree with :Z
+# (private SELinux category) and two consumers on one path fight over the label.
+#
+# Uses npm-43 because it carries *.usesend.gbrown.org, the SAN that validates
+# smtp.usesend.gbrown.org. A wildcard matches exactly ONE label, so *.gbrown.org
+# alone would not cover it.
+set -euo pipefail
+
+SRC=$HOME/Server/Nginx_Proxy_Manager/volumes/letsencrypt/live/npm-43
+DST=$HOME/Server/UseSend/certs
+CONTAINER=usesend-smtp-server
+
+fp()     { openssl x509 -in "$1" -noout -fingerprint -sha256 2>/dev/null | cut -d= -f2; }
+certpub(){ openssl x509 -in "$1" -noout -pubkey 2>/dev/null | openssl sha256 | awk '{print $2}'; }
+keypub() { openssl pkey -in "$1" -pubout   2>/dev/null | openssl sha256 | awk '{print $2}'; }
+
+[ -r "$SRC/fullchain.pem" ] && [ -r "$SRC/privkey.pem" ] || {
+  echo "source cert unreadable under $SRC -- did NPM move it?" >&2; exit 1; }
+
+src_fp=$(fp "$SRC/fullchain.pem")
+dst_fp=$(fp "$DST/fullchain.pem" 2>/dev/null || true)
+
+if [ -n "$dst_fp" ] && [ "$src_fp" = "$dst_fp" ]; then
+  echo "cert unchanged (${src_fp:0:17}...); nothing to do"
+  exit 0
+fi
+
+# Refuse to install a half-renewed pair: certbot writes the two files separately, so
+# a run that lands between them would leave the proxy with a key that is not its cert.
+if [ "$(certpub "$SRC/fullchain.pem")" != "$(keypub "$SRC/privkey.pem")" ]; then
+  echo "source cert and key do not match -- renewal probably mid-flight, retrying next run" >&2
+  exit 0
+fi
+
+echo "cert changed (${dst_fp:0:17}... -> ${src_fp:0:17}...), installing"
+# -L dereferences: live/ entries are symlinks into ../../archive/, outside this copy.
+cp -L "$SRC/fullchain.pem" "$DST/fullchain.pem.new"
+cp -L "$SRC/privkey.pem"   "$DST/privkey.pem.new"
+chmod 644 "$DST/fullchain.pem.new"
+chmod 640 "$DST/privkey.pem.new"
+mv -f "$DST/fullchain.pem.new" "$DST/fullchain.pem"
+mv -f "$DST/privkey.pem.new"   "$DST/privkey.pem"
+
+echo "restarting $CONTAINER"
+podman restart "$CONTAINER" >/dev/null
+echo "done; cert now valid until $(openssl x509 -in "$DST/fullchain.pem" -noout -enddate | cut -d= -f2)"

server/vps/UseSend/usesend-cert-refresh.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Refresh UseSend SMTP proxy TLS certificate from NPM's Let's Encrypt store
+Documentation=file://%h/Server/UseSend/README.md
+After=podman-usesend.service
+
+[Service]
+Type=oneshot
+ExecStart=%h/Server/UseSend/refresh-smtp-certs
+# The script calls `podman restart`, which leaves conmon and the container's
+# supervising processes inside THIS unit's cgroup. With the default KillMode,
+# systemd waits for that cgroup to empty and the unit times out and reports
+# `failed` even though ExecStart returned 0 and the cert was installed correctly.
+# KillMode=process makes systemd track only the script itself.
+KillMode=process
+TimeoutStartSec=300

server/vps/UseSend/usesend-cert-refresh.timer

@@ -0,0 +1,12 @@
+[Unit]
+Description=Daily check for a renewed UseSend SMTP proxy certificate
+
+[Timer]
+OnCalendar=daily
+# Spread the load and avoid landing exactly when certbot is writing.
+RandomizedDelaySec=2h
+# Catch up after the host has been off; the cert does not care that we were asleep.
+Persistent=true
+
+[Install]
+WantedBy=timers.target

server/vps/Vaultwarden/podman-vaultwarden.service

@@ -1,4 +1,5 @@
 [Unit]
+RequiresMountsFor=/home/gib/Media
 Description=Podman Compose: Vaultwarden
 After=network-online.target podman.socket
 Wants=network-online.target

server/vps/backup

@@ -0,0 +1,194 @@
+#!/usr/bin/env bash
+#
+# Daily backup of ~/Server to the home server, as hardlinked dated snapshots.
+#
+# WHAT THIS FIXES vs. the old version (2026-09-17):
+#   1. OWNERSHIP. Rootless podman leaves most container data owned by subuids
+#      (gitea = 525288, postgres data = the userns root mapping). Plain `rsync -a`
+#      running as gib cannot chown on the far side, so every previous backup landed
+#      100% owned by uid 1000. Restoring that gives you the right bytes with the wrong
+#      owners: postgres refuses to start unless PGDATA is owned correctly, and gitea
+#      fails on its own data dir. Fixed with `-M--fake-super`, which makes the REMOTE
+#      rsync stash mode/uid/gid in a user.rsync.%stat xattr (verified working on the
+#      home server's ZFS). Restore with --fake-super on the read side to put them back.
+#   2. CONSISTENCY. 13 SQLite DBs and a live postgres were copied hot, mid-write. Now
+#      dumped properly first (sqlite3 .backup / pg_dumpall) into _backup_state/.
+#   3. SPACE / HISTORY. Every run was a full ~19G copy into a new dated directory.
+#      Now it rsyncs into ONE stable directory, `current/`, and versioning is left to
+#      the home server: Media/Server_Backup is its own ZFS dataset with zfs-auto-snapshot
+#      keeping 14 dailies. ZFS snapshots are block-level, so a database that changes a
+#      few pages costs a few pages -- not a whole new copy the way dated dirs (or even
+#      rsync --link-dest hardlinks, which are per-file) would.
+#
+# Containers keep running throughout. That is safe *because* of the dumps above --
+# the dumps, not the raw data dirs, are the authoritative restore path.
+#
+# See RESTORE.md for how to bring this back on a new machine.
+
+set -uo pipefail
+
+REMOTE_HOST="[email protected]"
+REMOTE_ROOT="/home/gib/Public/Server_Backup/Containers/lilman.gib"
+SOURCE="$HOME/Server/"
+STATE_DIR="$HOME/Server/_backup_state"
+DEST_NAME="current"                # One stable directory. ZFS snapshots provide history.
+LOG_DIR="$HOME/Server/logs"
+LOG="$LOG_DIR/backup.log"
+LOCK="${XDG_RUNTIME_DIR:-/tmp}/server-backup.lock"
+
+# Versioning is ZFS's job, not ours -- see the header. We only sanity-check that it is
+# actually still happening, so this can never silently degrade into a single mirror with
+# no history. Warn if the newest snapshot of the dataset is older than this.
+REMOTE_DATASET="Media/Server_Backup"
+SNAPSHOT_MAX_AGE_HOURS=48
+
+mkdir -p "$LOG_DIR"
+log() { printf '%s  %s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG"; }
+die() { log "FATAL: $*"; exit 1; }
+
+# Refuse to overlap with a still-running backup rather than corrupting a snapshot.
+exec 9>"$LOCK" || die "cannot open lock $LOCK"
+flock -n 9 || die "another backup is already running"
+
+EXCLUSIONS=(
+    # --- build noise -------------------------------------------------------
+    # NOTE: these are bare patterns, so they match at ANY depth. Today they only
+    # catch genuine scratch dirs (checked 2026-09-17). If you add a service whose
+    # real data lives in a dir called cache/build/dist/tmp, it will be silently
+    # skipped -- re-check this list when adding services.
+    "node_modules" ".git" "*.log" "*.tmp" ".cache" "cache" "tmp" "temp"
+    ".npm" ".yarn" "dist" "build" "target" ".venv" "venv" "__pycache__"
+    "*.pyc" ".DS_Store" "Thumbs.db" "vendor" "*.mp4" "*.mkv"
+
+    # --- unreadable to gib, and genuinely disposable ------------------------
+    # Redis/valkey cache dumps: mode 0600 under a subuid because the container
+    # chowns its own data. Pure cache (usesend + infisical), regenerated on boot.
+    "dump.rdb"
+    # Gitea runs WITHOUT userns_mode (s6 needs container-root -- see AGENTS.md §3),
+    # so these are subuid-owned and mode 0700. Excluding them is what lets this run
+    # without sudo. Nothing here is needed to restore gitea:
+    #   sessions/     ephemeral web logins; worst case everyone logs in again
+    #   issues.bleve  search index, rebuilt from the DB on demand
+    #   Gitea/ssh     a 25-byte `environment` file for the git shell
+    #
+    # ⚠️ The SSH HOST KEYS are NOT here -- they live in Gitea/volume/ssh, are
+    # gib-owned, and DO get backed up. Losing those = host-key mismatch for every client.
+    "sessions" "issues.bleve" "Gitea/ssh"
+
+    # Mode 0600 under subuid 525288, so unreadable to gib -- this single file is what
+    # made every previous run exit 23. It is NOT disposable (it holds the MCP bridge's
+    # registered OAuth clients), so it is captured via `podman exec` into
+    # _backup_state/files/ instead of being read off disk.
+    "BillTracker/data/mcp/bridge.json"
+
+    # Stirling-PDF writes this 0600 under subuid 525287 (since its 2026-09-25 update),
+    # which made every run since exit 23. Excluded rather than captured: losing it only
+    # means re-entering any credentials stored in Stirling-PDF.
+    "pdf-tools/extraConfigs/credential-encryption.key"
+)
+
+EXCLUDE_ARGS=()
+for e in "${EXCLUSIONS[@]}"; do EXCLUDE_ARGS+=( "--exclude=$e" ); done
+
+# ---------------------------------------------------------------------------
+# Phase 1: consistent dumps of everything transactional.
+# ---------------------------------------------------------------------------
+dump_state() {
+    log "Dumping databases to _backup_state/ ..."
+    rm -rf "$STATE_DIR"
+    mkdir -p "$STATE_DIR"/{postgres,sqlite,files}
+    printf 'Generated %s by ~/Server/backup. Authoritative restore source -- see RESTORE.md\n' \
+        "$(date -Is)" > "$STATE_DIR/README"
+
+    # Live postgres (serves npm, authentik, usesend, ... -- pg_dumpall gets all of them).
+    if podman ps --format '{{.Names}}' | grep -qx postgresql; then
+        if podman exec postgresql pg_dumpall -U npm 2>>"$LOG" | gzip -9 > "$STATE_DIR/postgres/all.sql.gz"; then
+            log "  postgres: $(du -h "$STATE_DIR/postgres/all.sql.gz" | cut -f1)"
+        else
+            log "  WARNING: pg_dumpall failed -- raw pg_data is still copied, but is a hot copy"
+        fi
+    else
+        log "  WARNING: postgresql container not running, skipping pg_dumpall"
+    fi
+
+    # SQLite: .backup uses the online backup API, so this is safe while containers write.
+    local n=0
+    while IFS= read -r db; do
+        # *.db also matches stirling-pdf's H2 database, which is not SQLite. Check magic.
+        head -c 16 "$db" 2>/dev/null | grep -q "SQLite format 3" || continue
+        local rel="${db#"$HOME"/Server/}" out
+        out="$STATE_DIR/sqlite/$rel"
+        mkdir -p "$(dirname "$out")"
+        if sqlite3 -cmd ".timeout 30000" "$db" ".backup '$out'" 2>>"$LOG"; then n=$((n+1)); else log "  WARNING: sqlite dump failed: $rel"; fi
+    done < <(find "$HOME/Server" -path "$STATE_DIR" -prune -o \
+                  -type f \( -name '*.db' -o -name '*.sqlite' -o -name '*.sqlite3' \) -print 2>/dev/null)
+    log "  sqlite: $n databases"
+
+    # Files gib cannot read off disk, pulled through their container instead.
+    if podman ps --format '{{.Names}}' | grep -qx bill-tracker-mcp; then
+        podman exec bill-tracker-mcp cat /data/bridge.json \
+            > "$STATE_DIR/files/BillTracker--data--mcp--bridge.json" 2>>"$LOG" \
+            && log "  bridge.json captured" \
+            || log "  WARNING: could not capture bridge.json"
+    fi
+}
+
+# ---------------------------------------------------------------------------
+# Phase 2: confirm ZFS is still providing history.
+#
+# We deliberately overwrite one directory instead of keeping dated copies, which
+# means ZFS snapshots are the ONLY thing standing between us and "a mirror of a
+# mistake". If auto-snapshot ever stops, that must be loud, not silent.
+# ---------------------------------------------------------------------------
+check_zfs_history() {
+    local newest now age_h
+    newest=$(ssh -o BatchMode=yes "$REMOTE_HOST" \
+        "zfs list -H -p -o creation -t snapshot -s creation -d 1 '$REMOTE_DATASET' 2>/dev/null | tail -1" || true)
+    if [ -z "$newest" ]; then
+        log "  ⚠️  WARNING: no ZFS snapshots on $REMOTE_DATASET -- this backup has NO history."
+        log "      It is a plain mirror right now. Check zfs-auto-snapshot on the home server."
+        return
+    fi
+    now=$(date +%s)
+    age_h=$(( (now - newest) / 3600 ))
+    if [ "$age_h" -gt "$SNAPSHOT_MAX_AGE_HOURS" ]; then
+        log "  ⚠️  WARNING: newest ZFS snapshot is ${age_h}h old (limit ${SNAPSHOT_MAX_AGE_HOURS}h) -- versioning may be broken."
+    else
+        log "  ZFS history OK (newest snapshot ${age_h}h old, 14 dailies retained)"
+    fi
+}
+
+# ---------------------------------------------------------------------------
+# Phase 3: mirror.
+# ---------------------------------------------------------------------------
+log "=== backup start -> $REMOTE_HOST:$REMOTE_ROOT/$DEST_NAME ==="
+ssh -o BatchMode=yes -o ConnectTimeout=15 "$REMOTE_HOST" "mkdir -p '$REMOTE_ROOT/$DEST_NAME'" \
+    || die "cannot reach $REMOTE_HOST"
+
+dump_state
+check_zfs_history
+
+# --delete/--delete-excluded keep `current` an exact mirror of the source. Anything
+# removed here still exists in yesterday's ZFS snapshot, so this is not destructive.
+rsync -a --numeric-ids -M--fake-super --delete --delete-excluded \
+      --human-readable --stats \
+      "${EXCLUDE_ARGS[@]}" \
+      --rsync-path="mkdir -p '$REMOTE_ROOT/$DEST_NAME' && rsync" \
+      "$SOURCE" "$REMOTE_HOST:$REMOTE_ROOT/$DEST_NAME/" 2>&1 | tee -a "$LOG"
+status=${PIPESTATUS[0]}
+
+# 24 = "source files vanished during transfer". Normal on a live system, not a failure.
+if [ "$status" -ne 0 ] && [ "$status" -ne 24 ]; then
+    die "rsync failed with exit $status -- 'current' may be half-updated; yesterday's ZFS snapshot is still good"
+fi
+[ "$status" -eq 24 ] && log "NOTE: rsync exit 24 (files vanished mid-run) -- expected on a live system"
+
+# Optional heartbeat so a silent failure cannot go unnoticed. Create the file with an
+# uptime-kuma push URL in it and this pings on success only; absent, nothing happens.
+if [ -r "$HOME/Server/.backup-heartbeat-url" ]; then
+    url=$(< "$HOME/Server/.backup-heartbeat-url")
+    curl -fsS -m 20 "$url" >/dev/null 2>&1 \
+        && log "Heartbeat sent" || log "NOTE: heartbeat ping failed (backup itself was fine)"
+fi
+
+log "=== backup complete ==="

server/vps/server-backup.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=Back up ~/Server to the home server
+Documentation=file:///home/gib/Server/RESTORE.md
+After=network-online.target
+Wants=network-online.target
+# The backup reads podman-managed data and shells into containers for dumps.
+RequiresMountsFor=/home/gib/Media
+
+[Service]
+Type=oneshot
+ExecStart=%h/Server/backup
+# A stalled rsync should fail loudly, not hang until the next timer fires.
+TimeoutStartSec=6h
+# This box is IO-bound before it is CPU-bound (~18%% avg60 pressure), so stay
+# out of the way of the containers. best-effort/7 rather than idle, which can
+# starve the transfer outright when the volume is busy.
+Nice=10
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7
+
+[Install]
+WantedBy=default.target

server/vps/server-backup.timer

@@ -0,0 +1,18 @@
+[Unit]
+Description=Daily backup of ~/Server to the home server
+
+[Timer]
+# MUST finish before the home server's cron.daily fires zfs-auto-snapshot, which has
+# been landing between 03:22 and 03:52. If the snapshot catches a half-finished rsync,
+# that day's history entry is a partial backup. 01:00 leaves >2h of margin; the first
+# full run took well under that and daily deltas are far smaller.
+OnCalendar=*-*-* 01:00:00
+# Small jitter only -- must not push us into the snapshot window.
+RandomizedDelaySec=10m
+# If the VPS was down at 01:00, run once after it comes back rather than
+# silently skipping a day.
+Persistent=true
+Unit=server-backup.service
+
+[Install]
+WantedBy=timers.target

server/vps/verify-rootless

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+# Post-reboot verification for the rootless podman stack.
+# Run as gib. Compares against ~/premigration_backup_2026-08-10/post_migration_baseline.txt
+set -uo pipefail
+
+fail=0
+ok()   { printf '  \033[32mOK\033[0m   %s\n' "$1"; }
+bad()  { printf '  \033[31mFAIL\033[0m %s\n' "$1"; fail=$((fail+1)); }
+
+echo "== linger =="
+[[ "$(loginctl show-user gib --property=Linger --value 2>/dev/null)" == "yes" ]] \
+  && ok "Linger=yes" || bad "linger is OFF - user units will not start at boot"
+
+echo "== unprivileged port start =="
+p=$(sysctl -n net.ipv4.ip_unprivileged_port_start 2>/dev/null)
+[[ "$p" == "80" ]] && ok "ip_unprivileged_port_start=80" \
+  || bad "ip_unprivileged_port_start=$p (NPM cannot bind :80/:443)"
+
+echo "== user units =="
+for s in postgresql bang completeuphoria sierraandtyler uptime portainer beszel authentik tools isponsorblocktv vaultwarden infisical rustdesk nginx-proxy-manager gbrown-agent; do
+  st=$(systemctl --user is-active "podman-$s.service" 2>/dev/null)
+  [[ "$st" == "active" ]] && ok "$s" || bad "$s is $st"
+done
+
+echo "== containers =="
+for c in postgresql nginx-proxy-manager uptime portainer bang completeuphoria sierraandtyler beszel beszel-agent authentik-server authentik-worker convertx it-tools pdf-tools bento-pdf iSponsorBlockTV vaultwarden infisical-redis infisical-backend rustdesk-server gbrown-agent; do
+  if podman ps --format '{{.Names}}' 2>/dev/null | grep -qx "$c"; then ok "$c"; else bad "$c not running"; fi
+done
+
+echo "== postgres health =="
+h=$(podman inspect postgresql --format '{{.State.Health.Status}}' 2>/dev/null)
+[[ "$h" == "healthy" ]] && ok "postgres healthy" || bad "postgres health=$h"
+
+echo "== endpoints =="
+chk() { local n=$1 exp=$2; shift 2; c=$(curl -sk -o /dev/null -w '%{http_code}' "$@" 2>/dev/null)
+        [[ "$c" == "$exp" ]] && ok "$n -> $c" || bad "$n -> $c (expected $exp)"; }
+# Admin ports moved to the WireGuard address 2026-08-12 (AGENTS.md §13.2).
+# They no longer listen on localhost, so probe 192.168.2.2.
+chk "NPM :81"   200 http://192.168.2.2:81
+chk "NPM :80"   200 http://localhost:80
+chk "NPM :443"  200 --resolve nginx.gibbyb.com:443:127.0.0.1 https://nginx.gibbyb.com
+chk "Uptime"    302 http://192.168.2.2:3001
+chk "Portainer" 200 https://192.168.2.2:9443
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Boot-enablement check (added 2026-08-12)
+#
+# Five stacks were once found RUNNING but DISABLED — healthy in `podman ps`, yet they
+# would have disappeared on the next reboot. Starting a stack and enabling it are
+# separate actions; this catches the gap. See AGENTS.md §13.3.
+# ─────────────────────────────────────────────────────────────────────────────
+echo
+echo "== boot-enablement =="
+# podman's own units are meant to stay disabled; n8n is intentionally down.
+# podman's own units stay disabled by design; n8n is intentionally down;
+# podman-update.service is `static` (the timer activates it, so it is never enabled);
+# watchtower was RETIRED 2026-08-17 and must stay disabled (AGENTS.md §13.15).
+SKIP='podman-auto-update|podman-restart|podman-clean-transient|podman-kube@|podman-user-wait-network-online|podman-n8n|podman-update|podman-watchtower'
+bad=0
+for u in $(systemctl --user list-unit-files 'podman-*.service' --no-legend 2>/dev/null | awk '{print $1}'); do
+    echo "$u" | grep -qE "$SKIP" && continue
+    state=$(systemctl --user is-enabled "$u" 2>/dev/null)
+    if [ "$state" != "enabled" ]; then
+        echo "  WILL NOT SURVIVE REBOOT: $u ($state)  -> systemctl --user enable $u"
+        bad=$((bad+1))
+    fi
+done
+[ "$bad" -eq 0 ] && echo "  all service units enabled"
+
+echo
+echo "== nightly image updates (replaces watchtower — AGENTS.md §13.15) =="
+if [ "$(systemctl --user is-enabled podman-update.timer 2>/dev/null)" = "enabled" ]; then
+    echo "  timer enabled, next: $(systemctl --user list-timers podman-update.timer --no-pager 2>/dev/null | awk 'NR==2{print $1,$2,$3,$4}')"
+else
+    echo "  WARNING: podman-update.timer is NOT enabled — nothing will update"
+fi
+if [ "$(systemctl --user is-active podman-watchtower.service 2>/dev/null)" = "active" ]; then
+    echo "  WARNING: watchtower is RUNNING — it recreates containers outside their pod (§13.13)"
+fi
+# A container outside its compose pod is the signature of a watchtower-style recreate.
+podless=""
+for c in $(podman ps --format '{{.Names}}' 2>/dev/null); do
+    [ -z "$(podman inspect "$c" --format '{{.Pod}}' 2>/dev/null)" ] && podless="$podless $c"
+done
+[ -n "$podless" ] && echo "  PODLESS (rebuild with systemctl --user restart):$podless" || echo "  all containers in their pod"
+
+echo
+echo "== admin ports bound to 0.0.0.0 (should be 192.168.2.2 — AGENTS.md §13.2) =="
+podman ps --format '{{.Names}} {{.Ports}}' | grep -E '0\.0\.0\.0:(81|80-81|3001|9443|5432)(->|-)' \
+    | sed 's/^/  EXPOSED: /' || echo "  none"
+
+echo
+if [[ $fail -eq 0 ]]; then
+  printf '\033[32mALL CHECKS PASSED\033[0m\n'
+else
+  printf '\033[31m%d CHECK(S) FAILED\033[0m\n' "$fail"
+  echo "Investigate: journalctl --user -u podman-<name>.service -n 50"
+fi
+exit $fail

setup/scripts/link-mcp

@@ -23,10 +23,13 @@
 # failure mode than leaving two lines for them to paste once.
 #
 # opencode has no CLI for this either, but its config is plain JSON, so this
-# stage merges two keys into it with jq: one mcp entry per server row, and the
-# rules directory under instructions, because opencode reads ~/.claude/CLAUDE.md
-# and ~/.claude/skills on its own but not ~/.claude/rules. Every other key in
-# that file is left as it was.
+# stage writes it whole from user/agents/opencode/config.json -- the tracked
+# template carrying everything that is not a secret -- plus the MCP block built
+# here from the server rows. The whole file is generated rather than merged into
+# because the point of the template is that every machine ends up with the same
+# opencode experience, and a merged file drifts the moment one machine is edited
+# by hand. The output says so at the top, and anything worth changing goes in
+# the template, where it reaches the other machines too.
 
 set -euo pipefail
 
@@ -37,6 +40,8 @@ PANAMA_PATH="${PANAMA_PATH:-$HOME/.local/share/Panama}"
 MCP_DIR="$PANAMA_PATH/user/agents/mcp"
 SERVERS="$MCP_DIR/servers"
 ENV_FILE="$MCP_DIR/env"
+TEMPLATE="$PANAMA_PATH/user/agents/opencode/config.json"
+PANAMA_OLD="$PANAMA_PATH/config/old"
 STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/panama"
 DECISION="$STATE_DIR/user-content"
 
@@ -177,29 +182,62 @@ fi
 
 $has_opencode || exit 0
 
-# opencode reads opencode.jsonc or opencode.json, whichever this machine has.
-# The file holds tokens once this runs, so it is written private and swapped in
-# whole, the way ~/.claude.json already holds them for Claude Code.
+# opencode reads opencode.jsonc. The file holds tokens once this runs, so it is
+# written private and swapped in whole, the way ~/.claude.json already holds
+# them for Claude Code.
 OPENCODE_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/opencode"
 opencode_config="$OPENCODE_DIR/opencode.jsonc"
-[[ -e "$opencode_config" ]] || opencode_config="$OPENCODE_DIR/opencode.json"
-rules="$HOME/.claude/rules/*.md"
 
-mkdir -p "$OPENCODE_DIR"
-current='{"$schema": "https://opencode.ai/config.json"}'
-[[ -e "$opencode_config" ]] && current="$(cat "$opencode_config")"
-
-# jq cannot read JSONC comments. A file that has them is left alone rather
-# than rewritten without them.
-if ! merged="$(jq --argjson mcp "$opencode_mcp" --arg rules "$rules" \
-    '.mcp = ((.mcp // {}) + $mcp) | .instructions = (((.instructions // []) - [$rules]) + [$rules])' \
-    <<<"$current" 2>/dev/null)"; then
-    warn "Could not parse $opencode_config as JSON; opencode MCP servers not written."
+[[ -r "$TEMPLATE" ]] || { warn "No $TEMPLATE; opencode config not written."; exit 0; }
+if ! jq -e . "$TEMPLATE" >/dev/null 2>&1; then
+    warn "$TEMPLATE is not valid JSON; opencode config not written."
     exit 0
 fi
 
+# Nothing at the destination is destroyed, the promise link-user and link-skills
+# make. A file this stage did not write is somebody's own work and goes to
+# config/old/ under a name that says where it came from. One carrying the marker
+# was written by an earlier run and is replaced in place, so re-running never
+# buries its own output and config/old/ does not grow on every upgrade.
+MARKER='generated by setup/scripts/link-mcp'
+displace() {
+    local destination="$1" backup
+    [[ -e "$destination" || -L "$destination" ]] || return 0
+    if [[ -L "$destination" ]]; then
+        rm -f "$destination"
+        return 0
+    fi
+    grep -qF "$MARKER" "$destination" 2>/dev/null && return 0
+    backup="$PANAMA_OLD/mcp-$(printf '%s' "${destination#"$HOME"/}" | tr '/' '-')"
+    [[ -e "$backup" ]] && backup="$backup.$(date +%s)"
+    mv "$destination" "$backup"
+    log "Moved existing $destination to $backup"
+}
+
+mkdir -p "$OPENCODE_DIR" "$PANAMA_OLD"
+# Two names for one file would leave opencode choosing between them, and the one
+# this stage does not write would win or lose on discovery order.
+displace "${opencode_config%.jsonc}.json"
+displace "$opencode_config"
+
+# The template is portable prose: it names the rules directory as ~/ so it does
+# not carry this machine's login. opencode resolves a leading ~/ on its own, but
+# expanding it here keeps the generated file readable the way a person would
+# have written it by hand when they went looking for where the path came from.
+merged="$(jq --argjson mcp "$opencode_mcp" --arg home "$HOME" \
+    '.mcp = $mcp
+     | .instructions = [(.instructions // [])[]
+                         | if startswith("~/") then $home + .[1:] else . end]' \
+    <<<"$(cat "$TEMPLATE")")"
+
 tmp="$(mktemp "$OPENCODE_DIR/.opencode-config.XXXXXX")"
+{
+    printf '// %s\n' "$MARKER"
+    printf '// Edit user/agents/opencode/config.json in the Panama checkout instead.\n'
+    printf '// This file is replaced whenever ./install or panama update runs.\n'
+    printf '%s\n' "$merged"
+} >"$tmp"
 chmod 600 "$tmp"
-printf '%s\n' "$merged" >"$tmp"
 mv "$tmp" "$opencode_config"
-log "opencode: $(jq -r 'length' <<<"$opencode_mcp") MCP servers and the rules written to $opencode_config."
+
+log "opencode: $(jq -r 'length' <<<"$opencode_mcp") MCP servers and the template written to $opencode_config."

tests/contracts.manifest

@@ -270,6 +270,7 @@ hermetic tests/setup/launcher-search-contract
 hermetic tests/setup/lid-contract
 hermetic tests/setup/migrations-contract
 hermetic tests/setup/nautilus-extensions-contract
+hermetic tests/setup/opencode-config-contract
 hermetic tests/setup/package-lists-contract
 # Provenance uses local signed fixtures and stubs every network/package operation.
 hermetic tests/setup/package-provenance-contract

tests/quickshell/declared-dependencies-contract

@@ -59,7 +59,10 @@ SELF_INSTALLED='^(bun|claude|codex|node|npm|pnpm)$'
 # fresh machine to keep five aliases company. Where docker is absent the
 # aliases fail by naming the missing command, which is the honest outcome.
 # Anything added here needs that same property: absence must be loud.
-OPTIONAL='^(docker)$'
+# opencode is the other kind of optional: link-mcp probes for it with
+# `command -v` and writes its config only where someone installed it with
+# opencode's own installer, the same way it treats claude.
+OPTIONAL='^(docker|opencode)$'
 
 # Provided by a package swap rather than a list entry. ffmpeg cannot be a list
 # entry: RPM Fusion's ffmpeg CONFLICTS with the ffmpeg-free that Fedora ships

tests/server/containers-shape-contract

@@ -3,9 +3,12 @@
 # Every service in the catalog has the same shape, because panama-server and
 # the update timer both depend on it blindly:
 #
-#   * a compose.yml, which is the only file enable symlinks into ~/Server
+#   * a compose.yml, linked into ~/Server along with any helper script
 #   * exactly one podman-*.service unit -- its filename is the unit's
-#     identity, so two would be an ambiguity and zero an unenableable service
+#     identity, so two would be an ambiguity and zero an unenableable service.
+#     Other *.service and *.timer files are helpers (a cert refresh, say)
+#     that enable links beside it; each helper service needs a timer or a
+#     unit that pulls it in, so an orphan helper is flagged
 #   * a WorkingDirectory of %h/Server/<Name>, matching the directory enable
 #     creates -- a unit pointing anywhere else starts compose against a
 #     directory that has no .env and no data
@@ -31,20 +34,21 @@ for dir in "$containers_dir"/*/; do
 
     [[ -f "$dir/compose.yml" ]] || note "$name has no compose.yml"
 
-    units=("$dir"/*.service)
+    units=("$dir"/podman-*.service)
     if (( ${#units[@]} == 0 )); then
-        note "$name has no unit file"
+        note "$name has no podman-<name>.service unit"
         continue
     elif (( ${#units[@]} > 1 )); then
-        note "$name has ${#units[@]} unit files; its identity is ambiguous"
+        note "$name has ${#units[@]} podman-*.service units; its identity is ambiguous"
         continue
     fi
     unit="${units[0]}"
 
-    case "$(basename "$unit")" in
-        podman-*.service) ;;
-        *) note "$name's unit is not named podman-<name>.service: $(basename "$unit")" ;;
-    esac
+    for helper in "$dir"/*.service; do
+        [[ "$helper" == "$unit" ]] && continue
+        [[ -f "${helper%.service}.timer" ]] \
+            || note "$name's helper $(basename "$helper") has no matching .timer"
+    done
 
     grep -q "^WorkingDirectory=%h/Server/$name\$" "$unit" \
         || note "$name's unit does not work in %h/Server/$name"

tests/server/panama-server-contract

@@ -6,9 +6,10 @@
 #   * enable seeds .env from .env.example and REFUSES to start while a
 #     CHANGE_ME placeholder survives -- a service brought up with the
 #     placeholder as its database password runs, wrongly, forever
-#   * a completed enable links exactly the tracked files (compose.yml and the
-#     unit), reloads systemd, and enables the unit
-#   * disable removes the unit and nothing else: the .env and data stay
+#   * a completed enable links the tracked files (compose.yml, helper scripts,
+#     the unit and any helper units), reloads systemd, enables the unit and
+#     each helper timer, and never links .env.example
+#   * disable removes the units and nothing else: the .env and data stay
 #   * relink refreshes links for enabled services only, and names a service
 #     whose definition changed since the machine last looked -- without
 #     restarting anything
@@ -55,6 +56,21 @@ RemainAfterExit=yes
 [Install]
 WantedBy=default.target
 UNIT
+cat >"$catalog/Example/refresh-example" <<'SH'
+#!/usr/bin/env bash
+SH
+chmod +x "$catalog/Example/refresh-example"
+cat >"$catalog/Example/example-refresh.service" <<'UNIT'
+[Service]
+Type=oneshot
+ExecStart=%h/Server/Example/refresh-example
+UNIT
+cat >"$catalog/Example/example-refresh.timer" <<'UNIT'
+[Timer]
+OnCalendar=daily
+[Install]
+WantedBy=timers.target
+UNIT
 
 # The stub records every invocation; is-active answers 'inactive' so nothing
 # here depends on a real systemd.
@@ -81,6 +97,8 @@ run() {
 }
 
 unit_link="$home/.config/systemd/user/podman-example.service"
+helper_link="$home/.config/systemd/user/example-refresh.service"
+timer_link="$home/.config/systemd/user/example-refresh.timer"
 
 # ── enable stops on placeholders ─────────────────────────────────────────────
 
@@ -111,6 +129,14 @@ grep -q 'systemctl --user enable --now podman-example.service' "$calls" \
     || note 'enable did not enable --now the unit'
 [[ "$(cat "$home/Server/Example/.env")" == "DB_PASSWORD=s3cret" ]] \
     || note 'enable rewrote an .env that already existed'
+[[ -L "$home/Server/Example/refresh-example" ]] \
+    || note 'enable did not link the helper script into ~/Server'
+[[ -e "$home/Server/Example/.env.example" ]] \
+    && note 'enable linked .env.example into ~/Server'
+[[ -L "$helper_link" && -L "$timer_link" ]] \
+    || note 'enable did not link the helper units'
+grep -q 'systemctl --user enable --now example-refresh.timer' "$calls" \
+    || note 'enable did not enable the helper timer'
 
 # ── relink names a changed definition, restarts nothing ──────────────────────
 
@@ -136,6 +162,10 @@ run disable Example >/dev/null 2>&1 || note 'disable failed'
 
 [[ -e "$unit_link" ]] \
     && note 'disable left the unit installed'
+[[ -e "$helper_link" || -e "$timer_link" ]] \
+    && note 'disable left a helper unit installed'
+grep -q 'systemctl --user disable --now example-refresh.timer' "$calls" \
+    || note 'disable did not stop the helper timer'
 grep -q 'systemctl --user disable --now podman-example.service' "$calls" \
     || note 'disable did not stop the unit'
 [[ -f "$home/Server/Example/.env" ]] \

tests/setup/opencode-config-contract

@@ -0,0 +1,159 @@
+#!/usr/bin/env bash
+
+# The opencode config, generated rather than linked.
+#
+# Every other personal destination is a symlink or a one-time copy, so getting
+# one wrong is visible and cheap to undo. This one is written whole by
+# setup/scripts/link-mcp, and it is the only file in the set that carries
+# secrets -- which is the reason it is generated at all. Three properties are
+# worth pinning:
+#
+#   1. The tracked template is the source of truth and holds no secret. It is
+#      readable by anyone who finds the repository, so a token in it is a
+#      disclosure rather than a mistake.
+#   2. The generation is faithful: the template's keys survive, the MCP block
+#      appears, and a leading ~/ is resolved so the output is readable.
+#   3. Nothing is destroyed and nothing churns. A file somebody wrote by hand
+#      goes to config/old/, and running the stage again does not fill it up.
+
+set -uo pipefail
+
+repo_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+linker="$repo_dir/setup/scripts/link-mcp"
+template="$repo_dir/user/agents/opencode/config.json"
+servers="$repo_dir/user/agents/mcp/servers"
+
+findings=()
+note() { findings+=("$1"); }
+
+[[ -x "$linker" ]] || { printf 'opencode config contract: %s is not executable\n' "$linker" >&2; exit 1; }
+
+# ── 1. The template holds no secret ──────────────────────────────────────────
+
+if [[ ! -r "$template" ]]; then
+    note 'user/agents/opencode/config.json is missing, so nothing generates an opencode config'
+else
+    jq -e . "$template" >/dev/null 2>&1 \
+        || note 'user/agents/opencode/config.json is not valid JSON'
+
+    if grep -qE 'BEGIN [A-Z ]*PRIVATE KEY|sk-ant-[A-Za-z0-9]|ghp_[A-Za-z0-9]{20}|xox[baprs]-[A-Za-z0-9]' "$template" 2>/dev/null; then
+        note 'user/agents/opencode/config.json carries something that looks like a token'
+    fi
+fi
+
+[[ -r "$servers" ]] || note 'user/agents/mcp/servers is missing, so no MCP servers can be registered'
+
+# ── 2 and 3, against a throwaway home ────────────────────────────────────────
+
+work="$(mktemp -d)"
+trap 'rm -rf "$work"' EXIT
+
+checkout="$work/Panama"
+mkdir -p "$checkout/setup/scripts" "$checkout/user/agents/mcp/bin" "$checkout/user/agents/opencode"
+cp "$linker" "$checkout/setup/scripts/link-mcp"
+cp "$template" "$checkout/user/agents/opencode/config.json"
+printf '#!/usr/bin/env bash\nexit 0\n' >"$checkout/user/agents/mcp/bin/gamma"
+chmod +x "$checkout/user/agents/mcp/bin/gamma"
+cat >"$checkout/user/agents/mcp/servers" <<'FIXTURE'
+# a comment, then a row that needs a token, one that needs none, and a stdio one
+
+alpha  http  https://alpha.example/mcp  ALPHA_TOKEN
+beta   http  https://beta.example/mcp
+gamma  stdio  bin/gamma
+FIXTURE
+printf 'ALPHA_TOKEN=sekrit-token-value\n' >"$checkout/user/agents/mcp/env"
+
+# link-mcp registers through whichever runtime it finds, and gives up entirely
+# when it finds neither. A stub is enough to prove the config generation.
+mkdir -p "$work/bin"
+printf '#!/usr/bin/env bash\nexit 0\n' >"$work/bin/opencode"
+chmod +x "$work/bin/opencode"
+
+home="$work/home"
+state="$work/state"
+old="$checkout/config/old"
+# Every XDG directory is pinned to the throwaway home. XDG_CONFIG_HOME is
+# usually set in the environment this runs from, and leaving it alone would send
+# the generated config at the real ~/.config/opencode rather than at the fixture.
+mkdir -p "$home/.config/opencode" "$state/panama"
+
+run() {
+    HOME="$home" XDG_CONFIG_HOME="$home/.config" XDG_STATE_HOME="$state" \
+        PANAMA_PATH="$checkout" PATH="$work/bin:$PATH" \
+        "$checkout/setup/scripts/link-mcp" >"$work/log" 2>&1
+}
+
+generated="$home/.config/opencode/opencode.jsonc"
+
+# A file somebody wrote by hand is there before this stage ever runs.
+printf '{"model": "someone-elses-hand-edit"}\n' >"$generated"
+
+printf 'yes\n' >"$state/panama/user-content"
+if ! run; then
+    note "link-mcp failed against a throwaway home: $(tail -1 "$work/log")"
+fi
+
+# ── The hand-edit is preserved, not deleted ──────────────────────────────────
+
+if ! grep -rq 'someone-elses-hand-edit' "$old" 2>/dev/null; then
+    note 'a config written by hand was not preserved in config/old/'
+fi
+
+# ── The generation is faithful ───────────────────────────────────────────────
+
+body="$(sed '/^\/\//d' "$generated" 2>/dev/null)"
+
+jq -e . <<<"$body" >/dev/null 2>&1 \
+    || note 'the generated opencode config is not valid JSON once its header comment is stripped'
+grep -qF 'generated by setup/scripts/link-mcp' "$generated" 2>/dev/null \
+    || note 'the generated opencode config does not say who wrote it or where to edit instead'
+
+if ! jq -e --arg home "$home" '.instructions | index($home + "/.claude/rules/*.md")' <<<"$body" >/dev/null 2>&1; then
+    note 'the generated opencode config did not resolve the ~/ rules path to this home'
+fi
+jq -e '.permissions | length > 0' <<<"$body" >/dev/null 2>&1 \
+    || note 'the template permissions did not reach the generated opencode config'
+
+jq -e '.mcp.alpha.headers.Authorization == "sekrit-token-value"' <<<"$body" >/dev/null 2>&1 \
+    || note 'a token-carrying server was registered without its Authorization header'
+jq -e '.mcp.beta | has("headers") | not' <<<"$body" >/dev/null 2>&1 \
+    || note 'a server that needs no token was given a header anyway'
+jq -e --arg launcher "$checkout/user/agents/mcp/bin/gamma" '.mcp.gamma.command == [$launcher]' <<<"$body" >/dev/null 2>&1 \
+    || note 'a stdio server was not registered against its launcher in this checkout'
+
+# The file holds tokens, so it is private. Checked on the file rather than on
+# the umask, because the umask is not what a later reader sees.
+mode="$(stat -c '%a' "$generated" 2>/dev/null)"
+[[ "$mode" == 600 ]] || note "the generated opencode config is mode $mode, not 600"
+
+# The template is readable by anybody who finds the repository. The tokens live
+# in the generated file and in user/agents/mcp/env, never here.
+if grep -qF 'sekrit-token-value' "$template" 2>/dev/null; then
+    note 'the tracked opencode template carries an MCP token'
+fi
+
+# ── Re-running replaces, and does not churn ──────────────────────────────────
+
+before="$(find "$old" -type f 2>/dev/null | wc -l)"
+run
+after="$(find "$old" -type f 2>/dev/null | wc -l)"
+[[ "$before" == "$after" ]] \
+    || note 're-running link-mcp buried its own output in config/old/, so it grows every upgrade'
+grep -qF 'generated by setup/scripts/link-mcp' "$generated" 2>/dev/null \
+    || note 're-running link-mcp did not rewrite the config it generates'
+
+# A machine that never said yes is left alone, the same gate every other
+# personal destination obeys.
+rm -f "$generated"
+printf 'no\n' >"$state/panama/user-content"
+run
+[[ ! -e "$generated" ]] \
+    || note 'an opencode config was generated on a machine that never said yes'
+
+if (( ${#findings[@]} > 0 )); then
+    printf 'opencode config contract: %d finding(s)\n' "${#findings[@]}" >&2
+    printf '  - %s\n' "${findings[@]}" >&2
+    exit 1
+fi
+
+printf 'opencode config contract: PASS\n'

user/README.md

@@ -14,8 +14,12 @@ and one file says where each piece goes.
 | --- | --- | --- |
 | `agents/AGENTS.md` | `~/.claude/CLAUDE.md`, `~/.codex/AGENTS.md`, `~/.config/opencode/AGENTS.md` | Three tools, one file. These were byte-identical copies before this, waiting to disagree. |
 | `agents/skills/` | `~/.agents/skills`, `~/.claude/skills` | Personal skills live once in the checkout. Both destinations use `linkdir` so Panama's shipped skills can live beside them. opencode reads both directories itself. |
-| `agents/rules/` | `~/.claude/rules` | opencode loads these through `instructions` in its config, which `setup/scripts/link-mcp` writes. |
+| `agents/rules/` | `~/.claude/rules` | opencode loads these through `instructions` in its config, which `setup/scripts/link-mcp` generates. |
+| `agents/commands/` | `~/.config/opencode/commands` | Claude Code carries its commands as plugins, which opencode cannot read. These are the same commands in opencode's own format. |
+| `agents/agents/` | `~/.config/opencode/agents` | The same again for subagents. |
+| `agents/opencode/config.json` | read by `setup/scripts/link-mcp` | Not linked. The opencode config is generated from it, because that file also has to carry the MCP servers and their tokens, and a symlink cannot be half written. |
 | `ssh/config` | `~/.ssh/config` | Host aliases only. Keys are per-machine and are never tracked. |
+| `bin/pwswitch` | `~/.local/bin/pwswitch` | `pwswitch off` makes sudo passwordless for an agent session, `pwswitch on` restores the password. |
 | `espanso/identity.yml` | `~/.config/espanso/match/identity.yml` | Copied, not linked, because a machine may add its own triggers. |
 
 `manifest` is the authority; this table is a summary of it.
@@ -44,6 +48,23 @@ Two consequences, recorded rather than fixed:
   A tool that installs directly into a home skill directory creates a
   machine-local skill until it is moved into the checkout.
 
+## What link-mcp generates
+
+The opencode config is the one destination here that is generated rather than linked or
+copied. It cannot be a symlink because it has to carry the MCP servers and their tokens,
+and a token is exactly what cannot live in this repository. So `setup/scripts/link-mcp`
+builds the whole file: `agents/opencode/config.json` supplies everything that is not a
+secret, `agents/mcp/servers` supplies the server list, and the machine-local
+`agents/mcp/env` supplies the tokens.
+
+The result is `~/.config/opencode/opencode.jsonc`, written private and replaced in
+place. It opens with a comment saying where to edit it, and that comment is the whole
+warning: **hand-edits to the generated file do not survive an upgrade.** Put the change in
+`agents/opencode/config.json` and it reaches every machine.
+
+Anything already at that path that this stage did not write moves to `config/old/`, the
+same promise everything else here makes.
+
 ## It is off unless you say yes
 
 The installer asks, naming the destinations, and the default is no. Nothing here

user/agents/AGENTS.md

@@ -105,6 +105,68 @@ and record who decided what, because that history is useful.
   - When it comes to personal projects, I am much less strict, but I prefer a title that makes it very clear what the MR is about.
     - No confusing or complicated language.
 
+---
+
+# Operating context
+
+Where things live, what is sensitive, and who may see it. This used to sit in one tool's
+settings file, so only whichever agent read that file ever knew it.
+
+## Source control and hosting
+
+- I keep my own projects on `git.gbrown.org`, my self-hosted Git host. Ksense work lives in
+  the client's own GitLab, and that is where the merge requests go.
+- `agentchat.gbrown.org` is my agent-to-agent message hub. It is personal infrastructure,
+  and it is not a place to publish anything sensitive outside the trust boundary.
+- I have no internal package registry and no trusted cloud buckets. Nothing gets uploaded
+  to a bucket.
+
+## Secrets
+
+- Infisical carries staging and development secrets.
+- `JIRA_CREDENTIALS` comes from `~/.bashrc`. Jira ticket work always uses my own
+  credentials, never the Jira credentials staging Infisical carries.
+- `.claude/docs/env/prod/.env` and `.claude/docs/env/ci/.env` are read-only, and every
+  read needs explicit permission. Ask every time, even when I approved the same read
+  earlier in the same session.
+
+## Sensitive locations
+
+These are mine alone plus the cleared Ksense coworkers named at the end of this file, and
+only at a "named and specific" bar, meaning I say which file and why. They never go to a
+paste site, a snippet host, or an external tool:
+
+- Jira credentials and the prod/CI env files above.
+- `~/.infisical/secrets-backup/`.
+- `~/.config/context7/credentials.json`.
+- `~/.claude/.credentials.json`.
+- Client project env files, such as `~/Documents/Code/GibSend/.env*` and
+  `~/Documents/Code/StPeteIT/.env*`.
+
+## Network
+
+My personal homelab. Everything under `gbrown.org` is inside the trust boundary:
+`plex.`, `git.`, `chat.`, `radarr.`, `immich.`, `jellyfin.`, `home.`, and
+`agentchat.`. Local hosts are `macnano.gib` and `nano.gib`, and the LAN is
+`192.168.1.2` and `192.168.2.2`. The NanoKVM boards on those two hosts give an agent
+console access to the machine, including what SSH cannot reach, so treat that access as
+root-equivalent.
+
+## Targets that need my explicit say-so
+
+- Any host, namespace, or resource where `prod` or `production` is a whole word or a whole
+  name segment. Command Center production is the one I care about most.
+- IAM, RBAC, networking, quota, and node-pool resources in any IaC, and anything whose
+  name or tag carries `prod` or `production`.
+
+## CLIs I expect you to know
+
+- `ctx7` for current library documentation.
+- `infisical` for secrets.
+- `claude`, `clauded`, and `claudedr` are Claude Code wrappers.
+
+---
+
 # Coworkers & Others that you will be working with & Interacting with when working on Ksense Projects
 
 - Conrad Rohleder - Project Manager
@@ -115,7 +177,8 @@ and record who decided what, because that history is useful.
     - For example, he once asked for a JSON blob of our schema so he could better understand it.
   - Conrad is awesome & is great to work with.
   - He is somewhat skeptical of AI & its ability to complete work.
-- Henry Nguyen - Tech Owner (Tech Lead)
+- Henry Nguyen - Tech Owner (Tech Lead) for Command Center only
+  - Henry is only involved in Command Center. On other projects (for example 360 Shop View), don't route reviews or sign-offs through him.
   - Henry is who I interact with second most, just behind Conrad.
   - Henry reviews all of my code & up until recently, would also write all the dev reviews for all the stories I completed as well. Nowadays, I write them myself & he reviews those too.
   - Henry likes work to be very considerate & he always prefers solutions that result in 0 downtime.

user/agents/agents/code-simplifier.md

@@ -0,0 +1,35 @@
+---
+description: Simplify recently changed code for clarity while leaving behaviour identical
+mode: subagent
+---
+
+You simplify code. You never change what it does, only how it does it.
+
+Work only on code changed in this session, or on the specific files you are pointed at.
+Anything wider is out of scope unless you are asked for it.
+
+What you do:
+
+- Cut complexity that is not earning its keep: nesting that flattens, abstractions with one
+  caller, names that need a comment to be understood, values threaded through layers that
+  never use them.
+- Keep the behaviour exactly as it was. Same outputs, same errors, same edge cases, same
+  failure modes.
+- Follow the guidance that governs the code you are touching. Where that guidance and your
+  own instinct disagree, the guidance wins.
+- Prefer explicit over compact. A clear line beats a clever one, and a name that says what
+  something is beats a shorter name that does not.
+- Delete comments that restate the code. Keep comments that explain a reason, and update
+  them when the code they describe changed.
+
+What you do not do:
+
+- You do not refactor beyond the change. Renaming a public API, reshuffling a module, or
+  tidying a file you were not asked about is somebody else's job.
+- You do not merge concerns into one expression to save a line. Nested ternaries and dense
+  one-liners are not simplification.
+- You do not remove an abstraction just because it has one caller today.
+- You do not add tests, types, or documentation as a side effect of simplifying.
+
+Report what you changed and why, briefly. When a simplification you considered would have
+changed behaviour, say that you left it alone and why.

user/agents/commands/code-review.md

@@ -0,0 +1,38 @@
+---
+description: Review a diff or merge request for real bugs and missed requirements
+---
+
+Review the change for real problems. Do not edit anything.
+
+Target: $ARGUMENTS
+
+When the target names a merge request or pull request, review that change. When it names a
+file or directory, review what is uncommitted there. When it is empty, review the current
+working diff.
+
+How to review:
+
+1. Read the change, then read the code around it. Context you did not read is context you
+   will misjudge.
+2. Look for real problems only: logic that is wrong, a case that was missed, a contract
+   broken, data lost, an error swallowed, a race introduced, something the change claims to
+   do that it does not do.
+3. Check the change against whatever guidance governs the code it touches. Guidance is
+   guidance for writing code, so not all of it applies to a review.
+4. Read the git history of the lines being modified. A change that looks fine in isolation
+   often breaks a reason the code is the way it is.
+5. Leave these out: style, naming, formatting, import order, missing tests, missing
+   documentation, and anything a linter, typechecker, or compiler will catch. CI runs those.
+   Also leave out problems on lines the change did not touch.
+
+Output:
+
+- Order findings by severity, highest first.
+- Each finding gets the file and line, what is wrong, and why it matters. Link to code
+  rather than quoting long stretches of it.
+- Say which requirement or guideline is broken when that is the reason.
+- When a finding is a judgement call rather than a definite bug, say so plainly instead of
+  dressing it up as certain.
+- If there is nothing worth reporting, say that and stop. Do not invent findings to fill
+  the space.
+

user/agents/mcp/bin/playwright

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# Playwright's MCP server, for driving a real browser: pages, screenshots,
+# forms, end-to-end runs. This is what the ticket skill's proof screenshots are
+# captured with, so it belongs on every machine rather than only where a Claude
+# Code plugin happened to be installed.
+#
+# link-mcp registers this file as a stdio server. It takes no token, so its
+# row in user/agents/mcp/servers names no variable and nothing arrives in
+# MCP_TOKEN.
+set -euo pipefail
+exec npx -y @playwright/mcp@latest

user/agents/mcp/servers

@@ -38,6 +38,11 @@ penpot          http  https://penpot.gbrown.org/mcp/stream?userToken={TOKEN}  PE
 # The hosted mcp.sentry.dev only speaks to sentry.io.
 sentry-gbrown   stdio  bin/sentry-gbrown  SENTRY_GBROWN_TOKEN
 
+# Playwright's MCP server. Browser automation: pages, screenshots, forms,
+# end-to-end runs. The ticket skill captures its proof screenshots this way.
+# No token, hence no variable in the last column.
+playwright      stdio  bin/playwright
+
 # T3 Code environments, one per machine. Agents read projects and threads and,
 # with more than read-only access, start and message threads there. No token:
 # each is an OAuth server that asks for a pairing code from that machine on

user/agents/opencode/config.json

@@ -0,0 +1,15 @@
+{
+  "$schema": "https://opencode.ai/config.json",
+  "instructions": ["~/.claude/rules/*.md"],
+  "permissions": [
+    { "action": "read", "resource": "/etc/**", "effect": "allow" },
+    { "action": "shell", "resource": "coredumpctl list*", "effect": "allow" },
+    { "action": "shell", "resource": "coredumpctl info*", "effect": "allow" },
+    { "action": "shell", "resource": "journalctl*", "effect": "allow" },
+    { "action": "shell", "resource": "rpm -q*", "effect": "allow" },
+    { "action": "shell", "resource": "panama doctor*", "effect": "allow" },
+    { "action": "shell", "resource": "lspci*", "effect": "allow" },
+    { "action": "shell", "resource": "lsusb*", "effect": "allow" },
+    { "action": "shell", "resource": "getent hosts*", "effect": "allow" }
+  ]
+}

user/agents/skills/frontend-design/SKILL.md

@@ -0,0 +1,71 @@
+---
+name: frontend-design
+description: Make deliberate visual design choices for new UI or a reshaping of an existing one, and run the mock-then-decide workflow before touching real components.
+---
+
+# Frontend Design
+
+Distinctive UI comes from decisions made on purpose. Templated UI comes from decisions made
+by habit. This is about catching the second kind before it ships.
+
+## Start from the thing itself
+
+If the brief does not say what the product is, who it is for, and what the screen is
+supposed to accomplish, work that out first and confirm it before designing anything. A
+dashboard for people reconciling invoices and a booking page for a surf school want
+different palettes, different type, different density, and pretending otherwise is how you
+end up with the same page twice.
+
+Name the subject in one sentence, name the audience in one, name the job of the screen in
+one. Design against those three sentences. When a choice cannot be justified from them, it
+is decoration.
+
+## Choices to make deliberately
+
+**Type carries the personality.** Pick the family on purpose rather than reaching for the
+usual one. Decide a scale up front and hold to it, so a heading is a heading everywhere and
+not a heading in one place and a slightly larger paragraph in another. You rarely need more
+than two families.
+
+**Colour is a system, not a pick.** Name the roles before the values: what is background,
+what is surface, what is text, what is the one accent. The accent is the only one that gets
+to be loud, and it earns that by being used sparingly.
+
+**Structure is information.** Borders, rules, labels, numbering, eyebrows and dividers
+should be telling the reader something about the content. If a device is only there to fill
+a gap, it is ornament. Numbered markers in particular only make sense when the content is
+genuinely a sequence.
+
+**Motion answers the user.** Motion that reacts to what somebody did is useful, because it
+shows what changed. Motion that runs on its own is filling space. Do not use continuously
+repainting effects such as pulse, shimmer, blur or spinners. They peg the GPU on high
+refresh displays and read as generated.
+
+## Tells to avoid
+
+These are the shapes that show up again and again in generated UI, and they are the fastest
+way to make a page look like nobody chose it:
+
+- A cream background with a serif display face and a warm terracotta accent.
+- A near-black background with one acid green or vermilion accent.
+- A headline with exactly one word italicised, bolded or coloured.
+- All-caps labels used as decoration rather than because the label is short and needs it.
+- A label above every block of content, naming what the content already makes obvious.
+- The same fade-and-slide entrance applied to every section, and a hover lift on every card.
+- A hero that is a big number, a small label and a gradient, when the subject has something
+  more characteristic to show.
+
+## Workflow
+
+This is the rule from my preferences, spelled out:
+
+1. Build several genuinely distinct directions, not three variations of one idea. Different
+   palettes, different type, different structure. If two mocks would be described the same
+   way in one sentence, they are the same mock.
+2. Build them as static pages. Real content, not grey boxes labelled "image here", because
+   the content is where most of the design decisions turn out to matter.
+3. Publish them with a simple http server, report the URL, and stop.
+4. Wait for a decision before implementing anything in real components.
+
+Do not edit real components first. Do not keep polishing past the point where a choice
+could be made, and do not pick a winner yourself and build it.

user/bin/pwswitch

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Toggle passwordless sudo for the current user.
+#   pwswitch on   -> sudo asks for a password again (default behavior)
+#   pwswitch off  -> sudo runs without a password
+# Works by adding/removing a drop-in file in /etc/sudoers.d.
+set -euo pipefail
+
+user="${SUDO_USER:-$USER}"
+# "zz-" prefix so it sorts last and overrides earlier rules.
+dropin="/etc/sudoers.d/zz-passwordswitch-${user}"
+
+case "${1:-}" in
+off)
+  tmp="$(mktemp)"
+  trap 'rm -f "$tmp"' EXIT
+  echo "${user} ALL=(ALL) NOPASSWD: ALL" >"$tmp"
+  # Validate before installing so a bad file can never lock sudo.
+  sudo visudo -cqf "$tmp"
+  sudo install -m 0440 -o root -g root "$tmp" "$dropin"
+  echo "Passwordless sudo ENABLED for ${user}. Run '$(basename "$0") on' when done."
+  ;;
+on)
+  sudo rm -f "$dropin"
+  sudo -k
+  echo "Password required for sudo again."
+  ;;
+*)
+  echo "Usage: $(basename "$0") on|off" >&2
+  exit 1
+  ;;
+esac

user/manifest

@@ -46,9 +46,22 @@ link  agents/rules       ~/.claude/rules
 # what the SSH Hosts launcher command reads.
 link  ssh/config         ~/.ssh/config
 
+# ── Scripts ──────────────────────────────────────────────────────────────────
+# Personal commands wanted on every machine, desktop or server. ~/.local/bin is
+# on PATH in both shell configs. pwswitch toggles passwordless sudo through a
+# sudoers.d drop-in, so an agent can run sudo and the password comes back after.
+link  bin/pwswitch       ~/.local/bin/pwswitch
+
 # ── Espanso ──────────────────────────────────────────────────────────────────
 # The personal expansion triggers. Copied rather than linked, because this is a
 # file each machine may add its own triggers to. It lands before setup-identity
 # runs, and that stage only seeds a file that does not exist yet, so what is
 # tracked here wins over the name and email the interview would have written.
 copy  espanso/identity.yml  ~/.config/espanso/match/identity.yml
+
+# ── OpenCode ─────────────────────────────────────────────────────────────────
+# Claude Code carries its commands and agents as plugins, which opencode cannot
+# read. These two directories hold the same capabilities in opencode's own
+# format, so the command I type in one tool exists in the other.
+linkdir  agents/commands  ~/.config/opencode/commands
+linkdir  agents/agents    ~/.config/opencode/agents

user/ssh/config

@@ -1,3 +1,5 @@
+Include ~/.ssh/config.local
+
 Host server.gib
     HostName server.gib
     User gib