Let a server service carry helper scripts and timers, starting with UseSend's cert refresh

bin/panama-server

@@ -4,8 +4,10 @@
 #
 # The repository carries the catalog: server/vps/<Name>/ holds a
 # compose.yml, a podman-<name>.service user unit, and an .env.example naming
-# what the service needs told. The machine carries the rest: ~/Server/<Name>/
+# what the service needs told. A service may also carry helper units (any
-# is a real directory owning the .env (secrets never sit inside the checkout,
+# other *.service or *.timer, e.g. a cert refresh) and helper files (scripts,
+# a README) beside them. The machine carries the rest: ~/Server/<Name>/ is a
+# real directory owning the .env (secrets never sit inside the checkout,
 # gitignored or not) and the bind-mounted data. Only the tracked files are
 # symlinked, so a `git clean` in the repo can never reach a database.
 #
@@ -83,12 +85,12 @@ require_service() {
   fi
 }
 
-# The one unit file a service directory carries. Its filename is its identity
+# The one compose unit a service directory carries. Its filename is its
-# -- podman-<name>.service, exactly what lands in the systemd user directory --
+# identity -- podman-<name>.service, exactly what lands in the systemd user
-# so nothing here invents a name that could drift from the file's.
+# directory -- so nothing here invents a name that could drift from the file's.
 unit_path_for() {
   local name="$1" unit
-  for unit in "$SERVER_DIR/$name"/*.service; do
+  for unit in "$SERVER_DIR/$name"/podman-*.service; do
     [[ -e "$unit" ]] || continue
     printf '%s' "$unit"
     return 0
@@ -96,6 +98,44 @@ unit_path_for() {
   return 1
 }
 
+# Every other unit in the directory: helper services and the timers that fire
+# them. They are linked beside the compose unit, and each timer is enabled.
+helper_units_for() {
+  local name="$1" unit
+  for unit in "$SERVER_DIR/$name"/*.service "$SERVER_DIR/$name"/*.timer; do
+    [[ -e "$unit" ]] || continue
+    [[ "$(basename "$unit")" == podman-*.service ]] && continue
+    printf '%s\n' "$unit"
+  done
+}
+
+# The files linked into ~/Server/<Name>/: every visible top-level file that is
+# not a unit. That is compose.yml plus any helper script or README; dotfiles
+# are skipped so .env.example (and a stray .env) never become links.
+live_files_for() {
+  local name="$1" file
+  for file in "$SERVER_DIR/$name"/*; do
+    [[ -f "$file" ]] || continue
+    case "$file" in *.service|*.timer) continue ;; esac
+    printf '%s\n' "$file"
+  done
+}
+
+link_service_files() {
+  local name="$1" file unit
+  mkdir -p "$TARGET_DIR/$name" "$UNIT_DIR"
+  while IFS= read -r file; do
+    link_file "$file" "$TARGET_DIR/$name/$(basename "$file")"
+  done < <(live_files_for "$name")
+  while IFS= read -r unit; do
+    link_file "$unit" "$UNIT_DIR/$(basename "$unit")"
+  done < <(helper_units_for "$name")
+}
+
+helper_timers_for() {
+  helper_units_for "$1" | grep '\.timer$' | xargs -r -n1 basename
+}
+
 unit_name_for() {
   local unit
   unit="$(unit_path_for "$1")" || return 1
@@ -111,10 +151,12 @@ unit_installed() {
   [[ -L "$UNIT_DIR/$unit_name" ]]
 }
 
+# Everything this machine links for the service, so a change to a helper
+# script or timer is reported by relink just like a compose change.
 definition_hash() {
-  local name="$1" unit
+  local name="$1"
-  unit="$(unit_path_for "$name")" || unit=/dev/null
+  { live_files_for "$name"; unit_path_for "$name" && echo; helper_units_for "$name"; } \
-  cat "$SERVER_DIR/$name/compose.yml" "$unit" 2>/dev/null | sha256sum | cut -d' ' -f1
+    | sort | xargs -r -d '\n' cat 2>/dev/null | sha256sum | cut -d' ' -f1
 }
 
 recorded_hash() {
@@ -198,6 +240,7 @@ cmd_enable() {
   fi
 
   mkdir -p "$UNIT_DIR"
+  link_service_files "$name"
   link_file "$unit_path" "$UNIT_DIR/$unit_name"
   systemctl --user daemon-reload
   if systemctl --user enable --now "$unit_name"; then
@@ -207,6 +250,10 @@ cmd_enable() {
     err "$unit_name did not start; see: systemctl --user status $unit_name"
     exit 1
   fi
+  local timer
+  for timer in $(helper_timers_for "$name"); do
+    systemctl --user enable --now "$timer" && ok "Enabled $timer"
+  done
 }
 
 cmd_disable() {
@@ -216,11 +263,15 @@ cmd_disable() {
   local unit_name
   unit_name="$(unit_name_for "$name")" || { err "$name has no unit file in the catalog"; exit 1; }
 
+  local timer unit
+  for timer in $(helper_timers_for "$name"); do
+    systemctl --user disable --now "$timer" 2>/dev/null || true
+  done
   systemctl --user disable --now "$unit_name" 2>/dev/null || true
-  if [[ -L "$UNIT_DIR/$unit_name" ]]; then
+  for unit in "$UNIT_DIR/$unit_name" $(helper_units_for "$name" | xargs -r -n1 basename | sed "s|^|$UNIT_DIR/|"); do
-    rm "$UNIT_DIR/$unit_name"
+    [[ -L "$unit" ]] && rm "$unit"
-    systemctl --user daemon-reload
+  done
-  fi
+  systemctl --user daemon-reload
   ok "$name disabled. ~/Server/$name, its .env and its data were not touched."
 }
 
@@ -267,8 +318,7 @@ cmd_relink() {
     unit_installed "$name" || continue
 
     link_file "$unit_path" "$UNIT_DIR/$unit_name"
-    mkdir -p "$TARGET_DIR/$name"
+    link_service_files "$name"
-    link_file "$SERVER_DIR/$name/compose.yml" "$TARGET_DIR/$name/compose.yml"
     refreshed=$((refreshed + 1))
 
     # Changed since the last time this machine looked, which is what makes a

server/vps/UseSend/.env.example

@@ -8,14 +8,17 @@ AWS_DEFAULT_REGION=us-east-2
 DATABASE_URL=CHANGE_ME
 GITHUB_ID=Ov23liz34colyJed9t9S
 GITHUB_SECRET=CHANGE_ME
-MINIO_ROOT_USER=usesend
-MINIO_ROOT_PASSWORD=CHANGE_ME
 NEXTAUTH_SECRET=CHANGE_ME
 NEXT_PUBLIC_IS_CLOUD=false
 POSTGRES_USER=usesend
 POSTGRES_PASSWORD=CHANGE_ME
 POSTGRES_DB=usesend
 REDIS_URL=redis://usesend-redis:6379
+RUSTFS_ACCESS_KEY=CHANGE_ME
+RUSTFS_SECRET_KEY=CHANGE_ME
+S3_COMPATIBLE_API_URL=https://storage.usesend.gbrown.org
+S3_COMPATIBLE_PUBLIC_URL=https://storage.usesend.gbrown.org/unsend
+S3_COMPATIBLE_BUCKET=unsend
 SMTP_HOST=smtp.usesend.gbrown.org
 SMTP_USER=usesend
 USESEND_PORT=3000

server/vps/UseSend/README.md

@@ -0,0 +1,39 @@
+# TLS certificate for the UseSend SMTP proxy
+
+The `usesend/smtp-proxy` image ships a default certificate that **cannot complete a
+TLS 1.3 handshake** — OpenSSL rejects it with `no suitable signature algorithm`.
+Because the proxy also refuses `AUTH` until `STARTTLS` has succeeded, that default
+cert made *all* inbound SMTP impossible. These files replace it.
+
+## What lives in certs/
+
+`fullchain.pem` and `privkey.pem`, copied from NPM's Let's Encrypt store:
+
+    ~/Server/Nginx_Proxy_Manager/volumes/letsencrypt/live/npm-43/
+
+That certificate carries `*.usesend.gbrown.org`, which is what lets clients validate
+the proxy as `smtp.usesend.gbrown.org`. A wildcard matches exactly one label, so
+`*.gbrown.org` alone would not cover it.
+
+Clients reach that name two ways, neither of them public DNS:
+
+- containers on `nginx-bridge` get the network alias set in `compose.yml`
+- LAN and home-server clients get a UniFi DNS record pointing it at `192.168.2.2`
+
+## Why copies instead of a bind mount
+
+NPM mounts that letsencrypt tree with `:Z`, which gives it a *private* SELinux MCS
+category. A second container mounting the same path would relabel it and the two
+would fight over it. Copying keeps the stacks independent.
+
+## Keeping the copies fresh
+
+NPM renews on its own schedule, so `usesend-cert-refresh.timer` runs
+`refresh-smtp-certs` daily. It copies the cert when NPM's fingerprint changes,
+refuses a half-renewed cert/key pair, and restarts the proxy.
+
+If the copies ever expire, every app sending mail through the proxy fails at once,
+with a TLS error rather than anything that mentions certificates. Check with:
+
+    openssl x509 -in ~/Server/UseSend/certs/fullchain.pem -noout -enddate
+    systemctl --user list-timers usesend-cert-refresh.timer

server/vps/UseSend/compose.yml

@@ -21,25 +21,69 @@ services:
     tty: true
     stdin_open: true
     restart: unless-stopped
-  minio:
+  # S3-compatible storage for images uploaded in the email editor, following
-    image: minio/minio
+  # upstream's docker/prod/compose.yml (RustFS replaced MinIO, whose images are
+  # no longer published). The browser uploads straight to S3_COMPATIBLE_API_URL
+  # and sent emails link to S3_COMPATIBLE_PUBLIC_URL, so NPM proxies
+  # storage.usesend.gbrown.org -> http://usesend-storage:9000. No host ports:
+  # the console (9001) is only reachable from inside nginx-bridge.
+  # Pinned like upstream. Bump the tag deliberately when upstream does.
+  rustfs:
+    image: docker.io/rustfs/rustfs:1.0.4
     container_name: usesend-storage
     hostname: usesend-storage
     networks:
     - nginx-bridge
-    env_file: .env
+    # RustFS runs as uid 10001; keep-id keeps ./volumes/rustfs gib-owned (AGENTS.md §3.2)
+    userns_mode: keep-id:uid=10001,gid=10001
+    security_opt:
+    - no-new-privileges:true
     environment:
-      MINIO_ROOT_USER: ${MINIO_ROOT_USER}
+      RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY:?err}
-      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
+      RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY:?err}
-    labels:
+      RUSTFS_ADDRESS: 0.0.0.0:9000
-      com.centurylinklabs.watchtower.enable: 'true'
+      RUSTFS_CONSOLE_ADDRESS: 0.0.0.0:9001
+      RUSTFS_CONSOLE_ENABLE: 'true'
+      RUSTFS_OBS_LOGGER_LEVEL: error
+      TZ: America/New_York
     volumes:
-    - ./volumes/minio:/data:Z
+    - ./volumes/rustfs:/data:Z
-    entrypoint: sh
+    command: ["/data"]
-    command: -c 'mkdir -p /data/unsend && minio server /data --console-address ':9001' --address ':9002''
+    healthcheck:
-    tty: true
+      test: ["CMD", "curl", "-fsS", "http://127.0.0.1:9000/health"]
-    stdin_open: true
+      interval: 20s
+      timeout: 10s
+      retries: 5
+      start_period: 40s
     restart: unless-stopped
+  # One-shot: creates the bucket, allows browser uploads from the app (CORS) and
+  # makes objects publicly readable so images load in sent emails. Reruns on
+  # every unit start, which is harmless.
+  rustfs-init:
+    image: docker.io/amazon/aws-cli:2.34.30
+    container_name: usesend-storage-init
+    networks:
+    - nginx-bridge
+    entrypoint: ["/bin/sh", "-ec"]
+    environment:
+      AWS_ACCESS_KEY_ID: ${RUSTFS_ACCESS_KEY:?err}
+      AWS_SECRET_ACCESS_KEY: ${RUSTFS_SECRET_KEY:?err}
+      AWS_DEFAULT_REGION: us-east-1
+      AWS_EC2_METADATA_DISABLED: 'true'
+      BUCKET: ${S3_COMPATIBLE_BUCKET:-unsend}
+      APP_ORIGIN: ${USESEND_URL:?err}
+    depends_on:
+      rustfs:
+        condition: service_healthy
+    command:
+    - |
+      s3="aws --endpoint-url http://usesend-storage:9000 s3api"
+      $$s3 head-bucket --bucket "$$BUCKET" >/dev/null 2>&1 || $$s3 create-bucket --bucket "$$BUCKET"
+      $$s3 put-bucket-cors --bucket "$$BUCKET" --cors-configuration \
+        '{"CORSRules":[{"AllowedOrigins":["'"$$APP_ORIGIN"'"],"AllowedMethods":["GET","HEAD","PUT"],"AllowedHeaders":["*"],"ExposeHeaders":["ETag"],"MaxAgeSeconds":3600}]}'
+      $$s3 put-bucket-policy --bucket "$$BUCKET" --policy \
+        '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::'"$$BUCKET"'/*"]}]}'
+    restart: 'no'
   smtp-server:
     image: usesend/smtp-proxy:latest
     container_name: usesend-smtp-server
@@ -47,26 +91,15 @@ services:
     networks:
       nginx-bridge:
         # The proxy REQUIRES STARTTLS before it will accept AUTH, so clients must
-        # reach it by a name the certificate actually covers. The wildcard is
+        # reach it by a name the certificate actually covers: smtp.usesend.gbrown.org,
-        # *.gbrown.org, which matches ONE label -- hence smtp-usesend, not
+        # validated by the *.usesend.gbrown.org SAN on npm-43 (see certs/README.md).
-        # smtp.usesend.
         #
         # Resolution: podman's aardvark-dns answers this name authoritatively for
         # containers ON nginx-bridge, handing back the container IP. NPM is not
-        # involved at all -- it only reverse-proxies HTTP(S), and SMTP ports are
+        # involved at all -- it only reverse-proxies HTTP(S). Cloudflare is not
-        # never published to the host, so the proxy is unreachable from outside.
+        # involved either: there is deliberately no public DNS record for this name,
-        #
+        # so if the container is stopped, lookups fail fast with NXDOMAIN.
-        # NOTE: there IS a wildcard *.gbrown.org record in Cloudflare, so this name
-        # also resolves publicly (to Cloudflare, which serves nothing for it). The
-        # consequence: if this container is STOPPED, aardvark-dns stops answering,
-        # the lookup falls through to that public record, and clients dial
-        # Cloudflare on 587 and hang until timeout instead of failing fast.
         aliases:
-        # Both names resolve to this container and BOTH validate against npm-43,
-        # which carries *.gbrown.org and *.usesend.gbrown.org. smtp.usesend is the
-        # name UseSend's own UI advertises and is the target to standardise on;
-        # smtp-usesend stays until every consumer has been moved across.
-        - smtp-usesend.gbrown.org
         - smtp.usesend.gbrown.org
     env_file: .env
     environment:
@@ -83,8 +116,8 @@ services:
     # brute-force it, and contradicts the Cloudflare-only posture in §7.
     # The `Home` wg interface sits in firewalld's `trusted` zone, so no rule is needed.
     # Rootless can bind these because net.ipv4.ip_unprivileged_port_start = 80.
-    # Consumers on the home server reach it as smtp-usesend.gbrown.org (UniFi DNS maps
+    # Consumers on the home server reach it as smtp.usesend.gbrown.org (a UniFi DNS record
-    # that name to 192.168.2.2) so the *.gbrown.org cert still validates by hostname.
+    # maps that name to 192.168.2.2) so the cert still validates by hostname.
     - "192.168.2.2:587:587"
     - "192.168.2.2:465:465"
     volumes:
@@ -121,6 +154,12 @@ services:
     - API_RATE_LIMIT=${API_RATE_LIMIT:-1}
     - SMTP_HOST=${SMTP_HOST:-smtp.usesend.gbrown.org}
     - SMTP_USER=${SMTP_USER:-usesend}
+    # Editor image uploads (the rustfs service above). Uploads are disabled if any is unset.
+    - S3_COMPATIBLE_ACCESS_KEY=${RUSTFS_ACCESS_KEY:?err}
+    - S3_COMPATIBLE_SECRET_KEY=${RUSTFS_SECRET_KEY:?err}
+    - S3_COMPATIBLE_API_URL=${S3_COMPATIBLE_API_URL:?err}
+    - S3_COMPATIBLE_PUBLIC_URL=${S3_COMPATIBLE_PUBLIC_URL:?err}
+    - S3_COMPATIBLE_BUCKET=${S3_COMPATIBLE_BUCKET:-unsend}
     labels:
       com.centurylinklabs.watchtower.enable: 'true'
     depends_on:

server/vps/UseSend/refresh-smtp-certs

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Keep the UseSend SMTP proxy's TLS certificate in step with NPM's Let's Encrypt store.
+#
+# The proxy refuses AUTH until STARTTLS succeeds, so an expired or mismatched cert
+# takes down ALL outbound mail for every app that sends through it -- and it fails
+# with a bare TLS error that never mentions certificates. Hence this timer.
+#
+# Copies rather than bind-mounts because NPM mounts its letsencrypt tree with :Z
+# (private SELinux category) and two consumers on one path fight over the label.
+#
+# Uses npm-43 because it carries *.usesend.gbrown.org, the SAN that validates
+# smtp.usesend.gbrown.org. A wildcard matches exactly ONE label, so *.gbrown.org
+# alone would not cover it.
+set -euo pipefail
+
+SRC=$HOME/Server/Nginx_Proxy_Manager/volumes/letsencrypt/live/npm-43
+DST=$HOME/Server/UseSend/certs
+CONTAINER=usesend-smtp-server
+
+fp()     { openssl x509 -in "$1" -noout -fingerprint -sha256 2>/dev/null | cut -d= -f2; }
+certpub(){ openssl x509 -in "$1" -noout -pubkey 2>/dev/null | openssl sha256 | awk '{print $2}'; }
+keypub() { openssl pkey -in "$1" -pubout   2>/dev/null | openssl sha256 | awk '{print $2}'; }
+
+[ -r "$SRC/fullchain.pem" ] && [ -r "$SRC/privkey.pem" ] || {
+  echo "source cert unreadable under $SRC -- did NPM move it?" >&2; exit 1; }
+
+src_fp=$(fp "$SRC/fullchain.pem")
+dst_fp=$(fp "$DST/fullchain.pem" 2>/dev/null || true)
+
+if [ -n "$dst_fp" ] && [ "$src_fp" = "$dst_fp" ]; then
+  echo "cert unchanged (${src_fp:0:17}...); nothing to do"
+  exit 0
+fi
+
+# Refuse to install a half-renewed pair: certbot writes the two files separately, so
+# a run that lands between them would leave the proxy with a key that is not its cert.
+if [ "$(certpub "$SRC/fullchain.pem")" != "$(keypub "$SRC/privkey.pem")" ]; then
+  echo "source cert and key do not match -- renewal probably mid-flight, retrying next run" >&2
+  exit 0
+fi
+
+echo "cert changed (${dst_fp:0:17}... -> ${src_fp:0:17}...), installing"
+# -L dereferences: live/ entries are symlinks into ../../archive/, outside this copy.
+cp -L "$SRC/fullchain.pem" "$DST/fullchain.pem.new"
+cp -L "$SRC/privkey.pem"   "$DST/privkey.pem.new"
+chmod 644 "$DST/fullchain.pem.new"
+chmod 640 "$DST/privkey.pem.new"
+mv -f "$DST/fullchain.pem.new" "$DST/fullchain.pem"
+mv -f "$DST/privkey.pem.new"   "$DST/privkey.pem"
+
+echo "restarting $CONTAINER"
+podman restart "$CONTAINER" >/dev/null
+echo "done; cert now valid until $(openssl x509 -in "$DST/fullchain.pem" -noout -enddate | cut -d= -f2)"

server/vps/UseSend/usesend-cert-refresh.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Refresh UseSend SMTP proxy TLS certificate from NPM's Let's Encrypt store
+Documentation=file://%h/Server/UseSend/README.md
+After=podman-usesend.service
+
+[Service]
+Type=oneshot
+ExecStart=%h/Server/UseSend/refresh-smtp-certs
+# The script calls `podman restart`, which leaves conmon and the container's
+# supervising processes inside THIS unit's cgroup. With the default KillMode,
+# systemd waits for that cgroup to empty and the unit times out and reports
+# `failed` even though ExecStart returned 0 and the cert was installed correctly.
+# KillMode=process makes systemd track only the script itself.
+KillMode=process
+TimeoutStartSec=300

server/vps/UseSend/usesend-cert-refresh.timer

@@ -0,0 +1,12 @@
+[Unit]
+Description=Daily check for a renewed UseSend SMTP proxy certificate
+
+[Timer]
+OnCalendar=daily
+# Spread the load and avoid landing exactly when certbot is writing.
+RandomizedDelaySec=2h
+# Catch up after the host has been off; the cert does not care that we were asleep.
+Persistent=true
+
+[Install]
+WantedBy=timers.target

tests/server/containers-shape-contract

@@ -3,9 +3,12 @@
 # Every service in the catalog has the same shape, because panama-server and
 # the update timer both depend on it blindly:
 #
-#   * a compose.yml, which is the only file enable symlinks into ~/Server
+#   * a compose.yml, linked into ~/Server along with any helper script
 #   * exactly one podman-*.service unit -- its filename is the unit's
-#     identity, so two would be an ambiguity and zero an unenableable service
+#     identity, so two would be an ambiguity and zero an unenableable service.
+#     Other *.service and *.timer files are helpers (a cert refresh, say)
+#     that enable links beside it; each helper service needs a timer or a
+#     unit that pulls it in, so an orphan helper is flagged
 #   * a WorkingDirectory of %h/Server/<Name>, matching the directory enable
 #     creates -- a unit pointing anywhere else starts compose against a
 #     directory that has no .env and no data
@@ -31,20 +34,21 @@ for dir in "$containers_dir"/*/; do
 
     [[ -f "$dir/compose.yml" ]] || note "$name has no compose.yml"
 
-    units=("$dir"/*.service)
+    units=("$dir"/podman-*.service)
     if (( ${#units[@]} == 0 )); then
-        note "$name has no unit file"
+        note "$name has no podman-<name>.service unit"
         continue
     elif (( ${#units[@]} > 1 )); then
-        note "$name has ${#units[@]} unit files; its identity is ambiguous"
+        note "$name has ${#units[@]} podman-*.service units; its identity is ambiguous"
         continue
     fi
     unit="${units[0]}"
 
-    case "$(basename "$unit")" in
+    for helper in "$dir"/*.service; do
-        podman-*.service) ;;
+        [[ "$helper" == "$unit" ]] && continue
-        *) note "$name's unit is not named podman-<name>.service: $(basename "$unit")" ;;
+        [[ -f "${helper%.service}.timer" ]] \
-    esac
+            || note "$name's helper $(basename "$helper") has no matching .timer"
+    done
 
     grep -q "^WorkingDirectory=%h/Server/$name\$" "$unit" \
         || note "$name's unit does not work in %h/Server/$name"

tests/server/panama-server-contract

@@ -6,9 +6,10 @@
 #   * enable seeds .env from .env.example and REFUSES to start while a
 #     CHANGE_ME placeholder survives -- a service brought up with the
 #     placeholder as its database password runs, wrongly, forever
-#   * a completed enable links exactly the tracked files (compose.yml and the
+#   * a completed enable links the tracked files (compose.yml, helper scripts,
-#     unit), reloads systemd, and enables the unit
+#     the unit and any helper units), reloads systemd, enables the unit and
-#   * disable removes the unit and nothing else: the .env and data stay
+#     each helper timer, and never links .env.example
+#   * disable removes the units and nothing else: the .env and data stay
 #   * relink refreshes links for enabled services only, and names a service
 #     whose definition changed since the machine last looked -- without
 #     restarting anything
@@ -55,6 +56,21 @@ RemainAfterExit=yes
 [Install]
 WantedBy=default.target
 UNIT
+cat >"$catalog/Example/refresh-example" <<'SH'
+#!/usr/bin/env bash
+SH
+chmod +x "$catalog/Example/refresh-example"
+cat >"$catalog/Example/example-refresh.service" <<'UNIT'
+[Service]
+Type=oneshot
+ExecStart=%h/Server/Example/refresh-example
+UNIT
+cat >"$catalog/Example/example-refresh.timer" <<'UNIT'
+[Timer]
+OnCalendar=daily
+[Install]
+WantedBy=timers.target
+UNIT
 
 # The stub records every invocation; is-active answers 'inactive' so nothing
 # here depends on a real systemd.
@@ -81,6 +97,8 @@ run() {
 }
 
 unit_link="$home/.config/systemd/user/podman-example.service"
+helper_link="$home/.config/systemd/user/example-refresh.service"
+timer_link="$home/.config/systemd/user/example-refresh.timer"
 
 # ── enable stops on placeholders ─────────────────────────────────────────────
 
@@ -111,6 +129,14 @@ grep -q 'systemctl --user enable --now podman-example.service' "$calls" \
     || note 'enable did not enable --now the unit'
 [[ "$(cat "$home/Server/Example/.env")" == "DB_PASSWORD=s3cret" ]] \
     || note 'enable rewrote an .env that already existed'
+[[ -L "$home/Server/Example/refresh-example" ]] \
+    || note 'enable did not link the helper script into ~/Server'
+[[ -e "$home/Server/Example/.env.example" ]] \
+    && note 'enable linked .env.example into ~/Server'
+[[ -L "$helper_link" && -L "$timer_link" ]] \
+    || note 'enable did not link the helper units'
+grep -q 'systemctl --user enable --now example-refresh.timer' "$calls" \
+    || note 'enable did not enable the helper timer'
 
 # ── relink names a changed definition, restarts nothing ──────────────────────
 
@@ -136,6 +162,10 @@ run disable Example >/dev/null 2>&1 || note 'disable failed'
 
 [[ -e "$unit_link" ]] \
     && note 'disable left the unit installed'
+[[ -e "$helper_link" || -e "$timer_link" ]] \
+    && note 'disable left a helper unit installed'
+grep -q 'systemctl --user disable --now example-refresh.timer' "$calls" \
+    || note 'disable did not stop the helper timer'
 grep -q 'systemctl --user disable --now podman-example.service' "$calls" \
     || note 'disable did not stop the unit'
 [[ -f "$home/Server/Example/.env" ]] \