Deploy from CI with Infisical and ship Payload changes as migrations

CI reads Infisical staging as the read-only convexmonorepo-ci machine identity, deploys the Convex functions, builds and pushes the image, and asks the VPS to redeploy. Payload schema changes now ship as migrations that production runs on startup; the baseline matches the production schema and is recorded as applied there. The local stack moves to its own ports and syncs the Convex runtime env, and the scripts pick up Infisical account switching, dev-kill and a cache-safe CI install.

scripts/build-next-bundle

@@ -0,0 +1,23 @@
+#!/usr/bin/env sh
+set -eu
+
+ROOT_DIR="$(cd -- "$(dirname -- "$0")/.." && pwd)"
+
+if [ -n "${SENTRY_AUTH_TOKEN:-}" ] &&
+  [ -n "${NEXT_PUBLIC_SENTRY_URL:-}" ] &&
+  ! curl \
+    --connect-timeout 3 \
+    --max-time 10 \
+    --silent \
+    --show-error \
+    --output /dev/null \
+    "${NEXT_PUBLIC_SENTRY_URL%/}/api/0/"; then
+  echo "Sentry is unreachable; continuing without source-map upload." >&2
+  SENTRY_AUTH_TOKEN=""
+  SKIP_ENV_VALIDATION=true
+  NEXT_PUBLIC_SENTRY_URL="http://127.0.0.1:9"
+  export NEXT_PUBLIC_SENTRY_URL SENTRY_AUTH_TOKEN SKIP_ENV_VALIDATION
+fi
+
+cd "$ROOT_DIR/apps/next"
+exec bun run build:docker

scripts/ci-export-env.mjs

@@ -0,0 +1,62 @@
+#!/usr/bin/env node
+// ci-export-env.mjs [environment]
+//
+// CI's counterpart to scripts/export-env: prints an Infisical environment
+// (default `staging`) as dotenv on stdout, in the same quoting export-env uses.
+// It logs in as the `convexmonorepo-ci` machine identity through Universal Auth, using
+// INFISICAL_CLIENT_ID and INFISICAL_CLIENT_SECRET from the job's Gitea secrets,
+// and reads the project and host from .infisical.json. Plain Node, no CLI to
+// install, so it runs in every job whether or not Bun is set up.
+import { readFile } from 'node:fs/promises';
+
+const environment = process.argv[2] ?? 'staging';
+const { INFISICAL_CLIENT_ID: clientId, INFISICAL_CLIENT_SECRET: clientSecret } =
+  process.env;
+
+const fail = (message) => {
+  console.error(`ci-export-env: ${message}`);
+  process.exit(1);
+};
+
+if (!clientId || !clientSecret)
+  fail('INFISICAL_CLIENT_ID and INFISICAL_CLIENT_SECRET must be set.');
+
+const { workspaceId, domain } = JSON.parse(
+  await readFile(new URL('../.infisical.json', import.meta.url), 'utf8'),
+);
+
+const request = async (path, init) => {
+  const response = await fetch(`${domain}${path}`, init);
+  if (!response.ok)
+    fail(`${init?.method ?? 'GET'} ${path} returned ${response.status}.`);
+  return response.json();
+};
+
+const { accessToken } = await request('/v1/auth/universal-auth/login', {
+  method: 'POST',
+  headers: { 'Content-Type': 'application/json' },
+  body: JSON.stringify({ clientId, clientSecret }),
+});
+
+const query = new URLSearchParams({
+  workspaceId,
+  environment,
+  secretPath: '/',
+  expandSecretReferences: 'true',
+  include_imports: 'true',
+});
+const { secrets, imports = [] } = await request(`/v3/secrets/raw?${query}`, {
+  headers: { Authorization: `Bearer ${accessToken}` },
+});
+
+// Imported secrets first so the environment's own values win.
+const values = new Map();
+for (const { secretKey, secretValue } of [
+  ...imports.flatMap((folder) => folder.secrets),
+  ...secrets,
+])
+  values.set(secretKey, secretValue);
+
+const quote = (value) => `'${value.replaceAll("'", "'\\''")}'`;
+for (const [key, value] of [...values].sort(([a], [b]) => a.localeCompare(b)))
+  process.stdout.write(`${key}=${quote(value)}\n`);

scripts/ci-install

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Install on a runner, and survive a half-written cache entry.
+#
+# Runners have failed twice with "Fail extracting tarball" on a different
+# package each time, on different machines, and passed on a re-run. The cache
+# is shared between jobs on a host, so a job that dies mid-write leaves an
+# entry the next job cannot read. Clearing it and installing again costs a
+# couple of minutes; a human re-running the job costs however long it takes
+# someone to notice.
+set -euo pipefail
+if bun install --frozen-lockfile; then exit 0; fi
+echo "Install failed. Clearing the bun cache and trying once more." >&2
+rm -rf "${BUN_INSTALL_CACHE_DIR:-$HOME/.bun/install/cache}"
+bun install --frozen-lockfile

scripts/db/up

@@ -51,9 +51,9 @@ for i in $(seq 1 30); do
   sleep 1
 done
 
-info "Waiting for Convex at http://localhost:${BACKEND_PORT:-3210}"
+info "Waiting for Convex at http://localhost:${BACKEND_PORT:-3240}"
 for i in $(seq 1 60); do
-  curl -fs "http://localhost:${BACKEND_PORT:-3210}/version" >/dev/null 2>&1 && break
+  curl -fs "http://localhost:${BACKEND_PORT:-3240}/version" >/dev/null 2>&1 && break
   [ "$i" -lt 60 ] || die "Convex did not become ready."
   sleep 2
 done
@@ -69,6 +69,11 @@ fi
 info "Deploying Convex schema and functions"
 (cd "$ROOT_DIR/packages/backend" && bun run setup)
 
+# Env sync runs after the deploy: changing deployment env re-analyzes the
+# deployed modules, which fails on a fresh volume before the first deploy.
+info "Synchronizing the Convex function runtime environment"
+sh "$ROOT_DIR/scripts/with-env" dev -- bash "$ROOT_DIR/scripts/sync-local-convex-env"
+
 convex_env_names="$(
   sh "$ROOT_DIR/scripts/with-env" dev -- bash -c \
     'cd packages/backend && bunx convex env list' 2>/dev/null \
@@ -92,4 +97,4 @@ fi
 info "Seeding local Payload from snapshot if needed"
 bash "$ROOT_DIR/scripts/db/seed-payload" --yes
 
-printf '\nLocal stack ready:\n  App: http://localhost:3000\n  Convex: http://localhost:%s\n  Dashboard: http://localhost:%s\n  Payload Postgres: localhost:%s\n' "${BACKEND_PORT:-3210}" "${DASHBOARD_PORT:-6791}" "${POSTGRES_PORT:-5432}"
+printf '\nLocal stack ready:\n  App: http://localhost:3000\n  Convex: http://localhost:%s\n  Dashboard: http://localhost:%s\n  Payload Postgres: localhost:%s\n' "${BACKEND_PORT:-3240}" "${DASHBOARD_PORT:-6794}" "${POSTGRES_PORT:-5436}"

scripts/dev-kill

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+#
+# Kill any lingering host-level dev servers (Next.js, Expo Metro, Convex CLI
+# watchers, turbo orchestrators) that survived a Ctrl-C. Does not touch the
+# dockerized Convex/Postgres stack — use `bun db:down` for that.
+#
+# Usage: scripts/dev-kill   (wired as `bun dev:kill`)
+set -euo pipefail
+
+info() { printf '\033[36m▶ %s\033[0m\n' "$*"; }
+ok()   { printf '\033[32m✓ %s\033[0m\n' "$*"; }
+
+PORTS=(3000 3001 8081 19000 19001 19002)
+PATTERNS=('next-server' 'next dev' 'expo start' 'turbo run dev' 'convex dev')
+
+killed=false
+
+for port in "${PORTS[@]}"; do
+  pids="$(lsof -ti "tcp:$port" -sTCP:LISTEN 2>/dev/null || true)"
+  [ -z "$pids" ] && continue
+  for pid in $pids; do
+    cmd="$(ps -p "$pid" -o comm= 2>/dev/null || true)"
+    info "Killing $cmd (pid $pid) on port $port"
+    kill "$pid" 2>/dev/null || true
+    killed=true
+  done
+done
+
+for pattern in "${PATTERNS[@]}"; do
+  pids="$(pgrep -f "$pattern" 2>/dev/null || true)"
+  [ -z "$pids" ] && continue
+  for pid in $pids; do
+    info "Killing '$pattern' (pid $pid)"
+    kill "$pid" 2>/dev/null || true
+    killed=true
+  done
+done
+
+if [ "$killed" = true ]; then
+  sleep 1
+  ok "Dev servers stopped."
+else
+  ok "No dev servers were running."
+fi

scripts/export-env

@@ -16,6 +16,12 @@ fi
 [ -f "$ROOT_DIR/.infisical.json" ] || { echo "export-env: run 'infisical init' in this repository." >&2; exit 1; }
 command -v infisical >/dev/null 2>&1 || { echo "export-env: Infisical CLI is required." >&2; exit 1; }
 
+if [ -f "$ROOT_DIR/scripts/infisical-profile" ]; then
+  # shellcheck source=/dev/null
+  . "$ROOT_DIR/scripts/infisical-profile"
+  ensure_infisical_profile "$ROOT_DIR/scripts/flags/infisical.env"
+fi
+
 (cd "$ROOT_DIR" && infisical export --env="$ENVIRONMENT" --format=dotenv --silent) || {
   echo "export-env: failed to export '$ENVIRONMENT'; check login and project access." >&2
   exit 1

scripts/infisical-profile

@@ -0,0 +1,117 @@
+#!/usr/bin/env sh
+#
+# ensure_infisical_profile <profile_env_file>
+#
+# Infisical's CLI keeps a single active login (~/.infisical/infisical-config.json)
+# shared across every project on the machine. If you juggle multiple Infisical
+# accounts (e.g. a client org vs. a personal org), running with-env in one repo
+# can silently pull secrets while logged in as the wrong user.
+#
+# This switches the active `loggedInUserEmail` to the email configured in
+# scripts/flags/infisical.env (INFISICAL_EMAIL=...) before infisical runs,
+# provided that email is already logged in locally (`infisical login`).
+# It is a no-op if the flag file is absent, only one user is logged in, or the
+# expected user isn't logged in yet.
+#
+# POSIX sh, no bashisms: matches the rest of scripts/ which must stay
+# busybox-compatible.
+
+warn_infisical_profile() {
+  echo "Warning: infisical-profile: $*" >&2
+}
+
+ensure_infisical_profile() {
+  profile_env_file="${1:-}"
+
+  if [ -z "$profile_env_file" ] || [ ! -f "$profile_env_file" ]; then
+    return 0
+  fi
+
+  set -a
+  # shellcheck source=/dev/null
+  if ! . "$profile_env_file"; then
+    set +a
+    warn_infisical_profile "could not read $profile_env_file. Continuing with current Infisical profile."
+    return 0
+  fi
+  set +a
+
+  expected_email="${INFISICAL_EMAIL:-}"
+  if [ -z "$expected_email" ]; then
+    return 0
+  fi
+
+  config="${INFISICAL_CONFIG_PATH:-$HOME/.infisical/infisical-config.json}"
+  lock="${INFISICAL_LOCK_PATH:-$HOME/.infisical/.switch.lock}"
+
+  if [ ! -f "$config" ]; then
+    warn_infisical_profile "Infisical config not found at $config. Continuing with current Infisical profile."
+    return 0
+  fi
+
+  if ! command -v jq >/dev/null 2>&1; then
+    warn_infisical_profile "jq is required to select Infisical profile. Continuing with current Infisical profile."
+    return 0
+  fi
+
+  if ! command -v flock >/dev/null 2>&1; then
+    warn_infisical_profile "flock is required to safely select Infisical profile. Continuing with current Infisical profile."
+    return 0
+  fi
+
+  if ! mkdir -p "$(dirname "$lock")"; then
+    warn_infisical_profile "could not create Infisical lock directory. Continuing with current Infisical profile."
+    return 0
+  fi
+
+  (
+    if ! flock 9; then
+      warn_infisical_profile "could not lock Infisical profile config. Continuing with current Infisical profile."
+      return 0
+    fi
+
+    user_count="$(jq -r '(.loggedInUsers // []) | length' "$config" 2>/dev/null)" || {
+      warn_infisical_profile "could not parse Infisical config. Continuing with current Infisical profile."
+      return 0
+    }
+
+    if [ "$user_count" -lt 2 ]; then
+      return 0
+    fi
+
+    current_email="$(jq -r '.loggedInUserEmail // ""' "$config" 2>/dev/null)" || {
+      warn_infisical_profile "could not read current Infisical profile. Continuing with current Infisical profile."
+      return 0
+    }
+
+    if [ "$current_email" = "$expected_email" ]; then
+      return 0
+    fi
+
+    domain="$(
+      jq -er --arg email "$expected_email" \
+        '.loggedInUsers[] | select(.email == $email) | .domain' \
+        "$config" 2>/dev/null
+    )" || {
+      warn_infisical_profile "Infisical profile $expected_email is not logged in locally. Continuing with current Infisical profile."
+      return 0
+    }
+
+    tmp="$(mktemp)" || {
+      warn_infisical_profile "could not create temp file for Infisical profile switch. Continuing with current Infisical profile."
+      return 0
+    }
+
+    if jq --arg email "$expected_email" --arg domain "$domain" \
+      '.loggedInUserEmail = $email | .LoggedInUserDomain = $domain' \
+      "$config" > "$tmp"; then
+      chmod 600 "$tmp"
+      mv "$tmp" "$config"
+    else
+      rm -f "$tmp"
+      warn_infisical_profile "failed to update Infisical profile. Continuing with current Infisical profile."
+    fi
+  ) 9>"$lock"
+
+  return 0
+}

scripts/sync-local-convex-env

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd -- "$SCRIPT_DIR/.." && pwd)"
+VALUE_FILE=""
+
+readonly RUNTIME_ENV_NAMES=(
+  AUTH_AUTHENTIK_ID
+  AUTH_AUTHENTIK_SECRET
+  AUTH_AUTHENTIK_ISSUER
+  USESEND_API_KEY
+  USESEND_URL
+  USESEND_FROM_EMAIL
+)
+readonly DEPLOYMENT_ENV_NAMES=(
+  CONVEX_SELF_HOSTED_URL
+  CONVEX_SELF_HOSTED_ADMIN_KEY
+)
+
+die() { printf 'sync-local-convex-env: %s\n' "$*" >&2; exit 1; }
+cleanup() {
+  if [ -n "$VALUE_FILE" ] && [ -f "$VALUE_FILE" ]; then
+    rm -f "$VALUE_FILE"
+  fi
+}
+trap cleanup EXIT INT TERM HUP
+
+for name in "${DEPLOYMENT_ENV_NAMES[@]}" "${RUNTIME_ENV_NAMES[@]}"; do
+  [ -n "${!name:-}" ] || die "Missing required environment variable: $name"
+done
+
+VALUE_FILE="$(mktemp "${TMPDIR:-/tmp}/convex-monorepo-convex-env.value.XXXXXX")"
+chmod 600 "$VALUE_FILE"
+
+for name in "${RUNTIME_ENV_NAMES[@]}"; do
+  printf '%s' "${!name}" > "$VALUE_FILE"
+  (
+    cd "$ROOT_DIR/packages/backend"
+    env -u CONVEX_DEPLOY_KEY -u CONVEX_DEPLOYMENT \
+      CONVEX_SELF_HOSTED_URL="$CONVEX_SELF_HOSTED_URL" \
+      CONVEX_SELF_HOSTED_ADMIN_KEY="$CONVEX_SELF_HOSTED_ADMIN_KEY" \
+      bunx convex env set "$name" --from-file "$VALUE_FILE" >/dev/null
+  )
+done
+
+printf 'Local Convex runtime environment synchronized.\n'