Move to infisical. Create local dev environment. Add ci gates. Modernize repo

2026-06-21 14:04:02 -05:00
parent 86e2fdc82e
commit a12bf6071b
79 changed files with 1612 additions and 42168 deletions
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd -- "$SCRIPT_DIR/../.." && pwd)"
+COMPOSE_FILE="$ROOT_DIR/docker/compose.local.yml"
+STATE_FILE="$ROOT_DIR/.local/dev.generated.env"
+PAYLOAD_SEED_MARKER="$ROOT_DIR/.local/payload-seed-state.env"
+WIPE=false
+[ "${1:-}" = "--wipe" ] && WIPE=true
+
+if command -v docker >/dev/null 2>&1; then RUNTIME=docker
+elif command -v podman >/dev/null 2>&1; then RUNTIME=podman
+else echo "Docker or Podman not found; nothing to stop." >&2; exit 0; fi
+
+ENV_FILE="$(mktemp "${TMPDIR:-/tmp}/convex-monorepo-local.XXXXXX.env")"
+trap 'rm -f "$ENV_FILE"' EXIT
+sh "$ROOT_DIR/scripts/export-env" dev > "$ENV_FILE"
+
+if [ "$WIPE" = true ]; then
+  "$RUNTIME" compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" down -v
+  if [ -f "$STATE_FILE" ]; then
+    tmp="${STATE_FILE}.tmp"
+    grep -v '^CONVEX_SELF_HOSTED_ADMIN_KEY=' "$STATE_FILE" > "$tmp" || true
+    mv "$tmp" "$STATE_FILE"
+  fi
+  rm -f "$PAYLOAD_SEED_MARKER"
+  echo "Local stack and both data volumes removed; generated admin key and Payload seed marker cleared."
+else
+  "$RUNTIME" compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" down
+  echo "Local stack stopped; Payload and Convex data preserved."
+fi
@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+# Restore the local Payload database from the local snapshot only when the
+# database has not already been seeded. Does not contact staging/production.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd -- "$SCRIPT_DIR/../.." && pwd)"
+COMPOSE_FILE="$ROOT_DIR/docker/compose.local.yml"
+SNAPSHOT="$ROOT_DIR/.local/payload-staging.dump"
+MARKER="$ROOT_DIR/.local/payload-seed-state.env"
+DEV_ENV=""
+FORCE=false
+ASSUME_YES=false
+
+for arg in "$@"; do
+  case "$arg" in
+    --force) FORCE=true ;;
+    --yes) ASSUME_YES=true ;;
+    *) printf 'usage: seed-payload [--force] [--yes]\n' >&2; exit 2 ;;
+  esac
+done
+
+info() { printf '▶ %s\n' "$*"; }
+die() { printf 'Error: %s\n' "$*" >&2; exit 1; }
+
+if command -v docker >/dev/null 2>&1; then RUNTIME=docker
+elif command -v podman >/dev/null 2>&1; then RUNTIME=podman
+else die "Docker or Podman is required."; fi
+"$RUNTIME" info >/dev/null 2>&1 || die "$RUNTIME is not usable."
+
+cleanup() { [ -z "$DEV_ENV" ] || rm -f "$DEV_ENV"; }
+trap cleanup EXIT INT TERM HUP
+
+if [ ! -s "$SNAPSHOT" ]; then
+  echo "No local Payload snapshot found at .local/payload-staging.dump; skipping Payload seed."
+  exit 0
+fi
+
+mkdir -p "$ROOT_DIR/.local"
+chmod 700 "$ROOT_DIR/.local"
+
+next_is_running=false
+if command -v ss >/dev/null 2>&1; then
+  ss -ltnH 'sport = :3000' | grep -q . && next_is_running=true
+elif command -v lsof >/dev/null 2>&1; then
+  lsof -nP -iTCP:3000 -sTCP:LISTEN >/dev/null 2>&1 && next_is_running=true
+elif curl -sS --max-time 1 http://localhost:3000 >/dev/null 2>&1; then
+  next_is_running=true
+fi
+if [ "$next_is_running" = true ]; then
+  die "Next is running on port 3000. Stop bun dev:next before replacing Payload data."
+fi
+
+DEV_ENV="$(mktemp "${TMPDIR:-/tmp}/convex-monorepo-dev.XXXXXX.env")"
+sh "$ROOT_DIR/scripts/export-env" dev > "$DEV_ENV"
+
+bunx dotenv -e "$DEV_ENV" -- node -e '
+  const raw = process.env.PAYLOAD_DB_URL;
+  if (!raw) process.exit(2);
+  const url = new URL(raw);
+  const local = url.hostname === "localhost" || url.hostname === "127.0.0.1" || url.hostname === "::1";
+  if (!local) process.exit(3);
+' || die "Refusing to seed: dev PAYLOAD_DB_URL is not localhost."
+
+dc() { "$RUNTIME" compose --env-file "$DEV_ENV" -f "$COMPOSE_FILE" "$@"; }
+POSTGRES_USER="$(bunx dotenv -e "$DEV_ENV" -- sh -c 'printf %s "$POSTGRES_USER"')"
+POSTGRES_DB="$(bunx dotenv -e "$DEV_ENV" -- sh -c 'printf %s "$POSTGRES_DB"')"
+[ -n "$POSTGRES_USER" ] && [ -n "$POSTGRES_DB" ] || die "Local Postgres configuration is incomplete."
+
+info "Ensuring local Payload Postgres is running"
+dc up -d postgres >/dev/null
+for i in $(seq 1 30); do
+  if dc exec -T postgres pg_isready -U "$POSTGRES_USER" >/dev/null 2>&1; then break; fi
+  [ "$i" -lt 30 ] || die "Local Payload Postgres did not become ready."
+  sleep 1
+done
+
+SNAPSHOT_HASH="$(sha256sum "$SNAPSHOT" | awk '{print $1}')"
+marker_exists=false
+
+# Earlier versions stored this marker in the Payload database. Payload/Drizzle
+# treats unmanaged tables as drift and prompts to delete them during dev startup,
+# so keep seed state in .local/ and remove the legacy table from local DBs.
+dc exec -T postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=1 \
+  -c 'DROP TABLE IF EXISTS _local_seed_state' >/dev/null
+
+if [ -s "$MARKER" ] && grep -qx "PAYLOAD_SNAPSHOT_SHA256=$SNAPSHOT_HASH" "$MARKER"; then
+  marker_exists=true
+fi
+
+if [ "$marker_exists" = true ] && [ "$FORCE" != true ]; then
+  echo "Payload snapshot seed already applied; skipping. Use --force to restore again."
+  exit 0
+fi
+
+if [ "$FORCE" = true ] && [ "$ASSUME_YES" != true ]; then
+  printf 'This will replace the LOCAL Payload database from .local/payload-staging.dump.\n'
+  read -r -p 'Continue? [y/N] ' answer
+  case "$answer" in y|Y|yes|YES) ;; *) echo "Cancelled."; exit 0 ;; esac
+fi
+
+info "Restoring local Payload database from .local/payload-staging.dump"
+dc exec -T postgres dropdb --force --if-exists -U "$POSTGRES_USER" "$POSTGRES_DB"
+dc exec -T postgres createdb -U "$POSTGRES_USER" -O "$POSTGRES_USER" "$POSTGRES_DB"
+dc exec -T postgres pg_restore -U "$POSTGRES_USER" -d "$POSTGRES_DB" \
+  --no-owner --no-acl --exit-on-error < "$SNAPSHOT"
+
+{
+  printf 'PAYLOAD_SNAPSHOT_SHA256=%s\n' "$SNAPSHOT_HASH"
+  printf 'PAYLOAD_SNAPSHOT_SEEDED_AT=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+} > "$MARKER"
+chmod 600 "$MARKER"
+
+echo "Payload snapshot seed applied."
@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+# Refresh the local Payload seed snapshot from staging, then force-apply it to
+# the local development Payload database. Normal db:up never calls staging.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd -- "$SCRIPT_DIR/../.." && pwd)"
+COMPOSE_FILE="$ROOT_DIR/docker/compose.local.yml"
+SNAPSHOT="$ROOT_DIR/.local/payload-staging.dump"
+STAGING_ENV=""
+DEV_ENV=""
+SOURCE_PG_ENV=""
+ASSUME_YES=false
+[ "${1:-}" = "--yes" ] && ASSUME_YES=true
+
+info() { printf '▶ %s\n' "$*"; }
+die() { printf 'Error: %s\n' "$*" >&2; exit 1; }
+
+if command -v docker >/dev/null 2>&1; then RUNTIME=docker
+elif command -v podman >/dev/null 2>&1; then RUNTIME=podman
+else die "Docker or Podman is required."; fi
+"$RUNTIME" info >/dev/null 2>&1 || die "$RUNTIME is not usable."
+
+mkdir -p "$ROOT_DIR/.local"
+chmod 700 "$ROOT_DIR/.local"
+cleanup() {
+  [ -z "$STAGING_ENV" ] || rm -f "$STAGING_ENV"
+  [ -z "$DEV_ENV" ] || rm -f "$DEV_ENV"
+  [ -z "$SOURCE_PG_ENV" ] || rm -f "$SOURCE_PG_ENV"
+  rm -f "${SNAPSHOT}.tmp"
+}
+trap cleanup EXIT INT TERM HUP
+
+STAGING_ENV="$(mktemp "${TMPDIR:-/tmp}/convex-monorepo-staging.XXXXXX.env")"
+DEV_ENV="$(mktemp "${TMPDIR:-/tmp}/convex-monorepo-dev.XXXXXX.env")"
+sh "$ROOT_DIR/scripts/export-env" staging > "$STAGING_ENV"
+sh "$ROOT_DIR/scripts/export-env" dev > "$DEV_ENV"
+
+validate_url() {
+  local env_file="$1" expected="$2"
+  bunx dotenv -e "$env_file" -- node -e '
+    const expected = process.argv[1];
+    const raw = process.env.PAYLOAD_DB_URL;
+    if (!raw) process.exit(2);
+    const url = new URL(raw);
+    const local = url.hostname === "localhost" || url.hostname === "127.0.0.1" || url.hostname === "::1";
+    if ((expected === "local") !== local) process.exit(3);
+  ' "$expected" || die "PAYLOAD_DB_URL safety check failed for the $expected environment."
+}
+
+validate_url "$STAGING_ENV" remote
+validate_url "$DEV_ENV" local
+
+next_is_running=false
+if command -v ss >/dev/null 2>&1; then
+  ss -ltnH 'sport = :3000' | grep -q . && next_is_running=true
+elif command -v lsof >/dev/null 2>&1; then
+  lsof -nP -iTCP:3000 -sTCP:LISTEN >/dev/null 2>&1 && next_is_running=true
+elif curl -sS --max-time 1 http://localhost:3000 >/dev/null 2>&1; then
+  next_is_running=true
+fi
+if [ "$next_is_running" = true ]; then
+  die "Next is running on port 3000. Stop bun dev:next before replacing Payload data."
+fi
+
+SOURCE_PG_ENV="$(mktemp "${TMPDIR:-/tmp}/convex-monorepo-pg.XXXXXX.env")"
+PG_ENV_FILE="$SOURCE_PG_ENV" bunx dotenv -e "$STAGING_ENV" -- sh -c '
+  umask 077
+  printf "PGDATABASE=%s\n" "$PAYLOAD_DB_URL" > "$PG_ENV_FILE"
+'
+
+if [ "$ASSUME_YES" != true ]; then
+  printf 'This will download staging Payload data and replace the LOCAL Payload database.\n'
+  printf 'The snapshot may contain user records and password hashes; it stays under .local/.\n'
+  read -r -p 'Continue? [y/N] ' answer
+  case "$answer" in y|Y|yes|YES) ;; *) echo "Cancelled."; exit 0 ;; esac
+fi
+
+info "Exporting a read-only snapshot from staging Payload Postgres"
+"$RUNTIME" run --rm --network host --env-file "$SOURCE_PG_ENV" \
+  docker.io/library/postgres:17 \
+  sh -c 'exec pg_dump --dbname="$PGDATABASE" --format=custom --no-owner --no-acl' \
+  > "${SNAPSHOT}.tmp"
+mv "${SNAPSHOT}.tmp" "$SNAPSHOT"
+chmod 600 "$SNAPSHOT"
+
+info "Ensuring local Payload Postgres is running"
+bash "$ROOT_DIR/scripts/db/seed-payload" --force --yes
+
+printf '\nLocal Payload seed snapshot refreshed and applied.\n'
+printf 'Snapshot: .local/payload-staging.dump\n'
+printf 'Normal db:up will reuse this snapshot and skip after .local/payload-seed-state.env exists.\n'
@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd -- "$SCRIPT_DIR/../.." && pwd)"
+COMPOSE_FILE="$ROOT_DIR/docker/compose.local.yml"
+STATE_FILE="$ROOT_DIR/.local/dev.generated.env"
+ENV_FILE=""
+
+info() { printf '▶ %s\n' "$*"; }
+die() { printf 'Error: %s\n' "$*" >&2; exit 1; }
+
+if command -v docker >/dev/null 2>&1; then RUNTIME=docker
+elif command -v podman >/dev/null 2>&1; then RUNTIME=podman
+else die "Docker or Podman is required."; fi
+"$RUNTIME" info >/dev/null 2>&1 || die "$RUNTIME is not usable."
+
+mkdir -p "$ROOT_DIR/.local"
+cleanup() { [ -z "$ENV_FILE" ] || rm -f "$ENV_FILE"; }
+trap cleanup EXIT
+
+refresh_env() {
+  local next
+  next="$(mktemp "${TMPDIR:-/tmp}/convex-monorepo-local.XXXXXX.env")"
+  sh "$ROOT_DIR/scripts/export-env" dev > "$next" || { rm -f "$next"; die "Unable to export Infisical dev."; }
+  [ -z "$ENV_FILE" ] || rm -f "$ENV_FILE"
+  ENV_FILE="$next"
+  set -a
+  # shellcheck disable=SC1090
+  source "$ENV_FILE"
+  set +a
+}
+dc() { "$RUNTIME" compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@"; }
+upsert_state() {
+  local key="$1" value="$2" tmp escaped
+  tmp="$(mktemp "${TMPDIR:-/tmp}/convex-monorepo-state.XXXXXX.env")"
+  [ ! -f "$STATE_FILE" ] || grep -v "^${key}=" "$STATE_FILE" > "$tmp" || true
+  escaped="$(printf '%s' "$value" | sed "s/'/'\\\\''/g")"
+  printf "%s='%s'\n" "$key" "$escaped" >> "$tmp"
+  mv "$tmp" "$STATE_FILE"
+}
+
+refresh_env
+info "Starting local Payload Postgres, Convex, and dashboard"
+dc up -d
+
+info "Waiting for Payload Postgres"
+for i in $(seq 1 30); do
+  dc exec -T postgres pg_isready -U "${POSTGRES_USER:-convexmonorepo}" >/dev/null 2>&1 && break
+  [ "$i" -lt 30 ] || die "Postgres did not become ready."
+  sleep 1
+done
+
+info "Waiting for Convex at http://localhost:${BACKEND_PORT:-3210}"
+for i in $(seq 1 60); do
+  curl -fs "http://localhost:${BACKEND_PORT:-3210}/version" >/dev/null 2>&1 && break
+  [ "$i" -lt 60 ] || die "Convex did not become ready."
+  sleep 2
+done
+
+if [ -z "${CONVEX_SELF_HOSTED_ADMIN_KEY:-}" ]; then
+  admin_key="$(dc exec -T convex-backend ./generate_admin_key.sh 2>/dev/null | grep -E '.+\|.+' | tail -n1 | tr -d '\r')"
+  [ -n "$admin_key" ] || die "Unable to generate the Convex admin key."
+  upsert_state CONVEX_SELF_HOSTED_ADMIN_KEY "$admin_key"
+  refresh_env
+  info "Generated the machine-local Convex admin key"
+fi
+
+info "Deploying Convex schema and functions"
+(cd "$ROOT_DIR/packages/backend" && bun run setup)
+
+convex_env_names="$(
+  sh "$ROOT_DIR/scripts/with-env" dev -- bash -c \
+    'cd packages/backend && bunx convex env list' 2>/dev/null \
+    | sed -n 's/^\([A-Za-z_][A-Za-z0-9_]*\)=.*/\1/p'
+)"
+if ! printf '%s\n' "$convex_env_names" | grep -qx 'JWT_PRIVATE_KEY' \
+  || ! printf '%s\n' "$convex_env_names" | grep -qx 'JWKS' \
+  || ! printf '%s\n' "$convex_env_names" | grep -qx 'SITE_URL'; then
+  info "Configuring local Convex Auth keys"
+  auth_keys="$(node "$ROOT_DIR/scripts/generate-convex-auth-keys.mjs")"
+  jwt="$(printf '%s\n' "$auth_keys" | sed -n 's/^JWT_PRIVATE_KEY="\(.*\)"$/\1/p')"
+  jwks="$(printf '%s\n' "$auth_keys" | sed -n 's/^JWKS=//p')"
+  JWT_VAL="$jwt" JWKS_VAL="$jwks" sh "$ROOT_DIR/scripts/with-env" dev -- bash -c '
+    cd packages/backend
+    bunx convex env set "JWT_PRIVATE_KEY=$JWT_VAL" >/dev/null
+    bunx convex env set "JWKS=$JWKS_VAL" >/dev/null
+    bunx convex env set "SITE_URL=http://localhost:3000" >/dev/null
+  '
+fi
+
+info "Seeding local Payload from snapshot if needed"
+bash "$ROOT_DIR/scripts/db/seed-payload" --yes
+
+printf '\nLocal stack ready:\n  App: http://localhost:3000\n  Convex: http://localhost:%s\n  Dashboard: http://localhost:%s\n  Payload Postgres: localhost:%s\n' "${BACKEND_PORT:-3210}" "${DASHBOARD_PORT:-6791}" "${POSTGRES_PORT:-5432}"